diff --git a/.github/workflows/build_tools.yml b/.github/workflows/build_tools.yml deleted file mode 100644 index bf764cf49..000000000 --- a/.github/workflows/build_tools.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: Build Fireball and Management Tools -env: - AWS_REGION: eu-west-2 -permissions: - id-token: write - contents: read - -on: - push: - branches: [master] - pull_request: - # The branches below must be a subset of the branches above - branches: [master] -jobs: - local_tests: - name: local test stage - runs-on: win10 - steps: - - name: "☁️ checkout repository" - uses: actions/checkout@v4 - - name: testing - run: | - echo "hello from test" - conda activate ukmon-admin - dir - build_usermgmt: - name: build usermanagement - runs-on: win10 - needs: local_tests - steps: - - name: building - run: | - echo "running build step" - conda activate ukmon-admin - cd usermgmt\windows - pyinstaller ./stationMaint2.py --noconsole --onefile --windowed --icon .\camera.ico - dir dist - package_usermgmt: - name: package usermanagement - runs-on: win10 - needs: - - build_usermgmt - steps: - - name: packaging - uses: Minionguyjpro/Inno-Setup-Action@v1.2.2 - with: - path: usermgmt\windows\cam_management_setup.iss - release_usermgmt: - name: release usermanagement - runs-on: win10 - needs: - - package_usermgmt - steps: - - name: release - uses: xresloader/upload-to-github-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.MM_PAT }} - with: - file: e:\temp\setup_cameraMgmt.exe - tags: true - draft: true - overwrite: true - tag_name: 2025.11.1 - default_release_name: User Management Tool - default_release_body_path: usermgmt\windows\release_notes.md - cleanup_usermgmt: - name: clean up temp cameramgmt files - runs-on: win10 - continue-on-error: true - needs: - - release_usermgmt - steps: - - name: delete temp cameramgmt package - run: | - del e:\temp\setup_cameraMgmt.exe - build_fbtool: - name: build fbcollector - runs-on: win10 - needs: local_tests - steps: - - name: building - run: | - echo "running build step" - conda activate fbcollector - cd fbCollector - pyinstaller ./fireballCollector.py --onefile --windowed --icon .\ukmda.ico - markdown2 ./README.md > README.html - package_fbtool: - name: package fbcollector - runs-on: win10 - needs: - - build_fbtool - steps: - - name: packaging - uses: Minionguyjpro/Inno-Setup-Action@v1.2.2 - with: - path: .\fbCollector\fireballcollector_setup.iss - release_fbtool: - name: release fbcollector - runs-on: win10 - needs: - - package_fbtool - steps: - - name: release - uses: xresloader/upload-to-github-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.MM_PAT }} - with: - file: e:\temp\setup_fireballCollector.exe - tags: true - draft: true - overwrite: true - tag_name: 2025.11.1 - default_release_name: Tools to Gather Fireball data and Manage UKMON Cameras - default_release_body_path: fbCollector\README.md - cleanup_fbtool: - name: clean up temp fbcollector files - runs-on: win10 - continue-on-error: true - needs: - - release_fbtool - steps: - - name: delete package - run: | - del e:\temp\setup_fireballCollector.exe diff --git a/usermgmt/deploy-server.yml b/archive/deployment/deploy-usermgmt.yml similarity index 64% rename from usermgmt/deploy-server.yml rename to archive/deployment/deploy-usermgmt.yml index db5f0ef94..8429780d3 100644 --- a/usermgmt/deploy-server.yml +++ b/archive/deployment/deploy-usermgmt.yml @@ -3,22 +3,16 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/usermgmt/server" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive/usermgmt" + destdir: /home/ec2-user/keymgmt tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - name: Ensures {{destdir}} exists file: path={{destdir}} state=directory - tags: [dev,prod] - name: Copy files - tags: [dev,prod] copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} with_items: - {src: '{{srcdir}}/addSftpUser.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } - {src: '{{srcdir}}/updateAwsKey.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } - - {src: '{{srcdir}}/ukmon.ini', dest: '{{destdir}}/', mode: '644', backup: yes, directory_mode: no } + - {src: '{{srcdir}}/addAdminUser.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } + - {src: '{{srcdir}}/addFBApiKey.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } diff --git a/archive/ukmon_pylib/reports/createSearchableFormat.py b/archive/ukmon_pylib/reports/createSearchableFormat.py index 2e47d56f7..e2df79c2b 100644 --- a/archive/ukmon_pylib/reports/createSearchableFormat.py +++ b/archive/ukmon_pylib/reports/createSearchableFormat.py @@ -9,10 +9,27 @@ import os import pandas as pd import datetime +import boto3 + + +def checkUrl(s3, url): + siteroot = os.getenv('WEBSITEBUCKET', default='s3://ukmda-website') + retval = url + tmpurl = url + if url[0]==',': + tmpurl = url[1:] + try: + _ = s3.head_object(Bucket=siteroot[5:], Key=tmpurl[1:]) + except Exception: + #print(e) + retval = '/img/missing-white.png' + print(url, retval) + return retval def convertSingletoSrchable(datadir, year, newonly=True): print(datetime.datetime.now(), 'single-detection searchable index start') + s3 = boto3.client('s3') # load the single-station combined data if newonly is False: @@ -34,6 +51,10 @@ def convertSingletoSrchable(datadir, year, newonly=True): uadata['fn']=[f'/img/single/{y}/{y}{m:02d}/'+f.replace('.fits','.jpg') for f,y,m in zip(uadata.Filename, uadata.Y, uadata.M)] + print(datetime.datetime.now(), 'checking target urls exist') + uadata['targfn'] = [checkUrl(s3, x) for x in uadata.fn] + print(datetime.datetime.now(), 'done') + # create array for source print(datetime.datetime.now(), 'add source column') srcs = ['2Single']*len(uadata.Filename) @@ -44,41 +65,13 @@ def convertSingletoSrchable(datadir, year, newonly=True): print(datetime.datetime.now(), 'create interim dataframe') hdr=['eventtime','source','shower','Mag','loccam','url','imgs', 'loctime', 'Y','M'] resdf = pd.DataFrame(zip(uadata.Dtstamp, srcs, uadata.Shwr, - uadata.Mag, uadata.ID, uadata.fn, uadata.fn, uadata.LocalTime, + uadata.Mag, uadata.ID, uadata.targfn, uadata.targfn, uadata.LocalTime, uadata.Y, uadata.M), columns=hdr) # fix up some mangled historical data resdf.loc[resdf.loccam=='Ringwood_N_UK000S', 'loccam'] = 'UK000S' resdf.loc[resdf.loccam=='Tackley_SW_UK0006', 'loccam'] = 'UK0006' - # select the RMS data out, its good now - # FIXME - needs to select for "not FF_UK9" so we can include non-UK cameras - rmsdata=resdf[resdf.url.str.contains('FF_UK0')] - rmsdata = rmsdata.drop(columns=['Y','M','loctime']) - - # now select out the UFO dta and fix it up - ufodata=resdf[resdf.url.str.contains('FF_UK9')] - #fix up Clanfield cameras - ufodata.loc[ufodata.loccam=='UK9990', 'loccam'] = 'Clanfield_NE' - ufodata.loc[ufodata.loccam=='UK9989', 'loccam'] = 'Clanfield_NW' - ufodata.loc[ufodata.loccam=='UK9988', 'loccam'] = 'Clanfield_SE' - ufodata = ufodata.drop(columns=['url','imgs']) - - # create the URL and imgs fields - ufodata['url']=[f'/img/single/{y}/{y}{m:02d}/M{lt}_{f}P.jpg' - for f,y,m,lt in zip(ufodata.loccam, ufodata.Y, ufodata.M, ufodata.loctime)] - ufodata['imgs'] = ufodata.url - ufodata = ufodata.drop(columns=['loctime','Y','M']) - - # annoying special case for UK0001, H and S which do not upload JPGs - rmsdata.loc[rmsdata.loccam=='UK0001','url']='/img/missing-white.png' - rmsdata.loc[rmsdata.loccam=='UK0001','imgs']='/img/missing-white.png' - rmsdata.loc[rmsdata.loccam=='UK000H','url']='/img/missing-white.png' - rmsdata.loc[rmsdata.loccam=='UK000H','imgs']='/img/missing-white.png' - rmsdata.loc[rmsdata.loccam=='UK000S','url']='/img/missing-white.png' - rmsdata.loc[rmsdata.loccam=='UK000S','imgs']='/img/missing-white.png' - - resdf = pd.concat([rmsdata,ufodata]) if newonly is True: return resdf, rmsuafile else: @@ -114,19 +107,22 @@ def convertMatchToSrchable(datadir, year, newonly=True): if __name__ == '__main__': if len(sys.argv) < 3: - print('usage: python createSearchableFormat.py year dest mode') + print('usage: python createSearchableFormat.py year mode outdir') exit(1) else: datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') year = sys.argv[1] mode = sys.argv[2] + outdir = os.path.join(datadir, 'searchidx') + if len(sys.argv) > 3: + outdir = sys.argv[3] # create a set of single-station data and merge with last match set if mode == 'singles': print(datetime.datetime.now(), 'converting single-station data') newsingles, fname = convertSingletoSrchable(datadir, year, True) - outfile = os.path.join(datadir, 'searchidx', '{:s}-singles-new.csv'.format(year)) + outfile = os.path.join(outdir, '{:s}-singles-new.csv'.format(year)) if newsingles is not None: newsingles.to_csv(outfile, index=False, header=False) if fname is not None: @@ -136,11 +132,11 @@ def convertMatchToSrchable(datadir, year, newonly=True): elif mode == 'matches': print(datetime.datetime.now(), 'converting match data') newmatches, fname = convertMatchToSrchable(datadir, year, True) - outfile = os.path.join(datadir, 'searchidx', '{:s}-matches-new.csv'.format(year)) + outfile = os.path.join(outdir, '{:s}-matches-new.csv'.format(year)) if newmatches is not None: newmatches.to_csv(outfile, index=False, header=False) if fname is not None: os.remove(fname) else: - print('usage: createSearchableFormat yyyy matches_or_singles') + print('usage: createSearchableFormat year mode outdir') diff --git a/usermgmt/addAdminUser.ps1 b/archive/usermgmt/addAdminUser.sh similarity index 96% rename from usermgmt/addAdminUser.ps1 rename to archive/usermgmt/addAdminUser.sh index 3158fb995..eb02ec10c 100644 --- a/usermgmt/addAdminUser.ps1 +++ b/archive/usermgmt/addAdminUser.sh @@ -1,5 +1,6 @@ -# simple script to create a new user-maintenance account -# and save the access key locally. Can only be used by an account with IAM Admin permissions. -$username = $args[0] -aws iam create-user --user-name $username --profile ukmda_admin -aws iam add-user-to-group --user-name $username --user-group Administrators --profile ukmda_admin +#!/bin/bash +# simple script to create a new user-maintenance account +# and save the access key locally. Can only be used by an account with IAM Admin permissions. +$username = $args[0] +aws iam create-user --user-name $username --profile ukmda_admin +aws iam add-user-to-group --user-name $username --user-group Administrators --profile ukmda_admin diff --git a/usermgmt/addFBApiKey.ps1 b/archive/usermgmt/addFBApiKey.sh similarity index 58% rename from usermgmt/addFBApiKey.ps1 rename to archive/usermgmt/addFBApiKey.sh index c903c2eb4..eb957f893 100644 --- a/usermgmt/addFBApiKey.ps1 +++ b/archive/usermgmt/addFBApiKey.sh @@ -1,6 +1,7 @@ -# copyright Mark McIntyre, 2024- - -# script to add an api key for a user - -$username = $args[0] -aws apigateway create-api-key --name "$username" --region eu-west-1 --profile ukmda_admin --description "key for $username" --enabled \ No newline at end of file +#!/bin/bash +# copyright Mark McIntyre, 2024- + +# script to add an api key for a user + +$username = $1 +aws apigateway create-api-key --name "$username" --region eu-west-1 --description "key for $username" --enabled --profile ukmda_admin \ No newline at end of file diff --git a/usermgmt/server/addSftpUser.sh b/archive/usermgmt/addSftpUser.sh similarity index 93% rename from usermgmt/server/addSftpUser.sh rename to archive/usermgmt/addSftpUser.sh index 8ddb730cd..82fdcbd26 100644 --- a/usermgmt/server/addSftpUser.sh +++ b/archive/usermgmt/addSftpUser.sh @@ -44,7 +44,7 @@ if [ ! -f $keydir/keys/$shortid_l.key ] ; then fi # add a unix user and set their homedir to /var/sftp/userid -grep $userid /etc/passwd +grep -w $userid /etc/passwd if [ $? -eq 1 ] ; then dt=$(date +%Y-%m-%d) logger -s -t addSftpUser "Creating unix user $userid" @@ -91,6 +91,10 @@ if [ ! -z $oldloc ] ; then logger -s -t addSftpUser "Moving $oldloc to $userid" sudo cp /var/sftp/$oldloc/ukmon.ini /var/sftp/$oldloc/ukmon.ini.bkp sudo cp $keydir/inifs/$userid.ini /var/sftp/$oldloc/ukmon.ini + # and copy the ssh authorized keys file + sudo cp $oldloc/.ssh/authorized_keys /var/sftp/$userid/.ssh/ + sudo chown -R $userid:$userid /var/sftp/$userid/.ssh/authorized_keys + sudo chmod 644 /var/sftp/$userid/.ssh/authorized_keys fi logger -s -t addSftpUser "Finished" diff --git a/usermgmt/windows/camTableMaintenance.py b/archive/usermgmt/camTableMaintenance.py similarity index 100% rename from usermgmt/windows/camTableMaintenance.py rename to archive/usermgmt/camTableMaintenance.py diff --git a/usermgmt/windows/testkeys.py b/archive/usermgmt/testkeys.py similarity index 100% rename from usermgmt/windows/testkeys.py rename to archive/usermgmt/testkeys.py diff --git a/usermgmt/server/updateAwsKey.sh b/archive/usermgmt/updateAwsKey.sh similarity index 100% rename from usermgmt/server/updateAwsKey.sh rename to archive/usermgmt/updateAwsKey.sh diff --git a/usermgmt/server/updateKeysForSplit.py b/archive/usermgmt/updateKeysForSplit.py similarity index 100% rename from usermgmt/server/updateKeysForSplit.py rename to archive/usermgmt/updateKeysForSplit.py diff --git a/fbCollector/README.md b/fbCollector/README.md deleted file mode 100644 index 6cd3c7596..000000000 --- a/fbCollector/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# A Fireball Collector and Analyser for UKMON and GMN - -This tool allows authorised users to collect fireball data from UKMON and GMN and then to reduce and solve it. - -## Prerequisites - -* You have installed [WMPL](https://github.com/wmpg/WesternMeteorPyLib/), used to solve trajectories. -* You have installed [RMS](https://github.com/CroatianMeteorNetwork/RMS), used to reduce raw data. -* You have a local folder where you will store fireball data. This is called `basedir` in this documentation. - -The above are sufficient to collect data from UKMON and to analyse data either collected from UKMON or by other means. There are additional configuration options that can be used to collect data from GMN and upload solutions to UKMON. These are described inline below. - -## Installation -* Install the package `setup_fireballCollector.exe` from [here](https://github.com/ukmda/ukmda-dataprocessing/releases). -* Install WMPL and RMS and verify they're working. - -## Configuring the App -* Launch the app from the desktop icon or start menu. After a few seconds a text editor will appear to allow you to configure the application. -* Update the values of `basedir`, `rms_loc`, `rms_env`, `wmpl_loc`, and `wmpl_env` as appropriate and save. - -## Using the App -* Enter a date and time in the format `YYYYMMDD_HHMMSS` into the `Image Selection` box, then click "Get Images". - * After a few seconds, the listbox below should be populated with images from around the time you selected, provided the UKMON live feed captured something. - * If nothing appears, check the UKMON [livestream](https://archive.ukmeteors.co.uk/live/index.html) to make sure you chose a suitable time. - -* Go through the image list and click `Remove` or press the `Delete` key to delete any that are not of the event you are interested in. -* Once you've whittled the list down to just the interesting events, you can select `Get ECSVs` from the `Raw` menu. This will attempt to get raw data for each image. You can also click `Get Videos` to collect any video data thats available. - -* Now you can click `Solve` from the `Solve` menu. This will invoke WMPL and will attempt to find a trajectory that matches the raw data. It may take some time. -* If the solver is successful, you can view the solution by selecting `View Solution` from the `Solve` menu. The left-hand list will now show the output of the solver so you can examine it. You can switch back to viewing the raw images by selecting `Review Images` from the `Review` menu. - -* If the solve process fails or if the solution seems very poor then try excluding some detections. To do this, select `Excl/Incl ECSV` from the `Raw` menu then rerun the Solver. -* If the solution was bad its also worth deleting it before attempting a rerun via `Delete Solution` from the `Solve` menu. - -## Manually Reducing an Image -If you've installed RMS and you have a FITS or FR file from RMS along with the camera's config and platepar files, you can reduce the data using RMS's `SkyFit2` tool. Select an image in the file list then select `Reduce Selected Image` from the `Raw` menu. - -## Sending Solutions to UKMON -Once you have a solution, select `Upload orbit` from the `Solve` menu. This will create a Zip file that bundles the created trajectory pickle file with any images and videos. You can then upload the file to Dropbox, Google Drive or another file-sharing site and email a link to [us](fireballdata@ukmeteornetwork.org) where one of our team will check and upload it to our Archive. - -#### API Key -Members of the UKMON team who frequently create solutions can request an API key to directly upload solutions less than 10 MB in size. We'll contact you if we think this is applicable. - -## Sharing Raw Data -If you've configured a raw data location in the configuration file, `Share Raw Data` on the raw menu will create a zip file of all the raw data and copy it to the location. For example, if you have Dropbox installed and set the share location to a folder in your Dropbox, then the zip file will be copied there. - -### Collecting data from GMN -Members of the GMN Coordinators group who have permission from Denis Vida can use this tool to collect raw data directly from GMN. - -If you fall into this category you can fill in the `[gmn]` section of the config file with the name of your SSH private key and other details. This will activate additional menu options to `Get GMN Raw Data` and to use the `Watchlist`. - -*Note that you will need WSL2 enabled and the rsync tool installed in WSL2 to run this process.* - -## Logs -You can view the logs from the `File` menu. The logs are created in [%TEMP%/fbcollector](%temp%/fbcollector). - - diff --git a/fbCollector/config.ini.sample b/fbCollector/config.ini.sample deleted file mode 100644 index 2128f65dd..000000000 --- a/fbCollector/config.ini.sample +++ /dev/null @@ -1,32 +0,0 @@ -# tailor this config file to your needs -# Note that $HOME will be translated into your homedir (~ on linux, $env:userprofile on windows) - -[Fireballs] -# the location to which fireball data will be downloaded eg c:\fireballs. -basedir= - -[reduction] -# To use the reduction options RMS must be installed and an RMS python conda environment created -rms_loc= -rms_env=RMS - -[solver] -# To use the solver options WMPL must be installed and a WMPL python conda environment created -wmpl_loc= -wmpl_env=wmpl - -[sharing] -# create a folder in Dropbox or similar if you want to share raw data with others and then put -# its *local* path here eg c:\users\yourname\dropbox\meteors -shrfldr= - -[gmn] -# the GMN SSH key needs to be configured with Denis Vida -# if you have permission you can then collect data from the GMN server -gmnkey= -gmnuser= -gmnserver=gmn.uwo.ca - -[ukmon] -apikey= - diff --git a/fbCollector/download_events.sh b/fbCollector/download_events.sh deleted file mode 100644 index c491c37b4..000000000 --- a/fbCollector/download_events.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" - -dt=$1 -basedir=$2 -getall=$3 - -mkdir -p $basedir/$dt -pushd $basedir/$dt - -rsync -avz gmn.uwo.ca:/home/uk*/files/event_monitor/*${dt}*.bz2 . -rsync -avz gmn.uwo.ca:/home/be*/files/event_monitor/*${dt}*.bz2 . -rsync -avz gmn.uwo.ca:/home/ie*/files/event_monitor/*${dt}*.bz2 . -rsync -avz gmn.uwo.ca:/home/nl*/files/event_monitor/*${dt}*.bz2 . -if [ "$getall" == "all" ] ; then - rsync -avz gmn.uwo.ca:/home/fr*/files/event_monitor/*${dt}*.bz2 . - rsync -avz gmn.uwo.ca:/home/de*/files/event_monitor/*${dt}*.bz2 . - rsync -avz gmn.uwo.ca:/home/es*/files/event_monitor/*${dt}*.bz2 . - rsync -avz gmn.uwo.ca:/home/ch*/files/event_monitor/*${dt}*.bz2 . - rsync -avz gmn.uwo.ca:/home/it*/files/event_monitor/*${dt}*.bz2 . - rsync -avz gmn.uwo.ca:/home/cz*/files/event_monitor/*${dt}*.bz2 . - rsync -avz gmn.uwo.ca:/home/hr*/files/event_monitor/*${dt}*.bz2 . - rsync -avz gmn.uwo.ca:/home/sk*/files/event_monitor/*${dt}*.bz2 . -fi - -for f in *.bz2 ; do tar -xvf $f ; done -if [ -d UKMON ] ; then - sites=$(ls -1 UKMON) - for site in $sites ; do - cams=$(ls -1 UKMON/$site) - for cam in $cams ; do - if [ -d ./$cam ] ; then rm -Rf $cam ; fi - mv -f UKMON/$site/$cam . - done - done - rm -Rf UKMON -fi -if [ -d NEMETODE ] ; then - sites=$(ls -1 NEMETODE) - for site in $sites ; do - cams=$(ls -1 NEMETODE/$site) - for cam in $cams ; do - if [ -d ./$cam ] ; then rm -Rf $cam ; fi - mv -f NEMETODE/$site/$cam . - done - done - rm -Rf NEMETODE -fi -if [ -d "UKMON,NEMETODE" ] ; then - sites=$(ls -1 "UKMON,NEMETODE") - for site in $sites ; do - cams=$(ls -1 "UKMON,NEMETODE/$site") - for cam in $cams ; do - if [ -d ./$cam ] ; then rm -Rf $cam ; fi - mv -f "UKMON,NEMETODE/$site/$cam" . - done - done - rm -Rf "UKMON,NEMETODE" -fi -mkdir -p ./stacks -mkdir -p ./jpgs -mkdir -p ./mp4s -cp -f */*.jpg ./jpgs -mv -f ./jpgs/*captured_stack* ./stacks -cp -f */*.mp4 ./mp4s -popd \ No newline at end of file diff --git a/fbCollector/fbCollector.ps1 b/fbCollector/fbCollector.ps1 deleted file mode 100644 index 52d6e2c17..000000000 --- a/fbCollector/fbCollector.ps1 +++ /dev/null @@ -1,25 +0,0 @@ -# -# powershell script to launch the fireball data collector tool -# Copyright (C) 2018-2023 Mark McIntyre -# -push-location $PSScriptRoot - -$wmpl_env=((select-string .\config.ini -pattern "wmpl_env" -list).line).split('=')[1] -$wmpl_loc=((select-string .\config.ini -pattern "wmpl_loc" -list).line).split('=')[1] - -conda activate $wmpl_env - -$wmpl_loc = $wmpl_loc.replace('$HOME',$env:userprofile) -$wmpl_loc = $wmpl_loc.replace('\','/') - -$env:pythonpath="$wmpl_loc" - -if ($args.count -lt 1) { - python $PSScriptRoot/fireballCollector.py -}else { - python $PSScriptRoot/fireballCollector.py -d $args[0] -} - -Pop-Location - - diff --git a/fbCollector/fbcollector.yml b/fbCollector/fbcollector.yml deleted file mode 100644 index 83463e685..000000000 --- a/fbCollector/fbcollector.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -- hosts: ukmonhelper2 - gather_facts: no - tasks: - - name: import variables - include_vars: ./vars.yml - tags: [dev,prod] - - name: Ensures {{destdir}} exists - file: path={{destdir}} state=directory - delegate_to: localhost - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - delegate_to: localhost - with_items: - - {src: '{{srcdir}}/fbCollector.ps1', dest: '{{destdir}}/', mode: '755', backup: no, directory_mode: no } - - {src: '{{srcdir}}/fireballCollector.py', dest: '{{destdir}}/', mode: '644', backup: no, directory_mode: no } - - {src: '{{srcdir}}/download_events.sh', dest: '{{destdir}}/', mode: '755', backup: no, directory_mode: no } - - {src: '{{srcdir}}/config.ini.sample', dest: '{{destdir}}/', mode: '644', backup: no, directory_mode: no } - - {src: '{{srcdir}}/noimage.jpg', dest: '{{destdir}}/', mode: '644', backup: no, directory_mode: no } - - {src: '{{srcdir}}/ukmda.ico', dest: '{{destdir}}/', mode: '644', backup: no, directory_mode: no } - - {src: '{{srcdir}}/requirements.txt', dest: '{{destdir}}/', mode: '644', backup: no, directory_mode: no } - diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py deleted file mode 100644 index 00d7c92f8..000000000 --- a/fbCollector/fireballCollector.py +++ /dev/null @@ -1,1244 +0,0 @@ -# -# UI to manage fireball data collection -# Copyright (C) 2018-2023 Mark McIntyre -# -import os -import sys -import argparse -import logging -import logging.handlers -import datetime -import configparser -import shutil -import platform -import subprocess -import glob -import xmltodict -from PIL import Image -import requests -import pandas as pd -import webbrowser - -import paramiko -from scp import SCPClient - - -import tkinter as tk -import tkinter.filedialog as tkFileDialog -import tkinter.messagebox as tkMessageBox -from tkinter.simpledialog import askstring -from tkinter import StringVar, Frame, ACTIVE, END, Listbox, Menu, Entry, Button -from tkinter.ttk import Label, Style, LabelFrame, Scrollbar - -from PIL import Image as img -from PIL import ImageTk - - -config_file = '' -noimg_file = '' -global_bg = "Black" -global_fg = "Gray" - - -def quitApp(): - # Cleanly exits the app - root.quit() - root.destroy() - - -def log_timestamp(): - """ Returns timestamp for logging. - """ - return datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S') - - -def showConfig(): - if platform.system() == 'Darwin': # macOS - procid = subprocess.Popen(('open', config_file)) - elif platform.system() == 'Windows': # Windows - procid = subprocess.Popen(('cmd','/c',config_file)) - else: # linux variants - procid = subprocess.Popen(('xdg-open', config_file)) - procid.wait() - - -class StyledButton(Button): - """ Button with style. - """ - def __init__(self, *args, **kwargs): - Button.__init__(self, *args, **kwargs) - - self.configure(foreground = global_fg, background = global_bg, borderwidth = 3) - - -class StyledEntry(Entry): - """ Entry box with style. - """ - def __init__(self, *args, **kwargs): - Entry.__init__(self, *args, **kwargs) - - self.configure(foreground = global_fg, background = global_bg, insertbackground = global_fg, disabledbackground = global_bg, disabledforeground = "DimGray") - - -class ConstrainedEntry(StyledEntry): - """ Entry box with constrained values which can be input (e.g. 0-255). - """ - def __init__(self, *args, **kwargs): - StyledEntry.__init__(self, *args, **kwargs) - self.maxvalue = 255 - vcmd = (self.register(self.on_validate), "%P") - self.configure(validate="key", validatecommand=vcmd) - # self.configure(foreground = global_fg, background = global_bg, insertbackground = global_fg) - - def disallow(self): - """ Pings a bell on values which are out of bound. - """ - self.bell() - - def update_value(self, maxvalue): - """ Updates values in the entry box. - """ - self.maxvalue = maxvalue - vcmd = (self.register(self.on_validate), "%P") - self.configure(validate="key", validatecommand=vcmd) - - def on_validate(self, new_value): - """ Checks if entered value is within bounds. - """ - try: - if new_value.strip() == "": - return True - value = int(new_value) - if value < 0 or value > self.maxvalue: - self.disallow() - return False - except ValueError: - self.disallow() - return False - - return True - - -def getECSVs(stationID, dateStr, savefiles=False, outdir='.'): - """ - Retrieve a detection in ECSV format for the specified date - """ - apiurl='https://api.ukmeteors.co.uk/getecsv?stat={}&dt={}' - res = requests.get(apiurl.format(stationID, dateStr)) - ecsvlines='' - if res.status_code == 200: - rawdata=res.text.strip() - if len(rawdata) > 10: - ecsvlines=rawdata.split('\n') # convert the raw data into a python list - if savefiles is True: - os.makedirs(outdir, exist_ok=True) - fnamebase = dateStr.replace(':','_').replace('.','_') # create an output filename - j=0 - outf = False - for li in ecsvlines: - if 'issue getting data' in li: - print(li) - return li - if '# %ECSV' in li: - if outf is not False: - outf.close() - j=j+1 - fname = fnamebase + f'_ukmda_{stationID}_M{j:03d}.ecsv' - outf = open(os.path.join(outdir, fname), 'w') - print('saving to ', os.path.join(outdir,fname)) - if outf: - outf.write(f'{li}\n') - else: - print('no ECSV marker found in data') - else: - print('no error, but no data returned') - else: - print(res.status_code) - return ecsvlines - - -def _download(url, outdir, fname=None): - get_response = requests.get(url, stream=True) - if fname is None: - fname = url.split('?')[0].split("/")[-1] - with open(os.path.join(outdir, fname), 'wb') as f: - for chunk in get_response.iter_content(chunk_size=4096): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - return fname - - -def getLiveJpgs(dtstr, outdir=None): - """ - Retrieve live images from the ukmon website that match a pattern - """ - if outdir is None: - outdir = dtstr - os.makedirs(outdir, exist_ok=True) - - apiurl = 'https://api.ukmeteors.co.uk/liveimages/getlive' - - while len(dtstr) < 15: - dtstr = dtstr + '0' - isodt1 = datetime.datetime.strptime(dtstr[:15],'%Y%m%d_%H%M%S') - fromdstr = isodt1.isoformat()[:19]+'.000Z' - isodt2 = isodt1 + datetime.timedelta(minutes=1) - todstr = isodt2.isoformat()[:19]+'.000Z' - liveimgs = pd.read_json(f'{apiurl}?dtstr={fromdstr}&enddtstr={todstr}&fmt=withxml') - - for _, thisimg in liveimgs.iterrows(): - try: - jpgurl = thisimg .urls['url'] - fname = _download(jpgurl, outdir) - log.info(f'retrieved {fname}') - except: - log.warning(f'{img.image_name} unavailable') - - -class fbCollector(Frame): - - def __init__(self, parent, patt=None): - Frame.__init__(self, parent, bg = global_bg) - parent.configure(bg = global_bg) # Set backgound color - parent.grid_columnconfigure(0, weight=1) - parent.grid_rowconfigure(0, weight=1) - - self.grid(sticky="NSEW") # Expand frame to all directions - self.parent = parent - - self.fb_dir = '' - self.gmn_key = '' - self.gmn_user = '' - self.gmn_server = '' - self.wmpl_loc = '' - self.wmpl_env = '' - self.rms_loc = '' - self.rms_env = '' - self.api_key = None - self.share_loc = '' - self.selected = {} - self.evtMonTriggered = None - self.review_stack = False - self.soln_outputdir = None - self.log_files_to_keep = 30 - self.script_loc = os.path.split(config_file)[0] - - self.readConfig() - - self.patt = patt - if patt is None: - self.dir_path = self.fb_dir.strip() - else: - self.dir_path = os.path.join(self.fb_dir, patt) - log.info(f'Fireball folder is {self.fb_dir}') - log.info(f'Scripts folder is {self.script_loc}') - - self.initUI() - if self.dir_path != self.fb_dir: - bin_list = self.get_bin_list() - for b in bin_list: - self.selected[b] = (0, '') - self.update_listbox(bin_list) - - # Update UI changes - parent.update_idletasks() - parent.update() - - return - - def readConfig(self): - localcfg = configparser.ConfigParser() - localcfg.read(config_file) - self.fb_dir = os.path.expanduser(localcfg['Fireballs']['basedir'].replace('$HOME','~')).replace('\\','/') - os.makedirs(self.fb_dir, exist_ok=True) - - self.gmn_key = None - self.gmn_user = None - self.gmn_server = None - if localcfg.has_option('gmn','gmnkey'): - self.gmn_key = localcfg['gmn']['gmnkey'] - self.gmn_user = localcfg['gmn']['gmnuser'] - self.gmn_server = localcfg['gmn']['gmnserver'] - - self.api_key = None - if localcfg.has_option('ukmon','apikey'): - self.api_key = localcfg['ukmon']['apikey'] - if os.path.isfile(os.path.expanduser(self.api_key)): - key = open(os.path.expanduser(self.api_key), 'r').readlines()[0].strip() - else: - key = self.api_key - if len(key) < 40: - key = None - self.api_key = key - - self.wmpl_loc = None - if localcfg.has_option('solver','wmpl_loc'): - self.wmpl_loc = os.path.expanduser(localcfg['solver']['wmpl_loc'].replace('$HOME','~')).replace('\\','/') - self.wmpl_env= localcfg['solver']['wmpl_env'] - log.info(f'wmpl_loc {self.wmpl_loc}') - - self.rms_loc = None - if localcfg.has_option('reduction','rms_loc'): - self.rms_loc = os.path.expanduser(localcfg['reduction']['rms_loc'].replace('$HOME','~')).replace('\\','/') - self.rms_env = localcfg['reduction']['rms_env'] - - - self.share_loc = None - if localcfg.has_option('sharing','shrfldr'): - self.share_loc = os.path.expanduser(localcfg['sharing']['shrfldr'].replace('$HOME','~')).replace('\\','/') - - def quitApplication(self): - print('quitting') - logdir = os.path.join(os.getenv('TMP'), 'fbcollector') - logfiles = os.listdir(logdir) - numtokeep = self.log_files_to_keep - if len(logfiles) > numtokeep: - logfiles.sort() - for i in range(len(logfiles)-numtokeep): - try: - os.remove(os.path.join(logdir, logfiles[i])) - except: - pass - quitApp() - - def initUI(self): - """ Initialize GUI elements. - """ - - self.parent.title("Fireball Downloader") - - # Configure the style of each element - s = Style() - s.configure("TButton", padding=(0, 5, 0, 5), font='serif 10') - s.configure('TLabelframe.Label', foreground=global_fg, background=global_bg) - s.configure('TLabelframe', foreground =global_fg, background=global_bg, padding=(3, 3, 3, 3)) - s.configure("TRadiobutton", foreground = global_fg, background = global_bg) - s.configure("TLabel", foreground = global_fg, background = global_bg) - s.configure("TCheckbutton", foreground = global_fg, background = global_bg) - s.configure("Vertical.TScrollbar", background=global_bg, troughcolor = global_bg) - - self.columnconfigure(0, pad=3) - self.columnconfigure(1, pad=3) - self.columnconfigure(2, pad=3) - self.columnconfigure(3, pad=3) - self.columnconfigure(4, pad=3) - self.columnconfigure(5, pad=3) - self.columnconfigure(6, pad=3) - self.columnconfigure(7, pad=3) - self.columnconfigure(8, pad=3) - - self.rowconfigure(0, pad=3) - self.rowconfigure(1, pad=3) - self.rowconfigure(2, pad=3) - self.rowconfigure(3, pad=3) - self.rowconfigure(4, pad=3) - self.rowconfigure(5, pad=3) - self.rowconfigure(6, pad=3) - self.rowconfigure(7, pad=3) - self.rowconfigure(8, pad=3) - self.rowconfigure(9, pad=3) - self.rowconfigure(10, pad=3) - - # Make menu - self.menuBar = Menu(self.parent) - self.parent.config(menu=self.menuBar) - - # File menu - fileMenu = Menu(self.menuBar, tearoff=0) - fileMenu.add_command(label="Load Folder", command=self.loadFolder) - fileMenu.add_command(label="Archive This Folder", command=self.archiveFolder) - fileMenu.add_command(label="Open in Explorer", command=self.openFolder) - fileMenu.add_separator() - fileMenu.add_command(label="Delete This Folder", command=self.delFolder) - fileMenu.add_separator() - fileMenu.add_command(label="Configuration", command=self.reviewConfig) - fileMenu.add_command(label="View Logs", command=self.viewLogs) - fileMenu.add_separator() - fileMenu.add_command(label="Exit", command=self.quitApplication) - self.menuBar.add_cascade(label="File", underline=0, menu=fileMenu) - - if self.rms_loc: - rmsavailable = 'active' - else: - rmsavailable = 'disabled' - if self.share_loc: - shareavailable = 'active' - else: - shareavailable = 'disabled' - if self.gmn_key: - gmnavailable ='active' - else: - gmnavailable ='disabled' - if self.wmpl_loc: - wmplavailable = 'active' - else: - wmplavailable = 'disabled' - - log.info(f'rms available {rmsavailable}') - log.info(f'share available {shareavailable}') - log.info(f'gmn available {gmnavailable}') - log.info(f'wmpl available {wmplavailable}') - - rawMenu = Menu(self.menuBar, tearoff=0) - rawMenu.add_command(label="Get Live Images", command=self.getData) - rawMenu.add_command(label="Get Videos", command=self.getVids) - rawMenu.add_separator() - rawMenu.add_command(label="Get ECSVs", command=self.getRequestedECSVs) - rawMenu.add_command(label="Excl/Incl ECSV", command=self.ignoreCamera) - rawMenu.add_command(label="Reduce Selected Image", command=self.reduceCamera, state=rmsavailable) - rawMenu.add_separator() - rawMenu.add_command(label="Share Raw Data", command=self.uploadRaw, state=shareavailable) - rawMenu.add_separator() - watchMenu = Menu(self.menuBar, tearoff=0) - watchMenu.add_command(label="Get GMN Raw Data", command=self.getGMNData, state=gmnavailable) - watchMenu.add_separator() - watchMenu.add_command(label="Get Watchlist", command=self.getWatchlist, state=gmnavailable) - watchMenu.add_command(label="View Watchlist", command=self.viewWatchlist, state=gmnavailable) - watchMenu.add_command(label="Upload Watchlist", command=self.putWatchlist, state=gmnavailable) - watchMenu.add_separator() - watchMenu.add_command(label="Fetch Watchlist Event", command=self.getEventData, state=gmnavailable) - rawMenu.add_cascade(label='GMN', menu=watchMenu, state=gmnavailable) - - self.menuBar.add_cascade(label="Raw", underline=0, menu=rawMenu) - - revMenu = Menu(self.menuBar, tearoff=0) - revMenu.add_command(label="Review Stacks", command=self.checkStacks) - revMenu.add_command(label="Review Images", command=self.viewData) - revMenu.add_command(label="Clean Folder", command=self.clean_folder) - self.menuBar.add_cascade(label="Review", underline=0, menu=revMenu) - - solveMenu = Menu(self.menuBar, tearoff=0) - solveMenu.add_command(label="Solve", command=self.solveOrbit, state=wmplavailable) - solveMenu.add_separator() - solveMenu.add_command(label="View Solution", command=self.viewSolution) - solveMenu.add_command(label="Delete Solution", command=self.removeSolution) - solveMenu.add_separator() - solveMenu.add_command(label="Upload Solution", command=self.uploadOrbit) - self.menuBar.add_cascade(label="Solve", underline=0, menu=solveMenu) - - otherMenu = Menu(self.menuBar, tearoff=0) - otherMenu.add_command(label="Get Traj Pickle", command=self.getTrajpickle) - otherMenu.add_command(label="Add Image/Vid", command=self.addImageVideo) - otherMenu.add_separator() - otherMenu.add_command(label="Delete Orbit", command=self.delOrbit) - self.menuBar.add_cascade(label="Other", underline=0, menu=otherMenu) - - helpMenu = Menu(self.menuBar, tearoff=0) - helpMenu.add_command(label="Documentation", command=self.showDocs) - self.menuBar.add_cascade(label="Help", underline=0, menu=helpMenu) - # buttons - self.save_panel = LabelFrame(self, text=' Image Selection ') - self.save_panel.grid(row = 1, columnspan = 2, sticky='WE') - - self.newpatt = StringVar() - self.newpatt.set(self.patt) - - self.patt_entry = StyledEntry(self.save_panel, textvariable = self.newpatt, width = 20) - self.patt_entry.grid(row = 1, column = 1, columnspan = 2, sticky = "W") - save_bmp = StyledButton(self.save_panel, text="Get Images", width = 8, command = lambda: self.getData()) - save_bmp.grid(row = 1, column = 3) - - save_bmp = StyledButton(self.save_panel, text="Remove", width = 8, command = lambda: self.remove_image()) - save_bmp.grid(row = 1, column = 4) - - # Listbox - self.scrollbar = Scrollbar(self) - self.listbox = Listbox(self, width = 47, yscrollcommand=self.scrollbar.set, exportselection=0, activestyle = "none", bg = global_bg, fg = global_fg) - self.listbox.config(height = 25) # Listbox size - self.listbox.grid(row = 4, column = 0, rowspan = 7, columnspan = 2, sticky = "NS") # Listbox position - self.scrollbar.grid(row = 4, column = 2, rowspan = 7, sticky = "NS") # Scrollbar size - - self.listbox.bind('<>', self.update_image) - self.scrollbar.config(command = self.listbox.yview) - - # bind delete key - self.listbox.bind("", lambda x: self.remove_image()) - - # IMAGE - try: - # Show the TV test card image on program start - #noimage_data = img.open('noimage.bin').resize((640,360)) - noimgdata = img.open(noimg_file).resize((640,360)) - noimage = ImageTk.PhotoImage(noimgdata) - except: - noimage = None - - self.imagelabel = Label(self, image = noimage) - self.imagelabel.image = noimage - self.imagelabel.grid(row=3, column=3, rowspan = 4, columnspan = 3) - - # Timestamp label - self.timestamp_label = Label(self, text = "CCNNNN YYYY-MM-DD HH:MM.SS.mms", font=("Courier", 12)) - self.timestamp_label.grid(row = 7, column = 3, sticky = "E") - - def reviewConfig(self): - showConfig() - self.readConfig() - self.initUI() - - def showDocs(self): - webbrowser.open('README.html') - return - - def reduceCamera(self): - current_image = self.listbox.get(ACTIVE) - if current_image == '': - return - camid = current_image[3:9] - print('selected camera is', camid) - dirname = os.path.join(self.dir_path, camid) - tmpscr = os.path.join(os.getenv('TEMP'), 'reduce.ps1') - with open(tmpscr, 'w') as outf: - outf.write(f'cd {self.rms_loc}\nconda activate {self.rms_env}\npython -m Utils.SkyFit2 {dirname} -c {dirname}/.config\n') - _ = subprocess.run(['powershell.exe', tmpscr]) - frs = glob.glob(os.path.join(dirname, 'FR*.bin')) - if len(frs) > 0: - if tkMessageBox.askyesno("Rerun", f'{len(frs)} FR files detected - rerun?'): - for fr in frs: - with open(tmpscr, 'w') as outf: - outf.write(f'cd {self.rms_loc}\nconda activate {self.rms_env}\npython -m Utils.SkyFit2 {fr} -c {dirname}/.config\n') - _ = subprocess.run(['powershell.exe', tmpscr]) - try: - os.remove(tmpscr) - except: - pass - os.makedirs(os.path.join(self.dir_path,'ecsvs'), exist_ok=True) - ecsvfs = glob.glob1(dirname, '*.ecsv') - for ecsv in ecsvfs: - shutil.copyfile(os.path.join(dirname, ecsv), os.path.join(self.dir_path, 'ecsvs', ecsv)) - - return - - def ignoreCamera(self): - current_image = self.listbox.get(ACTIVE) - if current_image == '': - return - camid = current_image[3:9] - log.info(f'selected camera is {camid}') - dirname = os.path.join(self.dir_path, camid) - allecsvs = glob.glob(os.path.join(dirname, '*.ecsv')) - ecsvs = [e for e in allecsvs if 'REJECT' not in e.upper()] - rejs = [e for e in allecsvs if 'REJECT' in e.upper()] - if len(ecsvs) == 0 and len(rejs) == 0: - log.info('no files to reject or include') - return - elif len(ecsvs) > 0 and len(rejs) == 0: - for ecsv in ecsvs: - os.rename(ecsv, ecsv.replace('.ecsv','_REJECT.ecsv')) - jpgnames = glob.glob(os.path.join(self.dir_path, 'jpgs', current_image)) - for jpgname in jpgnames: - os.rename(jpgname, jpgname.replace('.jpg','_REJECT.jpg')) - elif len(ecsvs) == 0 and len(rejs) > 0: - for rej in rejs: - os.rename(rej, rej.replace('_REJECT.ecsv','.ecsv')) - jpgnames = glob.glob(os.path.join(self.dir_path, 'jpgs', current_image)) - for jpgname in jpgnames: - os.rename(jpgname, jpgname.replace('_REJECT.jpg','.jpg')) - else: - # more than one ECSV or REJ file to handle, urk - log.info('urk') - return - bin_list = [line for line in os.listdir(os.path.join(self.dir_path, 'jpgs')) if self.correct_datafile_name(line)] - #print(bin_list) - for b in bin_list: - self.selected[b] = (0, '') - self.update_listbox(bin_list) - return - - def solveOrbit(self): - log.info('Using ECSV files:') - ecsv_names = [] - ecsv_loc = os.path.join(self.dir_path,'ecsvs') - os.makedirs(ecsv_loc, exist_ok=True) - for entry in sorted(os.walk(self.dir_path), key=lambda x: x[0]): - dir_name, _, file_names = entry - if 'ecsvs' in dir_name: - continue - for fn in file_names: - if fn.lower().endswith(".ecsv"): - shutil.copyfile(os.path.join(dir_name, fn), os.path.join(ecsv_loc, fn)) - if fn.lower().endswith(".ecsv") and 'REJECT' not in dir_name.upper() and 'REJECT' not in fn.upper(): - # Add ECSV file, but skip duplicates - if fn not in ecsv_names: - ecsv_names.append(fn) - log.info(fn) - if len(ecsv_names) < 2: - tkMessageBox.showinfo('Warning', 'Need at least two ECSV files') - return - - tmpscr = os.path.join(os.getenv('TEMP'), 'solve.ps1') - with open(tmpscr, 'w') as outf: - mcruns = 20 - outf.write(f'cd {self.wmpl_loc}\nconda activate {self.wmpl_env}\npython -m wmpl.Formats.ECSV {ecsv_loc} -l -x -r {mcruns} -w -t 15\n') - _ = subprocess.run(['powershell.exe', tmpscr]) - fldrs = os.listdir(ecsv_loc) - print(fldrs) - fldrs = [f for f in fldrs if os.path.isdir(os.path.join(ecsv_loc, f))] - if len(fldrs) > 0: - log.info(f'solved into {fldrs[0]}') - self.soln_outputdir = os.path.join(self.dir_path, fldrs[0]) - if os.path.isdir(self.soln_outputdir): - shutil.rmtree(self.soln_outputdir) - log.info(f'moving {os.path.join(ecsv_loc, fldrs[0])} to {self.dir_path}') - shutil.move(os.path.join(ecsv_loc, fldrs[0]), self.dir_path) - tkMessageBox.showinfo('Info', 'Solver Finished') - return - - def viewSolution(self): - self.review_stack = False - if not self.soln_outputdir: - solndir = glob.glob1(self.dir_path, os.path.split(self.dir_path)[1][:8]+'*') - solndir = [f for f in solndir if os.path.isdir(os.path.join(self.dir_path, f))] - if len(solndir) == 0: - tkMessageBox.showinfo('Warning', 'No solution to review') - return - solndir = os.path.join(self.dir_path, solndir[0]) - self.soln_outputdir = solndir - bin_list = [line for line in os.listdir(self.soln_outputdir) if self.correct_datafile_name(line)] - for b in bin_list: - self.selected[b] = (0, '') - self.update_listbox(bin_list) - return - - def removeSolution(self): - if not self.soln_outputdir: - solndir = glob.glob1(self.dir_path, os.path.split(self.dir_path)[1][:8]+'*') - if len(solndir) == 0: - tkMessageBox.showinfo('Warning', 'No solution to remove') - return - solndir = os.path.join(self.dir_path, solndir[0]) - self.soln_outputdir = solndir - if not tkMessageBox.askyesno("Delete file", f"delete {self.soln_outputdir}?"): - return - shutil.rmtree(os.path.join(self.dir_path, self.soln_outputdir)) - return - - def uploadOrbit(self): - return uploadOrbitGeneric(self.dir_path, self.api_key) - - def uploadRaw(self): - zfname = os.path.join(os.getenv('TMP'), os.path.basename(self.dir_path)) - log.info(f'zfname is {zfname}') - shutil.make_archive(zfname,'zip',self.dir_path) - try: - targname = os.path.join(self.share_loc, os.path.basename(zfname)+'.zip') - log.info(f'targname is {targname}') - shutil.copyfile(zfname+'.zip', targname) - tkMessageBox.showinfo('Info', 'Raw Data Uploaded to Dropbox') - subprocess.Popen(f'explorer "{self.share_loc}"') - except Exception: - tkMessageBox.showinfo('Warning', 'Problem with upload') - return - - def viewData(self): - self.review_stack = False - self.soln_outputdir = None - print(self.dir_path) - bin_list = self.get_bin_list() - for b in bin_list: - self.selected[b] = (0, '') - self.update_listbox(bin_list) - return - - def getRequestedECSVs(self): - notgotlist=[] - img_list = self.get_bin_list() - for current_image in img_list: - if not self.getOneEcsv(current_image): - notgotlist.append(current_image) - if len(notgotlist) > 0: - tkMessageBox.showinfo('Info', f'No ECSVs for {notgotlist}') - return - - def getOneEcsv(self, current_image): - if current_image[:1] == 'M': - datestr = current_image[1:16] - statid = current_image[-11:-5] - elif current_image[:2] == 'FF': - statid = current_image[3:9] - datestr = current_image[10:29] - else: - return - #dtval = datetime.datetime.strptime(datestr, '%Y%m%d_%H%M%S') - #datestr = dtval.strftime('%Y-%m-%dT%H:%M:%S') - try: - lis = getECSVs(statid, datestr, savefiles=True, outdir=os.path.join(self.dir_path, statid)) - for li in lis: - if 'issue getting data' in li: - return False - os.makedirs(os.path.join(self.dir_path,'ecsvs'), exist_ok=True) - ecsvfs = glob.glob1(os.path.join(self.dir_path, statid), '*.ecsv') - for ecsv in ecsvfs: - shutil.copyfile(os.path.join(self.dir_path, statid, ecsv), os.path.join(self.dir_path, 'ecsvs', ecsv)) - - shutil.copyfile() - ## finish here - return True - except Exception: - return False - - def get_bin_list(self): - """ Get a list of image files in a given directory. - """ - if self.dir_path is None: - dirname = tkFileDialog.askdirectory(parent=root,initialdir=self.fb_dir, - title='Please select a directory') - if not dirname: - return - _, thispatt = os.path.split(dirname) - self.dir_path = dirname - self.patt = thispatt - self.newpatt.set(self.patt) - - if self.review_stack is False: - targdir = os.path.join(self.dir_path, 'jpgs') - else: - targdir = os.path.join(self.dir_path, 'stacks') - if os.path.isdir(targdir): - bin_list = [line for line in os.listdir(targdir) if self.correct_datafile_name(line)] - else: - log.info('no jpgs available') - bin_list = [] - return bin_list - - def update_listbox(self, bin_list): - """ Updates the listbox with the current entries. - """ - self.listbox.delete(0, END) - for line in sorted(bin_list): - self.listbox.insert(END, line) - if line not in self.selected: - self.selected[line] = (0, '') - if self.selected[line][0] == 1: - self.listbox.itemconfig(END, fg = 'green') - - def checkStacks(self): - self.review_stack = True - bin_list = self.get_bin_list() - if len(bin_list) > 0: - for b in bin_list: - self.selected[b] = (0, '') - self.update_listbox(bin_list) - else: - tkMessageBox.showinfo('Warning', 'No stacks to review') - - def loadFolder(self): - self.review_stack = False - self.dir_path = None - self.soln_outputdir = None - bin_list = self.get_bin_list() - for b in bin_list: - self.selected[b] = (0, '') - self.update_listbox(bin_list) - - def openFolder(self): - dir_path = self.dir_path.replace("/","\\") - os.system(f'explorer.exe {dir_path}') - - def viewLogs(self): - logdir = os.path.join(os.getenv('TMP'), 'fbcollector') - os.system(f'explorer.exe {logdir}') - - def archiveFolder(self): - noimgdata = img.open(noimg_file).resize((640,360)) - noimage = ImageTk.PhotoImage(noimgdata) - self.imagelabel.configure(image = noimage) - self.imagelabel.image = noimage - if self.dir_path == self.fb_dir.strip(): - self.loadFolder() - if self.dir_path is not None and self.dir_path != self.fb_dir: - try: - _, fldr = os.path.split(os.path.normpath(self.dir_path)) - yr = fldr[:4] - archdir = os.path.join(self.fb_dir, yr) - os.makedirs(archdir, exist_ok=True) - zfname = os.path.join(archdir, fldr) - shutil.make_archive(zfname,'zip',self.fb_dir, fldr) - except Exception as e: - log.warning(f'unable to create archive {self.dir_path}') - log.warning(e) - try: - shutil.rmtree(self.dir_path) - except Exception as e: - log.warning(f'unable to remove folder, please do it manually {self.dir_path}') - log.warning(e) - self.dir_path = self.fb_dir - self.dir_path = self.fb_dir - self.viewData() - - def delFolder(self): - noimgdata = img.open(noimg_file).resize((640,360)) - noimage = ImageTk.PhotoImage(noimgdata) - self.imagelabel.configure(image = noimage) - self.imagelabel.image = noimage - if self.dir_path is not None and self.dir_path != self.fb_dir: - try: - shutil.rmtree(self.dir_path) - self.dir_path = self.fb_dir - except Exception as e: - log.warning(f'unable to remove {self.dir_path}') - log.warning(e) - - def correct_datafile_name(self, line): - if ('.jpg' in line or '.png' in line) and 'noimage' not in line: - return True - return False - - def removeRelated(self, imgname): - camid = imgname[:6] - mp4s = os.listdir(os.path.join(self.dir_path, 'mp4s')) - badones = [x for x in mp4s if camid in x] - for ba in badones: - os.remove(os.path.join(self.dir_path, 'mp4s', ba)) - jpgs = os.listdir(os.path.join(self.dir_path, 'jpgs')) - badones = [x for x in jpgs if camid in x] - for ba in badones: - os.remove(os.path.join(self.dir_path, 'jpgs', ba)) - fldrs = os.listdir(self.dir_path) - badones = [x for x in fldrs if camid in x] - for ba in badones: - try: - shutil.rmtree(os.path.join(self.dir_path, ba)) - except Exception: - os.remove(os.path.join(self.dir_path, ba)) - os.remove(os.path.join(self.dir_path, 'stacks', imgname)) - - def remove_image(self): - """ Remove the selected image from disk - """ - current_image = self.listbox.get(ACTIVE) - curr_row = self.listbox.curselection() - if current_image == '': - return - if not tkMessageBox.askyesno("Delete file", f"delete {current_image}?"): - return - log.info(f'removing {current_image}') - if self.review_stack: - self.removeRelated(current_image) - else: - try: - os.remove(os.path.join(self.dir_path, 'jpgs', current_image)) - except Exception: - pass - xmlf = current_image.replace('P.jpg', '.xml') - try: - os.remove(os.path.join(self.dir_path, 'jpgs', xmlf)) - except Exception: - pass - self.selected[current_image] = (0,'') - self.update_listbox(self.get_bin_list()) - maxrows = self.listbox.index("end") - curr_row = (max(min(maxrows-1, curr_row[0]),0),) - self.listbox.select_set(curr_row) - self.listbox.activate(curr_row) - self.update_image('x') - - def update_image(self, thing): - """ When selected, load a new image - """ - try: - # Check if the list is empty. If it is, do nothing. - if self.review_stack: - self.current_image = os.path.join(self.dir_path, 'stacks', self.listbox.get(self.listbox.curselection()[0])) - elif self.soln_outputdir: - self.current_image = os.path.join(self.soln_outputdir, self.listbox.get(self.listbox.curselection()[0])) - else: - self.current_image = os.path.join(self.dir_path, 'jpgs', self.listbox.get(self.listbox.curselection()[0])) - except: - return 0 - - with img.open(self.current_image).resize((640,360)) as imgdata: - thisimage = ImageTk.PhotoImage(imgdata) - self.imagelabel.configure(image = thisimage) - self.imagelabel.image = thisimage - - self.timestamp_label.configure(text = os.path.split(self.current_image)[1]) - return - - def clean_folder(self): - stacklist = os.listdir(os.path.join(self.dir_path, 'stacks')) - camlist = [x[:6] for x in stacklist if 'stack.jpg' in x] - datalist = os.listdir(self.dir_path) - datalist = [x for x in datalist if 'jpgs' not in x and 'mp4s' not in x and 'stacks' not in x] - for d in datalist: - keep = False - for c in camlist: - if c in d: - keep = True - if keep is False: - try: - os.remove(d) - except Exception: - pass - return - - def getData(self): - thispatt = self.newpatt.get().strip() - self.patt = thispatt.ljust(15,'0') - self.dir_path = os.path.join(self.fb_dir, self.newpatt.get().strip()) - log.info(f'getting data matching {thispatt}') - os.makedirs(os.path.join(self.dir_path, 'jpgs'), exist_ok=True) - reqdate = datetime.datetime.strptime(self.patt, '%Y%m%d_%H%M%S') - reqdate = reqdate + datetime.timedelta(seconds=-30) - getLiveJpgs(reqdate.strftime('%Y%m%d_%H%M%S'), outdir=os.path.join(self.dir_path, 'jpgs')) - self.renameImages(self.dir_path) - self.update_listbox(self.get_bin_list()) - - def getTrajpickle(self): - if self.dir_path: - basepatt = os.path.split(self.dir_path)[1] - else: - basepatt = '' - fullpatt = askstring('Trajectory Name', 'eg 20240101_010203.345_UK', initialvalue=basepatt) - if not fullpatt: - return - ymd = fullpatt[:8] - trajpick = f'{fullpatt[:15]}_trajectory.pickle' - url = f'https://archive.ukmeteors.co.uk/reports/{ymd[:4]}/orbits/{ymd[:6]}/{ymd}/{fullpatt}/{trajpick}' - log.info(f'{url}') - if self.patt is None: - self.dir_path = os.path.join(self.fb_dir, fullpatt[:15]) - else: - self.dir_path = os.path.join(self.fb_dir, self.patt) - log.info(self.dir_path) - os.makedirs(self.dir_path, exist_ok=True) - get_response = requests.get(url, stream=True) - if get_response.status_code == 200: - log.info(f'retrieved {trajpick}') - with open(os.path.join(self.dir_path, trajpick), 'wb') as f: - for chunk in get_response.iter_content(chunk_size=4096): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - if self.patt is None: - self.patt = fullpatt[:15] - self.newpatt.set(self.patt) - else: - log.info(f'unable to retrieve {trajpick}, {get_response.status_code}') - return - - def delOrbit(self): - orbit = None - orbit = askstring('Trajectory Name', 'eg 20240101_010203.345_UK', initialvalue='') - if not orbit: - return - tmpfname = os.path.join(self.fb_dir, f'{orbit[:15]}.delete') - open(tmpfname, 'w').write(orbit) - if self.api_key: - headers = {'Content-type': 'text/plain', 'Slug': orbit[:15], 'apikey': self.api_key} - url = f'https://api.ukmeteors.co.uk/fireballfiles?orbitfile={orbit[:15]}.delete' - r = requests.put(url, data=open(tmpfname, 'r'), headers=headers) - if r.status_code != 200: - tkMessageBox.showinfo('Warning', f'Problem with request, {r.status_code}') - else: - tkMessageBox.showinfo("Info", f'deleted {orbit}') - return - else: - tkMessageBox.showinfo('Info', "Can't delete without api key") - os.remove(tmpfname) - return - - def addImageVideo(self): - orbit = None - dirname = tkFileDialog.askdirectory(parent=root,initialdir=self.fb_dir, - title='Please select a directory') - if not dirname: - return - _, orbit = os.path.split(dirname) - ymd = orbit[:8] - pickname = f'{orbit[:15]}_trajectory.pickle' - os.makedirs(os.path.join(dirname, 'jpgs'), exist_ok=True) - os.makedirs(os.path.join(dirname, 'mp4s'), exist_ok=True) - for ext in ['jpg','mp4']: - files = glob.glob(f'{dirname}/*.{ext}') - for fil in files: - _, barename = os.path.split(fil) - shutil.move(fil, os.path.join(dirname,f'{ext}s',barename)) - url = f'https://archive.ukmeteors.co.uk/reports/{ymd[:4]}/orbits/{ymd[:6]}/{ymd[:8]}/{orbit}/{pickname}' - count = 0 - log.info(url) - get_response = requests.get(url, stream=True) - if get_response.status_code == 200: - log.info(f'retrieved {pickname}') - count += 1 - with open(os.path.join(dirname, pickname), 'wb') as f: - for chunk in get_response.iter_content(chunk_size=4096): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - uploadOrbitGeneric(dirname, self.api_key) - tkMessageBox.showinfo("Info", f'Added data to {orbit}') - return - - def getVids(self): - jpglist = glob.glob1(os.path.join(self.dir_path,'jpgs'), 'FF*.jpg') - os.makedirs(os.path.join(self.dir_path, 'mp4s'), exist_ok=True) - count = 0 - for jpg in jpglist: - mp4 = jpg.replace('.jpg','.mp4') - ym = mp4[10:16] - url = f'https://archive.ukmeteors.co.uk/img/mp4/{ym[:4]}/{ym}/{mp4}' - get_response = requests.get(url, stream=True) - if get_response.status_code == 200: - log.info(f'retrieved {mp4}') - count += 1 - with open(os.path.join(self.dir_path, 'mp4s', mp4), 'wb') as f: - for chunk in get_response.iter_content(chunk_size=4096): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - log.info(f'retrieved {count} videos') - tkMessageBox.showinfo("Info", f'Retrieved {count} videos') - return - - def renameImages(self, dir_path): - xmllist = glob.glob(os.path.join(dir_path,'jpgs', 'M*.xml')) - for xmlf in xmllist: - jpgf = xmlf.replace('.xml','P.jpg') - if os.path.isfile(jpgf): - xmld = xmltodict.parse(open(xmlf).read()) - realfname = xmld['ufocapture_record']['@cap'].replace('.fits','.jpg') - if not os.path.isfile(os.path.join(dir_path, 'jpgs', realfname)): - os.rename(jpgf, os.path.join(dir_path, 'jpgs', realfname)) - else: - os.remove(jpgf) - os.remove(xmlf) - return - - def viewWatchlist(self): - evtfile = os.path.join(self.fb_dir,'event_watchlist.txt') - if platform.system() == 'Darwin': # macOS - procid = subprocess.Popen(('open', evtfile)) - elif platform.system() == 'Windows': # Windows - procid = subprocess.Popen(('cmd','/c',evtfile)) - else: # linux variants - procid = subprocess.Popen(('xdg-open', evtfile)) - procid.wait() - if not tkMessageBox.askyesno("Upload File", "Upload event watchlist?"): - return - else: - self.putWatchlist() - - def getWatchlist(self): - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key)) - c = paramiko.SSHClient() - server=self.gmn_server - user=self.gmn_user - log.info(f'trying {user}@{server} with {self.gmn_key}') - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - c.connect(hostname = server, username = user, pkey = k) - scpcli = SCPClient(c.get_transport()) - log.info('getting Watchlist') - scpcli.get('./event_watchlist.txt', self.fb_dir) - self.viewWatchlist() - - def putWatchlist(self): - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key)) - c = paramiko.SSHClient() - server=self.gmn_server - user=self.gmn_user - log.info(f'trying {user}@{server} with {self.gmn_key}') - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - c.connect(hostname = server, username = user, pkey = k) - scpcli = SCPClient(c.get_transport()) - log.info('Uploading watchlist') - evtfile = os.path.join(self.fb_dir,'event_watchlist.txt') - scpcli.put(evtfile, 'event_watchlist.txt') - self.evtMonTriggered = datetime.datetime.now() + datetime.timedelta(minutes=5) - return - - def getEventData(self): - evtdate = self.newpatt.get().strip() - if len(evtdate) < 15: - tkMessageBox.showinfo("Warning", f'Need seconds in the event date field {evtdate}') - return - fbdir = self.fb_dir - if ':' in fbdir: - drv = fbdir[0].lower() - fbdir = '/mnt/' + drv + fbdir[2:] - fbdir = fbdir.replace('\\','/') - - cmd = os.path.join(self.script_loc, 'download_events.sh') + f' {evtdate} {fbdir} 1' - if ':' in cmd: - drv = cmd[0].lower() - cmd = '/mnt/' + drv + cmd[2:] - cmd = cmd.replace('\\','/') - log.info(f'executing {cmd}') - if self.evtMonTriggered is None: - ret = tkMessageBox.askyesno("Warning", 'Event Monitor has not been triggered, continue?') - if ret is False: - return - elif datetime.datetime.now() < self.evtMonTriggered: - ret = tkMessageBox.askyesno("Wait", f'Should wait till {self.evtMonTriggered.strftime("%H:%M:%S")} - continue anyway?') - if ret is False: - return - log.info(f'getting data for {evtdate}') - procid = subprocess.Popen(('bash','-c', cmd)) - procid.wait() - tkMessageBox.showinfo("Info", 'Done') - return - - def getGMNData(self): - print(self.dir_path) - camlist = [line for line in os.listdir(os.path.join(self.dir_path,'jpgs')) if self.correct_datafile_name(line)] - dts=[] - camids=[] - for cam in camlist: - camid,_ = os.path.splitext(cam) - spls = camid.split('_') - if camid[:2] == 'FF': - camids.append(spls[1]) - dts.append(camid[10:25]) - else: - camids.append(spls[-1][:6]) - dts.append(camid[1:16]) - dts.sort() - dtstr = f'{dts[0]}.000000' - stationlist = ','.join(map(str, camids)) - server=self.gmn_server - user=self.gmn_user - log.info('getting data from GMN') - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key)) - c = paramiko.SSHClient() - log.info(f'trying {user}@{server} with {self.gmn_key}') - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - c.connect(hostname = server, username = user, pkey = k) - command = f'source ~/anaconda3/etc/profile.d/conda.sh && conda activate wmpl && python scripts/extract_fireball.py {dtstr} {stationlist}' - log.info(f'running {command}') - - _, stdout, stderr = c.exec_command(command, timeout=900) - for line in iter(stdout.readline, ""): - log.info(line) - for line in iter(stderr.readline, ""): - print(line, end="") - scpcli = SCPClient(c.get_transport()) - log.info('done, collecting output') - indir = os.path.join(f'event_extract/{dtstr}/') - scpcli.get(indir, self.dir_path, recursive=True) - command = f'rm -Rf event_extract/{dtstr}' - log.info(f'running {command}') - _, stdout, stderr = c.exec_command(command, timeout=120) - for line in iter(stdout.readline, ""): - log.info(line) - for line in iter(stderr.readline, ""): - log.info(line) - dirs = os.listdir(os.path.join(self.dir_path, dtstr)) - for d in dirs: - srcdir = os.path.join(self.dir_path, dtstr, d) - targ = os.path.join(self.dir_path, d) - os.makedirs(targ, exist_ok=True) - for f in os.listdir(srcdir): - shutil.copy(os.path.join(srcdir, f), targ) - try: - shutil.rmtree(os.path.join(self.dir_path, dtstr)) - except Exception: - pass - tkMessageBox.showinfo("Data Collected", 'data collected from GMN') - self.update_listbox(self.get_bin_list()) - return - - -def uploadOrbitGeneric(orbdir, api_key): - pickles=[] - for path, _, files in os.walk(orbdir): - for name in files: - if '.pickle' in name and '_mc_' not in name and 'tmpzip' not in path: - log.info(f'adding pickle {name}') - pickles.append(os.path.join(path, name)) - if name.lower().endswith(".ecsv") and 'ecsvs' not in path: - log.info(f'copying {name}') - shutil.copyfile(os.path.join(path, name), os.path.join(orbdir, 'ecsvs', name)) - - if len(pickles) == 0: - return - pickles = list(set(pickles)) - if len(pickles) == 1: - pickfile = pickles[0] - else: - pickfile = tkFileDialog.askopenfilename(title='Select Orbit Pickle', defaultextension='*.pickle', - initialdir=orbdir, initialfile='*.pickle', - filetypes=[('pickles','*.pickle')]) - if not pickfile: - return - orbname = os.path.split(pickfile)[1] - tmpdir = os.path.join(orbdir, 'tmpzip') - os.makedirs(tmpdir, exist_ok=True) - shutil.copyfile(pickfile, os.path.join(tmpdir, orbname)) - if os.path.isdir(os.path.join(orbdir, 'jpgs')): - shutil.copytree(os.path.join(orbdir, 'jpgs'), os.path.join(tmpdir, 'jpgs'), dirs_exist_ok=True) - if os.path.isdir(os.path.join(orbdir, 'mp4s')): - shutil.copytree(os.path.join(orbdir, 'mp4s'), os.path.join(tmpdir, 'mp4s'), dirs_exist_ok=True) - if os.path.isdir(os.path.join(orbdir, 'ecsvs')): - shutil.copytree(os.path.join(orbdir, 'ecsvs'), os.path.join(tmpdir, 'ecsvs'), dirs_exist_ok=True) - for path, _, files in os.walk(orbdir): - for name in files: - if '_dyn_mass_fit' in name: - im = Image.open(os.path.join(path, name)).convert("RGB") - im.save(os.path.join(tmpdir,'jpgs', name[:-4] + '.jpg')) - zfname = os.path.join(orbdir, orbname[:15]) - shutil.make_archive(zfname,'zip',tmpdir) - try: - shutil.rmtree(tmpdir) - except Exception: - pass - - if api_key: - headers = {'Content-type': 'application/zip', 'Slug': orbname[:15], 'apikey': api_key} - url = f'https://api.ukmeteors.co.uk/fireballfiles?orbitfile={orbname[:15]}.zip' - r = requests.put(url, data=open(zfname+'.zip', 'rb'), headers=headers) #, auth=('username', 'pass')) - #print(r.text) - if r.status_code != 200: - tkMessageBox.showinfo('Warning', f'Problem with upload, {r.status_code}') - else: - tkMessageBox.showinfo('Info', 'Orbit Uploaded') - return - else: - tkMessageBox.showinfo('Info', 'Zip File created') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("-d", "--datepatt", type=str, help="date pattern to retrieve") - args = parser.parse_args() - - dir_ = os.getcwd() - config_file = os.path.join(dir_, 'config.ini') - if not os.path.isfile(config_file): - shutil.copyfile(os.path.join(dir_, 'config.ini.sample'), config_file) - tkMessageBox.showinfo("Config Missing", 'Please configure before using') - showConfig() - - noimg_file = os.path.join(dir_, 'noimage.jpg') - - log = logging.getLogger(__name__) - log.setLevel(logging.INFO) - - logdir = os.path.join(os.getenv('TMP'), 'fbcollector') - os.makedirs(logdir, exist_ok=True) - log_file = os.path.join(logdir, log_timestamp() + '.log') - handler = logging.handlers.TimedRotatingFileHandler(log_file, when='D', interval=1) # Log to a different file each day - handler.setLevel(logging.INFO) - - formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s-%(module)s-line:%(lineno)d - %(message)s', datefmt='%Y/%m/%d %H:%M:%S') - handler.setFormatter(formatter) - log.addHandler(handler) - - ch = logging.StreamHandler(sys.stdout) - ch.setLevel(logging.DEBUG) - formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s: %(message)s', datefmt='%Y/%m/%d %H:%M:%S') - ch.setFormatter(formatter) - log.addHandler(ch) - - # Log program start - log.info("Program start") - log.info(f'config file is {config_file}') - - - # Initialize main window - root = tk.Tk() - root.geometry('+0+0') - targdir = args.datepatt - log.info(f'patt is {targdir}') - - app = fbCollector(root, patt=targdir) - root.iconbitmap(os.path.join(dir_,'ukmda.ico')) - root.protocol('WM_DELETE_WINDOW', app.quitApplication) - - root.mainloop() diff --git a/fbCollector/fireballcollector_setup.iss b/fbCollector/fireballcollector_setup.iss deleted file mode 100644 index d9cafc188..000000000 --- a/fbCollector/fireballcollector_setup.iss +++ /dev/null @@ -1,31 +0,0 @@ -#define MyAppName "UKMON Fireball Analyser" - -[Setup] -AppName=UKMON Fireball Analyser -AppVersion="1.0.0" -AppPublisher=UK Meteor Network -AppPublisherURL=https://www.ukmeteornetwork.org -DefaultDirName={localappdata}\programs\fbAnalyser -DefaultGroupName=UKMON -UninstallDisplayIcon={app}\fireballCollector.exe -Compression=lzma2 -SolidCompression=yes -OutputDir=e:\temp -OutputBaseFilename=setup_fireballCollector -PrivilegesRequired=lowest - -[Files] -Source: ".\dist\fireballCollector.exe"; DestDir: "{app}" -Source: ".\noimage.jpg"; DestDir: "{app}" -Source: ".\ukmda.ico"; DestDir: "{app}" -Source: ".\download_events.sh"; DestDir: "{app}" -Source: ".\config.ini.sample"; DestDir: "{app}"; Permissions: users-modify -Source: ".\README.html"; DestDir: "{app}" - -[Tasks] -Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; - -[Icons] -Name: "{group}\Fireball Analyser"; Filename: "{app}\fireballCollector.exe"; IconFilename: "{app}\ukmda.ico" -Name: "{group}\{cm:UninstallProgram,{#MyAppName}}"; Filename: "{uninstallexe}" -Name: "{userdesktop}\{#MyAppName}"; Filename: "{app}\fireballCollector.exe"; Tasks: desktopicon; WorkingDir: {app}; IconFilename: "{app}\ukmda.ico" diff --git a/fbCollector/noimage.jpg b/fbCollector/noimage.jpg deleted file mode 100644 index 80e1f661b..000000000 Binary files a/fbCollector/noimage.jpg and /dev/null differ diff --git a/fbCollector/requirements.txt b/fbCollector/requirements.txt deleted file mode 100644 index 124471433..000000000 --- a/fbCollector/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -# requirements for the fbCollector tool -# Copyright (C) 2018-2023 Mark McIntyre -Pillow -paramiko -scp -pyqtgraph -xmltodict -requests -pandas -markdown2 \ No newline at end of file diff --git a/fbCollector/ukmda.ico b/fbCollector/ukmda.ico deleted file mode 100644 index e788188d8..000000000 Binary files a/fbCollector/ukmda.ico and /dev/null differ diff --git a/fbCollector/vars.yml b/fbCollector/vars.yml deleted file mode 100644 index e7e28879c..000000000 --- a/fbCollector/vars.yml +++ /dev/null @@ -1,2 +0,0 @@ - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/fbCollector" - destdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/meteors/fbCollector" diff --git a/usermgmt/README.md b/usermgmt/README.md deleted file mode 100644 index d966a3530..000000000 --- a/usermgmt/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# UKMDA User Management - -This Windows application is used to add/maintain new cameras to the network. Its use is restricted to members of the UKMON -admin team. - -## Basic Principles -A camera consists of an RMS ID, location and pointing direction. - -Each _location_ is assigned a unique *AWS* user with credentials that grant suitable permissions to the S3 storage. For example, a contributor in Toytown might be assigned a unique AWS user "Toytown". - -AWS credentials are not shared between locations. If a location's credentials are compromised, they can be disabled without impacting the rest of the network and new credentials can then be issued. The *ukmon-pitools* toolset is designed so that new credentials will be picked up the next time *refreshTools.sh* is run on the impacted Pi (either at reboot or manually). - -Each _camera_ is allocated a unique *Unix* ID. The public ssh key provided by the operator is used to enable *ukmon-pitools* to login and collect configuration information for the camera. For example if the contributor at Toytown had two cameras facing North and South, the cameras might be named "toytown_s" and "toytown_n". Note that camera names are in lower case, must be unique and that SSH keys should not be shared between cameras. If the operator chooses to share keys they do so at their own risk and it means that if the key is compromised, all their cameras will be unable to connect. - - -# User Management tool -Unix and AWS User creation and configuration are managed by *stationMaint.exe* which is a compiled python programme that uses native AWS and Unix libraries to execute the required commands. - -## Setup -Reminder: To use the app you will have to be permissioned on our servers by one of the Admin team. - -* Download the latest zip file from [here](https://github.com/ukmda/ukmda-dataprocessing/releases) and expand it to a location of your choosing on your computer, for example `%userprofile%\ukmon\usermgmt` -* Create a desktop shortcut pointing to `dist\stationmaint2.exe` in this folder. - -When you first run the app you will be prompted to provide the credentials we gave you to connect to our server. - -### Linux or MacOS -In principle the code should work on Linux-like OSes. You'd need to copy the python file, icon file, sample ini file, requirements file and this README to a folder of your choosing then create a python virtual environment, activate it and install the requirements. - -## Adding a new camera -To add a new camera the camera operator must supply the following: -* RMS ID (from the RMS .config file) -* town or village (eg Toytown) -* approx pointing direction (eg SW) -* the SSH public key generated by *ukmon-pitools*. -* Human name and email address of the operator. - -RMS ID, pointing direction and SSH key are unique to each camera. Other values will generally be shared -by other cameras at the same location. - -Once the information has been gathered, select Camera/Add and fill in the boxes. Note that the boxes are part-prepopulated with values from whatever row your cursor was on, so that if you're adding another camera at an existing location, you can save a bit of typing. - -## Amending a camera -To change location or pointing direction follow the process to move a camera. -All other values can be amended by selecting the line and updating the values. - -## Moving a camera -IMPORTANT NOTE: the camera owner should NOT make any changes - the changes will automatically flow down to their station. - -To move a camera to a new location, select the row containing the camera then select Camera/Move. After you fill in any new information, a new unix and AWS user will be created and the configuration files for the old user will be updated. The old camera must then be marked disabled as explained next. - -## Disabling and Re-enabling a camera -To disable a camera, change the Active column from 1 to the last active date in YYYYMMDD format eg 20220715. This removes the camera from current reporting and disables the unix user, but retains the details for any historical reporting. A camera can be reactivated by setting the Active column back to 1. - -## Disabling a location -Disable all cameras at that location, and then update the AWS key. We never remove a location as there would be a risk of losing historic data. - -## Other Functionality -The tool also provides some other functionality: -* Owners menu - search for station details and owner details. -* Camera menu - * Check camera - this does a quick sanity test to see when the camera last connected to our server. - * Download the current plateper for a camera and upload a new one. The new plate will be automaticlly installed on the camera the following morning. - * Update the SSH key - if a user sends a new SSH key, we need to add it to the server. - * Update the AWS key - force an update of the location's AWS key. Note that keys are autorolled every 60 days so this would only be needed if the key had been compromised. - -# Copyright -All code Copyright (C) 2018-2023 Mark McIntyre \ No newline at end of file diff --git a/usermgmt/deploy-local.yml b/usermgmt/deploy-local.yml deleted file mode 100644 index 76677f582..000000000 --- a/usermgmt/deploy-local.yml +++ /dev/null @@ -1,76 +0,0 @@ ---- -- hosts: ukmonhelper2 - gather_facts: no - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{windestdir}} exists - file: path={{windestdir}} state=directory - delegate_to: localhost - tags: [dev,prod] - - name: Ensures {{windestdir}}/caminfo exists - file: path={{windestdir}}/caminfo state=directory - delegate_to: localhost - tags: [dev,prod] - - - name: build exe - ansible.builtin.shell: | - conda activate ukmon-admin - cd windows - pyinstaller ./stationMaint2.py --noconsole --onefile --windowed --icon .\camera.ico - args: - executable: pwsh.exe - delegate_to: localhost - register: outdata - tags: [dev,prod] - - - name: print debug - debug: - var: outdata.stdout - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - delegate_to: localhost - tags: [dev,prod] - with_items: - - {src: '{{winsrcdir}}/dist/StationMaint2.exe', dest: '{{windestdir}}/', mode: '754', backup: yes, directory_mode: no } - - {src: '{{winsrcdir}}/camera.ico', dest: '{{windestdir}}/', mode: '644', backup: yes, directory_mode: no } - - {src: '{{winsrcdir}}/stationmaint.cfg', dest: '{{windestdir}}/', mode: '644', backup: yes, directory_mode: no } - - name: update source bucket - lineinfile: - path: '{{windestdir}}/stationmaint.cfg' - regexp: 'SRCBUCKET=' - line: 'SRCBUCKET={{srcbucket}}' - delegate_to: localhost - tags: [dev,prod] - - name: update live bucket - lineinfile: - path: '{{windestdir}}/stationmaint.cfg' - regexp: 'LIVEBUCKET=' - line: 'LIVEBUCKET={{livebucket}}' - delegate_to: localhost - tags: [dev,prod] - - name: update web bucket - lineinfile: - path: '{{windestdir}}/stationmaint.cfg' - regexp: 'WEBSITEBUCKET=' - line: 'WEBSITEBUCKET={{websitebucket}}' - delegate_to: localhost - tags: [dev,prod] - - name: update helper ipaddress - lineinfile: - path: '{{windestdir}}/stationmaint.cfg' - regexp: 'HELPERIP=' - line: 'HELPERIP={{serverip}}' - delegate_to: localhost - tags: [dev,prod] - - name: update helper remotedir - lineinfile: - path: '{{windestdir}}/stationmaint.cfg' - regexp: 'REMOTEDIR=' - line: 'REMOTEDIR={{remotedir}}' - delegate_to: localhost - tags: [dev,prod] diff --git a/usermgmt/dev-vars.yml b/usermgmt/dev-vars.yml deleted file mode 100644 index a459a2431..000000000 --- a/usermgmt/dev-vars.yml +++ /dev/null @@ -1,11 +0,0 @@ - winsrcdir: /mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/usermgmt/windows - windestdir: /mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/testing/usermgmt - destdir: /home/ec2-user/dev/keymgmt - srcbucket: mjmm-ukmon-shared - livebucket: mjmm-ukmon-live - websitebucket: mjmm-ukmonarchive.co.uk - liveprofile: default - archprofile: ukmda_admin - helper: ukmonhelper2 - serverip: "3.11.55.160" - remotedir: /home/ec2-user/dev/data \ No newline at end of file diff --git a/usermgmt/prod-vars.yml b/usermgmt/prod-vars.yml deleted file mode 100644 index bd561ec78..000000000 --- a/usermgmt/prod-vars.yml +++ /dev/null @@ -1,9 +0,0 @@ - winsrcdir: /mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/usermgmt/windows - windestdir: /mnt/f/videos/MeteorCam/usermgmt - destdir: /home/ec2-user/keymgmt - srcbucket: ukmda-shared - livebucket: ukmda-live - websitebucket: ukmda-website - helper: ukmonhelper2 - serverip: "3.11.55.160" - remotedir: /home/ec2-user/prod/data \ No newline at end of file diff --git a/usermgmt/server/ukmon.ini b/usermgmt/server/ukmon.ini deleted file mode 100644 index 59c2a1061..000000000 --- a/usermgmt/server/ukmon.ini +++ /dev/null @@ -1,5 +0,0 @@ -# config data for this station -export LOCATION=STATIONLOCATION -export UKMONHELPER=3.11.55.160 -export UKMONKEY=~/.ssh/ukmon -export RMSCFG=~/source/RMS/.config diff --git a/usermgmt/windows/build.ps1 b/usermgmt/windows/build.ps1 deleted file mode 100644 index 325846018..000000000 --- a/usermgmt/windows/build.ps1 +++ /dev/null @@ -1 +0,0 @@ -pyinstaller .\stationMaint2.py --noconsole --onefile --windowed --icon .\camera.ico --clean diff --git a/usermgmt/windows/cam_management_setup.iss b/usermgmt/windows/cam_management_setup.iss deleted file mode 100644 index bb9adfa73..000000000 --- a/usermgmt/windows/cam_management_setup.iss +++ /dev/null @@ -1,29 +0,0 @@ -#define MyAppName "UKMON Camera Management" - -[Setup] -AppName=UKMON Camera Management -AppVersion="2.0.0" -AppPublisher=UK Meteor Network -AppPublisherURL=https://www.ukmeteornetwork.org -DefaultDirName={localappdata}\programs\cameraMgmt -DefaultGroupName=UKMON -UninstallDisplayIcon={app}\stationMaint2.exe -Compression=lzma2 -SolidCompression=yes -OutputDir=e:\temp -OutputBaseFilename=setup_cameraMgmt -PrivilegesRequired=lowest - -[Files] -Source: ".\dist\stationMaint2.exe"; DestDir: "{app}" -Source: ".\camera.ico"; DestDir: "{app}" -Source: ".\stationmaint.ini.sample"; DestDir: "{app}"; Permissions: users-modify -Source: "..\README.md"; DestDir: "{app}"; Permissions: users-modify - -[Tasks] -Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; - -[Icons] -Name: "{group}\Camera Management"; Filename: "{app}\stationMaint2.exe"; IconFilename: "{app}\camera.ico" -Name: "{group}\{cm:UninstallProgram,{#MyAppName}}"; Filename: "{uninstallexe}" -Name: "{userdesktop}\{#MyAppName}"; Filename: "{app}\stationMaint2.exe"; Tasks: desktopicon; WorkingDir: {app}; IconFilename: "{app}\camera.ico" diff --git a/usermgmt/windows/camera.ico b/usermgmt/windows/camera.ico deleted file mode 100644 index 33fcafac1..000000000 Binary files a/usermgmt/windows/camera.ico and /dev/null differ diff --git a/usermgmt/windows/release_notes.md b/usermgmt/windows/release_notes.md deleted file mode 100644 index 979fd1c87..000000000 --- a/usermgmt/windows/release_notes.md +++ /dev/null @@ -1,19 +0,0 @@ -# The UKMON Station Management Tool - -This application is used to add, modify and disable RMS stations linked to UKMON. The app is intended for use only by the UKMON committee members responsible for managing users. - -## Before Installation -First you should contact me, Mark McIntyre so we can set up a userid for you. - -## Installation -Download the zip file and expand it to a location of your choice, for example `c:\programdata\ukmon` -Create an SSH keypair. -Edit the config file and add the userid i provided, plus the full path to the ssh private key. -Save the config file and run the app. - -## Use -The menus should be pretty self-explanatory. - -## Release Notes -2024.04.2 - first release. -2025.10.1 - overhaul to fix a plethora of bugs and add new features. \ No newline at end of file diff --git a/usermgmt/windows/requirements.txt b/usermgmt/windows/requirements.txt deleted file mode 100644 index 3894e9b5f..000000000 --- a/usermgmt/windows/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (C) 2018-2023 Mark McIntyre -boto3 -openpyxl -pandas -scp -tksheet -paramiko -pyinstaller diff --git a/usermgmt/windows/stationMaint.ps1 b/usermgmt/windows/stationMaint.ps1 deleted file mode 100644 index 23b12ae38..000000000 --- a/usermgmt/windows/stationMaint.ps1 +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (C) 2018-2023 Mark McIntyre -push-Location $psscriptroot - -# create conda env if not aleady there -$isadm=(conda env list | select-string "ukmon-admin") -if ($isadm.count -eq 0) { - conda create -n ukmon-admin python=3.11 - pip install -r requirements.txt -} -conda activate ukmon-admin - -bash -c "rsync -avz --delete ukmonhelper2:keymgmt/sshkeys/ ./sshkeys" -bash -c "rsync -avz --delete ukmonhelper2:keymgmt/keys/ ./keys" -bash -c "rsync -avz --delete ukmonhelper2:keymgmt/csvkeys/ ./csvkeys" -bash -c "rsync -avz --delete ukmonhelper2:keymgmt/inifs/ ./inifs" - -python stationMaint2.py -Pop-Location \ No newline at end of file diff --git a/usermgmt/windows/stationMaint2.py b/usermgmt/windows/stationMaint2.py deleted file mode 100644 index 7da203ff0..000000000 --- a/usermgmt/windows/stationMaint2.py +++ /dev/null @@ -1,969 +0,0 @@ -# Copyright (C) 2018-2023 Mark McIntyre - -from tksheet import Sheet -import tkinter as tk -from tkinter import ttk -from tkinter import Frame, Menu -from tkinter import simpledialog -import tkinter.messagebox as tkMessageBox -from tkinter.filedialog import askopenfilename -import boto3 -import os -import sys -import paramiko -import json -import time -import datetime -import tempfile -from scp import SCPClient -from boto3.dynamodb.conditions import Key -import pandas as pd -from configparser import ConfigParser -import logging -import logging.handlers -import shutil -import platform -import subprocess - - -log = logging.getLogger("logger") - -config_file = '' - - -def loadConfig(cfgdir): - config_file = os.path.join(cfgdir, 'stationmaint.ini') - if not os.path.isfile(config_file): - tkMessageBox.showinfo("Config Missing", 'Please configure before using') - shutil.copyfile(f'{config_file}.sample', config_file) - if platform.system() == 'Darwin': # macOS - procid = subprocess.Popen(('open', config_file)) - elif platform.system() == 'Windows': # Windows - procid = subprocess.Popen(('cmd','/c',config_file)) - else: # linux variants - procid = subprocess.Popen(('xdg-open', config_file)) - procid.wait() - cfg = ConfigParser() - cfg.read(config_file) - server = cfg['helper']['helperip'] - user = cfg['helper']['user'] - keyfile = cfg['helper']['sshkey'] - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(keyfile)) - c = paramiko.SSHClient() - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - c.connect(hostname = server, username = user, pkey = k) - scpcli = c.open_sftp() - try: - handle, tmpfnam = tempfile.mkstemp() - scpcli.get(f'{user}.csv', tmpfnam) - except Exception: - log.warning('unable to find AWS key') - scpcli.close() - c.close() - try: - lis = open(tmpfnam, 'r').readlines() - os.close(handle) - os.remove(tmpfnam) - key, sec = lis[1].split(',') - except Exception: - log.warning('malformed AWS key') - if key: - log.info('retrieved key details') - awskeys = {'key': key.strip(), 'secret': sec.strip()} - else: - awskeys = {} - return cfg, awskeys - - -def addRow(newdata=None, stationid=None, site=None, user=None, email=None, ddb=None, - direction=None, camtype=None, active=None, createdate=None, tblname='camdetails'): - ''' - add a row to the CamDetails table - ''' - if not ddb: - ddb = boto3.resource('dynamodb', region_name='eu-west-2') - if not newdata: - newdata = {'stationid': stationid, 'site': site, 'humanName':user, 'eMail': email, - 'direction': direction, 'camtype': camtype, 'active': active, 'created': createdate, - 'oldcode': stationid} - table = ddb.Table(tblname) - response = table.put_item(Item=newdata) - if response['ResponseMetadata']['HTTPStatusCode'] != 200: - log.info(response) - return - - -def loadLocationDetails(table='camdetails', ddb=None, loadall=False): - if not ddb: - ddb = boto3.resource('dynamodb', region_name='eu-west-2') - table = ddb.Table(table) - res = table.scan() - # strictly, should check for LastEvaluatedKey here, in case there was more than 1MB of data, - # however that equates to around 30,000 users which i hope we never have... - values = res.get('Items', []) - camdets = pd.DataFrame(values) - camdets.sort_values(by=['stationid'], inplace=True) - if not loadall: - camdets.dropna(inplace=True, subset=['eMail','humanName','site']) - camdets['camtype'] = camdets['camtype'].astype(int) - camdets['active'] = camdets['active'].astype(int) - return camdets - - -def findLocationInfo(srchstring, ddb=None, statdets=None): - if statdets is None: - statdets = loadLocationDetails(ddb=ddb) - statdets = statdets[statdets.active==1] - s1 = statdets[statdets.stationid.str.contains(srchstring)] - s2 = statdets[statdets.eMail.str.contains(srchstring)] - s3 = statdets[statdets.humanName.str.contains(srchstring)] - s4 = statdets[statdets.site.str.contains(srchstring)] - srchres = pd.concat([s1, s2, s3, s4]) - srchres.drop(columns=['oldcode','active','camtype'], inplace=True) - return srchres - - -def cameraExists(stationid=None, location=None, direction=None, ddb=None, statdets=None): - if statdets is None: - statdets = loadLocationDetails(ddb=ddb) - statdets = statdets[statdets.active==1] - if stationid: - if len(statdets[statdets.stationid == stationid]) > 0: - return True - if location and direction: - s1 = statdets[statdets.site == location] - if len(s1) > 0: - if len(s1[s1.direction == direction]) > 0: - return True - return False - - -def dumpCamTable(outdir, statdets=None, ddb=None, exportmindets=False): - if statdets is None: - statdets = loadLocationDetails(ddb=ddb) - statdets = statdets[statdets.active==1] - if exportmindets: - statdets = statdets[['stationid', 'eMail']] - statdets.to_csv(os.path.join(outdir,'camtable.csv'), index=False) - - -def getCamUpdateDate(camid, ddb=None): - if not ddb: - ddb = boto3.resource('dynamodb', region_name='eu-west-2') - table = ddb.Table('LiveBrightness') - resp = table.query(KeyConditionExpression=Key('camid').eq(camid), - IndexName = 'camid-CaptureNight-index', - ScanIndexForward=False, - Limit=1, - Select='SPECIFIC_ATTRIBUTES', - ProjectionExpression='CaptureNight') - if len(resp['Items']) > 0: - return int(resp['Items'][0]['CaptureNight']) - else: - return 0 - - -class srcResBox(tk.Toplevel): - ''' - A class for the search dialog - ''' - def __init__(self, title, message): - tk.Toplevel.__init__(self) - self.details_expanded = False - self.title(title) - self.geometry('600x250') - self.minsize(700, 250) - self.maxsize(700, 550) - self.rowconfigure(0, weight=0) - self.rowconfigure(1, weight=1) - self.columnconfigure(0, weight=1) - - button_frame = tk.Frame(self) - button_frame.grid(row=0, column=0, sticky='nsew') - button_frame.columnconfigure(0, weight=1) - button_frame.columnconfigure(1, weight=1) - - text_frame = tk.Frame(self) - text_frame.grid(row=1, column=0, padx=(7, 7), pady=(7, 7), sticky='nsew') - text_frame.rowconfigure(0, weight=1) - text_frame.columnconfigure(0, weight=1) - - self.textbox = tk.Text(text_frame, height=6) - self.textbox.insert('1.0', message) - self.textbox.grid(row=0, column=0, sticky='nsew') - self.geometry('700x550') - self.details_expanded = True - - -class infoDialog(simpledialog.Dialog): - ''' - A class to gather or display info on a camera - ''' - def __init__(self, parent, title, location, user, email, sshkey='', id=''): - self.data = [] - self.data.append(id) - self.data.append(location) - self.data.append('') - self.data.append(user) - self.data.append(email) - self.data.append(sshkey) - self.parent = parent - - super().__init__(parent, title) - - def body(self, frame): - self.camid_label = tk.Label(frame, width=25, text="RMS ID") - self.camid_label.pack() - self.camid_box = tk.Entry(frame, width=25) - self.camid_box.insert(tk.END, self.data[0]) - self.camid_box.pack() - - self.location_label = tk.Label(frame, width=25, text="Location") - self.location_label.pack() - self.location_box = tk.Entry(frame, width=25) - self.location_box.insert(tk.END, self.data[1]) - self.location_box.pack() - - self.direction_label = tk.Label(frame, width=25, text="direction") - self.direction_label.pack() - self.direction_box = tk.Entry(frame, width=25) - self.direction_box.pack() - - self.ownername_label = tk.Label(frame, width=25, text="owner name") - self.ownername_label.pack() - self.ownername_box = tk.Entry(frame, width=25) - self.ownername_box.insert(tk.END, self.data[3]) - self.ownername_box.pack() - - self.email_label = tk.Label(frame, width=25, text="email address") - self.email_label.pack() - self.email_box = tk.Entry(frame, width=25) - self.email_box.insert(tk.END, self.data[4]) - self.email_box.pack() - - self.sshkey_label = tk.Label(frame, width=25, text="SSH key") - self.sshkey_label.pack() - self.sshkey_box = tk.Entry(frame, width=50) - self.sshkey_box.insert(tk.END, self.data[5]) - self.sshkey_box.pack() - - def ok_pressed(self): - self.data[0] = self.camid_box.get().strip() - self.data[1] = self.location_box.get().strip().capitalize() - self.data[2] = self.direction_box.get().strip().upper() - self.data[3] = self.ownername_box.get().strip() - self.data[4] = self.email_box.get().strip() - self.data[5] = self.sshkey_box.get().strip() - if cameraExists(location=self.data[1],direction=self.data[2], statdets=self.parent.stationdetails): - msg = f'{self.data[1]}_{self.data[2]} already exists' - tk.messagebox.showinfo(title="Information", message=msg) - else: - self.destroy() - - def cancel_pressed(self): - self.data[0] = '' - self.destroy() - - def buttonbox(self): - self.ok_button = tk.Button(self, text='OK', width=5, command=self.ok_pressed) - self.ok_button.pack(side="left") - cancel_button = tk.Button(self, text='Cancel', width=5, command=self.cancel_pressed) - cancel_button.pack(side="right") - self.bind("", lambda event: self.ok_pressed()) - self.bind("", lambda event: self.cancel_pressed()) - - -class statOwnerDialog(simpledialog.Dialog): - ''' - A class to display station ownership data - ''' - def __init__(self, parent): - self.stationdetails = parent.stationdetails - self.parent = parent - super().__init__(parent, 'Owner Info') - - def body(self, frame): - columns = ('#1','#2','#3','#4') - tree = ttk.Treeview(frame, columns=columns, show='headings') - tree.heading('#1', text='camid') - tree.heading('#2', text='site') - tree.heading('#3', text='email') - tree.heading('#4', text='humanName') - contacts = [] - for _, li in self.stationdetails.iterrows(): - contacts.append((li['stationid'], li['site'], li['eMail'], li['humanName'])) - for contact in contacts: - tree.insert('', tk.END, values=contact) - tree.grid(row=0, column=0, sticky='nsew') - scrollbar = ttk.Scrollbar(root, orient=tk.VERTICAL, command=tree.yview) - tree.configure(yscroll=scrollbar.set) - scrollbar.grid(row=0, column=1, sticky='ns') - return - - -class CamMaintenance(Frame): - ''' - The main camera maintenance window class - ''' - def __init__(self, parent, cfgdir): - self.parent = parent - Frame.__init__(self, parent) - - self.config_dir = cfgdir - self.cfg, awskeys = loadConfig(cfgdir) - self.conn = boto3.Session(aws_access_key_id=awskeys['key'], aws_secret_access_key=awskeys['secret']) - self.bucket_name = self.cfg['store']['srcbucket'] - - os.makedirs('keys', exist_ok=True) - os.makedirs('jsonkeys', exist_ok=True) - os.makedirs('csvkeys', exist_ok=True) - os.makedirs('users', exist_ok=True) - os.makedirs('inifs', exist_ok=True) - os.makedirs('sshkeys', exist_ok=True) - - self.ddb = self.conn.resource('dynamodb', region_name='eu-west-2') - try: - self.stationdetails = loadLocationDetails(ddb=self.ddb) - except Exception: - log.info('unable to get operator details - probably wrong AWS profile') - exit(1) - tmpdf = self.stationdetails[['site','stationid','direction','camtype','active','humanName','eMail','oldcode','created']] - tmpdf = tmpdf.sort_values(by=['active','site','stationid'], ascending=[True,True,True]) - self.data = tmpdf.values.tolist() - self.hdrs = tmpdf.columns.tolist() - - self.parent.title('Station Maintenance') - - # Make menu - self.menuBar = Menu(self.parent) - self.parent.config(menu=self.menuBar) - - # File menu - fileMenu = Menu(self.menuBar, tearoff=0) - fileMenu.add_command(label="Exit", command=self.on_closing) - - camMenu = Menu(self.menuBar, tearoff=0) - camMenu.add_command(label = "Add Camera", command = self.addCamera) - camMenu.add_command(label = "Relocate Camera", command = self.moveCamera) - camMenu.add_separator() - camMenu.add_command(label = "Check Camera", command = self.checkLastUpdate) - camMenu.add_separator() - camMenu.add_command(label = "Download Platepar", command = self.getPlate) - camMenu.add_command(label = "Update Platepar", command = self.newPlate) - camMenu.add_separator() - camMenu.add_command(label = "Update SSH Key", command = self.newSSHKey) - camMenu.add_command(label = "Update AWS Key", command = self.newAWSKey) - - ownMenu = Menu(self.menuBar, tearoff=0) - ownMenu.add_command(label = "View Owner Data", command = self.viewOwnerData) - ownMenu.add_command(label = "Search Data", command = self.searchOwnerData) - - helpMenu = Menu(self.menuBar, tearoff=0) - helpMenu.add_command(label = "Documentation", command = self.viewReadme) - helpMenu.add_command(label = "About", command = self.aboutBox) - - self.menuBar.add_cascade(label="File", underline=0, menu=fileMenu) - self.menuBar.add_cascade(label="Camera", underline=0, menu=camMenu) - self.menuBar.add_cascade(label="Owners", underline=0, menu=ownMenu) - self.menuBar.add_cascade(label="Help", underline=0, menu=helpMenu) - - parent.grid_columnconfigure(0, weight = 1) - parent.grid_rowconfigure(0, weight = 1) - self.frame = tk.Frame(self) - - self.frame.grid_columnconfigure(0, weight = 1) - self.frame.grid_rowconfigure(0, weight = 1) - self.sheet = Sheet(self.parent, - page_up_down_select_row = True, - expand_sheet_if_paste_too_big = True, - #empty_vertical = 0, - column_width = 120, - startup_select = (0,1,"rows"), - data = self.data, - headers = self.hdrs, - height = 700, - width = 700) - - self.sheet.enable_bindings(("single_select", - "drag_select", - "select_all", - "column_width_resize", - "double_click_column_resize", - "row_width_resize", - "column_height_resize", - "arrowkeys", - "row_height_resize", - "double_click_row_resize", - "right_click_popup_menu", - "rc_delete_row", - "copy", - "cut", - "paste", - "delete", - "undo", - "edit_cell" - )) - self.sheet.popup_menu_add_command("Sort by this Column", self.columns_sort) - - self.frame.grid(row = 1, column = 0, sticky = "nswe") - self.sheet.grid(row = 0, column = 0, sticky = "nswe") - - self.sheet.change_theme("light green") - - self.sheet.set_all_column_widths() - - #self.sheet.extra_bindings("end_delete_rows", self.end_delete_rows) - #self.sheet.extra_bindings("column_select", self.column_select) - self.sheet.extra_bindings([("all_select_events", self.all_extra_bindings), - ("begin_edit_cell", self.begin_edit_cell), - ("end_edit_cell", self.end_edit_cell)]) - - self.sheet.popup_menu_add_command('Sort', self.columns_sort, table_menu = False, index_menu = False) - self.parent.title('Please wait, syncing data...') - self.resyncLocalFiles() - self.parent.title('Station Maintenance') - return - - def hide_columns_right_click(self, event = None): - currently_displayed = self.sheet.display_columns() - exclude = set(currently_displayed[c] for c in self.sheet.get_selected_columns()) - indexes = [c for c in currently_displayed if c not in exclude] - self.sheet.display_columns(indexes = indexes, enable = True, refresh = True) - - def all_extra_bindings(self, event): - pass - - def begin_edit_cell(self, event): - self.oldval = self.data[event[0]][event[1]] - return self.oldval - - def end_edit_cell(self, event): - #log.info(event) - if event[3] != self.oldval: - data = self.data[event[0]] - data[event[1]] = event[3] - newdata = {'stationid': data[1], 'site': data[0], 'humanName':data[5], 'eMail': data[6], - 'direction': data[2], 'camtype': str(data[3]), 'active': int(data[4]), 'oldcode': data[1], - 'created': data[8]} - addRow(newdata, ddb=self.ddb) - if int(data[4]) > 1: - self.disableEnableUnixUser(data[0], data[2], False) - else: - self.disableEnableUnixUser(data[0], data[2], True) - return event[3] - - def end_delete_rows(self, event): - pass - - def window_resized(self, event): - pass - - def mouse_motion(self, event): - pass - - def deselect(self, event): - log.info(f'{event} {self.sheet.get_selected_cells()}') - - def rc(self, event): - log.info(event) - - def cell_select(self, response): - pass - - def shift_select_cells(self, response): - pass - - def drag_select_cells(self, response): - pass - - def ctrl_a(self, response): - pass - - def row_select(self, response): - pass - - def shift_select_rows(self, response): - log.info(response) - - def drag_select_rows(self, response): - pass - - def columns_sort(self): - cursel = self.sheet.get_selected_cells() - col = list(cursel)[0][1] - log.info(self.hdrs[col]) - pass - - def column_select(self, response): - pass - - def shift_select_columns(self, response): - pass - - def drag_select_columns(self, response): - pass - - def on_closing(self): - outdir = 'stationdetails' - os.makedirs(outdir, exist_ok=True) - dumpCamTable(outdir=outdir, statdets=self.stationdetails, exportmindets=False) - log.info('quitting') - self.destroy() - self.parent.quit() - self.parent.destroy() - - def viewOwnerData(self): - statOwnerDialog(self) - return - - def searchOwnerData(self): - srchstring = simpledialog.askstring("Some_Name", "Search String",parent=root) - srchres = findLocationInfo(srchstring, statdets=self.stationdetails) - msgtext = '' - for _, li in srchres.iterrows(): - msgtext = msgtext + f'{li.stationid:10s}{li.site:20s}{li.eMail:30s}{li.humanName:20s}\n' - srcResBox(title="Results", message=msgtext) - return - - def moveCamera(self): - self.addCopyCamera(move=True) - return - - def checkLastUpdate(self): - cursel = self.sheet.get_selected_cells() - cr = list(cursel)[0][0] - curdata = self.data[cr] - camid = curdata[1] - lastupd = getCamUpdateDate(camid, ddb=self.ddb) - msg = f'{camid} last sent a live image on {lastupd}' - tk.messagebox.showinfo(title="Information", message=msg) - return - - def addCamera(self): - self.addCopyCamera(move=False) - return - - def addCopyCamera(self, move=False): - cursel = self.sheet.get_selected_cells() - cr = list(cursel)[0][0] - curdata = self.data[cr] - user = curdata[5] - email = curdata[6] - if move is True: - sshkey = self.getSSHkey(curdata[0], curdata[2]) - id = curdata[1] - title = 'Move Camera' - oldloc = curdata[0].lower() + '_' + curdata[2].lower() - created = curdata[7] - else: - sshkey = '' - id = '' - title = 'Add Camera' - oldloc = '' - created = datetime.datetime.now().strftime('%Y%m%d') - answer = infoDialog(self, title, curdata[0], user, email, sshkey, id) - if answer.data[0].strip() != '': - d = answer.data - rmsid = str(d[0]).upper() - location = str(d[1]).capitalize() - cameraname = d[1].lower() + '_' + d[2].lower() - with open(os.path.join('sshkeys', cameraname + '.pub'), 'w') as outf: - outf.write(d[5]) - rowdata=[d[1],d[0],d[2],'2','1',d[3],d[4],d[0]] - self.sheet.insert_row(values=rowdata, idx=0) - self.addNewAwsUser(location) - self.createIniFile(cameraname) - self.addNewUnixUser(location, cameraname, oldloc) - self.addNewOwner(rmsid, location, str(d[3]), str(d[4]), str(d[2]), '2','1', created) - log.info('done') - return - - def newSSHKey(self): - cursel = self.sheet.get_selected_cells() - cr = list(cursel)[0][0] - curdata = self.data[cr] - user,email = self.getUserDetails(self.stationdetails, curdata[1]) - sshkey = '' - id = '' - title = 'Update SSH Key' - answer = infoDialog(self, title, curdata[0], user, email, sshkey, id) - if answer.data[0].strip() != '': - d = answer.data - location = str(d[1]).capitalize() - cameraname = d[1].lower() + '_' + d[2].lower() - with open(os.path.join('sshkeys', cameraname + '.pub'), 'w') as outf: - outf.write(d[5]) - self.addNewUnixUser(location, cameraname, updatemode=2) - self.datachanged = True - return - - def getPlate(self): - cursel = self.sheet.get_selected_cells() - cr = list(cursel)[0][0] - curdata = self.data[cr] - ppdir = self.cfg['helper']['platepardir'] - ppdir = os.path.join(ppdir, curdata[1]) - os.makedirs(ppdir, exist_ok=True) - ppfile = 'platepar_cmn2010.cal' - site = curdata[0].capitalize() - camid = curdata[1].upper() - - server = self.cfg['helper']['helperip'] - user='ec2-user' - keyfile = self.cfg['helper']['sshkey'] - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(keyfile)) - c = paramiko.SSHClient() - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - c.connect(hostname = server, username = user, pkey = k) - scpcli = SCPClient(c.get_transport()) - remotedir = self.cfg['helper']['remotedir'] - remotef=f'{remotedir}/consolidated/platepars/{camid}.json' - localf = os.path.join(ppdir, ppfile) - scpcli.get(remotef, localf) - scpcli.close() - c.close() - - s3 = boto3.client('s3') - res = s3.list_objects_v2(Bucket=self.bucket_name, Prefix=f'archive/{site}/{camid}/2023/202312/') - if res['KeyCount'] > 0: - keys=res['Contents'] - fitsfiles = [x['Key'] for x in keys if 'fits' in x['Key']] - fitsfiles.sort() - fitsfiles = fitsfiles[-10:] - for ff in fitsfiles: - _, ffname = os.path.split(ff) - dlf = os.path.join(ppdir, ffname) - s3.download_file(self.bucket_name, ff, dlf) - cfgfiles = [x['Key'] for x in keys if '.config' in x['Key']] - cfgfiles.sort() - dlf = os.path.join(ppdir,'.config') - s3.download_file(self.bucket_name, cfgfiles[-1], dlf) - return - - def newPlate(self): - cursel = self.sheet.get_selected_cells() - cr = list(cursel)[0][0] - curdata = self.data[cr] - ppdir =self.cfg['helper']['platepardir'] - ppdir = os.path.join(ppdir, curdata[1]) - ppfile = 'platepar_cmn2010.cal' - plate = '' - title = 'Select New Platepar File' - plate = askopenfilename(title=title, defaultextension='*.cal',initialdir=ppdir, initialfile=ppfile) - if plate: - self.uploadPlatepar(curdata, plate) - log.info(plate) - return - - def resyncLocalFiles(self): - server = self.cfg['helper']['helperip'] - user='ec2-user' - keyfile = self.cfg['helper']['sshkey'] - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(keyfile)) - c = paramiko.SSHClient() - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - c.connect(hostname = server, username = user, pkey = k) - scpcli = c.open_sftp() - for fldr in ['csvkeys','inifs','keys','sshkeys']: - targdir = f'/home/{user}/keymgmt/{fldr}' - flist = scpcli.listdir_attr(targdir) - for fil in flist: - copyme = True - fname = fil.filename - localfname = os.path.join(fldr, fname) - if os.path.isfile(localfname): - mtime = os.path.getmtime(localfname) - if fil.st_mtime - mtime > 1: - copyme = True - else: - copyme = False - if copyme: - scpcli.get(f'{targdir}/{fname}', localfname) - print(f'Updated {fname}') - scpcli.close() - c.close() - - return - - def newAWSKey(self): - cursel = self.sheet.get_selected_cells() - cr = list(cursel)[0][0] - curdata = self.data[cr] - location = curdata[0] - self.createNewAwsKey(location, self.stationdetails) - - def uploadPlatepar(self, camdets, plateparfile): - server = self.cfg['helper']['helperip'] - user='ec2-user' - uplfile = f'/tmp/platepar_cmn2010_{camdets[1]}.cal' - camname = f'{camdets[0]}_{camdets[3]}'.lower() - keyfile = self.cfg['helper']['sshkey'] - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(keyfile)) - c = paramiko.SSHClient() - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - c.connect(hostname = server, username = user, pkey = k) - scpcli = SCPClient(c.get_transport()) - scpcli.put(plateparfile, uplfile) - command = f'sudo mkdir -p /var/sftp/{camname}/platepar/ && sudo chown {camname}:{camname} /var/sftp/{camname}/platepar' - log.info(f'running {command}') - _, stdout, stderr = c.exec_command(command, timeout=10) - for line in iter(stdout.readline, ""): - log.info(line) - for line in iter(stderr.readline, ""): - log.info(line) - command = f'sudo mv {uplfile} /var/sftp/{camname}/platepar/platepar_cmn2010.cal' - log.info(f'running {command}') - _, stdout, stderr = c.exec_command(command, timeout=10) - for line in iter(stdout.readline, ""): - log.info(line) - for line in iter(stderr.readline, ""): - log.info(line) - command = f'sudo chown {camname}:{camname} /var/sftp/{camname}/platepar/platepar_cmn2010.cal' - log.info(f'running {command}') - _, stdout, stderr = c.exec_command(command, timeout=10) - for line in iter(stdout.readline, ""): - log.info(line) - for line in iter(stderr.readline, ""): - log.info(line) - scpcli.close() - c.close() - return - - def addNewOwner(self, rmsid, location, user, email, direction, camtype, active, created): - log.info(f'adding new owner {user} with {email} for {rmsid} at {location}') - newdata = {'stationid': rmsid, 'site': location, 'humanName':user, 'eMail': email, - 'direction': direction, 'camtype': camtype, 'active': int(active), 'oldcode': rmsid, - 'created': created} - addRow(newdata=newdata, ddb=self.ddb) - return - - def addNewAwsUser(self, location): - log.info(f'adding new location {location} to AWS') - archkeyf = 'jsonkeys/' + location + '.key' - archuserdets = 'users/' + location + '.txt' - archcsvf = os.path.join('csvkeys', location + '.csv') - os.makedirs('jsonkeys', exist_ok=True) - os.makedirs('csvkeys', exist_ok=True) - os.makedirs('users', exist_ok=True) - - iamc = self.conn.client('iam') - try: - _ = iamc.get_user(UserName=location) - log.info('location exists, not adding it') - archkey = None - except iamc.exceptions.NoSuchEntityException: - log.info('new location') - usr = iamc.create_user(UserName=location) - with open(archuserdets, 'w') as outf: - outf.write(str(usr)) - archkey = iamc.create_access_key(UserName=location) - with open(archkeyf, 'w') as outf: - outf.write(json.dumps(archkey, indent=4, sort_keys=True, default=str)) - with open(archcsvf,'w') as outf: - outf.write('Access key ID,Secret access key\n') - outf.write('{},{}\n'.format(archkey['AccessKey']['AccessKeyId'], archkey['AccessKey']['SecretAccessKey'])) - _ = iamc.add_user_to_group(GroupName='cameras', UserName=location) - if archkey is not None: - self.createKeyFile(location) - return - - def createNewAwsKey(self, location, caminfo): - server = self.cfg['helper']['helperip'] - user='ec2-user' - keyfile = self.cfg['helper']['sshkey'] - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(keyfile)) - c = paramiko.SSHClient() - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - c.connect(hostname = server, username = user, pkey = k) - command = f'/home/{user}/keymgmt/updateAwsKey.sh {location} force' - log.info(f'running {command}') - _, stdout, stderr = c.exec_command(command, timeout=60) - for line in iter(stderr.readline, ""): - log.info(line) - for line in iter(stdout.readline, ""): - log.info(line) - tkMessageBox.showinfo('Updated', line.split(' to')[0]) - log.info('done') - c.close() - return - - - def getSSHkey(self, loc, dir): - server= self.cfg['helper']['helperip'] - user='ec2-user' - tmpdir=os.getenv('TEMP', default='c:/temp') - cameraname = (loc + '_' + dir).lower() - keyfile = self.cfg['helper']['sshkey'] - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(keyfile)) - c = paramiko.SSHClient() - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - c.connect(hostname = server, username = user, pkey = k) - scpcli = SCPClient(c.get_transport()) - tmpfil = os.path.join(tmpdir,'./tmp.txt') - # dont use os.path.join - source is on unix we are on windows! - scpcli.get(f'keymgmt/sshkeys/{cameraname}.pub', tmpfil) - scpcli.close() - c.close() - - with open(tmpfil, 'r') as inf: - lis = inf.readlines() - #os.remove('./tmp.txt') - return lis[0].strip() - - - def getUserDetails(self, stationdetails, camid): - reqdf = stationdetails[stationdetails.stationid == camid] - if len(reqdf) == 0: - return '','' - return reqdf.eMail.iloc[0], reqdf.humanName.iloc[0] - - - def addNewUnixUser(self, location, cameraname, oldcamname='', updatemode=0): - server = self.cfg['helper']['helperip'] - user='ec2-user' - log.info(f'adding new Unix user {cameraname}') - keyfile = self.cfg['helper']['sshkey'] - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(keyfile)) - c = paramiko.SSHClient() - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - try: - c.connect(hostname = server, username = user, pkey = k) - except Exception: - c.connect(hostname = server+'.', username = user, pkey = k) - scpcli = SCPClient(c.get_transport()) - scpcli.put(os.path.join('sshkeys', cameraname + '.pub'), 'keymgmt/sshkeys/') - scpcli.put(os.path.join('keys', location.lower() + '.key'), 'keymgmt/keys/') - scpcli.put(os.path.join('csvkeys', location + '.csv'), 'keymgmt/csvkeys/') - scpcli.put(os.path.join('inifs', cameraname + '.ini'), 'keymgmt/inifs/') - command = f'/home/{user}/keymgmt/addSftpUser.sh {cameraname} {location} {updatemode} {oldcamname}' - log.info(f'running {command}') - _, stdout, stderr = c.exec_command(command, timeout=60) - for line in iter(stdout.readline, ""): - log.info(line) - for line in iter(stderr.readline, ""): - log.info(line) - - log.info('done, collecting output') - infname = os.path.join('keymgmt/inifs/',cameraname + '.ini') - outfname = os.path.join('./inifs', cameraname + '.ini') - while os.path.isfile(outfname) is False: - try: - time.sleep(3) - scpcli.get(infname, outfname) - except Exception: - continue - scpcli.close() - c.close() - return - - def disableEnableUnixUser(self, loc, dir, enable): - cameraname = loc.lower() + '_' + dir.lower() - server = self.cfg['helper']['helperip'] - user='ec2-user' - log.info(f'updating Unix user {cameraname}') - keyfile = self.cfg['helper']['sshkey'] - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(keyfile)) - c = paramiko.SSHClient() - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - try: - c.connect(hostname = server, username = user, pkey = k) - except Exception: - c.connect(hostname = server+'.', username = user, pkey = k) - if enable is False: - command = f'/usr/bin/sudo /usr/sbin/usermod --expiredate 1 {cameraname}' - else: - command = f'/usr/bin/sudo /usr/sbin/usermod --expiredate "" {cameraname}' - log.info(f'running {command}') - line = '' - _, stdout, stderr = c.exec_command(command, timeout=60) - for line in iter(stderr.readline, ""): - log.info(line) - for line in iter(stdout.readline, ""): - log.info(line) - if enable is False: - tkMessageBox.showinfo('Disabled', f'{cameraname} {line}') - return - - def createKeyFile(self, location): - archbucket = self.cfg['store']['srcbucket'] - livebucket = self.cfg['store']['livebucket'] - webbucket = self.cfg['store']['websitebucket'] - - os.makedirs('keys', exist_ok=True) - outf = 'keys/' + location.lower() + '.key' - with open(outf, 'w') as ouf: - ouf.write('export AWS_DEFAULT_REGION=eu-west-1\n') - ouf.write(f'export CAMLOC="{location}"\n') - ouf.write(f'export S3FOLDER="archive/{location}/"\n') - ouf.write(f'export ARCHBUCKET={archbucket}\n') - ouf.write(f'export LIVEBUCKET={livebucket}\n') - ouf.write(f'export WEBBUCKET={webbucket}\n') - ouf.write('export ARCHREGION=eu-west-2\n') - ouf.write('export LIVEREGION=eu-west-1\n') - ouf.write('export MATCHDIR=matches/RMSCorrelate\n') - return - - - def createIniFile(self, cameraname): - helperip = self.cfg['helper']['helperip'] - os.makedirs('inifs', exist_ok=True) - outf = 'inifs/' + cameraname + '.ini' - with open(outf, 'w') as outf: - outf.write('# config data for this station\n') - outf.write(f'export LOCATION={cameraname}\n') - outf.write(f'export UKMONHELPER={helperip}\n') - outf.write('export UKMONKEY=~/.ssh/ukmon\n') - outf.write('export RMSCFG=~/source/RMS/.config\n') - return - - def viewReadme(self): - docfile = os.path.join(self.config_dir, 'README.md') - if platform.system() == 'Darwin': # macOS - procid = subprocess.Popen(('open', docfile)) - elif platform.system() == 'Windows': # Windows - procid = subprocess.Popen(('cmd','/c',f'notepad {docfile}')) - else: # linux variants - procid = subprocess.Popen(('xdg-open', docfile)) - procid.wait() - - def aboutBox(self): - tkMessageBox.showinfo('About', - 'UKMON user / camera maintenance tool.\nAll rights reserved, Mark McIntyre, 2024') - - -def log_timestamp(): - """ Returns timestamp for logging. - """ - return datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S') - - -if __name__ == '__main__': - # Initialize main window - dir_ = os.getcwd() - - log.setLevel(logging.INFO) - - os.makedirs(os.path.join(dir_, 'logs'), exist_ok=True) - log_file = os.path.join(dir_, 'logs', log_timestamp() + '.log') - handler = logging.handlers.TimedRotatingFileHandler(log_file, when='D', interval=1) # Log to a different file each day - handler.setLevel(logging.INFO) - - formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s-%(module)s-line:%(lineno)d - %(message)s', datefmt='%Y/%m/%d %H:%M:%S') - handler.setFormatter(formatter) - log.addHandler(handler) - - ch = logging.StreamHandler(sys.stdout) - ch.setLevel(logging.DEBUG) - formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s: %(message)s', datefmt='%Y/%m/%d %H:%M:%S') - ch.setFormatter(formatter) - log.addHandler(ch) - - # Log program start - log.info("Program start") - - root = tk.Tk() - root.minsize(400, 100) - app = CamMaintenance(root, dir_) - root.iconbitmap(os.path.join(dir_,'camera.ico')) - root.protocol("WM_DELETE_WINDOW", app.on_closing) - app.mainloop() diff --git a/usermgmt/windows/stationmaint.ini.sample b/usermgmt/windows/stationmaint.ini.sample deleted file mode 100644 index 0a6b99346..000000000 --- a/usermgmt/windows/stationmaint.ini.sample +++ /dev/null @@ -1,11 +0,0 @@ -[helper] -USER= -SSHKEY= -HELPERIP=3.11.55.160 -REMOTEDIR=/home/ec2-user/prod/data -PLATEPARDIR= - -[store] -SRCBUCKET=ukmda-shared -LIVEBUCKET=ukmda-live -WEBSITEBUCKET=ukmda-website