diff --git a/.gitignore b/.gitignore index 531416e0f..e46843f57 100644 --- a/.gitignore +++ b/.gitignore @@ -593,10 +593,19 @@ archive/samfunctions*/camDetails/pythoncode/camDetails.py archive/samfunctions*/camDetails/pythoncode/awskeys archive/containers/RMS-docker/rms_docker.tar archive/containers/RMS-docker/Dockerfile_orig.txt + +usermgmt/windows/csvkeys +usermgmt/windows/inifs +usermgmt/windows/jsonkeys +usermgmt/windows/keys +usermgmt/windows/sshkeys +usermgmt/windows/stationdetails +usermgmt/windows/users usermgmt/gmailkeys/* usermgmt/server/ukmonfundraising.txt usermgmt/server/ukmoncommittee.txt usermgmt/server/ukmon*.json + **/awskeys.test **/trajsolver_old/* archive/tmp/* diff --git a/archive/cronjobs/nightlyJob.sh b/archive/cronjobs/nightlyJob.sh index e96dcfddf..7f184a5fd 100644 --- a/archive/cronjobs/nightlyJob.sh +++ b/archive/cronjobs/nightlyJob.sh @@ -29,6 +29,7 @@ mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} log2cw $NJLOGGRP $NJLOGSTREAM "updating the camera location/dir/fov database" nightlyJob python -c "from reports.CameraDetails import updateCamLocDirFovDB; updateCamLocDirFovDB();" aws s3 cp $DATADIR/admin/cameraLocs.json $UKMONSHAREDBUCKET/admin/ --profile ukmonshared --quiet +aws s3 cp $DATADIR/admin/cameraLocs.json $WEBSITEBUCKET/browse/ --profile ukmonshared --quiet aws s3 sync $UKMONSHAREDBUCKET/admin/ $DATADIR/admin --profile ukmonshared --quiet # create the CSV file of camera info and the html versions for search functions on the website diff --git a/archive/deployment/analysis.yml b/archive/deployment/analysis.yml index faeb27f7b..8e217f41d 100644 --- a/archive/deployment/analysis.yml +++ b/archive/deployment/analysis.yml @@ -1,7 +1,9 @@ --- - hosts: "{{host | default ('ukmonhelper2')}}" + vars_files: + - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/config.yml b/archive/deployment/config.yml index 2fa3d319e..b9823e58a 100644 --- a/archive/deployment/config.yml +++ b/archive/deployment/config.yml @@ -1,6 +1,8 @@ - hosts: "{{host | default ('ukmonhelper2')}}" + vars_files: + - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: /mnt/e/dev/meteorhunting/ukmda-datapocessing/archive + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: dev-vars.yml diff --git a/archive/deployment/cronjobs.yml b/archive/deployment/cronjobs.yml index 1232a22ec..95ee560d0 100644 --- a/archive/deployment/cronjobs.yml +++ b/archive/deployment/cronjobs.yml @@ -1,7 +1,9 @@ --- - hosts: "{{host | default ('ukmonhelper2')}}" + vars_files: + - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/database.yml b/archive/deployment/database.yml index 1baceec67..1b75b1f5f 100644 --- a/archive/deployment/database.yml +++ b/archive/deployment/database.yml @@ -1,7 +1,9 @@ --- - hosts: "{{host | default ('ukmonhelper2')}}" + vars_files: + - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/pylib.yml b/archive/deployment/pylib.yml index 90ae4e01d..162f4ab67 100644 --- a/archive/deployment/pylib.yml +++ b/archive/deployment/pylib.yml @@ -1,7 +1,9 @@ --- - hosts: "{{host | default ('ukmonhelper2')}}" + vars_files: + - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/server_setup.yml b/archive/deployment/server_setup.yml index 4682ee3b5..f621dc278 100644 --- a/archive/deployment/server_setup.yml +++ b/archive/deployment/server_setup.yml @@ -1,7 +1,9 @@ --- - hosts: "{{host | default ('ukmonhelper2')}}" + vars_files: + - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/shwrinfo.yml b/archive/deployment/shwrinfo.yml index 0c5dbc78c..17a6110ee 100644 --- a/archive/deployment/shwrinfo.yml +++ b/archive/deployment/shwrinfo.yml @@ -1,7 +1,9 @@ --- - hosts: "{{host | default ('ukmonhelper2')}}" + vars_files: + - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/utils.yml b/archive/deployment/utils.yml index e6a47f723..f122d7cb5 100644 --- a/archive/deployment/utils.yml +++ b/archive/deployment/utils.yml @@ -1,7 +1,9 @@ --- - hosts: "{{host | default ('ukmonhelper2')}}" + vars_files: + - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/website.yml b/archive/deployment/website.yml index 427008f1d..1ee42364e 100644 --- a/archive/deployment/website.yml +++ b/archive/deployment/website.yml @@ -1,7 +1,9 @@ --- - hosts: "{{host | default ('ukmonhelper2')}}" + vars_files: + - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/website_static.yml b/archive/deployment/website_static.yml index edb791d7b..cbe146679 100644 --- a/archive/deployment/website_static.yml +++ b/archive/deployment/website_static.yml @@ -1,7 +1,9 @@ --- - hosts: "{{host | default ('ukmonhelper2')}}" + vars_files: + - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml @@ -18,7 +20,7 @@ with_items: - {src: '{{srcdir}}/static_content/browse/', dest: '{{destdir}}/static_content/browse', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/css/', dest: '{{destdir}}/static_content/css', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/data/', dest: '{{destdir}}/static_content/data', mode: '754', backup: no, directory_mode: yes } + #- {src: '{{srcdir}}/static_content/data/', dest: '{{destdir}}/static_content/data', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/docs/', dest: '{{destdir}}/static_content/docs', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/fonts/', dest: '{{destdir}}/static_content/fonts', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/img/', dest: '{{destdir}}/static_content/img', mode: '754', backup: no, directory_mode: yes } diff --git a/archive/static_content/browse/annual/index.html b/archive/static_content/browse/annual/index.html index b71378c10..e775af6d4 100644 --- a/archive/static_content/browse/annual/index.html +++ b/archive/static_content/browse/annual/index.html @@ -40,6 +40,7 @@
+

The data here are released under the CC BY 4.0 license, so if you are using the data whether for scientific or other purposes, your must reference this web site in your work. diff --git a/archive/static_content/browse/daily/index.html b/archive/static_content/browse/daily/index.html index e03a52b93..0d6cc5807 100644 --- a/archive/static_content/browse/daily/index.html +++ b/archive/static_content/browse/daily/index.html @@ -63,6 +63,7 @@

+

Daily Simplified Report of Detections

This page provides a list of all detections in the last 72 hours from the camera network, primarily for the purposes of data exchange with diff --git a/archive/static_content/browse/index.html b/archive/static_content/browse/index.html index ceff96e73..23f8d2e43 100644 --- a/archive/static_content/browse/index.html +++ b/archive/static_content/browse/index.html @@ -41,6 +41,7 @@

+

Browse the Archive.

Follow the links below to access downloadable datasets from our archive.

You can also use our REST APIs to retrieve some data programmatically. If you're interested in diff --git a/archive/static_content/browse/monthly/index.html b/archive/static_content/browse/monthly/index.html index cd1f5f72a..e0f8cf24b 100644 --- a/archive/static_content/browse/monthly/index.html +++ b/archive/static_content/browse/monthly/index.html @@ -41,6 +41,7 @@

+

The data here are released under the CC BY 4.0 license, so if you are using the data whether for scientific or other purposes, your must reference this web site in your work. diff --git a/archive/static_content/css/dragontail.css b/archive/static_content/css/dragontail.css index 35ca1de7a..ddefb159e 100644 --- a/archive/static_content/css/dragontail.css +++ b/archive/static_content/css/dragontail.css @@ -141,7 +141,7 @@ body { .sticky + .content { padding-top: 60px; } - + .sidebar .sidebar-nav.navbar-collapse { padding-right: 0; padding-left: 0; @@ -524,6 +524,12 @@ li > a:only-child:after { content: ''; } font-size: 15px; } +@media screen and (max-width: 1100px) { + .sidenav { + display: none !important; + } +} + .table { /*color: whitesmoke;*/ background-color: white; diff --git a/archive/static_content/docs/Code_of_Conduct.html b/archive/static_content/docs/Code_of_Conduct.html index 8d6024a51..20c9502d7 100644 --- a/archive/static_content/docs/Code_of_Conduct.html +++ b/archive/static_content/docs/Code_of_Conduct.html @@ -43,6 +43,7 @@

+

Contributor Covenant Code of Conduct

Our Pledge

diff --git a/archive/static_content/docs/Committee.html b/archive/static_content/docs/Committee.html index 68bdaccf3..9ce4053af 100644 --- a/archive/static_content/docs/Committee.html +++ b/archive/static_content/docs/Committee.html @@ -53,13 +53,14 @@
+

The Committee

The Society is managed by a committee consisting of the following offices:

- + diff --git a/archive/static_content/docs/index.html b/archive/static_content/docs/index.html index c6003069b..4e32063c5 100644 --- a/archive/static_content/docs/index.html +++ b/archive/static_content/docs/index.html @@ -43,6 +43,7 @@
+

Documents.

UK Meteors is an unincorporated society following the standard model for Astronomy clubs.

Our main website is here and diff --git a/archive/static_content/docs/otherdocs.html b/archive/static_content/docs/otherdocs.html index 8b9ad7833..1979c9508 100644 --- a/archive/static_content/docs/otherdocs.html +++ b/archive/static_content/docs/otherdocs.html @@ -43,6 +43,7 @@

+

Other Documents.

A repository of the Society's public documents.

diff --git a/archive/static_content/fundraising.html b/archive/static_content/fundraising.html index f7fe7e06a..c000f9455 100644 --- a/archive/static_content/fundraising.html +++ b/archive/static_content/fundraising.html @@ -40,6 +40,7 @@

+

Fundraising

If you'd still like to help with data analysis costs, you can donate directly via our bank account. This has the advantage that we are not charged administration fees by diff --git a/archive/static_content/img/ukmon-logo-jpg-crp.jpg b/archive/static_content/img/ukmon-logo-jpg-crp.jpg new file mode 100644 index 000000000..fd9c5fd59 Binary files /dev/null and b/archive/static_content/img/ukmon-logo-jpg-crp.jpg differ diff --git a/archive/static_content/img/ukmon-logo-sm.png b/archive/static_content/img/ukmon-logo-sm.png new file mode 100644 index 000000000..5cbf17f0c Binary files /dev/null and b/archive/static_content/img/ukmon-logo-sm.png differ diff --git a/archive/static_content/privacy_data.html b/archive/static_content/privacy_data.html index b56fe3613..c7e0f55f9 100644 --- a/archive/static_content/privacy_data.html +++ b/archive/static_content/privacy_data.html @@ -39,8 +39,9 @@
+

Privacy and Data Protection Policy

-

Last reviewed 2023-08-15

+

Last reviewed 2025-06-13

This page explains how we handle any personal and sensitive data that we collect.

Camera ID, name and geographic location of each camera. @@ -48,7 +49,7 @@ We collect the Camera ID, general location name and precise geographic coordinates of each camera for the following purposes: -
  • For Scientific data analysis such as statistical reporting of showers, orbit and trajectory +
  • For Scientific data analysis such as statistical reporting of showers, and for orbit and trajectory calculations of meteors detected by the camera. The results of this analysis are made available on this website.
    • @@ -66,7 +67,8 @@

    For example, we might collect Camera ID UK1234, location name Sunthorp*, geographic coordinates 53.62501N 0.08567E, altitude -20m. If a paper is published which makes use of data from this - camera, the author might mention the camera ID and approximate location in the paper. + camera, the author might mention the camera ID and location name in the paper. The precise + geographic coordinates will not be shared without your permission.

    This information will be retained indefinitely even if you cease to contribute. @@ -92,9 +94,9 @@

    Postal Address

    - If you have asked us to build a camera for you, the camera builder will also ask for + If you have asked us to build a camera for you, the camera builder will ask for your full Postal Address so that the items can be posted. We do not retain this information - anywhere on our systems. + anywhere on our online systems and the camera builder will only share it with your permission.

    Publications @@ -128,7 +130,7 @@ this data with other meteor networks if they wish to do so.
  • By agreeing to contribute, you are agreeing that Level 1 and Level 2 data may be used - by the network coordinators to manually produce other products such as Level 3 data, and that + by the network and its coordinators to manually or automatically produce other products such as Level 3 data, and that publication of such results may include any Level 2 data that was used, where this is necessary to explain or validate the Level 3 data.
    For example, when Level 2 data is combined to calculate an orbit or trajectory (Level 3 data), then the relevant Level 2 data may be made diff --git a/archive/static_content/search/index.html b/archive/static_content/search/index.html index c4d6d25a9..5fa067bec 100644 --- a/archive/static_content/search/index.html +++ b/archive/static_content/search/index.html @@ -75,6 +75,7 @@

    Search the Archive.

    +

    This page allows you to search the archive for specific events in the matched and single-station detections.
    Time should be entered as local time, so in the UK summer you should enter the time in BST. diff --git a/archive/static_content/templates/navbar.html b/archive/static_content/templates/navbar.html index fdd38bf7b..061b0d599 100644 --- a/archive/static_content/templates/navbar.html +++ b/archive/static_content/templates/navbar.html @@ -2,9 +2,9 @@

    diff --git a/archive/terraform/mjmm/iam.tf b/archive/terraform/mjmm/iam.tf index 597b828ca..766e2d668 100644 --- a/archive/terraform/mjmm/iam.tf +++ b/archive/terraform/mjmm/iam.tf @@ -70,6 +70,14 @@ resource "aws_iam_policy" "pol4s3fullaccess" { "*", ] }, + { + Action = [ + "iam:GetCredentialReport", + "iam:GenerateCredentialReport" + ] + Effect = "Allow" + Resource = [ "*" ] + }, { Sid = "PassRolePermission" Effect = "Allow" diff --git a/archive/terraform/ukmda/loggingbucket.tf b/archive/terraform/ukmda/loggingbucket.tf index 0bab24ff9..7938ec4a1 100644 --- a/archive/terraform/ukmda/loggingbucket.tf +++ b/archive/terraform/ukmda/loggingbucket.tf @@ -44,7 +44,7 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalogslcp" { } filter { - prefix = "archsite/" + prefix = "website/" } noncurrent_version_expiration { diff --git a/archive/ukmon_pylib/analysis/analyseGmnData.py b/archive/ukmon_pylib/analysis/analyseGmnData.py new file mode 100644 index 000000000..77920b99c --- /dev/null +++ b/archive/ukmon_pylib/analysis/analyseGmnData.py @@ -0,0 +1,20 @@ +# simple scripts to analyse the GMN data in python + +import pandas as pd +import os + +from converters.gmnTxtToPandas import dirpath + + +def findDuplicates(yr, mth=None): + if mth: + datafile = os.path.join(dirpath, 'parquet', 'monthly', f'gmn_{yr:04d}{mth:02d}.parquet.snap') + else: + datafile = os.path.join(dirpath, 'parquet', f'gmn_{yr:04d}.parquet.snap') + df = pd.read_parquet(datafile) + df['dupe']=df.duplicated(subset=['id']) + dupeids = df[df.dupe].sort_values(by=['id']).id + duperows = df[df.id.isin(dupeids)] + print(duperows) + print(len(df)) + return duperows diff --git a/archive/ukmon_pylib/analysis/findCamerasNearPoint.py b/archive/ukmon_pylib/analysis/findCamerasNearPoint.py deleted file mode 100644 index 8f786999d..000000000 --- a/archive/ukmon_pylib/analysis/findCamerasNearPoint.py +++ /dev/null @@ -1,49 +0,0 @@ -# copyright Mark McIntyre, 2024- - -# find cameras near to a given point (lat/lon in degrees) - -import os -import sys -import json -import numpy as np -import requests - -from meteortools.utils.Math import greatCircleDistance - -def camerasNearTo(lat, lon, distFromPt): - l1 = np.radians(lat) - g1 = np.radians(lon) - cams = [] - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') - camdb = os.path.join(datadir, 'admin', 'cameraLocs.json') - caminfo = json.load(open(camdb)) - for cam in caminfo: - l2 = np.radians(caminfo[cam]['lat']) - g2 = np.radians(caminfo[cam]['lon']) - dist = greatCircleDistance(l1, g1, l2, g2) - # print(dist) - if dist < distFromPt: - url = f'https://api.ukmeteors.co.uk/camdetails?camid={cam}' - res = requests.get(url) - if res.status_code == 200: - dta = json.loads(res.text) - cams.append(dta[0]['eMail']) - cams = list(set(cams)) - camstr = '' - for cam in cams: - camstr = camstr + cam + ';' - return camstr - - -def getCoordsFromTraj(trajname): - lat = 0 - lon = 0 - return lat, lon - - -if __name__ == '__main__': - lat = float(sys.argv[1]) - lon = float(sys.argv[2]) - dist = float(sys.argv[3]) - - print(camerasNearTo(lat, lon, dist)) \ No newline at end of file diff --git a/archive/ukmon_pylib/analysis/gatherDetectionData.py b/archive/ukmon_pylib/analysis/gatherDetectionData.py index eeaed0a60..b9031167b 100644 --- a/archive/ukmon_pylib/analysis/gatherDetectionData.py +++ b/archive/ukmon_pylib/analysis/gatherDetectionData.py @@ -6,6 +6,7 @@ import os import sys import datetime +import pandas as pd from meteortools.ukmondb import getECSVs, getLiveJpgs, getFBfiles, getDetections @@ -35,6 +36,21 @@ def getRawData(idlist, outpth): return +def updateSingleDB(yr, consdt): + datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + sngls = pd.read_csv(os.path.join(datadir, 'single', f'singles-{yr}.csv')) + consdf = pd.read_csv(os.path.join(datadir, 'single', 'used', f'consumed_{consdt}.txt'), names=['Filename']) + conslist = list(consdf.Filename) + for ff in conslist: + mtch = sngls.Filename == ff + if mtch.any(): + print(f'updating {ff}') + sngls.loc[mtch, 'status'] = 'Y' + else: + print(f'skipping {ff}') + sngls.to_csv(os.path.join(datadir, 'single', f'singles-{yr}.csv'), index=False) + + if __name__ == '__main__': outpth = f'./{sys.argv[1]}' os.makedirs(outpth, exist_ok=True) diff --git a/archive/ukmon_pylib/converters/gmnTxtToPandas.py b/archive/ukmon_pylib/converters/gmnTxtToPandas.py index e897c7c4e..958d64b8e 100644 --- a/archive/ukmon_pylib/converters/gmnTxtToPandas.py +++ b/archive/ukmon_pylib/converters/gmnTxtToPandas.py @@ -5,6 +5,7 @@ import os import glob import sys +import datetime colhdrs = ['id','jd_beg','utc_beg','iau_no','iau_code','sollon','lst', @@ -16,6 +17,8 @@ 'Lat1','Lat1sd','Lon1','Lon1sd','H1','H1sd','Lat2','Lat2sd','Lon2','Lon2sd','H2','H2sd', 'Dur','Amag','PkHt','F1','mass','Qc','MedianFitErr','BegIn','EndIn','NumStat','stats'] +dirpath='F:/videos/MeteorCam/gmndata' + def loadOneFile(fname): print(f'processing {fname}') @@ -45,14 +48,92 @@ def loadOneFile(fname): return df +def compareTwoFiles(file1, file2): + df1 = loadOneFile(file1) + df2 = loadOneFile(file2) + + df1['dupe']=df1.duplicated(subset=['id']) + print(f'file 1 contains {len(df1[df1.dupe==True].id.unique())} duplicates') + dupe1 = df1[df1.dupe==True] # noqa: E712 + df1.drop(columns=['dupe']) + + df2['dupe']=df2.duplicated(subset=['id']) + print(f'file 2 contains {len(df2[df2.dupe==True].id.unique())} duplicates') + dupe2 = df2[df2.dupe==True] # noqa: E712 + df2.drop(columns=['dupe']) + + mrg = df1.merge(df2.drop_duplicates(), on=['id'], how='left', indicator=True) + old_not_new = mrg[mrg._merge!='both'] + print(f'there are {len(old_not_new)} rows in the first file not in the second') + + mrg = df2.merge(df1.drop_duplicates(), on=['id'], how='left', indicator=True) + new_not_old = mrg[mrg._merge!='both'] + print(f'there are {len(new_not_old)} rows in the second file not in the first') + + print(old_not_new) + old_not_new.to_csv('./old_not_new.csv') + new_not_old.to_csv('./old_not_new.csv') + return df1,df2, old_not_new, new_not_old, dupe1, dupe2 + + +def aggregateOneMonth(yr, mth): + startdt = datetime.datetime(yr, mth, 1) - datetime.timedelta(days=1) + syr = startdt.year + smth = startdt.month + sday = startdt.day + datafiles = glob.glob(os.path.join(dirpath, 'daily', f'traj_summary_{syr:04d}{smth:02d}{sday:02d}*.txt')) + prevfile = datafiles[0] + datafiles = [prevfile] + glob.glob(os.path.join(dirpath, 'daily', f'traj_summary_{yr:04d}{mth:02d}*.txt')) + mthlydata = None + for datfile in datafiles: + newdata = loadOneFile(datfile) + if mthlydata is None: + mthlydata = newdata + else: + mthlydata = pd.concat([mthlydata, newdata], sort=True) + mthlydata = mthlydata[mthlydata.utc_beg >= datetime.datetime(yr,mth,1,0,0,0)] + return mthlydata + + def doYear(yr): - dirpath='F:/videos/MeteorCam/gmndata' - datafiles = glob.glob(os.path.join(dirpath, f'traj_summary_monthly_{yr}*.txt')) + datafiles = glob.glob(os.path.join(dirpath, 'monthly', f'traj_summary_monthly_{yr}*.txt')) for datfile in datafiles: + print(f'processing {datfile}') newdata = loadOneFile(datfile) fn, _ = os.path.splitext(datfile) ym = fn[-6:] - newdata.to_parquet(os.path.join(dirpath, 'parquet', f'gmn_{ym}.parquet.snap'), index=False) + newdata.to_parquet(os.path.join(dirpath, 'parquet', 'monthly', f'gmn_{ym}.parquet.snap'), index=False) + + datafiles = glob.glob(os.path.join(dirpath, f'traj_summary_yearly_{yr}.txt')) + for datfile in datafiles: + print(f'processing {datfile}') + newdata = loadOneFile(datfile) + newdata.to_parquet(os.path.join(dirpath, 'parquet', f'gmn_{yr}.parquet.snap'), index=False) + return + + +def getStats(): + for yr in range(2022,2025): + df = pd.read_parquet(f'gmn_{yr}01.parquet.snap') + for mth in range(2,13): + try: + df2 = pd.read_parquet(f'gmn_{yr}{mth:02d}.parquet.snap') + df = pd.concat(df, df2) + except: + pass +# df = df[df.Amag < -3.99] + print(f'{yr}, {len(df)}, {df.Lat1.mean():.1f}, {df.Lat1.max():.1f}, {df.Lat1.min():.1f}, {df.Lat1.std():.1f}') + print('-----') + for yr in range(2022,2025): + df = pd.read_parquet(f'gmn_{yr}01.parquet.snap') + for mth in range(2,13): + try: + df2 = pd.read_parquet(f'gmn_{yr}{mth:02d}.parquet.snap') + df = pd.concat(df, df2) + except: + pass + df = df[df.Amag < -3.99] + print(f'{yr}, {len(df)}, {df.Lat1.mean():.1f}, {df.Lat1.max():.1f}, {df.Lat1.min():.1f}, {df.Lat1.std():.1f}') if __name__ == '__main__': diff --git a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py index 0e7f99532..cb74b0163 100644 --- a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py +++ b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py @@ -40,7 +40,9 @@ def getImgList(outdir, traj): imglist = os.listdir(os.path.join(outdir, '..', 'jpgs')) return imglist orbname=traj.output_dir.replace('\\','/').split('/')[-1] - testdt = datetime.datetime.strptime(orbname.replace('-','_')[:15], '%Y%m%d_%h%m%s') + if orbname == '.': + orbname = os.path.split(outdir)[1] + testdt = datetime.datetime.strptime(orbname.replace('-','_')[:15], '%Y%m%d_%H%M%S') testdt = testdt + datetime.timedelta(seconds=-10) dtstr = testdt.strftime('%Y-%m-%dT%H:%M:%S.000Z') dtstr2 = (testdt + datetime.timedelta(seconds=30)).strftime('%Y-%m-%dT%H:%M:%S.000Z') diff --git a/archive/ukmon_pylib/metrics/getMatchStats.py b/archive/ukmon_pylib/metrics/getMatchStats.py index da0c3c714..8114c4e3a 100644 --- a/archive/ukmon_pylib/metrics/getMatchStats.py +++ b/archive/ukmon_pylib/metrics/getMatchStats.py @@ -7,6 +7,32 @@ import sys +def getDailyObsCounts(logf): + loglines = open(logf).readlines() + addlines = [line.strip() for line in loglines if ('Added' in line and 'observations' in line) or 'Processing station' in line] + res=[0,0,0,0] + stn = '' + offs = 0 + totalval = 0 + uniquestns = 0 + for i in range(len(addlines)): + if 'Processing' in addlines[i]: + print(addlines[i]) + newstn = addlines[i].split(' ')[-1] + if stn != newstn: + stn = newstn + offs = 0 + uniquestns += 1 + if 'Processing' in addlines[i+1]: + continue + val = int(addlines[i+1].split(' ')[1]) + totalval += val + res[offs] += val + offs += 1 + print(res, uniquestns) + return + + def getMatchStats(logf): with open(logf) as inf: loglines = inf.readlines() diff --git a/archive/ukmon_pylib/reports/meteoriteTools.py b/archive/ukmon_pylib/reports/meteoriteTools.py new file mode 100644 index 000000000..4f7c72e7b --- /dev/null +++ b/archive/ukmon_pylib/reports/meteoriteTools.py @@ -0,0 +1,55 @@ +# Find cameras near to a point and other tools relevant to meteorite finds + +# Copyright (C) 2018-2023 Mark McIntyre + +import sys +import json +import pandas as pd +import numpy as np +import requests + +from meteortools.utils.Math import greatCircleDistance + + +def stationsNearPoint(lat, lon, dist=75, email_only=True): + lat = np.radians(lat) + lon = np.radians(lon) + res = requests.get('https://archive.ukmeteors.co.uk/browse/cameraLocs.json') + if res.status_code != 200: + return None + jsdata = json.loads(res.text) + camlocs = pd.DataFrame(jsdata).transpose() + camlocs.drop(columns=['ele','az','alt','fov_h','fov_v','rot'], inplace=True) + camlocs['dist'] = [greatCircleDistance(np.radians(statlat), np.radians(statlon), lat, lon) + for statlat,statlon in zip(camlocs.lat, camlocs.lon)] + nearby = camlocs[camlocs.dist < dist] + cams = {} + for cam in nearby.index: + res = requests.get(f'https://api.ukmeteors.co.uk/camdetails?camid={cam}') + if res.status_code == 200: + dta = json.loads(res.text) + if len(dta) > 0: + cams[cam] = {'email': dta[0]['eMail']} + else: + cams[cam] = {'email':'unknown'} + else: + cams[cam] = {'email':'unknown'} + camlist = nearby.merge(pd.DataFrame(cams).transpose(), how='left', left_index=True, right_index=True) + if email_only: + camlist.drop(columns=['lat','lon','dist'], inplace=True) + camlist.drop_duplicates(inplace=True) + return camlist + + +def getCoordsFromTraj(trajname): + lat = 0 + lon = 0 + return lat, lon + + +if __name__ == '__main__': + lat = float(sys.argv[1]) + lon = float(sys.argv[2]) + dist = float(sys.argv[3]) + + print(stationsNearPoint(lat, lon, dist)) diff --git a/archive/utils/createTestDataSet.sh b/archive/utils/createTestDataSet.sh index 66601bc74..499bf238d 100644 --- a/archive/utils/createTestDataSet.sh +++ b/archive/utils/createTestDataSet.sh @@ -4,14 +4,30 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 -YYMMDD=20220120 +if [ "$1" == "" ] ; then + echo bad input + exit 0 +fi +dt=$1 +numdays=$2 basepth=~/dev/data/RMSCorrelate -mkdir $basepth +mkdir -p $basepth -fldrs=$(aws s3 ls s3://ukmda-shared/matches/RMSCorrelate/ | egrep "UK|BE|IE" | awk '{print $2}') +fldrs=$(aws s3 ls s3://ukmda-shared/matches/RMSCorrelate/ | egrep "UK|BE|IE|NL" | awk '{print $2}') for fldr in $fldrs ; do - camid=${fldr:0:6} - aws s3 sync s3://ukmda-shared/matches/RMSCorrelate/${fldr} ${basepth}/${fldr} --exclude "*" --include "${camid}_${YYMMDD}*" + for i in $(seq 1 $numdays) ; do + d2=$(python -c "import datetime;d1=datetime.datetime.strptime('$dt', '%Y%m%d')+datetime.timedelta(days=$i-1);print(d1.strftime('%Y%m%d'))") + camid=${fldr:0:6} + echo checking $camid for $d2 + aws s3 sync s3://ukmda-shared/matches/RMSCorrelate/${fldr} ${basepth}/${fldr} --exclude "*" --include "${camid}_${d2}*" + done done + +for i in $(seq 1 $numdays) ; do + d2=$(python -c "import datetime;d1=datetime.datetime.strptime('$dt', '%Y%m%d')+datetime.timedelta(days=$i-1);print(d1.strftime('%Y%m%d'))") + echo getting trajectories for $d2 + fldr="trajectories/${d2:0:4}/${d2:0:6}/$d2" + aws s3 sync s3://ukmda-shared/matches/RMSCorrelate/${fldr} ${basepth}/${fldr} --exclude "*" --include "${d2}*" +done diff --git a/archive/website/cameraStatusReport.sh b/archive/website/cameraStatusReport.sh index e2923f5fb..537a7b2fe 100644 --- a/archive/website/cameraStatusReport.sh +++ b/archive/website/cameraStatusReport.sh @@ -34,7 +34,7 @@ aws s3 cp $DATADIR/reports/stationlogins.txt $WEBSITEBUCKET/reports/stationlogin cp $TEMPLATES/header.html $DATADIR/reports/statrep.html -echo "

    Camera status report for the network.

    This page provides a status report " >> $DATADIR/reports/statrep.html +echo "

    Camera status report for the network.

    This page provides a status report " >> $DATADIR/reports/statrep.html echo "on the feed of daily data from cameras in the network. RMS cameras are reported red if more than three days " >> $DATADIR/reports/statrep.html echo "late. UFO cameras are reported red if more than 14 days late." >> $DATADIR/reports/statrep.html echo "The date & time are that of the start of the last data capture run recieved." >> $DATADIR/reports/statrep.html diff --git a/archive/website/costReport.sh b/archive/website/costReport.sh index b0aa30606..cd60ec06f 100644 --- a/archive/website/costReport.sh +++ b/archive/website/costReport.sh @@ -47,7 +47,7 @@ v3=$(cat $DATADIR/costs/costs-183798037734-last.csv) lastfullcost=$v3 cp $TEMPLATES/header.html $costfile -echo "

    Daily running costs

    " >> $costfile +echo "

    Daily running costs

    " >> $costfile echo "

    This page shows daily running costs by service of the Archive and Calculation Engine " >> $costfile echo "in USD.
    " >> $costfile echo "The last complete day's cost is \$${lastfullcost}" >> $costfile diff --git a/archive/website/createReportIndex.sh b/archive/website/createReportIndex.sh index 378543409..edf24c4f8 100644 --- a/archive/website/createReportIndex.sh +++ b/archive/website/createReportIndex.sh @@ -28,6 +28,7 @@ else fi cd ${DATADIR}/reports +mkdir -p ${DATADIR}/reports/${fldr} repidx=$fldr/reportindex.js echo "\$(function() {" > $repidx diff --git a/archive/website/templates/coverage-maps.html b/archive/website/templates/coverage-maps.html index 66f02c93d..aedfedc8d 100644 --- a/archive/website/templates/coverage-maps.html +++ b/archive/website/templates/coverage-maps.html @@ -77,6 +77,7 @@

    +

    Station Coverage Maps

    To use this page, select a camera or cameras from the dropdown, then select an altitude. diff --git a/archive/website/templates/fbreportindex.html b/archive/website/templates/fbreportindex.html index d9da20b4f..1ec505246 100644 --- a/archive/website/templates/fbreportindex.html +++ b/archive/website/templates/fbreportindex.html @@ -48,7 +48,7 @@

    - +

    The table below shows a list of fireballs and other possibly interesting events.

    The data here are released under the CC BY 4.0 license, so if you are using the data diff --git a/archive/website/templates/frontpage.html b/archive/website/templates/frontpage.html index d08ab16df..4ff72d8d6 100644 --- a/archive/website/templates/frontpage.html +++ b/archive/website/templates/frontpage.html @@ -8,7 +8,7 @@ - + @@ -45,10 +45,15 @@

    -

    Welcome to the UK Meteor data archive, the data archive of the UK Meteor Network.

    +
    +
    +

    Welcome to the data archive of the UK Meteor Network (UKMON).

    This site hosts our data archive. Our main public website is here and you can find us on Social Media here

    +

    + To view our livestream of images from the camera network, click the here or on Live on the menu above. +

    Use the menus above or click a link in the table below the graph to access reports on annual activity, shower activity and individual stations in our network. You can also search for detections matching diff --git a/archive/website/templates/reportindex.html b/archive/website/templates/reportindex.html index f25d6ab23..3b5c7d9f4 100644 --- a/archive/website/templates/reportindex.html +++ b/archive/website/templates/reportindex.html @@ -50,7 +50,7 @@

    - +

    The table below provides access to a summary report by year and by shower, and reports on the trajectories and orbits of matched events.

    The Annual and Shower reports provide a summary of the number of meteors detected, diff --git a/archive/website/templates/searchdialog.js b/archive/website/templates/searchdialog.js index a53350829..e85d171cf 100644 --- a/archive/website/templates/searchdialog.js +++ b/archive/website/templates/searchdialog.js @@ -98,6 +98,7 @@ function myFunc(myObj) { for (x in myObj) { console.log(myObj[x]); var dtarr = myObj[x].split(','); + var cams = dtarr[4].replaceAll(";", "; "); if (dtarr[0] > 0 ){ txt +='

    • PresidentCurrently VacantPeter Campbell-Burns
    Secretary
    '; dt = new Date(dtarr[0]*1000); @@ -109,7 +110,7 @@ function myFunc(myObj) { txt += ""; txt += dtarr[3]; txt += ""; - txt += dtarr[4]; + txt += cams; txt += ""; errimg = "onerror=\"this.onerror=null;this.src='/img/missing.png';\""; txt += "\"Image"; diff --git a/archive/website/templates/shwrcsvindex.html b/archive/website/templates/shwrcsvindex.html index 64a4a87e0..8ea0a6659 100644 --- a/archive/website/templates/shwrcsvindex.html +++ b/archive/website/templates/shwrcsvindex.html @@ -48,6 +48,7 @@
    +

    The data here are released under the CC BY 4.0 license, so if you are using the data whether for scientific or other purposes, your must reference this web site in your work. diff --git a/archive/website/templates/statreportindex.html b/archive/website/templates/statreportindex.html index a0fdae7fe..b62bd482a 100644 --- a/archive/website/templates/statreportindex.html +++ b/archive/website/templates/statreportindex.html @@ -46,7 +46,7 @@

    - +

    The table below provides access to a summary report by station.

    The data here are released under the CC BY 4.0 license, so if you are using the data diff --git a/tests/test_apis.py b/tests/test_apis.py index 7c81fe892..4394b5d16 100644 --- a/tests/test_apis.py +++ b/tests/test_apis.py @@ -172,7 +172,7 @@ def test_getECSV(): apiurl='https://api.ukmeteors.co.uk/getecsv?stat={}&dt={}' res = requests.get(apiurl.format(stationID, dateStr)) assert res.status_code == 200 - assert res.text.split()[-1] == 'time' + assert res.text.split()[-1] == '593.580' stationID = 'UK0080' dateStr = '2024-02-25T20:02:00.1' @@ -188,6 +188,6 @@ def test_getECSVbadDate(): dateStr = '2024-02-25T20:02:' apiurl='https://api.ukmeteors.co.uk/getecsv?stat={}&dt={}' res = requests.get(apiurl.format(stationID, dateStr)) - assert res.status_code == 200 - assert res.text.split()[-1] == 'format' + assert res.status_code == 502 + #assert res.text.split()[-1] == 'format' return diff --git a/usermgmt/deploy-server.yml b/usermgmt/deploy-server.yml index ff6ebb521..db5f0ef94 100644 --- a/usermgmt/deploy-server.yml +++ b/usermgmt/deploy-server.yml @@ -1,7 +1,9 @@ --- - hosts: "{{host | default ('ukmonhelper2')}}" + vars_files: + - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/usermgmt/server + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/usermgmt/server" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/usermgmt/dev-vars.yml b/usermgmt/dev-vars.yml index 5dfce2f67..a459a2431 100644 --- a/usermgmt/dev-vars.yml +++ b/usermgmt/dev-vars.yml @@ -1,5 +1,5 @@ - winsrcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/usermgmt/windows - windestdir: /mnt/e/dev/meteorhunting/testing/usermgmt + winsrcdir: /mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/usermgmt/windows + windestdir: /mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/testing/usermgmt destdir: /home/ec2-user/dev/keymgmt srcbucket: mjmm-ukmon-shared livebucket: mjmm-ukmon-live diff --git a/usermgmt/prod-vars.yml b/usermgmt/prod-vars.yml index 63b88f793..bd561ec78 100644 --- a/usermgmt/prod-vars.yml +++ b/usermgmt/prod-vars.yml @@ -1,4 +1,4 @@ - winsrcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/usermgmt/windows + winsrcdir: /mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/usermgmt/windows windestdir: /mnt/f/videos/MeteorCam/usermgmt destdir: /home/ec2-user/keymgmt srcbucket: ukmda-shared diff --git a/usermgmt/build.ps1 b/usermgmt/windows/build.ps1 similarity index 100% rename from usermgmt/build.ps1 rename to usermgmt/windows/build.ps1 diff --git a/usermgmt/windows/stationMaint.ps1 b/usermgmt/windows/stationMaint.ps1 index 28eeee7ef..23b12ae38 100644 --- a/usermgmt/windows/stationMaint.ps1 +++ b/usermgmt/windows/stationMaint.ps1 @@ -4,15 +4,15 @@ push-Location $psscriptroot # create conda env if not aleady there $isadm=(conda env list | select-string "ukmon-admin") if ($isadm.count -eq 0) { - conda env create -n ukmon-admin python=3.8 + conda create -n ukmon-admin python=3.11 pip install -r requirements.txt } conda activate ukmon-admin bash -c "rsync -avz --delete ukmonhelper2:keymgmt/sshkeys/ ./sshkeys" -bash -c "rsync -avz --delete ukmonhelper2:keymgmt/sshkeys/ ./keys" -bash -c "rsync -avz --delete ukmonhelper2:keymgmt/sshkeys/ ./csvkeys" -bash -c "rsync -avz --delete ukmonhelper2:keymgmt/sshkeys/ ./inifs" +bash -c "rsync -avz --delete ukmonhelper2:keymgmt/keys/ ./keys" +bash -c "rsync -avz --delete ukmonhelper2:keymgmt/csvkeys/ ./csvkeys" +bash -c "rsync -avz --delete ukmonhelper2:keymgmt/inifs/ ./inifs" python stationMaint2.py Pop-Location \ No newline at end of file diff --git a/usermgmt/windows/stationMaint2.py b/usermgmt/windows/stationMaint2.py index 5e5c8a8c8..cf6a81d3e 100644 --- a/usermgmt/windows/stationMaint2.py +++ b/usermgmt/windows/stationMaint2.py @@ -724,9 +724,9 @@ def addNewOwner(self, rmsid, location, user, email, direction, camtype, active, def addNewAwsUser(self, location): log.info(f'adding new location {location} to AWS') - archkeyf = 'jsonkeys/' + location + '_arch.key' - archuserdets = 'users/' + location + '_arch.txt' - archcsvf = os.path.join('csvkeys', location.lower() + '_arch.csv') + archkeyf = 'jsonkeys/' + location + '.key' + archuserdets = 'users/' + location + '.txt' + archcsvf = os.path.join('csvkeys', location + '.csv') os.makedirs('jsonkeys', exist_ok=True) os.makedirs('csvkeys', exist_ok=True) os.makedirs('users', exist_ok=True) @@ -736,7 +736,7 @@ def addNewAwsUser(self, location): _ = iamc.get_user(UserName=location) log.info('location exists, not adding it') archkey = None - except Exception: + except iamc.exceptions.NoSuchEntityException: log.info('new location') usr = iamc.create_user(UserName=location) with open(archuserdets, 'w') as outf: diff --git a/usermgmt/windows/stationmaint.cfg b/usermgmt/windows/stationmaint.cfg index 883b6cd0f..210eb98de 100644 --- a/usermgmt/windows/stationmaint.cfg +++ b/usermgmt/windows/stationmaint.cfg @@ -4,6 +4,7 @@ SSHKEY=~/.ssh/ukmonhelper HELPERIP=3.11.55.160 REMOTEDIR=/home/ec2-user/prod/data PLATEPARDIR= + [store] SRCBUCKET=ukmda-shared LIVEBUCKET=ukmda-live