Various bugfixes and enhancements

.gitignore

@@ -593,10 +593,19 @@ archive/samfunctions*/camDetails/pythoncode/camDetails.py
 archive/samfunctions*/camDetails/pythoncode/awskeys
 archive/containers/RMS-docker/rms_docker.tar
 archive/containers/RMS-docker/Dockerfile_orig.txt
+
+usermgmt/windows/csvkeys
+usermgmt/windows/inifs
+usermgmt/windows/jsonkeys
+usermgmt/windows/keys
+usermgmt/windows/sshkeys
+usermgmt/windows/stationdetails
+usermgmt/windows/users
 usermgmt/gmailkeys/*
 usermgmt/server/ukmonfundraising.txt
 usermgmt/server/ukmoncommittee.txt
 usermgmt/server/ukmon*.json
+
 **/awskeys.test
 **/trajsolver_old/*
 archive/tmp/*

archive/cronjobs/nightlyJob.sh

@@ -29,6 +29,7 @@ mkdir -p $DATADIR/browse/{annual,monthly,daily,showers}
 log2cw $NJLOGGRP $NJLOGSTREAM "updating the camera location/dir/fov database" nightlyJob
 python -c "from reports.CameraDetails import updateCamLocDirFovDB; updateCamLocDirFovDB();"
 aws s3 cp $DATADIR/admin/cameraLocs.json $UKMONSHAREDBUCKET/admin/ --profile ukmonshared --quiet
+aws s3 cp $DATADIR/admin/cameraLocs.json $WEBSITEBUCKET/browse/ --profile ukmonshared --quiet
 aws s3 sync $UKMONSHAREDBUCKET/admin/ $DATADIR/admin --profile ukmonshared --quiet
 
 # create the CSV file of camera info and the html versions for search functions on the website

archive/deployment/analysis.yml

@@ -1,7 +1,9 @@
 ---
 - hosts: "{{host | default ('ukmonhelper2')}}"
+  vars_files: 
+    - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc
   vars:
-    srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive
+    srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive"
   tasks:
   - name: import dev variables 
     include_vars: ./dev-vars.yml

archive/deployment/config.yml

@@ -1,6 +1,8 @@
 - hosts: "{{host | default ('ukmonhelper2')}}"
+  vars_files: 
+    - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc
   vars:
-    srcdir: /mnt/e/dev/meteorhunting/ukmda-datapocessing/archive
+    srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive"
   tasks:
   - name: import dev variables 
     include_vars: dev-vars.yml

archive/deployment/cronjobs.yml

@@ -1,7 +1,9 @@
 ---
 - hosts: "{{host | default ('ukmonhelper2')}}"
+  vars_files: 
+    - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc
   vars:
-    srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive
+    srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive"
   tasks:
   - name: import dev variables 
     include_vars: ./dev-vars.yml

archive/deployment/database.yml

@@ -1,7 +1,9 @@
 ---
 - hosts: "{{host | default ('ukmonhelper2')}}"
+  vars_files: 
+    - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc
   vars:
-    srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive
+    srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive"
   tasks:
   - name: import dev variables 
     include_vars: ./dev-vars.yml

archive/deployment/pylib.yml

@@ -1,7 +1,9 @@
 ---
 - hosts: "{{host | default ('ukmonhelper2')}}"
+  vars_files: 
+    - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc
   vars:
-    srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive
+    srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive"
   tasks:
   - name: import dev variables 
     include_vars: ./dev-vars.yml

archive/deployment/server_setup.yml

@@ -1,7 +1,9 @@
 ---
 - hosts: "{{host | default ('ukmonhelper2')}}"
+  vars_files: 
+    - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc
   vars:
-    srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive
+    srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive"
   tasks:
   - name: import dev variables 
     include_vars: ./dev-vars.yml

archive/deployment/shwrinfo.yml

@@ -1,7 +1,9 @@
 ---
 - hosts: "{{host | default ('ukmonhelper2')}}"
+  vars_files: 
+    - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc
   vars:
-    srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive
+    srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive"
   tasks:
   - name: import dev variables 
     include_vars: ./dev-vars.yml

archive/deployment/utils.yml

@@ -1,7 +1,9 @@
 ---
 - hosts: "{{host | default ('ukmonhelper2')}}"
+  vars_files: 
+    - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc
   vars:
-    srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive
+    srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive"
   tasks:
   - name: import dev variables 
     include_vars: ./dev-vars.yml

archive/deployment/website.yml

@@ -1,7 +1,9 @@
 ---
 - hosts: "{{host | default ('ukmonhelper2')}}"
+  vars_files: 
+    - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc
   vars:
-    srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive
+    srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive"
   tasks:
   - name: import dev variables 
     include_vars: ./dev-vars.yml

archive/deployment/website_static.yml

@@ -1,7 +1,9 @@
 ---
 - hosts: "{{host | default ('ukmonhelper2')}}"
+  vars_files: 
+    - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc
   vars:
-    srcdir: /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive
+    srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive"
   tasks:
   - name: import dev variables 
     include_vars: ./dev-vars.yml
@@ -18,7 +20,7 @@
     with_items:
     - {src: '{{srcdir}}/static_content/browse/', dest: '{{destdir}}/static_content/browse', mode: '754', backup: no, directory_mode: yes }
     - {src: '{{srcdir}}/static_content/css/', dest: '{{destdir}}/static_content/css', mode: '754', backup: no, directory_mode: yes }
-    - {src: '{{srcdir}}/static_content/data/', dest: '{{destdir}}/static_content/data', mode: '754', backup: no, directory_mode: yes }
+    #- {src: '{{srcdir}}/static_content/data/', dest: '{{destdir}}/static_content/data', mode: '754', backup: no, directory_mode: yes }
     - {src: '{{srcdir}}/static_content/docs/', dest: '{{destdir}}/static_content/docs', mode: '754', backup: no, directory_mode: yes }
     - {src: '{{srcdir}}/static_content/fonts/', dest: '{{destdir}}/static_content/fonts', mode: '754', backup: no, directory_mode: yes }
     - {src: '{{srcdir}}/static_content/img/', dest: '{{destdir}}/static_content/img', mode: '754', backup: no, directory_mode: yes }

archive/static_content/browse/annual/index.html

@@ -40,6 +40,7 @@
   </script>
 
   <div class="container">
+    <br><br>
     <p>
       The data here are released under the <a href= "https://creativecommons.org/licenses/by/4.0/">CC BY 4.0</a> license, so if you are using the data 
       whether for scientific or other purposes, your must reference this web site in your work.

archive/static_content/browse/daily/index.html

@@ -63,6 +63,7 @@
   </script>
 
   <div class="container">
+    <br><br>
     <h2>Daily Simplified Report of Detections</h2>
     <p>This page provides a list of all detections in the last 72 hours
       from the camera network, primarily for the purposes of data exchange with

archive/static_content/browse/index.html

@@ -41,6 +41,7 @@
 
 
   <div class="container">
+    <br><br>
     <h2>Browse the Archive.</h2>
     <p>Follow the links below to access downloadable datasets from our archive. </p>
     <p>You can also use our REST APIs to retrieve some data programmatically. If you're interested in 

archive/static_content/browse/monthly/index.html

@@ -41,6 +41,7 @@
   </script>
 
   <div class="container">
+    <br><br>
     <p>
       The data here are released under the <a href= "https://creativecommons.org/licenses/by/4.0/">CC BY 4.0</a> license, so if you are using the data 
       whether for scientific or other purposes, your must reference this web site in your work.

archive/static_content/css/dragontail.css

@@ -141,7 +141,7 @@ body {
 .sticky + .content {
     padding-top: 60px;
   }
-  
+
 .sidebar .sidebar-nav.navbar-collapse {
     padding-right: 0;
     padding-left: 0;
@@ -524,6 +524,12 @@ li > a:only-child:after { content: ''; }
     font-size: 15px;
 }
 
+@media screen and (max-width: 1100px) {
+    .sidenav {
+        display: none !important;
+   }
+}
+
 .table {
     /*color: whitesmoke;*/
     background-color: white;

archive/static_content/docs/Code_of_Conduct.html

@@ -43,6 +43,7 @@
 
 
   <div class="container">
+    <br><br>
     <h2>Contributor Covenant Code of Conduct</h2>
 
     <h3>Our Pledge</h3>

archive/static_content/docs/Committee.html

@@ -53,13 +53,14 @@
 
 
     <div class="container">
+        <br><br>
         <h2>The Committee</h2>
         <p>The Society is managed by a committee consisting of the following offices:</p>
         <p>
         <table>
             <tr>
                 <td>President</td>
-                <td>Currently Vacant</td>
+                <td>Peter Campbell-Burns</td>
             </tr>
             <tr>
                 <td>Secretary</td>

archive/static_content/docs/index.html

@@ -43,6 +43,7 @@
 
 
   <div class="container">
+    <br><br>
     <h2>Documents.</h2>
     <p>UK Meteors is an unincorporated society following the standard model for Astronomy clubs. 
     </p><p>Our main website is <a href=https://ukmeteornetwork.org>here</a> and 

archive/static_content/docs/otherdocs.html

@@ -43,6 +43,7 @@
 
 
   <div class="container">
+    <br><br>
     <h2>Other Documents.</h2>
     <p>A repository of the Society's public documents. </p>
     <p>

archive/static_content/fundraising.html

@@ -40,6 +40,7 @@
     </script>
 
     <div class="container">
+        <br><br>
         <h2>Fundraising</h2>
             If you'd still like to help with data analysis costs, you can donate
             directly via our bank account. This has the advantage that we are not charged administration fees by

archive/static_content/privacy_data.html

@@ -39,16 +39,17 @@
     </script>
 
     <div class="container">
+        <br><br>
         <p><b>Privacy and Data Protection Policy</b></p>
-        <p><em>Last reviewed 2023-08-15</em></p>
+        <p><em>Last reviewed 2025-06-13</em></p>
         <p>This page explains how we handle any personal and sensitive data that we collect. </p>
         <hr>
         <b>Camera ID, name and geographic location of each camera.</b>
         <p>
             We collect the Camera ID, general location name and precise geographic coordinates
             of each camera for the following purposes:
 
-            <li>For Scientific data analysis such as statistical reporting of showers, orbit and trajectory
+            <li>For Scientific data analysis such as statistical reporting of showers, and for orbit and trajectory
                 calculations of meteors detected by the camera. The results of this analysis are made available
                 on this website.
             </li>
@@ -66,7 +67,8 @@
         <p>
             For example, we might collect Camera ID UK1234, location name Sunthorp*, geographic coordinates
             53.62501N 0.08567E, altitude -20m. If a paper is published which makes use of data from this
-            camera, the author might mention the camera ID and approximate location in the paper.
+            camera, the author might mention the camera ID and location name in the paper. The precise
+            geographic coordinates will not be shared without your permission. 
         </p>
         <p>
             This information will be retained indefinitely even if you cease to contribute.
@@ -92,9 +94,9 @@
         <hr>
         <b>Postal Address</b>
         <p>
-            If you have asked us to build a camera for you, the camera builder will also ask for
+            If you have asked us to build a camera for you, the camera builder will ask for
             your full Postal Address so that the items can be posted. We do not retain this information
-            anywhere on our systems.
+            anywhere on our online systems and the camera builder will only share it with your permission.
         </p>
         <hr>
         <b>Publications</b>
@@ -128,7 +130,7 @@
             this data with other meteor networks if they wish to do so. 
             </li>
             <li>By agreeing to contribute, you are agreeing that Level 1 and Level 2 data may be used 
-            by the network coordinators to manually produce other products such as Level 3 data, and that
+            by the network and its coordinators to manually or automatically produce other products such as Level 3 data, and that
             publication of such results may include any Level 2 data that was used, where this is necessary to explain
             or validate the Level 3 data. <br> For example, when Level 2 
             data is combined to calculate an orbit or trajectory (Level 3 data), then the relevant Level 2 data may be made 

archive/static_content/search/index.html

@@ -75,6 +75,7 @@
 
   <div class="container">
     <h2>Search the Archive.</h2>
+    <br><br>
     <p>
       This page allows you to search the archive for specific events in the matched and single-station detections. <br>
       Time should be entered as <em>local time</em>, so in the UK summer you should enter the time in BST.    

archive/static_content/templates/navbar.html

@@ -2,9 +2,9 @@
     <!-- Code Copyright (C) 2018-2023 Mark McIntyre -->
     <nav class="navbar navbar-default navbar-static-top" role="navigation" >
         <div class="navbar-header">
-            <a class="navbar-brand" href="/index.html"><img src="/img/ukmda-logo.png" alt="UKMeteors Logo"
-                    height="65" /></a>
-        </div>
+            <a class="navbar-brand" href="/index.html"><img src="/img/ukmon-logo.png" alt="UKMON Logo" height="30" /></a>
+            <a class="navbar-brand" href="/index.html"><img src="/img/ukmda-logo.png" alt="UKMeteors Logo" height="45" /></a>
+            </div>
 
         <ul class="nav navbar-top-links navbar-right">
             <li class="divider"></li>
@@ -14,7 +14,7 @@
             <li class="dropdown"><a class="nav-item" href="/search/index.html">Search</a></li>
             <li class="dropdown"><a class="nav-item" href="/browse/index.html">Data</a></li>
             <li class="dropdown"><a class="nav-item" href="/docs/index.html">About Us</a></li>
-            <li class="dropdown"><a class="nav-item" href="/live/index.html">Livestream</a></li>
+            <li class="dropdown"><a class="nav-item" href="/live/index.html">Live</a></li>
             <!-- li class="dropdown"><a class="nav-item" href="/privacy_data.html">Privacy</a></li -->
         </ul>
     </nav>

archive/terraform/mjmm/iam.tf

@@ -70,6 +70,14 @@ resource "aws_iam_policy" "pol4s3fullaccess" {
             "*",
           ]
         },
+        {
+            Action = [
+                "iam:GetCredentialReport",
+                "iam:GenerateCredentialReport"
+            ]
+            Effect =  "Allow"
+            Resource = [ "*" ]
+        },
         {
           Sid      = "PassRolePermission"
           Effect   = "Allow"

archive/terraform/ukmda/loggingbucket.tf

@@ -44,7 +44,7 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalogslcp" {
     }
 
     filter {
-      prefix = "archsite/"
+      prefix = "website/"
     }
 
     noncurrent_version_expiration {

archive/ukmon_pylib/analysis/analyseGmnData.py

@@ -0,0 +1,20 @@
+# simple scripts to analyse the GMN data in python
+
+import pandas as pd
+import os
+
+from converters.gmnTxtToPandas import dirpath
+
+
+def findDuplicates(yr, mth=None):
+    if mth:
+        datafile = os.path.join(dirpath, 'parquet', 'monthly', f'gmn_{yr:04d}{mth:02d}.parquet.snap')
+    else:
+        datafile = os.path.join(dirpath, 'parquet', f'gmn_{yr:04d}.parquet.snap')
+    df = pd.read_parquet(datafile)
+    df['dupe']=df.duplicated(subset=['id'])
+    dupeids = df[df.dupe].sort_values(by=['id']).id
+    duperows = df[df.id.isin(dupeids)]
+    print(duperows)
+    print(len(df))
+    return duperows