Merge pull request #394 from ukmda/master

reverse merge from master

Author: ukmda

.github/workflows/build_usermgmt.yml

@@ -0,0 +1,71 @@
+name: build_usermgmt
+env:
+    AWS_REGION: eu-west-2
+permissions:
+    id-token: write
+    contents: read
+
+on:
+    push:
+        branches: [ dev, markmac99, master ]
+    pull_request:
+        # The branches below must be a subset of the branches above
+        branches: [ master ]
+jobs:
+    local_tests:
+        name: local test stage
+        runs-on: win10
+        steps:
+        - name: "☁️ checkout repository"
+          uses: actions/checkout@v4
+        - name: testing
+          run: | 
+            echo "hello from test"
+            conda activate ukmon-admin
+            dir usermgmt\windows
+    build:
+        name: build stage
+        runs-on: win10
+        needs: local_tests
+        steps:
+        - name: building
+          run: |
+            echo "running build step"
+            conda activate ukmon-admin
+            cd usermgmt\windows
+            pyinstaller ./stationMaint2.py --noconsole --onefile --windowed --icon .\camera.ico
+            dir dist
+    package:
+        name: package app
+        runs-on: win10
+        needs: 
+        - local_tests
+        - build
+        steps:
+        - name: packaging
+          run: | 
+            echo "packaging the app now"
+            cd usermgmt\windows\dist
+            copy ../camera.ico .
+            ((get-content ..\stationmaint.cfg) -replace "adm_mark","") -replace "~/.ssh/ukmonhelper","" | set-content -path ./stationmaint.cfg
+            compress-archive -path . -update -destinationpath c:\temp\ukmon_usermgmt.zip
+    release:
+        name: release package
+        runs-on: win10
+        needs:
+        - local_tests
+        - build
+        - package
+        steps:
+        - name: release
+          uses: xresloader/upload-to-github-release@v1
+          env:
+            GITHUB_TOKEN: ${{ secrets.MM_PAT }}
+          with:
+            file: c:\temp\ukmon_usermgmt.zip
+            #tags: true
+            draft: true 
+            overwrite: true
+            tag_name: 2024.04.02
+            default_release_name: User Management Tool 
+            default_release_body_path: usermgmt\windows\release_notes.md

README.md

@@ -1,4 +1,5 @@
 # UK Meteor Data Analysis Shared code and libraries
+version: 2024.04.2
 
 This repository contains the code behind the UK Meteors data archive and data processing pipeline. 
 

api_examples/api_examples.py

@@ -0,0 +1,97 @@
+import pandas as pd
+import json 
+import requests
+
+
+def detectionsExample():
+    apiurl='https://api.ukmeteors.co.uk/getecsv?stat={}&dt={}'
+
+    # retrieve details of a single-station detection at a specific time
+    stat ='UK0025'
+    dt = '2021-07-17T02:41:05.05'
+    res = requests.get(apiurl.format(stat, dt))
+    if res.status_code == 200:
+        rawdata=res.text
+        ecsvlines=rawdata.split('\n') # convert the raw data into a python list
+        numecsvs = len([e for e in ecsvlines if '# %ECSV' in e]) # find out how many meteors 
+        fnamebase = dt.replace(':','_').replace('.','_') # create an output filename
+        if numecsvs == 1:
+            with open(fnamebase + '.ecsv', 'w') as outf:
+                outf.writelines(ecsvlines)
+        else:
+            outf = None
+            j=1
+            for i in range(len(ecsvlines)):
+                if '# %ECSV' in ecsvlines[i]:
+                    if outf is not None:
+                        outf.close()
+                        j=j+1
+                    outf = open(fnamebase + f'_M{j:03d}.ecsv', 'w')
+                outf.write(f'{ecsvlines[i]}\n')
+
+
+def matchExampleA():
+    apiurl = 'https://api.ukmeteors.co.uk/matches'
+
+    # get all matched events for a given date
+    reqtyp = 'matches'
+    reqval = '20231121'
+    apicall = '{}?reqtyp={}&reqval={}'.format(apiurl, reqtyp, reqval)
+    matchlist = pd.read_json(apicall, lines=True)
+    print(matchlist)
+
+    # get details of the 6th event in matchlist
+    reqtyp = 'detail'
+    reqval = matchlist.iloc[5].orbname
+    apicall = '{}?reqtyp={}&reqval={}'.format(apiurl, reqtyp, reqval)
+    evtdetail = pd.read_json(apicall, typ='series')
+    print(evtdetail)
+
+    # get details for the first five events in the match list
+    # and put them in a pandas dataframe, then sort by brightest
+    details=[]
+    for id in matchlist.head(5).orbname:
+        reqval = id
+        apicall = '{}?reqtyp={}&reqval={}'.format(apiurl, reqtyp, reqval)
+        details.append(pd.read_json(apicall, typ='series'))
+    df = pd.DataFrame(details)
+    df = df.sort_values(by=['_mag'])
+    print(df)
+
+
+def matchSummaryExample():
+    apiurl = 'https://api.ukmeteors.co.uk/matches'
+    
+    # get summary of all events from a specific date
+    reqtyp = 'summary'
+    reqval = '20231121'
+    apicall = '{}?reqtyp={}&reqval={}'.format(apiurl, reqtyp, reqval)
+    data=requests.get(apicall)
+    if data.status_code == 200:
+        df = pd.DataFrame(json.loads(data.text))
+        print(df._lat1)
+        print(df.orbname)
+        rec = df[df._localtime =='_20231121_212437']
+        print(rec._lat1)
+
+
+def trajectoryExample():
+    apiurl = 'https://api.ukmeteors.co.uk/pickle/getpickle'
+
+    # get the complete raw and processed data for a given solution
+    trajid = '20220814_205940.252_UK'
+    apicall = '{}?reqval={}'.format(apiurl, trajid)
+    data=requests.get(apicall)
+    if data.status_code == 200:
+        traj = json.loads(data.text)
+        print(traj['jdt_ref'])
+
+        # observations contains the raw data
+        observations = traj['observations']
+        print(len(observations))
+        for obs in observations:
+            k = list(obs.keys())[0]
+            print(obs[k]['station_id'])
+
+        # print all fields
+        print(traj.keys())

archive/README.md

@@ -12,14 +12,13 @@ This diagram shows the overall flow of data from Cameras to websites and out to
     P -- next day --> D;
     D --> E[matching engine];
     E --> F[reports generator];
-    F --> G[opt-in email of matches];
+    F --> G[match report on groups.io];
     F --> I[bad-data alerts];
     F --> H[Archive];
-    H -- on demand --> J[other networks];
-    H --> M[public];
-    C --> M[public];
+    H --> M[website];
+    C --> M[website];
     C --> N{bright event?};
-    N -->|yes| K[social media];
+    N --> K[social media];
     H -- manual --> K;
     I --> O[camera owners];
     G --> O;
@@ -43,10 +42,12 @@ This diagram shows the various scripts and processes called, in order
     rep3 --> summ[Create summary date for homepage]
     summ --> camst[Create Camera status reports]
     camst --> exch[Create files for exchange with other networks]
-    exch --> stats[Create Station Reports]
-    stats --> mets[Create Metrics]
-    mets --> badc[Send out bad camera emails]
-    badc --> d[FINISHED]
+    exch --> badc[Send out bad camera emails]
+    badc --> cost[Create Cost Report]
+    cost --> stats[Create Station Reports]
+    stats --> clsp[Clear space]
+    clsp --> mdb[update mariadb tables]
+    mdb --> d[FINISHED]
 ```
 
 Find All Matches
@@ -56,33 +57,28 @@ Flow in findAllMatches.sh
 ```mermaid
     flowchart TD
 
-    setd[Set the Dates] --> rmssngl[Create RMS single station data files]
-    rmssngl --> bkp1[Backup the current trajectory database]
-    bkp1 --> rmat[Invoke runDistrib]
-    rmat --> start[runDistrib Start calc engine]
-    start --> creat[runDistrib Create batch script and copy to server]
-    creat --> sync1[runDistrib script syncs new ftpdetect files]
-    sync1 --> run[runDistrib  Then runs the the Phase 1 solver]
-    run --> sync2[runDistrib Then syncs back to S3]
-    sync2 --> done[runDistrib Stop calc engine]
-    done --> stage2[run the distributed solver phase 2 using ECS and Fargate]
-    stage2 --> consol[wait for containers to finish then consolidate the data]
-    consol --> repo[Create file of latest matches]
-    repo --> stats[Create Stats and sync back to S3]
-    stats --> idxpg[Update website index pages]
-    idxpg --> gzi[Backup and gzip old trajectory databases]
-    gzi --> d[FINISHED]
+    setd[findAllMatches sets up the date range] --> bkp1[findAllMatches saves the current trajectory database]
+    bkp1 --> rmat[findAllMatches then invokes runDistrib]
+    rmat --> start[runDistrib Starts the calc engine server]
+    start --> creat[runDistrib Creates batch script and copies to calc engine]
+    creat --> sync1[runDistrib syncs new ftpdetect files to the calc engine]
+    sync1 --> run[runDistrib runs the the Phase 1 solver. This identifies candidate groups]
+    run --> sync2[runDistrib syncs data back to the data store]
+    sync2 --> done[runDistrib stops calc engine for now, to save cost]
+    done --> stage2[runDistrib starts the distributed solver phase 2 using ECS and Fargate]
+    stage2 --> consol[runDistrib waits for containers to finish then consolidate the data]
+    consol --> repo[runDistrib creates file of latest matches]
+    repo --> stats[runDistrib syncs the solved orbit files back to S3 and backs up the data]
+    stats --> idxpg[findAllMatches creates some stats, updates website index pages and sends the daily report]
+    idxpg --> d[FINISHED]
 ```
 
 Flow in terms of files
 ======================
 ```mermaid
     flowchart LR
 
-    nightly[cronjobs/nightlyJob.sh] --> datasync[utils/dataSync.sh]
-    nightly --> findmatches[analysis/findAllMatches.sh]
-    findmatches --> rmssngl[analysis/getRMSSingleData.sh]
-    findmatches --> srchabl[analysis/createSearchable.sh]
+    nightly[cronjobs/nightlyJob.sh] --> findmatches[analysis/findAllMatches.sh]
     findmatches --> rundist[analysis/runDistrib.sh]
     rundist --> trajcont[launches docker containers which solve and post directly to the website]
     findmatches --> daily[creates daily report and stats]
@@ -106,8 +102,8 @@ Flow in terms of files
     nightly --> statstat[analysis/getBadStations.sh]
     nightly --> costs[website/costReport.sh]
     nightly --> statreps[analysis/stationReports.sh]
-    nightly --> databack[utils/dataSyncBack.sh ]
     nightly --> clearsp[utils/clearSpace.sh ]
+    nightly --> mariadb[utils/loadMatchCsvDB.sh ]
     nightly --> getlogs[analysis/getLogData.sh]
     nightly --> done[done]
 

archive/analysis/getLogData.sh

@@ -16,8 +16,7 @@ fi
 # create performance metrics
 cd $SRC/logs
 python -m metrics.timingMetrics $rundate
-mv -f $rundate-perfNightly.jpg $DATADIR/
-aws s3 cp $DATADIR/$rundate-perfNightly.jpg $WEBSITEBUCKET/reports/batchcharts/ --quiet
+aws s3 cp $DATADIR/batchcharts/$rundate-perfNightly.jpg $WEBSITEBUCKET/reports/batchcharts/ --quiet
 
 #lastmtch=$(ls -1tr $SRC/logs/matches-${rundate}-*.log | tail -1)
 if [ $rundate != $(date +%Y%m%d) ] ; then

archive/analysis/getRMSSingleData.sh

@@ -33,10 +33,10 @@ aws s3 mv $indir $outdir --recursive --exclude "*" --include "ukmda_??????_${yr}
 mrgfile=$DATADIR/single/singles-${yr}.csv
 newsngl=$DATADIR/single/singles-${yr}-new.csv
 if [ ! -f $mrgfile ] ; then 
-    echo "Ver,Y,M,D,h,mi,s,Mag,Dur,Az1,Alt1,Az2,Alt2,Ra1,Dec1,Ra2,Dec2,ID,Long,Lat,Alt,Tz,AngVel,Shwr,Filename,Dtstamp" > $mrgfile
+    cat $SRC/analysis/templates/ukmon-single.txt > $mrgfile
 fi 
 # file containing only new data
-echo "Ver,Y,M,D,h,mi,s,Mag,Dur,Az1,Alt1,Az2,Alt2,Ra1,Dec1,Ra2,Dec2,ID,Long,Lat,Alt,Tz,AngVel,Shwr,Filename,Dtstamp" > $newsngl
+cat $SRC/analysis/templates/ukmon-single.txt >> $newsngl
 
 ls -1 $outdir/ukmda_??????_${yr}*.csv | while read i
 do