Merge pull request #414 from ukmda/markmac99

Merge in a bunch of updates from my dev branch

Author: markmac99

.github/workflows/build_tools.yml

@@ -3,22 +3,16 @@
   vars_files: 
     - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc
   vars:
-    srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/usermgmt/server"
+    srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive/usermgmt"
+    destdir: /home/ec2-user/keymgmt
   tasks:
-  - name: import dev variables 
-    include_vars: ./dev-vars.yml
-    tags: dev
-  - name: import prod variables
-    include_vars: ./prod-vars.yml
-    tags: prod
   - name: Ensures {{destdir}} exists
     file: path={{destdir}} state=directory  
-    tags: [dev,prod]
 
   - name: Copy files
-    tags: [dev,prod]
     copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }}
     with_items:
     - {src: '{{srcdir}}/addSftpUser.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no }
     - {src: '{{srcdir}}/updateAwsKey.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no }
-    - {src: '{{srcdir}}/ukmon.ini', dest: '{{destdir}}/', mode: '644', backup: yes, directory_mode: no }
+    - {src: '{{srcdir}}/addAdminUser.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no }
+    - {src: '{{srcdir}}/addFBApiKey.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no }

usermgmt/deploy-server.yml archive/deployment/deploy-usermgmt.ymlusermgmt/deploy-server.yml renamed to archive/deployment/deploy-usermgmt.yml

@@ -9,10 +9,27 @@
 import os
 import pandas as pd
 import datetime 
+import boto3
+
+
+def checkUrl(s3, url):
+    siteroot = os.getenv('WEBSITEBUCKET', default='s3://ukmda-website')
+    retval = url
+    tmpurl = url
+    if url[0]==',':
+        tmpurl = url[1:]
+    try:
+        _ = s3.head_object(Bucket=siteroot[5:], Key=tmpurl[1:])
+    except Exception:
+        #print(e)
+        retval = '/img/missing-white.png'
+    print(url, retval)
+    return retval
 
 
 def convertSingletoSrchable(datadir, year, newonly=True):
     print(datetime.datetime.now(), 'single-detection searchable index start')
+    s3 = boto3.client('s3')
 
     # load the single-station combined data
     if newonly is False:
@@ -34,6 +51,10 @@ def convertSingletoSrchable(datadir, year, newonly=True):
     uadata['fn']=[f'/img/single/{y}/{y}{m:02d}/'+f.replace('.fits','.jpg') 
         for f,y,m in zip(uadata.Filename, uadata.Y, uadata.M)]
 
+    print(datetime.datetime.now(), 'checking target urls exist')
+    uadata['targfn'] = [checkUrl(s3, x) for x in uadata.fn]
+    print(datetime.datetime.now(), 'done')
+
     # create array for source
     print(datetime.datetime.now(), 'add source column')
     srcs = ['2Single']*len(uadata.Filename)
@@ -44,41 +65,13 @@ def convertSingletoSrchable(datadir, year, newonly=True):
     print(datetime.datetime.now(), 'create interim dataframe')
     hdr=['eventtime','source','shower','Mag','loccam','url','imgs', 'loctime', 'Y','M']
     resdf = pd.DataFrame(zip(uadata.Dtstamp, srcs, uadata.Shwr, 
-        uadata.Mag, uadata.ID, uadata.fn, uadata.fn, uadata.LocalTime,
+        uadata.Mag, uadata.ID, uadata.targfn, uadata.targfn, uadata.LocalTime,
         uadata.Y, uadata.M), columns=hdr)
 
     # fix up some mangled historical data
     resdf.loc[resdf.loccam=='Ringwood_N_UK000S', 'loccam'] = 'UK000S'
     resdf.loc[resdf.loccam=='Tackley_SW_UK0006', 'loccam'] = 'UK0006'
 
-    # select the RMS data out, its good now
-    # FIXME - needs to select for "not FF_UK9" so we can include non-UK cameras
-    rmsdata=resdf[resdf.url.str.contains('FF_UK0')]
-    rmsdata = rmsdata.drop(columns=['Y','M','loctime'])
-
-    # now select out the UFO dta and fix it up
-    ufodata=resdf[resdf.url.str.contains('FF_UK9')]
-    #fix up Clanfield cameras
-    ufodata.loc[ufodata.loccam=='UK9990', 'loccam'] = 'Clanfield_NE'
-    ufodata.loc[ufodata.loccam=='UK9989', 'loccam'] = 'Clanfield_NW'
-    ufodata.loc[ufodata.loccam=='UK9988', 'loccam'] = 'Clanfield_SE'
-    ufodata = ufodata.drop(columns=['url','imgs'])
-
-    # create the URL and imgs fields
-    ufodata['url']=[f'/img/single/{y}/{y}{m:02d}/M{lt}_{f}P.jpg'
-        for f,y,m,lt in zip(ufodata.loccam, ufodata.Y, ufodata.M, ufodata.loctime)]
-    ufodata['imgs'] = ufodata.url
-    ufodata = ufodata.drop(columns=['loctime','Y','M'])
-
-    # annoying special case for UK0001, H and S which do not upload JPGs
-    rmsdata.loc[rmsdata.loccam=='UK0001','url']='/img/missing-white.png'
-    rmsdata.loc[rmsdata.loccam=='UK0001','imgs']='/img/missing-white.png'
-    rmsdata.loc[rmsdata.loccam=='UK000H','url']='/img/missing-white.png'
-    rmsdata.loc[rmsdata.loccam=='UK000H','imgs']='/img/missing-white.png'
-    rmsdata.loc[rmsdata.loccam=='UK000S','url']='/img/missing-white.png'
-    rmsdata.loc[rmsdata.loccam=='UK000S','imgs']='/img/missing-white.png'
-    
-    resdf = pd.concat([rmsdata,ufodata])
     if newonly is True:
         return resdf, rmsuafile
     else:
@@ -114,19 +107,22 @@ def convertMatchToSrchable(datadir, year, newonly=True):
 
 if __name__ == '__main__':
     if len(sys.argv) < 3:
-        print('usage: python createSearchableFormat.py year dest mode')
+        print('usage: python createSearchableFormat.py year mode outdir')
         exit(1)
     else:
         datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data')
 
         year = sys.argv[1]
         mode = sys.argv[2]
+        outdir = os.path.join(datadir, 'searchidx')
+        if len(sys.argv) > 3:
+            outdir = sys.argv[3]
 
         # create a set of single-station data and merge with last match set
         if mode == 'singles':
             print(datetime.datetime.now(), 'converting single-station data')
             newsingles, fname = convertSingletoSrchable(datadir, year, True)
-            outfile = os.path.join(datadir, 'searchidx', '{:s}-singles-new.csv'.format(year))
+            outfile = os.path.join(outdir, '{:s}-singles-new.csv'.format(year))
             if newsingles is not None: 
                 newsingles.to_csv(outfile, index=False, header=False)
             if fname is not None:
@@ -136,11 +132,11 @@ def convertMatchToSrchable(datadir, year, newonly=True):
         elif mode == 'matches':
             print(datetime.datetime.now(), 'converting match data')
             newmatches, fname = convertMatchToSrchable(datadir, year, True)
-            outfile = os.path.join(datadir, 'searchidx', '{:s}-matches-new.csv'.format(year))
+            outfile = os.path.join(outdir, '{:s}-matches-new.csv'.format(year))
             if newmatches is not None: 
                 newmatches.to_csv(outfile, index=False, header=False)
             if fname is not None:
                 os.remove(fname)
 
         else:
-            print('usage: createSearchableFormat yyyy matches_or_singles')
+            print('usage: createSearchableFormat year mode outdir')

archive/ukmon_pylib/reports/createSearchableFormat.py

@@ -1,5 +1,6 @@
-# simple script to create a new user-maintenance account
-# and save the access key locally. Can only be used by an account with IAM Admin permissions. 
-$username = $args[0]
-aws iam create-user --user-name $username --profile ukmda_admin
-aws iam add-user-to-group --user-name $username --user-group Administrators --profile ukmda_admin
+#!/bin/bash
+# simple script to create a new user-maintenance account
+# and save the access key locally. Can only be used by an account with IAM Admin permissions. 
+$username = $args[0]
+aws iam create-user --user-name $username --profile ukmda_admin
+aws iam add-user-to-group --user-name $username --user-group Administrators --profile ukmda_admin

usermgmt/addAdminUser.ps1 archive/usermgmt/addAdminUser.shusermgmt/addAdminUser.ps1 renamed to archive/usermgmt/addAdminUser.sh

@@ -1,6 +1,7 @@
-# copyright Mark McIntyre, 2024-
-
-# script to add an api key for a user
-
-$username = $args[0]
-aws apigateway create-api-key --name "$username" --region eu-west-1 --profile ukmda_admin --description "key for $username" --enabled
+#!/bin/bash
+# copyright Mark McIntyre, 2024-
+
+# script to add an api key for a user
+
+$username = $1
+aws apigateway create-api-key --name "$username" --region eu-west-1 --description "key for $username" --enabled  --profile ukmda_admin

usermgmt/addFBApiKey.ps1 archive/usermgmt/addFBApiKey.shusermgmt/addFBApiKey.ps1 renamed to archive/usermgmt/addFBApiKey.sh

@@ -44,7 +44,7 @@ if [ ! -f $keydir/keys/$shortid_l.key ] ; then
 fi 
 
 # add a unix user and set their homedir to /var/sftp/userid
-grep $userid /etc/passwd
+grep -w $userid /etc/passwd
 if [ $? -eq 1 ] ; then 
     dt=$(date +%Y-%m-%d)
     logger -s -t addSftpUser "Creating unix user $userid"
@@ -91,6 +91,10 @@ if [ ! -z $oldloc ] ; then
     logger -s -t addSftpUser "Moving $oldloc to $userid"
     sudo cp /var/sftp/$oldloc/ukmon.ini /var/sftp/$oldloc/ukmon.ini.bkp
     sudo cp $keydir/inifs/$userid.ini /var/sftp/$oldloc/ukmon.ini
+    # and copy the ssh authorized keys file
+    sudo cp $oldloc/.ssh/authorized_keys /var/sftp/$userid/.ssh/
+    sudo chown -R $userid:$userid /var/sftp/$userid/.ssh/authorized_keys
+    sudo chmod 644 /var/sftp/$userid/.ssh/authorized_keys
 fi 
 
 logger -s -t addSftpUser "Finished"