diff --git a/archive/deployment/deploy-usermgmt.yml b/archive/deployment/deploy-usermgmt.yml deleted file mode 100644 index 8429780d3..000000000 --- a/archive/deployment/deploy-usermgmt.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive/usermgmt" - destdir: /home/ec2-user/keymgmt - tasks: - - name: Ensures {{destdir}} exists - file: path={{destdir}} state=directory - - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - with_items: - - {src: '{{srcdir}}/addSftpUser.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } - - {src: '{{srcdir}}/updateAwsKey.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } - - {src: '{{srcdir}}/addAdminUser.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } - - {src: '{{srcdir}}/addFBApiKey.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } diff --git a/archive/server_setup/.bashrc b/archive/server_setup/.bashrc index b772d578b..79e18c870 100644 --- a/archive/server_setup/.bashrc +++ b/archive/server_setup/.bashrc @@ -22,8 +22,7 @@ export PATH=$PATH:/usr/local/geos/bin:/usr/local/proj4/bin if shopt -q login_shell ; then echo "" - echo "Type 'dev' to activate the dev environment" - echo "Type 'prd' to activate the dev environment" + echo "THIS IS THE PROD SERVER!!" echo "" echo " Some handy aliases that work in either environment are" echo " logs => go to the logs folder" @@ -34,8 +33,9 @@ if shopt -q login_shell ; then echo " tnj => tail the nightly job log" echo " spacecalc => display space usage in the current folder" echo "" - fi - export NVM_DIR="$HOME/.nvm" + prd +fi +export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion export PATH=$PATH:$(dirname $(which node)) diff --git a/archive/usermgmt/addAdminUser.sh b/archive/usermgmt/addAdminUser.sh deleted file mode 100644 index eb02ec10c..000000000 --- a/archive/usermgmt/addAdminUser.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# simple script to create a new user-maintenance account -# and save the access key locally. Can only be used by an account with IAM Admin permissions. -$username = $args[0] -aws iam create-user --user-name $username --profile ukmda_admin -aws iam add-user-to-group --user-name $username --user-group Administrators --profile ukmda_admin diff --git a/archive/usermgmt/addFBApiKey.sh b/archive/usermgmt/addFBApiKey.sh deleted file mode 100644 index eb957f893..000000000 --- a/archive/usermgmt/addFBApiKey.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# copyright Mark McIntyre, 2024- - -# script to add an api key for a user - -$username = $1 -aws apigateway create-api-key --name "$username" --region eu-west-1 --description "key for $username" --enabled --profile ukmda_admin \ No newline at end of file diff --git a/archive/usermgmt/addSftpUser.sh b/archive/usermgmt/addSftpUser.sh deleted file mode 100644 index 82fdcbd26..000000000 --- a/archive/usermgmt/addSftpUser.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# bash script to create new sftp user -# -userid=$1 -shortid=$2 -updatemode=$3 -oldloc=$4 - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -cd $here - -if [ -f ../config.ini ] ; then - source ~/dev/config.ini - keydir=/home/ec2-user/dev/keymgmt -else - source ~/prod/config.ini - keydir=/home/ec2-user/keymgmt - -fi -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -cd $keydir - -logger -s -t addSftpUser "adding user $userid $shortid $oldloc" - -# wait for the file uploads to complete. For some reason paramiko does this asynchronously -counter=0 -# all-lowercase versions of the names -userid="${userid,,}" -shortid_l="${shortid,,}" - -ls $keydir/$shortid_l.key > /dev/null 2>&1 -while [[ $? -ne 0 && $counter -ne 5 ]] ; do - sleep 2 - logger -s -t addSftpUser "waiting for raw key for $shortid" - ls $keydir/keys/$shortid_l.key > /dev/null 2>&1 - counter=$((counter + 1)) -done -if [ ! -f $keydir/keys/$shortid_l.key ] ; then - logger -s -t addSftpUser "missing keyfile for $shortid" - exit 1 -fi - -# add a unix user and set their homedir to /var/sftp/userid -grep -w $userid /etc/passwd -if [ $? -eq 1 ] ; then - dt=$(date +%Y-%m-%d) - logger -s -t addSftpUser "Creating unix user $userid" - sudo useradd --system --shell /usr/sbin/nologin --groups sftp --home /var/sftp/$userid --comment '${dt}' $userid - sudo mkdir /var/sftp/$userid - sudo chown root:sftp /var/sftp/$userid - sudo chmod 751 /var/sftp/$userid - # create the .ssh folder, platepar folder and empty client copy of the ini file - sudo mkdir /var/sftp/$userid/.ssh - sudo mkdir /var/sftp/$userid/platepar - sudo touch /var/sftp/$userid/ukmon.ini.client - # make these three writeable by the client - sudo chown $userid:$userid /var/sftp/$userid/platepar /var/sftp/$userid/.ssh /var/sftp/$userid/ukmon.ini.client -else - logger -s -t addSftpUser "Unix user $userid already exists" -fi -# copy the public key to the right place -logger -s -t addSftpUser "Applying the public key $userid" -dos2unix $keydir/sshkeys/$userid.pub -sudo cp $keydir/sshkeys/$userid.pub /var/sftp/$userid/.ssh/authorized_keys -sudo chown -R $userid:$userid /var/sftp/$userid/.ssh/authorized_keys -sudo chmod 644 /var/sftp/$userid/.ssh/authorized_keys - -# add the key to logupload's authorized_keys file -logger -s -t addSftpUser "Adding the key to loguploads authorized_keys" -awk 1 $keydir/sshkeys/*.pub > /tmp/logul.pub -sudo cp /tmp/logul.pub /var/sftp/logupload/.ssh/authorized_keys -sudo chown logupload:logupload /var/sftp/logupload/.ssh/authorized_keys -rm /tmp/logul.pub - -# copy the ini and aws key files -logger -s -t addSftpUser "Copying the ini file and aws keyfile" -sudo cp $keydir/inifs/$userid.ini /var/sftp/$userid/ukmon.ini -sudo cp $keydir/keys/$shortid_l.key /var/sftp/$userid/live.key -sudo cp $keydir/csvkeys/${shortid}.csv /var/sftp/$userid/$userid.csv -sudo dos2unix /var/sftp/$userid/live.key -sudo dos2unix /var/sftp/$userid/ukmon.ini -sudo dos2unix /var/sftp/$userid/$userid.csv -sudo chown $userid:$userid /var/sftp/$userid/$userid.csv /var/sftp/$userid/ukmon.ini /var/sftp/$userid/live.key - -# if we are moving a station, update the old ini file -# so that the next run points to the new loc -if [ ! -z $oldloc ] ; then - logger -s -t addSftpUser "Moving $oldloc to $userid" - sudo cp /var/sftp/$oldloc/ukmon.ini /var/sftp/$oldloc/ukmon.ini.bkp - sudo cp $keydir/inifs/$userid.ini /var/sftp/$oldloc/ukmon.ini - # and copy the ssh authorized keys file - sudo cp $oldloc/.ssh/authorized_keys /var/sftp/$userid/.ssh/ - sudo chown -R $userid:$userid /var/sftp/$userid/.ssh/authorized_keys - sudo chmod 644 /var/sftp/$userid/.ssh/authorized_keys -fi - -logger -s -t addSftpUser "Finished" - -# run these as root to create the structure -# groupadd sftp -# mkdir -p /var/sftp -# chown root:root /var/sftp -# chmod 751 /var/sftp - -# Add this to /etc/ssh/sshd_config -# Match group sftp -# ChrootDirectory /var/sftp/%u -# AllowTCPForwarding no -# X11Forwarding no -# ForceCommand internal-sftp -# -# and then reload sshd -# service sshd reload diff --git a/archive/usermgmt/camTableMaintenance.py b/archive/usermgmt/camTableMaintenance.py deleted file mode 100644 index 30e27b54b..000000000 --- a/archive/usermgmt/camTableMaintenance.py +++ /dev/null @@ -1,49 +0,0 @@ -# -# Create and access dynamodb tables containing camera upload timings etc -# -# Copyright (C) 2018-2023 Mark McIntyre - -import boto3 -import os - -from stationMaint2 import addRow, loadLocationDetails - - -def addCreatedDate(): - bucket = 'ukmda-shared' - s3 = boto3.client('s3') - camdets = loadLocationDetails() - for _, cam in camdets.iterrows(): - res = s3.list_objects_v2(Bucket=bucket,Prefix=f'matches/RMSCorrelate/{cam["stationid"]}/', Delimiter='/') - if 'CommonPrefixes' in res: - earliest_fldr = os.path.split(res['CommonPrefixes'][0]['Prefix'][:-1])[1] - created = earliest_fldr.split('_')[1] - else: - created = '' - newdata = {'stationid': cam['stationid'], 'site': cam['site'], 'humanName':cam['humanName'], 'eMail': cam['eMail'], - 'direction': cam['direction'], 'camtype': cam['camtype'], 'active': cam['active'], - 'created': created, 'oldcode': cam['stationid']} - print(newdata) - addRow(newdata) - return - - -# remove a row from the table keyed on stationid adn datestamp in yyyymmdd_hhmmss format -def deleteRow(stationid, ddb=None): - if not ddb: - ddb = boto3.resource('dynamodb', region_name='eu-west-2') - table = ddb.Table('camdetails') - try: - table.delete_item(Key={'stationid': stationid}) - except Exception: - pass - return - - -def dumpCamTable(outdir, statdets=None, ddb=None, exportmindets=False): - if statdets is None: - statdets = loadLocationDetails(ddb=ddb) - statdets = statdets[statdets.active==1] - if exportmindets: - statdets = statdets[['stationid', 'eMail']] - statdets.to_csv(os.path.join(outdir,'camtable.csv'), index=False) diff --git a/archive/usermgmt/testkeys.py b/archive/usermgmt/testkeys.py deleted file mode 100644 index 398d9beed..000000000 --- a/archive/usermgmt/testkeys.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2018-2023 Mark McIntyre -# test that the keys provide read and write access - -import boto3 -import os -import sys - - -def readKeyFile(filename): - if not os.path.isfile(filename): - print('credentials file missing, cannot continue') - exit(1) - with open(filename, 'r') as fin: - lis = fin.readlines() - vals = {} - for li in lis: - if li[0]=='#': - continue - if '=' in li: - valstr = li.split(' ')[1] - data = valstr.split('=') - val = data[1].strip() - if val[0]=='"': - val = val[1:len(val)-1] - vals[data[0]] = val - if 'S3FOLDER' not in vals and 'CAMLOC' in vals: - vals['S3FOLDER'] = f'archive/{vals["CAMLOC"]}' - if 'S3FOLDER' in vals and vals['S3FOLDER'][-1] == '/': - vals['S3FOLDER'] = vals['S3FOLDER'][:-1] - if 'ARCHBUCKET' not in vals: - vals['ARCHBUCKET'] = 'ukmda-shared' - if 'LIVEBUCKET' not in vals: - vals['LIVEBUCKET'] = 'ukmda-live' - if 'WEBBUCKET' not in vals: - vals['WEBBUCKET'] = 'ukmda-website' - if 'ARCHREGION' not in vals: - vals['ARCHREGION'] = 'eu-west-2' - if 'LIVEREGION' not in vals: - vals['LIVEREGION'] = 'eu-west-1' - if 'MATCHDIR' not in vals: - vals['MATCHDIR'] = 'matches/RMSCorrelate' - #print(vals) - return vals - - -def uploadTest(keyfile): - myloc = os.path.split(os.path.abspath(__file__))[0] - filename = os.path.join(myloc, 'keys', keyfile) - keys = readKeyFile(filename) - target = keys['ARCHBUCKET'] - archreg = keys['ARCHREGION'] - livereg = keys['ARCHREGION'] - awskey = keys['AWS_ACCESS_KEY_ID'] - awssec = keys['AWS_SECRET_ACCESS_KEY'] - - with open('test.txt', 'w') as f: - f.write('test') - try: - conn = boto3.Session(aws_access_key_id=awskey, aws_secret_access_key=awssec, region_name=archreg) - s3 = conn.resource('s3') - target='ukmda-shared' - s3.meta.client.upload_file('test.txt', target, 'archive/test.txt') - key = {'Objects': []} - key['Objects'] = [{'Key': 'archive/test.txt'}] - s3.meta.client.delete_objects(Bucket=target, Delete=key) - print(f'{keyfile} archive test successful') - except Exception: - print(f'{keyfile} archive test FAILED') - - try: - conn = boto3.Session(aws_access_key_id=awskey, aws_secret_access_key=awssec, region_name=livereg) - s3 = conn.resource('s3') - target = 'ukmda-live' - s3.meta.client.upload_file('test.txt', target, 'test.txt') - key = {'Objects': []} - key['Objects'] = [{'Key': 'test.txt'}] - s3.meta.client.delete_objects(Bucket=target, Delete=key) - print(f'{keyfile} live test successful') - except Exception: - print(f'{keyfile} live test FAILED') - - os.remove('test.txt') - - -if __name__ == '__main__': - if len(sys.argv) < 2: - print("usage: python testKeys.py keyname\n") - exit(1) - else: - k = sys.argv[1] - uploadTest(k) diff --git a/archive/usermgmt/updateAwsKey.sh b/archive/usermgmt/updateAwsKey.sh deleted file mode 100644 index ff508560d..000000000 --- a/archive/usermgmt/updateAwsKey.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# bash script to update the keyfile to remove plaintext AWS security details -# - -targ=$1 -force=$2 - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -if [ -f ../config.ini ] ; then - source ~/dev/config.ini - keydir=/home/ec2-user/dev/keymgmt -else - source ~/prod/config.ini - keydir=/home/ec2-user/keymgmt - -fi -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -export AWS_PROFILE=ukmonshared -python -m maintenance.getUserAndKeyInfo update $targ $force -export AWS_PROFILE= \ No newline at end of file diff --git a/archive/usermgmt/updateKeysForSplit.py b/archive/usermgmt/updateKeysForSplit.py deleted file mode 100644 index f2025a0df..000000000 --- a/archive/usermgmt/updateKeysForSplit.py +++ /dev/null @@ -1,110 +0,0 @@ -import os -import boto3 -import json -import sys -import shutil - - -def createKeyFile(livekey, archkey, location): - archbucket = os.getenv('SRCBUCKET', default='ukmda-shared') - livebucket = os.getenv('LIVEBUCKET', default='ukmda-live') - webbucket = os.getenv('WEBSITEBUCKET', default='ukmda-website') - outf = 'keys/' + location.lower() + '.key' - if archkey is None: - return outf - with open(outf, 'w') as ouf: - ouf.write(f"export AWS_ACCESS_KEY_ID={archkey['AccessKey']['AccessKeyId']}\n") - ouf.write(f"export AWS_SECRET_ACCESS_KEY={archkey['AccessKey']['SecretAccessKey']}\n") - ouf.write(f"export LIVE_ACCESS_KEY_ID={livekey['AccessKey']['AccessKeyId']}\n") - ouf.write(f"export LIVE_SECRET_ACCESS_KEY={livekey['AccessKey']['SecretAccessKey']}\n") - ouf.write('export AWS_DEFAULT_REGION=eu-west-1\n') - ouf.write(f'export CAMLOC="{location}"\n') - ouf.write(f'export S3FOLDER="archive/{location}/"\n') - ouf.write(f'export ARCHBUCKET={archbucket}\n') - ouf.write(f'export LIVEBUCKET={livebucket}\n') - ouf.write(f'export WEBBUCKET={webbucket}\n') - ouf.write('export ARCHREGION=eu-west-2\n') - ouf.write('export LIVEREGION=eu-west-1\n') - ouf.write('export MATCHDIR=matches/RMSCorrelate\n') - os.chmod(outf, 0o666) - return outf - - -def readOldKeyFile(camid): - keyf= f'/var/sftp/{camid}/live.key' - if not os.path.isfile(keyf): - print(f'invalid camid {camid}') - exit() - flines = open(keyf, 'r').readlines() - camloc = None - for li in flines: - if 'AWS_ACCESS_KEY_ID' in li: - oldkey = li.split('=')[1].strip() - elif 'AWS_SECRET_ACCESS_KEY' in li: - oldsec= li.split('=')[1].strip() - elif 'CAMLOC' in li: - camloc= li.split('=')[1].strip().strip('"') - elif 'LIVE_ACCESS_KEY_ID' in li: - print('already processed') - exit() - if camloc is None: - print('malformed key file - no camloc') - exit() - return {'AccessKey': {'AccessKeyId': oldkey, 'SecretAccessKey': oldsec}}, camloc - - -def createNewUser(location): - archuserdets = 'users/' + location + '_arch.txt' - archkeyf = 'rawkeys/arch/' + location + '.key' - archprof = os.getenv('ARCH_PROFILE', default='ukmda_admin') - archconn = boto3.Session(profile_name=archprof) - iamc = archconn.client('iam') - sts = archconn.client('sts') - acct = sts.get_caller_identity()['Account'] - policyarn = 'arn:aws:iam::' + acct + ':policy/UkmonLive' - policyarn2 = 'arn:aws:iam::' + acct + ':policy/UKMDA-shared' - try: - _ = iamc.get_user(UserName=location) - print('location exists, not adding it') - archkey = None - except Exception: - print('new location') - usr = iamc.create_user(UserName=location) - with open(archuserdets, 'w') as outf: - outf.write(str(usr)) - archkey = iamc.create_access_key(UserName=location) - with open(archkeyf, 'w') as outf: - outf.write(json.dumps(archkey, indent=4, sort_keys=True, default=str)) - with open(os.path.join('csvkeys', location + '_arch.csv'),'w') as outf: - outf.write('Access key ID,Secret access key\n') - outf.write('{},{}\n'.format(archkey['AccessKey']['AccessKeyId'], archkey['AccessKey']['SecretAccessKey'])) - _ = iamc.attach_user_policy(UserName=location, PolicyArn=policyarn) - _ = iamc.attach_user_policy(UserName=location, PolicyArn=policyarn2) - os.chmod(archuserdets, 0o666) - os.chmod(archkeyf, 0o666) - return archkey - - -def copyKeyFileToTarget(newkeyfile, camid): - targkeyf = f'/var/sftp/{camid}/live.key' - bkpkeyf = newkeyfile + '.bkp' - if not os.path.isfile(bkpkeyf): - shutil.copyfile(targkeyf, bkpkeyf) - shutil.copyfile(newkeyfile, targkeyf) - return - - -if __name__ == '__main__': - os.chdir('/home/ec2-user/keymgmt') - os.makedirs('users/', exist_ok=True) - os.makedirs('csvkeys/', exist_ok=True) - os.makedirs('rawkeys/arch/', exist_ok=True) - os.makedirs('keys/', exist_ok=True) - - camid = sys.argv[1] - print(f'processing {camid}... ', end='') - sys.stdout.flush() - oldkey, location = readOldKeyFile(camid) - archkey = createNewUser(location) - newkeyfile = createKeyFile(oldkey, archkey, location) - copyKeyFileToTarget(newkeyfile, camid)