From 7b715ba9ea97d35b14838b2c9e05052c0c8846a8 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 11 Dec 2025 23:46:56 +0000 Subject: [PATCH 01/15] make the prod server prod-only --- archive/server_setup/.bashrc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/archive/server_setup/.bashrc b/archive/server_setup/.bashrc index b772d578b..79e18c870 100644 --- a/archive/server_setup/.bashrc +++ b/archive/server_setup/.bashrc @@ -22,8 +22,7 @@ export PATH=$PATH:/usr/local/geos/bin:/usr/local/proj4/bin if shopt -q login_shell ; then echo "" - echo "Type 'dev' to activate the dev environment" - echo "Type 'prd' to activate the dev environment" + echo "THIS IS THE PROD SERVER!!" echo "" echo " Some handy aliases that work in either environment are" echo " logs => go to the logs folder" @@ -34,8 +33,9 @@ if shopt -q login_shell ; then echo " tnj => tail the nightly job log" echo " spacecalc => display space usage in the current folder" echo "" - fi - export NVM_DIR="$HOME/.nvm" + prd +fi +export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion export PATH=$PATH:$(dirname $(which node)) From 3d24d2d2b3e92a6c1d8585ead2f4d12cb8970ced Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 11 Dec 2025 23:53:50 +0000 Subject: [PATCH 02/15] Remove camera and fireball tool code --- archive/deployment/deploy-usermgmt.yml | 18 ---- archive/usermgmt/addAdminUser.sh | 6 -- archive/usermgmt/addFBApiKey.sh | 7 -- archive/usermgmt/addSftpUser.sh | 116 ------------------------ archive/usermgmt/camTableMaintenance.py | 49 ---------- archive/usermgmt/testkeys.py | 91 ------------------- archive/usermgmt/updateAwsKey.sh | 23 ----- archive/usermgmt/updateKeysForSplit.py | 110 ---------------------- 8 files changed, 420 deletions(-) delete mode 100644 archive/deployment/deploy-usermgmt.yml delete mode 100644 archive/usermgmt/addAdminUser.sh delete mode 100644 archive/usermgmt/addFBApiKey.sh delete mode 100644 archive/usermgmt/addSftpUser.sh delete mode 100644 archive/usermgmt/camTableMaintenance.py delete mode 100644 archive/usermgmt/testkeys.py delete mode 100644 archive/usermgmt/updateAwsKey.sh delete mode 100644 archive/usermgmt/updateKeysForSplit.py diff --git a/archive/deployment/deploy-usermgmt.yml b/archive/deployment/deploy-usermgmt.yml deleted file mode 100644 index 8429780d3..000000000 --- a/archive/deployment/deploy-usermgmt.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive/usermgmt" - destdir: /home/ec2-user/keymgmt - tasks: - - name: Ensures {{destdir}} exists - file: path={{destdir}} state=directory - - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - with_items: - - {src: '{{srcdir}}/addSftpUser.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } - - {src: '{{srcdir}}/updateAwsKey.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } - - {src: '{{srcdir}}/addAdminUser.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } - - {src: '{{srcdir}}/addFBApiKey.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } diff --git a/archive/usermgmt/addAdminUser.sh b/archive/usermgmt/addAdminUser.sh deleted file mode 100644 index eb02ec10c..000000000 --- a/archive/usermgmt/addAdminUser.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# simple script to create a new user-maintenance account -# and save the access key locally. Can only be used by an account with IAM Admin permissions. -$username = $args[0] -aws iam create-user --user-name $username --profile ukmda_admin -aws iam add-user-to-group --user-name $username --user-group Administrators --profile ukmda_admin diff --git a/archive/usermgmt/addFBApiKey.sh b/archive/usermgmt/addFBApiKey.sh deleted file mode 100644 index eb957f893..000000000 --- a/archive/usermgmt/addFBApiKey.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# copyright Mark McIntyre, 2024- - -# script to add an api key for a user - -$username = $1 -aws apigateway create-api-key --name "$username" --region eu-west-1 --description "key for $username" --enabled --profile ukmda_admin \ No newline at end of file diff --git a/archive/usermgmt/addSftpUser.sh b/archive/usermgmt/addSftpUser.sh deleted file mode 100644 index 82fdcbd26..000000000 --- a/archive/usermgmt/addSftpUser.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# bash script to create new sftp user -# -userid=$1 -shortid=$2 -updatemode=$3 -oldloc=$4 - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -cd $here - -if [ -f ../config.ini ] ; then - source ~/dev/config.ini - keydir=/home/ec2-user/dev/keymgmt -else - source ~/prod/config.ini - keydir=/home/ec2-user/keymgmt - -fi -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -cd $keydir - -logger -s -t addSftpUser "adding user $userid $shortid $oldloc" - -# wait for the file uploads to complete. For some reason paramiko does this asynchronously -counter=0 -# all-lowercase versions of the names -userid="${userid,,}" -shortid_l="${shortid,,}" - -ls $keydir/$shortid_l.key > /dev/null 2>&1 -while [[ $? -ne 0 && $counter -ne 5 ]] ; do - sleep 2 - logger -s -t addSftpUser "waiting for raw key for $shortid" - ls $keydir/keys/$shortid_l.key > /dev/null 2>&1 - counter=$((counter + 1)) -done -if [ ! -f $keydir/keys/$shortid_l.key ] ; then - logger -s -t addSftpUser "missing keyfile for $shortid" - exit 1 -fi - -# add a unix user and set their homedir to /var/sftp/userid -grep -w $userid /etc/passwd -if [ $? -eq 1 ] ; then - dt=$(date +%Y-%m-%d) - logger -s -t addSftpUser "Creating unix user $userid" - sudo useradd --system --shell /usr/sbin/nologin --groups sftp --home /var/sftp/$userid --comment '${dt}' $userid - sudo mkdir /var/sftp/$userid - sudo chown root:sftp /var/sftp/$userid - sudo chmod 751 /var/sftp/$userid - # create the .ssh folder, platepar folder and empty client copy of the ini file - sudo mkdir /var/sftp/$userid/.ssh - sudo mkdir /var/sftp/$userid/platepar - sudo touch /var/sftp/$userid/ukmon.ini.client - # make these three writeable by the client - sudo chown $userid:$userid /var/sftp/$userid/platepar /var/sftp/$userid/.ssh /var/sftp/$userid/ukmon.ini.client -else - logger -s -t addSftpUser "Unix user $userid already exists" -fi -# copy the public key to the right place -logger -s -t addSftpUser "Applying the public key $userid" -dos2unix $keydir/sshkeys/$userid.pub -sudo cp $keydir/sshkeys/$userid.pub /var/sftp/$userid/.ssh/authorized_keys -sudo chown -R $userid:$userid /var/sftp/$userid/.ssh/authorized_keys -sudo chmod 644 /var/sftp/$userid/.ssh/authorized_keys - -# add the key to logupload's authorized_keys file -logger -s -t addSftpUser "Adding the key to loguploads authorized_keys" -awk 1 $keydir/sshkeys/*.pub > /tmp/logul.pub -sudo cp /tmp/logul.pub /var/sftp/logupload/.ssh/authorized_keys -sudo chown logupload:logupload /var/sftp/logupload/.ssh/authorized_keys -rm /tmp/logul.pub - -# copy the ini and aws key files -logger -s -t addSftpUser "Copying the ini file and aws keyfile" -sudo cp $keydir/inifs/$userid.ini /var/sftp/$userid/ukmon.ini -sudo cp $keydir/keys/$shortid_l.key /var/sftp/$userid/live.key -sudo cp $keydir/csvkeys/${shortid}.csv /var/sftp/$userid/$userid.csv -sudo dos2unix /var/sftp/$userid/live.key -sudo dos2unix /var/sftp/$userid/ukmon.ini -sudo dos2unix /var/sftp/$userid/$userid.csv -sudo chown $userid:$userid /var/sftp/$userid/$userid.csv /var/sftp/$userid/ukmon.ini /var/sftp/$userid/live.key - -# if we are moving a station, update the old ini file -# so that the next run points to the new loc -if [ ! -z $oldloc ] ; then - logger -s -t addSftpUser "Moving $oldloc to $userid" - sudo cp /var/sftp/$oldloc/ukmon.ini /var/sftp/$oldloc/ukmon.ini.bkp - sudo cp $keydir/inifs/$userid.ini /var/sftp/$oldloc/ukmon.ini - # and copy the ssh authorized keys file - sudo cp $oldloc/.ssh/authorized_keys /var/sftp/$userid/.ssh/ - sudo chown -R $userid:$userid /var/sftp/$userid/.ssh/authorized_keys - sudo chmod 644 /var/sftp/$userid/.ssh/authorized_keys -fi - -logger -s -t addSftpUser "Finished" - -# run these as root to create the structure -# groupadd sftp -# mkdir -p /var/sftp -# chown root:root /var/sftp -# chmod 751 /var/sftp - -# Add this to /etc/ssh/sshd_config -# Match group sftp -# ChrootDirectory /var/sftp/%u -# AllowTCPForwarding no -# X11Forwarding no -# ForceCommand internal-sftp -# -# and then reload sshd -# service sshd reload diff --git a/archive/usermgmt/camTableMaintenance.py b/archive/usermgmt/camTableMaintenance.py deleted file mode 100644 index 30e27b54b..000000000 --- a/archive/usermgmt/camTableMaintenance.py +++ /dev/null @@ -1,49 +0,0 @@ -# -# Create and access dynamodb tables containing camera upload timings etc -# -# Copyright (C) 2018-2023 Mark McIntyre - -import boto3 -import os - -from stationMaint2 import addRow, loadLocationDetails - - -def addCreatedDate(): - bucket = 'ukmda-shared' - s3 = boto3.client('s3') - camdets = loadLocationDetails() - for _, cam in camdets.iterrows(): - res = s3.list_objects_v2(Bucket=bucket,Prefix=f'matches/RMSCorrelate/{cam["stationid"]}/', Delimiter='/') - if 'CommonPrefixes' in res: - earliest_fldr = os.path.split(res['CommonPrefixes'][0]['Prefix'][:-1])[1] - created = earliest_fldr.split('_')[1] - else: - created = '' - newdata = {'stationid': cam['stationid'], 'site': cam['site'], 'humanName':cam['humanName'], 'eMail': cam['eMail'], - 'direction': cam['direction'], 'camtype': cam['camtype'], 'active': cam['active'], - 'created': created, 'oldcode': cam['stationid']} - print(newdata) - addRow(newdata) - return - - -# remove a row from the table keyed on stationid adn datestamp in yyyymmdd_hhmmss format -def deleteRow(stationid, ddb=None): - if not ddb: - ddb = boto3.resource('dynamodb', region_name='eu-west-2') - table = ddb.Table('camdetails') - try: - table.delete_item(Key={'stationid': stationid}) - except Exception: - pass - return - - -def dumpCamTable(outdir, statdets=None, ddb=None, exportmindets=False): - if statdets is None: - statdets = loadLocationDetails(ddb=ddb) - statdets = statdets[statdets.active==1] - if exportmindets: - statdets = statdets[['stationid', 'eMail']] - statdets.to_csv(os.path.join(outdir,'camtable.csv'), index=False) diff --git a/archive/usermgmt/testkeys.py b/archive/usermgmt/testkeys.py deleted file mode 100644 index 398d9beed..000000000 --- a/archive/usermgmt/testkeys.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2018-2023 Mark McIntyre -# test that the keys provide read and write access - -import boto3 -import os -import sys - - -def readKeyFile(filename): - if not os.path.isfile(filename): - print('credentials file missing, cannot continue') - exit(1) - with open(filename, 'r') as fin: - lis = fin.readlines() - vals = {} - for li in lis: - if li[0]=='#': - continue - if '=' in li: - valstr = li.split(' ')[1] - data = valstr.split('=') - val = data[1].strip() - if val[0]=='"': - val = val[1:len(val)-1] - vals[data[0]] = val - if 'S3FOLDER' not in vals and 'CAMLOC' in vals: - vals['S3FOLDER'] = f'archive/{vals["CAMLOC"]}' - if 'S3FOLDER' in vals and vals['S3FOLDER'][-1] == '/': - vals['S3FOLDER'] = vals['S3FOLDER'][:-1] - if 'ARCHBUCKET' not in vals: - vals['ARCHBUCKET'] = 'ukmda-shared' - if 'LIVEBUCKET' not in vals: - vals['LIVEBUCKET'] = 'ukmda-live' - if 'WEBBUCKET' not in vals: - vals['WEBBUCKET'] = 'ukmda-website' - if 'ARCHREGION' not in vals: - vals['ARCHREGION'] = 'eu-west-2' - if 'LIVEREGION' not in vals: - vals['LIVEREGION'] = 'eu-west-1' - if 'MATCHDIR' not in vals: - vals['MATCHDIR'] = 'matches/RMSCorrelate' - #print(vals) - return vals - - -def uploadTest(keyfile): - myloc = os.path.split(os.path.abspath(__file__))[0] - filename = os.path.join(myloc, 'keys', keyfile) - keys = readKeyFile(filename) - target = keys['ARCHBUCKET'] - archreg = keys['ARCHREGION'] - livereg = keys['ARCHREGION'] - awskey = keys['AWS_ACCESS_KEY_ID'] - awssec = keys['AWS_SECRET_ACCESS_KEY'] - - with open('test.txt', 'w') as f: - f.write('test') - try: - conn = boto3.Session(aws_access_key_id=awskey, aws_secret_access_key=awssec, region_name=archreg) - s3 = conn.resource('s3') - target='ukmda-shared' - s3.meta.client.upload_file('test.txt', target, 'archive/test.txt') - key = {'Objects': []} - key['Objects'] = [{'Key': 'archive/test.txt'}] - s3.meta.client.delete_objects(Bucket=target, Delete=key) - print(f'{keyfile} archive test successful') - except Exception: - print(f'{keyfile} archive test FAILED') - - try: - conn = boto3.Session(aws_access_key_id=awskey, aws_secret_access_key=awssec, region_name=livereg) - s3 = conn.resource('s3') - target = 'ukmda-live' - s3.meta.client.upload_file('test.txt', target, 'test.txt') - key = {'Objects': []} - key['Objects'] = [{'Key': 'test.txt'}] - s3.meta.client.delete_objects(Bucket=target, Delete=key) - print(f'{keyfile} live test successful') - except Exception: - print(f'{keyfile} live test FAILED') - - os.remove('test.txt') - - -if __name__ == '__main__': - if len(sys.argv) < 2: - print("usage: python testKeys.py keyname\n") - exit(1) - else: - k = sys.argv[1] - uploadTest(k) diff --git a/archive/usermgmt/updateAwsKey.sh b/archive/usermgmt/updateAwsKey.sh deleted file mode 100644 index ff508560d..000000000 --- a/archive/usermgmt/updateAwsKey.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# bash script to update the keyfile to remove plaintext AWS security details -# - -targ=$1 -force=$2 - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -if [ -f ../config.ini ] ; then - source ~/dev/config.ini - keydir=/home/ec2-user/dev/keymgmt -else - source ~/prod/config.ini - keydir=/home/ec2-user/keymgmt - -fi -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -export AWS_PROFILE=ukmonshared -python -m maintenance.getUserAndKeyInfo update $targ $force -export AWS_PROFILE= \ No newline at end of file diff --git a/archive/usermgmt/updateKeysForSplit.py b/archive/usermgmt/updateKeysForSplit.py deleted file mode 100644 index f2025a0df..000000000 --- a/archive/usermgmt/updateKeysForSplit.py +++ /dev/null @@ -1,110 +0,0 @@ -import os -import boto3 -import json -import sys -import shutil - - -def createKeyFile(livekey, archkey, location): - archbucket = os.getenv('SRCBUCKET', default='ukmda-shared') - livebucket = os.getenv('LIVEBUCKET', default='ukmda-live') - webbucket = os.getenv('WEBSITEBUCKET', default='ukmda-website') - outf = 'keys/' + location.lower() + '.key' - if archkey is None: - return outf - with open(outf, 'w') as ouf: - ouf.write(f"export AWS_ACCESS_KEY_ID={archkey['AccessKey']['AccessKeyId']}\n") - ouf.write(f"export AWS_SECRET_ACCESS_KEY={archkey['AccessKey']['SecretAccessKey']}\n") - ouf.write(f"export LIVE_ACCESS_KEY_ID={livekey['AccessKey']['AccessKeyId']}\n") - ouf.write(f"export LIVE_SECRET_ACCESS_KEY={livekey['AccessKey']['SecretAccessKey']}\n") - ouf.write('export AWS_DEFAULT_REGION=eu-west-1\n') - ouf.write(f'export CAMLOC="{location}"\n') - ouf.write(f'export S3FOLDER="archive/{location}/"\n') - ouf.write(f'export ARCHBUCKET={archbucket}\n') - ouf.write(f'export LIVEBUCKET={livebucket}\n') - ouf.write(f'export WEBBUCKET={webbucket}\n') - ouf.write('export ARCHREGION=eu-west-2\n') - ouf.write('export LIVEREGION=eu-west-1\n') - ouf.write('export MATCHDIR=matches/RMSCorrelate\n') - os.chmod(outf, 0o666) - return outf - - -def readOldKeyFile(camid): - keyf= f'/var/sftp/{camid}/live.key' - if not os.path.isfile(keyf): - print(f'invalid camid {camid}') - exit() - flines = open(keyf, 'r').readlines() - camloc = None - for li in flines: - if 'AWS_ACCESS_KEY_ID' in li: - oldkey = li.split('=')[1].strip() - elif 'AWS_SECRET_ACCESS_KEY' in li: - oldsec= li.split('=')[1].strip() - elif 'CAMLOC' in li: - camloc= li.split('=')[1].strip().strip('"') - elif 'LIVE_ACCESS_KEY_ID' in li: - print('already processed') - exit() - if camloc is None: - print('malformed key file - no camloc') - exit() - return {'AccessKey': {'AccessKeyId': oldkey, 'SecretAccessKey': oldsec}}, camloc - - -def createNewUser(location): - archuserdets = 'users/' + location + '_arch.txt' - archkeyf = 'rawkeys/arch/' + location + '.key' - archprof = os.getenv('ARCH_PROFILE', default='ukmda_admin') - archconn = boto3.Session(profile_name=archprof) - iamc = archconn.client('iam') - sts = archconn.client('sts') - acct = sts.get_caller_identity()['Account'] - policyarn = 'arn:aws:iam::' + acct + ':policy/UkmonLive' - policyarn2 = 'arn:aws:iam::' + acct + ':policy/UKMDA-shared' - try: - _ = iamc.get_user(UserName=location) - print('location exists, not adding it') - archkey = None - except Exception: - print('new location') - usr = iamc.create_user(UserName=location) - with open(archuserdets, 'w') as outf: - outf.write(str(usr)) - archkey = iamc.create_access_key(UserName=location) - with open(archkeyf, 'w') as outf: - outf.write(json.dumps(archkey, indent=4, sort_keys=True, default=str)) - with open(os.path.join('csvkeys', location + '_arch.csv'),'w') as outf: - outf.write('Access key ID,Secret access key\n') - outf.write('{},{}\n'.format(archkey['AccessKey']['AccessKeyId'], archkey['AccessKey']['SecretAccessKey'])) - _ = iamc.attach_user_policy(UserName=location, PolicyArn=policyarn) - _ = iamc.attach_user_policy(UserName=location, PolicyArn=policyarn2) - os.chmod(archuserdets, 0o666) - os.chmod(archkeyf, 0o666) - return archkey - - -def copyKeyFileToTarget(newkeyfile, camid): - targkeyf = f'/var/sftp/{camid}/live.key' - bkpkeyf = newkeyfile + '.bkp' - if not os.path.isfile(bkpkeyf): - shutil.copyfile(targkeyf, bkpkeyf) - shutil.copyfile(newkeyfile, targkeyf) - return - - -if __name__ == '__main__': - os.chdir('/home/ec2-user/keymgmt') - os.makedirs('users/', exist_ok=True) - os.makedirs('csvkeys/', exist_ok=True) - os.makedirs('rawkeys/arch/', exist_ok=True) - os.makedirs('keys/', exist_ok=True) - - camid = sys.argv[1] - print(f'processing {camid}... ', end='') - sys.stdout.flush() - oldkey, location = readOldKeyFile(camid) - archkey = createNewUser(location) - newkeyfile = createKeyFile(oldkey, archkey, location) - copyKeyFileToTarget(newkeyfile, camid) From bafdb181b547d1eb168d80e3b2276690e29ee9f8 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 14 Dec 2025 15:24:07 +0000 Subject: [PATCH 03/15] fix for #417 failure to archive json database --- archive/ukmon_pylib/traj/jsonDbMaintenance.py | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/archive/ukmon_pylib/traj/jsonDbMaintenance.py b/archive/ukmon_pylib/traj/jsonDbMaintenance.py index e5aa984c3..140d9a51e 100644 --- a/archive/ukmon_pylib/traj/jsonDbMaintenance.py +++ b/archive/ukmon_pylib/traj/jsonDbMaintenance.py @@ -33,12 +33,12 @@ def __init__(self, station, obs_id): for traj in [t for t in db.trajectories if t < archdate_jd]: if traj < archdate_jd: archdb.addTrajectory(None, db.trajectories[traj], False) - db.removeTrajectory(db.trajectories[traj]) + del db.trajectories[traj] for traj in [t for t in db.failed_trajectories if t < archdate_jd]: if traj < archdate_jd: archdb.addTrajectory(None, db.failed_trajectories[traj], True) - db.removeTrajectory(db.failed_trajectories[traj]) + del db.failed_trajectories[traj] for station in db.processed_dirs: arch_processed = [dirname for dirname in db.processed_dirs[station] if @@ -55,6 +55,47 @@ def __init__(self, station, obs_id): db.paired_obs[station].remove(obs_id) archdb.save() + db.save() + return db + + +def clearDownHistoricArchive(database_path, mthsback=1): + db = DatabaseJSON(database_path) + + if len(list(db.failed_trajectories.keys()))==0 and len(list(db.trajectories.keys()))==0: + print('nothing to do') + return + elif len(list(db.failed_trajectories.keys()))==0: + latest = jd2Date(max(list(db.trajectories.keys())), dt_obj=True) + elif len(list(db.trajectories.keys()))==0: + latest = jd2Date(max(list(db.failed_trajectories.keys())), dt_obj=True) + else: + latest = jd2Date(max(max(list(db.trajectories.keys())), max(list(db.failed_trajectories.keys()))), dt_obj=True) + print('processing', database_path) + archdate = latest - relativedelta(months=mthsback) + archdate = archdate.replace(day=1, hour=12, minute=0, second=0).replace(tzinfo=datetime.timezone.utc) + archdate_jd = datetime2JD(archdate) + + for traj in [t for t in db.trajectories if t < archdate_jd]: + if traj < archdate_jd: + del db.trajectories[traj] + + for traj in [t for t in db.failed_trajectories if t < archdate_jd]: + if traj < archdate_jd: + del db.failed_trajectories[traj] + + for station in db.processed_dirs: + arch_processed = [dirname for dirname in db.processed_dirs[station] if + datetime.datetime.strptime(dirname[14:22], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] + for dirname in arch_processed: + db.processed_dirs[station].remove(dirname) + + for station in db.paired_obs: + arch_processed = [obs_id for obs_id in db.paired_obs[station] if + datetime.datetime.strptime(obs_id[7:15], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] + for obs_id in arch_processed: + db.paired_obs[station].remove(obs_id) + db.save() return @@ -64,9 +105,8 @@ def __init__(self, station, obs_id): database_path = os.path.join(db_dir, JSON_DB_NAME) db = DatabaseJSON(database_path) soonest = jd2Date(min(min(list(db.trajectories.keys())), min(list(db.failed_trajectories.keys()))), dt_obj=True) - nowdt = datetime.datetime.now() mthsback = int((nowdt - soonest).days/30) for i in range(mthsback, 2, -1): print(f'archiving {i} months back') - archiveOldRecords(db, db_dir, i) + db = archiveOldRecords(db, db_dir, i) From 10748a208a8f52b0555ccb20fa443971d8e317ca Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 14 Dec 2025 15:25:26 +0000 Subject: [PATCH 04/15] pass rundate to createExecConsol and avoid doing yearly plots every day --- archive/analysis/runDistrib.sh | 4 ++-- archive/ukmon_pylib/traj/createDistribMatchingSh.py | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 468f02987..5eb4b5ab8 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -135,7 +135,7 @@ if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then execcons=execconsol.sh execConsolsh=/tmp/$execcons - python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh')" + python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh', $rundate)" chmod +x $execConsolsh log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib @@ -160,7 +160,7 @@ if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then python -m traj.jsonDbMaintenance $DATADIR/distrib/ aws s3 cp $DATADIR/distrib/processed_trajectories.json $UKMONSHAREDBUCKET/matches/distrib/ --quiet - log2cw $NJLOGGRP $NJLOGSTREAM "compressing the procssed data" runDistrib + log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib gzip < $DATADIR/distrib/processed_trajectories.json > $DATADIR/trajdb/processed_trajectories.json.${rundate}.gz aws s3 mv $UKMONSHAREDBUCKET/matches/distrib/${rundate}.pickle $DATADIR/distrib --quiet tar czvf $DATADIR/distrib/${rundate}.tgz $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index 97555ca96..8ca5aec51 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -124,12 +124,15 @@ def gatherUsedImageList(outf, matchstart, matchend, shbucket): return -def createExecConsolSh(matchstart, matchend, execconsolsh): +def createExecConsolSh(matchstart, matchend, execconsolsh, rundt): shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') webbucket = os.getenv('WEBSITEBUCKET', default='s3://ukmda-website') calcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' # hardcoded! _, outpath, _ = getTrajsolverPaths() enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend) + includeyear = False + if enddt.day == 30: + includeyear = True with open(execconsolsh, 'w') as outf: outf.write('#!/bin/bash\n') @@ -139,12 +142,12 @@ def createExecConsolSh(matchstart, matchend, execconsolsh): outf.write(f'cd {calcdir}\n') outf.write('logger -s -t execConsol start\n') - outf.write('python -m traj.consolidateDistTraj ~/data/distrib/ ~/data/distrib/processed_trajectories.json\n') + outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/ ~/data/distrib/processed_trajectories.json {rundt}\n') outf.write('logger -s -t execConsol syncing any updated trajectories from shared S3\n') refreshTrajectories(outf, matchstart, matchend, outpath) outf.write('logger -s -t execConsol creating density plots\n') - createDensityPlots(outf, calcdir, enddt) + createDensityPlots(outf, calcdir, enddt, includeyear) outf.write('logger -s -t execConsol pushing data back to S3\n') pushUpdatedTrajectoriesShared(outf, matchstart, matchend, shbucket) pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webbucket) From 420b7d138f9efe8b4aa8205d82ba9466d15667ff Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 14 Dec 2025 15:25:41 +0000 Subject: [PATCH 05/15] make container names three digits --- archive/ukmon_pylib/traj/distributeCandidates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index 3030c9b08..91f185f41 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -101,7 +101,7 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): s3 = boto3.client('s3') for i in range(0, numbucks): - bucknames[i] = buckroot + f'_{i:02d}' + bucknames[i] = buckroot + f'_{i:03d}' filelist = flist[i::numbucks] for fli in filelist: src = os.path.join(srcdir, fli) From 275843c6f1f84a47bf80e51c70ac487e1784b341 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 26 Dec 2025 16:19:52 +0000 Subject: [PATCH 06/15] add support to request period of the day for matches --- .../samfunctions/matchDataApi/matchDataApi.py | 83 ++++++++++--------- .../samfunctions/matchDataApi/template.yml | 2 + .../matchDataApi/testMatchesPeriod.json | 8 ++ 3 files changed, 56 insertions(+), 37 deletions(-) create mode 100644 archive/samfunctions/matchDataApi/testMatchesPeriod.json diff --git a/archive/samfunctions/matchDataApi/matchDataApi.py b/archive/samfunctions/matchDataApi/matchDataApi.py index b7c2023ef..da2048d15 100644 --- a/archive/samfunctions/matchDataApi/matchDataApi.py +++ b/archive/samfunctions/matchDataApi/matchDataApi.py @@ -9,6 +9,33 @@ import pymysql.cursors +def fileToJsonString(flis): + hdr = ['No','statid','ign','t','jd','m1','m2','az','alt','azl','altl','rao', + 'deco','ral','decl','X','Y','Z','lat','lon','H','range','length','svd', + 'lag','vel','pvel', 'hres','vres','ares','vmag','amag'] + ptsarray='[' + gotpts = False + for fli in flis: + if 'Points' in fli: + gotpts = True + continue + elif '------' in fli or ' No' in fli: + continue + elif gotpts is True and (len(fli) < 2 or 'Notes' in fli): + gotpts=False + break + elif gotpts is True: + spls = fli.split(',') + thisrow = '{' + for h, s in zip(hdr, spls): + thisrow = thisrow + f'"{h}": "{s.strip()}",' + thisrow = thisrow[:-1] + '},' + ptsarray = ptsarray + thisrow + ptsarray = ptsarray[:-1] + ']' + ptsarray = '{' + f'"points": {ptsarray}' + '}' + return ptsarray + + def getSqlLoginDetails(): # retrieve password and host from SSM. This allows me to manage them from Terraform ssm = boto3.client('ssm', region_name='eu-west-1') @@ -22,15 +49,25 @@ def getSqlLoginDetails(): return host, user, password, db -def getStationData(statid, dtstr): +def periodToSqlFragment(period): + if period == 'am': + frag = "and s._h_ut < 12" + elif period == 'pm': + frag = "and s._h_ut >= 12" + elif '-' in period: + sh,eh = period.split('-') + frag = f"and s._h_ut >={sh} and s._h_ut < {eh}" + return frag + + +def getStationData(statid, dtstr, period=None): host, user, passwd, db = getSqlLoginDetails() connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) try: + statfrag = f"and s.stations like '%{statid}%' " if statid is not None else "" + perfrag = periodToSqlFragment(period) if period is not None else "" with connection.cursor() as cursor: - if statid is None: - sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%'" - else: - sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' and s.stations like '%{statid}%'" + sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' {statfrag} {perfrag}" cursor.execute(sql) result = cursor.fetchall() finally: @@ -88,12 +125,10 @@ def lambda_handler(event, context): if qs is None: return { 'statusCode': 200, - 'body': 'usage: detections?reqtyp=xxx&reqval=yyyy[&points=1]' + 'body': 'usage: detections?reqtyp=xxx&reqval=yyyy[&points=1][&period=am|pm|h-h]' } reqtyp = qs['reqtyp'] - points = False - if 'points' in qs: - points = True + points = True if 'points' in qs else False if reqtyp == 'station': statid = qs['statid'] @@ -106,8 +141,9 @@ def lambda_handler(event, context): res = getSummaryData(dtstr) elif reqtyp == 'matches': dtstr = qs['reqval'] + period = qs['period'] if 'period' in qs else None print(f'match data requested for {dtstr}') - res = getStationData(None, dtstr) + res = getStationData(None, dtstr, period) elif reqtyp == 'detail': orbname = qs['reqval'] print(f'detail requested for {orbname}') @@ -140,30 +176,3 @@ def lambda_handler(event, context): 'statusCode': 200, 'body': res } - - -def fileToJsonString(flis): - hdr = ['No','statid','ign','t','jd','m1','m2','az','alt','azl','altl','rao', - 'deco','ral','decl','X','Y','Z','lat','lon','H','range','length','svd', - 'lag','vel','pvel', 'hres','vres','ares','vmag','amag'] - ptsarray='[' - gotpts = False - for fli in flis: - if 'Points' in fli: - gotpts = True - continue - elif '------' in fli or ' No' in fli: - continue - elif gotpts is True and (len(fli) < 2 or 'Notes' in fli): - gotpts=False - break - elif gotpts is True: - spls = fli.split(',') - thisrow = '{' - for h, s in zip(hdr, spls): - thisrow = thisrow + f'"{h}": "{s.strip()}",' - thisrow = thisrow[:-1] + '},' - ptsarray = ptsarray + thisrow - ptsarray = ptsarray[:-1] + ']' - ptsarray = '{' + f'"points": {ptsarray}' + '}' - return ptsarray diff --git a/archive/samfunctions/matchDataApi/template.yml b/archive/samfunctions/matchDataApi/template.yml index d07529b3a..83f98f961 100644 --- a/archive/samfunctions/matchDataApi/template.yml +++ b/archive/samfunctions/matchDataApi/template.yml @@ -31,6 +31,8 @@ Resources: Required: false - method.request.querystring.statid: Required: false + - method.request.querystring.period: + Required: false Tags: billingtag: "ukmda" typetag: "api" diff --git a/archive/samfunctions/matchDataApi/testMatchesPeriod.json b/archive/samfunctions/matchDataApi/testMatchesPeriod.json new file mode 100644 index 000000000..6f61467d2 --- /dev/null +++ b/archive/samfunctions/matchDataApi/testMatchesPeriod.json @@ -0,0 +1,8 @@ +{ + "httpMethod": "GET", + "queryStringParameters": { + "reqtyp": "matches", + "reqval": "20251213", + "period": "0-3" + } +} \ No newline at end of file From 2ea5ecfb718c136da710962290fa83f8e9beb140 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 3 Jan 2026 15:57:30 +0000 Subject: [PATCH 07/15] updaring policy assumerole for radio data --- archive/terraform/mjmm/iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/terraform/mjmm/iam.tf b/archive/terraform/mjmm/iam.tf index 766e2d668..10e44de3a 100644 --- a/archive/terraform/mjmm/iam.tf +++ b/archive/terraform/mjmm/iam.tf @@ -198,7 +198,7 @@ resource "aws_iam_role_policy" "stsAssumeLambda" { Statement = { Action = "sts:AssumeRole" Effect = "Allow" - Resource = "arn:aws:iam::822069317839:role/service-role/S3FullAccess" + Resource = "arn:aws:iam::183798037734:policy/s3PolicyForRadio" } Version = "2012-10-17" } From 3c3ba62d879e025e22364a4a0c445f4c70878171 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 3 Jan 2026 15:57:44 +0000 Subject: [PATCH 08/15] updating calcserver disk size --- archive/terraform/ukmda/ec2.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/terraform/ukmda/ec2.tf b/archive/terraform/ukmda/ec2.tf index 29c5fd71c..af11b8276 100644 --- a/archive/terraform/ukmda/ec2.tf +++ b/archive/terraform/ukmda/ec2.tf @@ -17,7 +17,7 @@ resource "aws_instance" "calc_server" { "Name" = "calcengine" "billingtag" = "ukmda" } - volume_size = 100 + volume_size = 120 } primary_network_interface { network_interface_id = aws_network_interface.calcserver_if.id @@ -58,7 +58,7 @@ resource "aws_instance" "ubuntu_calc_server" { } root_block_device { tags = { - "Name" = "calcengine2" + "Name" = "calcengine_ub" "billingtag" = "ukmda" } volume_size = 100 From 540d6509adcca650aaa2d6e37f752cd97b1baca7 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 8 Jan 2026 21:08:54 +0000 Subject: [PATCH 09/15] renamed samfunctions to lambdas --- .gitignore | 20 ++++++++--------- archive/{samfunctions => lambdas}/README.md | 0 .../camDetails/camDetails.py | 0 .../camDetails/samconfig.toml | 0 .../camDetails/template.yml | 0 .../fetchECSV/Math.py | 0 .../fetchECSV/MeteorObservation.py | 0 .../fetchECSV/fetchECSV.py | 0 .../fetchECSV/ftpDetectInfo.py | 0 .../fetchECSV/requirements.txt | 0 .../fetchECSV/samconfig.toml | 0 .../fetchECSV/template.yml | 0 ...tectinfo_UK005U_20240809_201258_058797.txt | 0 .../tests/platepars_all_recalibrated.json | 0 .../fireballApi/getFireballFiles.py | 0 .../fireballApi/requirements.txt | 0 .../fireballApi/samconfig.toml | 0 .../fireballApi/template.yml | 0 .../ftpToUkmon/.gitignore | 0 .../ftpToUkmon/README.md | 0 .../ftpToUkmon/__init__.py | 0 .../ftpToUkmon/pythoncode/Dockerfile | 0 .../pythoncode/ShowerAssociation.py | 0 .../ftpToUkmon/pythoncode/Showers.py | 0 .../ftpToUkmon/pythoncode/__init__.py | 0 .../pythoncode/established_showers.csv | 0 .../ftpToUkmon/pythoncode/ftpToUkmon1.py | 0 .../ftpToUkmon/pythoncode/requirements.txt | 0 .../ftpToUkmon/pythoncode/supportFuncs.py | 0 .../ftpToUkmon/samconfig.toml | 0 .../ftpToUkmon/template.yaml | 0 .../ftpToUkmon/tests/expected_results.txt | 0 .../ftpToUkmon/tests/localTest.ps1 | 0 .../ftpToUkmon/tests/new_data.txt | 0 .../ftpToUkmon/tests/remoteResult.txt | 0 .../ftpToUkmon/tests/remoteTest.ps1 | 0 .../ftpToUkmon/tests/testEvent.json | 0 .../getApiDetails.ps1 | 0 .../getExtraFilesV2/.gitignore | 0 .../getExtraFilesV2/README.md | 0 .../getExtraFilesV2/__init__.py | 0 .../getExtraFilesV2/pythoncode/Dockerfile | 0 .../WesternMeteorPyLib/requirements.txt | 22 +++++++++++++++++++ .../getExtraFilesV2/pythoncode/__init__.py | 0 .../pythoncode/createOrbitPageIndex.py | 0 .../pythoncode/getExtraFiles.py | 0 .../pythoncode/pickleAnalysis.py | 0 .../pythoncode/requirements.txt | 0 .../getExtraFilesV2/samconfig.toml | 0 .../getExtraFilesV2/template.yaml | 0 .../getExtraFilesV2/tests/localTest.ps1 | 0 .../getExtraFilesV2/tests/remoteTest.ps1 | 0 .../getExtraFilesV2/tests/templateEvent.json | 0 .../getExtraFilesV2/tests/testEvent.json | 0 .../getExtraFilesV2/tests/testEvent2.json | 0 .../getExtraFilesV2/update_wmpl.sh | 0 .../getExtraFilesV2/wmpl__init__.py_fixed.py | 0 .../getLiveImages/getLiveImages.py | 0 .../getLiveImages/requirements.txt | 0 .../getLiveImages/samconfig.toml | 0 .../getLiveImages/template.yml | 0 .../matchDataApi/localTest.ps1 | 0 .../matchDataApi/matchDataApi.py | 0 .../matchDataApi/requirements.txt | 0 .../matchDataApi/samconfig.toml | 0 .../matchDataApi/template.yml | 0 .../matchDataApi/testDetail.json | 0 .../matchDataApi/testMatches.json | 0 .../matchDataApi/testMatchesPeriod.json | 0 .../matchDataApi/testPoints.json | 0 .../matchDataApi/testStation.json | 0 .../matchDataApi/testSummary.json | 0 .../matchPickle/__init__.py | 0 .../matchPickle/localTest.ps1 | 0 .../matchPickle/pythoncode/Dockerfile | 0 .../matchPickle/pythoncode/__init__.py | 0 .../matchPickle/pythoncode/matchPickleApi.py | 0 .../matchPickle/pythoncode/requirements.txt | 0 .../matchPickle/samconfig.toml | 0 .../matchPickle/template.yml | 0 .../matchPickle/testEvent.json | 0 .../searchArchive/requirements.txt | 0 .../searchArchive/samconfig.toml | 0 .../searchArchive/searchArchive.py | 0 .../searchArchive/template.yml | 0 .../searchArchive/testEvent.json | 0 .../updateDnsRecords/lambda_function.py | 0 .../updateDnsRecords/samconfig.toml | 0 .../updateDnsRecords/template.yaml | 0 89 files changed, 32 insertions(+), 10 deletions(-) rename archive/{samfunctions => lambdas}/README.md (100%) rename archive/{samfunctions => lambdas}/camDetails/camDetails.py (100%) rename archive/{samfunctions => lambdas}/camDetails/samconfig.toml (100%) rename archive/{samfunctions => lambdas}/camDetails/template.yml (100%) rename archive/{samfunctions => lambdas}/fetchECSV/Math.py (100%) rename archive/{samfunctions => lambdas}/fetchECSV/MeteorObservation.py (100%) rename archive/{samfunctions => lambdas}/fetchECSV/fetchECSV.py (100%) rename archive/{samfunctions => lambdas}/fetchECSV/ftpDetectInfo.py (100%) rename archive/{samfunctions => lambdas}/fetchECSV/requirements.txt (100%) rename archive/{samfunctions => lambdas}/fetchECSV/samconfig.toml (100%) rename archive/{samfunctions => lambdas}/fetchECSV/template.yml (100%) rename archive/{samfunctions => lambdas}/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt (100%) rename archive/{samfunctions => lambdas}/fetchECSV/tests/platepars_all_recalibrated.json (100%) rename archive/{samfunctions => lambdas}/fireballApi/getFireballFiles.py (100%) rename archive/{samfunctions => lambdas}/fireballApi/requirements.txt (100%) rename archive/{samfunctions => lambdas}/fireballApi/samconfig.toml (100%) rename archive/{samfunctions => lambdas}/fireballApi/template.yml (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/.gitignore (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/README.md (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/__init__.py (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/pythoncode/Dockerfile (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/pythoncode/ShowerAssociation.py (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/pythoncode/Showers.py (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/pythoncode/__init__.py (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/pythoncode/established_showers.csv (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/pythoncode/ftpToUkmon1.py (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/pythoncode/requirements.txt (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/pythoncode/supportFuncs.py (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/samconfig.toml (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/template.yaml (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/tests/expected_results.txt (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/tests/localTest.ps1 (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/tests/new_data.txt (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/tests/remoteResult.txt (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/tests/remoteTest.ps1 (100%) rename archive/{samfunctions => lambdas}/ftpToUkmon/tests/testEvent.json (100%) rename archive/{samfunctions => lambdas}/getApiDetails.ps1 (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/.gitignore (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/README.md (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/__init__.py (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/pythoncode/Dockerfile (100%) create mode 100644 archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/requirements.txt rename archive/{samfunctions => lambdas}/getExtraFilesV2/pythoncode/__init__.py (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/pythoncode/createOrbitPageIndex.py (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/pythoncode/getExtraFiles.py (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/pythoncode/pickleAnalysis.py (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/pythoncode/requirements.txt (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/samconfig.toml (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/template.yaml (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/tests/localTest.ps1 (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/tests/remoteTest.ps1 (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/tests/templateEvent.json (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/tests/testEvent.json (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/tests/testEvent2.json (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/update_wmpl.sh (100%) rename archive/{samfunctions => lambdas}/getExtraFilesV2/wmpl__init__.py_fixed.py (100%) rename archive/{samfunctions => lambdas}/getLiveImages/getLiveImages.py (100%) rename archive/{samfunctions => lambdas}/getLiveImages/requirements.txt (100%) rename archive/{samfunctions => lambdas}/getLiveImages/samconfig.toml (100%) rename archive/{samfunctions => lambdas}/getLiveImages/template.yml (100%) rename archive/{samfunctions => lambdas}/matchDataApi/localTest.ps1 (100%) rename archive/{samfunctions => lambdas}/matchDataApi/matchDataApi.py (100%) rename archive/{samfunctions => lambdas}/matchDataApi/requirements.txt (100%) rename archive/{samfunctions => lambdas}/matchDataApi/samconfig.toml (100%) rename archive/{samfunctions => lambdas}/matchDataApi/template.yml (100%) rename archive/{samfunctions => lambdas}/matchDataApi/testDetail.json (100%) rename archive/{samfunctions => lambdas}/matchDataApi/testMatches.json (100%) rename archive/{samfunctions => lambdas}/matchDataApi/testMatchesPeriod.json (100%) rename archive/{samfunctions => lambdas}/matchDataApi/testPoints.json (100%) rename archive/{samfunctions => lambdas}/matchDataApi/testStation.json (100%) rename archive/{samfunctions => lambdas}/matchDataApi/testSummary.json (100%) rename archive/{samfunctions => lambdas}/matchPickle/__init__.py (100%) rename archive/{samfunctions => lambdas}/matchPickle/localTest.ps1 (100%) rename archive/{samfunctions => lambdas}/matchPickle/pythoncode/Dockerfile (100%) rename archive/{samfunctions => lambdas}/matchPickle/pythoncode/__init__.py (100%) rename archive/{samfunctions => lambdas}/matchPickle/pythoncode/matchPickleApi.py (100%) rename archive/{samfunctions => lambdas}/matchPickle/pythoncode/requirements.txt (100%) rename archive/{samfunctions => lambdas}/matchPickle/samconfig.toml (100%) rename archive/{samfunctions => lambdas}/matchPickle/template.yml (100%) rename archive/{samfunctions => lambdas}/matchPickle/testEvent.json (100%) rename archive/{samfunctions => lambdas}/searchArchive/requirements.txt (100%) rename archive/{samfunctions => lambdas}/searchArchive/samconfig.toml (100%) rename archive/{samfunctions => lambdas}/searchArchive/searchArchive.py (100%) rename archive/{samfunctions => lambdas}/searchArchive/template.yml (100%) rename archive/{samfunctions => lambdas}/searchArchive/testEvent.json (100%) rename archive/{samfunctions => lambdas}/updateDnsRecords/lambda_function.py (100%) rename archive/{samfunctions => lambdas}/updateDnsRecords/samconfig.toml (100%) rename archive/{samfunctions => lambdas}/updateDnsRecords/template.yaml (100%) diff --git a/.gitignore b/.gitignore index e897bccc6..035e1394a 100644 --- a/.gitignore +++ b/.gitignore @@ -531,13 +531,13 @@ StandaloneCurator/.installed analysis/orbitSolver/testing analysis/FormatConverters/testing */test_data/* -archive/samfunctions*/pytz* +archive/lambdas*/pytz* archive/static_content/data/searchdialog.js ToDo* archive/costs/**/*.csv archive/costs/**/*.json archive/costs/plots/*.png -archive/samfunctions*/.aws-sam/build.toml +archive/lambdas*/.aws-sam/build.toml archive/costs/S3-size-empireelements.xlsx archive/ukmon_pylib/share/maps/BM*.jpg archive/ukmon_pylib/share/maps/BM*.png @@ -556,8 +556,8 @@ archive/unused/** ukmon_pylib/tests/20220217_032206.832_UK/* unused/* *tfstate.lock.info -archive/samfunctions*/getExtraFilesV2/pythoncode/WesternMeteorPyLib/* -archive/samfunctions*/getExtraFilesForEE/pythoncode/WesternMeteorPyLib/* +archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/* +archive/lambdas/getExtraFilesForEE/pythoncode/WesternMeteorPyLib/* archive/containers/trajsolver/WesternMeteorPyLib/* archive/containers/trajsolver/awskeys archive/containers/trajsolvertest/WesternMeteorPyLib/* @@ -568,11 +568,11 @@ archive/containers/gui/awskeys archive/terraform/*/ukmonreadonly.key .cache_ggshield replacements.txt -archive/samfunctions*/apis-mm.txt -archive/samfunctions*/apis-ee.txt +archive/lambdas/apis-mm.txt +archive/lambdas/apis-ee.txt usermgmt/windows/createUserKeyAWS.ps1 -archive/samfunctions*/matchPickle/pythoncode/WesternMeteorPyLib/** -archive/samfunctions*/matchPickle/pythoncode/*.npy +archive/lambdas/matchPickle/pythoncode/WesternMeteorPyLib/** +archive/lambdas/matchPickle/pythoncode/*.npy .terraform/ archive/utils/tmp.json usermgmt/windows/caminfo/camera-details-new.csv @@ -589,8 +589,8 @@ ukmon_pylib/tests/usertools/2021-07-17T02_41_05_05_M002.ecsv ukmon_pylib/tests/usertools/2021-07-17T02_41_05_05_M004.ecsv ukmon_pylib/tests/usertools/20230202_014115.kml archive/containers*/simpleUI/awskeys -archive/samfunctions*/camDetails/pythoncode/camDetails.py -archive/samfunctions*/camDetails/pythoncode/awskeys +archive/lambdas*/camDetails/pythoncode/camDetails.py +archive/lambdas*/camDetails/pythoncode/awskeys archive/containers/RMS-docker/rms_docker.tar archive/containers/RMS-docker/Dockerfile_orig.txt diff --git a/archive/samfunctions/README.md b/archive/lambdas/README.md similarity index 100% rename from archive/samfunctions/README.md rename to archive/lambdas/README.md diff --git a/archive/samfunctions/camDetails/camDetails.py b/archive/lambdas/camDetails/camDetails.py similarity index 100% rename from archive/samfunctions/camDetails/camDetails.py rename to archive/lambdas/camDetails/camDetails.py diff --git a/archive/samfunctions/camDetails/samconfig.toml b/archive/lambdas/camDetails/samconfig.toml similarity index 100% rename from archive/samfunctions/camDetails/samconfig.toml rename to archive/lambdas/camDetails/samconfig.toml diff --git a/archive/samfunctions/camDetails/template.yml b/archive/lambdas/camDetails/template.yml similarity index 100% rename from archive/samfunctions/camDetails/template.yml rename to archive/lambdas/camDetails/template.yml diff --git a/archive/samfunctions/fetchECSV/Math.py b/archive/lambdas/fetchECSV/Math.py similarity index 100% rename from archive/samfunctions/fetchECSV/Math.py rename to archive/lambdas/fetchECSV/Math.py diff --git a/archive/samfunctions/fetchECSV/MeteorObservation.py b/archive/lambdas/fetchECSV/MeteorObservation.py similarity index 100% rename from archive/samfunctions/fetchECSV/MeteorObservation.py rename to archive/lambdas/fetchECSV/MeteorObservation.py diff --git a/archive/samfunctions/fetchECSV/fetchECSV.py b/archive/lambdas/fetchECSV/fetchECSV.py similarity index 100% rename from archive/samfunctions/fetchECSV/fetchECSV.py rename to archive/lambdas/fetchECSV/fetchECSV.py diff --git a/archive/samfunctions/fetchECSV/ftpDetectInfo.py b/archive/lambdas/fetchECSV/ftpDetectInfo.py similarity index 100% rename from archive/samfunctions/fetchECSV/ftpDetectInfo.py rename to archive/lambdas/fetchECSV/ftpDetectInfo.py diff --git a/archive/samfunctions/fetchECSV/requirements.txt b/archive/lambdas/fetchECSV/requirements.txt similarity index 100% rename from archive/samfunctions/fetchECSV/requirements.txt rename to archive/lambdas/fetchECSV/requirements.txt diff --git a/archive/samfunctions/fetchECSV/samconfig.toml b/archive/lambdas/fetchECSV/samconfig.toml similarity index 100% rename from archive/samfunctions/fetchECSV/samconfig.toml rename to archive/lambdas/fetchECSV/samconfig.toml diff --git a/archive/samfunctions/fetchECSV/template.yml b/archive/lambdas/fetchECSV/template.yml similarity index 100% rename from archive/samfunctions/fetchECSV/template.yml rename to archive/lambdas/fetchECSV/template.yml diff --git a/archive/samfunctions/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt b/archive/lambdas/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt similarity index 100% rename from archive/samfunctions/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt rename to archive/lambdas/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt diff --git a/archive/samfunctions/fetchECSV/tests/platepars_all_recalibrated.json b/archive/lambdas/fetchECSV/tests/platepars_all_recalibrated.json similarity index 100% rename from archive/samfunctions/fetchECSV/tests/platepars_all_recalibrated.json rename to archive/lambdas/fetchECSV/tests/platepars_all_recalibrated.json diff --git a/archive/samfunctions/fireballApi/getFireballFiles.py b/archive/lambdas/fireballApi/getFireballFiles.py similarity index 100% rename from archive/samfunctions/fireballApi/getFireballFiles.py rename to archive/lambdas/fireballApi/getFireballFiles.py diff --git a/archive/samfunctions/fireballApi/requirements.txt b/archive/lambdas/fireballApi/requirements.txt similarity index 100% rename from archive/samfunctions/fireballApi/requirements.txt rename to archive/lambdas/fireballApi/requirements.txt diff --git a/archive/samfunctions/fireballApi/samconfig.toml b/archive/lambdas/fireballApi/samconfig.toml similarity index 100% rename from archive/samfunctions/fireballApi/samconfig.toml rename to archive/lambdas/fireballApi/samconfig.toml diff --git a/archive/samfunctions/fireballApi/template.yml b/archive/lambdas/fireballApi/template.yml similarity index 100% rename from archive/samfunctions/fireballApi/template.yml rename to archive/lambdas/fireballApi/template.yml diff --git a/archive/samfunctions/ftpToUkmon/.gitignore b/archive/lambdas/ftpToUkmon/.gitignore similarity index 100% rename from archive/samfunctions/ftpToUkmon/.gitignore rename to archive/lambdas/ftpToUkmon/.gitignore diff --git a/archive/samfunctions/ftpToUkmon/README.md b/archive/lambdas/ftpToUkmon/README.md similarity index 100% rename from archive/samfunctions/ftpToUkmon/README.md rename to archive/lambdas/ftpToUkmon/README.md diff --git a/archive/samfunctions/ftpToUkmon/__init__.py b/archive/lambdas/ftpToUkmon/__init__.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/__init__.py rename to archive/lambdas/ftpToUkmon/__init__.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/Dockerfile b/archive/lambdas/ftpToUkmon/pythoncode/Dockerfile similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/Dockerfile rename to archive/lambdas/ftpToUkmon/pythoncode/Dockerfile diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/ShowerAssociation.py b/archive/lambdas/ftpToUkmon/pythoncode/ShowerAssociation.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/ShowerAssociation.py rename to archive/lambdas/ftpToUkmon/pythoncode/ShowerAssociation.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/Showers.py b/archive/lambdas/ftpToUkmon/pythoncode/Showers.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/Showers.py rename to archive/lambdas/ftpToUkmon/pythoncode/Showers.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/__init__.py b/archive/lambdas/ftpToUkmon/pythoncode/__init__.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/__init__.py rename to archive/lambdas/ftpToUkmon/pythoncode/__init__.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/established_showers.csv b/archive/lambdas/ftpToUkmon/pythoncode/established_showers.csv similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/established_showers.csv rename to archive/lambdas/ftpToUkmon/pythoncode/established_showers.csv diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/ftpToUkmon1.py b/archive/lambdas/ftpToUkmon/pythoncode/ftpToUkmon1.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/ftpToUkmon1.py rename to archive/lambdas/ftpToUkmon/pythoncode/ftpToUkmon1.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/requirements.txt b/archive/lambdas/ftpToUkmon/pythoncode/requirements.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/requirements.txt rename to archive/lambdas/ftpToUkmon/pythoncode/requirements.txt diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/supportFuncs.py b/archive/lambdas/ftpToUkmon/pythoncode/supportFuncs.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/supportFuncs.py rename to archive/lambdas/ftpToUkmon/pythoncode/supportFuncs.py diff --git a/archive/samfunctions/ftpToUkmon/samconfig.toml b/archive/lambdas/ftpToUkmon/samconfig.toml similarity index 100% rename from archive/samfunctions/ftpToUkmon/samconfig.toml rename to archive/lambdas/ftpToUkmon/samconfig.toml diff --git a/archive/samfunctions/ftpToUkmon/template.yaml b/archive/lambdas/ftpToUkmon/template.yaml similarity index 100% rename from archive/samfunctions/ftpToUkmon/template.yaml rename to archive/lambdas/ftpToUkmon/template.yaml diff --git a/archive/samfunctions/ftpToUkmon/tests/expected_results.txt b/archive/lambdas/ftpToUkmon/tests/expected_results.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/expected_results.txt rename to archive/lambdas/ftpToUkmon/tests/expected_results.txt diff --git a/archive/samfunctions/ftpToUkmon/tests/localTest.ps1 b/archive/lambdas/ftpToUkmon/tests/localTest.ps1 similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/localTest.ps1 rename to archive/lambdas/ftpToUkmon/tests/localTest.ps1 diff --git a/archive/samfunctions/ftpToUkmon/tests/new_data.txt b/archive/lambdas/ftpToUkmon/tests/new_data.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/new_data.txt rename to archive/lambdas/ftpToUkmon/tests/new_data.txt diff --git a/archive/samfunctions/ftpToUkmon/tests/remoteResult.txt b/archive/lambdas/ftpToUkmon/tests/remoteResult.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/remoteResult.txt rename to archive/lambdas/ftpToUkmon/tests/remoteResult.txt diff --git a/archive/samfunctions/ftpToUkmon/tests/remoteTest.ps1 b/archive/lambdas/ftpToUkmon/tests/remoteTest.ps1 similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/remoteTest.ps1 rename to archive/lambdas/ftpToUkmon/tests/remoteTest.ps1 diff --git a/archive/samfunctions/ftpToUkmon/tests/testEvent.json b/archive/lambdas/ftpToUkmon/tests/testEvent.json similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/testEvent.json rename to archive/lambdas/ftpToUkmon/tests/testEvent.json diff --git a/archive/samfunctions/getApiDetails.ps1 b/archive/lambdas/getApiDetails.ps1 similarity index 100% rename from archive/samfunctions/getApiDetails.ps1 rename to archive/lambdas/getApiDetails.ps1 diff --git a/archive/samfunctions/getExtraFilesV2/.gitignore b/archive/lambdas/getExtraFilesV2/.gitignore similarity index 100% rename from archive/samfunctions/getExtraFilesV2/.gitignore rename to archive/lambdas/getExtraFilesV2/.gitignore diff --git a/archive/samfunctions/getExtraFilesV2/README.md b/archive/lambdas/getExtraFilesV2/README.md similarity index 100% rename from archive/samfunctions/getExtraFilesV2/README.md rename to archive/lambdas/getExtraFilesV2/README.md diff --git a/archive/samfunctions/getExtraFilesV2/__init__.py b/archive/lambdas/getExtraFilesV2/__init__.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/__init__.py rename to archive/lambdas/getExtraFilesV2/__init__.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile b/archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile rename to archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile diff --git a/archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/requirements.txt b/archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/requirements.txt new file mode 100644 index 000000000..81af2bc95 --- /dev/null +++ b/archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/requirements.txt @@ -0,0 +1,22 @@ +numpy +cython +scipy +matplotlib==3.1.2 ; platform_machine != 'aarch64' +matplotlib==3.3.2 ; platform_machine == 'aarch64' +jplephem +pyephem +https://github.com/matplotlib/basemap/archive/master.zip ; sys_platform != 'win32' and platform_machine != 'aarch64' +basemap ; sys_platform == 'win32' or platform_machine == 'aarch64' +PyQt5 ; platform_machine != 'aarch64' +pyyaml +pyswarms +ml-dtypes +keras +pytz +pandas +cartopy +basemap-data-hires +gitpython +numba +watchdog==3.0.0; python_version == '3.7' +watchdog; python_version >= '3.8' \ No newline at end of file diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/__init__.py b/archive/lambdas/getExtraFilesV2/pythoncode/__init__.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/__init__.py rename to archive/lambdas/getExtraFilesV2/pythoncode/__init__.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py b/archive/lambdas/getExtraFilesV2/pythoncode/createOrbitPageIndex.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py rename to archive/lambdas/getExtraFilesV2/pythoncode/createOrbitPageIndex.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py b/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py rename to archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/pickleAnalysis.py b/archive/lambdas/getExtraFilesV2/pythoncode/pickleAnalysis.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/pickleAnalysis.py rename to archive/lambdas/getExtraFilesV2/pythoncode/pickleAnalysis.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/requirements.txt b/archive/lambdas/getExtraFilesV2/pythoncode/requirements.txt similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/requirements.txt rename to archive/lambdas/getExtraFilesV2/pythoncode/requirements.txt diff --git a/archive/samfunctions/getExtraFilesV2/samconfig.toml b/archive/lambdas/getExtraFilesV2/samconfig.toml similarity index 100% rename from archive/samfunctions/getExtraFilesV2/samconfig.toml rename to archive/lambdas/getExtraFilesV2/samconfig.toml diff --git a/archive/samfunctions/getExtraFilesV2/template.yaml b/archive/lambdas/getExtraFilesV2/template.yaml similarity index 100% rename from archive/samfunctions/getExtraFilesV2/template.yaml rename to archive/lambdas/getExtraFilesV2/template.yaml diff --git a/archive/samfunctions/getExtraFilesV2/tests/localTest.ps1 b/archive/lambdas/getExtraFilesV2/tests/localTest.ps1 similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/localTest.ps1 rename to archive/lambdas/getExtraFilesV2/tests/localTest.ps1 diff --git a/archive/samfunctions/getExtraFilesV2/tests/remoteTest.ps1 b/archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/remoteTest.ps1 rename to archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 diff --git a/archive/samfunctions/getExtraFilesV2/tests/templateEvent.json b/archive/lambdas/getExtraFilesV2/tests/templateEvent.json similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/templateEvent.json rename to archive/lambdas/getExtraFilesV2/tests/templateEvent.json diff --git a/archive/samfunctions/getExtraFilesV2/tests/testEvent.json b/archive/lambdas/getExtraFilesV2/tests/testEvent.json similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/testEvent.json rename to archive/lambdas/getExtraFilesV2/tests/testEvent.json diff --git a/archive/samfunctions/getExtraFilesV2/tests/testEvent2.json b/archive/lambdas/getExtraFilesV2/tests/testEvent2.json similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/testEvent2.json rename to archive/lambdas/getExtraFilesV2/tests/testEvent2.json diff --git a/archive/samfunctions/getExtraFilesV2/update_wmpl.sh b/archive/lambdas/getExtraFilesV2/update_wmpl.sh similarity index 100% rename from archive/samfunctions/getExtraFilesV2/update_wmpl.sh rename to archive/lambdas/getExtraFilesV2/update_wmpl.sh diff --git a/archive/samfunctions/getExtraFilesV2/wmpl__init__.py_fixed.py b/archive/lambdas/getExtraFilesV2/wmpl__init__.py_fixed.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/wmpl__init__.py_fixed.py rename to archive/lambdas/getExtraFilesV2/wmpl__init__.py_fixed.py diff --git a/archive/samfunctions/getLiveImages/getLiveImages.py b/archive/lambdas/getLiveImages/getLiveImages.py similarity index 100% rename from archive/samfunctions/getLiveImages/getLiveImages.py rename to archive/lambdas/getLiveImages/getLiveImages.py diff --git a/archive/samfunctions/getLiveImages/requirements.txt b/archive/lambdas/getLiveImages/requirements.txt similarity index 100% rename from archive/samfunctions/getLiveImages/requirements.txt rename to archive/lambdas/getLiveImages/requirements.txt diff --git a/archive/samfunctions/getLiveImages/samconfig.toml b/archive/lambdas/getLiveImages/samconfig.toml similarity index 100% rename from archive/samfunctions/getLiveImages/samconfig.toml rename to archive/lambdas/getLiveImages/samconfig.toml diff --git a/archive/samfunctions/getLiveImages/template.yml b/archive/lambdas/getLiveImages/template.yml similarity index 100% rename from archive/samfunctions/getLiveImages/template.yml rename to archive/lambdas/getLiveImages/template.yml diff --git a/archive/samfunctions/matchDataApi/localTest.ps1 b/archive/lambdas/matchDataApi/localTest.ps1 similarity index 100% rename from archive/samfunctions/matchDataApi/localTest.ps1 rename to archive/lambdas/matchDataApi/localTest.ps1 diff --git a/archive/samfunctions/matchDataApi/matchDataApi.py b/archive/lambdas/matchDataApi/matchDataApi.py similarity index 100% rename from archive/samfunctions/matchDataApi/matchDataApi.py rename to archive/lambdas/matchDataApi/matchDataApi.py diff --git a/archive/samfunctions/matchDataApi/requirements.txt b/archive/lambdas/matchDataApi/requirements.txt similarity index 100% rename from archive/samfunctions/matchDataApi/requirements.txt rename to archive/lambdas/matchDataApi/requirements.txt diff --git a/archive/samfunctions/matchDataApi/samconfig.toml b/archive/lambdas/matchDataApi/samconfig.toml similarity index 100% rename from archive/samfunctions/matchDataApi/samconfig.toml rename to archive/lambdas/matchDataApi/samconfig.toml diff --git a/archive/samfunctions/matchDataApi/template.yml b/archive/lambdas/matchDataApi/template.yml similarity index 100% rename from archive/samfunctions/matchDataApi/template.yml rename to archive/lambdas/matchDataApi/template.yml diff --git a/archive/samfunctions/matchDataApi/testDetail.json b/archive/lambdas/matchDataApi/testDetail.json similarity index 100% rename from archive/samfunctions/matchDataApi/testDetail.json rename to archive/lambdas/matchDataApi/testDetail.json diff --git a/archive/samfunctions/matchDataApi/testMatches.json b/archive/lambdas/matchDataApi/testMatches.json similarity index 100% rename from archive/samfunctions/matchDataApi/testMatches.json rename to archive/lambdas/matchDataApi/testMatches.json diff --git a/archive/samfunctions/matchDataApi/testMatchesPeriod.json b/archive/lambdas/matchDataApi/testMatchesPeriod.json similarity index 100% rename from archive/samfunctions/matchDataApi/testMatchesPeriod.json rename to archive/lambdas/matchDataApi/testMatchesPeriod.json diff --git a/archive/samfunctions/matchDataApi/testPoints.json b/archive/lambdas/matchDataApi/testPoints.json similarity index 100% rename from archive/samfunctions/matchDataApi/testPoints.json rename to archive/lambdas/matchDataApi/testPoints.json diff --git a/archive/samfunctions/matchDataApi/testStation.json b/archive/lambdas/matchDataApi/testStation.json similarity index 100% rename from archive/samfunctions/matchDataApi/testStation.json rename to archive/lambdas/matchDataApi/testStation.json diff --git a/archive/samfunctions/matchDataApi/testSummary.json b/archive/lambdas/matchDataApi/testSummary.json similarity index 100% rename from archive/samfunctions/matchDataApi/testSummary.json rename to archive/lambdas/matchDataApi/testSummary.json diff --git a/archive/samfunctions/matchPickle/__init__.py b/archive/lambdas/matchPickle/__init__.py similarity index 100% rename from archive/samfunctions/matchPickle/__init__.py rename to archive/lambdas/matchPickle/__init__.py diff --git a/archive/samfunctions/matchPickle/localTest.ps1 b/archive/lambdas/matchPickle/localTest.ps1 similarity index 100% rename from archive/samfunctions/matchPickle/localTest.ps1 rename to archive/lambdas/matchPickle/localTest.ps1 diff --git a/archive/samfunctions/matchPickle/pythoncode/Dockerfile b/archive/lambdas/matchPickle/pythoncode/Dockerfile similarity index 100% rename from archive/samfunctions/matchPickle/pythoncode/Dockerfile rename to archive/lambdas/matchPickle/pythoncode/Dockerfile diff --git a/archive/samfunctions/matchPickle/pythoncode/__init__.py b/archive/lambdas/matchPickle/pythoncode/__init__.py similarity index 100% rename from archive/samfunctions/matchPickle/pythoncode/__init__.py rename to archive/lambdas/matchPickle/pythoncode/__init__.py diff --git a/archive/samfunctions/matchPickle/pythoncode/matchPickleApi.py b/archive/lambdas/matchPickle/pythoncode/matchPickleApi.py similarity index 100% rename from archive/samfunctions/matchPickle/pythoncode/matchPickleApi.py rename to archive/lambdas/matchPickle/pythoncode/matchPickleApi.py diff --git a/archive/samfunctions/matchPickle/pythoncode/requirements.txt b/archive/lambdas/matchPickle/pythoncode/requirements.txt similarity index 100% rename from archive/samfunctions/matchPickle/pythoncode/requirements.txt rename to archive/lambdas/matchPickle/pythoncode/requirements.txt diff --git a/archive/samfunctions/matchPickle/samconfig.toml b/archive/lambdas/matchPickle/samconfig.toml similarity index 100% rename from archive/samfunctions/matchPickle/samconfig.toml rename to archive/lambdas/matchPickle/samconfig.toml diff --git a/archive/samfunctions/matchPickle/template.yml b/archive/lambdas/matchPickle/template.yml similarity index 100% rename from archive/samfunctions/matchPickle/template.yml rename to archive/lambdas/matchPickle/template.yml diff --git a/archive/samfunctions/matchPickle/testEvent.json b/archive/lambdas/matchPickle/testEvent.json similarity index 100% rename from archive/samfunctions/matchPickle/testEvent.json rename to archive/lambdas/matchPickle/testEvent.json diff --git a/archive/samfunctions/searchArchive/requirements.txt b/archive/lambdas/searchArchive/requirements.txt similarity index 100% rename from archive/samfunctions/searchArchive/requirements.txt rename to archive/lambdas/searchArchive/requirements.txt diff --git a/archive/samfunctions/searchArchive/samconfig.toml b/archive/lambdas/searchArchive/samconfig.toml similarity index 100% rename from archive/samfunctions/searchArchive/samconfig.toml rename to archive/lambdas/searchArchive/samconfig.toml diff --git a/archive/samfunctions/searchArchive/searchArchive.py b/archive/lambdas/searchArchive/searchArchive.py similarity index 100% rename from archive/samfunctions/searchArchive/searchArchive.py rename to archive/lambdas/searchArchive/searchArchive.py diff --git a/archive/samfunctions/searchArchive/template.yml b/archive/lambdas/searchArchive/template.yml similarity index 100% rename from archive/samfunctions/searchArchive/template.yml rename to archive/lambdas/searchArchive/template.yml diff --git a/archive/samfunctions/searchArchive/testEvent.json b/archive/lambdas/searchArchive/testEvent.json similarity index 100% rename from archive/samfunctions/searchArchive/testEvent.json rename to archive/lambdas/searchArchive/testEvent.json diff --git a/archive/samfunctions/updateDnsRecords/lambda_function.py b/archive/lambdas/updateDnsRecords/lambda_function.py similarity index 100% rename from archive/samfunctions/updateDnsRecords/lambda_function.py rename to archive/lambdas/updateDnsRecords/lambda_function.py diff --git a/archive/samfunctions/updateDnsRecords/samconfig.toml b/archive/lambdas/updateDnsRecords/samconfig.toml similarity index 100% rename from archive/samfunctions/updateDnsRecords/samconfig.toml rename to archive/lambdas/updateDnsRecords/samconfig.toml diff --git a/archive/samfunctions/updateDnsRecords/template.yaml b/archive/lambdas/updateDnsRecords/template.yaml similarity index 100% rename from archive/samfunctions/updateDnsRecords/template.yaml rename to archive/lambdas/updateDnsRecords/template.yaml From 846761aeec78ef0498c4c1657d8cd9f52734b433 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 6 Apr 2026 22:04:07 +0100 Subject: [PATCH 10/15] March 2026 release (#423) * remove disused code * update livestream to accept normal RMS filenames * search dialog - fix bug and set default date range * small change to explain default dates * improvement in live image search * remove redundant file * update deployment scripts * improvements in routine to convert GMN report to pandas * more analysis of gmn data --- archive/deployment/website.yml | 1 - .../lambdas/getLiveImages/getLiveImages.py | 11 +- archive/static_content/js/searchdialog.js | 14 ++ archive/static_content/search/index.html | 3 +- .../files/monitorLiveFeed/monitorLiveFeed.py | 42 +++--- archive/terraform/ukmda/ukmdalive_bucket.tf | 52 -------- .../ukmon_pylib/analysis/analyseGmnData.py | 87 ++++++++++++- .../ukmon_pylib/converters/gmnTxtToPandas.py | 10 +- archive/website/templates/searchdialog.js | 122 ------------------ 9 files changed, 134 insertions(+), 208 deletions(-) delete mode 100644 archive/website/templates/searchdialog.js diff --git a/archive/deployment/website.yml b/archive/deployment/website.yml index 1ee42364e..3d429b597 100644 --- a/archive/deployment/website.yml +++ b/archive/deployment/website.yml @@ -37,6 +37,5 @@ - {src: '{{srcdir}}/website/templates/frontpage.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - {src: '{{srcdir}}/website/templates/header.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - {src: '{{srcdir}}/website/templates/reportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/searchdialog.js', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - {src: '{{srcdir}}/website/templates/shwrcsvindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - {src: '{{srcdir}}/website/templates/statreportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } diff --git a/archive/lambdas/getLiveImages/getLiveImages.py b/archive/lambdas/getLiveImages/getLiveImages.py index 164ecc605..5d589131d 100644 --- a/archive/lambdas/getLiveImages/getLiveImages.py +++ b/archive/lambdas/getLiveImages/getLiveImages.py @@ -16,7 +16,10 @@ def getLiveImages(dtstr, ddb=None): resp = table.query(IndexName='month-image_name-index', KeyConditionExpression=Key('month').eq(dtstr[4:6]) & Key('image_name').begins_with(f'M{dtstr}'), ProjectionExpression='image_name') - return resp + resp2 = table.query(IndexName='month-image_name-index', + KeyConditionExpression=Key('month').eq(dtstr[4:6]) & Key('image_name').begins_with(f'FF_{dtstr}'), + ProjectionExpression='image_name') + return resp['Items'] + resp2['Items'] def filterImages(d1, d2, statid=None, maxitems=-1, ddb=None): @@ -103,10 +106,10 @@ def lambda_handler(event, context): patt = qs['pattern'] print(f'searching for {patt}') ecsvstr = getLiveImages(patt, ddb=ddb) - print(f"found {ecsvstr['Items']}") + print(f"found {len(ecsvstr)} items") return { 'statusCode': 200, - 'body': json.dumps(ecsvstr['Items']) + 'body': json.dumps(ecsvstr) } else: if 'dtstr' in qs: @@ -119,7 +122,7 @@ def lambda_handler(event, context): if 'enddtstr' in qs: dtstr2 = qs['enddtstr'] else: - dtstr2 = datetime.datetime.now().strftime('M%Y%m%d_%H') + dtstr2 = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S.000Z') statid = None if 'statid' in qs: statid = qs['statid'] diff --git a/archive/static_content/js/searchdialog.js b/archive/static_content/js/searchdialog.js index 5b9e0ea52..d5716b036 100644 --- a/archive/static_content/js/searchdialog.js +++ b/archive/static_content/js/searchdialog.js @@ -70,6 +70,17 @@ form.addEventListener("submit", function (event) { op = op + "l:" + strstat + "_"; } //jQuery.support.cors = true; + if (d1 == "" ){ + console.log("missing start date"); + const stdt = new Date(Date.now()); + stdt.setHours(stdt.getHours() - 12); + d1 = stdt.toISOString(); + } + if (d2 == "" ){ + console.log("missing end date"); + const enddt = new Date(Date.now()); + d2 = enddt.toISOString(); + } payload = {"d1": d1, "d2": d2, "opts": op }; console.log(payload); document.getElementById("searchresults").innerHTML = "Searching...."; @@ -96,6 +107,9 @@ function myFunc(myObj) { txt = "" txt += ""; txt += ""; + if (myObj.length == 0){ + alert("No data available, check dates"); + } for (x in myObj) { console.log(myObj[x]); var dtarr = myObj[x].split(','); diff --git a/archive/static_content/search/index.html b/archive/static_content/search/index.html index 5fa067bec..391ede95f 100644 --- a/archive/static_content/search/index.html +++ b/archive/static_content/search/index.html @@ -78,8 +78,9 @@

Search the Archive.



This page allows you to search the archive for specific events in the matched and single-station detections.
- Time should be entered as local time, so in the UK summer you should enter the time in BST. + Time should be entered as local time, so in the UK summer you should enter the time in BST.

+

NOTE - default date range is the last 24 hours. This might be a HUGE amount of data during busy showers.

To search the livestream, go over to this page.

diff --git a/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py b/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py index 654ad06a9..40c3a4214 100644 --- a/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py +++ b/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py @@ -61,28 +61,19 @@ def storeInDDb(evtdets, camdets): return response['ResponseMetadata']['HTTPStatusCode'] -def updateLiveTable(event, dtval): - record = event['Records'][0] - fname = record['s3']['object']['key'] - _, barefname = os.path.split(fname) - if 'P.jpg' not in barefname: - print(f'{barefname} not a jpg') - return +def updateLiveTable(evtdets, camdets, fname): + statname = camdets['camid'] + dtval = evtdets['dtval'] expdate = int((dtval + datetime.timedelta(days=90)).timestamp()) tstamp = str(int(dtval.timestamp()*1000)) yr = dtval.strftime('%Y') mth = dtval.strftime('%m') ddb = boto3.resource('dynamodb', region_name='eu-west-2') table = ddb.Table('live') - if barefname[0] == 'M': - statname = barefname[17:].replace('P.jpg','').replace('_',' ') - else: - statname = barefname[3:9] - - print(f'inserting {barefname} with timestamp {dtval}') + print(f'inserting {fname} with timestamp {dtval}') response = table.put_item( Item={ - 'image_name': barefname, + 'image_name': fname, 'timestamp': tstamp, 'image_timestamp': tstamp, 'station_name': statname, @@ -94,19 +85,15 @@ def updateLiveTable(event, dtval): return response['ResponseMetadata']['HTTPStatusCode'] -def processXml(event): - record = event['Records'][0] - fname = record['s3']['object']['key'] +def processXml(remote_xmlname): buck = 'ukmda-live' s3 = boto3.resource('s3') - if '.xml' not in fname: - fname = fname.replace('P.jpg','.xml') - _, barefname = os.path.split(fname) + _, barefname = os.path.split(remote_xmlname) tmpdir = mkdtemp() xmlname = os.path.join(tmpdir, barefname) try: - s3.meta.client.download_file(buck, fname, xmlname) + s3.meta.client.download_file(buck, remote_xmlname, xmlname) except Exception: print('xml file not available') return None, None @@ -157,7 +144,12 @@ def processXml(event): def lambda_handler(event, context): - evtdets, camdets = processXml(event) - if evtdets is not None: - storeInDDb(evtdets, camdets) - updateLiveTable(event, evtdets['dtval']) + record = event['Records'][0] + fname = record['s3']['object']['key'] + _, barefname = os.path.split(fname) + if ('M' in barefname and 'P.jpg' in barefname) or ('FF' in barefname and '.jpg' in barefname): + xmlname = fname.replace('P.jpg','.xml').replace('.jpg', '.xml') + evtdets, camdets = processXml(xmlname) + if evtdets is not None: + storeInDDb(evtdets, camdets) + updateLiveTable(evtdets, camdets, barefname) diff --git a/archive/terraform/ukmda/ukmdalive_bucket.tf b/archive/terraform/ukmda/ukmdalive_bucket.tf index 7e1c405bb..45ea7d50c 100644 --- a/archive/terraform/ukmda/ukmdalive_bucket.tf +++ b/archive/terraform/ukmda/ukmdalive_bucket.tf @@ -9,58 +9,6 @@ resource "aws_s3_bucket" "ukmdalive" { } provider = aws.eu-west-1-prov } -/* -resource "aws_s3_bucket_policy" "ukmdalivebp" { - bucket = aws_s3_bucket.ukmdalive.id - provider = aws.eu-west-1-prov - policy = jsonencode( - { - Id = "ukmda-live-bp" - Statement = [ - { - "Sid": "DataSyncCreateS3LocationAndTaskAccess", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::${var.eeaccountid}:role/DataSyncBetweenAccounts" - }, - "Action": [ - "s3:GetBucketLocation", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:AbortMultipartUpload", - "s3:DeleteObject", - "s3:GetObject", - "s3:ListMultipartUploadParts", - "s3:PutObject", - "s3:GetObjectTagging", - "s3:PutObjectTagging" - ], - "Resource": [ - "${aws_s3_bucket.ukmdalive.arn}", - "${aws_s3_bucket.ukmdalive.arn}/*" - ] - }, - { - "Sid": "replicatelive", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::${var.eeaccountid}:role/service-role/replicatelive" - }, - "Action": [ - "s3:Put*", - "s3:Get*" - ], - "Resource": [ - "${aws_s3_bucket.ukmdalive.arn}", - "${aws_s3_bucket.ukmdalive.arn}/*" - ] - } - ] - Version = "2008-10-17" - } - ) -} -*/ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalivelcp" { bucket = aws_s3_bucket.ukmdalive.id diff --git a/archive/ukmon_pylib/analysis/analyseGmnData.py b/archive/ukmon_pylib/analysis/analyseGmnData.py index 77920b99c..749216f3c 100644 --- a/archive/ukmon_pylib/analysis/analyseGmnData.py +++ b/archive/ukmon_pylib/analysis/analyseGmnData.py @@ -2,11 +2,40 @@ import pandas as pd import os +import sys +import datetime from converters.gmnTxtToPandas import dirpath -def findDuplicates(yr, mth=None): +def analyseAMonth(fulldf, yr, mth): + if mth != 0: + beg = datetime.datetime(yr, mth, 1) + if mth < 12: + end = datetime.datetime(yr, mth+1, 1) + else: + end = datetime.datetime(yr+1, 1, 1) + mthdf = fulldf[fulldf.utc_beg >= beg] + mthdf = mthdf[mthdf.utc_beg < end] + else: + mthdf = fulldf + tot_num = len(mthdf) + uktraj = mthdf[mthdf.stats.str.contains('UK')] + num_uk_traj = len(uktraj) + allstats = ','.join(uktraj.stats) + stats = list(set(allstats.split(','))) + ukstats = [x for x in stats if 'UK' in x] + otherstats = [x for x in stats if 'UK' not in x] + distinctuk = [x for x in ukstats if '_' not in x] + distinctother = [x for x in otherstats if '_' not in x] + ctrys = list(set([x[:2] for x in otherstats])) + otherctrydf = uktraj[uktraj.stats.str.contains('|'.join(ctrys))] + if mth == 0: + uktraj.to_csv('uk-trajectories.csv', index=False) + return len(distinctuk), len(distinctother), num_uk_traj, tot_num, len(otherctrydf) + + +def findDuplicatesById(yr, mth=None): if mth: datafile = os.path.join(dirpath, 'parquet', 'monthly', f'gmn_{yr:04d}{mth:02d}.parquet.snap') else: @@ -18,3 +47,59 @@ def findDuplicates(yr, mth=None): print(duperows) print(len(df)) return duperows + + +def atleastOneStation(stats,statsnext): + if stats is None or statsnext is None: + return False + stats = stats.split(',') + statsnext = statsnext.split(',') + return any(i in statsnext for i in stats) + + +def findDuplicatesByStatAndJD(yr, mth=None, df=None): + if not df: + dirpath = '.' + if mth: + datafile = os.path.join(dirpath, 'parquet', 'monthly', f'gmn_{yr:04d}{mth:02d}.parquet.snap') + else: + datafile = os.path.join(dirpath, 'parquet', f'gmn_{yr:04d}.parquet.snap') + df = pd.read_parquet(datafile) + df['statsnext'] = df.stats.shift(-1) + df['jd_next'] = df.jd_beg.shift(-1) + df['Lat1_next'] = df.Lat1.shift(-1) + df['Lon1_next'] = df.Lon1.shift(-1) + nearjd = df.query('abs(jd_beg - jd_next) < (0.5/86400)') + nearlat = nearjd.query('abs(Lat1 - Lat1_next) < 1 and abs(Lon1 - Lon1_next) < 1') + samestats = nearlat.query('stats == statsnext') + nearlat['overlapstats'] = nearlat.apply(lambda row: atleastOneStation(row.stats, row.statsnext), axis=1) + commonstats = nearlat[nearlat.overlapstats] + print(f'there are {len(df)} trajectories in the period') + print(f'there are {len(nearjd)} ({round(100*len(nearjd)/len(df),2)}%) events within 1s') + print(f'there are {len(commonstats)} ({round(100*len(commonstats)/len(df),2)}%) with at least one common station') + print(f'there are {len(samestats)} ({round(100*len(samestats)/len(df),2)}%) with the same stations') + return nearjd, commonstats, samestats + + +if __name__ == '__main__': + yr = int(sys.argv[1]) + fulldf = None + data = [] + for mth in range(1,13): + print(f'processing month {mth}') + df = pd.read_parquet(f'parquet/monthly/gmn_{yr}{mth:02d}.parquet.snap') + cams, othercams, traj, totaltraj, othertraj = analyseAMonth(df, yr, mth) + data.append([mth, cams, othercams, traj, totaltraj, othertraj]) + fulldf = df if fulldf is None else pd.concat([fulldf, df]) + + result = pd.DataFrame(data, columns=['Mth','Camcount','OtherCams','TrajCount','TotalTraj','NonUKTraj']) + tmpres = result.set_index('Mth') + plot = tmpres.plot(title='UK Statistics') + fig = plot.get_figure() + fig.savefig(f'uk-analysis-{yr}.jpg') + + cams, othercams, traj, totaltraj, otherctry, = analyseAMonth(fulldf, yr, 0) + data.append([0, cams, othercams, traj, totaltraj, otherctry]) + + result = pd.DataFrame(data, columns=['Mth','Camcount','OtherCams','TrajCount','TotalTraj','NonUKTraj']) + result.to_csv('2025-stats.csv', index=False) diff --git a/archive/ukmon_pylib/converters/gmnTxtToPandas.py b/archive/ukmon_pylib/converters/gmnTxtToPandas.py index ad98886f9..d274779c1 100644 --- a/archive/ukmon_pylib/converters/gmnTxtToPandas.py +++ b/archive/ukmon_pylib/converters/gmnTxtToPandas.py @@ -37,7 +37,7 @@ def loadOneFile(fname): df = df.fillna(value=np.nan) df['jd_beg'] = df.jd_beg.astype(float) df['utc_beg'] = pd.to_datetime(df.utc_beg) - df['ian_uo'] = df.iau_no.astype(int) + df['iau_no'] = df.iau_no.astype(int) for c in range(5,82): try: df[colhdrs[c]] = df[colhdrs[c]].astype(float) @@ -138,4 +138,10 @@ def getStats(): if __name__ == '__main__': - doYear(sys.argv[1]) + if sys.argv[1] == 'doYear': + doYear(sys.argv[2]) + if sys.argv[1] == 'convert': + df = loadOneFile(sys.argv[2]) + fn, _ = os.path.splitext(sys.argv[2]) + df.to_parquet(f'{fn}.parquet.snap', index=False) + diff --git a/archive/website/templates/searchdialog.js b/archive/website/templates/searchdialog.js deleted file mode 100644 index e85d171cf..000000000 --- a/archive/website/templates/searchdialog.js +++ /dev/null @@ -1,122 +0,0 @@ -// initialize input widgets first -// Copyright (C) 2018-2023 Mark McIntyre - -$('#selectInterval .time').timepicker({ - 'showDuration': true, - 'timeFormat': 'g:ia' -}); - -$('#selectInterval .date').datepicker({ - 'format': 'd/m/yyyy', - 'autoclose': true, - 'endDate': '0' - -}); - -// initialize datepair -var res = document.getElementById("selectInterval"); -var dateSelect = new Datepair(res, { - 'defaultDateDelta': 0, // days - 'defaultTimeDelta': 900000 // milliseconds = 15 minutes -}); - -$('#selectInterval').on('rangeSelected', function(){ - var startdate = $('#selectInterval .date:first').datepicker('getDate'); - var starttime = $('#selectInterval .time:first').timepicker('getTime', startdate); - if(starttime != null){ - var dateval = starttime.getTime(); - var timediff = dateSelect.getTimeDiff(); - var endval = dateval + timediff; - endtime = new Date(); - endtime.setTime(endval); - $('#datestart').text(starttime.toISOString()); - $('#dateend').text(endtime.toISOString()); - if(timediff > 0){ - $('#statusfield').text('Valid range selected'); - }else{ - $('#statusfield').text('Date range must be > 0'); - } - } -}); - -var apiurl = 'https://api.ukmeteors.co.uk/detections'; -var form = document.querySelector("form"); -form.addEventListener("submit", function (event) { - //console.log("Saving value", form.elements.value.value); - var d1 = document.getElementById("datestart").innerHTML; - var d2 = document.getElementById("dateend").innerHTML; - var matchOnly = document.getElementById("matchesOnly").checked; - console.log(matchOnly) - var op = ""; - if (matchOnly == true ) { - op = op + "t:_"; - } - var magSelect = document.getElementById("magselect").value; - if (magSelect != 1 ) { - if(magSelect == 2) {op = op + "m:0_"} - if(magSelect == 3) {op = op + "m:-4_"} - } - var shwrSelect = document.getElementById("shwrselect").value; - if (shwrSelect != 1 ) { - var e = document.getElementById("shwrselect"); - var strshwr = e.options[e.selectedIndex].text; - op = op + "s:" + strshwr + "_"; - } - var statSelect = document.getElementById("statselect").value; - if (statSelect != 1 ) { - var e = document.getElementById("statselect"); - var strstat = e.options[e.selectedIndex].text; - op = op + "l:" + strstat + "_"; - } - //jQuery.support.cors = true; - payload = {"d1": d1, "d2": d2, "opts": op }; - console.log(payload); - document.getElementById("searchresults").innerHTML = "Searching...."; - $.ajax({ - url: apiurl, - type: "GET", - data: payload, - dataType: 'jsonp', - error: function (xhr, status, ex ) { - if (status === 'error' ) { - alert("Too much data, try a narrower range"); - console.log(xhr.status); - } - }, - complete: function (xhr, status) { - } - }); - event.preventDefault();} - ); - -function myFunc(myObj) { - console.log(myObj); - var x, txt = ""; - txt = "
DateTimeSourceShowerMagCameraLink
" - txt += ""; - txt += ""; - for (x in myObj) { - console.log(myObj[x]); - var dtarr = myObj[x].split(','); - var cams = dtarr[4].replaceAll(";", "; "); - if (dtarr[0] > 0 ){ - txt +='"; - } - } - txt += "
DateTimeSourceShowerMagCameraLink
'; - dt = new Date(dtarr[0]*1000); - txt += dt.toISOString(); - txt += ""; - txt += dtarr[1]; - txt += ""; - txt += dtarr[2]; - txt += ""; - txt += dtarr[3]; - txt += ""; - txt += cams; - txt += ""; - errimg = "onerror=\"this.onerror=null;this.src='/img/missing.png';\""; - txt += "\"Image"; - txt += "
"; - document.getElementById("searchresults").innerHTML = txt; -} From 783e5ba045493de72d26d86ce1334f02ddd5e522 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 23:24:53 +0100 Subject: [PATCH 11/15] April 2026 upgrade (#432) * remove disused code * update livestream to accept normal RMS filenames * search dialog - fix bug and set default date range * small change to explain default dates * improvement in live image search * remove redundant file * update deployment scripts * improvements in routine to convert GMN report to pandas * more analysis of gmn data * Update the terraform a little * remove hardcoded reference to ec2-user * add SERVERUSERID env /ssm variable for calcserver user id * update container for new distrib-processing method * use calcserver SERVERUSERID variable * support for test ECS containers * fix paths in deployment scripts * remove unused file * missed some templates * update dev/test env settings * remove disused file * make sure paths are expanded * fixing various config issues * updates to makeConfig * updates to distributed processing to align with newest wmpl * add a comment to explain what the code does * remove json db * update consolidation to handle sqlite dbs * remove unused function * create data for daily report * remove unused file * update to call new version of reportOfLatestMatches * add test variables * mildly improve doco * put test logs in the right CW log group * handle missing log groups better * extra value in TrajQualityParams * bugfixes revealed in testing * fix issues in reportOfLatestMatches * put container dbs in a sensible place * catch logging errors * tidy up file locations * simplify calcserver start up * sorting out errors in container logging * update container logging info * avoid unnecessary error * fix bugs in log capture process * add MAX_STNS so i can experiement with limits * add logging. Add max_stns to constraints and set to 9999 push logs to s3 copy calcserver logs to ukmonhelper * don't pull or push the databases from S3 during distribution but do back them up after consolidation * catch error if trajpickle can't be found * set rundate more safely move some processes into nightlyJob to make testing easier * tidy up backup file names and housekeep backup folder * move some processes from findAllMatches to nightlyJob to make testing easier * support for testing of createExecReplot * pass testmode to createExecReplot * ensure logs are unique and capture container logs * realigning createFireballPage with prod. Updating gitignore * remove verbosity flag from one tar operation * fixup to handle new style log with datestamps * get privip after restarting calcserver * temporarily don't run rerunFailedLambdas in test * update to work in a test env too * add 'period' to the match summary api * add period to match summary api * re-enabling rerunFailedLambdas * quieten an S3 transfer * include the foldername when reporting on container progress * put daily trajdb in a safe place * be more specific when finding the daily log * only tar the current container logs * save daily obsdb in the right place * rename ec2.tf to calcserver.tf * quieten some s3 transfers * rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is * update SSM params to reflect new serverid * bugfix in distributeCandidates - using wrong path * add newline to getCostMetrics * weird bug in statsToMqtt * bugfix in update_wmpl.sh for traj container * Script to create an AMI for the calcengine * exclude test data when syncing from s3 * writing databases and logs to the wrong target * bad test on istest * typo in serveruserid * adding to server migration notes * purge old rawcsvs and fullcsvs older than 180 days * improved housekeeping of old data * make sure we scoop up data for 31/12 * stop creation of an unused set of files * change default locations for container data * put container logs in the right place * update dockerfile syntax * change locations of test container data and output * remove incorrect comments * Add script to clean up deleted/duplicate trajs * simplify the stats gathering and move to python fix bug in getMatchStats * comment out unused files * update reportOfLatestMatches to exclude deleted trajectories * add scp to requirements * add SQL database maintenance to dataMaintenance * incorrect default for datadir * update call to pymysql.connect * ensure consolidateOutput rerunnable for prior years * upgrading WMPL * update gitignore a bit * bugfix to catch data from 31/12 each year * add paramiko to dependencies for matchPickle to support wmpl upgrade * update getpickle to newest WMPL * bugfix in api tests --- .gitignore | 11 +- archive/analysis/README.md | 16 +- archive/analysis/consolidateOutput.sh | 16 +- archive/analysis/findAllMatches.sh | 33 +- archive/analysis/getRMSSingleData.sh | 5 +- archive/analysis/onlyConsolDistrib.sh | 122 - archive/analysis/runDistrib.sh | 157 +- archive/analysis/updatePlotsAndDetStatus.sh | 10 +- archive/containers/trajsolver/Dockerfile | 2 + archive/containers/trajsolver/build.ps1 | 2 +- archive/containers/trajsolver/trajsolver.py | 111 +- archive/containers/trajsolver/update_wmpl.sh | 4 +- archive/cronjobs/nightlyJob.sh | 24 +- archive/deployment/analysis.yml | 3 +- archive/deployment/config.yml | 2 +- archive/deployment/database.yml | 2 +- archive/deployment/dev-vars.yml | 3 - archive/deployment/prod-vars.yml | 3 - archive/deployment/pylib.yml | 4 +- archive/deployment/server_setup.yml | 10 +- archive/deployment/shwrinfo.yml | 2 +- archive/deployment/utils.yml | 2 +- archive/deployment/website.yml | 2 +- archive/deployment/website_static.yml | 3 +- .../getExtraFilesV2/pythoncode/Dockerfile | 5 +- .../WesternMeteorPyLib/requirements.txt | 22 - .../pythoncode/getExtraFiles.py | 63 +- .../getExtraFilesV2/tests/remoteTest.ps1 | 1 - .../getExtraFilesV2/tests/testEvent2.json | 2 +- .../lambdas/getExtraFilesV2/update_wmpl.sh | 8 +- archive/lambdas/matchDataApi/matchDataApi.py | 10 +- .../lambdas/matchDataApi/testSumPeriod.json | 8 + .../lambdas/matchPickle/pythoncode/Dockerfile | 8 +- .../matchPickle/pythoncode/requirements.txt | 3 +- archive/lambdas/matchPickle/remoteTest.ps1 | 5 + archive/lambdas/matchPickle/testresult.json | 4157 +++++++++++++++++ archive/server_setup/.bash_aliases | 8 +- archive/server_setup/migratingBatchServer.md | 25 +- archive/terraform/mjmm/dev_ssm_parameters.tf | 13 +- archive/terraform/mjmm/iam.tf | 2 +- archive/terraform/mjmm/s3.tf | 9 +- archive/terraform/mjmm/ssm_parameters.tf | 13 +- .../mjmm/{ec2.tf => ukmonhelper_ec2.tf} | 0 archive/terraform/mjmm/variables.tf | 6 +- .../terraform/ukmda/{ec2.tf => calcserver.tf} | 4 +- archive/terraform/ukmda/ecs_vpc.tf | 4 - archive/terraform/ukmda/ecscluster-test.tf | 73 + archive/terraform/ukmda/ecscluster.tf | 4 + .../analysis/gatherDetectionData.py | 4 +- .../ukmon_pylib/analysis/showerAnalysis.py | 2 +- .../ukmon_pylib/analysis/stationAnalysis.py | 4 +- .../ukmon_pylib/analysis/summaryAnalysis.py | 8 +- .../ukmon_pylib/converters/gmnTxtToPandas.py | 2 +- .../maintenance/dataMaintenance.py | 96 + .../maintenance/getUserAndKeyInfo.py | 4 +- archive/ukmon_pylib/maintenance/manageTraj.py | 2 +- .../maintenance/plotStationsOnMap.py | 6 +- .../maintenance/rerunFailedLambdas.py | 6 +- archive/ukmon_pylib/metrics/camMetrics.py | 9 +- archive/ukmon_pylib/metrics/costMetrics.py | 2 +- archive/ukmon_pylib/metrics/getMatchStats.py | 79 +- archive/ukmon_pylib/metrics/timingMetrics.py | 4 +- archive/ukmon_pylib/reports/CameraDetails.py | 6 +- .../ukmon_pylib/reports/cameraStatusReport.py | 4 +- .../reports/createAnnualBarChart.py | 2 +- .../reports/createExchangeFiles.py | 2 +- .../reports/createSearchableFormat.py | 2 +- .../ukmon_pylib/reports/createSummaryTable.py | 2 +- archive/ukmon_pylib/reports/dailyReport.py | 2 +- archive/ukmon_pylib/reports/extractors.py | 8 +- archive/ukmon_pylib/reports/findBestMp4s.py | 4 +- archive/ukmon_pylib/reports/findFireballs.py | 4 +- .../ukmon_pylib/reports/getLivestreamData.py | 2 +- .../ukmon_pylib/reports/makeCoverageMap.py | 4 +- .../reports/reportActiveShowers.py | 4 +- .../ukmon_pylib/reports/reportBadCameras.py | 2 +- .../reports/reportOfLatestMatches.py | 131 +- archive/ukmon_pylib/requirements.txt | 3 +- archive/ukmon_pylib/traj/clusdetails-ee.txt | 6 - .../traj/clusdetailstest-ukmda.txt | 6 + .../ukmon_pylib/traj/consolidateDistTraj.py | 157 +- .../traj/createDistribMatchingSh.py | 165 +- .../ukmon_pylib/traj/distributeCandidates.py | 292 +- archive/ukmon_pylib/traj/jsonDbMaintenance.py | 112 - archive/ukmon_pylib/traj/manualBulkReruns.py | 4 +- archive/ukmon_pylib/traj/pickleAnalyser.py | 29 +- archive/ukmon_pylib/traj/taskrunner.json | 4 + archive/ukmon_pylib/traj/taskrunner_test.json | 48 + archive/utils/cleanupDeletedTrajs.sh | 102 + archive/utils/getCostMetrics.sh | 2 +- archive/utils/loadMatchCsvMDB.sh | 1 - archive/utils/loadSingleCsvMDB.sh | 1 - archive/utils/makeConfig.sh | 20 +- archive/utils/statsToMqtt.sh | 4 +- archive/utils/updateDb.sh | 3 +- archive/website/createFireballPage.sh | 2 +- archive/website/createOrbitIndex.sh | 6 +- tests/test_apis.py | 2 +- utils/reimageCalcServer.ps1 | 9 + 99 files changed, 5414 insertions(+), 973 deletions(-) delete mode 100644 archive/analysis/onlyConsolDistrib.sh delete mode 100644 archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/requirements.txt create mode 100644 archive/lambdas/matchDataApi/testSumPeriod.json create mode 100644 archive/lambdas/matchPickle/remoteTest.ps1 create mode 100644 archive/lambdas/matchPickle/testresult.json rename archive/terraform/mjmm/{ec2.tf => ukmonhelper_ec2.tf} (100%) rename archive/terraform/ukmda/{ec2.tf => calcserver.tf} (98%) create mode 100644 archive/terraform/ukmda/ecscluster-test.tf delete mode 100644 archive/ukmon_pylib/traj/clusdetails-ee.txt create mode 100644 archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt delete mode 100644 archive/ukmon_pylib/traj/jsonDbMaintenance.py create mode 100644 archive/ukmon_pylib/traj/taskrunner_test.json create mode 100644 archive/utils/cleanupDeletedTrajs.sh create mode 100644 utils/reimageCalcServer.ps1 diff --git a/.gitignore b/.gitignore index 035e1394a..d266d9591 100644 --- a/.gitignore +++ b/.gitignore @@ -556,11 +556,10 @@ archive/unused/** ukmon_pylib/tests/20220217_032206.832_UK/* unused/* *tfstate.lock.info -archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/* -archive/lambdas/getExtraFilesForEE/pythoncode/WesternMeteorPyLib/* -archive/containers/trajsolver/WesternMeteorPyLib/* +archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/** +archive/containers/trajsolver/WesternMeteorPyLib/** archive/containers/trajsolver/awskeys -archive/containers/trajsolvertest/WesternMeteorPyLib/* +archive/containers/trajsolvertest/WesternMeteorPyLib/** archive/containers/trajsolvertest/awskeys #ukmon_pylib/traj/clusdetails.txt **/*.pickle @@ -609,11 +608,11 @@ usermgmt/server/ukmon*.json **/awskeys.test **/trajsolver_old/* archive/tmp/* -servercopy/files/* +servercopy/* tests/testing/RMS/ tests/testing/WesternMeteorPyLib/ archive/ukmon_pylib/tests/testdata.tar.gz -servercopybkp/* +servercopydev/* fbcollector/config.ini usermgmt/windows/stationmaint.ini usermgmt/windows/README.md diff --git a/archive/analysis/README.md b/archive/analysis/README.md index a260be3d5..a20d70e2b 100644 --- a/archive/analysis/README.md +++ b/archive/analysis/README.md @@ -1,17 +1,20 @@ # analysis -This folder contains the batch scripts that perform various analyses of the data. They're mostly triggere +This folder contains the batch scripts that perform various analyses of the data. They're mostly triggered from the nightly batch but can also be triggered manually as needed. +## findAllMatches +The heart of the matching engine. Reads in all single station for the date ranged provided (default three days) and runs the distributed matching engine process. + +## runDistrib +Called by findAllMatches to execute the distributed processing engine. This file is designed to be run standalone so you can rerun distributed matching. + ## consolidateOutput Collects all single station and match data and consolidates it into two files in parquet format. These are used for all detailed analysis. ## createSearchable Creates a single file for the search engine, by consolidating the required information from the match and single station data. -## findAllMatches -The heart of the matching engine. Reads in all single station for the date ranged provided (default three days) and runs the distributed matching engine process. - ## getBadStations Checks for stations that failed quality tests such as too many detections, not uploaded for a few days etc. @@ -24,9 +27,6 @@ Creates a UFO-analyser compatible version of the RMS single-station detections. ## reportActiveShowers Creates a shower report for any active showers, by calling showerReport for each active shower year-to-date. -## runDistrib -Called by findAllMatches to execute the distributed processing engine. - ## showerReport Creates a report for one or more showers. @@ -34,4 +34,4 @@ Creates a report for one or more showers. Creates a report of data for one or all stations, for a month or year to date, which is then pushed to the website. ## Copyright -All code Copyright (C) 2018-2023 Mark McIntyre# \ No newline at end of file +All code Copyright (C) 2018- Mark McIntyre# \ No newline at end of file diff --git a/archive/analysis/consolidateOutput.sh b/archive/analysis/consolidateOutput.sh index 34743434c..51aa0418f 100644 --- a/archive/analysis/consolidateOutput.sh +++ b/archive/analysis/consolidateOutput.sh @@ -46,12 +46,12 @@ do typ=${bn:0:3} if [ "$typ" != "M20" ] ; then pref="P" - yr=${bn:7:4} + snglyr=${bn:7:4} else pref="M" - yr=${bn:1:4} + snglyr=${bn:1:4} fi - mrgfile=${DATADIR}/consolidated/${pref}_${yr}-unified.csv + mrgfile=${DATADIR}/consolidated/${pref}_${snglyr}-unified.csv if [ ! -f $mrgfile ] ; then cat $csvf >> $mrgfile else @@ -62,6 +62,10 @@ do fi done +logger -s -t consolidateOutput "purging older raw data which is on S3 anyway" +find ${DATADIR}/single/rawcsvs -mtime +180 -exec rm -f {} \; + + logger -s -t consolidateOutput "pushing consolidated information back" aws s3 sync ${DATADIR}/consolidated ${UKMONSHAREDBUCKET}/consolidated/ --exclude 'UKMON*' --quiet @@ -90,6 +94,12 @@ cat ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv >> ${DATADIR}/matched/matches-full-$y cat ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv >> ${DATADIR}/searchidx/matches-full-$yr-new.csv mv ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv ${DATADIR}/orbits/${yr}/fullcsv/processed +logger -s -t consolidateOutput "purging older raw data" +find ${DATADIR}/orbits/${yr}/fullcsv/processed -mtime +180 -exec rm -f {} \; +# and last year, because the data is split across dated folders +lyr=$(date -d 'last year' +%Y) +find ${DATADIR}/orbits/${lyr}/fullcsv/processed -mtime +180 -exec rm -f {} \; + python << EOD3 import pandas as pd df = pd.read_csv('${DATADIR}/matched/matches-full-${yr}.csv', skipinitialspace=True) diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index db2b35cd6..2fba436f8 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -29,7 +29,8 @@ if [ "$NJLOGSTREAM" == "" ]; then aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared fi log2cw $NJLOGGRP $NJLOGSTREAM "start findAllMatches" findAllMatches -rundate=$(cat $DATADIR/rundate.txt) + +[ -f $DATADIR/rundate.txt ] && rundate=$(cat $DATADIR/rundate.txt) || rundate=$(date +%Y%m%d) # read start/end dates from commandline if rerunning for historical date if [ $# -gt 0 ] ; then @@ -47,15 +48,7 @@ if [ $# -gt 0 ] ; then fi # folder for logs -mkdir -p $SRC/logs > /dev/null 2>&1 - -log2cw $NJLOGGRP $NJLOGSTREAM "start getRMSSingleData" findAllMatches -# this creates the parquet table for Athena -$SRC/analysis/getRMSSingleData.sh - -log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 1" findAllMatches -yr=$(date +%Y) -$SRC/analysis/createSearchable.sh $yr singles +mkdir -p $SRC/logs/distrib > /dev/null 2>&1 startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') enddt=$(date --date="-$MATCHEND days" '+%Y%m%d-080000') @@ -64,6 +57,9 @@ log2cw $NJLOGGRP $NJLOGSTREAM "solving for ${startdt} to ${enddt}" findAllMatche log2cw $NJLOGGRP $NJLOGSTREAM "start runDistrib" findAllMatches $SRC/analysis/runDistrib.sh $MATCHSTART $MATCHEND +log2cw $NJLOGGRP $NJLOGSTREAM "clean duplicate/deleted trajs" findAllMatches +$SRC/utils/cleanupDeletedTrajs.sh + log2cw $NJLOGGRP $NJLOGSTREAM "start checkForFailures" findAllMatches success=$(grep "Total run time:" $SRC/logs/matchJob.log) @@ -79,28 +75,15 @@ python -m maintenance.rerunFailedLambdas cd $here log2cw $NJLOGGRP $NJLOGSTREAM "start reportOfLatestMatches" findAllMatches -python -m reports.reportOfLatestMatches $DATADIR/distrib $DATADIR $MATCHEND $rundate processed_trajectories.json - -log2cw $NJLOGGRP $NJLOGSTREAM "start getMatchStats" findAllMatches -dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1) -trajlist=$(cat $dailyrep | awk -F, '{print $2}') - matchlog=${SRC}/logs/matchJob.log -vals=$(python -m metrics.getMatchStats $matchlog ) -evts=$(echo $vals | awk '{print $2}') -trajs=$(echo $vals | awk '{print $6}') -matches=$(wc -l $dailyrep | awk '{print $1}') -rtim=$(echo $vals | awk '{print $7}') -echo $(basename $dailyrep) $evts $trajs $matches $rtim >> $DATADIR/dailyreports/stats.txt +python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $MATCHEND $rundate +python -m metrics.getMatchStats $matchlog # copy stats to S3 so the daily report can run if [ "$RUNTIME_ENV" == "PROD" ] ; then aws s3 sync $DATADIR/dailyreports/ $UKMONSHAREDBUCKET/matches/RMSCorrelate/dailyreports/ --quiet fi -log2cw $NJLOGGRP $NJLOGSTREAM "start updateIndexPages" findAllMatches -$SRC/website/updateIndexPages.sh $dailyrep - log2cw $NJLOGGRP $NJLOGSTREAM "start purgeLogs" findAllMatches find $SRC/logs -name "matches*" -mtime +7 -exec gzip {} \; find $SRC/logs -name "matches*" -mtime +30 -exec rm -f {} \; diff --git a/archive/analysis/getRMSSingleData.sh b/archive/analysis/getRMSSingleData.sh index 4879e4b36..0174a815c 100644 --- a/archive/analysis/getRMSSingleData.sh +++ b/archive/analysis/getRMSSingleData.sh @@ -56,7 +56,10 @@ fi # push to S3 bucket for future use by AWS tools logger -s -t getRMSSingleData "copy to S3 bucket" -aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/single/ --exclude "*" --include "*.csv" --exclude "new/*" --quiet +aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/single/ --exclude "*" --include "*.csv" --exclude "new/*" --exclude "rawcsvs/*" --exclude "used/*" --quiet aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/singlepq/ --exclude "*" --include "*.parquet.snap" --exclude "*new.parquet.snap" --quiet +logger -s -t getRMSSingleData "purge processed data" +find $outdir/processed -mtime +180 -exec rm -f {} \; + logger -s -t getRMSSingleData "finished" \ No newline at end of file diff --git a/archive/analysis/onlyConsolDistrib.sh b/archive/analysis/onlyConsolDistrib.sh deleted file mode 100644 index b37254253..000000000 --- a/archive/analysis/onlyConsolDistrib.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# just consolidate solutions of distributed candidates -# -# Used if consolidating a bunch of candidates that were solved outside the normal process -# -# consumes: the trajdb and solution json files -# creates: an updated trajdb and hopefully additional orbits -# - -""" -To use this function, first create the candidate pickle files, then copy them to the candidates -folder on the calcserver and submit them to the distributed processing engine with the following: - -source $here/../config.ini >/dev/null 2>&1 -conda activate $HOME/miniconda3/envs/${WMPL_ENV} -export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib -export AWS_PROFILE=ukmonshared -cd /home/ec2-user/ukmon-shared/matches/RMSCorrelate/candidates -time python -m traj.distributeCandidates 20230113 /home/ec2-user/ukmon-shared/matches/RMSCorrelate/candidates $UKMONSHAREDBUCKET/matches/distrib - -when this completes, logoff the calcserver and shut it down again -On the ukmonhelper run this script to wait for the distrib processing to finish and then -merge in the data - - -""" - - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -logger -s -t onlyConsolDistrib "starting onlyConsolDistrib" - -# load the configuration -source $here/../config.ini >/dev/null 2>&1 -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -# set the profile to the EE account so we can run the server and monitor progress -export AWS_PROFILE=ukmonshared - - -if [ "$1" == "" ] ; then - rundate=$(date --date="-$MATCHEND days" '+%Y%m%d') -else - rundate=$1 -fi -logger -s -t onlyConsolDistrib "consolidating for $rundate" - -python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}'); " - -privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared) -while [ "$privip" == "" ] ; do - sleep 5 - echo "getting ipaddress" - privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared) -done - -if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then - aws s3 sync $UKMONSHAREDBUCKET/matches/distrib/ $DATADIR/distrib/ --exclude "*" --include "*.json" --quiet - cp -f $DATADIR/distrib/processed_trajectories.json $DATADIR/distrib/prev_processed_trajectories.json - - numtoconsol=$(ls -1 $DATADIR/distrib/${rundate}*.json | wc -l) - if [ $numtoconsol -gt 5 ] ; then - logger -s -t onlyConsolDistrib "restarting calcserver to consolidate results" - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text --profile ukmonshared) - if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID --profile ukmonshared - fi - logger -s -t onlyConsolDistrib "waiting for the server to be ready" - while [ "$stat" -ne 16 ]; do - sleep 30 - if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID --profile ukmonshared - fi - echo "checking - status is ${stat}" - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text --profile ukmonshared) - done - - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - while [ $? -ne 0 ] ; do - # in case the server isn't responding to ssh sessions yet - sleep 10 - echo "server not responding yet, retrying" - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - done - scp -i $SERVERSSHKEY $DATADIR/distrib/${rundate}*.json ec2-user@$privip:data/distrib - - echo "#!/bin/bash" > /tmp/execConsol.sh - echo "export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib" >> /tmp/execConsol.sh - echo "python -m traj.consolidateDistTraj ~/data/distrib/ ~/data/distrib/processed_trajectories.json ${rundate}" >> /tmp/execConsol.sh - chmod +x /tmp/execConsol.sh - scp -i $SERVERSSHKEY /tmp/execConsol.sh ec2-user@$privip:data/distrib - ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/execConsol.sh" - - scp -i $SERVERSSHKEY ec2-user@$privip:data/distrib/processed_trajectories.json $DATADIR/distrib - - ssh -i $SERVERSSHKEY ec2-user@$privip "rm -f data/distrib/*.json" - - logger -s -t runDistrib "stopping calcserver again" - aws ec2 stop-instances --instance-ids $SERVERINSTANCEID --profile ukmonshared - - python -c "from traj.consolidateDistTraj import patchTrajDB ; patchTrajDB('$DATADIR/distrib/processed_trajectories.json','/home/ec2-user/ukmon-shared/matches/RMSCorrelate', '/home/ec2-user/data/distrib');" - else - python -m traj.consolidateDistTraj $DATADIR/distrib $DATADIR/distrib/processed_trajectories.json $rundate - fi - # push the updated traj db to the S3 bucket - aws s3 cp $DATADIR/distrib/processed_trajectories.json $UKMONSHAREDBUCKET/matches/distrib/ --quiet - - logger -s -t onlyConsolDistrib "compressing the procssed data" - gzip < $DATADIR/distrib/processed_trajectories.json > $DATADIR/trajdb/processed_trajectories.json.${rundate}.gz - aws s3 mv $UKMONSHAREDBUCKET/matches/distrib/${rundate}.pickle $DATADIR/distrib --quiet - tar czvf $DATADIR/distrib/${rundate}.tgz $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 cp $DATADIR/distrib/${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib/done/ --quiet - rm -f $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 rm $UKMONSHAREDBUCKET/matches/distrib/ --exclude "*" --include "${rundate}*.json" --exclude "test/*" --recursive -else - echo "trajectory database is size zero... not proceeding with copy" -fi -python -m reports.reportOfLatestMatches $DATADIR/distrib $DATADIR $MATCHEND $rundate processed_trajectories.json -dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1) -$SRC/website/updateIndexPages.sh $dailyrep -logger -s -t onlyConsolDistrib "finished" diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 5eb4b5ab8..9350e5db3 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -27,7 +27,7 @@ if [ "$NJLOGSTREAM" == "" ]; then fi log2cw $NJLOGGRP $NJLOGSTREAM "starting runDistrib" runDistrib -# set the profile to the EE account so we can run the server and monitor progress +# set the profile to the UKMDA account so we can run the server and monitor progress export AWS_PROFILE=ukmonshared if [ $# -gt 0 ] ; then @@ -46,11 +46,9 @@ begdate=$(date --date="-$MATCHSTART days" '+%Y%m%d') rundate=$(date --date="-$MATCHEND days" '+%Y%m%d') log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" runDistrib -stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) -if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID -fi +aws ec2 start-instances --instance-ids $SERVERINSTANCEID +stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) while [ "$stat" -ne 16 ]; do sleep 5 log2cw $NJLOGGRP $NJLOGSTREAM "checking server status" runDistrib @@ -58,12 +56,14 @@ while [ "$stat" -ne 16 ]; do done log2cw $NJLOGGRP $NJLOGSTREAM "running phase 1 for dates ${begdate} to ${rundate}" runDistrib + conda activate $HOME/miniconda3/envs/${WMPL_ENV} log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" runDistrib + execdist=execdistrib.sh execMatchingsh=/tmp/$execdist -python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh +python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh $TESTMODE chmod +x $execMatchingsh log2cw $NJLOGGRP $NJLOGSTREAM "get server details" runDistrib @@ -76,100 +76,99 @@ done log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" runDistrib -scp -i $SERVERSSHKEY $execMatchingsh ec2-user@$privip:data/distrib/$execdist +scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$privip:data/distrib/$execdist while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" runDistrib - scp -i $SERVERSSHKEY $execMatchingsh ec2-user@$privip:data/distrib/$execdist + scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$privip:data/distrib/$execdist done -# push the python and templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* ec2-user@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner.json ec2-user@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py ec2-user@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py ec2-user@$privip:src/ukmon_pylib/traj +# push the python code and ECS templates required +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$privip:src/ukmon_pylib/traj # now run the script log2cw $NJLOGGRP $NJLOGSTREAM "start distributed processing" runDistrib -ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/$execdist" +ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execdist" + +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/candidates/processed/*.tgz $DATADIR/distrib/candidates + +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/ +ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;" log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" runDistrib aws ec2 stop-instances --instance-ids $SERVERINSTANCEID log2cw $NJLOGGRP $NJLOGSTREAM "monitoring and waiting for completion" runDistrib -python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}'); " +python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}', '${TESTMODE}'); " -log2cw $NJLOGGRP $NJLOGSTREAM "merging in the new json files" runDistrib mkdir -p $DATADIR/distrib cd $DATADIR/distrib -# make sure the database isn't corrupt before overwriting it !! -if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then - aws s3 sync $UKMONSHAREDBUCKET/matches/distrib/ $DATADIR/distrib/ --exclude "*" --include "*.json" --quiet - cp -f $DATADIR/distrib/processed_trajectories.json $DATADIR/distrib/prev_processed_trajectories.json - - log2cw $NJLOGGRP $NJLOGSTREAM "restarting server to consolidate results" runDistrib - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) +log2cw $NJLOGGRP $NJLOGSTREAM "restarting server to consolidate results" runDistrib +stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) +if [ $stat -eq 80 ]; then + aws ec2 start-instances --instance-ids $SERVERINSTANCEID +fi +log2cw $NJLOGGRP $NJLOGSTREAM "waiting for the server to be ready" runDistrib +while [ "$stat" -ne 16 ]; do + sleep 30 if [ $stat -eq 80 ]; then aws ec2 start-instances --instance-ids $SERVERINSTANCEID fi - log2cw $NJLOGGRP $NJLOGSTREAM "waiting for the server to be ready" runDistrib - while [ "$stat" -ne 16 ]; do - sleep 30 - if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID - fi - log2cw $NJLOGGRP $NJLOGSTREAM "checking - status is ${stat}" runDistrib - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) - done - - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - while [ $? -ne 0 ] ; do - # in case the server isn't responding to ssh sessions yet - sleep 10 - log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" runDistrib - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - done - scp -i $SERVERSSHKEY $DATADIR/distrib/${rundate}*.json ec2-user@$privip:data/distrib - - execcons=execconsol.sh - execConsolsh=/tmp/$execcons - python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh', $rundate)" - chmod +x $execConsolsh - - log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib - scp -i $SERVERSSHKEY $execConsolsh ec2-user@$privip:data/distrib/$execcons - ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/$execcons" - - log2cw $NJLOGGRP $NJLOGSTREAM "finished consolidation" runDistrib - scp -i $SERVERSSHKEY ec2-user@$privip:data/distrib/processed_trajectories.json $DATADIR/distrib - - ssh -i $SERVERSSHKEY ec2-user@$privip "rm -f data/distrib/*.json /tmp/processed_trajectories.json" - # remote temporary files - ssh -i $SERVERSSHKEY ec2-user@$privip "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" - # prune trajdb folder on calcserver - ssh -i $SERVERSSHKEY ec2-user@$privip "find ~/ukmon-shared/matches/RMSCorrelate/trajdb -maxdepth 1 -name "*.json*" -mtime +30 -exec rm -f {} \;" - - log2cw $NJLOGGRP $NJLOGSTREAM "stopping calcserver again" runDistrib - aws ec2 stop-instances --instance-ids $SERVERINSTANCEID - - python -c "from traj.consolidateDistTraj import patchTrajDB ; patchTrajDB('$DATADIR/distrib/processed_trajectories.json','/home/ec2-user/ukmon-shared/matches/RMSCorrelate', '/home/ec2-user/data/distrib');" - - # archive older data then push the updated traj db to the S3 bucket - python -m traj.jsonDbMaintenance $DATADIR/distrib/ - aws s3 cp $DATADIR/distrib/processed_trajectories.json $UKMONSHAREDBUCKET/matches/distrib/ --quiet - - log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib - gzip < $DATADIR/distrib/processed_trajectories.json > $DATADIR/trajdb/processed_trajectories.json.${rundate}.gz - aws s3 mv $UKMONSHAREDBUCKET/matches/distrib/${rundate}.pickle $DATADIR/distrib --quiet - tar czvf $DATADIR/distrib/${rundate}.tgz $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 cp $DATADIR/distrib/${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib/done/ --quiet - rm -f $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 rm $UKMONSHAREDBUCKET/matches/distrib/ --exclude "*" --include "${rundate}*.json" --exclude "test/*" --recursive -else - log2cw $NJLOGGRP $NJLOGSTREAM "trajectory database is size zero... not proceeding with copy" runDistrib -fi + log2cw $NJLOGGRP $NJLOGSTREAM "checking - status is ${stat}" runDistrib + stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) +done +privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) + +execcons=execconsol.sh +execConsolsh=/tmp/$execcons +python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh', '$TESTMODE')" +chmod +x $execConsolsh + +log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib +scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$privip:data/distrib/$execcons +ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execcons" + +log2cw $NJLOGGRP $NJLOGSTREAM "finished consolidation, copying databases" runDistrib +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib + +# remote temporary files +ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" + +log2cw $NJLOGGRP $NJLOGSTREAM "stopping calcserver again" runDistrib +aws ec2 stop-instances --instance-ids $SERVERINSTANCEID + +# grab a copy of the indvidual container dbs so we can get a list of new solutions +rm -Rf $DATADIR/latest/contdbs/ +mkdir -p $DATADIR/latest/contdbs/ +aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $DATADIR/latest/contdbs/ --exclude "*" --include "*.db" --exclude "dbs/*" --exclude "test/*" --quiet +aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "*${rundate}*.db" --exclude "test/*" --exclude "dbs/*" --recursive --quiet +aws s3 mv $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/${rundate}.pickle $DATADIR/distrib --quiet + +# grab a copy of the indvidual container logs - duplicated in monitorProgress, but never mind +aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $SRC/logs/distrib/ --exclude "*" --include "correl*.log" --exclude "logs/" --quiet +aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "correl*.log" --exclude "logs/" --recursive --quiet + +log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib + +mkdir -p $DATADIR/trajdb +tar czvf $DATADIR/trajdb/databases_${rundate}.tgz $DATADIR/distrib/*.db +mkdir -p $DATADIR/distrib/containers +tar czvf $DATADIR/distrib/containers/contdbs_${rundate}.tgz $DATADIR/latest/contdbs/*.db $DATADIR/distrib/${rundate}.pickle +aws s3 cp $DATADIR/distrib/containers/contdbs_${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/done/ --quiet + +tar czvf $DATADIR/distrib/containers/contlogs_${rundate}.tgz $SRC/logs/distrib/correlator_${rundate}*.log $SRC/logs/distrib/${rundate}_*.log + +find $DATADIR/distrib/containers/ -name "cont*.tgz" -mtime +30 -exec rm -f {} \; +find $DATADIR/distrib/ -maxdepth 1 -name "20*.tgz" -mtime +30 -exec rm -f {} \; +rm -f $DATADIR/distrib/${rundate}.pickle + + # and then clear the profile again unset AWS_PROFILE log2cw $NJLOGGRP $NJLOGSTREAM "finished runDistrib" runDistrib diff --git a/archive/analysis/updatePlotsAndDetStatus.sh b/archive/analysis/updatePlotsAndDetStatus.sh index 948846cb8..beed33373 100644 --- a/archive/analysis/updatePlotsAndDetStatus.sh +++ b/archive/analysis/updatePlotsAndDetStatus.sh @@ -59,7 +59,7 @@ conda activate $HOME/miniconda3/envs/${WMPL_ENV} log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" updatePlotsAndDetStatus execrerun=execreplot.sh execrerunsh=/tmp/$execrerun -python -c "from traj.createDistribMatchingSh import createExecReplotSh;createExecReplotSh($MATCHSTART, $MATCHEND, '$execrerunsh')" +python -c "from traj.createDistribMatchingSh import createExecReplotSh;createExecReplotSh($MATCHSTART, $MATCHEND, '$execrerunsh', '$TESTMODE')" chmod +x $execrerunsh log2cw $NJLOGGRP $NJLOGSTREAM "get server details" updatePlotsAndDetStatus @@ -72,18 +72,18 @@ done log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" updatePlotsAndDetStatus -scp -i $SERVERSSHKEY $execrerunsh ec2-user@$privip:data/distrib/$execrerun +scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$privip:data/distrib/$execrerun while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" updatePlotsAndDetStatus - scp -i $SERVERSSHKEY $execrerunsh ec2-user@$privip:data/distrib/$execrerun + scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$privip:data/distrib/$execrerun done # push the python and templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py ec2-user@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$privip:src/ukmon_pylib/traj # now run the script -ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/$execrerun" +ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execrerun" log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" updatePlotsAndDetStatus aws ec2 stop-instances --instance-ids $SERVERINSTANCEID diff --git a/archive/containers/trajsolver/Dockerfile b/archive/containers/trajsolver/Dockerfile index ff8b6ff5b..47f4ac3fa 100644 --- a/archive/containers/trajsolver/Dockerfile +++ b/archive/containers/trajsolver/Dockerfile @@ -11,6 +11,8 @@ RUN conda install -y matplotlib RUN pip install boto3 RUN apt-get update && apt-get -y upgrade RUN conda install -y -c conda-forge numba +RUN conda install -y -c conda-forge libsqlite sqlite +RUN conda install -y -c conda-forge paramiko COPY WesternMeteorPyLib/ ./WesternMeteorPyLib ENV PYTHONPATH=./WesternMeteorPyLib diff --git a/archive/containers/trajsolver/build.ps1 b/archive/containers/trajsolver/build.ps1 index 4a4dac048..c5bb2a3f5 100644 --- a/archive/containers/trajsolver/build.ps1 +++ b/archive/containers/trajsolver/build.ps1 @@ -15,7 +15,7 @@ $repo = "calcengine/${imagename}" write-output "building $imagename in $accid" $yn=read-host -prompt "update WMPL?" -if ($yn.tolower() -eq "y") { bash -c "./update_wmpl.sh $env" } +if ($yn.tolower() -eq "y") { bash -c "./update_wmpl.sh $env" } if ($env -eq "test") { move-item awskeys.test awskeys} diff --git a/archive/containers/trajsolver/trajsolver.py b/archive/containers/trajsolver/trajsolver.py index 2d42835f7..e9fdd9a9a 100644 --- a/archive/containers/trajsolver/trajsolver.py +++ b/archive/containers/trajsolver/trajsolver.py @@ -6,6 +6,8 @@ import boto3 import datetime import tempfile +import glob +import logging from wmpl.Trajectory.CorrelateRMS import RMSDataHandle from wmpl.Utils.Math import generateDatetimeBins @@ -15,8 +17,10 @@ from wmpl.Trajectory.CorrelateRMS import TrajectoryReduced, DatabaseJSON # noqa: F401 from wmpl.Trajectory.CorrelateRMS import MeteorObsRMS, PlateparDummy, MeteorPointRMS # noqa: F401 +log = logging.getLogger('traj_correlator') # must be the same name as the WMPL logger -def runCorrelator(dir_path, time_beg, time_end): + +def runCorrelator(dir_path, time_beg, time_end, max_stns=9999): # Init trajectory constraints maxtoffset = 10.0 maxstationdist = 600.0 @@ -25,7 +29,7 @@ def runCorrelator(dir_path, time_beg, time_end): saveplots = True velpart = 0.40 uncerttime = False - distribute = 2 + distribute = 3 trajectory_constraints = TrajectoryConstraints() trajectory_constraints.max_toffset = maxtoffset @@ -34,6 +38,7 @@ def runCorrelator(dir_path, time_beg, time_end): trajectory_constraints.run_mc = not disablemc trajectory_constraints.save_plots = saveplots trajectory_constraints.geometric_uncert = not uncerttime + trajectory_constraints.max_stations = max_stns # Clock for measuring script time t1 = datetime.datetime.now(datetime.timezone.utc) @@ -42,8 +47,8 @@ def runCorrelator(dir_path, time_beg, time_end): event_time_range = None # Extract time range - dt_beg = datetime.datetime.strptime(time_beg, "%Y%m%d-%H%M%S") - dt_end = datetime.datetime.strptime(time_end, "%Y%m%d-%H%M%S") + dt_beg = datetime.datetime.strptime(time_beg, "%Y%m%d-%H%M%S").replace(tzinfo=datetime.timezone.utc) + dt_end = datetime.datetime.strptime(time_end, "%Y%m%d-%H%M%S").replace(tzinfo=datetime.timezone.utc) print("Custom time range:") print(" BEG: {:s}".format(str(dt_beg))) @@ -52,17 +57,7 @@ def runCorrelator(dir_path, time_beg, time_end): event_time_range = [dt_beg, dt_end] # Init the data handle - dh = RMSDataHandle(dir_path, event_time_range) - - # If there is nothing to process, stop, unless we're in distributed - # processing mode 2 - if not dh.processing_list and distribute !=2: - print() - print("Nothing to process!") - print("Probably everything is already processed.") - print("Exiting...") - sys.exit() - + dh = RMSDataHandle(dir_path, dt_range=event_time_range, mcmode=distribute) ### GENERATE MONTHLY TIME BINS ### @@ -108,25 +103,26 @@ def runCorrelator(dir_path, time_beg, time_end): print("-----------------------------") print() - # Load data of unprocessed observations - dh.unpaired_observations = dh.loadUnpairedObservations(dh.processing_list, dt_range=(bin_beg, bin_end)) - # Run the trajectory correlator - tc = TrajectoryCorrelator(dh, trajectory_constraints, velpart, data_in_j2000=True, distribute=distribute, enableOSM=True) - tc.run(event_time_range=event_time_range) + tc = TrajectoryCorrelator(dh, trajectory_constraints, velpart, data_in_j2000=True, enableOSM=True) + tc.run(event_time_range=event_time_range, mcmode=3) + dh.closeObservationsDatabase() + dh.closeTrajectoryDatabase() + print("Total run time: {:s}".format(str(datetime.datetime.now(datetime.timezone.utc) - t1))) + log.info("Total run time: {:s}".format(str(datetime.datetime.now(datetime.timezone.utc) - t1))) return # read the source bucket + folder and target buckets + folders from the environment def getSourceAndTargets(): - srcpth = os.getenv('SRCPATH', default='s3://ukmda-shared/matches/distrib') + srcpth = os.getenv('SRCPATH', default='s3://ukmda-shared/test/matches/distrib') srcpth = srcpth[5:] srcbucket = srcpth[:srcpth.find('/')] srcpth = srcpth[srcpth.find('/')+1:] - outpth = os.getenv('OUTPATH', default='s3://ukmda-shared/matches/distrib') + outpth = os.getenv('OUTPATH', default='s3://ukmda-shared/test/matches/RMSCorrelate') outpth = outpth[5:] outbucket = outpth[:outpth.find('/')] outpth = outpth[outpth.find('/')+1:] @@ -136,7 +132,9 @@ def getSourceAndTargets(): webbucket = webpth[:webpth.find('/')] webpth = webpth[webpth.find('/')+1:] - return srcbucket, srcpth, outbucket, outpth, webbucket, webpth + max_stns = int(os.getenv('MAX_STNS', default='9999')) + + return srcbucket, srcpth, outbucket, outpth, webbucket, webpth, max_stns # extra args for setting the MIME type when uploading to S3. @@ -162,6 +160,8 @@ def getExtraArgs(fname): ctyp = 'application/json' elif file_ext=='.zip': ctyp = 'application/zip' + elif file_ext=='.db': + ctyp = 'application/vnd.sqlite3' extraargs = {'ContentType': ctyp} return extraargs @@ -233,15 +233,37 @@ def getS3Client(): # starting point for the process def startup(srcfldr, startdt, enddt, isTest=False): + + localfldr = tempfile.mkdtemp() + + # Init the logger + log.setLevel(logging.DEBUG) + + # Init the log formatter + log_formatter = logging.Formatter( + fmt='%(asctime)s-%(levelname)-5s-%(module)-15s:%(lineno)-5d- %(message)s', + datefmt='%Y/%m/%d %H:%M:%S') + + # Init the file handler + datasetname = os.path.split(srcfldr)[-1] + log_file = os.path.join(localfldr, f'{datasetname}.log') + file_handler = logging.handlers.TimedRotatingFileHandler(log_file, when="midnight", backupCount=7) + file_handler.setFormatter(log_formatter) + log.addHandler(file_handler) + + # Init the console handler (i.e. print to console) + console_handler = logging.StreamHandler() + console_handler.setFormatter(log_formatter) + log.addHandler(console_handler) + print(f'processing {srcfldr}') print(f"Starting at {datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}") - localfldr = tempfile.mkdtemp() canddir = os.path.join(localfldr,'candidates') os.makedirs(canddir, exist_ok = True) s3 = getS3Client() - srcbucket, srcpth, outbucket, outpth, webbucket, webpth = getSourceAndTargets() + srcbucket, srcpth, outbucket, outpth, webbucket, webpth, max_stns = getSourceAndTargets() if isTest is True: outpth = os.path.join(outpth, 'test') @@ -249,7 +271,7 @@ def startup(srcfldr, startdt, enddt, isTest=False): print(f'fetching data from {srcbucket}/{srckey} saving to {outbucket} and {webbucket}') objlist = s3.meta.client.list_objects_v2(Bucket=srcbucket,Prefix=srckey) - print(objlist) + #print(objlist) if objlist['KeyCount'] > 0: keys = objlist['Contents'] for k in keys: @@ -257,24 +279,45 @@ def startup(srcfldr, startdt, enddt, isTest=False): if '.pickle' in fname: _, locfname = os.path.split(fname) targfile = os.path.join(canddir, locfname) - print(f'downloading {locfname}') + print(f'downloading {locfname} to {targfile}') s3.meta.client.download_file(srcbucket, fname, targfile) - runCorrelator(localfldr, startdt, enddt) + runCorrelator(localfldr, startdt, enddt, max_stns = max_stns) + + # flush and close the logs + file_handler.flush() + console_handler.flush() + logging.shutdown() print('uploading data to website') trajfldr = os.path.join(localfldr,'trajectories') - # reacquire tokens just in case the 1hour time limit on chained roles is exceeded + # reacquire tokens just in case the 1 hour time limit on chained roles is exceeded s3 = getS3Client() pushToWebsite(s3, trajfldr, webbucket, webpth, outbucket, outpth) - fname = f'{srcfldr}.json' - jsonfile = os.path.join(localfldr, 'processed_trajectories.json') - targkey = f'{srcpth}/{fname}' - print(f'uploading {jsonfile} to {srcbucket}/{srcpth}') - s3.meta.client.upload_file(jsonfile, srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) + + # databases get pushed back to source location not target + if srcfldr[:5] == 'test/': + srcfldr = srcfldr[5:] + for dbname in ['observations', 'trajectories', 'candidates']: + + fname = f'{dbname}_{srcfldr}.db' + localfile = os.path.join(localfldr, f'{dbname}.db') + if os.path.isfile(localfile): + targkey = f'{srcpth}/{fname}' + print(f'uploading {localfile} to {srcbucket}/{targkey}') + s3.meta.client.upload_file(localfile, srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) + + lognames = glob.glob(os.path.join(localfldr, '*.log')) + if len(lognames)> 0: + fname = f'correlator_{srcfldr}.log' + targkey = f'{srcpth}/{fname}' + print(f'uploading {lognames[0]} to {srcbucket}/{targkey}') + s3.meta.client.upload_file(lognames[0], srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) + else: print('no files found') + print(f"Finished at {datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}") return diff --git a/archive/containers/trajsolver/update_wmpl.sh b/archive/containers/trajsolver/update_wmpl.sh index 5fa92c4b2..1be9f5dc5 100644 --- a/archive/containers/trajsolver/update_wmpl.sh +++ b/archive/containers/trajsolver/update_wmpl.sh @@ -8,9 +8,9 @@ else fi cd /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive/containers/$targ/ -ssh ukmonhelper2 "cd src/wmpldev && git stash && git checkout forcontainer && git pull" +ssh ukmonhelper2 "cd ~/src/wmpldev && git stash && git checkout distrib_processing && git pull" rsync -avz ukmonhelper2:src/wmpldev/* ./WesternMeteorPyLib/ --exclude "build/" --exclude "*.egg*" --exclude "dist/" --exclude ".git/" --exclude "__pycache__/" -ssh ukmonhelper2 "cd src/wmpldev && git checkout - && git stash apply" +ssh ukmonhelper2 "cd ~/src/wmpldev && git checkout - && git stash apply" # remove modules and code that don't work in a container because theres no GUI rm -Rf ./WesternMeteorPyLib/wmpl/MetSim diff --git a/archive/cronjobs/nightlyJob.sh b/archive/cronjobs/nightlyJob.sh index 7f184a5fd..1c9eb1204 100644 --- a/archive/cronjobs/nightlyJob.sh +++ b/archive/cronjobs/nightlyJob.sh @@ -47,18 +47,40 @@ if [ -f $SRC/logs/$matchlog ] ; then suff=$(stat matchJob.log -c %X) mv $SRC/logs/$matchlog $SRC/logs/$matchlog-$suff fi + +log2cw $NJLOGGRP $NJLOGSTREAM "start getRMSSingleData" nightlyJob +# this creates the parquet table for Athena +$SRC/analysis/getRMSSingleData.sh +if [ "$(date +%m%d)" == "0101" ] ; then + # catch any data uploaded on 01/01 that is for 31/12 the previous year + $SRC/analysis/getRMSSingleData.sh $(date -d 'last year' +%Y) +fi + +log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 1" nightlyJob + +$SRC/analysis/createSearchable.sh $yr singles + # Run the match process - run this only once as it scoops up all unprocessed data ${SRC}/analysis/findAllMatches.sh > ${SRC}/logs/${matchlog} 2>&1 -# from here down, we're creating reports +log2cw $NJLOGGRP $NJLOGSTREAM "start updateIndexPages" nightlyJob +$SRC/website/updateIndexPages.sh + +# FIRST CONSOLIDATE THE DATA AND CREATE SEARCH INDEXES # consolidate the output of the match process for further analysos log2cw $NJLOGGRP $NJLOGSTREAM "start consolidateOutput" nightlyJob $SRC/analysis/consolidateOutput.sh ${yr} +if [ "$(date +%m%d)" == "0101" ] ; then + # catch any data uploaded on 01/01 that is for 31/12 the previous year + $SRC/analysis/consolidateOutput.sh $(date -d 'last year' +%Y) +fi # create the search indexes used on the website log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 2" nightlyJob $SRC/analysis/createSearchable.sh $yr matches +# FROM HERE DOWN WE'RE CREATING REPORTS + # add daily report to the website log2cw $NJLOGGRP $NJLOGSTREAM "start publishDailyReport" nightlyJob $SRC/website/publishDailyReport.sh diff --git a/archive/deployment/analysis.yml b/archive/deployment/analysis.yml index 8e217f41d..f65068693 100644 --- a/archive/deployment/analysis.yml +++ b/archive/deployment/analysis.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml @@ -25,7 +25,6 @@ - {src: '{{srcdir}}/analysis/getBadStations.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - {src: '{{srcdir}}/analysis/getLogData.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - {src: '{{srcdir}}/analysis/getRMSSingleData.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/onlyConsolDistrib.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - {src: '{{srcdir}}/analysis/reportActiveShowers.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - {src: '{{srcdir}}/analysis/runDistrib.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - {src: '{{srcdir}}/analysis/showerReport.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } diff --git a/archive/deployment/config.yml b/archive/deployment/config.yml index b9823e58a..88ad1408a 100644 --- a/archive/deployment/config.yml +++ b/archive/deployment/config.yml @@ -2,7 +2,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: dev-vars.yml diff --git a/archive/deployment/database.yml b/archive/deployment/database.yml index 1b75b1f5f..3abf9d787 100644 --- a/archive/deployment/database.yml +++ b/archive/deployment/database.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/dev-vars.yml b/archive/deployment/dev-vars.yml index fb60fbb1d..86dfcf575 100644 --- a/archive/deployment/dev-vars.yml +++ b/archive/deployment/dev-vars.yml @@ -1,6 +1,3 @@ destdir: /home/ec2-user/dev env: DEV -websitebucket: mjmm-ukmonarchive.co.uk -#api_id: 0zbnc358p0 -#api_svc: test s3profile: default diff --git a/archive/deployment/prod-vars.yml b/archive/deployment/prod-vars.yml index d26be9990..20171b912 100644 --- a/archive/deployment/prod-vars.yml +++ b/archive/deployment/prod-vars.yml @@ -1,6 +1,3 @@ destdir: /home/ec2-user/prod env: PROD -websitebucket: ukmda-website -#api_id: 40luvfh1od -#api_svc: Prod s3profile: ukmda_admin \ No newline at end of file diff --git a/archive/deployment/pylib.yml b/archive/deployment/pylib.yml index 46a4afc71..313173390 100644 --- a/archive/deployment/pylib.yml +++ b/archive/deployment/pylib.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml @@ -97,6 +97,7 @@ - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails-mda.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails-mm.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } + - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetailstest-ukmda.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/consolidateDistTraj.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/createDistribMatchingSh.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } @@ -107,4 +108,5 @@ - {src: '{{srcdir}}/ukmon_pylib/traj/plotOSMGroundTrack.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/ShowerAssociation.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/taskrunner.json', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } + - {src: '{{srcdir}}/ukmon_pylib/traj/taskrunner_test.json', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/README.md', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } diff --git a/archive/deployment/server_setup.yml b/archive/deployment/server_setup.yml index f621dc278..f53ea3a7b 100644 --- a/archive/deployment/server_setup.yml +++ b/archive/deployment/server_setup.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml @@ -24,10 +24,10 @@ - {src: '{{srcdir}}/server_setup/install-proj4.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - {src: '{{srcdir}}/server_setup/install-sqlite3.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - {src: '{{srcdir}}/server_setup/install_opencv.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libgeos-3.7.2.so', dest: '{{destdir}}/server_setu/libsp', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libgeos.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libgeos_c.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libproj.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } + #- {src: '{{srcdir}}/server_setup/libs/libgeos-3.7.2.so', dest: '{{destdir}}/server_setu/libsp', mode: '644', backup: no } + #- {src: '{{srcdir}}/server_setup/libs/libgeos.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } + #- {src: '{{srcdir}}/server_setup/libs/libgeos_c.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } + #- {src: '{{srcdir}}/server_setup/libs/libproj.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - {src: '{{srcdir}}/server_setup/migratingBatchServer.md', dest: '{{destdir}}/server_setup', mode: '644', backup: no } - {src: '{{srcdir}}/server_setup/newserver.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - {src: '{{srcdir}}/server_setup/remount-s3.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } diff --git a/archive/deployment/shwrinfo.yml b/archive/deployment/shwrinfo.yml index 17a6110ee..c7bbfd11f 100644 --- a/archive/deployment/shwrinfo.yml +++ b/archive/deployment/shwrinfo.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/utils.yml b/archive/deployment/utils.yml index f122d7cb5..73dd5f02f 100644 --- a/archive/deployment/utils.yml +++ b/archive/deployment/utils.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/website.yml b/archive/deployment/website.yml index 3d429b597..78956ddaa 100644 --- a/archive/deployment/website.yml +++ b/archive/deployment/website.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/website_static.yml b/archive/deployment/website_static.yml index cbe146679..c15044f48 100644 --- a/archive/deployment/website_static.yml +++ b/archive/deployment/website_static.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml @@ -27,7 +27,6 @@ - {src: '{{srcdir}}/static_content/js/', dest: '{{destdir}}/static_content/js', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/latest/', dest: '{{destdir}}/static_content/latest', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/live/', dest: '{{destdir}}/static_content/live', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/reports/', dest: '{{destdir}}/static_content/reports', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/search/', dest: '{{destdir}}/static_content/search', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/templates/', dest: '{{destdir}}/static_content/templates', mode: '754', backup: no, directory_mode: yes } diff --git a/archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile b/archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile index 26f9034fa..03a07a133 100644 --- a/archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile +++ b/archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile @@ -6,10 +6,11 @@ COPY requirements.txt ./ RUN python -m pip install -r requirements.txt -t . RUN python -m pip install numba watchdog basemap-data-hires -t . RUN python -m pip install cartopy -t . -#RUN yum update -y +RUN python -m pip install paramiko -t . + ENV LD_LIBRARY_PATH=./ COPY WesternMeteorPyLib/ ./WesternMeteorPyLib -#COPY *.npy ./WesternMeteorPyLib/wmpl/share/ + COPY *.py ./ ENV PYTHONPATH=./WesternMeteorPyLib ENV PROJ_LIB=./ diff --git a/archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/requirements.txt b/archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/requirements.txt deleted file mode 100644 index 81af2bc95..000000000 --- a/archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/requirements.txt +++ /dev/null @@ -1,22 +0,0 @@ -numpy -cython -scipy -matplotlib==3.1.2 ; platform_machine != 'aarch64' -matplotlib==3.3.2 ; platform_machine == 'aarch64' -jplephem -pyephem -https://github.com/matplotlib/basemap/archive/master.zip ; sys_platform != 'win32' and platform_machine != 'aarch64' -basemap ; sys_platform == 'win32' or platform_machine == 'aarch64' -PyQt5 ; platform_machine != 'aarch64' -pyyaml -pyswarms -ml-dtypes -keras -pytz -pandas -cartopy -basemap-data-hires -gitpython -numba -watchdog==3.0.0; python_version == '3.7' -watchdog; python_version >= '3.8' \ No newline at end of file diff --git a/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py b/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py index 8c71bd7b3..83fb956cc 100644 --- a/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py +++ b/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py @@ -36,10 +36,15 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3): yr = orbname[:4] ym = orbname[:6] ymd= orbname[:8] + istest = False if int(yr) > 2020: webpth = f'reports/{yr}/orbits/{ym}/{ymd}/{orbname}/' else: webpth = f'reports/{yr}/orbits/{ym}/{orbname}/' + + if 'test' in key: + webpth = f'dummy/{yr}/orbits/{ym}/{ymd}/{orbname}/' + istest = True outdir = os.path.join(tmpdir, orbname) try: @@ -162,7 +167,7 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3): pass print('pushing files back') - pushFilesBack(outdir, archbucket, websitebucket, fuloutdir, s3) + pushFilesBack(outdir, archbucket, websitebucket, fuloutdir, s3, istest=istest) print('updating the index page') createOrbitPageIndex(outdir, websitebucket, s3) @@ -241,7 +246,7 @@ def findOtherFiles(evtdt, archbucket, websitebucket, outdir, fldr, s3): return -def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3): +def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3, istest=False): flist = os.listdir(outdir) _, pth =os.path.split(outdir) @@ -253,26 +258,32 @@ def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3): else: webpth = f'reports/{yr}/orbits/{ym}/{pth}/' + if istest: + webpth = f'dummy/{yr}/orbits/{ym}/{ymd}/{pth}/' + + for f in flist: print(f) locfname = os.path.join(outdir, f) + extraargs = getExtraArgs(locfname) # some files need to be pushed to the website, some to the archive bucket if '3dtrack' in f or '2dtrack' in f: key = os.path.join(webpth, f) - extraargs = getExtraArgs(locfname) s3.meta.client.upload_file(locfname, websitebucket, key, ExtraArgs=extraargs) elif '.lst' in f: - key = os.path.join(f'matches/RMSCorrelate/trajectories/{yr}/{ym}/{ymd}/{pth}', f) - extraargs = getExtraArgs(locfname) + if istest: + key = os.path.join(f'matches/distrib/test/trajectories/{yr}/{ym}/{ymd}/{pth}', f) + else: + key = os.path.join(f'matches/RMSCorrelate/trajectories/{yr}/{ym}/{ymd}/{pth}', f) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) elif 'summary' in f: key = os.path.join(fldr, f) - # print(locfname, key) - extraargs = getExtraArgs(locfname) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) elif 'orbit_full.csv' in f: - key = os.path.join(f'matches/{yr}/fullcsv', f) - extraargs = getExtraArgs(locfname) + if istest: + key = os.path.join(f'matches/distrib/test/{yr}/fullcsv', f) + else: + key = os.path.join(f'matches/{yr}/fullcsv', f) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) return @@ -345,38 +356,8 @@ def getExtraArgs(fname): def lambda_handler(event, context): - # NB: IN THE CONSOLE, ADD TRUST RELATIONSHIP - # "AWS": "arn:aws:iam::317976261112:root" - # on the target role (replace by this lambda's ARN). - - # Note: if the lambda is connected to a VPC you must - # create a private subnet for the lambda - # create a NAT gateway on a public subnet - # create a routing table on the VPC to send 0.0.0.0/0 to the NAT gw - # attach the new subnet to the NAT gw - # - - sts_client = boto3.client('sts') - response = sts_client.get_caller_identity()['Account'] - if response == '317976261112': - assumed_role_object=sts_client.assume_role( - RoleArn="arn:aws:iam::183798037734:role/service-role/S3FullAccess", - RoleSessionName="GetExtraFilesV2") - - credentials=assumed_role_object['Credentials'] - - # Use the temporary credentials that AssumeRole returns to connections - s3 = boto3.resource('s3', - aws_access_key_id=credentials['AccessKeyId'], - aws_secret_access_key=credentials['SecretAccessKey'], - aws_session_token=credentials['SessionToken']) - ddb = boto3.resource('dynamodb', region_name='eu-west-2', - aws_access_key_id=credentials['AccessKeyId'], - aws_secret_access_key=credentials['SecretAccessKey'], - aws_session_token=credentials['SessionToken']) - else: - s3 = boto3.resource('s3') - ddb = boto3.resource('dynamodb', region_name='eu-west-2') + s3 = boto3.resource('s3') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') websitebucket = os.getenv('WEBSITEBUCKET') if websitebucket[:3] == 's3:': diff --git a/archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 b/archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 index 0a8098317..fa8c832dd 100644 --- a/archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 +++ b/archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 @@ -15,4 +15,3 @@ else { aws lambda invoke --profile ukmonshared --function-name getExtraOrbitFilesV2 --log-type Tail --cli-binary-format raw-in-base64-out --payload file://tests/dummy.json --region eu-west-2 ./ftpdetect.log Remove-Item tests\dummy.json } - diff --git a/archive/lambdas/getExtraFilesV2/tests/testEvent2.json b/archive/lambdas/getExtraFilesV2/tests/testEvent2.json index fd940082b..9aa49b7f2 100644 --- a/archive/lambdas/getExtraFilesV2/tests/testEvent2.json +++ b/archive/lambdas/getExtraFilesV2/tests/testEvent2.json @@ -7,7 +7,7 @@ "arn": "arn:aws:s3:::ukmda-shared" }, "object": { - "key": "matches/RMSCorrelate/trajectories/2024/202407/20240714/20240714_000041.319_UK/20240714_000041_trajectory.pickle" + "key": "matches/distrib/test/trajectories/2026/202604/20260412/20260412_232324.162_UK/20260412_232324_trajectory.pickle" } } } diff --git a/archive/lambdas/getExtraFilesV2/update_wmpl.sh b/archive/lambdas/getExtraFilesV2/update_wmpl.sh index 57d3fb6fa..002e1f93d 100644 --- a/archive/lambdas/getExtraFilesV2/update_wmpl.sh +++ b/archive/lambdas/getExtraFilesV2/update_wmpl.sh @@ -4,14 +4,14 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" cd $here/pythoncode -ssh ukmonhelper2 "cd src/wmpldev && git stash && git checkout forcontainer && git pull" +ssh ukmonhelper2 "cd ~/src/wmpldev && git stash && git checkout distrib_processing && git pull" rsync -avz ukmonhelper2:src/wmpldev/* ./WesternMeteorPyLib/ --exclude "build/" --exclude "*.egg*" --exclude "dist/" --exclude ".git/" --exclude "__pycache__/" ssh ukmonhelper2 "cd src/wmpldev && git checkout - && git stash apply" -mv -f ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so.0 ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so +rm -f ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so +cp -f ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so.0 ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so # remove modules and code that don't work in a container because theres no GUI rm -Rf ./WesternMeteorPyLib/wmpl/MetSim rm -Rf ./WesternMeteorPyLib/wmpl/CAMO rm -f ./WesternMeteorPyLib/wmpl/Utils/DynamicMassFit.py -rm -f ./WesternMeteorPyLib/wmpl/Utils/PlotMap_OSM.py -cp ./wmpl__init__.py_fixed.py ./WesternMeteorPyLib/wmpl/__init__.py +#cp ./wmpl__init__.py_fixed.py ./WesternMeteorPyLib/wmpl/__init__.py diff --git a/archive/lambdas/matchDataApi/matchDataApi.py b/archive/lambdas/matchDataApi/matchDataApi.py index da2048d15..3000292de 100644 --- a/archive/lambdas/matchDataApi/matchDataApi.py +++ b/archive/lambdas/matchDataApi/matchDataApi.py @@ -62,7 +62,7 @@ def periodToSqlFragment(period): def getStationData(statid, dtstr, period=None): host, user, passwd, db = getSqlLoginDetails() - connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) + connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) try: statfrag = f"and s.stations like '%{statid}%' " if statid is not None else "" perfrag = periodToSqlFragment(period) if period is not None else "" @@ -78,13 +78,14 @@ def getStationData(statid, dtstr, period=None): return res -def getSummaryData(dtstr): +def getSummaryData(dtstr, period=None): host, user, passwd, db = getSqlLoginDetails() + perfrag = periodToSqlFragment(period) if period is not None else "" connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) fieldlist = '_localtime,_mjd,_sol,_ID1,_amag,_ra_o,_dc_o,_ra_t,_dc_t,_elng,_elat,_vo,_vi,_vg,_vs,_a,_q,_e,_p,_peri,_node,_incl,'\ '_stream,_mag,_dur,_lng1,_lat1,_H1,_lng2,_lat2,_H2,_LD21,_az1r,_ev1r,_Nts,_Nos,_leap,_tme,_dt,'\ 'dtstamp,orbname,iau,shwrname as name,mass,pi,Q,true_anom,EA,MA,Tj,T,last_peri,jacchia1,Jacchia2,numstats,stations' - expr = f"SELECT {fieldlist} from matches s where s._localtime like '_{dtstr}%'" + expr = f"SELECT {fieldlist} from matches s where s._localtime like '_{dtstr}%' {perfrag}" result=[] try: with connection.cursor() as cursor: @@ -137,8 +138,9 @@ def lambda_handler(event, context): res = getStationData(statid, dtstr) elif reqtyp == 'summary': dtstr = qs['reqval'] + period = qs['period'] if 'period' in qs else None print(f'summary data requested for {dtstr}') - res = getSummaryData(dtstr) + res = getSummaryData(dtstr, period) elif reqtyp == 'matches': dtstr = qs['reqval'] period = qs['period'] if 'period' in qs else None diff --git a/archive/lambdas/matchDataApi/testSumPeriod.json b/archive/lambdas/matchDataApi/testSumPeriod.json new file mode 100644 index 000000000..81c4b4194 --- /dev/null +++ b/archive/lambdas/matchDataApi/testSumPeriod.json @@ -0,0 +1,8 @@ +{ + "httpMethod": "GET", + "queryStringParameters": { + "reqtyp": "summary", + "reqval": "20251213", + "period": "0-3" + } +} \ No newline at end of file diff --git a/archive/lambdas/matchPickle/pythoncode/Dockerfile b/archive/lambdas/matchPickle/pythoncode/Dockerfile index a7caae450..cb6745d50 100644 --- a/archive/lambdas/matchPickle/pythoncode/Dockerfile +++ b/archive/lambdas/matchPickle/pythoncode/Dockerfile @@ -3,13 +3,13 @@ FROM public.ecr.aws/lambda/python:3.8 COPY requirements.txt ./ RUN python3.8 -m pip install -r requirements.txt -t . -ENV LD_LIBRARY_PATH ./ +ENV LD_LIBRARY_PATH=./ # WMPL required even tho only one function called, because # pickle files reference structures defined in the library COPY WesternMeteorPyLib/ ./WesternMeteorPyLib -COPY *.npy ./WesternMeteorPyLib/wmpl/share/ +#COPY *.npy ./WesternMeteorPyLib/wmpl/share/ COPY *.py ./ -ENV PYTHONPATH ./WesternMeteorPyLib -ENV PROJ_LIB ./ +ENV PYTHONPATH=./WesternMeteorPyLib +ENV PROJ_LIB=./ CMD ["matchPickleApi.lambda_handler"] diff --git a/archive/lambdas/matchPickle/pythoncode/requirements.txt b/archive/lambdas/matchPickle/pythoncode/requirements.txt index 58aa7aa9c..92834776d 100644 --- a/archive/lambdas/matchPickle/pythoncode/requirements.txt +++ b/archive/lambdas/matchPickle/pythoncode/requirements.txt @@ -12,4 +12,5 @@ PyQt5 ; platform_machine != 'aarch64' pyyaml pyswarms keras -pytz \ No newline at end of file +pytz +paramiko \ No newline at end of file diff --git a/archive/lambdas/matchPickle/remoteTest.ps1 b/archive/lambdas/matchPickle/remoteTest.ps1 new file mode 100644 index 000000000..861de301a --- /dev/null +++ b/archive/lambdas/matchPickle/remoteTest.ps1 @@ -0,0 +1,5 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# test + + curl "https://api.ukmeteors.co.uk/pickle/getpickle?reqval=20260421_000017.759_UK" > ./testresult.json + diff --git a/archive/lambdas/matchPickle/testresult.json b/archive/lambdas/matchPickle/testresult.json new file mode 100644 index 000000000..7c1230d27 --- /dev/null +++ b/archive/lambdas/matchPickle/testresult.json @@ -0,0 +1,4157 @@ +{ + "jdt_ref": 2461151.5002055545, + "meastype": 2, + "output_dir": ".", + "max_toffset": 10.0, + "verbose": false, + "v_init_part": 0.4, + "v_init_ht": null, + "fixed_time_offsets": {}, + "fixed_time_offsets_copy": {}, + "estimate_timing_vel": true, + "fixed_times": null, + "monte_carlo": true, + "mc_runs": 10, + "mc_runs_max": 20, + "mc_pick_multiplier": 1, + "mc_noise_std": 1.0, + "enable_OSM_plot": true, + "geometric_uncert": true, + "filter_picks": false, + "calc_orbit": true, + "show_plots": false, + "show_jacchia": false, + "save_results": false, + "gravity_correction": true, + "gravity_factor": 1.0, + "plot_all_spatial_residuals": false, + "plot_file_type": "png", + "traj_id": "20260421000017_W5MFS", + "reject_n_sigma_outliers": 2, + "mc_cores": 2, + "file_name": "20260421_000017", + "meas_count": 19, + "observations": [ + { + "ObservedPoints.UK002Z": { + "meas1": [ + 2.3968813154069024, + 2.398794550755332, + 2.401040877732498, + 2.403248309562171, + 2.405262341887118, + 2.407007598336504, + 2.4083661435454724, + 2.4104380988088447, + 2.4124495492369737, + 2.414010003498822, + 2.4151889792750527, + 2.4171917886793013, + 2.417909396727613, + 2.42033600533781, + 2.4223435953587114, + 2.427942162403076, + 2.4300097550164326, + 2.4318363974955406, + 2.433899657791362, + 2.436516452224417, + 2.437238417883184 + ], + "meas2": [ + 0.44034695311399874, + 0.4392383275873981, + 0.4382869800294502, + 0.43791610200467623, + 0.43703226417084373, + 0.4362760437170148, + 0.4357724143366082, + 0.4343759101176623, + 0.4336724668163452, + 0.4324539154277791, + 0.4328902971191608, + 0.43031636173280585, + 0.43144285337433486, + 0.43049915328211963, + 0.4295252455056575, + 0.42636260506048096, + 0.4263728887871292, + 0.4259131817698758, + 0.4244646349323693, + 0.42476734916852354, + 0.4238170894671063 + ], + "jdt_ref": 2461151.500205556, + "time_data": [ + 0.2905716440411789, + 0.33053564404117924, + 0.41047964404117904, + 0.4504116440411789, + 0.4903636440411791, + 0.5703196440411791, + 0.6102876440411793, + 0.6502516440411792, + 0.690195644041179, + 0.7301756440411791, + 0.7701196440411788, + 0.8101196440411789, + 0.850059644041179, + 0.889999644041179, + 0.929951644041179, + 1.0498356440411787, + 1.0897596440411788, + 1.169703644041179, + 1.209671644041179, + 1.249563644041179, + 1.2895636440411786 + ], + "ignore_station": false, + "ignore_list": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "kmeas": 21, + "JD_data": [ + 2461151.500208891, + 2461151.5002093534, + 2461151.5002102787, + 2461151.500210741, + 2461151.5002112035, + 2461151.5002121287, + 2461151.500212591, + 2461151.500213054, + 2461151.500213516, + 2461151.500213979, + 2461151.500214441, + 2461151.500214904, + 2461151.5002153665, + 2461151.500215829, + 2461151.5002162913, + 2461151.5002176785, + 2461151.500218141, + 2461151.500219066, + 2461151.5002195286, + 2461151.5002199905, + 2461151.5002204534 + ], + "lat": 0.9344075681486413, + "lon": -0.006680073179083097, + "ele": 34.0, + "station_id": "UK002Z", + "azim_data": [ + 4.737799374816502, + 4.735428543040449, + 4.732911263205364, + 4.730867396557772, + 4.728583356038361, + 4.726610236803339, + 4.725138590050284, + 4.722411691527732, + 4.720262091239325, + 4.718078528987862, + 4.717466569845322, + 4.713889250782277, + 4.7141741061193105, + 4.711500673159142, + 4.709139283371754, + 4.702204122090407, + 4.700540864881191, + 4.698711574889019, + 4.695932771463546, + 4.69404324715463, + 4.692730245169177 + ], + "elev_data": [ + 0.5397212284763535, + 0.540021754973832, + 0.5406386453085916, + 0.5416703853261442, + 0.5422000352665981, + 0.5426661854062742, + 0.5430931842237807, + 0.5432695876107108, + 0.5439333938247444, + 0.5439396765993363, + 0.5449702160484455, + 0.5442139525051555, + 0.54549253181179, + 0.5462208350060278, + 0.5466770226216804, + 0.5476093934732065, + 0.5488458699002796, + 0.549583029220384, + 0.5497111435989721, + 0.5514955495795086, + 0.5512040280201354 + ], + "ra_data": [ + 2.396881315406902, + 2.3987945507553325, + 2.401040877732498, + 2.403248309562171, + 2.4052623418871177, + 2.407007598336504, + 2.4083661435454733, + 2.4104380988088443, + 2.4124495492369737, + 2.4140100034988214, + 2.4151889792750527, + 2.417191788679302, + 2.417909396727613, + 2.4203360053378105, + 2.422343595358711, + 2.427942162403076, + 2.430009755016433, + 2.431836397495541, + 2.433899657791363, + 2.4365164522244163, + 2.4372384178831847 + ], + "dec_data": [ + 0.44034695311399874, + 0.43923832758739784, + 0.4382869800294501, + 0.43791610200467607, + 0.43703226417084373, + 0.4362760437170146, + 0.43577241433660796, + 0.4343759101176626, + 0.4336724668163451, + 0.43245391542777895, + 0.432890297119161, + 0.43031636173280596, + 0.43144285337433513, + 0.4304991532821195, + 0.4295252455056574, + 0.42636260506048085, + 0.42637288878712926, + 0.42591318176987575, + 0.4244646349323693, + 0.4247673491685236, + 0.4238170894671061 + ], + "magnitudes": [ + 3.6, + 5.02, + 3.44, + 3.24, + 4.28, + 5.1, + 5.72, + 3.45, + 3.98, + 5.66, + 4.45, + 5.81, + 3.55, + 4.2, + 5.66, + 4.01, + 4.53, + 5.21, + 3.72, + 5.72, + 6.0 + ], + "fov_beg": true, + "fov_end": true, + "obs_id": "UK002Z_20260421-000018.239614_8851", + "comment": "{\"ff_name\": \"FF_UK002Z_20260421_000016_194_0375808.fits\"}", + "incident_angle": 0.5002051863784102, + "weight": 0.23002152819250413, + "h_residuals": [ + -3.8052913962245976, + 20.398349033358095, + 20.60146887676896, + -17.142668686892375, + -12.648791991966329, + -13.029467986359393, + -17.831075981704, + 20.861748677458234, + 11.816255498445493, + 51.257765767545706, + -14.989881371214677, + 106.20898562360125, + 5.1376366928819195, + 0.8473289069201014, + 9.980312274554471, + 63.8895794707902, + 3.432361870992362, + -21.53507262422365, + 17.085715282877228, + -78.20881756637142, + -36.77494843909438 + ], + "h_res_rms": 29.722498774935328, + "v_residuals": [ + 6.309529854796034, + -33.877011893083605, + -34.22656685546831, + 28.4211657875748, + 20.979912654651937, + 21.618339676041263, + 29.581808972747485, + -34.69571283606128, + -19.644738726302496, + -85.4334144263602, + 24.889885886707884, + -177.67230920382528, + -8.544025956103757, + -1.4090186969998264, + -16.60820767834652, + -106.71982725101019, + -5.713621282471512, + 35.80745108911481, + -28.480033854767836, + 129.65104400143866, + 61.12507390508546 + ], + "v_res_rms": 49.431213522476874, + "velocities": [ + 0.0, + 16249.106425374774, + 8967.863527321228, + 15682.132184664548, + 15881.2387686525, + 6875.364245819561, + 10338.63052400254, + 17524.6942293953, + 14943.888606668643, + 13556.02313277404, + 6169.26635208947, + 20213.790509050847, + 1179.9960627586386, + 17843.383954063283, + 15237.361660009743, + 14427.322364972191, + 12382.078647459066, + 6167.611210861438, + 16369.803445691245, + 14542.207740346299, + 6974.687234953151 + ], + "velocities_prev_point": [ + 11765.84594020067, + 11765.84594020067, + 11765.84594020067, + 11765.84594020067, + 12515.334991192343, + 11548.839904206608, + 11165.164595519736, + 11379.517422838991, + 11657.45887779439, + 11886.958654011225, + 11826.65374047157, + 12038.458636794907, + 11892.690809748638, + 11932.525875899712, + 12027.59652074097, + 12255.042346574837, + 12396.814201326875, + 12294.978224650129, + 12284.761091501927, + 12304.68522942524, + 12262.39262157771 + ], + "length": [ + 0.0, + 649.3792891836829, + 1366.3061710118493, + 1992.5250734098718, + 2627.0123246950798, + 3176.7389483338284, + 3589.953333117164, + 4290.310213300716, + 4887.228899805485, + 5429.198704653792, + 5675.623879821653, + 6484.175500183687, + 6531.304542930267, + 7243.9692980555565, + 7852.7323710962655, + 9582.337485498587, + 10076.679593419743, + 10569.743104060852, + 11224.01140817824, + 11804.129159356135, + 12083.116648754258 + ], + "state_vect_dist": [ + 3587.0536516011, + 4236.432940784783, + 4953.35982261295, + 5579.5787250109715, + 6214.0659762961795, + 6763.792599934928, + 7177.006984718264, + 7877.363864901816, + 8474.282551406584, + 9016.252356254892, + 9262.677531422753, + 10071.229151784788, + 10118.358194531367, + 10831.022949656655, + 11439.786022697366, + 13169.391137099687, + 13663.733245020843, + 14156.796755661951, + 14811.06505977934, + 15391.182810957234, + 15670.170300355358 + ], + "lag": [ + 63.37447151468132, + 194.5118468950268, + -125.25258238414244, + -16.860626788108675, + 99.54032331926737, + -387.5799767748267, + -492.65937666999525, + -310.54441028977544, + -231.60828321249573, + -208.08787566819592, + -479.64525992782546, + -189.8023912456356, + -660.6040370513838, + -465.86997047842124, + -375.1931986155396, + -200.2100838728802, + -223.59118100337764, + -767.2189814696103, + -631.2444620307251, + -568.4349489031392, + -808.1562111848561 + ], + "lag_line": [ + 11370.947134066559, + -3165.029756068875 + ], + "v_init": 11370.947134066559, + "v_init_stddev": null, + "jacchia_fit": [ + 63.688001193520016, + 1.8900846614547042 + ], + "model_ra": [ + 2.396881593167252, + 2.3989304099368014, + 2.4011844016151613, + 2.4031860875196687, + 2.4052274845571198, + 2.406976087071311, + 2.4083103572784346, + 2.4106035391564897, + 2.412566930909117, + 2.4143572664202093, + 2.41515920660715, + 2.417869111447515, + 2.418000831463778, + 2.420405613794107, + 2.4224695120777024, + 2.428399440764061, + 2.430109452127189, + 2.4317929026796112, + 2.4340906148854335, + 2.4361376027592625, + 2.4371099642448937 + ], + "model_dec": [ + 0.440302678909872, + 0.4394764728451123, + 0.43852840306686475, + 0.4377147344407611, + 0.43688315067561767, + 0.4361219869381554, + 0.4355611737515261, + 0.4346241691042197, + 0.4338134534871204, + 0.4330683131149763, + 0.43271070454145943, + 0.431599113939899, + 0.43150463507581566, + 0.4305093179638242, + 0.4296461972321007, + 0.4271459610813783, + 0.4264149242154197, + 0.42564839310551755, + 0.42467559658948373, + 0.42380137844967836, + 0.4233614958813678 + ], + "model_azim": [ + 4.73778196828684, + 4.735521732614828, + 4.7330053965160985, + 4.730788912250429, + 4.728525435294195, + 4.726550570784113, + 4.725056923942933, + 4.722507072008819, + 4.720316125854996, + 4.7183126731808445, + 4.717397902721562, + 4.714373400873156, + 4.714197584063138, + 4.711504522278045, + 4.709184901171225, + 4.702495517776749, + 4.700556524882403, + 4.69861306303917, + 4.696010730582082, + 4.6936850389188365, + 4.692562119253269 + ], + "model_elev": [ + 0.5396755224600758, + 0.5402679008661397, + 0.5408884116776695, + 0.5414620307648099, + 0.5420456131089348, + 0.5425065260896258, + 0.5428741613485486, + 0.5435273909554157, + 0.5440798903382158, + 0.5445789635092166, + 0.5447834940136214, + 0.5455517621083557, + 0.5455568612650257, + 0.5462314278769935, + 0.5468031898818537, + 0.548429156889144, + 0.5488898437745016, + 0.5493059467086585, + 0.549932237410441, + 0.5504839878814398, + 0.5507264098873919 + ], + "model_fit1": [ + 4.73778196828684, + 4.735521732614828, + 4.7330053965160985, + 4.730788912250429, + 4.728525435294195, + 4.726550570784113, + 4.725056923942933, + 4.722507072008819, + 4.720316125854996, + 4.7183126731808445, + 4.717397902721562, + 4.714373400873156, + 4.714197584063138, + 4.711504522278045, + 4.709184901171225, + 4.702495517776749, + 4.700556524882403, + 4.69861306303917, + 4.696010730582082, + 4.6936850389188365, + 4.692562119253269 + ], + "model_fit2": [ + 0.5396755224600758, + 0.5402679008661397, + 0.5408884116776695, + 0.5414620307648099, + 0.5420456131089348, + 0.5425065260896258, + 0.5428741613485486, + 0.5435273909554157, + 0.5440798903382158, + 0.5445789635092166, + 0.5447834940136214, + 0.5455517621083557, + 0.5455568612650257, + 0.5462314278769935, + 0.5468031898818537, + 0.548429156889144, + 0.5488898437745016, + 0.5493059467086585, + 0.549932237410441, + 0.5504839878814398, + 0.5507264098873919 + ], + "meas_eci": [ + [ + -0.6651403376230054, + 0.6131039197684128, + 0.42625334583010854 + ], + [ + -0.6666597964124698, + 0.6121494659158085, + 0.42525021719721046 + ], + [ + -0.6683315036211338, + 0.610923065694228, + 0.4243889832103093 + ], + [ + -0.6697948071375551, + 0.6095521783058309, + 0.424053131404759 + ], + [ + -0.6712985427876046, + 0.6084534187936955, + 0.42325252935984004 + ], + [ + -0.6725967633254724, + 0.6074952687408567, + 0.42256726378361675 + ], + [ + -0.6735794940210694, + 0.6067233106956108, + 0.4221107550073526 + ], + [ + -0.6752733035008325, + 0.6057194052358751, + 0.42084435091821104 + ], + [ + -0.6767109144101443, + 0.6045569835569059, + 0.42020613031078136 + ], + [ + -0.6780353559270645, + 0.6038403657657955, + 0.41910007013205 + ], + [ + -0.6786100121852274, + 0.6029190303333297, + 0.41949623862900476 + ], + [ + -0.6806225551669195, + 0.602272242271424, + 0.41715834354283843 + ], + [ + -0.6807019943374223, + 0.6014721244956676, + 0.4181818723470973 + ], + [ + -0.6824555864862213, + 0.6000788818635572, + 0.4173244637151418 + ], + [ + -0.6839643602217045, + 0.5989750663284349, + 0.4164392198909243 + ], + [ + -0.6882992447918803, + 0.5959956375357374, + 0.41356178457069387 + ], + [ + -0.6895268279492358, + 0.5945684648394516, + 0.41357114763571556 + ], + [ + -0.6907558791802513, + 0.5934317844296477, + 0.4131525536730498 + ], + [ + -0.6924328423375028, + 0.592393747301069, + 0.41183298436502674 + ], + [ + -0.6938856684688789, + 0.5904989435368124, + 0.4121088166678862 + ], + [ + -0.6946099128507072, + 0.5902511457747254, + 0.41124281620613995 + ] + ], + "meas_eci_los": [ + [ + -0.6651532201448609, + 0.6130899435642051, + 0.42625334583010854 + ], + [ + -0.6666744422176303, + 0.6121335155564811, + 0.42525021719721046 + ], + [ + -0.668349681390559, + 0.6109031791657809, + 0.4243889832103093 + ], + [ + -0.669814719865365, + 0.6095302968659562, + 0.424053131404759 + ], + [ + -0.6713201921678998, + 0.6084295324671147, + 0.42325252935984004 + ], + [ + -0.6726219198904363, + 0.6074674151436691, + 0.42256726378361675 + ], + [ + -0.6736063861027676, + 0.6066934539853648, + 0.4221107550073526 + ], + [ + -0.6753019174198893, + 0.6056875041052527, + 0.42084435091821104 + ], + [ + -0.6767412328073986, + 0.6045230449433344, + 0.42020613031078136 + ], + [ + -0.6780673992284806, + 0.6038043833219787, + 0.41910007013205 + ], + [ + -0.6786437629715607, + 0.602881040302245, + 0.41949623862900476 + ], + [ + -0.6806580260024921, + 0.6022321546139711, + 0.41715834354283843 + ], + [ + -0.6807391702046832, + 0.6014300489577449, + 0.4181818723470973 + ], + [ + -0.6824944243246932, + 0.6000347096215143, + 0.4173244637151418 + ], + [ + -0.6840048714786812, + 0.598928803723841, + 0.4164392198909243 + ], + [ + -0.6883447630503693, + 0.5959430656730842, + 0.41356178457069387 + ], + [ + -0.6895739692035289, + 0.5945137902859511, + 0.41357114763571556 + ], + [ + -0.6908063894172288, + 0.5933729853420778, + 0.4131525536730498 + ], + [ + -0.6924849898642252, + 0.5923327880522399, + 0.41183298436502674 + ], + [ + -0.693939367574386, + 0.5904358367808104, + 0.4121088166678862 + ], + [ + -0.6946653105480682, + 0.590185947342021, + 0.41124281620613995 + ] + ], + "model_eci": [ + [ + -3431648.7433502977, + -1734673.2501258308, + 5171746.249168466 + ], + [ + -3431502.168458378, + -1735198.4026242748, + 5171393.320457905 + ], + [ + -3431340.2471023854, + -1735778.206261865, + 5171003.513213945 + ], + [ + -3431198.8672727705, + -1736284.6365797631, + 5170663.117429066 + ], + [ + -3431055.612694895, + -1736797.753523634, + 5170318.214222535 + ], + [ + -3430931.3319215793, + -1737242.3632842037, + 5170019.113061614 + ], + [ + -3430837.965680444, + -1737576.5501712172, + 5169794.374518736 + ], + [ + -3430679.813194559, + -1738142.9361567015, + 5169413.623255734 + ], + [ + -3430544.9854288357, + -1738625.679263341, + 5169089.050559572 + ], + [ + -3430422.545034539, + -1739063.9846602748, + 5168794.316972965 + ], + [ + -3430366.765328334, + -1739263.305532724, + 5168660.1248029135 + ], + [ + -3430184.1694901455, + -1739917.1765316478, + 5168220.538002146 + ], + [ + -3430173.328761385, + -1739955.342499328, + 5168194.58388284 + ], + [ + -3430012.3423658106, + -1740531.6864444455, + 5167807.052084736 + ], + [ + -3429874.786286485, + -1741024.0084887384, + 5167475.953263732 + ], + [ + -3429483.8746772027, + -1742422.7843117185, + 5166535.0961491605 + ], + [ + -3429372.090910218, + -1742822.589670693, + 5166266.09232802 + ], + [ + -3429260.304079202, + -1743221.4092456542, + 5165997.30074232 + ], + [ + -3429112.4197078072, + -1743750.5190035156, + 5165641.380819524 + ], + [ + -3428981.246454044, + -1744219.694618384, + 5165325.712035401 + ], + [ + -3428918.0015465165, + -1744445.3384067302, + 5165173.638196536 + ] + ], + "meas_lat": [ + 0.9346450225936334, + 0.9345970660398534, + 0.9345465778299221, + 0.9345060097444198, + 0.9344609020518903, + 0.9344222008936756, + 0.9343935294446254, + 0.9343405561033522, + 0.9342993156178634, + 0.9342574203614202, + 0.93424620996589, + 0.9341778905737628, + 0.9341839571465814, + 0.9341340643559439, + 0.9340902381901774, + 0.9339630599390059, + 0.9339337530240859, + 0.9339012154522863, + 0.933851362550423, + 0.9338192346503082, + 0.9337956204733615 + ], + "meas_lon": [ + -0.04088550365419589, + -0.04075548711489885, + -0.04060781279524608, + -0.04046702814824051, + -0.04033520755003031, + -0.04022319461679362, + -0.040136396159376767, + -0.03999918021151192, + -0.03987197780910561, + -0.03976888838620386, + -0.03970217229821398, + -0.03956286802268136, + -0.03952989329711795, + -0.03937914973331329, + -0.039254062049537145, + -0.03890628729167373, + -0.0387879885582501, + -0.03868188951868613, + -0.03855458929929464, + -0.038409166927020906, + -0.03836284386104872 + ], + "meas_ht": [ + 80166.50920069027, + 79907.89847155106, + 79663.6709640296, + 79509.71227376115, + 79287.06050564267, + 79100.56625424455, + 78967.5446651115, + 78669.14774349949, + 78480.50679384157, + 78234.48391237574, + 78254.36663415054, + 77789.32418704791, + 77932.08594644774, + 77696.87267580134, + 77475.89917794727, + 76804.02455018043, + 76731.24392918951, + 76602.42998524354, + 76320.02753907315, + 76272.04281635238, + 76112.63523459999 + ], + "meas_range": [ + 153339.33047024682, + 152779.23373854544, + 152169.97299907313, + 151631.42779261048, + 151086.25079201206, + 150624.91058615095, + 150273.74436083622, + 149672.74252612054, + 149162.45846822963, + 148700.46758205735, + 148495.59961464474, + 147802.94181198507, + 147771.35775808938, + 147162.96362779627, + 146645.26441739433, + 145179.17570669635, + 144762.42956183938, + 144356.17304021947, + 143802.8283496073, + 143313.7343373524, + 143083.5198313615 + ], + "model_lat": [ + 0.9346446667677631, + 0.9345989704981107, + 0.9345485008239762, + 0.9345044075468767, + 0.9344597200105587, + 0.9344209834416608, + 0.9343918634192264, + 0.9343425020668983, + 0.9343004177428308, + 0.9342621973971849, + 0.9342448099303645, + 0.9341877726487242, + 0.9341844358943037, + 0.9341341428363436, + 0.9340911683984181, + 0.9339690073179128, + 0.9339340724672882, + 0.9338992059698612, + 0.9338529539754011, + 0.9338119290348905, + 0.9337921890458436 + ], + "model_lon": [ + -0.04088647065639903, + -0.0407503038625055, + -0.04060257766467196, + -0.040471381075913575, + -0.0403384201294265, + -0.04022650428958335, + -0.04014092463279356, + -0.03999387866505028, + -0.039868976417382226, + -0.03975584834373117, + -0.03970598017255938, + -0.03953580064279074, + -0.03952858935807945, + -0.0393789360113246, + -0.03925152725986469, + -0.03889002516887339, + -0.038787118004688446, + -0.038687359650227564, + -0.03855024718026426, + -0.03842899481740237, + -0.038372180120937446 + ], + "model_ht": [ + 80160.57129329837, + 79939.75564689818, + 79695.8571107965, + 79482.97408082303, + 79267.32029777888, + 79080.2240190833, + 78939.71062025194, + 78701.77767818491, + 78498.9785627082, + 78314.84618101093, + 78230.94224464067, + 77956.46774587498, + 77940.11468171058, + 77698.18875523485, + 77491.51556920519, + 76904.43232644908, + 76736.61042515974, + 76568.72510363306, + 76346.81825810629, + 76150.02295057768, + 76055.10224458351 + ], + "model_range": [ + 153339.33064750832, + 152779.23885477215, + 152169.9782410282, + 151631.43142680972, + 151086.25277939526, + 150624.9127025781, + 150273.74833255954, + 149672.74799874792, + 149162.46022825968, + 148700.50095141708, + 148495.6024593428, + 147803.08674493877, + 147771.3580936107, + 147162.9636368488, + 146645.26569589667, + 145179.22897841557, + 144762.42971472227, + 144356.17909111048, + 143802.8321819995, + 143313.81433522768, + 143083.53761941093 + ], + "rbeg_lat": 0.9346446667677631, + "rbeg_lon": -0.04088647065639903, + "rbeg_ele": 80160.57129329837, + "rbeg_jd": 2461151.500208891, + "rend_lat": 0.9337921890458436, + "rend_lon": -0.038372180120937446, + "rend_ele": 76055.10224458351, + "rend_jd": 2461151.5002204534, + "htmin_lat": 0.9337921890458436, + "htmin_lon": -0.038372180120937446, + "htmin_ele": 76055.10224458351, + "htmin_jd": 2461151.5002204534, + "absolute_magnitudes": [ + 2.671732184227931, + 4.099678289776929, + 2.528355107417953, + 2.3360538259627734, + 3.3838752499018385, + 4.210515958921451, + 4.835584403455401, + 2.5742863380034136, + 3.111702310827387, + 4.798437841987752, + 3.591432036646971, + 4.961582479831281, + 2.7020486763676725, + 3.3610073736334165, + 4.828659766224398, + 3.200477571944522, + 3.726720681188872, + 4.412823108180669, + 2.931162802418613, + 4.938559724900953, + 5.222051654133499 + ], + "ra_data_los": [ + 2.396902327617516, + 2.3988184762780778, + 2.401070632813468, + 2.403280977947147, + 2.4052979235842242, + 2.4070490095918355, + 2.408410468104811, + 2.4104853396142762, + 2.4124997004205735, + 2.414063070928515, + 2.415244960008752, + 2.417250685659096, + 2.4179712070195376, + 2.420400728941866, + 2.42241123226677, + 2.42801853924753, + 2.43008904516489, + 2.431921517202225, + 2.433987690810178, + 2.4366073956132723, + 2.4373322775181343 + ], + "dec_data_los": [ + 0.44034695311399874, + 0.43923832758739784, + 0.4382869800294501, + 0.43791610200467607, + 0.43703226417084373, + 0.4362760437170146, + 0.43577241433660796, + 0.4343759101176626, + 0.4336724668163451, + 0.43245391542777895, + 0.432890297119161, + 0.43031636173280596, + 0.43144285337433513, + 0.4304991532821195, + 0.4295252455056574, + 0.42636260506048085, + 0.42637288878712926, + 0.42591318176987575, + 0.4244646349323693, + 0.4247673491685236, + 0.4238170894671061 + ], + "x_eci": [ + -0.6651403376230054, + -0.6666597964124698, + -0.6683315036211338, + -0.6697948071375551, + -0.6712985427876046, + -0.6725967633254724, + -0.6735794940210694, + -0.6752733035008325, + -0.6767109144101443, + -0.6780353559270645, + -0.6786100121852274, + -0.6806225551669195, + -0.6807019943374223, + -0.6824555864862213, + -0.6839643602217045, + -0.6882992447918803, + -0.6895268279492358, + -0.6907558791802513, + -0.6924328423375028, + -0.6938856684688789, + -0.6946099128507072 + ], + "y_eci": [ + 0.6131039197684128, + 0.6121494659158085, + 0.610923065694228, + 0.6095521783058309, + 0.6084534187936955, + 0.6074952687408567, + 0.6067233106956108, + 0.6057194052358751, + 0.6045569835569059, + 0.6038403657657955, + 0.6029190303333297, + 0.602272242271424, + 0.6014721244956676, + 0.6000788818635572, + 0.5989750663284349, + 0.5959956375357374, + 0.5945684648394516, + 0.5934317844296477, + 0.592393747301069, + 0.5904989435368124, + 0.5902511457747254 + ], + "z_eci": [ + 0.42625334583010854, + 0.42525021719721046, + 0.4243889832103093, + 0.424053131404759, + 0.42325252935984004, + 0.42256726378361675, + 0.4221107550073526, + 0.42084435091821104, + 0.42020613031078136, + 0.41910007013205, + 0.41949623862900476, + 0.41715834354283843, + 0.4181818723470973, + 0.4173244637151418, + 0.4164392198909243, + 0.41356178457069387, + 0.41357114763571556, + 0.4131525536730498, + 0.41183298436502674, + 0.4121088166678862, + 0.41124281620613995 + ], + "x_eci_los": [ + -0.6651532201448609, + -0.6666744422176303, + -0.668349681390559, + -0.669814719865365, + -0.6713201921678998, + -0.6726219198904363, + -0.6736063861027676, + -0.6753019174198893, + -0.6767412328073986, + -0.6780673992284806, + -0.6786437629715607, + -0.6806580260024921, + -0.6807391702046832, + -0.6824944243246932, + -0.6840048714786812, + -0.6883447630503693, + -0.6895739692035289, + -0.6908063894172288, + -0.6924849898642252, + -0.693939367574386, + -0.6946653105480682 + ], + "y_eci_los": [ + 0.6130899435642051, + 0.6121335155564811, + 0.6109031791657809, + 0.6095302968659562, + 0.6084295324671147, + 0.6074674151436691, + 0.6066934539853648, + 0.6056875041052527, + 0.6045230449433344, + 0.6038043833219787, + 0.602881040302245, + 0.6022321546139711, + 0.6014300489577449, + 0.6000347096215143, + 0.598928803723841, + 0.5959430656730842, + 0.5945137902859511, + 0.5933729853420778, + 0.5923327880522399, + 0.5904358367808104, + 0.590185947342021 + ], + "z_eci_los": [ + 0.42625334583010854, + 0.42525021719721046, + 0.4243889832103093, + 0.424053131404759, + 0.42325252935984004, + 0.42256726378361675, + 0.4221107550073526, + 0.42084435091821104, + 0.42020613031078136, + 0.41910007013205, + 0.41949623862900476, + 0.41715834354283843, + 0.4181818723470973, + 0.4173244637151418, + 0.4164392198909243, + 0.41356178457069387, + 0.41357114763571556, + 0.4131525536730498, + 0.41183298436502674, + 0.4121088166678862, + 0.41124281620613995 + ], + "x_stat": -3329692.8394425996, + "y_stat": -1828618.1638918526, + "z_stat": 5106390.987831098, + "stat_eci": [ + -3329692.8394425996, + -1828618.1638918526, + 5106390.987831098 + ], + "stat_eci_los": [ + [ + -3329654.415397559, + -1828688.127695391, + 5106390.987831098 + ], + [ + -3329649.0878441236, + -1828697.8280102296, + 5106390.987831098 + ], + [ + -3329638.4272870813, + -1828717.2383623847, + 5106390.987831098 + ], + [ + -3329633.099663696, + -1828726.9386035914, + 5106390.987831098 + ], + [ + -3329627.771997192, + -1828736.6388563307, + 5106390.987831098 + ], + [ + -3329617.1112139, + -1828756.049084223, + 5106390.987831098 + ], + [ + -3329611.783477447, + -1828765.7492633294, + 5106390.987831098 + ], + [ + -3329606.450332315, + -1828775.4592229126, + 5106390.987831098 + ], + [ + -3329601.1278900444, + -1828785.1496290863, + 5106390.987831098 + ], + [ + -3329595.7946883356, + -1828794.859557595, + 5106390.987831098 + ], + [ + -3329590.466838816, + -1828804.5596746, + 5106390.987831098 + ], + [ + -3329585.1335805035, + -1828814.2695720182, + 5106390.987831098 + ], + [ + -3329579.805659577, + -1828823.9696850183, + 5106390.987831098 + ], + [ + -3329574.4777103914, + -1828833.6697824954, + 5106390.987831098 + ], + [ + -3329569.1497478094, + -1828843.369837391, + 5106390.987831098 + ], + [ + -3329553.1656459146, + -1828872.4699901317, + 5106390.987831098 + ], + [ + -3329547.837570296, + -1828882.1699829372, + 5106390.987831098 + ], + [ + -3329537.17593862, + -1828901.5797448508, + 5106390.987831098 + ], + [ + -3329531.847763332, + -1828911.2797181348, + 5106390.987831098 + ], + [ + -3329526.5249406365, + -1828920.9698801264, + 5106390.987831098 + ], + [ + -3329521.191342843, + -1828930.679591074, + 5106390.987831098 + ] + ], + "plane_N": [ + -0.010758941735242446, + 0.5626841701025131, + -0.8266019416192922 + ], + "excluded_time": null, + "excluded_indx_range": [], + "ang_res": [ + 4.808344044228582e-05, + 0.00025879609182125133, + 0.0002624805684568251, + 0.0002189398121328159, + 0.00016219719514936625, + 0.00016763637746942077, + 0.0002299126917867094, + 0.0002704217170759464, + 0.0001536191301664089, + 0.000669935090213715, + 0.0001957385228524681, + 0.0014004153194447633, + 6.738763283318817e-05, + 1.1091740167342496e-05, + 0.00013204797495846036, + 0.0008566651982668407, + 4.595851975374786e-05, + 0.0002895389345591846, + 0.0002308693235996496, + 0.0010565990502203683, + 0.0004986368036220028 + ], + "ang_res_std": 0.0003960746824252553 + } + }, + { + "ObservedPoints.UK00AS": { + "meas1": [ + 4.2684960666937455, + 4.275705660567851, + 4.282316120282617, + 4.287776816608948, + 4.295439857062819, + 4.301929096125738, + 4.308588575262349, + 4.314596261320367, + 4.3215806860608525, + 4.327444662401953, + 4.331822069286927, + 4.335799177502339 + ], + "meas2": [ + 1.0876892604784882, + 1.082754431923786, + 1.0781467943009044, + 1.0734864670857704, + 1.0676655293254065, + 1.0625353098743915, + 1.0568295338648426, + 1.0530700757470863, + 1.047988416812209, + 1.0419285874153617, + 1.037607171800031, + 1.0343662231599273 + ], + "jdt_ref": 2461151.500205556, + "time_data": [ + 0.8142981686851195, + 0.8543421686851195, + 0.8943901686851182, + 0.934414168685119, + 0.97446616868512, + 1.0145101686851197, + 1.054550168685119, + 1.09461416868512, + 1.1346821686851198, + 1.1747101686851193, + 1.2147341686851183, + 1.2547741686851195 + ], + "ignore_station": false, + "ignore_list": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "kmeas": 12, + "JD_data": [ + 2461151.500214981, + 2461151.5002154447, + 2461151.500215908, + 2461151.5002163714, + 2461151.5002168347, + 2461151.5002172985, + 2461151.500217762, + 2461151.5002182256, + 2461151.5002186894, + 2461151.5002191523, + 2461151.5002196156, + 2461151.500220079 + ], + "lat": 0.9311541522498763, + "lon": -0.045808173347230974, + "ele": 57.0, + "station_id": "UK00AS", + "azim_data": [ + 0.8832534008354802, + 0.8974325902214364, + 0.9104409422636692, + 0.9232847050629855, + 0.9391342537659964, + 0.9528309723365957, + 0.9677775061336478, + 0.9774560061825399, + 0.9902964441331861, + 1.0053988251477577, + 1.01598915152872, + 1.0237734767338456 + ], + "elev_data": [ + 1.191408858155571, + 1.1885973861086219, + 1.1859092541919156, + 1.1836589415935101, + 1.1802988113469086, + 1.1773284999232583, + 1.1741538069154087, + 1.1711813003457707, + 1.1676161578412807, + 1.164451512475476, + 1.162008573079885, + 1.1597749824215797 + ], + "ra_data": [ + 4.2684960666937455, + 4.275705660567851, + 4.282316120282618, + 4.2877768166089485, + 4.295439857062819, + 4.301929096125738, + 4.308588575262349, + 4.314596261320368, + 4.3215806860608525, + 4.327444662401954, + 4.331822069286927, + 4.33579917750234 + ], + "dec_data": [ + 1.0876892604784882, + 1.082754431923786, + 1.0781467943009044, + 1.07348646708577, + 1.0676655293254065, + 1.0625353098743917, + 1.0568295338648421, + 1.0530700757470863, + 1.0479884168122084, + 1.0419285874153617, + 1.037607171800031, + 1.0343662231599273 + ], + "magnitudes": [ + 2.07, + 2.55, + 2.28, + 2.28, + 2.54, + 2.07, + 2.44, + 2.91, + 1.92, + 2.53, + 4.48, + 3.45 + ], + "fov_beg": true, + "fov_end": true, + "obs_id": "UK00AS_20260421-000018.455603_8295", + "comment": "{\"ff_name\": \"FF_UK00AS_20260421_000008_555_0363264.fits\"}", + "incident_angle": 1.265480375315784, + "weight": 0.909642928791794, + "h_residuals": [ + 5.292367312510447, + -1.2325180959965676, + -8.34854165501151, + 18.532634348422874, + 21.298045268951483, + 26.66314727338721, + 45.7974867531929, + 1.1143548526059945, + -25.281726091833455, + 20.538142904874615, + 42.62245203257627, + 32.87553355400295 + ], + "h_res_rms": 25.272385757689367, + "v_residuals": [ + 1.948174626316228, + -0.453757025563914, + -3.0739954328321524, + 6.815581499143073, + 7.830632268485128, + 9.7997429292699, + 16.81714105751497, + 0.40987265024026254, + -9.307426018837527, + 7.546207273811804, + 15.64451435403657, + 12.069949587749356 + ], + "v_res_rms": 9.28371695850286, + "velocities": [ + 0.0, + 12969.297700466464, + 12063.810480597225, + 11535.301400186596, + 14810.349609688103, + 12948.52617841113, + 14077.237634374165, + 10248.745079827027, + 13145.40515860947, + 14305.426366502534, + 10388.296265044806, + 8232.259675064872 + ], + "velocities_prev_point": [ + 12177.028722194334, + 12177.028722194334, + 12177.028722194334, + 12177.028722194334, + 12635.927988514766, + 12811.691915261166, + 13010.049718796889, + 12862.45411623616, + 12812.459983391418, + 12868.413677161267, + 12786.266891221863, + 12563.748843977468 + ], + "length": [ + 0.0, + 519.3425571174787, + 1002.4740392444213, + 1464.162942485498, + 2057.347065052742, + 2575.8578473410325, + 3139.510442221366, + 3550.116165099566, + 4076.826258994727, + 4649.4438655930835, + 5065.225035305226, + 5394.844712694833 + ], + "state_vect_dist": [ + 9921.354086256408, + 10440.696643373885, + 10923.828125500828, + 11385.517028741906, + 11978.70115130915, + 12497.21193359744, + 13060.864528477774, + 13471.470251355973, + 13998.180345251134, + 14570.79795184949, + 14986.579121561634, + 15316.19879895124 + ], + "lag": [ + -393.8633893220631, + -393.8001635112796, + -429.99988356618087, + -487.3309572559665, + -413.52990774576574, + -414.29845676416335, + -369.87332231534674, + -478.8062851196937, + -471.68674778223067, + -418.1409889898896, + -521.3797962085864, + -710.987579250519 + ], + "lag_line": [ + 12179.051046767432, + -9899.353326496881 + ], + "v_init": 12179.051046767432, + "v_init_stddev": null, + "jacchia_fit": [ + 169.25151905025842, + 0.9545719833694175 + ], + "model_ra": [ + 4.268675306470282, + 4.275740112640083, + 4.282193626963838, + 4.288259757151979, + 4.295985212896367, + 4.302594155354585, + 4.3096754275247084, + 4.3147005165367975, + 4.321109894501091, + 4.327977157888565, + 4.332834122852471, + 4.336600836612544 + ], + "model_dec": [ + 1.0877269738198503, + 1.0827457055324, + 1.078088202406111, + 1.073615583501862, + 1.067812547679924, + 1.0627178988374508, + 1.0571404830083215, + 1.0530775705703026, + 1.0478192667242059, + 1.0420648001571293, + 1.0378881555178725, + 1.0345818657151975 + ], + "model_azim": [ + 0.8831700010505981, + 0.8974524277636791, + 0.9105774389431591, + 0.9229769805934369, + 0.9387740337613688, + 0.9523731955673052, + 0.9669789957103925, + 0.9774364487803786, + 0.9907468066797422, + 1.0050282483732862, + 1.0152133154617018, + 1.0231714421513614 + ], + "model_elev": [ + 1.1913491255580793, + 1.188611241077182, + 1.1860024765005819, + 1.1834532326767415, + 1.1800641024878358, + 1.1770365570460057, + 1.1736559939503042, + 1.171169280976094, + 1.1678878621525661, + 1.1642326384892834, + 1.161556949829732, + 1.1594281762863772 + ], + "model_fit1": [ + 0.8831700010505981, + 0.8974524277636791, + 0.9105774389431591, + 0.9229769805934369, + 0.9387740337613688, + 0.9523731955673052, + 0.9669789957103925, + 0.9774364487803786, + 0.9907468066797422, + 1.0050282483732862, + 1.0152133154617018, + 1.0231714421513614 + ], + "model_fit2": [ + 1.1913491255580793, + 1.188611241077182, + 1.1860024765005819, + 1.1834532326767415, + 1.1800641024878358, + 1.1770365570460057, + 1.1736559939503042, + 1.171169280976094, + 1.1678878621525661, + 1.1642326384892834, + 1.161556949829732, + 1.1594281762863772 + ], + "meas_eci": [ + [ + -0.19949752969882315, + -0.41951346273949935, + 0.885555865106421 + ], + [ + -0.19831370967000816, + -0.42489567883123996, + 0.8832527014775893 + ], + [ + -0.19719537224748762, + -0.42989189519454285, + 0.8810828244894001 + ], + [ + -0.19653440624741567, + -0.43469911459331356, + 0.8788691068257918 + ], + [ + -0.19526604393742353, + -0.44086253314794654, + 0.8760772790977996 + ], + [ + -0.19419198311969488, + -0.44623568023084337, + 0.8735921195735208 + ], + [ + -0.1931713762708322, + -0.45209532151521414, + 0.8708011481697114 + ], + [ + -0.19171870682741307, + -0.45626258010862597, + 0.8689467161138501 + ], + [ + -0.19020697856453836, + -0.4616673137779947, + 0.8664205657152649 + ], + [ + -0.18946461302037865, + -0.4676323484404185, + 0.8633789128216748 + ], + [ + -0.18879990840657096, + -0.4719169246748595, + 0.8611904613911451 + ], + [ + -0.18794699339414994, + -0.4752570683607163, + 0.8595386242905415 + ] + ], + "meas_eci_los": [ + [ + -0.19947261818984635, + -0.41952530837615254, + 0.885555865106421 + ], + [ + -0.19828723693595823, + -0.4249080335800251, + 0.8832527014775893 + ], + [ + -0.1971673332669236, + -0.42990475582643994, + 0.8810828244894001 + ], + [ + -0.19650478472685406, + -0.4347125057399289, + 0.8788691068257918 + ], + [ + -0.19523471543822826, + -0.44087640777926657, + 0.8760772790977996 + ], + [ + -0.19415896881448527, + -0.4462500458800414, + 0.8735921195735208 + ], + [ + -0.19313660866544666, + -0.4521101754434669, + 0.8708011481697114 + ], + [ + -0.19168228546062463, + -0.4562778824308138, + 0.8689467161138501 + ], + [ + -0.19016877668854015, + -0.4616830511058688, + 0.8664205657152649 + ], + [ + -0.18942455378532386, + -0.4676485767305336, + 0.8633789128216748 + ], + [ + -0.18875810448810454, + -0.4719336470299235, + 0.8611904613911451 + ], + [ + -0.18790350619325405, + -0.4752742636762536, + 0.8595386242905415 + ] + ], + "model_eci": [ + [ + -3430218.4459287277, + -1739794.238366333, + 5168303.099138084 + ], + [ + -3430101.130186622, + -1740214.0687953206, + 5168020.730054632 + ], + [ + -3429991.976360051, + -1740604.6083564344, + 5167758.024153548 + ], + [ + -3429887.652948811, + -1740977.7996632662, + 5167506.958303962 + ], + [ + -3429753.6593147092, + -1741457.3125123843, + 5167184.447316998 + ], + [ + -3429636.5043168305, + -1741876.432179816, + 5166902.494989177 + ], + [ + -3429509.159525174, + -1742332.044553453, + 5166596.010709642 + ], + [ + -3429416.332842804, + -1742663.8893094896, + 5166372.661447595 + ], + [ + -3429297.3084989507, + -1743089.6082505668, + 5166086.229687476 + ], + [ + -3429167.922097079, + -1743552.438223741, + 5165774.850941512 + ], + [ + -3429073.9084734865, + -1743888.437613092, + 5165548.665302157 + ], + [ + -3428999.3243624857, + -1744154.7560413252, + 5165369.277862051 + ] + ], + "meas_lat": [ + 0.9341977291845097, + 0.9341620422361886, + 0.9341289887201897, + 0.9340925154995868, + 0.9340502484747027, + 0.9340128678846222, + 0.9339702969792177, + 0.9339477745675356, + 0.9339143956069487, + 0.9338673009272427, + 0.9338347231487523, + 0.9338128429395532 + ], + "meas_lon": [ + -0.039569337714437845, + -0.0394604803690885, + -0.039359318234566246, + -0.03926606572328635, + -0.03914232800703642, + -0.039034834123200954, + -0.03891903030012694, + -0.03882974941824365, + -0.03871741853890708, + -0.038602319686930085, + -0.03851846883407553, + -0.03844994217817617 + ], + "meas_ht": [ + 78009.79883316693, + 77831.20354207544, + 77664.69644717604, + 77517.24209363233, + 77316.91530664213, + 77142.80788407566, + 76958.1916129066, + 76803.36867867864, + 76615.55707637589, + 76437.24094479789, + 76303.77256128346, + 76188.47275826824 + ], + "meas_range": [ + 83839.35913329039, + 83740.33676324955, + 83650.64655986936, + 83567.18288250422, + 83464.48024344677, + 83377.45673232341, + 83286.60977747764, + 83221.62612094023, + 83142.13428744262, + 83059.64575911379, + 83000.93009824904, + 82954.89554656077 + ], + "model_lat": [ + 0.9341984959671219, + 0.9341618633579815, + 0.9341277785486444, + 0.9340952007177374, + 0.934053334269257, + 0.9340167308727484, + 0.9339769318673277, + 0.9339479355618667, + 0.9339107317015435, + 0.9338702759470549, + 0.9338408971786679, + 0.933817604920053 + ], + "model_lon": [ + -0.039568822172874184, + -0.039460600665898946, + -0.03936013225744678, + -0.03926425828503005, + -0.03914025008272261, + -0.03903223179007913, + -0.03891455786744328, + -0.03882964093347483, + -0.03871988735385722, + -0.03860031305219093, + -0.03851430205713854, + -0.03844672807852617 + ], + "model_ht": [ + 78007.96659065987, + 77831.63104327311, + 77667.58915264205, + 77510.83050120615, + 77309.5485524352, + 77133.5883195504, + 76942.36899348421, + 76802.98414373519, + 76624.31618154436, + 76430.14134564155, + 76289.05225369317, + 76177.11611285266 + ], + "model_range": [ + 83839.35932285733, + 83740.33677357878, + 83650.64703317253, + 83567.18521487652, + 83464.4833274639, + 83377.46157053398, + 83286.62406500315, + 83221.62612936008, + 83142.13865346192, + 83059.64864007127, + 83000.94251389406, + 82954.90293702218 + ], + "rbeg_lat": 0.9341984959671219, + "rbeg_lon": -0.039568822172874184, + "rbeg_ele": 78007.96659065987, + "rbeg_jd": 2461151.500214981, + "rend_lat": 0.933817604920053, + "rend_lon": -0.03844672807852617, + "rend_ele": 76177.11611285266, + "rend_jd": 2461151.500220079, + "htmin_lat": 0.933817604920053, + "htmin_lon": -0.03844672807852617, + "htmin_ele": 76177.11611285266, + "htmin_jd": 2461151.500220079, + "absolute_magnitudes": [ + 2.4527602455285358, + 2.935326484323726, + 2.667653477222837, + 2.6698211294198684, + 2.9324914534964, + 2.4647566555887437, + 2.837123706123827, + 3.3088190122834766, + 2.3208940419532453, + 2.9330495519000843, + 4.884584879911243, + 3.855789702473728 + ], + "ra_data_los": [ + 4.268555447761289, + 4.2757679637412895, + 4.282381342636115, + 4.287844958142504, + 4.295510917776434, + 4.302003078945247, + 4.3086654772619175, + 4.314676085433959, + 4.321663432280336, + 4.327530324867534, + 4.331910650932565, + 4.3358906783280355 + ], + "dec_data_los": [ + 1.0876892604784882, + 1.082754431923786, + 1.0781467943009044, + 1.07348646708577, + 1.0676655293254065, + 1.0625353098743917, + 1.0568295338648421, + 1.0530700757470863, + 1.0479884168122084, + 1.0419285874153617, + 1.037607171800031, + 1.0343662231599273 + ], + "x_eci": [ + -0.19949752969882315, + -0.19831370967000816, + -0.19719537224748762, + -0.19653440624741567, + -0.19526604393742353, + -0.19419198311969488, + -0.1931713762708322, + -0.19171870682741307, + -0.19020697856453836, + -0.18946461302037865, + -0.18879990840657096, + -0.18794699339414994 + ], + "y_eci": [ + -0.41951346273949935, + -0.42489567883123996, + -0.42989189519454285, + -0.43469911459331356, + -0.44086253314794654, + -0.44623568023084337, + -0.45209532151521414, + -0.45626258010862597, + -0.4616673137779947, + -0.4676323484404185, + -0.4719169246748595, + -0.4752570683607163 + ], + "z_eci": [ + 0.885555865106421, + 0.8832527014775893, + 0.8810828244894001, + 0.8788691068257918, + 0.8760772790977996, + 0.8735921195735208, + 0.8708011481697114, + 0.8689467161138501, + 0.8664205657152649, + 0.8633789128216748, + 0.8611904613911451, + 0.8595386242905415 + ], + "x_eci_los": [ + -0.19947261818984635, + -0.19828723693595823, + -0.1971673332669236, + -0.19650478472685406, + -0.19523471543822826, + -0.19415896881448527, + -0.19313660866544666, + -0.19168228546062463, + -0.19016877668854015, + -0.18942455378532386, + -0.18875810448810454, + -0.18790350619325405 + ], + "y_eci_los": [ + -0.41952530837615254, + -0.4249080335800251, + -0.42990475582643994, + -0.4347125057399289, + -0.44087640777926657, + -0.4462500458800414, + -0.4521101754434669, + -0.4562778824308138, + -0.4616830511058688, + -0.4676485767305336, + -0.4719336470299235, + -0.4752742636762536 + ], + "z_eci_los": [ + 0.885555865106421, + 0.8832527014775893, + 0.8810828244894001, + 0.8788691068257918, + 0.8760772790977996, + 0.8735921195735208, + 0.8708011481697114, + 0.8689467161138501, + 0.8664205657152649, + 0.8633789128216748, + 0.8611904613911451, + 0.8595386242905415 + ], + "x_stat": -3413601.4235978713, + "y_stat": -1704419.3296659163, + "z_stat": 5094057.194026681, + "stat_eci": [ + -3413601.4235978713, + -1704419.3296659163, + 5094057.194026681 + ], + "stat_eci_los": [ + [ + -3413500.207340225, + -1704622.0299575112, + 5094057.194026681 + ], + [ + -3413495.2262395704, + -1704632.004559309, + 5094057.194026681 + ], + [ + -3413490.2500972687, + -1704641.969159245, + 5094057.194026681 + ], + [ + -3413485.273925883, + -1704651.9337446468, + 5094057.194026681 + ], + [ + -3413480.2977254046, + -1704661.89831553, + 5094057.194026681 + ], + [ + -3413475.316508251, + -1704671.8728591497, + 5094057.194026681 + ], + [ + -3413470.340249567, + -1704681.8374009654, + 5094057.194026681 + ], + [ + -3413465.358960294, + -1704691.8119432305, + 5094057.194026681 + ], + [ + -3413460.37765573, + -1704701.7864431958, + 5094057.194026681 + ], + [ + -3413455.4063112848, + -1704711.7409264494, + 5094057.194026681 + ], + [ + -3413450.4299362227, + -1704721.7054101357, + 5094057.194026681 + ], + [ + -3413445.4535320713, + -1704731.6698792966, + 5094057.194026681 + ] + ], + "plane_N": [ + -0.9633203118488703, + -0.08160610934519683, + -0.2556451049735453 + ], + "excluded_time": null, + "excluded_indx_range": [], + "ang_res": [ + 6.724691477696396e-05, + 1.5706629728077543e-05, + 0.00010637754358927273, + 0.0002362632170843941, + 0.00027184572472634675, + 0.00034066933402676907, + 0.0005857413387279998, + 1.4225497938347832e-05, + 0.0003240762673397417, + 0.0002633833955488076, + 0.0005469632219021634, + 0.00042211398237902714 + ], + "ang_res_std": 0.0003236383693663661 + } + }, + { + "ObservedPoints.UK00CD": { + "meas1": [ + 3.2350197840765165, + 3.2477072959811264, + 3.2552292551574604, + 3.2758807935362744, + 3.293926737375255, + 3.3099222963095567, + 3.3239878762449746, + 3.3380897824049134, + 3.357389348527242, + 3.371805691892109, + 3.3873157784796977, + 3.4044770059058163, + 3.4212650205456225, + 3.4374557844393463 + ], + "meas2": [ + 1.2609999783504144, + 1.2612395779929737, + 1.2595198256953215, + 1.260472338963183, + 1.2592765366352765, + 1.2592426249824373, + 1.2591634241814451, + 1.2581291561405523, + 1.2570818128175514, + 1.2569329596745575, + 1.2553525946153048, + 1.2547119136331997, + 1.2533115917530044, + 1.253010663890778 + ], + "jdt_ref": 2461151.500205556, + "time_data": [ + 6.187651883518428e-05, + 0.0399378765188352, + 0.0798018765188349, + 0.11962187651883464, + 0.15939787651883444, + 0.1992378765188352, + 0.2390978765188352, + 0.27892987651883483, + 0.3187098765188352, + 0.3585698765188352, + 0.3983738765188345, + 0.4381978765188348, + 0.4779978765188344, + 0.5178458765188345 + ], + "ignore_station": false, + "ignore_list": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "kmeas": 14, + "JD_data": [ + 2461151.500205559, + 2461151.5002060207, + 2461151.500206482, + 2461151.500206943, + 2461151.500207403, + 2461151.500207864, + 2461151.5002083257, + 2461151.5002087867, + 2461151.500209247, + 2461151.5002097087, + 2461151.5002101692, + 2461151.5002106302, + 2461151.500211091, + 2461151.500211552 + ], + "lat": 0.9302909996683025, + "lon": -0.039105358527994384, + "ele": 80.0, + "station_id": "UK00CD", + "azim_data": [ + 5.966419815139067, + 5.97542519016121, + 5.977778599822299, + 5.993954146606859, + 6.005275633071222, + 6.017243491584776, + 6.02792804365347, + 6.0375395619757395, + 6.051683132695393, + 6.063393040906969, + 6.074541567562709, + 6.0886289147110215, + 6.101951058174785, + 6.116263187282847 + ], + "elev_data": [ + 1.2028290600570986, + 1.2049672270465874, + 1.2077118383916157, + 1.2105467452617509, + 1.214565742676128, + 1.2171622436903107, + 1.2193966508469822, + 1.2223914732590124, + 1.226048237869094, + 1.2281100846973485, + 1.2315217824941145, + 1.2341667869259583, + 1.2373483805420769, + 1.2393083048928824 + ], + "ra_data": [ + 3.2350197840765156, + 3.2477072959811255, + 3.2552292551574595, + 3.275880793536273, + 3.293926737375254, + 3.3099222963095567, + 3.323987876244974, + 3.3380897824049134, + 3.3573893485272417, + 3.371805691892108, + 3.387315778479697, + 3.404477005905816, + 3.421265020545622, + 3.4374557844393454 + ], + "dec_data": [ + 1.2609999783504147, + 1.2612395779929737, + 1.2595198256953215, + 1.2604723389631833, + 1.2592765366352765, + 1.2592426249824373, + 1.2591634241814453, + 1.2581291561405523, + 1.2570818128175514, + 1.2569329596745575, + 1.2553525946153052, + 1.2547119136331992, + 1.2533115917530042, + 1.2530106638907785 + ], + "magnitudes": [ + 4.86, + 4.32, + 5.89, + 4.98, + 2.84, + 2.54, + 2.93, + 2.47, + 2.12, + 2.31, + 2.72, + 2.1, + 2.7, + 2.14 + ], + "fov_beg": true, + "fov_end": true, + "obs_id": "UK00CD_20260421-000017.700500_9470", + "comment": "{\"ff_name\": \"FF_UK00CD_20260421_000013_479_0364800.fits\"}", + "incident_angle": 0.9673542667254906, + "weight": 0.6779664749397492, + "h_residuals": [ + 10.60965356642866, + -24.36076247965052, + 105.75947002249839, + -7.220710828385594, + 52.4106292421939, + 13.980469426264557, + -21.11566161888541, + 16.385999544642043, + 31.159896386556547, + -15.566908035957312, + 42.18031417134474, + 11.207776909509969, + 34.794840093942796, + -31.51281526197431 + ], + "h_res_rms": 38.65757618379624, + "v_residuals": [ + 3.2446947715231613, + -7.4621198683776795, + 32.17069118416699, + -2.20894625977602, + 15.980407194763492, + 4.270405033174591, + -6.460331591938528, + 5.002493265014071, + 9.503554253729916, + -4.7583795692783655, + 12.852019409716682, + 3.4197354351588864, + 10.601440013163899, + -9.632696675175211 + ], + "v_res_rms": 11.775952423901554, + "velocities": [ + 0.0, + 10471.680103624529, + 6537.224031186331, + 16633.282812297697, + 14826.52683173458, + 12873.383577289029, + 11286.270727626033, + 11554.888061040885, + 15564.781179446425, + 11390.115494189105, + 12773.925244016458, + 13582.991536140642, + 13615.88234822533, + 12542.882660933778 + ], + "velocities_prev_point": [ + 10743.85866816658, + 10743.85866816658, + 10743.85866816658, + 10743.85866816658, + 12007.055445941307, + 12492.235701727113, + 12536.279099510239, + 12472.845218100547, + 12649.080739643105, + 12670.109215040093, + 12686.294204401467, + 12730.115433503099, + 12785.779374174901, + 12813.830897857222 + ], + "length": [ + 0.0, + 417.56871581213187, + 678.1686145913418, + 1340.5059361770318, + 1930.2458674361035, + 2443.121469155308, + 2892.992220358482, + 3353.246521605858, + 3972.4135169242427, + 4426.4235205226205, + 4934.8768409354425, + 5475.8058958707115, + 6017.718013330074, + 6517.526801602965 + ], + "state_vect_dist": [ + 37.76837659225623, + 379.80033921987564, + 640.4002379990856, + 1302.7375595847757, + 1892.4774908438474, + 2405.3530925630516, + 2855.223843766226, + 3315.4781450136015, + 3934.6451403319866, + 4388.655143930364, + 4897.108464343187, + 5438.037519278456, + 5979.949636737819, + 6479.758425010708 + ], + "lag": [ + 281.3381680637624, + 106.26937614174375, + -150.0758670031762, + -4.113107714768603, + 69.82284087386734, + 66.06452591993548, + -0.9579939258560444, + -57.2338676012655, + 46.077274171508634, + -16.805993279080212, + -24.51975166286138, + -0.017129900050349534, + 25.779779637870888, + 8.85090948729794 + ], + "lag_line": [ + 12015.663931514431, + -83.75025379184109 + ], + "v_init": 12015.663931514431, + "v_init_stddev": null, + "jacchia_fit": [ + 0.0, + 0.0 + ], + "model_ra": [ + 3.2350317233883077, + 3.2476714150887465, + 3.255438140444483, + 3.2758702514549296, + 3.2941153488378436, + 3.3099924216832672, + 3.323910252516376, + 3.3381929059797426, + 3.3575930020386857, + 3.371733421626219, + 3.3876369233400556, + 3.404593711003503, + 3.4215849629232125, + 3.437217005959056 + ], + "model_dec": [ + 1.2611273499274973, + 1.2609464139140631, + 1.260793448991946, + 1.2603850642422985, + 1.259911335665489, + 1.2594122969591925, + 1.2589067455139373, + 1.2583285427696425, + 1.25746144668329, + 1.2567431069600323, + 1.2558671504145162, + 1.254848700705218, + 1.253736238689485, + 1.252625998752515 + ], + "model_azim": [ + 5.966643957880864, + 5.974911280194201, + 5.9800178789208704, + 5.993802064108692, + 6.006378939289549, + 6.017536643092251, + 6.0274870372718645, + 6.03788102220137, + 6.052329074285987, + 6.0630724131073555, + 6.075406678946376, + 6.088856757848179, + 6.102652853784429, + 6.115634949478984 + ], + "model_elev": [ + 1.202730398539852, + 1.2051959385233475, + 1.2067159430592809, + 1.210615868504016, + 1.2140581450105756, + 1.2170252460038622, + 1.2196057067116586, + 1.2222277443684135, + 1.225732770549091, + 1.2282693521279788, + 1.2310859268705145, + 1.234049575685928, + 1.2369803907988324, + 1.239645469881098 + ], + "model_fit1": [ + 5.966643957880864, + 5.974911280194201, + 5.9800178789208704, + 5.993802064108692, + 6.006378939289549, + 6.017536643092251, + 6.0274870372718645, + 6.03788102220137, + 6.052329074285987, + 6.0630724131073555, + 6.075406678946376, + 6.088856757848179, + 6.102652853784429, + 6.115634949478984 + ], + "model_fit2": [ + 1.202730398539852, + 1.2051959385233475, + 1.2067159430592809, + 1.210615868504016, + 1.2140581450105756, + 1.2170252460038622, + 1.2196057067116586, + 1.2222277443684135, + 1.225732770549091, + 1.2282693521279788, + 1.2310859268705145, + 1.234049575685928, + 1.2369803907988324, + 1.239645469881098 + ], + "meas_eci": [ + [ + -0.3035351281043153, + -0.02844121514996133, + 0.9523956758025997 + ], + [ + -0.3029229405820039, + -0.03226575808186291, + 0.9524686939341127 + ], + [ + -0.3042986687014469, + -0.034729084187786134, + 0.9519433864143417 + ], + [ + -0.3026179106160443, + -0.04088405047247195, + 0.9522346846189463 + ], + [ + -0.30295616797952424, + -0.046510880186298745, + 0.9518688451186214 + ], + [ + -0.30220529917399286, + -0.05135608475551451, + 0.9518584504587548 + ], + [ + -0.30152721261512533, + -0.05561523142857777, + 0.9518341694253778 + ], + [ + -0.30167832324719224, + -0.06005383655938584, + 0.9515165400544865 + ], + [ + -0.30143648571506654, + -0.0660779059164778, + 0.9511938579645285 + ], + [ + -0.3005904430157494, + -0.07044880838966484, + 0.9511479122429234 + ], + [ + -0.30091929891322594, + -0.07546789342433696, + 0.9506588097754449 + ], + [ + -0.3001680129514678, + -0.08077893212098153, + 0.9504598508754388 + ], + [ + -0.3000486040229262, + -0.08617384954180235, + 0.9500236328002793 + ], + [ + -0.2988875713280024, + -0.09110370995523792, + 0.9499296467307672 + ] + ], + "meas_eci_los": [ + [ + -0.30353512752019685, + -0.028441221383888878, + 0.9523956758025997 + ], + [ + -0.3029228461071481, + -0.03226664503462707, + 0.9524686939341127 + ], + [ + -0.30429846603725963, + -0.03473085989960501, + 0.9519433864143417 + ], + [ + -0.3026175532804857, + -0.040886695332201205, + 0.9522346846189463 + ], + [ + -0.30295562663884373, + -0.04651440615989257, + 0.9518688451186214 + ], + [ + -0.30220455226634335, + -0.05136047974617341, + 0.9518584504587548 + ], + [ + -0.301526242054227, + -0.05562049322827857, + 0.9518341694253778 + ], + [ + -0.30167710078673465, + -0.06005997721985251, + 0.9515165400544865 + ], + [ + -0.3014349488853584, + -0.06608491628985995, + 0.9511938579645285 + ], + [ + -0.30058859968688656, + -0.07045667302821534, + 0.9511479122429234 + ], + [ + -0.30091710526470994, + -0.07547663979962246, + 0.9506588097754449 + ], + [ + -0.30016543028477377, + -0.08078852849133539, + 0.9504598508754388 + ], + [ + -0.30004559881096554, + -0.08618431269743403, + 0.9500236328002793 + ], + [ + -0.2988841295616315, + -0.09111500073064646, + 0.9499296467307672 + ] + ], + "model_eci": [ + [ + -3432466.7540115756, + -1731741.8935464248, + 5173716.011629781 + ], + [ + -3432372.547769191, + -1732079.5683393998, + 5173489.145111228 + ], + [ + -3432313.743999133, + -1732290.3063399354, + 5173347.5431581 + ], + [ + -3432164.301102949, + -1732825.91596933, + 5172987.668751478 + ], + [ + -3432031.227056555, + -1733302.8168760038, + 5172667.221548597 + ], + [ + -3431915.4844218343, + -1733717.5583187286, + 5172388.519694615 + ], + [ + -3431813.9466015864, + -1734081.3479430394, + 5172144.033616263 + ], + [ + -3431710.0576202665, + -1734453.532469572, + 5171893.892688835 + ], + [ + -3431570.3102787603, + -1734954.2219146248, + 5171557.403865901 + ], + [ + -3431467.812772388, + -1735321.3520204683, + 5171310.628589792 + ], + [ + -3431353.024745032, + -1735732.5059074648, + 5171034.262150875 + ], + [ + -3431230.901985564, + -1736169.9185074358, + 5170740.239346225 + ], + [ + -3431108.549160772, + -1736608.122839488, + 5170445.669753482 + ], + [ + -3430995.6861842214, + -1737012.276212167, + 5170173.961306149 + ] + ], + "meas_lat": [ + 0.9348979840563066, + 0.9348736638383488, + 0.9348366981997408, + 0.9348063479155164, + 0.9347563423139719, + 0.9347257901580164, + 0.9346991840939688, + 0.9346614322637298, + 0.9346157521163507, + 0.9345905006711562, + 0.934546433761751, + 0.9345127904652669, + 0.9344712488826498, + 0.9344455500907388 + ], + "meas_lon": [ + -0.04164301451607974, + -0.04155291066409006, + -0.041513252115616614, + -0.04136276158768624, + -0.04124541609544114, + -0.041134549051738925, + -0.041037524018862015, + -0.040945674933504726, + -0.040817480073928235, + -0.040718389405427956, + -0.040618299375673295, + -0.04050220635063718, + -0.040391477328569035, + -0.04028058930887304 + ], + "meas_ht": [ + 81396.9675099336, + 81244.77789502541, + 81193.34068859647, + 80935.62631653334, + 80752.07851757792, + 80566.58131972801, + 80403.44378474863, + 80257.65942465847, + 80051.33834901436, + 79883.51877213769, + 79727.19354210321, + 79534.42486416693, + 79356.9731703905, + 79168.05194906335 + ], + "meas_range": [ + 87069.72999388228, + 86836.55913972364, + 86692.05673251014, + 86325.53975271186, + 86002.07075931023, + 85723.10812734772, + 85480.18898973276, + 85233.27293127988, + 84903.47168914345, + 84663.79807745767, + 84397.2537452506, + 84115.98487302613, + 83836.60845417317, + 83581.14830424251 + ], + "model_lat": [ + 0.934899505502217, + 0.9348701702014254, + 0.9348518594457105, + 0.9348053124719798, + 0.9347638562520294, + 0.9347277946223416, + 0.934696156319921, + 0.934663781456173, + 0.9346202190594269, + 0.9345882687471724, + 0.9345524799775028, + 0.9345143970336911, + 0.9344762361830826, + 0.9344410323178205 + ], + "model_lon": [ + -0.04164193466936269, + -0.04155538966397436, + -0.041502471430448676, + -0.04136349692543704, + -0.04124007338001663, + -0.04113312415058746, + -0.04103967585574465, + -0.04094400396225524, + -0.04081430089807224, + -0.040719977257233045, + -0.04061399294714445, + -0.0405010622126221, + -0.04038792316631062, + -0.0402838067317805 + ], + "model_ht": [ + 81393.91662581416, + 81251.79438980817, + 81163.08976953034, + 80937.70349190966, + 80737.05100419743, + 80562.56553501167, + 80409.51917280334, + 80252.95499555573, + 80042.40065984003, + 79887.99406011793, + 79715.1061188881, + 79531.20864436727, + 79347.00196171085, + 79177.11246162215 + ], + "model_range": [ + 87069.7307007453, + 86836.56287739017, + 86692.12721183259, + 86325.54008297449, + 86002.08821370563, + 85723.10937370706, + 85480.19184196735, + 85233.27465311192, + 84903.4779387753, + 84663.79964238926, + 84397.26526402768, + 84115.98568912731, + 83836.61634464488, + 83581.15480030599 + ], + "rbeg_lat": 0.934899505502217, + "rbeg_lon": -0.04164193466936269, + "rbeg_ele": 81393.91662581416, + "rbeg_jd": 2461151.500205559, + "rend_lat": 0.9344410323178205, + "rend_lon": -0.0402838067317805, + "rend_ele": 79177.11246162215, + "rend_jd": 2461151.500211552, + "htmin_lat": 0.9344410323178205, + "htmin_lon": -0.0402838067317805, + "htmin_ele": 79177.11246162215, + "htmin_jd": 2461151.500211552, + "absolute_magnitudes": [ + 5.160663993871118, + 4.6264868745522065, + 6.2001017019905555, + 5.2993034793540605, + 3.1674550176961667, + 2.8745104224398865, + 3.270672559130019, + 2.816954128451293, + 2.47537259597214, + 2.6715112232095644, + 3.0883581282452037, + 2.4756073077614653, + 3.082831290906207, + 2.529458163572881 + ], + "ra_data_los": [ + 3.235019804614262, + 3.247710223963076, + 3.2552350905836143, + 3.275889533472668, + 3.293938375945835, + 3.3099368393903763, + 3.324005326769998, + 3.3381101374401774, + 3.3574126051387827, + 3.371831855939726, + 3.387344844103591, + 3.404508976039951, + 3.4212998922560343, + 3.4374935606518706 + ], + "dec_data_los": [ + 1.2609999783504147, + 1.2612395779929737, + 1.2595198256953215, + 1.2604723389631833, + 1.2592765366352765, + 1.2592426249824373, + 1.2591634241814453, + 1.2581291561405523, + 1.2570818128175514, + 1.2569329596745575, + 1.2553525946153052, + 1.2547119136331992, + 1.2533115917530042, + 1.2530106638907785 + ], + "x_eci": [ + -0.3035351281043153, + -0.3029229405820039, + -0.3042986687014469, + -0.3026179106160443, + -0.30295616797952424, + -0.30220529917399286, + -0.30152721261512533, + -0.30167832324719224, + -0.30143648571506654, + -0.3005904430157494, + -0.30091929891322594, + -0.3001680129514678, + -0.3000486040229262, + -0.2988875713280024 + ], + "y_eci": [ + -0.02844121514996133, + -0.03226575808186291, + -0.034729084187786134, + -0.04088405047247195, + -0.046510880186298745, + -0.05135608475551451, + -0.05561523142857777, + -0.06005383655938584, + -0.0660779059164778, + -0.07044880838966484, + -0.07546789342433696, + -0.08077893212098153, + -0.08617384954180235, + -0.09110370995523792 + ], + "z_eci": [ + 0.9523956758025997, + 0.9524686939341127, + 0.9519433864143417, + 0.9522346846189463, + 0.9518688451186214, + 0.9518584504587548, + 0.9518341694253778, + 0.9515165400544865, + 0.9511938579645285, + 0.9511479122429234, + 0.9506588097754449, + 0.9504598508754388, + 0.9500236328002793, + 0.9499296467307672 + ], + "x_eci_los": [ + -0.30353512752019685, + -0.3029228461071481, + -0.30429846603725963, + -0.3026175532804857, + -0.30295562663884373, + -0.30220455226634335, + -0.301526242054227, + -0.30167710078673465, + -0.3014349488853584, + -0.30058859968688656, + -0.30091710526470994, + -0.30016543028477377, + -0.30004559881096554, + -0.2988841295616315 + ], + "y_eci_los": [ + -0.028441221383888878, + -0.03226664503462707, + -0.03473085989960501, + -0.040886695332201205, + -0.04651440615989257, + -0.05136047974617341, + -0.05562049322827857, + -0.06005997721985251, + -0.06608491628985995, + -0.07045667302821534, + -0.07547663979962246, + -0.08078852849133539, + -0.08618431269743403, + -0.09111500073064646 + ], + "z_eci_los": [ + 0.9523956758025997, + 0.9524686939341127, + 0.9519433864143417, + 0.9522346846189463, + 0.9518688451186214, + 0.9518584504587548, + 0.9518341694253778, + 0.9515165400544865, + 0.9511938579645285, + 0.9511479122429234, + 0.9506588097754449, + 0.9504598508754388, + 0.9500236328002793, + 0.9499296467307672 + ], + "x_stat": -3406048.6136386427, + "y_stat": -1729266.1246223755, + "z_stat": 5090787.796276712, + "stat_eci": [ + -3406048.6136386427, + -1729266.1246223755, + 5090787.796276712 + ], + "stat_eci_los": [ + [ + -3406048.5781234126, + -1729266.1945749386, + 5090787.796276712 + ], + [ + -3406043.5503640417, + -1729276.0974638269, + 5090787.796276712 + ], + [ + -3406038.5225758795, + -1729286.0003380955, + 5090787.796276712 + ], + [ + -3406033.4998326153, + -1729295.8932045705, + 5090787.796276712 + ], + [ + -3406028.4872221127, + -1729305.766042447, + 5090787.796276712 + ], + [ + -3406023.464421443, + -1729315.6588797665, + 5090787.796276712 + ], + [ + -3406018.4365182575, + -1729325.5616956379, + 5090787.796276712 + ], + [ + -3406013.413660088, + -1729335.454503771, + 5090787.796276712 + ], + [ + -3406008.39584702, + -1729345.3373042115, + 5090787.796276712 + ], + [ + -3406003.3678716016, + -1729355.24004859, + 5090787.796276712 + ], + [ + -3405998.350001125, + -1729365.1228198803, + 5090787.796276712 + ], + [ + -3405993.3270280496, + -1729375.0155696734, + 5090787.796276712 + ], + [ + -3405988.309100192, + -1729384.8983118308, + 5090787.796276712 + ], + [ + -3405983.286083734, + -1729394.791004777, + 5090787.796276712 + ] + ], + "plane_N": [ + -0.9508405323585105, + -0.0543339152478599, + -0.30487720098102244 + ], + "excluded_time": null, + "excluded_indx_range": [], + "ang_res": [ + 0.00012742337477004607, + 0.0002934025890114486, + 0.001275134678871608, + 8.747317337504222e-05, + 0.000637107786152886, + 0.00017052484520883212, + 0.00025833001748669024, + 0.00020100450053262754, + 0.0003836891026201303, + 0.00019227104764286688, + 0.0005224611490777694, + 0.00013929892102162514, + 0.00043386000256940454, + 0.00039426294165447807 + ], + "ang_res_std": 0.0004704327154024583 + } + } + ], + "los_mini_status": true, + "t_ref_station": 2, + "time_diffs_final": [ + 0.0, + -0.0019011817811903332, + -0.00014963188985710543 + ], + "intersection_list": [ + { + "PlaneIntersection.UK002Z_UK00AS": { + "conv_angle": 1.3977829287229966, + "radiant_eci": [ + -0.2119721838741445, + 0.805179044558886, + 0.5538542222163385 + ], + "radiant_eq": [ + 1.8282164010696587, + 0.5869862134998625 + ], + "w1": [ + -0.9772483431518503, + -0.17887979587700067, + -0.1139635662567909 + ], + "w2": [ + -0.16103171664643293, + -0.5877633547814152, + 0.792844893412324 + ], + "rcpa_stat1": [ + -73631.49434216082, + -13477.82963291005, + -8586.668622006024 + ], + "rcpa_stat2": [ + -13242.80545538603, + -48336.04163995439, + 65201.38329524699 + ], + "cpa_eci": [ + -3403324.3337847604, + -1842095.9935247626, + 5097804.319209092 + ], + "weight": 0.0024214538252984794 + } + }, + { + "PlaneIntersection.UK002Z_UK00CD": { + "conv_angle": 1.3408619526894816, + "radiant_eci": [ + -0.2224668519112711, + 0.8032092732613852, + 0.5525969264731806 + ], + "radiant_eq": [ + 1.8409953958825356, + 0.5854769115887319 + ], + "w1": [ + -0.9749128323753357, + -0.18752636752381052, + -0.11991176236393772 + ], + "w2": [ + -0.2150761118164841, + -0.5932692097626505, + 0.7757408786917855 + ], + "rcpa_stat1": [ + -72116.41498877588, + -13871.731802662609, + -8870.132928298488 + ], + "rcpa_stat2": [ + -15374.393477834303, + -42408.9602147798, + 55452.67396327532 + ], + "cpa_eci": [ + -3401809.2544313753, + -1842489.8956945152, + 5097520.854902799 + ], + "weight": 0.002371116245910391 + } + }, + { + "PlaneIntersection.UK00AS_UK00CD": { + "conv_angle": 0.0578872564894713, + "radiant_eci": [ + -0.17933529144848642, + 0.8982410335263312, + 0.4012504192279862 + ], + "radiant_eq": [ + 1.767856898383979, + 0.4128815721461222 + ], + "w1": [ + -0.19672893468820898, + -0.4323680153644721, + 0.8799747868809822 + ], + "w2": [ + -0.25216678471696985, + -0.43620698939268593, + 0.8637912798184956 + ], + "rcpa_stat1": [ + -16494.934626699258, + -36252.32942686249, + 73782.36763061871 + ], + "rcpa_stat2": [ + -19567.098351078548, + -33847.856181592375, + 67026.62662722637 + ], + "cpa_eci": [ + -3430096.3582245708, + -1740671.6590927788, + 5167839.561657299 + ], + "weight": 1.3053179448789485e-05 + } + } + ], + "rbeg_lat": 0.934899505502217, + "rbeg_lon": -0.04164193466936269, + "rbeg_ele": 81393.91662581416, + "rbeg_ele_wgs84": 81445.53494089004, + "rbeg_jd": 2461151.500205559, + "rend_lat": 0.9337921890458436, + "rend_lon": -0.038372180120937446, + "rend_ele": 76055.10224458351, + "rend_ele_wgs84": 76106.4147786973, + "rend_jd": 2461151.5002204534, + "htmin_lat": 0.9337921890458436, + "htmin_lon": -0.038372180120937446, + "htmin_ele": 76055.10224458351, + "htmin_ele_wgs84": 76106.4147786973, + "htmin_jd": 2461151.5002204534, + "state_vect": [ + -3432369.342207372, + -1731768.1585065187, + 5173694.939644102 + ], + "incident_angles": [ + 0.5027492433632643, + 1.2638605479641993, + 0.9659504756451831 + ], + "state_vect_mini": [ + -3432458.2336270753, + -1731772.435690051, + 5173695.492595176 + ], + "radiant_eci_mini": [ + -0.22559573032168215, + 0.8086697491907738, + 0.5432861154165067 + ], + "radiant_eq_mini": [ + 1.842850952886513, + 0.5743463248927236 + ], + "v_init": 12967.718791996132, + "v0z": -4413.997141557832, + "v_avg": 12312.85904311762, + "timing_minimization_successful": true, + "velocity_fit": [ + 12967.718791996132, + -244.37218876758848 + ], + "jacchia_fit": [ + 43.65312744412464, + 2.1924329546040098 + ], + "timing_res": 7.041064582452035e-05, + "timing_stddev": 0.008391105161092927, + "state_vect_avg": [ + -3430549.6763656787, + -1738607.335042225, + 5169100.684310519 + ], + "jd_avg": 2461151.5002130065, + "orbit": { + "Orbit": { + "ra": 1.842850952886513, + "dec": 0.5743463248927236, + "azimuth_apparent": 5.215879750489071, + "elevation_apparent": 0.3473192026194278, + "v_avg": 12312.85904311762, + "v_init": 12967.718791996132, + "v_init_stddev": 10.13578330407507, + "ra_norot": 1.837726586059179, + "dec_norot": 0.5860709340821146, + "azimuth_apparent_norot": 5.227200613194143, + "elevation_apparent_norot": 0.3538606337963674, + "v_avg_norot": 12082.953555046177, + "v_init_norot": 12737.81330392469, + "jd_ref": 2461151.500205559, + "jd_dyn": 2461151.5010063, + "lst_ref": 3.6088563643950953, + "lon_ref": -0.0416338437576483, + "lat_ref": 0.9348968523981452, + "ht_ref": 81381.06170645228, + "ht_ref_wgs84": 81432.6793071907, + "lat_geocentric": 0.9317248461260144, + "zc": 1.2250089673888134, + "zg": 1.6682930856125058, + "v_inf": 12967.718791996132, + "v_g": 6669.855942091, + "ra_g": 1.560157053440776, + "dec_g": 0.21033070922631758, + "L_g": 1.560182664795458, + "B_g": -0.19871170575289931, + "meteor_pos": [ + -129457249.42153406, + -76394902.82501395, + 10345.503483171698 + ], + "v_h": 35477.25261044726, + "v_h_x": 14.598935218756163, + "v_h_y": -32.30740600750524, + "v_h_z": 1.3183551608964306, + "L_h": 1.9952090815491528, + "B_h": -0.037169134413427715, + "la_sun": 0.5331351764388832, + "a": 1.749324945375272, + "e": 0.4367632022782812, + "i": 0.037382200258429005, + "peri": 5.925529697530141, + "node": 3.6728731864642414, + "pi": 3.3152175768147956, + "b": -0.0130840548179203, + "q": 0.9852841804078888, + "Q": 2.513365710342655, + "true_anomaly": 0.35949713851885046, + "eccentric_anomaly": 0.22656761811223144, + "mean_anomaly": 0.12845566895513721, + "last_perihelion": "2026-04-03 17:22:02.404589", + "n": 0.007434915611796666, + "Tj": 4.017379995926152, + "T": 2.313696397548099 + } + }, + "uncertainties": { + "MCUncertainties": { + "ci": 95, + "mc_traj_list": null, + "state_vect_mini": [ + 23.302927645551236, + 37.2070223921984, + 47.8951574781733 + ], + "state_vect_mini_ci": [ + [ + -3432495.617382003, + -3432423.837728241 + ], + [ + -1731807.3913284293, + -1731696.1065867343 + ], + [ + 5173640.759083595, + 5173792.756573728 + ] + ], + "x": 23.302927645551236, + "x_ci": [ + -3432495.617382003, + -3432423.837728241 + ], + "y": 37.2070223921984, + "y_ci": [ + -1731807.3913284293, + -1731696.1065867343 + ], + "z": 47.8951574781733, + "z_ci": [ + 5173640.759083595, + 5173792.756573728 + ], + "vx": 13.858206008974927, + "vx_ci": [ + -2938.2878636842593, + -2861.212929938162 + ], + "vy": 59.54333260289223, + "vy_ci": [ + 10483.746446609672, + 10492.11339262698 + ], + "vz": 57.58213998290366, + "vz_ci": [ + 6945.863075051403, + 7122.386599949237 + ], + "radiant_eci_mini": [ + 0.0016920905173864757, + 0.002356945317379389, + 0.002939081393705436 + ], + "radiant_eci_mini_ci": [ + [ + -0.22617399930035348, + -0.2214846318885415 + ], + [ + 0.8059754297844866, + 0.8128571058966497 + ], + [ + 0.5381818809865024, + 0.5473502583390504 + ] + ], + "rbeg_lon": 2.2088929690296427e-06, + "rbeg_lon_ci": [ + 6.2415409901294865, + 6.241546962406092 + ], + "rbeg_lon_m": 8.38540985943337, + "rbeg_lat": 2.638433710383294e-06, + "rbeg_lat_ci": [ + 0.9348949134986161, + 0.9349036888013773 + ], + "rbeg_lat_m": 16.864870452262917, + "rbeg_ele": 45.558677417463386, + "rbeg_ele_ci": [ + 81325.7017569644, + 81464.2043463111 + ], + "rbeg_ele_wgs84": 45.5588599685256, + "rbeg_ele_wgs84_ci": [ + 81377.31980654877, + 81515.822891308 + ], + "rend_lon": 3.407449543509572e-06, + "rend_lon_ci": [ + 6.244800260516159, + 6.244811360843754 + ], + "rend_lon_m": 12.954725654542509, + "rend_lat": 1.0453520900156793e-06, + "rend_lat_ci": [ + 0.933790446617204, + 0.9337939283069673 + ], + "rend_lat_m": 6.681867643440438, + "rend_ele": 32.15807998869947, + "rend_ele_ci": [ + 76036.61348161136, + 76133.00603886494 + ], + "rend_ele_wgs84": 32.15826496087123, + "rend_ele_wgs84_ci": [ + 76087.92684692098, + 76184.31958180107 + ], + "htmin_lon": 3.407449543509572e-06, + "htmin_lon_ci": [ + 6.244800260516159, + 6.244811360843754 + ], + "htmin_lon_m": 12.954725654542509, + "htmin_lat": 1.0453520900156793e-06, + "htmin_lat_ci": [ + 0.933790446617204, + 0.9337939283069673 + ], + "htmin_lat_m": 6.681867643440438, + "htmin_ele": 32.15807998869947, + "htmin_ele_ci": [ + 76036.61348161136, + 76133.00603886494 + ], + "htmin_ele_wgs84": 32.15826496087123, + "htmin_ele_wgs84_ci": [ + 76087.92684692098, + 76184.31958180107 + ], + "ra": 0.002611503709103583, + "ra_ci": [ + 1.8370325403202352, + 1.8440354813447923 + ], + "dec": 0.003499436816389006, + "dec_ci": [ + 0.5682786814957027, + 0.579194832887812 + ], + "azimuth_apparent": 0.0011488927073448102, + "azimuth_apparent_ci": [ + 5.2149278320957935, + 5.218617334102857 + ], + "elevation_apparent": 0.003985943562293819, + "elevation_apparent_ci": [ + 0.3401376381944182, + 0.35146749322636645 + ], + "v_avg": 25.363986414427266, + "v_avg_ci": [ + 12263.698119250277, + 12331.272064862434 + ], + "v_init": 35.83555406641995, + "v_init_ci": [ + 12906.156161188343, + 13012.85741906931 + ], + "lon_ref": 9.746851701222101e-06, + "lon_ref_ci": [ + 6.241530852167341, + 6.241562416968685 + ], + "lat_ref": 4.037368389248857e-06, + "lat_ref_ci": [ + 0.9348930977264116, + 0.9349048456367035 + ], + "ht_ref": 46.23031409446649, + "ht_ref_ci": [ + 81327.7139370313, + 81465.6113865276 + ], + "lat_geocentric": 4.058469875306498e-06, + "lat_geocentric_ci": [ + 0.9317210791689499, + 0.9317328894806066 + ], + "zc": 0.003984415255032236, + "zc_ci": [ + 1.2208682718192776, + 1.2321888843316788 + ], + "zg": 0.010437731100424239, + "zg_ci": [ + 1.6565960535519642, + 1.6867956381526732 + ], + "v_inf": 35.83555406641995, + "v_inf_ci": [ + 12906.156161188343, + 13012.85741906931 + ], + "v_g": 69.86456353616636, + "v_g_ci": [ + 6549.278440215807, + 6757.316401539255 + ], + "ra_g": 0.005218583891039907, + "ra_g_ci": [ + 1.550331619290657, + 1.5642537989451053 + ], + "dec_g": 0.009175375088677046, + "dec_g_ci": [ + 0.19423320526384985, + 0.22168953806563507 + ], + "L_g": 0.005261542789721588, + "L_g_ci": [ + 1.550262999253746, + 1.5642996777101477 + ], + "B_g": 0.009147790639302643, + "B_g_ci": [ + -0.2147525041002339, + -0.18736660983219178 + ], + "meteor_pos": [ + 0.023402287765977765, + 0.04500344767424416, + 0.04074976099956943 + ], + "meteor_pos_ci": [ + [ + -129457249.45909907, + -129457249.3870311 + ], + [ + -76394902.87402098, + -76394902.73235357 + ], + [ + 10345.446462113892, + 10345.578152204636 + ] + ], + "v_h": 85.48609930680463, + "v_h_ci": [ + 35328.09302105182, + 35577.45936832089 + ], + "v_h_x": 0.03321975068903373, + "v_h_x_ci": [ + 14.53657918185901, + 14.62520287469976 + ], + "v_h_y": 0.08104467976059955, + "v_h_y_ci": [ + -32.40782446857918, + -32.16672196346313 + ], + "v_h_z": 0.04568102257541548, + "v_h_z_ci": [ + 1.2602424833462527, + 1.3973518963978855 + ], + "L_h": 0.00021478363911639652, + "L_h_ci": [ + 1.9947075813557262, + 1.9953602674305384 + ], + "B_h": 0.0013798257879346354, + "B_h_ci": [ + -0.03956396089577628, + -0.035430681123816095 + ], + "la_sun": 0.0, + "la_sun_ci": [ + 0.5331351764388832, + 0.5331351764388832 + ], + "a": 0.020764918399534503, + "a_ci": [ + 1.7136415146239252, + 1.7742356355051931 + ], + "e": 0.00662985086508511, + "e_ci": [ + 0.4252246127134539, + 0.44463172993888717 + ], + "i": 0.0013875907380813083, + "i_ci": [ + 0.03563537240700222, + 0.03979064362004738 + ], + "peri": 0.0036974620554850617, + "peri_ci": [ + 5.918617880304961, + 5.928510238183485 + ], + "node": 6.736619195035191e-05, + "node_ci": [ + 3.6727828832139906, + 3.6729846226879324 + ], + "pi": 0.0036316227110077645, + "pi_ci": [ + 3.308403222800056, + 3.3181100489377644 + ], + "b": 0.0006164034149704704, + "b_ci": [ + -0.01417379179358153, + -0.01237568509476685 + ], + "q": 0.0001887248437724549, + "q_ci": [ + 0.9848735253457906, + 0.9853733247776466 + ], + "Q": 0.04135376621516677, + "Q_ci": [ + 2.4423263887544406, + 2.563117103921014 + ], + "true_anomaly": 0.003631669498448071, + "true_anomaly_ci": [ + 0.3566046072044634, + 0.3663115643921616 + ], + "eccentric_anomaly": 0.00415997564946805, + "eccentric_anomaly_ci": [ + 0.22255528058867302, + 0.23405984987107875 + ], + "mean_anomaly": 0.0038950653377452164, + "mean_anomaly_ci": [ + 0.12441501198245487, + 0.13542723129154152 + ], + "last_perihelion": 0.21162646186227413, + "last_perihelion_ci": [ + 2461133.838177647, + 2461134.4084006962 + ], + "n": 0.00013353213752772863, + "n_ci": [ + 0.00727888466701047, + 0.007668381602780202 + ], + "T": 0.04112839790029051, + "T_ci": [ + 2.243267062584157, + 2.3632930877217384 + ], + "Tj": 0.03300359258971792, + "Tj_ci": [ + 3.978562778709054, + 4.074873090067962 + ], + "ra_norot": 0.0026819047506464275, + "ra_norot_ci": [ + 1.831751910635052, + 1.8389425768418368 + ], + "dec_norot": 0.003535179504345732, + "dec_norot_ci": [ + 0.5799393752075872, + 0.590966744099576 + ], + "v_avg_norot": 25.73922847929732, + "v_avg_norot_ci": [ + 12033.412162374418, + 12102.036345823391 + ], + "v_init_norot": 36.25000387087065, + "v_init_norot_ci": [ + 12675.549576533595, + 12783.621700030268 + ], + "azimuth_apparent_norot": 0.0011626219913197264, + "azimuth_apparent_norot_ci": [ + 5.226249501183861, + 5.229971656717279 + ], + "elevation_apparent_norot": 0.004037622005774978, + "elevation_apparent_norot_ci": [ + 0.3465823432479639, + 0.35805015466321544 + ], + "ht_ref_wgs84": 46.23066689867216, + "ht_ref_wgs84_ci": [ + 81379.33149963012, + 81517.23000879795 + ] + } + }, + "orbit_cov": [ + [ + 4.839509762694922e-05, + 1.2839048270422571e-06, + 0.001529597070645031, + -2.8134845951441427e-05, + 0.0015235437188076953, + -0.0005796707377483675 + ], + [ + 1.2839048270422571e-06, + 3.92532878805147e-08, + 4.283334419962881e-05, + -7.366720739670027e-07, + 4.318110319692552e-05, + -1.5189142360366112e-05 + ], + [ + 0.001529597070645031, + 4.283334419962881e-05, + 0.04932644359606424, + -0.0008848174909047756, + 0.049355279485603636, + -0.018239993412701383 + ], + [ + -2.8134845951441427e-05, + -7.366720739670027e-07, + -0.0008848174909047754, + 1.640345974762046e-05, + -0.0008803958803679891, + 0.00033761695055935486 + ], + [ + 0.0015235437188076953, + 4.318110319692551e-05, + 0.04935527948560364, + -0.0008803958803679891, + 0.049434753433829214, + -0.018149168163731216 + ], + [ + -0.0005796707377483675, + -1.5189142360366112e-05, + -0.018239993412701383, + 0.00033761695055935486, + -0.018149168163731216, + 0.006957026666261073 + ] + ], + "state_vect_cov": [ + [ + 594.8872861480645, + -256.66487170381055, + -967.7624393923977, + 583.6215942851601, + 26.71726004970638, + -985.1200079805594 + ], + [ + -256.66487170381055, + 1522.6343263519718, + 718.6734107124669, + 359.1627911110367, + 46.990250811313, + -641.6507010433324 + ], + [ + -967.7624393923977, + 718.673410712467, + 2514.098708757615, + -931.2754297849413, + -11.982008124620437, + 2312.506952848055 + ], + [ + 583.6215942851601, + 359.1627911110367, + -931.2754297849413, + 923.3221697191502, + 62.170755663341666, + -1621.7192914483069 + ], + [ + 26.71726004970638, + 46.99025081131299, + -11.982008124620435, + 62.170755663341666, + 7.208948184864825, + -91.24287656294936 + ], + [ + -985.1200079805594, + -641.6507010433324, + 2312.5069528480544, + -1621.719291448307, + -91.24287656294938, + 3635.949624000401 + ] + ], + "phase_1_only": false, + "avg_radiant": [ + -0.21707459130810688, + 0.8045078859660982, + 0.5528523159278415 + ], + "radiant_eq": [ + 1.8343430182556113, + 0.5857833676735065 + ], + "best_conv_inter": { + "PlaneIntersection.UK002Z_UK00AS": { + "conv_angle": 1.3977829287229966, + "radiant_eci": [ + -0.2119721838741445, + 0.805179044558886, + 0.5538542222163385 + ], + "radiant_eq": [ + 1.8282164010696587, + 0.5869862134998625 + ], + "w1": [ + -0.9772483431518503, + -0.17887979587700067, + -0.1139635662567909 + ], + "w2": [ + -0.16103171664643293, + -0.5877633547814152, + 0.792844893412324 + ], + "rcpa_stat1": [ + -73631.49434216082, + -13477.82963291005, + -8586.668622006024 + ], + "rcpa_stat2": [ + -13242.80545538603, + -48336.04163995439, + 65201.38329524699 + ], + "cpa_eci": [ + -3403324.3337847604, + -1842095.9935247626, + 5097804.319209092 + ], + "weight": 0.0024214538252984794 + } + }, + "stations_time_dict": { + "OrderedDict": {} + }, + "stations_time_dict_copy": { + "OrderedDict": {} + }, + "v_init_stddev": 10.13578330407507, + "time_diffs": [ + 0.0, + 0.00026701713965955896, + 6.187651883518428e-05 + ], + "longname": "20260421_000017.759_UK", + "pre_mc_longname": "20260421_000017.759_UK", + "save_date": "2026-04-21 09:33:47.459946+00:00" +} \ No newline at end of file diff --git a/archive/server_setup/.bash_aliases b/archive/server_setup/.bash_aliases index 7676e445e..00d569938 100644 --- a/archive/server_setup/.bash_aliases +++ b/archive/server_setup/.bash_aliases @@ -16,8 +16,8 @@ alias stats='if [ "$DATADIR" == "" ] ; then echo select env first; else tail $DA alias matchstatus='if [ "$SRC" == "" ] ; then echo select env first; else grep "Running" $SRC/logs/matchJob.log && grep TRAJ $SRC/logs/matchJob.log | grep SOLVING && echo -n "Completed " && grep Observations: $SRC/logs/matchJob.log | wc -l && grep "nightlyJob" $SRC/logs/nightlyJob.log ; fi ' alias spacecalc='ls -1 | egrep -v "ukmon-shared" | while read i ; do \du -s $i ; done | sort -n' -alias startcalc='~/prod/utils/stopstart-calcengine.sh start' -alias stopcalc='~/prod/utils/stopstart-calcengine.sh stop' +alias startcalc='$SRC/utils/stopstart-calcengine.sh start' +alias stopcalc='$SRC/utils/stopstart-calcengine.sh stop' function dev { source ~/dev/config.ini >/dev/null @@ -37,9 +37,9 @@ function calcserver { ipaddr=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared) isrunning=$(echo $sts | cut -d " " -f 1) if [ $isrunning -ne 16 ] ; then - /home/ec2-user/prod/utils/stopstart-calcengine.sh start + $HOME/prod/utils/stopstart-calcengine.sh start echo "starting server on ${ipaddr}... waiting 10s..." sleep 10 fi - ssh -i ~/.ssh/markskey.pem ec2-user@$ipaddr + ssh -i ~/.ssh/markskey.pem $SERVERUSERID@$ipaddr } diff --git a/archive/server_setup/migratingBatchServer.md b/archive/server_setup/migratingBatchServer.md index cd413ef7e..584a4d90b 100644 --- a/archive/server_setup/migratingBatchServer.md +++ b/archive/server_setup/migratingBatchServer.md @@ -1,4 +1,21 @@ -# how to move accounts to a new server +# Replacing the UKMON helper server +The ukmon helper server provides two services. +* camera authentication and key management +* batch processing + +# how to move batch processing +The best approach is to reinstall all the code using the ansible deployment scripts. +Both dev and prod envs should be created, though arguably the dev env should be on a separate server. +Additionally, WMPL will need to be reinstalled. +After doing this, the additional UKMON python requirements can be added. + +The contents of ~/prod/data should be replicated to the new server. +The contents of ~/keymgmt should be replicated to the new server. + +Finally, double check that all required SSM variables exist in the account holding the server. These +are used to build the config file, but originally were only in Mark McIntyre's account. + +## how to move accounts to a new server The basic process is to extract the user accounts from the current server along with group and password info, then import it back in on the new server. Its important to avoid accidentally overwriting system or otherwise-existing accounts on the new server. @@ -8,7 +25,7 @@ new accounts starting at 1000 and working both upwards and downwards! Steps in brief. NB must all be done as root, of course. -## On the old server +### On the old server ``` bash mkdir /root/move/ export UGIDLIMIT=500 @@ -24,7 +41,7 @@ scp /root/move/* newserver:/tmp ``` -## On the new server +### On the new server In summary: backup the existing files, remove any accounts from the .mig files that are already present in the target, then append the filtered data. @@ -33,7 +50,7 @@ When comparing groups, remember new ids will get added to the sftp group. ``` bash mkdir -p /root/move/bkp -mv /tmp/*.mig /tmp/arsftp.tar.gz /root/move +mv /tmp/*.mig /tmp/varsftp.tar.gz /root/move cp /etc/passwd /etc/group /etc/shadow /etc/gshadow /root/move/bkp cd / diff --git a/archive/terraform/mjmm/dev_ssm_parameters.tf b/archive/terraform/mjmm/dev_ssm_parameters.tf index e622d5475..a9216604f 100644 --- a/archive/terraform/mjmm/dev_ssm_parameters.tf +++ b/archive/terraform/mjmm/dev_ssm_parameters.tf @@ -32,7 +32,7 @@ resource "aws_ssm_parameter" "dev_livebucket" { resource "aws_ssm_parameter" "dev_siteurl" { name = "dev_siteurl" type = "String" - value = "http://mjmm-ukmonarchive.co.uk.s3-website.eu-west-2.amazonaws.com" + value = "https://www.ukmeteors.co.uk/dummy/" tags = { "billingtag" = "ukmon" } @@ -50,7 +50,16 @@ resource "aws_ssm_parameter" "dev_envname" { resource "aws_ssm_parameter" "dev_calcinstance" { name = "dev_calcinstance" type = "String" - value = "i-08cd1d5f6e1056f6b" #"i-0da38ed8aea1a1d85" + value = "i-0ab47af23705beb31" + tags = { + "billingtag" = "ukmon" + } +} + +resource "aws_ssm_parameter" "dev_calcuser" { + name = "dev_calcuser" + type = "String" + value = "ubuntu" tags = { "billingtag" = "ukmon" } diff --git a/archive/terraform/mjmm/iam.tf b/archive/terraform/mjmm/iam.tf index 10e44de3a..1da045ac5 100644 --- a/archive/terraform/mjmm/iam.tf +++ b/archive/terraform/mjmm/iam.tf @@ -198,7 +198,7 @@ resource "aws_iam_role_policy" "stsAssumeLambda" { Statement = { Action = "sts:AssumeRole" Effect = "Allow" - Resource = "arn:aws:iam::183798037734:policy/s3PolicyForRadio" + Resource = "arn:aws:iam::183798037734:role/s3AccessForRadio" } Version = "2012-10-17" } diff --git a/archive/terraform/mjmm/s3.tf b/archive/terraform/mjmm/s3.tf index 740ba4cc6..5a05aef5a 100644 --- a/archive/terraform/mjmm/s3.tf +++ b/archive/terraform/mjmm/s3.tf @@ -1,5 +1,6 @@ # Copyright (C) 2018-2023 Mark McIntyre ######################################################################## +/* resource "aws_s3_bucket" "mjmm-ukmonarchive-co-uk" { bucket = "mjmm-ukmonarchive.co.uk" tags = { @@ -52,8 +53,9 @@ data "aws_iam_policy_document" "websiteacesspolicy" { resources = ["${aws_s3_bucket.mjmm-ukmonarchive-co-uk.arn}/*"] } } - +*/ ######################################################################## +/* resource "aws_s3_bucket" "mjmm-ukmon-shared" { bucket = "mjmm-ukmon-shared" timeouts {} @@ -113,8 +115,9 @@ resource "aws_s3_bucket_versioning" "shared_versioning" { status = "Suspended" } } - +*/ ######################################################################## +/* resource "aws_s3_bucket" "mjmm-ukmon-live" { bucket = "mjmm-ukmon-live" tags = { @@ -175,4 +178,4 @@ resource "aws_s3_bucket_versioning" "live_versioning" { status = "Suspended" } } - +*/ diff --git a/archive/terraform/mjmm/ssm_parameters.tf b/archive/terraform/mjmm/ssm_parameters.tf index 683a1367c..73dff36bc 100644 --- a/archive/terraform/mjmm/ssm_parameters.tf +++ b/archive/terraform/mjmm/ssm_parameters.tf @@ -50,7 +50,18 @@ resource "aws_ssm_parameter" "prod_envname" { resource "aws_ssm_parameter" "prod_calcinstance" { name = "prod_calcinstance" type = "String" - value = "i-04cd701c3cfc980f5" # "i-08cd1d5f6e1056f6b" + value = "i-0ab47af23705beb31" + #value = "i-04cd701c3cfc980f5" + tags = { + "billingtag" = "ukmon" + } +} + +resource "aws_ssm_parameter" "prod_calcuser" { + name = "prod_calcuser" + type = "String" + value = "ubuntu" + #value = "ec2-user" # "ubuntu" tags = { "billingtag" = "ukmon" } diff --git a/archive/terraform/mjmm/ec2.tf b/archive/terraform/mjmm/ukmonhelper_ec2.tf similarity index 100% rename from archive/terraform/mjmm/ec2.tf rename to archive/terraform/mjmm/ukmonhelper_ec2.tf diff --git a/archive/terraform/mjmm/variables.tf b/archive/terraform/mjmm/variables.tf index f1d68fb92..bad991dce 100644 --- a/archive/terraform/mjmm/variables.tf +++ b/archive/terraform/mjmm/variables.tf @@ -11,8 +11,8 @@ variable "webbucket" {default = "ukmda-website"} variable "sharedbucket" {default = "ukmda-shared"} variable "livebucket" {default = "ukmda-live"} -variable "dev_sharedbucket" { default = "mjmm-ukmon-shared" } -variable "dev_webbucket" { default = "mjmm-ukmonarchive.co.uk" } -variable "dev_livebucket" { default = "mjmm-ukmon-live" } +variable "dev_webbucket" { default = "ukmda-website" } +variable "dev_sharedbucket" { default = "ukmda-shared" } +variable "dev_livebucket" { default = "ukmda-live" } variable "vpc_id" { default = "vpc-a19015c8" } diff --git a/archive/terraform/ukmda/ec2.tf b/archive/terraform/ukmda/calcserver.tf similarity index 98% rename from archive/terraform/ukmda/ec2.tf rename to archive/terraform/ukmda/calcserver.tf index af11b8276..fe1248458 100644 --- a/archive/terraform/ukmda/ec2.tf +++ b/archive/terraform/ukmda/calcserver.tf @@ -47,7 +47,7 @@ resource "aws_network_interface" "calcserver_if" { resource "aws_instance" "ubuntu_calc_server" { ami = "ami-0bdf149a42243bde8" - instance_type = "c6g.4xlarge" + instance_type = "c8g.2xlarge" iam_instance_profile = aws_iam_instance_profile.calcserverrole.name key_name = aws_key_pair.marks_key.key_name tags = { @@ -61,7 +61,7 @@ resource "aws_instance" "ubuntu_calc_server" { "Name" = "calcengine_ub" "billingtag" = "ukmda" } - volume_size = 100 + volume_size = 120 } primary_network_interface { network_interface_id = aws_network_interface.ubuntu_calcserver_if.id diff --git a/archive/terraform/ukmda/ecs_vpc.tf b/archive/terraform/ukmda/ecs_vpc.tf index 6c7243599..3f88c92a9 100644 --- a/archive/terraform/ukmda/ecs_vpc.tf +++ b/archive/terraform/ukmda/ecs_vpc.tf @@ -5,10 +5,6 @@ #data used by the code in several places data "aws_region" "current" {} -# some variables for the cluster and task defns -variable "ecsloggroup" { default = "/ecs/trajcont" } -variable "containername" { default = "trajcont" } - # create a VPC for the cluster resource "aws_vpc" "ecs_vpc" { cidr_block = "172.128.0.0/16" diff --git a/archive/terraform/ukmda/ecscluster-test.tf b/archive/terraform/ukmda/ecscluster-test.tf new file mode 100644 index 000000000..44c532659 --- /dev/null +++ b/archive/terraform/ukmda/ecscluster-test.tf @@ -0,0 +1,73 @@ +# terraform to create ECS cluster +# Copyright (C) 2018-2023 Mark McIntyre + +variable "ecsloggrouptest" { default = "/ecs/trajcontest" } +variable "containernametest" { default = "trajconttest" } + +# create a cluster +resource "aws_ecs_cluster" "trajsolvertest" { + name = "trajsolvertest" + tags = { + "billingtag" = "ukmda" + } +} + +# declare the capacity provider type, in this case FARGATE +resource "aws_ecs_cluster_capacity_providers" "trajsolvertest_cap" { + cluster_name = aws_ecs_cluster.trajsolvertest.name + capacity_providers = ["FARGATE", "FARGATE_SPOT"] +} + +data "template_file" "tasktest_json_template" { + template = file("files/trajsolver/trajsolver_container.json") + vars = { + acctid = data.aws_caller_identity.current.account_id + regionid = "eu-west-2" + repoid = "calcengine/trajsolvertest" + contname = var.containernametest + loggrp = var.ecsloggrouptest + } +} + +# define the task +# the definition of the container it runs are in the webapp.json file +resource "aws_ecs_task_definition" "trajsolvertest_task" { + family = "trajsolvertest" + container_definitions = data.template_file.tasktest_json_template.rendered + cpu = 4096 + memory = 8192 + network_mode = "awsvpc" + tags = { + billingtag = "ukmda" + } + requires_compatibilities = ["FARGATE"] + execution_role_arn = aws_iam_role.ecstaskrole.arn + task_role_arn = aws_iam_role.ecstaskrole.arn + runtime_platform { + operating_system_family = "LINUX" + } +} + +# create a local file containing the clustername and a few other details +# +resource "null_resource" "createECSdetailstest" { + triggers = { + clusname = join(",", tolist([aws_ecs_cluster.trajsolvertest.name, + aws_subnet.ecs_subnet.id, + aws_security_group.ecssecgrp.id, + aws_iam_role.ecstaskrole.arn, var.ecsloggrouptest, + var.containernametest])) + } + provisioner "local-exec" { + command = "echo $env:CLUSNAME $env:SECGRP $env:SUBNET $env:IAMROLE $env:LOGGRP $env:CONTNAME > ../../ukmon_pylib/traj/clusdetailstest-ukmda.txt" + interpreter = ["pwsh.exe", "-command"] + environment = { + CLUSNAME = "${aws_ecs_cluster.trajsolvertest.name}" + SECGRP = "${aws_security_group.ecssecgrp.id}" + SUBNET = "${aws_subnet.ecs_subnet.id}" + IAMROLE = "${aws_iam_role.ecstaskrole.arn}" + LOGGRP = "${var.ecsloggrouptest}" + CONTNAME = "${var.containernametest}" + } + } +} diff --git a/archive/terraform/ukmda/ecscluster.tf b/archive/terraform/ukmda/ecscluster.tf index 974d4c596..2dbb16566 100644 --- a/archive/terraform/ukmda/ecscluster.tf +++ b/archive/terraform/ukmda/ecscluster.tf @@ -1,6 +1,10 @@ # terraform to create ECS cluster # Copyright (C) 2018-2023 Mark McIntyre +# some variables for the cluster and task defns +variable "ecsloggroup" { default = "/ecs/trajcont" } +variable "containername" { default = "trajcont" } + # create a cluster resource "aws_ecs_cluster" "trajsolver" { name = "trajsolver" diff --git a/archive/ukmon_pylib/analysis/gatherDetectionData.py b/archive/ukmon_pylib/analysis/gatherDetectionData.py index b9031167b..d7e45b9b8 100644 --- a/archive/ukmon_pylib/analysis/gatherDetectionData.py +++ b/archive/ukmon_pylib/analysis/gatherDetectionData.py @@ -12,7 +12,7 @@ def getUncalibratedImageList(dtstr=None): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') flines = open(logfile, 'r').readlines() uncal = [f for f in flines if 'not recalibrated' in f] @@ -37,7 +37,7 @@ def getRawData(idlist, outpth): def updateSingleDB(yr, consdt): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) sngls = pd.read_csv(os.path.join(datadir, 'single', f'singles-{yr}.csv')) consdf = pd.read_csv(os.path.join(datadir, 'single', 'used', f'consumed_{consdt}.txt'), names=['Filename']) conslist = list(consdf.Filename) diff --git a/archive/ukmon_pylib/analysis/showerAnalysis.py b/archive/ukmon_pylib/analysis/showerAnalysis.py index 29636148f..02551d78a 100644 --- a/archive/ukmon_pylib/analysis/showerAnalysis.py +++ b/archive/ukmon_pylib/analysis/showerAnalysis.py @@ -467,7 +467,7 @@ def magDistributionVis(dta, shwrname, outdir, binwidth=0.2): def showerAnalysis(shwr, dtstr): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) # set up paths, files etc # check if month was passed in diff --git a/archive/ukmon_pylib/analysis/stationAnalysis.py b/archive/ukmon_pylib/analysis/stationAnalysis.py index 519bf4790..fb7a2e9d5 100644 --- a/archive/ukmon_pylib/analysis/stationAnalysis.py +++ b/archive/ukmon_pylib/analysis/stationAnalysis.py @@ -140,7 +140,7 @@ def reportOneSite(yr, mth, loc, sngl, mful, idlist, outdir): when = f'{mth:02d}-{yr}' idxfile = os.path.join(outdir,'index.html') - templatedir=os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') + templatedir=os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) shutil.copyfile(os.path.join(templatedir, 'header.html'), idxfile) outf = open(idxfile, 'a+') @@ -291,7 +291,7 @@ def pushToWebsite(fuloutdir, outdir, websitebucket): matchcols = ['_Y_ut','_M_ut','_mag','_mjd','mjd','_stream','orbname','stations'] snglcols = ['ID','Shwr','Dtstamp','Ver', 'M', 'Y'] - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) camlist = loadLocationDetails() sngl = pd.read_parquet(os.path.join(datadir, 'single', f'singles-{yr}.parquet.snap'), columns=snglcols) diff --git a/archive/ukmon_pylib/analysis/summaryAnalysis.py b/archive/ukmon_pylib/analysis/summaryAnalysis.py index 1aa6117a4..5df930768 100644 --- a/archive/ukmon_pylib/analysis/summaryAnalysis.py +++ b/archive/ukmon_pylib/analysis/summaryAnalysis.py @@ -13,7 +13,7 @@ def showerSummaryByPeriod(dtstr): yr = int(dtstr[:4]) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) cols = ['_stream','_M_ut'] filt = None matchfile = os.path.join(datadir, 'matched', 'matches-full-{}.parquet.snap'.format(yr)) @@ -34,7 +34,7 @@ def showerSummaryByPeriod(dtstr): def createSummWebpage(dtstr, outdir=None): yr = dtstr[:4] mth = None - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if len(dtstr) > 4: mth = dtstr[4:6] if outdir is None: @@ -43,7 +43,7 @@ def createSummWebpage(dtstr, outdir=None): if outdir is None: outdir = os.path.join(datadir, 'reports', yr, 'showers') os.makedirs(outdir, exist_ok=True) - templatedir=os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') + templatedir=os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) idxfile = os.path.join(outdir, 'index.html') shutil.copyfile(os.path.join(templatedir, 'header.html'), idxfile) with open(idxfile, 'a+') as outf: @@ -73,7 +73,7 @@ def createSummWebpage(dtstr, outdir=None): def createSummJS(dtstr, shwrdata, maxlines=None): yr = dtstr[:4] mth = None - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if len(dtstr) > 4: mth = dtstr[4:6] outdir = os.path.join(datadir, 'reports', yr, 'showers', mth) diff --git a/archive/ukmon_pylib/converters/gmnTxtToPandas.py b/archive/ukmon_pylib/converters/gmnTxtToPandas.py index d274779c1..4e759b6b9 100644 --- a/archive/ukmon_pylib/converters/gmnTxtToPandas.py +++ b/archive/ukmon_pylib/converters/gmnTxtToPandas.py @@ -17,7 +17,7 @@ 'Lat1','Lat1sd','Lon1','Lon1sd','H1','H1sd','Lat2','Lat2sd','Lon2','Lon2sd','H2','H2sd', 'Dur','Amag','PkHt','F1','mass','Qc','MedianFitErr','BegIn','EndIn','NumStat','stats'] -datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') +datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) dirpath = os.path.join(datadir, 'gmndata') diff --git a/archive/ukmon_pylib/maintenance/dataMaintenance.py b/archive/ukmon_pylib/maintenance/dataMaintenance.py index 96d24dab9..911abcb3e 100644 --- a/archive/ukmon_pylib/maintenance/dataMaintenance.py +++ b/archive/ukmon_pylib/maintenance/dataMaintenance.py @@ -9,6 +9,13 @@ import paramiko from scp import SCPClient from time import sleep +import pandas as pd +import datetime + +import pymysql.cursors + +from wmpl.Utils.TrajConversions import datetime2JD +from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase def findInputDataByMonth(yyyymm, archbucket, outdir): @@ -97,6 +104,95 @@ def deleteFromCalcServerByMonth(outfname): return +def removeDeletedTraj(csvfile): + """ + Remove deleted trajectories from the consolidated match CSV and Parquet files + and from the search index. + """ + + csvdata = open(csvfile, 'r').readlines() + if 'search' in csvfile: + ts_end = float(csvdata[-1].split(',')[0]) + dt_end = datetime.datetime.fromtimestamp(ts_end, tz=datetime.timezone.utc) + jdt_end = datetime2JD(dt_end) + else: + jdt_end = float(csvdata[-1].split(',')[3]) + 2400000.5 + + jdt_beg =jdt_end - 21 + + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + masterdb_path = os.path.join(datadir, 'distrib') + masterdb = TrajectoryDatabase(db_path=masterdb_path) + cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') + deltrajs = cur.fetchall() + masterdb.closeTrajDatabase() + + i=0 + for traj in deltrajs: + fldr = os.path.basename(os.path.dirname(traj[0])) + match = [tr for tr in csvdata if fldr in tr] + if len(match) > 0: + for thismtch in match: + print(f'removing {fldr}') + idx = csvdata.index(thismtch) + _ = csvdata.pop(idx) + i += 1 + print(f'removed {i} trajectories') + + open(csvfile, 'w').writelines(csvdata) + + if 'search' not in csvfile: + df = pd.read_csv(csvfile, skipinitialspace=True) + df = df.drop_duplicates(subset=['_mjd','_sol','_ID1','_ra_o','_dc_o','_amag','_ra_t','_dc_t']) + df.to_csv(csvfile, index=False) + + return + + +def getSqlLoginDetails(): + # retrieve password and host from SSM. This allows me to manage them from Terraform + ssm = boto3.client('ssm', region_name='eu-west-1') + res = ssm.get_parameter(Name='prod_dbpw', WithDecryption=True) + password = res['Parameter']['Value'] + res = ssm.get_parameter(Name='prod_dbhost') + host = res['Parameter']['Value'] + # should really do these too but they won't change often if at all + user = 'batch' + db = 'ukmon' + return host, user, password, db + + +def removeDelTrajFromDb(): + dt_end = datetime.datetime.now(tz=datetime.timezone.utc) + jdt_end = datetime2JD(dt_end) + jdt_beg =jdt_end - 7 + + # get list of deleted trajectories + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + masterdb_path = os.path.join(datadir, 'distrib') + masterdb = TrajectoryDatabase(db_path=masterdb_path) + cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') + deltrajs = cur.fetchall() + masterdb.closeTrajDatabase() + + # get connection to the SQL database + host, user, passwd, db = getSqlLoginDetails() + connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) + + count = 0 + for traj in deltrajs: + fldr = os.path.basename(os.path.dirname(traj[0])) + sqlstr = f"delete from matches where orbname like '{fldr}%'" + with connection.cursor() as cursor: + cursor.execute(sqlstr) + result = cursor.fetchall() + count += len(result) + connection.commit() + connection.close() + print(f'cleaned up {count} trajectories') + return + + if __name__ == '__main__': arg_parser = argparse.ArgumentParser(description="Find and clear down historical raw data.") diff --git a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py index f1b477283..2dadf8cb6 100644 --- a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py +++ b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py @@ -25,8 +25,8 @@ csvdir = 'f:/videos/meteorcam/usermgmt/csvkeys' lnxdir='/home/ec2-user/keymgmt/csvkeys' else: - csvdir='/home/ec2-user/keymgmt/csvkeys' - lnxdir='/home/ec2-user/keymgmt/csvkeys' + csvdir=os.path.expanduser('~/keymgmt/csvkeys') + csvdir=os.path.expanduser('~/keymgmt/csvkeys') def checkIfOnGMN(camid): diff --git a/archive/ukmon_pylib/maintenance/manageTraj.py b/archive/ukmon_pylib/maintenance/manageTraj.py index fae24d735..0a684babd 100644 --- a/archive/ukmon_pylib/maintenance/manageTraj.py +++ b/archive/ukmon_pylib/maintenance/manageTraj.py @@ -8,7 +8,7 @@ # delete an orbit from the database def deleteDuplicate(trajname, jd=None): - datadir = os.getenv('DATADIR', default='/home/pi/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) yr=trajname[:4] if int(yr) > 2021: fname = os.path.join(datadir, 'matched','matches-full-{}.parquet.snap'.format(yr)) diff --git a/archive/ukmon_pylib/maintenance/plotStationsOnMap.py b/archive/ukmon_pylib/maintenance/plotStationsOnMap.py index 28d7b9c02..59b37d3b8 100644 --- a/archive/ukmon_pylib/maintenance/plotStationsOnMap.py +++ b/archive/ukmon_pylib/maintenance/plotStationsOnMap.py @@ -107,7 +107,7 @@ def plotMap(srcpath, intersect=False, plotlabels=False): # ax.add_feature(cfeature.LAKES, edgecolor='black') # ax.add_feature(cfeature.RIVERS) ax.gridlines() - pylib=os.getenv('PYLIB', default='/home/ec2-user/prod/ukmon_pylib') + pylib=os.getenv('PYLIB', default=os.path.expanduser('~/prod/ukmon_pylib')) os.environ['CARTOPY_USER_BACKGROUNDS'] = os.path.join(pylib, 'share','maps') ax.background_img(name='BM', resolution='high', extent= [minn, maxn, mina, maxa]) @@ -143,7 +143,7 @@ def plotMap(srcpath, intersect=False, plotlabels=False): plt.plot(yi, xi, 'bx', markersize=10, linewidth=1) plt.tight_layout() - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if datadir is None: datadir='.' plt.savefig(os.path.join(datadir, 'stations.png'), dpi=200) @@ -158,7 +158,7 @@ def plotMap(srcpath, intersect=False, plotlabels=False): plotlabels = False else: plotlabels = True - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) srcfile = os.path.join(datadir, 'admin', 'cameraLocs.json') intersect = False if len(sys.argv) > 2: diff --git a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py index feb2a5079..24107d46b 100644 --- a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py +++ b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py @@ -14,7 +14,7 @@ def findOtherBadEvents(): evts = [] - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'dailyreports', 'latest.txt') with open(lastf, 'r') as inf: lis = inf.readlines() @@ -33,7 +33,7 @@ def findOtherBadEvents(): def findFailedEvents(prof='ukmonshared'): #session = boto3.Session(profile_name=prof) logcli = boto3.client('logs', region_name='eu-west-2') - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'orbits', 'lastorbitcheck.txt') if os.path.isfile(lastf): with open(lastf, 'r') as inf: @@ -102,7 +102,7 @@ def rerunFails(fails, prof='ukmonshared'): session=boto3.Session(profile_name=prof) lambd = session.client('lambda', region_name='eu-west-2') - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'orbits', 'lastorbitcheck.txt') with open(lastf, 'w') as outf: rundt = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index 09c961dce..e85ff3d66 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -71,7 +71,7 @@ def findRowCamTimings(stationid, uploaddate, ddb=None): def getDayCamTimings(uploaddate, ddb=None, outfile=None, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if not ddb: ddb = boto3.resource('dynamodb', region_name='eu-west-2') #, endpoint_url="http://thelinux:8000") table = ddb.Table('uploadtimes') @@ -136,9 +136,10 @@ def deleteRowCamTimings(stationid, dtstamp, ddb=None): def backPopulate(stationid): s3bucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] - fldrs = glob.glob1(f'/home/ec2-user/ukmon-shared/matches/RMSCorrelate/{stationid}/', '*') + basepath = os.path.expanduser('~/prod/ukmon-shared/matches/RMSCorrelate') + fldrs = glob.glob1(os.path.join(basepath, stationid), '*') for fldr in fldrs: - s3objects = glob.glob1(f'/home/ec2-user/ukmon-shared/matches/RMSCorrelate/{stationid}/{fldr}/', 'FTPd*') + s3objects = glob.glob1(os.path.join(basepath, stationid. fldr), 'FTPd*') if len(s3objects) > 0: s3obj = s3objects[0] fullobj = f'matches/RMSCorrelate/{stationid}/{fldr}/{s3obj}' @@ -147,7 +148,7 @@ def backPopulate(stationid): if __name__ == '__main__': - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) ddb = boto3.resource('dynamodb', region_name='eu-west-2') if os.path.isfile('/sys/devices/virtual/dmi/id/board_asset_tag'): # crude check for EC2 diff --git a/archive/ukmon_pylib/metrics/costMetrics.py b/archive/ukmon_pylib/metrics/costMetrics.py index 36f83dd2b..9b0236fe6 100644 --- a/archive/ukmon_pylib/metrics/costMetrics.py +++ b/archive/ukmon_pylib/metrics/costMetrics.py @@ -17,7 +17,7 @@ def monthlyCostByService(dtstr, acctid): mth = int(dtstr[4:6]) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) df = pd.read_csv(os.path.join(datadir, 'costs', f'costs-{acctid}-90-days.csv')) if acctid != '183798037734': df = df[(df.Tag.str.contains('ukmda')) | (df.Tag.str.contains('ukmon'))] diff --git a/archive/ukmon_pylib/metrics/getMatchStats.py b/archive/ukmon_pylib/metrics/getMatchStats.py index 8114c4e3a..e1dcd4f96 100644 --- a/archive/ukmon_pylib/metrics/getMatchStats.py +++ b/archive/ukmon_pylib/metrics/getMatchStats.py @@ -5,6 +5,9 @@ from datetime import datetime import sys +import os +import glob +from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase def getDailyObsCounts(logf): @@ -34,27 +37,42 @@ def getDailyObsCounts(logf): def getMatchStats(logf): - with open(logf) as inf: - loglines = inf.readlines() + loglines = open(logf).readlines() + + events = [line.strip() for line in loglines if 'Analysing' in line and 'observations' in line] + if len(events) > 0: + # new-style log + addlines = [x for x in events if ' 0 observations' not in x] + addoffs = -5 + oldstyle = False + else: + addlines = [line.strip() for line in loglines if 'Added' in line and 'observations' in line] + addoffs = 1 + oldstyle = True - addlines = [line.strip() for line in loglines if 'Added' in line and 'observations' in line] - nocallines = [line.strip() for line in loglines if 'Skipping' in line and 'recalibrated' in line] - misdflines = [line.strip() for line in loglines if 'Skipping' in line and 'missing data' in line] - uncal = len(nocallines) - missdf = len(misdflines) + uncal = len([line.strip() for line in loglines if 'Skipping' in line and 'recalibrated' in line]) + missdf = len([line.strip() for line in loglines if 'Skipping' in line and 'missing data' in line]) + added=0 for li in addlines: spls=li.split(' ') - added = added + int(spls[1]) + added += int(spls[addoffs]) beglowr = len([line.strip() for line in loglines if 'Begin height lower than the end height' in line]) badalti = len([line.strip() for line in loglines if 'Meteor heights outside allowed range' in line]) badvelo = len([line.strip() for line in loglines if 'Velocity difference too high' in line]) badangl = len([line.strip() for line in loglines if 'Max convergence angle too small' in line]) - trajs = [line.strip() for line in loglines if 'SAVING' in line and 'CANDIDATES' in line] - spls = trajs[0].split(' ') - trajs = int(spls[1]) + cands = 0 + if oldstyle: + cands = [line.strip() for line in loglines if 'SAVING' in line and 'CANDIDATES' in line] + spls = cands[0].split(' ') + cands = int(spls[1]) + else: + trajlines = [line.strip() for line in loglines if 'Saved' in line and ' 0 candidates' not in line] + for li in trajlines: + spls=li.split(' ') + cands += int(spls[-2]) nonphys = beglowr + badalti + badvelo + badangl tot = added + uncal + missdf @@ -73,9 +91,42 @@ def getMatchStats(logf): d2=datetime.strptime(etim,'%H:%M:%S') cstime = str(d2 - d1) - return tot, added, uncal, missdf, nonphys, trajs, runtime, cstime + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + rundate=os.getenv('rundate') + if not rundate: + rundate = open(os.path.join(datadir, 'rundate.txt'),'r').readline().strip() + dailydbdir = os.path.join(datadir, 'latest','dailydbs') + trajdb = TrajectoryDatabase(dailydbdir, f'{rundate}_trajectories.db') + trajs = len(trajdb.getTrajBasics('.',[0,9999999])) + + return tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime + + +def updateStats(obscount, candcount, trajcount, runtime): + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + rundate=os.getenv('rundate') + if not rundate: + rundate = open(os.path.join(datadir, 'rundate.txt'),'r').readline().strip() + statsdir = os.path.join(datadir, 'dailyreports') + reports = glob.glob(os.path.join(statsdir, f'{rundate}*.txt')) + + # only update stats if the daily report exists already + if len(reports) > 0: + dailyrep = os.path.split(reports[-1])[1] + statsdata = open(os.path.join(statsdir,'stats.txt'), 'r').readlines() + + # remove any existing entry for today + currentstats = [li for li in statsdata if rundate in li] + if len(currentstats) > 0: + statsdata.pop(statsdata.index(currentstats[0])) + # add the new entry and save + statsdata.append(f'{dailyrep} {obscount} {candcount} {trajcount} {runtime}\n') + open(os.path.join(statsdir,'stats.txt'), 'w').writelines(statsdata) + return if __name__ == '__main__': - tot, added, uncal, missdf, nonphys, trajs, runtime, cstime = getMatchStats(sys.argv[1]) - print(tot, added, uncal, missdf, nonphys, trajs, runtime, cstime) + tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime = getMatchStats(sys.argv[1]) + if len(sys.argv) < 3: + updateStats(added, cands, trajs, runtime) + print(tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime) diff --git a/archive/ukmon_pylib/metrics/timingMetrics.py b/archive/ukmon_pylib/metrics/timingMetrics.py index 0f4606da5..d88273d09 100644 --- a/archive/ukmon_pylib/metrics/timingMetrics.py +++ b/archive/ukmon_pylib/metrics/timingMetrics.py @@ -47,7 +47,7 @@ def graphOfData(logf, dtstr): plt.grid(axis='x') plt.gca().invert_yaxis() logname, _ = os.path.splitext(os.path.basename(logf)) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) os.makedirs(os.path.join(datadir, 'batchcharts'), exist_ok=True) plt.savefig(os.path.join(datadir, 'batchcharts',f'./{dtstr}-{logname}.jpg'), dpi=100) plt.close() @@ -130,7 +130,7 @@ def getLogStats(nightlogf, matchlogf, thisdy): if __name__ == '__main__': dtstr = sys.argv[1] - logdir = os.path.join(os.getenv('SRC', default='/home/ec2-user/prod'), 'logs') + logdir = os.path.join(os.getenv('SRC',default=os.path.expanduser('~/prod')), 'logs') nowdt = datetime.datetime.now().strftime('%Y%m%d') if nowdt == dtstr: nightf = os.path.join(logdir, 'nightlyJob.log') diff --git a/archive/ukmon_pylib/reports/CameraDetails.py b/archive/ukmon_pylib/reports/CameraDetails.py index 74ce7ac86..ab3fd50d9 100644 --- a/archive/ukmon_pylib/reports/CameraDetails.py +++ b/archive/ukmon_pylib/reports/CameraDetails.py @@ -13,7 +13,7 @@ def getCamLocDirFov(camid, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data/') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) camdb = json.load(open(os.path.join(datadir, 'admin', 'cameraLocs.json'))) if camid not in camdb.keys(): return False @@ -22,7 +22,7 @@ def getCamLocDirFov(camid, datadir=None): def updateCamLocDirFovDB(datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data/') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) camdb = {} ppfiles = glob.glob(os.path.join(datadir, 'consolidated','platepars', '*.json')) for ppf in ppfiles: @@ -80,7 +80,7 @@ def createCDCsv(targetloc): cd4csv = cd4csv[['site','stationid','oldcode','direction','camtype','dummycode','active']] cd4csv = cd4csv.rename(columns={'stationid':'camid', 'oldcode':'lid', 'direction':'sid'}) cd4csv.sort_values(by=['active','camid'], inplace=True) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data/') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) outfname = os.path.join(datadir, targetloc, 'camera-details.csv') cd4csv.to_csv(outfname, index=False) with open(os.path.join(datadir, 'activecamcount.txt'), 'w') as outf: diff --git a/archive/ukmon_pylib/reports/cameraStatusReport.py b/archive/ukmon_pylib/reports/cameraStatusReport.py index 0872feff7..256ad0765 100644 --- a/archive/ukmon_pylib/reports/cameraStatusReport.py +++ b/archive/ukmon_pylib/reports/cameraStatusReport.py @@ -25,7 +25,7 @@ def getLastUpdateDate(datadir=None, camfname=None, ddb=None): caminfo = caminfo.drop(columns=['direction','oldcode','active','camtype','eMail', 'humanName']) caminfo.rename(columns={'site':'Site'}, inplace=True) if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) tmplist = pd.read_csv(os.path.join(datadir,'reports','camuploadtimes.csv'), index_col=False) fldrlist = pd.merge(tmplist, caminfo, on=['stationid'], how='inner') @@ -54,7 +54,7 @@ def getLastUpdateDate(datadir=None, camfname=None, ddb=None): def createStatusReportJSfile(stati, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) os.makedirs(os.path.join(datadir, 'reports'), exist_ok=True) repfile = os.path.join(datadir, 'reports','camrep.js') with open(repfile, 'w') as outf: diff --git a/archive/ukmon_pylib/reports/createAnnualBarChart.py b/archive/ukmon_pylib/reports/createAnnualBarChart.py index 05aa32aa5..9426f5d6d 100644 --- a/archive/ukmon_pylib/reports/createAnnualBarChart.py +++ b/archive/ukmon_pylib/reports/createAnnualBarChart.py @@ -15,7 +15,7 @@ def createBarChart(datadir=None, yr=None): if yr is None: yr=datetime.datetime.now().year if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) fname = os.path.join(datadir, 'matched', 'matches-full-{}.parquet.snap'.format(yr)) if not os.path.isfile(fname): print('{} missing', fname) diff --git a/archive/ukmon_pylib/reports/createExchangeFiles.py b/archive/ukmon_pylib/reports/createExchangeFiles.py index 820650ab5..e11a5588a 100644 --- a/archive/ukmon_pylib/reports/createExchangeFiles.py +++ b/archive/ukmon_pylib/reports/createExchangeFiles.py @@ -164,7 +164,7 @@ def createCameraFile(datadir): def createAll(targdate=None, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/datas') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if targdate is None: targdate = datetime.datetime.now() createCameraFile(datadir) diff --git a/archive/ukmon_pylib/reports/createSearchableFormat.py b/archive/ukmon_pylib/reports/createSearchableFormat.py index e2df79c2b..4424cc4bf 100644 --- a/archive/ukmon_pylib/reports/createSearchableFormat.py +++ b/archive/ukmon_pylib/reports/createSearchableFormat.py @@ -110,7 +110,7 @@ def convertMatchToSrchable(datadir, year, newonly=True): print('usage: python createSearchableFormat.py year mode outdir') exit(1) else: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) year = sys.argv[1] mode = sys.argv[2] diff --git a/archive/ukmon_pylib/reports/createSummaryTable.py b/archive/ukmon_pylib/reports/createSummaryTable.py index cff8e40ce..a0d861bb1 100644 --- a/archive/ukmon_pylib/reports/createSummaryTable.py +++ b/archive/ukmon_pylib/reports/createSummaryTable.py @@ -18,7 +18,7 @@ def createSummaryTable(curryr=None, datadir=None): if curryr is None: curryr = str(datetime.datetime.now().year) if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) fname = os.path.join(datadir, 'summarytable.js') with open(fname, 'w') as f: f.write('$(function() {\n') diff --git a/archive/ukmon_pylib/reports/dailyReport.py b/archive/ukmon_pylib/reports/dailyReport.py index 8c7852070..a511c4055 100644 --- a/archive/ukmon_pylib/reports/dailyReport.py +++ b/archive/ukmon_pylib/reports/dailyReport.py @@ -115,7 +115,7 @@ def LookForMatchesRMS(doff, dayfile, statsfile): # now send the post to groups.io targeturl='https://archive.ukmeteors.co.uk' - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) recs = open(os.path.join(datadir, 'admin','dailyReportRecips.txt'), 'r').readlines() mailFrom = 'markmcintyre99@googlemail.com' mailRecip = recs[0].strip() diff --git a/archive/ukmon_pylib/reports/extractors.py b/archive/ukmon_pylib/reports/extractors.py index d7fc4bb80..65670b38d 100644 --- a/archive/ukmon_pylib/reports/extractors.py +++ b/archive/ukmon_pylib/reports/extractors.py @@ -19,7 +19,7 @@ def createSplitMatchFile(yr, mth=None, shwr=None, matches=None): shwr (string): optional shower code. If provided, the data will be filtered """ - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if matches is None: infname = os.path.join(datadir, 'matched',f'matches-full-{yr}.parquet.snap') if not os.path.isfile(infname): @@ -58,7 +58,7 @@ def createUFOSingleMonthlyExtract(yr, mth=None, shwr=None, dta=None): """ # print('ufo singles file') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if dta is None: fname = os.path.join(datadir, 'consolidated','M_{}-unified.csv'.format(yr)) if not os.path.isfile(fname): @@ -94,7 +94,7 @@ def createRMSSingleMonthlyExtract(yr, mth=None, shwr=None, dta=None, withshower= """ #print(f'rms singles file, withshower {withshower}') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if dta is None: fname = os.path.join(datadir, 'single','singles-{}.parquet.snap'.format(yr)) if not os.path.isfile(fname): @@ -147,7 +147,7 @@ def extractAllShowersData(ymd): showerlist = sl.getMajorShowers(True, True).strip().split(' ') print(f'processing data for {ymd}') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) infname = os.path.join(datadir, 'matched',f'matches-full-{yr}.parquet.snap') if not os.path.isfile(infname): print(f'unable to open {infname}') diff --git a/archive/ukmon_pylib/reports/findBestMp4s.py b/archive/ukmon_pylib/reports/findBestMp4s.py index a07611ddd..22f533700 100644 --- a/archive/ukmon_pylib/reports/findBestMp4s.py +++ b/archive/ukmon_pylib/reports/findBestMp4s.py @@ -20,7 +20,7 @@ def getBestNMatches(reqdate=None, numtoget=10): tod = reqdate + datetime.timedelta(days=1) yr = reqdate.year - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) mf = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap') # select only the columns we need @@ -81,7 +81,7 @@ def getUrlFromFilename(fname): def getBestNMp4s(yr, mth, numtoget): - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) mf = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap') # select only the columns we need diff --git a/archive/ukmon_pylib/reports/findFireballs.py b/archive/ukmon_pylib/reports/findFireballs.py index 60b7af0c0..d30949fa6 100644 --- a/archive/ukmon_pylib/reports/findFireballs.py +++ b/archive/ukmon_pylib/reports/findFireballs.py @@ -19,7 +19,7 @@ # Manually mark a trajectoriy as a "fireball" # def markAsFireball(trajname, tof=True): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) yr=trajname[:4] if int(yr) > 2021: fname = os.path.join(datadir, 'matched','matches-full-{}.parquet.snap'.format(yr)) @@ -142,7 +142,7 @@ def findFBPre2020(df, mag=-4): def findFireballs(dtval, shwr, minmag=-3.99, matchdataset = None): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) orbdir = 'matches' # check if month was passed in diff --git a/archive/ukmon_pylib/reports/getLivestreamData.py b/archive/ukmon_pylib/reports/getLivestreamData.py index 3db0dfe24..18c15b1d3 100644 --- a/archive/ukmon_pylib/reports/getLivestreamData.py +++ b/archive/ukmon_pylib/reports/getLivestreamData.py @@ -72,7 +72,7 @@ def getLatestLiveFiles(daysback=None, df=None): if __name__ == '__main__': - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) df = getLatestLiveFiles(-30) prvfile = os.path.join(datadir, 'ukmonlive', 'livefeed.csv') if os.path.isfile(prvfile): diff --git a/archive/ukmon_pylib/reports/makeCoverageMap.py b/archive/ukmon_pylib/reports/makeCoverageMap.py index 2bdb89c52..bbb7f0fba 100644 --- a/archive/ukmon_pylib/reports/makeCoverageMap.py +++ b/archive/ukmon_pylib/reports/makeCoverageMap.py @@ -58,8 +58,8 @@ def makeCoverageMap(kmlsource, outdir, showMarker=False, useName=False): def createCoveragePage(): apikey = os.getenv('APIKEY') apikey = decodeApiKey(apikey) - templdir = os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + templdir = os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) with open(os.path.join(templdir, 'coverage-maps.html'), 'r') as inf: lis = inf.readlines() with open(os.path.join(datadir, 'latest','coverage-maps.html'), 'w') as outf: diff --git a/archive/ukmon_pylib/reports/reportActiveShowers.py b/archive/ukmon_pylib/reports/reportActiveShowers.py index 7786eafa6..7a58d91fc 100644 --- a/archive/ukmon_pylib/reports/reportActiveShowers.py +++ b/archive/ukmon_pylib/reports/reportActiveShowers.py @@ -15,7 +15,7 @@ def createShowerIndexPage(dtstr, shwr, shwrname, outdir, datadir): - templdir = os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') + templdir = os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) idxfile = os.path.join(outdir, 'index.html') shutil.copyfile(os.path.join(templdir,'header.html'), idxfile) @@ -175,7 +175,7 @@ def reportActiveShowers(ymd, thisshower=None, thismth=None, includeMinor=False): else: shwrlist = [thisshower] - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) # pltdir=os.path.join(datadir, 'showerplots') dtstr = ymd[:4] if thismth is not None: diff --git a/archive/ukmon_pylib/reports/reportBadCameras.py b/archive/ukmon_pylib/reports/reportBadCameras.py index bdf4f6ef2..cc0938ee8 100644 --- a/archive/ukmon_pylib/reports/reportBadCameras.py +++ b/archive/ukmon_pylib/reports/reportBadCameras.py @@ -19,7 +19,7 @@ print('usage: python reportBadCameras.py daysback') exit(0) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if datadir is None: print('define DATADIR first') exit(1) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index f7eeed01a..c8ffdc040 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -8,14 +8,16 @@ import datetime import numpy import csv -import json import shutil import tempfile import boto3 +import glob from traj.pickleAnalyser import getVMagCodeAndStations from reports.CameraDetails import findSite, loadLocationDetails +from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase, ObservationsDatabase + def processLocalFolder(trajdir, basedir): # load camera details @@ -39,30 +41,84 @@ def processLocalFolder(trajdir, basedir): return outstr -def getTrajPaths(trajdict): - trajpaths=[] - fullnames=[] - for traj in trajdict: - fullnames.append(trajdict[traj]['traj_file_path']) - pth, _ = os.path.split(trajdict[traj]['traj_file_path']) - trajpaths.append(pth) - return trajpaths, fullnames - - -def getListOfNewMatches(dir_path, tfile, prevtfile): - with open(os.path.join(dir_path, tfile), 'r') as inf: - trajs = json.load(inf) - with open(os.path.join(dir_path, prevtfile), 'r') as inf: - ptrajs = json.load(inf) - newtrajs = {k:v for k,v in trajs['trajectories'].items() if k not in ptrajs['trajectories']} - #print(len(newtrajs)) - _, newdirs = getTrajPaths(newtrajs) +def getListOfNewMatches(dir_path, db_path='/tmp', rundate=None): + os.makedirs(db_path, exist_ok=True) + db_name = f'{rundate}_trajectories.db' if rundate else 'trajectories.db' + dailydb = TrajectoryDatabase(db_path=db_path, db_name=db_name, purge_records=True) + flist = glob.glob(os.path.join(dir_path, 'trajectories_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if dailydb.mergeTrajDatabase(fl): + os.remove(fl) + else: + print('error') + + if os.getenv('TESTMODE').lower() == 'true': + trajdir = 'matches/distrib/test' + else: + trajdir = 'matches/RMSCorrelate' + + cur = dailydb.dbhandle.execute('select traj_file_path from trajectories where status=1') + newtrajs = cur.fetchall() + + if len(newtrajs) > 0: + # now get a list of logically-deleted trajs from the current master DB + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + + # get the date range for the new trajectories + cur = dailydb.dbhandle.execute('select min(jdt_ref), max(jdt_ref) from trajectories where status=1') + vals = cur.fetchall() + jdt_beg = float(vals[0][0]) + jdt_end = float(vals[0][1]) + + # retrieve a list of logically-deleted trajectories from within that date range + masterdb_path = os.path.join(datadir, 'distrib') + masterdb = TrajectoryDatabase(db_path=masterdb_path) + cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') + deltrajs = cur.fetchall() + masterdb.closeTrajDatabase() + + # iterate over the delete list and update the daily db and new traj list accordingly + for testtr in deltrajs: + if testtr in newtrajs: + sqlstr = f'update trajectories set status=0 where "traj_file_path={testtr[0]}";' + dailydb.dbhandle.execute(sqlstr) + newtrajs.pop(newtrajs.index(testtr)) + + dailydb.closeTrajDatabase() + + newdirs = [] + for traj in newtrajs: + newdirs.append(os.path.join(trajdir, traj[0])) + return newdirs -def findNewMatches(dir_path, out_path, offset, repdtstr, dbname): - prevdbname = 'prev_' + dbname - newdirs = getListOfNewMatches(dir_path, dbname, prevdbname) +def updatePairedDB(dir_path, db_path='/tmp', rundate=None): + os.makedirs(db_path, exist_ok=True) + db_name = f'{rundate}_observations.db' if rundate else 'observations.db' + obsdb = ObservationsDatabase(db_path=db_path, db_name=db_name, purge_records=True) + flist = glob.glob(os.path.join(dir_path, 'observations_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if obsdb.mergeObsDatabase(fl): + os.remove(fl) + else: + print('error') + + cur = obsdb.dbhandle.execute('select count(*) from paired_obs where status=1') + obscount = cur.fetchall() + + return obscount + + +def findNewMatches(dir_path, out_path, offset, repdtstr): + daily_path = os.path.join(os.path.split(dir_path)[0], 'dailydbs') + newdirs = getListOfNewMatches(dir_path, daily_path, rundate=repdtstr) # load camera details caminfo = loadLocationDetails() caminfo = caminfo[caminfo.active==1] @@ -72,15 +128,15 @@ def findNewMatches(dir_path, out_path, offset, repdtstr, dbname): else: repdt = datetime.datetime.now() - datetime.timedelta(int(offset)) - os.makedirs(os.path.join(out_path, 'dailyreports'), exist_ok=True) + os.makedirs(out_path, exist_ok=True) # create filename. Allow for three reruns in a day - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d.txt')) if os.path.isfile(matchlist) is True: - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d_1.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d_1.txt')) if os.path.isfile(matchlist) is True: - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d_2.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d_2.txt')) if os.path.isfile(matchlist) is True: - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d_3.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d_3.txt')) s3 = boto3.client('s3') srcbucket=os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] @@ -90,10 +146,11 @@ def findNewMatches(dir_path, out_path, offset, repdtstr, dbname): trajdir = trajdir[trajdir.find('matches'):] trajpath, picklename = os.path.split(trajdir) localpick = os.path.join(tmpdir, picklename) - s3.download_file(srcbucket, trajdir, localpick) try: + s3.download_file(srcbucket, trajdir, localpick) bestvmag, shwr, stationids = getVMagCodeAndStations(localpick) except: + print(f'unable to find {trajdir}') bestvmag, shwr, stationids = 0,'',[''] stations=[] for statid in stationids: @@ -127,20 +184,22 @@ def findNewMatches(dir_path, out_path, offset, repdtstr, dbname): outf.write('\n') # finally, create a "latest.txt" as well - latestlist = os.path.join(out_path, 'dailyreports', 'latest.txt') + latestlist = os.path.join(out_path, 'latest.txt') shutil.copy(matchlist, latestlist) return if __name__ == '__main__': repdtstr = None - dbname = None if len(sys.argv) > 4: repdtstr = sys.argv[4] - if len(sys.argv) > 5: - dbname = sys.argv[5] - else: - dbname = 'processed_trajectories.json.bigserver' + + cand_db_dir = sys.argv[1] + daily_db_dir = sys.argv[2] + offset = sys.argv[3] - # arguments dblocation, datadir, days ago, rundate eg 20220524, full path to database - findNewMatches(sys.argv[1], sys.argv[2], sys.argv[3], repdtstr, dbname) + # arguments dblocation, datadir, days ago, rundate eg 20220524 + findNewMatches(cand_db_dir, daily_db_dir, offset, repdtstr) + # update the daily database of paired observations + daily_db_dir = os.path.join(os.path.split(cand_db_dir)[0], 'dailydbs') + updatePairedDB(cand_db_dir, daily_db_dir, repdtstr) diff --git a/archive/ukmon_pylib/requirements.txt b/archive/ukmon_pylib/requirements.txt index aeaba31b9..8b9f56c62 100644 --- a/archive/ukmon_pylib/requirements.txt +++ b/archive/ukmon_pylib/requirements.txt @@ -48,4 +48,5 @@ dynamodb_json pytest pytest-cov requests -paho-mqtt \ No newline at end of file +paho-mqtt +scp \ No newline at end of file diff --git a/archive/ukmon_pylib/traj/clusdetails-ee.txt b/archive/ukmon_pylib/traj/clusdetails-ee.txt deleted file mode 100644 index 1e17ed3f7..000000000 --- a/archive/ukmon_pylib/traj/clusdetails-ee.txt +++ /dev/null @@ -1,6 +0,0 @@ -trajsolver -sg-0d37a3b8ee1a3a1c6 -subnet-0c224d5642fb71023 -arn:aws:iam::822069317839:role/ecsTaskExecutionRole -/ecs/trajcont -trajcont diff --git a/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt b/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt new file mode 100644 index 000000000..d3be6424f --- /dev/null +++ b/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt @@ -0,0 +1,6 @@ +trajsolvertest +sg-06fb7bbee3b4f5e94 +subnet-019eef41d5eaf419b +arn:aws:iam::183798037734:role/ecsTaskExecutionRole +/ecs/trajcontest +trajconttest diff --git a/archive/ukmon_pylib/traj/consolidateDistTraj.py b/archive/ukmon_pylib/traj/consolidateDistTraj.py index b213f4b45..848feeef1 100644 --- a/archive/ukmon_pylib/traj/consolidateDistTraj.py +++ b/archive/ukmon_pylib/traj/consolidateDistTraj.py @@ -3,123 +3,56 @@ import os import sys import glob -import shutil import datetime -from wmpl.Trajectory.CorrelateRMS import TrajectoryReduced, DatabaseJSON -#from wmpl.Trajectory.CorrelateRMS import MeteorObsRMS, PlateparDummy, MeteorPointRMS # noqa: F401 - - -# support class so i can add any paired obs objects -class dummyMeteorObsRMS(object): - def __init__(self, station_code, id): - self.station_code = station_code - self.id = id - - -# merge a distributed engine DB back into the master DB -def mergeDatabases(srcdir, srcdb, masterpth, masterfile, mastdb = None): - newdb = os.path.join(srcdir, srcdb) - mergedb = DatabaseJSON(newdb) - if mastdb is None: - mastdb = DatabaseJSON(masterfile) - mastdb.db_file_path = masterfile - # merge successful trajectories - for traj in mergedb.trajectories: - traj_obj = TrajectoryReduced(traj, json_dict = mergedb.trajectories[traj].__dict__) - traj_file_path = traj_obj.traj_file_path - traj_file_path = masterpth + '/' + traj_file_path[traj_file_path.find('trajectories'):] - traj_obj.traj_file_path = traj_file_path - mastdb.addTrajectory(traj_file_path, traj_obj=traj_obj) - # merge failed trajectories - for traj in mergedb.failed_trajectories: - traj_obj = TrajectoryReduced(traj, json_dict = mergedb.failed_trajectories[traj].__dict__) - traj_file_path = traj_obj.traj_file_path - traj_file_path = masterpth + '/' + traj_file_path[traj_file_path.find('trajectories'):] - traj_obj.traj_file_path = traj_file_path - mastdb.addTrajectory(traj_file_path, traj_obj=traj_obj, failed=True) - # merge paired obs data - for p in mergedb.paired_obs: - ids = mergedb.paired_obs[p] - for id in ids: - met_obs = dummyMeteorObsRMS(p, id) - mastdb.addPairedObservation(met_obs) - # save the master DB again - mastdb.save() - return mastdb - - -# utility to patch the database to have the right trajectory folder -def patchTrajDB(dbfile, targpath, oldstr='/home/ec2-user/data/RMSCorrelate'): - - with open(dbfile, 'r') as inf: - with open(os.path.join('/tmp/processed_trajectories.json'), 'w') as outf: - for lin in inf: - outf.write(lin.replace(oldstr, targpath)) - shutil.copyfile('/tmp/processed_trajectories.json', dbfile) - return - - -# utility to count how many of each type of observation was in a database -def countDataInDb(path_to_db): - mergedb = DatabaseJSON(path_to_db) - trajs = len(mergedb.trajectories) - failed = len(mergedb.failed_trajectories) - pairs = len(mergedb.paired_obs) - print(path_to_db, trajs, failed, pairs) - return - - -# utility to dump the detection dates (JDs) to a file -def dumpJDs(path_to_db, fname): - mergedb = DatabaseJSON(path_to_db) - with open(fname,'w') as outf: - outf.write('traj\n') - for t in mergedb.trajectories: - outf.write(f'{t}\n') - outf.write('failed\n') - for t in mergedb.failed_trajectories: - outf.write(f'{t}\n') - - -# utility to compare two databases and dump the JDs of any new events -def dumpNewEntries(db1, db2, fname): - olddb = DatabaseJSON(db1) - newdb = DatabaseJSON(db2) - newtraj = set(newdb.trajectories) - set(olddb.trajectories) - newfail = set(newdb.failed_trajectories) - set(olddb.failed_trajectories) - print(len(newtraj), len(newfail)) - with open(fname,'w') as outf: - outf.write('traj\n') - for t in newtraj: - outf.write(f'{t}\n') - outf.write('failed\n') - for t in newfail: - outf.write(f'{t}\n') +from wmpl.Trajectory.CorrelateDB import ObservationsDatabase, TrajectoryDatabase, CandidateDatabase + + +def mergeDatabases(srcdir, dbdir, ignore_missing=False, purge_records=False): + targdb = os.path.join(dbdir, 'observations.db') + if os.path.isfile(targdb) or ignore_missing: + obsdb = ObservationsDatabase(dbdir, purge_records=purge_records) + flist = glob.glob(os.path.join(srcdir, 'observations_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if obsdb.mergeObsDatabase(fl): + os.remove(fl) + obsdb.closeObsDatabase() + + targdb = os.path.join(dbdir, 'trajectories.db') + if os.path.isfile(targdb) or ignore_missing: + trajdb = TrajectoryDatabase(dbdir, purge_records=purge_records) + flist = glob.glob(os.path.join(srcdir, 'trajectories_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if trajdb.mergeTrajDatabase(fl): + os.remove(fl) + trajdb.closeTrajDatabase() + + # should never need to run this part as the candidates are all made in one place + targdb = os.path.join(dbdir, 'candidates.db') + if os.path.isfile(targdb): + canddb = CandidateDatabase(dbdir) + flist = glob.glob(os.path.join(srcdir, 'candidates_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if canddb.mergeCandDatabase(fl): + os.remove(fl) + canddb.closeCandDatabase() + + print('done') if __name__ == '__main__': if len(sys.argv) < 3: - print('usage: consolidateDistTraj folder_containing_srcdbs full_path_to_targb') + print('usage: consolidateDistTraj folder_containing_srcdbs targ_dbdir') exit(0) srcdir = sys.argv[1] - masterdb = sys.argv[2] - if len(sys.argv) > 3: - rundt = sys.argv[3] - else: - rundt = datetime.datetime.now().strftime('%Y%m%d') - # real path to the trajectories as per the master database - masterpth = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' - - flist = glob.glob1(srcdir, f'{rundt}*.json') - flist.sort() - mastdb = None - for fl in flist: - #countDataInDb(os.path.join(srcdir, fl)) - tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') - print(f'{tstamp} processing {fl}') - sys.stdout.flush() - mastdb = mergeDatabases(srcdir, fl, masterpth, masterdb, mastdb) - - print('patching path in mastdb') - patchTrajDB(masterdb, masterpth, oldstr='/home/ec2-user/prod/data/distrib') + dbdir = sys.argv[2] + mergeDatabases(srcdir, dbdir) diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index 8ca5aec51..d3b6c99c7 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -6,21 +6,8 @@ import os import sys import datetime -import json - -# read the task template to determine the paths to write any data to -# -def getTrajsolverPaths(): - templdir,_ = os.path.split(__file__) - with open(os.path.join(templdir, 'taskrunner.json'), 'r') as inf: - taskdets = json.load(inf) - taskenv = taskdets['overrides']['containerOverrides'][0]['environment'] - srcpath = [x for x in taskenv if x['name']=='SRCPATH'][0]['value'] # path from which trajsolver will consume candidates - outpath = [x for x in taskenv if x['name']=='OUTPATH'][0]['value'] # path to which trajsolver will publish trajectories - webpath = [x for x in taskenv if x['name']=='WEBPATH'][0]['value'] # web loc to which trajsolver will publish trajectories - - return srcpath, outpath, webpath +from traj.distributeCandidates import getTrajsolverPaths # make sure the local trajectories folder is synced with the master copy @@ -36,7 +23,7 @@ def refreshTrajectories(outf, matchstart, matchend, trajpath): return -# make sure the master copy is updated with any new locally updated trajectories +# make sure the shared bucket is updated with any new locally updated trajectories # def pushUpdatedTrajectoriesShared(outf, matchstart, matchend, targpath): for d in range(matchend, matchstart+1): @@ -46,15 +33,17 @@ def pushUpdatedTrajectoriesShared(outf, matchstart, matchend, targpath): ymd=thisdt.strftime('%Y%m%d') trajloc=f'trajectories/{yr}/{ym}/{ymd}' outf.write(f'if [ -d {trajloc} ] ; then \n') - outf.write(f'aws s3 sync {trajloc} {targpath}/matches/RMSCorrelate/{trajloc} --exclude "*" --include "*.pickle" --include "*report.txt" --quiet\n') - outf.write(f'aws s3 sync {trajloc}/plots {targpath}/matches/RMSCorrelate/{trajloc}/plots --quiet\n') + outf.write(f'aws s3 sync {trajloc} {targpath}/{trajloc} --exclude "*" --include "*.pickle" --include "*report.txt" --quiet\n') + outf.write(f'if [ -d {trajloc}/plots ] ; then \n') + outf.write(f'aws s3 sync {trajloc}/plots {targpath}/{trajloc}/plots --quiet\n') + outf.write('fi\n') outf.write('fi\n') - outf.write(f'aws s3 sync trajectories/{yr}/plots {targpath}/matches/RMSCorrelate/trajectories/{yr}/plots --quiet\n') - outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {targpath}/matches/RMSCorrelate/trajectories/{yr}/{ym}/plots --quiet\n') + outf.write(f'aws s3 sync trajectories/{yr}/plots {targpath}/trajectories/{yr}/plots --quiet\n') + outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {targpath}/trajectories/{yr}/{ym}/plots --quiet\n') return -# make sure the master copy is updated with any new locally updated trajectories +# make sure the website is updated with any new locally updated trajectories # def pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webpath): for d in range(matchend, matchstart+1): @@ -63,16 +52,18 @@ def pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webpath): ym=thisdt.strftime('%Y%m') ymd=thisdt.strftime('%Y%m%d') trajloc=f'trajectories/{yr}/{ym}/{ymd}' - targloc=f'reports/{yr}/orbits/{ym}/{ymd}' + targloc=f'{yr}/orbits/{ym}/{ymd}' outf.write(f'if [ -d {trajloc} ] ; then \n') outf.write(f'aws s3 sync {trajloc} {webpath}/{targloc} --quiet\n') outf.write('fi\n') outf.write(f'aws s3 sync {trajloc}/plots {webpath}/{targloc}/plots --quiet\n') - outf.write(f'aws s3 sync trajectories/{yr}/plots {webpath}/reports/{yr}/orbits/plots --quiet\n') - outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {webpath}/reports/{yr}/orbits/{ym}/plots --quiet\n') + outf.write(f'aws s3 sync trajectories/{yr}/plots {webpath}/{yr}/orbits/plots --quiet\n') + outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {webpath}/{yr}/orbits/{ym}/plots --quiet\n') return +# Create the density plots showing meteor showers +# def createDensityPlots(outf, calcdir, enddt, includeyear=True): yr = enddt.year ym = enddt.strftime('%Y%m') @@ -98,18 +89,24 @@ def createDensityPlots(outf, calcdir, enddt, includeyear=True): return -def SyncRawData(outf, matchstart, matchend, shbucket, calcdir): +# Sync the raw camera data from shared storage to the local disk +# +def SyncRawData(outf, matchstart, matchend, shbucket): # camera data - no need to replicate it for an historical date - outf.write(f'targdirs=$(aws s3 ls {shbucket}/matches/RMSCorrelate/ | egrep -v "traj|daily|test|plot|proce"|grep PRE | awk \'{{print $2}}\')\n') + outf.write(f'targdirs=$(aws s3 ls {shbucket}/ | egrep -v "traj|daily|test|plot|proce|dbs"|grep PRE | awk \'{{print $2}}\')\n') outf.write('for td in $targdirs ; do\n') for d in range(matchend+1, matchstart+2): thisdt=datetime.datetime.now() + datetime.timedelta(days=-d) trgdy=thisdt.strftime('%Y%m%d') - outf.write(f' aws s3 sync {shbucket}/matches/RMSCorrelate/$td {calcdir}/$td --exclude "*" --include "${{td:0:6}}_{trgdy}*" --quiet\n') + outf.write(f' aws s3 sync {shbucket}/$td ./$td --exclude "*" --include "${{td:0:6}}_{trgdy}*" --quiet\n') outf.write('done\n') return +# +# Get a list of images that are used by the solutions +# + def gatherUsedImageList(outf, matchstart, matchend, shbucket): for d in range(matchend, matchstart+1): thisdt=datetime.datetime.now() + datetime.timedelta(days=-d) @@ -117,18 +114,25 @@ def gatherUsedImageList(outf, matchstart, matchend, shbucket): mth = thisdt.month dy = thisdt.day trajloc = f'trajectories/{yr}/{yr}{mth:02d}/{yr}{mth:02d}{dy:02d}' - out_dir = '/home/ec2-user/data/distrib' + out_dir = '~/data/distrib' outf.write(f'python -c "from traj.pickleAnalyser import getAllImages;getAllImages(\'{trajloc}\', \'{out_dir}\');"\n') - outf.write(f'aws s3 sync {out_dir} {shbucket}/matches/consumed/ --exclude "*" --include "consumed_*.txt"\n') + outf.write(f'aws s3 sync {out_dir} {shbucket}/matches/consumed/ --exclude "*" --include "consumed_*.txt --quiet"\n') outf.write(f'rm {out_dir}/consumed_*.txt\n') return -def createExecConsolSh(matchstart, matchend, execconsolsh, rundt): - shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - webbucket = os.getenv('WEBSITEBUCKET', default='s3://ukmda-website') - calcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' # hardcoded! - _, outpath, _ = getTrajsolverPaths() +# +# Create the bash script that consolidates the generated data and makes sure the website and shared area are updated + +def createExecConsolSh(matchstart, matchend, execconsolsh, istest=''): + + istest = True if istest.lower()=='true' else False + print(f'istest is {istest}') + + srcpath, shbucket, webbucket = getTrajsolverPaths(istest=istest) + csuser = os.getenv('SERVERUSERID', default='ec2-user') + calcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate' + enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend) includeyear = False if enddt.day == 30: @@ -136,16 +140,19 @@ def createExecConsolSh(matchstart, matchend, execconsolsh, rundt): with open(execconsolsh, 'w') as outf: outf.write('#!/bin/bash\n') - outf.write('source /home/ec2-user/venvs/wmpl/bin/activate\n') - outf.write('export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib\n') - # outf.write('export AWS_PROFILE=ukmonshared\n') + outf.write(f'source /home/{csuser}/venvs/wmpl/bin/activate\n') + outf.write(f'export PYTHONPATH=/home/{csuser}/src/WesternMeteorPyLib:/home/{csuser}/src/ukmon_pylib\n') + outf.write(f'cd {calcdir}\n') outf.write('logger -s -t execConsol start\n') + outf.write(f'aws s3 sync {srcpath}/ ~/data/distrib/canddbs/ --exclude "*" --include "*.db" --exclude "dbs/*" --quiet\n') - outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/ ~/data/distrib/processed_trajectories.json {rundt}\n') + outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/canddbs/ {calcdir}/dbs/\n') + + outf.write(f'aws s3 sync {calcdir}/dbs/ {srcpath}/dbs/ --exclude "*" --include "*.db" --quiet\n') outf.write('logger -s -t execConsol syncing any updated trajectories from shared S3\n') - refreshTrajectories(outf, matchstart, matchend, outpath) + refreshTrajectories(outf, matchstart, matchend, shbucket) outf.write('logger -s -t execConsol creating density plots\n') createDensityPlots(outf, calcdir, enddt, includeyear) outf.write('logger -s -t execConsol pushing data back to S3\n') @@ -153,34 +160,39 @@ def createExecConsolSh(matchstart, matchend, execconsolsh, rundt): pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webbucket) outf.write('logger -s -t execConsol getting the image list\n') gatherUsedImageList(outf, matchstart, matchend, shbucket) - #outf.write('unset AWS_PROFILE\n') + outf.write('logger -s -t execConsol done\n') return +# +# Create a bash script to replot the density charts if needed + + +def createExecReplotSh(matchstart, matchend, execconsolsh, istest=''): + + istest = True if istest.lower()=='true' else False -def createExecReplotSh(matchstart, matchend, execconsolsh): shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - calcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' # hardcoded! - _, outpath, _ = getTrajsolverPaths() + csuser = os.getenv('SERVERUSERID', default='ec2-user') + calcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate' + _, outpath, _ = getTrajsolverPaths(istest=istest) + enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend) with open(execconsolsh, 'w') as outf: outf.write('#!/bin/bash\n') - outf.write('source /home/ec2-user/venvs/wmpl/bin/activate\n') - outf.write('export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib\n') - #outf.write('export AWS_PROFILE=ukmonshared\n') + outf.write(f'source /home/{csuser}/venvs/wmpl/bin/activate\n') + outf.write(f'export PYTHONPATH=/home/{csuser}/src/WesternMeteorPyLib:/home/{csuser}/src/ukmon_pylib\n') outf.write(f'cd {calcdir}\n') outf.write('logger -s -t execReplot start\n') refreshTrajectories(outf, matchstart, matchend, outpath) createDensityPlots(outf, calcdir, enddt, False) gatherUsedImageList(outf, matchstart, matchend, shbucket) - #outf.write('unset AWS_PROFILE\n') outf.write('logger -s -t execReplot done\n') return -def createDistribMatchingSh(matchstart, matchend, execmatchingsh): - shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - webbucket = os.getenv('WEBSITEBUCKET', default='s3://ukmda-website') +def createDistribMatchingSh(matchstart, matchend, execmatchingsh, istest=False): + csuser = os.getenv('SERVERUSERID', default='ec2-user') startdt = datetime.datetime.now() + datetime.timedelta(days=-matchstart) enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend) @@ -189,66 +201,63 @@ def createDistribMatchingSh(matchstart, matchend, execmatchingsh): enddtstr = enddt.strftime('%Y%m%d-080000') rundatestr = enddt.strftime('%Y%m%d') - calcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' # hardcoded! + calcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate' + + _, outpath, webpath = getTrajsolverPaths(istest=istest) - srcpath, outpath, _ = getTrajsolverPaths() + srcpath = os.getenv('UKMONSHAREDBUCKET') + '/matches/RMSCorrelate' with open(execmatchingsh, 'w') as outf: outf.write('#!/bin/bash\n') - outf.write('source /home/ec2-user/venvs/wmpl/bin/activate\n') - outf.write('export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib\n') - #outf.write('export AWS_PROFILE=ukmonshared\n') + outf.write(f'source /home/{csuser}/venvs/wmpl/bin/activate\n') + outf.write(f'export PYTHONPATH=/home/{csuser}/src/WesternMeteorPyLib:/home/{csuser}/src/ukmon_pylib\n') outf.write(f'cd {calcdir}\n') outf.write('df -h . \n') # fetch anything thats new from S3 + outf.write('logger -s -t execdistrib syncing latest trajectories from shared S3\n') + # in test, we seeded the test area with prod trajectories in April 2026 refreshTrajectories(outf, matchstart, matchend, outpath) outf.write('logger -s -t execdistrib syncing the raw data from shared S3\n') - outf.write(f'aws s3 cp {srcpath}/processed_trajectories.json {calcdir}/processed_trajectories.json --quiet\n') - outf.write(f'ls -ltr {calcdir}/*.json\n') - - SyncRawData(outf, matchstart, matchend, shbucket, calcdir) + SyncRawData(outf, matchstart, matchend, srcpath) outf.write('logger -s -t execdistrib starting correlator to update existing matches and create candidates\n') - outf.write(f'mkdir -p {calcdir}/candidates\n') - outf.write(f'rm {calcdir}/candidates/*.pickle >/dev/null 2>&1\n') - outf.write(f'time python -m wmpl.Trajectory.CorrelateRMS {calcdir} -i 1 -l -r \"({startdtstr},{enddtstr})\"\n') + outf.write('mkdir -p ./candidates/processed\n') + outf.write('rm ./candidates/*.pickle >/dev/null 2>&1\n') + outf.write(f'time python -m wmpl.Trajectory.CorrelateRMS . --dbdir ./dbs --logdir ./logs --mcmode 4 -l -r \"({startdtstr},{enddtstr})\"\n') # backup the raw candidates in case i need to reprocess some by hand - outf.write(f'mkdir -p {calcdir}/candidates/bkp\n') - outf.write(f'tar cvfz {calcdir}/candidates/bkp/{rundatestr}.tgz {calcdir}/candidates/*.pickle\n') - outf.write(f'find {calcdir}/candidates/bkp/ -name "*.tgz" -mtime +14 -exec rm -f ' + '{} \\;\n') + outf.write(f'tar czf ./candidates/processed/{rundatestr}.tgz ./candidates/*.pickle\n') + outf.write('find ./candidates/processed/ -name "*.tgz" -mtime +14 -exec rm -f ' + '{} \\;\n') + outf.write('find ./logs/ -mtime +28 -exec rm -f ' + '{} \\;\n') outf.write('logger -s -t execdistrib backing up the database to trajdb\n') - outf.write(f'cp {calcdir}/processed_trajectories.json {calcdir}/trajdb/processed_trajectories.json.{rundatestr}\n') - - outf.write('logger -s -t execdistrib Syncing the database back to shared S3\n') - outf.write(f'if [ -s {calcdir}/processed_trajectories.json ] ; then\n') - outf.write(f'aws s3 cp {calcdir}/processed_trajectories.json {srcpath}/processed_trajectories.json --quiet\n') - outf.write('else echo "bad database file" ; fi \n') + outf.write(f'tar cvzf ./trajdb/databases_{rundatestr}.tgz dbs/observations.db dbs/trajectories.db dbs/candidates.db\n') + outf.write('find ./trajdb/ -name "*" -mtime +14 -exec rm -f ' + '{} \\;\n') outf.write('logger -s -t execdistrib distributing candidates and launching containers\n') - outf.write(f'time python -m traj.distributeCandidates {rundatestr} {calcdir}/candidates {srcpath}\n') + outf.write(f'time python -m traj.distributeCandidates {rundatestr} ./candidates {istest}\n') # do this again to fetch todays results outf.write('logger -s -t execdistrib refetch latest trajectories\n') refreshTrajectories(outf, matchstart, matchend, outpath) - - outf.write('logger -s -t execdistrib and sync the back to S3 as well\n') - pushUpdatedTrajectoriesShared(outf, matchstart, matchend, shbucket) - pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webbucket) + + outf.write('logger -s -t execdistrib and sync back to S3 as well\n') + pushUpdatedTrajectoriesShared(outf, matchstart, matchend, outpath) + pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webpath) - #outf.write('unset AWS_PROFILE\n') outf.write('logger -s -t execdistrib done\n') if __name__ == '__main__': if len(sys.argv) < 4: - print('Usage: createDistribMatchingSh day1 day2 outfile') + print('Usage: createDistribMatchingSh day1 day2 outfile optional_istest') exit(1) matchstart = int(sys.argv[1]) matchend = int(sys.argv[2]) outfname = sys.argv[3] + if len(sys.argv) > 4: + istest = True if sys.argv[4].lower()=='true' else False - createDistribMatchingSh(matchstart, matchend, outfname) + createDistribMatchingSh(matchstart, matchend, outfname, istest) diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index 91f185f41..e81d2f1e2 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -3,6 +3,8 @@ # Used when running the correlator in 'distributed' mode. # Pick up new groups of candidate trajectories and distribute them, # then wait for completion and gather the logs. + +# NB this runs on the calculation engine server # import json @@ -16,13 +18,30 @@ import pickle -def getClusterDetails(templdir): - sts = boto3.client('sts') - accid = sts.get_caller_identity()['Account'] - if accid == '183798037734': - clusdetails = os.path.join(templdir, 'clusdetails-mda.txt') +# read the task template to determine the paths to write any data to +# +def getTrajsolverPaths(istest=False): + templdir,_ = os.path.split(__file__) + if istest: + taskjson = 'taskrunner_test.json' + else: + taskjson = 'taskrunner.json' + with open(os.path.join(templdir, taskjson), 'r') as inf: + taskdets = json.load(inf) + taskenv = taskdets['overrides']['containerOverrides'][0]['environment'] + srcpath = [x for x in taskenv if x['name']=='SRCPATH'][0]['value'] # path from which trajsolver will consume candidates + outpath = [x for x in taskenv if x['name']=='OUTPATH'][0]['value'] # path to which trajsolver will publish trajectories + webpath = [x for x in taskenv if x['name']=='WEBPATH'][0]['value'] # web loc to which trajsolver will publish trajectories + + return srcpath, outpath, webpath + + +def getClusterDetails(istest=False): + templdir,_ = os.path.split(__file__) + if istest: + clusdetails = os.path.join(templdir, 'clusdetailstest-ukmda.txt') else: - clusdetails = os.path.join(templdir, 'clusdetails-mm.txt') + clusdetails = os.path.join(templdir, 'clusdetails-mda.txt') with open(clusdetails) as inf: lis = inf.readlines() clusname = lis[0].strip() @@ -34,12 +53,16 @@ def getClusterDetails(templdir): return [clusname, secgrp, subnet, exrolearn, loggrp, contname] -def createTaskTemplate(rundate, buckname, clusdets, spandays=3): +def createTaskTemplate(rundate, buckname, clusdets, spandays=3, istest=False): d1 = (rundate + datetime.timedelta(days = -(spandays-1))).strftime('%Y%m%d') d2 = (rundate + datetime.timedelta(days = 1)).strftime('%Y%m%d') templdir,_ = os.path.split(__file__) - templatefile = os.path.join(templdir, 'taskrunner.json') + if istest: + taskjson = 'taskrunner_test.json' + else: + taskjson = 'taskrunner.json' + templatefile = os.path.join(templdir, taskjson) clusname = clusdets[0] secgrp = clusdets[1] subnet = clusdets[2] @@ -65,16 +88,26 @@ def getDebugStatus(): return False -def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): +def distributeCandidates(rundate, srcdir, maxcount=20, istest=False): + + clusdets = getClusterDetails(istest=istest) + # the target for distribution is the source for the trajsolver + targdir, _, _ = getTrajsolverPaths(istest=istest) clusname = clusdets[0] + targdir = targdir[5:] + outbucket=targdir[:targdir.find('/')] + targdir = targdir[targdir.find('/')+1:] + ecsclient = boto3.client('ecs', region_name='eu-west-2') status = ecsclient.describe_clusters(clusters=[clusname]) if len(status['clusters']) == 0: print('cluster not running!') return False + print(clusdets, targdir) + print(f'Reading from {srcdir}') # obtain a list of picklefiles and sort by name flist = glob.glob1(srcdir, '*.pickle') @@ -85,12 +118,10 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): # work out how many buckets i need numcands = len(flist) + print(f'processing {numcands} candidates') numbucks = int(math.ceil(numcands/maxcount)) # create buckets - targdir = targdir[5:] - outbucket=targdir[:targdir.find('/')] - targdir = targdir[targdir.find('/')+1:] buckroot = os.path.join(targdir, rundate.strftime('%Y%m%d')) taskarns = [None] * numbucks @@ -108,7 +139,7 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): dst = os.path.join(bucknames[i], fli) s3.upload_file(src, outbucket, dst) - taskjson = createTaskTemplate(rundate, bucknames[i], clusdets) + taskjson = createTaskTemplate(rundate, bucknames[i], clusdets, istest=istest) response = ecsclient.run_task(**taskjson) while len(response['tasks']) == 0: @@ -117,7 +148,7 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): taskarn = response['tasks'][0]['taskArn'] taskarns[i] = taskarn jsontempls[i] = taskjson - print(taskarn[51:]) + print(taskarn.split('/')[-1]) if isDbg is True and i == 3: break @@ -143,23 +174,31 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): return True -def monitorProgress(rundate): +def monitorProgress(rundatestr, istest=''): + + istest = False if (istest=='' or istest.lower() == 'false') else True client = boto3.client('ecs', region_name='eu-west-2') s3 = boto3.client('s3') - archbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - templdir,_ = os.path.split(__file__) - clusdets = getClusterDetails(templdir) + clusdets = getClusterDetails(istest=istest) + targdir, _, _ = getTrajsolverPaths(istest=istest) + targdir = targdir[5:] + outbucket=targdir[:targdir.find('/')] + targdir = targdir[targdir.find('/')+1:] # load the buckets, tasks and cluster name from the dump file - rundate = datetime.datetime.strptime(rundate, '%Y%m%d') - picklefile = os.path.join(datadir, 'distrib', rundate.strftime('%Y%m%d') + '.pickle') - rempickle = f"matches/distrib/{rundate.strftime('%Y%m%d')}.pickle" + rundate = datetime.datetime.strptime(rundatestr, '%Y%m%d') + if istest: + picklefile = os.path.join(datadir, 'distrib', 'test', rundatestr + '.pickle') + else: + picklefile = os.path.join(datadir, 'distrib', rundatestr + '.pickle') + rempickle = f"{targdir}/{rundatestr}.pickle" + os.makedirs(os.path.join(datadir, 'distrib', 'test'), exist_ok=True) try: - s3.download_file(archbucket, rempickle, picklefile) + s3.download_file(outbucket, rempickle, picklefile) except: - print('no containers to monitor') + print(f'no containers to monitor in {outbucket}/{rempickle}') return dumpdata = pickle.load(open(picklefile,'rb')) bucknames = dumpdata[0] @@ -190,59 +229,49 @@ def monitorProgress(rundate): thisbuck = bucknames.pop(idx) taskcount -= 1 _, thisbuck = os.path.split(thisbuck) + taskid = tsk["arn"].split('/')[-1] try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print(f'task {tsk["arn"][51:]} completed already') + print(f'{thisbuck}: task {taskid} completed already') + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) + for tsk in sts['tasks']: - print(f'checking {tsk["taskArn"][51:]}') + taskid = tsk["taskArn"].split('/')[-1] idx = taskarns.index(tsk['taskArn']) + _, thisbuck = os.path.split(bucknames[idx]) + print(f'{thisbuck}: checking {taskid} ') #print(tsk['taskArn'], tsk['lastStatus']) if tsk['lastStatus'] == 'STOPPED': if tsk['stopCode'] != 'EssentialContainerExited': # retry the task - thisjson = createTaskTemplate(rundate, bucknames[idx]) + thisjson = createTaskTemplate(rundate, bucknames[idx], clusdets, istest=istest) response = client.run_task(**thisjson) taskarns[idx] = response['tasks'][0]['taskArn'] - print('task restarted') + print(f'{thisbuck}: task restarted') else: thisarn=taskarns.pop(idx) thisbuck = bucknames.pop(idx) taskcount -= 1 _, thisbuck = os.path.split(thisbuck) try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print('task completed') + print(f'{thisbuck}: task completed') # collect the logs from CloudWatch - realfname=None - logdir = os.path.join(datadir, '..', 'logs', 'distrib') - os.makedirs(logdir, exist_ok=True) - tmpfname = os.path.join(logdir, f'{thisarn[51:]}.log') - with open(tmpfname, 'w') as outf: - for events in getLogDetails(loggrp, thisarn[51:], contname): - for evt in events: - evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) - msg = evt['message'] - outf.write(f'{evtdt} {msg}\n') - if msg[:10] == 'processing': - realfname = msg[11:].strip() - - locname = os.path.join(logdir, f'{realfname}.log') - os.rename(tmpfname, locname) - remlog = f'matches/distrib/logs/{realfname}.log' - s3.upload_file(Filename=locname, Bucket=archbucket, Key=remlog) + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) + if len(taskarns) > 99: sts = client.describe_tasks(cluster=clusname, tasks=taskarns[99:199]) for tsk in sts['failures']: @@ -252,87 +281,92 @@ def monitorProgress(rundate): thisbuck = bucknames.pop(idx) taskcount -= 1 _, thisbuck = os.path.split(thisbuck) + taskid = tsk["arn"].split('/')[-1] try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print(f'task {tsk["arn"][51:]} completed already') + print(f'{thisbuck}: task {taskid} completed already') + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) + for tsk in sts['tasks']: - print(f'checking {tsk["taskArn"][51:]}') + taskid = tsk["taskArn"].split('/')[-1] idx = taskarns.index(tsk['taskArn']) + _, thisbuck = os.path.split(bucknames[idx]) + print(f'{thisbuck} : checking {taskid} ') if tsk['lastStatus'] == 'STOPPED': if tsk['stopCode'] != 'EssentialContainerExited': # retry the task - thisjson = createTaskTemplate(rundate, bucknames[idx]) + thisjson = createTaskTemplate(rundate, bucknames[idx], clusdets, istest=istest) response = client.run_task(**thisjson) taskarns[idx] = response['tasks'][0]['taskArn'] - print('task restarted') + print(f'{thisbuck}: task restarted') else: thisarn=taskarns.pop(idx) thisbuck = bucknames.pop(idx) taskcount -= 1 _, thisbuck = os.path.split(thisbuck) try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print('task completed') - - # collect the logs from CloudWatch - realfname=None - logdir = os.path.join(datadir, '..', 'logs', 'distrib') - os.makedirs(logdir, exist_ok=True) - tmpfname = os.path.join(logdir, f'{thisarn[51:]}.log') - with open(tmpfname, 'w') as outf: - for events in getLogDetails(loggrp, thisarn[51:], contname): - for evt in events: - evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) - msg = evt['message'] - outf.write(f'{evtdt} {msg}\n') - if msg[:10] == 'processing': - realfname = msg[11:].strip() - - locname = os.path.join(logdir, f'{realfname}.log') - os.rename(tmpfname, locname) - remlog = f'matches/distrib/logs/{realfname}.log' - s3.upload_file(Filename=locname, Bucket=archbucket, Key=remlog) + print(f'{thisbuck}: task completed') + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) + if taskcount > 0: # wait 60s before checking again - print('sleeping for 60s') - time.sleep(60.0) + print('sleeping for 30s') + time.sleep(30.0) return -def getMissedLogs(picklefile): +def getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir): + # collect the logs from CloudWatch + realfname=None + logdir = os.path.join(datadir, '..', 'logs', 'distrib') + os.makedirs(logdir, exist_ok=True) + tmpfname = os.path.join(logdir, f'{thisarn.split("/")[-1]}.log') + with open(tmpfname, 'w') as outf: + for events in getLogDetails(loggrp, thisarn.split("/")[-1], contname): + for evt in events: + evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) + msg = evt['message'] + outf.write(f'{evtdt} {msg}\n') + if msg[:10] == 'processing': + realfname = msg[11:].strip() + if realfname: + locname = os.path.join(logdir, f'{realfname}.log') + os.rename(tmpfname, locname) + remlog = f'{targdir}/logs/{realfname}.log' + s3.upload_file(Filename=locname, Bucket=outbucket, Key=remlog) + return + + +def getMissedLogs(picklefile, istest=False): + + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) s3 = boto3.client('s3') - archbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] + _, targdir, _ = getTrajsolverPaths(istest=istest) + targdir = targdir[5:] + outbucket=targdir[:targdir.find('/')] + targdir = targdir[targdir.find('/')+1:] + + clusdets = getClusterDetails(istest=istest) + contname = clusdets[5] + dumpdata = pickle.load(open(picklefile,'rb')) taskarns = dumpdata[1] - loggrp = '/ecs/trajcont' - contname = 'trajcont' - logdir = '/home/ec2-user/prod/logs/distrib' + loggrp = '/ecs/trajcont' for thisarn in taskarns: - tmpfname = os.path.join(logdir, f'{thisarn[51:]}.log') - with open(tmpfname, 'w') as outf: - for events in getLogDetails(loggrp, thisarn[51:], contname): - for evt in events: - evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) - msg = evt['message'] - outf.write(f'{evtdt} {msg}\n') - if msg[:10] == 'processing': - realfname = msg[11:].strip() - locname = os.path.join(logdir, f'{realfname}.log') - os.rename(tmpfname, locname) - remlog = f'matches/distrib/logs/{realfname}.log' - s3.upload_file(Filename=locname, Bucket=archbucket, Key=remlog) + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) def getLogDetails(loggrp, thisarn, contname, region_name='eu-west-2'): @@ -341,39 +375,43 @@ def getLogDetails(loggrp, thisarn, contname, region_name='eu-west-2'): """ client = boto3.client('logs', region_name=region_name) + logstreamname = f'ecs/{contname}/{thisarn}' # first request - response = client.get_log_events( - logGroupName=loggrp, - logStreamName=f'ecs/{contname}/{thisarn}', - startFromHead=True) - yield response['events'] - - # second and later - while True: - prev_token = response['nextForwardToken'] + #print(f'looking for {logstreamname} in {loggrp}') + try: response = client.get_log_events( logGroupName=loggrp, - logStreamName=f'ecs/{contname}/{thisarn}', - nextToken=prev_token) - # same token then break - if response['nextForwardToken'] == prev_token: - break + logStreamName=logstreamname, + startFromHead=True) yield response['events'] + # second and later + while True: + prev_token = response['nextForwardToken'] + response = client.get_log_events( + logGroupName=loggrp, + logStreamName=logstreamname, + nextToken=prev_token) + # same token then break + if response['nextForwardToken'] == prev_token: + break + yield response['events'] + except Exception: + dtval = int(datetime.datetime.now().timestamp() * 1000) + msg = f'log for {thisarn} not available' + yield [{'timestamp': dtval, 'message': msg}] + if __name__ == '__main__': - if len(sys.argv) < 4: - rundt = datetime.datetime(2022,4,21) - srcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate/candidates' # hardcoded on calcserver - buck = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - targdir = f'{buck}/matches/distrib' + if len(sys.argv) < 3: + print('usage: python distributeCandidates yyyymmdd ./candidates istest\n') + print(' optional istest if True then test mode used') else: rundt = datetime.datetime.strptime(sys.argv[1], '%Y%m%d') srcdir = sys.argv[2] - targdir = sys.argv[3] - - templdir,_ = os.path.split(__file__) - clusdets = getClusterDetails(templdir) - print(clusdets) - distributeCandidates(rundt, srcdir, targdir, clusdets) + if len(sys.argv)>3: + istest = True if sys.argv[3].lower()=='true' else False + else: + istest = False + distributeCandidates(rundt, srcdir, istest=istest) diff --git a/archive/ukmon_pylib/traj/jsonDbMaintenance.py b/archive/ukmon_pylib/traj/jsonDbMaintenance.py deleted file mode 100644 index 140d9a51e..000000000 --- a/archive/ukmon_pylib/traj/jsonDbMaintenance.py +++ /dev/null @@ -1,112 +0,0 @@ -# small script to archive off old data from the JSON database -import datetime -import os -import sys -from dateutil.relativedelta import relativedelta - -from wmpl.Utils.TrajConversions import datetime2JD, jd2Date -from wmpl.Trajectory.CorrelateRMS import DatabaseJSON - -# Name of json file with the list of processed directories -JSON_DB_NAME = "processed_trajectories.json" - - -def archiveOldRecords(db, db_dir, older_than=3): - """ - Archive off old records to keep the database size down - - Keyword Arguments: - older_than: [int] number of months to keep, default 3 - """ - class DummyMetObs(): - def __init__(self, station, obs_id): - self.station_code = station - self.id = obs_id - - archdate = datetime.datetime.now(datetime.timezone.utc) - relativedelta(months=older_than) - archdate_jd = datetime2JD(archdate) - - arch_db_path = os.path.join(db_dir, 'archive', f'{archdate.strftime("%Y%m")}_{JSON_DB_NAME}') - os.makedirs(os.path.join(db_dir, 'archive'), exist_ok=True) - archdb = DatabaseJSON(arch_db_path) - - for traj in [t for t in db.trajectories if t < archdate_jd]: - if traj < archdate_jd: - archdb.addTrajectory(None, db.trajectories[traj], False) - del db.trajectories[traj] - - for traj in [t for t in db.failed_trajectories if t < archdate_jd]: - if traj < archdate_jd: - archdb.addTrajectory(None, db.failed_trajectories[traj], True) - del db.failed_trajectories[traj] - - for station in db.processed_dirs: - arch_processed = [dirname for dirname in db.processed_dirs[station] if - datetime.datetime.strptime(dirname[14:22], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] - for dirname in arch_processed: - archdb.addProcessedDir(station, dirname) - db.processed_dirs[station].remove(dirname) - - for station in db.paired_obs: - arch_processed = [obs_id for obs_id in db.paired_obs[station] if - datetime.datetime.strptime(obs_id[7:15], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] - for obs_id in arch_processed: - archdb.addPairedObservation(DummyMetObs(station, obs_id)) - db.paired_obs[station].remove(obs_id) - - archdb.save() - db.save() - return db - - -def clearDownHistoricArchive(database_path, mthsback=1): - db = DatabaseJSON(database_path) - - if len(list(db.failed_trajectories.keys()))==0 and len(list(db.trajectories.keys()))==0: - print('nothing to do') - return - elif len(list(db.failed_trajectories.keys()))==0: - latest = jd2Date(max(list(db.trajectories.keys())), dt_obj=True) - elif len(list(db.trajectories.keys()))==0: - latest = jd2Date(max(list(db.failed_trajectories.keys())), dt_obj=True) - else: - latest = jd2Date(max(max(list(db.trajectories.keys())), max(list(db.failed_trajectories.keys()))), dt_obj=True) - print('processing', database_path) - archdate = latest - relativedelta(months=mthsback) - archdate = archdate.replace(day=1, hour=12, minute=0, second=0).replace(tzinfo=datetime.timezone.utc) - archdate_jd = datetime2JD(archdate) - - for traj in [t for t in db.trajectories if t < archdate_jd]: - if traj < archdate_jd: - del db.trajectories[traj] - - for traj in [t for t in db.failed_trajectories if t < archdate_jd]: - if traj < archdate_jd: - del db.failed_trajectories[traj] - - for station in db.processed_dirs: - arch_processed = [dirname for dirname in db.processed_dirs[station] if - datetime.datetime.strptime(dirname[14:22], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] - for dirname in arch_processed: - db.processed_dirs[station].remove(dirname) - - for station in db.paired_obs: - arch_processed = [obs_id for obs_id in db.paired_obs[station] if - datetime.datetime.strptime(obs_id[7:15], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] - for obs_id in arch_processed: - db.paired_obs[station].remove(obs_id) - - db.save() - return - - -if __name__ == '__main__': - db_dir = sys.argv[1] - database_path = os.path.join(db_dir, JSON_DB_NAME) - db = DatabaseJSON(database_path) - soonest = jd2Date(min(min(list(db.trajectories.keys())), min(list(db.failed_trajectories.keys()))), dt_obj=True) - nowdt = datetime.datetime.now() - mthsback = int((nowdt - soonest).days/30) - for i in range(mthsback, 2, -1): - print(f'archiving {i} months back') - db = archiveOldRecords(db, db_dir, i) diff --git a/archive/ukmon_pylib/traj/manualBulkReruns.py b/archive/ukmon_pylib/traj/manualBulkReruns.py index d94510575..497310da0 100644 --- a/archive/ukmon_pylib/traj/manualBulkReruns.py +++ b/archive/ukmon_pylib/traj/manualBulkReruns.py @@ -406,7 +406,7 @@ def prepDataForRange(startdt, enddt): # working out what has been used, then collecting the FTPs and platepars, # and finally removing the already-used meteors from the FTPs # - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) sdt = datetime.datetime.strptime(startdt, '%Y%m') edt = datetime.datetime.strptime(enddt, '%Y%m') smth = sdt.month @@ -430,7 +430,7 @@ def prepDataForRange(startdt, enddt): def moveInterestingCands(): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) pickledir = os.path.join(datadir, 'historic', 'candidates') goodcandr = os.path.join(datadir, 'historic', 'goodcands') os.makedirs(goodcandr, exist_ok=True) diff --git a/archive/ukmon_pylib/traj/pickleAnalyser.py b/archive/ukmon_pylib/traj/pickleAnalyser.py index ddfe6fbc6..5afb69442 100644 --- a/archive/ukmon_pylib/traj/pickleAnalyser.py +++ b/archive/ukmon_pylib/traj/pickleAnalyser.py @@ -122,6 +122,8 @@ def getShowerDets(shwr): def createUFOOrbitFile(traj, outdir, amag, mass, shower_obj): + outdir = os.path.expanduser(outdir) + #print('Creating UFO Orbit style output file') orb = traj.orbit if shower_obj is None: @@ -378,6 +380,7 @@ def computeAbsoluteMagnitudes(traj, meteor_list): def draw3Dmap(traj, outdir): + outdir = os.path.expanduser(outdir) lats = [] lons = [] alts = [] @@ -493,6 +496,8 @@ def calcAdditionalValues(traj): def createAdditionalOutput(traj, outdir): + outdir = os.path.expanduser(outdir) + # calculate the values amag, vmag, mass, id, cod, shwrname, orb, shower_obj, lg, bg, vg, _ = calcAdditionalValues(traj) @@ -582,16 +587,18 @@ def getListOfImages(picklename): class TrajQualityParams(object): def __init__(self): - self. min_traj_points = 6 - self. min_qc = 5.0 - self. max_e = 1.5 - self. max_radiant_err = 2.0 - self. max_vg_err = 10.0 - self. max_begin_ht = 160 - self. min_end_ht = 20 - - -def getAllImages(dir_path, out_path): + self.min_traj_points = 6 + self.min_qc = 5.0 + self.max_e = 1.5 + self.max_radiant_err = 2.0 + self.max_vg_err = 10.0 + self.max_vg = 120.0 + self.max_begin_ht = 160 + self.min_end_ht = 20 + + +def getAllImages(dir_path, outdir): + outdir = os.path.expanduser(outdir) traj_quality_params = TrajQualityParams() trajs = loadTrajectoryPickles(dir_path, traj_quality_params, verbose=True) imglist = [] @@ -604,6 +611,6 @@ def getAllImages(dir_path, out_path): except Exception: pass outfname = os.path.split(dir_path)[1] - with open(os.path.join(out_path, f'consumed_{outfname}.txt'), 'w') as outf: + with open(os.path.join(outdir, f'consumed_{outfname}.txt'), 'w') as outf: for img in imglist: outf.write(f'{img}\n') diff --git a/archive/ukmon_pylib/traj/taskrunner.json b/archive/ukmon_pylib/traj/taskrunner.json index 630b61384..be99acf10 100644 --- a/archive/ukmon_pylib/traj/taskrunner.json +++ b/archive/ukmon_pylib/traj/taskrunner.json @@ -29,6 +29,10 @@ { "name": "WEBPATH", "value": "s3://ukmda-website/reports" + }, + { + "name": "MAX_STNS", + "value": "9999" } ] } diff --git a/archive/ukmon_pylib/traj/taskrunner_test.json b/archive/ukmon_pylib/traj/taskrunner_test.json new file mode 100644 index 000000000..d834b6e2f --- /dev/null +++ b/archive/ukmon_pylib/traj/taskrunner_test.json @@ -0,0 +1,48 @@ +{ + "cluster": "trajsolvertest", + "count": 1, + "enableECSManagedTags": true, + "enableExecuteCommand": true, + "group": "trajsolvergrp", + "launchType": "FARGATE", + "networkConfiguration": { + "awsvpcConfiguration": { + "subnets":[], + "securityGroups": [], + "assignPublicIp": "ENABLED" + } + }, + "overrides": { + "containerOverrides": [ + { + "name": "trajconttest", + "command": [], + "environment": [ + { + "name": "SRCPATH", + "value": "s3://ukmda-shared/test/matches/distrib" + }, + { + "name": "OUTPATH", + "value": "s3://ukmda-shared/test/matches/RMSCorrelate" + }, + { + "name": "WEBPATH", + "value": "s3://ukmda-website/dummy" + }, + { + "name": "MAX_STNS", + "value": "9999" + } + ] + } + ], + "executionRoleArn": "", + "taskRoleArn": "" + }, + "platformVersion": "LATEST", + "propagateTags": "TASK_DEFINITION", + "referenceId": "", + "startedBy": "cli", + "taskDefinition": "trajsolvertest" +} diff --git a/archive/utils/cleanupDeletedTrajs.sh b/archive/utils/cleanupDeletedTrajs.sh new file mode 100644 index 000000000..ed5fa0d3f --- /dev/null +++ b/archive/utils/cleanupDeletedTrajs.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# Copyright (C) 2018-2023 Mark McIntyre +# +# this script reads a list of logically-deleted trajectories from the sqlite database +# and moves the corresponding pickle and report data to a backup area +# These trajectories are generally duplicated events + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +source $here/../config.ini >/dev/null 2>&1 +conda activate $HOME/miniconda3/envs/${WMPL_ENV} + +cd ${DATADIR}/distrib + +startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') +jdt_min=$(python -c "from wmpl.Utils.TrajConversions import datetime2JD;import datetime;print(datetime2JD(datetime.datetime.strptime('$startdt', '%Y%m%d-%H%M%S')))") + +echo "checking main storage" +sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do + trajdir=$(dirname $traj) + echo $trajdir + moved=0 + aws s3 ls ${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv ${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir/$fname ${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir/$fname --quiet + moved=1 + done + [ $moved == 1 ] && echo moved $trajdir +done + +echo "checking website" +sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do + trajdir=$(dirname $traj) + yr=${trajdir:13:4} + trajpth=${trajdir:18} + echo $trajdir + webloc=$WEBSITEBUCKET/reports/${yr}/orbits/$trajpth + newloc=${UKMONSHAREDBUCKET}/matches/duplicates/reports/${yr}/orbits/$trajpth + moved=0 + aws s3 ls $webloc/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv $webloc/$fname $newloc/$fname --quiet + moved=1 + done + [ $moved == 1 ] && echo moved $trajdir +done + +echo "checking fullcsv" +sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do + trajdir=$(dirname $traj) + yr=${trajdir:13:4} + trajpth=${trajdir:34:19} + csvname=$(echo $trajpth | sed 's/_/-/g') + echo $csvname + csvloc=${UKMONSHAREDBUCKET}/matches/${yr}/fullcsv + newloc=${UKMONSHAREDBUCKET}/matches/duplicates/csvs/${yr} + moved=0 + aws s3 ls $csvloc/ | awk -F " " '{print $4}' | grep $csvname | while read fname; do + aws s3 mv $csvloc/$fname $newloc/$fname --quiet + moved=1 + done + [ $moved == 1 ] && echo moved $trajdir +done + +echo "checking historic fullcsv just in case" +sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do + trajdir=$(dirname $traj) + yr=${trajdir:13:4} + trajpth=$(basename $trajdir) + csvname=$(echo $trajpth | sed 's/_/-/g') + echo $csvname + csvloc=$DATADIR/orbits/${yr}/fullcsv/processed + newloc=${UKMONSHAREDBUCKET}/matches/duplicates/csvs/${yr} + moved=0 + ls -1 $csvloc/ | grep $csvname | while read fname; do + aws s3 mv $csvloc/$fname $newloc/$fname --quiet + moved=1 + done + [ $moved == 1 ] && echo moved $trajdir + export trajpth +done + +echo "checking consolidated matches" +lasttraj=$(sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | tail -1) +trajdir=$(dirname $lasttraj) +yr=${trajdir:13:4} +trajpth=$(basename $trajdir) +matchcsv=$DATADIR/matched/matches-full-${yr}.csv +if [ $? == 0 ] ; then + python -c "from maintenance.dataMaintenance import removeDeletedTraj;removeDeletedTraj('$matchcsv')" + python -m converters.toParquet $matchcsv + aws s3 sync $DATADIR/matched/ $UKMONSHAREDBUCKET/matches/matched/ --include "*" --exclude "*.snap" --exclude "*.bkp" --exclude "*.gzip" --quiet + aws s3 sync $DATADIR/matched/ $UKMONSHAREDBUCKET/matches/matchedpq/ --exclude "*" --include "*.snap" --exclude "*.bkp" --exclude "*.gzip" --quiet + aws s3 sync $DATADIR/matched/ $WEBSITEBUCKET/browse/parquet/ --exclude "*" --include "*.snap" --exclude "*.bkp" --exclude "*.gzip" --quiet + + srchcsv=$DATADIR/searchidx/${yr}-allevents.csv + python -c "from maintenance.dataMaintenance import removeDeletedTraj;removeDeletedTraj('$srchcsv')" + aws s3 sync $DATADIR/searchidx/ $WEBSITEBUCKET/search/indexes/ --exclude "*" --include "*allevents.csv" --quiet +fi + +export AWS_PROFILE=ukmonshared # needed for mariadb connection details +echo "checking Mariadb Database" +python -c "from maintenance.dataMaintenance import removeDelTrajFromDb;removeDelTrajFromDb()" +unset AWS_PROFILE +done \ No newline at end of file diff --git a/archive/utils/getCostMetrics.sh b/archive/utils/getCostMetrics.sh index 699bb2577..99e7fc8a7 100644 --- a/archive/utils/getCostMetrics.sh +++ b/archive/utils/getCostMetrics.sh @@ -26,4 +26,4 @@ python $PYLIB/metrics/costMetrics.py . eu-west-1 $thismth #export AWS_PROFILE=realukms #python $PYLIB/metrics/costMetrics.py . eu-west-1 $thismth -export AWS_PROFILE= \ No newline at end of file +export AWS_PROFILE= diff --git a/archive/utils/loadMatchCsvMDB.sh b/archive/utils/loadMatchCsvMDB.sh index 94363039c..b2fd845f7 100644 --- a/archive/utils/loadMatchCsvMDB.sh +++ b/archive/utils/loadMatchCsvMDB.sh @@ -1,7 +1,6 @@ #!/bin/bash # Copyright (C) 2018-2023 Mark McIntyre # -# script to clear out old log files and other old data here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 diff --git a/archive/utils/loadSingleCsvMDB.sh b/archive/utils/loadSingleCsvMDB.sh index a3b9c8018..09fdaaaab 100644 --- a/archive/utils/loadSingleCsvMDB.sh +++ b/archive/utils/loadSingleCsvMDB.sh @@ -1,7 +1,6 @@ #!/bin/bash # Copyright (C) 2018-2023 Mark McIntyre # -# script to clear out old log files and other old data here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 diff --git a/archive/utils/makeConfig.sh b/archive/utils/makeConfig.sh index a5687db67..a0aa9d567 100644 --- a/archive/utils/makeConfig.sh +++ b/archive/utils/makeConfig.sh @@ -8,6 +8,8 @@ fi RUNTIME_ENV=$1 envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') +export AWS_PROFILE=default + # read from AWS SSM Parameterset SRC=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_srcdir --query Parameters[0].Value | tr -d '"') SITEURL=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_siteurl --query Parameters[0].Value | tr -d '"') @@ -20,6 +22,8 @@ SERVERINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_ BKPINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_backupinstance --query Parameters[0].Value | tr -d '"') SERVERSSHKEY=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_sshkey --query Parameters[0].Value | tr -d '"') NJLOGGRP=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_batchloggroup --query Parameters[0].Value | tr -d '"') +SERVERUSERID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcuser --query Parameters[0].Value | tr -d '"') +unset AWS_PROFILE # hardcoded PYLIB=$SRC/ukmon_pylib @@ -34,6 +38,15 @@ WMPL_ENV=wmpl APIKEY=$(cat ~/.ssh/gmapsapikey) KMLTEMPLATE=*70km.kml +# variables that can be used in scripts and python files to ensure test data doesnt overwrite live data +if [ "$RUNTIME_ENV" == "DEV" ] ; then + TESTMODE="true" + TESTSUFF="/test" +else + TESTMODE="false" + TESTSUFF="" +fi + # create the config file now=$(date +%Y-%m-%d-%H:%M:%S) CFGFILE=~/${envname}/config.ini @@ -50,6 +63,7 @@ echo "TEMPLATES=${TEMPLATES}" >> ${CFGFILE} echo "RCODEDIR=${RCODEDIR}" >> ${CFGFILE} echo "DATADIR=${DATADIR}" >> ${CFGFILE} echo "SERVERINSTANCEID=${SERVERINSTANCEID}" >> ${CFGFILE} +echo "SERVERUSERID=${SERVERUSERID}" >> ${CFGFILE} echo "BKPINSTANCEID=${BKPINSTANCEID}" >> ${CFGFILE} echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> ${CFGFILE} echo "RMS_ENV=${RMS_ENV}" >> ${CFGFILE} @@ -62,6 +76,8 @@ echo "SERVERSSHKEY=${SERVERSSHKEY}" >> ${CFGFILE} echo "APIKEY=${APIKEY}" >> ${CFGFILE} echo "KMLTEMPLATE=${KMLTEMPLATE}" >> ${CFGFILE} echo "NJLOGGRP=${NJLOGGRP}" >> ${CFGFILE} +echo "TESTMODE=${TESTMODE}" >> ${CFGFILE} +echo "TESTSUFF=${TESTSUFF}" >> ${CFGFILE} echo "" >> ${CFGFILE} echo "export RUNTIME_ENV SRC SITEURL" >> ${CFGFILE} echo "export WEBSITEBUCKET UKMONSHAREDBUCKET" >> ${CFGFILE} @@ -69,8 +85,10 @@ echo "export PYLIB TEMPLATES RCODEDIR DATADIR BKPINSTANCEID AWS_DEFAULT_REGION" echo "export RMS_ENV RMS_LOC WMPL_ENV WMPL_LOC" >> ${CFGFILE} echo "export PYTHONPATH=${RMS_LOC}:${WMPL_LOC}:${PYLIB}:${SRC}/share" >> ${CFGFILE} echo "export MATCHSTART MATCHEND SERVERSSHKEY" >> ${CFGFILE} -echo "export APIKEY KMLTEMPLATE SERVERINSTANCEID NJLOGGRP" >> ${CFGFILE} +echo "export APIKEY KMLTEMPLATE SERVERINSTANCEID SERVERUSERID NJLOGGRP" >> ${CFGFILE} echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib" >> ${CFGFILE} +echo "export TESTMODE TESTSUFF" >> ${CFGFILE} + # enable miniconda echo "source ~/.condaon" >> ${CFGFILE} # function to log to cloudwatch diff --git a/archive/utils/statsToMqtt.sh b/archive/utils/statsToMqtt.sh index 3235281a2..e2334bfee 100644 --- a/archive/utils/statsToMqtt.sh +++ b/archive/utils/statsToMqtt.sh @@ -5,7 +5,7 @@ if [ "$(hostname)" == "wordpresssite" ] ; then source /home/bitnami/venvs/openhabstuff/bin/activate python $here/statsToMqtt.py diskpct=$(df / |tail -1| awk '{print $5 }' | sed 's/%//g') - if [ $diskpct -gt 9 ] ; then + if [ $diskpct -gt 90 ] ; then /home/bitnami/src/maintenance/mysql_checks.sh msg="$(hostname) diskspace was ${diskpct}%" subj="Diskspace alert for $(hostname)" @@ -13,7 +13,7 @@ if [ "$(hostname)" == "wordpresssite" ] ; then python -c "from meteortools.utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" fi elif [ "$(hostname)" == "ukmcalcserver" ] ; then - source /home/ec2-user/venvs/wmpl/bin/activate + source $HOME/venvs/wmpl/bin/activate python $here/statsToMqtt.py diskpct=$(df / |tail -1| awk '{print $5 }' | sed 's/%//g') if [ $diskpct -gt 80 ] ; then diff --git a/archive/utils/updateDb.sh b/archive/utils/updateDb.sh index 2b8df8c27..501497f0b 100644 --- a/archive/utils/updateDb.sh +++ b/archive/utils/updateDb.sh @@ -11,10 +11,11 @@ else rundt=$1 fi +cd ~/prod/data/brightness mysql -u batch -p$(cat ~/.ssh/db_batch.passwd) -h localhost << EOD use ukmon; select count(*) from ukmon.brightness; -load data local infile '/home/ec2-user/prod/data/brightness/CaptureNight_${rundt}.csv' +load data local infile './CaptureNight_${rundt}.csv' into table brightness fields terminated by ','; select count(*) from ukmon.brightness; EOD diff --git a/archive/website/createFireballPage.sh b/archive/website/createFireballPage.sh index 7c63ba4bb..396aeb65d 100644 --- a/archive/website/createFireballPage.sh +++ b/archive/website/createFireballPage.sh @@ -88,7 +88,7 @@ echo "\$(document).ready(function() {" >> reportindex.js # need single quotes here to allow the hash to be printed echo '$("#fbtableid").DataTable({' >> reportindex.js echo "columnDefs : [" >> reportindex.js -echo "{ Type : \"numeric\", targets : [2]}" >> reportindex.js +echo "{ Type : \"numeric\", targets : [1]}" >> reportindex.js echo " ]," >> reportindex.js echo "order : [[ 1, \"asc\"],[2,\"desc\"]]," >> reportindex.js echo "paging: false" >> reportindex.js diff --git a/archive/website/createOrbitIndex.sh b/archive/website/createOrbitIndex.sh index 2d92cec21..3b05e1e8f 100644 --- a/archive/website/createOrbitIndex.sh +++ b/archive/website/createOrbitIndex.sh @@ -90,9 +90,9 @@ else j=0 echo "" >> $idxfile fi - if [ $domth -eq 1 ] ; then - echo "$i" >> $DATADIR/orbits/$yr/$ym.txt - fi + #if [ $domth -eq 1 ] ; then + # echo "$i" >> $DATADIR/orbits/$yr/$ym.txt + #fi done fi echo "" >> $idxfile diff --git a/tests/test_apis.py b/tests/test_apis.py index 4394b5d16..3d98c763d 100644 --- a/tests/test_apis.py +++ b/tests/test_apis.py @@ -56,7 +56,7 @@ def test_getMatchPickle(): for chunk in res.iter_content(chunk_size=4096): data = data + chunk jsd = json.loads(data) - assert len(jsd) == 62 + assert len(jsd) == 64 assert len(jsd['observations']) == 5 else: assert False diff --git a/utils/reimageCalcServer.ps1 b/utils/reimageCalcServer.ps1 new file mode 100644 index 000000000..9f776af31 --- /dev/null +++ b/utils/reimageCalcServer.ps1 @@ -0,0 +1,9 @@ +# script to re-image the UKMON calc server +# copyright Mark McIntyre, 2026- + +$dtstr=(get-date -uformat "%Y%m%d") + +$instdetails=(aws ec2 describe-instances --filters Name="tag:Name",Values="calcengine_ub" --profile ukmonshared --region eu-west-2) +$instanceid= (Write-Output $instdetails| convertfrom-json)[0].reservations.instances.instanceid + +aws ec2 create-image --instance-id $instanceid --name "calcengine_${dtstr}" --description "Latest Calc Engine image" --profile ukmonshared --region eu-west-2 \ No newline at end of file From 0cd2bbe7a7b7658cd242b26816ef752ef44aa6af Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 17:04:58 +0100 Subject: [PATCH 12/15] Update version (#433) * remove disused code * update livestream to accept normal RMS filenames * search dialog - fix bug and set default date range * small change to explain default dates * improvement in live image search * remove redundant file * update deployment scripts * improvements in routine to convert GMN report to pandas * more analysis of gmn data * Update the terraform a little * remove hardcoded reference to ec2-user * add SERVERUSERID env /ssm variable for calcserver user id * update container for new distrib-processing method * use calcserver SERVERUSERID variable * support for test ECS containers * fix paths in deployment scripts * remove unused file * missed some templates * update dev/test env settings * remove disused file * make sure paths are expanded * fixing various config issues * updates to makeConfig * updates to distributed processing to align with newest wmpl * add a comment to explain what the code does * remove json db * update consolidation to handle sqlite dbs * remove unused function * create data for daily report * remove unused file * update to call new version of reportOfLatestMatches * add test variables * mildly improve doco * put test logs in the right CW log group * handle missing log groups better * extra value in TrajQualityParams * bugfixes revealed in testing * fix issues in reportOfLatestMatches * put container dbs in a sensible place * catch logging errors * tidy up file locations * simplify calcserver start up * sorting out errors in container logging * update container logging info * avoid unnecessary error * fix bugs in log capture process * add MAX_STNS so i can experiement with limits * add logging. Add max_stns to constraints and set to 9999 push logs to s3 copy calcserver logs to ukmonhelper * don't pull or push the databases from S3 during distribution but do back them up after consolidation * catch error if trajpickle can't be found * set rundate more safely move some processes into nightlyJob to make testing easier * tidy up backup file names and housekeep backup folder * move some processes from findAllMatches to nightlyJob to make testing easier * support for testing of createExecReplot * pass testmode to createExecReplot * ensure logs are unique and capture container logs * realigning createFireballPage with prod. Updating gitignore * remove verbosity flag from one tar operation * fixup to handle new style log with datestamps * get privip after restarting calcserver * temporarily don't run rerunFailedLambdas in test * update to work in a test env too * add 'period' to the match summary api * add period to match summary api * re-enabling rerunFailedLambdas * quieten an S3 transfer * include the foldername when reporting on container progress * put daily trajdb in a safe place * be more specific when finding the daily log * only tar the current container logs * save daily obsdb in the right place * rename ec2.tf to calcserver.tf * quieten some s3 transfers * rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is * update SSM params to reflect new serverid * bugfix in distributeCandidates - using wrong path * add newline to getCostMetrics * weird bug in statsToMqtt * bugfix in update_wmpl.sh for traj container * Script to create an AMI for the calcengine * exclude test data when syncing from s3 * writing databases and logs to the wrong target * bad test on istest * typo in serveruserid * adding to server migration notes * purge old rawcsvs and fullcsvs older than 180 days * improved housekeeping of old data * make sure we scoop up data for 31/12 * stop creation of an unused set of files * change default locations for container data * put container logs in the right place * update dockerfile syntax * change locations of test container data and output * remove incorrect comments * Add script to clean up deleted/duplicate trajs * simplify the stats gathering and move to python fix bug in getMatchStats * comment out unused files * update reportOfLatestMatches to exclude deleted trajectories * add scp to requirements * add SQL database maintenance to dataMaintenance * incorrect default for datadir * update call to pymysql.connect * ensure consolidateOutput rerunnable for prior years * upgrading WMPL * update gitignore a bit * bugfix to catch data from 31/12 each year * add paramiko to dependencies for matchPickle to support wmpl upgrade * update getpickle to newest WMPL * bugfix in api tests * bugfix - changes to dailydb not saved * improvements to process to cleanup deleted trajs * after merging, check and remove any duplicate trajectories from the sqlite db and disk * Adding version * bump version 2025.11.1 -> 2026.04.0 * update files containing vesion * bump version 2026.04.0 -> 2026.04.1 * unmangling side effets of automerge * correct usage message --- README.md | 2 +- archive/README.md | 3 + .../reports/reportOfLatestMatches.py | 21 +++++-- .../ukmon_pylib/traj/consolidateDistTraj.py | 38 +++++++++++- archive/utils/cleanupDeletedTrajs.sh | 58 +++++-------------- bumpver.toml | 6 +- 6 files changed, 74 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 226593df5..595de7c54 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # UK Meteor Data Analysis Shared code and libraries -version: 2024.04.2 +version: 2026.04.1 This repository contains the code behind the UK Meteors data archive and data processing pipeline. diff --git a/archive/README.md b/archive/README.md index 9fefc6338..108c7fbf6 100644 --- a/archive/README.md +++ b/archive/README.md @@ -1,5 +1,8 @@ Data Processing and Flows ========================== + +version: 2026.04.1 + This diagram shows the overall flow of data from Cameras to websites and out to the public. ```mermaid diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index c8ffdc040..2d8738584 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -81,10 +81,15 @@ def getListOfNewMatches(dir_path, db_path='/tmp', rundate=None): masterdb.closeTrajDatabase() # iterate over the delete list and update the daily db and new traj list accordingly + print('cleaning up deleted trajectories in dailydb') for testtr in deltrajs: if testtr in newtrajs: - sqlstr = f'update trajectories set status=0 where "traj_file_path={testtr[0]}";' + sqlstr = f'update trajectories set status=0 where traj_file_path="{testtr[0]}"' dailydb.dbhandle.execute(sqlstr) + dailydb.dbhandle.commit() + cur = dailydb.dbhandle.execute(f'select traj_id, status from trajectories where traj_file_path="{testtr[0]}"') + msg = cur.fetchone() + print(f'updated {msg}') newtrajs.pop(newtrajs.index(testtr)) dailydb.closeTrajDatabase() @@ -198,8 +203,12 @@ def findNewMatches(dir_path, out_path, offset, repdtstr): daily_db_dir = sys.argv[2] offset = sys.argv[3] - # arguments dblocation, datadir, days ago, rundate eg 20220524 - findNewMatches(cand_db_dir, daily_db_dir, offset, repdtstr) - # update the daily database of paired observations - daily_db_dir = os.path.join(os.path.split(cand_db_dir)[0], 'dailydbs') - updatePairedDB(cand_db_dir, daily_db_dir, repdtstr) + flist = glob.glob(os.path.join(cand_db_dir, 'trajectories_*.db')) + if len(flist) == 0: + print('no container databases to process, aborting') + else: + # arguments dblocation, datadir, days ago, rundate eg 20220524 + findNewMatches(cand_db_dir, daily_db_dir, offset, repdtstr) + # update the daily database of paired observations + daily_db_dir = os.path.join(os.path.split(cand_db_dir)[0], 'dailydbs') + updatePairedDB(cand_db_dir, daily_db_dir, repdtstr) diff --git a/archive/ukmon_pylib/traj/consolidateDistTraj.py b/archive/ukmon_pylib/traj/consolidateDistTraj.py index 848feeef1..c1fee16df 100644 --- a/archive/ukmon_pylib/traj/consolidateDistTraj.py +++ b/archive/ukmon_pylib/traj/consolidateDistTraj.py @@ -6,9 +6,26 @@ import datetime from wmpl.Trajectory.CorrelateDB import ObservationsDatabase, TrajectoryDatabase, CandidateDatabase +from wmpl.Trajectory.CorrelateRMS import RMSDataHandle +from wmpl.Utils.TrajConversions import jd2Date -def mergeDatabases(srcdir, dbdir, ignore_missing=False, purge_records=False): +def mergeDatabases(srcdir, dbdir, basedir, ignore_missing=False, purge_records=False, matchstart=3): + """ + merge container databases into the master database, looking for and cleaning deleted trajectories + + arguments: + srcdir location of container databases + dbdir location of master databases + basedir location of raw data and trajectories + + keyword args: + ignore_missing default false: if true, create master DBs if not present + purge_records default false: if true, purge records from master DBs + matchstart default 3: range of days to scan for duplicate and deleted trajectories + + """ + targdb = os.path.join(dbdir, 'observations.db') if os.path.isfile(targdb) or ignore_missing: obsdb = ObservationsDatabase(dbdir, purge_records=purge_records) @@ -31,8 +48,24 @@ def mergeDatabases(srcdir, dbdir, ignore_missing=False, purge_records=False): print(f'{tstamp} processing {fl}') if trajdb.mergeTrajDatabase(fl): os.remove(fl) + + # get the latest date in the database for use below + cur = trajdb.dbhandle.execute('select max(jdt_ref) from trajectories where status=1') + vals = cur.fetchall() + jdt_end = float(vals[0][0]) trajdb.closeTrajDatabase() + # find and remove duplicates using WMPL's built-in routine to clean up the trajectory DB + # NB: can only be run on the calcserver where trajectory folders are present + + # select range midday on the latest date to matchstart days earlier + dt_end = jd2Date(int(jdt_end) + 1, dt_obj=True, tzinfo=datetime.timezone.utc) + dt_beg = jd2Date(int(jdt_end) + 1 - matchstart, dt_obj=True, tzinfo=datetime.timezone.utc) + event_time_range = [dt_beg, dt_end] + + dh = RMSDataHandle(basedir, dt_range=event_time_range, db_dir=dbdir, output_dir=basedir,mcmode=1, archivemonths=0) + dh.updateTrajectoryDatabase(dt_range=event_time_range) + # should never need to run this part as the candidates are all made in one place targdb = os.path.join(dbdir, 'candidates.db') if os.path.isfile(targdb): @@ -55,4 +88,5 @@ def mergeDatabases(srcdir, dbdir, ignore_missing=False, purge_records=False): exit(0) srcdir = sys.argv[1] dbdir = sys.argv[2] - mergeDatabases(srcdir, dbdir) + basedir = os.path.dirname(os.path.normpath(dbdir)) + mergeDatabases(srcdir, dbdir, basedir) diff --git a/archive/utils/cleanupDeletedTrajs.sh b/archive/utils/cleanupDeletedTrajs.sh index ed5fa0d3f..bd0836b9a 100644 --- a/archive/utils/cleanupDeletedTrajs.sh +++ b/archive/utils/cleanupDeletedTrajs.sh @@ -14,67 +14,41 @@ cd ${DATADIR}/distrib startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') jdt_min=$(python -c "from wmpl.Utils.TrajConversions import datetime2JD;import datetime;print(datetime2JD(datetime.datetime.strptime('$startdt', '%Y%m%d-%H%M%S')))") -echo "checking main storage" +echo "checking main storage, website and csvfiles" sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do + export moved=0 trajdir=$(dirname $traj) - echo $trajdir - moved=0 - aws s3 ls ${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir/ | awk -F " " '{print $4}' | while read fname; do - aws s3 mv ${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir/$fname ${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir/$fname --quiet - moved=1 + #echo $trajdir + srcloc=${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir + trgloc=${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir + aws s3 ls $srcloc/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv ${srcloc}/$fname ${trgloc}/$fname --quiet + export moved=1 done - [ $moved == 1 ] && echo moved $trajdir -done -echo "checking website" -sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do - trajdir=$(dirname $traj) yr=${trajdir:13:4} - trajpth=${trajdir:18} - echo $trajdir + trajpth=$(basename $trajdir) webloc=$WEBSITEBUCKET/reports/${yr}/orbits/$trajpth newloc=${UKMONSHAREDBUCKET}/matches/duplicates/reports/${yr}/orbits/$trajpth - moved=0 aws s3 ls $webloc/ | awk -F " " '{print $4}' | while read fname; do aws s3 mv $webloc/$fname $newloc/$fname --quiet - moved=1 + export moved=1 done - [ $moved == 1 ] && echo moved $trajdir -done -echo "checking fullcsv" -sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do - trajdir=$(dirname $traj) - yr=${trajdir:13:4} - trajpth=${trajdir:34:19} - csvname=$(echo $trajpth | sed 's/_/-/g') - echo $csvname + trajpth1=${trajdir:34:19} + csvname=$(echo $trajpth1 | sed 's/_/-/g') csvloc=${UKMONSHAREDBUCKET}/matches/${yr}/fullcsv newloc=${UKMONSHAREDBUCKET}/matches/duplicates/csvs/${yr} - moved=0 aws s3 ls $csvloc/ | awk -F " " '{print $4}' | grep $csvname | while read fname; do aws s3 mv $csvloc/$fname $newloc/$fname --quiet - moved=1 + export moved=1 done - [ $moved == 1 ] && echo moved $trajdir -done - -echo "checking historic fullcsv just in case" -sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do - trajdir=$(dirname $traj) - yr=${trajdir:13:4} - trajpth=$(basename $trajdir) - csvname=$(echo $trajpth | sed 's/_/-/g') - echo $csvname csvloc=$DATADIR/orbits/${yr}/fullcsv/processed - newloc=${UKMONSHAREDBUCKET}/matches/duplicates/csvs/${yr} - moved=0 ls -1 $csvloc/ | grep $csvname | while read fname; do aws s3 mv $csvloc/$fname $newloc/$fname --quiet - moved=1 + export moved=1 done - [ $moved == 1 ] && echo moved $trajdir - export trajpth + [ $moved == 1 ] && echo "moved $trajpth" done echo "checking consolidated matches" @@ -83,6 +57,7 @@ trajdir=$(dirname $lasttraj) yr=${trajdir:13:4} trajpth=$(basename $trajdir) matchcsv=$DATADIR/matched/matches-full-${yr}.csv +grep $trajpth $matchcsv if [ $? == 0 ] ; then python -c "from maintenance.dataMaintenance import removeDeletedTraj;removeDeletedTraj('$matchcsv')" python -m converters.toParquet $matchcsv @@ -99,4 +74,3 @@ export AWS_PROFILE=ukmonshared # needed for mariadb connection details echo "checking Mariadb Database" python -c "from maintenance.dataMaintenance import removeDelTrajFromDb;removeDelTrajFromDb()" unset AWS_PROFILE -done \ No newline at end of file diff --git a/bumpver.toml b/bumpver.toml index 9cab7605b..f8ef5f0f4 100644 --- a/bumpver.toml +++ b/bumpver.toml @@ -1,5 +1,5 @@ [bumpver] -current_version = "2024.04.2" +current_version = "2026.04.1" version_pattern = "YYYY.0M.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true @@ -13,6 +13,6 @@ push = true "bumpver.toml" = [ 'current_version = "{version}"', ] -".github/workflows/build_usermgmt.yml" = [ - 'tag_name: {version}' +"archive/README.md" = [ + "version: {version}" ] From 8a4a7f2904a715d8f1e11b19c98365fb53a99973 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 19:45:59 +0000 Subject: [PATCH 13/15] adding install script --- install_or_update.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 install_or_update.sh diff --git a/install_or_update.sh b/install_or_update.sh new file mode 100755 index 000000000..8efa03f02 --- /dev/null +++ b/install_or_update.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (C) 2018-2023 Mark McIntyre + +if [[ "$1" != "PROD" && "$1" != "DEV" ]] ; then + echo "must provide runtime env of PROD or DEV" + exit -1 +fi +RUNTIME_ENV=$1 +envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +mkdir -p ~/${envname} + +cd $here/archive + +for loc in analysis ukmon_pylib website cronjobs utils share static_content +do + rsync -avz $loc/ ~/${envname}/$loc + chmod +x ~/${envname}/$loc/*.sh > /dev/null 2>&1 +done +DATADIR=~/$envname/data +mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls} +mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajdb,videos} +mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} +mkdir -p ~/$envname/logs + From 5e52f0783b575fa9b948c917cf2dcdede64ff84e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 5 May 2026 12:17:30 +0100 Subject: [PATCH 14/15] Migration of both calc and batch to new hardware (#466) * updates to makeConfig * updates to distributed processing to align with newest wmpl * add a comment to explain what the code does * remove json db * update consolidation to handle sqlite dbs * remove unused function * create data for daily report * remove unused file * update to call new version of reportOfLatestMatches * add test variables * mildly improve doco * put test logs in the right CW log group * handle missing log groups better * extra value in TrajQualityParams * bugfixes revealed in testing * fix issues in reportOfLatestMatches * put container dbs in a sensible place * catch logging errors * tidy up file locations * simplify calcserver start up * sorting out errors in container logging * update container logging info * avoid unnecessary error * fix bugs in log capture process * add MAX_STNS so i can experiement with limits * add logging. Add max_stns to constraints and set to 9999 push logs to s3 copy calcserver logs to ukmonhelper * don't pull or push the databases from S3 during distribution but do back them up after consolidation * catch error if trajpickle can't be found * set rundate more safely move some processes into nightlyJob to make testing easier * tidy up backup file names and housekeep backup folder * move some processes from findAllMatches to nightlyJob to make testing easier * support for testing of createExecReplot * pass testmode to createExecReplot * ensure logs are unique and capture container logs * realigning createFireballPage with prod. Updating gitignore * remove verbosity flag from one tar operation * fixup to handle new style log with datestamps * get privip after restarting calcserver * temporarily don't run rerunFailedLambdas in test * update to work in a test env too * add 'period' to the match summary api * add period to match summary api * re-enabling rerunFailedLambdas * quieten an S3 transfer * include the foldername when reporting on container progress * put daily trajdb in a safe place * be more specific when finding the daily log * only tar the current container logs * save daily obsdb in the right place * rename ec2.tf to calcserver.tf * quieten some s3 transfers * rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is * update SSM params to reflect new serverid * bugfix in distributeCandidates - using wrong path * add newline to getCostMetrics * weird bug in statsToMqtt * bugfix in update_wmpl.sh for traj container * Script to create an AMI for the calcengine * exclude test data when syncing from s3 * writing databases and logs to the wrong target * bad test on istest * typo in serveruserid * adding to server migration notes * purge old rawcsvs and fullcsvs older than 180 days * improved housekeeping of old data * make sure we scoop up data for 31/12 * stop creation of an unused set of files * change default locations for container data * put container logs in the right place * update dockerfile syntax * change locations of test container data and output * remove incorrect comments * Add script to clean up deleted/duplicate trajs * simplify the stats gathering and move to python fix bug in getMatchStats * comment out unused files * update reportOfLatestMatches to exclude deleted trajectories * add scp to requirements * add SQL database maintenance to dataMaintenance * incorrect default for datadir * update call to pymysql.connect * ensure consolidateOutput rerunnable for prior years * upgrading WMPL * update gitignore a bit * bugfix to catch data from 31/12 each year * add paramiko to dependencies for matchPickle to support wmpl upgrade * update getpickle to newest WMPL * bugfix in api tests * bugfix - changes to dailydb not saved * improvements to process to cleanup deleted trajs * after merging, check and remove any duplicate trajectories from the sqlite db and disk * Adding version * bump version 2025.11.1 -> 2026.04.0 * update files containing vesion * bump version 2026.04.0 -> 2026.04.1 * unmangling side effets of automerge * correct usage message * decommission old calcserver and add new batchserver * add calcserver IP to ssm variables and use it in scripts * Update my branch with the latest from master (#436) * Update version (#433) * remove disused code * update livestream to accept normal RMS filenames * search dialog - fix bug and set default date range * small change to explain default dates * improvement in live image search * remove redundant file * update deployment scripts * improvements in routine to convert GMN report to pandas * more analysis of gmn data * Update the terraform a little * remove hardcoded reference to ec2-user * add SERVERUSERID env /ssm variable for calcserver user id * update container for new distrib-processing method * use calcserver SERVERUSERID variable * support for test ECS containers * fix paths in deployment scripts * remove unused file * missed some templates * update dev/test env settings * remove disused file * make sure paths are expanded * fixing various config issues * updates to makeConfig * updates to distributed processing to align with newest wmpl * add a comment to explain what the code does * remove json db * update consolidation to handle sqlite dbs * remove unused function * create data for daily report * remove unused file * update to call new version of reportOfLatestMatches * add test variables * mildly improve doco * put test logs in the right CW log group * handle missing log groups better * extra value in TrajQualityParams * bugfixes revealed in testing * fix issues in reportOfLatestMatches * put container dbs in a sensible place * catch logging errors * tidy up file locations * simplify calcserver start up * sorting out errors in container logging * update container logging info * avoid unnecessary error * fix bugs in log capture process * add MAX_STNS so i can experiement with limits * add logging. Add max_stns to constraints and set to 9999 push logs to s3 copy calcserver logs to ukmonhelper * don't pull or push the databases from S3 during distribution but do back them up after consolidation * catch error if trajpickle can't be found * set rundate more safely move some processes into nightlyJob to make testing easier * tidy up backup file names and housekeep backup folder * move some processes from findAllMatches to nightlyJob to make testing easier * support for testing of createExecReplot * pass testmode to createExecReplot * ensure logs are unique and capture container logs * realigning createFireballPage with prod. Updating gitignore * remove verbosity flag from one tar operation * fixup to handle new style log with datestamps * get privip after restarting calcserver * temporarily don't run rerunFailedLambdas in test * update to work in a test env too * add 'period' to the match summary api * add period to match summary api * re-enabling rerunFailedLambdas * quieten an S3 transfer * include the foldername when reporting on container progress * put daily trajdb in a safe place * be more specific when finding the daily log * only tar the current container logs * save daily obsdb in the right place * rename ec2.tf to calcserver.tf * quieten some s3 transfers * rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is * update SSM params to reflect new serverid * bugfix in distributeCandidates - using wrong path * add newline to getCostMetrics * weird bug in statsToMqtt * bugfix in update_wmpl.sh for traj container * Script to create an AMI for the calcengine * exclude test data when syncing from s3 * writing databases and logs to the wrong target * bad test on istest * typo in serveruserid * adding to server migration notes * purge old rawcsvs and fullcsvs older than 180 days * improved housekeeping of old data * make sure we scoop up data for 31/12 * stop creation of an unused set of files * change default locations for container data * put container logs in the right place * update dockerfile syntax * change locations of test container data and output * remove incorrect comments * Add script to clean up deleted/duplicate trajs * simplify the stats gathering and move to python fix bug in getMatchStats * comment out unused files * update reportOfLatestMatches to exclude deleted trajectories * add scp to requirements * add SQL database maintenance to dataMaintenance * incorrect default for datadir * update call to pymysql.connect * ensure consolidateOutput rerunnable for prior years * upgrading WMPL * update gitignore a bit * bugfix to catch data from 31/12 each year * add paramiko to dependencies for matchPickle to support wmpl upgrade * update getpickle to newest WMPL * bugfix in api tests * bugfix - changes to dailydb not saved * improvements to process to cleanup deleted trajs * after merging, check and remove any duplicate trajectories from the sqlite db and disk * Adding version * bump version 2025.11.1 -> 2026.04.0 * update files containing vesion * bump version 2026.04.0 -> 2026.04.1 * unmangling side effets of automerge * correct usage message * adding install script * changes to ansible scripts * remove irrelevant file * Adding SSM variables for the new batchserver in the ukmda account * attempt to add logging to consolidateDistTraj * remove reference to ee account * temporarily add mariadb deets to ssm in mjmm account * bugfixes in dataMaintenance and cleanupDeletedTrajs to avoid deleting disk files when trajs share same folder * updating migration instructions * more work on deduplication for ukmon * remove username-specific stuff * remove the old deployment scripts * update install script * updates to improve server migration process * add bashrc to the installer * slight tweak to installer * add a few installer messages * remove an echo * be less verbose in the rsyncs * oops need -a for one of the rsyncs * rename requirements file for ukmda * remove dependencies on MeteorTools * add utils to ukmon_pylib * more removal of dependencies on MeteorTools * updating how shower data is obtained * corercting location of IMO and gmn shower data * bugfixes * fixes in imoWorkinShowerlist * small error in install_or_update * bugfix when no data found in imo set * retrieve IMO data after each code update * fixing up getShowerDets * improve installer * fix createShowerExtracts * update extractors and showeranalysis * bug in extractor thats existed since 2023 but nobody noticed... * trying to fix matplotlib bug * more working on matplotlib issue * oops overfixed a bug * another incorrect bugfix! * make bars wider * more removal of MeteorTools * bugfix in createannualbarchart * working on bug * more work on lenght bug * remove some debugging * oops forgot to push the fixed version of createReportIndex * segregating function to get list of images that weren't calibrated by RMS * reorganize stuff a bit * add email sending functionality to ukmon_pylib * use send email functionality in code * add database and gmaps keys/passwords as secure variables * update to use secure keys * remove dbpw from environment * remove dbpw from environment * add script to retrieve db passwd * add script to retrieve db passwords * bug in createTables * updates to migration instructions * include pip installs in the script * tweak installer * remove glob.glob1 * move unused scripts - will delete later * fix mariadb connections * typo * tidy up docs * fixing mysql calls * small changes to documentation * suppress pip messages if already installed * tweaks to the installer * add logrotate stuff * bugfixes in installer * tidy up installer * bugfix in key mgmt * move some functionality into new script to tidy up nightly job and simplify rerunning * improve install script * update location of camera count file * move annual bar charts into a more logical location * update test script * add gmaps param to my acct * relocate coverage maps and front page index * tidy up useraudit process * bugfixes in testing * bugfix in createAnnualBarchart * bug in testscript * another small bug in createSummaryTable * still fighting that bug * tidy up lastlog data * remove unused file * update cam status reporting * clear down the old brightness data * simplification of logging and of the nightlyJob script * bugfixes in process to gather consumed image ifo * update getUncal to work for historic data * typo * improvements in getUncalImages * bugfix * add python to get unused images * move script to right place * comment out unnecessary echo * bugfixes in camMetrics while migrating user accts * fix #448 * make sure window title is sensible * bugfix in replot and get unused * bugfix in removeDeletedTraj for search index * remove deprecation warning issue #450 * whoops naive vs tzaware dates * remove warning in cameraStatusReport issue #453 * remove warning issue #454 * widen constraints on analysing pickles to get images * add migration script * add sudo * actually do it * update sftp migration script * more changes to migration script * moving sftp migration script * update messages and docos * slight tweak to migration script * further work on sftp account migration * update ukmon ini files after migration * typo * collect old server login records integration with new * update migration instructions * update cam metrics report to include hostname * bugfix as $USER not available in cron * removing unused files * some tidying up * motre reorg * remove disused template rename file to follow convention * simplifying logging and removing unused aws profile * removing profile from python * adding executable bit * typo * bugfix to work around typo * print message when done * bugfix to get trajectories from s3 before trying to check for missing trajectories... * improve to support running for past dates * errors in bashrc and logrotate config * prevent email being sent for anything other than current days report * bugfix: make sure we use the most recent stats * improvement in reportOfLatestMatches to handle past dates * bugfix * small tweak to support reportOfLatestMatches * tweak getMatchStats to handle historic data better * add mariadb and ping to the sec grp so that the database can be accessed by apis * update the db servername * bugfix in match stats * prevent bogus renaming of matchJob.log * bumping version * bump version 2026.04.1 -> 2026.05.0 --- README.md | 2 +- archive/README.md | 2 +- archive/analysis/findAllMatches.sh | 32 +- archive/analysis/getLogData.sh | 6 +- archive/analysis/reportActiveShowers.sh | 2 +- archive/analysis/runDistrib.sh | 92 ++-- archive/analysis/updatePlotsAndDetStatus.sh | 53 +- archive/cronjobs/captureBright.sh | 12 +- archive/cronjobs/gatherMonthlyVideos.sh | 4 +- archive/cronjobs/getImoWSfile.sh | 2 +- archive/cronjobs/nightlyJob.sh | 108 ++-- archive/database/Setup.md | 43 +- archive/database/ddl/create_tables.sql | 5 +- archive/deployment/analysis.yml | 37 -- archive/deployment/config.yml | 29 - archive/deployment/cronjobs.yml | 27 - archive/deployment/database.yml | 28 - archive/deployment/dev-vars.yml | 3 - archive/deployment/full_deployment.yml | 9 - archive/deployment/prod-vars.yml | 3 - archive/deployment/pylib.yml | 112 ---- archive/deployment/server_setup.yml | 36 -- archive/deployment/shwrinfo.yml | 41 -- archive/deployment/utils.yml | 43 -- archive/deployment/website.yml | 41 -- archive/deployment/website_static.yml | 42 -- archive/lambdas/matchDataApi/matchDataApi.py | 2 +- archive/server_setup/.bash_aliases | 25 +- archive/server_setup/.bashrc | 20 +- archive/server_setup/.condaon | 9 +- archive/server_setup/.vimrc | 1 + archive/server_setup/logs.conf | 24 + archive/server_setup/migrateSftpAccts.sh | 44 ++ archive/server_setup/migratingBatchServer.md | 194 +++++-- archive/server_setup/ssh-setup.sh | 15 - .../{ => unused}/install-cartopy.sh | 0 .../server_setup/{ => unused}/install-geos.sh | 0 .../{ => unused}/install-proj4.sh | 0 .../{ => unused}/install-sqlite3.sh | 0 .../{ => unused}/install_opencv.sh | 0 .../server_setup/{ => unused}/newserver.sh | 0 .../server_setup/{ => unused}/remount-s3.sh | 0 .../server_setup/{ => unused}/yum-installs.sh | 0 archive/shwrinfo/costnotes.html | 17 +- archive/terraform/mjmm/aws.tf | 6 +- archive/terraform/mjmm/crossacct-role.tf | 2 - archive/terraform/mjmm/dev_ssm_parameters.tf | 10 + archive/terraform/mjmm/ssm_parameters.tf | 55 ++ archive/terraform/ukmda/batchserver.tf | 47 ++ archive/terraform/ukmda/calcserver.tf | 41 -- archive/terraform/ukmda/cloudwatch.tf | 2 +- archive/terraform/ukmda/dev_ssm_variables.tf | 191 +++++++ archive/terraform/ukmda/secgrp.tf | 40 +- archive/terraform/ukmda/ssm_variables.tf | 159 +++++- archive/terraform/ukmda/variables.tf | 7 +- ...ements.txt => additional_requirements.txt} | 27 +- archive/ukmon_pylib/analysis/README.md | 7 +- .../analysis/gatherDetectionData.py | 65 --- archive/ukmon_pylib/analysis/getUsedUnused.py | 65 +++ .../ukmon_pylib/analysis/showerAnalysis.py | 37 +- .../ukmon_pylib/analysis/stationAnalysis.py | 6 +- archive/ukmon_pylib/converters/toParquet.py | 2 +- .../ukmon_pylib/maintenance/cleanDdbTables.py | 8 +- .../maintenance/dataMaintenance.py | 154 +++--- .../maintenance/getNextBatchStart.py | 2 +- .../maintenance/getUserAndKeyInfo.py | 2 +- .../maintenance/plotStationsOnMap.py | 13 +- .../maintenance/recreateOrbitPages.py | 6 +- .../maintenance/rerunFailedLambdas.py | 13 +- archive/ukmon_pylib/metrics/camMetrics.py | 109 ++-- archive/ukmon_pylib/metrics/getMatchStats.py | 21 +- archive/ukmon_pylib/reports/CameraDetails.py | 26 +- .../ukmon_pylib/reports/cameraStatusReport.py | 2 +- .../reports/createAnnualBarChart.py | 12 +- .../reports/createSearchableFormat.py | 2 +- .../ukmon_pylib/reports/createSummaryTable.py | 7 +- archive/ukmon_pylib/reports/dailyReport.py | 23 +- archive/ukmon_pylib/reports/extractors.py | 16 +- .../ukmon_pylib/reports/findFailedMatches.py | 2 +- archive/ukmon_pylib/reports/findFireballs.py | 2 +- .../ukmon_pylib/reports/makeCoverageMap.py | 29 +- archive/ukmon_pylib/reports/meteoriteTools.py | 2 +- .../reports/reportActiveShowers.py | 6 +- .../ukmon_pylib/reports/reportBadCameras.py | 2 +- .../reports/reportOfLatestMatches.py | 38 +- .../ukmon_pylib/reports/trackThumbnails.py | 2 +- .../test_reports_createAnnualBarChart.py | 6 +- .../ukmon_pylib/tests/test_reports_various.py | 5 +- .../ukmon_pylib/traj/consolidateDistTraj.py | 8 + .../traj/createDistribMatchingSh.py | 9 +- .../ukmon_pylib/traj/distributeCandidates.py | 2 +- archive/ukmon_pylib/traj/manualBulkReruns.py | 7 +- archive/ukmon_pylib/traj/pickleAnalyser.py | 23 +- archive/ukmon_pylib/utils/__init__.py | 1 + .../{analysis => utils}/analyseGmnData.py | 0 .../compareBrightnessData.py | 4 +- archive/ukmon_pylib/utils/convertSolLon.py | 41 ++ archive/ukmon_pylib/utils/ftpDetectInfo.py | 506 ++++++++++++++++++ archive/ukmon_pylib/utils/getActiveShowers.py | 139 +++++ archive/ukmon_pylib/utils/getRiseSet.py | 32 ++ archive/ukmon_pylib/utils/getUncalImages.py | 25 + .../ukmon_pylib/utils/imoWorkingShowerList.py | 223 ++++++++ archive/ukmon_pylib/utils/kmlHandlers.py | 149 ++++++ .../{analysis => utils}/meteorFlux.py | 0 archive/ukmon_pylib/utils/sendAnEmail.py | 192 +++++++ archive/utils/README.md | 21 +- archive/utils/checkCalcServerRunTime.sh | 66 --- archive/utils/cleanupDeletedTrajs.sh | 83 +-- archive/utils/liveevent.template | 15 - .../{updateDb.sh => loadBrightnessCsvMDB.sh} | 6 +- archive/utils/loadMatchCsvMDB.sh | 5 +- archive/utils/loadSingleCsvMDB.sh | 5 +- archive/utils/makeConfig.sh | 18 +- archive/utils/monthlyCleardown.sh | 157 ------ archive/utils/rerunFTPtoUKMONlambda.sh | 2 +- archive/utils/statsToMqtt.sh | 4 +- ...t-calcengine.sh => stopstartCalcengine.sh} | 2 +- archive/utils/userAudit.sh | 18 +- archive/website/cameraStatusReport.sh | 21 +- archive/website/costReport.sh | 10 +- archive/website/createReportIndex.sh | 2 +- archive/website/createShwrExtracts.sh | 2 +- archive/website/createSummaryTable.sh | 37 +- archive/website/updateCameraDets.sh | 37 ++ bumpver.toml | 2 +- install_or_update.sh | 55 +- using_vault.sh | 49 -- utils/reimageBatchServer.ps1 | 9 + 128 files changed, 2808 insertions(+), 1623 deletions(-) delete mode 100644 archive/deployment/analysis.yml delete mode 100644 archive/deployment/config.yml delete mode 100644 archive/deployment/cronjobs.yml delete mode 100644 archive/deployment/database.yml delete mode 100644 archive/deployment/dev-vars.yml delete mode 100644 archive/deployment/full_deployment.yml delete mode 100644 archive/deployment/prod-vars.yml delete mode 100644 archive/deployment/pylib.yml delete mode 100644 archive/deployment/server_setup.yml delete mode 100644 archive/deployment/shwrinfo.yml delete mode 100644 archive/deployment/utils.yml delete mode 100644 archive/deployment/website.yml delete mode 100644 archive/deployment/website_static.yml create mode 100644 archive/server_setup/.vimrc create mode 100644 archive/server_setup/logs.conf create mode 100755 archive/server_setup/migrateSftpAccts.sh delete mode 100644 archive/server_setup/ssh-setup.sh rename archive/server_setup/{ => unused}/install-cartopy.sh (100%) rename archive/server_setup/{ => unused}/install-geos.sh (100%) rename archive/server_setup/{ => unused}/install-proj4.sh (100%) rename archive/server_setup/{ => unused}/install-sqlite3.sh (100%) rename archive/server_setup/{ => unused}/install_opencv.sh (100%) rename archive/server_setup/{ => unused}/newserver.sh (100%) rename archive/server_setup/{ => unused}/remount-s3.sh (100%) rename archive/server_setup/{ => unused}/yum-installs.sh (100%) create mode 100644 archive/terraform/ukmda/batchserver.tf create mode 100644 archive/terraform/ukmda/dev_ssm_variables.tf rename archive/ukmon_pylib/{requirements.txt => additional_requirements.txt} (67%) delete mode 100644 archive/ukmon_pylib/analysis/gatherDetectionData.py create mode 100644 archive/ukmon_pylib/analysis/getUsedUnused.py create mode 100644 archive/ukmon_pylib/utils/__init__.py rename archive/ukmon_pylib/{analysis => utils}/analyseGmnData.py (100%) rename archive/ukmon_pylib/{analysis => utils}/compareBrightnessData.py (91%) create mode 100644 archive/ukmon_pylib/utils/convertSolLon.py create mode 100644 archive/ukmon_pylib/utils/ftpDetectInfo.py create mode 100644 archive/ukmon_pylib/utils/getActiveShowers.py create mode 100644 archive/ukmon_pylib/utils/getRiseSet.py create mode 100644 archive/ukmon_pylib/utils/getUncalImages.py create mode 100644 archive/ukmon_pylib/utils/imoWorkingShowerList.py create mode 100644 archive/ukmon_pylib/utils/kmlHandlers.py rename archive/ukmon_pylib/{analysis => utils}/meteorFlux.py (100%) create mode 100644 archive/ukmon_pylib/utils/sendAnEmail.py delete mode 100644 archive/utils/checkCalcServerRunTime.sh delete mode 100644 archive/utils/liveevent.template rename archive/utils/{updateDb.sh => loadBrightnessCsvMDB.sh} (69%) delete mode 100644 archive/utils/monthlyCleardown.sh rename archive/utils/{stopstart-calcengine.sh => stopstartCalcengine.sh} (90%) create mode 100644 archive/website/updateCameraDets.sh delete mode 100644 using_vault.sh create mode 100644 utils/reimageBatchServer.ps1 diff --git a/README.md b/README.md index 595de7c54..9f059c11c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # UK Meteor Data Analysis Shared code and libraries -version: 2026.04.1 +version: 2026.05.0 This repository contains the code behind the UK Meteors data archive and data processing pipeline. diff --git a/archive/README.md b/archive/README.md index 108c7fbf6..c37b478af 100644 --- a/archive/README.md +++ b/archive/README.md @@ -1,7 +1,7 @@ Data Processing and Flows ========================== -version: 2026.04.1 +version: 2026.05.0 This diagram shows the overall flow of data from Cameras to websites and out to the public. diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index 2fba436f8..59872560d 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -23,12 +23,7 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 conda activate $HOME/miniconda3/envs/${WMPL_ENV} -# logstream name inherited from parent environment but set it if not -if [ "$NJLOGSTREAM" == "" ]; then - NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) - aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared -fi -log2cw $NJLOGGRP $NJLOGSTREAM "start findAllMatches" findAllMatches +logger -s -t findAllMatches "starting" [ -f $DATADIR/rundate.txt ] && rundate=$(cat $DATADIR/rundate.txt) || rundate=$(date +%Y%m%d) @@ -52,40 +47,39 @@ mkdir -p $SRC/logs/distrib > /dev/null 2>&1 startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') enddt=$(date --date="-$MATCHEND days" '+%Y%m%d-080000') -log2cw $NJLOGGRP $NJLOGSTREAM "solving for ${startdt} to ${enddt}" findAllMatches -log2cw $NJLOGGRP $NJLOGSTREAM "start runDistrib" findAllMatches -$SRC/analysis/runDistrib.sh $MATCHSTART $MATCHEND +logger -s -t findAllMatches "solving for ${startdt} to ${enddt}" +logger -s -t findAllMatches "start runDistrib" -log2cw $NJLOGGRP $NJLOGSTREAM "clean duplicate/deleted trajs" findAllMatches +$SRC/analysis/runDistrib.sh $MATCHSTART $MATCHEND $SRC/utils/cleanupDeletedTrajs.sh -log2cw $NJLOGGRP $NJLOGSTREAM "start checkForFailures" findAllMatches +logger -s -t findAllMatches "Solving Run Done" + success=$(grep "Total run time:" $SRC/logs/matchJob.log) if [ "$success" == "" ] then - python -c "from meteortools.utils import sendAnEmail ; sendAnEmail('markmcintyre99@googlemail.com','problem with matching','Error in UKMON matching', mailfrom='ukmonhelper@ukmeteors.co.uk')" + python -c "from utils.sendAnEmail import sendAnEmail ; sendAnEmail('markmcintyre99@googlemail.com','problem with matching','Error in UKMON matching', mailfrom='ukmonhelper@ukmeteors.co.uk')" echo problems with solver fi -log2cw $NJLOGGRP $NJLOGSTREAM "Solving Run Done" findAllMatches -log2cw $NJLOGGRP $NJLOGSTREAM "start rerunFailedLambdas" findAllMatches python -m maintenance.rerunFailedLambdas cd $here -log2cw $NJLOGGRP $NJLOGSTREAM "start reportOfLatestMatches" findAllMatches + +logger -s -t findAllMatches "start reportOfLatestMatches" + matchlog=${SRC}/logs/matchJob.log -python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $MATCHEND $rundate -python -m metrics.getMatchStats $matchlog +python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $rundate +python -m metrics.getMatchStats $matchlog $rundate # copy stats to S3 so the daily report can run if [ "$RUNTIME_ENV" == "PROD" ] ; then aws s3 sync $DATADIR/dailyreports/ $UKMONSHAREDBUCKET/matches/RMSCorrelate/dailyreports/ --quiet fi -log2cw $NJLOGGRP $NJLOGSTREAM "start purgeLogs" findAllMatches find $SRC/logs -name "matches*" -mtime +7 -exec gzip {} \; find $SRC/logs -name "matches*" -mtime +30 -exec rm -f {} \; -log2cw $NJLOGGRP $NJLOGSTREAM "finished findAllMatches" findAllMatches +logger -s -t findAllMatches "finished" diff --git a/archive/analysis/getLogData.sh b/archive/analysis/getLogData.sh index 1f46b6962..67342a511 100644 --- a/archive/analysis/getLogData.sh +++ b/archive/analysis/getLogData.sh @@ -10,7 +10,7 @@ if [ "$1" != "" ] ; then logfile=$DATADIR/lastlogs/lastlog-${rundate}.html else rundate=$(date +%Y%m%d) - logfile=$DATADIR/lastlog.html + logfile=$DATADIR/lastlogs/lastlog.html fi # create performance metrics @@ -78,7 +78,7 @@ echo "

> $logfile -if [ ! -d $DATADIR/lastlogs ] ; then mkdir $DATADIR/lastlogs ; fi +mkdir -p $DATADIR/lastlogs if [ "$1" == "" ] ; then aws s3 cp $logfile $WEBSITEBUCKET/reports/ --quiet cp $logfile $DATADIR/lastlogs/lastlog-${rundate}.html @@ -92,3 +92,5 @@ ls -1r $DATADIR/lastlogs/last*.html | head -90 | while read i ; do done cat $TEMPLATES/footer.html >> $DATADIR/lastlogs/index.html aws s3 cp $DATADIR/lastlogs/index.html $WEBSITEBUCKET/reports/lastlogs/ --quiet + +find $DATADIR/lastlogs -name "lastlog*" -mtime +90 -exec rm -f {} \; diff --git a/archive/analysis/reportActiveShowers.sh b/archive/analysis/reportActiveShowers.sh index a42c90d3e..e26e04638 100644 --- a/archive/analysis/reportActiveShowers.sh +++ b/archive/analysis/reportActiveShowers.sh @@ -30,7 +30,7 @@ fi logger -s -t reportActiveShowers "report on active showers" python -m reports.reportActiveShowers -m -python -c "from meteortools.utils import getActiveShowers; getActiveShowers('$rundt', inclMinor=True)" | while read shwr +python -c "from utils.getActiveShowers import getActiveShowers;getActiveShowers('$rundt', inclMinor=True)" | while read shwr do aws s3 sync $DATADIR/reports/${yr}/$shwr $WEBSITEBUCKET/reports/${yr}/${shwr} --quiet done diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 9350e5db3..1cf23ab35 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -20,129 +20,110 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # load the configuration source $here/../config.ini >/dev/null 2>&1 -# logstream name inherited from parent environment but set it if not -if [ "$NJLOGSTREAM" == "" ]; then - NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) - aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared -fi -log2cw $NJLOGGRP $NJLOGSTREAM "starting runDistrib" runDistrib - -# set the profile to the UKMDA account so we can run the server and monitor progress -export AWS_PROFILE=ukmonshared +logger -s -t runDistrib "starting runDistrib" if [ $# -gt 0 ] ; then if [ "$1" != "" ] ; then - log2cw $NJLOGGRP $NJLOGSTREAM "selecting range" runDistrib MATCHSTART=$1 fi if [ "$2" != "" ] ; then MATCHEND=$2 else - log2cw $NJLOGGRP $NJLOGSTREAM "matchend was not supplied, using 2 days" runDistrib MATCHEND=$(( $MATCHSTART - 2 )) fi fi begdate=$(date --date="-$MATCHSTART days" '+%Y%m%d') rundate=$(date --date="-$MATCHEND days" '+%Y%m%d') -log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" runDistrib +logger -s -t runDistrib "running phase 1 for dates ${begdate} to ${rundate}" +logger -s -t runDistrib "start correlation server" aws ec2 start-instances --instance-ids $SERVERINSTANCEID stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) while [ "$stat" -ne 16 ]; do sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "checking server status" runDistrib stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) done -log2cw $NJLOGGRP $NJLOGSTREAM "running phase 1 for dates ${begdate} to ${rundate}" runDistrib - conda activate $HOME/miniconda3/envs/${WMPL_ENV} -log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" runDistrib +logger -s -t runDistrib "creating the run script" execdist=execdistrib.sh execMatchingsh=/tmp/$execdist python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh $TESTMODE chmod +x $execMatchingsh -log2cw $NJLOGGRP $NJLOGSTREAM "get server details" runDistrib -privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -while [ "$privip" == "" ] ; do - sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "getting IP address" runDistrib - privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -done - -log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" runDistrib +logger -s -t runDistrib "deploy the script to the server $CALCSERVERIP and run it" -scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$privip:data/distrib/$execdist +scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execdist while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 - log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" runDistrib - scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$privip:data/distrib/$execdist + scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execdist done # push the python code and ECS templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/ShowerAssociation.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/utils/convertSolLon.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/utils/ # now run the script -log2cw $NJLOGGRP $NJLOGSTREAM "start distributed processing" runDistrib -ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execdist" +logger -s -t runDistrib "start distributed processing" +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execdist" -rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/candidates/processed/*.tgz $DATADIR/distrib/candidates +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/candidates/processed/*.tgz $DATADIR/distrib/candidates -rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/ -ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;" +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/ +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;" -log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" runDistrib +logger -s -t runDistrib "job run, stop the server again" aws ec2 stop-instances --instance-ids $SERVERINSTANCEID -log2cw $NJLOGGRP $NJLOGSTREAM "monitoring and waiting for completion" runDistrib +logger -s -t runDistrib "monitoring and waiting for completion" python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}', '${TESTMODE}'); " mkdir -p $DATADIR/distrib cd $DATADIR/distrib -log2cw $NJLOGGRP $NJLOGSTREAM "restarting server to consolidate results" runDistrib +logger -s -t runDistrib "restarting server to consolidate results" + stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) if [ $stat -eq 80 ]; then aws ec2 start-instances --instance-ids $SERVERINSTANCEID fi -log2cw $NJLOGGRP $NJLOGSTREAM "waiting for the server to be ready" runDistrib while [ "$stat" -ne 16 ]; do sleep 30 if [ $stat -eq 80 ]; then aws ec2 start-instances --instance-ids $SERVERINSTANCEID fi - log2cw $NJLOGGRP $NJLOGSTREAM "checking - status is ${stat}" runDistrib stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) done -privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) execcons=execconsol.sh execConsolsh=/tmp/$execcons python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh', '$TESTMODE')" chmod +x $execConsolsh -log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib -scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$privip:data/distrib/$execcons -ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execcons" +logger -s -t runDistrib "running consolidation" + +scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execcons +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execcons" -log2cw $NJLOGGRP $NJLOGSTREAM "finished consolidation, copying databases" runDistrib -rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib +logger -s -t runDistrib "finished consolidation, copying databases" -# remote temporary files -ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" -log2cw $NJLOGGRP $NJLOGSTREAM "stopping calcserver again" runDistrib +logger -s -t runDistrib "stopping calcserver again" aws ec2 stop-instances --instance-ids $SERVERINSTANCEID +logger -s -t runDistrib "copying data to batch server and tidying up" + # grab a copy of the indvidual container dbs so we can get a list of new solutions rm -Rf $DATADIR/latest/contdbs/ mkdir -p $DATADIR/latest/contdbs/ @@ -154,8 +135,6 @@ aws s3 mv $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/${rundate}.pickle $DATAD aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $SRC/logs/distrib/ --exclude "*" --include "correl*.log" --exclude "logs/" --quiet aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "correl*.log" --exclude "logs/" --recursive --quiet -log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib - mkdir -p $DATADIR/trajdb tar czvf $DATADIR/trajdb/databases_${rundate}.tgz $DATADIR/distrib/*.db mkdir -p $DATADIR/distrib/containers @@ -168,7 +147,4 @@ find $DATADIR/distrib/containers/ -name "cont*.tgz" -mtime +30 -exec rm -f {} \; find $DATADIR/distrib/ -maxdepth 1 -name "20*.tgz" -mtime +30 -exec rm -f {} \; rm -f $DATADIR/distrib/${rundate}.pickle - -# and then clear the profile again -unset AWS_PROFILE -log2cw $NJLOGGRP $NJLOGSTREAM "finished runDistrib" runDistrib +logger -s -t "finished runDistrib" diff --git a/archive/analysis/updatePlotsAndDetStatus.sh b/archive/analysis/updatePlotsAndDetStatus.sh index beed33373..1ef92a489 100644 --- a/archive/analysis/updatePlotsAndDetStatus.sh +++ b/archive/analysis/updatePlotsAndDetStatus.sh @@ -17,86 +17,67 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # load the configuration source $here/../config.ini >/dev/null 2>&1 -# logstream name inherited from parent environment but set it if not -if [ "$NJLOGSTREAM" == "" ]; then - NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) - aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared -fi -log2cw $NJLOGGRP $NJLOGSTREAM "starting updatePlotsAndDetStatus" updatePlotsAndDetStatus - -# set the profile to the EE account so we can run the server and monitor progress -export AWS_PROFILE=ukmonshared +logger -s -t updatePlotsAndDetStatus "starting updatePlotsAndDetStatus" if [ $# -gt 0 ] ; then if [ "$1" != "" ] ; then - log2cw $NJLOGGRP $NJLOGSTREAM "selecting range" updatePlotsAndDetStatus MATCHSTART=$1 fi if [ "$2" != "" ] ; then MATCHEND=$2 else - log2cw $NJLOGGRP $NJLOGSTREAM "matchend was not supplied, using 2 days" updatePlotsAndDetStatus MATCHEND=$(( $MATCHSTART - 2 )) fi fi begdate=$(date --date="-$MATCHSTART days" '+%Y%m%d') rundate=$(date --date="-$MATCHEND days" '+%Y%m%d') -log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "updating plots etc for dates ${begdate} to ${rundate}" +logger -s -t updatePlotsAndDetStatus "start correlation server" + stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) if [ $stat -eq 80 ]; then aws ec2 start-instances --instance-ids $SERVERINSTANCEID fi -log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" updatePlotsAndDetStatus while [ "$stat" -ne 16 ]; do sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "checking server status" updatePlotsAndDetStatus stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) done -log2cw $NJLOGGRP $NJLOGSTREAM "updating plots etc for dates ${begdate} to ${rundate}" updatePlotsAndDetStatus + conda activate $HOME/miniconda3/envs/${WMPL_ENV} -log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "creating the run script" execrerun=execreplot.sh execrerunsh=/tmp/$execrerun python -c "from traj.createDistribMatchingSh import createExecReplotSh;createExecReplotSh($MATCHSTART, $MATCHEND, '$execrerunsh', '$TESTMODE')" chmod +x $execrerunsh -log2cw $NJLOGGRP $NJLOGSTREAM "get server details" updatePlotsAndDetStatus -privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -while [ "$privip" == "" ] ; do - sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "getting IP address" updatePlotsAndDetStatus - privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -done - -log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "deploy the script to the server $CALCSERVERIP and run it" -scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$privip:data/distrib/$execrerun +scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execrerun while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 - log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" updatePlotsAndDetStatus - scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$privip:data/distrib/$execrerun + scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execrerun done # push the python and templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj # now run the script -ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execrerun" +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execrerun" + +logger -s -t updatePlotsAndDetStatus "job run, stop the server again" -log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" updatePlotsAndDetStatus aws ec2 stop-instances --instance-ids $SERVERINSTANCEID -log2cw $NJLOGGRP $NJLOGSTREAM "get a list of uncalibrated data" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "get a list of uncalibrated data" + aws s3 sync $UKMONSHAREDBUCKET/matches/consumed/ $DATADIR/single/used/ --exclude "*" --include "*.txt" --quiet rundate=$(cat $DATADIR/rundate.txt) -python -c "from analysis.gatherDetectionData import getUncalibratedImageList;getUncalibratedImageList('$rundate');" +python -c "from utils.getUncalImages import getUncalibratedImageList;getUncalibratedImageList('$rundate');" -# and then clear the profile again -unset AWS_PROFILE # refresh the website index pages just in case any new data dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1) $SRC/website/updateIndexPages.sh $dailyrep -log2cw $NJLOGGRP $NJLOGSTREAM "finished updatePlotsAndDetStatus" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "finished updatePlotsAndDetStatus" diff --git a/archive/cronjobs/captureBright.sh b/archive/cronjobs/captureBright.sh index 06ce1561f..b06ed51d2 100644 --- a/archive/cronjobs/captureBright.sh +++ b/archive/cronjobs/captureBright.sh @@ -6,15 +6,9 @@ conda activate $HOME/miniconda3/envs/${WMPL_ENV} cd $DATADIR/brightness rundt=$(date -d "yesterday" +%Y%m%d) -python -m analysis.compareBrightnessData $rundt +python -m utils.compareBrightnessData $rundt -mysql -u batch -p$(cat ~/.ssh/db_batch.passwd) -h localhost << EOD -use ukmon; -select count(*) from ukmon.brightness; -load data local infile '${DATADIR}/brightness/CaptureNight_${rundt}.csv' -into table brightness fields terminated by ','; -select count(*) from ukmon.brightness; -EOD +$SRC/utils/loadBrightnessCsvMDB.sh find ${DATADIR}/brightness -name "CaptureNight*" -mtime +30 -exec rm -f {} \; - +find ${DATADIR}/brightness -name "matcheddata*" -mtime +30 -exec rm -f {} \; \ No newline at end of file diff --git a/archive/cronjobs/gatherMonthlyVideos.sh b/archive/cronjobs/gatherMonthlyVideos.sh index 52f670486..e8c9c1464 100644 --- a/archive/cronjobs/gatherMonthlyVideos.sh +++ b/archive/cronjobs/gatherMonthlyVideos.sh @@ -70,7 +70,7 @@ echo "mkdir \$1 ; cd \$1" >> $outdir/getVideos.ps1 echo "foreach(\$line in Get-Content ../best_\$1.txt) { wget https://archive.ukmeteors.co.uk/\$line }" >> $outdir/getVideos.ps1 echo "cd .." >> $outdir/getVideos.ps1 -aws s3 sync $outdir $UKMONSHAREDBUCKET/videos/ --profile ukmonshared --quiet --exclude "*" --include "getVideos*" --include "README*" +aws s3 sync $outdir $UKMONSHAREDBUCKET/videos/ --quiet --exclude "*" --include "getVideos*" --include "README*" -aws s3 sync $outdir $WEBSITEBUCKET/data/bestvideos/ --profile ukmonshared --quiet --include "*.txt" +aws s3 sync $outdir $WEBSITEBUCKET/data/bestvideos/ --quiet --include "*.txt" diff --git a/archive/cronjobs/getImoWSfile.sh b/archive/cronjobs/getImoWSfile.sh index 021b6bad6..d7cb03996 100644 --- a/archive/cronjobs/getImoWSfile.sh +++ b/archive/cronjobs/getImoWSfile.sh @@ -19,7 +19,7 @@ git checkout wmpl/share/streamfulldata.csv #git checkout wmpl/share/ShowerLookUpTable.txt git checkout wmpl/share/gmn_shower_table_20230518.txt -python -c "from meteortools.utils.getShowerDates import numpifyShowerData; numpifyShowerData()" +python -c "from utils.getActiveShowers import numpifyShowerData; numpifyShowerData()" logger -s -t getImoWSfile "finished" diff --git a/archive/cronjobs/nightlyJob.sh b/archive/cronjobs/nightlyJob.sh index 1c9eb1204..5d63a02aa 100644 --- a/archive/cronjobs/nightlyJob.sh +++ b/archive/cronjobs/nightlyJob.sh @@ -7,12 +7,7 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 conda activate $HOME/miniconda3/envs/${WMPL_ENV} -NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) -export NJLOGSTREAM -aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared - -# log2cw is a function defined in config.ini and logs to cloudwatch, syslog and console -log2cw $NJLOGGRP $NJLOGSTREAM "start nightlyJob" nightlyJob +logger -s -t nightlyJob "start nightlyJob" # dates to process for rundate=$(date +%Y%m%d) @@ -25,139 +20,118 @@ mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls} mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajdb,videos} mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} -# update the JSON file containing camera location details. This is used by the website -log2cw $NJLOGGRP $NJLOGSTREAM "updating the camera location/dir/fov database" nightlyJob -python -c "from reports.CameraDetails import updateCamLocDirFovDB; updateCamLocDirFovDB();" -aws s3 cp $DATADIR/admin/cameraLocs.json $UKMONSHAREDBUCKET/admin/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/admin/cameraLocs.json $WEBSITEBUCKET/browse/ --profile ukmonshared --quiet -aws s3 sync $UKMONSHAREDBUCKET/admin/ $DATADIR/admin --profile ukmonshared --quiet - -# create the CSV file of camera info and the html versions for search functions on the website -log2cw $NJLOGGRP $NJLOGSTREAM "updating the camera details files for searching" nightlyJob -python -c "from reports.CameraDetails import createCDCsv; createCDCsv('consolidated');" -aws s3 cp $DATADIR/consolidated/camera-details.csv $UKMONSHAREDBUCKET/consolidated/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/statopts.html $WEBSITEBUCKET/search/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/activestatopts.html $WEBSITEBUCKET/search/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/activestatlocs.html $WEBSITEBUCKET/search/ --profile ukmonshared --quiet +#################################################################################### +# START OF DATA PROCESSING. FIRST WE UPDATE THE SEARCH PAGE WITH SINGLE-STATION +# AND CAMERA DETAILS SO IT CAN BE SEARCHED WITHOUT WAITING FOR THE MATCHING ENGINE +#################################################################################### -# set up logging for the match process -log2cw $NJLOGGRP $NJLOGSTREAM "start findAllMatches" nightlyJob -matchlog=matchJob.log -if [ -f $SRC/logs/$matchlog ] ; then - suff=$(stat matchJob.log -c %X) - mv $SRC/logs/$matchlog $SRC/logs/$matchlog-$suff -fi +# update the JSON and html files containing camera and location details, used by the website +$SRC/website/updateCameraDets.sh -log2cw $NJLOGGRP $NJLOGSTREAM "start getRMSSingleData" nightlyJob -# this creates the parquet table for Athena +# consolidate the single-station data - we do this here so the search index can be updated earlier +# and we can search for single-station events without waiting for the main batch $SRC/analysis/getRMSSingleData.sh if [ "$(date +%m%d)" == "0101" ] ; then # catch any data uploaded on 01/01 that is for 31/12 the previous year $SRC/analysis/getRMSSingleData.sh $(date -d 'last year' +%Y) fi -log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 1" nightlyJob - +# now update the search index with single-station data $SRC/analysis/createSearchable.sh $yr singles +#################################################################################### +# NOW THE RUN THE MATCHING ENGINE VIA findAllmatches.sh +# Take care rerunning it as it will re-create the daily report +#################################################################################### + +# set up logging for the match process +matchlog=matchJob.log + +# save the existing log, in case the process is being rerun on the same day +if [ "$(find $SRC/logs -name $matchlog -mmin +1380 -ls)" != "" ] ; then + dt=$(stat $SRC/logs/$matchlog -c %y) + suff=$(date --date ${dt:0:10} +%Y%m%d) + mv -f $SRC/logs/$matchlog $SRC/logs/$matchlog-$suff +fi + +logger -s -t nightlyJob "start findAllMatches" # Run the match process - run this only once as it scoops up all unprocessed data ${SRC}/analysis/findAllMatches.sh > ${SRC}/logs/${matchlog} 2>&1 -log2cw $NJLOGGRP $NJLOGSTREAM "start updateIndexPages" nightlyJob + +#################################################################################### +# FROM HERE DOWN WE'RE CONSOLIDATING DATA AND CREATING REPORTS +# and everything can be rerun safely provided the data are present +#################################################################################### + +# update the website daily, monthly and annual index pages where needed $SRC/website/updateIndexPages.sh -# FIRST CONSOLIDATE THE DATA AND CREATE SEARCH INDEXES -# consolidate the output of the match process for further analysos -log2cw $NJLOGGRP $NJLOGSTREAM "start consolidateOutput" nightlyJob +# consolidate the output of the match process for further analysis $SRC/analysis/consolidateOutput.sh ${yr} if [ "$(date +%m%d)" == "0101" ] ; then # catch any data uploaded on 01/01 that is for 31/12 the previous year $SRC/analysis/consolidateOutput.sh $(date -d 'last year' +%Y) fi -# create the search indexes used on the website -log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 2" nightlyJob +# update the search indexes used on the website $SRC/analysis/createSearchable.sh $yr matches -# FROM HERE DOWN WE'RE CREATING REPORTS - # add daily report to the website -log2cw $NJLOGGRP $NJLOGSTREAM "start publishDailyReport" nightlyJob $SRC/website/publishDailyReport.sh # create monthly and per-shower CSV extracts of the data -log2cw $NJLOGGRP $NJLOGSTREAM "start createMthlyExtracts" nightlyJob ${SRC}/website/createMthlyExtracts.sh ${mth} - -log2cw $NJLOGGRP $NJLOGSTREAM "start createShwrExtracts" nightlyJob ${SRC}/website/createShwrExtracts.sh ${rundate} # create the fireballs page -log2cw $NJLOGGRP $NJLOGSTREAM "start createFireballPage" nightlyJob #requires search index to have been updated first ${SRC}/website/createFireballPage.sh ${yr} -3.99 # create a report of activity for the current month and whole year -log2cw $NJLOGGRP $NJLOGSTREAM "start showerReport ALL $mth" nightlyJob $SRC/analysis/showerReport.sh ALL ${mth} force - -log2cw $NJLOGGRP $NJLOGSTREAM "start showerReport ALL $yr" nightlyJob $SRC/analysis/showerReport.sh ALL ${yr} force # if we ran on the 1st of the month we need to catch any late-arrivals for last month if [ $(date +%d) -eq 1 ] ; then lastmth=$(date -d '-1 month' +%Y%m) - log2cw $NJLOGGRP $NJLOGSTREAM "start showerMthlyExtracts ALL $lastmth" nightlyJob ${SRC}/website/createMthlyExtracts.sh ${lastmth} - log2cw $NJLOGGRP $NJLOGSTREAM "start createShwrExtracts ALL $lastmth" nightlyJob ${SRC}/website/createShwrExtracts.sh ${lastmth}28 - log2cw $NJLOGGRP $NJLOGSTREAM "start showerReport ALL $lastmth" nightlyJob $SRC/analysis/showerReport.sh ALL ${lastmth} force fi # create a per-shower report for any currently active showers -log2cw $NJLOGGRP $NJLOGSTREAM "start reportActiveShowers" nightlyJob ${SRC}/analysis/reportActiveShowers.sh ${yr} # create the website front page -log2cw $NJLOGGRP $NJLOGSTREAM "start createSummaryTable" nightlyJob ${SRC}/website/createSummaryTable.sh # create the camera status reports -log2cw $NJLOGGRP $NJLOGSTREAM "start cameraStatusReport" nightlyJob ${SRC}/website/cameraStatusReport.sh $rundate -log2cw $NJLOGGRP $NJLOGSTREAM "start createExchangeFiles" nightlyJob +# create the files for exchange with other networks - i think this is unused python -c "from reports.createExchangeFiles import createAll;createAll();" -aws s3 sync $DATADIR/browse/daily/ $WEBSITEBUCKET/browse/daily/ --region eu-west-2 --quiet +aws s3 sync $DATADIR/browse/daily/ $WEBSITEBUCKET/browse/daily/ --quiet cd $DATADIR # do this manually when on PC required as it requires too much memory for the batch server; closes #61 # python $PYLIB/maintenance/plotStationsOnMap.py False -aws s3 cp $DATADIR/stations.png $WEBSITEBUCKET/ --region eu-west-2 --quiet +aws s3 cp $DATADIR/stations.png $WEBSITEBUCKET/ --quiet rm -f $SRC/data/.nightly_running # various reports for management - bad stations, costs, next batch start time. -log2cw $NJLOGGRP $NJLOGSTREAM "start getBadStations" nightlyJob $SRC/analysis/getBadStations.sh - -log2cw $NJLOGGRP $NJLOGSTREAM "start costReport" nightlyJob $SRC/website/costReport.sh - -# set time of next run python $PYLIB/maintenance/getNextBatchStart.py 150 # create station reports. This takes a while hence done after everything else -log2cw $NJLOGGRP $NJLOGSTREAM "start stationReports" nightlyJob $SRC/analysis/stationReports.sh # clear down space where possible -log2cw $NJLOGGRP $NJLOGSTREAM "start clearSpace" nightlyJob $SRC/utils/clearSpace.sh # load the MariaDB with the latest data. The mariadb database isn't used much -log2cw $NJLOGGRP $NJLOGSTREAM "update MariaDB tables" nightlyJob $SRC/utils/loadMatchCsvMDB.sh $SRC/utils/loadSingleCsvMDB.sh @@ -165,12 +139,10 @@ $SRC/utils/loadSingleCsvMDB.sh # has to be run quite late as not all trajectories have synced to the website earlier $SRC/analysis/updatePlotsAndDetStatus.sh +# push the API data dictionary to the website for end-user use aws s3 sync $SRC/share/ s3://ukmda-website/browse --exclude "*" --include "datadictionary.xlsx" --quiet -log2cw $NJLOGGRP $NJLOGSTREAM "finished nightlyJob" nightlyJob +logger -s -t nightlyJog "finished nightlyJob" # grab the logs for the website - run this last to capture the above Finished message $SRC/analysis/getLogData.sh - - - diff --git a/archive/database/Setup.md b/archive/database/Setup.md index 1aef81b10..328cab345 100644 --- a/archive/database/Setup.md +++ b/archive/database/Setup.md @@ -1,42 +1,16 @@ # Installing MariaDB -## Create /etc/yum.repos.d/MariaDB.repo - -[mariadb] -name = MariaDB -baseurl = http://yum.mariadb.org/10.3/centos7-amd64 -gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB -gpgcheck=1 - -## Install packages - -yum update -yum install mariadb-server mariadb-client mariadb-devel mariadb-shared mariadb-common - -## Create Databases and Tables -``` -mysql -u root -h localhost -set root password with -update mysql.user set password= PASSWORD('newpassword') where user = 'root'; -flush privileges; -exit; -``` -execute contents of create_dbs_and_users.sql -execute contents of create_tables.sql - -## migrating to new server -make sure you've installed mariadb on the new server and created databases and users -make sure you can ssh from old to new -then on the old server run -``` -mysqldump -u batch -p'passwd' ukmon | ssh ukmonhelper2 mysql -u batch -p'passwd' ukmon -``` +## Installing and Migrating +For installation and migration instructions see `migratingBatchServer.md` in the `server_setup` folder ## Connecting from Python +These packages are installed as part of the ukmda tooling. +``` bash pip install mysql-connector-python pip install pymysql - +``` +``` python import pymysql.cursors connection = pymysql.connect(host='localhost', @@ -46,12 +20,13 @@ pip install pymysql cursorclass=pymysql.cursors.DictCursor) try: with connection.cursor() as cursor: - sql = "INSERT INTO books VALUES ({},'{}', {}, {})".format(sys.argv[1],'foo',34,56) + #sql = "INSERT INTO books VALUES ({},'{}', {}, {})".format(sys.argv[1],'foo',34,56) cursor.execute(sql) connection.commit() - sql = "SELECT * from `books` " + sql = "SELECT * from matches limit 10 " cursor.execute(sql) result = cursor.fetchall() print(result) finally: connection.close() +``` \ No newline at end of file diff --git a/archive/database/ddl/create_tables.sql b/archive/database/ddl/create_tables.sql index c51027569..ba036622e 100644 --- a/archive/database/ddl/create_tables.sql +++ b/archive/database/ddl/create_tables.sql @@ -28,8 +28,9 @@ create table singles( AngVel float, Shwr varchar(6), filname varchar(64), - Dtstamp double(18,6)), - status varchar(3); + Dtstamp double(18,6), + status varchar(3) + ); alter ignore table ukmon.singles add unique uniq_Dtstamp (Dtstamp); diff --git a/archive/deployment/analysis.yml b/archive/deployment/analysis.yml deleted file mode 100644 index f65068693..000000000 --- a/archive/deployment/analysis.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/analysis exists - file: path={{destdir}}/analysis state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/analysis/updatePlotsAndDetStatus.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/consolidateOutput.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/createSearchable.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/findAllMatches.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/getBadStations.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/getLogData.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/getRMSSingleData.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/reportActiveShowers.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/runDistrib.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/showerReport.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/stationReports.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/templates/extracsv.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/match_hdr_full.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/UA_header.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/UO_header.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/ukmon-single.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/README.md', dest: '{{destdir}}/analysis', mode: '644', backup: no } diff --git a/archive/deployment/config.yml b/archive/deployment/config.yml deleted file mode 100644 index 88ad1408a..000000000 --- a/archive/deployment/config.yml +++ /dev/null @@ -1,29 +0,0 @@ -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: dev-vars.yml - tags: dev - - name: import prod variables - include_vars: prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/utils exists - file: path={{destdir}}/utils state=directory - tags: [dev,prod] - - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/utils/makeConfig.sh', dest: '{{destdir}}/utils/', mode: '754', backup: yes } - - {src: '{{srcdir}}/utils/stopstart-calcengine.sh', dest: '{{destdir}}/utils/', mode: '754', backup: yes } - - {src: '{{srcdir}}/server_setup/.bashrc', dest: '/home/ec2-user', mode: '644', backup: yes } - - {src: '{{srcdir}}/server_setup/.bash_aliases', dest: '/home/ec2-user', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/.condaon', dest: '/home/ec2-user', mode: '644', backup: no } - - - name: create Config file - shell: '{{destdir}}/utils/makeConfig.sh {{env}}' - tags: [dev,prod] diff --git a/archive/deployment/cronjobs.yml b/archive/deployment/cronjobs.yml deleted file mode 100644 index 6a80c1b5e..000000000 --- a/archive/deployment/cronjobs.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/Dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/cronjobs exists - file: path={{destdir}}/cronjobs state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/cronjobs/nightlyJob.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/captureBright.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/getImoWSfile.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/mergeNewOrbit.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/deleteDuplicateOrbit.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/gatherMonthlyVideos.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/README.md', dest: '{{destdir}}/cronjobs', mode: '644', backup: no } diff --git a/archive/deployment/database.yml b/archive/deployment/database.yml deleted file mode 100644 index 3abf9d787..000000000 --- a/archive/deployment/database.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/database exists - file: path={{destdir}}/database state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - - {src: '{{srcdir}}/database/ddl/create_brightness_table.sql', dest: '{{destdir}}/database/ddl', mode: '644', backup: no } - - {src: '{{srcdir}}/database/ddl/create_dbs_and_users.sql', dest: '{{destdir}}/database/ddl', mode: '644', backup: no } - - {src: '{{srcdir}}/database/ddl/create_tables.sql', dest: '{{destdir}}/database/ddl', mode: '644', backup: no } - - {src: '{{srcdir}}/database/dml/load_data.sql', dest: '{{destdir}}/database/dml', mode: '644', backup: no } - - {src: '{{srcdir}}/database/Setup.md', dest: '{{destdir}}/database', mode: '644', backup: no } - - {src: '{{srcdir}}/database/testDB.py', dest: '{{destdir}}/database', mode: '644', backup: no } - - {src: '{{srcdir}}/database/README.md', dest: '{{destdir}}/database', mode: '644', backup: no } diff --git a/archive/deployment/dev-vars.yml b/archive/deployment/dev-vars.yml deleted file mode 100644 index 86dfcf575..000000000 --- a/archive/deployment/dev-vars.yml +++ /dev/null @@ -1,3 +0,0 @@ -destdir: /home/ec2-user/dev -env: DEV -s3profile: default diff --git a/archive/deployment/full_deployment.yml b/archive/deployment/full_deployment.yml deleted file mode 100644 index b682b18d1..000000000 --- a/archive/deployment/full_deployment.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- import_playbook: cronjobs.yml -- import_playbook: analysis.yml -- import_playbook: website.yml -- import_playbook: database.yml -- import_playbook: pylib.yml -- import_playbook: server_setup.yml -- import_playbook: shwrinfo.yml -- import_playbook: utils.yml diff --git a/archive/deployment/prod-vars.yml b/archive/deployment/prod-vars.yml deleted file mode 100644 index 20171b912..000000000 --- a/archive/deployment/prod-vars.yml +++ /dev/null @@ -1,3 +0,0 @@ -destdir: /home/ec2-user/prod -env: PROD -s3profile: ukmda_admin \ No newline at end of file diff --git a/archive/deployment/pylib.yml b/archive/deployment/pylib.yml deleted file mode 100644 index 313173390..000000000 --- a/archive/deployment/pylib.yml +++ /dev/null @@ -1,112 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/ukmon_pylib exists - file: path={{destdir}}/ukmon_pylib state=directory - tags: [dev,prod] - - name: clean local filesystem - ansible.builtin.command: 'pyclean {{srcdir}}/ukmon_pylib -v --debris all' - delegate_to: localhost - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - - {src: '{{srcdir}}/ukmon_pylib/analysis/compareBrightnessData.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/gatherDetectionData.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/meteorFlux.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/showerAnalysis.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/stationAnalysis.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/summaryAnalysis.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/__init__.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/README.md', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/converters/gmnTxtToPandas.py', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/converters/toParquet.py', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/converters/__init__.py', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/converters/README.md', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/maintenance/compressOldArchiveData.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/cleanDdbTables.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/dataMaintenance.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/deDuplicate.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/getNextBatchStart.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/getUserAndKeyInfo.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/hcVaultActions.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/manageTraj.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/plotStationsOnMap.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/purgeOldImages.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/purgeOldOrbits.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/recreateOrbitPages.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/rerunFailedLambdas.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/sortToDateDirs.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/README.md', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/metrics/camMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/costMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/getMatchStats.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/statsToMqtt.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/timingMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/volumeMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/README.md', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/reports/CameraDetails.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/cameraStatusReport.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createAnnualBarChart.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createExchangeFiles.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createSearchableFormat.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createSummaryTable.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/dailyReport.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/extractors.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/filterBylocDirBri.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/findBestMp4s.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/findFailedMatches.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/findFireballs.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/getLivestreamData.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/getSolutionStati.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/makeCoverageMap.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/reportActiveShowers.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/reportBadCameras.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/reportOfLatestMatches.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/trackThumbnails.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/__init__.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/README.md', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/requirements.txt', dest: '{{destdir}}/ukmon_pylib', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/README.md', dest: '{{destdir}}/ukmon_pylib', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BM.jpg', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BMNG_hirez.png', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BMUK_hirez.png', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BMWE_hirez.png', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BM_medres.jpg', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/images.json', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/population.jpg', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/README.md', dest: '{{destdir}}/ukmon_pylib/share', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails-mda.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails-mm.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetailstest-ukmda.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/consolidateDistTraj.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/createDistribMatchingSh.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/distributeCandidates.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/gatherMissedPlatepars.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/manualBulkReruns.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/pickleAnalyser.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/plotOSMGroundTrack.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/ShowerAssociation.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/taskrunner.json', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/taskrunner_test.json', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/README.md', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } diff --git a/archive/deployment/server_setup.yml b/archive/deployment/server_setup.yml deleted file mode 100644 index f53ea3a7b..000000000 --- a/archive/deployment/server_setup.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/server_setup exists - file: path={{destdir}}/server_setup state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/server_setup/copyTestData.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-cartopy.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-geos.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-proj4.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-sqlite3.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install_opencv.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - #- {src: '{{srcdir}}/server_setup/libs/libgeos-3.7.2.so', dest: '{{destdir}}/server_setu/libsp', mode: '644', backup: no } - #- {src: '{{srcdir}}/server_setup/libs/libgeos.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - #- {src: '{{srcdir}}/server_setup/libs/libgeos_c.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - #- {src: '{{srcdir}}/server_setup/libs/libproj.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/migratingBatchServer.md', dest: '{{destdir}}/server_setup', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/newserver.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/remount-s3.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/ssh-setup.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/yum-installs.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/README.md', dest: '{{destdir}}/server_setup', mode: '644', backup: no } diff --git a/archive/deployment/shwrinfo.yml b/archive/deployment/shwrinfo.yml deleted file mode 100644 index c7bbfd11f..000000000 --- a/archive/deployment/shwrinfo.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/data/shwrinfo exists - file: path={{destdir}}/data/shwrinfo state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/shwrinfo/AUR.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/CAP.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/COM.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/costnotes.html', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/DAD.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/ETA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/GEM.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/HYD.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/KCG.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/LEO.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/LYR.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/NOO.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/NTA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/ORI.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/PER.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/QUA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/SDA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/STA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/template.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/URS.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/README.md', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } diff --git a/archive/deployment/utils.yml b/archive/deployment/utils.yml deleted file mode 100644 index 73dd5f02f..000000000 --- a/archive/deployment/utils.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/utils exists - file: path={{destdir}}/utils state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/utils/calcserver/clearSpace.sh', dest: '{{destdir}}/utils/calcserver', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/cftpd_templ.json', dest: '{{destdir}}/utils', mode: '644', backup: no } - - {src: '{{srcdir}}/utils/checkAndRollKeys.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/clearCaches.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/clearSpace.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/createTestDataSet.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/deleteOrbit.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/getCostMetrics.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/liveevent.template', dest: '{{destdir}}/utils', mode: '644', backup: no } - - {src: '{{srcdir}}/utils/loadMatchCsvMDB.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/loadSingleCsvMDB.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/makeConfig.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/monthlyCleardown.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/rerunFTPtoUKMONlambda.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/rerunGetExtraFiles.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/ses-msg-templ.json', dest: '{{destdir}}/utils', mode: '644', backup: no } - - {src: '{{srcdir}}/utils/statsToMqtt.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/stopstart-calcengine.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/updateDb.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/updateFireballFlag.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/updateFireballImage.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/userAudit.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/README.md', dest: '{{destdir}}/utils', mode: '644', backup: no } diff --git a/archive/deployment/website.yml b/archive/deployment/website.yml deleted file mode 100644 index 78956ddaa..000000000 --- a/archive/deployment/website.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/website exists - file: path={{destdir}}/website state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/website/cameraStatusReport.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/costReport.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createFireballPage.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createMthlyExtracts.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createOrbitIndex.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createReportIndex.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createShwrExtracts.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createSummaryTable.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/publishDailyReport.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/pushStatic.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/README.md', dest: '{{destdir}}/website', mode: '644', backup: no } - - {src: '{{srcdir}}/website/updateIndexPages.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - - {src: '{{srcdir}}/website/templates/coverage-maps.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/fbreportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/footer.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/frontpage.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/header.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/reportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/shwrcsvindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/statreportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } diff --git a/archive/deployment/website_static.yml b/archive/deployment/website_static.yml deleted file mode 100644 index c15044f48..000000000 --- a/archive/deployment/website_static.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/static_content exists - file: path={{destdir}}/static_content state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/static_content/browse/', dest: '{{destdir}}/static_content/browse', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/css/', dest: '{{destdir}}/static_content/css', mode: '754', backup: no, directory_mode: yes } - #- {src: '{{srcdir}}/static_content/data/', dest: '{{destdir}}/static_content/data', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/docs/', dest: '{{destdir}}/static_content/docs', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/fonts/', dest: '{{destdir}}/static_content/fonts', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/img/', dest: '{{destdir}}/static_content/img', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/js/', dest: '{{destdir}}/static_content/js', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/latest/', dest: '{{destdir}}/static_content/latest', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/live/', dest: '{{destdir}}/static_content/live', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/search/', dest: '{{destdir}}/static_content/search', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/templates/', dest: '{{destdir}}/static_content/templates', mode: '754', backup: no, directory_mode: yes } - - - {src: '{{srcdir}}/static_content/favicon.ico', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/fundraising.html', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/privacy_data.html', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/README.md', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/robots.txt', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - - #- name: Update Static Content on the S3 website bucket - # shell: '{{destdir}}/website/pushStatic.sh {{env}}' - # tags: [dev,prod] diff --git a/archive/lambdas/matchDataApi/matchDataApi.py b/archive/lambdas/matchDataApi/matchDataApi.py index 3000292de..73c393db9 100644 --- a/archive/lambdas/matchDataApi/matchDataApi.py +++ b/archive/lambdas/matchDataApi/matchDataApi.py @@ -81,7 +81,7 @@ def getStationData(statid, dtstr, period=None): def getSummaryData(dtstr, period=None): host, user, passwd, db = getSqlLoginDetails() perfrag = periodToSqlFragment(period) if period is not None else "" - connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) + connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) fieldlist = '_localtime,_mjd,_sol,_ID1,_amag,_ra_o,_dc_o,_ra_t,_dc_t,_elng,_elat,_vo,_vi,_vg,_vs,_a,_q,_e,_p,_peri,_node,_incl,'\ '_stream,_mag,_dur,_lng1,_lat1,_H1,_lng2,_lat2,_H2,_LD21,_az1r,_ev1r,_Nts,_Nos,_leap,_tme,_dt,'\ 'dtstamp,orbname,iau,shwrname as name,mass,pi,Q,true_anom,EA,MA,Tj,T,last_peri,jacchia1,Jacchia2,numstats,stations' diff --git a/archive/server_setup/.bash_aliases b/archive/server_setup/.bash_aliases index 00d569938..fa6eb3529 100644 --- a/archive/server_setup/.bash_aliases +++ b/archive/server_setup/.bash_aliases @@ -4,6 +4,7 @@ alias h='history' alias df='df -h' alias du='du -h' +alias ls='ls --color=auto' alias data='if [ "$SRC" == "" ] ; then echo select env first; else cd $SRC/data && pwd ; fi' alias logs='if [ "$SRC" == "" ] ; then echo select env first; else cd $SRC/logs && pwd ; fi' @@ -16,28 +17,30 @@ alias stats='if [ "$DATADIR" == "" ] ; then echo select env first; else tail $DA alias matchstatus='if [ "$SRC" == "" ] ; then echo select env first; else grep "Running" $SRC/logs/matchJob.log && grep TRAJ $SRC/logs/matchJob.log | grep SOLVING && echo -n "Completed " && grep Observations: $SRC/logs/matchJob.log | wc -l && grep "nightlyJob" $SRC/logs/nightlyJob.log ; fi ' alias spacecalc='ls -1 | egrep -v "ukmon-shared" | while read i ; do \du -s $i ; done | sort -n' -alias startcalc='$SRC/utils/stopstart-calcengine.sh start' -alias stopcalc='$SRC/utils/stopstart-calcengine.sh stop' +alias startcalc='$SRC/utils/stopstartCalcengine.sh start' +alias stopcalc='$SRC/utils/stopstartCalcengine.sh stop' function dev { - source ~/dev/config.ini >/dev/null - conda activate $HOME/miniconda3/envs/${WMPL_ENV} - PS1="(wmpl) (dev) [\W]\$ " + source ~/dev/config.ini + #conda activate $HOME/miniconda3/envs/${WMPL_ENV} + conda activate ${WMPL_ENV} + PS1="(dev) [\W]\$ " cd ~/dev } function prd { - source ~/prod/config.ini >/dev/null - conda activate $HOME/miniconda3/envs/${WMPL_ENV} - PS1="(wmpl) (prd) [\W]\$ " + source ~/prod/config.ini #>/dev/null + #conda activate $HOME/miniconda3/envs/${WMPL_ENV} + conda activate ${WMPL_ENV} + PS1="(prd) [\W]\$ " cd ~/prod } function calcserver { - sts=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State --output text --profile ukmonshared) - ipaddr=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared) + sts=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State --output text) + ipaddr=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) isrunning=$(echo $sts | cut -d " " -f 1) if [ $isrunning -ne 16 ] ; then - $HOME/prod/utils/stopstart-calcengine.sh start + $HOME/prod/utils/stopstartCalcengine.sh start echo "starting server on ${ipaddr}... waiting 10s..." sleep 10 fi diff --git a/archive/server_setup/.bashrc b/archive/server_setup/.bashrc index 79e18c870..a7f39fa35 100644 --- a/archive/server_setup/.bashrc +++ b/archive/server_setup/.bashrc @@ -42,15 +42,27 @@ export PATH=$PATH:$(dirname $(which node)) # >>> conda initialize >>> # !! Contents within this block are managed by 'conda init' !! -__conda_setup="$('/home/ec2-user/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +__conda_setup="$('/home/ubuntu/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" if [ $? -eq 0 ]; then eval "$__conda_setup" else - if [ -f "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" ]; then - . "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" + if [ -f "/home/ubuntu/miniconda3/etc/profile.d/conda.sh" ]; then + . "/home/ubuntu/miniconda3/etc/profile.d/conda.sh" else - export PATH="/home/ec2-user/miniconda3/bin:$PATH" + export PATH="/home/ubuntu/miniconda3/bin:$PATH" fi fi unset __conda_setup # <<< conda initialize <<< + +# If this is an xterm set the title to user@host:dir +case "$TERM" in + xterm*|rxvt*) + PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1" + ;; + *) + ;; +esac +export rundate=$(date +%Y%m%d) +conda activate wmpl + diff --git a/archive/server_setup/.condaon b/archive/server_setup/.condaon index b69b186cd..a8766953f 100644 --- a/archive/server_setup/.condaon +++ b/archive/server_setup/.condaon @@ -1,13 +1,14 @@ # >>> conda initialize >>> -__conda_setup="$('/home/ec2-user/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +__conda_setup="$('~/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" if [ $? -eq 0 ]; then eval "$__conda_setup" else - if [ -f "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" ]; then - . "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" + if [ -f "~/miniconda3/etc/profile.d/conda.sh" ]; then + . "~/miniconda3/etc/profile.d/conda.sh" else - export PATH="/home/ec2-user/miniconda3/bin:$PATH" + export PATH="~/miniconda3/bin:$PATH" fi fi unset __conda_setup +. ~/miniconda3/etc/profile.d/conda.sh # <<< conda initialize <<< diff --git a/archive/server_setup/.vimrc b/archive/server_setup/.vimrc new file mode 100644 index 000000000..2fd7a2b6c --- /dev/null +++ b/archive/server_setup/.vimrc @@ -0,0 +1 @@ +colo slate diff --git a/archive/server_setup/logs.conf b/archive/server_setup/logs.conf new file mode 100644 index 000000000..910c3bf2b --- /dev/null +++ b/archive/server_setup/logs.conf @@ -0,0 +1,24 @@ +daily +rotate 7 +notifempty +~/prod/logs/*.log { + daily + rotate 45 + compress + delaycompress + dateext + notifempty + maxage 45 + # create +} + +~/dev/logs/*.log { + daily + rotate 45 + compress + delaycompress + dateext + notifempty + maxage 45 + # create +} diff --git a/archive/server_setup/migrateSftpAccts.sh b/archive/server_setup/migrateSftpAccts.sh new file mode 100755 index 000000000..52d5add0f --- /dev/null +++ b/archive/server_setup/migrateSftpAccts.sh @@ -0,0 +1,44 @@ +#!/bin/bash + + +addOneUser() { + userid=$1 + srchost=$2 + grep -w $userid /etc/passwd + if [ $? -eq 1 ] ; then + dt=$(date +%Y-%m-%d) + logger -s -t addSftpUser "Creating unix user $userid" + sudo useradd --system --shell /usr/sbin/nologin --groups sftp --home /var/sftp/$userid --comment "${dt}" $userid + sudo mkdir /var/sftp/$userid + sudo chown root:sftp /var/sftp/$userid + sudo chmod 751 /var/sftp/$userid + # create the .ssh folder, platepar folder and empty client copy of the ini file + sudo mkdir /var/sftp/$userid/.ssh + sudo mkdir /var/sftp/$userid/platepar + sudo touch /var/sftp/$userid/ukmon.ini.client + # make these three writeable by the client + sudo chown $userid:$userid /var/sftp/$userid/platepar /var/sftp/$userid/.ssh /var/sftp/$userid/ukmon.ini.client + else + logger -s -t addSftpUser "Unix user $userid already exists" + fi + sudo rsync -av $srchost:/var/sftp/$userid/ /var/sftp/$userid + sudo cat /var/sftp/$userid/ukmon.ini | sed 's/3.11.55.160/batchserver.ukmeteors.co.uk/g' > /tmp/$userid.ini + sudo mv /tmp/$userid.ini /var/sftp/$userid/ukmon.ini + sudo scp /var/sftp/$userid/ukmon.ini $srchost:/var/sftp/$userid/ +} + +if [ $# -lt 2 ] ; then + echo "Usage: ./migrateSftpAccounts.sh oldservername userfile" + exit +fi + +echo "Warning: this must only be run on the new server" +read -p "press ctrl-c to quit or enter to continue" + +oldserver=$1 +srcfile=$2 + +cat $srcfile | while read stn +do + addOneUser $stn $oldserver +done \ No newline at end of file diff --git a/archive/server_setup/migratingBatchServer.md b/archive/server_setup/migratingBatchServer.md index 584a4d90b..1bf7db16e 100644 --- a/archive/server_setup/migratingBatchServer.md +++ b/archive/server_setup/migratingBatchServer.md @@ -1,68 +1,172 @@ # Replacing the UKMON helper server The ukmon helper server provides two services. -* camera authentication and key management * batch processing +* camera authentication and key management -# how to move batch processing -The best approach is to reinstall all the code using the ansible deployment scripts. -Both dev and prod envs should be created, though arguably the dev env should be on a separate server. -Additionally, WMPL will need to be reinstalled. -After doing this, the additional UKMON python requirements can be added. +## how to move batch processing +* Build a new Ubuntu server from the Terraform by cloning the existing batch server details in `terraform/ukmda/batchserver.tf` and making any necessary changes then deploying it with terraform in the normal way. +* I do not recommend using Amazon Linux as this doesn't contain many of the base requirements like C++, development libraries, GEOS and PROJ, so you would need to install and/or build these from source which is tedious. +* If you want to set the hostname to eg `batchserver1`, then do the following + * edit `/etc/cloud/cloud.cfg` and set `preserve_hostname: true` + * run `sudo hostnamctl hostname batchserver1` + * now the new hostname should be preserved through boot -The contents of ~/prod/data should be replicated to the new server. -The contents of ~/keymgmt should be replicated to the new server. +### Prerequisites +* install some prerequisites via apt +``` bash +sudo apt-get install unzip net-tools dos2unix mariadb-server +``` -Finally, double check that all required SSM variables exist in the account holding the server. These -are used to build the config file, but originally were only in Mark McIntyre's account. +### Miniconda +* Install miniconda with the default settings -## how to move accounts to a new server -The basic process is to extract the user accounts from the current server along with -group and password info, then import it back in on the new server. Its important to avoid -accidentally overwriting system or otherwise-existing accounts on the new server. +### WMPL +* Install WMPL + * create a 'wmpl' conda environment with at least python 3.13 + * install the python requirements + * See note below for how to get PyQt5 working. + * Alternatively you can delete subfolders of wmpl that rely on QT (`CAM0` `MetSIM` and `Utils/DynamicMassFit.py`). +``` bash +conda create -n wmpl python=3.13 +mkdir -p ~/src +cd ~/src +git clone --recursive git@github.com:markmac99/WesternMeteorPyLib.git +cd WesternMeteorPyLib +conda activate wmpl +pip install -r requirements.txt +python setup.py +``` -On most systems, accoutns below 500 are system accounts. Check though, as some AWS servers create -new accounts starting at 1000 and working both upwards and downwards! +### UKMDA Dataprocessing +* Clone the ukmda git repo and then install all the code using the deployment script. Both dev and prod envs should be created, though arguably the dev env should be on a separate server. +``` bash +cd ~/src +git clone git@github.com:ukmda/ukmda-dataprocessing.git +cd ukmda-dataprocessing +./install_or_upgrade.sh PROD +``` +* double check that all required SSM variables exist in the account holding the server. These +are used to build the config file and are deployed via terraform so should be present! For example: +``` bash +aws ssm get-parameters --region eu-west-2 --names prod_siteurl --query Parameters[0].Value +"https://archive.ukmeteors.co.uk" +``` -Steps in brief. NB must all be done as root, of course. +### SSH and other API keys +copy the below files from ~/.ssh on the old server, and make sure permissions are correct (should all be 0600). At a minimum you need the following files +``` bash +gmailcreds.json +gmailtoken.json +``` +These are used to authenticate against gmail. The keys are currently specific to Mark McIntyre's account and will need to be replaced with keys for `ukmeteors@gmail.com` + +You may also need the `github` keys though this depends on how you authenticate with GitHub. -### On the old server +## data +Replicate `~/prod/data`, `~/prod/logs` and `~/keymgmt` to the new server. (once for prod and once for dev). + + This will have to be repeated every day till golive (strictly, the key data only needs to be replicated if a new camera is added). Also keep the MariaDB SQL database up to date. ``` bash -mkdir /root/move/ -export UGIDLIMIT=500 -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/passwd > /root/move/passwd.mig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/group > /root/move/group.mig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534) {print $1}' /etc/passwd | tee - |egrep -f - /etc/shadow > /root/move/shadow.mig -cp /etc/gshadow /root/move/gshadow.mig +cd $DATADIR +rsync -avz ukmonhelper2:prod/data/ . +cd $SRC/logs +rsync -avz ukmonhelper2:prod/logs/ . +cd ~ +rsync -avz ukmonhelper2:keymgmt/ ./keymgmt +$SRC/utils/loadSingleCsvMDB.sh +$SRC/utils/loadMatchCsvMDB.sh + +``` +## Mariadb database +Find the mariadb config file (normally in /etc, might ber called `my.cnf`) and ensure that `bind-address` is set to `0.0.0.0`. The default on many installations is to bind to 127.0.0.1 which allows traffic only from localhost. The UKMON APIs access the database externally. After changing the config file, restart the database. -# Also backup the user homedirs. In our case, they're all in /var/sftp -tar cvfz /root/move/varsftp.tar.gz /var/sftp -# now copy the files to target server, -scp /root/move/* newserver:/tmp +Now sudo to root and run the following to set a root password +``` bash +mysql -u root -p +ALTER USER 'root'@'localhost' IDENTIFIED VIA mysql_native_password USING PASSWORD('ROOTPASSWORD'); +FLUSH PRIVILEGES; +quit ``` +Pbviously, replace `ROOTPASSWORD` with the password in the SSM variable `prod_dbpw` -### On the new server -In summary: backup the existing files, remove any accounts from the .mig files -that are already present in the target, then append the filtered data. +Exit the root shell, and as a 'normal' user execute the following: +``` bash +cd $SRC/database/ +mysql -u root -pROOTPASSWORD < ddl/create_dbs_and_users.sql +mysql -u root -pROOTPASSWORD < ddl/create_tables.sql +mysql -u root -pROOTPASSWORD < ddl/create_brightness_table.sq +``` +Now dump and reload the databases from the old server. +Make sure the old server can connect to the new one then on the old server, run the following: +``` bash +mysqldump -u batch -p'passwd' ukmon | ssh newserver mysql -u batch -p'passwd' ukmon +``` -When comparing groups, remember new ids will get added to the sftp group. +### To install PyQt5 +* needs at least 3GB memory so add 2GB swap if the server has less. +``` bash +sudo fallocate -l 2G /swapfile +sudo chmod 600 /swapfile +sudo mkswap /swapfile +sudo swapon /swapfile +echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab + +sudo apt install qtbase5-dev qt5-qmake +pip install pyqt5 +``` + +### To get conda working in batch scripts +This is done by sourcing a file `.condaon` which is automatically installed by the ukmda deployment script and sourced from the environments' config.ini files. Make sure any scripts activate conda like this: + +``` bash +source $HOME/prod/config.ini >/dev/null 2>&1 +conda activate ${WMPL_ENV} +``` + +## how to move accounts to a new server + +### Install and configure sftp +First set up the SFTP server on the new host. +Run the following as root: + +``` bash +groupadd sftp +mkdir -p /var/sftp +chown root:root /var/sftp +chmod 751 /var/sftp +``` + +Add this to /etc/ssh/sshd_config +``` bash +Match group sftp +ChrootDirectory /var/sftp/%u +AllowTCPForwarding no +X11Forwarding no +ForceCommand internal-sftp +``` +and then reload sshd ``` bash -mkdir -p /root/move/bkp -mv /tmp/*.mig /tmp/varsftp.tar.gz /root/move -cp /etc/passwd /etc/group /etc/shadow /etc/gshadow /root/move/bkp +service sshd reload +``` -cd / -tar -xvf /root/move/varsftp.tar.gz . +### Now move the user accounts and update camera config -export UGIDLIMIT=500 -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/passwd > /root/move/passwd.orig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/group > /root/move/group.orig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534) {print $1}' /etc/passwd | tee - |egrep -f - /etc/shadow > /root/move/shadow.orig +First, ensure that root on the new server can connect to the old server as the batch user by creating a default SSH keypair for root, and adding its public half to root's authorized_keys file on the old server. -cd /root/move -diff passwd.orig passwd.mig -diff shadow.orig shadow.mig -diff group.orig group.mig +* on the new server as the standard batch user: + * create a folder `move` + * Create a list of the desired SFTP user accounts using +``` bash +ssh oldserver "sudo ls -1 /var/sftp" > ./move/sftp_accts.txt ``` + * Edit the list to exclude defunct accounts and other entries not related to a camera account. + * use `$SRC/$utils/migrateSftpAccts.sh` to create accounts on the new server and copy over the user data +``` bash +$SRC/utils/migrateSftpAccts.sh oldserverFQDN ./move/sftp_accts.txt +``` +This will also update the `ukmon.ini` file for each station on both server. Upon next connection, the station should update its local ini file and thereafter connect to the new server. + +This means that the old server must be kept running for a few days till all cameras have switched over. diff --git a/archive/server_setup/ssh-setup.sh b/archive/server_setup/ssh-setup.sh deleted file mode 100644 index f2c0cbf77..000000000 --- a/archive/server_setup/ssh-setup.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -echo $1 -scp -i ~/.ssh/markskey.pem ~/.ssh/ukmon-shared-keys $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/markskey.pem $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/config $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/gm* $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/ukmon* $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/marksk* $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/s3* $1:.ssh - -ssh -i ~/.ssh/markskey.pem $1 mkdir setup -ssh -i ~/.ssh/markskey.pem $1 mkdir data -ssh -i ~/.ssh/markskey.pem $1 mkdir src - diff --git a/archive/server_setup/install-cartopy.sh b/archive/server_setup/unused/install-cartopy.sh similarity index 100% rename from archive/server_setup/install-cartopy.sh rename to archive/server_setup/unused/install-cartopy.sh diff --git a/archive/server_setup/install-geos.sh b/archive/server_setup/unused/install-geos.sh similarity index 100% rename from archive/server_setup/install-geos.sh rename to archive/server_setup/unused/install-geos.sh diff --git a/archive/server_setup/install-proj4.sh b/archive/server_setup/unused/install-proj4.sh similarity index 100% rename from archive/server_setup/install-proj4.sh rename to archive/server_setup/unused/install-proj4.sh diff --git a/archive/server_setup/install-sqlite3.sh b/archive/server_setup/unused/install-sqlite3.sh similarity index 100% rename from archive/server_setup/install-sqlite3.sh rename to archive/server_setup/unused/install-sqlite3.sh diff --git a/archive/server_setup/install_opencv.sh b/archive/server_setup/unused/install_opencv.sh similarity index 100% rename from archive/server_setup/install_opencv.sh rename to archive/server_setup/unused/install_opencv.sh diff --git a/archive/server_setup/newserver.sh b/archive/server_setup/unused/newserver.sh similarity index 100% rename from archive/server_setup/newserver.sh rename to archive/server_setup/unused/newserver.sh diff --git a/archive/server_setup/remount-s3.sh b/archive/server_setup/unused/remount-s3.sh similarity index 100% rename from archive/server_setup/remount-s3.sh rename to archive/server_setup/unused/remount-s3.sh diff --git a/archive/server_setup/yum-installs.sh b/archive/server_setup/unused/yum-installs.sh similarity index 100% rename from archive/server_setup/yum-installs.sh rename to archive/server_setup/unused/yum-installs.sh diff --git a/archive/shwrinfo/costnotes.html b/archive/shwrinfo/costnotes.html index 09a5714c8..08b211c37 100644 --- a/archive/shwrinfo/costnotes.html +++ b/archive/shwrinfo/costnotes.html @@ -4,19 +4,4 @@

  • Amounts of less than $0.10 per day are consolidated under Other.
  • AWS cost data may lag by up to a day, so yesterday's costs data are incomplete.
  • AWS charge VAT at the start of the month for the previous month's total bill.
    • -
  • Until September 2023, costs were split across two accounts for historical reasons. We've now consolidated into one account.
    • -
    -2023 Costs prior to 13th September were as follows: -
  • S3: $1135
    • -
  • ECS: $212
    • -
  • EC2-other: $159
    • -
  • EC2-instance: $131
    • -
  • Cloudwatch: $44
    • -
  • Registrar: $29
    • -
  • KMS: $23
    • -
  • DynamoDB: $15
    • -
  • Route53: $6
    • -
  • ECR: $5
    • -
  • Other: £8
    • - -Total: $1768 + VAT +
    \ No newline at end of file diff --git a/archive/terraform/mjmm/aws.tf b/archive/terraform/mjmm/aws.tf index e18c7d50e..a9871b9e3 100644 --- a/archive/terraform/mjmm/aws.tf +++ b/archive/terraform/mjmm/aws.tf @@ -6,8 +6,8 @@ provider "aws" { } provider "aws" { - alias = "eeacct" - region = var.remote_region - profile = "ukmonshared" + profile = var.profile + region = "eu-west-1" + alias = "eu-west-1-prov" } diff --git a/archive/terraform/mjmm/crossacct-role.tf b/archive/terraform/mjmm/crossacct-role.tf index fabcd4ecd..858365ade 100644 --- a/archive/terraform/mjmm/crossacct-role.tf +++ b/archive/terraform/mjmm/crossacct-role.tf @@ -1,7 +1,5 @@ # Copyright (C) 2018-2023 Mark McIntyre -data "aws_caller_identity" "eeacct" {provider = aws.eeacct } - # RESTRICT THESE PERMISSIONS!! data "aws_iam_policy" "terraformpol" { name = "AdministratorAccess" diff --git a/archive/terraform/mjmm/dev_ssm_parameters.tf b/archive/terraform/mjmm/dev_ssm_parameters.tf index a9216604f..819844f67 100644 --- a/archive/terraform/mjmm/dev_ssm_parameters.tf +++ b/archive/terraform/mjmm/dev_ssm_parameters.tf @@ -65,6 +65,15 @@ resource "aws_ssm_parameter" "dev_calcuser" { } } +resource "aws_ssm_parameter" "dev_calcip" { + name = "dev_calcserverip" + type = "String" + value = "172.32.16.137" + tags = { + "billingtag" = "ukmon" + } +} + resource "aws_ssm_parameter" "dev_backupinstance" { name = "dev_backupinstance" type = "String" @@ -127,3 +136,4 @@ resource "aws_ssm_parameter" "dev_batchloggroup" { "billingtag" = "ukmon" } } + diff --git a/archive/terraform/mjmm/ssm_parameters.tf b/archive/terraform/mjmm/ssm_parameters.tf index 73dff36bc..8889f3a64 100644 --- a/archive/terraform/mjmm/ssm_parameters.tf +++ b/archive/terraform/mjmm/ssm_parameters.tf @@ -1,6 +1,45 @@ # Copyright (C) 2018-2023 Mark McIntyre # SSM parameters for use in the environment config +resource "aws_ssm_parameter" "prod_dbhost" { + provider = aws.eu-west-1-prov + name = "prod_dbhost" + type = "String" + value = "3.11.55.160" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_dbname" { + provider = aws.eu-west-1-prov + name = "prod_dbname" + type = "String" + value = "ukmon" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_dbpw" { + provider = aws.eu-west-1-prov + name = "prod_dbpw" + type = "SecureString" + value = "Batch33mdl" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_dbuser" { + provider = aws.eu-west-1-prov + name = "prod_dbuser" + type = "String" + value = "batch" + tags = { + "billingtag" = "ukmda" + } +} resource "aws_ssm_parameter" "prod_websitebucket" { name = "prod_websitebucket" @@ -67,6 +106,14 @@ resource "aws_ssm_parameter" "prod_calcuser" { } } +resource "aws_ssm_parameter" "prod_calcip" { + name = "prod_calcserverip" + type = "String" + value = "172.32.16.137" + tags = { + "billingtag" = "ukmon" + } +} resource "aws_ssm_parameter" "prod_backupinstance" { name = "prod_backupinstance" type = "String" @@ -130,3 +177,11 @@ resource "aws_ssm_parameter" "prod_batchloggroup" { } } +resource "aws_ssm_parameter" "prod_gmapsapikey" { + name = "prod_gmapsapikey" + type = "SecureString" + value = "AIzaSyBFadTuzvLfkUhz8CwY2CtRDJ_lYlHUYyA" + tags = { + "billingtag" = "ukmon" + } +} diff --git a/archive/terraform/ukmda/batchserver.tf b/archive/terraform/ukmda/batchserver.tf new file mode 100644 index 000000000..7ff5993f8 --- /dev/null +++ b/archive/terraform/ukmda/batchserver.tf @@ -0,0 +1,47 @@ +# Copyright (C) 2018-2023 Mark McIntyre + +resource "aws_instance" "batchserver" { + ami = "ami-06f0cf249257c89b3" + instance_type = "t4g.micro" + iam_instance_profile = aws_iam_instance_profile.calcserverrole.name + key_name = aws_key_pair.marks_key.key_name + force_destroy = false + + root_block_device { + tags = { + "Name" = "batchservervol" + "billingtag" = "ukmda" + } + volume_size = 50 + volume_type = "gp3" + throughput = 125 + iops = 3000 + encrypted = true + kms_key_id = aws_kms_key.container_key.arn + } + + tags = { + "Name" = "batchserver" + "billingtag" = "ukmda" + "Route53FQDN" = "batchserver.ukmeteors.co.uk" + "DNSRecordType" = "A" + } + primary_network_interface { + network_interface_id = aws_network_interface.batchserver_if.id + } + metadata_options { + http_tokens = "required" + } +} + +resource "aws_network_interface" "batchserver_if" { + subnet_id = aws_subnet.ec2_subnet.id + description = "Primary network interface" + private_ips = [var.batchserverip] + security_groups = [aws_security_group.ec2_secgrp.id] + ipv6_address_list_enabled = false + tags = { + "Name" = "batchserver" + "billingtag" = "ukmda" + } +} diff --git a/archive/terraform/ukmda/calcserver.tf b/archive/terraform/ukmda/calcserver.tf index fe1248458..a1aa2a468 100644 --- a/archive/terraform/ukmda/calcserver.tf +++ b/archive/terraform/ukmda/calcserver.tf @@ -1,46 +1,5 @@ # Copyright (C) 2018-2023 Mark McIntyre -resource "aws_instance" "calc_server" { - ami = "ami-0df2d8f6def0bd716" - instance_type = "c8g.2xlarge" - iam_instance_profile = aws_iam_instance_profile.calcserverrole.name - key_name = aws_key_pair.marks_key.key_name - force_destroy = false - tags = { - "Name" = "Calcengine" - "billingtag" = "ukmda" - "Route53FQDN" = "calcengine.ukmeteors.co.uk" - "DNSRecordType" = "A" - } - root_block_device { - tags = { - "Name" = "calcengine" - "billingtag" = "ukmda" - } - volume_size = 120 - } - primary_network_interface { - network_interface_id = aws_network_interface.calcserver_if.id - } - - metadata_options { - http_tokens = "required" - } -} - -# elastic network interface attached to the calc server - -resource "aws_network_interface" "calcserver_if" { - subnet_id = aws_subnet.ec2_subnet.id - description = "Primary network interface" - private_ips = [var.calcserverip] - security_groups = [aws_security_group.ec2_secgrp.id] - ipv6_address_list_enabled = false - tags = { - "Name" = "calcengine" - "billingtag" = "ukmda" - } -} ################################################ # Ubuntu calc server ################################################ diff --git a/archive/terraform/ukmda/cloudwatch.tf b/archive/terraform/ukmda/cloudwatch.tf index 61ab690a0..ee336b1dd 100644 --- a/archive/terraform/ukmda/cloudwatch.tf +++ b/archive/terraform/ukmda/cloudwatch.tf @@ -21,7 +21,7 @@ resource "aws_cloudwatch_metric_alarm" "calcServerIdle" { "arn:aws:automate:${var.region}:ec2:stop", ] dimensions = { - "InstanceId" = aws_instance.calc_server.id + "InstanceId" = aws_instance.ubuntu_calc_server.id } tags = { "billingtag" = "ukmda" diff --git a/archive/terraform/ukmda/dev_ssm_variables.tf b/archive/terraform/ukmda/dev_ssm_variables.tf new file mode 100644 index 000000000..bca543597 --- /dev/null +++ b/archive/terraform/ukmda/dev_ssm_variables.tf @@ -0,0 +1,191 @@ +# Copyright (C) 2018-2023 Mark McIntyre + +# SSM parameters for use in Lambdas and the batch + +resource "aws_ssm_parameter" "dev_dbhost" { + provider = aws.eu-west-1-prov + name = "dev_dbhost" + type = "String" + value = "3.11.55.160" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_dbname" { + provider = aws.eu-west-1-prov + name = "dev_dbname" + type = "String" + value = "ukmon" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_dbpw" { + provider = aws.eu-west-1-prov + name = "dev_dbpw" + type = "SecureString" + value = "Batch33mdl" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_rootdbpw" { + provider = aws.eu-west-1-prov + name = "dev_rootdbpw" + type = "SecureString" + value = "Wombat33mdb" + tags = { + "billingtag" = "ukmda" + } +} + + +resource "aws_ssm_parameter" "dev_dbuser" { + provider = aws.eu-west-1-prov + name = "dev_dbuser" + type = "String" + value = "batch" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_websitebucket" { + name = "dev_websitebucket" + type = "String" + value = var.dev_websitebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_sharedbucket" { + name = "dev_sharedbucket" + type = "String" + value = var.dev_sharedbucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_livebucket" { + name = "dev_livebucket" + type = "String" + value = var.dev_livebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_siteurl" { + name = "dev_siteurl" + type = "String" + value = "https://www.ukmeteors.co.uk/dummy/" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_envname" { + name = "dev_envname" + type = "String" + value = "DEV" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_calcinstance" { + name = "dev_calcinstance" + type = "String" + value = "i-0ab47af23705beb31" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_calcuser" { + name = "dev_calcuser" + type = "String" + value = "ubuntu" + tags = { + "billingtag" = "ukmda" + } +} + + +resource "aws_ssm_parameter" "dev_calcserverip" { + name = "dev_calcserverip" + type = "String" + value = var.ubuntu_calcserverip + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_wmplhome" { + name = "dev_wmplhome" + type = "String" + value = "$HOME/src/wmpldev" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_rmshome" { + name = "dev_rmshome" + type = "String" + value = "$HOME/src/RMS" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_srcdir" { + name = "dev_srcdir" + type = "String" + value = "$HOME/dev" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_caminfo" { + name = "dev_caminfo" + type = "String" + value = "$HOME/dev/data/consolidated/camera-details.csv" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_sshkey" { + name = "dev_sshkey" + type = "String" + value = "$HOME/.ssh/markskey.pem" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_batchloggroup" { + name = "dev_batchloggroup" + type = "String" + value = "/ukmondev/nightlyjob" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_gmapsapikey" { + name = "dev_gmapsapikey" + type = "SecureString" + value = "AIzaSyBFadTuzvLfkUhz8CwY2CtRDJ_lYlHUYyA" + tags = { + "billingtag" = "ukmda" + } +} + diff --git a/archive/terraform/ukmda/secgrp.tf b/archive/terraform/ukmda/secgrp.tf index 1ae17dd00..a6aa5e1d4 100644 --- a/archive/terraform/ukmda/secgrp.tf +++ b/archive/terraform/ukmda/secgrp.tf @@ -46,8 +46,8 @@ resource "aws_security_group" "ec2_secgrp" { description = "Security Group for EC2 instances" vpc_id = aws_vpc.ec2_vpc.id ingress = [ -/* { - cidr_blocks = [data.aws_vpc.mjmm_ec2_vpc.cidr_block] + { + cidr_blocks = [aws_vpc.ec2_vpc.cidr_block] description = "SSH for Admin" from_port = 22 protocol = "tcp" @@ -56,9 +56,9 @@ resource "aws_security_group" "ec2_secgrp" { prefix_list_ids = [] security_groups = [] self = false - },*/ + }, { - cidr_blocks = [aws_vpc.ec2_vpc.cidr_block] + cidr_blocks = ["0.0.0.0/0"] description = "SSH for Admin" from_port = 22 protocol = "tcp" @@ -69,16 +69,38 @@ resource "aws_security_group" "ec2_secgrp" { self = false }, { - cidr_blocks = ["0.0.0.0/0"] - description = "SSH for Admin" - from_port = 22 + cidr_blocks = ["194.0.0.0/8"] + description = "MariaDB" + from_port = 3306 protocol = "tcp" - to_port = 22 + to_port = 3306 ipv6_cidr_blocks = [] prefix_list_ids = [] security_groups = [] self = false - } + }, + { + cidr_blocks = ["0.0.0.0/0"] + description = "icmp" + from_port = -1 + ipv6_cidr_blocks = [] + prefix_list_ids = [] + protocol = "icmp" + security_groups = [] + self = false + to_port = -1 + }, + { + cidr_blocks = ["0.0.0.0/0"] + description = "mariadb" + from_port = 3306 + ipv6_cidr_blocks = [] + prefix_list_ids = [] + protocol = "tcp" + security_groups = [] + self = false + to_port = 3306 + }, ] egress = [ { diff --git a/archive/terraform/ukmda/ssm_variables.tf b/archive/terraform/ukmda/ssm_variables.tf index 29fba5ac6..5661b5078 100644 --- a/archive/terraform/ukmda/ssm_variables.tf +++ b/archive/terraform/ukmda/ssm_variables.tf @@ -1,14 +1,14 @@ # Copyright (C) 2018-2023 Mark McIntyre -# SSM parameters for use in Lambdas +# SSM parameters for use in Lambdas and the batch resource "aws_ssm_parameter" "prod_dbhost" { provider = aws.eu-west-1-prov name = "prod_dbhost" type = "String" - value = "3.11.55.160" + value = "batchserver.ukmeteors.co.uk" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" } } @@ -18,7 +18,7 @@ resource "aws_ssm_parameter" "prod_dbname" { type = "String" value = "ukmon" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" } } @@ -28,7 +28,17 @@ resource "aws_ssm_parameter" "prod_dbpw" { type = "SecureString" value = "Batch33mdl" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_rootdbpw" { + provider = aws.eu-west-1-prov + name = "prod_rootdbpw" + type = "SecureString" + value = "Wombat33mdb" + tags = { + "billingtag" = "ukmda" } } @@ -38,6 +48,143 @@ resource "aws_ssm_parameter" "prod_dbuser" { type = "String" value = "batch" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_websitebucket" { + name = "prod_websitebucket" + type = "String" + value = var.websitebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_sharedbucket" { + name = "prod_sharedbucket" + type = "String" + value = var.sharedbucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_livebucket" { + name = "prod_livebucket" + type = "String" + value = var.livebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_siteurl" { + name = "prod_siteurl" + type = "String" + value = "https://archive.ukmeteors.co.uk" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_envname" { + name = "prod_envname" + type = "String" + value = "PROD" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_calcinstance" { + name = "prod_calcinstance" + type = "String" + value = "i-0ab47af23705beb31" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_calcuser" { + name = "prod_calcuser" + type = "String" + value = "ubuntu" + tags = { + "billingtag" = "ukmda" } } + + +resource "aws_ssm_parameter" "prod_calcserverip" { + name = "prod_calcserverip" + type = "String" + value = var.ubuntu_calcserverip + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_wmplhome" { + name = "prod_wmplhome" + type = "String" + value = "$HOME/src/WesternMeteorPyLib" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_rmshome" { + name = "prod_rmshome" + type = "String" + value = "$HOME/src/RMS" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_srcdir" { + name = "prod_srcdir" + type = "String" + value = "$HOME/prod" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_caminfo" { + name = "prod_caminfo" + type = "String" + value = "$HOME/prod/data/consolidated/camera-details.csv" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_sshkey" { + name = "prod_sshkey" + type = "String" + value = "$HOME/.ssh/markskey.pem" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_batchloggroup" { + name = "prod_batchloggroup" + type = "String" + value = "/ukmonbatch/nightlyjob" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_gmapsapikey" { + name = "prod_gmapsapikey" + type = "SecureString" + value = "AIzaSyBFadTuzvLfkUhz8CwY2CtRDJ_lYlHUYyA" + tags = { + "billingtag" = "ukmda" + } +} + diff --git a/archive/terraform/ukmda/variables.tf b/archive/terraform/ukmda/variables.tf index e73774644..01d6aef6b 100644 --- a/archive/terraform/ukmda/variables.tf +++ b/archive/terraform/ukmda/variables.tf @@ -8,6 +8,10 @@ variable "sharedbucket" { default = "ukmda-shared" } variable "livebucket" { default = "ukmda-live" } variable "adminbucket" { default = "ukmda-admin" } +variable "dev_websitebucket" { default = "ukmda-website" } +variable "dev_sharedbucket" { default = "ukmda-shared" } +variable "dev_livebucket" { default = "ukmda-live" } + variable "region" { default = "eu-west-2"} variable "liveregion" { default = "eu-west-1"} @@ -18,14 +22,13 @@ data "aws_canonical_user_id" "current" {} variable "remote_profile" { default = "default"} variable "remote_account_id" { default = "317976261112" } variable "remote_region" {default = "eu-west-2"} -#variable "eeaccountid" {default = "822069317839" } variable "main_cidr" {default = "172.32.0.0/16" } variable mgmt_cidr { default = "172.32.36.0/22" } variable lambda_cidr { default = "172.32.32.0/22" } variable ec2_cidr { default = "172.32.16.0/20" } -variable calcserverip { default = "172.32.16.136" } variable ubuntu_calcserverip { default = "172.32.16.137" } +variable batchserverip { default = "172.32.16.138" } variable archalias { default = "archive.ukmeteors.co.uk"} diff --git a/archive/ukmon_pylib/requirements.txt b/archive/ukmon_pylib/additional_requirements.txt similarity index 67% rename from archive/ukmon_pylib/requirements.txt rename to archive/ukmon_pylib/additional_requirements.txt index 8b9f56c62..c5942af22 100644 --- a/archive/ukmon_pylib/requirements.txt +++ b/archive/ukmon_pylib/additional_requirements.txt @@ -1,52 +1,37 @@ # Copyright (C) 2018-2023 Mark McIntyre #cartopy INSTALL THIS WITH CONDA -# skip this -#https://github.com/matplotlib/basemap/archive/master.zip -basemap basemap-data-hires boto3 -Cython gmplot google-auth google-auth-oauthlib google-api-python-client googleapis-common-protos -jplephem -Keras -matplotlib==3.3.2 -numpy oauthlib -pandas Pillow pycparser pyephem -pyproj==2.6.1.post1 -PyQt5 +pyproj pyswarms -pytz -PyYAML -scipy Shapely xmltodict -tweepy fastparquet cryptography python-crontab simplekml astropy -gitpython pymysql mysql-connector-python pre-commit s3fs -# pyarrow +dynamodb_json +requests +paho-mqtt +scp +# testing requirements pytest behave pdoc -dynamodb_json pytest pytest-cov -requests -paho-mqtt -scp \ No newline at end of file diff --git a/archive/ukmon_pylib/analysis/README.md b/archive/ukmon_pylib/analysis/README.md index 732ae6a45..a4b09f7d4 100644 --- a/archive/ukmon_pylib/analysis/README.md +++ b/archive/ukmon_pylib/analysis/README.md @@ -1,9 +1,8 @@ # Analysis -This folder contains functions to analyse the single station and matched data. +This folder contains functions to analyse the single station and matched data for website reporting. * ShowerAnalysis.py - analysis for a named shower for a specific year eg GEM 2022 * stationAnalysis.py - analysis for a named year or year+month, for one or all stations. -* meteorFlux - plot a graph of meteor flux -* gatherDetectionData.py -collects all available data on a detection or detections matching a pattern YYYYMMDD_HHMMSS. If the seconds are omitted, then all detections within the specified minute will be returned. - +* summaryAnalysis.py - summary analysis for a given period + diff --git a/archive/ukmon_pylib/analysis/gatherDetectionData.py b/archive/ukmon_pylib/analysis/gatherDetectionData.py deleted file mode 100644 index d7e45b9b8..000000000 --- a/archive/ukmon_pylib/analysis/gatherDetectionData.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Collect data for a potential match at a specific time -# -# Copyright (C) 2018-2023 Mark McIntyre -# -import os -import sys -import datetime -import pandas as pd - -from meteortools.ukmondb import getECSVs, getLiveJpgs, getFBfiles, getDetections - - -def getUncalibratedImageList(dtstr=None): - datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') - flines = open(logfile, 'r').readlines() - uncal = [f for f in flines if 'not recalibrated' in f] - imglist = [x.strip(' ').split(' ')[1][:-1] for x in uncal] - with open(os.path.join(datadir, 'single', 'used', f'uncal_{dtstr}.txt'), 'w') as outf: - for li in imglist: - outf.write(f'{li}\n') - return imglist - - -def getRawData(idlist, outpth): - for _,row in idlist.iterrows(): - stat =row.ID - dts = datetime.datetime.strptime(row.Dtstamp, '%Y%m%d_%H%M%S') - dt = dts.strftime('%Y-%m-%dT%H:%M:%S') - fboutdir = os.path.join(outpth, stat) - dtstr = row.Filename[10:25] - getLiveJpgs(dtstr, outpth, False) - getFBfiles(f'{stat}_{dtstr}', fboutdir) - getECSVs(stat, dt, True, fboutdir) - return - - -def updateSingleDB(yr, consdt): - datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - sngls = pd.read_csv(os.path.join(datadir, 'single', f'singles-{yr}.csv')) - consdf = pd.read_csv(os.path.join(datadir, 'single', 'used', f'consumed_{consdt}.txt'), names=['Filename']) - conslist = list(consdf.Filename) - for ff in conslist: - mtch = sngls.Filename == ff - if mtch.any(): - print(f'updating {ff}') - sngls.loc[mtch, 'status'] = 'Y' - else: - print(f'skipping {ff}') - sngls.to_csv(os.path.join(datadir, 'single', f'singles-{yr}.csv'), index=False) - - -if __name__ == '__main__': - outpth = f'./{sys.argv[1]}' - os.makedirs(outpth, exist_ok=True) - idlist = getDetections(sys.argv[1]) - if not isinstance(idlist, bool): - getRawData(idlist, outpth) - ids = list(idlist.ID) - with open(os.path.join(outpth, 'ids.txt'), 'w') as outf: - outf.writelines(line + '\n' for line in ids) - print(ids) - else: - print('no matches') diff --git a/archive/ukmon_pylib/analysis/getUsedUnused.py b/archive/ukmon_pylib/analysis/getUsedUnused.py new file mode 100644 index 000000000..0541ecad6 --- /dev/null +++ b/archive/ukmon_pylib/analysis/getUsedUnused.py @@ -0,0 +1,65 @@ +# +# Python code to analyse meteor shower data +# +# Copyright (C) 2018-2023 Mark McIntyre + +import os +import boto3 +import json +import pymysql.cursors + + +def getSqlLoginDetails(): + # retrieve password and host from SSM. This allows me to manage them from Terraform + ssm = boto3.client('ssm', region_name='eu-west-1') + res = ssm.get_parameter(Name='prod_dbpw', WithDecryption=True) + password = res['Parameter']['Value'] + res = ssm.get_parameter(Name='prod_dbhost') + host = res['Parameter']['Value'] + # should really do these too but they won't change often if at all + user = 'batch' + db = 'ukmon' + return host, user, password, db + + +def getAllImages(dtstr): + """ + Retrieve a list of all images for a given date from the SQL database + This is a list of all images containing detections, whether or not used + in a solution. + """ + host, user, passwd, db = getSqlLoginDetails() + connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) + sqlstr = f'select filname from singles where Y={dtstr[:4]} and M={int(dtstr[4:6])} and D={int(dtstr[6:8])}' + with connection.cursor() as cursor: + cursor.execute(sqlstr) + result = cursor.fetchall() + connection.close() + res=[] + for event in result: + res.append(event['filname']) + return res + + +def getKnownImages(dtstr, datatype='consumed'): + """ + Get a list of all images that were involved in a match or were uncalibrated, + depending on the value of datatype (consumed or uncal) + The raw data are generated by a process on the matching engine server + """ + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + srcfile = os.path.join(datadir,'single','used',f'{datatype}_{dtstr}.txt') + imglist = [] + if os.path.isfile(srcfile): + used = open(srcfile,'r').readlines() + for img in used: + imglist.append(img.strip()) + return imglist + + +def getUnusedImages(dtstr): + + allimages = getAllImages(dtstr) + uncal = getKnownImages(dtstr, 'uncal') + used = getKnownImages(dtstr, 'consumed') + unused = list(set(allimages)-set(used + uncal)) \ No newline at end of file diff --git a/archive/ukmon_pylib/analysis/showerAnalysis.py b/archive/ukmon_pylib/analysis/showerAnalysis.py index 02551d78a..7defd5930 100644 --- a/archive/ukmon_pylib/analysis/showerAnalysis.py +++ b/archive/ukmon_pylib/analysis/showerAnalysis.py @@ -10,8 +10,8 @@ from matplotlib import pyplot as plt import datetime -from meteortools.utils import getShowerDates as sd -from meteortools.fileformats import imoWorkingShowerList as imo +from utils.getActiveShowers import getShowerDets +from utils.imoWorkingShowerList import IMOshowerList SMALL_SIZE = 8 MEDIUM_SIZE = 10 @@ -97,8 +97,8 @@ def timeGraph(dta, shwrname, outdir, binmins=10): countcol = dta['Shwr'] # resample it - binned = countcol.resample('{}min'.format(binmins)).count() - binned.plot(kind='bar') + binned = countcol.resample(f'{binmins}min').count() + binned.plot(kind='bar', width=10) # set ticks and labels every 144 intervals nbins = max(len(binned)/144, 2) @@ -138,8 +138,8 @@ def matchesGraphs(dta, shwrname, outdir, binmins=60, startdt=None, enddt=None, t #select just the shower ID col mcountcol = mdta['_ID1'] # resample it - mbinned = mcountcol.resample('{}min'.format(binmins)).count() - mbinned.plot(kind='bar') + mbinned = mcountcol.resample(f'{binmins}min').count() + mbinned.plot(kind='bar', width=10) # set ticks and labels every 144 intervals ax = plt.gca() @@ -216,7 +216,7 @@ def velDistribution(dta, shwrname, outdir, vg_or_vs, binwidth=0.2): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -254,7 +254,7 @@ def durationDistribution(dta, shwrname, outdir, binwidth=0.2): # format x-axes x_labels=["%.1f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -292,7 +292,7 @@ def distanceDistribution(dta, shwrname, outdir, binwidth=1.0): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -356,13 +356,6 @@ def radiantDistribution(dta, shwrname, outdir): magdf.plot.scatter(x=idx, y=idx2) ax = plt.gca() ax.set(xlabel='RA (deg)', ylabel="Dec (deg)") - #plt.locator_params(axis='x', nbins=12) - #for lab in ax.get_xticklabels(): - # lab.set_fontsize(SMALL_SIZE) - - # format x-axes - #x_labels=["%.0f" % number for number in bins[:-1]] - #ax.set_xticklabels(x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -394,7 +387,7 @@ def semimajorDistribution(dta, shwrname, outdir, binwidth=0.5): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -424,7 +417,7 @@ def magDistributionAbs(dta, shwrname, outdir, binwidth=0.2): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -454,7 +447,7 @@ def magDistributionVis(dta, shwrname, outdir, binwidth=0.2): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -486,7 +479,7 @@ def showerAnalysis(shwr, dtstr): cols = ['Shwr','Dtstamp','Y','M','ID','Mag'] filt = None if shwr != 'ALL': - sl = imo.IMOshowerList() + sl = IMOshowerList() maxdt = sl.getEnd(shwr) + datetime.timedelta(days=10) mindt = sl.getStart(shwr) + datetime.timedelta(days=-10) @@ -498,7 +491,7 @@ def showerAnalysis(shwr, dtstr): # select the required data if shwr != 'ALL': - id, shwrname, sl, dt = sd.getShowerDets(shwr) + id, shwrname, sl, dt = getShowerDets(shwr) sngl = sngl[sngl['Shwr']==shwr] sngl = sngl[sngl.Dtstamp > mindt.timestamp()] sngl = sngl[sngl.Dtstamp < maxdt.timestamp()] @@ -531,7 +524,7 @@ def showerAnalysis(shwr, dtstr): # select the required data if shwr != 'ALL': - id, shwrname, sl, dt = sd.getShowerDets(shwr) + id, shwrname, sl, dt = getShowerDets(shwr) mtch = mtch[mtch['_stream']==shwr] mtch = mtch[mtch.dtstamp > mindt.timestamp()] mtch = mtch[mtch.dtstamp < maxdt.timestamp()] diff --git a/archive/ukmon_pylib/analysis/stationAnalysis.py b/archive/ukmon_pylib/analysis/stationAnalysis.py index fb7a2e9d5..7e888ae47 100644 --- a/archive/ukmon_pylib/analysis/stationAnalysis.py +++ b/archive/ukmon_pylib/analysis/stationAnalysis.py @@ -77,7 +77,7 @@ def timeGraph(dta, loc, outdir, when, sampleinterval): #select just the shower ID col countcol = dta['ID'] # resample it - binned = countcol.resample('{}'.format(sampleinterval)).count() + binned = countcol.resample(f'{sampleinterval}').count() binned.plot(kind='bar') # set ticks and labels every 144 intervals @@ -100,7 +100,7 @@ def timeGraph(dta, loc, outdir, when, sampleinterval): pass fname = os.path.join(outdir, 'station_plot_timeline_single.jpg') - plt.title('Observed stream activity {} intervals for {} in {}'.format(sampleinterval, loc, when)) + plt.title(f'Observed stream activity {sampleinterval} intervals for {loc} in {when}') try: plt.tight_layout() plt.savefig(fname) @@ -311,7 +311,7 @@ def pushToWebsite(fuloutdir, outdir, websitebucket): idlist = list(camlistfltr.stationid) if mth is None: - sampleinterval="1M" + sampleinterval="1ME" shortoutdir = os.path.join('reports', str(yr), 'stations', loc) outdir = os.path.join(datadir, shortoutdir) else: diff --git a/archive/ukmon_pylib/converters/toParquet.py b/archive/ukmon_pylib/converters/toParquet.py index c9454b7b8..a147501ce 100644 --- a/archive/ukmon_pylib/converters/toParquet.py +++ b/archive/ukmon_pylib/converters/toParquet.py @@ -15,7 +15,7 @@ if 'match' in fn: # fill in any #NAs in the mjd column - df.mjd.fillna(df._mjd, inplace=True) + df.fillna({'mjd': df._mjd}, inplace=True) if 'single' in fn: df = df.drop_duplicates() diff --git a/archive/ukmon_pylib/maintenance/cleanDdbTables.py b/archive/ukmon_pylib/maintenance/cleanDdbTables.py index 464d2be7f..26e17f80d 100644 --- a/archive/ukmon_pylib/maintenance/cleanDdbTables.py +++ b/archive/ukmon_pylib/maintenance/cleanDdbTables.py @@ -9,9 +9,7 @@ def deleteRows(): - archprof = os.getenv('ADM_PROFILE', default='ukmda_admin') - conn = boto3.Session(profile_name=archprof) - ddb = conn.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') tbl = 'uploadtimes' idx = 'uploaddate-stationid-index' table = ddb.Table(tbl) @@ -28,9 +26,7 @@ def deleteRows(): def updateMissingExpiryDate(): - archprof = os.getenv('ADM_PROFILE', default='ukmda_admin') - conn = boto3.Session(profile_name=archprof) - ddb = conn.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') tbl = 'uploadtimes' idx = 'uploaddate-stationid-index' table = ddb.Table(tbl) diff --git a/archive/ukmon_pylib/maintenance/dataMaintenance.py b/archive/ukmon_pylib/maintenance/dataMaintenance.py index 911abcb3e..32418b256 100644 --- a/archive/ukmon_pylib/maintenance/dataMaintenance.py +++ b/archive/ukmon_pylib/maintenance/dataMaintenance.py @@ -11,8 +11,8 @@ from time import sleep import pandas as pd import datetime - -import pymysql.cursors +import json +import operator from wmpl.Utils.TrajConversions import datetime2JD from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase @@ -55,18 +55,11 @@ def deleteS3FilesByMonth(flist, archbucket): def deleteFromCalcServerByMonth(outfname): - hname = os.getenv('HOSTNAME', default='none') env = os.getenv('RUNTIME_ENV', default='DEV').lower() - if hname != 'ukmonhelper2': - sess = boto3.Session(profile_name='default') - ssmc = sess.client('ssm', region_name='eu-west-2') - ec2 = boto3.client('ec2', region_name='eu-west-2') - else: - ssmc = boto3.client('ssm', region_name='eu-west-2') - prof = os.getenv('UKMPROFILE',default='ukmonshared') - sess = boto3.Session(profile_name=prof) - ec2 = sess.client('ec2', region_name='eu-west-2') + ssmc = boto3.client('ssm', region_name='eu-west-2') + ec2 = boto3.client('ec2', region_name='eu-west-2') + resp = ssmc.get_parameter(Name=f'{env}_calcinstance') instId = resp['Parameter']['Value'] print('clearing down calcserver') @@ -106,8 +99,7 @@ def deleteFromCalcServerByMonth(outfname): def removeDeletedTraj(csvfile): """ - Remove deleted trajectories from the consolidated match CSV and Parquet files - and from the search index. + Remove deleted trajectories from the consolidated match CSV, Parquet and search files """ csvdata = open(csvfile, 'r').readlines() @@ -117,28 +109,44 @@ def removeDeletedTraj(csvfile): jdt_end = datetime2JD(dt_end) else: jdt_end = float(csvdata[-1].split(',')[3]) + 2400000.5 - - jdt_beg =jdt_end - 21 + jdt_beg = jdt_end - 21 datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) masterdb_path = os.path.join(datadir, 'distrib') masterdb = TrajectoryDatabase(db_path=masterdb_path) cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') deltrajs = cur.fetchall() - masterdb.closeTrajDatabase() i=0 + # loop over the deleted trajectories, removing the corresponding one from the CSV file. + # Note that there's no need to update the MariaDB database, as it is populated + # from the CSV file *after* it has been purged + offs = 4 if 'search' in csvfile else 131 for traj in deltrajs: - fldr = os.path.basename(os.path.dirname(traj[0])) - match = [tr for tr in csvdata if fldr in tr] - if len(match) > 0: + cur = masterdb.dbhandle.execute(f'select jdt_ref, participating_stations from trajectories where traj_file_path="{traj[0]}" and status=0') + thistraj = cur.fetchall() + if len(thistraj) == 1: + fldr = os.path.basename(os.path.dirname(traj[0])) + + # find the rows in the CSV file that correspond to the deleted trajectory + # then go through them and compare obs_ids. The one with the same obs_ids + # is the one we want to remove + match = [tr for tr in csvdata if fldr in tr] + obs_ids = thistraj[0][1] + obs_ids_str = ';'.join(json.loads(obs_ids)) + ';' for thismtch in match: - print(f'removing {fldr}') - idx = csvdata.index(thismtch) - _ = csvdata.pop(idx) - i += 1 - print(f'removed {i} trajectories') + if thismtch.split(',')[offs] == obs_ids_str: + print(f'removing {fldr}') + idx = csvdata.index(thismtch) + _ = csvdata.pop(idx) + i += 1 + break + else: + print(f'skipping {traj[0]} as there are {len(thistraj)} active trajs with the same path') + masterdb.closeTrajDatabase() + print(f'removed {i} trajectories from the text file') + # save the CSV file again open(csvfile, 'w').writelines(csvdata) if 'search' not in csvfile: @@ -149,48 +157,64 @@ def removeDeletedTraj(csvfile): return -def getSqlLoginDetails(): - # retrieve password and host from SSM. This allows me to manage them from Terraform - ssm = boto3.client('ssm', region_name='eu-west-1') - res = ssm.get_parameter(Name='prod_dbpw', WithDecryption=True) - password = res['Parameter']['Value'] - res = ssm.get_parameter(Name='prod_dbhost') - host = res['Parameter']['Value'] - # should really do these too but they won't change often if at all - user = 'batch' - db = 'ukmon' - return host, user, password, db - - -def removeDelTrajFromDb(): - dt_end = datetime.datetime.now(tz=datetime.timezone.utc) - jdt_end = datetime2JD(dt_end) - jdt_beg =jdt_end - 7 - - # get list of deleted trajectories - datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - masterdb_path = os.path.join(datadir, 'distrib') - masterdb = TrajectoryDatabase(db_path=masterdb_path) - cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') - deltrajs = cur.fetchall() - masterdb.closeTrajDatabase() +def removeDeletedTrajCsv(csvloc, targloc): + """ + Remove csv files corresponding to deleted trajectories before they are + consolidated into the annual CSV and Parquet files + """ + srcbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] - # get connection to the SQL database - host, user, passwd, db = getSqlLoginDetails() - connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) + csvfiles = [] + s3 = boto3.client('s3') + print('checking in ', csvloc, 'sending to ', targloc) + res = s3.list_objects(Bucket=srcbucket, Prefix=csvloc) + if 'Contents' in res: + for k in res['Contents']: + csvname = k["Key"] + csv_dt = datetime.datetime.strptime(os.path.basename(k["Key"])[:22], "%Y%m%d-%H%M%S.%f") + csvfiles.append({"name":csvname, "dt":csv_dt}) + csvfiles.sort(key=operator.itemgetter('name')) + jdt_beg = datetime2JD(csvfiles[0]['dt']) + jdt_end = datetime2JD(csvfiles[-1]['dt']) + + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + masterdb_path = os.path.join(datadir, 'distrib') + masterdb = TrajectoryDatabase(db_path=masterdb_path) + cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') + deltrajs = cur.fetchall() + + i=0 + # loop over the deleted trajectories, removing the corresponding CSV file. + for del_traj in deltrajs: + cur = masterdb.dbhandle.execute(f'select jdt_ref, participating_stations from trajectories where traj_file_path="{del_traj[0]}" and status=0') + thistraj = cur.fetchall() + if len(thistraj) == 1: + obs_ids = thistraj[0][1] + obs_ids_str = ';'.join(json.loads(obs_ids)) + ';' + fldr = os.path.basename(os.path.dirname(del_traj[0]))[:19].replace('_', '-') + + matched_csvs = [x for x in csvfiles if fldr in x['name']] + for csv in matched_csvs: + csvdata = s3.get_object(Bucket=srcbucket, Key=csv['name'])['Body'].read() + csv_obs_ids = csvdata.decode('utf-8').split(',')[131].strip() + if csv_obs_ids == obs_ids_str: + # we have a match to within 0.001s and with the same observations + print(f'removing {fldr}') + bare_csv_name = os.path.basename(matched_csvs[0]['name']) + yr = bare_csv_name[:4] + destkey = f"{targloc}/{yr}/{bare_csv_name}" + s3.copy({"Bucket":srcbucket,"Key":csv['name']}, srcbucket, destkey) + s3.delete_object(Bucket=srcbucket, Key=matched_csvs[0]['name']) + i += 1 + break + else: + print(f'skipping {del_traj[0]} as there are {len(thistraj)} inactive trajs with the same path') + masterdb.closeTrajDatabase() + print(f'removed {i} trajectories from the text file') + else: + print('no fullcsv files to process') - count = 0 - for traj in deltrajs: - fldr = os.path.basename(os.path.dirname(traj[0])) - sqlstr = f"delete from matches where orbname like '{fldr}%'" - with connection.cursor() as cursor: - cursor.execute(sqlstr) - result = cursor.fetchall() - count += len(result) - connection.commit() - connection.close() - print(f'cleaned up {count} trajectories') - return + return if __name__ == '__main__': diff --git a/archive/ukmon_pylib/maintenance/getNextBatchStart.py b/archive/ukmon_pylib/maintenance/getNextBatchStart.py index 2dc8f39a6..d89b40304 100644 --- a/archive/ukmon_pylib/maintenance/getNextBatchStart.py +++ b/archive/ukmon_pylib/maintenance/getNextBatchStart.py @@ -6,7 +6,7 @@ import datetime import sys from crontab import CronTab -from meteortools.utils import getNextRiseSet +from utils.getRiseSet import getNextRiseSet offset = 90 diff --git a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py index 2dadf8cb6..85b3f04bc 100644 --- a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py +++ b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py @@ -26,7 +26,7 @@ lnxdir='/home/ec2-user/keymgmt/csvkeys' else: csvdir=os.path.expanduser('~/keymgmt/csvkeys') - csvdir=os.path.expanduser('~/keymgmt/csvkeys') + lnxdir=os.path.expanduser('~/keymgmt/csvkeys') def checkIfOnGMN(camid): diff --git a/archive/ukmon_pylib/maintenance/plotStationsOnMap.py b/archive/ukmon_pylib/maintenance/plotStationsOnMap.py index 59b37d3b8..e90cca86c 100644 --- a/archive/ukmon_pylib/maintenance/plotStationsOnMap.py +++ b/archive/ukmon_pylib/maintenance/plotStationsOnMap.py @@ -6,14 +6,11 @@ import numpy as np import os import sys -import fnmatch import matplotlib.pyplot as plt import json import cartopy.crs as ccrs -from meteortools.fileformats import UFOAnalyzerXML as ua - from PIL import Image @@ -21,15 +18,7 @@ def getBearingsForEvent(stns, fldr): brngs = np.zeros(len(stns)) listOfFiles = os.listdir(fldr) for entry in listOfFiles: - if fnmatch.fnmatch(entry, '*A.XML'): - fullname = os.path.join(fldr, entry) - xmlf = ua.UAXml(fullname) - sta, _, _, _, _, _ = xmlf.getStationDetails() - _, _, _, _, _, _, az1, _ = xmlf.getObjectStart(0) - i = stns.index(sta) - brngs[i] = az1 - - elif entry == 'FTPdetectinfo_UFO.txt': + if entry == 'FTPdetectinfo_UFO.txt': with open(os.path.join(fldr, 'FTPdetectinfo_UFO.txt'), 'r') as f: lis = f.readlines() for s in stns: diff --git a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py index 4f5f050cf..c64c746c5 100644 --- a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py +++ b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py @@ -71,8 +71,7 @@ def createExtraJpgtxt(outdir, traj, availableimages): def createExtraJpgHtml(outdir, parentfldr, yr, ym): - mdasess = boto3.Session(profile_name='ukmda_admin') - s3mda = mdasess.client('s3') + s3mda = boto3.client('s3') webfldr = f'img/single/{yr}/{ym}' with open(os.path.join(outdir, 'extrajpgs.html'), 'w') as outf: jpgfldr = os.path.join(parentfldr, 'jpgs') @@ -194,8 +193,7 @@ def recreateOrbitFiles(outdir, pickname, doupload=False): if doupload: files = os.listdir(outdir) - mdasess = boto3.Session(profile_name='ukmda_admin') - s3mda = mdasess.client('s3') + s3mda = boto3.client('s3') if int(yr) > 2020: webfldr = f'reports/{yr}/orbits/{ym}/{ymd}' else: diff --git a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py index 24107d46b..3ef031d57 100644 --- a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py +++ b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py @@ -30,8 +30,7 @@ def findOtherBadEvents(): return badevents -def findFailedEvents(prof='ukmonshared'): - #session = boto3.Session(profile_name=prof) +def findFailedEvents(): logcli = boto3.client('logs', region_name='eu-west-2') datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'orbits', 'lastorbitcheck.txt') @@ -98,9 +97,8 @@ def checkFails(fails): return realfails -def rerunFails(fails, prof='ukmonshared'): - session=boto3.Session(profile_name=prof) - lambd = session.client('lambda', region_name='eu-west-2') +def rerunFails(fails): + lambd = boto3.client('lambda', region_name='eu-west-2') datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'orbits', 'lastorbitcheck.txt') @@ -120,10 +118,9 @@ def rerunFails(fails, prof='ukmonshared'): return -def resubmitPicklesForDay(dtstr, prof='ukmonshared'): +def resubmitPicklesForDay(dtstr): s3 = boto3.client('s3') - session = boto3.Session(profile_name=prof) - lambd = session.client('lambda', region_name='eu-west-2') + lambd = boto3.client('lambda', region_name='eu-west-2') pref = f'matches/RMSCorrelate/trajectories/{dtstr[:4]}/{dtstr[:6]}/{dtstr}/' buck = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] res = s3.list_objects_v2(Bucket=buck, Prefix=pref) diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index e85ff3d66..36aeaf985 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -137,9 +137,9 @@ def backPopulate(stationid): s3bucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] basepath = os.path.expanduser('~/prod/ukmon-shared/matches/RMSCorrelate') - fldrs = glob.glob1(os.path.join(basepath, stationid), '*') + fldrs = glob.glob('*', root_dir=os.path.join(basepath, stationid)) for fldr in fldrs: - s3objects = glob.glob1(os.path.join(basepath, stationid. fldr), 'FTPd*') + s3objects = glob.glob('FTPd*', root_dir=os.path.join(basepath, stationid. fldr)) if len(s3objects) > 0: s3obj = s3objects[0] fullobj = f'matches/RMSCorrelate/{stationid}/{fldr}/{s3obj}' @@ -151,20 +151,6 @@ def backPopulate(stationid): datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) ddb = boto3.resource('dynamodb', region_name='eu-west-2') - if os.path.isfile('/sys/devices/virtual/dmi/id/board_asset_tag'): # crude check for EC2 - #print('asset tag file exists') - lis = open('/sys/devices/virtual/dmi/id/board_asset_tag').readlines() - if 'i-' in lis[0]: - sts_client = boto3.client('sts') - assumed_role_object=sts_client.assume_role( - RoleArn="arn:aws:iam::183798037734:role/service-role/S3FullAccess", - RoleSessionName="AssumeRoleSession1") - credentials=assumed_role_object['Credentials'] - - ddb = boto3.resource('dynamodb', region_name='eu-west-2', - aws_access_key_id=credentials['AccessKeyId'], - aws_secret_access_key=credentials['SecretAccessKey'], - aws_session_token=credentials['SessionToken']) s,d,t,m,r = getDayCamTimings(sys.argv[1], ddb=ddb) newdata=pd.DataFrame(zip(s,d,t,m,r), columns=['stationid','upddate','uploadtime','manual','rundate']) @@ -187,49 +173,48 @@ def backPopulate(stationid): pd.options.mode.chained_assignment = 'warn' caminfo = camlist.drop(columns=['site','direction','oldcode','active','camtype','eMail', 'humanName']) - logindf = pd.read_csv(os.path.join(datadir, 'reports', 'lastlogins.txt'), names=['dateval','timeval','siteid'], skipinitialspace=True) - logindf['timeval'] = logindf.timeval.astype('str').str.pad(6,fillchar='0') - logindf.dateval.fillna('Jan-01',inplace=True) - logindf.timeval.fillna('00:00:00',inplace=True) - # handle case round yearend where the log may have prev year's details as well as current year - nowdt = datetime.datetime.now() - yrval = str(nowdt.year) + '-' - yrvalback = str(nowdt.year-1) + '-' - logindf['lastseen'] = [datetime.datetime.strptime(x, '%Y-%b-%d_%H%M%S') for x in yrval + logindf.dateval+'_'+logindf.timeval] - try: # will fail on 29th Feb in a leapyear, as previous year is not leap - logindf['lastseen2'] = [datetime.datetime.strptime(x, '%Y-%b-%d_%H%M%S') for x in yrvalback + logindf.dateval+'_'+logindf.timeval] - except Exception: - logindf['lastseen2'] = [datetime.datetime.strptime(x, '%Y-%b-%d_%H%M%S') for x in yrval + logindf.dateval+'_'+logindf.timeval] - logindf.loc[logindf.lastseen > nowdt, 'lastseen'] = logindf.lastseen2 - logindf = logindf.sort_values(by=['lastseen']) - logindf.drop_duplicates(subset=['siteid'], inplace=True, keep='last') - logindf.rename(columns={'siteid':'location'}, inplace=True) - logindf.drop(columns = ['lastseen2'], inplace=True) - - # create a merged dataframe with siteid and stationid - intdf = pd.merge(logindf,caminfo, on=['location'], how='outer') - - df = pd.merge(intdf, fulldf, on=['stationid']) - df.dateval.fillna('Jan-01',inplace=True) - df.timeval.fillna('00:00:00',inplace=True) - df['uploadtime']=df.uploadtime.astype("str").str.pad(6,fillchar="0") - df['lastupload']=df.upddate.astype('str') + '_' +df.uploadtime - df.lastupload = [datetime.datetime.strptime(x, '%Y%m%d_%H%M%S') for x in df.lastupload] - df = df.drop(columns=['timeval','stationid','manual','rundate', 'upddate','uploadtime', 'dateval']) - df['dateval']=[x.strftime('%b-%d') for x in df.lastupload] - df = df.sort_values(by=['lastupload']) - - outfile=os.path.join(datadir, 'reports', 'stationlogins.txt') - zerodate = datetime.datetime(1970,1,1,0,0,0) - with open(outfile,'w') as outf: - outf.write('Last Upload, StationID, Last Login\n') - for _,rw in df.iterrows(): - dtval = rw.dateval - lastup = rw.lastupload.strftime('%H:%M:%S') - if pd.isnull(rw.lastseen): - lastseen = '> 1 month' - else: - lastseen = rw.lastseen.strftime('%b-%d %H:%M:%S') - if lastseen == 'Jan-01 00:00:00': - lastseen = '> 1 month' - outf.write(f'{dtval}, {lastup}, {rw.location:20s}, {lastseen}\n') + # process the last-login data from the SSHD log + lastlogs = open(os.path.join(datadir, 'reports', 'lastlogins.txt'),'r').readlines() + lodata = [] + now = datetime.datetime.now(tz=datetime.timezone.utc) + for li in lastlogs: + spls = li.split('ssh') + # could be ubuntu (auth.log) or amazon linux style log + dtstr = spls[0][spls[0].find(':')+1:][:19] + if ' ' in dtstr: + dtval = datetime.datetime.strptime(f'{now.year} {dtstr[:15]}', '%Y %b %d %H:%M:%S').replace(tzinfo=datetime.timezone.utc) + if dtval > now: + dtval = dtval.replace(year=now.year-1, tzinfo=datetime.timezone.utc) + targhost = 'ukmonhelper2' + else: + dtval = datetime.datetime.strptime(dtstr, '%Y-%m-%dT%H:%M:%S') + dtval = dtval.replace(tzinfo=datetime.timezone.utc) + targhost = 'batchserver' + location = spls[1].split(' for ')[1].split()[0] + lodata.append({'location':location, 'lastseen':dtval,'host': targhost}) + + + if len(lodata) > 0: + logindf = pd.DataFrame(lodata) + logindf = logindf.sort_values(by=['lastseen']) + logindf.drop_duplicates(subset=['location'], inplace=True, keep='last') + + # create a merged dataframe with siteid and stationid + intdf = pd.merge(logindf,caminfo, on=['location'], how='outer') + + df = pd.merge(intdf, fulldf, on=['stationid']) + df['uploadtime']=df.uploadtime.astype("str").str.pad(6,fillchar="0") + df['lastupload']=df.upddate.astype('str') + '_' +df.uploadtime + df.lastupload = [datetime.datetime.strptime(x, '%Y%m%d_%H%M%S') for x in df.lastupload] + df = df.drop(columns=['stationid','manual','rundate', 'upddate','uploadtime']) + df = df.sort_values(by=['lastupload']) + + outfile=os.path.join(datadir, 'reports', 'stationlogins.txt') + zerodate = datetime.datetime(1970,1,1,0,0,0) + with open(outfile,'w') as outf: + outf.write('Last Upload, StationID, Last Login, Via\n') + for _,rw in df.iterrows(): + lastup = '> 1 month' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S') + lastlo = '> 1 month' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S') + via = '' if pd.isnull(rw.host) else rw.host + outf.write(f'{lastup} , {rw.location:20s}, {lastlo}, {via}\n') diff --git a/archive/ukmon_pylib/metrics/getMatchStats.py b/archive/ukmon_pylib/metrics/getMatchStats.py index e1dcd4f96..05d098edf 100644 --- a/archive/ukmon_pylib/metrics/getMatchStats.py +++ b/archive/ukmon_pylib/metrics/getMatchStats.py @@ -36,7 +36,7 @@ def getDailyObsCounts(logf): return -def getMatchStats(logf): +def getMatchStats(logf, rundate): loglines = open(logf).readlines() events = [line.strip() for line in loglines if 'Analysing' in line and 'observations' in line] @@ -77,7 +77,7 @@ def getMatchStats(logf): nonphys = beglowr + badalti + badvelo + badangl tot = added + uncal + missdf - rtims = [line.strip() for line in loglines if 'runDistrib' in line] + rtims = [line.strip() for line in loglines if 'runDistrib' in line and '/prod/' not in line] stim = rtims[0][11:19] etim = rtims[-1][11:19] d1=datetime.strptime(stim,'%H:%M:%S') @@ -92,9 +92,6 @@ def getMatchStats(logf): cstime = str(d2 - d1) datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - rundate=os.getenv('rundate') - if not rundate: - rundate = open(os.path.join(datadir, 'rundate.txt'),'r').readline().strip() dailydbdir = os.path.join(datadir, 'latest','dailydbs') trajdb = TrajectoryDatabase(dailydbdir, f'{rundate}_trajectories.db') trajs = len(trajdb.getTrajBasics('.',[0,9999999])) @@ -102,11 +99,8 @@ def getMatchStats(logf): return tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime -def updateStats(obscount, candcount, trajcount, runtime): +def updateStats(obscount, candcount, trajcount, runtime, rundate): datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - rundate=os.getenv('rundate') - if not rundate: - rundate = open(os.path.join(datadir, 'rundate.txt'),'r').readline().strip() statsdir = os.path.join(datadir, 'dailyreports') reports = glob.glob(os.path.join(statsdir, f'{rundate}*.txt')) @@ -126,7 +120,10 @@ def updateStats(obscount, candcount, trajcount, runtime): if __name__ == '__main__': - tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime = getMatchStats(sys.argv[1]) - if len(sys.argv) < 3: - updateStats(added, cands, trajs, runtime) + matchfile = sys.argv[1] + rundate = sys.argv[2] + + tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime = getMatchStats(matchfile, rundate) + + updateStats(added, cands, trajs, runtime, rundate) print(tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime) diff --git a/archive/ukmon_pylib/reports/CameraDetails.py b/archive/ukmon_pylib/reports/CameraDetails.py index ab3fd50d9..3557c103d 100644 --- a/archive/ukmon_pylib/reports/CameraDetails.py +++ b/archive/ukmon_pylib/reports/CameraDetails.py @@ -41,9 +41,7 @@ def updateCamLocDirFovDB(datadir=None): def loadLocationDetails(table='camdetails', ddb=None, loadall=False): if not ddb: - prof = os.getenv('UKMPROFILE',default='ukmonshared') - conn = boto3.Session(profile_name=prof) - ddb = conn.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') table = ddb.Table(table) res = table.scan() # strictly, should check for LastEvaluatedKey here, in case there was more than 1MB of data, @@ -71,7 +69,7 @@ def findLocationInfo(srchstring, ddb=None, statdets=None): return srchres -def createCDCsv(targetloc): +def createCDCsv(targetloc, srchidxdir): camdets = loadLocationDetails() cd4csv = camdets[['site','stationid','oldcode','direction','camtype','active']] pd.options.mode.chained_assignment = None @@ -83,28 +81,32 @@ def createCDCsv(targetloc): datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) outfname = os.path.join(datadir, targetloc, 'camera-details.csv') cd4csv.to_csv(outfname, index=False) - with open(os.path.join(datadir, 'activecamcount.txt'), 'w') as outf: + with open(os.path.join(datadir, targetloc, 'activecamcount.txt'), 'w') as outf: outf.write(str(len(cd4csv[cd4csv.active==1]))) - createStatOptsHtml(camdets, datadir) - createStatOptsHtml(camdets, datadir, True) - createStatOptsHtml(camdets, datadir, True, True) + + outdir = os.path.join(datadir, srchidxdir) + os.makedirs(outdir, exist_ok=True) + + createStatOptsHtml(camdets, outdir) + createStatOptsHtml(camdets, outdir, True) + createStatOptsHtml(camdets, outdir, True, True) return -def createStatOptsHtml(sitedets, datadir, active=False, locs=False): +def createStatOptsHtml(sitedets, outdir, active=False, locs=False): """ Create an HTML file containing a list of stations or locations These files are used by the website search functions """ if active: if locs: - siteidx = os.path.join(datadir, 'activestatlocs.html') + siteidx = os.path.join(outdir, 'activestatlocs.html') sitedets = sitedets.sort_values(by=['site', 'stationid']) else: - siteidx = os.path.join(datadir, 'activestatopts.html') + siteidx = os.path.join(outdir, 'activestatopts.html') sitedets = sitedets[sitedets.active==1] else: - siteidx = os.path.join(datadir, 'statopts.html') + siteidx = os.path.join(outdir, 'statopts.html') with open(siteidx, 'w') as outf: if not active: outf.write('\n') diff --git a/archive/ukmon_pylib/reports/cameraStatusReport.py b/archive/ukmon_pylib/reports/cameraStatusReport.py index 256ad0765..83e4850e0 100644 --- a/archive/ukmon_pylib/reports/cameraStatusReport.py +++ b/archive/ukmon_pylib/reports/cameraStatusReport.py @@ -30,7 +30,7 @@ def getLastUpdateDate(datadir=None, camfname=None, ddb=None): fldrlist = pd.merge(tmplist, caminfo, on=['stationid'], how='inner') nowdt = datetime.datetime.now() - fldrlist.rundate.fillna(fldrlist.upddate.astype(str) + '_' + fldrlist.uploadtime.astype(str).str.pad(width=6,fillchar='0'), inplace=True) + fldrlist.fillna({'rundate': fldrlist.upddate.astype(str) + '_' + fldrlist.uploadtime.astype(str).str.pad(width=6,fillchar='0')}, inplace=True) fldrlist['dtstamp'] = [datetime.datetime.strptime(f,'%Y%m%d_%H%M%S') for f in fldrlist.rundate] fldrlist = fldrlist.sort_values(by=['stationid','dtstamp']) fldrlist.drop_duplicates(keep='last', subset=['stationid'], inplace=True) diff --git a/archive/ukmon_pylib/reports/createAnnualBarChart.py b/archive/ukmon_pylib/reports/createAnnualBarChart.py index 9426f5d6d..944c2ba03 100644 --- a/archive/ukmon_pylib/reports/createAnnualBarChart.py +++ b/archive/ukmon_pylib/reports/createAnnualBarChart.py @@ -8,7 +8,7 @@ import matplotlib.pyplot as plt import datetime -from meteortools.utils import jd2Date +from wmpl.Utils.TrajConversions import jd2Date def createBarChart(datadir=None, yr=None): @@ -16,7 +16,8 @@ def createBarChart(datadir=None, yr=None): yr=datetime.datetime.now().year if datadir is None: datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - fname = os.path.join(datadir, 'matched', 'matches-full-{}.parquet.snap'.format(yr)) + yr = str(yr) + fname = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap') if not os.path.isfile(fname): print('{} missing', fname) return None @@ -36,19 +37,20 @@ def createBarChart(datadir=None, yr=None): dts.append(dt) fig, ax = plt.subplots() - width = 0.35 + width = 0.35 nowdt=datetime.datetime.now() ax.set_ylabel('# matches') ax.set_xlabel('Date') ax.set_title('Number of matched events per day. Last updated {}'.format(nowdt.strftime('%Y-%m-%d %H:%M:%S'))) - li.append(0) # comes up one short + while len(li) < len(dts): + li.append(0) ax.bar(dts, li, width, label='Events') fig = plt.gcf() fig.set_size_inches(12, 5) fig.tight_layout() #plt.gca().invert_yaxis() - plt.savefig(os.path.join(datadir,f'Annual-{yr}.jpg'), dpi=100) + plt.savefig(os.path.join(datadir,'reports',yr, f'Annual-{yr}.jpg'), dpi=100) plt.close() return matches diff --git a/archive/ukmon_pylib/reports/createSearchableFormat.py b/archive/ukmon_pylib/reports/createSearchableFormat.py index 4424cc4bf..c567fa572 100644 --- a/archive/ukmon_pylib/reports/createSearchableFormat.py +++ b/archive/ukmon_pylib/reports/createSearchableFormat.py @@ -23,7 +23,7 @@ def checkUrl(s3, url): except Exception: #print(e) retval = '/img/missing-white.png' - print(url, retval) + #print(url, retval) return retval diff --git a/archive/ukmon_pylib/reports/createSummaryTable.py b/archive/ukmon_pylib/reports/createSummaryTable.py index a0d861bb1..b31500027 100644 --- a/archive/ukmon_pylib/reports/createSummaryTable.py +++ b/archive/ukmon_pylib/reports/createSummaryTable.py @@ -16,10 +16,13 @@ def createSummaryTable(curryr=None, datadir=None): """ if curryr is None: - curryr = str(datetime.datetime.now().year) + curryr = datetime.datetime.now().year if datadir is None: datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - fname = os.path.join(datadir, 'summarytable.js') + curryr = str(curryr) + outdir = os.path.join(datadir, 'reports', curryr) + os.makedirs(outdir, exist_ok=True) + fname = os.path.join(outdir, 'summarytable.js') with open(fname, 'w') as f: f.write('$(function() {\n') f.write('var table = document.createElement("table");\n') diff --git a/archive/ukmon_pylib/reports/dailyReport.py b/archive/ukmon_pylib/reports/dailyReport.py index a511c4055..3c5bc8979 100644 --- a/archive/ukmon_pylib/reports/dailyReport.py +++ b/archive/ukmon_pylib/reports/dailyReport.py @@ -6,7 +6,7 @@ import sys import datetime -from meteortools.utils import sendAnEmail +from utils.sendAnEmail import sendAnEmail def AddHeader(body, bodytext, stats): @@ -55,11 +55,7 @@ def AddRowRMS(body, bodytext, ele): return body, bodytext -def LookForMatchesRMS(doff, dayfile, statsfile): - # get stats - with open(statsfile, 'r') as inf: - lis = inf.readlines() - stats = lis[-1].strip().split(' ') +def LookForMatchesRMS(doff, dayfile, stats): bodytext = 'Daily notification of matches\n\n' body = '' @@ -100,7 +96,14 @@ def LookForMatchesRMS(doff, dayfile, statsfile): dailyreps.sort() dailyrep = dailyreps[-1] statfile = os.path.join(reppth, 'stats.txt') - _, _, body, bodytext = LookForMatchesRMS(doff, dailyrep, statfile) + lis = open(statfile, 'r').readlines() + stats = [li for li in lis if repdtstr in li] + if len(stats) > 0: + stats = stats[0].strip().split(' ') + else: + stats = lis[-1].strip().split(' ') + + _, _, body, bodytext = LookForMatchesRMS(doff, dailyrep, stats) #body = body.replace('assets/img/logo.svg', 'latest/dailyreports/dailyreportsidx.html') if doff == 1: @@ -123,5 +126,7 @@ def LookForMatchesRMS(doff, dayfile, statsfile): mailFrom = recs[-1].strip() #mailRecip = 'markmcintyre99@googlemail.com' # TODO for testing only yest = (datetime.date.today()-datetime.timedelta(days=doff)).strftime('%Y-%m-%d') - mailSubj = f'Latest Match Report for {yest}' - sendAnEmail(mailRecip, bodytext, mailSubj, mailFrom, msg_html=body.replace('href="/reports', f'href="{targeturl}/reports')) + # only send the email for the most recent report. + if doff == 1: + mailSubj = f'Latest Match Report for {yest}' + sendAnEmail(mailRecip, bodytext, mailSubj, mailFrom, msg_html=body.replace('href="/reports', f'href="{targeturl}/reports')) diff --git a/archive/ukmon_pylib/reports/extractors.py b/archive/ukmon_pylib/reports/extractors.py index 65670b38d..8541e7598 100644 --- a/archive/ukmon_pylib/reports/extractors.py +++ b/archive/ukmon_pylib/reports/extractors.py @@ -6,8 +6,8 @@ import os import sys import pandas as pd -from meteortools.fileformats import imoWorkingShowerList as imo -from meteortools.utils import getActiveShowers +from utils.imoWorkingShowerList import IMOshowerList +from utils.getActiveShowers import getActiveShowers def createSplitMatchFile(yr, mth=None, shwr=None, matches=None): @@ -143,7 +143,7 @@ def extractAllShowersData(ymd): showerlist = getActiveShowers(ymd, retlist=True) showerlist.append('spo') else: - sl = imo.IMOshowerList() + sl = IMOshowerList() showerlist = sl.getMajorShowers(True, True).strip().split(' ') print(f'processing data for {ymd}') @@ -159,8 +159,9 @@ def extractAllShowersData(ymd): fname = os.path.join(datadir, 'consolidated','M_{}-unified.csv'.format(yr)) if not os.path.isfile(fname): print(f'unable to open {fname}') - return - ufosingles = pd.read_csv(fname, skipinitialspace=True) + ufosingles = None + else: + ufosingles = pd.read_csv(fname, skipinitialspace=True) fname = os.path.join(datadir, 'single','singles-{}.parquet.snap'.format(yr)) if not os.path.isfile(fname): print(f'unable to open {fname}') @@ -174,8 +175,9 @@ def extractAllShowersData(ymd): print(f'processing RMS singles for {shwr}') createRMSSingleMonthlyExtract(yr, shwr=shwr, dta=rmssingles) createRMSSingleMonthlyExtract(yr, shwr=shwr, dta=rmssingles, withshower=True) - print(f'processing UFO singles for {shwr}') - createUFOSingleMonthlyExtract(yr, shwr=shwr, dta=ufosingles) + if ufosingles: + print(f'processing UFO singles for {shwr}') + createUFOSingleMonthlyExtract(yr, shwr=shwr, dta=ufosingles) return diff --git a/archive/ukmon_pylib/reports/findFailedMatches.py b/archive/ukmon_pylib/reports/findFailedMatches.py index c6005fe7e..1b110a1c6 100644 --- a/archive/ukmon_pylib/reports/findFailedMatches.py +++ b/archive/ukmon_pylib/reports/findFailedMatches.py @@ -46,7 +46,7 @@ def processOneLog(logfile, repfile): srcdir = os.getenv('SRC', default='.') logdir = os.path.join(srcdir, 'logs','distrib') - logs = glob.glob1(logdir, f'{repdt}*.log') + logs = glob.glob(f'{repdt}*.log', root_dir=logdir) for logf in logs: processOneLog(os.path.join(logdir, logf), reportfile) reportfile.close() diff --git a/archive/ukmon_pylib/reports/findFireballs.py b/archive/ukmon_pylib/reports/findFireballs.py index d30949fa6..178bd1247 100644 --- a/archive/ukmon_pylib/reports/findFireballs.py +++ b/archive/ukmon_pylib/reports/findFireballs.py @@ -12,7 +12,7 @@ from shutil import rmtree from traj.pickleAnalyser import getBestView -from meteortools.utils import jd2Date +from wmpl.Utils.TrajConversions import jd2Date # diff --git a/archive/ukmon_pylib/reports/makeCoverageMap.py b/archive/ukmon_pylib/reports/makeCoverageMap.py index bbb7f0fba..b25f8beca 100644 --- a/archive/ukmon_pylib/reports/makeCoverageMap.py +++ b/archive/ukmon_pylib/reports/makeCoverageMap.py @@ -6,8 +6,9 @@ import os import gmplot import glob +import boto3 from cryptography.fernet import Fernet -from meteortools.fileformats import readCameraKML +from utils.kmlHandlers import readCameraKML def decodeApiKey(enckey): @@ -24,16 +25,27 @@ def encodeApiKey(plainkey): return apikey.decode('utf-8') +def getApiKey(): + ssm = boto3.client('ssm') + res = ssm.get_parameter(Name='prod_gmapsapikey', WithDecryption=True) + if 'Parameter' in res: + return res['Parameter']['Value'] + else: + return False + + def makeCoverageMap(kmlsource, outdir, showMarker=False, useName=False): - apikey = os.getenv('APIKEY') - apikey = decodeApiKey(apikey) + + apikey = getApiKey() + if not apikey: + return kmltempl = os.getenv('KMLTEMPLATE', default='*70km.kml') heightval = kmltempl[1:-4] gmap = gmplot.GoogleMapPlotter(52, -1.0, 5, apikey=apikey, title=f'Camera Coverage at {heightval}', map_type='satellite') - flist = glob.glob1(kmlsource, kmltempl) + flist = glob.glob(kmltempl, root_dir=kmlsource) cols = list(gmplot.color._HTML_COLOR_CODES.keys()) for col, fn in zip(cols,flist): @@ -55,14 +67,15 @@ def makeCoverageMap(kmlsource, outdir, showMarker=False, useName=False): return -def createCoveragePage(): - apikey = os.getenv('APIKEY') - apikey = decodeApiKey(apikey) +def createCoveragePage(targdir): + apikey = getApiKey() + if not apikey: + return templdir = os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) with open(os.path.join(templdir, 'coverage-maps.html'), 'r') as inf: lis = inf.readlines() - with open(os.path.join(datadir, 'latest','coverage-maps.html'), 'w') as outf: + with open(os.path.join(datadir, targdir,'coverage-maps.html'), 'w') as outf: for li in lis: outf.write(li.replace('{{MAPSAPIKEY}}', apikey)) return diff --git a/archive/ukmon_pylib/reports/meteoriteTools.py b/archive/ukmon_pylib/reports/meteoriteTools.py index 4f7c72e7b..e699bb072 100644 --- a/archive/ukmon_pylib/reports/meteoriteTools.py +++ b/archive/ukmon_pylib/reports/meteoriteTools.py @@ -8,7 +8,7 @@ import numpy as np import requests -from meteortools.utils.Math import greatCircleDistance +from wmpl.Utils.Earth import greatCircleDistance def stationsNearPoint(lat, lon, dist=75, email_only=True): diff --git a/archive/ukmon_pylib/reports/reportActiveShowers.py b/archive/ukmon_pylib/reports/reportActiveShowers.py index 7a58d91fc..bf2a171a0 100644 --- a/archive/ukmon_pylib/reports/reportActiveShowers.py +++ b/archive/ukmon_pylib/reports/reportActiveShowers.py @@ -9,7 +9,7 @@ import shutil import argparse -from meteortools.utils import getActiveShowers +from utils.getActiveShowers import getActiveShowers from analysis.showerAnalysis import showerAnalysis from reports.findFireballs import findFireballs @@ -109,8 +109,8 @@ def createShowerIndexPage(dtstr, shwr, shwrname, outdir, datadir): outf.write('The graphs and histograms below show more information about the velocity, magnitude \n') outf.write('start and end altitude and other parameters. Click for larger view. \n') # add the charts and stuff - jpglist = glob.glob1(outdir, '*.jpg') - pnglist = glob.glob1(outdir, '*.png') + jpglist = glob.glob('*.jpg', root_dir=outdir) + pnglist = glob.glob('*.png', root_dir=outdir) outf.write('
    \n') for j in jpglist: outf.write(f'\n') diff --git a/archive/ukmon_pylib/reports/reportBadCameras.py b/archive/ukmon_pylib/reports/reportBadCameras.py index cc0938ee8..adf4d64ca 100644 --- a/archive/ukmon_pylib/reports/reportBadCameras.py +++ b/archive/ukmon_pylib/reports/reportBadCameras.py @@ -10,7 +10,7 @@ import textwrap from reports.CameraDetails import loadLocationDetails -from meteortools.utils import sendAnEmail +from utils.sendAnEmail import sendAnEmail mailfrom = 'ukmonhelper2@ukmeteors.co.uk' diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index 2d8738584..d79ddefff 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -12,6 +12,7 @@ import tempfile import boto3 import glob +import tarfile from traj.pickleAnalyser import getVMagCodeAndStations from reports.CameraDetails import findSite, loadLocationDetails @@ -41,11 +42,11 @@ def processLocalFolder(trajdir, basedir): return outstr -def getListOfNewMatches(dir_path, db_path='/tmp', rundate=None): +def getListOfNewMatches(dir_path, db_path, rundate): os.makedirs(db_path, exist_ok=True) - db_name = f'{rundate}_trajectories.db' if rundate else 'trajectories.db' + db_name = f'{rundate}_trajectories.db' dailydb = TrajectoryDatabase(db_path=db_path, db_name=db_name, purge_records=True) - flist = glob.glob(os.path.join(dir_path, 'trajectories_*.db')) + flist = glob.glob(os.path.join(dir_path, f'trajectories_{rundate}_*.db')) flist.sort() for fl in flist: tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') @@ -101,11 +102,11 @@ def getListOfNewMatches(dir_path, db_path='/tmp', rundate=None): return newdirs -def updatePairedDB(dir_path, db_path='/tmp', rundate=None): +def updatePairedDB(dir_path, db_path, rundate): os.makedirs(db_path, exist_ok=True) - db_name = f'{rundate}_observations.db' if rundate else 'observations.db' + db_name = f'{rundate}_observations.db' obsdb = ObservationsDatabase(db_path=db_path, db_name=db_name, purge_records=True) - flist = glob.glob(os.path.join(dir_path, 'observations_*.db')) + flist = glob.glob(os.path.join(dir_path, f'observations_{rundate}_*.db')) flist.sort() for fl in flist: tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') @@ -121,17 +122,14 @@ def updatePairedDB(dir_path, db_path='/tmp', rundate=None): return obscount -def findNewMatches(dir_path, out_path, offset, repdtstr): +def findNewMatches(dir_path, out_path, repdtstr): daily_path = os.path.join(os.path.split(dir_path)[0], 'dailydbs') newdirs = getListOfNewMatches(dir_path, daily_path, rundate=repdtstr) # load camera details caminfo = loadLocationDetails() caminfo = caminfo[caminfo.active==1] - if repdtstr is not None: - repdt = datetime.datetime.strptime(repdtstr, '%Y%m%d') - else: - repdt = datetime.datetime.now() - datetime.timedelta(int(offset)) + repdt = datetime.datetime.strptime(repdtstr, '%Y%m%d') os.makedirs(out_path, exist_ok=True) # create filename. Allow for three reruns in a day @@ -195,20 +193,28 @@ def findNewMatches(dir_path, out_path, offset, repdtstr): if __name__ == '__main__': - repdtstr = None - if len(sys.argv) > 4: - repdtstr = sys.argv[4] cand_db_dir = sys.argv[1] daily_db_dir = sys.argv[2] - offset = sys.argv[3] + repdtstr = sys.argv[3] + + if repdtstr != datetime.datetime.now().strftime('%Y%m%d'): + print(f'running for a past date, extracting the container dbs to {cand_db_dir}') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + tarf = tarfile.open(os.path.join(datadir, 'distrib','containers',f'contdbs_{repdtstr}.tgz'), 'r') + members = [m for m in tarf.getmembers() if '.db' in m.name] + for m in members: + m.name = os.path.basename(m.name) + tarf.extractall(cand_db_dir, filter='fully_trusted') + tarf.close() flist = glob.glob(os.path.join(cand_db_dir, 'trajectories_*.db')) if len(flist) == 0: print('no container databases to process, aborting') else: # arguments dblocation, datadir, days ago, rundate eg 20220524 - findNewMatches(cand_db_dir, daily_db_dir, offset, repdtstr) + findNewMatches(cand_db_dir, daily_db_dir, repdtstr) # update the daily database of paired observations daily_db_dir = os.path.join(os.path.split(cand_db_dir)[0], 'dailydbs') updatePairedDB(cand_db_dir, daily_db_dir, repdtstr) + print('reportOfLatestMatches finished') diff --git a/archive/ukmon_pylib/reports/trackThumbnails.py b/archive/ukmon_pylib/reports/trackThumbnails.py index 0e012e1db..65837803b 100644 --- a/archive/ukmon_pylib/reports/trackThumbnails.py +++ b/archive/ukmon_pylib/reports/trackThumbnails.py @@ -7,7 +7,7 @@ import argparse import requests from tempfile import mkdtemp -from meteortools.utils import greatCircleDistance +from wmpl.Utils.Earth import greatCircleDistance RAD2DEG=57.2958 diff --git a/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py b/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py index 793c3ee11..7dac55f6c 100644 --- a/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py +++ b/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py @@ -17,14 +17,14 @@ def test_createBarChart(): # pass year and datadir res = createBarChart(datadir=datadir, yr=yr) - outf=os.path.join(datadir, f'Annual-{yr}.jpg') + outf=os.path.join(datadir,'reports',str(yr), f'Annual-{yr}.jpg') print(f'looking for {outf}') assert os.path.isfile(outf) os.remove(outf) # pass datadir only - will fail if we're not in 2023 res = createBarChart(datadir=datadir, yr=None) - outf=os.path.join(datadir, f'Annual-{yr}.jpg') + outf=os.path.join(datadir,'reports',str(yr), f'Annual-{yr}.jpg') print(f'looking for {outf}') assert os.path.isfile(outf) is False try: @@ -35,7 +35,7 @@ def test_createBarChart(): # no params passed res = createBarChart() - outf=os.path.join(datadir, f'Annual-{yr}.jpg') + outf=os.path.join(datadir,'reports',str(yr), f'Annual-{yr}.jpg') print(f'looking for {outf}') assert os.path.isfile(outf) is False try: diff --git a/archive/ukmon_pylib/tests/test_reports_various.py b/archive/ukmon_pylib/tests/test_reports_various.py index d9aeac7b4..9ddc3ba64 100644 --- a/archive/ukmon_pylib/tests/test_reports_various.py +++ b/archive/ukmon_pylib/tests/test_reports_various.py @@ -10,8 +10,9 @@ def test_createSummaryTable(): - createSummaryTable(curryr=None, datadir=datadir) - fname = os.path.join(datadir, 'summarytable.js') + yr=2023 + createSummaryTable(curryr=yr, datadir=datadir) + fname = os.path.join(datadir,'reports',str(yr), 'summarytable.js') assert os.path.isfile(fname) lis = open(fname, 'r').readlines() assert 'cell.innerHTML = ' in lis[9] diff --git a/archive/ukmon_pylib/traj/consolidateDistTraj.py b/archive/ukmon_pylib/traj/consolidateDistTraj.py index c1fee16df..0202a06fe 100644 --- a/archive/ukmon_pylib/traj/consolidateDistTraj.py +++ b/archive/ukmon_pylib/traj/consolidateDistTraj.py @@ -4,11 +4,14 @@ import sys import glob import datetime +import logging from wmpl.Trajectory.CorrelateDB import ObservationsDatabase, TrajectoryDatabase, CandidateDatabase from wmpl.Trajectory.CorrelateRMS import RMSDataHandle from wmpl.Utils.TrajConversions import jd2Date +log = logging.getLogger("consol_disttraj") + def mergeDatabases(srcdir, dbdir, basedir, ignore_missing=False, purge_records=False, matchstart=3): """ @@ -63,6 +66,11 @@ def mergeDatabases(srcdir, dbdir, basedir, ignore_missing=False, purge_records=F dt_beg = jd2Date(int(jdt_end) + 1 - matchstart, dt_obj=True, tzinfo=datetime.timezone.utc) event_time_range = [dt_beg, dt_end] + # capture RMSDataHandle log messages to console + log.setLevel(logging.DEBUG) + console_handler = logging.StreamHandler() + log.addHandler(console_handler) + dh = RMSDataHandle(basedir, dt_range=event_time_range, db_dir=dbdir, output_dir=basedir,mcmode=1, archivemonths=0) dh.updateTrajectoryDatabase(dt_range=event_time_range) diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index d3b6c99c7..dfe4624aa 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -116,7 +116,7 @@ def gatherUsedImageList(outf, matchstart, matchend, shbucket): trajloc = f'trajectories/{yr}/{yr}{mth:02d}/{yr}{mth:02d}{dy:02d}' out_dir = '~/data/distrib' outf.write(f'python -c "from traj.pickleAnalyser import getAllImages;getAllImages(\'{trajloc}\', \'{out_dir}\');"\n') - outf.write(f'aws s3 sync {out_dir} {shbucket}/matches/consumed/ --exclude "*" --include "consumed_*.txt --quiet"\n') + outf.write(f'aws s3 sync {out_dir} {shbucket}/matches/consumed/ --exclude "*" --include "consumed_*.txt" --quiet\n') outf.write(f'rm {out_dir}/consumed_*.txt\n') return @@ -147,12 +147,15 @@ def createExecConsolSh(matchstart, matchend, execconsolsh, istest=''): outf.write('logger -s -t execConsol start\n') outf.write(f'aws s3 sync {srcpath}/ ~/data/distrib/canddbs/ --exclude "*" --include "*.db" --exclude "dbs/*" --quiet\n') + # must run this before consolidation as the process updates the database to remove any missing + # trajectories ie not on disk. + outf.write('logger -s -t execConsol syncing any updated trajectories from shared S3\n') + refreshTrajectories(outf, matchstart, matchend, shbucket) + outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/canddbs/ {calcdir}/dbs/\n') outf.write(f'aws s3 sync {calcdir}/dbs/ {srcpath}/dbs/ --exclude "*" --include "*.db" --quiet\n') - outf.write('logger -s -t execConsol syncing any updated trajectories from shared S3\n') - refreshTrajectories(outf, matchstart, matchend, shbucket) outf.write('logger -s -t execConsol creating density plots\n') createDensityPlots(outf, calcdir, enddt, includeyear) outf.write('logger -s -t execConsol pushing data back to S3\n') diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index e81d2f1e2..9255534e9 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -110,7 +110,7 @@ def distributeCandidates(rundate, srcdir, maxcount=20, istest=False): print(f'Reading from {srcdir}') # obtain a list of picklefiles and sort by name - flist = glob.glob1(srcdir, '*.pickle') + flist = glob.glob('*.pickle', root_dir=srcdir) if len(flist) == 0: print('no candidates to process') return False diff --git a/archive/ukmon_pylib/traj/manualBulkReruns.py b/archive/ukmon_pylib/traj/manualBulkReruns.py index 497310da0..98dd19b98 100644 --- a/archive/ukmon_pylib/traj/manualBulkReruns.py +++ b/archive/ukmon_pylib/traj/manualBulkReruns.py @@ -24,9 +24,10 @@ from wmpl.Utils.TrajConversions import geo2Cartesian from wmpl.Utils.TrajConversions import raDec2ECI from wmpl.Utils.Math import vectNorm, angleBetweenVectors, vectorFromPointDirectionAndAngle +from wmpl.Utils.TrajConversions import raDec2AltAz, altAz2RADec, datetime2JD +from wmpl.Utils.Earth import greatCircleDistance -from meteortools.utils import raDec2AltAz, altAz2RADec, datetime2JD, greatCircleDistance -from meteortools.fileformats import loadFTPDetectInfo, writeNewFTPFile +from utils.ftpDetectInfo import loadFTPDetectInfo, writeNewFTPFile # Dummy platepar structure needed by loadPlatePar @@ -199,7 +200,7 @@ def getOverlappingStations(datadir, stationid): checker = EventChecker(traj_constraints=trajcons) ppdir = os.path.join(datadir, 'consolidated','platepars') rp = loadPlatepar(ppdir, stationid) - ppfs = glob.glob1(ppdir, '*.json') + ppfs = glob.glob('*.json', root_dir=ppdir) overlaps=[] for ppf in ppfs: ppfname, _ = os.path.splitext(ppf) diff --git a/archive/ukmon_pylib/traj/pickleAnalyser.py b/archive/ukmon_pylib/traj/pickleAnalyser.py index 5afb69442..16f1fd4b4 100644 --- a/archive/ukmon_pylib/traj/pickleAnalyser.py +++ b/archive/ukmon_pylib/traj/pickleAnalyser.py @@ -14,12 +14,10 @@ import datetime import json +from utils.convertSolLon import sollon2jd +from traj.ShowerAssociation import associateShower + from wmpl.Utils.TrajConversions import jd2Date -try: - from meteortools.utils import sollon2jd - from traj.ShowerAssociation import associateShower -except: - pass from wmpl.Utils.Math import mergeClosePoints, angleBetweenSphericalCoords from wmpl.Utils.Physics import calcMass from wmpl.Utils.Pickling import loadPickle @@ -81,7 +79,7 @@ def getBestView(picklename): except Exception: pass beststatid = statids[vmags.index(bestvmag)] - imgfn = glob.glob1(outdir, '*{}*.jpg'.format(beststatid)) + imgfn = glob.glob(f'*{beststatid}*.jpg', root_dir=outdir) if len(imgfn) > 0: bestimg = imgfn[0] elif os.path.isfile(os.path.join(outdir, 'jpgs.lst')): @@ -587,19 +585,20 @@ def getListOfImages(picklename): class TrajQualityParams(object): def __init__(self): - self.min_traj_points = 6 - self.min_qc = 5.0 - self.max_e = 1.5 - self.max_radiant_err = 2.0 - self.max_vg_err = 10.0 + self.min_traj_points = 4 + self.min_qc = 3.0 + self.max_e = 2.5 + self.max_radiant_err = 5.0 + self.max_vg_err = 20.0 self.max_vg = 120.0 self.max_begin_ht = 160 - self.min_end_ht = 20 + self.min_end_ht = 0 def getAllImages(dir_path, outdir): outdir = os.path.expanduser(outdir) traj_quality_params = TrajQualityParams() + trajs = loadTrajectoryPickles(dir_path, traj_quality_params, verbose=True) imglist = [] for traj in trajs: diff --git a/archive/ukmon_pylib/utils/__init__.py b/archive/ukmon_pylib/utils/__init__.py new file mode 100644 index 000000000..1d590cf3f --- /dev/null +++ b/archive/ukmon_pylib/utils/__init__.py @@ -0,0 +1 @@ +# Copyright (C) 2018- Mark McIntyre diff --git a/archive/ukmon_pylib/analysis/analyseGmnData.py b/archive/ukmon_pylib/utils/analyseGmnData.py similarity index 100% rename from archive/ukmon_pylib/analysis/analyseGmnData.py rename to archive/ukmon_pylib/utils/analyseGmnData.py diff --git a/archive/ukmon_pylib/analysis/compareBrightnessData.py b/archive/ukmon_pylib/utils/compareBrightnessData.py similarity index 91% rename from archive/ukmon_pylib/analysis/compareBrightnessData.py rename to archive/ukmon_pylib/utils/compareBrightnessData.py index 1ace683cb..663fb9af3 100644 --- a/archive/ukmon_pylib/analysis/compareBrightnessData.py +++ b/archive/ukmon_pylib/utils/compareBrightnessData.py @@ -10,9 +10,7 @@ def getBrightnessData(yyyymmdd, outdir=None): - prof = os.getenv('UKMPROFILE',default='ukmonshared') - sess = boto3.Session(profile_name=prof) - ddb = sess.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') table = ddb.Table('LiveBrightness') resp = table.query(KeyConditionExpression=Key('CaptureNight').eq(yyyymmdd)) df = pd.DataFrame(ddbjson.loads(resp['Items'])) diff --git a/archive/ukmon_pylib/utils/convertSolLon.py b/archive/ukmon_pylib/utils/convertSolLon.py new file mode 100644 index 000000000..31e4f762c --- /dev/null +++ b/archive/ukmon_pylib/utils/convertSolLon.py @@ -0,0 +1,41 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# +import numpy as np +from wmpl.Utils.TrajConversions import date2JD + + +def sollon2jd(Year, Month, Long): + """ + Calculate the julian date corresponding to a solar longitude. + Because Solar Longitude is relative to the Spring equinox, the exact date + of a given LS varies from year to year. + + Parameters: + Year: [int] year you wish to calculate in. + Month: [int] month you wish to calculate in. + Long: [float] The solar longitude to convert. + + Returns: + [float] julian date + + Notes: + The function is only stable for date ranges from 1900-2100. + """ + + Long = np.radians(Long) + N = Year - 2000 + if abs(N) > 100: + print("Algorithm is not stable for years below 1900 or above 2100") + + JDM0 = 2451182.24736 + 365.25963575 * N + ApproxJD = date2JD(Year, Month, 15, 12, 0, 0) + DiffJD = ApproxJD-2451545 + + Dt = 1.94330 * np.sin(Long - 1.798135) + 0.01305272 * np.sin(2*Long + 2.634232) + 78.195268 + 58.13165 * Long - 0.0000089408 * DiffJD + + if abs(ApproxJD - (JDM0 + Dt))>50: + Dt = Dt + 365.2596 + + JD1 = JDM0 + Dt + + return JD1 diff --git a/archive/ukmon_pylib/utils/ftpDetectInfo.py b/archive/ukmon_pylib/utils/ftpDetectInfo.py new file mode 100644 index 000000000..0e78d1300 --- /dev/null +++ b/archive/ukmon_pylib/utils/ftpDetectInfo.py @@ -0,0 +1,506 @@ +# +# Load an FTPdetectInfo file - copied from RMS +# +# Copyright (C) 2018-2023 Mark McIntyre + +import os +import numpy as np +import configparser as crp +import json +import datetime + +from wmpl.Utils.TrajConversions import date2JD +from wmpl.Utils.Math import angleBetweenSphericalCoords + + +def filterFTPforSpecificTime(ftpfile, dtstr): + """ filter FTPdetect file for a specific event, by time, and copy it into a new file + + Arguments: + ftpfile - [string] full path to the ftpdetect file to filter + dtstr - [string] date/time of required event in yyyymmdd_hhmmss format + + Returns: + tuple containing + full name of the new file containing just matching events + the number of matching events + """ + meteor_list = loadFTPDetectInfo(ftpfile, time_offsets=None, join_broken_meteors=True, locdata=None) + refdt = datetime.datetime.strptime(dtstr, '%Y%m%d_%H%M%S') + #print(refdt) + new_met_list = [] + for met in meteor_list: + #print(met.ff_name) + dtpart = datetime.datetime.strptime(met.ff_name[10:25], '%Y%m%d_%H%M%S') + tdiff = (refdt - dtpart).seconds + #print(tdiff) + if abs(tdiff) < 21: + print('adding one entry') + new_met_list.append(met) + newname = writeNewFTPFile(ftpfile, new_met_list) + return newname, len(new_met_list) + + +def writeNewFTPFile(srcname, metlist): + """ creates a FTPDetect file from a list of MeteorObservation objects + + Arguments: + srcname - [string] full path to the original FTP file + metlist - list of MeteorObservation objects + + Returns: + the full path to the created file + + """ + outdir, fname = os.path.split(srcname) + newname = os.path.join(outdir, f'{fname}.old') + try: + os.rename(srcname, newname) + except: + pass + if os.path.isfile(srcname): + srcname = srcname[:-4] + '_new.txt' + with open(srcname, 'w') as ftpf: + _writeFTPHeader(ftpf, len(metlist), outdir, False) + metno = 1 + ffname = '' + for met in metlist: + if ffname == met.ff_name: + metno = metno + 1 + else: + metno = 1 + ffname = met.ff_name + _writeOneMeteor(ftpf, metno, met.station_id, met.time_data, len(met.frames), met.fps, met.frames, + np.degrees(met.ra_data), np.degrees(met.dec_data), + np.degrees(met.azim_data), np.degrees(met.elev_data), + None, met.mag_data, False, met.x_data, met.y_data, met.ff_name) + return srcname + + +def loadFTPDetectInfo(ftpdetectinfo_file_name, time_offsets=None, + join_broken_meteors=True, locdata=None): + """ Loads an FTPDEtect file into a list of MeteorObservation objects + + Arguments: + ftpdetectinfo_file_name: [str] Path to the FTPdetectinfo file. + stations: [dict] A dictionary where the keys are stations IDs, and values are lists of: + - latitude +N in radians + - longitude +E in radians + - height in meters + + + Keyword arguments: + time_offsets: [dict] (key, value) pairs of (stations_id, time_offset) for every station. None by + default. + join_broken_meteors: [bool] Join meteors broken across 2 FF files. + + + Return: + meteor_list: [list] A list of MeteorObservation objects filled with data from the FTPdetectinfo file. + + """ + stations={} + if locdata is None: + dirname, fname = os.path.split(ftpdetectinfo_file_name) + cfgfile = os.path.join(dirname, '.config') + cfg = crp.ConfigParser() + cfg.read(cfgfile) + try: + lat = float(cfg['System']['latitude'].split()[0]) + lon = float(cfg['System']['longitude'].split()[0]) + height = float(cfg['System']['elevation'].split()[0]) + except: + # try reading from platepars file + ppf = os.path.join(dirname, 'platepars_all_recalibrated.json') + if not os.path.isfile(ppf): + return [] + js = json.load(open(ppf, 'r')) + if len(js) < 10: + return [] + lat = js[list(js.keys())[0]]['lat'] + lon = js[list(js.keys())[0]]['lon'] + height = js[list(js.keys())[0]]['elev'] + statid= fname.split('_')[1] + else: + statid = locdata['station_code'] + lat = float(locdata['lat']) + lon = float(locdata['lon']) + height = float(locdata['elev']) + + stations[statid] = [np.radians(lat), np.radians(lon), height*1000] + meteor_list = [] + + with open(ftpdetectinfo_file_name) as f: + # Skip the header + for i in range(11): + next(f) + + current_meteor = None + + bin_name = False + cal_name = False + meteor_header = False + + for line in f: + # Skip comments + if line.startswith("#"): + continue + + line = line.replace('\n', '').replace('\r', '') + + # Skip the line if it is empty + if not line: + continue + + if '-----' in line: + # Mark that the next line is the bin name + bin_name = True + + # If the separator is read in, save the current meteor + if current_meteor is not None: + current_meteor._finish() + meteor_list.append(current_meteor) + continue + + if bin_name: + bin_name = False + + # Mark that the next line is the calibration file name + cal_name = True + + # Save the name of the FF file + ff_name = line + + # Extract the reference time from the FF bin file name + line = line.split('_') + + # Count the number of string segments, and determine if it the old or new CAMS format + if len(line) == 6: + sc = 1 + else: + sc = 0 + + ff_date = line[1 + sc] + ff_time = line[2 + sc] + milliseconds = line[3 + sc] + + year = ff_date[:4] + month = ff_date[4:6] + day = ff_date[6:8] + + hour = ff_time[:2] + minute = ff_time[2:4] + seconds = ff_time[4:6] + + year, month, day, hour, minute, seconds, milliseconds = map(int, [year, month, day, hour, + minute, seconds, milliseconds]) + + # Calculate the reference JD time + jdt_ref = date2JD(year, month, day, hour, minute, seconds, milliseconds) + continue + + if cal_name: + cal_name = False + # Mark that the next line is the meteor header + meteor_header = True + continue + + if meteor_header: + meteor_header = False + line = line.split() + + # Get the station ID and the FPS from the meteor header + station_id = line[0].strip() + fps = float(line[3]) + + # Try converting station ID to integer + try: + station_id = int(station_id) + except: + pass + + # If the time offsets were given, apply the correction to the JD + if time_offsets is not None: + if station_id in time_offsets: + print('Applying time offset for station {:s} of {:.2f} s'.format(str(station_id), + time_offsets[station_id])) + + jdt_ref += time_offsets[station_id]/86400.0 + else: + print('Time offset for given station not found!') + + # Get the station data + if station_id in stations: + lat, lon, height = stations[station_id] + else: + print('ERROR! No info for station ', station_id, ' found in CameraSites.txt file!') + print('Exiting...') + break + # Init a new meteor observation + current_meteor = MeteorObservation(jdt_ref, station_id, lat, lon, height, fps, + ff_name=ff_name) + continue + + # Read in the meteor observation point + if (current_meteor is not None) and (not bin_name) and (not cal_name) and (not meteor_header): + + line = line.replace('\n', '').split() + + # Read in the meteor frame, RA and Dec + frame_n = float(line[0]) + x = float(line[1]) + y = float(line[2]) + ra = float(line[3]) + dec = float(line[4]) + azim = float(line[5]) + elev = float(line[6]) + + # Read the visual magnitude, if present + if len(line) > 8: + mag = line[8] + if mag == 'inf': + mag = None + else: + mag = float(mag) + else: + mag = None + + # Add the measurement point to the current meteor + current_meteor._addPoint(frame_n, x, y, azim, elev, ra, dec, mag) + + # Add the last meteor the the meteor list + if current_meteor is not None: + current_meteor._finish() + meteor_list.append(current_meteor) + + # Concatenate observations across different FF files ### + if join_broken_meteors: + + # Go through all meteors and compare the next observation + merged_indices = [] + for i in range(len(meteor_list)): + + # If the next observation was merged, skip it + if (i + 1) in merged_indices: + continue + + # Get the current meteor observation + met1 = meteor_list[i] + + if i >= (len(meteor_list) - 1): + break + + # Get the next meteor observation + met2 = meteor_list[i + 1] + + # Compare only same station observations + if met1.station_id != met2.station_id: + continue + + # Extract frame number + met1_frame_no = int(met1.ff_name.split("_")[-1].split('.')[0]) + met2_frame_no = int(met2.ff_name.split("_")[-1].split('.')[0]) + + # Skip if the next FF is not exactly 256 frames later + if met2_frame_no != (met1_frame_no + 256): + continue + + + # Check for frame continouty + if (met1.frames[-1] < 254) or (met2.frames[0] > 2): + continue + + # Check if the next frame is close to the predicted position + + # Compute angular distance between the last 2 points on the first FF + ang_dist = angleBetweenSphericalCoords(met1.dec_data[-2], met1.ra_data[-2], met1.dec_data[-1], + met1.ra_data[-1]) + + # Compute frame difference between the last frame on the 1st FF and the first frame on the 2nd FF + df = met2.frames[0] + (256 - met1.frames[-1]) + + # Skip the pair if the angular distance between the last and first frames is 2x larger than the + # frame difference times the expected separation + ang_dist_between = angleBetweenSphericalCoords(met1.dec_data[-1], met1.ra_data[-1], + met2.dec_data[0], met2.ra_data[0]) + + if ang_dist_between > 2*df*ang_dist: + continue + + # If all checks have passed, merge observations ### + # Recompute the frames + frames = 256.0 + met2.frames + + # Recompute the time data + time_data = frames/met1.fps + + # Add the observations to first meteor object + met1.frames = np.append(met1.frames, frames) + met1.time_data = np.append(met1.time_data, time_data) + met1.x_data = np.append(met1.x_data, met2.x_data) + met1.y_data = np.append(met1.y_data, met2.y_data) + met1.azim_data = np.append(met1.azim_data, met2.azim_data) + met1.elev_data = np.append(met1.elev_data, met2.elev_data) + met1.ra_data = np.append(met1.ra_data, met2.ra_data) + met1.dec_data = np.append(met1.dec_data, met2.dec_data) + met1.mag_data = np.append(met1.mag_data, met2.mag_data) + + # Merge the FF file name and create a list + if (met1.ff_name is not None) and (met2.ff_name is not None): + met1.ff_name = met1.ff_name + ',' + met2.ff_name + + # Sort all observations by time + met1._finish() + + # Indicate that the next observation is to be skipped + merged_indices.append(i + 1) + + # Removed merged meteors from the list + meteor_list = [element for i, element in enumerate(meteor_list) if i not in merged_indices] + + return meteor_list + + +class MeteorObservation(object): + """ Container for meteor observations. + """ + def __init__(self, jdt_ref, station_id, latitude, longitude, height, fps, ff_name=None): + """ Construct the MeteorObservation object. + Arguments: + jdt_ref: [float] Reference Julian date when the relative time is t = 0s. + station_id: [str] Station ID. + latitude: [float] Latitude +N in radians. + longitude: [float] Longitude +E in radians. + height: [float] Elevation above sea level (MSL) in meters. + fps: [float] Frames per second. + + Keyword arguments: + ff_name: [str] Name of the originating FF file. + """ + self.jdt_ref = jdt_ref + self.station_id = station_id + self.latitude = latitude + self.longitude = longitude + self.height = height + self.fps = fps + self.ff_name = ff_name + self.frames = [] + self.time_data = [] + self.x_data = [] + self.y_data = [] + self.azim_data = [] + self.elev_data = [] + self.ra_data = [] + self.dec_data = [] + self.mag_data = [] + self.abs_mag_data = [] + + def _addPoint(self, frame_n, x, y, azim, elev, ra, dec, mag): + """ Adds the measurement point to the meteor. + + Arguments: + frame_n: [flaot] Frame number from the reference time. + x: [float] X image coordinate. + y: [float] X image coordinate. + azim: [float] Azimuth, J2000 in degrees. + elev: [float] Elevation angle, J2000 in degrees. + ra: [float] Right ascension, J2000 in degrees. + dec: [float] Declination, J2000 in degrees. + mag: [float] Visual magnitude. + """ + self.frames.append(frame_n) + + # Calculate the time in seconds w.r.t. to the reference JD + point_time = float(frame_n)/self.fps + self.time_data.append(point_time) + self.x_data.append(x) + self.y_data.append(y) + + # Angular coordinates converted to radians + self.azim_data.append(np.radians(azim)) + self.elev_data.append(np.radians(elev)) + self.ra_data.append(np.radians(ra)) + self.dec_data.append(np.radians(dec)) + self.mag_data.append(mag) + + def _finish(self): + """ When the initialization is done, convert data lists to numpy arrays. """ + self.frames = np.array(self.frames) + self.time_data = np.array(self.time_data) + self.x_data = np.array(self.x_data) + self.y_data = np.array(self.y_data) + self.azim_data = np.array(self.azim_data) + self.elev_data = np.array(self.elev_data) + self.ra_data = np.array(self.ra_data) + self.dec_data = np.array(self.dec_data) + self.mag_data = np.array(self.mag_data) + # Sort by frame + temp_arr = np.c_[self.frames, self.time_data, self.x_data, self.y_data, self.azim_data, + self.elev_data, self.ra_data, self.dec_data, self.mag_data] + temp_arr = temp_arr[np.argsort(temp_arr[:, 0])] + self.frames, self.time_data, self.x_data, self.y_data, self.azim_data, self.elev_data, self.ra_data, \ + self.dec_data, self.mag_data = temp_arr.T + + +def _writeFTPHeader(ftpf, metcount, fldr, ufo=True): + """ + Internal function to create the header of the FTPDetect file + """ + l1 = 'Meteor Count = {:06d}\n'.format(metcount) + ftpf.write(l1) + ftpf.write('-----------------------------------------------------\n') + if ufo is True: + ftpf.write('Processed with UFOAnalyser\n') + else: + ftpf.write('Processed with RMS 1.0\n') + ftpf.write('-----------------------------------------------------\n') + l1 = 'FF folder = {:s}\n'.format(fldr) + ftpf.write(l1) + l1 = 'CAL folder = {:s}\n'.format(fldr) + ftpf.write(l1) + ftpf.write('-----------------------------------------------------\n') + ftpf.write('FF file processed\n') + ftpf.write('CAL file processed\n') + ftpf.write('Cam# Meteor# #Segments fps hnr mle bin Pix/fm Rho Phi\n') + ftpf.write('Per segment: Frame# Col Row RA Dec Azim Elev Inten Mag\n') + + +def _writeOneMeteor(ftpf, metno, sta, evttime, fcount, fps, fno, ra, dec, az, alt, b, mag, + ufo=True, x=None, y=None, ffname = None): + """ Internal function to write one meteor event into the file in FTPDetectInfo style + """ + ftpf.write('-------------------------------------------------------\n') + if ffname is None: + ms = '{:03d}'.format(int(evttime.microsecond / 1000)) + + fname = 'FF_' + sta + '_' + evttime.strftime('%Y%m%d_%H%M%S_') + ms + '_0000000.fits\n' + else: + fname = ffname + '\n' + ftpf.write(fname) + + if ufo is True: + ftpf.write('UFO UKMON DATA Recalibrated on: ') + else: + ftpf.write('RMS data reprocessed on: ') + ftpf.write(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f UTC\n')) + li = f'{sta} {metno:04d} {fcount:04d} {fps:04.2f} 000.0 000.0 00.0 000.0 0000.0 0000.0\n' + ftpf.write(li) + + for i in range(len(fno)): + # 204.4909 0422.57 0353.46 262.3574 +16.6355 267.7148 +23.0996 000120 3.41 + bri = 0 + if b is not None: + bri = int(b[i]) + if ufo is True: + # UFO is timestamped as at the first detection + thisfn = fno[i] - fno[0] + thisx = 0 + thisy = 0 + else: + thisfn = fno[i] + thisx = x[i] + thisy = y[i] + li = f'{thisfn:.4f} {thisx:07.2f} {thisy:07.2f} ' + li = li + f'{ra[i]:8.4f} {dec[i]:+7.4f} {az[i]:8.4f} ' + li = li + f'{alt[i]:+7.4f} {bri:06d} {mag[i]:.02f}\n' + ftpf.write(li) diff --git a/archive/ukmon_pylib/utils/getActiveShowers.py b/archive/ukmon_pylib/utils/getActiveShowers.py new file mode 100644 index 000000000..74d03b4cc --- /dev/null +++ b/archive/ukmon_pylib/utils/getActiveShowers.py @@ -0,0 +1,139 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# +# simple script to get the active shower list from the IMO working list + +from utils.imoWorkingShowerList import IMOshowerList as iwsl +import datetime +import numpy as np +import os + + +def getActiveShowers(targdate, retlist=False, inclMinor=False): + """ + Return a list of showers active at the specified date + + Arguments: + targdate: [str] Date in YYYYMMDD format + + Keyword Arguments: + retlist: [bool] return a list, or print to console. Default False=print + inclMinor: [bool] include minor showers or only return major showers + + Returns: + If retlist is true, returns a python list of shower short-codes eg ['PER','LYR'] + + """ + sl = iwsl() + testdate = datetime.datetime.strptime(targdate, '%Y%m%d') + listofshowers=sl.getActiveShowers(testdate, True, inclMinor=inclMinor) + if retlist is False: + for shwr in listofshowers: + print(shwr) + else: + return listofshowers + + +def getActiveShowersStr(targdatestr): + """ + Prints a comma-separated list of showers active at the specified date + + Arguments: + targdate: [str] Date in YYYYMMDD format + + Returns: + nothing + + """ + shwrs = getActiveShowers(targdatestr, retlist=True) + shwrs.append('spo') + for s in shwrs: + print(s) + + +def getShowerDets(shwr, stringFmt=False, dataPth=None): + """ Get details of a shower + + Arguments: + shwr: [string] three-letter shower code eg PER + Keyword Arguments: + stringFmt [bool] default False, return a string rather than a list + dataPth [string] path to the datafiles. Default None means data read from internal files. + + Returns: + (id, full name, peak solar longitude, peak date mm-dd) + """ + sl = iwsl() + mtch = sl.getShowerByCode(shwr, useFull=True) + if len(mtch) > 0 and mtch['@id'] is not None: + id = int(mtch['@id']) + nam = mtch['name'] + pkdtstr = mtch['peak'] + dt = datetime.datetime.now() + yr = dt.year + pkdt = datetime.datetime.strptime(f'{yr} {pkdtstr}','%Y %b %d') + dtstr = pkdt.strftime('%m-%d') + pksollong = mtch['pksollon'] + else: + id, nam, pksollong, dtstr = 0, 'Unknown', 0, 'Unknown' + if stringFmt: + return f"{pksollong},{dtstr},{nam},{shwr}" + else: + return id, nam, pksollong, dtstr + + +def getShowerPeak(shwr): + """ Get date of a shower peak in MM-DD format + + Arguments: + shwr: [string] three-letter shower code eg PER + + Returns: + peak date mm-dd + """ + _, _, _, pk = getShowerDets(shwr) + return pk + + +def numpifyShowerData(): + """Refresh the numpy versions of the shower data files """ + srcdir = os.getenv('SRC', default=os.path.expanduser('~/prod')) + abs_path = os.getenv('WMPL_LOC', default=os.path.expanduser('~/src/WesternMeteorPyLib')) + + iau_shower_table_file = os.path.join(abs_path, 'wmpl', 'share', 'streamfulldata.csv') + iau_shower_table_npy = os.path.join(abs_path, 'wmpl', 'share', 'streamfulldata.npy') + iau_shower_list = np.loadtxt(iau_shower_table_file, delimiter="|", usecols=range(20), dtype=str) + np.save(iau_shower_table_npy, iau_shower_list) + + iau_shower_table_npy = os.path.join(srcdir, 'share', 'streamfulldata.npy') + np.save(iau_shower_table_npy, iau_shower_list) + + gmn_shower_table_file = os.path.join(abs_path, 'wmpl', 'share', 'gmn_shower_table_20230518.txt') + gmn_shower_table_npy = os.path.join(abs_path, 'wmpl', 'share', 'gmn_shower_table_20230518.npy') + gmn_shower_list = _loadGMNShowerTable(*os.path.split(gmn_shower_table_file)) + np.save(gmn_shower_table_npy, gmn_shower_list) + + gmn_shower_table_npy = os.path.join(srcdir, 'share', 'gmn_shower_table_20230518.npy') + np.save(gmn_shower_table_npy, gmn_shower_list) + + +def _loadGMNShowerTable(dir_path, file_name): + gmn_shower_list = [] + with open(os.path.join(dir_path, file_name), encoding='cp1252') as f: + for line in f: + if line.startswith('#'): + continue + line = line.strip() + line = line.replace('\n', '').replace('\r', '') + if not line: + continue + la_sun, L_g, B_g, v_g, dispersion, IAU_no, IAU_code = line.split() + gmn_shower_list.append([ + np.radians(float(la_sun)), + np.radians(float(L_g)), + np.radians(float(B_g)), + 1000*float(v_g), + np.radians(float(dispersion)), + int(IAU_no)] + + ) + return np.array(gmn_shower_list) diff --git a/archive/ukmon_pylib/utils/getRiseSet.py b/archive/ukmon_pylib/utils/getRiseSet.py new file mode 100644 index 000000000..081eeed98 --- /dev/null +++ b/archive/ukmon_pylib/utils/getRiseSet.py @@ -0,0 +1,32 @@ +# Copyright (C) 2018- Mark McIntyre +# +import ephem + + +def getNextRiseSet(lati, longi, elev, fordate=None): + """ Calculate the next rise and set times for a given lat, long, elev + + Paramters: + lati: [float] latitude in degrees + longi: [float] longitude in degrees (+E) + elev: [float] latitude in metres + fordate:[datetime] date to calculate for, today if none + + Returns: + rise, set: [date tuple] next rise and set as datetimes + + Note that set may be earlier than rise, if you're invoking the function during daytime. + + """ + obs = ephem.Observer() + obs.lat = float(lati) / 57.3 # convert to radians, close enough for this + obs.lon = float(longi) / 57.3 + obs.elev = float(elev) + obs.horizon = -6.0 / 57.3 # degrees below horizon for darkness + if fordate is not None: + obs.date = fordate + + sun = ephem.Sun() + rise = obs.next_rising(sun).datetime() + set = obs.next_setting(sun).datetime() + return rise, set diff --git a/archive/ukmon_pylib/utils/getUncalImages.py b/archive/ukmon_pylib/utils/getUncalImages.py new file mode 100644 index 000000000..381c96a7c --- /dev/null +++ b/archive/ukmon_pylib/utils/getUncalImages.py @@ -0,0 +1,25 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# + +import os +import datetime + + +def getUncalibratedImageList(dtstr=None): + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + now = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y%m%d') + if dtstr is not None and dtstr != now: + logfile = os.path.join(datadir, '..', 'logs', f'matchJob.log-{dtstr}') + else: + logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') + if os.path.isfile(logfile): + flines = open(logfile, 'r').readlines() + uncal = [f for f in flines if 'not recalibrated' in f] + imglist = [x.split('Skipping ')[1].split(',')[0] for x in uncal] + with open(os.path.join(datadir, 'single', 'used', f'uncal_{dtstr}.txt'), 'w') as outf: + for li in imglist: + outf.write(f'{li}\n') + else: + print('file not found', logfile) + imglist=[] + return imglist diff --git a/archive/ukmon_pylib/utils/imoWorkingShowerList.py b/archive/ukmon_pylib/utils/imoWorkingShowerList.py new file mode 100644 index 000000000..abc76760a --- /dev/null +++ b/archive/ukmon_pylib/utils/imoWorkingShowerList.py @@ -0,0 +1,223 @@ +# +# python module to read the IMO Working Shower short List +# +# Copyright (C) 2018-2023 Mark McIntyre + +import xmltodict +import datetime +import os +import numpy as np +import copy + +from wmpl.Utils.TrajConversions import jd2Date +from utils.convertSolLon import sollon2jd + +try: + # imported from $SRC/share + from majorminor import majorlist, minorlist +except Exception: + c = ['QUA', 'LYR', 'ETA', 'CAP', 'SDA', 'PER', 'AUR', 'ORI', 'NTA', 'STA', 'LEO', 'GEM', 'URS'] + minorlist = ['SPE','OCT','DRA','EGE','MON','HYD','COM','NOO'] + + +class IMOshowerList: + """ + Class that loads and parses the IMO Working Shower list, or if needed, the unconfirmed list. + Not all known showers are in the IMO working list. If a shower is not in the Working List then + this library will reference the full shower list curated by Peter Jenniskens which contains + debated and unconfirmed showers. + + """ + def __init__(self, fname=None, fullstreamname=None): + if fname is None: + srcdir = os.getenv('SRC', default=os.path.expanduser('~/prod')) + fname = os.path.join(srcdir, 'share', 'IMO_Working_Meteor_Shower_List.xml') + if not os.path.isfile(fname): + print('unable to open data file') + return + + tmplist = xmltodict.parse(open(fname, 'rb').read()) + self.showerlist = tmplist['meteor_shower_list']['shower'] + if fullstreamname is None: + fullstreamname = os.path.join(srcdir, 'share', 'streamfulldata.npy') + self.fullstreamdata = np.load(fullstreamname) + #print('initialised') + + + def getShowerByCode(self, iaucode, useFull=False): + ds = {'@id':None, 'IAU_code':None,'start':None, 'end':None, + 'peak':None, 'r':None, 'name':None, 'V':None, 'ZHR':None, 'RA':None, 'DE':None, 'pksollon': None} + ds2 = {'@id':None, 'IAU_code':None,'start':None, 'end':None, + 'peak':None, 'r':None, 'name':None, 'V':None, 'ZHR':None, 'RA':None, 'DE':None, 'pksollon': None} + for shower in self.showerlist: + if shower['IAU_code'] == iaucode: + ds = shower + if ds['@id'] is None: + ds['@id'] = -1 + pksollong = -1 + #print(ds) + subset = self.fullstreamdata[np.where(self.fullstreamdata[:,3]==iaucode)] + if subset is not None and len(subset) > 0: + mtch = [sh for sh in subset if int(sh[6]) > -1] + if len(mtch) > 0: + ds2 = copy.deepcopy(ds) + ds2['IAU_code'] = mtch[-1][3].strip() + ds2['name'] = mtch[-1][4].strip() + ds2['V'] = mtch[-1][12] + ds2['@id'] = mtch[-1][1] + ds2['RA'] = mtch[-1][8] + ds2['DE'] = mtch[-1][9] + + pksollong = float(mtch[-1][7]) + dt = datetime.datetime.now() + yr = dt.year + mth = dt.month + jd = sollon2jd(yr, mth, pksollong) + pkdt = jd2Date(jd, dt_obj=True) + ds2['peak'] = pkdt.strftime('%h %d') + # start/end pop idx, ZHR not available in the IAU data + ds2['start'] = (pkdt + datetime.timedelta(days=-2)).strftime('%h %d') + ds2['end'] = (pkdt + datetime.timedelta(days=2)).strftime('%h %d') + ds2['pksollon'] = pksollong + #print(ds2) + else: + # okay so its poor quality but lets try it anyway + mtch = subset + ds2 = copy.deepcopy(ds) + ds2['IAU_code'] = mtch[-1][3].strip() + ds2['name'] = mtch[-1][4].strip() + ds2['V'] = mtch[-1][12] + ds2['@id'] = mtch[-1][1] + ds2['RA'] = mtch[-1][8] + ds2['DE'] = mtch[-1][9] + + pksollong = float(mtch[-1][7]) + dt = datetime.datetime.now() + yr = dt.year + mth = dt.month + jd = sollon2jd(yr, mth, pksollong) + pkdt = jd2Date(jd, dt_obj=True) + ds2['peak'] = pkdt.strftime('%h %d') + # start/end pop idx, ZHR not available in the IAU data + ds2['start'] = (pkdt + datetime.timedelta(days=-2)).strftime('%h %d') + ds2['end'] = (pkdt + datetime.timedelta(days=2)).strftime('%h %d') + ds2['pksollon'] = pksollong + if useFull is False: + if 'pksollon' not in ds: + ds['pksollon'] = ds2['pksollon'] + elif ds['pksollon'] is None: + ds['pksollon'] = ds2['pksollon'] + if ds['peak'] is None: + ds['peak'] = ds2['peak'] + ds['@id'] = ds2['@id'] + return ds + else: + ds2['ZHR'] = ds['ZHR'] + ds2['r'] = ds['r'] + return ds2 + + def getStart(self, iaucode, currdt=None): + shower = self.getShowerByCode(iaucode) + if currdt is None: + now = datetime.datetime.today().year + mth = datetime.datetime.today().month + else: + now = datetime.datetime.strptime(str(currdt), '%Y%m%d') + mth = now.month + now = now.year + if shower['start'] is not None: + startdate = datetime.datetime.strptime(shower['start'], '%b %d') + else: + startdate = datetime.datetime.strptime(shower['peak'], '%b %d') + datetime.timedelta(days=-3) + if iaucode == 'QUA' and mth !=12: + # quadrantids straddle yearend + now = now - 1 + startdate = startdate.replace(year=now) + return startdate + + def getEnd(self, iaucode, currdt=None): + shower = self.getShowerByCode(iaucode) + if currdt is None: + now = datetime.datetime.today().year + mth = datetime.datetime.today().month + else: + now = datetime.datetime.strptime(str(currdt), '%Y%m%d') + mth = now.month + now = now.year + #print(shower) + if shower['end'] is not None: + enddate = datetime.datetime.strptime(shower['end'], '%b %d') + else: + enddate = datetime.datetime.strptime(shower['peak'], '%b %d') + datetime.timedelta(days=3) + if iaucode == 'QUA' and mth == 12: + # quadrantids straddle yearend + now = now + 1 + enddate = enddate.replace(year=now) + return enddate + + def getPeak(self, iaucode, currdt=None): + shower = self.getShowerByCode(iaucode) + if currdt is None: + now = datetime.datetime.today().year + mth = datetime.datetime.today().month + else: + now = datetime.datetime.strptime(str(currdt), '%Y%m%d') + mth = now.month + now = now.year + enddate = datetime.datetime.strptime(shower['peak'], '%b %d') + if iaucode == 'QUA' and mth == 12: + # quadrantids straddle yearend + now = now + 1 + enddate = enddate.replace(year=now) + return enddate + + def getRvalue(self, iaucode): + shower = self.getShowerByCode(iaucode) + return shower['r'] + + def getName(self, iaucode): + shower = self.getShowerByCode(iaucode) + return shower['name'] + + def getVelocity(self, iaucode): + shower = self.getShowerByCode(iaucode) + return shower['V'] + + def getZHR(self, iaucode): + shower = self.getShowerByCode(iaucode) + zhr = shower['ZHR'] + if zhr is None: + return -1 + else: + return int(zhr) + + def getRaDec(self, iaucode): + shower = self.getShowerByCode(iaucode) + return float(shower['RA']), float(shower['DE']) + + def getActiveShowers(self, datetotest, majorOnly=False, inclMinor=False): + activelist = [] + for shower in self.showerlist: + shwname = shower['IAU_code'] + if shwname == 'ANT': #skip the anthelion source, its not a real shower + continue + start = self.getStart(shwname, datetotest.strftime('%Y%m%d')) + #print(shwname, start,shower) + end = self.getEnd(shwname, datetotest.strftime('%Y%m%d')) + datetime.timedelta(days=3) + if datetotest > start and datetotest < end: + if majorOnly is False or (majorOnly is True and shwname in majorlist): + activelist.append(shwname) + elif inclMinor is True and shwname in minorlist: + activelist.append(shwname) + return activelist + + def getMajorShowers(self, includeSpo=False, stringFmt=False): + majlist = majorlist + if includeSpo is True: + majlist.append('spo') + if stringFmt is True: + tmplist = '' + for shwr in majlist: + tmplist = tmplist + shwr + ' ' + majlist = tmplist + return majlist diff --git a/archive/ukmon_pylib/utils/kmlHandlers.py b/archive/ukmon_pylib/utils/kmlHandlers.py new file mode 100644 index 000000000..60f958b24 --- /dev/null +++ b/archive/ukmon_pylib/utils/kmlHandlers.py @@ -0,0 +1,149 @@ +import xmltodict +from shapely.geometry import Polygon +import csv +import simplekml +import numpy as np +import pandas as pd +import os + + +def readCameraKML(kmlFilename, return_poly=False): + """ Load a KML file and return either a list of lats and longs, or a Shapely polygon + + Arguments: + kmlFilename: [string] full path to the KML file to consume + return_poly: [bool] return a Shapely polygon? Default False + + + Returns: + if return_poly is false, returns a tuple of (cameraname, lats, longs) where lats and longs are lists of the + latitudes and longitudes in the KML file. + + If return_poly is true, returns a tuple of (cameranamem, shapely Polygon) + """ + + with open(kmlFilename) as fd: + x = xmltodict.parse(fd.read()) + cname = x['kml']['Folder']['name'] + coords = x['kml']['Folder']['Placemark']['MultiGeometry']['Polygon']['outerBoundaryIs']['LinearRing']['coordinates'] + coords = coords.split('\n') + if return_poly is False: + lats = [] + lngs = [] + for lin in coords: + s = lin.split(',') + lngs.append(float(s[0])) + lats.append(float(s[1])) + return cname, lats, lngs + else: + ptsarr=[] + for lin in coords: + s = lin.split(',') + ptsarr.append((float(s[0]), float(s[1]))) + polyg = Polygon(ptsarr) + return cname, polyg + + +def trackCsvtoKML(trackcsvfile, trackdata=None, saveOutput=True, outdir=None): + """ + Either reads a CSV file containing lat, long, height of an event and + creates a 3d KML file from it or, if trackdata is populated, converts a Pandas dataframe containing + the same data. Output is written to disk unless saveOutput is false. + + Arguments: + trackcsvfile: [string] full path to the file to read from + trackdata: [array] pandas dataframe containing the data. Default None + saveOutput: [bool] write the KML file to disk. Default true + outdir: [string] where to save the file. Default same folder as the source file + + Returns: + the KML file as a tuple + """ + kml=simplekml.Kml() + kml.document.name = trackcsvfile + if trackdata is None: + inputfile = csv.reader(open(trackcsvfile)) + for row in inputfile: + #columns are lat, long, height, times + kml.newpoint(name='', coords=[(row[1], row[0], row[2])]) + else: + for i,r in trackdata.iterrows(): + kml.newpoint(name=f'{r[3]:.5f}', coords=[(r[1], r[0], r[2])], extrude=1, altitudemode='absolute') + if 'csv' in trackcsvfile: + outname = trackcsvfile.replace('.csv','.kml') + else: + outname = f'{trackcsvfile}.kml' + if saveOutput: + if outdir is None: + outdir, _ = os.path.split(trackcsvfile) + os.makedirs(outdir, exist_ok=True) + outname = os.path.join(outdir, outname) + kml.save(outname) + return kml + + +def trackKMLtoCsv(kmlfile, kmldata = None, saveOutput=True, outdir=None): + """ convert a track KML retrieved by ukmondb.trajectoryKML to a 3 dimensional CSV file + containing coordinates of points on the trajectory. + + Arguments: + kmlfile [string] full path to the KML file to read from. + kmldata [kml] the kml data if available. + saveOutput [bool] Default True, save the output to file. + outdir [string] where to save to if saveOutput is True. + + Note: if kmldata is supplied, then kmlfile is ignored. + + Returns: + a Pandas dataframe containing the lat, long, alt and time of each + point on the trajectory, sorted by time. + + """ + with open(kmlfile) as fd: + x = xmltodict.parse(fd.read()) + placemarks=x['kml']['Document']['Placemark'] + lats = [] + lons = [] + alts = [] + tims = [] + for pm in placemarks: + tims.append(float(pm['name'])) + coords = pm['Point']['coordinates'].split(',') + lons.append(float(coords[0])) + lats.append(float(coords[1])) + alts.append(float(coords[2])) + df = pd.DataFrame({"lats": lats, "lons": lons, "alts": alts, "times": tims}) + df = df.sort_values(by=['times', 'lats']) + if saveOutput: + fname = kmlfile + if outdir is None: + outdir, fname = os.path.split(kmlfile) + outf = os.path.join(outdir, fname).replace('.kml', '.csv').replace('.KML','.csv') + df.to_csv(outf, index=False) + return df + + +def getTrackDetails(traj): + """ Get track details from a WMPL trajectory object + + Arguments: + traj: a WMPL trajectory object containing observations + + Returns: + a Pandas dataframe containing the lat, long, alt and time of each point on the trajectory, sorted by time. + """ + lats = [] + lons = [] + alts = [] + lens = [] + # Go through observation from all stations + for obs in traj.observations: + # Go through all observed points + for i in range(obs.kmeas): + lats.append(np.degrees(obs.model_lat[i])) + lons.append(np.degrees(obs.model_lon[i])) + alts.append(obs.model_ht[i]) + lens.append(obs.time_data[i]) + df = pd.DataFrame({"lats": lats, "lons": lons, "alts": alts, "times": lens}) + df = df.sort_values(by=['times', 'lats']) + return df diff --git a/archive/ukmon_pylib/analysis/meteorFlux.py b/archive/ukmon_pylib/utils/meteorFlux.py similarity index 100% rename from archive/ukmon_pylib/analysis/meteorFlux.py rename to archive/ukmon_pylib/utils/meteorFlux.py diff --git a/archive/ukmon_pylib/utils/sendAnEmail.py b/archive/ukmon_pylib/utils/sendAnEmail.py new file mode 100644 index 000000000..a7d215fba --- /dev/null +++ b/archive/ukmon_pylib/utils/sendAnEmail.py @@ -0,0 +1,192 @@ +# Copyright (C) 2018-2023 Mark McIntyre +import os +import platform +import base64 +import email +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart + +from google.auth.transport.requests import Request +from google.oauth2.credentials import Credentials +from google_auth_oauthlib.flow import InstalledAppFlow +from googleapiclient.discovery import build + +SCOPES =['https://mail.google.com/'] + + +def _getGmailCreds(tokfile=None, crdfile=None): + creds = None + # The token stores the user's access and refresh tokens, and is + # created automatically when the authorization flow completes for the first + # time. + if tokfile is None: + tokfile = '~/.ssh/gmailtoken.json' + if crdfile is None: + crdfile = '~/.ssh/gmailcreds.json' + tokfile = os.path.expanduser(tokfile) + crdfile = os.path.expanduser(crdfile) + if os.path.exists(tokfile): + creds = Credentials.from_authorized_user_file(tokfile, SCOPES) + + # If there are no (valid) credentials available, ask the user to log in interactively. + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + if not os.path.isfile(crdfile): + print(f'to use this function you must have stored your Google OAUTH2 secret json file in {crdfile}') + print('To set up OAUTH2 go to your google cloud console, select APIs then Credentials, and add an OAUTH2 desktop client ID.') + return None + flow = InstalledAppFlow.from_client_secrets_file(crdfile, SCOPES) + creds = flow.run_local_server(port=0) + # Save the credentials for the next run + with open(tokfile, 'w') as token: + token.write(creds.to_json()) + return creds + + +def _refreshCreds(tokfile=None, crdfile=None): + creds = None + if tokfile is None: + tokfile = '~/.ssh/gmailtoken.json' + if crdfile is None: + crdfile = '~/.ssh/gmailcreds.json' + tokfile = os.path.expanduser(tokfile) + crdfile = os.path.expanduser(crdfile) + if os.path.exists(tokfile): + creds = Credentials.from_authorized_user_file(tokfile, SCOPES) + + # If there are no (valid) credentials available, let the user log in. + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + try: + creds.refresh(Request()) + except: + flow = InstalledAppFlow.from_client_secrets_file(crdfile, SCOPES) + creds = flow.run_local_server(port=0) + # Save the credentials for the next run + if creds.valid: + with open(tokfile, 'w') as token: + token.write(creds.to_json()) + else: + print('credentials not valid, try again') + return creds + + +def _create_message(sender, to, subject, message_text, msg_text_html=None): + if msg_text_html is None: + msg = MIMEText(message_text) + else: + msg = MIMEMultipart('alternative') + msg.attach(MIMEText(message_text, 'plain')) + msg.attach(MIMEText(msg_text_html, 'html')) # html must be the last part + msg['to'] = to + msg['from'] = sender + msg['subject'] = subject + return {'raw': base64.urlsafe_b64encode(msg.as_string().encode('utf-8')).decode('utf-8')} + + + +def sendAnEmail(mailrecip, message, subject, mailfrom, files=None, tokfile=None, crdfile=None, msg_html=None): + """ sends an email using gmail. + + Arguments: + mailrecip: [string] email address of recipient. + message: [string] the message to send. + subject: [string] Subject line. + mailfrom: [string] email address of sender. + Keyword Args: + files: [list] list of files to attach, not currently implemented. + tokfile: [string] full path to token file, if not ~/.ssh/gmailtoken.json + crdfile: [string] full path to credentials file, if not ~/.ssh/gmailcreds.json + msg_html: [string] HTML version of the message body, if any + + Returns: + Nothing, though a message is printed onscreen. + + Notes: + You must have gmail OAUTH2 set up. The gmail credentials default to gmailtoken.json and + gmailcreds.json in the $HOME/.ssh folder. + """ + + if subject is None: + subject = platform.uname()[1] + + # email a summary to the mailrecip + creds = _getGmailCreds(tokfile, crdfile) + if not creds: + return + service = build('gmail', 'v1', credentials=creds) + + subj = subject + mailmsg = _create_message(mailfrom, mailrecip, subj, message, msg_text_html=msg_html) + + try: + retval = (service.users().messages().send(userId='me', body=mailmsg).execute()) + print('Message Id: %s' % retval['id']) + except: + print('An error occurred sending the message') + + +def forwardAnEmail(reciplist, msgid=None, tokfile=None, crdfile=None): + """ Forward email from a gmail account to a list of recipients + + Arguments: + recplist: [list of strings] list of addresses to forward to + + Keyword arguments: + msgid: [string] gmail message id. If None, then all unread mail is forwarded + tokfile: [string] full path to a gmail oauth2 json token file + crdfile: [string] full path to a gmail oauth2 json credentials file for initial authorization + + To obtain the oauth2 credentials file, go to the google cloud console, select APIs and enable gmail. Then + go to Credentials and create an OAUTH2 Client ID for Desktop and download the JSON credentials file. + Upon first run, you'll be taken to the gmail authorisation screen and the token file will be created. + Subsequent runs will use the token file. The creds file will be used to reauthorise periodically. + """ + creds = _getGmailCreds(tokfile, crdfile) + if not creds: + return + service = build('gmail', 'v1', credentials=creds) + userid = 'me' + if msgid is None: + try: + msglist = (service.users().messages().list(userId=userid, labelIds=['INBOX','UNREAD']).execute()) + for msg in msglist['messages']: + msgid = msg['id'] + # retrieve raw mail + message = (service.users().messages().get(userId='me', id=msgid, format='raw').execute()) + # decode it from base64 + decmsg = base64.urlsafe_b64decode(message['raw']) + newmsg = email.message_from_bytes(decmsg) + newmsg.add_header('Reply-To',newmsg['from']) + newmsg.replace_header('Subject','Fwd: ' + newmsg['subject']) + # Send it + for recip in reciplist: + newmsg.replace_header('To', recip) + mailmsg = {'raw': base64.urlsafe_b64encode(newmsg.as_string().encode('utf-8')).decode('utf-8')} + retval = (service.users().messages().send(userId='me', body=mailmsg).execute()) + # This will mark the messagea as read + service.users().messages().modify(userId=userid, id=msgid, body={'removeLabelIds': ['UNREAD']}).execute() + print(retval) + except Exception: + print('Nothing to forward') + else: + try: + message = (service.users().messages().get(userId='me', id=msgid, format='raw').execute()) + # decode it from base64 + decmsg = base64.urlsafe_b64decode(message['raw']) + newmsg = email.message_from_bytes(decmsg) + newmsg.add_header('Reply-To',newmsg['from']) + newmsg.replace_header('Subject','Fwd: ' + newmsg['subject']) + # Send it + for recip in reciplist: + newmsg.replace_header('To', recip) + mailmsg = {'raw': base64.urlsafe_b64encode(newmsg.as_string().encode('utf-8')).decode('utf-8')} + retval = (service.users().messages().send(userId='me', body=mailmsg).execute()) + # This will mark the messagea as read + service.users().messages().modify(userId=userid, id=msgid, body={'removeLabelIds': ['UNREAD']}).execute() + print(retval) + except Exception: + print('Nothing to forward') + return diff --git a/archive/utils/README.md b/archive/utils/README.md index e179ca9f5..d1317bbe9 100644 --- a/archive/utils/README.md +++ b/archive/utils/README.md @@ -3,27 +3,36 @@ This folder contains some utility functions useful for managing the environment. ## Used by the Batch +* cleanupDeletedTrajs.sh - checks for and removes any logically-deleted trajectories eg duplicates +* getCostmetrics.sh - extract costs data from AWS and push to the website +* loadMatchCsvMDB.sh - loads the match data into MariaDB +* loadSingleCsvMDB.sh - loads the single-station data into MariaDB +* loadBrightCsvMDB.sh - loads brightness data into MariaDB * clearSpace.sh - used by the batch to delete old logs etc -* clearCaches.sh - clears the in-memory cache. Not used much. +* clearCaches.sh - clears the in-memory cache before and after running memory-intensive jobs. + +## Other routine maintenance +* checkAndRollKeys.sh - rolls station AWS Key/Secret pairs periodically to avoid stale keys +* statsToMqtt.sh - posts server space/memory etc statistics to MQ +* userAudit.sh - performs a user audit and emails a report ## User tools to maintain data * deleteOrbit.sh - remove an orbit from the database and website * updateFireballFlag.sh - marks or unmarks a match as a Fireball * updateFireballImage.sh - updates the image shown for a fireball -* stopstart-calcengine.sh - stops/starts the calculation engine server +* stopstartCalcengine.sh - stops/starts the calculation engine server ## Used by the deployment tool * makeConfig.sh - used by the deployment process to make the config file ## Used in case one of the Lambdas fails -* rerunconsolidateFTPdetect.sh - reruns the lambda -* rerunFTPtoUkMONlambra.sh - reruns the lambda -* rerun GetExtraFiles.sh - reruns the lambda +* rerunFTPtoUkMONlambra.sh - reruns FTPtoUKMON lambda +* rerunGetExtraFiles.sh - reruns GetExtraFiles lambda * cftpd_templ.json - template used by the above ## The following are work in progress -* monthlyClearDown.sh - should be run monthly to clear down old data * createTestDataSet.sh - work in progress +* getGmnData.sh - pulls GMN datasets from the GMN server ## Copyright All code Copyright (C) 2018-2023 Mark McIntyre diff --git a/archive/utils/checkCalcServerRunTime.sh b/archive/utils/checkCalcServerRunTime.sh deleted file mode 100644 index 8380d69f3..000000000 --- a/archive/utils/checkCalcServerRunTime.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# script to extract the start/stop times of the Calculation Engine server from the logs -# I have a spreadsheet that consumes this and works out the cost for various models of server (c8g.2xlarge, c8g.4xlarge etc) - -cd ~/prod/logs - -csl=/tmp/calcserver_timings.txt -csl2=/tmp/csl.txt -[ -f $csl ] && rm $csl -touch $csl - -ls -1tr nightly*.gz | while read i -do - mfz=matchJob${i:10:50} - gunzip $i - gunzip $mfz - nf=$(basename $i .gz) - mf=$(basename $mfz .gz) - egrep "start correlation server|stop the server again|restarting server to|stopping calcserver again" $mf $nf >> $csl - gzip $nf - gzip $mf -done -ls -1tr nightlyJob* | grep -v gz | while read nf -do - mf=matchJob${nf:10:50} - egrep "start correlation server|stop the server again|restarting server to|stopping calcserver again" $mf $nf >> $csl -done -[ -f $csl2 ] && rm $csl2 -touch $csl2 -cat $csl | while read i -do - parta=$(echo $i | awk -F ">" '{print $2}') - partb=$(echo $parta | awk -F " " '{printf("%s, %s, %s\n",$1,$2,$3)}') - echo $partb >> $csl2 -done -[ -f $csl ] && rm $csl -python << EOD -lis = open('$csl2','r').readlines() -data = [x.strip().split(',') for x in lis] -currdt = None -results = [] -res = [] -col = 0 -with open('/tmp/computeservertimes.csv','w') as outf: - for i in range(len(data)): - mth = data[i][0] - dy = data[i][1] - tim = data[i][2] - dtstr = f'{mth} {dy}' - if currdt is None: - currdt = dtstr - if currdt != dtstr: - outf.write(','.join(res) + '\n') - res = [] - col = 0 - if col == 0: - res.append(dtstr) - res.append(tim) - else: - res.append(tim) - col += 1 - currdt = dtstr - outf.write(','.join(res) + '\n') -EOD -[ -f $csl2 ] && rm $csl2 diff --git a/archive/utils/cleanupDeletedTrajs.sh b/archive/utils/cleanupDeletedTrajs.sh index bd0836b9a..70774dd1d 100644 --- a/archive/utils/cleanupDeletedTrajs.sh +++ b/archive/utils/cleanupDeletedTrajs.sh @@ -14,50 +14,56 @@ cd ${DATADIR}/distrib startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') jdt_min=$(python -c "from wmpl.Utils.TrajConversions import datetime2JD;import datetime;print(datetime2JD(datetime.datetime.strptime('$startdt', '%Y%m%d-%H%M%S')))") -echo "checking main storage, website and csvfiles" +logger -s -t cleanupDeletedTrajs "starting: checking main storage and website" sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do - export moved=0 - trajdir=$(dirname $traj) - #echo $trajdir - srcloc=${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir - trgloc=${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir - aws s3 ls $srcloc/ | awk -F " " '{print $4}' | while read fname; do - aws s3 mv ${srcloc}/$fname ${trgloc}/$fname --quiet - export moved=1 - done - yr=${trajdir:13:4} - trajpth=$(basename $trajdir) - webloc=$WEBSITEBUCKET/reports/${yr}/orbits/$trajpth - newloc=${UKMONSHAREDBUCKET}/matches/duplicates/reports/${yr}/orbits/$trajpth - aws s3 ls $webloc/ | awk -F " " '{print $4}' | while read fname; do - aws s3 mv $webloc/$fname $newloc/$fname --quiet - export moved=1 - done - - trajpth1=${trajdir:34:19} - csvname=$(echo $trajpth1 | sed 's/_/-/g') - csvloc=${UKMONSHAREDBUCKET}/matches/${yr}/fullcsv - newloc=${UKMONSHAREDBUCKET}/matches/duplicates/csvs/${yr} - aws s3 ls $csvloc/ | awk -F " " '{print $4}' | grep $csvname | while read fname; do - aws s3 mv $csvloc/$fname $newloc/$fname --quiet - export moved=1 - done - csvloc=$DATADIR/orbits/${yr}/fullcsv/processed - ls -1 $csvloc/ | grep $csvname | while read fname; do - aws s3 mv $csvloc/$fname $newloc/$fname --quiet - export moved=1 - done - [ $moved == 1 ] && echo "moved $trajpth" + # check if there are two trajectories with the same folder - we don't want to delete the active one + cnt=$(sqlite3 $DATADIR/distrib/trajectories.db "select count(traj_id) from trajectories where traj_file_path='$traj';") + moved=0 + if [ $cnt == 1 ] ; then + trajdir=$(dirname $traj) + srcloc=${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir + trgloc=${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir + aws s3 ls $srcloc/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv ${srcloc}/$fname ${trgloc}/$fname --quiet + moved=1 + done + yr=${trajdir:13:4} + trajpth=$(basename $trajdir) + webloc=$WEBSITEBUCKET/reports/${yr}/orbits/$trajpth + newloc=${UKMONSHAREDBUCKET}/matches/duplicates/reports/${yr}/orbits/$trajpth + aws s3 ls $webloc/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv $webloc/$fname $newloc/$fname --quiet + moved=1 + done + [ $moved == 1 ] && echo "moved $trajpth" + else + echo skipping $traj + fi done -echo "checking consolidated matches" +logger -s -t cleanupDeletedTrajs "checking raw fullcsv data files" +yr=${startdt:0:4} +csvloc=matches/${yr}/fullcsv +newloc=matches/duplicates/csvs +python -c "from maintenance.dataMaintenance import removeDeletedTrajCsv;removeDeletedTrajCsv('$csvloc', '$newloc')" + +# cater for year-end +newyr=$(date +%Y) +if [ $newyr != $yr ] ; then + csvloc=matches/${newyr}/fullcsv + newloc=matches/duplicates/csvs + python -c "from maintenance.dataMaintenance import removeDeletedTrajCsv;removeDeletedTrajCsv('$csvloc', '$newloc')" + +fi + +logger -s -t cleanupDeletedTrajs "checking consolidated matches" lasttraj=$(sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | tail -1) trajdir=$(dirname $lasttraj) yr=${trajdir:13:4} trajpth=$(basename $trajdir) matchcsv=$DATADIR/matched/matches-full-${yr}.csv -grep $trajpth $matchcsv +grep $trajpth $matchcsv > /dev/null if [ $? == 0 ] ; then python -c "from maintenance.dataMaintenance import removeDeletedTraj;removeDeletedTraj('$matchcsv')" python -m converters.toParquet $matchcsv @@ -70,7 +76,6 @@ if [ $? == 0 ] ; then aws s3 sync $DATADIR/searchidx/ $WEBSITEBUCKET/search/indexes/ --exclude "*" --include "*allevents.csv" --quiet fi -export AWS_PROFILE=ukmonshared # needed for mariadb connection details -echo "checking Mariadb Database" -python -c "from maintenance.dataMaintenance import removeDelTrajFromDb;removeDelTrajFromDb()" -unset AWS_PROFILE +# no need to check the SQL database in mariadb, as it is populated from the CSV file that +# we cleaned up above +logger -s -t cleanupDeletedTrajs "finished cleanupDeletedTrajs" diff --git a/archive/utils/liveevent.template b/archive/utils/liveevent.template deleted file mode 100644 index 02af78a63..000000000 --- a/archive/utils/liveevent.template +++ /dev/null @@ -1,15 +0,0 @@ -{ - "Records": [ - { - "s3": { - "bucket": { - "name": "ukmda-live", - "arn": "arn:aws:s3:::ukmda-live" - }, - "object": { - "key": "KEYHERE" - } - } - } - ] -} diff --git a/archive/utils/updateDb.sh b/archive/utils/loadBrightnessCsvMDB.sh similarity index 69% rename from archive/utils/updateDb.sh rename to archive/utils/loadBrightnessCsvMDB.sh index 501497f0b..b38b95c39 100644 --- a/archive/utils/updateDb.sh +++ b/archive/utils/loadBrightnessCsvMDB.sh @@ -11,8 +11,10 @@ else rundt=$1 fi -cd ~/prod/data/brightness -mysql -u batch -p$(cat ~/.ssh/db_batch.passwd) -h localhost << EOD + +cd $DATADIR/brightness +passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value| sed 's/"//g') +mysql -p$passwd -ubatch -h localhost << EOD use ukmon; select count(*) from ukmon.brightness; load data local infile './CaptureNight_${rundt}.csv' diff --git a/archive/utils/loadMatchCsvMDB.sh b/archive/utils/loadMatchCsvMDB.sh index b2fd845f7..4dfaccb51 100644 --- a/archive/utils/loadMatchCsvMDB.sh +++ b/archive/utils/loadMatchCsvMDB.sh @@ -17,8 +17,9 @@ if [ ! -f $csvname ] ; then echo "$csvname not found" ; exit ; fi echo "loading $csvname into mariadb" logger -s -t loadSQL "starting" user=batch -passwd=$(cat ~/.ssh/db_batch.passwd ) -mysql -u${user} -p${passwd} << EOD +# leading space to prevent being inserted into environment +passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value | sed 's/"//g') +mysql -p${passwd} -u${user} << EOD select count(*) from ukmon.matches; LOAD DATA LOCAL INFILE '$csvname' INTO TABLE ukmon.matches diff --git a/archive/utils/loadSingleCsvMDB.sh b/archive/utils/loadSingleCsvMDB.sh index 09fdaaaab..6d7b7b02b 100644 --- a/archive/utils/loadSingleCsvMDB.sh +++ b/archive/utils/loadSingleCsvMDB.sh @@ -17,8 +17,9 @@ if [ ! -f $csvname ] ; then echo "$csvname not found" ; exit ; fi echo "loading $csvname into mariadb" logger -s -t loadSQL "starting" user=batch -passwd=$(cat ~/.ssh/db_batch.passwd ) -mysql -u${user} -p${passwd} << EOD +# leading space to prevent being inserted into environment +passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value | sed 's/"//g') +mysql -p${passwd} -u${user} << EOD select count(*) from ukmon.singles; LOAD DATA LOCAL INFILE '$csvname' INTO TABLE ukmon.singles diff --git a/archive/utils/makeConfig.sh b/archive/utils/makeConfig.sh index a0aa9d567..d68f3f9cb 100644 --- a/archive/utils/makeConfig.sh +++ b/archive/utils/makeConfig.sh @@ -8,7 +8,7 @@ fi RUNTIME_ENV=$1 envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') -export AWS_PROFILE=default +#export AWS_PROFILE=default # read from AWS SSM Parameterset SRC=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_srcdir --query Parameters[0].Value | tr -d '"') @@ -19,23 +19,21 @@ UKMONLIVEBUCKET=s3://$(aws ssm get-parameters --region eu-west-2 --names ${envna RMS_LOC=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_rmshome --query Parameters[0].Value | tr -d '"') WMPL_LOC=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_wmplhome --query Parameters[0].Value | tr -d '"') SERVERINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcinstance --query Parameters[0].Value | tr -d '"') -BKPINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_backupinstance --query Parameters[0].Value | tr -d '"') SERVERSSHKEY=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_sshkey --query Parameters[0].Value | tr -d '"') NJLOGGRP=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_batchloggroup --query Parameters[0].Value | tr -d '"') SERVERUSERID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcuser --query Parameters[0].Value | tr -d '"') +CALCSERVERIP=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcserverip --query Parameters[0].Value | tr -d '"') unset AWS_PROFILE # hardcoded PYLIB=$SRC/ukmon_pylib TEMPLATES=$SRC/website/templates -RCODEDIR=$SRC/R DATADIR=$SRC/data AWS_DEFAULT_REGION=eu-west-2 MATCHSTART=2 MATCHEND=0 RMS_ENV=RMS WMPL_ENV=wmpl -APIKEY=$(cat ~/.ssh/gmapsapikey) KMLTEMPLATE=*70km.kml # variables that can be used in scripts and python files to ensure test data doesnt overwrite live data @@ -60,11 +58,10 @@ echo "UKMONSHAREDBUCKET=${UKMONSHAREDBUCKET}" >> ${CFGFILE} echo "UKMONLIVEBUCKET=${UKMONLIVEBUCKET}" >> ${CFGFILE} echo "PYLIB=${PYLIB}" >> ${CFGFILE} echo "TEMPLATES=${TEMPLATES}" >> ${CFGFILE} -echo "RCODEDIR=${RCODEDIR}" >> ${CFGFILE} echo "DATADIR=${DATADIR}" >> ${CFGFILE} echo "SERVERINSTANCEID=${SERVERINSTANCEID}" >> ${CFGFILE} echo "SERVERUSERID=${SERVERUSERID}" >> ${CFGFILE} -echo "BKPINSTANCEID=${BKPINSTANCEID}" >> ${CFGFILE} +echo "CALCSERVERIP=${CALCSERVERIP}" >> ${CFGFILE} echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> ${CFGFILE} echo "RMS_ENV=${RMS_ENV}" >> ${CFGFILE} echo "WMPL_ENV=${WMPL_ENV}" >> ${CFGFILE} @@ -73,7 +70,6 @@ echo "WMPL_LOC=${WMPL_LOC}" >> ${CFGFILE} echo "MATCHSTART=${MATCHSTART}" >> ${CFGFILE} echo "MATCHEND=${MATCHEND}" >> ${CFGFILE} echo "SERVERSSHKEY=${SERVERSSHKEY}" >> ${CFGFILE} -echo "APIKEY=${APIKEY}" >> ${CFGFILE} echo "KMLTEMPLATE=${KMLTEMPLATE}" >> ${CFGFILE} echo "NJLOGGRP=${NJLOGGRP}" >> ${CFGFILE} echo "TESTMODE=${TESTMODE}" >> ${CFGFILE} @@ -81,11 +77,11 @@ echo "TESTSUFF=${TESTSUFF}" >> ${CFGFILE} echo "" >> ${CFGFILE} echo "export RUNTIME_ENV SRC SITEURL" >> ${CFGFILE} echo "export WEBSITEBUCKET UKMONSHAREDBUCKET" >> ${CFGFILE} -echo "export PYLIB TEMPLATES RCODEDIR DATADIR BKPINSTANCEID AWS_DEFAULT_REGION" >> ${CFGFILE} +echo "export PYLIB TEMPLATES DATADIR AWS_DEFAULT_REGION" >> ${CFGFILE} echo "export RMS_ENV RMS_LOC WMPL_ENV WMPL_LOC" >> ${CFGFILE} echo "export PYTHONPATH=${RMS_LOC}:${WMPL_LOC}:${PYLIB}:${SRC}/share" >> ${CFGFILE} echo "export MATCHSTART MATCHEND SERVERSSHKEY" >> ${CFGFILE} -echo "export APIKEY KMLTEMPLATE SERVERINSTANCEID SERVERUSERID NJLOGGRP" >> ${CFGFILE} +echo "export KMLTEMPLATE SERVERINSTANCEID SERVERUSERID CALCSERVERIP NJLOGGRP" >> ${CFGFILE} echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib" >> ${CFGFILE} echo "export TESTMODE TESTSUFF" >> ${CFGFILE} @@ -93,8 +89,6 @@ echo "export TESTMODE TESTSUFF" >> ${CFGFILE} echo "source ~/.condaon" >> ${CFGFILE} # function to log to cloudwatch echo 'function log2cw() { ' >> ${CFGFILE} -echo 'aws logs put-log-events --log-group-name $1 --log-stream-name $2 --log-events "[{\"timestamp\": $(date +%s%3N), \"message\": \"$3\"}]" --profile ukmonshared > /dev/null ' >> ${CFGFILE} +echo 'aws logs put-log-events --log-group-name $1 --log-stream-name $2 --log-events "[{\"timestamp\": $(date +%s%3N), \"message\": \"$3\"}]" > /dev/null ' >> ${CFGFILE} echo 'logger -s -t $4 RUNTIME $SECONDS $3' >> ${CFGFILE} echo '}' >> ${CFGFILE} - - diff --git a/archive/utils/monthlyCleardown.sh b/archive/utils/monthlyCleardown.sh deleted file mode 100644 index 89ca8f7bf..000000000 --- a/archive/utils/monthlyCleardown.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# script to backup then clear down orbits from the local disk. - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -source $here/../config.ini >/dev/null 2>&1 -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -what=$1 -if [ "$what" == "" ] ; then - echo "usage: ./monthlyCleardown.sh single|raw|matched|reports|search|other" - exit -fi - -cyr=$(date +%Y) -cmt=$(date +%m) -lyr=$(date --date='last year' +%Y) -lym=$(date --date='last month' +%Y%m) - -mkdir -p $DATADIR/olddata - -# SINGLE STATION DATA -case "$what" in -single) - logger -s -t monthlyClearDown "single-station data" - cd $DATADIR/single/new/processed - # backup then remove last year's data if still present - if compgen -G "$DATADIR/single/new/processed/ukmon*_${lyr}????_*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-singlecsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-singlecsv.tar ukmon*_${lyr}????_*.csv - else - tar uvf $DATADIR/olddata/${lyr}-singlecsv.tar ukmon*_${lyr}????_*.csv - fi - rm -f $DATADIR/single/new/processed/ukmon*_${lyr}????_*.csv - else - echo "no ukmon-csv files from ${lyr} to archive" - fi - # backup last month's data - if compgen -G "$DATADIR/single/new/processed/ukmon*_${lym}??_*.csv" > /dev/null ; then - if [ $cmt -ne 1 ] ; then - if [ ! -f $DATADIR/olddata/${lym}-singlecsv.tar ] ; then - tar cvf $DATADIR/olddata/${lym}-singlecsv.tar ukmon*_${lym}??_*.csv - else - tar uvf $DATADIR/olddata/${lym}-singlecsv.tar ukmon*_${lym}??_*.csv - fi - fi - rm -f $DATADIR/single/new/processed/ukmon*_${lym}??_*.csv - else - echo "no ukmon-csv files from ${lym} to archive" - fi - ;; -raw) - logger -s -t monthlyClearDown "rawcsv data" - # raw CSV files from cameras - cd $DATADIR/single/rawcsvs - # backup then remove last year's data if still present - if compgen -G "$DATADIR/single/rawcsvs/*${lyr}????_??????_*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-rawcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lyr}????_??????_*.csv - else - tar uvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lyr}????_??????_*.csv - fi - find . -maxdepth 1 -name "*${lyr}????_??????_*.csv" -print0 | xargs -0 rm -f - else - echo "no rawcsv files from ${lyr} to archive" - fi - # backup last month's data - if compgen -G "$DATADIR/single/rawcsvs/*${lym}??_??????_*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-rawcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lym}??_??????_*.csv - else - tar uvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lym}??_??????_*.csv - fi - rm -f $DATADIR/single/rawcsvs/*${lym}??_??????_*.csv - else - echo "no rawcsv files from ${lym} to archive" - fi - ;; -matched) - logger -s -t monthlyClearDown "match data" - # MATCHES - # last year's fullcsv files - cd $DATADIR/orbits/$lyr/fullcsv/processed - if compgen -G "$DATADIR/orbits/${lyr}/fullcsv/processed/${lyr}*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-matchfullcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-matchfullcsv.tar ${lyr}*.csv - else - tar uvf $DATADIR/olddata/${lyr}-matchfullcsv.tar ${lyr}*.csv - fi - rm -f $DATADIR/orbits/${lyr}/fullcsv/processed/${lyr}*.csv - else - echo "no fullcsv files from ${lyr} to archive" - fi - - # and now last months fullcsv data - cd $DATADIR/orbits/$cyr/fullcsv/processed - if compgen -G "$DATADIR/orbits/${cyr}/fullcsv/processed/${lym}??-*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lym}-matchfullcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lym}-matchfullcsv.tar ${lym}??-*.csv - else - tar uvf $DATADIR/olddata/${lym}-matchfullcsv.tar ${lym}??-*.csv - fi - rm -f $DATADIR/orbits/${cyr}/fullcsv/processed/${lym}??-*.csv - else - echo "no fullcsv files from ${lym} to archive" - fi - ;; -reports) - logger -s -t monthlyClearDown "reports and so forth" - # SHOWER, MONTHLY AND ANNUAL REPORTS FROM PREVIOUS YEAR - cd $DATADIR/reports/${lyr} - nf=$(ls -1 | wc -l) - if [ $nf -ne 0 ] ; then - if [ ! -f $DATADIR/olddata/${lyr}reports.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-reports.tar * - else - tar uvf $DATADIR/olddata/${lyr}-reports.tar * - fi - rm -Rf * - else - echo "no report files from ${lyr} to archive" - fi - ;; -search) - logger -s -t monthlyClearDown "old search indexes" - # SEARCH INDEXES FOR PRIOR YEAR - cd $DATADIR/searchidx - if compgen -G "$DATADIR/searchidx/${lyr}-allevents.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-searchidx.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-searchidx.tar ${lyr}-allevents.csv - else - tar uvf $DATADIR/olddata/${lyr}-searchidx.tar ${lyr}-allevents.csv - fi - rm -Rf $DATADIR/searchidx/${lyr}-allevents.csv - else - echo "no search indexes from ${lyr} to archive" - fi - ;; -ftpdata) - logger -s -t monthlyClearDown "old ftpdetect and platepars_all files" - # cleardown the raw input data. This is already backed up elsewhere - inputym=$(date --date='3 months ago' +%Y%m) - #python $PYLIB/maintenance/dataMaintenance.py $inputym - ;; -*) - echo missing argument - ;; -esac -# now push it all to the offline backup -logger -s -t monthlyClearDown "syncing to backup bucket" -aws s3 sync $DATADIR/olddata/ s3://ukmda-admin/backups/ --exclude "*" --include "${lyr}*.tar" --quiet -rm -f $DATADIR/olddata/${lyr}*.tar - -aws s3 sync $DATADIR/olddata/ s3://ukmda-admin/backups/ --exclude "*" --include "${lym}*.tar" --quiet -# dont do this in case data arrives late -#rm -f $DATADIR/olddata/${lym}*.tar diff --git a/archive/utils/rerunFTPtoUKMONlambda.sh b/archive/utils/rerunFTPtoUKMONlambda.sh index 9c768d024..38d6b1f63 100644 --- a/archive/utils/rerunFTPtoUKMONlambda.sh +++ b/archive/utils/rerunFTPtoUKMONlambda.sh @@ -22,6 +22,6 @@ cat /tmp/ftp2ukmon.txt | while read i do fullname=${i} cat $SRC/utils/cftpd_templ.json | sed "s|KEYGOESHERE|${fullname}|g" > ./tmp.json - aws lambda invoke --profile ukmonshared --function-name ftpToUkmon --log-type Tail --payload file://./tmp.json --region eu-west-2 --cli-binary-format raw-in-base64-out res.log + aws lambda invoke --function-name ftpToUkmon --log-type Tail --payload file://./tmp.json --region eu-west-2 --cli-binary-format raw-in-base64-out res.log done rm ./tmp.json \ No newline at end of file diff --git a/archive/utils/statsToMqtt.sh b/archive/utils/statsToMqtt.sh index e2334bfee..170ebb0bf 100644 --- a/archive/utils/statsToMqtt.sh +++ b/archive/utils/statsToMqtt.sh @@ -10,7 +10,7 @@ if [ "$(hostname)" == "wordpresssite" ] ; then msg="$(hostname) diskspace was ${diskpct}%" subj="Diskspace alert for $(hostname)" hn="$(hostname)@aws" - python -c "from meteortools.utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" + python -c "from utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" fi elif [ "$(hostname)" == "ukmcalcserver" ] ; then source $HOME/venvs/wmpl/bin/activate @@ -31,6 +31,6 @@ else msg="$(hostname) diskspace is ${diskpct}%" subj="Diskspace alert for $(hostname)" hn="$(hostname)@aws" - python -c "from meteortools.utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" + python -c "from utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" fi fi diff --git a/archive/utils/stopstart-calcengine.sh b/archive/utils/stopstartCalcengine.sh similarity index 90% rename from archive/utils/stopstart-calcengine.sh rename to archive/utils/stopstartCalcengine.sh index 85d31f29c..366ed507f 100644 --- a/archive/utils/stopstart-calcengine.sh +++ b/archive/utils/stopstartCalcengine.sh @@ -5,4 +5,4 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 -aws ec2 $1-instances --instance-ids $SERVERINSTANCEID --region eu-west-2 --profile ukmonshared +aws ec2 $1-instances --instance-ids $SERVERINSTANCEID --region eu-west-2 diff --git a/archive/utils/userAudit.sh b/archive/utils/userAudit.sh index 205f7e209..98e7123f0 100644 --- a/archive/utils/userAudit.sh +++ b/archive/utils/userAudit.sh @@ -8,22 +8,24 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 conda activate $HOME/miniconda3/envs/${WMPL_ENV} -cd $DATADIR -echo "{\"Subject\": {\"Data\": \"User Audit\",\"Charset\": \"utf-8\"}," > auditmsg.json -echo -n "\"Body\": {\"Text\": {\"Data\": \"" >> auditmsg.json +echo "{\"Subject\": {\"Data\": \"User Audit\",\"Charset\": \"utf-8\"}," > $DATADIR/admin/auditmsg.json +echo -n "\"Body\": {\"Text\": {\"Data\": \"" >> $DATADIR/admin/auditmsg.json # general audit of all users export AWS_PROFILE=ukmonshared -python -m maintenance.getUserAndKeyInfo audit > $DATADIR/useraudit.txt +python -m maintenance.getUserAndKeyInfo audit > $DATADIR/admin/useraudit.txt # accounts not used for 30 days or more export MAXAGE=30 -python -m maintenance.getUserAndKeyInfo dormant >> $DATADIR/useraudit.txt +python -m maintenance.getUserAndKeyInfo dormant >> $DATADIR/admin/useraudit.txt -awk -v ORS='\\n' '1' useraudit.txt >> auditmsg.json -echo "\",\"Charset\": \"utf-8\"}}}" >> auditmsg.json +awk -v ORS='\\n' '1' $DATADIR/admin/useraudit.txt >> $DATADIR/admin/auditmsg.json +echo "\",\"Charset\": \"utf-8\"}}}" >> $DATADIR/admin/auditmsg.json + +pushd $DATADIR/admin aws ses send-email --destination ToAddresses="markmcintyre99@googlemail.com" --message file://auditmsg.json --from "noreply@ukmeteors.co.uk" --region eu-west-2 +popd +rm $DATADIR/admin/auditmsg.json unset MAXAGE unset AWS_PROFILE -rm auditmsg.json diff --git a/archive/website/cameraStatusReport.sh b/archive/website/cameraStatusReport.sh index 537a7b2fe..385fb0353 100644 --- a/archive/website/cameraStatusReport.sh +++ b/archive/website/cameraStatusReport.sh @@ -27,9 +27,28 @@ conda activate $HOME/miniconda3/envs/${WMPL_ENV} export PYTHONPATH=$PYLIB logger -s -t cameraStatusReport "starting" -sudo grep publickey /var/log/secure* | grep -v ec2-user | awk '{printf("%s-%s, %s, %s\n", $1, $2, $3,$9)}' | awk -F ":" '{print $2$3$4}' > $DATADIR/reports/lastlogins.txt +echo TEMPORARY HACK REMOVE CODE PERTAINING TO OLD SERVER +platform=$(grep "^NAME" /etc/os-release | awk -F'"' '{print $2}') +if [ "$platform" == "Ubuntu" ] ; then + authlog=/var/log/auth.log + batchuser=ubuntu +else + authlog=/var/log/secure + batchuser=ec2-user +fi +sudo grep publickey $authlog* | grep -v $batchuser > $DATADIR/reports/lastlogins.txt + +if [ "$platform" == "Ubuntu" ] +then + ssh ukmonhelper2 "sudo grep publickey /var/log/secure* | grep -v ec2-user" > $DATADIR/reports/lastlogins_old.txt +fi +cat $DATADIR/reports/lastlogins_old.txt >> $DATADIR/reports/lastlogins.txt +echo TO HERE python -m metrics.camMetrics $rundate +rm -f $DATADIR/reports/lastlogins.txt +rm -f $DATADIR/reports/lastlogins_old.txt + aws s3 cp $DATADIR/reports/stationlogins.txt $WEBSITEBUCKET/reports/stationlogins.txt --region eu-west-2 --quiet diff --git a/archive/website/costReport.sh b/archive/website/costReport.sh index cd60ec06f..1ad8efa04 100644 --- a/archive/website/costReport.sh +++ b/archive/website/costReport.sh @@ -27,10 +27,8 @@ if [ $# -gt 1 ] ; then numdays = $1 ; fi mkdir -p $DATADIR/costs > /dev/null -export AWS_PROFILE=ukmonshared logger -s -t costReport "getting data for $numdays for $AWS_PROFILE" python -m metrics.costMetrics $DATADIR/costs eu-west-2 $numdays -unset AWS_PROFILE tod=$(date +%d) if [ "$tod" == "03" ] ; then @@ -74,8 +72,8 @@ done echo "var outer_div = document.getElementById(\"docindex\"); outer_div.appendChild(table); })" >> $frjs logger -s -t costReport "publishing data" -aws s3 sync $DATADIR/costs/ $WEBSITEBUCKET/docs/financial/raw/ --exclude "*" --include "costs-18*.csv" --quiet --profile ukmonshared -aws s3 cp $costfile $WEBSITEBUCKET/reports/ --quiet --profile ukmonshared -aws s3 cp $DATADIR/$imgfile3 $WEBSITEBUCKET/reports/ --quiet --profile ukmonshared -aws s3 cp $frjs $WEBSITEBUCKET/docs/ --quiet --profile ukmonshared +aws s3 sync $DATADIR/costs/ $WEBSITEBUCKET/docs/financial/raw/ --exclude "*" --include "costs-18*.csv" --quiet +aws s3 cp $costfile $WEBSITEBUCKET/reports/ --quiet +aws s3 cp $DATADIR/$imgfile3 $WEBSITEBUCKET/reports/ --quiet +aws s3 cp $frjs $WEBSITEBUCKET/docs/ --quiet logger -s -t costReport "done" diff --git a/archive/website/createReportIndex.sh b/archive/website/createReportIndex.sh index edf24c4f8..f758ae1d2 100644 --- a/archive/website/createReportIndex.sh +++ b/archive/website/createReportIndex.sh @@ -60,7 +60,7 @@ if [ -f $curryr/tmp.txt ] ; then rm -f $curryr/tmp.txt ; fi aws s3 ls $WEBSITEBUCKET/reports/$curryr/ | grep PRE | egrep -v "ALL|orbits|stations|fireballs|showers|.js|.html" | awk '{print $2 }' | while read j do - python -c "from meteortools.utils import getShowerDets ; x=getShowerDets('${j:0:3}', True); print(x)" >> $curryr/tmp.txt + python -c "from utils.getActiveShowers import getShowerDets ; x=getShowerDets('${j:0:3}', True); print(x)" >> $curryr/tmp.txt done sort -n $curryr/tmp.txt > $curryr/shwrs.txt rm -f $curryr/tmp.txt diff --git a/archive/website/createShwrExtracts.sh b/archive/website/createShwrExtracts.sh index beb015b48..d90f0463e 100644 --- a/archive/website/createShwrExtracts.sh +++ b/archive/website/createShwrExtracts.sh @@ -44,7 +44,7 @@ cd $DATADIR/browse/showers/ # get a list of files on the website aws s3 ls $WEBSITEBUCKET/browse/showers/ | awk '{ print $4 }' | grep csv > /tmp/browseshwr.txt -shwrs=$(python -c "from meteortools.utils.getActiveShowers import getActiveShowersStr ; getActiveShowersStr('${ymd}')") +shwrs=$(python -c "from utils.getActiveShowers import getActiveShowersStr ; getActiveShowersStr('${ymd}')") for shwr in $shwrs do now=$(date '+%Y-%m-%d %H:%M:%S') diff --git a/archive/website/createSummaryTable.sh b/archive/website/createSummaryTable.sh index fab46348f..caa0fd83b 100644 --- a/archive/website/createSummaryTable.sh +++ b/archive/website/createSummaryTable.sh @@ -9,6 +9,7 @@ # Consumes # the single station and matched station data files in $DATADIR/single and $DATADIR/matched # the most recent match run logfile +# information about the number of active cameras # # Produces # a webpage showing the current summary - the site homepage @@ -32,37 +33,37 @@ python -c "from reports.createSummaryTable import createSummaryTable; createSumm logger -s -t createSummaryTable "create a coverage map from the kmls" # make sure correct version of GEOS and PROJ4 available for mapping routines -LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib -export LD_LIBRARY_PATH +#LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib +#export LD_LIBRARY_PATH mkdir -p $DATADIR/kmls aws s3 sync $WEBSITEBUCKET/img/kmls/ $DATADIR/kmls --quiet export KMLTEMPLATE="*25km.kml" -python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR +python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR/reports/${yr} export KMLTEMPLATE="*70km.kml" -python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR +python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR/reports/${yr} export KMLTEMPLATE="*100km.kml" -python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR +python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR/reports/${yr} -python -c "from reports.makeCoverageMap import createCoveragePage as ccp ; ccp() ;" +python -c "from reports.makeCoverageMap import createCoveragePage as ccp ; ccp('reports/${yr}') ;" logger -s -t createSummaryTable "create year-to-date barchart" -pushd $DATADIR python -c "from reports.createAnnualBarChart import createBarChart; createBarChart('${DATADIR}','${yr}')" -popd # update index page -numcams=$(cat $DATADIR/activecamcount.txt) -cat $TEMPLATES/frontpage.html | sed "s/#NUMCAMS#/$numcams/g" > $DATADIR/newindex.html +numcams=$(cat $DATADIR/consolidated/activecamcount.txt) +cat $TEMPLATES/frontpage.html | sed "s/#NUMCAMS#/$numcams/g" > $DATADIR/reports/${yr}/newindex.html logger -s -t createSummaryTable "copying to website" -aws s3 cp $DATADIR/latest/coverage-maps.html $WEBSITEBUCKET/latest/ --quiet -aws s3 cp $DATADIR/summarytable.js $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/coverage-100km.html $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/coverage-70km.html $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/coverage-70km.html $WEBSITEBUCKET/data/coverage.html --quiet -aws s3 cp $DATADIR/coverage-25km.html $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/Annual-${yr}.jpg $WEBSITEBUCKET/YearToDate.jpg --quiet -aws s3 cp $DATADIR/newindex.html $WEBSITEBUCKET/index.html --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-maps.html $WEBSITEBUCKET/latest/ --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-100km.html $WEBSITEBUCKET/data/ --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-70km.html $WEBSITEBUCKET/data/ --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-70km.html $WEBSITEBUCKET/data/coverage.html --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-25km.html $WEBSITEBUCKET/data/ --quiet + +aws s3 cp $DATADIR/reports/${yr}/Annual-${yr}.jpg $WEBSITEBUCKET/YearToDate.jpg --quiet + +aws s3 cp $DATADIR/reports/${yr}/newindex.html $WEBSITEBUCKET/index.html --quiet +aws s3 cp $DATADIR/reports/${yr}/summarytable.js $WEBSITEBUCKET/data/ --quiet aws s3 cp $SRC/website/templates/header.html $WEBSITEBUCKET/templates/ --quiet aws s3 cp $SRC/website/templates/footer.html $WEBSITEBUCKET/templates/ --quiet diff --git a/archive/website/updateCameraDets.sh b/archive/website/updateCameraDets.sh new file mode 100644 index 000000000..c620bde50 --- /dev/null +++ b/archive/website/updateCameraDets.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright (C) 2018-2023 Mark McIntyre +# create various html and js files of camera info for the website +# +# Parameters +# none +# +# Consumes +# +# +# Produces +# camera locations file cameraLocs.json +# camera details file camera-details.csv +# three html files used by the search page + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +# load the configuration +source $here/../config.ini >/dev/null 2>&1 +conda activate ${WMPL_ENV} + +logger -s -t updateCameraDets "starting" + +python -c "from reports.CameraDetails import updateCamLocDirFovDB; updateCamLocDirFovDB();" +aws s3 cp $DATADIR/admin/cameraLocs.json $UKMONSHAREDBUCKET/admin/ --quiet +aws s3 cp $DATADIR/admin/cameraLocs.json $WEBSITEBUCKET/browse/ --quiet +aws s3 sync $UKMONSHAREDBUCKET/admin/ $DATADIR/admin --quiet + +# create the CSV file of camera info and the html versions for search functions on the website + +python -c "from reports.CameraDetails import createCDCsv; createCDCsv('consolidated','searchidx');" +aws s3 cp $DATADIR/consolidated/camera-details.csv $UKMONSHAREDBUCKET/consolidated/ --quiet +aws s3 cp $DATADIR/searchidx/statopts.html $WEBSITEBUCKET/search/ --quiet +aws s3 cp $DATADIR/searchidx/activestatopts.html $WEBSITEBUCKET/search/ --quiet +aws s3 cp $DATADIR/searchidx/activestatlocs.html $WEBSITEBUCKET/search/ --quiet + +logger -s -t updateCameraDets "finished" diff --git a/bumpver.toml b/bumpver.toml index f8ef5f0f4..17d501fc7 100644 --- a/bumpver.toml +++ b/bumpver.toml @@ -1,5 +1,5 @@ [bumpver] -current_version = "2026.04.1" +current_version = "2026.05.0" version_pattern = "YYYY.0M.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true diff --git a/install_or_update.sh b/install_or_update.sh index 8efa03f02..83713e346 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -10,18 +10,63 @@ envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -mkdir -p ~/${envname} - +echo "Updating codebase..." cd $here/archive +git pull -for loc in analysis ukmon_pylib website cronjobs utils share static_content +echo "Updating code..." +mkdir -p ~/${envname} +[ -d ~/$envname/data ] && msg="upgrade" || msg="install" +for loc in analysis ukmon_pylib website cronjobs utils static_content do - rsync -avz $loc/ ~/${envname}/$loc + rsync -a --delete $loc/ ~/${envname}/$loc chmod +x ~/${envname}/$loc/*.sh > /dev/null 2>&1 done +rsync -a share/ ~/${envname}/share + +echo "Creating data folders..." DATADIR=~/$envname/data -mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls} +mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls,manualuploads} mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajdb,videos} mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs +mkdir -p ~/.logrotate +mkdir -p ~/.aws +echo "Checking conda environment..." +if [[ -f ~/.condaon && -d ~/miniconda3/envs/wmpl ]] +then + pushd $here + source ~/.condaon + conda activate wmpl + pip install -r archive/ukmon_pylib/additional_requirements.txt | grep -v "already satisfied" + echo "" + popd +fi + +echo "Updating meteor shower tables..." +# update the IMO and GMN meteor shower tables if missing +if [ ! -f ~/${envname}/share/IMO_Working_Meteor_Shower_List.xml ] +then + ~/$envname/cronjobs/getImoWSfile.sh + echo "" +fi +echo "" +read -n 1 -p "Update $envname configuration files? (y/N) " yesno +if [[ "$yesno" == "y" || "$yesno" == "Y" ]] +then + echo "" + echo "Updating config for $envname" + ~/$envname/utils/makeConfig.sh $RUNTIME_ENV + for fil in .bashrc .bash_aliases .vimrc .condaon + do + rsync -a server_setup/$fil ~ + done + mkdir -p ~/.logrotate + rsync -a server_setup/logs.conf ~/.logrotate +else + echo "" + echo skipping config and bashrc +fi +echo "" +echo "$msg complete" diff --git a/using_vault.sh b/using_vault.sh deleted file mode 100644 index b87bd80d5..000000000 --- a/using_vault.sh +++ /dev/null @@ -1,49 +0,0 @@ -# testing vault - -# set the default to JSON exports -export VAULT_FORMAT="json" - -# create an approle thats bound to your cidr blocks - vault write auth/approle/role/usermaint \ - secret_id_ttl=10m \ - token_num_uses=10 \ - token_ttl=20m \ - token_max_ttl=30m \ - secret_id_num_uses=40 \ - secret_id_bound_cidrs="192.168.1.0/24" - -# a policy must exist to allow the role to access the secret -echo 'path "secret/test" { capabilities = ["list"]}' > usermaint.hcl -echo 'path "secret/test" { capabilities = ["read"] }' >> usermaint.hcl -vault policy write usermaint usermaint.hcl -vault write auth/approle/role/usermaint policies=usermaint - -# a token must exist for the server to initially authenticate. -# This needs a policy which grants access the approle -# in a corporate environment this step would be done with LDAP or AD authentication - -echo 'path "auth/approle/role" { capabilities = ["list"]}' > server1.hcl -echo 'path "auth/approle/role/usermaint/*" { capabilities = ["read", "update"] }' >> server1.hcl -vault policy write server1 server1.hcl -servertoken=$(vault token create -policy=server1 -format=json | jq -r .auth.client_token) - - -# now on the target server we can use this unprivileged token to retrieve a more privileged one -export VAULT_TOKEN=$servertoken -# get the role ID - this could be statically stored, its not considered Secret -vault_role_id=$(vault read -format=json auth/approle/role/usermaint/role-id | jq -r .data.role_id) - -# then either get a wrapped token and unwrap it to get a secret -# this is more secure because the wrapped token is single-use and expires after 120s -wraptoken=$(vault write -format=json -wrap-ttl=120s -f auth/approle/role/usermaint/secret-id | jq -r .wrap_info.token) -vault_secret_id=$(vault unwrap -format=json $wraptoken | jq -r .data.secret_id) - -# or just straight up get the secret. -vault_secret_id=$(vault write -format=json -force auth/approle/role/usermaint/secret-id | jq -r .data.secret_id) - -# then login to get a token with permission to read the actual secret -vault_token=$(vault write -format=json auth/approle/login role_id=$vault_role_id secret_id=$vault_secret_id | jq -r .auth.client_token) - -# now you can use vault_token in subsequent calls eg -VAULT_TOKEN=$vault_token vault kv get -format=json -mount=secret test - diff --git a/utils/reimageBatchServer.ps1 b/utils/reimageBatchServer.ps1 new file mode 100644 index 000000000..8f513e4df --- /dev/null +++ b/utils/reimageBatchServer.ps1 @@ -0,0 +1,9 @@ +# script to re-image the UKMON calc server +# copyright Mark McIntyre, 2026- + +$dtstr=(get-date -uformat "%Y%m%d") + +$instdetails=(aws ec2 describe-instances --filters Name="tag:Name",Values="batchserver" --profile ukmonshared --region eu-west-2) +$instanceid= (Write-Output $instdetails| convertfrom-json)[0].reservations.instances.instanceid + +aws ec2 create-image --instance-id $instanceid --name "batchserver_${dtstr}" --description "Latest Batchserver image" --profile ukmonshared --region eu-west-2 \ No newline at end of file From ad58f619e6d906e4d5f0f72888b0d3500449bc89 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 5 May 2026 12:29:04 +0100 Subject: [PATCH 15/15] Checking master and my branch are synced (#467) * updates to makeConfig * updates to distributed processing to align with newest wmpl * add a comment to explain what the code does * remove json db * update consolidation to handle sqlite dbs * remove unused function * create data for daily report * remove unused file * update to call new version of reportOfLatestMatches * add test variables * mildly improve doco * put test logs in the right CW log group * handle missing log groups better * extra value in TrajQualityParams * bugfixes revealed in testing * fix issues in reportOfLatestMatches * put container dbs in a sensible place * catch logging errors * tidy up file locations * simplify calcserver start up * sorting out errors in container logging * update container logging info * avoid unnecessary error * fix bugs in log capture process * add MAX_STNS so i can experiement with limits * add logging. Add max_stns to constraints and set to 9999 push logs to s3 copy calcserver logs to ukmonhelper * don't pull or push the databases from S3 during distribution but do back them up after consolidation * catch error if trajpickle can't be found * set rundate more safely move some processes into nightlyJob to make testing easier * tidy up backup file names and housekeep backup folder * move some processes from findAllMatches to nightlyJob to make testing easier * support for testing of createExecReplot * pass testmode to createExecReplot * ensure logs are unique and capture container logs * realigning createFireballPage with prod. Updating gitignore * remove verbosity flag from one tar operation * fixup to handle new style log with datestamps * get privip after restarting calcserver * temporarily don't run rerunFailedLambdas in test * update to work in a test env too * add 'period' to the match summary api * add period to match summary api * re-enabling rerunFailedLambdas * quieten an S3 transfer * include the foldername when reporting on container progress * put daily trajdb in a safe place * be more specific when finding the daily log * only tar the current container logs * save daily obsdb in the right place * rename ec2.tf to calcserver.tf * quieten some s3 transfers * rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is * update SSM params to reflect new serverid * bugfix in distributeCandidates - using wrong path * add newline to getCostMetrics * weird bug in statsToMqtt * bugfix in update_wmpl.sh for traj container * Script to create an AMI for the calcengine * exclude test data when syncing from s3 * writing databases and logs to the wrong target * bad test on istest * typo in serveruserid * adding to server migration notes * purge old rawcsvs and fullcsvs older than 180 days * improved housekeeping of old data * make sure we scoop up data for 31/12 * stop creation of an unused set of files * change default locations for container data * put container logs in the right place * update dockerfile syntax * change locations of test container data and output * remove incorrect comments * Add script to clean up deleted/duplicate trajs * simplify the stats gathering and move to python fix bug in getMatchStats * comment out unused files * update reportOfLatestMatches to exclude deleted trajectories * add scp to requirements * add SQL database maintenance to dataMaintenance * incorrect default for datadir * update call to pymysql.connect * ensure consolidateOutput rerunnable for prior years * upgrading WMPL * update gitignore a bit * bugfix to catch data from 31/12 each year * add paramiko to dependencies for matchPickle to support wmpl upgrade * update getpickle to newest WMPL * bugfix in api tests * bugfix - changes to dailydb not saved * improvements to process to cleanup deleted trajs * after merging, check and remove any duplicate trajectories from the sqlite db and disk * Adding version * bump version 2025.11.1 -> 2026.04.0 * update files containing vesion * bump version 2026.04.0 -> 2026.04.1 * unmangling side effets of automerge * correct usage message * decommission old calcserver and add new batchserver * add calcserver IP to ssm variables and use it in scripts * Update my branch with the latest from master (#436) * Update version (#433) * remove disused code * update livestream to accept normal RMS filenames * search dialog - fix bug and set default date range * small change to explain default dates * improvement in live image search * remove redundant file * update deployment scripts * improvements in routine to convert GMN report to pandas * more analysis of gmn data * Update the terraform a little * remove hardcoded reference to ec2-user * add SERVERUSERID env /ssm variable for calcserver user id * update container for new distrib-processing method * use calcserver SERVERUSERID variable * support for test ECS containers * fix paths in deployment scripts * remove unused file * missed some templates * update dev/test env settings * remove disused file * make sure paths are expanded * fixing various config issues * updates to makeConfig * updates to distributed processing to align with newest wmpl * add a comment to explain what the code does * remove json db * update consolidation to handle sqlite dbs * remove unused function * create data for daily report * remove unused file * update to call new version of reportOfLatestMatches * add test variables * mildly improve doco * put test logs in the right CW log group * handle missing log groups better * extra value in TrajQualityParams * bugfixes revealed in testing * fix issues in reportOfLatestMatches * put container dbs in a sensible place * catch logging errors * tidy up file locations * simplify calcserver start up * sorting out errors in container logging * update container logging info * avoid unnecessary error * fix bugs in log capture process * add MAX_STNS so i can experiement with limits * add logging. Add max_stns to constraints and set to 9999 push logs to s3 copy calcserver logs to ukmonhelper * don't pull or push the databases from S3 during distribution but do back them up after consolidation * catch error if trajpickle can't be found * set rundate more safely move some processes into nightlyJob to make testing easier * tidy up backup file names and housekeep backup folder * move some processes from findAllMatches to nightlyJob to make testing easier * support for testing of createExecReplot * pass testmode to createExecReplot * ensure logs are unique and capture container logs * realigning createFireballPage with prod. Updating gitignore * remove verbosity flag from one tar operation * fixup to handle new style log with datestamps * get privip after restarting calcserver * temporarily don't run rerunFailedLambdas in test * update to work in a test env too * add 'period' to the match summary api * add period to match summary api * re-enabling rerunFailedLambdas * quieten an S3 transfer * include the foldername when reporting on container progress * put daily trajdb in a safe place * be more specific when finding the daily log * only tar the current container logs * save daily obsdb in the right place * rename ec2.tf to calcserver.tf * quieten some s3 transfers * rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is * update SSM params to reflect new serverid * bugfix in distributeCandidates - using wrong path * add newline to getCostMetrics * weird bug in statsToMqtt * bugfix in update_wmpl.sh for traj container * Script to create an AMI for the calcengine * exclude test data when syncing from s3 * writing databases and logs to the wrong target * bad test on istest * typo in serveruserid * adding to server migration notes * purge old rawcsvs and fullcsvs older than 180 days * improved housekeeping of old data * make sure we scoop up data for 31/12 * stop creation of an unused set of files * change default locations for container data * put container logs in the right place * update dockerfile syntax * change locations of test container data and output * remove incorrect comments * Add script to clean up deleted/duplicate trajs * simplify the stats gathering and move to python fix bug in getMatchStats * comment out unused files * update reportOfLatestMatches to exclude deleted trajectories * add scp to requirements * add SQL database maintenance to dataMaintenance * incorrect default for datadir * update call to pymysql.connect * ensure consolidateOutput rerunnable for prior years * upgrading WMPL * update gitignore a bit * bugfix to catch data from 31/12 each year * add paramiko to dependencies for matchPickle to support wmpl upgrade * update getpickle to newest WMPL * bugfix in api tests * bugfix - changes to dailydb not saved * improvements to process to cleanup deleted trajs * after merging, check and remove any duplicate trajectories from the sqlite db and disk * Adding version * bump version 2025.11.1 -> 2026.04.0 * update files containing vesion * bump version 2026.04.0 -> 2026.04.1 * unmangling side effets of automerge * correct usage message * adding install script * changes to ansible scripts * remove irrelevant file * Adding SSM variables for the new batchserver in the ukmda account * attempt to add logging to consolidateDistTraj * remove reference to ee account * temporarily add mariadb deets to ssm in mjmm account * bugfixes in dataMaintenance and cleanupDeletedTrajs to avoid deleting disk files when trajs share same folder * updating migration instructions * more work on deduplication for ukmon * remove username-specific stuff * remove the old deployment scripts * update install script * updates to improve server migration process * add bashrc to the installer * slight tweak to installer * add a few installer messages * remove an echo * be less verbose in the rsyncs * oops need -a for one of the rsyncs * rename requirements file for ukmda * remove dependencies on MeteorTools * add utils to ukmon_pylib * more removal of dependencies on MeteorTools * updating how shower data is obtained * corercting location of IMO and gmn shower data * bugfixes * fixes in imoWorkinShowerlist * small error in install_or_update * bugfix when no data found in imo set * retrieve IMO data after each code update * fixing up getShowerDets * improve installer * fix createShowerExtracts * update extractors and showeranalysis * bug in extractor thats existed since 2023 but nobody noticed... * trying to fix matplotlib bug * more working on matplotlib issue * oops overfixed a bug * another incorrect bugfix! * make bars wider * more removal of MeteorTools * bugfix in createannualbarchart * working on bug * more work on lenght bug * remove some debugging * oops forgot to push the fixed version of createReportIndex * segregating function to get list of images that weren't calibrated by RMS * reorganize stuff a bit * add email sending functionality to ukmon_pylib * use send email functionality in code * add database and gmaps keys/passwords as secure variables * update to use secure keys * remove dbpw from environment * remove dbpw from environment * add script to retrieve db passwd * add script to retrieve db passwords * bug in createTables * updates to migration instructions * include pip installs in the script * tweak installer * remove glob.glob1 * move unused scripts - will delete later * fix mariadb connections * typo * tidy up docs * fixing mysql calls * small changes to documentation * suppress pip messages if already installed * tweaks to the installer * add logrotate stuff * bugfixes in installer * tidy up installer * bugfix in key mgmt * move some functionality into new script to tidy up nightly job and simplify rerunning * improve install script * update location of camera count file * move annual bar charts into a more logical location * update test script * add gmaps param to my acct * relocate coverage maps and front page index * tidy up useraudit process * bugfixes in testing * bugfix in createAnnualBarchart * bug in testscript * another small bug in createSummaryTable * still fighting that bug * tidy up lastlog data * remove unused file * update cam status reporting * clear down the old brightness data * simplification of logging and of the nightlyJob script * bugfixes in process to gather consumed image ifo * update getUncal to work for historic data * typo * improvements in getUncalImages * bugfix * add python to get unused images * move script to right place * comment out unnecessary echo * bugfixes in camMetrics while migrating user accts * fix #448 * make sure window title is sensible * bugfix in replot and get unused * bugfix in removeDeletedTraj for search index * remove deprecation warning issue #450 * whoops naive vs tzaware dates * remove warning in cameraStatusReport issue #453 * remove warning issue #454 * widen constraints on analysing pickles to get images * add migration script * add sudo * actually do it * update sftp migration script * more changes to migration script * moving sftp migration script * update messages and docos * slight tweak to migration script * further work on sftp account migration * update ukmon ini files after migration * typo * collect old server login records integration with new * update migration instructions * update cam metrics report to include hostname * bugfix as $USER not available in cron * removing unused files * some tidying up * motre reorg * remove disused template rename file to follow convention * simplifying logging and removing unused aws profile * removing profile from python * adding executable bit * typo * bugfix to work around typo * print message when done * bugfix to get trajectories from s3 before trying to check for missing trajectories... * improve to support running for past dates * errors in bashrc and logrotate config * prevent email being sent for anything other than current days report * bugfix: make sure we use the most recent stats * improvement in reportOfLatestMatches to handle past dates * bugfix * small tweak to support reportOfLatestMatches * tweak getMatchStats to handle historic data better * add mariadb and ping to the sec grp so that the database can be accessed by apis * update the db servername * bugfix in match stats * prevent bogus renaming of matchJob.log * bumping version * bump version 2026.04.1 -> 2026.05.0