diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 77e99e8a1..bc121630d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,11 +39,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -54,7 +54,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v4 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v4 diff --git a/README.md b/README.md index 9f059c11c..2e9d9281e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # UK Meteor Data Analysis Shared code and libraries -version: 2026.05.0 +version: 2026.05.2 This repository contains the code behind the UK Meteors data archive and data processing pipeline. diff --git a/archive/README.md b/archive/README.md index c37b478af..e8cdc90fb 100644 --- a/archive/README.md +++ b/archive/README.md @@ -1,7 +1,7 @@ Data Processing and Flows ========================== -version: 2026.05.0 +version: 2026.05.2 This diagram shows the overall flow of data from Cameras to websites and out to the public. diff --git a/archive/server_setup/acctStatusChecks.sh b/archive/server_setup/acctStatusChecks.sh new file mode 100644 index 000000000..6c3463872 --- /dev/null +++ b/archive/server_setup/acctStatusChecks.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +cd $here + +sudo ls -1 /var/sftp > done.txt +grep ukmonh /home/ubuntu/prod/data/reports/stationlogins.txt |awk '{print $3}'| sort > pending.txt + +ssh ukmonhelper2 "~/prod/server_setup/get-nbd.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > not-being-done.txt +ssh ukmonhelper2 "~/prod/server_setup/get-all.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > all-accounts.txt + +python ~/src/ukmda-dataprocessing/archive/server_setup/checkSftpAccounts.py diff --git a/archive/server_setup/backupSftpAccts.sh b/archive/server_setup/backupSftpAccts.sh new file mode 100644 index 000000000..df09ff641 --- /dev/null +++ b/archive/server_setup/backupSftpAccts.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +cd $here + +bkpOneUser() { + userid=$1 + srchost=$2 + mkdir -p ./backup/$userid + sudo rsync -av $srchost:/var/sftp/$userid/ ./backup/$userid + sudo chown -R $USER:$USER ./backup/$userid + cat ./backup/$userid/ukmon.ini | sed 's/3.11.55.160/batchserver.ukmeteors.co.uk/g' > /tmp/$userid.ini + mv -f /tmp/$userid.ini ./backup/$userid/ukmon.ini +} + +if [ $# -lt 2 ] ; then + echo "Usage: ./backupSftpAccounts.sh oldservername userfile" + exit +fi + +echo "Warning: this must only be run on the new server" +read -p "press ctrl-c to quit or enter to continue" + +oldserver=$1 +srcfile=$2 + +cat $srcfile | while read stn +do + bkpOneUser $stn $oldserver +done \ No newline at end of file diff --git a/archive/server_setup/checkSftpAccounts.py b/archive/server_setup/checkSftpAccounts.py new file mode 100644 index 000000000..8149c0185 --- /dev/null +++ b/archive/server_setup/checkSftpAccounts.py @@ -0,0 +1,65 @@ +import pandas as pd +import datetime + + +data=open('/home/ubuntu/prod/data/reports/stationlogins.txt').readlines() +camlist = [] +not_live = [] +still_upl = [] +not_upl = [] +livenames = [] + +livedate = datetime.datetime.now() - datetime.timedelta(days=10) +for li in data: + if 'Last Upload' in li: + continue + lastup = li[:19] + lastlo = li[21:40] + loc = li[42:61] + gmnid = li[64:73] + via = li[76:].strip() + camlist.append([lastup, lastlo, loc, gmnid, via]) + + if ">" in lastup and ">" in lastlo: + not_live.append([loc, gmnid, lastup, lastlo, via]) + + elif ">" not in lastup: + lastupdt = datetime.datetime.strptime(lastup, '%Y-%m-%dT%H:%M:%S') + if lastupdt >= livedate: + still_upl.append([loc, gmnid, lastup, lastlo, via]) + livenames.append(loc.strip()) + else: + if '>' in lastlo: + not_live.append([loc, gmnid, lastup, lastlo, via]) + else: + lastlodt = datetime.datetime.strptime(lastlo, '%Y-%m-%dT%H:%M:%S') + if lastlodt >= livedate: + not_upl.append([loc, gmnid, lastup, lastlo, via]) + else: + not_live.append([loc, gmnid, lastup, lastlo, via]) + else: # '>' not in lastlo + lastlodt = datetime.datetime.strptime(lastlo, '%Y-%m-%dT%H:%M:%S') + if lastlodt >= livedate: + not_upl.append([loc, gmnid, lastup, lastlo, via]) + else: + not_live.append([loc, gmnid, lastup, lastlo, via]) + +with open('still-live.txt','w') as outf: + for cam in still_upl: + outf.write(','.join(cam) + '\n') + +with open('not_uploading.txt','w') as outf: + for cam in not_upl: + outf.write(','.join(cam) + '\n') + +with open('not_live.txt','w') as outf: + for cam in not_live: + outf.write(','.join(cam) + '\n') + +donelist = open('done.txt', 'r').readlines() +donelist = [x.strip() for x in donelist] +with open('todo.txt', 'w') as outf: + for nam in livenames: + if nam.strip() not in donelist: + print('done list is missing', nam) + outf.write(f'{nam}\n') \ No newline at end of file diff --git a/archive/server_setup/copyTestData.sh b/archive/server_setup/copyTestData.sh deleted file mode 100644 index e2c041c5a..000000000 --- a/archive/server_setup/copyTestData.sh +++ /dev/null @@ -1,14 +0,0 @@ -# prep some test data -$yr=$(date +%Y) - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -source $here/../config.ini >/dev/null 2>&1 - -rsync -avz ~/prod/data/single/ $DATADIR/single/ -rsync -avz ~/prod/data/matched/*${yr}* $DATADIR/matched/ -rsync -avz ~/prod/data/consolidated/*${yr}* $DATADIR/consolidated/ -rsync -avz ~/prod/data/dailyreports/stats.txt $DATADIR/dailyreports/ -rsync -avz ~/prod/data/dailyreports/${yr}*.txt $DATADIR/dailyreports/ -rsync -avz ~/prod/logs/*${yr}*.txt $SRC/logs/ -rsync -avz ~/prod/data/*${yr}*.jpg $DATADIR/ -rsync -avz ~/prod/data/*.png $DATADIR/ \ No newline at end of file diff --git a/archive/server_setup/migrateSftpAccts.sh b/archive/server_setup/migrateSftpAccts.sh index 52d5add0f..823444955 100755 --- a/archive/server_setup/migrateSftpAccts.sh +++ b/archive/server_setup/migrateSftpAccts.sh @@ -1,5 +1,7 @@ #!/bin/bash +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +cd $here addOneUser() { userid=$1 diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index 36aeaf985..6824db9f9 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -206,15 +206,15 @@ def backPopulate(stationid): df['uploadtime']=df.uploadtime.astype("str").str.pad(6,fillchar="0") df['lastupload']=df.upddate.astype('str') + '_' +df.uploadtime df.lastupload = [datetime.datetime.strptime(x, '%Y%m%d_%H%M%S') for x in df.lastupload] - df = df.drop(columns=['stationid','manual','rundate', 'upddate','uploadtime']) + df = df.drop(columns=['manual','rundate', 'upddate','uploadtime']) df = df.sort_values(by=['lastupload']) outfile=os.path.join(datadir, 'reports', 'stationlogins.txt') zerodate = datetime.datetime(1970,1,1,0,0,0) with open(outfile,'w') as outf: - outf.write('Last Upload, StationID, Last Login, Via\n') + outf.write(f'{"Last Upload":19s} {"Last Connect":19s} {"StationID":20s} {"GMN ID":10s} Via\n') for _,rw in df.iterrows(): - lastup = '> 1 month' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S') - lastlo = '> 1 month' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S') + lastup = '> 1 month ago' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S') + lastlo = '> 1 month ago' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S') via = '' if pd.isnull(rw.host) else rw.host - outf.write(f'{lastup} , {rw.location:20s}, {lastlo}, {via}\n') + outf.write(f'{lastup:19s} {lastlo:19s} {rw.location:20s} {rw.stationid:10s} {via}\n') diff --git a/bumpver.toml b/bumpver.toml index 17d501fc7..cc67f3028 100644 --- a/bumpver.toml +++ b/bumpver.toml @@ -1,5 +1,5 @@ [bumpver] -current_version = "2026.05.0" +current_version = "2026.05.2" version_pattern = "YYYY.0M.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true diff --git a/install_or_update.sh b/install_or_update.sh index 83713e346..c903adce7 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -10,8 +10,8 @@ envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +pushd $here/archive echo "Updating codebase..." -cd $here/archive git pull echo "Updating code..." @@ -32,6 +32,11 @@ mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs mkdir -p ~/.logrotate mkdir -p ~/.aws +mkdir -p ~/server_setup + +rsync -a server_setup/*.sh ~/server_setup +rsync -a server_setup/*.py ~/server_setup +chmod +x ~/server_setup/*.sh echo "Checking conda environment..." if [[ -f ~/.condaon && -d ~/miniconda3/envs/wmpl ]] @@ -69,4 +74,5 @@ else echo skipping config and bashrc fi echo "" +popd echo "$msg complete"