From 7ccf1d9a68d0a4c8672ca2e6a08c3de2e6441eb6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 5 May 2026 22:28:40 +0100 Subject: [PATCH 1/4] Minor tweaks (#469) * add a comment to explain what the code does * remove json db * update consolidation to handle sqlite dbs * remove unused function * create data for daily report * remove unused file * update to call new version of reportOfLatestMatches * add test variables * mildly improve doco * put test logs in the right CW log group * handle missing log groups better * extra value in TrajQualityParams * bugfixes revealed in testing * fix issues in reportOfLatestMatches * put container dbs in a sensible place * catch logging errors * tidy up file locations * simplify calcserver start up * sorting out errors in container logging * update container logging info * avoid unnecessary error * fix bugs in log capture process * add MAX_STNS so i can experiement with limits * add logging. Add max_stns to constraints and set to 9999 push logs to s3 copy calcserver logs to ukmonhelper * don't pull or push the databases from S3 during distribution but do back them up after consolidation * catch error if trajpickle can't be found * set rundate more safely move some processes into nightlyJob to make testing easier * tidy up backup file names and housekeep backup folder * move some processes from findAllMatches to nightlyJob to make testing easier * support for testing of createExecReplot * pass testmode to createExecReplot * ensure logs are unique and capture container logs * realigning createFireballPage with prod. Updating gitignore * remove verbosity flag from one tar operation * fixup to handle new style log with datestamps * get privip after restarting calcserver * temporarily don't run rerunFailedLambdas in test * update to work in a test env too * add 'period' to the match summary api * add period to match summary api * re-enabling rerunFailedLambdas * quieten an S3 transfer * include the foldername when reporting on container progress * put daily trajdb in a safe place * be more specific when finding the daily log * only tar the current container logs * save daily obsdb in the right place * rename ec2.tf to calcserver.tf * quieten some s3 transfers * rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is * update SSM params to reflect new serverid * bugfix in distributeCandidates - using wrong path * add newline to getCostMetrics * weird bug in statsToMqtt * bugfix in update_wmpl.sh for traj container * Script to create an AMI for the calcengine * exclude test data when syncing from s3 * writing databases and logs to the wrong target * bad test on istest * typo in serveruserid * adding to server migration notes * purge old rawcsvs and fullcsvs older than 180 days * improved housekeeping of old data * make sure we scoop up data for 31/12 * stop creation of an unused set of files * change default locations for container data * put container logs in the right place * update dockerfile syntax * change locations of test container data and output * remove incorrect comments * Add script to clean up deleted/duplicate trajs * simplify the stats gathering and move to python fix bug in getMatchStats * comment out unused files * update reportOfLatestMatches to exclude deleted trajectories * add scp to requirements * add SQL database maintenance to dataMaintenance * incorrect default for datadir * update call to pymysql.connect * ensure consolidateOutput rerunnable for prior years * upgrading WMPL * update gitignore a bit * bugfix to catch data from 31/12 each year * add paramiko to dependencies for matchPickle to support wmpl upgrade * update getpickle to newest WMPL * bugfix in api tests * bugfix - changes to dailydb not saved * improvements to process to cleanup deleted trajs * after merging, check and remove any duplicate trajectories from the sqlite db and disk * Adding version * bump version 2025.11.1 -> 2026.04.0 * update files containing vesion * bump version 2026.04.0 -> 2026.04.1 * unmangling side effets of automerge * correct usage message * decommission old calcserver and add new batchserver * add calcserver IP to ssm variables and use it in scripts * Update my branch with the latest from master (#436) * Update version (#433) * remove disused code * update livestream to accept normal RMS filenames * search dialog - fix bug and set default date range * small change to explain default dates * improvement in live image search * remove redundant file * update deployment scripts * improvements in routine to convert GMN report to pandas * more analysis of gmn data * Update the terraform a little * remove hardcoded reference to ec2-user * add SERVERUSERID env /ssm variable for calcserver user id * update container for new distrib-processing method * use calcserver SERVERUSERID variable * support for test ECS containers * fix paths in deployment scripts * remove unused file * missed some templates * update dev/test env settings * remove disused file * make sure paths are expanded * fixing various config issues * updates to makeConfig * updates to distributed processing to align with newest wmpl * add a comment to explain what the code does * remove json db * update consolidation to handle sqlite dbs * remove unused function * create data for daily report * remove unused file * update to call new version of reportOfLatestMatches * add test variables * mildly improve doco * put test logs in the right CW log group * handle missing log groups better * extra value in TrajQualityParams * bugfixes revealed in testing * fix issues in reportOfLatestMatches * put container dbs in a sensible place * catch logging errors * tidy up file locations * simplify calcserver start up * sorting out errors in container logging * update container logging info * avoid unnecessary error * fix bugs in log capture process * add MAX_STNS so i can experiement with limits * add logging. Add max_stns to constraints and set to 9999 push logs to s3 copy calcserver logs to ukmonhelper * don't pull or push the databases from S3 during distribution but do back them up after consolidation * catch error if trajpickle can't be found * set rundate more safely move some processes into nightlyJob to make testing easier * tidy up backup file names and housekeep backup folder * move some processes from findAllMatches to nightlyJob to make testing easier * support for testing of createExecReplot * pass testmode to createExecReplot * ensure logs are unique and capture container logs * realigning createFireballPage with prod. Updating gitignore * remove verbosity flag from one tar operation * fixup to handle new style log with datestamps * get privip after restarting calcserver * temporarily don't run rerunFailedLambdas in test * update to work in a test env too * add 'period' to the match summary api * add period to match summary api * re-enabling rerunFailedLambdas * quieten an S3 transfer * include the foldername when reporting on container progress * put daily trajdb in a safe place * be more specific when finding the daily log * only tar the current container logs * save daily obsdb in the right place * rename ec2.tf to calcserver.tf * quieten some s3 transfers * rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is * update SSM params to reflect new serverid * bugfix in distributeCandidates - using wrong path * add newline to getCostMetrics * weird bug in statsToMqtt * bugfix in update_wmpl.sh for traj container * Script to create an AMI for the calcengine * exclude test data when syncing from s3 * writing databases and logs to the wrong target * bad test on istest * typo in serveruserid * adding to server migration notes * purge old rawcsvs and fullcsvs older than 180 days * improved housekeeping of old data * make sure we scoop up data for 31/12 * stop creation of an unused set of files * change default locations for container data * put container logs in the right place * update dockerfile syntax * change locations of test container data and output * remove incorrect comments * Add script to clean up deleted/duplicate trajs * simplify the stats gathering and move to python fix bug in getMatchStats * comment out unused files * update reportOfLatestMatches to exclude deleted trajectories * add scp to requirements * add SQL database maintenance to dataMaintenance * incorrect default for datadir * update call to pymysql.connect * ensure consolidateOutput rerunnable for prior years * upgrading WMPL * update gitignore a bit * bugfix to catch data from 31/12 each year * add paramiko to dependencies for matchPickle to support wmpl upgrade * update getpickle to newest WMPL * bugfix in api tests * bugfix - changes to dailydb not saved * improvements to process to cleanup deleted trajs * after merging, check and remove any duplicate trajectories from the sqlite db and disk * Adding version * bump version 2025.11.1 -> 2026.04.0 * update files containing vesion * bump version 2026.04.0 -> 2026.04.1 * unmangling side effets of automerge * correct usage message * adding install script * changes to ansible scripts * remove irrelevant file * Adding SSM variables for the new batchserver in the ukmda account * attempt to add logging to consolidateDistTraj * remove reference to ee account * temporarily add mariadb deets to ssm in mjmm account * bugfixes in dataMaintenance and cleanupDeletedTrajs to avoid deleting disk files when trajs share same folder * updating migration instructions * more work on deduplication for ukmon * remove username-specific stuff * remove the old deployment scripts * update install script * updates to improve server migration process * add bashrc to the installer * slight tweak to installer * add a few installer messages * remove an echo * be less verbose in the rsyncs * oops need -a for one of the rsyncs * rename requirements file for ukmda * remove dependencies on MeteorTools * add utils to ukmon_pylib * more removal of dependencies on MeteorTools * updating how shower data is obtained * corercting location of IMO and gmn shower data * bugfixes * fixes in imoWorkinShowerlist * small error in install_or_update * bugfix when no data found in imo set * retrieve IMO data after each code update * fixing up getShowerDets * improve installer * fix createShowerExtracts * update extractors and showeranalysis * bug in extractor thats existed since 2023 but nobody noticed... * trying to fix matplotlib bug * more working on matplotlib issue * oops overfixed a bug * another incorrect bugfix! * make bars wider * more removal of MeteorTools * bugfix in createannualbarchart * working on bug * more work on lenght bug * remove some debugging * oops forgot to push the fixed version of createReportIndex * segregating function to get list of images that weren't calibrated by RMS * reorganize stuff a bit * add email sending functionality to ukmon_pylib * use send email functionality in code * add database and gmaps keys/passwords as secure variables * update to use secure keys * remove dbpw from environment * remove dbpw from environment * add script to retrieve db passwd * add script to retrieve db passwords * bug in createTables * updates to migration instructions * include pip installs in the script * tweak installer * remove glob.glob1 * move unused scripts - will delete later * fix mariadb connections * typo * tidy up docs * fixing mysql calls * small changes to documentation * suppress pip messages if already installed * tweaks to the installer * add logrotate stuff * bugfixes in installer * tidy up installer * bugfix in key mgmt * move some functionality into new script to tidy up nightly job and simplify rerunning * improve install script * update location of camera count file * move annual bar charts into a more logical location * update test script * add gmaps param to my acct * relocate coverage maps and front page index * tidy up useraudit process * bugfixes in testing * bugfix in createAnnualBarchart * bug in testscript * another small bug in createSummaryTable * still fighting that bug * tidy up lastlog data * remove unused file * update cam status reporting * clear down the old brightness data * simplification of logging and of the nightlyJob script * bugfixes in process to gather consumed image ifo * update getUncal to work for historic data * typo * improvements in getUncalImages * bugfix * add python to get unused images * move script to right place * comment out unnecessary echo * bugfixes in camMetrics while migrating user accts * fix #448 * make sure window title is sensible * bugfix in replot and get unused * bugfix in removeDeletedTraj for search index * remove deprecation warning issue #450 * whoops naive vs tzaware dates * remove warning in cameraStatusReport issue #453 * remove warning issue #454 * widen constraints on analysing pickles to get images * add migration script * add sudo * actually do it * update sftp migration script * more changes to migration script * moving sftp migration script * update messages and docos * slight tweak to migration script * further work on sftp account migration * update ukmon ini files after migration * typo * collect old server login records integration with new * update migration instructions * update cam metrics report to include hostname * bugfix as $USER not available in cron * removing unused files * some tidying up * motre reorg * remove disused template rename file to follow convention * simplifying logging and removing unused aws profile * removing profile from python * adding executable bit * typo * bugfix to work around typo * print message when done * bugfix to get trajectories from s3 before trying to check for missing trajectories... * improve to support running for past dates * errors in bashrc and logrotate config * prevent email being sent for anything other than current days report * bugfix: make sure we use the most recent stats * improvement in reportOfLatestMatches to handle past dates * bugfix * small tweak to support reportOfLatestMatches * tweak getMatchStats to handle historic data better * add mariadb and ping to the sec grp so that the database can be accessed by apis * update the db servername * bugfix in match stats * prevent bogus renaming of matchJob.log * bumping version * bump version 2026.04.1 -> 2026.05.0 * update matchapi to use indexed column * add api readme --- api_examples/README.md | 6 ++++++ archive/lambdas/matchDataApi/matchDataApi.py | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 api_examples/README.md diff --git a/api_examples/README.md b/api_examples/README.md new file mode 100644 index 000000000..4de90ef88 --- /dev/null +++ b/api_examples/README.md @@ -0,0 +1,6 @@ +#Available APIs and Examples + +We have a number of APIs which can be used to retrieve our data programatically, either using commandline tools like curl and wget, or from programming languages such as Python. + +Data are returned in JSON format which can readily be converted into a python list, array or Pandas DataFrame. Simple examples are shown on our main website (here)[https://ukmeteornetwork.org/our-data-apis/], and a set of Python examples are available here. A Data Dictionary in Excel format can be downloaded from (here)[https://archive.ukmeteors.co.uk/browse/datadictionary.xlsx]. + diff --git a/archive/lambdas/matchDataApi/matchDataApi.py b/archive/lambdas/matchDataApi/matchDataApi.py index 73c393db9..3f4d1f178 100644 --- a/archive/lambdas/matchDataApi/matchDataApi.py +++ b/archive/lambdas/matchDataApi/matchDataApi.py @@ -67,7 +67,7 @@ def getStationData(statid, dtstr, period=None): statfrag = f"and s.stations like '%{statid}%' " if statid is not None else "" perfrag = periodToSqlFragment(period) if period is not None else "" with connection.cursor() as cursor: - sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' {statfrag} {perfrag}" + sql = f"SELECT s.orbname from matches s where s.orbname like '{dtstr}%' {statfrag} {perfrag}" cursor.execute(sql) result = cursor.fetchall() finally: @@ -85,7 +85,7 @@ def getSummaryData(dtstr, period=None): fieldlist = '_localtime,_mjd,_sol,_ID1,_amag,_ra_o,_dc_o,_ra_t,_dc_t,_elng,_elat,_vo,_vi,_vg,_vs,_a,_q,_e,_p,_peri,_node,_incl,'\ '_stream,_mag,_dur,_lng1,_lat1,_H1,_lng2,_lat2,_H2,_LD21,_az1r,_ev1r,_Nts,_Nos,_leap,_tme,_dt,'\ 'dtstamp,orbname,iau,shwrname as name,mass,pi,Q,true_anom,EA,MA,Tj,T,last_peri,jacchia1,Jacchia2,numstats,stations' - expr = f"SELECT {fieldlist} from matches s where s._localtime like '_{dtstr}%' {perfrag}" + expr = f"SELECT {fieldlist} from matches s where s.orbname like '{dtstr}%' {perfrag}" result=[] try: with connection.cursor() as cursor: @@ -171,7 +171,7 @@ def lambda_handler(event, context): res = '{"points": "unavailable"}' else: - res = '{"invalid request type - must be one of \'matches\', \'details\', \'station\',\'summary\'"}' + res = '{"invalid request type - must be one of \'matches\', \'detail\', \'station\',\'summary\'"}' print(res) return { From d13be369c540ff4ea226510e8e4563eb41d4ee93 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 7 May 2026 01:04:24 +0100 Subject: [PATCH 2/4] Latest updates (#470) * simplify calcserver start up * sorting out errors in container logging * update container logging info * avoid unnecessary error * fix bugs in log capture process * add MAX_STNS so i can experiement with limits * add logging. Add max_stns to constraints and set to 9999 push logs to s3 copy calcserver logs to ukmonhelper * don't pull or push the databases from S3 during distribution but do back them up after consolidation * catch error if trajpickle can't be found * set rundate more safely move some processes into nightlyJob to make testing easier * tidy up backup file names and housekeep backup folder * move some processes from findAllMatches to nightlyJob to make testing easier * support for testing of createExecReplot * pass testmode to createExecReplot * ensure logs are unique and capture container logs * realigning createFireballPage with prod. Updating gitignore * remove verbosity flag from one tar operation * fixup to handle new style log with datestamps * get privip after restarting calcserver * temporarily don't run rerunFailedLambdas in test * update to work in a test env too * add 'period' to the match summary api * add period to match summary api * re-enabling rerunFailedLambdas * quieten an S3 transfer * include the foldername when reporting on container progress * put daily trajdb in a safe place * be more specific when finding the daily log * only tar the current container logs * save daily obsdb in the right place * rename ec2.tf to calcserver.tf * quieten some s3 transfers * rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is * update SSM params to reflect new serverid * bugfix in distributeCandidates - using wrong path * add newline to getCostMetrics * weird bug in statsToMqtt * bugfix in update_wmpl.sh for traj container * Script to create an AMI for the calcengine * exclude test data when syncing from s3 * writing databases and logs to the wrong target * bad test on istest * typo in serveruserid * adding to server migration notes * purge old rawcsvs and fullcsvs older than 180 days * improved housekeeping of old data * make sure we scoop up data for 31/12 * stop creation of an unused set of files * change default locations for container data * put container logs in the right place * update dockerfile syntax * change locations of test container data and output * remove incorrect comments * Add script to clean up deleted/duplicate trajs * simplify the stats gathering and move to python fix bug in getMatchStats * comment out unused files * update reportOfLatestMatches to exclude deleted trajectories * add scp to requirements * add SQL database maintenance to dataMaintenance * incorrect default for datadir * update call to pymysql.connect * ensure consolidateOutput rerunnable for prior years * upgrading WMPL * update gitignore a bit * bugfix to catch data from 31/12 each year * add paramiko to dependencies for matchPickle to support wmpl upgrade * update getpickle to newest WMPL * bugfix in api tests * bugfix - changes to dailydb not saved * improvements to process to cleanup deleted trajs * after merging, check and remove any duplicate trajectories from the sqlite db and disk * Adding version * bump version 2025.11.1 -> 2026.04.0 * update files containing vesion * bump version 2026.04.0 -> 2026.04.1 * unmangling side effets of automerge * correct usage message * decommission old calcserver and add new batchserver * add calcserver IP to ssm variables and use it in scripts * Update my branch with the latest from master (#436) * Update version (#433) * remove disused code * update livestream to accept normal RMS filenames * search dialog - fix bug and set default date range * small change to explain default dates * improvement in live image search * remove redundant file * update deployment scripts * improvements in routine to convert GMN report to pandas * more analysis of gmn data * Update the terraform a little * remove hardcoded reference to ec2-user * add SERVERUSERID env /ssm variable for calcserver user id * update container for new distrib-processing method * use calcserver SERVERUSERID variable * support for test ECS containers * fix paths in deployment scripts * remove unused file * missed some templates * update dev/test env settings * remove disused file * make sure paths are expanded * fixing various config issues * updates to makeConfig * updates to distributed processing to align with newest wmpl * add a comment to explain what the code does * remove json db * update consolidation to handle sqlite dbs * remove unused function * create data for daily report * remove unused file * update to call new version of reportOfLatestMatches * add test variables * mildly improve doco * put test logs in the right CW log group * handle missing log groups better * extra value in TrajQualityParams * bugfixes revealed in testing * fix issues in reportOfLatestMatches * put container dbs in a sensible place * catch logging errors * tidy up file locations * simplify calcserver start up * sorting out errors in container logging * update container logging info * avoid unnecessary error * fix bugs in log capture process * add MAX_STNS so i can experiement with limits * add logging. Add max_stns to constraints and set to 9999 push logs to s3 copy calcserver logs to ukmonhelper * don't pull or push the databases from S3 during distribution but do back them up after consolidation * catch error if trajpickle can't be found * set rundate more safely move some processes into nightlyJob to make testing easier * tidy up backup file names and housekeep backup folder * move some processes from findAllMatches to nightlyJob to make testing easier * support for testing of createExecReplot * pass testmode to createExecReplot * ensure logs are unique and capture container logs * realigning createFireballPage with prod. Updating gitignore * remove verbosity flag from one tar operation * fixup to handle new style log with datestamps * get privip after restarting calcserver * temporarily don't run rerunFailedLambdas in test * update to work in a test env too * add 'period' to the match summary api * add period to match summary api * re-enabling rerunFailedLambdas * quieten an S3 transfer * include the foldername when reporting on container progress * put daily trajdb in a safe place * be more specific when finding the daily log * only tar the current container logs * save daily obsdb in the right place * rename ec2.tf to calcserver.tf * quieten some s3 transfers * rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is * update SSM params to reflect new serverid * bugfix in distributeCandidates - using wrong path * add newline to getCostMetrics * weird bug in statsToMqtt * bugfix in update_wmpl.sh for traj container * Script to create an AMI for the calcengine * exclude test data when syncing from s3 * writing databases and logs to the wrong target * bad test on istest * typo in serveruserid * adding to server migration notes * purge old rawcsvs and fullcsvs older than 180 days * improved housekeeping of old data * make sure we scoop up data for 31/12 * stop creation of an unused set of files * change default locations for container data * put container logs in the right place * update dockerfile syntax * change locations of test container data and output * remove incorrect comments * Add script to clean up deleted/duplicate trajs * simplify the stats gathering and move to python fix bug in getMatchStats * comment out unused files * update reportOfLatestMatches to exclude deleted trajectories * add scp to requirements * add SQL database maintenance to dataMaintenance * incorrect default for datadir * update call to pymysql.connect * ensure consolidateOutput rerunnable for prior years * upgrading WMPL * update gitignore a bit * bugfix to catch data from 31/12 each year * add paramiko to dependencies for matchPickle to support wmpl upgrade * update getpickle to newest WMPL * bugfix in api tests * bugfix - changes to dailydb not saved * improvements to process to cleanup deleted trajs * after merging, check and remove any duplicate trajectories from the sqlite db and disk * Adding version * bump version 2025.11.1 -> 2026.04.0 * update files containing vesion * bump version 2026.04.0 -> 2026.04.1 * unmangling side effets of automerge * correct usage message * adding install script * changes to ansible scripts * remove irrelevant file * Adding SSM variables for the new batchserver in the ukmda account * attempt to add logging to consolidateDistTraj * remove reference to ee account * temporarily add mariadb deets to ssm in mjmm account * bugfixes in dataMaintenance and cleanupDeletedTrajs to avoid deleting disk files when trajs share same folder * updating migration instructions * more work on deduplication for ukmon * remove username-specific stuff * remove the old deployment scripts * update install script * updates to improve server migration process * add bashrc to the installer * slight tweak to installer * add a few installer messages * remove an echo * be less verbose in the rsyncs * oops need -a for one of the rsyncs * rename requirements file for ukmda * remove dependencies on MeteorTools * add utils to ukmon_pylib * more removal of dependencies on MeteorTools * updating how shower data is obtained * corercting location of IMO and gmn shower data * bugfixes * fixes in imoWorkinShowerlist * small error in install_or_update * bugfix when no data found in imo set * retrieve IMO data after each code update * fixing up getShowerDets * improve installer * fix createShowerExtracts * update extractors and showeranalysis * bug in extractor thats existed since 2023 but nobody noticed... * trying to fix matplotlib bug * more working on matplotlib issue * oops overfixed a bug * another incorrect bugfix! * make bars wider * more removal of MeteorTools * bugfix in createannualbarchart * working on bug * more work on lenght bug * remove some debugging * oops forgot to push the fixed version of createReportIndex * segregating function to get list of images that weren't calibrated by RMS * reorganize stuff a bit * add email sending functionality to ukmon_pylib * use send email functionality in code * add database and gmaps keys/passwords as secure variables * update to use secure keys * remove dbpw from environment * remove dbpw from environment * add script to retrieve db passwd * add script to retrieve db passwords * bug in createTables * updates to migration instructions * include pip installs in the script * tweak installer * remove glob.glob1 * move unused scripts - will delete later * fix mariadb connections * typo * tidy up docs * fixing mysql calls * small changes to documentation * suppress pip messages if already installed * tweaks to the installer * add logrotate stuff * bugfixes in installer * tidy up installer * bugfix in key mgmt * move some functionality into new script to tidy up nightly job and simplify rerunning * improve install script * update location of camera count file * move annual bar charts into a more logical location * update test script * add gmaps param to my acct * relocate coverage maps and front page index * tidy up useraudit process * bugfixes in testing * bugfix in createAnnualBarchart * bug in testscript * another small bug in createSummaryTable * still fighting that bug * tidy up lastlog data * remove unused file * update cam status reporting * clear down the old brightness data * simplification of logging and of the nightlyJob script * bugfixes in process to gather consumed image ifo * update getUncal to work for historic data * typo * improvements in getUncalImages * bugfix * add python to get unused images * move script to right place * comment out unnecessary echo * bugfixes in camMetrics while migrating user accts * fix #448 * make sure window title is sensible * bugfix in replot and get unused * bugfix in removeDeletedTraj for search index * remove deprecation warning issue #450 * whoops naive vs tzaware dates * remove warning in cameraStatusReport issue #453 * remove warning issue #454 * widen constraints on analysing pickles to get images * add migration script * add sudo * actually do it * update sftp migration script * more changes to migration script * moving sftp migration script * update messages and docos * slight tweak to migration script * further work on sftp account migration * update ukmon ini files after migration * typo * collect old server login records integration with new * update migration instructions * update cam metrics report to include hostname * bugfix as $USER not available in cron * removing unused files * some tidying up * motre reorg * remove disused template rename file to follow convention * simplifying logging and removing unused aws profile * removing profile from python * adding executable bit * typo * bugfix to work around typo * print message when done * bugfix to get trajectories from s3 before trying to check for missing trajectories... * improve to support running for past dates * errors in bashrc and logrotate config * prevent email being sent for anything other than current days report * bugfix: make sure we use the most recent stats * improvement in reportOfLatestMatches to handle past dates * bugfix * small tweak to support reportOfLatestMatches * tweak getMatchStats to handle historic data better * add mariadb and ping to the sec grp so that the database can be accessed by apis * update the db servername * bugfix in match stats * prevent bogus renaming of matchJob.log * bumping version * bump version 2026.04.1 -> 2026.05.0 * update matchapi to use indexed column * add api readme * tweak to cammetrics to enhance stationlogins report * slight tweak to formatting * minor tweak to avoid unexpectedly changing directory * add support script for during server migrations * skip 1st row * add check for todos * oops strip whitespace * darn whitespace * working on migration tasks * minor tweaks * daft typo * bump version 2026.05.0 -> 2026.05.1 * bump version 2026.05.1 -> 2026.05.2 * update CodeQL to v3 * bump codeQL to v4 --- .github/workflows/codeql-analysis.yml | 8 +-- README.md | 2 +- archive/README.md | 2 +- archive/server_setup/acctStatusChecks.sh | 13 +++++ archive/server_setup/backupSftpAccts.sh | 30 +++++++++++ archive/server_setup/checkSftpAccounts.py | 65 +++++++++++++++++++++++ archive/server_setup/copyTestData.sh | 14 ----- archive/server_setup/migrateSftpAccts.sh | 2 + archive/ukmon_pylib/metrics/camMetrics.py | 10 ++-- bumpver.toml | 2 +- install_or_update.sh | 8 ++- 11 files changed, 129 insertions(+), 27 deletions(-) create mode 100644 archive/server_setup/acctStatusChecks.sh create mode 100644 archive/server_setup/backupSftpAccts.sh create mode 100644 archive/server_setup/checkSftpAccounts.py delete mode 100644 archive/server_setup/copyTestData.sh diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 77e99e8a1..bc121630d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,11 +39,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -54,7 +54,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v4 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v4 diff --git a/README.md b/README.md index 9f059c11c..2e9d9281e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # UK Meteor Data Analysis Shared code and libraries -version: 2026.05.0 +version: 2026.05.2 This repository contains the code behind the UK Meteors data archive and data processing pipeline. diff --git a/archive/README.md b/archive/README.md index c37b478af..e8cdc90fb 100644 --- a/archive/README.md +++ b/archive/README.md @@ -1,7 +1,7 @@ Data Processing and Flows ========================== -version: 2026.05.0 +version: 2026.05.2 This diagram shows the overall flow of data from Cameras to websites and out to the public. diff --git a/archive/server_setup/acctStatusChecks.sh b/archive/server_setup/acctStatusChecks.sh new file mode 100644 index 000000000..6c3463872 --- /dev/null +++ b/archive/server_setup/acctStatusChecks.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +cd $here + +sudo ls -1 /var/sftp > done.txt +grep ukmonh /home/ubuntu/prod/data/reports/stationlogins.txt |awk '{print $3}'| sort > pending.txt + +ssh ukmonhelper2 "~/prod/server_setup/get-nbd.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > not-being-done.txt +ssh ukmonhelper2 "~/prod/server_setup/get-all.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > all-accounts.txt + +python ~/src/ukmda-dataprocessing/archive/server_setup/checkSftpAccounts.py diff --git a/archive/server_setup/backupSftpAccts.sh b/archive/server_setup/backupSftpAccts.sh new file mode 100644 index 000000000..df09ff641 --- /dev/null +++ b/archive/server_setup/backupSftpAccts.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +cd $here + +bkpOneUser() { + userid=$1 + srchost=$2 + mkdir -p ./backup/$userid + sudo rsync -av $srchost:/var/sftp/$userid/ ./backup/$userid + sudo chown -R $USER:$USER ./backup/$userid + cat ./backup/$userid/ukmon.ini | sed 's/3.11.55.160/batchserver.ukmeteors.co.uk/g' > /tmp/$userid.ini + mv -f /tmp/$userid.ini ./backup/$userid/ukmon.ini +} + +if [ $# -lt 2 ] ; then + echo "Usage: ./backupSftpAccounts.sh oldservername userfile" + exit +fi + +echo "Warning: this must only be run on the new server" +read -p "press ctrl-c to quit or enter to continue" + +oldserver=$1 +srcfile=$2 + +cat $srcfile | while read stn +do + bkpOneUser $stn $oldserver +done \ No newline at end of file diff --git a/archive/server_setup/checkSftpAccounts.py b/archive/server_setup/checkSftpAccounts.py new file mode 100644 index 000000000..8149c0185 --- /dev/null +++ b/archive/server_setup/checkSftpAccounts.py @@ -0,0 +1,65 @@ +import pandas as pd +import datetime + + +data=open('/home/ubuntu/prod/data/reports/stationlogins.txt').readlines() +camlist = [] +not_live = [] +still_upl = [] +not_upl = [] +livenames = [] + +livedate = datetime.datetime.now() - datetime.timedelta(days=10) +for li in data: + if 'Last Upload' in li: + continue + lastup = li[:19] + lastlo = li[21:40] + loc = li[42:61] + gmnid = li[64:73] + via = li[76:].strip() + camlist.append([lastup, lastlo, loc, gmnid, via]) + + if ">" in lastup and ">" in lastlo: + not_live.append([loc, gmnid, lastup, lastlo, via]) + + elif ">" not in lastup: + lastupdt = datetime.datetime.strptime(lastup, '%Y-%m-%dT%H:%M:%S') + if lastupdt >= livedate: + still_upl.append([loc, gmnid, lastup, lastlo, via]) + livenames.append(loc.strip()) + else: + if '>' in lastlo: + not_live.append([loc, gmnid, lastup, lastlo, via]) + else: + lastlodt = datetime.datetime.strptime(lastlo, '%Y-%m-%dT%H:%M:%S') + if lastlodt >= livedate: + not_upl.append([loc, gmnid, lastup, lastlo, via]) + else: + not_live.append([loc, gmnid, lastup, lastlo, via]) + else: # '>' not in lastlo + lastlodt = datetime.datetime.strptime(lastlo, '%Y-%m-%dT%H:%M:%S') + if lastlodt >= livedate: + not_upl.append([loc, gmnid, lastup, lastlo, via]) + else: + not_live.append([loc, gmnid, lastup, lastlo, via]) + +with open('still-live.txt','w') as outf: + for cam in still_upl: + outf.write(','.join(cam) + '\n') + +with open('not_uploading.txt','w') as outf: + for cam in not_upl: + outf.write(','.join(cam) + '\n') + +with open('not_live.txt','w') as outf: + for cam in not_live: + outf.write(','.join(cam) + '\n') + +donelist = open('done.txt', 'r').readlines() +donelist = [x.strip() for x in donelist] +with open('todo.txt', 'w') as outf: + for nam in livenames: + if nam.strip() not in donelist: + print('done list is missing', nam) + outf.write(f'{nam}\n') \ No newline at end of file diff --git a/archive/server_setup/copyTestData.sh b/archive/server_setup/copyTestData.sh deleted file mode 100644 index e2c041c5a..000000000 --- a/archive/server_setup/copyTestData.sh +++ /dev/null @@ -1,14 +0,0 @@ -# prep some test data -$yr=$(date +%Y) - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -source $here/../config.ini >/dev/null 2>&1 - -rsync -avz ~/prod/data/single/ $DATADIR/single/ -rsync -avz ~/prod/data/matched/*${yr}* $DATADIR/matched/ -rsync -avz ~/prod/data/consolidated/*${yr}* $DATADIR/consolidated/ -rsync -avz ~/prod/data/dailyreports/stats.txt $DATADIR/dailyreports/ -rsync -avz ~/prod/data/dailyreports/${yr}*.txt $DATADIR/dailyreports/ -rsync -avz ~/prod/logs/*${yr}*.txt $SRC/logs/ -rsync -avz ~/prod/data/*${yr}*.jpg $DATADIR/ -rsync -avz ~/prod/data/*.png $DATADIR/ \ No newline at end of file diff --git a/archive/server_setup/migrateSftpAccts.sh b/archive/server_setup/migrateSftpAccts.sh index 52d5add0f..823444955 100755 --- a/archive/server_setup/migrateSftpAccts.sh +++ b/archive/server_setup/migrateSftpAccts.sh @@ -1,5 +1,7 @@ #!/bin/bash +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +cd $here addOneUser() { userid=$1 diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index 36aeaf985..6824db9f9 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -206,15 +206,15 @@ def backPopulate(stationid): df['uploadtime']=df.uploadtime.astype("str").str.pad(6,fillchar="0") df['lastupload']=df.upddate.astype('str') + '_' +df.uploadtime df.lastupload = [datetime.datetime.strptime(x, '%Y%m%d_%H%M%S') for x in df.lastupload] - df = df.drop(columns=['stationid','manual','rundate', 'upddate','uploadtime']) + df = df.drop(columns=['manual','rundate', 'upddate','uploadtime']) df = df.sort_values(by=['lastupload']) outfile=os.path.join(datadir, 'reports', 'stationlogins.txt') zerodate = datetime.datetime(1970,1,1,0,0,0) with open(outfile,'w') as outf: - outf.write('Last Upload, StationID, Last Login, Via\n') + outf.write(f'{"Last Upload":19s} {"Last Connect":19s} {"StationID":20s} {"GMN ID":10s} Via\n') for _,rw in df.iterrows(): - lastup = '> 1 month' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S') - lastlo = '> 1 month' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S') + lastup = '> 1 month ago' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S') + lastlo = '> 1 month ago' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S') via = '' if pd.isnull(rw.host) else rw.host - outf.write(f'{lastup} , {rw.location:20s}, {lastlo}, {via}\n') + outf.write(f'{lastup:19s} {lastlo:19s} {rw.location:20s} {rw.stationid:10s} {via}\n') diff --git a/bumpver.toml b/bumpver.toml index 17d501fc7..cc67f3028 100644 --- a/bumpver.toml +++ b/bumpver.toml @@ -1,5 +1,5 @@ [bumpver] -current_version = "2026.05.0" +current_version = "2026.05.2" version_pattern = "YYYY.0M.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true diff --git a/install_or_update.sh b/install_or_update.sh index 83713e346..c903adce7 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -10,8 +10,8 @@ envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +pushd $here/archive echo "Updating codebase..." -cd $here/archive git pull echo "Updating code..." @@ -32,6 +32,11 @@ mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs mkdir -p ~/.logrotate mkdir -p ~/.aws +mkdir -p ~/server_setup + +rsync -a server_setup/*.sh ~/server_setup +rsync -a server_setup/*.py ~/server_setup +chmod +x ~/server_setup/*.sh echo "Checking conda environment..." if [[ -f ~/.condaon && -d ~/miniconda3/envs/wmpl ]] @@ -69,4 +74,5 @@ else echo skipping config and bashrc fi echo "" +popd echo "$msg complete" From 91e6978e8683af43fcd9bbe97797e70d13c888d9 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 7 May 2026 23:50:38 +0100 Subject: [PATCH 3/4] Trivial changes mainly to automation scripts (#471) * add MAX_STNS so i can experiement with limits * add logging. Add max_stns to constraints and set to 9999 push logs to s3 copy calcserver logs to ukmonhelper * don't pull or push the databases from S3 during distribution but do back them up after consolidation * catch error if trajpickle can't be found * set rundate more safely move some processes into nightlyJob to make testing easier * tidy up backup file names and housekeep backup folder * move some processes from findAllMatches to nightlyJob to make testing easier * support for testing of createExecReplot * pass testmode to createExecReplot * ensure logs are unique and capture container logs * realigning createFireballPage with prod. Updating gitignore * remove verbosity flag from one tar operation * fixup to handle new style log with datestamps * get privip after restarting calcserver * temporarily don't run rerunFailedLambdas in test * update to work in a test env too * add 'period' to the match summary api * add period to match summary api * re-enabling rerunFailedLambdas * quieten an S3 transfer * include the foldername when reporting on container progress * put daily trajdb in a safe place * be more specific when finding the daily log * only tar the current container logs * save daily obsdb in the right place * rename ec2.tf to calcserver.tf * quieten some s3 transfers * rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is * update SSM params to reflect new serverid * bugfix in distributeCandidates - using wrong path * add newline to getCostMetrics * weird bug in statsToMqtt * bugfix in update_wmpl.sh for traj container * Script to create an AMI for the calcengine * exclude test data when syncing from s3 * writing databases and logs to the wrong target * bad test on istest * typo in serveruserid * adding to server migration notes * purge old rawcsvs and fullcsvs older than 180 days * improved housekeeping of old data * make sure we scoop up data for 31/12 * stop creation of an unused set of files * change default locations for container data * put container logs in the right place * update dockerfile syntax * change locations of test container data and output * remove incorrect comments * Add script to clean up deleted/duplicate trajs * simplify the stats gathering and move to python fix bug in getMatchStats * comment out unused files * update reportOfLatestMatches to exclude deleted trajectories * add scp to requirements * add SQL database maintenance to dataMaintenance * incorrect default for datadir * update call to pymysql.connect * ensure consolidateOutput rerunnable for prior years * upgrading WMPL * update gitignore a bit * bugfix to catch data from 31/12 each year * add paramiko to dependencies for matchPickle to support wmpl upgrade * update getpickle to newest WMPL * bugfix in api tests * bugfix - changes to dailydb not saved * improvements to process to cleanup deleted trajs * after merging, check and remove any duplicate trajectories from the sqlite db and disk * Adding version * bump version 2025.11.1 -> 2026.04.0 * update files containing vesion * bump version 2026.04.0 -> 2026.04.1 * unmangling side effets of automerge * correct usage message * decommission old calcserver and add new batchserver * add calcserver IP to ssm variables and use it in scripts * Update my branch with the latest from master (#436) * Update version (#433) * remove disused code * update livestream to accept normal RMS filenames * search dialog - fix bug and set default date range * small change to explain default dates * improvement in live image search * remove redundant file * update deployment scripts * improvements in routine to convert GMN report to pandas * more analysis of gmn data * Update the terraform a little * remove hardcoded reference to ec2-user * add SERVERUSERID env /ssm variable for calcserver user id * update container for new distrib-processing method * use calcserver SERVERUSERID variable * support for test ECS containers * fix paths in deployment scripts * remove unused file * missed some templates * update dev/test env settings * remove disused file * make sure paths are expanded * fixing various config issues * updates to makeConfig * updates to distributed processing to align with newest wmpl * add a comment to explain what the code does * remove json db * update consolidation to handle sqlite dbs * remove unused function * create data for daily report * remove unused file * update to call new version of reportOfLatestMatches * add test variables * mildly improve doco * put test logs in the right CW log group * handle missing log groups better * extra value in TrajQualityParams * bugfixes revealed in testing * fix issues in reportOfLatestMatches * put container dbs in a sensible place * catch logging errors * tidy up file locations * simplify calcserver start up * sorting out errors in container logging * update container logging info * avoid unnecessary error * fix bugs in log capture process * add MAX_STNS so i can experiement with limits * add logging. Add max_stns to constraints and set to 9999 push logs to s3 copy calcserver logs to ukmonhelper * don't pull or push the databases from S3 during distribution but do back them up after consolidation * catch error if trajpickle can't be found * set rundate more safely move some processes into nightlyJob to make testing easier * tidy up backup file names and housekeep backup folder * move some processes from findAllMatches to nightlyJob to make testing easier * support for testing of createExecReplot * pass testmode to createExecReplot * ensure logs are unique and capture container logs * realigning createFireballPage with prod. Updating gitignore * remove verbosity flag from one tar operation * fixup to handle new style log with datestamps * get privip after restarting calcserver * temporarily don't run rerunFailedLambdas in test * update to work in a test env too * add 'period' to the match summary api * add period to match summary api * re-enabling rerunFailedLambdas * quieten an S3 transfer * include the foldername when reporting on container progress * put daily trajdb in a safe place * be more specific when finding the daily log * only tar the current container logs * save daily obsdb in the right place * rename ec2.tf to calcserver.tf * quieten some s3 transfers * rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is * update SSM params to reflect new serverid * bugfix in distributeCandidates - using wrong path * add newline to getCostMetrics * weird bug in statsToMqtt * bugfix in update_wmpl.sh for traj container * Script to create an AMI for the calcengine * exclude test data when syncing from s3 * writing databases and logs to the wrong target * bad test on istest * typo in serveruserid * adding to server migration notes * purge old rawcsvs and fullcsvs older than 180 days * improved housekeeping of old data * make sure we scoop up data for 31/12 * stop creation of an unused set of files * change default locations for container data * put container logs in the right place * update dockerfile syntax * change locations of test container data and output * remove incorrect comments * Add script to clean up deleted/duplicate trajs * simplify the stats gathering and move to python fix bug in getMatchStats * comment out unused files * update reportOfLatestMatches to exclude deleted trajectories * add scp to requirements * add SQL database maintenance to dataMaintenance * incorrect default for datadir * update call to pymysql.connect * ensure consolidateOutput rerunnable for prior years * upgrading WMPL * update gitignore a bit * bugfix to catch data from 31/12 each year * add paramiko to dependencies for matchPickle to support wmpl upgrade * update getpickle to newest WMPL * bugfix in api tests * bugfix - changes to dailydb not saved * improvements to process to cleanup deleted trajs * after merging, check and remove any duplicate trajectories from the sqlite db and disk * Adding version * bump version 2025.11.1 -> 2026.04.0 * update files containing vesion * bump version 2026.04.0 -> 2026.04.1 * unmangling side effets of automerge * correct usage message * adding install script * changes to ansible scripts * remove irrelevant file * Adding SSM variables for the new batchserver in the ukmda account * attempt to add logging to consolidateDistTraj * remove reference to ee account * temporarily add mariadb deets to ssm in mjmm account * bugfixes in dataMaintenance and cleanupDeletedTrajs to avoid deleting disk files when trajs share same folder * updating migration instructions * more work on deduplication for ukmon * remove username-specific stuff * remove the old deployment scripts * update install script * updates to improve server migration process * add bashrc to the installer * slight tweak to installer * add a few installer messages * remove an echo * be less verbose in the rsyncs * oops need -a for one of the rsyncs * rename requirements file for ukmda * remove dependencies on MeteorTools * add utils to ukmon_pylib * more removal of dependencies on MeteorTools * updating how shower data is obtained * corercting location of IMO and gmn shower data * bugfixes * fixes in imoWorkinShowerlist * small error in install_or_update * bugfix when no data found in imo set * retrieve IMO data after each code update * fixing up getShowerDets * improve installer * fix createShowerExtracts * update extractors and showeranalysis * bug in extractor thats existed since 2023 but nobody noticed... * trying to fix matplotlib bug * more working on matplotlib issue * oops overfixed a bug * another incorrect bugfix! * make bars wider * more removal of MeteorTools * bugfix in createannualbarchart * working on bug * more work on lenght bug * remove some debugging * oops forgot to push the fixed version of createReportIndex * segregating function to get list of images that weren't calibrated by RMS * reorganize stuff a bit * add email sending functionality to ukmon_pylib * use send email functionality in code * add database and gmaps keys/passwords as secure variables * update to use secure keys * remove dbpw from environment * remove dbpw from environment * add script to retrieve db passwd * add script to retrieve db passwords * bug in createTables * updates to migration instructions * include pip installs in the script * tweak installer * remove glob.glob1 * move unused scripts - will delete later * fix mariadb connections * typo * tidy up docs * fixing mysql calls * small changes to documentation * suppress pip messages if already installed * tweaks to the installer * add logrotate stuff * bugfixes in installer * tidy up installer * bugfix in key mgmt * move some functionality into new script to tidy up nightly job and simplify rerunning * improve install script * update location of camera count file * move annual bar charts into a more logical location * update test script * add gmaps param to my acct * relocate coverage maps and front page index * tidy up useraudit process * bugfixes in testing * bugfix in createAnnualBarchart * bug in testscript * another small bug in createSummaryTable * still fighting that bug * tidy up lastlog data * remove unused file * update cam status reporting * clear down the old brightness data * simplification of logging and of the nightlyJob script * bugfixes in process to gather consumed image ifo * update getUncal to work for historic data * typo * improvements in getUncalImages * bugfix * add python to get unused images * move script to right place * comment out unnecessary echo * bugfixes in camMetrics while migrating user accts * fix #448 * make sure window title is sensible * bugfix in replot and get unused * bugfix in removeDeletedTraj for search index * remove deprecation warning issue #450 * whoops naive vs tzaware dates * remove warning in cameraStatusReport issue #453 * remove warning issue #454 * widen constraints on analysing pickles to get images * add migration script * add sudo * actually do it * update sftp migration script * more changes to migration script * moving sftp migration script * update messages and docos * slight tweak to migration script * further work on sftp account migration * update ukmon ini files after migration * typo * collect old server login records integration with new * update migration instructions * update cam metrics report to include hostname * bugfix as $USER not available in cron * removing unused files * some tidying up * motre reorg * remove disused template rename file to follow convention * simplifying logging and removing unused aws profile * removing profile from python * adding executable bit * typo * bugfix to work around typo * print message when done * bugfix to get trajectories from s3 before trying to check for missing trajectories... * improve to support running for past dates * errors in bashrc and logrotate config * prevent email being sent for anything other than current days report * bugfix: make sure we use the most recent stats * improvement in reportOfLatestMatches to handle past dates * bugfix * small tweak to support reportOfLatestMatches * tweak getMatchStats to handle historic data better * add mariadb and ping to the sec grp so that the database can be accessed by apis * update the db servername * bugfix in match stats * prevent bogus renaming of matchJob.log * bumping version * bump version 2026.04.1 -> 2026.05.0 * update matchapi to use indexed column * add api readme * tweak to cammetrics to enhance stationlogins report * slight tweak to formatting * minor tweak to avoid unexpectedly changing directory * add support script for during server migrations * skip 1st row * add check for todos * oops strip whitespace * darn whitespace * working on migration tasks * minor tweaks * daft typo * bump version 2026.05.0 -> 2026.05.1 * bump version 2026.05.1 -> 2026.05.2 * update CodeQL to v3 * bump codeQL to v4 * tweak to script supporting server migration * tweak to scripts supporting server migration * add some stats on migration status * tweak --- archive/server_setup/acctStatusChecks.sh | 14 +++++++++++++- archive/server_setup/checkSftpAccounts.py | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/archive/server_setup/acctStatusChecks.sh b/archive/server_setup/acctStatusChecks.sh index 6c3463872..6665041a7 100644 --- a/archive/server_setup/acctStatusChecks.sh +++ b/archive/server_setup/acctStatusChecks.sh @@ -4,10 +4,22 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" cd $here -sudo ls -1 /var/sftp > done.txt +sudo ls -1 /var/sftp > moved.txt +grep batchs /home/ubuntu/prod/data/reports/stationlogins.txt |awk '{print $3}'| sort > switched.txt grep ukmonh /home/ubuntu/prod/data/reports/stationlogins.txt |awk '{print $3}'| sort > pending.txt ssh ukmonhelper2 "~/prod/server_setup/get-nbd.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > not-being-done.txt ssh ukmonhelper2 "~/prod/server_setup/get-all.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > all-accounts.txt python ~/src/ukmda-dataprocessing/archive/server_setup/checkSftpAccounts.py + +echo "Moved: $(wc -l moved.txt | awk '{print $1}')" +echo "Switched: $(wc -l switched.txt | awk '{print $1}')" +echo "Pending: $(wc -l pending.txt | awk '{print $1}')" +echo "Total: $(wc -l still-live.txt | awk '{print $1}')" +echo "" +echo "Not Live: $(wc -l not_live.txt | awk '{print $1}')" +echo "Not Upl: $(wc -l not_uploading.txt | awk '{print $1}')" +echo "" +echo "Dead: $(wc -l not-being-done.txt | awk '{print $1}')" +echo "" diff --git a/archive/server_setup/checkSftpAccounts.py b/archive/server_setup/checkSftpAccounts.py index 8149c0185..6e4f5e90c 100644 --- a/archive/server_setup/checkSftpAccounts.py +++ b/archive/server_setup/checkSftpAccounts.py @@ -56,7 +56,7 @@ for cam in not_live: outf.write(','.join(cam) + '\n') -donelist = open('done.txt', 'r').readlines() +donelist = open('moved.txt', 'r').readlines() donelist = [x.strip() for x in donelist] with open('todo.txt', 'w') as outf: for nam in livenames: From 63f063a98aa699a8aae3afb4b249edde179b1e1a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 14 May 2026 21:43:55 +0100 Subject: [PATCH 4/4] Latest patch release (#473) * save daily obsdb in the right place * rename ec2.tf to calcserver.tf * quieten some s3 transfers * rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is * update SSM params to reflect new serverid * bugfix in distributeCandidates - using wrong path * add newline to getCostMetrics * weird bug in statsToMqtt * bugfix in update_wmpl.sh for traj container * Script to create an AMI for the calcengine * exclude test data when syncing from s3 * writing databases and logs to the wrong target * bad test on istest * typo in serveruserid * adding to server migration notes * purge old rawcsvs and fullcsvs older than 180 days * improved housekeeping of old data * make sure we scoop up data for 31/12 * stop creation of an unused set of files * change default locations for container data * put container logs in the right place * update dockerfile syntax * change locations of test container data and output * remove incorrect comments * Add script to clean up deleted/duplicate trajs * simplify the stats gathering and move to python fix bug in getMatchStats * comment out unused files * update reportOfLatestMatches to exclude deleted trajectories * add scp to requirements * add SQL database maintenance to dataMaintenance * incorrect default for datadir * update call to pymysql.connect * ensure consolidateOutput rerunnable for prior years * upgrading WMPL * update gitignore a bit * bugfix to catch data from 31/12 each year * add paramiko to dependencies for matchPickle to support wmpl upgrade * update getpickle to newest WMPL * bugfix in api tests * bugfix - changes to dailydb not saved * improvements to process to cleanup deleted trajs * after merging, check and remove any duplicate trajectories from the sqlite db and disk * Adding version * bump version 2025.11.1 -> 2026.04.0 * update files containing vesion * bump version 2026.04.0 -> 2026.04.1 * unmangling side effets of automerge * correct usage message * decommission old calcserver and add new batchserver * add calcserver IP to ssm variables and use it in scripts * Update my branch with the latest from master (#436) * Update version (#433) * remove disused code * update livestream to accept normal RMS filenames * search dialog - fix bug and set default date range * small change to explain default dates * improvement in live image search * remove redundant file * update deployment scripts * improvements in routine to convert GMN report to pandas * more analysis of gmn data * Update the terraform a little * remove hardcoded reference to ec2-user * add SERVERUSERID env /ssm variable for calcserver user id * update container for new distrib-processing method * use calcserver SERVERUSERID variable * support for test ECS containers * fix paths in deployment scripts * remove unused file * missed some templates * update dev/test env settings * remove disused file * make sure paths are expanded * fixing various config issues * updates to makeConfig * updates to distributed processing to align with newest wmpl * add a comment to explain what the code does * remove json db * update consolidation to handle sqlite dbs * remove unused function * create data for daily report * remove unused file * update to call new version of reportOfLatestMatches * add test variables * mildly improve doco * put test logs in the right CW log group * handle missing log groups better * extra value in TrajQualityParams * bugfixes revealed in testing * fix issues in reportOfLatestMatches * put container dbs in a sensible place * catch logging errors * tidy up file locations * simplify calcserver start up * sorting out errors in container logging * update container logging info * avoid unnecessary error * fix bugs in log capture process * add MAX_STNS so i can experiement with limits * add logging. Add max_stns to constraints and set to 9999 push logs to s3 copy calcserver logs to ukmonhelper * don't pull or push the databases from S3 during distribution but do back them up after consolidation * catch error if trajpickle can't be found * set rundate more safely move some processes into nightlyJob to make testing easier * tidy up backup file names and housekeep backup folder * move some processes from findAllMatches to nightlyJob to make testing easier * support for testing of createExecReplot * pass testmode to createExecReplot * ensure logs are unique and capture container logs * realigning createFireballPage with prod. Updating gitignore * remove verbosity flag from one tar operation * fixup to handle new style log with datestamps * get privip after restarting calcserver * temporarily don't run rerunFailedLambdas in test * update to work in a test env too * add 'period' to the match summary api * add period to match summary api * re-enabling rerunFailedLambdas * quieten an S3 transfer * include the foldername when reporting on container progress * put daily trajdb in a safe place * be more specific when finding the daily log * only tar the current container logs * save daily obsdb in the right place * rename ec2.tf to calcserver.tf * quieten some s3 transfers * rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is * update SSM params to reflect new serverid * bugfix in distributeCandidates - using wrong path * add newline to getCostMetrics * weird bug in statsToMqtt * bugfix in update_wmpl.sh for traj container * Script to create an AMI for the calcengine * exclude test data when syncing from s3 * writing databases and logs to the wrong target * bad test on istest * typo in serveruserid * adding to server migration notes * purge old rawcsvs and fullcsvs older than 180 days * improved housekeeping of old data * make sure we scoop up data for 31/12 * stop creation of an unused set of files * change default locations for container data * put container logs in the right place * update dockerfile syntax * change locations of test container data and output * remove incorrect comments * Add script to clean up deleted/duplicate trajs * simplify the stats gathering and move to python fix bug in getMatchStats * comment out unused files * update reportOfLatestMatches to exclude deleted trajectories * add scp to requirements * add SQL database maintenance to dataMaintenance * incorrect default for datadir * update call to pymysql.connect * ensure consolidateOutput rerunnable for prior years * upgrading WMPL * update gitignore a bit * bugfix to catch data from 31/12 each year * add paramiko to dependencies for matchPickle to support wmpl upgrade * update getpickle to newest WMPL * bugfix in api tests * bugfix - changes to dailydb not saved * improvements to process to cleanup deleted trajs * after merging, check and remove any duplicate trajectories from the sqlite db and disk * Adding version * bump version 2025.11.1 -> 2026.04.0 * update files containing vesion * bump version 2026.04.0 -> 2026.04.1 * unmangling side effets of automerge * correct usage message * adding install script * changes to ansible scripts * remove irrelevant file * Adding SSM variables for the new batchserver in the ukmda account * attempt to add logging to consolidateDistTraj * remove reference to ee account * temporarily add mariadb deets to ssm in mjmm account * bugfixes in dataMaintenance and cleanupDeletedTrajs to avoid deleting disk files when trajs share same folder * updating migration instructions * more work on deduplication for ukmon * remove username-specific stuff * remove the old deployment scripts * update install script * updates to improve server migration process * add bashrc to the installer * slight tweak to installer * add a few installer messages * remove an echo * be less verbose in the rsyncs * oops need -a for one of the rsyncs * rename requirements file for ukmda * remove dependencies on MeteorTools * add utils to ukmon_pylib * more removal of dependencies on MeteorTools * updating how shower data is obtained * corercting location of IMO and gmn shower data * bugfixes * fixes in imoWorkinShowerlist * small error in install_or_update * bugfix when no data found in imo set * retrieve IMO data after each code update * fixing up getShowerDets * improve installer * fix createShowerExtracts * update extractors and showeranalysis * bug in extractor thats existed since 2023 but nobody noticed... * trying to fix matplotlib bug * more working on matplotlib issue * oops overfixed a bug * another incorrect bugfix! * make bars wider * more removal of MeteorTools * bugfix in createannualbarchart * working on bug * more work on lenght bug * remove some debugging * oops forgot to push the fixed version of createReportIndex * segregating function to get list of images that weren't calibrated by RMS * reorganize stuff a bit * add email sending functionality to ukmon_pylib * use send email functionality in code * add database and gmaps keys/passwords as secure variables * update to use secure keys * remove dbpw from environment * remove dbpw from environment * add script to retrieve db passwd * add script to retrieve db passwords * bug in createTables * updates to migration instructions * include pip installs in the script * tweak installer * remove glob.glob1 * move unused scripts - will delete later * fix mariadb connections * typo * tidy up docs * fixing mysql calls * small changes to documentation * suppress pip messages if already installed * tweaks to the installer * add logrotate stuff * bugfixes in installer * tidy up installer * bugfix in key mgmt * move some functionality into new script to tidy up nightly job and simplify rerunning * improve install script * update location of camera count file * move annual bar charts into a more logical location * update test script * add gmaps param to my acct * relocate coverage maps and front page index * tidy up useraudit process * bugfixes in testing * bugfix in createAnnualBarchart * bug in testscript * another small bug in createSummaryTable * still fighting that bug * tidy up lastlog data * remove unused file * update cam status reporting * clear down the old brightness data * simplification of logging and of the nightlyJob script * bugfixes in process to gather consumed image ifo * update getUncal to work for historic data * typo * improvements in getUncalImages * bugfix * add python to get unused images * move script to right place * comment out unnecessary echo * bugfixes in camMetrics while migrating user accts * fix #448 * make sure window title is sensible * bugfix in replot and get unused * bugfix in removeDeletedTraj for search index * remove deprecation warning issue #450 * whoops naive vs tzaware dates * remove warning in cameraStatusReport issue #453 * remove warning issue #454 * widen constraints on analysing pickles to get images * add migration script * add sudo * actually do it * update sftp migration script * more changes to migration script * moving sftp migration script * update messages and docos * slight tweak to migration script * further work on sftp account migration * update ukmon ini files after migration * typo * collect old server login records integration with new * update migration instructions * update cam metrics report to include hostname * bugfix as $USER not available in cron * removing unused files * some tidying up * motre reorg * remove disused template rename file to follow convention * simplifying logging and removing unused aws profile * removing profile from python * adding executable bit * typo * bugfix to work around typo * print message when done * bugfix to get trajectories from s3 before trying to check for missing trajectories... * improve to support running for past dates * errors in bashrc and logrotate config * prevent email being sent for anything other than current days report * bugfix: make sure we use the most recent stats * improvement in reportOfLatestMatches to handle past dates * bugfix * small tweak to support reportOfLatestMatches * tweak getMatchStats to handle historic data better * add mariadb and ping to the sec grp so that the database can be accessed by apis * update the db servername * bugfix in match stats * prevent bogus renaming of matchJob.log * bumping version * bump version 2026.04.1 -> 2026.05.0 * update matchapi to use indexed column * add api readme * tweak to cammetrics to enhance stationlogins report * slight tweak to formatting * minor tweak to avoid unexpectedly changing directory * add support script for during server migrations * skip 1st row * add check for todos * oops strip whitespace * darn whitespace * working on migration tasks * minor tweaks * daft typo * bump version 2026.05.0 -> 2026.05.1 * bump version 2026.05.1 -> 2026.05.2 * update CodeQL to v3 * bump codeQL to v4 * tweak to script supporting server migration * tweak to scripts supporting server migration * add some stats on migration status * tweak * get cursor keys working in python * set via even if no recent login * improved reporting on migration * tweaks to migr scripts * another small tweak * make lists unique * uniq and sort backwards * testing fix for #460 * more work on status reports * bugfix in failed matches report * found the bug * attempt to fix fails report * another tweak * okay, got it * remove debug * fix timing metrics data * small tweak * looks good * add EIP for the batchserver * bugfix to handle bad data in login details * add function to email problem cameras with bespoke message * bump version 2026.05.2 -> 2026.05.3 --- README.md | 2 +- archive/README.md | 2 +- archive/analysis/getLogData.sh | 6 +- archive/server_setup/.bashrc | 5 +- archive/server_setup/acctStatusChecks.sh | 28 +++--- archive/server_setup/checkSftpAccounts.py | 12 ++- archive/terraform/ukmda/batchserver.tf | 8 ++ archive/ukmon_pylib/metrics/camMetrics.py | 2 +- archive/ukmon_pylib/metrics/timingMetrics.py | 96 ++++++------------- .../ukmon_pylib/reports/findFailedMatches.py | 14 ++- .../ukmon_pylib/reports/getSolutionStati.py | 93 ++---------------- .../ukmon_pylib/reports/reportBadCameras.py | 40 ++++++++ archive/website/cameraStatusReport.sh | 4 +- bumpver.toml | 2 +- 14 files changed, 132 insertions(+), 182 deletions(-) diff --git a/README.md b/README.md index 2e9d9281e..2365d4136 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # UK Meteor Data Analysis Shared code and libraries -version: 2026.05.2 +version: 2026.05.3 This repository contains the code behind the UK Meteors data archive and data processing pipeline. diff --git a/archive/README.md b/archive/README.md index e8cdc90fb..4c9acc4ff 100644 --- a/archive/README.md +++ b/archive/README.md @@ -1,7 +1,7 @@ Data Processing and Flows ========================== -version: 2026.05.2 +version: 2026.05.3 This diagram shows the overall flow of data from Cameras to websites and out to the public. diff --git a/archive/analysis/getLogData.sh b/archive/analysis/getLogData.sh index 67342a511..9a4c9c929 100644 --- a/archive/analysis/getLogData.sh +++ b/archive/analysis/getLogData.sh @@ -62,7 +62,7 @@ echo "the entire night was uncalibrated (indicated by the 'skipping' warning).To interpret these data, see " >> $logfile echo "here

" >> $logfile echo "
" >> $logfile
-grep "Skipping" $lastmtch >> $logfile
+grep "Skipping" $lastmtch | egrep -v "directory|pickle" >> $logfile
 echo "
" >> $logfile echo "

Failed Match Report

" >> $logfile @@ -72,6 +72,7 @@ echo "" >> $logfile cat $DATADIR/failed/${rundate}_failed.txt >> $logfile echo "" >> $logfile + echo "

Chart of Batch Element Runtimes

" >> $logfile echo "

This section shows the runtime of the different jobs in the batch

" >> $logfile echo "

" >> $logfile @@ -93,4 +94,7 @@ done cat $TEMPLATES/footer.html >> $DATADIR/lastlogs/index.html aws s3 cp $DATADIR/lastlogs/index.html $WEBSITEBUCKET/reports/lastlogs/ --quiet +# housekeep the fails reports +find $DATADIR/failed -mtime +90 -exec rm -f {} \; + find $DATADIR/lastlogs -name "lastlog*" -mtime +90 -exec rm -f {} \; diff --git a/archive/server_setup/.bashrc b/archive/server_setup/.bashrc index a7f39fa35..b8c939f24 100644 --- a/archive/server_setup/.bashrc +++ b/archive/server_setup/.bashrc @@ -55,9 +55,12 @@ fi unset __conda_setup # <<< conda initialize <<< +# make the function keys work properly in python +export TERM=putty + # If this is an xterm set the title to user@host:dir case "$TERM" in - xterm*|rxvt*) + xterm*|rxvt*|putty*) PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1" ;; *) diff --git a/archive/server_setup/acctStatusChecks.sh b/archive/server_setup/acctStatusChecks.sh index 6665041a7..eded44c2c 100644 --- a/archive/server_setup/acctStatusChecks.sh +++ b/archive/server_setup/acctStatusChecks.sh @@ -4,22 +4,24 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" cd $here -sudo ls -1 /var/sftp > moved.txt -grep batchs /home/ubuntu/prod/data/reports/stationlogins.txt |awk '{print $3}'| sort > switched.txt -grep ukmonh /home/ubuntu/prod/data/reports/stationlogins.txt |awk '{print $3}'| sort > pending.txt +sudo ls -1 /var/sftp | egrep -v "test|logup" > moved.txt +grep batchs /home/ubuntu/prod/data/reports/stationlogins.txt | cut -c 42-61 | sed 's/ //g' | sort | uniq > switched.txt +grep ukmonh /home/ubuntu/prod/data/reports/stationlogins.txt | cut -c 42-61 | sed 's/ //g' | sort | uniq > pending.txt ssh ukmonhelper2 "~/prod/server_setup/get-nbd.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > not-being-done.txt ssh ukmonhelper2 "~/prod/server_setup/get-all.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > all-accounts.txt python ~/src/ukmda-dataprocessing/archive/server_setup/checkSftpAccounts.py -echo "Moved: $(wc -l moved.txt | awk '{print $1}')" -echo "Switched: $(wc -l switched.txt | awk '{print $1}')" -echo "Pending: $(wc -l pending.txt | awk '{print $1}')" -echo "Total: $(wc -l still-live.txt | awk '{print $1}')" -echo "" -echo "Not Live: $(wc -l not_live.txt | awk '{print $1}')" -echo "Not Upl: $(wc -l not_uploading.txt | awk '{print $1}')" -echo "" -echo "Dead: $(wc -l not-being-done.txt | awk '{print $1}')" -echo "" + +echo "Moved: $(wc -l moved.txt | awk '{print $1}')" > statusreport.txt +echo "Switched: $(wc -l switched.txt | awk '{print $1}')" >> statusreport.txt +echo "Pending: $(wc -l pending.txt | awk '{print $1}')" >> statusreport.txt +echo "Total: $(wc -l still-live.txt | awk '{print $1}')" >> statusreport.txt +echo "" >> statusreport.txt +echo "Not Live: $(wc -l not_live.txt | awk '{print $1}')" >> statusreport.txt +echo "Not Upl: $(wc -l not_uploading.txt | awk '{print $1}')" >> statusreport.txt +echo "" >> statusreport.txt +echo "Dead: $(wc -l not-being-done.txt | awk '{print $1}')" >> statusreport.txt +echo "" >> statusreport.txt +cat statusreport.txt diff --git a/archive/server_setup/checkSftpAccounts.py b/archive/server_setup/checkSftpAccounts.py index 6e4f5e90c..de6b7696a 100644 --- a/archive/server_setup/checkSftpAccounts.py +++ b/archive/server_setup/checkSftpAccounts.py @@ -62,4 +62,14 @@ for nam in livenames: if nam.strip() not in donelist: print('done list is missing', nam) - outf.write(f'{nam}\n') \ No newline at end of file + outf.write(f'{nam}\n') + + +moved = open('moved.txt').readlines() +pending = open('pending.txt').readlines() +switched = open('switched.txt').readlines() + +pendingnotmoved = [x for x in pending if x not in moved] + +print('stations marked pending but not moved - check if need done') +print(pendingnotmoved) \ No newline at end of file diff --git a/archive/terraform/ukmda/batchserver.tf b/archive/terraform/ukmda/batchserver.tf index 7ff5993f8..833338e4c 100644 --- a/archive/terraform/ukmda/batchserver.tf +++ b/archive/terraform/ukmda/batchserver.tf @@ -45,3 +45,11 @@ resource "aws_network_interface" "batchserver_if" { "billingtag" = "ukmda" } } + +resource "aws_eip" "batchserver" { + instance = aws_instance.batchserver.id + tags = { + billingtag = "ukmda" + Name = "ukmonhelper_eip" + } +} diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index 6824db9f9..b6198aa5a 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -216,5 +216,5 @@ def backPopulate(stationid): for _,rw in df.iterrows(): lastup = '> 1 month ago' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S') lastlo = '> 1 month ago' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S') - via = '' if pd.isnull(rw.host) else rw.host + via = 'ukmonhelper2' if pd.isnull(rw.host) else rw.host outf.write(f'{lastup:19s} {lastlo:19s} {rw.location:20s} {rw.stationid:10s} {via}\n') diff --git a/archive/ukmon_pylib/metrics/timingMetrics.py b/archive/ukmon_pylib/metrics/timingMetrics.py index d88273d09..cef8b4b0c 100644 --- a/archive/ukmon_pylib/metrics/timingMetrics.py +++ b/archive/ukmon_pylib/metrics/timingMetrics.py @@ -12,6 +12,7 @@ def graphOfData(logf, dtstr): + lis = open(logf,'r').readlines() dta = [li for li in lis if li[:8]==dtstr] @@ -55,78 +56,35 @@ def graphOfData(logf, dtstr): def getLogStats(nightlogf, matchlogf, thisdy): - with open(nightlogf,'r') as inf: - loglines = inf.readlines() - - #logf = os.path.basename(nightlogf) - #spls = logf.split('-') - #thisdy= spls[1] - matchstr = 'RUNTIME' - - # <13>Nov 1 11:49:55 nightlyJob: RUNTIME 10554 showerReport ALL 202211 - startFAM = 0 - startRD = 0 - dts = [] - tss = [] - tsks = [] - secs = [] - msgs = [] - for li in loglines: - if matchstr in li: - li = ' '.join(li.split()) # remove double-spaces eg when date is ' 6 Jan' as opposed to '16 Jan' - spls = li.split() - msg = '' - for s in range(6, len(spls)): - msg = msg + ' ' + spls[s] - if 'start findAllMatches' in msg: - startFAM = int(spls[5]) - dts.append(thisdy) - tss.append(spls[2]) - tsks.append(spls[3].replace(':','')) - secs.append(int(spls[5])) - msgs.append(msg.strip()) - - with open(matchlogf,'r') as inf: - loglines = inf.readlines() + # logline example + # <13>May 8 06:10:02 nightlyJob: start nightlyJob + outdir = os.path.split(nightlogf)[0] - # scan for findAllMatches data first - for li in loglines: - if matchstr in li: - spls = li.split() - if 'runDistrib' in spls[3]: - continue - msg = '' - for s in range(6, len(spls)): - msg = msg + ' ' + spls[s] - if 'start runDistrib' in msg: - startRD = int(spls[5]) - dts.append(thisdy) - tss.append(spls[2]) - tsks.append(spls[3].replace(':','')) - secs.append(int(spls[5])+startFAM) - msgs.append(msg.strip()) + loglines = open(nightlogf,'r').readlines() + bsfs = [x for x in loglines if 'start'in x or 'finish' in x or 'end' in x] + bsfs = [x for x in bsfs if x[0]=='<'] - # rescan for runDistrib data - for li in loglines: - if matchstr in li: - spls = li.split() - if 'runDistrib' not in spls[3]: - continue - msg = '' - for s in range(6, len(spls)): - msg = msg + ' ' + spls[s] - dts.append(thisdy) - tss.append(spls[2]) - tsks.append(spls[3].replace(':','')) - secs.append(int(spls[5]) + startFAM + startRD) - msgs.append(msg.strip()) + loglines = open(matchlogf,'r').readlines() + msfs = [x for x in loglines if 'start'in x or 'finish' in x or 'end' in x] + msfs = [x for x in msfs if x[0]=='<'] - df = pd.DataFrame(zip(dts,tss,secs,tsks,msgs), columns=['dts','tss','secs','tsk','msgs']) - df = df.sort_values(by=['tss']) - #print(df) - outdir = os.path.split(nightlogf)[0] - df.to_csv(os.path.join(outdir, 'perfNightly.csv'), mode='a', header=False, index=False) - + alldata = msfs + bsfs + alldata.sort() + starttime = None + yr = datetime.datetime.now().year + with open(os.path.join(outdir, 'perfNightly.csv'), 'a+') as outf: + for rw in alldata: + dtpart = rw[4:19] + evtdt = datetime.datetime.strptime(f'{yr} {dtpart}', '%Y %b %d %H:%M:%S') + if 'start nightlyJob' in rw: + starttime = evtdt + elapsed_secs = (evtdt - starttime).seconds + txtpart = rw[20:] + task = txtpart[:txtpart.find(':')] + msg = txtpart[txtpart.find(':')+1:].replace(',', ' ') + outstr = f'{evtdt.strftime("%Y%m%d,%H:%M:%S")},{elapsed_secs},{task},{msg}' + outf.write(outstr) + if __name__ == '__main__': dtstr = sys.argv[1] diff --git a/archive/ukmon_pylib/reports/findFailedMatches.py b/archive/ukmon_pylib/reports/findFailedMatches.py index 1b110a1c6..c7c0082d4 100644 --- a/archive/ukmon_pylib/reports/findFailedMatches.py +++ b/archive/ukmon_pylib/reports/findFailedMatches.py @@ -16,24 +16,27 @@ def saveData(observation, repfile): def processOneLog(logfile, repfile): - with open(logfile, 'r') as inf: - lis = inf.readlines() + lis = open(logfile, 'r').readlines() obs = False observation = [] for li in lis: - if 'Observations:' in li: + if 'Observations ' in li: obs = True if obs is True: observation.append(li) - if 'Shower:' in li: + if 'Saving trajectory:' in li or 'Updating database' in li: obs = False observation = [] - if '-----------------------' in li: + if 'Updating database' in li or 'added to fails' in li: if obs is True: + #print('got fail') + observation.append('--------------\n\n') saveData(observation, repfile) obs = False observation = [] + if "SOLVING RUN DONE" in li: + break return @@ -48,5 +51,6 @@ def processOneLog(logfile, repfile): logdir = os.path.join(srcdir, 'logs','distrib') logs = glob.glob(f'{repdt}*.log', root_dir=logdir) for logf in logs: + #print(logf) processOneLog(os.path.join(logdir, logf), reportfile) reportfile.close() diff --git a/archive/ukmon_pylib/reports/getSolutionStati.py b/archive/ukmon_pylib/reports/getSolutionStati.py index 1031688ca..e4b440971 100644 --- a/archive/ukmon_pylib/reports/getSolutionStati.py +++ b/archive/ukmon_pylib/reports/getSolutionStati.py @@ -5,93 +5,14 @@ import sys -def getSolutionStati(fname): - with open(fname, 'r') as inf: - lis = inf.readlines() - # find start of interesting data - totnew = 0 - totchk = 0 - gotnew = 0 - gotchk = 0 - chktraj = False - newtraj = False - for li in lis: - # check for new data for existing solns - if 'Checking trajectory at ' in li and chktraj is False: - thistraj = li - chktraj = True - totchk += 1 - continue - if 'Checking trajectory at ' in li and chktraj is True: - spls = thistraj.strip().split(' ') - print(f'{spls[3]} {spls[4]} no new data') - totchk += 1 - thistraj = li - chktraj = True - continue - if "New Trajectory solution failed" in li and chktraj is True: - spls = thistraj.strip().split(' ') - print(f'{spls[3]} {spls[4]} no improvement') - chktraj = False - continue - if "Shower:" in li and chktraj is True: - spls = thistraj.strip().split(' ') - spl2 = li.strip().split(' ') - try: - if spl2[3] == '...': - shwr = 'Spo' - else: - shwr = spl2[3] - except: - if spl2[1] == '...': - shwr = 'Spo' - else: - shwr = spl2[1] - - print(f'{spls[3]} {spls[4]} updated as {shwr}') - gotchk += 1 - chktraj = False - continue - - # check for new groups - if "Observations:" in li: - thistraj = None - newtraj = True - totnew += 1 - continue - if "Observations:" not in li and newtraj is True and thistraj is None: - thistraj = li - continue - if "-------" in li and newtraj is True: - spls = thistraj.strip().split(' ') - print(f'{spls[3]} {spls[4]} not matched') - newtraj = False - thistraj = None - continue - if "Shower:" in li and newtraj is True: - spls = thistraj.strip().split(' ') - spl2 = li.strip().split(' ') - try: - if spl2[3] == '...': - shwr = 'Spo' - else: - shwr = spl2[3] - except: - if spl2[1] == '...': - shwr = 'Spo' - else: - shwr = spl2[1] - print(f'{spls[3]} {spls[4]} solved as {shwr}') - gotnew += 1 - newtraj = False - thistraj = None - continue - if "SOLVING RUN DONE" in li: - break - - return totnew, totchk, gotnew, gotchk +def parseDistriblog(logname): + lis = open(logname,'r').readlines() + dta = [x for x in lis if ('added to fails' in x or ('saved' in x and 'to' in x)) and ('CorrelateEngine' in x or 'CorrelateRMS' in x)] + offset = 49 if 'match' in logname else 76 + for dd in dta: + print(dd[offset:].strip()) if __name__ == '__main__': fname = sys.argv[1] - print(getSolutionStati(fname)) + parseDistriblog(fname) diff --git a/archive/ukmon_pylib/reports/reportBadCameras.py b/archive/ukmon_pylib/reports/reportBadCameras.py index adf4d64ca..54e8070b2 100644 --- a/archive/ukmon_pylib/reports/reportBadCameras.py +++ b/archive/ukmon_pylib/reports/reportBadCameras.py @@ -14,6 +14,46 @@ mailfrom = 'ukmonhelper2@ukmeteors.co.uk' + +def sendBespokeMessage(targlist, messagefile, subj): + + messagecontent = open(messagefile, 'r').read() + targets = open(targlist, 'r').readlines() + camowners = loadLocationDetails() + camowners['location'] = [ x.lower() + '_' + y.lower() for x,y in zip(camowners.site,camowners.direction)] + camowners = camowners[camowners.active==1] + camowners.drop(columns=['oldcode', 'site', 'camtype', 'active', 'humanName','direction','created'], inplace=True) + #camowners = camowners.groupby('eMail', as_index=False).agg(lambda x: ','.join(x.tolist())) + owners = [] + camids = [] + gmnids = [] + for target in targets: + locationid = target.strip() + if locationid[:1] == '#': + continue + thisrow = camowners[camowners.location.str.contains(locationid)] + if len(thisrow) > 0: + thisowner = thisrow.values[0][0] + if thisowner in owners: + idx = owners.index(thisowner) + gmnids[idx] = f'{gmnids[idx]},{thisrow.values[0][1]}' + camids[idx] = f'{camids[idx]},{thisrow.values[0][2]}' + continue + else: + owners.append(thisowner) + gmnids.append(thisrow.values[0][1]) + camids.append(thisrow.values[0][2]) + df = pd.DataFrame({'owner':owners, 'camids':camids, 'gmnids':gmnids}) + print(df) + for _,rw in df.iterrows(): + mailrecip = rw.owner + statid = rw.gmnids + locationid = rw.camids + print(f'emailing {mailrecip} concerning {statid} {locationid} subject {subj}') + sendAnEmail(mailrecip, messagecontent.format(statid, locationid), subj, mailfrom) + return + + if __name__ == '__main__': if len(sys.argv) < 2: print('usage: python reportBadCameras.py daysback') diff --git a/archive/website/cameraStatusReport.sh b/archive/website/cameraStatusReport.sh index 385fb0353..d76dc602d 100644 --- a/archive/website/cameraStatusReport.sh +++ b/archive/website/cameraStatusReport.sh @@ -36,11 +36,11 @@ else authlog=/var/log/secure batchuser=ec2-user fi -sudo grep publickey $authlog* | grep -v $batchuser > $DATADIR/reports/lastlogins.txt +sudo grep publickey $authlog* | grep -v $batchuser | grep -v fatal > $DATADIR/reports/lastlogins.txt if [ "$platform" == "Ubuntu" ] then - ssh ukmonhelper2 "sudo grep publickey /var/log/secure* | grep -v ec2-user" > $DATADIR/reports/lastlogins_old.txt + ssh ukmonhelper2 "sudo grep publickey /var/log/secure* | egrep -v 'ec2-user|fatal'" > $DATADIR/reports/lastlogins_old.txt fi cat $DATADIR/reports/lastlogins_old.txt >> $DATADIR/reports/lastlogins.txt echo TO HERE diff --git a/bumpver.toml b/bumpver.toml index cc67f3028..f8fc75bba 100644 --- a/bumpver.toml +++ b/bumpver.toml @@ -1,5 +1,5 @@ [bumpver] -current_version = "2026.05.2" +current_version = "2026.05.3" version_pattern = "YYYY.0M.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true