diff --git a/.gitignore b/.gitignore index e897bccc6..035e1394a 100644 --- a/.gitignore +++ b/.gitignore @@ -531,13 +531,13 @@ StandaloneCurator/.installed analysis/orbitSolver/testing analysis/FormatConverters/testing */test_data/* -archive/samfunctions*/pytz* +archive/lambdas*/pytz* archive/static_content/data/searchdialog.js ToDo* archive/costs/**/*.csv archive/costs/**/*.json archive/costs/plots/*.png -archive/samfunctions*/.aws-sam/build.toml +archive/lambdas*/.aws-sam/build.toml archive/costs/S3-size-empireelements.xlsx archive/ukmon_pylib/share/maps/BM*.jpg archive/ukmon_pylib/share/maps/BM*.png @@ -556,8 +556,8 @@ archive/unused/** ukmon_pylib/tests/20220217_032206.832_UK/* unused/* *tfstate.lock.info -archive/samfunctions*/getExtraFilesV2/pythoncode/WesternMeteorPyLib/* -archive/samfunctions*/getExtraFilesForEE/pythoncode/WesternMeteorPyLib/* +archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/* +archive/lambdas/getExtraFilesForEE/pythoncode/WesternMeteorPyLib/* archive/containers/trajsolver/WesternMeteorPyLib/* archive/containers/trajsolver/awskeys archive/containers/trajsolvertest/WesternMeteorPyLib/* @@ -568,11 +568,11 @@ archive/containers/gui/awskeys archive/terraform/*/ukmonreadonly.key .cache_ggshield replacements.txt -archive/samfunctions*/apis-mm.txt -archive/samfunctions*/apis-ee.txt +archive/lambdas/apis-mm.txt +archive/lambdas/apis-ee.txt usermgmt/windows/createUserKeyAWS.ps1 -archive/samfunctions*/matchPickle/pythoncode/WesternMeteorPyLib/** -archive/samfunctions*/matchPickle/pythoncode/*.npy +archive/lambdas/matchPickle/pythoncode/WesternMeteorPyLib/** +archive/lambdas/matchPickle/pythoncode/*.npy .terraform/ archive/utils/tmp.json usermgmt/windows/caminfo/camera-details-new.csv @@ -589,8 +589,8 @@ ukmon_pylib/tests/usertools/2021-07-17T02_41_05_05_M002.ecsv ukmon_pylib/tests/usertools/2021-07-17T02_41_05_05_M004.ecsv ukmon_pylib/tests/usertools/20230202_014115.kml archive/containers*/simpleUI/awskeys -archive/samfunctions*/camDetails/pythoncode/camDetails.py -archive/samfunctions*/camDetails/pythoncode/awskeys +archive/lambdas*/camDetails/pythoncode/camDetails.py +archive/lambdas*/camDetails/pythoncode/awskeys archive/containers/RMS-docker/rms_docker.tar archive/containers/RMS-docker/Dockerfile_orig.txt diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 468f02987..5eb4b5ab8 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -135,7 +135,7 @@ if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then execcons=execconsol.sh execConsolsh=/tmp/$execcons - python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh')" + python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh', $rundate)" chmod +x $execConsolsh log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib @@ -160,7 +160,7 @@ if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then python -m traj.jsonDbMaintenance $DATADIR/distrib/ aws s3 cp $DATADIR/distrib/processed_trajectories.json $UKMONSHAREDBUCKET/matches/distrib/ --quiet - log2cw $NJLOGGRP $NJLOGSTREAM "compressing the procssed data" runDistrib + log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib gzip < $DATADIR/distrib/processed_trajectories.json > $DATADIR/trajdb/processed_trajectories.json.${rundate}.gz aws s3 mv $UKMONSHAREDBUCKET/matches/distrib/${rundate}.pickle $DATADIR/distrib --quiet tar czvf $DATADIR/distrib/${rundate}.tgz $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle diff --git a/archive/samfunctions/README.md b/archive/lambdas/README.md similarity index 100% rename from archive/samfunctions/README.md rename to archive/lambdas/README.md diff --git a/archive/samfunctions/camDetails/camDetails.py b/archive/lambdas/camDetails/camDetails.py similarity index 100% rename from archive/samfunctions/camDetails/camDetails.py rename to archive/lambdas/camDetails/camDetails.py diff --git a/archive/samfunctions/camDetails/samconfig.toml b/archive/lambdas/camDetails/samconfig.toml similarity index 100% rename from archive/samfunctions/camDetails/samconfig.toml rename to archive/lambdas/camDetails/samconfig.toml diff --git a/archive/samfunctions/camDetails/template.yml b/archive/lambdas/camDetails/template.yml similarity index 100% rename from archive/samfunctions/camDetails/template.yml rename to archive/lambdas/camDetails/template.yml diff --git a/archive/samfunctions/fetchECSV/Math.py b/archive/lambdas/fetchECSV/Math.py similarity index 100% rename from archive/samfunctions/fetchECSV/Math.py rename to archive/lambdas/fetchECSV/Math.py diff --git a/archive/samfunctions/fetchECSV/MeteorObservation.py b/archive/lambdas/fetchECSV/MeteorObservation.py similarity index 100% rename from archive/samfunctions/fetchECSV/MeteorObservation.py rename to archive/lambdas/fetchECSV/MeteorObservation.py diff --git a/archive/samfunctions/fetchECSV/fetchECSV.py b/archive/lambdas/fetchECSV/fetchECSV.py similarity index 100% rename from archive/samfunctions/fetchECSV/fetchECSV.py rename to archive/lambdas/fetchECSV/fetchECSV.py diff --git a/archive/samfunctions/fetchECSV/ftpDetectInfo.py b/archive/lambdas/fetchECSV/ftpDetectInfo.py similarity index 100% rename from archive/samfunctions/fetchECSV/ftpDetectInfo.py rename to archive/lambdas/fetchECSV/ftpDetectInfo.py diff --git a/archive/samfunctions/fetchECSV/requirements.txt b/archive/lambdas/fetchECSV/requirements.txt similarity index 100% rename from archive/samfunctions/fetchECSV/requirements.txt rename to archive/lambdas/fetchECSV/requirements.txt diff --git a/archive/samfunctions/fetchECSV/samconfig.toml b/archive/lambdas/fetchECSV/samconfig.toml similarity index 100% rename from archive/samfunctions/fetchECSV/samconfig.toml rename to archive/lambdas/fetchECSV/samconfig.toml diff --git a/archive/samfunctions/fetchECSV/template.yml b/archive/lambdas/fetchECSV/template.yml similarity index 100% rename from archive/samfunctions/fetchECSV/template.yml rename to archive/lambdas/fetchECSV/template.yml diff --git a/archive/samfunctions/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt b/archive/lambdas/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt similarity index 100% rename from archive/samfunctions/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt rename to archive/lambdas/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt diff --git a/archive/samfunctions/fetchECSV/tests/platepars_all_recalibrated.json b/archive/lambdas/fetchECSV/tests/platepars_all_recalibrated.json similarity index 100% rename from archive/samfunctions/fetchECSV/tests/platepars_all_recalibrated.json rename to archive/lambdas/fetchECSV/tests/platepars_all_recalibrated.json diff --git a/archive/samfunctions/fireballApi/getFireballFiles.py b/archive/lambdas/fireballApi/getFireballFiles.py similarity index 100% rename from archive/samfunctions/fireballApi/getFireballFiles.py rename to archive/lambdas/fireballApi/getFireballFiles.py diff --git a/archive/samfunctions/fireballApi/requirements.txt b/archive/lambdas/fireballApi/requirements.txt similarity index 100% rename from archive/samfunctions/fireballApi/requirements.txt rename to archive/lambdas/fireballApi/requirements.txt diff --git a/archive/samfunctions/fireballApi/samconfig.toml b/archive/lambdas/fireballApi/samconfig.toml similarity index 100% rename from archive/samfunctions/fireballApi/samconfig.toml rename to archive/lambdas/fireballApi/samconfig.toml diff --git a/archive/samfunctions/fireballApi/template.yml b/archive/lambdas/fireballApi/template.yml similarity index 100% rename from archive/samfunctions/fireballApi/template.yml rename to archive/lambdas/fireballApi/template.yml diff --git a/archive/samfunctions/ftpToUkmon/.gitignore b/archive/lambdas/ftpToUkmon/.gitignore similarity index 100% rename from archive/samfunctions/ftpToUkmon/.gitignore rename to archive/lambdas/ftpToUkmon/.gitignore diff --git a/archive/samfunctions/ftpToUkmon/README.md b/archive/lambdas/ftpToUkmon/README.md similarity index 100% rename from archive/samfunctions/ftpToUkmon/README.md rename to archive/lambdas/ftpToUkmon/README.md diff --git a/archive/samfunctions/ftpToUkmon/__init__.py b/archive/lambdas/ftpToUkmon/__init__.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/__init__.py rename to archive/lambdas/ftpToUkmon/__init__.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/Dockerfile b/archive/lambdas/ftpToUkmon/pythoncode/Dockerfile similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/Dockerfile rename to archive/lambdas/ftpToUkmon/pythoncode/Dockerfile diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/ShowerAssociation.py b/archive/lambdas/ftpToUkmon/pythoncode/ShowerAssociation.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/ShowerAssociation.py rename to archive/lambdas/ftpToUkmon/pythoncode/ShowerAssociation.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/Showers.py b/archive/lambdas/ftpToUkmon/pythoncode/Showers.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/Showers.py rename to archive/lambdas/ftpToUkmon/pythoncode/Showers.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/__init__.py b/archive/lambdas/ftpToUkmon/pythoncode/__init__.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/__init__.py rename to archive/lambdas/ftpToUkmon/pythoncode/__init__.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/established_showers.csv b/archive/lambdas/ftpToUkmon/pythoncode/established_showers.csv similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/established_showers.csv rename to archive/lambdas/ftpToUkmon/pythoncode/established_showers.csv diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/ftpToUkmon1.py b/archive/lambdas/ftpToUkmon/pythoncode/ftpToUkmon1.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/ftpToUkmon1.py rename to archive/lambdas/ftpToUkmon/pythoncode/ftpToUkmon1.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/requirements.txt b/archive/lambdas/ftpToUkmon/pythoncode/requirements.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/requirements.txt rename to archive/lambdas/ftpToUkmon/pythoncode/requirements.txt diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/supportFuncs.py b/archive/lambdas/ftpToUkmon/pythoncode/supportFuncs.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/supportFuncs.py rename to archive/lambdas/ftpToUkmon/pythoncode/supportFuncs.py diff --git a/archive/samfunctions/ftpToUkmon/samconfig.toml b/archive/lambdas/ftpToUkmon/samconfig.toml similarity index 100% rename from archive/samfunctions/ftpToUkmon/samconfig.toml rename to archive/lambdas/ftpToUkmon/samconfig.toml diff --git a/archive/samfunctions/ftpToUkmon/template.yaml b/archive/lambdas/ftpToUkmon/template.yaml similarity index 100% rename from archive/samfunctions/ftpToUkmon/template.yaml rename to archive/lambdas/ftpToUkmon/template.yaml diff --git a/archive/samfunctions/ftpToUkmon/tests/expected_results.txt b/archive/lambdas/ftpToUkmon/tests/expected_results.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/expected_results.txt rename to archive/lambdas/ftpToUkmon/tests/expected_results.txt diff --git a/archive/samfunctions/ftpToUkmon/tests/localTest.ps1 b/archive/lambdas/ftpToUkmon/tests/localTest.ps1 similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/localTest.ps1 rename to archive/lambdas/ftpToUkmon/tests/localTest.ps1 diff --git a/archive/samfunctions/ftpToUkmon/tests/new_data.txt b/archive/lambdas/ftpToUkmon/tests/new_data.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/new_data.txt rename to archive/lambdas/ftpToUkmon/tests/new_data.txt diff --git a/archive/samfunctions/ftpToUkmon/tests/remoteResult.txt b/archive/lambdas/ftpToUkmon/tests/remoteResult.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/remoteResult.txt rename to archive/lambdas/ftpToUkmon/tests/remoteResult.txt diff --git a/archive/samfunctions/ftpToUkmon/tests/remoteTest.ps1 b/archive/lambdas/ftpToUkmon/tests/remoteTest.ps1 similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/remoteTest.ps1 rename to archive/lambdas/ftpToUkmon/tests/remoteTest.ps1 diff --git a/archive/samfunctions/ftpToUkmon/tests/testEvent.json b/archive/lambdas/ftpToUkmon/tests/testEvent.json similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/testEvent.json rename to archive/lambdas/ftpToUkmon/tests/testEvent.json diff --git a/archive/samfunctions/getApiDetails.ps1 b/archive/lambdas/getApiDetails.ps1 similarity index 100% rename from archive/samfunctions/getApiDetails.ps1 rename to archive/lambdas/getApiDetails.ps1 diff --git a/archive/samfunctions/getExtraFilesV2/.gitignore b/archive/lambdas/getExtraFilesV2/.gitignore similarity index 100% rename from archive/samfunctions/getExtraFilesV2/.gitignore rename to archive/lambdas/getExtraFilesV2/.gitignore diff --git a/archive/samfunctions/getExtraFilesV2/README.md b/archive/lambdas/getExtraFilesV2/README.md similarity index 100% rename from archive/samfunctions/getExtraFilesV2/README.md rename to archive/lambdas/getExtraFilesV2/README.md diff --git a/archive/samfunctions/getExtraFilesV2/__init__.py b/archive/lambdas/getExtraFilesV2/__init__.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/__init__.py rename to archive/lambdas/getExtraFilesV2/__init__.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile b/archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile rename to archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile diff --git a/archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/requirements.txt b/archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/requirements.txt new file mode 100644 index 000000000..81af2bc95 --- /dev/null +++ b/archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/requirements.txt @@ -0,0 +1,22 @@ +numpy +cython +scipy +matplotlib==3.1.2 ; platform_machine != 'aarch64' +matplotlib==3.3.2 ; platform_machine == 'aarch64' +jplephem +pyephem +https://github.com/matplotlib/basemap/archive/master.zip ; sys_platform != 'win32' and platform_machine != 'aarch64' +basemap ; sys_platform == 'win32' or platform_machine == 'aarch64' +PyQt5 ; platform_machine != 'aarch64' +pyyaml +pyswarms +ml-dtypes +keras +pytz +pandas +cartopy +basemap-data-hires +gitpython +numba +watchdog==3.0.0; python_version == '3.7' +watchdog; python_version >= '3.8' \ No newline at end of file diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/__init__.py b/archive/lambdas/getExtraFilesV2/pythoncode/__init__.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/__init__.py rename to archive/lambdas/getExtraFilesV2/pythoncode/__init__.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py b/archive/lambdas/getExtraFilesV2/pythoncode/createOrbitPageIndex.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py rename to archive/lambdas/getExtraFilesV2/pythoncode/createOrbitPageIndex.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py b/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py rename to archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/pickleAnalysis.py b/archive/lambdas/getExtraFilesV2/pythoncode/pickleAnalysis.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/pickleAnalysis.py rename to archive/lambdas/getExtraFilesV2/pythoncode/pickleAnalysis.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/requirements.txt b/archive/lambdas/getExtraFilesV2/pythoncode/requirements.txt similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/requirements.txt rename to archive/lambdas/getExtraFilesV2/pythoncode/requirements.txt diff --git a/archive/samfunctions/getExtraFilesV2/samconfig.toml b/archive/lambdas/getExtraFilesV2/samconfig.toml similarity index 100% rename from archive/samfunctions/getExtraFilesV2/samconfig.toml rename to archive/lambdas/getExtraFilesV2/samconfig.toml diff --git a/archive/samfunctions/getExtraFilesV2/template.yaml b/archive/lambdas/getExtraFilesV2/template.yaml similarity index 100% rename from archive/samfunctions/getExtraFilesV2/template.yaml rename to archive/lambdas/getExtraFilesV2/template.yaml diff --git a/archive/samfunctions/getExtraFilesV2/tests/localTest.ps1 b/archive/lambdas/getExtraFilesV2/tests/localTest.ps1 similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/localTest.ps1 rename to archive/lambdas/getExtraFilesV2/tests/localTest.ps1 diff --git a/archive/samfunctions/getExtraFilesV2/tests/remoteTest.ps1 b/archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/remoteTest.ps1 rename to archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 diff --git a/archive/samfunctions/getExtraFilesV2/tests/templateEvent.json b/archive/lambdas/getExtraFilesV2/tests/templateEvent.json similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/templateEvent.json rename to archive/lambdas/getExtraFilesV2/tests/templateEvent.json diff --git a/archive/samfunctions/getExtraFilesV2/tests/testEvent.json b/archive/lambdas/getExtraFilesV2/tests/testEvent.json similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/testEvent.json rename to archive/lambdas/getExtraFilesV2/tests/testEvent.json diff --git a/archive/samfunctions/getExtraFilesV2/tests/testEvent2.json b/archive/lambdas/getExtraFilesV2/tests/testEvent2.json similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/testEvent2.json rename to archive/lambdas/getExtraFilesV2/tests/testEvent2.json diff --git a/archive/samfunctions/getExtraFilesV2/update_wmpl.sh b/archive/lambdas/getExtraFilesV2/update_wmpl.sh similarity index 100% rename from archive/samfunctions/getExtraFilesV2/update_wmpl.sh rename to archive/lambdas/getExtraFilesV2/update_wmpl.sh diff --git a/archive/samfunctions/getExtraFilesV2/wmpl__init__.py_fixed.py b/archive/lambdas/getExtraFilesV2/wmpl__init__.py_fixed.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/wmpl__init__.py_fixed.py rename to archive/lambdas/getExtraFilesV2/wmpl__init__.py_fixed.py diff --git a/archive/samfunctions/getLiveImages/getLiveImages.py b/archive/lambdas/getLiveImages/getLiveImages.py similarity index 100% rename from archive/samfunctions/getLiveImages/getLiveImages.py rename to archive/lambdas/getLiveImages/getLiveImages.py diff --git a/archive/samfunctions/getLiveImages/requirements.txt b/archive/lambdas/getLiveImages/requirements.txt similarity index 100% rename from archive/samfunctions/getLiveImages/requirements.txt rename to archive/lambdas/getLiveImages/requirements.txt diff --git a/archive/samfunctions/getLiveImages/samconfig.toml b/archive/lambdas/getLiveImages/samconfig.toml similarity index 100% rename from archive/samfunctions/getLiveImages/samconfig.toml rename to archive/lambdas/getLiveImages/samconfig.toml diff --git a/archive/samfunctions/getLiveImages/template.yml b/archive/lambdas/getLiveImages/template.yml similarity index 100% rename from archive/samfunctions/getLiveImages/template.yml rename to archive/lambdas/getLiveImages/template.yml diff --git a/archive/samfunctions/matchDataApi/localTest.ps1 b/archive/lambdas/matchDataApi/localTest.ps1 similarity index 100% rename from archive/samfunctions/matchDataApi/localTest.ps1 rename to archive/lambdas/matchDataApi/localTest.ps1 diff --git a/archive/samfunctions/matchDataApi/matchDataApi.py b/archive/lambdas/matchDataApi/matchDataApi.py similarity index 87% rename from archive/samfunctions/matchDataApi/matchDataApi.py rename to archive/lambdas/matchDataApi/matchDataApi.py index b7c2023ef..da2048d15 100644 --- a/archive/samfunctions/matchDataApi/matchDataApi.py +++ b/archive/lambdas/matchDataApi/matchDataApi.py @@ -9,6 +9,33 @@ import pymysql.cursors +def fileToJsonString(flis): + hdr = ['No','statid','ign','t','jd','m1','m2','az','alt','azl','altl','rao', + 'deco','ral','decl','X','Y','Z','lat','lon','H','range','length','svd', + 'lag','vel','pvel', 'hres','vres','ares','vmag','amag'] + ptsarray='[' + gotpts = False + for fli in flis: + if 'Points' in fli: + gotpts = True + continue + elif '------' in fli or ' No' in fli: + continue + elif gotpts is True and (len(fli) < 2 or 'Notes' in fli): + gotpts=False + break + elif gotpts is True: + spls = fli.split(',') + thisrow = '{' + for h, s in zip(hdr, spls): + thisrow = thisrow + f'"{h}": "{s.strip()}",' + thisrow = thisrow[:-1] + '},' + ptsarray = ptsarray + thisrow + ptsarray = ptsarray[:-1] + ']' + ptsarray = '{' + f'"points": {ptsarray}' + '}' + return ptsarray + + def getSqlLoginDetails(): # retrieve password and host from SSM. This allows me to manage them from Terraform ssm = boto3.client('ssm', region_name='eu-west-1') @@ -22,15 +49,25 @@ def getSqlLoginDetails(): return host, user, password, db -def getStationData(statid, dtstr): +def periodToSqlFragment(period): + if period == 'am': + frag = "and s._h_ut < 12" + elif period == 'pm': + frag = "and s._h_ut >= 12" + elif '-' in period: + sh,eh = period.split('-') + frag = f"and s._h_ut >={sh} and s._h_ut < {eh}" + return frag + + +def getStationData(statid, dtstr, period=None): host, user, passwd, db = getSqlLoginDetails() connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) try: + statfrag = f"and s.stations like '%{statid}%' " if statid is not None else "" + perfrag = periodToSqlFragment(period) if period is not None else "" with connection.cursor() as cursor: - if statid is None: - sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%'" - else: - sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' and s.stations like '%{statid}%'" + sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' {statfrag} {perfrag}" cursor.execute(sql) result = cursor.fetchall() finally: @@ -88,12 +125,10 @@ def lambda_handler(event, context): if qs is None: return { 'statusCode': 200, - 'body': 'usage: detections?reqtyp=xxx&reqval=yyyy[&points=1]' + 'body': 'usage: detections?reqtyp=xxx&reqval=yyyy[&points=1][&period=am|pm|h-h]' } reqtyp = qs['reqtyp'] - points = False - if 'points' in qs: - points = True + points = True if 'points' in qs else False if reqtyp == 'station': statid = qs['statid'] @@ -106,8 +141,9 @@ def lambda_handler(event, context): res = getSummaryData(dtstr) elif reqtyp == 'matches': dtstr = qs['reqval'] + period = qs['period'] if 'period' in qs else None print(f'match data requested for {dtstr}') - res = getStationData(None, dtstr) + res = getStationData(None, dtstr, period) elif reqtyp == 'detail': orbname = qs['reqval'] print(f'detail requested for {orbname}') @@ -140,30 +176,3 @@ def lambda_handler(event, context): 'statusCode': 200, 'body': res } - - -def fileToJsonString(flis): - hdr = ['No','statid','ign','t','jd','m1','m2','az','alt','azl','altl','rao', - 'deco','ral','decl','X','Y','Z','lat','lon','H','range','length','svd', - 'lag','vel','pvel', 'hres','vres','ares','vmag','amag'] - ptsarray='[' - gotpts = False - for fli in flis: - if 'Points' in fli: - gotpts = True - continue - elif '------' in fli or ' No' in fli: - continue - elif gotpts is True and (len(fli) < 2 or 'Notes' in fli): - gotpts=False - break - elif gotpts is True: - spls = fli.split(',') - thisrow = '{' - for h, s in zip(hdr, spls): - thisrow = thisrow + f'"{h}": "{s.strip()}",' - thisrow = thisrow[:-1] + '},' - ptsarray = ptsarray + thisrow - ptsarray = ptsarray[:-1] + ']' - ptsarray = '{' + f'"points": {ptsarray}' + '}' - return ptsarray diff --git a/archive/samfunctions/matchDataApi/requirements.txt b/archive/lambdas/matchDataApi/requirements.txt similarity index 100% rename from archive/samfunctions/matchDataApi/requirements.txt rename to archive/lambdas/matchDataApi/requirements.txt diff --git a/archive/samfunctions/matchDataApi/samconfig.toml b/archive/lambdas/matchDataApi/samconfig.toml similarity index 100% rename from archive/samfunctions/matchDataApi/samconfig.toml rename to archive/lambdas/matchDataApi/samconfig.toml diff --git a/archive/samfunctions/matchDataApi/template.yml b/archive/lambdas/matchDataApi/template.yml similarity index 93% rename from archive/samfunctions/matchDataApi/template.yml rename to archive/lambdas/matchDataApi/template.yml index d07529b3a..83f98f961 100644 --- a/archive/samfunctions/matchDataApi/template.yml +++ b/archive/lambdas/matchDataApi/template.yml @@ -31,6 +31,8 @@ Resources: Required: false - method.request.querystring.statid: Required: false + - method.request.querystring.period: + Required: false Tags: billingtag: "ukmda" typetag: "api" diff --git a/archive/samfunctions/matchDataApi/testDetail.json b/archive/lambdas/matchDataApi/testDetail.json similarity index 100% rename from archive/samfunctions/matchDataApi/testDetail.json rename to archive/lambdas/matchDataApi/testDetail.json diff --git a/archive/samfunctions/matchDataApi/testMatches.json b/archive/lambdas/matchDataApi/testMatches.json similarity index 100% rename from archive/samfunctions/matchDataApi/testMatches.json rename to archive/lambdas/matchDataApi/testMatches.json diff --git a/archive/lambdas/matchDataApi/testMatchesPeriod.json b/archive/lambdas/matchDataApi/testMatchesPeriod.json new file mode 100644 index 000000000..6f61467d2 --- /dev/null +++ b/archive/lambdas/matchDataApi/testMatchesPeriod.json @@ -0,0 +1,8 @@ +{ + "httpMethod": "GET", + "queryStringParameters": { + "reqtyp": "matches", + "reqval": "20251213", + "period": "0-3" + } +} \ No newline at end of file diff --git a/archive/samfunctions/matchDataApi/testPoints.json b/archive/lambdas/matchDataApi/testPoints.json similarity index 100% rename from archive/samfunctions/matchDataApi/testPoints.json rename to archive/lambdas/matchDataApi/testPoints.json diff --git a/archive/samfunctions/matchDataApi/testStation.json b/archive/lambdas/matchDataApi/testStation.json similarity index 100% rename from archive/samfunctions/matchDataApi/testStation.json rename to archive/lambdas/matchDataApi/testStation.json diff --git a/archive/samfunctions/matchDataApi/testSummary.json b/archive/lambdas/matchDataApi/testSummary.json similarity index 100% rename from archive/samfunctions/matchDataApi/testSummary.json rename to archive/lambdas/matchDataApi/testSummary.json diff --git a/archive/samfunctions/matchPickle/__init__.py b/archive/lambdas/matchPickle/__init__.py similarity index 100% rename from archive/samfunctions/matchPickle/__init__.py rename to archive/lambdas/matchPickle/__init__.py diff --git a/archive/samfunctions/matchPickle/localTest.ps1 b/archive/lambdas/matchPickle/localTest.ps1 similarity index 100% rename from archive/samfunctions/matchPickle/localTest.ps1 rename to archive/lambdas/matchPickle/localTest.ps1 diff --git a/archive/samfunctions/matchPickle/pythoncode/Dockerfile b/archive/lambdas/matchPickle/pythoncode/Dockerfile similarity index 100% rename from archive/samfunctions/matchPickle/pythoncode/Dockerfile rename to archive/lambdas/matchPickle/pythoncode/Dockerfile diff --git a/archive/samfunctions/matchPickle/pythoncode/__init__.py b/archive/lambdas/matchPickle/pythoncode/__init__.py similarity index 100% rename from archive/samfunctions/matchPickle/pythoncode/__init__.py rename to archive/lambdas/matchPickle/pythoncode/__init__.py diff --git a/archive/samfunctions/matchPickle/pythoncode/matchPickleApi.py b/archive/lambdas/matchPickle/pythoncode/matchPickleApi.py similarity index 100% rename from archive/samfunctions/matchPickle/pythoncode/matchPickleApi.py rename to archive/lambdas/matchPickle/pythoncode/matchPickleApi.py diff --git a/archive/samfunctions/matchPickle/pythoncode/requirements.txt b/archive/lambdas/matchPickle/pythoncode/requirements.txt similarity index 100% rename from archive/samfunctions/matchPickle/pythoncode/requirements.txt rename to archive/lambdas/matchPickle/pythoncode/requirements.txt diff --git a/archive/samfunctions/matchPickle/samconfig.toml b/archive/lambdas/matchPickle/samconfig.toml similarity index 100% rename from archive/samfunctions/matchPickle/samconfig.toml rename to archive/lambdas/matchPickle/samconfig.toml diff --git a/archive/samfunctions/matchPickle/template.yml b/archive/lambdas/matchPickle/template.yml similarity index 100% rename from archive/samfunctions/matchPickle/template.yml rename to archive/lambdas/matchPickle/template.yml diff --git a/archive/samfunctions/matchPickle/testEvent.json b/archive/lambdas/matchPickle/testEvent.json similarity index 100% rename from archive/samfunctions/matchPickle/testEvent.json rename to archive/lambdas/matchPickle/testEvent.json diff --git a/archive/samfunctions/searchArchive/requirements.txt b/archive/lambdas/searchArchive/requirements.txt similarity index 100% rename from archive/samfunctions/searchArchive/requirements.txt rename to archive/lambdas/searchArchive/requirements.txt diff --git a/archive/samfunctions/searchArchive/samconfig.toml b/archive/lambdas/searchArchive/samconfig.toml similarity index 100% rename from archive/samfunctions/searchArchive/samconfig.toml rename to archive/lambdas/searchArchive/samconfig.toml diff --git a/archive/samfunctions/searchArchive/searchArchive.py b/archive/lambdas/searchArchive/searchArchive.py similarity index 100% rename from archive/samfunctions/searchArchive/searchArchive.py rename to archive/lambdas/searchArchive/searchArchive.py diff --git a/archive/samfunctions/searchArchive/template.yml b/archive/lambdas/searchArchive/template.yml similarity index 100% rename from archive/samfunctions/searchArchive/template.yml rename to archive/lambdas/searchArchive/template.yml diff --git a/archive/samfunctions/searchArchive/testEvent.json b/archive/lambdas/searchArchive/testEvent.json similarity index 100% rename from archive/samfunctions/searchArchive/testEvent.json rename to archive/lambdas/searchArchive/testEvent.json diff --git a/archive/samfunctions/updateDnsRecords/lambda_function.py b/archive/lambdas/updateDnsRecords/lambda_function.py similarity index 100% rename from archive/samfunctions/updateDnsRecords/lambda_function.py rename to archive/lambdas/updateDnsRecords/lambda_function.py diff --git a/archive/samfunctions/updateDnsRecords/samconfig.toml b/archive/lambdas/updateDnsRecords/samconfig.toml similarity index 100% rename from archive/samfunctions/updateDnsRecords/samconfig.toml rename to archive/lambdas/updateDnsRecords/samconfig.toml diff --git a/archive/samfunctions/updateDnsRecords/template.yaml b/archive/lambdas/updateDnsRecords/template.yaml similarity index 100% rename from archive/samfunctions/updateDnsRecords/template.yaml rename to archive/lambdas/updateDnsRecords/template.yaml diff --git a/archive/terraform/mjmm/iam.tf b/archive/terraform/mjmm/iam.tf index 766e2d668..10e44de3a 100644 --- a/archive/terraform/mjmm/iam.tf +++ b/archive/terraform/mjmm/iam.tf @@ -198,7 +198,7 @@ resource "aws_iam_role_policy" "stsAssumeLambda" { Statement = { Action = "sts:AssumeRole" Effect = "Allow" - Resource = "arn:aws:iam::822069317839:role/service-role/S3FullAccess" + Resource = "arn:aws:iam::183798037734:policy/s3PolicyForRadio" } Version = "2012-10-17" } diff --git a/archive/terraform/ukmda/ec2.tf b/archive/terraform/ukmda/ec2.tf index 29c5fd71c..af11b8276 100644 --- a/archive/terraform/ukmda/ec2.tf +++ b/archive/terraform/ukmda/ec2.tf @@ -17,7 +17,7 @@ resource "aws_instance" "calc_server" { "Name" = "calcengine" "billingtag" = "ukmda" } - volume_size = 100 + volume_size = 120 } primary_network_interface { network_interface_id = aws_network_interface.calcserver_if.id @@ -58,7 +58,7 @@ resource "aws_instance" "ubuntu_calc_server" { } root_block_device { tags = { - "Name" = "calcengine2" + "Name" = "calcengine_ub" "billingtag" = "ukmda" } volume_size = 100 diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index 97555ca96..8ca5aec51 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -124,12 +124,15 @@ def gatherUsedImageList(outf, matchstart, matchend, shbucket): return -def createExecConsolSh(matchstart, matchend, execconsolsh): +def createExecConsolSh(matchstart, matchend, execconsolsh, rundt): shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') webbucket = os.getenv('WEBSITEBUCKET', default='s3://ukmda-website') calcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' # hardcoded! _, outpath, _ = getTrajsolverPaths() enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend) + includeyear = False + if enddt.day == 30: + includeyear = True with open(execconsolsh, 'w') as outf: outf.write('#!/bin/bash\n') @@ -139,12 +142,12 @@ def createExecConsolSh(matchstart, matchend, execconsolsh): outf.write(f'cd {calcdir}\n') outf.write('logger -s -t execConsol start\n') - outf.write('python -m traj.consolidateDistTraj ~/data/distrib/ ~/data/distrib/processed_trajectories.json\n') + outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/ ~/data/distrib/processed_trajectories.json {rundt}\n') outf.write('logger -s -t execConsol syncing any updated trajectories from shared S3\n') refreshTrajectories(outf, matchstart, matchend, outpath) outf.write('logger -s -t execConsol creating density plots\n') - createDensityPlots(outf, calcdir, enddt) + createDensityPlots(outf, calcdir, enddt, includeyear) outf.write('logger -s -t execConsol pushing data back to S3\n') pushUpdatedTrajectoriesShared(outf, matchstart, matchend, shbucket) pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webbucket) diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index 3030c9b08..91f185f41 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -101,7 +101,7 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): s3 = boto3.client('s3') for i in range(0, numbucks): - bucknames[i] = buckroot + f'_{i:02d}' + bucknames[i] = buckroot + f'_{i:03d}' filelist = flist[i::numbucks] for fli in filelist: src = os.path.join(srcdir, fli) diff --git a/archive/ukmon_pylib/traj/jsonDbMaintenance.py b/archive/ukmon_pylib/traj/jsonDbMaintenance.py index e5aa984c3..140d9a51e 100644 --- a/archive/ukmon_pylib/traj/jsonDbMaintenance.py +++ b/archive/ukmon_pylib/traj/jsonDbMaintenance.py @@ -33,12 +33,12 @@ def __init__(self, station, obs_id): for traj in [t for t in db.trajectories if t < archdate_jd]: if traj < archdate_jd: archdb.addTrajectory(None, db.trajectories[traj], False) - db.removeTrajectory(db.trajectories[traj]) + del db.trajectories[traj] for traj in [t for t in db.failed_trajectories if t < archdate_jd]: if traj < archdate_jd: archdb.addTrajectory(None, db.failed_trajectories[traj], True) - db.removeTrajectory(db.failed_trajectories[traj]) + del db.failed_trajectories[traj] for station in db.processed_dirs: arch_processed = [dirname for dirname in db.processed_dirs[station] if @@ -55,6 +55,47 @@ def __init__(self, station, obs_id): db.paired_obs[station].remove(obs_id) archdb.save() + db.save() + return db + + +def clearDownHistoricArchive(database_path, mthsback=1): + db = DatabaseJSON(database_path) + + if len(list(db.failed_trajectories.keys()))==0 and len(list(db.trajectories.keys()))==0: + print('nothing to do') + return + elif len(list(db.failed_trajectories.keys()))==0: + latest = jd2Date(max(list(db.trajectories.keys())), dt_obj=True) + elif len(list(db.trajectories.keys()))==0: + latest = jd2Date(max(list(db.failed_trajectories.keys())), dt_obj=True) + else: + latest = jd2Date(max(max(list(db.trajectories.keys())), max(list(db.failed_trajectories.keys()))), dt_obj=True) + print('processing', database_path) + archdate = latest - relativedelta(months=mthsback) + archdate = archdate.replace(day=1, hour=12, minute=0, second=0).replace(tzinfo=datetime.timezone.utc) + archdate_jd = datetime2JD(archdate) + + for traj in [t for t in db.trajectories if t < archdate_jd]: + if traj < archdate_jd: + del db.trajectories[traj] + + for traj in [t for t in db.failed_trajectories if t < archdate_jd]: + if traj < archdate_jd: + del db.failed_trajectories[traj] + + for station in db.processed_dirs: + arch_processed = [dirname for dirname in db.processed_dirs[station] if + datetime.datetime.strptime(dirname[14:22], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] + for dirname in arch_processed: + db.processed_dirs[station].remove(dirname) + + for station in db.paired_obs: + arch_processed = [obs_id for obs_id in db.paired_obs[station] if + datetime.datetime.strptime(obs_id[7:15], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] + for obs_id in arch_processed: + db.paired_obs[station].remove(obs_id) + db.save() return @@ -64,9 +105,8 @@ def __init__(self, station, obs_id): database_path = os.path.join(db_dir, JSON_DB_NAME) db = DatabaseJSON(database_path) soonest = jd2Date(min(min(list(db.trajectories.keys())), min(list(db.failed_trajectories.keys()))), dt_obj=True) - nowdt = datetime.datetime.now() mthsback = int((nowdt - soonest).days/30) for i in range(mthsback, 2, -1): print(f'archiving {i} months back') - archiveOldRecords(db, db_dir, i) + db = archiveOldRecords(db, db_dir, i)