ukmda · markmac99 · Jan 8, 2026 · Dec 14, 2025 · Dec 14, 2025 · Dec 14, 2025
@@ -531,13 +531,13 @@ StandaloneCurator/.installed
 analysis/orbitSolver/testing
 analysis/FormatConverters/testing
 */test_data/*
-archive/samfunctions*/pytz*
+archive/lambdas*/pytz*
 archive/static_content/data/searchdialog.js
 ToDo*
 archive/costs/**/*.csv
 archive/costs/**/*.json
 archive/costs/plots/*.png
-archive/samfunctions*/.aws-sam/build.toml
+archive/lambdas*/.aws-sam/build.toml
 archive/costs/S3-size-empireelements.xlsx
 archive/ukmon_pylib/share/maps/BM*.jpg
 archive/ukmon_pylib/share/maps/BM*.png
@@ -556,8 +556,8 @@ archive/unused/**
 ukmon_pylib/tests/20220217_032206.832_UK/*
 unused/*
 *tfstate.lock.info
-archive/samfunctions*/getExtraFilesV2/pythoncode/WesternMeteorPyLib/*
-archive/samfunctions*/getExtraFilesForEE/pythoncode/WesternMeteorPyLib/*
+archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/*
+archive/lambdas/getExtraFilesForEE/pythoncode/WesternMeteorPyLib/*
 archive/containers/trajsolver/WesternMeteorPyLib/*
 archive/containers/trajsolver/awskeys
 archive/containers/trajsolvertest/WesternMeteorPyLib/*
@@ -568,11 +568,11 @@ archive/containers/gui/awskeys
 archive/terraform/*/ukmonreadonly.key
 .cache_ggshield
 replacements.txt
-archive/samfunctions*/apis-mm.txt
-archive/samfunctions*/apis-ee.txt
+archive/lambdas/apis-mm.txt
+archive/lambdas/apis-ee.txt
 usermgmt/windows/createUserKeyAWS.ps1
-archive/samfunctions*/matchPickle/pythoncode/WesternMeteorPyLib/**
-archive/samfunctions*/matchPickle/pythoncode/*.npy
+archive/lambdas/matchPickle/pythoncode/WesternMeteorPyLib/**
+archive/lambdas/matchPickle/pythoncode/*.npy
 .terraform/
 archive/utils/tmp.json
 usermgmt/windows/caminfo/camera-details-new.csv
@@ -589,8 +589,8 @@ ukmon_pylib/tests/usertools/2021-07-17T02_41_05_05_M002.ecsv
 ukmon_pylib/tests/usertools/2021-07-17T02_41_05_05_M004.ecsv
 ukmon_pylib/tests/usertools/20230202_014115.kml
 archive/containers*/simpleUI/awskeys
-archive/samfunctions*/camDetails/pythoncode/camDetails.py
-archive/samfunctions*/camDetails/pythoncode/awskeys
+archive/lambdas*/camDetails/pythoncode/camDetails.py
+archive/lambdas*/camDetails/pythoncode/awskeys
 archive/containers/RMS-docker/rms_docker.tar
 archive/containers/RMS-docker/Dockerfile_orig.txt
 

@@ -135,7 +135,7 @@ if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then
 
     execcons=execconsol.sh
     execConsolsh=/tmp/$execcons
-    python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh')"
+    python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh', $rundate)"
     chmod +x $execConsolsh
 
     log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib
@@ -160,7 +160,7 @@ if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then
     python -m traj.jsonDbMaintenance $DATADIR/distrib/
     aws s3 cp $DATADIR/distrib/processed_trajectories.json $UKMONSHAREDBUCKET/matches/distrib/ --quiet
 
-    log2cw $NJLOGGRP $NJLOGSTREAM "compressing the procssed data" runDistrib
+    log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib
     gzip < $DATADIR/distrib/processed_trajectories.json > $DATADIR/trajdb/processed_trajectories.json.${rundate}.gz
     aws s3 mv $UKMONSHAREDBUCKET/matches/distrib/${rundate}.pickle $DATADIR/distrib --quiet
     tar czvf $DATADIR/distrib/${rundate}.tgz $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle

@@ -0,0 +1,22 @@
+numpy
+cython
+scipy
+matplotlib==3.1.2 ; platform_machine != 'aarch64'
+matplotlib==3.3.2 ; platform_machine == 'aarch64'
+jplephem
+pyephem
+https://github.com/matplotlib/basemap/archive/master.zip ; sys_platform != 'win32' and platform_machine != 'aarch64'
+basemap ; sys_platform == 'win32' or platform_machine == 'aarch64'
+PyQt5 ; platform_machine != 'aarch64'
+pyyaml
+pyswarms
+ml-dtypes
+keras
+pytz
+pandas
+cartopy
+basemap-data-hires
+gitpython
+numba
+watchdog==3.0.0; python_version == '3.7'
+watchdog; python_version >= '3.8'
@@ -9,6 +9,33 @@
 import pymysql.cursors
 
 
+def fileToJsonString(flis):
+    hdr = ['No','statid','ign','t','jd','m1','m2','az','alt','azl','altl','rao', 
+           'deco','ral','decl','X','Y','Z','lat','lon','H','range','length','svd',
+           'lag','vel','pvel', 'hres','vres','ares','vmag','amag']
+    ptsarray='['
+    gotpts = False
+    for fli in flis:
+        if 'Points' in fli:
+            gotpts = True
+            continue
+        elif '------' in fli or ' No' in fli:
+            continue
+        elif gotpts is True and (len(fli) < 2 or 'Notes' in fli):
+            gotpts=False
+            break
+        elif gotpts is True:
+            spls = fli.split(',')
+            thisrow = '{'
+            for h, s in zip(hdr, spls):
+                thisrow = thisrow + f'"{h}": "{s.strip()}",'
+            thisrow = thisrow[:-1] + '},'
+            ptsarray = ptsarray + thisrow
+    ptsarray = ptsarray[:-1] + ']'
+    ptsarray = '{' + f'"points": {ptsarray}' + '}'
+    return ptsarray
+
+
 def getSqlLoginDetails():
     # retrieve password and host from SSM. This allows me to manage them from Terraform
     ssm = boto3.client('ssm', region_name='eu-west-1')
@@ -22,15 +49,25 @@ def getSqlLoginDetails():
     return host, user, password, db
 
 
-def getStationData(statid, dtstr):
+def periodToSqlFragment(period):
+    if period == 'am':
+        frag = "and s._h_ut < 12"
+    elif period == 'pm':
+        frag = "and s._h_ut >= 12"
+    elif '-' in period:
+        sh,eh = period.split('-')
+        frag = f"and s._h_ut >={sh} and s._h_ut < {eh}"
+    return frag
+
+
+def getStationData(statid, dtstr, period=None):
     host, user, passwd, db = getSqlLoginDetails()
     connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor)  
     try:
+        statfrag = f"and s.stations like '%{statid}%' " if statid is not None else ""
+        perfrag = periodToSqlFragment(period) if period is not None else ""
         with connection.cursor() as cursor:
-            if statid is None:
-                sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%'"
-            else:
-                sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' and s.stations like '%{statid}%'"
+            sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' {statfrag} {perfrag}"
             cursor.execute(sql)
             result = cursor.fetchall()
     finally:
@@ -88,12 +125,10 @@ def lambda_handler(event, context):
     if qs is None:
         return {
             'statusCode': 200,
-            'body': 'usage: detections?reqtyp=xxx&reqval=yyyy[&points=1]'
+            'body': 'usage: detections?reqtyp=xxx&reqval=yyyy[&points=1][&period=am|pm|h-h]'
         }
     reqtyp = qs['reqtyp']
-    points = False
-    if 'points' in qs:
-        points = True
+    points = True if 'points' in qs else False
 
     if reqtyp == 'station':
         statid = qs['statid']
@@ -106,8 +141,9 @@ def lambda_handler(event, context):
         res = getSummaryData(dtstr)
     elif reqtyp == 'matches':
         dtstr = qs['reqval']
+        period = qs['period'] if 'period' in qs else None
         print(f'match data requested for {dtstr}')
-        res = getStationData(None, dtstr)
+        res = getStationData(None, dtstr, period)
     elif reqtyp == 'detail':
         orbname = qs['reqval']
         print(f'detail requested for {orbname}')
@@ -140,30 +176,3 @@ def lambda_handler(event, context):
         'statusCode': 200,
         'body': res
     }
-
-
-def fileToJsonString(flis):
-    hdr = ['No','statid','ign','t','jd','m1','m2','az','alt','azl','altl','rao', 
-           'deco','ral','decl','X','Y','Z','lat','lon','H','range','length','svd',
-           'lag','vel','pvel', 'hres','vres','ares','vmag','amag']
-    ptsarray='['
-    gotpts = False
-    for fli in flis:
-        if 'Points' in fli:
-            gotpts = True
-            continue
-        elif '------' in fli or ' No' in fli:
-            continue
-        elif gotpts is True and (len(fli) < 2 or 'Notes' in fli):
-            gotpts=False
-            break
-        elif gotpts is True:
-            spls = fli.split(',')
-            thisrow = '{'
-            for h, s in zip(hdr, spls):
-                thisrow = thisrow + f'"{h}": "{s.strip()}",'
-            thisrow = thisrow[:-1] + '},'
-            ptsarray = ptsarray + thisrow
-    ptsarray = ptsarray[:-1] + ']'
-    ptsarray = '{' + f'"points": {ptsarray}' + '}'
-    return ptsarray
@@ -31,6 +31,8 @@ Resources:
                   Required: false
               - method.request.querystring.statid: 
                   Required: false
+              - method.request.querystring.period: 
+                  Required: false
       Tags:
         billingtag: "ukmda"
         typetag: "api"

@@ -0,0 +1,8 @@
+{
+  "httpMethod": "GET",
+  "queryStringParameters": {
+    "reqtyp": "matches",
+    "reqval": "20251213",
+    "period": "0-3"
+  }
+}
@@ -198,7 +198,7 @@ resource "aws_iam_role_policy" "stsAssumeLambda" {
       Statement = {
         Action   = "sts:AssumeRole"
         Effect   = "Allow"
-        Resource = "arn:aws:iam::822069317839:role/service-role/S3FullAccess"
+        Resource = "arn:aws:iam::183798037734:policy/s3PolicyForRadio"
       }
       Version = "2012-10-17"
     }

@@ -17,7 +17,7 @@ resource "aws_instance" "calc_server" {
       "Name"       = "calcengine"
       "billingtag" = "ukmda"
     }
-    volume_size = 100
+    volume_size = 120
   }
   primary_network_interface {
     network_interface_id = aws_network_interface.calcserver_if.id
@@ -58,7 +58,7 @@ resource "aws_instance" "ubuntu_calc_server" {
   }
   root_block_device {
     tags = {
-      "Name"       = "calcengine2"
+      "Name"       = "calcengine_ub"
       "billingtag" = "ukmda"
     }
     volume_size = 100

@@ -124,12 +124,15 @@ def gatherUsedImageList(outf, matchstart, matchend, shbucket):
     return 
 
 
-def createExecConsolSh(matchstart, matchend, execconsolsh):
+def createExecConsolSh(matchstart, matchend, execconsolsh, rundt):
     shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')
     webbucket = os.getenv('WEBSITEBUCKET', default='s3://ukmda-website')
     calcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' # hardcoded!
     _, outpath, _ = getTrajsolverPaths()
     enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend)
+    includeyear = False
+    if enddt.day == 30:
+        includeyear = True
 
     with open(execconsolsh, 'w') as outf:
         outf.write('#!/bin/bash\n')
@@ -139,12 +142,12 @@ def createExecConsolSh(matchstart, matchend, execconsolsh):
         outf.write(f'cd {calcdir}\n')
         outf.write('logger -s -t execConsol start\n')
 
-        outf.write('python -m traj.consolidateDistTraj ~/data/distrib/ ~/data/distrib/processed_trajectories.json\n')
+        outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/ ~/data/distrib/processed_trajectories.json {rundt}\n')
 
         outf.write('logger -s -t execConsol syncing any updated trajectories from shared S3\n')
         refreshTrajectories(outf, matchstart, matchend, outpath)
         outf.write('logger -s -t execConsol creating density plots\n')
-        createDensityPlots(outf, calcdir, enddt)
+        createDensityPlots(outf, calcdir, enddt, includeyear)
         outf.write('logger -s -t execConsol pushing data back to S3\n')
         pushUpdatedTrajectoriesShared(outf, matchstart, matchend, shbucket)
         pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webbucket)

@@ -101,7 +101,7 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20):
 
     s3 = boto3.client('s3')
     for i in range(0, numbucks):
-        bucknames[i] = buckroot + f'_{i:02d}'
+        bucknames[i] = buckroot + f'_{i:03d}'
         filelist = flist[i::numbucks]
         for fli in filelist:
             src = os.path.join(srcdir, fli)

@@ -33,12 +33,12 @@ def __init__(self, station, obs_id):
     for traj in [t for t in db.trajectories if t < archdate_jd]:
         if traj < archdate_jd:
             archdb.addTrajectory(None, db.trajectories[traj], False)
-            db.removeTrajectory(db.trajectories[traj])
+            del db.trajectories[traj]
 
     for traj in [t for t in db.failed_trajectories if t < archdate_jd]:
         if traj < archdate_jd:
             archdb.addTrajectory(None, db.failed_trajectories[traj], True)
-            db.removeTrajectory(db.failed_trajectories[traj])
+            del db.failed_trajectories[traj]
 
     for station in db.processed_dirs:
         arch_processed = [dirname for dirname in db.processed_dirs[station] if 
@@ -55,6 +55,47 @@ def __init__(self, station, obs_id):
             db.paired_obs[station].remove(obs_id)
 
     archdb.save()
+    db.save()
+    return db
+
+
+def clearDownHistoricArchive(database_path, mthsback=1):
+    db = DatabaseJSON(database_path)
+
+    if len(list(db.failed_trajectories.keys()))==0 and len(list(db.trajectories.keys()))==0:
+        print('nothing to do')
+        return
+    elif len(list(db.failed_trajectories.keys()))==0:
+        latest = jd2Date(max(list(db.trajectories.keys())), dt_obj=True)
+    elif len(list(db.trajectories.keys()))==0:
+        latest = jd2Date(max(list(db.failed_trajectories.keys())), dt_obj=True)
+    else:
+        latest = jd2Date(max(max(list(db.trajectories.keys())), max(list(db.failed_trajectories.keys()))), dt_obj=True)
+    print('processing', database_path)
+    archdate = latest - relativedelta(months=mthsback)
+    archdate = archdate.replace(day=1, hour=12, minute=0, second=0).replace(tzinfo=datetime.timezone.utc)
+    archdate_jd = datetime2JD(archdate)
+
+    for traj in [t for t in db.trajectories if t < archdate_jd]:
+        if traj < archdate_jd:
+            del db.trajectories[traj]
+
+    for traj in [t for t in db.failed_trajectories if t < archdate_jd]:
+        if traj < archdate_jd:
+            del db.failed_trajectories[traj]
+
+    for station in db.processed_dirs:
+        arch_processed = [dirname for dirname in db.processed_dirs[station] if 
+                            datetime.datetime.strptime(dirname[14:22], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate]
+        for dirname in arch_processed:
+            db.processed_dirs[station].remove(dirname)
+
+    for station in db.paired_obs:
+        arch_processed = [obs_id for obs_id in db.paired_obs[station] if 
+                            datetime.datetime.strptime(obs_id[7:15], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate]
+        for obs_id in arch_processed:
+            db.paired_obs[station].remove(obs_id)
+
     db.save()
     return 
 
@@ -64,9 +105,8 @@ def __init__(self, station, obs_id):
     database_path = os.path.join(db_dir, JSON_DB_NAME)
     db = DatabaseJSON(database_path)
     soonest = jd2Date(min(min(list(db.trajectories.keys())), min(list(db.failed_trajectories.keys()))), dt_obj=True)
-
     nowdt = datetime.datetime.now()
     mthsback = int((nowdt - soonest).days/30)
     for i in range(mthsback, 2, -1):
         print(f'archiving {i} months back')
-        archiveOldRecords(db, db_dir, i)
+        db = archiveOldRecords(db, db_dir, i)