October updates

.github/workflows/automated-testing.yml

@@ -10,7 +10,7 @@ on:
     branches: [dev, markmac99]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [master]
+    branches: [dev, markmac99]
 jobs:
   conttests:
     runs-on: thelinux
@@ -32,17 +32,27 @@ jobs:
           aws-region: ${{ env.AWS_REGION }}
       - name: remote tests
         run: |
-          chmod +x /tests.sh
           if [ ! -f /data/admin/cameraLocs.json ] ; then 
             echo "getting test data"
             curl https://archive.ukmeteors.co.uk/browse/testdata/testdata.tar.gz  -o /data/testdata.tar.gz
             cd /data && tar -xzf ./testdata.tar.gz
           else
             echo "test data available"
           fi 
-          mkdir -p ~/.aws
-          cp /data/profile/credentials ~/.aws/
-          /tests.sh
+          aws sts get-caller-identity 
+          pip install --upgrade MeteorTools | grep -v 'already satisfied'
+          pip install pytest pytest-cov | grep -v 'already satisfied'
+          [ -d ./ukmda-dataprocessing ] && rm -Rf ./ukmda-dataprocessing
+          git clone https://github.com/ukmda/ukmda-dataprocessing.git
+          cd ./ukmda-dataprocessing/
+          git checkout $BRANCH
+          cd ./archive/ukmon_pylib/
+          curdir=$(pwd -P)
+          export DATADIR=/data
+          echo DATADIR is $DATADIR curdir is $curdir
+          export PYTHONPATH=/WesternMeteorPyLib:/RMS:${curdir}
+          pytest -v ./tests --cov=. --cov-report=term-missing
+
   apitests:
     runs-on: ubuntu-latest
     steps:

.github/workflows/build_tools.yml

@@ -7,7 +7,7 @@ permissions:
 
 on:
   push:
-    branches: [dev, markmac99, master]
+    branches: [master]
   pull_request:
     # The branches below must be a subset of the branches above
     branches: [master]

archive/analysis/consolidateOutput.sh

@@ -37,23 +37,29 @@ consdir=${DATADIR}/consolidated/temp
 mkdir -p ${DATADIR}/single/rawcsvs 
 ls -1 $consdir/*.csv | while read csvf
 do
-    bn=$(basename $csvf)
-    typ=${bn:0:3}
-    if [ "$typ" != "M20" ] ; then 
-        pref="P"
-        yr=${bn:7:4}
+    flen=$(wc -l $csvf | awk '{print $1}')
+    if [ $flen -lt 2 ] ; then 
+        logger -s -t consolidateOutput "skipping empty file $csvf"
+        rm $csvf 
     else
-        pref="M"
-        yr=${bn:1:4}
-    fi 
-    mrgfile=${DATADIR}/consolidated/${pref}_${yr}-unified.csv
-    if [ ! -f $mrgfile ] ; then
-        cat $csvf >> $mrgfile
-    else
-        #echo $bn $mrgfile
-        sed '1d' $csvf >> $mrgfile
+        bn=$(basename $csvf)
+        typ=${bn:0:3}
+        if [ "$typ" != "M20" ] ; then 
+            pref="P"
+            yr=${bn:7:4}
+        else
+            pref="M"
+            yr=${bn:1:4}
+        fi 
+        mrgfile=${DATADIR}/consolidated/${pref}_${yr}-unified.csv
+        if [ ! -f $mrgfile ] ; then
+            cat $csvf >> $mrgfile
+        else
+            #echo $bn $mrgfile
+            sed '1d' $csvf >> $mrgfile
+        fi
+        mv $csvf ${DATADIR}/single/rawcsvs
     fi
-    mv $csvf ${DATADIR}/single/rawcsvs
 done
 
 logger -s -t consolidateOutput "pushing consolidated information back"

archive/cronjobs/deleteDuplicateOrbit.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+source $here/../config.ini >/dev/null 2>&1
+conda activate $HOME/miniconda3/envs/${WMPL_ENV}
+
+chkrunning=.delrunning
+mkdir -p $DATADIR/manualuploads
+cd $DATADIR/manualuploads
+if [[ -f ./$chkrunning || -f ./running ]] ; then
+   echo "update already running"
+   exit
+fi
+touch ./$chkrunning
+aws s3 sync $UKMONSHAREDBUCKET/fireballs/uploads . --exclude "*" --include "*.delete" --exclude "*.done"
+
+newfiles=$(ls -1 *.delete 2> /dev/null)
+if [ "$newfiles" != "" ] ; then 
+    for fil in $newfiles ; do
+        orbname=$(cat $fil)
+        echo  $(date +%Y-%m-%dT%H:%M:%SZ) removing $orbname
+        $SRC/utils/deleteOrbit.sh $orbname
+        aws s3 mv $UKMONSHAREDBUCKET/fireballs/uploads/$fil $UKMONSHAREDBUCKET/fireballs/uploads/processed/$fil.done
+        rm $fil
+    done
+fi
+rm $DATADIR/manualuploads/$chkrunning
+find $SRC/logs -name "deleteDuplicateOrbit*" -mtime +10 -exec rm -f {} \;

archive/cronjobs/mergeNewOrbit.sh

@@ -5,7 +5,8 @@ source $here/../config.ini >/dev/null 2>&1
 conda activate $HOME/miniconda3/envs/${WMPL_ENV}
 
 mkdir -p $DATADIR/manualuploads
-cd $DATADIR/manualuploadsif [ -f ./running ] ; then
+cd $DATADIR/manualuploads
+if [ -f ./running ] ; then
    echo "already running"
    exit
 fi
@@ -62,4 +63,4 @@ EOD
 #    echo nothing to process
 fi
 rm $DATADIR/manualuploads/.running
-find $SRC/logs -name "mergeNewOrbit*" -mtime +10 -exec rm -f {} \;
+find $SRC/logs -name "mergeNewOrbit*" -mtime +10 -exec rm -f {} \;

archive/deployment/cronjobs.yml

@@ -3,7 +3,7 @@
   vars_files: 
     - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc
   vars:
-    srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive"
+    srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/Dev/meteorhunting/ukmda-dataprocessing/archive"
   tasks:
   - name: import dev variables 
     include_vars: ./dev-vars.yml
@@ -21,6 +21,7 @@
         - {src: '{{srcdir}}/cronjobs/nightlyJob.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no }
         - {src: '{{srcdir}}/cronjobs/captureBright.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no }
         - {src: '{{srcdir}}/cronjobs/getImoWSfile.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no }
-        - {src: '{{srcdir}}//cronjobs/mergeNewOrbit.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no }
-        - {src: '{{srcdir}}//cronjobs/gatherMonthlyVideos.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no }
+        - {src: '{{srcdir}}/cronjobs/mergeNewOrbit.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no }
+        - {src: '{{srcdir}}/cronjobs/deleteDuplicateOrbit.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no }
+        - {src: '{{srcdir}}/cronjobs/gatherMonthlyVideos.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no }
         - {src: '{{srcdir}}/cronjobs/README.md', dest: '{{destdir}}/cronjobs', mode: '644', backup: no }