Merge pull request #415 from ukmda/master

Merge in changes from Master

Author: markmac99

.github/workflows/automated-testing.yml

@@ -1,4 +1,4 @@
-name: automated_testing
+name: Automated Testing
 env:
   AWS_REGION: eu-west-2
 permissions:
@@ -7,10 +7,10 @@ permissions:
 
 on:
   push:
-    branches: [ dev, markmac99 ]
+    branches: [dev, markmac99]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ master ]
+    branches: [dev, markmac99]
 jobs:
   conttests:
     runs-on: thelinux
@@ -20,7 +20,7 @@ jobs:
         username: ${{ secrets.DH_LOGIN_MM }}
         password: ${{ secrets.DH_PATT_MM }}
       env:
-        BRANCH: ${{ github.head_ref || github.ref_name }} 
+        BRANCH: ${{ github.head_ref || github.ref_name }}
       volumes:
         - testdata:/data
     steps:
@@ -32,21 +32,31 @@ jobs:
           aws-region: ${{ env.AWS_REGION }}
       - name: remote tests
         run: |
-          chmod +x /tests.sh
           if [ ! -f /data/admin/cameraLocs.json ] ; then 
             echo "getting test data"
             curl https://archive.ukmeteors.co.uk/browse/testdata/testdata.tar.gz  -o /data/testdata.tar.gz
             cd /data && tar -xzf ./testdata.tar.gz
           else
             echo "test data available"
           fi 
-          mkdir -p ~/.aws
-          cp /data/profile/credentials ~/.aws/
-          /tests.sh
+          aws sts get-caller-identity 
+          pip install --upgrade MeteorTools | grep -v 'already satisfied'
+          pip install pytest pytest-cov | grep -v 'already satisfied'
+          [ -d ./ukmda-dataprocessing ] && rm -Rf ./ukmda-dataprocessing
+          git clone https://github.com/ukmda/ukmda-dataprocessing.git
+          cd ./ukmda-dataprocessing/
+          git checkout $BRANCH
+          cd ./archive/ukmon_pylib/
+          curdir=$(pwd -P)
+          export DATADIR=/data
+          echo DATADIR is $DATADIR curdir is $curdir
+          export PYTHONPATH=/WesternMeteorPyLib:/RMS:${curdir}
+          pytest -v ./tests --cov=. --cov-report=term-missing
+
   apitests:
     runs-on: ubuntu-latest
     steps:
-      - name: checkout code 
+      - name: checkout code
         uses: actions/checkout@v3
       - name: run api tests
         run: |

.github/workflows/build_usermgmt.yml

@@ -13,12 +13,12 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ master ]
+    branches: [master]
   schedule:
-    - cron: '26 14 * * 6'
+    - cron: "26 14 * * 6"
 
 jobs:
   analyze:
@@ -32,40 +32,40 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: [ 'javascript', 'python' ]
+        language: ["javascript", "python"]
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
         # Learn more:
         # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
+      - name: Checkout repository
+        uses: actions/checkout@v3
 
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+          # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
 
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
 
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
 
-    #- run: |
-    #   make bootstrap
-    #   make release
+      #- run: |
+      #   make bootstrap
+      #   make release
 
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2

.github/workflows/codeql-analysis.yml

@@ -593,10 +593,19 @@ archive/samfunctions*/camDetails/pythoncode/camDetails.py
 archive/samfunctions*/camDetails/pythoncode/awskeys
 archive/containers/RMS-docker/rms_docker.tar
 archive/containers/RMS-docker/Dockerfile_orig.txt
+
+usermgmt/windows/csvkeys
+usermgmt/windows/inifs
+usermgmt/windows/jsonkeys
+usermgmt/windows/keys
+usermgmt/windows/sshkeys
+usermgmt/windows/stationdetails
+usermgmt/windows/users
 usermgmt/gmailkeys/*
 usermgmt/server/ukmonfundraising.txt
 usermgmt/server/ukmoncommittee.txt
 usermgmt/server/ukmon*.json
+
 **/awskeys.test
 **/trajsolver_old/*
 archive/tmp/*
@@ -605,3 +614,7 @@ tests/testing/RMS/
 tests/testing/WesternMeteorPyLib/
 archive/ukmon_pylib/tests/testdata.tar.gz
 servercopybkp/*
+fbcollector/config.ini
+usermgmt/windows/stationmaint.ini
+usermgmt/windows/README.md
+fbCollector/README.html

.gitignore

@@ -5,8 +5,7 @@
             "includePath": [
                 "${workspaceFolder}/**",
                 "${vcpkgRoot}/x64-windows/include",
-                "${vcpkgRoot}/x86-windows/include",
-                "e:/dev/opencv3/opencv-3.4.11/build/install/include"
+                "${vcpkgRoot}/x86-windows/include"
             ],
             "defines": [
                 "_DEBUG",

.vscode/c_cpp_properties.json

@@ -37,23 +37,29 @@ consdir=${DATADIR}/consolidated/temp
 mkdir -p ${DATADIR}/single/rawcsvs 
 ls -1 $consdir/*.csv | while read csvf
 do
-    bn=$(basename $csvf)
-    typ=${bn:0:3}
-    if [ "$typ" != "M20" ] ; then 
-        pref="P"
-        yr=${bn:7:4}
+    flen=$(wc -l $csvf | awk '{print $1}')
+    if [ $flen -lt 2 ] ; then 
+        logger -s -t consolidateOutput "skipping empty file $csvf"
+        rm $csvf 
     else
-        pref="M"
-        yr=${bn:1:4}
-    fi 
-    mrgfile=${DATADIR}/consolidated/${pref}_${yr}-unified.csv
-    if [ ! -f $mrgfile ] ; then
-        cat $csvf >> $mrgfile
-    else
-        #echo $bn $mrgfile
-        sed '1d' $csvf >> $mrgfile
+        bn=$(basename $csvf)
+        typ=${bn:0:3}
+        if [ "$typ" != "M20" ] ; then 
+            pref="P"
+            yr=${bn:7:4}
+        else
+            pref="M"
+            yr=${bn:1:4}
+        fi 
+        mrgfile=${DATADIR}/consolidated/${pref}_${yr}-unified.csv
+        if [ ! -f $mrgfile ] ; then
+            cat $csvf >> $mrgfile
+        else
+            #echo $bn $mrgfile
+            sed '1d' $csvf >> $mrgfile
+        fi
+        mv $csvf ${DATADIR}/single/rawcsvs
     fi
-    mv $csvf ${DATADIR}/single/rawcsvs
 done
 
 logger -s -t consolidateOutput "pushing consolidated information back"

archive/analysis/consolidateOutput.sh

@@ -1,18 +1,20 @@
 # Copyright (C) 2018-2023 Mark McIntyre
-FROM continuumio/miniconda3:4.11.0
+#FROM continuumio/miniconda3:4.11.0
+FROM continuumio/miniconda3:24.7.1-0
 
-RUN conda install -y -c conda-forge "numpy<1.25.0" scipy cython pytz pyqt
+#RUN conda install -y -c conda-forge "numpy<1.25.0" scipy cython pytz pyqt
+RUN conda install -y -c conda-forge "numpy<2" scipy cython pytz pyqt
 RUN conda install -y -c conda-forge jplephem ephem
 RUN conda install -y -c conda-forge basemap basemap-data-hires
 RUN conda install -y -c conda-forge pandas cartopy
-RUN conda install -y matplotlib=3.4.3 
+RUN conda install -y matplotlib
 RUN pip install boto3
 RUN apt-get update && apt-get -y upgrade 
 RUN conda install -y -c conda-forge numba
 
 COPY WesternMeteorPyLib/ ./WesternMeteorPyLib
-ENV PYTHONPATH ./WesternMeteorPyLib
-ENV PROJ_LIB ./
+ENV PYTHONPATH=./WesternMeteorPyLib
+ENV PROJ_LIB=./
 COPY *.py ./
 #ENV NUMPY_EXPERIMENTAL_DTYPE_API 1
 

archive/containers/trajsolver/Dockerfile

@@ -17,7 +17,7 @@ write-output "building $imagename in $accid"
 $yn=read-host -prompt "update WMPL?"
 if ($yn.tolower() -eq "y") { bash -c "./update_wmpl.sh $env" } 
 
-if ($env -eq "test") { copy-item awskeys.test awskeys}
+if ($env -eq "test") { move-item awskeys.test awskeys}
 
 docker build . -t ${imagename}
 if (! $?)
@@ -26,7 +26,7 @@ if (! $?)
     exit
 }
 set-location $loc
-if ($env -eq "test") { Remove-Item awskeys}
+if ($env -eq "test") { move-item awskeys awskeys.test }
 
 $yn=read-host -prompt "upload to ECR?"
 if ($yn.tolower() -eq "y") {