Merge pull request #372 from ukmda/master

Updates and Enhancements April 2024

Author: ukmda

archive/.bash_aliases

@@ -8,12 +8,12 @@ alias du='du -h'
 alias data='if [ "$SRC" == "" ] ; then echo select env first; else cd $SRC/data && pwd ; fi'
 alias logs='if [ "$SRC" == "" ] ; then echo select env first; else cd $SRC/logs && pwd ; fi'
 
-alias tml='if [ "$SRC" == "" ] ; then echo select env first; else tail -f $(ls -1 $SRC/logs/matches-*.log | tail -1) ; fi'
-alias tnj='if [ "$SRC" == "" ] ; then echo select env first; else tail -f $(ls -1 $SRC/logs/nigh*.log | tail -1) ; fi'
+alias tml='if [ "$SRC" == "" ] ; then echo select env first; else tail -f $(ls -1 $SRC/logs/matchJob.log | tail -1) ; fi'
+alias tnj='if [ "$SRC" == "" ] ; then echo select env first; else tail -f $(ls -1 $SRC/logs/nightlyJob.log | tail -1) ; fi'
 
 alias stats='if [ "$DATADIR" == "" ] ; then echo select env first; else tail $DATADIR/dailyreports/stats.txt ; fi'
 
-alias matchstatus='if [ "$SRC" == "" ] ; then echo select env first; else grep "Running" $(ls -1 $SRC/logs/matches-*.log| tail -1 ) && grep TRAJ $(ls -1 $SRC/logs/matches-*.log | tail -1)|grep SOLVING && echo -n "Completed " && grep Observations: $(ls -1 $SRC/logs/matches-*.log | tail -1) | wc -l && nj=$(ls -1 $SRC/logs/nightly* | tail -1) &&  grep "nightlyJob" $nj | tail -1 ; fi '
+alias matchstatus='if [ "$SRC" == "" ] ; then echo select env first; else grep "Running" $SRC/logs/matchJob.log && grep TRAJ $SRC/logs/matchJob.log | grep SOLVING && echo -n "Completed " && grep Observations: $SRC/logs/matchJob.log | wc -l &&  grep "nightlyJob" $SRC/logs/nightlyJob.log ; fi '
 alias spacecalc='ls -1 | egrep -v "ukmon-shared" | while read i ; do \du -s $i ; done | sort -n'
 
 alias startcalc='~/prod/utils/stopstart-calcengine.sh start'
@@ -34,7 +34,7 @@ function prd {
 
 function calcserver { 
 	sts=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State --output text --profile ukmonshared)
-	ipaddr=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PublicIpAddress --output text --profile ukmonshared)
+	ipaddr=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared)
 	isrunning=$(echo $sts | cut -d " " -f 1)
 	if [ $isrunning -ne 16 ] ; then
 		/home/ec2-user/prod/utils/stopstart-calcengine.sh start

archive/analysis/findAllMatches.sh

@@ -65,8 +65,7 @@ log2cw $NJLOGGRP $NJLOGSTREAM "start runDistrib" findAllMatches
 $SRC/analysis/runDistrib.sh $MATCHSTART $MATCHEND
 
 log2cw $NJLOGGRP $NJLOGSTREAM "start checkForFailures" findAllMatches
-logf=$(ls -1tr $SRC/logs/matches-*.log | tail -1)
-success=$(grep "Total run time:" $logf)
+success=$(grep "Total run time:" $SRC/logs/matchJob.log)
 
 if [ "$success" == "" ]
 then
@@ -86,7 +85,7 @@ log2cw $NJLOGGRP $NJLOGSTREAM "start getMatchStats" findAllMatches
 dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1)
 trajlist=$(cat $dailyrep | awk -F, '{print $2}')
 
-matchlog=$( ls -1 ${SRC}/logs/matches-*.log | tail -1)
+matchlog=${SRC}/logs/matchJob.log
 vals=$(python -m metrics.getMatchStats $matchlog )
 evts=$(echo $vals | awk '{print $2}')
 trajs=$(echo $vals | awk '{print $6}')

archive/analysis/getLogData.sh

@@ -16,9 +16,15 @@ fi
 # create performance metrics
 cd $SRC/logs
 python -m metrics.timingMetrics $rundate
-aws s3 cp $rundate-perfNightly.jpg $WEBSITEBUCKET/reports/batchcharts/ --quiet
+mv -f $rundate-perfNightly.jpg $DATADIR/
+aws s3 cp $DATADIR/$rundate-perfNightly.jpg $WEBSITEBUCKET/reports/batchcharts/ --quiet
 
-lastmtch=$(ls -1tr $SRC/logs/matches-${rundate}-*.log | tail -1)
+#lastmtch=$(ls -1tr $SRC/logs/matches-${rundate}-*.log | tail -1)
+if [ $rundate != $(date +%Y%m%d) ] ; then
+    lastmtch=matchJob.log-$rundate
+else
+    lastmtch=matchJob.log
+fi 
 
 python -m reports.findFailedMatches $rundate
 

archive/analysis/onlyConsolDistrib.sh

@@ -48,11 +48,11 @@ logger -s -t onlyConsolDistrib "consolidating for $rundate"
 
 python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}'); "
 
-privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PublicIpAddress --output text --profile ukmonshared)
+privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared)
 while [ "$privip" == "" ] ; do
     sleep 5
     echo "getting ipaddress"
-    privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PublicIpAddress --output text --profile ukmonshared)
+    privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared)
 done
 
 if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then 

archive/analysis/runDistrib.sh

@@ -65,11 +65,11 @@ python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh
 chmod +x $execMatchingsh
 
 log2cw $NJLOGGRP $NJLOGSTREAM "get server details" runDistrib
-privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PublicIpAddress --output text)
+privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text)
 while [ "$privip" == "" ] ; do
     sleep 5
     log2cw $NJLOGGRP $NJLOGSTREAM "getting IP address" runDistrib
-    privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PublicIpAddress --output text)
+    privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text)
 done
 
 log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" runDistrib

archive/cronjobs/nightlyJob.sh

@@ -41,7 +41,11 @@ aws s3 cp $DATADIR/activestatlocs.html $WEBSITEBUCKET/search/ --profile ukmonsha
 
 # run this only once as it scoops up all unprocessed data
 log2cw $NJLOGGRP $NJLOGSTREAM "start findAllMatches" nightlyJob
-matchlog=matches-$(date +%Y%m%d-%H%M%S).log
+matchlog=matchJob.log
+if [ -f $SRC/logs/$matchlog ] ; then
+    suff=$(stat matchJob.log -c %X)
+    mv $SRC/logs/$matchlog $SRC/logs/$matchlog-$suff
+fi 
 ${SRC}/analysis/findAllMatches.sh > ${SRC}/logs/${matchlog} 2>&1
 
 log2cw $NJLOGGRP $NJLOGSTREAM "start consolidateOutput" nightlyJob 

archive/deploy-analysis.yml

@@ -14,12 +14,27 @@
     tags: [dev,prod]
   - name: Ensures {{destdir}}/logs exists
     file: path={{destdir}}/logs state=directory  
+  - name: clean local filesystem
+    ansible.builtin.command: 'pyclean {{srcdir}}/archive/ukmon_pylib -v --debris all'
+    delegate_to: localhost
+    tags: [dev,prod]
   - name: Copy files
     copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }}
     tags: [dev,prod]
     with_items:
     - {src: '{{srcdir}}/archive/static_content/', dest: '{{destdir}}/static_content/', mode: '754', backup: no, directory_mode: yes }
-    - {src: '{{srcdir}}/archive/ukmon_pylib/', dest: '{{destdir}}/ukmon_pylib/', mode: '644', backup: no, directory_mode: yes }
+
+    - {src: '{{srcdir}}/archive/ukmon_pylib/analysis/', dest: '{{destdir}}/ukmon_pylib/analysis/', mode: '644', backup: no, directory_mode: yes }
+    - {src: '{{srcdir}}/archive/ukmon_pylib/converters/', dest: '{{destdir}}/ukmon_pylib/converters/', mode: '644', backup: no, directory_mode: yes }
+    - {src: '{{srcdir}}/archive/ukmon_pylib/maintenance/', dest: '{{destdir}}/ukmon_pylib/maintenance/', mode: '644', backup: no, directory_mode: yes }
+    - {src: '{{srcdir}}/archive/ukmon_pylib/metrics/', dest: '{{destdir}}/ukmon_pylib/metrics/', mode: '644', backup: no, directory_mode: yes }
+    - {src: '{{srcdir}}/archive/ukmon_pylib/reports/', dest: '{{destdir}}/ukmon_pylib/reports/', mode: '644', backup: no, directory_mode: yes }
+    - {src: '{{srcdir}}/archive/ukmon_pylib/share/', dest: '{{destdir}}/ukmon_pylib/share/', mode: '644', backup: no, directory_mode: yes }
+    - {src: '{{srcdir}}/archive/ukmon_pylib/tests/', dest: '{{destdir}}/ukmon_pylib/tests/', mode: '644', backup: no, directory_mode: yes }
+    - {src: '{{srcdir}}/archive/ukmon_pylib/traj/', dest: '{{destdir}}/ukmon_pylib/traj/', mode: '644', backup: no, directory_mode: yes }
+    - {src: '{{srcdir}}/archive/ukmon_pylib/README.md', dest: '{{destdir}}/ukmon_pylib/', mode: '644', backup: no, directory_mode: no }
+    - {src: '{{srcdir}}/archive/ukmon_pylib/requirements.txt', dest: '{{destdir}}/ukmon_pylib/', mode: '644', backup: no, directory_mode: no }
+
     - {src: '{{srcdir}}/archive/analysis/', dest: '{{destdir}}/analysis/', mode: '754', backup: no, directory_mode: yes }
     - {src: '{{srcdir}}/archive/cronjobs/', dest: '{{destdir}}/cronjobs/', mode: '754', backup: no, directory_mode: yes }
     - {src: '{{srcdir}}/archive/utils/', dest: '{{destdir}}/utils/', mode: '754', backup: no, directory_mode: yes }

archive/terraform/mjmm/vpc.tf

@@ -65,7 +65,7 @@ resource "aws_route_table" "default" {
   }
   route {
     cidr_block                = "172.32.0.0/16"
-    vpc_peering_connection_id = "pcx-04bcbf8428c045637"
+    vpc_peering_connection_id = "pcx-0beef413172ec795e"
   }
 
   tags = {
@@ -82,14 +82,6 @@ resource "aws_internet_gateway" "main_igw" {
   }
 }
 
-resource "aws_vpc_peering_connection_accepter" "eetommpeering" {
-  vpc_peering_connection_id = "pcx-04bcbf8428c045637"
-  tags = {
-    "Name"       = "ee-to-mm-peering"
-    "billingtag" = "ukmon"
-  }
-}
-
 resource "aws_vpc_peering_connection_accepter" "mdatommpeering" {
   vpc_peering_connection_id = "pcx-0beef413172ec795e"
   tags = {

archive/terraform/ukmda/secgrp.tf

@@ -46,7 +46,7 @@ resource "aws_security_group" "ec2_secgrp" {
   description = "Security Group for EC2 instances"
   vpc_id      = aws_vpc.ec2_vpc.id
   ingress = [
-    {
+/*    {
       cidr_blocks      = [data.aws_vpc.mjmm_ec2_vpc.cidr_block]
       description      = "SSH for Admin"
       from_port        = 22
@@ -56,7 +56,7 @@ resource "aws_security_group" "ec2_secgrp" {
       prefix_list_ids  = []
       security_groups  = []
       self             = false
-    },
+    },*/
     {
       cidr_blocks      = [aws_vpc.ec2_vpc.cidr_block]
       description      = "SSH for Admin"

archive/ukmon_pylib/maintenance/README.md

@@ -2,9 +2,13 @@
 
 Routines used to maintain and keep data organized
 
-* dataMaintenance.py - cleardown and space freeing routines
-* sortToDateDirs.py - one off tool to convert the WMPL trajectory database to a new layout
-* getNextBatchStart.py - update the crontab to start the batch at the right time
+* dataMaintenance.py    - cleardown and space freeing routines
+* sortToDateDirs.py     - one off tool to convert the WMPL trajectory database to a new layout
+* getNextBatchStart.py  - update the crontab to start the batch at the right time
 * rerunFailedLambdas.py - rerun any failed lambdas
-* plotStationsOnMap.py - plot the locations of all stations on a map
-* manageTraj - functions to manage the trajectory database
+* plotStationsOnMap.py  - plot the locations of all stations on a map
+* manageTraj.py         - functions to manage the trajectory database
+* getUserAndKeyInfo.py  - functions to audit and manage users and keys
+
+getUserAndKeyInfo is used by a scheduled process to roll keys every 90 days, and to 
+report on accounts that have been inactive for 90 days.