cloudformation.yaml

Description: "Deploy monitor-ontap-services."
#
# This just formats the page that prompts for the parameters when using the AWS Console to deploy your stack.
Metadata: 
  AWS::CloudFormation::Interface: 
    ParameterGroups:
      - Label: 
          default: "Configuration Parameters"
        Parameters: 
          - OntapAdminSever
          - s3BucketName
          - subNetIds
          - securityGroupIds
          - snsTopicArn
          - cloudWatchLogGroupArn
          - secretArn
          - secretUsernameKey
          - secretPasswordKey
          - checkInterval
          - createWatchdogAlarm
          - createSecretsManagerEndpoint
          - createSNSEndpoint
          - createCloudWatchLogsEndpoint
          - createS3Endpoint
          - routeTableIds
          - vpcId
          - endpointSecurityGroupIds
          - LambdaRoleArn
          - SchedulerRoleArn
          - watchdogRoleArn
      - Label: 
          default: "Alert Parameters"
        Parameters: 
          - versionChangeAlert
          - failoverAlert
          - emsEventsAlert
          - snapMirrorLagTimeAlert
          - snapMirrorLagTimePercentAlert
          - snapMirrorStalledAlert
          - snapMirrorHealthAlert
          - fileSystemUtilizationWarnAlert
          - fileSystemUtilizationCriticalAlert
          - volumeUtilizationWarnAlert
          - volumeUtilizationCriticalAlert
          - volumeFileUtilizationWarnAlert
          - volumeFileUtilizationCriticalAlert
          - volumeOfflineAlert
          - softQuotaUtilizationAlert
          - hardQuotaUtilizationAlert
          - inodesQuotaUtilizationAlert
          - vserverStateAlert
          - vserverNFSProtocolStateAlert
          - vserverCIFSProtocolStateAlert

Parameters:
  OntapAdminSever:
    Description: "The DNS name, or IP address, of the management endpoint of the FSxN file system to be monitored."
    Type: String

  s3BucketName:
    Description: "The name of the S3 bucket to use to store the status and configuration files. Must also be holding the lambda_layer.zip file."
    Type: String

  subNetIds:
    Description: "The subnet IDs where you want to deploy the Lambda function. Must have connectivity to the FSxN file system to be monitored."
    Type: "List<AWS::EC2::Subnet::Id>"

  securityGroupIds:
    Description: "The security group IDs to associate with the Lambda function. Must allow outbound traffic over TCP port 443 to the FSxN file system, and the AWS service endpoints."
    Type: "List<AWS::EC2::SecurityGroup::Id>"
    Default: ""

  snsTopicArn:
    Description: "The ARN of the SNS topic where you want alerts sent to."
    Type: String

  cloudWatchLogGroupArn:
    Description: "The ARN of the CloudWatch log group to send alerts to. If left blank, alerts will not be sent to CloudWatch. Note that the log group must already exist. Also note that the ARN should end with ':*'."
    Type: String
    Default: ""

  secretArn:
    Description: "The ARN of the Secrets Manager secret that holds the FSxN credentials to use."
    Type: String

  secretUsernameKey:
    Description: "The key in the secret that holds the username."
    Type: String
    Default: "username"

  secretPasswordKey:
    Description: "The key in the secret that holds the password."
    Type: String
    Default: "password"

  createWatchdogAlarm:
    Description: "Create a CloudWatch alarm to monitor the Lambda function. It will alert you if the function fails to run successfully."
    Type: String
    Default: "true"
    AllowedValues: ["true", "false"]

  watchdogRoleArn:
    Description: "The ARN of the role to use for the Lambda function that will publish messages to the SNS topic if the monitoring function doesn't run properly. This is only needed if you are having the CloudWatch alarm created and if you want to provide an existing role, otherwise an appropriate one will be created for you."
    Type: String
    Default: ""

  createSecretsManagerEndpoint:
    Description: "Create a Secrets Manager endpoint."
    Type: String
    Default: "false"
    AllowedValues: ["true", "false"]

  createSNSEndpoint:
    Description: "Create an SNS endpoint."
    Type: String
    Default: "false"
    AllowedValues: ["true", "false"]

  createCloudWatchLogsEndpoint:
    Description: "Create a CloudWatch logs endpoint."
    Type: String
    Default: "false"
    AllowedValues: ["true", "false"]

  createS3Endpoint:
    Description: "Create an S3 endpoint."
    Type: String
    Default: "false"
    AllowedValues: ["true", "false"]

  routeTableIds:
    Description: "The route table IDs, comma separated, to update to use the S3 endpoint. Since the S3 endpoint is of type 'Gateway' route tables have to be updated to use it. This parameter is only needed if createS3Endpoint is set to 'true'."
    Type: CommaDelimitedList
    Default: ""

  vpcId:
    Description: "The VPC ID where the FSxN file system is located. This is only needed if you are creating an endpoint."
    Type: String
    Default: ""

  endpointSecurityGroupIds:
    Description: "The security group IDs, comma separated list, to associate with the SNS, SecretsManager and/or CloudWatch Logs endpoints. Must allow traffic from from the Lambda function over TCP port 443. This parameter is only needed if you are creating the SNS, SecretsManager, or CloudWatch Logs endpoint."
    Type: CommaDelimitedList
    Default: ""

  LambdaRoleArn:
    Description: "The ARN of the role to use for the Lambda function. This is only needed if you want to provide an existing role, otherwise an appropriate one will be created for you."
    Type: String
    Default: ""

  SchedulerRoleArn:
    Description: "The ARN of the role to use for the Lambda scheduler. This is only needed if you want to provide an existing role, otherwise an appropriate one will be created for you."
    Type: String
    Default: ""

  checkInterval:
    Description: "The interval, in minutes, between checks."
    Type: Number
    Default: 15
    MinValue: 1

  versionChangeAlert:
    Description: "Alert when the ONTAP version changes."
    Type: String
    Default: "true"
    AllowedValues: ["true", "false"]

  failoverAlert:
    Description: "Alert when a failover occurs."
    Type: String
    Default: "true"
    AllowedValues: ["true", "false"]

  emsEventsAlert:
    Description: "Alert for EMS messages."
    Type: String
    Default: "true"
    AllowedValues: ["true", "false"]

  snapMirrorLagTimeAlert:
    Description: "Alert when a SnapMirror update time is more than the specified seconds. Set to 0 to disable this alert. Recommended to set both snapMirrorLagTimeAlert and snapMirrorLagTimePercentAlert."
    Type: Number
    Default: 86400

  snapMirrorLagTimePercentAlert:
    Description: "Alert when the last succesful SnapMirror update time is more than this percent of the amount of time since the last scheduled update. Must be more than 100. A value of 200 means 2 times the update interval. Set to 0 to disable this alert."
    Type: Number
    MinValue: 100
    Default: 200

  snapMirrorStalledAlert:
    Description: "Alert when a SnapMirror update hasn't transferred any new data in the specified seconds. Set to 0 to disable this alert."
    Type: Number
    Default: 600
    MinValue: 60

  snapMirrorHealthAlert:
    Description: "Alert when the SnapMirror relationship is not healthy."
    Type: String
    Default: "true"
    AllowedValues: ["true", "false"]

  fileSystemUtilizationWarnAlert:
    Description: "Alert when the file system utilization exceeds this threshold in percentage."
    Type: Number
    Default: 80

  fileSystemUtilizationCriticalAlert:
    Description: "Alert when the file system utilization exceeds this threshold in percentage."
    Type: Number
    Default: 90

  volumeUtilizationWarnAlert:
    Description: "Alert when a volume utilization exceeds this threshold in percentage."
    Type: Number
    Default: 90

  volumeUtilizationCriticalAlert:
    Description: "Alert when a volume utilization exceeds this threshold in percentage."
    Type: Number
    Default: 95

  volumeFileUtilizationWarnAlert:
    Description: "Alert when a volume inode utilization exceeds this threshold in percentage."
    Type: Number
    Default: 90

  volumeFileUtilizationCriticalAlert:
    Description: "Alert when a volume inode utilization exceeds this threshold in percentage."
    Type: Number
    Default: 95

  volumeOfflineAlert:
    Description: "Alert when a volume goes offline."
    Type: String
    AllowedValues: ["true", "false"]
    Default: "true"

  softQuotaUtilizationAlert:
    Description: "Alert when a soft quota exceeds this threshold in percentage."
    Type: Number
    Default: 100

  hardQuotaUtilizationAlert:
    Description: "Alert when a hard quota exceeds this threshold in percentage."
    Type: Number
    Default: 80

  inodesQuotaUtilizationAlert:
    Description: "Alert when an inode quota exceeds this threshold in percentage."
    Type: Number
    Default: 80

  vserverStateAlert:
    Description: "Alert when a vserver goes offline."
    Type: String
    AllowedValues: ["true", "false"]
    Default: "true"

  vserverNFSProtocolStateAlert:
    Description: "Alert when a vserver's NFS protocol goes offline."
    Type: String
    AllowedValues: ["true", "false"]
    Default: "true"

  vserverCIFSProtocolStateAlert:
    Description: "Alert when a vserver's CIFS protocol goes offline."
    Type: String
    AllowedValues: ["true", "false"]
    Default: "true"

Conditions:
  CreateSecretsManagerEndpoint: !Equals [!Ref createSecretsManagerEndpoint, "true"]
  CreateSNSEndpoint: !Equals [!Ref createSNSEndpoint, "true"]
  CreateS3Endpoint: !Equals [!Ref createS3Endpoint, "true"]
  CreateCloudWatchLogsEndpoint: !Equals [!Ref createCloudWatchLogsEndpoint, "true"]
  CreateWatchdogAlarm: !Equals [!Ref createWatchdogAlarm, "true"]
  CreateLambdaRoleWithCW: !And [!Equals [!Ref LambdaRoleArn, ""], !Not [!Equals [!Ref cloudWatchLogGroupArn, ""]]]
  CreateLambdaRoleWithoutCW: !And [!Equals [!Ref LambdaRoleArn, ""], !Equals [!Ref cloudWatchLogGroupArn, ""]]
  CreateSchedulerRole: !Equals [!Ref SchedulerRoleArn, ""]
  CreateWatchdogRole: !Equals [!Ref watchdogRoleArn, ""]

Resources:
  SecretManagerEndpoint:
    Type: AWS::EC2::VPCEndpoint
    Condition: CreateSecretsManagerEndpoint
    Properties:
      VpcId: !Ref vpcId
      ServiceName: !Sub "com.amazonaws.${AWS::Region}.secretsmanager"
      VpcEndpointType: 'Interface'
      PrivateDnsEnabled: true
      SubnetIds: !Ref subNetIds
      SecurityGroupIds: !Ref endpointSecurityGroupIds

  CWEndpoint:
    Type: AWS::EC2::VPCEndpoint
    Condition: CreateCloudWatchLogsEndpoint
    Properties:
      VpcId: !Ref vpcId
      ServiceName: !Sub "com.amazonaws.${AWS::Region}.logs"
      VpcEndpointType: 'Interface'
      PrivateDnsEnabled: true
      SubnetIds: !Ref subNetIds
      SecurityGroupIds: !Ref endpointSecurityGroupIds

  SNSEndpoint:
    Type: AWS::EC2::VPCEndpoint
    Condition: CreateSNSEndpoint
    Properties:
      VpcId: !Ref vpcId
      ServiceName: !Sub "com.amazonaws.${AWS::Region}.sns"
      VpcEndpointType: 'Interface'
      PrivateDnsEnabled: true
      SubnetIds: !Ref subNetIds
      SecurityGroupIds: !Ref endpointSecurityGroupIds

  S3Endpoint:
    Type: AWS::EC2::VPCEndpoint
    Condition: CreateS3Endpoint
    Properties:
      VpcId: !Ref vpcId
      ServiceName: !Sub "com.amazonaws.${AWS::Region}.s3"
      VpcEndpointType: 'Gateway'
      RouteTableIds: !Ref routeTableIds
  #
  # Allow the Watchdog Lambda function to publish to the SNS topic.
  LambdaRoleWatchdog:
    Type: "AWS::IAM::Role"
    Condition: CreateWatchdogRole
    Properties:
      RoleName: !Sub "mon-ontap-services-watchdog-${AWS::StackName}"
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: "Allow"
            Principal:
              Service: "lambda.amazonaws.com"
            Action: "sts:AssumeRole"
      Policies:
        - PolicyName: "LambdaPolicyWatchdog"
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: "Allow"
                Action:
                  - "sns:Publish"
                Resource: !Ref snsTopicArn
      ManagedPolicyArns:
        - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
  #
  # This allows the Watchdog CloudWatch alarm to invoke the Lambda function.
  resourceBasedPermission:
    Type: "AWS::Lambda::Permission"
    Condition: CreateWatchdogAlarm
    Properties:
      Action: "lambda:InvokeFunction"
      FunctionName: !Sub "monitor-ontap-services-watchdog-${AWS::StackName}"
      Principal: "lambda.alarms.cloudwatch.amazonaws.com"
      SourceArn: !GetAtt watchdogAlarm.Arn
  #
  # Use a Lambda function to publish to an SNS topic so it can reside in a different region.
  watchdogLambdaFunction: 
    Type: "AWS::Lambda::Function"
    Condition: CreateWatchdogAlarm
    Properties:
      FunctionName: !Sub "monitor-ontap-services-watchdog-${AWS::StackName}"
      PackageType: "Zip"
      Runtime: "python3.12"
      Handler: "index.lambda_handler"
      Timeout: 10
      Role: !If [CreateWatchdogRole, !GetAtt LambdaRoleWatchdog.Arn, !Ref watchdogRoleArn]
      Environment:
        Variables:
          snsTopicArn: !Ref snsTopicArn
      Code:
        ZipFile: |
          import boto3
          import os
          
          def lambda_handler(event, context):
              snsTopicArn = os.environ.get('snsTopicArn')
              if snsTopicArn is not None:
                  region = snsTopicArn.split(":")[3]
                  snsClient = boto3.client('sns', region_name=region)
                  #
                  # This is for future developement when the monitor-ontap-services
                  # Lambda function will be able to send messages to the SNS topic.
                  cmd = event.get("cmd")
                  #
                  # If the cmd is None, then assume a CloudWatch alarm triggered this function.
                  if cmd is None: 
                      message = f'Error! Lambda function {event["alarmData"]["alarmName"].replace("-watchdog-", "")} failed to execute properly.'
                      snsClient.publish(
                          TopicArn = snsTopicArn,
                          Subject  = 'Error! Monitoring ONTAP services has failed to execute',
                          Message  = message
                      )
                  elif cmd == "sendSns":
                    message = event.get("message")
                    subject = event.get("subject")
                    snsClient.publish(
                        TopicArn = snsTopicArn,
                        Subject  = subject,
                        Message  = message
                    )
  #
  # This is the CloudWatch alarm that will trigger when the monitor-ontap-services
  # Lambda function fails to run successfully. It will invoke the watchdogLambdaFunction
  # to send a message to the SNS topic.
  watchdogAlarm:
    Type: "AWS::CloudWatch::Alarm"
    Condition: CreateWatchdogAlarm
    Properties:
      AlarmName: !Sub "monitor-ontap-services-watchdog-${AWS::StackName}"
      AlarmDescription: !Sub "Watchdog alarm for the monitor-ontap-services-${AWS::StackName} Lambda function."
      Namespace: "AWS/Lambda"
      MetricName: "Errors"
      Dimensions:
        - Name: "FunctionName"
          Value: !Sub "monitor-ontap-services-${AWS::StackName}"
      Statistic: "Maximum"
      Period: 300
      EvaluationPeriods: 1
      TreatMissingData: "ignore"
      Threshold: 0.5
      ComparisonOperator: "GreaterThanThreshold"
      AlarmActions:
        - !GetAtt watchdogLambdaFunction.Arn

  LambdaRoleWithoutCW:
    Type: "AWS::IAM::Role"
    Condition: CreateLambdaRoleWithoutCW
    Properties:
      RoleName: !Sub "mon-ontap-services-${AWS::StackName}"
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: "Allow"
            Principal:
              Service: "lambda.amazonaws.com"
            Action: "sts:AssumeRole"

      ManagedPolicyArns:
        - "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"

      Policies:
        - PolicyName: "LambdaPolicy"
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: "Allow"
                Action:
                  - "secretsManager:GetSecretValue"
                  - "sns:Publish"
                  - "s3:GetObject"
                  - "s3:PutObject"
                  - "s3:ListBucket"
                Resource:
                  - !Ref secretArn
                  - !Ref snsTopicArn
                  - !Sub "arn:aws:s3:::${s3BucketName}"
                  - !Sub "arn:aws:s3:::${s3BucketName}/*"

  LambdaRoleWithCW:
    Type: "AWS::IAM::Role"
    Condition: CreateLambdaRoleWithCW
    Properties:
      RoleName: !Sub "mon-ontap-services-${AWS::StackName}"
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: "Allow"
            Principal:
              Service: "lambda.amazonaws.com"
            Action: "sts:AssumeRole"

      ManagedPolicyArns:
        - "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"

      Policies:
        - PolicyName: "LambdaPolicy"
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: "Allow"
                Action:
                  - "secretsManager:GetSecretValue"
                  - "sns:Publish"
                  - "logs:PutLogEvents"
                  - "logs:DescribeLogStreams"
                  - "logs:CreateLogStream"
                  - "s3:GetObject"
                  - "s3:PutObject"
                  - "s3:ListBucket"
                Resource: 
                  - !Ref secretArn
                  - !Ref snsTopicArn
                  - !Sub "arn:aws:s3:::${s3BucketName}"
                  - !Sub "arn:aws:s3:::${s3BucketName}/*"
                  - !Ref cloudWatchLogGroupArn

  SchedulerRole:
    Type: "AWS::IAM::Role"
    Condition: CreateSchedulerRole
    Properties:
      RoleName: !Sub "SchedulerRole-${AWS::StackName}"
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: "Allow"
            Principal:
              Service: "scheduler.amazonaws.com"
            Action: "sts:AssumeRole"

      Policies:
        - PolicyName: "SchedulerPolicy"
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: "Allow"
                Action:
                  - "lambda:InvokeFunction"
                Resource: !GetAtt LambdaFunction.Arn

  LambdaScheduler:
    Type: "AWS::Scheduler::Schedule"
    Properties:
      Description: "Schedule the monitor-ontap-services Lambda function."
      Name: !Sub "monitor-ontap-services-scheduler-${AWS::StackName}"
      FlexibleTimeWindow:
        Mode: "OFF"
      ScheduleExpression: !Sub "rate(${checkInterval} minutes)"
      Target:
        Arn: !GetAtt LambdaFunction.Arn
        RoleArn: !If [CreateSchedulerRole, !GetAtt SchedulerRole.Arn, !Ref SchedulerRoleArn]

  LambdaLayer:
    Type: "AWS::Lambda::LayerVersion"
    Properties:
      LayerName: !Sub "MOS-${AWS::StackName}"
      Content:
        S3Bucket: !Ref s3BucketName
        S3Key: "lambda_layer.zip"
      CompatibleRuntimes:
        - "python3.12"

  LambdaFunction:
    Type: "AWS::Lambda::Function"
    Properties:
      FunctionName: !Sub "monitor-ontap-services-${AWS::StackName}"
      Role: !If [CreateLambdaRoleWithCW, !GetAtt LambdaRoleWithCW.Arn, !If [CreateLambdaRoleWithoutCW, !GetAtt LambdaRoleWithoutCW.Arn, !Ref LambdaRoleArn]]
      VpcConfig:
        SecurityGroupIds: !Ref securityGroupIds
        SubnetIds: !Ref subNetIds
      PackageType: "Zip"
      Runtime: "python3.12"
      Layers:
        - !GetAtt LambdaLayer.LayerVersionArn
      Handler: "index.lambda_handler"
      Timeout: 60
      Environment:
        Variables:
          s3BucketRegion: !Ref AWS::Region
          s3BucketName: !Ref s3BucketName
          OntapAdminServer: !Ref OntapAdminSever
          secretArn: !Ref secretArn
          secretUsernameKey: !Ref secretUsernameKey
          secretPasswordKey: !Ref secretPasswordKey
          snsTopicArn: !Ref snsTopicArn
          sendSnsFunctionName: !Sub "monitor-ontap-services-watchdog-${AWS::StackName}"
          cloudWatchLogGroupArn: !Ref cloudWatchLogGroupArn

          initialVersionChangeAlert: !Ref versionChangeAlert
          initialFailoverAlert: !Ref failoverAlert
          initialEmsEventsAlert: !Ref emsEventsAlert
          initialSnapMirrorLagTimeAlert: !Ref snapMirrorLagTimeAlert
          initialSnapMirrorLagTimePercentAlert: !Ref snapMirrorLagTimePercentAlert
          initialSnapMirrorStalledAlert: !Ref snapMirrorStalledAlert
          initialSnapMirrorHealthAlert: !Ref snapMirrorHealthAlert
          initialFileSystemUtilizationWarnAlert: !Ref fileSystemUtilizationWarnAlert
          initialFileSystemUtilizationCriticalAlert: !Ref fileSystemUtilizationCriticalAlert
          initialVolumeUtilizationWarnAlert: !Ref volumeUtilizationWarnAlert
          initialVolumeUtilizationCriticalAlert: !Ref volumeUtilizationCriticalAlert
          initialVolumeFileUtilizationWarnAlert: !Ref volumeFileUtilizationWarnAlert
          initialVolumeFileUtilizationCriticalAlert: !Ref volumeFileUtilizationCriticalAlert
          initialVolumeOfflineAlert: !Ref volumeOfflineAlert
          initialSoftQuotaUtilizationAlert: !Ref softQuotaUtilizationAlert
          initialHardQuotaUtilizationAlert: !Ref hardQuotaUtilizationAlert
          initialInodesQuotaUtilizationAlert: !Ref inodesQuotaUtilizationAlert
          initialVserverStateAlert: !Ref vserverStateAlert
          initialVserverNFSProtocolStateAlert: !Ref vserverNFSProtocolStateAlert
          initialVserverCIFSProtocolStateAlert: !Ref vserverCIFSProtocolStateAlert
      Code:
        ZipFile: |
          #!/bin/python3
          #
          ################################################################################
          # THIS SOFTWARE IS PROVIDED BY NETAPP "AS IS" AND ANY EXPRESS OR IMPLIED
          # WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
          # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
          # EVENT SHALL NETAPP BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
          # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
          # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
          # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
          # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR'
          # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
          # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
          ################################################################################
          #
          ################################################################################
          # This program is used to monitor some of Data ONTAP services (EMS Message,
          # Snapmirror relationships, quotas) running under AMS, and alert on any
          # "matching conditions."  It is intended to be run as a Lambda function, but
          # can be run as a standalone program.
          #
          # Version: v2.16
          # Date: 2025-05-19-21:24:55
          ################################################################################
          
          import json
          import re
          import os
          import datetime
          import pytz
          import logging
          from logging.handlers import SysLogHandler
          from cronsim import CronSim
          import urllib3
          from urllib3.util import Retry
          import botocore
          import boto3
          
          eventResilience = 4 # Times an event has to be missing before it is removed
                              # from the alert history.
                              # This was added since the Ontap API that returns EMS
                              # events would often drop some events and then including
                              # them in the subsequent calls. If I don't "age" the
                              # alert history duplicate alerts will be sent.
          initialVersion = "Initial Run"  # The version to store if this is the first
                                          # time the program has been run against a
                                          # FSxN.
          
          ################################################################################
          # This function is used to extract the one-, two-, or three-digit number from
          # the string passed in, starting at the 'start' character. Then, multiple it
          # by the unit after the number:
          # D = Day = 60*60*24
          # H = Hour = 60*60
          # M = Minutes = 60
          #
          # It returns a tuple that has the extracted number and the end position.
          ################################################################################
          def getNumber(string, start):
          
              if len(string) <= start:
                  return (0, start)
              #
              # Check to see if it is a 1, 2 or 3 digit number.
              startp1=start+1   # Single digit
              startp2=start+2   # Double digit
              startp3=start+3   # Triple digit
              if re.search('[0-9]', string[startp1:startp2]) and re.search('[0-9]', string[startp2:startp3]):
                  end=startp3
              elif re.search('[0-9]', string[startp1:startp2]):
                  end=startp2
              else:
                  end=startp1
          
              num=int(string[start:end])
          
              endp1=end+1
              if string[end:endp1] == "D":
                  num=num*60*60*24
              elif string[end:endp1] == "H":
                  num=num*60*60
              elif string[end:endp1] == "M":
                  num=num*60
              elif string[end:endp1] != "S":
                  logger.warning(f'Unknown lag time specifier "{string[end:endp1]}".')
          
              return (num, endp1)
          
          ################################################################################
          # This function is used to parse the lag time string returned by the
          # ONTAP API and return the equivalent seconds it represents.
          # The input string is assumed to follow this pattern "P#DT#H#M#S" where
          # each of those "#" can be one to three digits long. Also, if the lag isn't
          # more than 24 hours, then the "#D" isn't there and the string simply starts
          # with "PT". Similarly, if the lag time isn't more than an hour then the "#H"
          # string is missing.
          ################################################################################
          def parseLagTime(string):
              #
              num=0
              #
              # First check to see if the Day field is there, by checking to see if the
              # second character is a digit. If not, it is assumed to be 'T'.
              includesDay=False
              if re.search('[0-9]', string[1:2]):
                  includesDay=True
                  start=1
              else:
                  start=2
              data=getNumber(string, start)
              num += data[0]
          
              start=data[1]
              #
              # If there is a 'D', then there is a 'T' between the D and the # of hours
              # so skip pass it.
              if includesDay:
                  start += 1
              data=getNumber(string, start)
              num += data[0]
          
              start=data[1]
              data=getNumber(string, start)
              num += data[0]
          
              start=data[1]
              data=getNumber(string, start)
              num += data[0]
          
              return(num)
          
          ################################################################################
          # This function checks to see if an event is in the events array based on
          # the unique Identifier passed in. It will also update the "refresh" field on
          # any matches.
          ################################################################################
          def eventExist (events, uniqueIdentifier):
              for event in events:
                  if event["index"] == uniqueIdentifier:
                      event["refresh"] = eventResilience
                      return True
          
              return False
          
          ################################################################################
          # This function makes an API call to the FSxN to ensure it is up. If the
          # errors out, then it sends an alert, and returns 'False'. Otherwise it returns
          # 'True'.
          ################################################################################
          def checkSystem():
              global config, s3Client, snsClient, http, headers, clusterName, clusterVersion, logger, clusterTimezone
          
              changedEvents = False
              #
              # Get the previous status.
              try:
                  data = s3Client.get_object(Key=config["systemStatusFilename"], Bucket=config["s3BucketName"])
              except botocore.exceptions.ClientError as err:
                  # If the error is that the object doesn't exist, then this must be the
                  # first time this script has run against thie filesystem so create an
                  # initial status structure.
                  if err.response['Error']['Code'] == "NoSuchKey":
                      fsxStatus = {
                          "systemHealth": True,
                          "version" : initialVersion,
                          "numberNodes" : 2,
                          "downInterfaces" : []
                      }
                      changedEvents = True
                  else:
                      raise err
              else:
                  fsxStatus = json.loads(data["Body"].read().decode('UTF-8'))
              #
              # Get the cluster name, ONTAP version and timezone from the FSxN.
              # This is also a way to test that the FSxN cluster is accessible.
              badHTTPStatus = False
              try:
                  endpoint = f'https://{config["OntapAdminServer"]}/api/cluster?fields=version,name,timezone'
                  response = http.request('GET', endpoint, headers=headers, timeout=5.0)
                  if response.status == 200:
                      if not fsxStatus["systemHealth"]:
                          fsxStatus["systemHealth"] = True
                          changedEvents = True
          
                      data = json.loads(response.data)
                      if config["awsAccountId"] != None:
                          clusterName = f'{data["name"]}({config["awsAccountId"]})'
                      else:
                          clusterName = data['name']
                      #
                      # The following assumes that the format of the "full" version
                      # looks like: "NetApp Release 9.13.1P6: Tue Dec 05 16:06:25 UTC 2023".
                      # The reason for looking at the "full" instead of the individual
                      # keys (generation, major, minor) is because they don't provide
                      # the patch level. :-(
                      clusterVersion = data["version"]["full"].split()[2].replace(":", "")
                      if fsxStatus["version"] == initialVersion:
                          fsxStatus["version"] = clusterVersion
                      #
                      # Get the Timezone for SnapMirror lag time calculations.
                      clusterTimezone = data["timezone"]["name"]
                  else:
                      badHTTPStatus = True
                      raise Exception(f'API call to {endpoint} failed. HTTP status code: {response.status}.')
              except:
                  if fsxStatus["systemHealth"]:
                      if config["awsAccountId"] != None:
                          clusterName = f'{config["OntapAdminServer"]}({config["awsAccountId"]})'
                      else:
                          clusterName = config["OntapAdminServer"]
                      if badHTTPStatus:
                          message = f'CRITICAL: Received a non 200 HTTP status code ({response.status}) when trying to access {clusterName}.'
                      else:
                          message = f'CRITICAL: Failed to issue API against {clusterName}. Cluster could be down.'
                      sendAlert(message, "CRITICAL")
                      fsxStatus["systemHealth"] = False
                      changedEvents = True
          
              if changedEvents:
                  s3Client.put_object(Key=config["systemStatusFilename"], Bucket=config["s3BucketName"], Body=json.dumps(fsxStatus).encode('UTF-8'))
              #
              # If the cluster is done, return false so the program can exit cleanly.
              return(fsxStatus["systemHealth"])
          
          ################################################################################
          # This function checks the following things:
          #   o If the ONTAP version has changed.
          #   o If one of the nodes are down.
          #   o If a network interface is down.
          #
          # ASSUMPTIONS: That checkSystem() has been called before it.
          ################################################################################
          def checkSystemHealth(service):
              global config, s3Client, snsClient, http, headers, clusterName, clusterVersion, logger
          
              changedEvents = False
              #
              # Get the previous status.
              # Shouldn't have to check for status of the get_object() call, to see if the object exist or not,
              # since "checkSystem()" should already have been called and it creates the object if it doesn't
              # already exist. So, if there is a failure, it should be something else than "non-existent".
              data = s3Client.get_object(Key=config["systemStatusFilename"], Bucket=config["s3BucketName"])
              fsxStatus = json.loads(data["Body"].read().decode('UTF-8'))
          
              for rule in service["rules"]:
                  for key in rule.keys():
                      lkey = key.lower()
                      if lkey == "versionchange":
                          if rule[key] and clusterVersion != fsxStatus["version"]:
                              message = f'NOTICE: The ONTAP vesion changed on cluster {clusterName} from {fsxStatus["version"]} to {clusterVersion}.'
                              sendAlert(message, "INFO")
                              fsxStatus["version"] = clusterVersion
                              changedEvents = True
                      elif lkey == "failover":
                          #
                          # Check that both nodes are available.
                          # Using the CLI passthrough API because I couldn't find the equivalent API call.
                          if rule[key]:
                              endpoint = f'https://{config["OntapAdminServer"]}/api/private/cli/system/node/virtual-machine/instance/show-settings'
                              response = http.request('GET', endpoint, headers=headers)
                              if response.status == 200:
                                  data = json.loads(response.data)
                                  if data["num_records"] != fsxStatus["numberNodes"]:
                                      message = f'Alert: The number of nodes on cluster {clusterName} went from {fsxStatus["numberNodes"]} to {data["num_records"]}.'
                                      sendAlert(message, "INFO")
                                      fsxStatus["numberNodes"] = data["num_records"]
                                      changedEvents = True
                              else:
                                  logger.warning(f'API call to {endpoint} failed. HTTP status code: {response.status}.')
                      elif lkey == "networkinterfaces":
                          if rule[key]:
                              endpoint = f'https://{config["OntapAdminServer"]}/api/network/ip/interfaces?fields=state'
                              response = http.request('GET', endpoint, headers=headers)
                              if response.status == 200:
                                  #
                                  # Decrement the refresh field to know if any events have really gone away.
                                  for interface in fsxStatus["downInterfaces"]:
                                      interface["refresh"] -= 1
          
                                  data = json.loads(response.data)
                                  for interface in data["records"]:
                                      if interface.get("state") != None and interface["state"] != "up":
                                          uniqueIdentifier = interface["name"]
                                          if(not eventExist(fsxStatus["downInterfaces"], uniqueIdentifier)): # Resets the refresh key.
                                              message = f'Alert: Network interface {interface["name"]} on cluster {clusterName} is down.'
                                              sendAlert(message, "WARNING")
                                              event = {
                                                  "index": uniqueIdentifier,
                                                  "refresh": eventResilience
                                              }
                                              fsxStatus["downInterfaces"].append(event)
                                              changedEvents = True
                                  #
                                  # After processing the records, see if any events need to be removed.
                                  i = 0
                                  while i < len(fsxStatus["downInterfaces"]):
                                      if fsxStatus["downInterfaces"][i]["refresh"] <= 0:
                                          logger.debug(f'Deleting interface: {fsxStatus["downInterfaces"][i]["index"]}')
                                          del fsxStatus["downInterfaces"][i]
                                          changedEvents = True
                                      else:
                                          if fsxStatus["downInterfaces"][i]["refresh"] != eventResilience:
                                              changedEvents = True
                                          i += 1
                              else:
                                  logger.warning(f'API call to {endpoint} failed. HTTP status code: {response.status}.')
                      else:
                          logger.warning(f'Unknown System Health alert type: "{key}".')
          
              if changedEvents:
                  s3Client.put_object(Key=config["systemStatusFilename"], Bucket=config["s3BucketName"], Body=json.dumps(fsxStatus).encode('UTF-8'))
          
          ################################################################################
          # This function processes the EMS events.
          ################################################################################
          def processEMSEvents(service):
              global config, s3Client, snsClient, http, headers, clusterName, clusterVersion, logger
          
              changedEvents = False
              #
              # Get the saved events so we can ensure we are only reporting on new ones.
              try:
                  data = s3Client.get_object(Key=config["emsEventsFilename"], Bucket=config["s3BucketName"])
              except botocore.exceptions.ClientError as err:
                  # If the error is that the object doesn't exist, then it will get created once an alert it sent.
                  if err.response['Error']['Code'] == "NoSuchKey":
                      events = []
                  else:
                      raise err
              else:
                  events = json.loads(data["Body"].read().decode('UTF-8'))
              #
              # Decrement the refresh field to know if any records have really gone away.
              for event in events:
                  event["refresh"] -= 1
              #
              # Run the API call to get the current list of EMS events.
              endpoint = f'https://{config["OntapAdminServer"]}/api/support/ems/events?return_timeout=15'
              response = http.request('GET', endpoint, headers=headers)
              if response.status == 200:
                  data = json.loads(response.data)
                  #
                  # Process the events to see if there are any new ones.
                  print(f'Received {len(data["records"])} EMS records.')
                  logger.debug(f'Received {len(data["records"])} EMS records.')
                  for record in data["records"]:
                      for rule in service["rules"]:
                          messageFilter = rule.get("filter")
                          if messageFilter == None or messageFilter == "":
                              messageFilter = "ThisShouldn'tMatchAnything"
          
                          if (not re.search(messageFilter, record["log_message"]) and
                              re.search(rule["name"], record["message"]["name"]) and
                              re.search(rule["severity"], record["message"]["severity"]) and
                              re.search(rule["message"], record["log_message"])):
                              if (not eventExist (events, record["index"])):  # This resets the "refresh" field if found.
                                  message = f'{record["time"]} : {clusterName} {record["message"]["name"]}({record["message"]["severity"]}) - {record["log_message"]}'
                                  useverity=record["message"]["severity"].upper()
                                  if useverity == "EMERGENCY":
                                      sendAlert(message, "CRITICAL")
                                  elif useverity == "ALERT":
                                      sendAlert(message, "ERROR")
                                  elif useverity == "ERROR":
                                      sendAlert(message, "WARNING")
                                  elif useverity == "NOTICE" or useverity == "INFORMATIONAL":
                                      sendAlert(message, "INFO")
                                  elif useverity == "DEBUG":
                                      sendAlert(message, "DEBUG")
                                  else:
                                      sendAlert(f'Received unknown severity from ONTAP "{record["message"]["severity"]}". The message received is next.', "INFO")
                                      sendAlert(message, "INFO")
          
                                  changedEvents = True
                                  event = {
                                          "index": record["index"],
                                          "time": record["time"],
                                          "messageName": record["message"]["name"],
                                          "message": record["log_message"],
                                          "refresh": eventResilience
                                          }
                                  events.append(event)
                  #
                  # Now that we have processed all the events, check to see if any events should be deleted.
                  i = 0
                  while i < len(events):
                      if events[i]["refresh"] <= 0:
                          logger.debug(f'Deleting event: {events[i]["time"]} : {events[i]["message"]}')
                          del events[i]
                          changedEvents = True
                      else:
                          # If an event wasn't refreshed, then we need to save the new refresh count.
                          if events[i]["refresh"] != eventResilience:
                              changedEvents = True
                          i += 1
                  #
                  # If the events array changed, save it.
                  if changedEvents:
                      s3Client.put_object(Key=config["emsEventsFilename"], Bucket=config["s3BucketName"], Body=json.dumps(events).encode('UTF-8'))
              else:
                  logger.warning(f'API call to {endpoint} failed. HTTP status code: {response.status}.')
          
          ################################################################################
          # This function is used to find an existing SM relationship based on the source
          # and destinatino path passed in. It returns None if one isn't found
          ################################################################################
          def getPreviousSMRecord(relationShips, uuid):
              for relationship in relationShips:
                  if relationship.get('uuid') == uuid:
                      relationship['refresh'] = True
                      return(relationship)
          
              return(None)
          
          ################################################################################
          # This function will convert seconds into an ascii string of number days, hours,
          # minutes, and seconds. It will return the string.
          ################################################################################
          def lagTimeStr(seconds):
              days = seconds // (60 * 60 * 24)
              seconds = seconds - (days * (60 * 60 * 24))
              hours = seconds // (60 * 60)
              seconds = seconds - (hours * (60 * 60))
              minutes = seconds // 60
              seconds = seconds - (minutes * 60)
          
              timeStr=""
              if days > 0:
                  plural = "s" if days != 1 else ""
                  timeStr = f'{days} day{plural} '
              if hours > 0 or days > 0:
                  plural = "s" if hours != 1 else ""
                  timeStr += f'{hours} hour{plural} '
              if minutes > 0 or days > 0 or hours > 0:
                  plural = "s" if minutes != 1 else ""
                  timeStr += f'{minutes} minute{plural} and '
              plural = "s" if seconds != 1 else ""
              timeStr += f'{seconds} second{plural}'
              return timeStr
          
          ################################################################################
          # This function converts an array of numbers to a comma separated string. If
          # the array is empty, it returns "*".
          ################################################################################
          def convertArrayToString(array):
          
              text = ""
              for item in array:
                  if text != "":
                       text += ","
                  text += str(item)
          
              return text if text != "" else "*"
          
          ################################################################################
          # This function takes a schedule dictionary and returns the last time it should
          # run. It returns the time in seconds since the UNIX epoch.
          ################################################################################
          def getLastRunTime(scheduleUUID):
              global config, http, headers, clusterName, clusterVersion, logger, clusterTimezone
          
              minutes = ""
              hours = ""
              months = ""
              daysOfMonth = ""
              daysOfWeek = ""
              #
              # Run the API call to get the schedule information.
              endpoint = f'https://{config["OntapAdminServer"]}/api/cluster/schedules/{scheduleUUID}?fields=*&return_timeout=15'
              response = http.request('GET', endpoint, headers=headers)
              if response.status == 200:
                  schedule = json.loads(response.data)
          
                  if schedule['cron'].get("minutes") is not None:
                      minutes = convertArrayToString(schedule['cron']['minutes'])
                  else:
                      minutes = "*"
          
                  if schedule['cron'].get("hours") is not None:
                      hours = convertArrayToString(schedule['cron']['hours'])
                  else:
                      hours = "*"
          
                  if schedule['cron'].get("days") is not None:
                      daysOfMonth = convertArrayToString(schedule['cron']['days'])
                  else:
                      daysOfMonth = "*"
          
                  if schedule['cron'].get("months") is not None:
                      months = convertArrayToString(schedule['cron']['months'])
                  else:
                      months = "*"
          
                  if schedule['cron'].get("weekdays") is not None:
                      daysOfWeek = convertArrayToString(schedule['cron']['weekdays'])
                  else:
                      daysOfWeek = "*"
                  #
                  # Create the cron expression.
                  cron_expression = f"{minutes} {hours} {daysOfMonth} {months} {daysOfWeek}"
                  #
                  # Initialize CronSim with the cron expression and current time.
                  curTime = datetime.datetime.now(pytz.timezone(clusterTimezone) if clusterTimezone != None else datetime.timezone.utc)
                  curTimeSec = curTime.timestamp()
                  it = CronSim(cron_expression, curTime, reverse=True)
                  #
                  # Get the last run time.
                  lastRunTime = next(it)
                  lastRunTimeSec = lastRunTime.timestamp()
                  return int(lastRunTimeSec)
              else:
                  logger.error(f'API call to {endpoint} failed. HTTP status code: {response.status}.')
                  return -1
          
          ################################################################################
          ################################################################################
          def getPolicySchedule(policyUUID):
              global config, http, headers, clusterName, clusterVersion, logger
          
              # Run the API call to get the policy information.
              endpoint = f'https://{config["OntapAdminServer"]}/api/snapmirror/policies/{policyUUID}?fields=*&return_timeout=15'
              response = http.request('GET', endpoint, headers=headers)
              if response.status == 200:
                  data = json.loads(response.data)
                  if data.get('transfer_schedule') != None:
                      return data['transfer_schedule']['uuid']
                  else:
                      return None
              else:
                  logger.error(f'API call to {endpoint} failed. HTTP status code: {response.status}.')
                  return None
          
          ################################################################################
          # This function is used to find the last time a SnapMirror relationship should
          # have been updated. It returns the time in seconds since the UNIX epoch.
          ################################################################################
          def getLastScheduledUpdate(record):
              global config, http, headers, clusterName, clusterVersion, logger
              #
              # First check to see if there is a schedule associated with the SM relationship.
              if record.get("transfer_schedule") is not None:
                  lastRunTime = getLastRunTime(record["transfer_schedule"]["uuid"])
              else:
                  #
                  # If there is no schedule at the relationship level, check to see
                  # if the policy has one.
                  scheduleUUID = getPolicySchedule(record["policy"]["uuid"])
                  if scheduleUUID is not None:
                      lastRunTime = getLastRunTime(scheduleUUID)
                  else:
                      lastRunTime = -1
              return lastRunTime
          
          ################################################################################
          # This function is used to check SnapMirror relationships.
          ################################################################################
          def processSnapMirrorRelationships(service):
              global config, s3Client, snsClient, http, headers, clusterName, clusterVersion, logger, clusterTimezone
              #
              # Get the saved events so we can ensure we are only reporting on new ones.
              try:
                  data = s3Client.get_object(Key=config["smEventsFilename"], Bucket=config["s3BucketName"])
              except botocore.exceptions.ClientError as err:
                  # If the error is that the object doesn't exist, then it will get created once an alert is sent.
                  if err.response['Error']['Code'] == "NoSuchKey":
                      events = []
                  else:
                      raise err
              else:
                  events = json.loads(data["Body"].read().decode('UTF-8'))
              #
              # Decrement the refresh field to know if any records have really gone away.
              for event in events:
                  event["refresh"] -= 1
          
              changedEvents=False
              #
              # Get the saved SM relationships.
              try:
                  data = s3Client.get_object(Key=config["smRelationshipsFilename"], Bucket=config["s3BucketName"])
              except botocore.exceptions.ClientError as err:
                  # If the error is that the object doesn't exist, then it will get created once an alert is sent.
                  if err.response['Error']['Code'] == "NoSuchKey":
                      smRelationships = []
                  else:
                      raise err
              else:
                  smRelationships = json.loads(data["Body"].read().decode('UTF-8'))
              #
              # Set the refresh to False to know if any of the relationships still exist.
              for relationship in smRelationships:
                  relationship["refresh"] = False
          
              updateRelationships = False
              #
              # Get the current time in seconds since UNIX epoch 01/01/1970.
              curTimeSeconds = int(datetime.datetime.now(pytz.timezone(clusterTimezone) if clusterTimezone != None else datetime.timezone.utc).timestamp())
              #
              # Consolidate all the rules so we can decide how to process lagtime.
              maxLagtime = None
              maxLagTimePercent = None
              healthy = None
              stalledTransferSeconds = None
              offline = None
              for rule in service["rules"]:
                  for key in rule.keys():
                      lkey = key.lower()
                      if lkey == "maxlagtime":
                          maxLagTime = rule[key]
                          maxLagTimeKey = key
                      elif lkey == "maxlagtimepercent":
                          maxLagTimePercent = rule[key]
                          maxLagTimePercentKey = key
                      elif lkey == "healthy":
                          healthy = rule[key]
                          healthyKey = key
                      elif lkey == "stalledtransferseconds":
                          stalledTransferSeconds = rule[key]
                          stalledTransferSecondsKey = key
                      else:
                          logger.warning(f'Unknown snapmirror alert type: "{key}".')
              #
              # Run the API call to get the current state of all the snapmirror relationships.
              endpoint = f'https://{config["OntapAdminServer"]}/api/snapmirror/relationships?fields=*&return_timeout=15'
              response = http.request('GET', endpoint, headers=headers)
              if response.status == 200:
                  data = json.loads(response.data)
                  for record in data["records"]:
                      #
                      # Since there are multiple ways to process lag time, make sure to only do it one way for each relationship.
                      processedLagTime = False
                      #
                      # If the source cluster isn't defined, then assume it is a local SM relationship.
                      if record['source'].get('cluster') is None:
                          sourceClusterName = clusterName
                      else:
                          sourceClusterName = record['source']['cluster']['name']
                      #
                      # For lag time if maxLagTimePercent is defined check to see if there is a schedule,
                      # if there is a schedule alert on that otherrwise alert on the maxLagTime.
                      # But, first check that lag_time is defined, and that the state is not "uninitialized",
                      # since the lag_time is set to the oldest snapshot of the source volume which would
                      # cause a false positive.
                      if record.get("lag_time") is not None and record["state"].lower() != "uninitialized":
                          lagSeconds = parseLagTime(record["lag_time"])
                          if maxLagTimePercent is not None:
                              lastScheduledUpdate = getLastScheduledUpdate(record)
                              if lastScheduledUpdate != -1:
                                  processedLagTime = True
                                  if lagSeconds > ((curTimeSeconds - lastScheduledUpdate) * maxLagTimePercent/100):
                                      #
                                      # If the transfer is in progress, and they have stalled transfer alert enabled, we don't need to alert on the lag time.
                                      if not (record.get("transfer") is not None and record["transfer"]["state"].lower() in ["transferring", "finalizing", "preparing", "fasttransferring"] and stalledTransferSeconds is not None):
                                          uniqueIdentifier = record["uuid"] + "_" + maxLagTimePercentKey
                                          if not eventExist(events, uniqueIdentifier):  # This resets the "refresh" field if found.
                                              timeStr = lagTimeStr(lagSeconds)
                                              asciiTime = datetime.datetime.fromtimestamp(lastScheduledUpdate).strftime('%Y-%m-%d %H:%M:%S')
                                              message = f'Snapmirror Lag Alert: {sourceClusterName}::{record["source"]["path"]} -> {clusterName}::{record["destination"]["path"]} has a lag time of {lagSeconds} seconds ({timeStr}) which is more than {maxLagTimePercent}% of its last scheduled update at {asciiTime}.'
                                              sendAlert(message, "WARNING")
                                              changedEvents=True
                                              event = {
                                                  "index": uniqueIdentifier,
                                                  "message": message,
                                                  "refresh": eventResilience
                                              }
                                              events.append(event)
          
                          if maxLagTime is not None and not processedLagTime:
                              if lagSeconds > maxLagTime:
                                  uniqueIdentifier = record["uuid"] + "_" + maxLagTimeKey
                                  if not eventExist(events, uniqueIdentifier):  # This resets the "refresh" field if found.
                                      timeStr = lagTimeStr(lagSeconds)
                                      message = f'Snapmirror Lag Alert: {sourceClusterName}::{record["source"]["path"]} -> {clusterName}::{record["destination"]["path"]} has a lag time of {lagSeconds} seconds, or {timeStr} which is more than {maxLagTime}.'
                                      sendAlert(message, "WARNING")
                                      changedEvents=True
                                      event = {
                                          "index": uniqueIdentifier,
                                          "message": message,
                                          "refresh": eventResilience
                                      }
                                      events.append(event)
          
                      if healthy is not None:
                          if not healthy and not record["healthy"]: # Report on "not healthy" and the status is "not healthy"
                              uniqueIdentifier = record["uuid"] + "_" + healthyKey
                              if not eventExist(events, uniqueIdentifier):  # This resets the "refresh" field if found.
                                  message = f'Snapmirror Health Alert: {sourceClusterName}::{record["source"]["path"]} {clusterName}::{record["destination"]["path"]} has a status of {record["healthy"]}.'
                                  for reason in record["unhealthy_reason"]:
                                      message += "\n" + reason["message"]
                                  sendAlert(message, "WARNING")
                                  changedEvents=True
                                  event = {
                                      "index": uniqueIdentifier,
                                      "message": message,
                                      "refresh": eventResilience
                                  }
                                  events.append(event)
          
                      if stalledTransferSeconds is not None:
                          if record.get('transfer') is not None and record['transfer']['state'].lower() == "transferring":
                              transferUuid = record['transfer']['uuid']
                              bytesTransferred = record['transfer']['bytes_transferred']
                              prevRec =  getPreviousSMRecord(smRelationships, transferUuid) # This reset the "refresh" field if found.
                              if prevRec != None:
                                  timeDiff=curTimeSeconds - prevRec["time"]
                                  if prevRec['bytesTransferred'] == bytesTransferred:
                                      if (curTimeSeconds - prevRec['time']) > stalledTransferSeconds:
                                          uniqueIdentifier = record['uuid'] + "_" + "transfer"
          
                                          if not eventExist(events, uniqueIdentifier):
                                              message = f"Snapmiorror transfer has stalled: {sourceClusterName}::{record['source']['path']} -> {clusterName}::{record['destination']['path']}."
                                              sendAlert(message, "WARNING")
                                              changedEvents=True
                                              event = {
                                                  "index": uniqueIdentifier,
                                                  "message": message,
                                                  "refresh": eventResilience
                                              }
                                              events.append(event)
                                  else:
                                      prevRec['time'] = curTimeSeconds
                                      prevRec['refresh'] = True
                                      prevRec['bytesTransferred'] = bytesTransferred
                                      updateRelationships = True
                              else:
                                  prevRec = {
                                      "time": curTimeSeconds,
                                      "refresh": True,
                                      "bytesTransferred": bytesTransferred,
                                      "uuid": transferUuid
                                  }
                                  updateRelationships = True
                                  smRelationships.append(prevRec)
                  #
                  # After processing the records, see if any SM relationships need to be removed.
                  i = 0
                  while i < len(smRelationships):
                      if not smRelationships[i]["refresh"]:
                          relationshipId = smRelationships[i].get("uuid")
                          if relationshipId is None:
                              id="Old format"
                          else:
                              id = relationshipId
                          logger.debug(f'Deleting smRelationship: {id}')
                          del smRelationships[i]
                          updateRelationships = True
                      else:
                          i += 1
                  #
                  # If any of the SM relationships changed, save it.
                  if(updateRelationships):
                      s3Client.put_object(Key=config["smRelationshipsFilename"], Bucket=config["s3BucketName"], Body=json.dumps(smRelationships).encode('UTF-8'))
                  #
                  # After processing the records, see if any events need to be removed.
                  i = 0
                  while i < len(events):
                      if events[i]["refresh"] <= 0:
                          logger.debug(f'Deleting event: {events[i]["message"]}')
                          del events[i]
                          changedEvents = True
                      else:
                          # If an event wasn't refreshed, then we need to save the new refresh count.
                          if events[i]["refresh"] != eventResilience:
                              changedEvents = True
                          i += 1
                  #
                  # If the events array changed, save it.
                  if(changedEvents):
                      s3Client.put_object(Key=config["smEventsFilename"], Bucket=config["s3BucketName"], Body=json.dumps(events).encode('UTF-8'))
              else:
                  logger.warning(f'API call to {endpoint} failed. HTTP status code {response.status}.')
          
          ################################################################################
          # This function is used to check all the volume and aggregate utlization.
          ################################################################################
          def processStorageUtilization(service):
              global config, s3Client, snsClient, http, headers, clusterName, clusterVersion, logger
          
              changedEvents=False
              #
              # Get the saved events so we can ensure we are only reporting on new ones.
              try:
                  data = s3Client.get_object(Key=config["storageEventsFilename"], Bucket=config["s3BucketName"])
              except botocore.exceptions.ClientError as err:
                  # If the error is that the object doesn't exist, then it will get created once an alert it sent.
                  if err.response['Error']['Code'] == "NoSuchKey":
                      events = []
                  else:
                      raise err
              else:
                  events = json.loads(data["Body"].read().decode('UTF-8'))
              #
              # Decrement the refresh field to know if any records have really gone away.
              for event in events:
                  event["refresh"] -= 1
              #
              # Run the API call to get the physical storage used.
              endpoint = f'https://{config["OntapAdminServer"]}/api/storage/aggregates?fields=space&return_timeout=15'
              aggrResponse = http.request('GET', endpoint, headers=headers)
              if aggrResponse.status != 200:
                  logger.error(f'API call to {endpoint} failed. HTTP status code {aggrResponse.status}.')
                  aggrResponse = None
              #
              # Run the API call to get the volume information.
              endpoint = f'https://{config["OntapAdminServer"]}/api/storage/volumes?fields=space,files,svm,state&return_timeout=15'
              volumeResponse = http.request('GET', endpoint, headers=headers)
              if volumeResponse.status != 200:
                  logger.error(f'API call to {endpoint} failed. HTTP status code {volumeResponse.status}.')
                  volumeResponse = None
                  volumeRecords = None
              else:
                  volumeRecords = json.loads(volumeResponse.data).get("records")
                  #
                  # Now get the constituent volumes.
                  endpoint = f'https://{config["OntapAdminServer"]}/api/storage/volumes?is_constituent=true&fields=space,files,svm,state&return_timeout=15'
                  volumeResponse = http.request('GET', endpoint, headers=headers)
                  if volumeResponse.status != 200:
                      logger.error(f'API call to {endpoint} failed. HTTP status code {volumeResponse.status}.')
                  else:
                      volumeRecords.extend(json.loads(volumeResponse.data).get("records"))
              #
              # If both API calls failed, no point on continuing.
              if volumeResponse is None and aggrResponse is None:
                  return
          
              for rule in service["rules"]:
                  for key in rule.keys():
                      lkey=key.lower()
                      if lkey == "aggrwarnpercentused" or lkey == 'aggrcriticalpercentused':
                          if aggrResponse is not None:
                              data = json.loads(aggrResponse.data)
                              for aggr in data["records"]:
                                  if aggr["space"]["block_storage"]["used_percent"] >= rule[key]:
                                      uniqueIdentifier = aggr["uuid"] + "_" + key
                                      if not eventExist(events, uniqueIdentifier):  # This resets the "refresh" field if found.
                                          alertType = 'Warning' if lkey == "aggrwarnpercentused" else 'Critical'
                                          message = f'Aggregate {alertType} Alert: Aggregate {aggr["name"]} on {clusterName} is {aggr["space"]["block_storage"]["used_percent"]}% full, which is more or equal to {rule[key]}% full.'
                                          sendAlert(message, "WARNING")
                                          changedEvents = True
                                          event = {
                                                  "index": uniqueIdentifier,
                                                  "message": message,
                                                  "refresh": eventResilience
                                              }
                                          logger.debug(event)
                                          events.append(event)
                      elif lkey == "volumewarnpercentused" or lkey == "volumecriticalpercentused":
                          if volumeResponse is not None:
                              for record in volumeRecords:
                                  if record["space"].get("percent_used"):
                                      if record["space"]["percent_used"] >= rule[key]:
                                          uniqueIdentifier = record["uuid"] + "_" + key
                                          if not eventExist(events, uniqueIdentifier):  # This resets the "refresh" field if found.
                                              alertType = 'Warning' if lkey == "volumewarnpercentused" else 'Critical'
                                              message = f'Volume Usage {alertType} Alert: volume {record["svm"]["name"]}:{record["name"]} on {clusterName} is {record["space"]["percent_used"]}% full, which is more or equal to {rule[key]}% full.'
                                              sendAlert(message, "WARNING")
                                              changedEvents = True
                                              event = {
                                                      "index": uniqueIdentifier,
                                                      "message": message,
                                                      "refresh": eventResilience
                                                  }
                                              events.append(event)
                      elif lkey == "volumewarnfilespercentused" or lkey == "volumecriticalfilespercentused":
                          if volumeResponse is not None:
                              for record in volumeRecords:
                                  #
                                  # If a volume is offline, the API will not report the "files" information.
                                  if record.get("files") is not None:
                                      maxFiles = record["files"].get("maximum")
                                      usedFiles = record["files"].get("used")
                                      if maxFiles != None and usedFiles != None:
                                          percentUsed = (usedFiles / maxFiles) * 100
                                          if percentUsed >= rule[key]:
                                              uniqueIdentifier = record["uuid"] + "_" + key
                                              if not eventExist(events, uniqueIdentifier):
                                                  alertType = 'Warning' if lkey == "volumewarnfilespercentused" else 'Critical'
                                                  message = f"Volume File (inode) Usage {alertType} Alert: volume {record['svm']['name']}:{record['name']} on {clusterName} is using {percentUsed:.0f}% of it's inodes, which is more or equal to {rule[key]}% utilization."
                                                  sendAlert(message, "WARNING")
                                                  changedEvents = True
                                                  event = {
                                                          "index": uniqueIdentifier,
                                                          "message": message,
                                                          "refresh": eventResilience
                                                      }
                                                  events.append(event)
                      elif lkey == "offline":
                          for record in volumeRecords:
                              if rule[key] and record["state"].lower() == "offline":
                                  uniqueIdentifier = f'{record["uuid"]}_{key}_{rule[key]}'
                                  if not eventExist(events, uniqueIdentifier):  # This resets the "refresh" field if found.
                                      message = f"Volume Offline Alert: volume {record['svm']['name']}:{record['name']} on {clusterName} is offline."
                                      sendAlert(message, "WARNING")
                                      changedEvents=True
                                      event = {
                                          "index": uniqueIdentifier,
                                          "message": message,
                                          "refresh": eventResilience
                                      }
                                      events.append(event)
                      else:
                          message = f'Unknown storage alert type: "{key}".'
                          logger.warning(message)
              #
              # After processing the records, see if any events need to be removed.
              i = 0
              while i < len(events):
                  if events[i]["refresh"] <= 0:
                      logger.debug(f'Deleting event: {events[i]["message"]}')
                      del events[i]
                      changedEvents = True
                  else:
                      # If an event wasn't refreshed, then we need to save the new refresh count.
                      if events[i]["refresh"] != eventResilience:
                          changedEvents = True
                      i += 1
              #
              # If the events array changed, save it.
              if(changedEvents):
                  s3Client.put_object(Key=config["storageEventsFilename"], Bucket=config["s3BucketName"], Body=json.dumps(events).encode('UTF-8'))
          
          ################################################################################
          # This function sends the message to the various alerting systems.
          ################################################################################
          def sendAlert(message, severity):
              global config, snsClient, logger, cloudWatchClient
          
              if severity == "CRITICAL":
                  logger.critical(message)
              elif severity == "ERROR":
                  logger.error(message)
              elif severity == "WARNING":
                  logger.warning(message)
              elif severity == "INFO":
                  logger.info(message)
              elif severity == "DEBUG":
                  logger.debug(message)
              else:
                  logger.info(message)
          
              snsClient.publish(TopicArn=config["snsTopicArn"], Message=message, Subject=f'{severity}: Monitor ONTAP Services Alert for cluster {clusterName}')
          
              if cloudWatchClient is not None:
                  #
                  # Create a new log stream for the current day if it doesn't exist.
                  dateStr = datetime.datetime.now().strftime("%Y-%m-%d")
                  logStreamName = f'{clusterName}-monitor-ontap-services-{dateStr}'
                  #
                  # Don't ask me why AWS puts a ":*" at the end of the log group ARN, but they do.
                  logGroupName = config["cloudWatchLogGroupArn"].split(":")[-2] if config["cloudWatchLogGroupArn"].endswith(":*") else config["cloudWatchLogGroupArn"].split(":")[-1]
                  #
                  # Check to see if the log stream already exists.
                  logStreams = cloudWatchClient.describe_log_streams(logGroupName=logGroupName, logStreamNamePrefix=logStreamName)
                  if len(logStreams["logStreams"]) == 0:
                      cloudWatchClient.create_log_stream(
                          logGroupName=logGroupName,
                          logStreamName=logStreamName)
                  #
                  # Send the message to CloudWatch.
                  cloudWatchClient.put_log_events(
                      logGroupName=logGroupName,
                      logStreamName=logStreamName,
                      logEvents=[
                          {
                              'timestamp': int(datetime.datetime.now().timestamp() * 1000),
                              'message': message
                          },
                      ]
                  )
          
          ################################################################################
          # This function is used to check utilization of quota limits.
          ################################################################################
          def processQuotaUtilization(service):
              global config, s3Client, snsClient, http, headers, clusterName, clusterVersion, logger
          
              changedEvents=False
              #
              # Get the saved events so we can ensure we are only reporting on new ones.
              try:
                  data = s3Client.get_object(Key=config["quotaEventsFilename"], Bucket=config["s3BucketName"])
              except botocore.exceptions.ClientError as err:
                  # If the error is that the object doesn't exist, then it will get created once an alert it sent.
                  if err.response['Error']['Code'] == "NoSuchKey":
                      events = []
                  else:
                      raise err
              else:
                  events = json.loads(data["Body"].read().decode('UTF-8'))
              #
              # Decrement the refresh field to know if any records have really gone away.
              for event in events:
                  event["refresh"] -= 1
              #
              # Run the API call to get the quota report.
              endpoint = f'https://{config["OntapAdminServer"]}/api/storage/quota/reports?fields=*&return_timeout=30'
              response = http.request('GET', endpoint, headers=headers)
              if response.status == 200:
                  data = json.loads(response.data)
                  for record in data["records"]:
                      for rule in service["rules"]:
                          for key in rule.keys():
                              lkey = key.lower() # Convert to all lower case so the key can be case insensitive.
                              if lkey == "maxquotainodespercentused":
                                  #
                                  # Since the quota report might not have the files key, and even if it does, it might not have
                                  # the hard_limit_percent" key, need to check for their existencae first.
                                  if(record.get("files") is not None and record["files"]["used"].get("hard_limit_percent") is not None and
                                          record["files"]["used"]["hard_limit_percent"] > rule[key]):
                                      uniqueIdentifier = str(record["index"]) + "_" + key
                                      if not eventExist(events, uniqueIdentifier):  # This resets the "refresh" field if found.
                                          if record.get("qtree") is not None:
                                              qtree=f' under qtree: {record["qtree"]["name"]} '
                                          else:
                                              qtree=' '
                                          if record.get("users") is not None:
                                              users=None
                                              for user in record["users"]:
                                                  if users is None:
                                                      users = user["name"]
                                                  else:
                                                      users += ',{user["name"]}'
                                              user=f'associated with user(s) "{users}" '
                                          else:
                                              user=''
                                          message = f'Quota Inode Usage Alert: Quota of type "{record["type"]}" on {record["svm"]["name"]}:/{record["volume"]["name"]}{qtree}{user}on {clusterName} is using {record["files"]["used"]["hard_limit_percent"]}% which is more than {rule[key]}% of its inodes.'
                                          sendAlert(message, "WARNING")
                                          changedEvents=True
                                          event = {
                                                  "index": uniqueIdentifier,
                                                  "message": message,
                                                  "refresh": eventResilience
                                                  }
                                          logger.debug(message)
                                          events.append(event)
                              elif lkey == "maxhardquotaspacepercentused":
                                  if(record.get("space") is not None and record["space"]["used"].get("hard_limit_percent") and
                                          record["space"]["used"]["hard_limit_percent"] >= rule[key]):
                                      uniqueIdentifier = str(record["index"]) + "_" + key
                                      if not eventExist(events, uniqueIdentifier):  # This resets the "refresh" field if found.
                                          if record.get("qtree") is not None:
                                              qtree=f' under qtree: {record["qtree"]["name"]} '
                                          else:
                                              qtree=" "
                                          if record.get("users") is not None:
                                              users=None
                                              for user in record["users"]:
                                                  if users is None:
                                                      users = user["name"]
                                                  else:
                                                      users += ',{user["name"]}'
                                              user=f'associated with user(s) "{users}" '
                                          else:
                                              user=''
                                          message = f'Quota Space Usage Alert: Hard quota of type "{record["type"]}" on {record["svm"]["name"]}:/{record["volume"]["name"]}{qtree}{user}on {clusterName} is using {record["space"]["used"]["hard_limit_percent"]}% which is more than {rule[key]}% of its allocaed space.'
                                          sendAlert(message, "WARNING")
                                          changedEvents=True
                                          event = {
                                                  "index": uniqueIdentifier,
                                                  "message": message,
                                                  "refresh": eventResilience
                                                  }
                                          logger.debug(message)
                                          events.append(event)
                              elif lkey == "maxsoftquotaspacepercentused":
                                  if(record.get("space") is not None and record["space"]["used"].get("soft_limit_percent") and
                                          record["space"]["used"]["soft_limit_percent"] >= rule[key]):
                                      uniqueIdentifier = str(record["index"]) + "_" + key
                                      if not eventExist(events, uniqueIdentifier):  # This resets the "refresh" field if found.
                                          if record.get("qtree") is not None:
                                              qtree=f' under qtree: {record["qtree"]["name"]} '
                                          else:
                                              qtree=" "
                                          if record.get("users") is not None:
                                              users=None
                                              for user in record["users"]:
                                                  if users is None:
                                                      users = user["name"]
                                                  else:
                                                      users += ',{user["name"]}'
                                              user=f'associated with user(s) "{users}" '
                                          else:
                                              user=''
                                          message = f'Quota Space Usage Alert: Soft quota of type "{record["type"]}" on {record["svm"]["name"]}:/{record["volume"]["name"]}{qtree}{user}on {clusterName} is using {record["space"]["used"]["soft_limit_percent"]}% which is more than {rule[key]}% of its allocaed space.'
                                          sendAlert(message, "WARNING")
                                          changedEvents=True
                                          event = {
                                              "index": uniqueIdentifier,
                                              "message": message,
                                              "refresh": eventResilience
                                          }
                                          logger.debug(message)
                                          events.append(event)
                              else:
                                  message = f'Unknown quota matching condition type "{key}".'
                                  logger.warning(message)
                  #
                  # After processing the records, see if any events need to be removed.
                  i=0
                  while i < len(events):
                      if events[i]["refresh"] <= 0:
                          logger.debug(f'Deleting event: {events[i]["message"]}')
                          del events[i]
                          changedEvents = True
                      else:
                          # If an event wasn't refreshed, then we need to save the new refresh count.
                          if events[i]["refresh"] != eventResilience:
                              changedEvents = True
                          i += 1
                  #
                  # If the events array changed, save it.
                  if(changedEvents):
                      s3Client.put_object(Key=config["quotaEventsFilename"], Bucket=config["s3BucketName"], Body=json.dumps(events).encode('UTF-8'))
              else:
                  logger.error(f'API call to {endpoint} failed. HTTP status code {response.status}.')
          
          ################################################################################
          ################################################################################
          def processVserver(service):
              global config, s3Client, snsClient, http, headers, clusterName, logger
          
              changedEvents=False
              #
              # Get the saved events so we can ensure we are only reporting on new ones.
              try:
                  data = s3Client.get_object(Key=config["vserverEventsFilename"], Bucket=config["s3BucketName"])
              except botocore.exceptions.ClientError as err:
                  # If the error is that the object doesn't exist, then it will get created once an alert it sent.
                  if err.response['Error']['Code'] == "NoSuchKey":
                      events = []
                  else:
                      raise err
              else:
                  events = json.loads(data["Body"].read().decode('UTF-8'))
              #
              # Decrement the refresh field to know if any records have really gone away.
              for event in events:
                  event["refresh"] -= 1
              #
              # Consolidate the rules
              vserverState = None
              nfsProtocolState = None
              cifsProtocolState = None
              for rule in service["rules"]:
                  for key in rule.keys():
                      lkey = key.lower() # Convert to all lower case so the key can be case insensitive.
                      if lkey == "vserverstate":
                          vserverState = rule[key]
                          vserverStateKey = key
                      elif lkey == "nfsprotocolstate":
                          nfsProtocolState = rule[key]
                          nfsProtocolStateKey = key
                      elif lkey == "cifsprotocolstate":
                          cifsProtocolState = rule[key]
                          cifsProtocolStateKey = key
              #
              # Check for any vservers that are down.
              if vserverState is not None and vserverState:
                  #
                  # Run the API call to get the vserver state for each vserver.
                  endpoint = f'https://{config["OntapAdminServer"]}/api/svm/svms?fields=state&return_timeout=15'
                  response = http.request('GET', endpoint, headers=headers)
                  if response.status == 200:
                      data = json.loads(response.data)
                      for record in data["records"]:
                          if record["state"].lower() != "running":
                              uniqueIdentifier = str(record["uuid"]) + "_" + vserverStateKey
                              if not eventExist(events, uniqueIdentifier):
                                  message = f'SVM State Alert: SVM {record["name"]} on {clusterName} is not online.'
                                  sendAlert(message, "WARNING")
                                  changedEvents=True
                                  event = {
                                          "index": uniqueIdentifier,
                                          "message": message,
                                          "refresh": eventResilience
                                          }
                                  events.append(event)
                  else:
                      logger.error(f'API call to {endpoint} failed. HTTP status code {response.status}.')
          
              if nfsProtocolState is not None and nfsProtocolState:
                  #
                  # Run the API call to get the NFS protocol state for each vserver.
                  endpoint = f'https://{config["OntapAdminServer"]}/api/protocols/nfs/services?fields=state&return_timeout=15'
                  response = http.request('GET', endpoint, headers=headers)
                  if response.status == 200:
                      data = json.loads(response.data)
                      for record in data["records"]:
                          if record["state"].lower() != "online":
                              uniqueIdentifier = str(record["svm"]["uuid"]) + "_" + nfsProtocolStateKey
                              if not eventExist(events, uniqueIdentifier):
                                  message = f'NFS Protocol State Alert: NFS protocol on {record["svm"]["name"]} on {clusterName} is not online.'
                                  sendAlert(message, "WARNING")
                                  changedEvents=True
                                  event = {
                                          "index": uniqueIdentifier,
                                          "message": message,
                                          "refresh": eventResilience
                                          }
                                  events.append(event) 
                  else:
                      logger.error(f'API call to {endpoint} failed. HTTP status code {response.status}.')
          
              if cifsProtocolState is not None and cifsProtocolState:
                  #
                  # Run the API call to get the NFS protocol state for each vserver.
                  endpoint = f'https://{config["OntapAdminServer"]}/api/protocols/cifs/services?fields=enabled&return_timeout=15'
                  response = http.request('GET', endpoint, headers=headers)
                  if response.status == 200:
                      data = json.loads(response.data)
                      for record in data["records"]:
                          if not record["enabled"]:
                              uniqueIdentifier = str(record["svm"]["uuid"]) + "_" + cifsProtocolStateKey
                              if not eventExist(events, uniqueIdentifier):
                                  message = f'CIFS Protocol State Alert: CIFS protocol on {record["svm"]["name"]} on {clusterName} is not online.'
                                  sendAlert(message, "WARNING")
                                  changedEvents=True
                                  event = {
                                          "index": uniqueIdentifier,
                                          "message": message,
                                          "refresh": eventResilience
                                          }
                                  events.append(event) 
                  else:
                      logger.error(f'API call to {endpoint} failed. HTTP status code {response.status}.')
          
              #
              # After processing the records, see if any events need to be removed.
              i=0
              while i < len(events):
                  if events[i]["refresh"] <= 0:
                      logger.debug(f'Deleting event: {events[i]["message"]}')
                      del events[i]
                      changedEvents = True
                  else:
                      # If an event wasn't refreshed, then we need to save the new refresh count.
                      if events[i]["refresh"] != eventResilience:
                          changedEvents = True
                      i += 1
              #
              # If the events array changed, save it.
              if(changedEvents):
                  s3Client.put_object(Key=config["vserverEventsFilename"], Bucket=config["s3BucketName"], Body=json.dumps(events).encode('UTF-8'))
          
          ################################################################################
          # This function returns the index of the service in the conditions dictionary.
          ################################################################################
          def getServiceIndex(targetService, conditions):
          
              i = 0
              while i < len(conditions["services"]):
                  if conditions["services"][i]["name"] == targetService:
                      return i
                  i += 1
          
              return None
          
          ################################################################################
          # This function builds a default matching conditions dictionary based on the
          # environment variables passed in.
          ################################################################################
          def buildDefaultMatchingConditions():
              #
              # Define global variables so we don't have to pass them to all the functions.
              global config, s3Client, snsClient, http, headers, clusterName, clusterVersion, logger
              #
              # Define an empty matching conditions dictionary.
              conditions = { "services": [
                  {"name": "systemHealth", "rules": []},
                  {"name": "ems", "rules": []},
                  {"name": "snapmirror", "rules": []},
                  {"name": "storage", "rules": []},
                  {"name": "quota", "rules": []},
                  {"name": "vserver", "rules": []}
              ]}
              #
              # Now, add rules based on the environment variables.
              for name, value in os.environ.items():
                  if name == "initialVersionChangeAlert":
                      if value == "true":
                          conditions["services"][getServiceIndex("systemHealth", conditions)]["rules"].append({"versionChange": True})
                      else:
                          conditions["services"][getServiceIndex("systemHealth", conditions)]["rules"].append({"versionChange": False})
                  elif name == "initialFailoverAlert":
                      if value == "true":
                          conditions["services"][getServiceIndex("systemHealth", conditions)]["rules"].append({"failover": True})
                      else:
                          conditions["services"][getServiceIndex("systemHealth", conditions)]["rules"].append({"failover": False})
                  elif name == "initialNetworkInterfacesAlert":
                      if value == "true":
                          conditions["services"][getServiceIndex("systemHealth", conditions)]["rules"].append({"networkInterfaces": True})
                      else:
                          conditions["services"][getServiceIndex("systemHealth", conditions)]["rules"].append({"networkInterfaces": False})
                  elif name == "initialEmsEventsAlert":
                      if value == "true":
                          conditions["services"][getServiceIndex("ems", conditions)]["rules"].append({"name": "", "severity": "error|alert|emergency", "message": ""})
                  elif name == "initialSnapMirrorHealthAlert":
                      if value == "true":
                          conditions["services"][getServiceIndex("snapmirror", conditions)]["rules"].append({"Healthy": False})  # This is what it matches on, so it is interesting when the health is false.
                      else:
                          conditions["services"][getServiceIndex("snapmirror", conditions)]["rules"].append({"Healthy": True})
                  elif name == "initialSnapMirrorLagTimeAlert":
                      value = int(value)
                      if value > 0:
                          conditions["services"][getServiceIndex("snapmirror", conditions)]["rules"].append({"maxLagTime": value})
                  elif name == "initialSnapMirrorLagTimePercentAlert":
                      value = int(value)
                      if value > 0:
                          conditions["services"][getServiceIndex("snapmirror", conditions)]["rules"].append({"maxLagTimePercent": value})
                  elif name == "initialSnapMirrorStalledAlert":
                      value = int(value)
                      if value > 0:
                          conditions["services"][getServiceIndex("snapmirror", conditions)]["rules"].append({"stalledTransferSeconds": value})
                  elif name == "initialFileSystemUtilizationWarnAlert":
                      value = int(value)
                      if value > 0:
                          conditions["services"][getServiceIndex("storage", conditions)]["rules"].append({"aggrWarnPercentUsed": value})
                  elif name == "initialFileSystemUtilizationCriticalAlert":
                      value = int(value)
                      if value > 0:
                          conditions["services"][getServiceIndex("storage", conditions)]["rules"].append({"aggrCriticalPercentUsed": value})
                  elif name == "initialVolumeUtilizationWarnAlert":
                      value = int(value)
                      if value > 0:
                          conditions["services"][getServiceIndex("storage", conditions)]["rules"].append({"volumeWarnPercentUsed": value})
                  elif name == "initialVolumeUtilizationCriticalAlert":
                      value = int(value)
                      if value > 0:
                          conditions["services"][getServiceIndex("storage", conditions)]["rules"].append({"volumeCriticalPercentUsed": value})
                  elif name == "initialVolumeFileUtilizationWarnAlert":
                      value = int(value)
                      if value > 0:
                          conditions["services"][getServiceIndex("storage", conditions)]["rules"].append({"volumeWarnFilesPercentUsed": value})
                  elif name == "initialVolumeFileUtilizationCriticalAlert":
                      value = int(value)
                      if value > 0:
                          conditions["services"][getServiceIndex("storage", conditions)]["rules"].append({"volumeCriticalFilesPercentUsed": value})
                  elif name == "initialVolumeOfflineAlert":
                      if value == "true":
                          conditions["services"][getServiceIndex("storage", conditions)]["rules"].append({"offline": True})
                      else:
                          conditions["services"][getServiceIndex("storage", conditions)]["rules"].append({"offline": False})
                  elif name == "initialSoftQuotaUtilizationAlert":
                      value = int(value)
                      if value > 0:
                          conditions["services"][getServiceIndex("quota", conditions)]["rules"].append({"maxSoftQuotaSpacePercentUsed": value})
                  elif name == "initialHardQuotaUtilizationAlert":
                      value = int(value)
                      if value > 0:
                          conditions["services"][getServiceIndex("quota", conditions)]["rules"].append({"maxHardQuotaSpacePercentUsed": value})
                  elif name == "initialInodesQuotaUtilizationAlert":
                      value = int(value)
                      if value > 0:
                          conditions["services"][getServiceIndex("quota", conditions)]["rules"].append({"maxQuotaInodesPercentUsed": value})
                  elif name == "initialVserverStateAlert":
                      if value == "true":
                          conditions["services"][getServiceIndex("vserver", conditions)]["rules"].append({"vserverState": True})
                      else:
                          conditions["services"][getServiceIndex("vserver", conditions)]["rules"].append({"vserverState": False})
                  elif name == "initialVserverNFSProtocolStateAlert":
                      if value == "true":
                          conditions["services"][getServiceIndex("vserver", conditions)]["rules"].append({"nfsProtocolState": True})
                      else:
                          conditions["services"][getServiceIndex("vserver", conditions)]["rules"].append({"nfsProtocolState": False})
                  elif name == "initialVserverCIFSProtocolStateAlert":
                      if value == "true":
                          conditions["services"][getServiceIndex("vserver", conditions)]["rules"].append({"cifsProtocolState": True})
                      else:
                          conditions["services"][getServiceIndex("vserver", conditions)]["rules"].append({"cifsProtocolState": False})
          
              return conditions
          
          ################################################################################
          # This function is used to read in all the configuration parameters from the
          # various places:
          #   Environment Variables
          #   Config File
          #   Calculated
          ################################################################################
          def readInConfig():
              #
              # Define global variables so we don't have to pass them to all the functions.
              global config, s3Client, snsClient, http, headers, clusterName, clusterVersion, logger
              #
              # Define a dictionary with all the required variables so we can
              # easily add them and check for their existence.
              requiredEnvVariables = {
                  "OntapAdminServer": None,
                  "s3BucketName": None,
                  "s3BucketRegion": None
                  }
          
              optionalVariables = {
                  "configFilename": None,
                  "secretsManagerEndPointHostname": None,
                  "snsEndPointHostname": None,
                  "cloudWatchLogsEndPointHostname": None,
                  "syslogIP": None,
                  "cloudWatchLogGroupArn": None,
                  "awsAccountId": None
                  }
          
              filenameVariables = {
                  "emsEventsFilename": None,
                  "smEventsFilename": None,
                  "smRelationshipsFilename": None,
                  "conditionsFilename": None,
                  "storageEventsFilename": None,
                  "quotaEventsFilename": None,
                  "systemStatusFilename": None,
                  "vserverEventsFilename": None
                  }
          
              config = {
                  "snsTopicArn": None,
                  "secretArn": None,
                  "secretUsernameKey": None,
                  "secretPasswordKey": None
                  }
              config.update(filenameVariables)
              config.update(optionalVariables)
              config.update(requiredEnvVariables)
              #
              # Get the required, and any additional, paramaters from the environment.
              for var in config:
                  config[var] = os.environ.get(var)
              #
              # Since the CloudFormation template will set the environment variables
              # to an empty string if someone doesn't provide a value, reset the
              # values back to None.
              for var in config:
                  if config[var] == "":
                      config[var] = None
              #
              # Since CloudFormation has to pass an ARN, get the Bucket name from it.
              # Too bad the bucket ARN doesn't include the region, like most (all?) the others do.
              if config["s3BucketName"] is None and os.environ.get("s3BucketArn") is not None:
                  config["s3BucketName"] = os.environ.get("s3BucketArn").split(":")[-1]
              #
              # Check that required environmental variables are there.
              for var in requiredEnvVariables:
                  if config[var] is None:
                      raise Exception (f'\n\nMissing required environment variable "{var}".')
              #
              # Open a client to the s3 service.
              s3Client = boto3.client('s3', config["s3BucketRegion"])
              #
              # Calculate the config filename if it hasn't already been provided.
              defaultConfigFilename = config["OntapAdminServer"] + "-config"
              if config["configFilename"] is None:
                  config["configFilename"] = defaultConfigFilename
              #
              # Process the config file if it exist.
              try:
                  lines = s3Client.get_object(Key=config["configFilename"], Bucket=config["s3BucketName"])['Body'].iter_lines()
              except botocore.exceptions.ClientError as err:
                  if err.response['Error']['Code'] != "NoSuchKey":
                      raise err
                  else:
                      if config["configFilename"] != defaultConfigFilename:
                          logger.warning(f"Warning, did not find file '{config['configFilename']}' in s3 bucket '{config['s3BucketName']}' in region '{config['s3BucketRegion']}'.")
              else:
                  #
                  # While iterating through the file, get rid of any "export ", comments, blank lines, or anything else that isn't key=value.
                  for line in lines:
                      line = line.decode('utf-8')
                      if line[0:7] == "export ":
                          line = line[7:]
                      comment = line.split("#")
                      line=comment[0].strip().replace('"', '')
                      x = line.split("=")
                      if len(x) == 2:
                          (key, value) = line.split("=")
                      key = key.strip()
                      value = value.strip()
                      if len(value) == 0:
                          logger.warning(f"Warning, empty value for key '{key}'. Ignored.")
                      else:
                          #
                          # Preserve any environment variables settings.
                          if key in config:
                              if config[key] is None:
                                  config[key] = value
                          else:
                              logger.warning(f"Warning, unknown config parameter '{key}'.")
              #
              # Now, fill in the filenames for any that aren't already defined.
              for filename in filenameVariables:
                  if config[filename] is None:
                      config[filename] = config["OntapAdminServer"] + "-" + filename.replace("Filename", "")
              #
              # Define endpoints if alternates weren't provided.
              if config.get("secretArn") is not None and config["secretsManagerEndPointHostname"] is None:
                  secretRegion = config["secretArn"].split(":")[3]
                  config["secretsManagerEndPointHostname"] = f'secretsmanager.{secretRegion}.amazonaws.com'
          
              if config.get("snsTopicArn") is not None and config["snsEndPointHostname"] is None:
                  snsRegion = config["snsTopicArn"].split(":")[3]
                  config["snsEndPointHostname"] = f'sns.{snsRegion}.amazonaws.com'
          
              if config.get("cloudWatchLogGroupArn") is not None and config["cloudWatchLogsEndPointHostname"] is None:
                  cloudWatchRegion = config["cloudWatchLogGroupArn"].split(":")[3]
                  config["cloudWatchLogsEndPointHostname"] = f'logs.{cloudWatchRegion}.amazonaws.com'
              #
              # Now, check that all the configuration parameters have been set.
              for key in config:
                  if config[key] is None and key not in optionalVariables:
                      raise Exception(f'\n\nMissing configuration parameter "{key}".\n\n')
          
          ################################################################################
          # Main logic
          ################################################################################
          def lambda_handler(event, context):
              #
              # Define global variables so we don't have to pass them to all the functions.
              global config, s3Client, snsClient, http, headers, clusterName, clusterVersion, logger, cloudWatchClient, clusterTimezone
              #
              # Set up logging.
              logger = logging.getLogger("mon_fsxn_service")
              if lambdaFunction:
                  logger.setLevel(logging.INFO)       # Anything at this level and above this get logged.
              else: # Assume we are running in a test environment.
                  logger.setLevel(logging.DEBUG)      # Anything at this level and above this get logged.
                  formatter = logging.Formatter(
                          fmt="%(name)s:%(funcName)s - Level:%(levelname)s - Message:%(message)s",
                          datefmt="%Y-%m-%d %H:%M:%S"
                      )
                  loggerscreen = logging.StreamHandler()
                  loggerscreen.setFormatter(formatter)
                  logger.addHandler(loggerscreen)
              #
              # Read in the configuraiton.
              readInConfig()   # This defines the s3Client variable.
              #
              # Set up the logger to log to a file and to syslog.
              if config["syslogIP"] is not None:
                  #
                  # Due to a bug with the SysLogHandler() of not sending proper framing with a message
                  # when using TCP (it should end it with a LF and not a NUL like it does now) you must add
                  # an additional frame delimiter to the receiving syslog server. With rsyslog, you add
                  # a AddtlFrameDelimiter="0" directive to the "input()" line where they have it listen
                  # to a TCP port. For example:
                  #
                  #  # provides TCP syslog reception
                  #  module(load="imtcp")
                  #  input(type="imtcp" port="514" AddtlFrameDelimiter="0")
                  #
                  # Because of this bug, I am going to stick with UDP, the default protocol used by
                  # the syslog handler. If TCP is required, then the above changes will have to be made
                  # to the syslog server. Or, the program will have to handle closing and opening the
                  # connection for each message. The following will do that:
                  #    handler.flush()
                  #    handler.close()
                  #    logger.removeHandler(handler)
                  #    handler = logging.handlers.SysLogHandler(facility=SysLogHandler.LOG_LOCAL0, address=(syslogIP, 514), socktype=socket.SOCK_STREAM)
                  #    handler.setFormatter(formatter)
                  #    logger.addHandler(handler)
                  #
                  # You might get away with a simple handler.open() after the close(), without having to
                  # remove and add the handler. I didn't test that.
                  handler = logging.handlers.SysLogHandler(facility=SysLogHandler.LOG_LOCAL0, address=(config["syslogIP"], 514))
                  formatter = logging.Formatter(
                          fmt="%(name)s:%(funcName)s - Level:%(levelname)s - Message:%(message)s",
                          datefmt="%Y-%m-%d %H:%M:%S"
                      )
                  handler.setFormatter(formatter)
                  logger.addHandler(handler)
              #
              # Create a Secrets Manager client.
              session = boto3.session.Session()
              secretRegion = config["secretArn"].split(":")[3]
              client = session.client(service_name='secretsmanager', region_name=secretRegion, endpoint_url=f'https://{config["secretsManagerEndPointHostname"]}')
              #
              # Get the username and password of the ONTAP/FSxN system.
              secretsInfo = client.get_secret_value(SecretId=config["secretArn"])
              secrets = json.loads(secretsInfo['SecretString'])
              if secrets.get(config['secretUsernameKey']) is None:
                  logger.critical(f'Error, "{config["secretUsernameKey"]}" not found in secret "{config["secretArn"]}".')
                  return
          
              if secrets.get(config['secretPasswordKey']) is None:
                  logger.critical(f'Error, "{config["secretPasswordKey"]}" not found in secret "{config["secretArn"]}".')
                  return
          
              username = secrets[config['secretUsernameKey']]
              password = secrets[config['secretPasswordKey']]
              #
              # Create clients to the other AWS services we will be using.
              #s3Client = boto3.client('s3', config["s3BucketRegion"])  # Defined in readInConfig()
              snsRegion = config["snsTopicArn"].split(":")[3]
              snsClient = boto3.client('sns', region_name=snsRegion, endpoint_url=f'https://{config["snsEndPointHostname"]}')
              cloudWatchClient = None
              if config["cloudWatchLogGroupArn"] is not None:
                  cloudWatchRegion = config["cloudWatchLogGroupArn"].split(":")[3]
                  cloudWatchClient = boto3.client('logs', region_name=cloudWatchRegion, endpoint_url=f'https://{config["cloudWatchLogsEndPointHostname"]}')
              #
              # Create a http handle to make ONTAP/FSxN API calls with.
              auth = urllib3.make_headers(basic_auth=f'{username}:{password}')
              headers = { **auth }
              #
              # Disable warning about connecting to servers with self-signed SSL certificates.
              urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
              retries = Retry(total=None, connect=1, read=1, redirect=10, status=0, other=0)  # pylint: disable=E1123
              http = urllib3.PoolManager(cert_reqs='CERT_NONE', retries=retries)
              #
              # Get the conditions we know what to alert on.
              try:
                  data = s3Client.get_object(Key=config["conditionsFilename"], Bucket=config["s3BucketName"])
                  matchingConditions = json.loads(data["Body"].read().decode('UTF-8'))
              except botocore.exceptions.ClientError as err:
                  if err.response['Error']['Code'] != "NoSuchKey":
                      logger.error(f'Error, could not retrieve configuration file {config["conditionsFilename"]} from: s3://{config["s3BucketName"]}.\nBelow is additional information:')
                      raise err
                  else:
                      matchingConditions = buildDefaultMatchingConditions()
                      s3Client.put_object(Key=config["conditionsFilename"], Bucket=config["s3BucketName"], Body=json.dumps(matchingConditions, indent=4).encode('UTF-8'))
              except json.decoder.JSONDecodeError as err:
                  logger.error(f'Error, could not decode JSON from configuration file "{config["conditionsFilename"]}". The error message from the decoder:\n{err}\n')
                  return
          
              if(checkSystem()):
                  #
                  # Loop on all the configured ONTAP services we want to check on.
                  for service in matchingConditions["services"]:
                      if service["name"].lower() == "systemhealth":
                          checkSystemHealth(service)
                      elif service["name"].lower() == "ems":
                          processEMSEvents(service)
                      elif (service["name"].lower() == "snapmirror"):
                          processSnapMirrorRelationships(service)
                      elif service["name"].lower() == "storage":
                          processStorageUtilization(service)
                      elif service["name"].lower() == "quota":
                          processQuotaUtilization(service)
                      elif service["name"].lower() == "vserver":
                          processVserver(service)
                      else:
                          logger.warning(f'Unknown service "{service["name"]}".')
              return
          
          if os.environ.get('AWS_LAMBDA_FUNCTION_NAME') is None:
              lambdaFunction = False
              lambda_handler(None, None)
          else:
              lambdaFunction = True