template.yaml

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: MIT-0
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of this
# software and associated documentation files (the "Software"), to deal in the Software
# without restriction, including without limitation the rights to use, copy, modify,
# merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31
Description: >
  aws-batch-job

  Sample SAM Template for aws-batch-job

Resources:
  BatchJobData:
    Type: AWS::DynamoDB::Table
    Properties:
      AttributeDefinitions:
        - AttributeName: jobId
          AttributeType: S
      BillingMode: PAY_PER_REQUEST
      ContributorInsightsSpecification:
        Enabled: false
      KeySchema:
        - AttributeName: jobId
          KeyType: HASH
      PointInTimeRecoverySpecification:
         PointInTimeRecoveryEnabled: true
      SSESpecification:
        SSEEnabled: true
      StreamSpecification:
        StreamViewType: NEW_IMAGE
      TableClass: STANDARD
      Tags:
      - Key: "Application"
        Value: "BatchOperationalDashboard"
      TimeToLiveSpecification:
        AttributeName: ExpirationTime
        Enabled: true

  BatchJobStateChangeLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
       RetentionInDays: 7

  BatchJobStateChange:
    Type: AWS::Serverless::StateMachine
    Properties:
      DefinitionSubstitutions:
        BatchJobData: !Ref BatchJobData
        ECSRegistrationTable: !Ref ECSInstanceRegistrationTable
      DefinitionUri: src/batch-job-states.json
      Events:
        CloudTrail:
          Type: CloudWatchEvent
          Description: "Capture AWS Batch jobs states transitions"
          Properties:
            Pattern:
              detail-type:
              - 'Batch Job State Change'
              source:
              - 'aws.batch'
      Logging:
        Destinations:
          - CloudWatchLogsLogGroup:
              LogGroupArn: !GetAtt BatchJobStateChangeLogGroup.Arn
        Level: 'ALL'
      Policies:
        - CloudWatchLogsFullAccess
        - DynamoDBWritePolicy:
            TableName: !Ref BatchJobData
        - DynamoDBReadPolicy:
            TableName: !Ref ECSInstanceRegistrationTable
      Tags:
        "Application": "BatchOperationalDashboard"
      Tracing:
        Enabled: true
      Type: EXPRESS

### ECS Instance Registration
  ECSInstanceRegistrationTable:
    Type: AWS::DynamoDB::Table
    Properties:
      AttributeDefinitions:
        - AttributeName: containerInstanceArn
          AttributeType: S
      BillingMode: PAY_PER_REQUEST
      ContributorInsightsSpecification:
        Enabled: false
      KeySchema:
        - AttributeName: containerInstanceArn
          KeyType: HASH
      PointInTimeRecoverySpecification:
         PointInTimeRecoveryEnabled: true
      SSESpecification:
        SSEEnabled: true
      StreamSpecification:
        StreamViewType: NEW_IMAGE
      TableClass: STANDARD
      Tags:
      - Key: "Application"
        Value: "BatchOperationalDashboard"
      TimeToLiveSpecification:
        AttributeName: ExpirationTime
        Enabled: true

  ECSInstanceRegistrationLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
       RetentionInDays: 7


  ECSInstanceRegistrationStateMachine:
    Type: AWS::Serverless::StateMachine
    Properties:
      DefinitionSubstitutions:
        ECSRegistrationTable: !Ref ECSInstanceRegistrationTable
      DefinitionUri: src/ecs-instance-registration.json
      Events:
        CloudTrail:
          Type: CloudWatchEvent
          Description: "Capture ECS instances registration and deregistration"
          Properties:
            Pattern:
              detail-type:
              - 'AWS API Call via CloudTrail'
              source:
              - 'aws.ecs'
              detail:
                eventName:
                - 'RegisterContainerInstance'
                - 'DeregisterContainerInstance'
                eventSource:
                - 'ecs.amazonaws.com'
      Logging:
        Destinations:
          - CloudWatchLogsLogGroup:
              LogGroupArn: !GetAtt ECSInstanceRegistrationLogGroup.Arn
        Level: 'ALL'
      Policies:
        - CloudWatchLogsFullAccess
        - DynamoDBWritePolicy:
            TableName: !Ref ECSInstanceRegistrationTable
        - EC2DescribePolicy: {}
      Tags:
        "Application": "BatchOperationalDashboard"
      Tracing:
        Enabled: true
      Type: EXPRESS

  AthenaSpillS3Bucket:
    Type: AWS::S3::Bucket
    DeletionPolicy: Retain
    Properties:
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: 'AES256'
      PublicAccessBlockConfiguration:
        BlockPublicAcls: True
        BlockPublicPolicy: True
        IgnorePublicAcls: True
        RestrictPublicBuckets: True
      Tags:
        - Key: "Application"
          Value: "BatchOperationalDashboard"

  AthenaSpillS3BucketPolicy:
    Type: AWS::S3::BucketPolicy
    Properties:
      Bucket: !Ref AthenaSpillS3Bucket
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Action:
              - 's3:*'
            Effect: Deny
            Resource:
              - !Join
                - ''
                - - 'arn:aws:s3:::'
                  - !Ref AthenaSpillS3Bucket
                  - /*
              - !Join
                - ''
                - - 'arn:aws:s3:::'
                  - !Ref AthenaSpillS3Bucket
            Principal: '*'
            Condition:
              Bool:
                'aws:SecureTransport': False

  AthenaDynamoDBConnector:
    Type: AWS::Serverless::Application
    Properties:
      Location:
        ApplicationId: arn:aws:serverlessrepo:us-east-1:292517598671:applications/AthenaDynamoDBConnector
        SemanticVersion: 2025.15.1
      Parameters:
        AthenaCatalogName: aws-batch-jobs-data
        # WARNING: If set to 'true' encryption for spilled data is disabled.
        # DisableSpillEncryption: 'false' # Uncomment to override default value
        # Lambda memory in MB (min 128 - 3008 max).
        # LambdaMemory: '3008' # Uncomment to override default value
        # Maximum Lambda invocation runtime in seconds. (min 1 - 900 max)
        # LambdaTimeout: '900' # Uncomment to override default value
        # The name of the bucket where this function can spill data.
        SpillBucket: !Ref AthenaSpillS3Bucket
        # The prefix within SpillBucket where this function can spill data.
        # SpillPrefix: 'athena-spill' # Uncomment to override default value

  BatchDataCatalog:
    DependsOn: AthenaDynamoDBConnector
    Type: AWS::Athena::DataCatalog
    Properties:
      Name: aws-batch-jobs-data
      Parameters:
        function: !Sub 'arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:aws-batch-jobs-data'
      Tags:
        - Key: "Application"
          Value: "BatchOperationalDashboard"
      Type: LAMBDA

  GrafanaWorkGroup:
    Type: AWS::Athena::WorkGroup
    Properties:
      Name: 'batch-wg'
      RecursiveDeleteOption: true
      State: ENABLED
      Tags:
        - Key: "Application"
          Value: "BatchOperationalDashboard"
      WorkGroupConfiguration:
        BytesScannedCutoffPerQuery: 1000000000
        EnforceWorkGroupConfiguration: false
        EngineVersion:
          EffectiveEngineVersion: 'AUTO'
          SelectedEngineVersion: 'Athena engine version 3'
        PublishCloudWatchMetricsEnabled: false
        RequesterPaysEnabled: false

  GrafanaAthenaAccess:
    Type: AWS::IAM::Policy
    Properties:
      PolicyName: 'GrafanaAthenaAccess'
      PolicyDocument:
        Statement:
          - Effect: Allow
            Action:
              - athena:GetDatabase
              - athena:GetDataCatalog
              - athena:GetTableMetadata
              - athena:ListDatabases
              - athena:ListDataCatalogs
              - athena:ListTableMetadata
              - athena:ListWorkGroups
            Resource:
              - '*'
          - Effect: Allow
            Action:
              - athena:GetQueryExecution
              - athena:GetQueryResults
              - athena:GetWorkGroup
              - athena:StartQueryExecution
              - athena:StopQueryExecution
            Resource:
              - !Join [ '', ['arn:aws:athena:', !Ref AWS::Region, ':', !Ref AWS::AccountId, ':workgroup/', !Ref GrafanaWorkGroup ]]
              - !Join [ '', ['arn:aws:athena:', !Ref AWS::Region, ':', !Ref AWS::AccountId, ':workgroup/primary' ]]
          - Effect: Allow
            Action:
              - s3:GetBucketLocation
              - s3:GetObject
              - s3:ListBucket
              - s3:ListBucketMultipartUploads
              - s3:ListMultipartUploadParts
              - s3:AbortMultipartUpload
              - s3:PutObject
              - s3:PutBucketPublicAccessBlock
            Resource:
              - !GetAtt AthenaSpillS3Bucket.Arn
              - !Join [ '', [ !GetAtt AthenaSpillS3Bucket.Arn, '/*' ]]
              - !GetAtt BucketCURBatchReport.Arn
              - !Join [ '', [ !GetAtt BucketCURBatchReport.Arn, '/*' ]]
          - Effect: Allow
            Action:
              - lambda:InvokeFunction
            Resource:
              - !Sub 'arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:aws-batch-jobs-data'
      Roles:
        - !Ref GrafanaRole

  GrafanaRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - grafana.amazonaws.com
            Action:
              - sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AmazonGrafanaCloudWatchAccess
        - arn:aws:iam::aws:policy/service-role/AmazonGrafanaAthenaAccess

  GrafanaBatch:
    Type: AWS::Grafana::Workspace
    Properties:
      AccountAccessType: CURRENT_ACCOUNT
      AuthenticationProviders:
        - AWS_SSO
      DataSources:
        - ATHENA
        - CLOUDWATCH
      GrafanaVersion: "9.4"
      Name: "aws-batch-operation-dashboard"
      PluginAdminEnabled: true
      RoleArn: !Ref GrafanaRole
      PermissionType: SERVICE_MANAGED

#### Cost
  BucketCURBatchReport:
    Type: AWS::S3::Bucket
    DeletionPolicy: Retain
    Properties:
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: 'AES256'
      PublicAccessBlockConfiguration:
        BlockPublicAcls: True
        BlockPublicPolicy: True
        IgnorePublicAcls: True
        RestrictPublicBuckets: True
      Tags:
        - Key: "Application"
          Value: "BatchOperationalDashboard"
        - Key: "Application"
          Value: "BatchCostReport"


  BucketCURBatchReportPolicy:
    Type: AWS::S3::BucketPolicy
    Properties:
      Bucket: !Ref BucketCURBatchReport
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - 's3:GetBucketAcl'
              - 's3:GetBucketPolicy'
            Resource:
              - !Join
                - ''
                - - 'arn:aws:s3:::'
                  - !Ref BucketCURBatchReport
            Principal:
              Service:
                - 'billingreports.amazonaws.com'
            Condition:
              StringEquals:
                aws:SourceArn: !Sub 'arn:aws:cur:${AWS::Region}:${AWS::AccountId}:definition/*'
                aws:SourceAccount: !Sub '${AWS::AccountId}'
          - Effect: Allow
            Action:
              - 's3:PutObject'
            Resource:
              - !Join
                - ''
                - - 'arn:aws:s3:::'
                  - !Ref BucketCURBatchReport
                  - /*
            Principal:
              Service:
                - 'billingreports.amazonaws.com'
            Condition:
              StringEquals:
                aws:SourceArn: !Sub 'arn:aws:cur:${AWS::Region}:${AWS::AccountId}:definition/*'
                aws:SourceAccount: !Sub '${AWS::AccountId}'

  CURAWSBatch:
    DependsOn: BucketCURBatchReportPolicy
    Type: AWS::CUR::ReportDefinition
    Properties:
      AdditionalArtifacts:
        - 'ATHENA'
      AdditionalSchemaElements:
        - 'RESOURCES'
        - 'SPLIT_COST_ALLOCATION_DATA'
      Compression: 'Parquet'
      Format: 'Parquet'
      RefreshClosedReports: True
      ReportName: 'AWSBatchCostReport'
      ReportVersioning: 'OVERWRITE_REPORT'
      S3Bucket: !Ref BucketCURBatchReport
      S3Prefix: 'aws-batch'
      S3Region: !Ref 'AWS::Region'
      TimeUnit: 'HOURLY'

### Glue and Athena for CUR
  AWSCURDatabase:
    Type: 'AWS::Glue::Database'
    Properties:
      DatabaseInput:
        Name: 'athenacurcfn_a_w_s_batch_cost_report'
      CatalogId: !Ref AWS::AccountId

  AWSCURCrawlerComponentFunction:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - glue.amazonaws.com
            Action:
              - 'sts:AssumeRole'
      Path: /
      ManagedPolicyArns:
        - !Sub 'arn:${AWS::Partition}:iam::aws:policy/service-role/AWSGlueServiceRole'
      Policies:
        - PolicyName: AWSCURCrawlerComponentFunction
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - 'logs:CreateLogGroup'
                  - 'logs:CreateLogStream'
                  - 'logs:PutLogEvents'
                Resource: !Sub 'arn:${AWS::Partition}:logs:*:*:*'
              - Effect: Allow
                Action:
                  - 'glue:UpdateDatabase'
                  - 'glue:UpdatePartition'
                  - 'glue:CreateTable'
                  - 'glue:UpdateTable'
                  - 'glue:ImportCatalogToGlue'
                Resource: '*'
              - Effect: Allow
                Action:
                  - 's3:GetObject'
                  - 's3:PutObject'
                Resource:
                  - !Join
                    - ''
                    - - 'arn:aws:s3:::'
                      - !Ref BucketCURBatchReport
                      - '/aws-batch/AWSBatchCostReport/AWSBatchCostReport*'

        - PolicyName: AWSCURKMSDecryption
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - 'kms:Decrypt'
                Resource: '*'


  AWSCURCrawlerLambdaExecutor:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - 'sts:AssumeRole'
      Path: /
      Policies:
        - PolicyName: AWSCURCrawlerLambdaExecutor
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - 'logs:CreateLogGroup'
                  - 'logs:CreateLogStream'
                  - 'logs:PutLogEvents'
                Resource: !Sub 'arn:${AWS::Partition}:logs:*:*:*'
              - Effect: Allow
                Action:
                  - 'glue:StartCrawler'
                Resource: '*'


  AWSCURCrawler:
    Type: 'AWS::Glue::Crawler'
    DependsOn:
      - AWSCURDatabase
      - AWSCURCrawlerComponentFunction
    Properties:
      Name: AWSCURCrawler-AWSBatchCostReport
      Description: A recurring crawler that keeps your CUR table in Athena up-to-date.
      Role: !GetAtt AWSCURCrawlerComponentFunction.Arn
      DatabaseName: !Ref AWSCURDatabase
      Targets:
        S3Targets:
          - Path: !Join ['', ['s3://', !Ref BucketCURBatchReport,'/aws-batch/AWSBatchCostReport/AWSBatchCostReport']]
            Exclusions:
              - '**.json'
              - '**.yml'
              - '**.sql'
              - '**.csv'
              - '**.gz'
              - '**.zip'
      SchemaChangePolicy:
        UpdateBehavior: UPDATE_IN_DATABASE
        DeleteBehavior: DELETE_FROM_DATABASE


  AWSCURInitializer:
    Type: 'AWS::Lambda::Function'
    DependsOn: AWSCURCrawler
    Properties:
      Code:
        ZipFile: |
          // AWS SDK v3 for Glue
          const { GlueClient, StartCrawlerCommand } = require('@aws-sdk/client-glue');
          const response = require('./cfn-response');
          exports.handler = function(event, context, callback) {
            if (event.RequestType === 'Delete') {
              response.send(event, context, response.SUCCESS);
            } else {
              // Create Glue client using SDK v3
              const glue = new GlueClient();
              // Create command for starting crawler
              const command = new StartCrawlerCommand({ Name: 'AWSCURCrawler-AWSBatchCostReport' });
    
              // Send the command
              glue.send(command).then(data => {
                if (event.ResponseURL) {
                  response.send(event, context, response.SUCCESS);
                } else {
                  callback(null, response.SUCCESS);
                }
             }).catch(err => {
                const responseData = err;
                if (err.name === 'CrawlerRunningException') {
                  callback(null, err.message);
               } else {
                  const responseString = JSON.stringify(responseData);
                  if (event.ResponseURL) {
                    response.send(event, context, response.FAILED,{ msg: responseString });
                 } else {
                    callback(responseString);
                 }
                }
              });
            }
          };
      Handler: 'index.handler'
      Timeout: 30
      Runtime: nodejs22.x
      ReservedConcurrentExecutions: 1
      Role: !GetAtt AWSCURCrawlerLambdaExecutor.Arn


  AWSStartCURCrawler:
    Type: 'Custom::AWSStartCURCrawler'
    Properties:
      ServiceToken: !GetAtt AWSCURInitializer.Arn


  AWSS3CUREventLambdaPermission:
    Type: AWS::Lambda::Permission
    Properties:
      Action: 'lambda:InvokeFunction'
      FunctionName: !GetAtt AWSCURInitializer.Arn
      Principal: 's3.amazonaws.com'
      SourceAccount: !Ref AWS::AccountId
      SourceArn: !GetAtt BucketCURBatchReport.Arn


  AWSS3CURLambdaExecutor:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - 'sts:AssumeRole'
      Path: /
      Policies:
        - PolicyName: AWSS3CURLambdaExecutor
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - 'logs:CreateLogGroup'
                  - 'logs:CreateLogStream'
                  - 'logs:PutLogEvents'
                Resource: !Sub 'arn:${AWS::Partition}:logs:*:*:*'
              - Effect: Allow
                Action:
                  - 's3:PutBucketNotification'
                Resource: !GetAtt BucketCURBatchReport.Arn


  AWSS3CURNotification:
    Type: 'AWS::Lambda::Function'
    DependsOn:
    - AWSCURInitializer
    - AWSS3CUREventLambdaPermission
    - AWSS3CURLambdaExecutor
    Properties:
      Code:
        ZipFile: |
          // Add S3 client import
          const { S3Client, PutBucketNotificationConfigurationCommand } = require('@aws-sdk/client-s3');
          const response = require('./cfn-response');
          exports.handler = function(event, context, callback) {
            const s3Client = new S3Client();
            const putConfigRequest = function(notificationConfiguration) {
              return new Promise(function(resolve, reject) {
                const command = new PutBucketNotificationConfigurationCommand({
                  Bucket: event.ResourceProperties.BucketName,
                  NotificationConfiguration: notificationConfiguration
                });
                s3Client.send(command)
                  .then(data => resolve(data))
                  .catch(err => reject({ msg: err.message, error: err, data: null }));
              });
            };
            const newNotificationConfig = {};
            if (event.RequestType !== 'Delete') {
              newNotificationConfig.LambdaFunctionConfigurations = [{
                Events: [ 's3:ObjectCreated:*' ],
                LambdaFunctionArn: event.ResourceProperties.TargetLambdaArn || 'missing arn',
                Filter: { Key: { FilterRules: [ { Name: 'prefix', Value: event.ResourceProperties.ReportKey } ] } }
              }];
            }
            putConfigRequest(newNotificationConfig).then(function(result) {
              response.send(event, context, response.SUCCESS, result);
              callback(null, result);
            }).catch(function(error) {
              response.send(event, context, response.FAILED, error);
              console.log(error);
              callback(error);
            });
          };
      Handler: 'index.handler'
      Timeout: 30
      Runtime: nodejs22.x
      ReservedConcurrentExecutions: 1
      Role: !GetAtt AWSS3CURLambdaExecutor.Arn


  AWSPutS3CURNotification:
    Type: 'Custom::AWSPutS3CURNotification'
    Properties:
      ServiceToken: !GetAtt AWSS3CURNotification.Arn
      TargetLambdaArn: !GetAtt AWSCURInitializer.Arn
      BucketName: !Ref BucketCURBatchReport
      ReportKey: 'aws-batch/AWSBatchCostReport/AWSBatchCostReport'


  AWSCURReportStatusTable:
    Type: 'AWS::Glue::Table'
    DependsOn: AWSCURDatabase
    Properties:
      DatabaseName: athenacurcfn_a_w_s_batch_cost_report
      CatalogId: !Ref AWS::AccountId
      TableInput:
        Name: 'cost_and_usage_data_status'
        TableType: 'EXTERNAL_TABLE'
        StorageDescriptor:
          Columns:
            - Name: status
              Type: 'string'
          InputFormat: 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
          OutputFormat: 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
          SerdeInfo:
            SerializationLibrary: 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
          Location: !Join ['', ['s3://', !Ref BucketCURBatchReport,'/aws-batch/AWSBatchCostReport/cost_and_usage_data_status/']]


###


Outputs:
  AthenaDataSource:
    Value: !Ref BatchJobData

  AthenaSpillBucket:
    Value: !Ref AthenaSpillS3Bucket

  GrafanaWorkspaceId:
    Value: !Ref GrafanaBatch

  CURBatchBucket:
    Value: !Ref BucketCURBatchReport