DH-5581 modularized terraform resources (Dataherald#490)

* DH-5581 modularized terraform resources

Author: DishenWang2023

.github/workflows/ai-deploy-branch-env.yml

@@ -18,6 +18,8 @@ env:
   SLACK_ICON: https://files.dataherald.com/random/github-actions.png
   SLACK_USERNAME: GitHub Actions
   SLACK_CHANNEL: eng-notifications
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
 
 jobs:
   extract-branch-names:
@@ -78,13 +80,13 @@ jobs:
       target_env: branch
       image_tag: ${{ needs.extract-branch-names.outputs.branch_name}}-${{ github.sha }}
 
-  deploy-backend-dbs-infra:
-    name: Deploy Backend and DBs infrastructure
+  deploy-backend:
+    name: Deploy Backend infrastructure
     needs: [extract-branch-names, build-engine, build-server]
     runs-on: ubuntu-latest
     defaults:
       run:
-        working-directory: ./apps/ai/server/terraform
+        working-directory: ./apps/ai/server/terraform/branch
     outputs:
       api_url: ${{ steps.get-api-url.outputs.api_url }}
     steps:
@@ -105,9 +107,12 @@ jobs:
           terraform_wrapper: false
       - name: Apply Terraform
         env:
-          PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
           TF_VAR_sha: ${{ github.sha }}
           TF_VAR_branch_name: ${{ needs.extract-branch-names.outputs.branch_name }}
+          TF_VAR_vpc_id: ${{ vars.VPC_ID }}
+          TF_VAR_subnet_1_id: ${{ vars.PRIVATE_SUBNET_1_ID }}
+          TF_VAR_subnet_2_id: ${{ vars.PRIVATE_SUBNET_2_ID }}
+          TF_VAR_ecs_security_group_id: ${{ vars.BRANCH_SECURITY_GROUP_ID }}
           TF_VAR_pinecone_index_name: ${{ needs.extract-branch-names.outputs.db_name }}
           TF_VAR_mongodb_uri: ${{ vars.EPHEMERAL_MONGODB_URI }}
           TF_VAR_mongodb_name: ${{ needs.extract-branch-names.outputs.db_name }}
@@ -116,8 +121,8 @@ jobs:
 
         run: |
           terraform init -upgrade -backend-config="key=${{ needs.extract-branch-names.outputs.branch_name }}"
-          terraform plan
-          terraform apply -auto-approve
+          terraform plan -target=module.aws_ecs
+          terraform apply -target=module.aws_ecs -auto-approve
       - name: Get API URL
         id: get-api-url
         run: |
@@ -135,7 +140,36 @@ jobs:
           if [ -n "$UNTAGGED_SERVER_IMAGES" ] && [ "$UNTAGGED_SERVER_IMAGES" != "[]" ]; then
             aws ecr batch-delete-image --repository-name $ECR_SERVER_REPOSITORY --image-ids "$UNTAGGED_SERVER_IMAGES" --region us-east-1
           fi
-
+  create-vector-db:
+    name: Create PineconeDB
+    needs: [extract-branch-names]
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./apps/ai/server/terraform/branch
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GH_ACCESS_TOKEN }}
+          submodules: true
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_REGION }}
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_wrapper: false
+      - name: Apply Terraform
+        env:
+          TF_VAR_pinecone_index_name: ${{ needs.extract-branch-names.outputs.db_name }}
+        run: |
+          terraform init -upgrade -backend-config="key=${{ needs.extract-branch-names.outputs.branch_name }}"
+          terraform plan -target=module.pinecone_index
+          terraform apply -target=module.pinecone_index -auto-approve
   populate-db:
     name: Populate MongoDB
     needs: [extract-branch-names, start-self-hosted-runner]
@@ -201,7 +235,7 @@ jobs:
         extract-branch-names,
         start-self-hosted-runner,
         populate-db,
-        deploy-backend-dbs-infra,
+        create-vector-db,
       ]
     runs-on: ${{ needs.start-self-hosted-runner.outputs.label }}
     defaults:
@@ -215,8 +249,6 @@ jobs:
           submodules: true
       - name: Populate PineconeDB
         env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
           PINECONE_INDEX_NAME: ${{ needs.extract-branch-names.outputs.db_name }}
           MONGODB_URI: ${{ vars.EPHEMERAL_MONGODB_URI }}
           MONGODB_USERNAME: ${{ secrets.MONGODB_USERNAME }}
@@ -231,7 +263,7 @@ jobs:
 
   deploy-frontend:
     name: Deploy Frontend
-    needs: [extract-branch-names, deploy-backend-dbs-infra]
+    needs: [extract-branch-names, deploy-backend]
     runs-on: ubuntu-latest
     outputs:
       url: ${{ steps.console_dns.outputs.url }}
@@ -252,10 +284,10 @@ jobs:
       - name: Deploy to Vercel and set custom domain alias
         run: |
           echo Deploying to Vercel with the following environment variables:
-          echo "NEXT_PUBLIC_API_URL=${{ needs.deploy-backend-dbs-infra.outputs.api_url }}"
+          echo "NEXT_PUBLIC_API_URL=${{ needs.deploy-backend.outputs.api_url }}"
           echo "AUTH0_BASE_URL=${{ steps.console_dns.outputs.url }}"
 
-          vercel deploy --yes --build-env NEXT_PUBLIC_API_URL=${{ needs.deploy-backend-dbs-infra.outputs.api_url }} --env AUTH0_BASE_URL=${{ steps.console_dns.outputs.url }} --token=${{ secrets.VERCEL_ACCESS_TOKEN }} >deployment-url.txt 2>error.txt
+          vercel deploy --yes --build-env NEXT_PUBLIC_API_URL=${{ needs.deploy-backend.outputs.api_url }} --env AUTH0_BASE_URL=${{ steps.console_dns.outputs.url }} --token=${{ secrets.VERCEL_ACCESS_TOKEN }} >deployment-url.txt 2>error.txt
 
           # check the exit code
           code=$?
@@ -274,7 +306,7 @@ jobs:
     needs:
       [
         extract-branch-names,
-        deploy-backend-dbs-infra,
+        deploy-backend,
         populate-db,
         populate-vector-db,
         deploy-frontend,
@@ -299,12 +331,12 @@ jobs:
     needs:
       [
         extract-branch-names,
-        deploy-backend-dbs-infra,
+        deploy-backend,
         populate-db,
         populate-vector-db,
         deploy-frontend,
       ]
-    if: needs.deploy-backend-dbs-infra.result == 'failure' || needs.populate-db.result == 'failure' || needs.populate-vector-db.result == 'failure' || needs.deploy-frontend.result == 'failure'
+    if: needs.deploy-backend.result == 'failure' || needs.populate-db.result == 'failure' || needs.populate-vector-db.result == 'failure' || needs.deploy-frontend.result == 'failure'
     runs-on: ubuntu-latest
     steps:
       - name: Send failed notification
@@ -324,12 +356,12 @@ jobs:
     needs:
       [
         extract-branch-names,
-        deploy-backend-dbs-infra,
+        deploy-backend,
         populate-db,
         populate-vector-db,
         deploy-frontend,
       ]
-    if: needs.deploy-backend-dbs-infra.result == 'success' && needs.populate-db.result == 'success' && needs.populate-vector-db.result == 'success' && needs.deploy-frontend.result == 'success'
+    if: needs.deploy-backend.result == 'success' && needs.populate-db.result == 'success' && needs.populate-vector-db.result == 'success' && needs.deploy-frontend.result == 'success'
     runs-on: ubuntu-latest
     steps:
       - name: Send success notification
@@ -341,7 +373,7 @@ jobs:
             The branch environment deployment succeeded. Here are the details:
 
             *Branch*: `${{ needs.extract-branch-names.outputs.branch }}`
-            *API URL*: ${{ needs.deploy-backend-dbs-infra.outputs.api_url }}
+            *API URL*: ${{ needs.deploy-backend.outputs.api_url }}
             *Console URL*: ${{ needs.deploy-frontend.outputs.url }}
           SLACK_COLOR: success
 

.github/workflows/ai-destroy-branch-env.yml

@@ -52,7 +52,7 @@ jobs:
       PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
     defaults:
       run:
-        working-directory: ./apps/ai/server
+        working-directory: ./apps/ai/server/terraform/branch
 
     steps:
       - name: Check out code
@@ -75,7 +75,6 @@ jobs:
           TF_VAR_mongodb_password: ${{ secrets.MONGODB_PASSWORD }}
         timeout-minutes: 3
         run: |
-          cd terraform
           terraform init -upgrade -backend-config="key=${{ needs.extract-branch-name.outputs.branch_name }}"
           terraform destroy -auto-approve
       - name: Delete images

apps/ai/server/terraform/main.tf …/server/terraform/branch/aws_ecs/main.tfapps/ai/server/terraform/main.tf renamed to apps/ai/server/terraform/branch/aws_ecs/main.tf apps/ai/server/terraform/main.tf renamed to apps/ai/server/terraform/branch/aws_ecs/main.tf

@@ -4,97 +4,19 @@ terraform {
       source  = "aws"
       version = "4.1.0"
     }
-
-    pinecone = {
-      source  = "skyscrapr/pinecone"
-      version = "0.5.1"
-    }
-  }
-  backend "s3" {
-    bucket = "terraform-states2"
-    region = "us-east-1"
-  }
-}
-
-
-provider "aws" {
-  region = "us-east-1"
-}
-provider "pinecone" {}
-
-variable "branch_name" { type= string }
-
-variable "sha" { type= string }
-
-variable "pinecone_index_name" {
-  description = "pinecone_index_name"
-  type        = string
-}
-
-variable "mongodb_uri" {
-  description = "mongodb_uri"
-  type        = string
-}
-
-variable "mongodb_name" {
-  description = "mongodb_name"
-  type        = string
-}
-
-variable "mongodb_username" {
-  description = "mongodb_username"
-  type        = string
-}
-
-variable "mongodb_password" {
-  description = "mongodb_password"
-  type        = string
-}
-
-variable "subnet_1_id" {
-  description = "subnet_1_id"
-  type        = string
-  default = "subnet-076afb4a159204349"
-}
-
-variable "subnet_2_id" {
-  description = "subnet_2_id"
-  type        = string
-  default = "subnet-0b6b9dbf631131b09"
-}
-
-variable "ecs_security_group_id" {
-  description = "security_group_id"
-  type        = string
-  default = "sg-07fac199a96aa3b65"
-}
-
-resource "pinecone_index" "my_index" {
-  name = var.pinecone_index_name
-  dimension = 1536
-  metric = "cosine"
-  spec = {
-    serverless = {
-      cloud = "aws"
-      region = "us-west-2"
-    }
   }
 }
 
-locals {
-  srv_connection_string = replace(var.mongodb_uri, "mongodb+srv://", "mongodb+srv://${var.mongodb_username}:${var.mongodb_password}@")
-}
-
 resource "aws_ecs_task_definition" "my_task_definition" {
-  family = "ai-backend-branch-${var.branch_name}"
-  task_role_arn = "arn:aws:iam::422486916789:role/ecsk2TaskExecutionRole"
-  execution_role_arn = "arn:aws:iam::422486916789:role/ecsk2TaskExecutionRole"
-  network_mode = "awsvpc"
+  family                   = "ai-backend-branch-${var.branch_name}"
+  task_role_arn            = "arn:aws:iam::422486916789:role/ecsk2TaskExecutionRole"
+  execution_role_arn       = "arn:aws:iam::422486916789:role/ecsk2TaskExecutionRole"
+  network_mode             = "awsvpc"
   requires_compatibilities = ["FARGATE"]
-  cpu = "2048"
-  memory = "4096"
-#  container_definitions = file("task_definition.json")
-  container_definitions =<<DEFINITION
+  cpu                      = "2048"
+  memory                   = "4096"
+  #  container_definitions = file("task_definition.json")
+  container_definitions = <<DEFINITION
 [
         {
             "name": "ai-engine-branch",
@@ -128,11 +50,11 @@ resource "aws_ecs_task_definition" "my_task_definition" {
               },
               {
                 "name": "MONGODB_URI",
-                "value": "${local.srv_connection_string}"
+                "value": "${var.mongodb_uri}"
               },
               {
                 "name": "GOLDEN_SQL_COLLECTION",
-                "value": "${var.pinecone_index_name}"
+                "value": "${var.index_name}"
               }
             ],
             "command": ["sh", "-c", "uvicorn dataherald.app:app --host 0.0.0.0 --port $CORE_PORT --log-config log_config.yml --log-level debug --reload"],
@@ -180,7 +102,7 @@ resource "aws_ecs_task_definition" "my_task_definition" {
               },
               {
                 "name": "MONGODB_URI",
-                "value": "${local.srv_connection_string}"
+                "value": "${var.mongodb_uri}"
               }
             ],
             "mountPoints": [],
@@ -200,19 +122,19 @@ resource "aws_ecs_task_definition" "my_task_definition" {
 }
 
 resource "aws_lb" "my_load_balancer" {
-  name               = "${var.branch_name}"
+  name               = var.branch_name
   internal           = false
   idle_timeout       = 300
   load_balancer_type = "application"
-  security_groups    = ["sg-07fac199a96aa3b65"]  # Replace with your security group ID
-  subnets            = ["subnet-076afb4a159204349", "subnet-0b6b9dbf631131b09"]  # Replace with your subnet IDs
+  security_groups    = [var.ecs_security_group_id]
+  subnets            = [var.subnet_1_id, var.subnet_2_id]
 }
 
 resource "aws_lb_target_group" "ecs_target_group" {
-  name        = "${var.branch_name}"
+  name        = var.branch_name
   port        = 80
   protocol    = "HTTP"
-  vpc_id      = "vpc-09c492a49b76fdf80"
+  vpc_id      = var.vpc_id
   target_type = "ip"
 
   health_check {
@@ -240,7 +162,7 @@ resource "aws_lb_listener" "https_listener" {
   load_balancer_arn = aws_lb.my_load_balancer.arn
   port              = 443
   protocol          = "HTTPS"
-  ssl_policy        = "ELBSecurityPolicy-TLS13-1-2-2021-06"  # Choose an appropriate SSL policy for your application
+  ssl_policy        = "ELBSecurityPolicy-TLS13-1-2-2021-06" # Choose an appropriate SSL policy for your application
 
   default_action {
     type             = "forward"
@@ -250,11 +172,11 @@ resource "aws_lb_listener" "https_listener" {
 }
 
 resource "aws_ecs_service" "my_service" {
-  name            = "ai-backend-branch-${var.branch_name}"
-  cluster         = "arn:aws:ecs:us-east-1:422486916789:cluster/ai"
-  task_definition = aws_ecs_task_definition.my_task_definition.arn
-  desired_count   = 1
-  launch_type     = "FARGATE"
+  name                    = "ai-backend-branch-${var.branch_name}"
+  cluster                 = "arn:aws:ecs:us-east-1:422486916789:cluster/ai"
+  task_definition         = aws_ecs_task_definition.my_task_definition.arn
+  desired_count           = 1
+  launch_type             = "FARGATE"
   enable_ecs_managed_tags = true
   wait_for_steady_state   = true
 
@@ -265,15 +187,15 @@ resource "aws_ecs_service" "my_service" {
   }
 
   network_configuration {
-    subnets         = [var.subnet_1_id, var.subnet_2_id]
-    security_groups = [var.ecs_security_group_id]
+    subnets          = [var.subnet_1_id, var.subnet_2_id]
+    security_groups  = [var.ecs_security_group_id]
     assign_public_ip = true
   }
 }
 
 resource "aws_route53_record" "my_load_balancer_record" {
-  zone_id = "Z07539241TW7P7NHVR11T"  # Replace with your Route 53 hosted zone ID
-  name    = "${var.branch_name}.api.dataherald.ai"   # Replace with the desired domain name
+  zone_id = "Z07539241TW7P7NHVR11T"                # Replace with your Route 53 hosted zone ID
+  name    = "${var.branch_name}.api.dataherald.ai" # Replace with the desired domain name
   type    = "A"
   alias {
     name                   = aws_lb.my_load_balancer.dns_name