Add support for config file in S3.

Author: TotalDwarf03

Makefile

@@ -17,8 +17,8 @@ clean: ## Clean the temporary files.
 
 .PHONY: format
 format:  ## Format the code.
-	poetry run black .
-	poetry run ruff check . --fix
+	poetry run black src
+	poetry run ruff check src --fix
 
 .PHONY: lint
 lint:  ## Run all linters (black/ruff/pylint/mypy).

README.md

@@ -18,6 +18,10 @@ A Python utility used to archive old, unused GitHub repositories from an organis
     - [Storing the Container on AWS Elastic Container Registry (ECR)](#storing-the-container-on-aws-elastic-container-registry-ecr)
     - [Deploying the Lambda](#deploying-the-lambda)
     - [Destroying / Removing the Lambda](#destroying--removing-the-lambda)
+    - [Deployments with Concourse](#deployments-with-concourse)
+      - [Allowlisting your IP](#allowlisting-your-ip)
+      - [Setting up a pipeline](#setting-up-a-pipeline)
+      - [Triggering a pipeline](#triggering-a-pipeline)
   - [Linting and Testing](#linting-and-testing)
     - [GitHub Actions](#github-actions)
     - [Running Tests Locally](#running-tests-locally)
@@ -112,19 +116,21 @@ Before the doing the following, make sure your Daemon is running. If using Colim
     -e AWS_SECRET_NAME=<secret_name> \
     -e GITHUB_ORG=<org> \
     -e GITHUB_APP_CLIENT_ID=<client_id> \
+    -e S3_BUCKET_NAME=<bucket_name>\
     -e AWS_LAMBDA_FUNCTION_TIMEOUT=300
     github-repository-archive-script
     ```
 
     When running the container, you are required to pass some environment variable.
 
-    | Variable                    | Description                                                                               |
-    |-----------------------------|-------------------------------------------------------------------------------------------|
-    | GITHUB_ORG                  | The organisation you would like to run the tool in.                                       |
-    | GITHUB_APP_CLIENT_ID        | The Client ID for the GitHub App which the tool uses to authenticate with the GitHub API. |
-    | AWS_DEFAULT_REGION          | The AWS Region which the Secret Manager Secret is in.                                     |
-    | AWS_SECRET_NAME             | The name of the AWS Secret Manager Secret to get.                                         |
-    | AWS_LAMBDA_FUNCTION_TIMEOUT | The timeout time in seconds (Default: 300s / 5 minutes).                                  |
+    | Variable                    | Description                                                                                        |
+    |-----------------------------|----------------------------------------------------------------------------------------------------|
+    | GITHUB_ORG                  | The organisation you would like to run the tool in.                                                |
+    | GITHUB_APP_CLIENT_ID        | The Client ID for the GitHub App which the tool uses to authenticate with the GitHub API.          |
+    | AWS_DEFAULT_REGION          | The AWS Region which the Secret Manager Secret is in.                                              |
+    | AWS_SECRET_NAME             | The name of the AWS Secret Manager Secret to get.                                                  |
+    | AWS_BUCKET_NAME             | The name of the S3 bucket which has the cloud config in (Only used when `use_local_config=False`). |
+    | AWS_LAMBDA_FUNCTION_TIMEOUT | The timeout time in seconds (Default: 300s / 5 minutes).                                           |
 
     Once the container is running, a local endpoint is created at `localhost:9000/2015-03-31/functions/function/invocations`.
 
@@ -177,6 +183,7 @@ To run the Lambda function outside of a container, we need to execute the `handl
     export AWS_SECRET_ACCESS_KEY=<secret_access_key>
     export AWS_DEFAULT_REGION=eu-west-2
     export AWS_SECRET_NAME=<secret_name>
+    export S3_BUCKET_NAME=<bucket_name>
     export GITHUB_ORG=<org>
     export GITHUB_APP_CLIENT_ID=<client_id>
     ```

config/config.json

@@ -1,6 +1,7 @@
 {
   "features": {
-    "show_log_locally": true
+    "show_log_locally": false,
+    "use_local_config": false
   },
   "archive_configuration": {
     "archive_threshold": 365,

src/logger.py

@@ -12,7 +12,7 @@ def __init__(self, debug: bool) -> None:
         self.logger.setLevel(logging.INFO)
 
         if debug:
-            logging.basicConfig(level=logging.DEBUG, filename="debug.log", filemode="w")
+            logging.basicConfig(filename="debug.log", filemode="w")
 
     def log_info(self, message: str) -> None:
         """Logs an info message to the logger.

src/main.py

@@ -512,6 +512,39 @@ def handler(event, context) -> str:  # type: ignore[no-untyped-def]
     features = get_dict_value(config, "features")
     archive_rules = get_dict_value(config, "archive_configuration")
 
+    # Create a Boto3 session
+
+    session = boto3.session.Session()
+
+    # Create Boto3 S3 client
+
+    s3 = session.client(service_name="s3")
+
+    # Check whether to use local config or cloud config
+
+    if not get_dict_value(features, "use_local_config"):
+
+        bucket_name = os.getenv("S3_BUCKET_NAME")
+
+        if not bucket_name:
+            error_message = "S3_BUCKET_NAME environment variable not set. Please check your environment variables."
+            raise Exception(error_message)
+
+        try:
+            response = s3.get_object(
+                Bucket=bucket_name,
+                Key="config.json",
+            )
+        except s3.exceptions.NoSuchKey as e:
+            error_message = (
+                f"Configuration file not found in S3 bucket {bucket_name}. Please check the bucket contents."
+            )
+            raise Exception(error_message) from e
+
+        config = json.loads(response["Body"].read().decode("utf-8"))
+        features = get_dict_value(config, "features")
+        archive_rules = get_dict_value(config, "archive_configuration")
+
     # Initialise logging
 
     debug = get_dict_value(features, "show_log_locally")
@@ -528,9 +561,7 @@ def handler(event, context) -> str:  # type: ignore[no-untyped-def]
 
     # Create Boto3 Secret Manager client
 
-    session = boto3.session.Session()
     secret_manager = session.client(service_name="secretsmanager", region_name=aws_default_region)
-
     logger.log_info("Boto3 Secret Manager client created.")
 
     # Create GitHub API interfaces (GraphQL and REST)
@@ -567,8 +598,8 @@ def handler(event, context) -> str:  # type: ignore[no-untyped-def]
         "## Important Notice \n\n",
         f"This repository has not been updated in over {archive_threshold} days and will be archived in {notification_period} days if no action is taken. \n",
         "## Actions Required to Prevent Archive \n\n",
-        f"1. Update the repository by creating/updating an exemption file. \n",
-        f"   - The exemption file should be named one of the following: \n",
+        "1. Update the repository by creating/updating an exemption file. \n",
+        "   - The exemption file should be named one of the following: \n",
         f"{''.join(formatted_filenames)}\n",
         "   - This file should contain the reason why the repository should not be archived. \n",
         "   - If the file already exists, please update it with the latest information. \n",

terraform/service/main.tf

@@ -52,6 +52,7 @@ resource "aws_lambda_function" "lambda_function" {
       GITHUB_APP_CLIENT_ID = var.github_app_client_id
       AWS_SECRET_NAME      = var.aws_secret_name
       AWS_ACCOUNT_NAME     = var.env_name
+      AWS_BUCKET_NAME      = var.aws_bucket_name
     }
   }
 }

terraform/service/variables.tf

@@ -18,6 +18,12 @@ variable "aws_secret_name" {
   type        = string
 }
 
+variable "aws_bucket_name" {
+  description = "The name of the S3 bucket which the cloud config is stored in"
+  type        = string
+  default     = "${env_name}-github-repository-archive-script"
+}
+
 variable "env_name" {
   description = "AWS environment"
   type        = string