diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c78cbfe..31078a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install Poetry - run: pipx install poetry==1.7.1 + run: pipx install poetry==2.3.2 - name: Set up Python uses: actions/setup-python@v5 @@ -37,10 +37,10 @@ jobs: run: python -c "import sys; print(sys.version)" - name: Install dependencies - run: make install-dev + run: poetry install --with dev --without alerts --no-root - name: Lint Python run: make lint - name: Cleanup residue file - run: make clean \ No newline at end of file + run: make clean diff --git a/README.md b/README.md index 7aeb0ca..30d44ce 100644 --- a/README.md +++ b/README.md @@ -86,15 +86,16 @@ poetry run flask --app app run --port=8000 This repo utilises PyTest for the testing. Please make sure you have installed dev dependencies before running tests. -To test you need a mock token. Visit the Cognito UI with the redirect URL set to your local environment. Once successully logged in, copy the `id_token`. +Local tests now mock AWS Secrets Manager, Cognito settings and Teams alert configuration in `testing/conftest.py`, so a real AWS session or Cognito token is not required for `pytest`. -Then import the token into your environment and set the email of the user you want to test with: +If you want to override the default mocked user in tests, set: ```bash -export MOCK_TOKEN= export MOCK_USER_EMAIL= ``` +`MOCK_TOKEN` is optional for local tests and defaults to `local-test-token`. + Make sure dev dependencies are installed: ```bash make install-dev @@ -105,8 +106,6 @@ When in root directory, run the testing command. If you are in `aws_lambda_scrip make pytest ``` -If all tests fail, please relogin to the Cognito and use a new token. - Once you have finished testing, clean the temp files with: ```bash make clean @@ -116,7 +115,7 @@ make clean View the Postman workspace for this project [here](https://www.postman.com/science-pilot-55892832/workspace/keh-tech-audit-tool-api/collection/38871441-e42f661e-6430-4f46-8182-083e9e0fd4ad?action=share&creator=38871441&active-environment=38871441-7c5e3795-74f5-46b3-9034-637561aba746). -Please read the description or README to understand how to use this workspace. You need to get a mock_token to authenticate yourself in each request. +Please read the description or README to understand how to use this workspace. Postman requests still require a real Cognito ID token in the `Authorization` header. ## MkDocs Documentation @@ -152,7 +151,7 @@ This will build the MkDocs documentation and deploy it to the `gh-pages` branch ## API Reference -Before testing the API, you need to use the above instructions at **Testing** to get a mock token, as all requests need to be authenticated. +Before calling the API manually, use a valid Cognito ID token in the `Authorization` header. Local `pytest` runs mock authentication automatically. | Header | Type | Description | | :-------- | :------- | :------------------------- | @@ -185,7 +184,7 @@ can be used multiple times to get a new id_token. GET /api/v1/projects ``` -Get's the projects associated with the users email. +Gets all projects currently stored by the API. ### Get a specific user project @@ -198,7 +197,7 @@ GET /api/v1/projects/ | `` | `string` | **Required**. The project you want to get | -Get's a specific project from the user. +Gets a specific project by name. ### Create a new project @@ -229,7 +228,7 @@ Send JSON in this format: "project_description": "Description" }] , - "developed": ["In-house", []], + "developed": ["In-house"], "source_control": [ { "type": "GitHub", @@ -280,12 +279,6 @@ Send JSON in this format: } }, "stage":"Development", - "project_dependencies":[ - { - "name": "string", - "description": "string" - } - ], "supporting_tools": { "code_editors": { "main": [], @@ -293,38 +286,39 @@ Send JSON in this format: "List of strings" ] }, - "ui_tools": { + "user_interface": { "main": [], "others": [ "List of strings" ] }, - "diagram_tools": { + "diagrams": { "main": [], "others": [ "List of strings" ] }, - "project_tracking_tools": "string", - "documentation_tools": { + "project_tracking": "string", + "documentation": { "main": [], "others": [ "List of strings" ] }, - "communication_tools": { + "communication": { "main": [], "others": [ "List of strings" ] }, - "collaboration_tools": { + "collaboration": { "main": [], "others": [ "List of strings" ] }, - "incident_management": "string" + "incident_management": "string", + "miscellaneous": [] } } ``` @@ -351,9 +345,9 @@ GET /api/v1/projects/filter | `` | `string` | What you want returned from the project. Multiple return filter seperated by a comma (,) | -Get's projects using a filter. +Gets projects using one or more query filters. -Filter can be one or more of: email, roles, name, developed, source_control, hosting, database, languages, frameworks, cicd, infrastructure. +Filter can be one or more of: email, roles, name, developed, source_control, hosting, database, languages, frameworks, cicd, environments, infrastructure, publishing. Return can be one or more of: user, details, developed, source_control, architecture. @@ -392,7 +386,7 @@ Send JSON in this format: "project_description": "Description" }] , - "developed": ["In-house", []], + "developed": ["In-house"], "source_control": [ { "type": "GitHub", @@ -443,12 +437,6 @@ Send JSON in this format: } }, "stage":"Development", - "project_dependencies":[ - { - "name": "string", - "description": "string" - } - ], "supporting_tools": { "code_editors": { "main": [], @@ -456,38 +444,39 @@ Send JSON in this format: "List of strings" ] }, - "ui_tools": { + "user_interface": { "main": [], "others": [ "List of strings" ] }, - "diagram_tools": { + "diagrams": { "main": [], "others": [ "List of strings" ] }, - "project_tracking_tools": "string", - "documentation_tools": { + "project_tracking": "string", + "documentation": { "main": [], "others": [ "List of strings" ] }, - "communication_tools": { + "communication": { "main": [], "others": [ "List of strings" ] }, - "collaboration_tools": { + "collaboration": { "main": [], "others": [ "List of strings" ] }, - "incident_management": "string" + "incident_management": "string", + "miscellaneous": [] } } ``` diff --git a/aws_lambda_script/Dockerfile b/aws_lambda_script/Dockerfile index 4ff9930..560a2c8 100644 --- a/aws_lambda_script/Dockerfile +++ b/aws_lambda_script/Dockerfile @@ -1,23 +1,40 @@ -# Use an official Python runtime as a parent image +FROM python:3.12-slim AS builder +WORKDIR /build + +RUN apt-get update && apt-get install -y --no-install-recommends git \ + && rm -rf /var/lib/apt/lists/* + +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +COPY requirements.txt . +RUN pip install --no-cache-dir --upgrade pip setuptools wheel +# Github token needed to install Github private dependencies, then token deleted. +RUN --mount=type=secret,id=github_token \ + set -e; \ + if [ -f /run/secrets/github_token ]; then \ + GITHUB_TOKEN="$(cat /run/secrets/github_token)"; \ + git config --global url."https://${GITHUB_TOKEN}:x-oauth-basic@github.com/".insteadOf "https://github.com/"; \ + fi; \ + pip install --no-cache-dir -r requirements.txt; \ + rm -f /root/.gitconfig 2>/dev/null || true + +# Stage 2: runtime (Lambda base) FROM public.ecr.aws/lambda/python:3.12 -# Set the working directory in the container WORKDIR /var/task -# Upgrade tooling and awslambdaric -RUN pip install --no-cache-dir --upgrade pip setuptools wheel && \ - pip install --no-cache-dir --upgrade awslambdaric -# Create a non-root user and group manually -RUN mkdir -p /home/appuser && \ - chown -R 1000:1000 /home/appuser -# Copy the requirements.txt first to leverage Docker's cache for dependencies -COPY requirements.txt . -# Install the dependencies -RUN pip install --no-cache-dir -r requirements.txt -# Copy the app folder (with your Flask app) and the lambda_handler.py + +# Copy dependencies into /var/task so Lambda always imports them +COPY --from=builder /opt/venv/lib/python3.12/site-packages/ /var/task/ + +# Ensure /var/task is on sys.path +ENV PYTHONPATH="/var/task:${PYTHONPATH}" + COPY app/ /var/task/app/ COPY main.py /var/task/ + # Change ownership of the application files to the non-root user RUN chown -R 1000:1000 /var/task # Switch to non-root user USER 1000 -# The CMD specifies the handler to use in AWS Lambda + CMD ["main.lambda_handler"] \ No newline at end of file diff --git a/aws_lambda_script/app/resources.py b/aws_lambda_script/app/resources.py index b4ada27..ff7c848 100644 --- a/aws_lambda_script/app/resources.py +++ b/aws_lambda_script/app/resources.py @@ -4,12 +4,15 @@ from collections import Counter import requests from flask_restx import Resource, Namespace, reqparse, abort +from werkzeug.exceptions import HTTPException from .api_models import get_project_model, get_refresh_model from .utils import ( read_data, write_data, verify_cognito_token, cognito_data, + send_teams_alert, + logger, ) # Set namespace as /api/ - each request has to be /api/v1/ @@ -29,10 +32,6 @@ required_param = {"Authorization": "ID Token required"} -# Set logger for AWS cloudwatch to return just errors -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - def get_user_attributes(args): """Get user attributes from Cognito token. @@ -43,14 +42,17 @@ def get_user_attributes(args): dict: User attributes extracted from the token. Raises: - Exception: If token verification fails. + HTTPException: Propagates HTTP authentication failures from token verification. """ token = args["Authorization"] try: user_attributes = verify_cognito_token(token) return user_attributes + except HTTPException: + raise except Exception as error: logger.exception("Error verifying token: %s", error) + send_teams_alert("Unauthorized access attempt with token") abort(401, description="Not authorized") def get_user_information() -> dict: @@ -69,6 +71,7 @@ def get_user_information() -> dict: if not owner_email: logger.error("No email found in user attributes") + send_teams_alert("Unauthorized access attempt without email in token") abort(401, description="Not authorized") return {"email": owner_email, "groups": user_groups} @@ -270,102 +273,109 @@ class Filter(Resource): responses={200: "Success", 401: "Authorization is required"}, ) def get(self): - get_user_email(parser.parse_args()) - args = filterParser.parse_args() - filter_params = {k: v for k, v in args.items() if k and v} - - data = read_data("new_project_data.json") - projects = data["projects"] - - # Define paths for different filter types - paths = { - "email": ["user", "email"], - "roles": ["user", "roles"], - "name": ["details", "name"], - "languages": ["architecture", "languages"], - "hosting": ["architecture", "hosting"], - "database": ["architecture", "database"], - "frameworks": ["architecture", "frameworks"], - "cicd": ["architecture", "CICD"], - "environments": ["architecture", "environments"], - "infrastructure": ["architecture", "infrastructure"], - "publishing": ["architecture", "publishing"], - } - - # Special case filters - def filter_developed(project): - if "developed" not in filter_params: - return True - - # Handle cases where developed field might be empty or malformed - if ( - not project.get("developed") - or not isinstance(project["developed"], list) - or len(project["developed"]) < 1 - ): - return False - - # Get project type - first element in the developed list - project_type = ( - project["developed"][0].lower() if project["developed"][0] else "" - ) + try: + get_user_information() + args = filterParser.parse_args() + filter_params = {k: v for k, v in args.items() if k and v} + + data = read_data("new_project_data.json") + projects = data["projects"] + + # Define paths for different filter types + paths = { + "email": ["user", "email"], + "roles": ["user", "roles"], + "name": ["details", "name"], + "languages": ["architecture", "languages"], + "hosting": ["architecture", "hosting"], + "database": ["architecture", "database"], + "frameworks": ["architecture", "frameworks"], + "cicd": ["architecture", "CICD"], + "environments": ["architecture", "environments"], + "infrastructure": ["architecture", "infrastructure"], + "publishing": ["architecture", "publishing"], + } + + # Special case filters + def filter_developed(project): + if "developed" not in filter_params: + return True + + # Handle cases where developed field might be empty or malformed + if ( + not project.get("developed") + or not isinstance(project["developed"], list) + or len(project["developed"]) < 1 + ): + return False + + # Get project type - first element in the developed list + project_type = ( + project["developed"][0].lower() if project["developed"][0] else "" + ) - # Get partners - second element in the developed list if it exists and is a list - partners = [] - if len(project["developed"]) > 1 and isinstance( - project["developed"][1], list - ): - partners = [p.lower() for p in project["developed"][1] if p] + # Get partners - second element in the developed list if it exists and is a list + partners = [] + if len(project["developed"]) > 1 and isinstance( + project["developed"][1], list + ): + partners = [p.lower() for p in project["developed"][1] if p] - return any( - f.lower() == project_type or any(f.lower() in p for p in partners) - for f in filter_params["developed"] - ) - - def filter_source_control(project): - if "source_control" not in filter_params: - return True - return all( - any( - f.lower() in sc["type"].lower() - or any( - f.lower() in link["description"].lower() - or f.lower() in link["url"].lower() - for link in sc["links"] - ) - for sc in project["source_control"] + return any( + f.lower() == project_type or any(f.lower() in p for p in partners) + for f in filter_params["developed"] ) - for f in filter_params["source_control"] - ) - # Filter projects - filtered_projects = [] - for project in projects: - # Apply standard filters - standard_filters_pass = all( - matches_filter( - get_nested_values(project, paths[key]), filter_params[key] + def filter_source_control(project): + if "source_control" not in filter_params: + return True + return all( + any( + f.lower() in sc["type"].lower() + or any( + f.lower() in link["description"].lower() + or f.lower() in link["url"].lower() + for link in sc["links"] + ) + for sc in project["source_control"] + ) + for f in filter_params["source_control"] ) - for key in filter_params - if key in paths - ) - - # Apply special case filters - if ( - standard_filters_pass - and filter_developed(project) - and filter_source_control(project) - ): - # Handle return parameter - if "return" in filter_params: - filtered_projects.append( - build_project_response(project, filter_params["return"]) + # Filter projects + filtered_projects = [] + for project in projects: + # Apply standard filters + standard_filters_pass = all( + matches_filter( + get_nested_values(project, paths[key]), filter_params[key] ) - else: - filtered_projects.append(project) + for key in filter_params + if key in paths + ) - return filtered_projects, 200 + # Apply special case filters + if ( + standard_filters_pass + and filter_developed(project) + and filter_source_control(project) + ): + + # Handle return parameter + if "return" in filter_params: + filtered_projects.append( + build_project_response(project, filter_params["return"]) + ) + else: + filtered_projects.append(project) + + return filtered_projects, 200 + except HTTPException: + raise + except Exception as error: + logger.exception("Error filtering projects: %s", error) + send_teams_alert("Error filtering projects endpoint.") + abort(500, description="Internal server error") @ns.route("/projects") @@ -376,14 +386,21 @@ class Projects(Resource): @ns.doc(responses={200: "Success", 401: "Authorization is required"}) # @ns.marshal_list_with(project_model) def get(self): - data = read_data("new_project_data.json") - user_projects = [proj for proj in data["projects"]] - return user_projects, 200 + try: + data = read_data("new_project_data.json") + user_projects = [proj for proj in data["projects"]] + return user_projects, 200 + except HTTPException: + raise + except Exception as error: + logger.exception("Error fetching projects: %s", error) + send_teams_alert("Error in projects get endpoint.") + abort(500, description="Internal server error") # Add a new project to the list of projects # needs certain fields to be present in the JSON payload, # the non required will be saved as null if string or emtpy if list - @ns.marshal_list_with(project_model) + @ns.marshal_with(project_model) @ns.doc( body=project_model, responses={ @@ -394,79 +411,86 @@ def get(self): }, ) def post(self): - user_info = get_user_information() - owner_email = user_info["email"] + try: + user_info = get_user_information() + owner_email = user_info["email"] - # Check that required fields are present in the JSON payload - new_project = ns.payload - - logger.info('POSTING PROJECT: "%s"', new_project["details"][0]["name"]) - if ( - "user" not in new_project - or "details" not in new_project - or "email" not in new_project["user"][0] - or "name" not in new_project["details"][0] - ): - logger.error("Missing JSON data") - abort(406, description="Missing JSON data") - - # Ensure the email is set to owner_email and add 'Editor' role if email matches owner_email - for user in new_project["user"]: - if "email" not in user or not user["email"]: - user["email"] = owner_email - if user["email"] == owner_email: - if "roles" not in user: - user["roles"] = [] - if "Editor" not in user["roles"]: - user["roles"].append("Editor") - if not any(user["email"] == owner_email for user in new_project["user"]): - new_project["user"].append( - {"email": owner_email, "roles": ["Editor"], "grade": ""} - ) + # Check that required fields are present in the JSON payload + new_project = ns.payload - # Validate environments - environments = new_project.get("architecture", {}).get("environments", {}) - if not isinstance(environments, dict): - abort(400, description="Invalid environments data: Must be a dictionary") - - # If no environments have been selected, skip further validation - if len(environments) > 0: - # If environments dict is present, validate each key - # Each key will exist with a boolean value if the user has visited the page - for key in ["dev", "int", "uat", "preprod", "prod", "postprod"]: - if key not in environments or not isinstance(environments[key], bool): - abort(400, description=f"Invalid environments data: '{key}' must be a boolean") - - data = read_data("new_project_data.json") - - # Check if project with same name exists and has any matching user emails - new_project_name = new_project["details"][0]["name"] - new_project_emails = {user["email"] for user in new_project["user"]} - - matching_projects = [ - proj - for proj in data["projects"] - if proj["details"][0]["name"] == new_project_name - and any(user["email"] in new_project_emails for user in proj["user"]) - ] - - if matching_projects: - proj = matching_projects[0] - matching_email = next( - user["email"] - for user in proj["user"] - if user["email"] in new_project_emails - ) - logger.error("PROJECT '%s' WITH EMAIL '%s' ALREADY EXISTS", new_project_name, matching_email) - abort( - 409, - description=f"Project with the same name '{new_project_name}', and owner '{matching_email}' already exists", - ) - data["projects"].append(new_project) - logger.info("PROJECT '%s' ADDED SUCCESSFULLY", new_project_name) - write_data(data, "new_project_data.json") + logger.info('POSTING PROJECT: "%s"', new_project["details"][0]["name"]) + if ( + "user" not in new_project + or "details" not in new_project + or "email" not in new_project["user"][0] + or "name" not in new_project["details"][0] + ): + logger.error("Missing JSON data") + abort(406, description="Missing JSON data") + + # Ensure the email is set to owner_email and add 'Editor' role if email matches owner_email + for user in new_project["user"]: + if "email" not in user or not user["email"]: + user["email"] = owner_email + if user["email"] == owner_email: + if "roles" not in user: + user["roles"] = [] + if "Editor" not in user["roles"]: + user["roles"].append("Editor") + if not any(user["email"] == owner_email for user in new_project["user"]): + new_project["user"].append( + {"email": owner_email, "roles": ["Editor"], "grade": ""} + ) - return new_project, 201 + # Validate environments + environments = new_project.get("architecture", {}).get("environments", {}) + if not isinstance(environments, dict): + abort(400, description="Invalid environments data: Must be a dictionary") + + # If no environments have been selected, skip further validation + if len(environments) > 0: + # If environments dict is present, validate each key + # Each key will exist with a boolean value if the user has visited the page + for key in ["dev", "int", "uat", "preprod", "prod", "postprod"]: + if key not in environments or not isinstance(environments[key], bool): + abort(400, description=f"Invalid environments data: '{key}' must be a boolean") + + data = read_data("new_project_data.json") + + # Check if project with same name exists and has any matching user emails + new_project_name = new_project["details"][0]["name"] + new_project_emails = {user["email"] for user in new_project["user"]} + + matching_projects = [ + proj + for proj in data["projects"] + if proj["details"][0]["name"] == new_project_name + and any(user["email"] in new_project_emails for user in proj["user"]) + ] + + if matching_projects: + proj = matching_projects[0] + matching_email = next( + user["email"] + for user in proj["user"] + if user["email"] in new_project_emails + ) + logger.error("PROJECT '%s' WITH EMAIL '%s' ALREADY EXISTS", new_project_name, matching_email) + abort( + 409, + description=f"Project with the same name '{new_project_name}', and owner '{matching_email}' already exists", + ) + data["projects"].append(new_project) + logger.info("PROJECT '%s' ADDED SUCCESSFULLY", new_project_name) + write_data(data, "new_project_data.json") + + return new_project, 201 + except HTTPException: + raise + except Exception as error: + logger.exception("Error creating project: %s", error) + send_teams_alert("Error in projects post endpoint") + abort(500, description="Internal server error") @ns.doc( @@ -482,30 +506,37 @@ class ProjectDetail(Resource): # the name and the user email in the first user item in the user list @ns.marshal_with(project_model) def get(self, project_name): - # Sanitize project_name by replacing '%20' with spaces - project_name = ( - project_name.replace("%20", " ").replace("\r\n", "").replace("\n", "") - ) + try: + # Sanitize project_name by replacing '%20' with spaces + project_name = ( + project_name.replace("%20", " ").replace("\r\n", "").replace("\n", "") + ) - logger.info('FETCHING PROJECT: "%s"', project_name) + logger.info('FETCHING PROJECT: "%s"', project_name) - data = read_data("new_project_data.json") - project = next( - ( - proj - for proj in data["projects"] - if proj["details"][0]["name"] == project_name - ), - None, - ) - if not project: - logger.error("PROJECT '%s' NOT FOUND", project_name) - abort(404, description="Project not found") - return project, 200 + data = read_data("new_project_data.json") + project = next( + ( + proj + for proj in data["projects"] + if proj["details"][0]["name"] == project_name + ), + None, + ) + if not project: + logger.error("PROJECT '%s' NOT FOUND", project_name) + abort(404, description="Project not found") + return project, 200 + except HTTPException: + raise + except Exception as error: + logger.exception("Error fetching project detail: %s", error) + send_teams_alert("Error in project detail get endpoint") + abort(500, description="Internal server error") # Edit a project by taking the whole schema and replacing # the existing project with the same name and owner - @ns.marshal_list_with(project_model) + @ns.marshal_with(project_model) @ns.doc( body=project_model, responses={ @@ -516,70 +547,74 @@ def get(self, project_name): }, ) def put(self, project_name): - user_info = get_user_information() - owner_email = user_info["email"] - user_groups = user_info["groups"] + try: + user_info = get_user_information() + owner_email = user_info["email"] + user_groups = user_info["groups"] - project_name = ( - project_name.replace("%20", " ").replace("\r\n", "").replace("\n", "") - ) - updated_project = ns.payload + project_name = ( + project_name.replace("%20", " ").replace("\r\n", "").replace("\n", "") + ) + updated_project = ns.payload + + logger.info('EDITING PROJECT: "%s"', project_name) - logger.info('EDITING PROJECT: "%s"', project_name) + if "user" not in updated_project or "details" not in updated_project: + logger.error("Missing JSON data") + abort(406, description="Missing JSON data") + + # Validate environments + environments = updated_project.get("architecture", {}).get("environments", {}) + if not isinstance(environments, dict): + abort(400, description="Invalid environments data: Must be a dictionary") + for key in ["dev", "int", "uat", "preprod", "prod", "postprod"]: + if key not in environments or not isinstance(environments[key], bool): + abort(400, description=f"Invalid environments data: '{key}' must be a boolean") - if "user" not in updated_project or "details" not in updated_project: - logger.error("Missing JSON data") - abort(406, description="Missing JSON data") + data = read_data("new_project_data.json") - # Validate environments - environments = updated_project.get("architecture", {}).get("environments", {}) - if not isinstance(environments, dict): - abort(400, description="Invalid environments data: Must be a dictionary") - for key in ["dev", "int", "uat", "preprod", "prod", "postprod"]: - if key not in environments or not isinstance(environments[key], bool): - abort(400, description=f"Invalid environments data: '{key}' must be a boolean") + logger.info(f"Authenticated user is in admin group: {is_auth_user_in_admin_group(user_groups)}") - # Ensure the email is set to owner_email + project = next( + ( + proj + for proj in data["projects"] + if proj["details"][0]["name"] == project_name + and (any(user["email"] == owner_email for user in proj["user"]) or is_auth_user_in_admin_group(user_groups)) + ), + None, + ) - data = read_data("new_project_data.json") + if not project: + logger.error("PROJECT '%s' NOT FOUND", project_name) + abort(404, description=f"{project_name} with user {owner_email} not found") - logger.info(f"Authenticated user is in admin group: {is_auth_user_in_admin_group(user_groups)}") + # Update the project details + project.update(updated_project) + logger.info("PROJECT '%s' UPDATED SUCCESSFULLY", project_name) + write_data(data, "duplicates.json") - project = next( - ( - proj - for proj in data["projects"] - if proj["details"][0]["name"] == project_name - and (any(user["email"] == owner_email for user in proj["user"]) or is_auth_user_in_admin_group(user_groups)) - ), - None, - ) - - if not project: - logger.error("PROJECT '%s' NOT FOUND", project_name) - abort(404, description=f"{project_name} with user {owner_email} not found") - - # Update the project details - project.update(updated_project) - logger.info("PROJECT '%s' UPDATED SUCCESSFULLY", project_name) - write_data(data, "duplicates.json") - - duplicate_data = read_data("duplicates.json") - duplicate_arr = [] - for proj in duplicate_data["projects"]: - duplicate_arr.append(proj["details"][0]["name"]) - - counts = Counter(duplicate_arr) - duplicate_arr = [item for item, count in counts.items() if count > 1] - if len(duplicate_arr) > 0: - write_data({"projects": []}, "duplicates.json") - return abort(409, description="Project with the same name already exists") + duplicate_data = read_data("duplicates.json") + duplicate_arr = [] + for proj in duplicate_data["projects"]: + duplicate_arr.append(proj["details"][0]["name"]) - write_data({"projects": []}, "duplicates.json") + counts = Counter(duplicate_arr) + duplicate_arr = [item for item, count in counts.items() if count > 1] + if len(duplicate_arr) > 0: + write_data({"projects": []}, "duplicates.json") + return abort(409, description="Project with the same name already exists") - write_data(data, "new_project_data.json") + write_data({"projects": []}, "duplicates.json") + write_data(data, "new_project_data.json") - return project, 200 + return project, 200 + except HTTPException: + raise + except Exception as error: + logger.exception("Error updating project detail: %s", error) + send_teams_alert("Error in project detail put endpoint.") + abort(500, description="Internal server error") # Read the client keys from S3 bucket for Cognito @@ -727,4 +762,4 @@ def exchange_code_for_tokens(code): raise Exception(f"Error: {response.status_code}, {response.json()}") # Return the parsed JSON response - return response.json() \ No newline at end of file + return response.json() diff --git a/aws_lambda_script/app/utils.py b/aws_lambda_script/app/utils.py index eb246df..190dd6b 100644 --- a/aws_lambda_script/app/utils.py +++ b/aws_lambda_script/app/utils.py @@ -3,20 +3,37 @@ import boto3 import jwt import requests +import logging +from typing import Any from flask_restx import abort from botocore.exceptions import ClientError from jwt.algorithms import RSAAlgorithm +try: + from keh_teams_alert import TeamsAlertClient +except ModuleNotFoundError: + TeamsAlertClient = None + +# Setup Logging +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +# global logger +logger = logging.getLogger(__name__) + # Connecting to S3 BUCKET_NAME = os.getenv("TECH_AUDIT_DATA_BUCKET", "sdp-dev-tech-audit-tool-api") -SECRET_NAME = os.getenv("TECH_AUDIT_SECRET_MANAGER", "sdp-dev-tech-audit-tool-api/secrets") +TA_SECRET_NAME = os.getenv("TECH_AUDIT_SECRET_MANAGER", "sdp-dev-tech-audit-tool-api/secrets") REGION_NAME = os.getenv("AWS_DEFAULT_REGION", "eu-west-2") OBJECT_NAME = "new_project_data.json" +AWS_ENV = os.getenv("AWS_ACCOUNT_NAME") +branch_name = os.getenv("BRANCH_NAME", "") +ALERTS_AZURE_SECRET_NAME = os.getenv("AZURE_SECRET_NAME", "") # Create an S3 client s3 = boto3.client("s3", region_name=REGION_NAME) -def read_cognito_data(): +def read_cognito_data(secret_name): """ Reads and returns Cognito data from AWS Secrets Manager. This function creates a Secrets Manager client using boto3, retrieves the secret value @@ -32,7 +49,7 @@ def read_cognito_data(): client = session.client(service_name="secretsmanager", region_name=REGION_NAME) try: - get_secret_value_response = client.get_secret_value(SecretId=SECRET_NAME) + get_secret_value_response = client.get_secret_value(SecretId=secret_name) except ClientError as e: # For a list of exceptions thrown, see # https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html @@ -40,7 +57,9 @@ def read_cognito_data(): return json.loads(get_secret_value_response["SecretString"]) -cognito_data = read_cognito_data() +cognito_data = read_cognito_data(TA_SECRET_NAME) + + # Used for the view project or get projects routes. This reads the data from the S3 bucket. def read_data(object_name): @@ -68,7 +87,8 @@ def read_data(object_name): if e.response["Error"]["Code"] == "NoSuchKey": data = {"projects": []} else: - abort(500, description=f"Error reading data: {e}") + logger.exception("Failed to read %s from S3", object_name) + abort(500, description="Internal server error") return data @@ -91,7 +111,9 @@ def write_data(new_data, object_name): Body=json.dumps(new_data, indent=4).encode("utf-8"), ) except ClientError as e: - abort(500, description=f"Error writing data: {e}") + logger.exception("S3 write failed for %s", object_name) + send_teams_alert(f"Failed to write data - {object_name} to S3 bucket") + abort(500, description="Internal server error") def get_cognito_jwks(): """ @@ -168,3 +190,101 @@ def verify_cognito_token(id_token): abort(401, description="Token is expired") except jwt.InvalidTokenError as e: abort(401, description=f"Invalid token: {str(e)}") + +# Initialize Teams Alert Client +tenant_id = "" +client_id = "" +client_secret = "" +scope = "" +alert_url = "" + +if ALERTS_AZURE_SECRET_NAME: + try: + logger.info("Attempting to read Azure credentials from Secrets Manager") + azure_secret_name = read_cognito_data(ALERTS_AZURE_SECRET_NAME) + if azure_secret_name: + logger.info("Azure credentials found. Initializing Teams Alert Client.") + tenant_id = azure_secret_name["azure_tenant_id"] + client_id = azure_secret_name["azure_client_id"] + client_secret = azure_secret_name["azure_client_secret"] + scope = azure_secret_name["azure_scope"] + alert_url = azure_secret_name["azure_webhook_url"] + else: + logger.warning("Azure credentials not found. Teams alerts will not be sent.") + except Exception as e: + logger.error(f"Error initializing Teams Alert Client: {e}, Teams alerts will not be sent.") +else: + logger.warning("ALERTS_AZURE_SECRET_NAME environment variable not set. Teams alerts will not be sent.") + + +def get_teams_alert_client() -> Any: + """Create and return a Teams alert client when the dependency is available. + + Returns: + Any: An initialized ``TeamsAlertClient`` instance, or ``None`` when the + package is unavailable or client initialization fails. + """ + if TeamsAlertClient is None: + logger.warning("keh_teams_alert is not installed. Teams alerts are disabled.") + return None + + try: + logger.info("Creating TeamsAlertClient instance") + teams_alert_client = TeamsAlertClient( + tenant_id=tenant_id, + client_id=client_id, + client_secret=client_secret, + scope=scope, + ) + return teams_alert_client + except Exception as e: + logger.error(f"Failed to initialize TeamsAlertClient: {e}") + return None + + +def setup_alert_message(message: str, aws_env: str | None = None) -> dict: + """Build the payload sent to the Teams alert webhook. + + Args: + message (str): The alert description body. + aws_env (str | None): Optional environment name override. + + Returns: + dict: A webhook payload containing channel and formatted message fields. + """ + env = aws_env or AWS_ENV or "Unknown Environment" + return { + "channel": "KEH Alerts", + "message": f"🚨 Tech Audit Tool API {env}🚨
Description: {message}", + } + + +def send_teams_alert(message) -> None: + """Send an alert message to Microsoft Teams when alerting is enabled. + + Alerts are only sent when a Teams client can be initialized and the current + branch is ``main``. + + Args: + message: The alert message body to send. + """ + logger.info("Preparing to send alert to Teams Channel") + teams_alert_client = get_teams_alert_client() + if ( + teams_alert_client and branch_name == "main" + ): # Only send alerts if client is initialized and on main branch + try: + alert_message = setup_alert_message(message) + teams_alert_client.post_to_webhook(alert_url, alert_message) + logger.info("Alert sent successfully to Teams Channel") + except Exception as e: + logger.error(f"Failed to send alert to Teams alert: {e}") + else: + if not teams_alert_client: + logger.warning("TeamsAlertClient is not initialized. Cannot send alert.") + elif branch_name != "main": + logger.warning( + f"Current branch is '{branch_name}'. Alerts are only sent from 'main' branch. Alert not sent." + ) + else: + logger.error("TeamsAlertClient is not available. Alert not sent.") \ No newline at end of file diff --git a/aws_lambda_script/requirements.txt b/aws_lambda_script/requirements.txt index c3b874b..f2b6392 100644 --- a/aws_lambda_script/requirements.txt +++ b/aws_lambda_script/requirements.txt @@ -1,8 +1,9 @@ -Flask==3.0.3 -boto3==1.35.33 -botocore==1.35.38 -flask-restx==1.3.0 +Flask==3.1.3 +boto3==1.42.88 +botocore==1.42.88 +flask-restx==1.3.2 aws-wsgi==0.2.7 -pyjwt>=2.7.0 -requests==2.32.4 -cryptography==46.0.5 \ No newline at end of file +pyjwt>=2.12.1 +requests==2.33.1 +cryptography==46.0.7 +keh-teams-alert @ git+https://github.com/ONS-Innovation/keh-teams-alert@v0.1.0 \ No newline at end of file diff --git a/concourse/ci.yml b/concourse/ci.yml index 9572e3c..fa3f86b 100644 --- a/concourse/ci.yml +++ b/concourse/ci.yml @@ -69,6 +69,64 @@ terraform-task: &terraform-task ./resource-repo/concourse/scripts/terraform_infra.sh timeout: 30m +notify-azure-webhook-task: ¬ify-azure-webhook-task + config: + platform: linux + params: + secrets: ((sdp_azure_webhook_secrets)) + branch: ((branch)) + env: ((env)) + image_resource: + type: registry-image + source: + repository: alpine + tag: latest + run: + path: sh + args: + - -euc + - | + if [ "$branch" != "main" ]; then + echo "Branch is '${branch}', skipping webhook notification." + exit 0 + fi + apk add --no-cache curl jq + + AZURE_CLIENT_ID=$(echo "$secrets" | jq -r .azure_client_id) + AZURE_CLIENT_SECRET=$(echo "$secrets" | jq -r .azure_client_secret) + AZURE_SCOPE=$(echo "$secrets" | jq -r .azure_scope) + AZURE_TENANT_ID=$(echo "$secrets" | jq -r .azure_tenant_id) + AZURE_WEBHOOK_URL=$(echo "$secrets" | jq -r .azure_webhook_url) + CHANNEL_ID=$(echo "$secrets" | jq -r .channel_id) + + token_json="$(curl -sS -X POST \ + "https://login.microsoftonline.com/${AZURE_TENANT_ID}/oauth2/v2.0/token" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "client_id=${AZURE_CLIENT_ID}" \ + --data-urlencode "client_secret=${AZURE_CLIENT_SECRET}" \ + --data-urlencode "grant_type=client_credentials" \ + --data-urlencode "scope=${AZURE_SCOPE}")" + + access_token="$(printf '%s' "$token_json" | jq -r '.access_token // empty')" + if [ -z "$access_token" ]; then + error_code="$(printf '%s' "$token_json" | jq -r '.error // "unknown_error"')" + error_description="$(printf '%s' "$token_json" | jq -r '.error_description // "No error description provided"')" + echo "Failed to get access token. error=${error_code}; description=${error_description}" + exit 1 + fi + + pipeline_msg="status:🚨 Concourse failure
service: Tech Audit Tool API
event: Pipeline failure in the ${env} environment." + + payload="$(cat < release-tag-output/tag - echo "Using tag: ${tag}" - chmod u+x ./resource-repo/concourse/scripts/assume_role.sh - chmod u+x ./resource-repo/concourse/scripts/build_image.sh - source ./resource-repo/concourse/scripts/assume_role.sh - ./resource-repo/concourse/scripts/build_image.sh - timeout: 10m - - put: images-to-shared-ecr - inputs: - - built-images - - release-tag-output - params: - image: built-images/tech_audit_tool_api.tar - additional_tags: release-tag-output/tag - - <<: *terraform-task - params: - secrets: ((sdp_((env))_secrets_tat_api)) - github_access_token: ((github_access_token)) - env: dev -- name: release-build-and-push-prod - public: true - plan: - - get: resource-repo - passed: [deploy-after-github-release-dev] - trigger: false # Manual trigger only - timeout: 5m - - get: github-release-tag - passed: [deploy-after-github-release-dev] - - task: build-image - privileged: true - config: - platform: linux - image_resource: - type: docker-image - source: - repository: hashicorp/terraform - inputs: - - name: resource-repo - - name: github-release-tag - params: - aws_account_id: ((aws_account_sdp_prod)) - aws_role_arn: arn:aws:iam::((aws_account_sdp_prod)):role/sdp-concourse-prod - secrets: ((sdp_prod_secrets_tat_api)) - env: prod - tag: github-release-tag/tag - repo_name: ((repo_name)) - branch: ((branch)) - run: - path: sh - args: + - do: + - get: github-release-tag + passed: [calculate-tag] + trigger: true + - get: resource-repo + timeout: 5m + - task: deploy-release + privileged: true + config: + platform: linux + image_resource: + type: docker-image + source: + repository: hashicorp/terraform + inputs: + - name: resource-repo + - name: github-release-tag + outputs: + - name: built-images + - name: release-tag-output + params: + aws_account_id: ((aws_account_sdp_((env)))) + aws_role_arn: arn:aws:iam::((aws_account_sdp_((env)))):role/sdp-concourse-((env)) + secrets: ((sdp_((env))_secrets_tat_api)) + env: dev + tag: github-release-tag/tag # Use the release tag from the resource + repo_name: ((repo_name)) + github_access_token: ((github_access_token)) + run: # binary used to build the image + path: sh + args: - -cx - | - if [[ "$branch" != "main" ]]; then - echo "Not on main branch, skipping build." - exit 0 - fi apk add --no-cache aws-cli podman jq iptables curl export repo_name=((repo_name)) export tag=$(cat github-release-tag/tag) - echo "Using release tag: ${tag}" + # Write the tag to a file for output + echo "${tag}" > release-tag-output/tag + echo "Using tag: ${tag}" chmod u+x ./resource-repo/concourse/scripts/assume_role.sh chmod u+x ./resource-repo/concourse/scripts/build_image.sh source ./resource-repo/concourse/scripts/assume_role.sh ./resource-repo/concourse/scripts/build_image.sh - timeout: 10m - - <<: *terraform-task - params: - github_access_token: ((github_access_token)) - env: prod - secrets: ((sdp_prod_secrets_tat_api)) + timeout: 10m + - put: images-to-shared-ecr + inputs: + - built-images + - release-tag-output + params: + image: built-images/tech_audit_tool_api.tar + additional_tags: release-tag-output/tag + - <<: *terraform-task + params: + secrets: ((sdp_((env))_secrets_tat_api)) + github_access_token: ((github_access_token)) + env: dev + on_failure: + do: + - task: notify-azure-webhook + <<: *notify-azure-webhook-task + params: + env: dev + +- name: release-build-and-push-prod + public: true + plan: + - do: + - get: resource-repo + passed: [deploy-after-github-release-dev] + trigger: false # Manual trigger only + timeout: 5m + - get: github-release-tag + passed: [deploy-after-github-release-dev] + - task: build-image + privileged: true + config: + platform: linux + image_resource: + type: docker-image + source: + repository: hashicorp/terraform + inputs: + - name: resource-repo + - name: github-release-tag + outputs: + - name: built-images + params: + aws_account_id: ((aws_account_sdp_prod)) + aws_role_arn: arn:aws:iam::((aws_account_sdp_prod)):role/sdp-concourse-prod + secrets: ((sdp_prod_secrets_tat_api)) + env: prod + tag: github-release-tag/tag + repo_name: ((repo_name)) + branch: ((branch)) + github_access_token: ((github_access_token)) + run: + path: sh + args: + - -cx + - | + if [[ "$branch" != "main" ]]; then + echo "Not on main branch, skipping build." + exit 0 + fi + apk add --no-cache aws-cli podman jq iptables curl + export repo_name=((repo_name)) + export tag=$(cat github-release-tag/tag) + echo "Using release tag: ${tag}" + chmod u+x ./resource-repo/concourse/scripts/assume_role.sh + chmod u+x ./resource-repo/concourse/scripts/build_image.sh + source ./resource-repo/concourse/scripts/assume_role.sh + ./resource-repo/concourse/scripts/build_image.sh + timeout: 10m + - <<: *terraform-task + params: + github_access_token: ((github_access_token)) + env: prod + secrets: ((sdp_prod_secrets_tat_api)) + on_failure: + do: + - task: notify-azure-webhook + <<: *notify-azure-webhook-task + params: + env: prod \ No newline at end of file diff --git a/concourse/scripts/build_image.sh b/concourse/scripts/build_image.sh index 0d8bb08..fd68d54 100755 --- a/concourse/scripts/build_image.sh +++ b/concourse/scripts/build_image.sh @@ -7,7 +7,17 @@ container_image=$(echo "$secrets" | jq -r .ecr_repository) aws ecr get-login-password --region eu-west-2 | podman --storage-driver=vfs login --username AWS --password-stdin ${aws_account_id}.dkr.ecr.eu-west-2.amazonaws.com -podman build -t ${container_image}:${tag} resource-repo/aws_lambda_script/ +# Write token to a temp file and mount as a build secret (only used when needed) +tmp_token_file="$(mktemp)" +chmod 600 "$tmp_token_file" +printf '%s' "${github_access_token}" > "$tmp_token_file" + +podman build \ + --secret id=github_token,src="$tmp_token_file" \ + -t ${container_image}:${tag} resource-repo/aws_lambda_script/ + +# Remove token file after build +rm -f "$tmp_token_file" podman tag ${container_image}:${tag} ${aws_account_id}.dkr.ecr.eu-west-2.amazonaws.com/${container_image}:${tag} diff --git a/concourse/scripts/terraform_infra.sh b/concourse/scripts/terraform_infra.sh index 87f5ba6..7c7f611 100644 --- a/concourse/scripts/terraform_infra.sh +++ b/concourse/scripts/terraform_infra.sh @@ -8,6 +8,10 @@ domain=$(echo "$secrets" | jq -r .domain) service_subdomain=$(echo "$secrets" | jq -r .service_subdomain) ecr_repository=$(echo "$secrets" | jq -r .ecr_repository) +azure_secret_name=$(echo "$secrets" | jq -r .azure_secret_name) +aws_account_name=$(echo "$secrets" | jq -r .domain) + +branch_name=$branch export AWS_ACCESS_KEY_ID=$aws_access_key_id export AWS_SECRET_ACCESS_KEY=$aws_secret_access_key @@ -29,6 +33,9 @@ terraform apply \ -var "service_subdomain=$service_subdomain" \ -var "container_ver=${tag}" \ -var "ecr_repository=$ecr_repository" \ +-var "azure_secret_name=$azure_secret_name" \ +-var "branch_name=$branch_name" \ +-var "aws_account_name=$aws_account_name" \ -auto-approve cd ../api_gateway diff --git a/docs/endpoints/auth.md b/docs/endpoints/auth.md index 61d0e3a..a1b6325 100644 --- a/docs/endpoints/auth.md +++ b/docs/endpoints/auth.md @@ -1,6 +1,6 @@ # Authentication -The API uses AWS Cognito for authentication. All API endpoints require a valid ID token in the Authorization header. +The API uses AWS Cognito for authentication. This endpoint exchanges an authorization code for tokens and does not require an `Authorization` header. ## Verify Token @@ -20,4 +20,3 @@ Exchange authorization code for tokens. |-----------|-------------| | `200` | Success. Response body contains the new ID token: `{"id_token": "", "refresh_token": ""}`. | | `400` | Bad Request | -| `401` | Authorization is required | diff --git a/docs/endpoints/filter.md b/docs/endpoints/filter.md index e29eb45..6a9e2af 100644 --- a/docs/endpoints/filter.md +++ b/docs/endpoints/filter.md @@ -21,14 +21,14 @@ Requires a valid Cognito ID token in the Authorization header. | `email` | string | User email to filter by (comma-separated) | False | | `roles` | string | Roles to filter by (comma-separated) | False | | `name` | string | Project name to filter by (comma-separated) | False | -| `developed` | string | Filter by development type ("In house", "Partnership", "Outsourced") | False | +| `developed` | string | Filter by development type (for example `In-house`, `Partnership`, `Outsourced`) | False | | `languages` | string | Programming languages to filter by (comma-separated) | False | | `source_control` | string | Source control systems to filter by (comma-separated) | False | | `hosting` | string | Hosting platforms to filter by (comma-separated) | False | | `database` | string | Database types to filter by (comma-separated) | False | | `frameworks` | string | Frameworks to filter by (comma-separated) | False | | `cicd` | string | CI/CD tools to filter by (comma-separated) | False | -| `environments` | dict | List of environments that project are deployed | False | +| `environments` | string | Environment names to filter by (comma-separated) | False | | `infrastructure` | string | Infrastructure tools to filter by (comma-separated) | False | | `publishing` | string | Publishing target to filter by (comma-separated) | False | | `return` | string | Sections to return in response (user, details, developed, source_control, architecture) | False | @@ -39,4 +39,4 @@ Requires a valid Cognito ID token in the Authorization header. |-------------|-----------------------------------------| | `200` | Returns filtered list of projects matching the specified criteria. If the `return` parameter is specified, only the requested sections will be included in the response. | | `401` | Authorization is required | -| `404` | Project not found | \ No newline at end of file +| `500` | Internal server error | \ No newline at end of file diff --git a/docs/endpoints/project.md b/docs/endpoints/project.md index 6c38b2c..3b68335 100644 --- a/docs/endpoints/project.md +++ b/docs/endpoints/project.md @@ -4,7 +4,7 @@ `GET /api/v1/projects/{project_name}` -Returns the project associated with the authenticated user. +Returns the project matching the supplied project name. ### Authorization @@ -27,6 +27,7 @@ Requires a valid Cognito ID token in the Authorization header. | `200` | Returns a project object. | | `401` | Authorization is required | | `404` | Project not found | +| `500` | Internal server error | ## Update a Project @@ -170,7 +171,7 @@ Requires a valid Cognito ID token in the Authorization header. "string" ] }, - "ui_tools": { + "user_interface": { "main": [ "string" ], @@ -178,7 +179,7 @@ Requires a valid Cognito ID token in the Authorization header. "string" ] }, - "diagram_tools": { + "diagrams": { "main": [ "string" ], @@ -186,8 +187,8 @@ Requires a valid Cognito ID token in the Authorization header. "string" ] }, - "project_tracking_tools": "string", - "documentation_tools": { + "project_tracking": "string", + "documentation": { "main": [ "string" ], @@ -195,7 +196,7 @@ Requires a valid Cognito ID token in the Authorization header. "string" ] }, - "communication_tools": { + "communication": { "main": [ "string" ], @@ -203,7 +204,7 @@ Requires a valid Cognito ID token in the Authorization header. "string" ] }, - "collaboration_tools": { + "collaboration": { "main": [ "string" ], @@ -211,7 +212,8 @@ Requires a valid Cognito ID token in the Authorization header. "string" ] }, - "incident_management": "string" + "incident_management": "string", + "miscellaneous": [] } } ``` @@ -221,6 +223,9 @@ Requires a valid Cognito ID token in the Authorization header. | Status Code | Description | |-------------|-----------------------------------------| | `200` | Returns the same project object as the request body. | +| `400` | Invalid environments data | | `401` | Authorization is required | | `404` | Project not found | -| `406` | Missing JSON data | \ No newline at end of file +| `406` | Missing JSON data | +| `409` | Project with the same name already exists | +| `500` | Internal server error | \ No newline at end of file diff --git a/docs/endpoints/projects.md b/docs/endpoints/projects.md index 1c59f4a..6e7b74c 100644 --- a/docs/endpoints/projects.md +++ b/docs/endpoints/projects.md @@ -4,7 +4,7 @@ `GET /api/v1/projects` -Returns all projects associated with the authenticated user. +Returns all projects currently stored by the API. ### Authorization @@ -20,7 +20,7 @@ Requires a valid Cognito ID token in the Authorization header. |-------------|-----------------------------------------| | `200` | Returns a list of project objects. | | `401` | Authorization is required | -| `404` | Projects not found | +| `500` | Internal server error | ## Create a Project @@ -157,7 +157,7 @@ Requires a valid Cognito ID token in the Authorization header. "string" ] }, - "ui_tools": { + "user_interface": { "main": [ "string" ], @@ -165,7 +165,7 @@ Requires a valid Cognito ID token in the Authorization header. "string" ] }, - "diagram_tools": { + "diagrams": { "main": [ "string" ], @@ -173,8 +173,8 @@ Requires a valid Cognito ID token in the Authorization header. "string" ] }, - "project_tracking_tools": "string", - "documentation_tools": { + "project_tracking": "string", + "documentation": { "main": [ "string" ], @@ -182,7 +182,7 @@ Requires a valid Cognito ID token in the Authorization header. "string" ] }, - "communication_tools": { + "communication": { "main": [ "string" ], @@ -190,7 +190,7 @@ Requires a valid Cognito ID token in the Authorization header. "string" ] }, - "collaboration_tools": { + "collaboration": { "main": [ "string" ], @@ -198,7 +198,8 @@ Requires a valid Cognito ID token in the Authorization header. "string" ] }, - "incident_management": "string" + "incident_management": "string", + "miscellaneous": [] } } ``` diff --git a/docs/endpoints/refresh.md b/docs/endpoints/refresh.md index 6913d00..c062188 100644 --- a/docs/endpoints/refresh.md +++ b/docs/endpoints/refresh.md @@ -1,22 +1,23 @@ # Authentication -The API uses AWS Cognito for authentication. All API endpoints require a valid ID token in the Authorization header. +The API uses AWS Cognito for authentication. This endpoint exchanges a refresh token for a new ID token and does not require an `Authorization` header. ## Refresh Token -`GET /api/v1/refresh` +`POST /api/v1/refresh` -Exchange authorization code for tokens. +Exchange a refresh token for a new ID token. -### Query Parameters +### Request Body -| Parameter | Description | -|-----------|-------------| -| `refresh_token` | Refresh token from Cognito callback | +| Parameter | Type | Description | Required | +|-----------|------|-------------|----------| +| `refresh_token` | string | Refresh token returned by Cognito | Yes | ### Response | Status Code | Description | |-------------|-------------| | `200` | Success. Response body contains the new ID token: `{"id_token": "string"}`. | -| `400` | Bad Request | \ No newline at end of file +| `400` | Bad Request | +| `401` | Failed to refresh token | \ No newline at end of file diff --git a/docs/endpoints/user.md b/docs/endpoints/user.md index a262eb3..e2561a5 100644 --- a/docs/endpoints/user.md +++ b/docs/endpoints/user.md @@ -4,7 +4,7 @@ `GET /api/v1/user` -Returns the email address of the authenticated user. +Returns the email address and Cognito groups of the authenticated user. ### Authorization @@ -17,6 +17,5 @@ Requires a valid Cognito ID token in the Authorization header. ### Responses | Status Code | Description | |--------|-------------| -| `200` | Success. Returns the user's email address. ` { "email": "string" }` | -| `401` | Authorization is required | -| `404` | User not found | \ No newline at end of file +| `200` | Success. Returns the user object. `{"email": "string", "groups": []}` | +| `401` | Authorization is required | \ No newline at end of file diff --git a/docs/infrastructure.md b/docs/infrastructure.md index 29d3fc1..4bde25d 100644 --- a/docs/infrastructure.md +++ b/docs/infrastructure.md @@ -1,58 +1,94 @@ # Infrastructure -## AWS Resources +## Overview -Before terraforming the API, you must have the API Python code containerised, using Docker, and have an elastic container registry (ECR) available. +This API is deployed on AWS using Terraform modules in the `terraform/` directory. -Please refer to the [deployment](deployment.md) guide for more information. +Before applying Terraform: -There are **5** AWS resources that are created by the terraform script: +- Build and push the Lambda container image to ECR. +- Ensure an ECR repository already exists. -- Secrets Manager (Secrets) -- S3 Bucket (Storage) -- Cognito User Pool (Authentication) -- Lambda Function (Lambda) -- API Gateway (api_gateway) +For container build and push steps, see the [deployment](deployment.md) guide. -Go through the list and deploy each resource one by one. +## Terraform Modules -For each resource, you will need to set the `domain` and `service_subdomain` variables in the `tfvars` file. +The Terraform configuration is split into five modules: -### 1. ECR Repository +- `terraform/secrets/` (Secrets Manager) +- `terraform/storage/` (S3) +- `terraform/authentication/` (Cognito) +- `terraform/lambda/` (Lambda) +- `terraform/api_gateway/` (API Gateway) -Make sure you have an ECR repository created in the AWS account. This will be used in the `S3 bucket` and `Lambda function`. +Each module has environment-specific files under `env//`. -### 2. Secrets Manager +For each module, set `domain` and `service_subdomain` in the tfvars file for your target environment. -Run like normal. Leave the `cognito_pool_id`, `cognito_client_id`, `cognito_client_secret`, and `redirect_uri` variables blank. +## Deployment Order -### 3. S3 Bucket +### 1. ECR Repository (prerequisite) -Set the `ecr_repository_name` variable in the `tfvars` file. Then run the terraform script. +Create the ECR repository first. It is referenced by the storage and lambda modules. -### 4. Cognito User Pool +### 2. Secrets Manager (`terraform/secrets/`) -Run like normal. +Apply this module first. -### 5. Lambda Function +For the first apply, leave these Cognito values blank in tfvars: -The tech audit S3 bucket and the secrets manager secret are created by the terraform script. The aws cognito token url is set by the terraform script. Then run the terraform script for the lambda function and this data is set in the lambda function. +- `cognito_pool_id` +- `cognito_client_id` +- `cognito_client_secret` +- `redirect_uri` -### 6. API Gateway +### 3. Storage (`terraform/storage/`) -Run like normal. Note down the URLs in the outputs. +Set `ecr_repository_name` in tfvars, then apply. -### 7. Secrets Manager Re-application +### 4. Authentication (`terraform/authentication/`) -Go back to the `Secrets Manager` resource and set the `cognito_pool_id`, `cognito_client_id`, `cognito_client_secret`, and `redirect_uri` variables. +Apply the Cognito module and capture outputs required by the secrets module. -### 8. Finished +### 5. Lambda (`terraform/lambda/`) -![AWS Resources](assets/explanation.png) +Set the following lambda-specific variables before apply: -## Terraform Configuration +- `ecr_repository` (ECR repository containing the Lambda image) +- `container_ver` (image tag to deploy) +- `azure_secret_name` (Secrets Manager secret name for Teams alert credentials) +- `branch_name` (alert gate: only `main` sends alerts) +- `aws_account_name` (environment label shown in alert messages derived from domain) -Flow chart explanation of the Terraform setup and infrastructure components. +The secret referenced by `azure_secret_name` must contain JSON in this shape: -![Infrastructure Diagram](assets/creation_flow.png) +```json +{ + "azure_tenant_id": "tenant-id", + "azure_client_id": "client-id", + "azure_client_secret": "client-secret", + "azure_scope": "https://graph.microsoft.com/.default", + "azure_webhook_url": "https://example.webhook.office.com/..." +} +``` + +### 6. API Gateway (`terraform/api_gateway/`) + +Apply the API Gateway module and note output URLs. + +### 7. Re-apply Secrets Manager (`terraform/secrets/`) + +Update and re-apply secrets with Cognito values populated: +- `cognito_pool_id` +- `cognito_client_id` +- `cognito_client_secret` +- `redirect_uri` + +### 8. Complete + +![AWS Resources](assets/explanation.png) + +## Terraform Configuration Diagram + +![Infrastructure Diagram](assets/creation_flow.png) diff --git a/poetry.lock b/poetry.lock index 9f66bb7..e3ba9ad 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. [[package]] name = "aniso8601" @@ -51,6 +51,45 @@ files = [ {file = "aws_wsgi-0.2.7-py2.py3-none-any.whl", hash = "sha256:37e16b6a211f251d9f72ab85018c4127a59dc0fc81263c19f312a6a8333f9908"}, ] +[[package]] +name = "azure-core" +version = "1.39.0" +description = "Microsoft Azure Core Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["alerts"] +files = [ + {file = "azure_core-1.39.0-py3-none-any.whl", hash = "sha256:4ac7b70fab5438c3f68770649a78daf97833caa83827f91df9c14e0e0ea7d34f"}, + {file = "azure_core-1.39.0.tar.gz", hash = "sha256:8a90a562998dd44ce84597590fff6249701b98c0e8797c95fcdd695b54c35d74"}, +] + +[package.dependencies] +requests = ">=2.21.0" +typing-extensions = ">=4.6.0" + +[package.extras] +aio = ["aiohttp (>=3.0)"] +tracing = ["opentelemetry-api (>=1.26,<2.0)"] + +[[package]] +name = "azure-identity" +version = "1.25.3" +description = "Microsoft Azure Identity Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["alerts"] +files = [ + {file = "azure_identity-1.25.3-py3-none-any.whl", hash = "sha256:f4d0b956a8146f30333e071374171f3cfa7bdb8073adb8c3814b65567aa7447c"}, + {file = "azure_identity-1.25.3.tar.gz", hash = "sha256:ab23c0d63015f50b630ef6c6cf395e7262f439ce06e5d07a64e874c724f8d9e6"}, +] + +[package.dependencies] +azure-core = ">=1.31.0" +cryptography = ">=2.5" +msal = ">=1.35.1" +msal-extensions = ">=1.2.0" +typing-extensions = ">=4.0.0" + [[package]] name = "babel" version = "2.17.0" @@ -198,7 +237,7 @@ version = "2025.11.12" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" -groups = ["main", "docs"] +groups = ["main", "alerts", "docs"] files = [ {file = "certifi-2025.11.12-py3-none-any.whl", hash = "sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b"}, {file = "certifi-2025.11.12.tar.gz", hash = "sha256:d8ab5478f2ecd78af242878415affce761ca6bc54a22a27e026d7c25357c3316"}, @@ -210,7 +249,7 @@ version = "2.0.0" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["main", "alerts"] markers = "platform_python_implementation != \"PyPy\"" files = [ {file = "cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44"}, @@ -308,7 +347,7 @@ version = "3.4.4" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" -groups = ["main", "docs"] +groups = ["main", "alerts", "docs"] files = [ {file = "charset_normalizer-3.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d"}, {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8"}, @@ -459,7 +498,7 @@ version = "46.0.5" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = "!=3.9.0,!=3.9.1,>=3.8" -groups = ["main"] +groups = ["main", "alerts"] files = [ {file = "cryptography-46.0.5-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad"}, {file = "cryptography-46.0.5-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b"}, @@ -632,7 +671,7 @@ version = "3.11" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.8" -groups = ["main", "docs"] +groups = ["main", "alerts", "docs"] files = [ {file = "idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea"}, {file = "idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902"}, @@ -745,7 +784,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.03.6" +jsonschema-specifications = ">=2023.3.6" referencing = ">=0.28.4" rpds-py = ">=0.7.1" @@ -768,6 +807,27 @@ files = [ [package.dependencies] referencing = ">=0.31.0" +[[package]] +name = "keh-teams-alert" +version = "1.0.0" +description = "Python Module for MS Teams" +optional = false +python-versions = ">=3.12" +groups = ["alerts"] +files = [] +develop = false + +[package.dependencies] +azure-core = ">=1.38.2,<2.0.0" +azure-identity = ">=1.25.2,<2.0.0" +requests = ">=2.31,<3.0" + +[package.source] +type = "git" +url = "https://github.com/ONS-Innovation/keh-teams-alert" +reference = "v0.1.0" +resolved_reference = "45cb39f30f4f1ec5ca633f7c4eb66b7e3c5a7fa8" + [[package]] name = "librt" version = "0.7.4" @@ -1158,6 +1218,44 @@ griffe = ">=0.49" mkdocs-autorefs = ">=1.2" mkdocstrings = ">=0.26" +[[package]] +name = "msal" +version = "1.36.0" +description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." +optional = false +python-versions = ">=3.8" +groups = ["alerts"] +files = [ + {file = "msal-1.36.0-py3-none-any.whl", hash = "sha256:36ecac30e2ff4322d956029aabce3c82301c29f0acb1ad89b94edcabb0e58ec4"}, + {file = "msal-1.36.0.tar.gz", hash = "sha256:3f6a4af2b036b476a4215111c4297b4e6e236ed186cd804faefba23e4990978b"}, +] + +[package.dependencies] +cryptography = ">=2.5,<49" +PyJWT = {version = ">=1.0.0,<3", extras = ["crypto"]} +requests = ">=2.0.0,<3" + +[package.extras] +broker = ["pymsalruntime (>=0.14,<0.21) ; python_version >= \"3.8\" and platform_system == \"Windows\"", "pymsalruntime (>=0.17,<0.21) ; python_version >= \"3.8\" and platform_system == \"Darwin\"", "pymsalruntime (>=0.18,<0.21) ; python_version >= \"3.8\" and platform_system == \"Linux\""] + +[[package]] +name = "msal-extensions" +version = "1.3.1" +description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." +optional = false +python-versions = ">=3.9" +groups = ["alerts"] +files = [ + {file = "msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca"}, + {file = "msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4"}, +] + +[package.dependencies] +msal = ">=1.29,<2" + +[package.extras] +portalocker = ["portalocker (>=1.4,<4)"] + [[package]] name = "mypy" version = "1.19.1" @@ -1316,7 +1414,7 @@ version = "2.23" description = "C parser in Python" optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["main", "alerts"] markers = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\"" files = [ {file = "pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934"}, @@ -1344,12 +1442,15 @@ version = "2.12.0" description = "JSON Web Token implementation in Python" optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["main", "alerts"] files = [ {file = "pyjwt-2.12.0-py3-none-any.whl", hash = "sha256:9bb459d1bdd0387967d287f5656bf7ec2b9a26645d1961628cda1764e087fd6e"}, {file = "pyjwt-2.12.0.tar.gz", hash = "sha256:2f62390b667cd8257de560b850bb5a883102a388829274147f1d724453f8fb02"}, ] +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} + [package.extras] crypto = ["cryptography (>=3.4.0)"] dev = ["coverage[toml] (==7.10.7)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=8.4.2,<9.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] @@ -1753,7 +1854,7 @@ version = "2.32.5" description = "Python HTTP for Humans." optional = false python-versions = ">=3.9" -groups = ["main", "docs"] +groups = ["main", "alerts", "docs"] files = [ {file = "requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6"}, {file = "requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf"}, @@ -1935,10 +2036,10 @@ files = [ ] [package.dependencies] -botocore = ">=1.37.4,<2.0a.0" +botocore = ">=1.37.4,<2.0a0" [package.extras] -crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] +crt = ["botocore[crt] (>=1.37.4,<2.0a0)"] [[package]] name = "six" @@ -1982,7 +2083,7 @@ version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" -groups = ["main", "dev", "docs"] +groups = ["main", "alerts", "dev", "docs"] files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, @@ -1995,7 +2096,7 @@ version = "2.6.3" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" -groups = ["main", "docs"] +groups = ["main", "alerts", "docs"] files = [ {file = "urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4"}, {file = "urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed"}, @@ -2071,4 +2172,4 @@ watchdog = ["watchdog (>=2.3)"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "a6b229784850e8959b4e5b4253e0f2b8787ae0112e9018878d13e74a328a5ecb" +content-hash = "fd693a46ffc52dcb73df307dd9e081ae57277ab0a8f5d18bf428da42772d5fce" diff --git a/pyproject.toml b/pyproject.toml index 4c90778..d680cc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,12 @@ pylint = "^3.3.1" mypy = "^1.11.2" pytest = "^8.3.3" +[tool.poetry.group.alerts] +optional = true + +[tool.poetry.group.alerts.dependencies] +keh-teams-alert = { git = "https://github.com/ONS-Innovation/keh-teams-alert", tag = "v0.1.0" } + [tool.poetry.group.docs.dependencies] mkdocs = "1.6.1" mkdocs-material = "9.5.34" diff --git a/terraform/README.md b/terraform/README.md index 49316be..b838695 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -64,6 +64,26 @@ Run like normal. ### 5. Lambda Function The tech audit S3 bucket and the secrets manager secret are created by the terraform script. The aws cognito token url is set by the terraform script. Then run the terraform script for the lambda function and this data is set in the lambda function. +Set these Lambda-specific variables in the `terraform/lambda/env//.tfvars` file before applying: + +- `ecr_repository` - ECR repository containing the Lambda image +- `container_ver` - image tag to deploy +- `azure_secret_name` - Secrets Manager secret name containing the Teams alert Azure credentials +- `branch_name` - branch identifier used to gate alert delivery (`main` sends alerts) +- `aws_account_name` - environment label shown in the Teams alert message - derived from domain + +The secret referenced by `azure_secret_name` must contain JSON in this shape: + +```json +{ + "azure_tenant_id": "tenant-id", + "azure_client_id": "client-id", + "azure_client_secret": "client-secret", + "azure_scope": "https://graph.microsoft.com/.default", + "azure_webhook_url": "https://example.webhook.office.com/..." +} +``` + ### 6. API Gateway Run like normal. Note down the URLs in the outputs. diff --git a/terraform/lambda/env/dev/example_tfvars.txt b/terraform/lambda/env/dev/example_tfvars.txt index 9411cf3..3d498eb 100644 --- a/terraform/lambda/env/dev/example_tfvars.txt +++ b/terraform/lambda/env/dev/example_tfvars.txt @@ -5,3 +5,4 @@ domain = "sdp-dev" ecr_repository = "tech-audit-tool-api" # ECR repository name where the lambda image is stored service_subdomain = "tech-audit-tool-api" container_ver = "container version" +azure_secret_name = "sdp-keh-team-azure/secrets" diff --git a/terraform/lambda/env/prod/example_tfvars.txt b/terraform/lambda/env/prod/example_tfvars.txt index 96793b9..8fd45b7 100644 --- a/terraform/lambda/env/prod/example_tfvars.txt +++ b/terraform/lambda/env/prod/example_tfvars.txt @@ -4,4 +4,5 @@ aws_secret_access_key = "" domain = "sdp-prod" ecr_repository = "tech-audit-tool-api" # ECR repository name where the lambda image is stored service_subdomain = "tech-audit-tool-api" -container_ver = "container version" \ No newline at end of file +container_ver = "container version" +azure_secret_name = "sdp-keh-team-azure/secrets" \ No newline at end of file diff --git a/terraform/lambda/main.tf b/terraform/lambda/main.tf index f9b2fc8..eeed9a3 100644 --- a/terraform/lambda/main.tf +++ b/terraform/lambda/main.tf @@ -72,7 +72,10 @@ resource "aws_iam_role_policy" "lambda_additional_permissions" { Action = [ "secretsmanager:GetSecretValue" ] - Resource = data.terraform_remote_state.secrets.outputs.secret_arn + Resource = [ + data.terraform_remote_state.secrets.outputs.secret_arn, + "arn:aws:secretsmanager:${var.region}:${var.aws_account_id}:secret:${var.azure_secret_name}*" + ] }, { Effect = "Allow" @@ -169,6 +172,9 @@ resource "aws_lambda_function" "tech_audit_lambda" { AWS_COGNITO_TOKEN_URL = "https://${var.domain}-${var.service_subdomain}.auth.eu-west-2.amazoncognito.com/oauth2/token" IMAGE_DIGEST = data.aws_ecr_image.lambda_image.image_digest IMAGE_TAG = var.container_ver + AZURE_SECRET_NAME = var.azure_secret_name + BRANCH_NAME = var.branch_name + AWS_ACCOUNT_NAME = var.aws_account_name } } diff --git a/terraform/lambda/variables.tf b/terraform/lambda/variables.tf index 5bda45d..e7a2fc0 100644 --- a/terraform/lambda/variables.tf +++ b/terraform/lambda/variables.tf @@ -65,4 +65,19 @@ variable "log_retention_days" { description = "Log retention days" type = number default = 365 -} \ No newline at end of file +} + +variable "azure_secret_name" { + description = "Name of the Azure secret containing the service principal credentials" + type = string +} + +variable "branch_name" { + description = "Branch name for the deployment" + type = string +} + +variable "aws_account_name" { + description = "AWS account/environment name used by the Lambda runtime" + type = string +} diff --git a/testing/conftest.py b/testing/conftest.py index 07dc541..32cee4b 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -3,14 +3,58 @@ from flask_restx import Api import sys import os +import json +from unittest.mock import MagicMock, patch # Add the app directory to the path sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../aws_lambda_script")) ) -# Import your Flask app/namespace from aws_lambda_script/app -from app import ns # Assuming ns is the namespace for your API +# Local test defaults so app imports do not depend on real cloud credentials. +os.environ.setdefault("TECH_AUDIT_SECRET_MANAGER", "local/test/secret") +os.environ.setdefault("AZURE_SECRET_NAME", "local/test/azure") +os.environ.setdefault("AWS_DEFAULT_REGION", "eu-west-2") +os.environ.setdefault("MOCK_TOKEN", "local-test-token") +os.environ.setdefault("REDIRECT_URI", "http://localhost:3000/callback") +os.environ.setdefault( + "AWS_COGNITO_TOKEN_URL", + "https://example.auth.eu-west-2.amazoncognito.com/oauth2/token", +) + +mock_cognito_secret = { + "COGNITO_POOL_ID": "eu-west-2_local", + "COGNITO_CLIENT_ID": "local-client-id", + "COGNITO_CLIENT_SECRET": "local-client-secret", + "REDIRECT_URI": "http://localhost:3000/callback", +} +mock_azure_secret = { + "azure_tenant_id": "tenant-id", + "azure_client_id": "client-id", + "azure_client_secret": "client-secret", + "azure_scope": "https://graph.microsoft.com/.default", + "azure_webhook_url": "https://example.invalid/webhook", +} + +mock_s3_client = MagicMock(name="mock_s3_client") + +with patch("boto3.client", return_value=mock_s3_client), patch( + "boto3.session.Session.client" +) as mock_session_client: + mock_secrets_client = MagicMock(name="mock_secrets_client") + + def _mock_get_secret_value(SecretId): + if SecretId == os.environ["TECH_AUDIT_SECRET_MANAGER"]: + return {"SecretString": json.dumps(mock_cognito_secret)} + if SecretId == os.environ["AZURE_SECRET_NAME"]: + return {"SecretString": json.dumps(mock_azure_secret)} + return {"SecretString": "{}"} + + mock_secrets_client.get_secret_value.side_effect = _mock_get_secret_value + mock_session_client.return_value = mock_secrets_client + + # Import your Flask app/namespace from aws_lambda_script/app + from app import ns @pytest.fixture diff --git a/testing/test_api.py b/testing/test_api.py index 00f7b8e..01e1fa5 100644 --- a/testing/test_api.py +++ b/testing/test_api.py @@ -2,7 +2,9 @@ import json import os import datetime +import copy from unittest.mock import patch +from werkzeug.exceptions import Unauthorized # Mock verify_cognito_token for authentication mocked_user_email = os.getenv("MOCK_USER_EMAIL", "test@ons.gov.uk") @@ -13,11 +15,11 @@ def mock_verify_cognito_token(token): # Mock reading data function -def mock_read_data(): +def mock_read_data(_object_name=None): return { "projects": [ { - "details": {"name": "Test Project"}, + "details": [{"name": "Test Project"}], "user": [{"email": mocked_user_email}], "architecture": { "languages": {"main": "python", "others": ["javascript"]} @@ -48,17 +50,11 @@ def client(): # Helper to get mock token from environment def get_mock_token(): - mock_token = os.getenv("MOCK_TOKEN") - print(mock_token) - if not mock_token: - raise EnvironmentError( - "MOCK_TOKEN environment variable is not set. Please set it to run the tests." - ) - return mock_token + return os.getenv("MOCK_TOKEN", "local-test-token") # Test for "/user" route -@patch("app.utils.verify_cognito_token", side_effect=mock_verify_cognito_token) +@patch("app.resources.verify_cognito_token", side_effect=mock_verify_cognito_token) def test_get_user(mock_verify_token, client): mock_token = get_mock_token() response = client.get("/api/v1/user", headers={"Authorization": f"{mock_token}"}) @@ -70,8 +66,8 @@ def test_get_user(mock_verify_token, client): # Test for "/projects" route - GET -@patch("app.utils.read_data", side_effect=mock_read_data) -@patch("app.utils.verify_cognito_token", side_effect=mock_verify_cognito_token) +@patch("app.resources.read_data", side_effect=mock_read_data) +@patch("app.resources.verify_cognito_token", side_effect=mock_verify_cognito_token) def test_get_projects(mock_verify_token, mock_read, client): mock_token = get_mock_token() response = client.get( @@ -84,14 +80,44 @@ def test_get_projects(mock_verify_token, mock_read, client): assert "Test Project" in data[-1]["details"][0]["name"] -# Test for POSTing a new project with a timestamp in the name -@patch("app.utils.read_data", side_effect=mock_read_data) -@patch("app.utils.verify_cognito_token", side_effect=mock_verify_cognito_token) -@patch("app.utils.write_data") -def test_post_and_get_project_with_timestamp( - mock_write_data, mock_verify_token, mock_read, client +@patch("app.resources.get_user_information", side_effect=Unauthorized("Not authorized")) +def test_filter_projects_preserves_http_exception(mock_get_user_information, client): + mock_token = get_mock_token() + response = client.get( + "/api/v1/projects/filter", headers={"Authorization": f"{mock_token}"} + ) + + assert ( + response.status_code == 401 + ), f"Unexpected status code: {response.status_code}, {response.data}" + + +@patch("app.resources.send_teams_alert") +@patch("app.resources.verify_cognito_token", side_effect=Unauthorized("Not authorized")) +def test_get_user_preserves_http_exception_from_token_verification( + mock_verify_token, mock_send_teams_alert, client ): mock_token = get_mock_token() + response = client.get("/api/v1/user", headers={"Authorization": f"{mock_token}"}) + + assert ( + response.status_code == 401 + ), f"Unexpected status code: {response.status_code}, {response.data}" + mock_send_teams_alert.assert_not_called() + + +# Test for POSTing a new project with a timestamp in the name +@patch("app.resources.verify_cognito_token", side_effect=mock_verify_cognito_token) +def test_post_and_get_project_with_timestamp(mock_verify_token, client): + mock_token = get_mock_token() + stored_data = mock_read_data() + + def mock_stateful_read_data(_object_name=None): + return copy.deepcopy(stored_data) + + def mock_stateful_write_data(new_data, _object_name=None): + stored_data.clear() + stored_data.update(copy.deepcopy(new_data)) # Create a unique project name using time.time() project_name = f"Test Project {datetime.datetime.now(datetime.UTC).strftime('%Y-%m-%d-%H-%M-%S')}" @@ -111,7 +137,7 @@ def test_post_and_get_project_with_timestamp( ], "details": [ { - "name": f"Test Project {datetime.datetime.now(datetime.UTC).strftime('%Y-%m-%d-%H-%M-%S')}", + "name": project_name, "short_name": "Principal", "documentation_link": ["https://hollis.biz.ons.gov.uk"], "project_description": "Operative hybrid instruction set", @@ -120,7 +146,7 @@ def test_post_and_get_project_with_timestamp( ], } ], - "developed": ["In-house", []], + "developed": ["In-house"], "source_control": [ { "type": "GitHub", @@ -139,39 +165,42 @@ def test_post_and_get_project_with_timestamp( }, "stage": "Development", "supporting_tools": { - "code_editors": ["VSCode"], - "ui_tools": ["Figma"], - "diagram_tools": ["Draw.io"], - "project_tracking_tools": ["Jira"], - "documentation_tools": ["Confluence"], - "communication_tools": ["Teams"], - "collaboration_tools": ["Github"], + "code_editors": {"main": ["VSCode"], "others": []}, + "user_interface": {"main": ["Figma"], "others": []}, + "diagrams": {"main": ["Draw.io"], "others": []}, + "project_tracking": "Jira", + "documentation": {"main": ["Confluence"], "others": []}, + "communication": {"main": ["Teams"], "others": []}, + "collaboration": {"main": ["Github"], "others": []}, "incident_management": "ServiceNow", + "miscellaneous": [], }, } - # POST the new project - post_response = client.post( - "/api/v1/projects", - data=json.dumps(new_project), - headers={"Authorization": f"{mock_token}", "Content-Type": "application/json"}, - ) - print(post_response.data) + with patch("app.resources.read_data", side_effect=mock_stateful_read_data), patch( + "app.resources.write_data", side_effect=mock_stateful_write_data + ): + # POST the new project + post_response = client.post( + "/api/v1/projects", + data=json.dumps(new_project), + headers={"Authorization": f"{mock_token}", "Content-Type": "application/json"}, + ) - assert ( - post_response.status_code == 201 - ), f"Unexpected status code: {post_response.status_code}, {post_response.data}" + assert ( + post_response.status_code == 201 + ), f"Unexpected status code: {post_response.status_code}, {post_response.data}" - # GET the project by name - get_response = client.get( - f"/api/v1/projects/{project_name}", headers={"Authorization": f"{mock_token}"} - ) - assert ( - get_response.status_code == 200 - ), f"Unexpected status code: {get_response.status_code}, {get_response.data}" - data = json.loads(get_response.data) - assert data["details"][0]["name"] == project_name - assert data["user"][0]["email"] == mocked_user_email + # GET the project by name + get_response = client.get( + f"/api/v1/projects/{project_name}", headers={"Authorization": f"{mock_token}"} + ) + assert ( + get_response.status_code == 200 + ), f"Unexpected status code: {get_response.status_code}, {get_response.data}" + data = json.loads(get_response.data) + assert data["details"][0]["name"] == project_name + assert data["user"][0]["email"] == mocked_user_email # Test for "/verify" route