Merge pull request #44 from ONS-Innovation/dataset_refresh

KEH-1020 | Dataset Refresh Button

Author: TotalDwarf03

docs/dashboard/index.md

@@ -42,6 +42,10 @@ Displays metrics on Dependabot alerts.
 - **Repository Overview:** Provides a high-level view of the repositories with Dependabot alerts, including the number of alerts and their severity.
 - **Repository Severity Proportion:** Displays the proportion of alerts by severity for each repository, allowing users to identify the most critical issues.
 
+### Data Refreshing
+
+- Users can refresh the backend data manually by clicking the "Refresh Data" button in the sidebar. This will trigger the Data Logger to collect the latest data from GitHub and update the S3 bucket. This functionality is considerate of GitHub's API rate limits, ensuring that there is enough rate limit remaining before attempting to refresh the data. If the rate limit is exceeded, the user will be informed and the refresh will not proceed.
+
 ## Data Collection Process
 
 The Dashboard relies on the data collected by the Data Logger, which is responsible for gathering information from GitHub repositories and storing it in a database. The Data Logger runs periodically to ensure that the Dashboard has up-to-date information.

src/app.py

@@ -1,6 +1,7 @@
 """The main application entry point for the GitHub Policy Dashboard."""
 
 import streamlit as st
+from src.refresh_data import refresh_data
 
 st.set_page_config(
     page_title="GitHub Policy Dashboard",
@@ -12,6 +13,24 @@
     st.Page("./repositories/repositories.py", title="Repositories", icon="📦"),
     st.Page("./secret_scanning/secret_scanning.py", title="Secret Scanning", icon="🔍"),
     st.Page("./dependabot/dependabot.py", title="Dependabot", icon="🤖"),
-])
+    ],
+    expanded=True,
+)
+
+if st.sidebar.button(
+    "Refresh Dataset",
+    key="refresh_dataset",
+    help="Click to refresh the dataset from GitHub. This may take a few minutes.",
+    icon="🔄",
+):
+    with st.spinner("Refreshing dataset... This may take a few minutes."):
+        status = refresh_data()
+
+        if status["status"] == "success":
+            # Clear cache to ensure fresh data is loaded
+            st.cache_data.clear()
+            st.success(status["message"])
+        else:
+            st.error(status["message"])
 
 pg.run()

src/refresh_data.py

@@ -0,0 +1,90 @@
+"""A Python script to refresh the dataset for the GitHub Policy Dashboard."""
+
+import botocore.config
+import boto3
+import botocore
+from requests import Response
+import datetime
+
+import utilities as utils
+
+def refresh_data() -> dict:
+    """A function to refresh the dataset for the GitHub Policy Dashboard.
+
+    Returns:
+        dict: A dictionary containing the status of the data refresh operation and a message.
+    """
+
+    # Check GitHub API rate limit
+    # If not enough rate limit, show error message and say when to try again
+    # If enough rate limit, proceed with data refresh
+
+    env = utils.get_environment_variables()
+
+    session = boto3.Session()
+    secret_manager = session.client("secretsmanager", region_name=env["secret_region"])
+
+    rest = utils.get_rest_interface(
+        secret_manager,
+        env["secret_name"],
+        env["org"],
+        env["client_id"]
+    )
+
+    response = rest.get("/rate_limit")
+
+    if type(response) is not Response:
+        return {"status": "error", "message": "Error fetching rate limit from GitHub API."}
+    
+    rate_limit = response.json()
+
+    remaining = {
+        "rest": {
+            "remaining": rate_limit["rate"]["remaining"],
+            "reset": rate_limit["rate"]["reset"]
+        },
+        "graphql": {
+            "remaining": rate_limit["resources"]["graphql"]["remaining"],
+            "reset": rate_limit["resources"]["graphql"]["reset"]
+        }
+    }
+
+    if remaining["rest"]["remaining"] < 5000:
+
+        reset_time = datetime.datetime.fromtimestamp(
+            remaining["rest"]["reset"]
+        ).strftime("%H:%M")
+
+        return {"status": "error", "message": f"GitHub API rate limit exceeded. Please try again after {reset_time}."}
+    
+    if remaining["graphql"]["remaining"] < 8000:
+
+        reset_time = datetime.datetime.fromtimestamp(
+            remaining["graphql"]["reset"]
+        ).strftime("%H:%M")
+
+        return {"status": "error", "message": f"GitHub GraphQL API rate limit exceeded. Please try again after {reset_time}."}
+    
+    # Proceed with data refresh
+
+    lambda_config = botocore.config.Config(
+        read_timeout=900,  # 15 minutes (Maximum timeout for Lambda)
+        retries={
+            "total_max_attempts": 1,
+        }
+    )
+
+    lambda_client = session.client("lambda", region_name=env["secret_region"], config=lambda_config)
+
+    response = lambda_client.invoke(
+        FunctionName="policy-dashboard-lambda",
+        InvocationType="RequestResponse",
+    )
+
+    if response["StatusCode"] != 200:
+        return {"status": "error", "message": "Error invoking Lambda function to refresh dataset."}
+    
+    if "FunctionError" in response:
+        return {"status": "error", "message": "Error in Lambda function execution."}
+
+    return {"status": "success", "message": "Dataset refreshed successfully!"}

src/repositories/repositories.py

@@ -347,5 +347,9 @@
 
         else:
             st.write("No point of contact available.")
+
+            if env["org"] == "ONS-Innovation":
+                st.write("Points of contact are not available for ONS Innovation repositories.")
+
 else:
     st.caption("Select a repository for more information.")

src/utilities.py

@@ -4,7 +4,7 @@
 import os
 import boto3
 import github_api_toolkit
-from datetime import timedelta
+from datetime import timedelta, timezone
 from requests import Response
 from typing import Tuple
 
@@ -49,6 +49,7 @@ def get_last_modified(s3: boto3.client, bucket: str, filename: str) -> str | Non
     response = s3.head_object(Bucket=bucket, Key=filename)
 
     last_modified = response['LastModified']
+    last_modified = last_modified.replace(tzinfo=timezone.utc).astimezone(tz=None)
 
     if not last_modified:
         return None