add mattermost notification

elasticroentgen · elasticroentgen · commit a56bf90c7647 · 2025-12-02T17:50:37.000+01:00
diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml
@@ -21,11 +21,40 @@ spec:
         image: service-monitor:latest  # Update this with your registry/image:tag
         imagePullPolicy: Always
         env:
-        - name: GITHUB_TOKEN
+        - name: SEMAPHORE_URL
           valueFrom:
             secretKeyRef:
-              name: github-token
-              key: GITHUB_TOKEN
+              name: semaphore-credentials
+              key: SEMAPHORE_URL
+        - name: SEMAPHORE_AUTH_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: semaphore-credentials
+              key: SEMAPHORE_AUTH_TOKEN
+        - name: TENANT
+          valueFrom:
+            secretKeyRef:
+              name: semaphore-credentials
+              key: TENANT
+              optional: true
+        - name: PROJECT
+          valueFrom:
+            secretKeyRef:
+              name: semaphore-credentials
+              key: PROJECT
+              optional: true
+        - name: MATTERMOST_WEBHOOK_URL
+          valueFrom:
+            secretKeyRef:
+              name: semaphore-credentials
+              key: MATTERMOST_WEBHOOK_URL
+              optional: true
+        - name: K8S_CLUSTER_NAME
+          valueFrom:
+            secretKeyRef:
+              name: semaphore-credentials
+              key: K8S_CLUSTER_NAME
+              optional: true
         resources:
           requests:
             cpu: "100m"
diff --git a/k8s/secret.yaml b/k8s/secret.yaml
@@ -1,10 +1,16 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: github-token
+  name: semaphore-credentials
   namespace: monitoring
 type: Opaque
 data:
-  # Replace this with your base64 encoded GitHub token
-  # Example: echo -n "your-token" | base64
-  GITHUB_TOKEN: "YOUR_BASE64_ENCODED_TOKEN"
+  # Replace these with your base64 encoded values
+  # Example: echo -n "your-value" | base64
+  SEMAPHORE_URL: "YOUR_BASE64_ENCODED_URL"
+  SEMAPHORE_AUTH_TOKEN: "YOUR_BASE64_ENCODED_TOKEN"
+  TENANT: "YOUR_BASE64_ENCODED_TENANT"
+  PROJECT: "YOUR_BASE64_ENCODED_PROJECT"
+  # Optional: Mattermost notifications
+  MATTERMOST_WEBHOOK_URL: "YOUR_BASE64_ENCODED_MATTERMOST_WEBHOOK"
+  K8S_CLUSTER_NAME: "YOUR_BASE64_ENCODED_CLUSTER_NAME"
diff --git a/requirements.txt b/requirements.txt
@@ -1,2 +1,2 @@
 kubernetes==29.0.0
-PyGithub==2.2.0
+requests==2.31.0
diff --git a/service_monitor_v2.py b/service_monitor_v2.py
@@ -8,9 +8,9 @@
 import sys
 from typing import Dict, Optional
 from pathlib import Path
+import requests
 from kubernetes import client, config, watch
 from kubernetes.client.exceptions import ApiException
-from github import Github
 from datetime import datetime, timedelta
 
 # Configure logging
@@ -20,13 +20,16 @@
 )
 logger = logging.getLogger(__name__)
 
-# GitHub configuration
-GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN')
-GITHUB_REPO = os.environ.get('GITHUB_REPO')
-WORKFLOW_FILE = os.environ.get('WORKFLOW_FILE')
+# Semaphore configuration
+SEMAPHORE_URL = os.environ.get('SEMAPHORE_URL')
+SEMAPHORE_AUTH_TOKEN = os.environ.get('SEMAPHORE_AUTH_TOKEN')
 TENANT = os.environ.get('TENANT')
 PROJECT = os.environ.get('PROJECT')
 
+# Mattermost configuration
+MATTERMOST_WEBHOOK_URL = os.environ.get('MATTERMOST_WEBHOOK_URL')
+K8S_CLUSTER_NAME = os.environ.get('K8S_CLUSTER_NAME', 'unknown')
+
 # Store the last trigger time globally
 last_trigger_global = 0
 DEBOUNCE_INTERVAL = 180  # 3 minutes in seconds
@@ -108,93 +111,118 @@ def get_initial_resource_version(v1) -> Optional[str]:
         logger.error(f"Failed to get initial resourceVersion: {e}")
         return None
 
-def trigger_github_workflow(gh_token: str, event_type: str, service_key: str) -> bool:
+def send_mattermost_notification(event_type: str, namespace: str, service_name: str) -> bool:
     """
-    Trigger GitHub Actions workflow with debouncing and detailed error logging
-    
+    Send a notification to Mattermost about the triggered Semaphore webhook.
+
+    Args:
+        event_type: Type of the Kubernetes event
+        namespace: Kubernetes namespace of the service
+        service_name: Name of the service
+
+    Returns:
+        bool: True if notification was sent, False otherwise
+    """
+    if not MATTERMOST_WEBHOOK_URL:
+        logger.debug("MATTERMOST_WEBHOOK_URL not set, skipping notification")
+        return False
+
+    message = (
+        f":rocket: **Loadbalancer update triggered**\n"
+        f"**Cluster** `{K8S_CLUSTER_NAME}` // **Namespace** `{namespace}`\n"
+        f"**Service** `{service_name}` // **Event** `{event_type}`"
+    )
+
+    payload = {"text": message}
+
+    try:
+        response = requests.post(
+            MATTERMOST_WEBHOOK_URL,
+            json=payload,
+            timeout=10
+        )
+
+        if response.status_code >= 200 and response.status_code < 300:
+            logger.info(f"Mattermost notification sent for {namespace}/{service_name}")
+            return True
+        else:
+            logger.warning(f"Mattermost notification failed with status {response.status_code}: {response.text}")
+            return False
+
+    except requests.exceptions.RequestException as e:
+        logger.warning(f"Failed to send Mattermost notification: {str(e)}")
+        return False
+
+def trigger_semaphore_webhook(event_type: str, service_key: str) -> bool:
+    """
+    Trigger Semaphore webhook with debouncing and detailed error logging
+
     Args:
-        gh_token: GitHub authentication token
         event_type: Type of the Kubernetes event
         service_key: Unique identifier for the service (namespace/name)
-        
+
     Returns:
-        bool: True if workflow was triggered, False otherwise
+        bool: True if webhook was triggered, False otherwise
     """
-    if not GITHUB_REPO:
-        logger.error("GITHUB_REPO environment variable not set")
+    if not SEMAPHORE_URL:
+        logger.error("SEMAPHORE_URL environment variable not set")
         return False
-    
-    if not WORKFLOW_FILE:
-        logger.error("WORKFLOW_FILE environment variable not set")
+
+    if not SEMAPHORE_AUTH_TOKEN:
+        logger.error("SEMAPHORE_AUTH_TOKEN environment variable not set")
         return False
 
     current_time = time.time()
     last_trigger_time = get_last_trigger()
-    
+
     # Check if enough time has passed since the last trigger
     time_since_last = current_time - last_trigger_time
     if time_since_last < DEBOUNCE_INTERVAL:
-        logger.info(f"Skipping workflow trigger for {service_key} due to debouncing (last trigger was {int(time_since_last)} seconds ago)")
+        logger.info(f"Skipping webhook trigger for {service_key} due to debouncing (last trigger was {int(time_since_last)} seconds ago)")
         return False
 
+    # Prepare payload
+    payload = {
+        "tenant": TENANT or "",
+        "project": PROJECT or "",
+        "branch": "main",
+        "type": "ansible_workload"
+    }
+
+    headers = {
+        "SEMAPHORE-AUTH": SEMAPHORE_AUTH_TOKEN,
+        "Content-Type": "application/json"
+    }
+
     try:
-        logger.info(f"Connecting to GitHub repo: {GITHUB_REPO}")
-        g = Github(gh_token)
-        repo = g.get_repo(GITHUB_REPO)
-        
-        # Get all workflows and log them for debugging
-        workflows = list(repo.get_workflows())
-        logger.info(f"Found {len(workflows)} workflows in repository")
-        for wf in workflows:
-            logger.info(f"Available workflow: {wf.path} (ID: {wf.id})")
-        
-        # Get the workflow by filename
-        workflow = None
-        for wf in workflows:
-            if wf.path.endswith(WORKFLOW_FILE):
-                workflow = wf
-                logger.info(f"Found matching workflow: {wf.path} (ID: {wf.id})")
-                break
-        
-        if workflow is None:
-            logger.error(f"Workflow {WORKFLOW_FILE} not found in repository {GITHUB_REPO}")
-            return False
+        logger.info(f"Triggering Semaphore webhook at {SEMAPHORE_URL} with payload: {payload}")
 
-        # Prepare inputs
-        inputs = {
-            "team": TENANT or "",
-            "project": PROJECT or ""
-        }
-        
-        logger.info(f"Triggering workflow dispatch for {workflow.path} with inputs: {inputs}")
-        
-        # Create workflow dispatch event with inputs
-        try:
-            workflow.create_dispatch(
-                ref="main",  # You might want to make this configurable
-                inputs=inputs
-            )
+        response = requests.post(
+            SEMAPHORE_URL,
+            json=payload,
+            headers=headers,
+            timeout=30
+        )
+
+        if response.status_code >= 200 and response.status_code < 300:
             # Update the last trigger time only on successful dispatch
             set_last_trigger(current_time)
-            logger.info(f"Successfully triggered workflow {workflow.path} for event: {event_type} (service: {service_key})")
+            logger.info(f"Successfully triggered Semaphore webhook for event: {event_type} (service: {service_key}), response: {response.status_code}")
+
+            # Send Mattermost notification
+            namespace, service_name = service_key.split('/', 1)
+            send_mattermost_notification(event_type, namespace, service_name)
+
             return True
-            
-        except Exception as dispatch_error:
-            logger.error(f"Failed to dispatch workflow: {str(dispatch_error)}")
-            # Try to get more details about the error
-            if hasattr(dispatch_error, 'response'):
-                response = getattr(dispatch_error, 'response')
-                if response:
-                    logger.error(f"GitHub API Response: {response.status_code} - {response.text}")
+        else:
+            logger.error(f"Semaphore webhook failed with status {response.status_code}: {response.text}")
             return False
-    
-    except Exception as e:
-        logger.error(f"Failed to trigger workflow: {str(e)}")
-        # Try to get more details about the error
-        if hasattr(e, 'response'):
-            response = getattr(e, 'response')
-            if response:
-                logger.error(f"GitHub API Response: {response.status_code} - {response.text}")
+
+    except requests.exceptions.Timeout:
+        logger.error("Semaphore webhook request timed out")
+        return False
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Failed to trigger Semaphore webhook: {str(e)}")
         return False
 
 def initialize_kubernetes_client():
@@ -263,13 +291,10 @@ def watch_services(timeout_seconds: Optional[int] = None):
             if service.spec.type == 'LoadBalancer':
                 logger.info(f"LoadBalancer service event: {event_type} - {service.metadata.namespace}/{service.metadata.name}")
 
-                # Trigger workflow for relevant events
+                # Trigger webhook for relevant events
                 if event_type in ['ADDED', 'MODIFIED', 'DELETED']:
-                    if GITHUB_TOKEN:
-                        service_key = f"{service.metadata.namespace}/{service.metadata.name}"
-                        trigger_github_workflow(GITHUB_TOKEN, event_type, service_key)
-                    else:
-                        logger.error("GITHUB_TOKEN environment variable not set")
+                    service_key = f"{service.metadata.namespace}/{service.metadata.name}"
+                    trigger_semaphore_webhook(event_type, service_key)
 
             # Check if we've been watching for too long and should restart
             if time.time() - start_time > MAX_WATCH_TIME:
@@ -321,8 +346,11 @@ def main():
     """
     Main function to start the service monitor with resilience
     """
-    if not GITHUB_TOKEN:
-        logger.error("GITHUB_TOKEN environment variable must be set")
+    if not SEMAPHORE_URL:
+        logger.error("SEMAPHORE_URL environment variable must be set")
+        exit(1)
+    if not SEMAPHORE_AUTH_TOKEN:
+        logger.error("SEMAPHORE_AUTH_TOKEN environment variable must be set")
         exit(1)
 
     # Initialize Kubernetes client and get initial resourceVersion

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`kubernetes==29.0.0`
`2`		`-PyGithub==2.2.0`
	`2`	`+requests==2.31.0`