Merge branch 'master' into custom_recource_monitoring

Author: Avi-Robusta

docs/configuration/holmesgpt/toolsets/datadog_logs.rst

@@ -69,6 +69,19 @@ Configuration
         kubernetes/logs:
           enabled: false # HolmesGPT's default logging mechanism MUST be disabled
 
+
+DataDog Kubernetes cluster label
+***********************************
+
+In multi cluster environments, HolmesGPT will route alerts investigations to the Kubernetes cluster the alert is firing on.
+
+Add the ``cluster`` label to your DataDog monitoring agent:
+
+    .. image:: /images/datadog-cluster-label.png
+        :width: 1000
+        :align: center
+
+
 Getting API and Application Keys
 ********************************
 

docs/configuration/sinks/mail.rst

@@ -2,12 +2,20 @@ Mail
 #################
 
 Robusta can report issues and events in your Kubernetes cluster by sending
-emails.
+emails using either SMTP servers or Amazon Simple Email Service (SES).
 
 Connecting the mail sink
 ------------------------------------------------
 
-To set up the mail sink, you need access to an SMTP server. You should also
+The mail sink supports two modes:
+
+1. **SMTP Mode** (default): Uses any SMTP server via the `Apprise library <https://github.com/caronc/apprise>`_
+2. **Amazon SES Mode**: Uses AWS Simple Email Service for improved reliability and deliverability
+
+SMTP Configuration
+------------------------------------------------
+
+To set up SMTP mode, you need access to an SMTP server. You should also
 set the sender and receiver(s) addresses.
 
 Robusta uses `Apprise library <https://github.com/caronc/apprise>`_ under the hood for running mail
@@ -18,10 +26,7 @@ the convenient and sophisticated syntax provided by Apprise. For more details
 .. image:: /images/mail_sink1.png
   :width: 640
 
-Configuring the mail sink
-------------------------------------------------
-
-.. admonition:: Add this to your generated_values.yaml
+.. admonition:: SMTP Configuration - Add this to your generated_values.yaml
 
     .. code-block:: yaml
 
@@ -31,12 +36,160 @@ Configuring the mail sink
             mailto: "mailtos://user:password@server&from=a@x&to=b@y,c@z"
             with_header: false  # optional
 
+Amazon SES Configuration
+------------------------------------------------
+
+Amazon SES provides better deliverability, detailed analytics, and is often more cost-effective than traditional SMTP servers.
+
+Prerequisites
+^^^^^^^^^^^^^
+
+1. **AWS Account**: Set up an AWS account with SES enabled
+2. **SES Setup**: Verify sender email addresses in SES console  
+3. **IAM Permissions**: Ensure your AWS credentials have ``ses:SendEmail`` and ``ses:SendRawEmail`` permissions
+4. **Production Access**: For production use, request SES production access (initially in sandbox mode)
+
+.. admonition:: SES Configuration - Add this to your generated_values.yaml
+
+    .. code-block:: yaml
+
+        sinksConfig:
+        - mail_sink:
+            name: ses_mail_sink
+            mailto: "mailtos://alerts@company.com"  # Recipient addresses
+            use_ses: true
+            aws_region: "us-east-1"
+            from_email: "robusta-alerts@company.com"
+            with_header: true  # optional
+            # Optional: explicit AWS credentials (prefer IAM roles)
+            # aws_access_key_id: "${AWS_ACCESS_KEY_ID}"
+            # aws_secret_access_key: "${AWS_SECRET_ACCESS_KEY}"
+            # configuration_set: "robusta-emails"  # optional
+
+SES Configuration Parameters
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. list-table::
+   :header-rows: 1
+   :widths: 25 15 60
+
+   * - Parameter
+     - Required
+     - Description
+   * - ``use_ses``
+     - Yes
+     - Set to ``true`` to enable SES mode
+   * - ``aws_region``
+     - Yes
+     - AWS region where SES is configured (e.g., ``us-east-1``)
+   * - ``from_email``
+     - Yes
+     - Verified sender email address in SES
+   * - ``mailto``
+     - Yes
+     - Recipient email addresses (same format as SMTP mode)
+   * - ``aws_access_key_id``
+     - No
+     - AWS access key (prefer IAM roles over explicit credentials)
+   * - ``aws_secret_access_key``
+     - No
+     - AWS secret key (prefer IAM roles over explicit credentials)
+   * - ``configuration_set``
+     - No
+     - SES configuration set for tracking and analytics
+
+Authentication Options
+^^^^^^^^^^^^^^^^^^^^^^
+
+**Option 1: IAM Roles (Recommended)**
+
+For clusters running in AWS (EKS), use IAM roles for service accounts:
+
+.. code-block:: yaml
+
+    sinksConfig:
+    - mail_sink:
+        name: ses_mail_sink
+        mailto: "mailtos://alerts@company.com"
+        use_ses: true
+        aws_region: "us-east-1"
+        from_email: "robusta@company.com"
+
+**Option 2: Environment Variables**
+
+Set AWS credentials as environment variables:
+
+.. code-block:: bash
+
+    export AWS_ACCESS_KEY_ID="your-access-key"
+    export AWS_SECRET_ACCESS_KEY="your-secret-key"
+
+**Option 3: Explicit Configuration**
+
+Include credentials directly in configuration (not recommended for production):
+
+.. code-block:: yaml
+
+    sinksConfig:
+    - mail_sink:
+        name: ses_mail_sink
+        mailto: "mailtos://alerts@company.com"
+        use_ses: true
+        aws_region: "us-east-1"
+        from_email: "robusta@company.com"
+        aws_access_key_id: "${AWS_ACCESS_KEY_ID}"
+        aws_secret_access_key: "${AWS_SECRET_ACCESS_KEY}"
+
+Multiple Recipients
+^^^^^^^^^^^^^^^^^^^
+
+SES mode supports multiple recipients using the same mailto format:
+
+.. code-block:: yaml
+
+    mailto: "mailtos://primary@company.com?to=secondary@company.com,third@company.com"
+
+Common Parameters
+------------------------------------------------
+
+The following parameters apply to both SMTP and SES modes:
+
+.. list-table::
+   :header-rows: 1
+   :widths: 25 15 60
+
+   * - Parameter
+     - Default
+     - Description
+   * - ``with_header``
+     - ``true``
+     - Include finding header, investigate button, and notification source
+   * - ``name``
+     - Required
+     - Unique name for this sink configuration
+
 The default value of the optional `with_header` parameter is `true`. If set to `false`, mails
 sent by this sink will *not* include header information, such as the finding header, investigate
 button and the source of the notification.
 
+Troubleshooting
+------------------------------------------------
+
+**SES Issues**
+
+- **Authentication errors**: Verify AWS credentials and IAM permissions
+- **Message rejected**: Check that sender email is verified in SES console
+- **Rate limiting**: SES has sending quotas; check your SES console for limits
+- **Sandbox mode**: In SES sandbox, you can only send to verified email addresses
+
+**SMTP Issues**
+
+- **Connection errors**: Verify SMTP server details and network connectivity
+- **Authentication failures**: Check username/password in mailto URL
+- **TLS/SSL issues**: Ensure correct protocol (``mailto://`` vs ``mailtos://``)
+
 .. note::
 
-    We highly recommend using the quotes around "mailto" to ensure special characters are handled correctly.
+    We highly recommend using quotes around "mailto" to ensure special characters are handled correctly.
 
-Then do a :ref:`Helm Upgrade <Simple Upgrade>`.
+Then do a :ref:`Helm Upgrade <Simple Upgrade>`.

docs/images/datadog-cluster-label.png

@@ -26,3 +26,47 @@ Node
 .. robusta-action:: playbooks.robusta_playbooks.node_actions.uncordon on_node_create
 
 .. robusta-action:: playbooks.robusta_playbooks.node_actions.drain on_node_create
+
+Kubectl
+*****************
+
+.. robusta-action:: playbooks.robusta_playbooks.kubectl_enrichments.kubectl_command
+
+    Use `kubectl_command` to run kubectl with dynamic placeholders:
+    - `$namespace`: resource namespace
+    - `$kind`: resource kind (e.g., Pod, Deployment)
+    - `$name`: resource name
+
+    Example: **Scale Down Deployment on Crash Loop**
+
+    .. code-block:: yaml
+
+        customPlaybooks:
+        - name: CrashLoopScaleDown
+          triggers:
+          - on_pod_crash_loop:
+              restart_count: 3
+          actions:
+            - kubectl_command:
+                description: "Scale Down Deployment"
+                command: kubectl scale --replicas=0 deployment/payment-processing-worker -n $namespace
+
+    If the pod is in the `production` namespace, the command will be:
+
+    .. code-block:: bash
+
+        kubectl scale --replicas=0 deployment/payment-processing-worker -n production
+
+    Example: **Delete Crashing Resource**
+
+    This deletes the crashing resource by kind, name, and namespace:
+
+    .. code-block:: bash
+
+        kubectl delete $kind $name -n $namespace
+
+    For example, deleting a crashing pod named `api-worker-1` in the `staging` namespace:
+
+    .. code-block:: bash
+
+        kubectl delete Pod api-worker-1 -n staging

docs/playbook-reference/actions/remediation.rst

@@ -7,11 +7,20 @@
 class MailSink(SinkBase):
     def __init__(self, sink_config: MailSinkConfigWrapper, registry):
         super().__init__(sink_config.mail_sink, registry)
+        params = sink_config.mail_sink
+
         self.sender = MailSender(
-            sink_config.mail_sink.mailto,
+            params.mailto,
             self.signing_key,
             self.account_id,
             self.cluster_name,
+            use_ses=params.use_ses,
+            aws_region=params.aws_region,
+            from_email=params.from_email,
+            aws_access_key_id=params.aws_access_key_id,
+            aws_secret_access_key=params.aws_secret_access_key,
+            configuration_set=params.configuration_set,
+            skip_ses_init=params.skip_ses_init,
         )
 
     def write_finding(self, finding: Finding, platform_enabled: bool):

src/robusta/core/sinks/mail/mail_sink.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import validator
+from pydantic import validator, root_validator
 
 from robusta.core.sinks.sink_base_params import SinkBaseParams
 from robusta.core.sinks.sink_config import SinkConfigBase
@@ -10,6 +10,15 @@ class MailSinkParams(SinkBaseParams):
     mailto: str
     with_header: Optional[bool] = True
 
+    # SES Configuration
+    use_ses: Optional[bool] = False
+    aws_region: Optional[str] = None
+    from_email: Optional[str] = None
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[str] = None
+    configuration_set: Optional[str] = None
+    skip_ses_init: Optional[bool] = False  # For testing purposes
+
     @classmethod
     def _get_sink_type(cls):
         return "mail"
@@ -22,6 +31,17 @@ def validate_mailto(cls, mailto):
             raise AttributeError(f"{mailto} is not a mailto(s) address")
         return mailto
 
+    @root_validator
+    def validate_ses_configuration(cls, values):
+        use_ses = values.get("use_ses", False)
+        if use_ses:
+            # Check for required fields when SES is enabled
+            if not values.get("from_email"):
+                raise ValueError("from_email is required when use_ses=True")
+            if not values.get("aws_region"):
+                raise ValueError("aws_region is required when use_ses=True")
+        return values
+
 
 class MailSinkConfigWrapper(SinkConfigBase):
     mail_sink: MailSinkParams