feat: Add logging to production

Author: tjtanjin

.github/workflows/deploy.yml

@@ -44,8 +44,6 @@ jobs:
           if [ "${{ inputs.target_tag }}" = "prod-latest" ]; then
             scp -o StrictHostKeyChecking=no docker-compose.prod.yml ${{ secrets.DEPLOYMENT_SSH_USER }}@${{ secrets.DEPLOYMENT_SERVER }}:/opt/rcb-deployments/${{ vars.PROJECT_NAME }}/docker-compose.override.yml
             scp -o StrictHostKeyChecking=no .env ${{ secrets.DEPLOYMENT_SSH_USER }}@${{ secrets.DEPLOYMENT_SERVER }}:/opt/rcb-deployments/${{ vars.PROJECT_NAME }}/.env.prod
-            echo "${{ secrets.HONEYCOMB_CONFIG }}" > honeycomb.toml
-            scp -o StrictHostKeyChecking=no honeycomb.toml ${{ secrets.DEPLOYMENT_SSH_USER }}@${{ secrets.DEPLOYMENT_SERVER }}:/opt/rcb-deployments/${{ vars.PROJECT_NAME }}/honeycomb.toml
           else
             scp -o StrictHostKeyChecking=no docker-compose.dev.yml ${{ secrets.DEPLOYMENT_SSH_USER }}@${{ secrets.DEPLOYMENT_SERVER }}:/opt/rcb-deployments/${{ vars.PROJECT_NAME }}/docker-compose.override.yml
             scp -o StrictHostKeyChecking=no .env ${{ secrets.DEPLOYMENT_SSH_USER }}@${{ secrets.DEPLOYMENT_SERVER }}:/opt/rcb-deployments/${{ vars.PROJECT_NAME }}/.env.development
@@ -64,8 +62,16 @@ jobs:
           IMAGE="ghcr.io/$OWNER/$REPO:${{ inputs.target_tag }}"
           echo "Deploying $IMAGE to VPS..."
           ssh -o StrictHostKeyChecking=no ${{ secrets.DEPLOYMENT_SSH_USER }}@${{ secrets.DEPLOYMENT_SERVER }} "\
+
+            # exports general variables
             export PROJECT_NAME='${{ vars.PROJECT_NAME }}' && \
             export GHCR_USER='${{ secrets.MACHINE_USER }}' && \
             export GHCR_PAT='${{ secrets.MACHINE_PAT }}' && \
             export APPLICATION_IMAGE='$IMAGE' && \
+
+            # applies only to production for logging
+            export HONEYCOMB_API_KEY='${{ secrets.HONEYCOMB_API_KEY }}' && \
+            export HONEYCOMB_DATASET='${{ secrets.HONEYCOMB_DATASET }}' && \
+
+            # runs deploy script
             /opt/rcb-deployments/${{ vars.PROJECT_NAME }}/deploy.sh $IMAGE"

docker-compose.override.yml

@@ -7,3 +7,18 @@ services:
     env_file: .env.local
   mysql:
     env_file: .env.local
+
+  # uncomment for testing locally, but you'll need your own honeycomb account and creds
+  # otel-collector:
+  #   image: otel/opentelemetry-collector:latest
+  #   container_name: rcb-discord-bot-otel-collector
+  #   environment:
+  #     HONEYCOMB_API_KEY: ${HONEYCOMB_API_KEY}
+  #     HONEYCOMB_DATASET: ${HONEYCOMB_DATASET}
+  #   volumes:
+  #     - ./otel-config.yaml:/etc/otel/config.yaml:ro
+  #   command: ["--config", "/etc/otel/config.yaml"]
+  #   ports:
+  #     - "4317:4317"
+  #   networks:
+  #     - app-network

docker-compose.prod.yml

@@ -8,12 +8,16 @@ services:
     env_file: .env.prod
 
   # add logging in production environment
-  honeycomb-agent:
-    image: honeycombio/honeycomb-agent:latest
-    container_name: honeycomb-agent
+  otel-collector:
+    image: otel/opentelemetry-collector:latest
+    container_name: rcb-discord-bot-otel-collector
+    environment:
+      HONEYCOMB_API_KEY: ${HONEYCOMB_API_KEY}
+      HONEYCOMB_DATASET: ${HONEYCOMB_DATASET}
     volumes:
-      - ./honeycomb.toml:/etc/honeycomb/honeycomb.toml:ro
+      - ./otel-config.yaml:/etc/otel/config.yaml:ro
+    command: ["--config", "/etc/otel/config.yaml"]
     ports:
-      - "514:514"
+      - "4317:4317"
     networks:
       - app-network

otel-config.yaml

@@ -0,0 +1,27 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+
+exporters:
+  otlp:
+    endpoint: api.honeycomb.io:443
+    headers:
+      x-honeycomb-team: ${HONEYCOMB_API_KEY}
+      x-honeycomb-dataset: ${HONEYCOMB_DATASET}
+    compression: gzip
+
+processors:
+  batch:
+
+service:
+  pipelines:
+    logs:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp]
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp]

requirements.txt

@@ -17,6 +17,14 @@ httpx==0.28.1
 hyperlink==21.0.0
 idna==3.10
 multidict==6.1.0
+opentelemetry-api==1.31.1
+opentelemetry-exporter-otlp==1.31.1
+opentelemetry-exporter-otlp-proto-common==1.31.1
+opentelemetry-exporter-otlp-proto-grpc==1.31.1
+opentelemetry-exporter-otlp-proto-http==1.31.1
+opentelemetry-proto==1.31.1
+opentelemetry-sdk==1.31.1
+opentelemetry-semantic-conventions==0.52b1
 packaging==24.2
 pydantic==2.10.6
 pydantic-settings==2.8.1

src/bot/utils/console_logger.py

@@ -1,14 +1,15 @@
 """
-Logger configuration module for console output.
+Logger configuration module for console output with OTEL integration.
 
-Initializes and configures a console logger using values from the
-logging config. Sets the log level, output format, and ensures no
-duplicate handlers are attached.
+This module sets up the standard console logger and attaches the OTEL handler
+(from otel_handler.py) so that logs are both printed to the console and sent
+to the OpenTelemetry Collector in a structured format.
 """
 
 import logging
 
 from bot.config.logging import logging_config
+from bot.utils.otel_handler import otel_handler
 
 # creates logger
 console_logger = logging.getLogger(logging_config.logger_prefix)
@@ -18,10 +19,10 @@
 console_logger.setLevel(log_level_num)
 
 # formats logger output
-handler = logging.StreamHandler()
 formatter = logging.Formatter(logging_config.logger_format)
-handler.setFormatter(formatter)
-if not console_logger.hasHandlers():
-    console_logger.addHandler(handler)
+
+# sets the formatter
+otel_handler.setFormatter(formatter)
+console_logger.addHandler(otel_handler)
 
 console_logger.info("✅ Logger is successfully configured.")

src/bot/utils/otel_handler.py

@@ -0,0 +1,42 @@
+"""
+OpenTelemetry Logging Handler Module.
+
+Sets up the OpenTelemetry LoggerProvider, exporter, and processor.
+Defines OTELHandler for Python's logging module that forwards logs
+to the OTEL Collector in structured format.
+
+The code snippet below is taken from the otel python example here:
+https://github.com/open-telemetry/opentelemetry-python/tree/main/docs/examples/logs
+"""
+
+import logging
+
+from opentelemetry import trace
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.exporter.otlp.proto.grpc._log_exporter import (
+    OTLPLogExporter,
+)
+from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+    BatchSpanProcessor,
+    ConsoleSpanExporter,
+)
+
+trace.set_tracer_provider(TracerProvider())
+trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
+
+logger_provider = LoggerProvider(
+    resource=Resource.create(
+        {
+            "service.name": "rcb-discord-bot",
+        }
+    ),
+)
+set_logger_provider(logger_provider)
+
+exporter = OTLPLogExporter(endpoint="otel-collector:4317", insecure=True)
+logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
+otel_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider)