Use alloy for collecting logs

Alloy is a recommended replacement for promtail and makes collecting docker logs convenient bypassing file exporter limitations (e.g. missing container_name label).
Logs now do not go via OTEL exporter by to loki directly.

Author: varmar05

deployment/common/telemetry/.gitignore

@@ -1,2 +1,3 @@
 loki-data
-dashboards
+dashboards
+alloy-data

deployment/common/telemetry/alloy-config.alloy

@@ -0,0 +1,35 @@
+discovery.docker "containers" {
+  host = "unix:///var/run/docker.sock"
+}
+
+discovery.relabel "containers" {
+  targets = discovery.docker.containers.targets
+
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container_name"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_log_stream"]
+    target_label  = "stream"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_image_name"]
+    target_label  = "image"
+  }
+}
+
+loki.source.docker "containers" {
+  host       = "unix:///var/run/docker.sock"
+  targets    = discovery.relabel.containers.output
+  forward_to = [loki.write.default.receiver]
+}
+
+loki.write "default" {
+  endpoint {
+    url = "http://merginmaps-loki:3100/loki/api/v1/push"
+  }
+}

deployment/common/telemetry/docker-compose.otel.yml

@@ -10,22 +10,30 @@ services:
     container_name: merginmaps-otel-collector
     volumes:
       - ./otel-config.yaml:/etc/otelcol-contrib/config.yaml
-      - /var/lib/docker/containers:/var/lib/docker/containers:ro
-      - /var/run/docker.sock:/var/run/docker.sock:ro
     networks:
       - merginmaps
-    user: "0:0"
-    privileged: true
     ports:
       - "8125:8125/udp" # StatsD (Metrics)
       - "4317:4317" # OTLP (Traces)
       - "8889:8889" # Prometheus Scrape Port
-      # - "8888:8889"
       - "55679:55679"
     depends_on:
       tempo:
         condition: service_started
 
+  alloy:
+    image: grafana/alloy:v1.8.3
+    container_name: merginmaps-alloy
+    volumes:
+      - ./alloy-config.alloy:/etc/alloy/config.alloy
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./alloy-data:/var/lib/alloy/data
+    networks:
+      - merginmaps
+    depends_on:
+      - loki
+    command: run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy
+
   tempo:
     image: grafana/tempo:2.8.3
     container_name: merginmaps-tempo

deployment/common/telemetry/loki-config.yaml

@@ -27,11 +27,13 @@ schema_config:
         period: 24h
 
 limits_config:
+  reject_old_samples: false
+  reject_old_samples_max_age: 336h
   allow_structured_metadata: true
-  reject_old_samples: false 
   otlp_config:
     resource_attributes:
       attributes_config:
         - action: index_label
           attributes:
-            - container_id
+            - container_name
+            - image

deployment/common/telemetry/otel-config.yaml

@@ -16,39 +16,8 @@ receivers:
     endpoint: "merginmaps-redis:6379"
     collection_interval: 10s
     #password: "${REDIS_PASSWORD}"
-  # This replaces Promtail
-  filelog:
-    include:
-      - /var/lib/docker/containers/*/*.log
-    start_at: end
-    include_file_path: true
-    operators:
-      - type: json_parser
-      - type: move
-        from: attributes.log
-        to: body
-      - type: regex_parser
-        parse_from: attributes["log.file.path"]
-        regex: '/var/lib/docker/containers/(?P<container_id>[a-f0-9]{12})'
-        on_error: send
-
 processors:
-  resourcedetection/docker:
-    detectors: [env, docker]
-    timeout: 2s
-    override: true
-  filter/exclude_collector:
-    logs:
-      log_record:
-        - 'IsMatch(body, ".*exporterhelper.*")'
-        - 'IsMatch(body, ".*Debug.*")'
-        - 'IsMatch(body, ".*kind.*exporter.*")'
   batch:
-  transform/logs:
-    log_statements:
-      - context: log
-        statements:
-          - set(resource.attributes["container_id"], attributes["container_id"]) where attributes["container_id"] != nil
   transform:
     metric_statements:
       - context: metric
@@ -67,13 +36,6 @@ processors:
     #     statements:
     #       - set(attributes["service_name"], attributes["docker_id"])
     #       - set(resource.attributes["service.name"], attributes["docker_id"])
-  attributes:
-    actions:
-      - key: service.name
-        action: insert
-        value: "merginmaps"
-
-
 exporters:
   prometheus:
     endpoint: "0.0.0.0:8889" # The Collector will "host" metrics here
@@ -84,21 +46,6 @@ exporters:
     endpoint: "merginmaps-tempo:4317"
     tls:
       insecure: true
-  loki:
-    endpoint: "http://merginmaps-loki:3100/loki/api/v1/push"
-    # This ensures the trace and span IDs are included in the log metadata
-    default_labels_enabled:
-      exporter: true
-      job: true
-      instance: true
-      level: true
-  debug:
-    verbosity: detailed
-  otlphttp/loki:
-    endpoint: "http://loki:3100/otlp"
-    tls:
-      insecure: true
-      
 service:
   extensions: [health_check]
   telemetry:
@@ -115,7 +62,3 @@ service:
       receivers: [otlp]
       processors: [batch]
       exporters: [otlp]
-    logs:
-      receivers: [filelog]
-      processors: [filter/exclude_collector, resourcedetection/docker, attributes, transform/logs, batch]
-      exporters: [otlphttp/loki]