Merge pull request #886 from myadla/myadla_auditlogging

Deploy a second Loki instance in telemetry logging jobs and configure log forwarding

Author: openshift-merge-bot[bot]

ci/deploy-logging-dependencies/files/audit_loki_secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: logging-loki-audit-s3
+stringData:
+  access_key_id: minio
+  access_key_secret: minio123
+  bucketnames: loki-audit
+  endpoint: http://minio.minio-dev.svc.cluster.local:9000

ci/deploy-logging-dependencies/files/audit_lokistack.yaml

@@ -0,0 +1,16 @@
+apiVersion: loki.grafana.com/v1
+kind: LokiStack
+metadata:
+  name: logging-loki-audit
+spec:
+  size: 1x.demo
+  storage:
+    schemas:
+    - version: v13
+      effectiveDate: "2025-06-01"
+    secret:
+      name: logging-loki-audit-s3
+      type: s3
+  storageClassName: crc-csi-hostpath-provisioner
+  tenants:
+    mode: openshift-logging

ci/deploy-logging-dependencies/files/cluster_log_forwarder.yaml

@@ -1,39 +1,76 @@
 apiVersion: observability.openshift.io/v1
 kind: ClusterLogForwarder
 metadata:
-  name: collector
+  name: collector-with-cadf-to-loki-audit
   namespace: openshift-logging
 spec:
   serviceAccount:
     name: collector
   inputs:
-    - name: syslog-receiver
-      type: receiver
-      receiver:
-        type: syslog
-        port: 10514
+  - name: dataplane
+    receiver:
+      port: 10514
+      type: syslog
+    type: receiver
+  managementState: Managed
   outputs:
-    - name: default-lokistack
-      type: lokiStack
-      lokiStack:
-        authentication:
-          token:
-            from: serviceAccount
-        labelKeys:
-          infrastructure:
-            labelKeys:
-              - host
-              - appname
-        target:
-          name: logging-loki
-          namespace: openshift-logging
-      tls:
-        ca:
-          key: service-ca.crt
-          configMapName: openshift-service-ca.crt
+  - name: default-lokistack
+    type: lokiStack
+    lokiStack:
+      authentication:
+        token:
+          from: serviceAccount
+      labelKeys:
+        infrastructure:
+          labelKeys:
+            - host
+            - appname
+      target:
+        name: logging-loki
+        namespace: openshift-logging
+    tls:
+      ca:
+        key: service-ca.crt
+        configMapName: openshift-service-ca.crt
+  - name: audit-lokistack
+    type: lokiStack
+    lokiStack:
+      authentication:
+        token:
+          from: serviceAccount
+      target:
+        name: logging-loki-audit
+        namespace: openshift-logging
+    tls:
+      ca:
+        key: service-ca.crt
+        configMapName: openshift-service-ca.crt
+  filters:
+  - name: to-loki-filter
+    type: drop
+    drop:
+    - test:
+      - field: .message
+        matches: "oslo.messaging.notification"
+  - name: to-loki-audit-filter
+    type: drop
+    drop:
+    - test:
+      - field: .message
+        notMatches: "oslo.messaging.notification"
   pipelines:
-    - name: my-syslog
-      inputRefs:
-        - syslog-receiver
-      outputRefs:
-        - default-lokistack
+  - name: to-loki
+    inputRefs:
+      - application
+      - dataplane
+    outputRefs:
+      - default-lokistack
+    filterRefs:
+      - to-loki-filter
+  - name: to-loki-audit
+    inputRefs:
+      - application
+    outputRefs:
+      - audit-lokistack
+    filterRefs:
+      - to-loki-audit-filter

ci/deploy-logging-dependencies/files/minio-dev.yaml

@@ -23,7 +23,7 @@ spec:
     - /bin/bash
     - -c
     - |
-      mkdir -p /data/loki && \
+      mkdir -p /data/loki /data/loki-audit && \
       minio server /data
     env:
     - name: MINIO_ACCESS_KEY

ci/deploy-logging-dependencies/tasks/deploy-cluster-logging.yml

@@ -15,7 +15,7 @@
 - name: Wait for cluster log forwarder to be created
   ansible.builtin.command:
     cmd:
-      oc get clusterlogforwarder --namespace=openshift-logging collector
+      oc get clusterlogforwarder --namespace=openshift-logging collector-with-cadf-to-loki-audit
   delay: 10
   retries: 20
   register: output
@@ -24,7 +24,7 @@
 - name: Wait for all conditions to be "True" in ClusterLogForwarder status
   ansible.builtin.shell:
     cmd: >
-      oc get clusterlogforwarder collector -n openshift-logging -o json |
+      oc get clusterlogforwarder collector-with-cadf-to-loki-audit -n openshift-logging -o json |
       jq '.status.conditions | length > 0 and all(.status == "True")'
   register: clf_conditions
   until: clf_conditions.stdout == "true"

ci/deploy-logging-dependencies/tasks/deploy-loki.yml

@@ -26,3 +26,31 @@
   until: loki_ready.stdout == "True"
   retries: 40
   delay: 15
+
+- name: Create audit loki S3 secret
+  ansible.builtin.shell:
+    cmd: |
+      oc apply -f {{ role_path }}/files/audit_loki_secret.yaml -n openshift-logging
+
+- name: Create audit loki stack
+  ansible.builtin.shell:
+    cmd: |
+      oc apply -f {{ role_path }}/files/audit_lokistack.yaml -n openshift-logging
+
+- name: Wait for audit lokistack to be created
+  ansible.builtin.command:
+    cmd:
+      oc get lokistacks --namespace=openshift-logging logging-loki-audit
+  delay: 10
+  retries: 20
+  register: output
+  until: output.stdout_lines | length != 0
+
+- name: Wait for the audit lokistack to be ready
+  ansible.builtin.command:
+    cmd:
+      oc get lokistack logging-loki-audit -n openshift-logging -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+  register: audit_loki_ready
+  until: audit_loki_ready.stdout == "True"
+  retries: 40
+  delay: 15