CNF-14592: add LokiStack configuration CRs

rdiscala · claude · rdiscala · commit 182ff45b48d6 · 2026-06-19T09:58:51.000+02:00
Add Loki Operator installation and LokiStack log storage CRs for the
telco-core optional logging configuration:

 - LokiStack CR with 1x.extra-small sizing, S3 storage, v13 schema,
   and 5-day retention
 - Loki Operator Namespace, OperatorGroup, Subscription, and
   installation status verification CR
 - S3 object storage Secret template
 - Update ClusterLogForwarder to enable local Loki storage by default
   with audit and infrastructure log forwarding, plus commented
   examples for external system integration (Kafka, Splunk, etc.)
 - Update core-overlay.yaml PolicyGenerator to demonstrate adding
   external forwarding to the base Loki configuration
 - Add kube-compare reference templates for validation

This implements "optional but enabled by default" pattern: CRs are in
the optional/ folder and commented in PolicyGenerators, but when
enabled provide a complete working Loki configuration that supports
both local CNF vendor troubleshooting and external OSS/BSS forwarding.

Co-authored-by: Claude Sonnet 4.5 &lt;noreply@anthropic.com&gt;
diff --git a/telco-core/configuration/core-baseline.yaml b/telco-core/configuration/core-baseline.yaml
@@ -47,6 +47,12 @@ policies:
                     status: "True"
                     reason: InstallSucceeded
 
+      # Loki operator (optional component)
+      # - path: reference-crs/optional/logging/LokiOperatorNS.yaml
+      # - path: reference-crs/optional/logging/LokiOperatorGroup.yaml
+      # - path: reference-crs/optional/logging/LokiSubscription.yaml
+      # - path: reference-crs/optional/logging/LokiOperatorStatus.yaml
+
       # Cert-manager operator (optional component)
       # - path: reference-crs/optional/cert-manager/certManagerNS.yaml
       # - path: reference-crs/optional/cert-manager/certManagerOperatorgroup.yaml
@@ -93,6 +99,10 @@ policies:
       - path: reference-crs/optional/logging/ClusterLogServiceAccountAuditBinding.yaml
       - path: reference-crs/optional/logging/ClusterLogServiceAccountInfrastructureBinding.yaml
 
+      # LokiStack log storage (optional component)
+      # - path: reference-crs/optional/logging/LokiSecret.yaml
+      # - path: reference-crs/optional/logging/LokiStack.yaml
+
       - path: reference-crs/required/networking/sriov/SriovOperatorConfig.yaml
         patches:
         - spec:
diff --git a/telco-core/configuration/core-overlay.yaml b/telco-core/configuration/core-overlay.yaml
@@ -174,10 +174,14 @@ policies:
       #           profile: custom-worker-profile
 
       # Cluster Logging
+      # The base ClusterLogForwarder.yaml includes local Loki storage by default.
+      # This example patch adds external Kafka forwarding with custom labels.
       - path: reference-crs/optional/logging/ClusterLogForwarder.yaml
         patches:
         - spec:
             outputs:
+            # Loki output is already configured in the base CR
+            # Add external output for OSS/BSS systems
             - type: "kafka"
               name: kafka-open
               kafka:
@@ -190,7 +194,9 @@ policies:
                 siteuuid: '{{hub fromConfigMap "" .ManagedClusterName "logging-uuid" | toLiteral hub}}'
                 label3: 'other data'
             pipelines:
-            - name: all-to-default
+            # Loki pipeline is already configured in the base CR
+            # Add pipeline for external forwarding
+            - name: all-to-external
               inputRefs:
               - audit
               - infrastructure
diff --git a/telco-core/configuration/reference-crs-kube-compare/compare_ignore b/telco-core/configuration/reference-crs-kube-compare/compare_ignore
@@ -13,6 +13,9 @@ required/machine-config/container-runtime.yaml
 ClusterLogging5Cleanup.yaml
 ClusterLogOperatorStatus.yaml
 
+# Utility object for Loki Operator installation verification
+LokiOperatorStatus.yaml
+
 # Utility objects to wait for and acknowledge cluster ugprades
 ClusterVersion.yaml
 upgrade-ack.yaml
diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogForwarder.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogForwarder.yaml
@@ -6,10 +6,7 @@ metadata:
 spec:
   {{- if .spec.outputs }}
   outputs:
-  - name: {{ (index .spec.outputs 0).name }}
-    type: kafka
-    kafka:
-      url: {{ (index .spec.outputs 0).kafka.url }}
+    {{- .spec.outputs | toYaml | nindent 2 }}
   {{- end }}
   {{- if .spec.filters }}
   filters:
diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiOperatorGroup.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiOperatorGroup.yaml
@@ -0,0 +1,11 @@
+apiVersion: operators.coreos.com/v1
+kind: OperatorGroup
+metadata:
+  name: loki-operator
+  namespace: openshift-operators-redhat
+  annotations:
+    operatorframework.io/bundle-unpack-min-retry-interval: 10m
+spec:
+  {{- if .spec.upgradeStrategy }}
+  upgradeStrategy: Default
+  {{- end }}
diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiOperatorNS.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiOperatorNS.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: openshift-operators-redhat
+  labels:
+    openshift.io/cluster-monitoring: "true"
diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiSecret.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiSecret.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: logging-loki-s3
+  namespace: openshift-logging
+stringData:
+  access_key_id: {{ .stringData.access_key_id }}
+  access_key_secret: {{ .stringData.access_key_secret }}
+  bucketnames: {{ .stringData.bucketnames }}
+  endpoint: {{ .stringData.endpoint }}
+  region: {{ .stringData.region }}
diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiStack.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiStack.yaml
@@ -0,0 +1,22 @@
+apiVersion: loki.grafana.com/v1
+kind: LokiStack
+metadata:
+  name: logging-loki
+  namespace: openshift-logging
+spec:
+  managementState: Managed
+  limits:
+    global:
+      retention:
+        days: {{ .spec.limits.global.retention.days }}
+  size: {{ .spec.size }}
+  storage:
+    schemas:
+    - version: v13
+      effectiveDate: {{ (index .spec.storage.schemas 0).effectiveDate }}
+    secret:
+      name: logging-loki-s3
+      type: s3
+  storageClassName: {{ .spec.storageClassName }}
+  tenants:
+    mode: openshift-logging
diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiSubscription.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiSubscription.yaml
@@ -0,0 +1,13 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: loki-operator
+  namespace: openshift-operators-redhat
+spec:
+  channel: "stable-6.5"
+  name: loki-operator
+  source: {{ .spec.source }}
+  sourceNamespace: openshift-marketplace
+  installPlanApproval: Manual
+status:
+  state: AtLatestKnown
diff --git a/telco-core/configuration/reference-crs/optional/logging/ClusterLogForwarder.yaml b/telco-core/configuration/reference-crs/optional/logging/ClusterLogForwarder.yaml
@@ -1,48 +1,54 @@
 ---
-apiVersion: "observability.openshift.io/v1"
+apiVersion: observability.openshift.io/v1
 kind: ClusterLogForwarder
 metadata:
   name: instance
   namespace: openshift-logging
 spec:
-  # outputs: $outputs
-  # pipelines: $pipelines
+  outputs:
+  # Local Loki storage for troubleshooting
+  - name: loki-local
+    type: lokiStack
+    lokiStack:
+      target:
+        name: logging-loki
+        namespace: openshift-logging
+      authentication:
+        token:
+          from: serviceAccount
+    tls:
+      ca:
+        key: service-ca.crt
+        configMapName: openshift-service-ca.crt
+
+  # External log aggregation (OSS/BSS systems)
+  # Uncomment and configure for external forwarding
+  # - name: external-logs
+  #   type: kafka  # or splunk, syslog, etc.
+  #   kafka:
+  #     url: tcp://10.11.12.13:9092/test
+
+  pipelines:
+  # Infrastructure and audit logs to local Loki
+  - name: infra-audit-to-local-loki
+    inputRefs:
+    - audit
+    - infrastructure
+    outputRefs:
+    - loki-local
+
+  # Optional: Forward to external system (uncomment if needed)
+  # - name: infra-audit-to-external
+  #   inputRefs:
+  #   - audit
+  #   - infrastructure
+  #   outputRefs:
+  #   - external-logs
+
   serviceAccount:
     name: collector
 status:
   # Verify that the configuration is correct
   conditions:
   - type: observability.openshift.io/Valid
     status: "True"
-
-# apiVersion: "observability.openshift.io/v1"
-# kind: ClusterLogForwarder
-# metadata:
-#   name: instance
-#   namespace: openshift-logging
-#  spec:
-#    outputs:
-#    - type: "kafka"
-#      name: kafka-open
-#      # below url is an example
-#      kafka:
-#        url: tcp://10.11.12.13:9092/test
-#    filters:
-#    - name: test-labels
-#      type: openshiftLabels
-#      openshiftLabels:
-#        label1: test1
-#        label2: test2
-#        label3: test3
-#        label4: test4
-#    pipelines:
-#    - name: all-to-default
-#      inputRefs:
-#      - audit
-#      - infrastructure
-#      filterRefs:
-#      - test-labels
-#      outputRefs:
-#      - kafka-open
-#    serviceAccount:
-#      name: collector
diff --git a/telco-core/configuration/reference-crs/optional/logging/LokiOperatorGroup.yaml b/telco-core/configuration/reference-crs/optional/logging/LokiOperatorGroup.yaml
@@ -0,0 +1,10 @@
+---
+apiVersion: operators.coreos.com/v1
+kind: OperatorGroup
+metadata:
+  name: loki-operator
+  namespace: openshift-operators-redhat
+  annotations:
+    operatorframework.io/bundle-unpack-min-retry-interval: 10m
+spec:
+  upgradeStrategy: Default
diff --git a/telco-core/configuration/reference-crs/optional/logging/LokiOperatorNS.yaml b/telco-core/configuration/reference-crs/optional/logging/LokiOperatorNS.yaml
@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: openshift-operators-redhat
+  labels:
+    # The cluster-monitoring label ensures Prometheus scrapes this namespace.
+    openshift.io/cluster-monitoring: "true"
diff --git a/telco-core/configuration/reference-crs/optional/logging/LokiOperatorStatus.yaml b/telco-core/configuration/reference-crs/optional/logging/LokiOperatorStatus.yaml
@@ -0,0 +1,28 @@
+# Operator installation verification
+# This CR can be used to verify the installation/upgrade of the
+# Loki Operator by including status fields as shown in
+# examples below
+---
+apiVersion: operators.coreos.com/v1
+kind: Operator
+metadata:
+  name: loki-operator.openshift-operators-redhat
+status:
+#  components:
+#    refs:
+#    - kind: Subscription
+#      namespace: openshift-operators-redhat
+#      conditions:
+#      - type: CatalogSourcesUnhealthy
+#        status: "False"
+#    - kind: InstallPlan
+#      namespace: openshift-operators-redhat
+#      conditions:
+#      - type: Installed
+#        status: "True"
+#    - kind: ClusterServiceVersion
+#      namespace: openshift-operators-redhat
+#      conditions:
+#      - type: Succeeded
+#        status: "True"
+#        reason: InstallSucceeded
diff --git a/telco-core/configuration/reference-crs/optional/logging/LokiSecret.yaml b/telco-core/configuration/reference-crs/optional/logging/LokiSecret.yaml
@@ -0,0 +1,18 @@
+# Object storage credentials template
+#
+# Note: In production environments, secrets should be managed via secure
+# methods such as the External Secrets Operator, Sealed Secrets, or a
+# secrets management platform (e.g., HashiCorp Vault, AWS Secrets Manager).
+# Avoid committing actual credentials to version control.
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: logging-loki-s3
+  namespace: openshift-logging
+stringData:
+  access_key_id: "<access key>"
+  access_key_secret: "<access key secret>"
+  bucketnames: "<bucket names>"
+  endpoint: "<storage endpoint>"  # e.g. <https://storage.endpoint.url>
+  region: "<region>"
diff --git a/telco-core/configuration/reference-crs/optional/logging/LokiStack.yaml b/telco-core/configuration/reference-crs/optional/logging/LokiStack.yaml
@@ -0,0 +1,43 @@
+# Important: do not reuse the same LokiStack that is used for logging
+# See: https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html/network_observability/installing-network-observability-operators
+---
+apiVersion: loki.grafana.com/v1
+kind: LokiStack
+metadata:
+  name: logging-loki
+  namespace: openshift-logging
+spec:
+  managementState: Managed
+  limits:
+    global:
+      retention:
+        days: 5  # Default retention period. User-configurable.
+  # Sizing: uses Loki's built-in defaults for PVC sizing, which are determined
+  # by the operator based on the size profile.
+  # Important: sufficient storage must be provided according to data
+  # requirements and the retention period configured above.
+  size: 1x.extra-small  # 14 vCPUs, 31Gi memory, 430Gi disk, 100GB/day
+  storage:
+    schemas:
+    - version: v13
+      # effectiveDate controls when this schema version becomes active.
+      # CRITICAL: Must be set to a date in the PAST (at least 2 months before
+      # deployment) to ensure the schema is immediately active when Loki starts.
+      # Setting it in the future will prevent Loki from ingesting any logs.
+      # Setting it too recently may cause Loki to reject logs with timestamps
+      # that predate the effectiveDate. Cannot be changed retroactively.
+      # Example: If deploying in May 2026, set to "2026-03-01" or earlier.
+      effectiveDate: "<yyyy-mm-dd>"  # e.g., "2026-03-01"
+    # Object storage configuration (LONG-TERM LOG STORAGE)
+    # This secret contains S3-compatible object storage credentials where
+    # compressed log chunks and indexes are stored permanently.
+    secret:
+      name: logging-loki-s3
+      type: s3
+  # Block storage configuration (SHORT-TERM BUFFERING)
+  # storageClassName provisions PVCs for Loki component StatefulSets.
+  # Used for: Write-Ahead Logs (WAL), temporary chunk caching, ingester buffering.
+  # This is INDEPENDENT from object storage - both are required.
+  storageClassName: "ocs-storagecluster-ceph-rbd"  # Set to the cluster's available StorageClass
+  tenants:
+    mode: openshift-logging
diff --git a/telco-core/configuration/reference-crs/optional/logging/LokiSubscription.yaml b/telco-core/configuration/reference-crs/optional/logging/LokiSubscription.yaml
@@ -0,0 +1,14 @@
+---
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: loki-operator
+  namespace: openshift-operators-redhat
+spec:
+  channel: "stable-6.5"  # Adapt for compatibility for OpenShift versions.
+  installPlanApproval: Manual
+  name: loki-operator
+  source: redhat-operators-disconnected  # For disconnected environments.
+  sourceNamespace: openshift-marketplace
+status:
+  state: AtLatestKnown
