diff --git a/telco-core/configuration/core-baseline.yaml b/telco-core/configuration/core-baseline.yaml index f71a08876..1a8af2bcb 100644 --- a/telco-core/configuration/core-baseline.yaml +++ b/telco-core/configuration/core-baseline.yaml @@ -47,6 +47,12 @@ policies: status: "True" reason: InstallSucceeded + # Loki operator (optional component) + # - path: reference-crs/optional/logging/LokiOperatorNS.yaml + # - path: reference-crs/optional/logging/LokiOperatorGroup.yaml + # - path: reference-crs/optional/logging/LokiSubscription.yaml + # - path: reference-crs/optional/logging/LokiOperatorStatus.yaml + # Cert-manager operator (optional component) # - path: reference-crs/optional/cert-manager/certManagerNS.yaml # - path: reference-crs/optional/cert-manager/certManagerOperatorgroup.yaml @@ -93,6 +99,10 @@ policies: - path: reference-crs/optional/logging/ClusterLogServiceAccountAuditBinding.yaml - path: reference-crs/optional/logging/ClusterLogServiceAccountInfrastructureBinding.yaml + # LokiStack log storage (optional component) + # - path: reference-crs/optional/logging/LokiSecret.yaml + # - path: reference-crs/optional/logging/LokiStack.yaml + - path: reference-crs/required/networking/sriov/SriovOperatorConfig.yaml patches: - spec: diff --git a/telco-core/configuration/core-overlay.yaml b/telco-core/configuration/core-overlay.yaml index ec625c5c6..e6c08d8d3 100644 --- a/telco-core/configuration/core-overlay.yaml +++ b/telco-core/configuration/core-overlay.yaml @@ -174,10 +174,14 @@ policies: # profile: custom-worker-profile # Cluster Logging + # The base ClusterLogForwarder.yaml includes local Loki storage by default. + # This example patch adds external Kafka forwarding with custom labels. - path: reference-crs/optional/logging/ClusterLogForwarder.yaml patches: - spec: outputs: + # Loki output is already configured in the base CR + # Add external output for OSS/BSS systems - type: "kafka" name: kafka-open kafka: @@ -190,7 +194,9 @@ policies: siteuuid: '{{hub fromConfigMap "" .ManagedClusterName "logging-uuid" | toLiteral hub}}' label3: 'other data' pipelines: - - name: all-to-default + # Loki pipeline is already configured in the base CR + # Add pipeline for external forwarding + - name: all-to-external inputRefs: - audit - infrastructure diff --git a/telco-core/configuration/reference-crs-kube-compare/compare_ignore b/telco-core/configuration/reference-crs-kube-compare/compare_ignore index a78b85a59..3b7fee830 100644 --- a/telco-core/configuration/reference-crs-kube-compare/compare_ignore +++ b/telco-core/configuration/reference-crs-kube-compare/compare_ignore @@ -13,6 +13,9 @@ required/machine-config/container-runtime.yaml ClusterLogging5Cleanup.yaml ClusterLogOperatorStatus.yaml +# Utility object for Loki Operator installation verification +LokiOperatorStatus.yaml + # Utility objects to wait for and acknowledge cluster ugprades ClusterVersion.yaml upgrade-ack.yaml diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogForwarder.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogForwarder.yaml index 5871ace29..7719446b3 100644 --- a/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogForwarder.yaml +++ b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogForwarder.yaml @@ -6,10 +6,7 @@ metadata: spec: {{- if .spec.outputs }} outputs: - - name: {{ (index .spec.outputs 0).name }} - type: kafka - kafka: - url: {{ (index .spec.outputs 0).kafka.url }} + {{- .spec.outputs | toYaml | nindent 2 }} {{- end }} {{- if .spec.filters }} filters: diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiOperatorGroup.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiOperatorGroup.yaml new file mode 100644 index 000000000..c0bc3acad --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiOperatorGroup.yaml @@ -0,0 +1,11 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: loki-operator + namespace: openshift-operators-redhat + annotations: + operatorframework.io/bundle-unpack-min-retry-interval: 10m +spec: + {{- if .spec.upgradeStrategy }} + upgradeStrategy: Default + {{- end }} diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiOperatorNS.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiOperatorNS.yaml new file mode 100644 index 000000000..5929c0d05 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiOperatorNS.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-operators-redhat + labels: + openshift.io/cluster-monitoring: "true" diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiSecret.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiSecret.yaml new file mode 100644 index 000000000..e858a62a8 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiSecret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: logging-loki-s3 + namespace: openshift-logging +stringData: + access_key_id: {{ .stringData.access_key_id }} + access_key_secret: {{ .stringData.access_key_secret }} + bucketnames: {{ .stringData.bucketnames }} + endpoint: {{ .stringData.endpoint }} + region: {{ .stringData.region }} diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiStack.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiStack.yaml new file mode 100644 index 000000000..d4b95edca --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiStack.yaml @@ -0,0 +1,22 @@ +apiVersion: loki.grafana.com/v1 +kind: LokiStack +metadata: + name: logging-loki + namespace: openshift-logging +spec: + managementState: Managed + limits: + global: + retention: + days: {{ .spec.limits.global.retention.days }} + size: {{ .spec.size }} + storage: + schemas: + - version: v13 + effectiveDate: {{ (index .spec.storage.schemas 0).effectiveDate }} + secret: + name: logging-loki-s3 + type: s3 + storageClassName: {{ .spec.storageClassName }} + tenants: + mode: openshift-logging diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiSubscription.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiSubscription.yaml new file mode 100644 index 000000000..e6d2156f6 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/logging/LokiSubscription.yaml @@ -0,0 +1,13 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: loki-operator + namespace: openshift-operators-redhat +spec: + channel: "stable-6.5" + name: loki-operator + source: {{ .spec.source }} + sourceNamespace: openshift-marketplace + installPlanApproval: Manual +status: + state: AtLatestKnown diff --git a/telco-core/configuration/reference-crs/optional/logging/ClusterLogForwarder.yaml b/telco-core/configuration/reference-crs/optional/logging/ClusterLogForwarder.yaml index cb29e2117..4e51add81 100644 --- a/telco-core/configuration/reference-crs/optional/logging/ClusterLogForwarder.yaml +++ b/telco-core/configuration/reference-crs/optional/logging/ClusterLogForwarder.yaml @@ -1,12 +1,50 @@ --- -apiVersion: "observability.openshift.io/v1" +apiVersion: observability.openshift.io/v1 kind: ClusterLogForwarder metadata: name: instance namespace: openshift-logging spec: - # outputs: $outputs - # pipelines: $pipelines + outputs: + # Local Loki storage for troubleshooting + - name: loki-local + type: lokiStack + lokiStack: + target: + name: logging-loki + namespace: openshift-logging + authentication: + token: + from: serviceAccount + tls: + ca: + key: service-ca.crt + configMapName: openshift-service-ca.crt + + # External log aggregation (OSS/BSS systems) + # Uncomment and configure for external forwarding + # - name: external-logs + # type: kafka # or splunk, syslog, etc. + # kafka: + # url: tcp://10.11.12.13:9092/test + + pipelines: + # Infrastructure and audit logs to local Loki + - name: infra-audit-to-local-loki + inputRefs: + - audit + - infrastructure + outputRefs: + - loki-local + + # Optional: Forward to external system (uncomment if needed) + # - name: infra-audit-to-external + # inputRefs: + # - audit + # - infrastructure + # outputRefs: + # - external-logs + serviceAccount: name: collector status: @@ -14,35 +52,3 @@ status: conditions: - type: observability.openshift.io/Valid status: "True" - -# apiVersion: "observability.openshift.io/v1" -# kind: ClusterLogForwarder -# metadata: -# name: instance -# namespace: openshift-logging -# spec: -# outputs: -# - type: "kafka" -# name: kafka-open -# # below url is an example -# kafka: -# url: tcp://10.11.12.13:9092/test -# filters: -# - name: test-labels -# type: openshiftLabels -# openshiftLabels: -# label1: test1 -# label2: test2 -# label3: test3 -# label4: test4 -# pipelines: -# - name: all-to-default -# inputRefs: -# - audit -# - infrastructure -# filterRefs: -# - test-labels -# outputRefs: -# - kafka-open -# serviceAccount: -# name: collector diff --git a/telco-core/configuration/reference-crs/optional/logging/LokiOperatorGroup.yaml b/telco-core/configuration/reference-crs/optional/logging/LokiOperatorGroup.yaml new file mode 100644 index 000000000..6c951840f --- /dev/null +++ b/telco-core/configuration/reference-crs/optional/logging/LokiOperatorGroup.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: loki-operator + namespace: openshift-operators-redhat + annotations: + operatorframework.io/bundle-unpack-min-retry-interval: 10m +spec: + upgradeStrategy: Default diff --git a/telco-core/configuration/reference-crs/optional/logging/LokiOperatorNS.yaml b/telco-core/configuration/reference-crs/optional/logging/LokiOperatorNS.yaml new file mode 100644 index 000000000..8b6b6e554 --- /dev/null +++ b/telco-core/configuration/reference-crs/optional/logging/LokiOperatorNS.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-operators-redhat + labels: + # The cluster-monitoring label ensures Prometheus scrapes this namespace. + openshift.io/cluster-monitoring: "true" diff --git a/telco-core/configuration/reference-crs/optional/logging/LokiOperatorStatus.yaml b/telco-core/configuration/reference-crs/optional/logging/LokiOperatorStatus.yaml new file mode 100644 index 000000000..98be02095 --- /dev/null +++ b/telco-core/configuration/reference-crs/optional/logging/LokiOperatorStatus.yaml @@ -0,0 +1,28 @@ +# Operator installation verification +# This CR can be used to verify the installation/upgrade of the +# Loki Operator by including status fields as shown in +# examples below +--- +apiVersion: operators.coreos.com/v1 +kind: Operator +metadata: + name: loki-operator.openshift-operators-redhat +status: +# components: +# refs: +# - kind: Subscription +# namespace: openshift-operators-redhat +# conditions: +# - type: CatalogSourcesUnhealthy +# status: "False" +# - kind: InstallPlan +# namespace: openshift-operators-redhat +# conditions: +# - type: Installed +# status: "True" +# - kind: ClusterServiceVersion +# namespace: openshift-operators-redhat +# conditions: +# - type: Succeeded +# status: "True" +# reason: InstallSucceeded diff --git a/telco-core/configuration/reference-crs/optional/logging/LokiSecret.yaml b/telco-core/configuration/reference-crs/optional/logging/LokiSecret.yaml new file mode 100644 index 000000000..75489f496 --- /dev/null +++ b/telco-core/configuration/reference-crs/optional/logging/LokiSecret.yaml @@ -0,0 +1,18 @@ +# Object storage credentials template +# +# Note: In production environments, secrets should be managed via secure +# methods such as the External Secrets Operator, Sealed Secrets, or a +# secrets management platform (e.g., HashiCorp Vault, AWS Secrets Manager). +# Avoid committing actual credentials to version control. +--- +apiVersion: v1 +kind: Secret +metadata: + name: logging-loki-s3 + namespace: openshift-logging +stringData: + access_key_id: "" + access_key_secret: "" + bucketnames: "" + endpoint: "" # e.g. + region: "" diff --git a/telco-core/configuration/reference-crs/optional/logging/LokiStack.yaml b/telco-core/configuration/reference-crs/optional/logging/LokiStack.yaml new file mode 100644 index 000000000..b474d440b --- /dev/null +++ b/telco-core/configuration/reference-crs/optional/logging/LokiStack.yaml @@ -0,0 +1,43 @@ +# Important: do not reuse the same LokiStack that is used for logging +# See: https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html/network_observability/installing-network-observability-operators +--- +apiVersion: loki.grafana.com/v1 +kind: LokiStack +metadata: + name: logging-loki + namespace: openshift-logging +spec: + managementState: Managed + limits: + global: + retention: + days: 5 # Default retention period. User-configurable. + # Sizing: uses Loki's built-in defaults for PVC sizing, which are determined + # by the operator based on the size profile. + # Important: sufficient storage must be provided according to data + # requirements and the retention period configured above. + size: 1x.extra-small # 14 vCPUs, 31Gi memory, 430Gi disk, 100GB/day + storage: + schemas: + - version: v13 + # effectiveDate controls when this schema version becomes active. + # CRITICAL: Must be set to a date in the PAST (at least 2 months before + # deployment) to ensure the schema is immediately active when Loki starts. + # Setting it in the future will prevent Loki from ingesting any logs. + # Setting it too recently may cause Loki to reject logs with timestamps + # that predate the effectiveDate. Cannot be changed retroactively. + # Example: If deploying in May 2026, set to "2026-03-01" or earlier. + effectiveDate: "" # e.g., "2026-03-01" + # Object storage configuration (LONG-TERM LOG STORAGE) + # This secret contains S3-compatible object storage credentials where + # compressed log chunks and indexes are stored permanently. + secret: + name: logging-loki-s3 + type: s3 + # Block storage configuration (SHORT-TERM BUFFERING) + # storageClassName provisions PVCs for Loki component StatefulSets. + # Used for: Write-Ahead Logs (WAL), temporary chunk caching, ingester buffering. + # This is INDEPENDENT from object storage - both are required. + storageClassName: "ocs-storagecluster-ceph-rbd" # Set to the cluster's available StorageClass + tenants: + mode: openshift-logging diff --git a/telco-core/configuration/reference-crs/optional/logging/LokiSubscription.yaml b/telco-core/configuration/reference-crs/optional/logging/LokiSubscription.yaml new file mode 100644 index 000000000..70a1a3b8c --- /dev/null +++ b/telco-core/configuration/reference-crs/optional/logging/LokiSubscription.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: loki-operator + namespace: openshift-operators-redhat +spec: + channel: "stable-6.5" # Adapt for compatibility for OpenShift versions. + installPlanApproval: Manual + name: loki-operator + source: redhat-operators-disconnected # For disconnected environments. + sourceNamespace: openshift-marketplace +status: + state: AtLatestKnown diff --git a/telco-core/configuration/reference-crs/optional/networking/firewall/node-disruption-policy.yaml b/telco-core/configuration/reference-crs/optional/networking/firewall/node-disruption-policy.yaml index 77842e92e..3eb2eca66 100644 --- a/telco-core/configuration/reference-crs/optional/networking/firewall/node-disruption-policy.yaml +++ b/telco-core/configuration/reference-crs/optional/networking/firewall/node-disruption-policy.yaml @@ -10,6 +10,7 @@ # If you aren't currently defining any nodeDisruptionPolicies, apply this file directly: # oc patch machineconfiguration cluster --type=merge --patch-file=node-disruption-policy.yaml # Otherwise, manually add these entries to .spec.nodeDisruptionPolicy.units and .spec.nodeDisruptionPolicy.files. +--- apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: diff --git a/telco-core/install/extra-manifests/Network.yaml b/telco-core/install/extra-manifests/Network.yaml index 96e1faed2..f3c402a2d 100644 --- a/telco-core/install/extra-manifests/Network.yaml +++ b/telco-core/install/extra-manifests/Network.yaml @@ -7,4 +7,4 @@ metadata: spec: defaultNetwork: ovnKubernetesConfig: - mtu: 8900 # The cluster MTU must be set lower than machine network MTU. Optional features like encryption may require further lowering the MTU. See the documentation for details: https://docs.redhat.com/en/documentation/openshift_container_platform/latest/html/advanced_networking/changing-cluster-network-mtu + mtu: 8900 # The cluster MTU must be set lower than machine network MTU. Optional features like encryption may require further lowering the MTU. See the documentation for details: https://docs.redhat.com/en/documentation/openshift_container_platform/latest/html/advanced_networking/changing-cluster-network-mtu diff --git a/telco-hub/install/openshift/openshift/Network.yaml b/telco-hub/install/openshift/openshift/Network.yaml index 96e1faed2..f3c402a2d 100644 --- a/telco-hub/install/openshift/openshift/Network.yaml +++ b/telco-hub/install/openshift/openshift/Network.yaml @@ -7,4 +7,4 @@ metadata: spec: defaultNetwork: ovnKubernetesConfig: - mtu: 8900 # The cluster MTU must be set lower than machine network MTU. Optional features like encryption may require further lowering the MTU. See the documentation for details: https://docs.redhat.com/en/documentation/openshift_container_platform/latest/html/advanced_networking/changing-cluster-network-mtu + mtu: 8900 # The cluster MTU must be set lower than machine network MTU. Optional features like encryption may require further lowering the MTU. See the documentation for details: https://docs.redhat.com/en/documentation/openshift_container_platform/latest/html/advanced_networking/changing-cluster-network-mtu