diff --git a/modules/gcs/standard/1.0/bucket.tf b/modules/gcs/standard/1.0/bucket.tf index c202f49f..80776ae9 100644 --- a/modules/gcs/standard/1.0/bucket.tf +++ b/modules/gcs/standard/1.0/bucket.tf @@ -36,7 +36,7 @@ resource "google_storage_bucket" "bucket" { name = local.bucket_name location = local.location storage_class = var.instance.spec.storage_class - project = var.inputs.cloud_account.attributes.project + project = var.inputs.cloud_account.attributes.project_id uniform_bucket_level_access = var.instance.spec.uniform_bucket_level_access requester_pays = var.instance.spec.requester_pays diff --git a/modules/gcs/standard/1.0/variables.tf b/modules/gcs/standard/1.0/variables.tf index 2f544e83..d57e1b9a 100644 --- a/modules/gcs/standard/1.0/variables.tf +++ b/modules/gcs/standard/1.0/variables.tf @@ -62,7 +62,7 @@ variable "inputs" { type = object({ cloud_account = object({ attributes = object({ - project = string + project_id = string region = string credentials = optional(string) }) diff --git a/modules/log_collector/loki_gcs/1.0/facets.yaml b/modules/log_collector/loki_gcs/1.0/facets.yaml new file mode 100644 index 00000000..78be91c0 --- /dev/null +++ b/modules/log_collector/loki_gcs/1.0/facets.yaml @@ -0,0 +1,405 @@ +intent: log_collector +flavor: loki_gcs +version: "1.0" +description: | + Deploys Grafana Loki on Kubernetes with GCS as the storage backend using the unified grafana/loki chart. + Includes Promtail (DaemonSet log scraper) and optional Loki Canary for end-to-end latency testing. +intentDetails: + type: Monitoring & Observability + description: Loki log aggregation stack with GCS backend on GKE + displayName: Loki (GCS) + iconUrl: https://raw.githubusercontent.com/Facets-cloud/facets-modules-redesign/main/icons/monitoring.svg +clouds: + - gcp + +inputs: + kubernetes_details: + type: "@facets/kubernetes-details" + optional: false + displayName: Kubernetes Cluster + description: GKE cluster — provides kubernetes, helm providers + providers: + - kubernetes + - helm + cloud_account: + type: "@facets/gcp_cloud_account" + optional: false + displayName: GCP Cloud Account + description: GCP project credentials — provides google provider + providers: + - google + storage: + type: "@facets/gcs-bucket" + optional: false + displayName: GCS Bucket + description: GCS bucket for Loki log storage (use storage/gcs module) + kubernetes_node_pool_details: + type: "@facets/kubernetes_nodepool" + optional: true + displayName: Node Pool + description: Target node pool for Loki and Promtail pods + +outputs: + default: + type: "@facets/loki" + title: Loki Log Collector + +spec: + title: Loki Log Collector Configuration + type: object + properties: + metadata: + type: object + title: Metadata + properties: + namespace: + type: string + title: Namespace + description: Kubernetes namespace for Loki deployment + default: facets + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + x-ui-placeholder: e.g. logging, loki, facets + + loki: + type: object + title: Loki Configuration + properties: + chart_version: + type: string + title: Chart Version + description: Helm chart version for grafana/loki (unified chart) + default: "6.55.0" + deployment_mode: + type: string + title: Deployment Mode + description: "Loki deployment topology: Distributed (individual components), SimpleScalable (write/read/backend), SingleBinary (all-in-one)" + default: SingleBinary + enum: + - SingleBinary + - Distributed + retention_hours: + type: integer + title: Retention (hours) + description: Log retention period in hours + default: 720 + single_binary_size: + type: object + title: SingleBinary Resource Sizing + description: Resource sizing and persistence for the SingleBinary Loki pod + x-ui-visible-if: + field: spec.loki.deployment_mode + values: + - SingleBinary + properties: + resources: + type: object + title: Resources + properties: + requests: + type: object + title: Resource Requests + properties: + cpu: + type: string + title: CPU Request + default: "1000m" + pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' + memory: + type: string + title: Memory Request + default: "2Gi" + pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' + limits: + type: object + title: Resource Limits + properties: + cpu: + type: string + title: CPU Limit + default: "2000m" + pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' + memory: + type: string + title: Memory Limit + default: "4Gi" + pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' + persistence_size: + type: string + title: Persistence Size + description: Persistent volume size for SingleBinary Loki pod + default: "20Gi" + pattern: '^[0-9]+(\.[0-9]+)?[G]i?$' + size: + type: object + title: Component Resource Sizing + description: Per-component CPU and memory requests/limits for each Loki component + x-ui-visible-if: + field: spec.loki.deployment_mode + values: + - Distributed + properties: + distributor: + type: object + title: Distributor Resources + properties: + resources: + type: object + properties: + requests: + type: object + properties: + cpu: { type: string, title: CPU Request, default: "300m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Request, default: "300Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + limits: + type: object + properties: + cpu: { type: string, title: CPU Limit, default: "1000m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Limit, default: "1000Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + ingester: + type: object + title: Ingester Resources + properties: + resources: + type: object + properties: + requests: + type: object + properties: + cpu: { type: string, title: CPU Request, default: "300m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Request, default: "500Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + limits: + type: object + properties: + cpu: { type: string, title: CPU Limit, default: "1000m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Limit, default: "4000Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + querier: + type: object + title: Querier Resources + properties: + resources: + type: object + properties: + requests: + type: object + properties: + cpu: { type: string, title: CPU Request, default: "300m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Request, default: "300Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + limits: + type: object + properties: + cpu: { type: string, title: CPU Limit, default: "1000m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Limit, default: "4000Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + queryFrontend: + type: object + title: Query Frontend Resources + properties: + resources: + type: object + properties: + requests: + type: object + properties: + cpu: { type: string, title: CPU Request, default: "300m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Request, default: "300Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + limits: + type: object + properties: + cpu: { type: string, title: CPU Limit, default: "1000m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Limit, default: "1000Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + compactor: + type: object + title: Compactor Resources + properties: + resources: + type: object + properties: + requests: + type: object + properties: + cpu: { type: string, title: CPU Request, default: "300m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Request, default: "300Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + limits: + type: object + properties: + cpu: { type: string, title: CPU Limit, default: "1000m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Limit, default: "1000Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + indexGateway: + type: object + title: Index Gateway Resources + properties: + resources: + type: object + properties: + requests: + type: object + properties: + cpu: { type: string, title: CPU Request, default: "300m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Request, default: "300Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + limits: + type: object + properties: + cpu: { type: string, title: CPU Limit, default: "1000m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Limit, default: "1000Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + gateway: + type: object + title: Gateway Resources + properties: + resources: + type: object + properties: + requests: + type: object + properties: + cpu: { type: string, title: CPU Request, default: "300m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Request, default: "300Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + limits: + type: object + properties: + cpu: { type: string, title: CPU Limit, default: "1000m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Limit, default: "1000Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + queryScheduler: + type: object + title: Query Scheduler Resources + properties: + resources: + type: object + properties: + requests: + type: object + properties: + cpu: { type: string, title: CPU Request, default: "100m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Request, default: "300Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + limits: + type: object + properties: + cpu: { type: string, title: CPU Limit, default: "500m", pattern: '^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$' } + memory: { type: string, title: Memory Limit, default: "1000Mi", pattern: '^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$' } + chunks_cache_memory_mb: + type: integer + title: Chunks Cache Memory (MB) + description: Memory allocated to the Loki chunks cache (memcached). Chart default is 9830MB — set lower if cluster resources are limited. + default: 1024 + values: + type: object + title: Additional Helm Values + description: Extra values deep-merged into the grafana/loki Helm release + x-ui-yaml-editor: true + + promtail: + type: object + title: Promtail Configuration + properties: + enabled: + type: boolean + title: Enable Promtail + description: Deploy Promtail DaemonSet for pod log collection + default: true + chart_version: + type: string + title: Chart Version + description: Helm chart version for promtail + default: "6.17.1" + x-ui-visible-if: + field: spec.promtail.enabled + values: + - true + size: + type: object + title: Resource Sizing + x-ui-visible-if: + field: spec.promtail.enabled + values: + - true + properties: + resources: + type: object + title: Resources + properties: + requests: + type: object + title: Requests + properties: + cpu: + type: string + title: CPU Request + default: "100m" + pattern: ^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$ + memory: + type: string + title: Memory Request + default: "100Mi" + pattern: ^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$ + limits: + type: object + title: Limits + properties: + cpu: + type: string + title: CPU Limit + default: "1000m" + pattern: ^([1-9]|[12][0-9]|3[0-2])$|^(3[0-1][0-9]{3}|[1-2][0-9]{4}|[1-9][0-9]{0,3}|32000)m$ + memory: + type: string + title: Memory Limit + default: "1000Mi" + pattern: ^([1-9]|[1-5][0-9]|6[0-4])Gi$|^([1-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-3][0-9]{3}|64000)Mi$ + values: + type: object + title: Additional Helm Values + description: Extra values deep-merged into the promtail Helm release + x-ui-yaml-editor: true + x-ui-visible-if: + field: spec.promtail.enabled + values: + - true + + loki_canary: + type: object + title: Loki Canary + properties: + enabled: + type: boolean + title: Enable Loki Canary + description: Deploy Loki Canary for end-to-end write/read latency testing (built into unified grafana/loki chart) + default: false + + x-ui-order: + - metadata + - loki + - promtail + - loki_canary + +sample: + $schema: https://facets-cloud.github.io/facets-schemas/schemas/log_collector/log_collector.schema.json + kind: log_collector + flavor: loki_gcs + version: "1.0" + disabled: true + spec: + metadata: + namespace: facets + loki: + chart_version: "6.55.0" + deployment_mode: SingleBinary + retention_hours: 720 + chunks_cache_memory_mb: 1024 + single_binary_size: + resources: + requests: { cpu: "1000m", memory: "2Gi" } + limits: { cpu: "2000m", memory: "4Gi" } + persistence_size: "20Gi" + values: {} + promtail: + enabled: true + chart_version: "6.17.1" + size: + resources: + requests: + cpu: "100m" + memory: "100Mi" + limits: + cpu: "1000m" + memory: "1000Mi" + values: {} + loki_canary: + enabled: false diff --git a/modules/log_collector/loki_gcs/1.0/locals.tf b/modules/log_collector/loki_gcs/1.0/locals.tf new file mode 100644 index 00000000..9a9045ed --- /dev/null +++ b/modules/log_collector/loki_gcs/1.0/locals.tf @@ -0,0 +1,474 @@ +locals { + spec = lookup(var.instance, "spec", {}) + metadata = lookup(local.spec, "metadata", {}) + + # ── Namespace ───────────────────────────────────────────────────────────── + namespace = lookup(local.metadata, "namespace", "facets") + + # ── GCP / GCS from inputs ───────────────────────────────────────────────── + project = var.inputs.cloud_account.attributes.project_id + bucket_name = var.inputs.storage.attributes.bucket_name + + # ── Loki spec ───────────────────────────────────────────────────────────── + loki_spec = lookup(local.spec, "loki", {}) + loki_chart_version = lookup(local.loki_spec, "chart_version", "6.55.0") + loki_deployment_mode = lookup(local.loki_spec, "deployment_mode", "SingleBinary") + loki_retention_hours = lookup(local.loki_spec, "retention_hours", 720) + loki_chunks_cache_memory = lookup(local.loki_spec, "chunks_cache_memory_mb", 1024) + loki_size = lookup(local.loki_spec, "size", {}) + + # Per-component resource lookups + loki_distributor_resources = lookup(lookup(local.loki_size, "distributor", {}), "resources", {}) + loki_distributor_requests = lookup(local.loki_distributor_resources, "requests", {}) + loki_distributor_limits = lookup(local.loki_distributor_resources, "limits", {}) + + loki_ingester_resources = lookup(lookup(local.loki_size, "ingester", {}), "resources", {}) + loki_ingester_requests = lookup(local.loki_ingester_resources, "requests", {}) + loki_ingester_limits = lookup(local.loki_ingester_resources, "limits", {}) + + loki_querier_resources = lookup(lookup(local.loki_size, "querier", {}), "resources", {}) + loki_querier_requests = lookup(local.loki_querier_resources, "requests", {}) + loki_querier_limits = lookup(local.loki_querier_resources, "limits", {}) + + loki_query_frontend_resources = lookup(lookup(local.loki_size, "queryFrontend", {}), "resources", {}) + loki_query_frontend_requests = lookup(local.loki_query_frontend_resources, "requests", {}) + loki_query_frontend_limits = lookup(local.loki_query_frontend_resources, "limits", {}) + + loki_compactor_resources = lookup(lookup(local.loki_size, "compactor", {}), "resources", {}) + loki_compactor_requests = lookup(local.loki_compactor_resources, "requests", {}) + loki_compactor_limits = lookup(local.loki_compactor_resources, "limits", {}) + + loki_index_gateway_resources = lookup(lookup(local.loki_size, "indexGateway", {}), "resources", {}) + loki_index_gateway_requests = lookup(local.loki_index_gateway_resources, "requests", {}) + loki_index_gateway_limits = lookup(local.loki_index_gateway_resources, "limits", {}) + + loki_gateway_resources = lookup(lookup(local.loki_size, "gateway", {}), "resources", {}) + loki_gateway_requests = lookup(local.loki_gateway_resources, "requests", {}) + loki_gateway_limits = lookup(local.loki_gateway_resources, "limits", {}) + + loki_query_scheduler_resources = lookup(lookup(local.loki_size, "queryScheduler", {}), "resources", {}) + loki_query_scheduler_requests = lookup(local.loki_query_scheduler_resources, "requests", {}) + loki_query_scheduler_limits = lookup(local.loki_query_scheduler_resources, "limits", {}) + loki_user_values = lookup(local.loki_spec, "values", {}) + + # SingleBinary + is_single_binary = local.loki_deployment_mode == "SingleBinary" + loki_sb_size = lookup(local.loki_spec, "single_binary_size", {}) + loki_sb_resources = lookup(local.loki_sb_size, "resources", {}) + loki_sb_requests = lookup(local.loki_sb_resources, "requests", {}) + loki_sb_limits = lookup(local.loki_sb_resources, "limits", {}) + loki_sb_persistence = lookup(local.loki_sb_size, "persistence_size", "20Gi") + + # ── Promtail spec ───────────────────────────────────────────────────────── + promtail_spec = lookup(local.spec, "promtail", {}) + promtail_enabled = lookup(local.promtail_spec, "enabled", true) + promtail_chart_version = lookup(local.promtail_spec, "chart_version", "6.17.1") + promtail_size = lookup(local.promtail_spec, "size", {}) + promtail_resources = lookup(local.promtail_size, "resources", {}) + promtail_requests = lookup(local.promtail_resources, "requests", {}) + promtail_limits = lookup(local.promtail_resources, "limits", {}) + promtail_user_values = lookup(local.promtail_spec, "values", {}) + + # ── Loki Canary spec — now controlled via unified chart lokiCanary.enabled ── + loki_canary_spec = lookup(local.spec, "loki_canary", {}) + loki_canary_enabled = lookup(local.loki_canary_spec, "enabled", false) + + # ── Node placement from node pool input ─────────────────────────────────── + nodepool_taints = try(var.inputs.kubernetes_node_pool_details.attributes.taints, []) + node_selector = try(var.inputs.kubernetes_node_pool_details.attributes.node_selector, {}) + + # Build explicit tolerations with operator=Equal from nodepool taints + tolerations = [for t in local.nodepool_taints : { + key = t.key + operator = "Equal" + value = t.value + effect = t.effect + }] + + # ── Service account name (max 28 chars for GCP) ─────────────────────────── + env_name = lookup(var.environment, "name", "default") + sa_name = substr(replace("loki-${var.instance_name}-${local.env_name}", "_", "-"), 0, 28) + + # ── Loki gateway endpoint (internal k8s service) ────────────────────────── + # SingleBinary: direct to loki service on port 3100 + # Distributed: via gateway service on port 80 + loki_endpoint = local.is_single_binary ? "${var.instance_name}.${local.namespace}.svc.cluster.local:3100" : "${var.instance_name}-gateway.${local.namespace}.svc.cluster.local" + + # ── Default Loki Helm values (unified grafana/loki chart, GCS backend) ──── + default_loki_values = { + deploymentMode = local.loki_deployment_mode + + # Workload Identity annotation at root serviceAccount level + serviceAccount = { + create = true + annotations = { + "iam.gke.io/gcp-service-account" = google_service_account.loki_gcs.email + } + } + + loki = { + auth_enabled = false + + podLabels = { + resourceName = var.instance_name + resourceType = "log_collector" + } + + # GCS storage backend + storage = { + type = "gcs" + bucketNames = { + chunks = local.bucket_name + ruler = local.bucket_name + admin = local.bucket_name + } + gcs = { + chunkBufferSize = 0 + requestTimeout = "0s" + enableHttp2 = true + } + } + + # Modern v13/tsdb schema (Loki 3.x) + schemaConfig = { + configs = [ + { + from = "2024-04-01" + store = "tsdb" + object_store = "gcs" + schema = "v13" + index = { + prefix = "index_" + period = "24h" + } + } + ] + } + + # Ingester chunking + ingester = { + chunk_target_size = 1572864 + chunk_encoding = "snappy" + max_chunk_age = "2h" + chunk_idle_period = "2h" + autoforget_unhealthy = true + } + + ingester_client = { + grpc_client_config = { + max_recv_msg_size = 104857600 + } + } + + # Server tuning + server = { + grpc_server_max_concurrent_streams = 1000 + grpc_server_max_recv_msg_size = 41943040 + grpc_server_max_send_msg_size = 41943040 + http_server_read_timeout = "310s" + http_server_write_timeout = "310s" + http_server_idle_timeout = "300s" + graceful_shutdown_timeout = "300s" + } + + # Limits + limits_config = { + reject_old_samples = true + reject_old_samples_max_age = "168h" + max_cache_freshness_per_query = "10m" + split_queries_by_interval = "15m" + query_timeout = "300s" + max_global_streams_per_user = 5000 + max_query_parallelism = 32 + volume_enabled = true + retention_period = "${local.loki_retention_hours}h" + } + + # Compactor config + compactor = { + working_directory = local.is_single_binary ? "/var/loki/compactor" : "/data/compactor" + compaction_interval = "10m" + retention_enabled = true + delete_request_store = "gcs" + } + + commonConfig = { + path_prefix = "/var/loki" + replication_factor = local.is_single_binary ? 1 : 3 + } + } + + # In SingleBinary mode, memberlist is not needed (single replica) + memberlist = local.is_single_binary ? { join_members = [] } : {} + + # ── SingleBinary sizing & placement ───────────────────────────────────── + singleBinary = { + replicas = local.is_single_binary ? 1 : 0 + tolerations = local.tolerations + nodeSelector = local.node_selector + persistence = { + enabled = true + size = local.loki_sb_persistence + } + resources = { + requests = { + cpu = lookup(local.loki_sb_requests, "cpu", "1000m") + memory = lookup(local.loki_sb_requests, "memory", "2Gi") + } + limits = { + cpu = lookup(local.loki_sb_limits, "cpu", "2000m") + memory = lookup(local.loki_sb_limits, "memory", "4Gi") + } + } + } + + # ── Distributed component sizing & placement ──────────────────────────── + compactor = { + enabled = true + tolerations = local.tolerations + nodeSelector = local.node_selector + resources = { + requests = { + memory = lookup(local.loki_compactor_requests, "memory", "300Mi") + cpu = lookup(local.loki_compactor_requests, "cpu", "300m") + } + limits = { + memory = lookup(local.loki_compactor_limits, "memory", "1000Mi") + cpu = lookup(local.loki_compactor_limits, "cpu", "1000m") + } + } + } + + distributor = { + replicas = local.is_single_binary ? 0 : 1 + tolerations = local.tolerations + nodeSelector = local.node_selector + resources = { + requests = { + memory = lookup(local.loki_distributor_requests, "memory", "300Mi") + cpu = lookup(local.loki_distributor_requests, "cpu", "300m") + } + limits = { + memory = lookup(local.loki_distributor_limits, "memory", "1000Mi") + cpu = lookup(local.loki_distributor_limits, "cpu", "1000m") + } + } + autoscaling = { + enabled = !local.is_single_binary + minReplicas = 1 + maxReplicas = 5 + targetCPUUtilizationPercentage = 60 + targetMemoryUtilizationPercentage = 80 + } + } + + ingester = { + replicas = local.is_single_binary ? 0 : 1 + tolerations = local.tolerations + nodeSelector = local.node_selector + persistence = { + enabled = true + size = "5Gi" + } + resources = { + requests = { + memory = lookup(local.loki_ingester_requests, "memory", "500Mi") + cpu = lookup(local.loki_ingester_requests, "cpu", "300m") + } + limits = { + memory = lookup(local.loki_ingester_limits, "memory", "4000Mi") + cpu = lookup(local.loki_ingester_limits, "cpu", "1000m") + } + } + autoscaling = { + enabled = !local.is_single_binary + minReplicas = 1 + maxReplicas = 3 + targetCPUUtilizationPercentage = 60 + targetMemoryUtilizationPercentage = 80 + } + } + + querier = { + replicas = local.is_single_binary ? 0 : 1 + tolerations = local.tolerations + nodeSelector = local.node_selector + persistence = { + enabled = true + size = "5Gi" + } + resources = { + requests = { + memory = lookup(local.loki_querier_requests, "memory", "300Mi") + cpu = lookup(local.loki_querier_requests, "cpu", "300m") + } + limits = { + memory = lookup(local.loki_querier_limits, "memory", "4000Mi") + cpu = lookup(local.loki_querier_limits, "cpu", "1000m") + } + } + autoscaling = { + enabled = !local.is_single_binary + minReplicas = 1 + maxReplicas = 10 + targetCPUUtilizationPercentage = 60 + targetMemoryUtilizationPercentage = 80 + } + } + + queryScheduler = { + replicas = local.is_single_binary ? 0 : 2 + tolerations = local.tolerations + nodeSelector = local.node_selector + resources = { + requests = { + memory = lookup(local.loki_query_scheduler_requests, "memory", "300Mi") + cpu = lookup(local.loki_query_scheduler_requests, "cpu", "100m") + } + limits = { + memory = lookup(local.loki_query_scheduler_limits, "memory", "1000Mi") + cpu = lookup(local.loki_query_scheduler_limits, "cpu", "500m") + } + } + } + + queryFrontend = { + replicas = local.is_single_binary ? 0 : 1 + tolerations = local.tolerations + nodeSelector = local.node_selector + resources = { + requests = { + memory = lookup(local.loki_query_frontend_requests, "memory", "300Mi") + cpu = lookup(local.loki_query_frontend_requests, "cpu", "300m") + } + limits = { + memory = lookup(local.loki_query_frontend_limits, "memory", "1000Mi") + cpu = lookup(local.loki_query_frontend_limits, "cpu", "1000m") + } + } + autoscaling = { + enabled = !local.is_single_binary + minReplicas = 1 + maxReplicas = 5 + targetCPUUtilizationPercentage = 60 + targetMemoryUtilizationPercentage = 80 + } + } + + indexGateway = { + enabled = !local.is_single_binary + replicas = local.is_single_binary ? 0 : 1 + tolerations = local.tolerations + nodeSelector = local.node_selector + resources = { + requests = { + memory = lookup(local.loki_index_gateway_requests, "memory", "300Mi") + cpu = lookup(local.loki_index_gateway_requests, "cpu", "300m") + } + limits = { + memory = lookup(local.loki_index_gateway_limits, "memory", "1000Mi") + cpu = lookup(local.loki_index_gateway_limits, "cpu", "1000m") + } + } + } + + gateway = { + enabled = !local.is_single_binary + tolerations = local.tolerations + nodeSelector = local.node_selector + resources = { + requests = { + memory = lookup(local.loki_gateway_requests, "memory", "300Mi") + cpu = lookup(local.loki_gateway_requests, "cpu", "300m") + } + limits = { + memory = lookup(local.loki_gateway_limits, "memory", "1000Mi") + cpu = lookup(local.loki_gateway_limits, "cpu", "1000m") + } + } + service = { type = "ClusterIP" } + } + + # Chunks cache — chart default requests 9830Mi; configurable via spec.loki.chunks_cache_memory_mb + chunksCache = { + allocatedMemory = local.loki_chunks_cache_memory + tolerations = local.tolerations + nodeSelector = local.node_selector + resources = { + requests = { cpu = "100m", memory = "${local.loki_chunks_cache_memory + 256}Mi" } + limits = { memory = "${local.loki_chunks_cache_memory + 256}Mi" } + } + } + + resultsCache = { + tolerations = local.tolerations + nodeSelector = local.node_selector + } + + # Disable SimpleScalable components in Distributed/SingleBinary modes + write = local.loki_deployment_mode == "SimpleScalable" ? {} : { replicas = 0 } + read = local.loki_deployment_mode == "SimpleScalable" ? {} : { replicas = 0 } + backend = local.loki_deployment_mode == "SimpleScalable" ? {} : { replicas = 0 } + + ruler = { + enabled = false + replicas = 0 + } + + # Canary now built into unified chart + # test.enabled must match lokiCanary.enabled — chart validate.yaml enforces this + lokiCanary = { + enabled = local.loki_canary_enabled + } + + test = { + enabled = local.loki_canary_enabled + } + + # ServiceMonitor moved under monitoring in unified chart + monitoring = { + serviceMonitor = { + enabled = true + interval = "15s" + } + dashboards = { enabled = false } + rules = { enabled = false } + } + } + + # ── Default Promtail Helm values ────────────────────────────────────────── + default_promtail_values = { + config = { + clients = [ + { + url = "http://${local.loki_endpoint}/loki/api/v1/push" + tenant_id = "facets" + } + ] + snippets = { + pipelineStages = [ + { cri = {} } + ] + } + } + podLabels = { + resourceName = var.instance_name + resourceType = "log_collector" + } + nodeSelector = { "kubernetes.io/os" = "linux" } + # Wildcard toleration — promtail is a log collector DaemonSet that must run on every node + # regardless of what taints are present (infra, jobs, keycloak, observability, critical, etc.) + tolerations = [{ operator = "Exists" }] + serviceMonitor = { enabled = true } + resources = { + requests = { + memory = lookup(local.promtail_requests, "memory", "100Mi") + cpu = lookup(local.promtail_requests, "cpu", "100m") + } + limits = { + memory = lookup(local.promtail_limits, "memory", "1000Mi") + cpu = lookup(local.promtail_limits, "cpu", "1000m") + } + } + } +} diff --git a/modules/log_collector/loki_gcs/1.0/main.tf b/modules/log_collector/loki_gcs/1.0/main.tf new file mode 100644 index 00000000..5c3e2462 --- /dev/null +++ b/modules/log_collector/loki_gcs/1.0/main.tf @@ -0,0 +1,120 @@ +# ── GCP Service Account ─────────────────────────────────────────────────────── +resource "google_service_account" "loki_gcs" { + account_id = local.sa_name + project = local.project + display_name = "Loki GCS backend service account" + + lifecycle { + ignore_changes = [account_id] + } +} + +# ── Custom IAM Role — minimal GCS permissions ───────────────────────────────── +resource "google_project_iam_custom_role" "loki_gcs" { + role_id = replace("${var.instance_name}_${local.env_name}_lokiGcsRole", "-", "_") + project = local.project + title = "Loki GCS backend role (${var.instance_name})" + description = "Minimal permissions for Loki to read/write GCS bucket" + permissions = [ + "storage.buckets.get", + "storage.objects.create", + "storage.objects.delete", + "storage.objects.get", + "storage.objects.list", + ] +} + +# ── Bind custom role to the service account at project level ────────────────── +resource "google_project_iam_member" "loki_gcs" { + project = local.project + role = google_project_iam_custom_role.loki_gcs.id + member = "serviceAccount:${google_service_account.loki_gcs.email}" +} + +# ── Bind GCS bucket IAM so the SA can access the bucket ─────────────────────── +resource "google_storage_bucket_iam_binding" "loki_gcs" { + bucket = local.bucket_name + role = google_project_iam_custom_role.loki_gcs.id + members = [ + "serviceAccount:${google_service_account.loki_gcs.email}", + ] +} + +# ── Workload Identity — allow k8s SA to impersonate the GCP SA ──────────────── +# Unified grafana/loki chart creates k8s SA named (the release name itself) +resource "google_service_account_iam_member" "loki_gcs_workload_identity" { + service_account_id = google_service_account.loki_gcs.id + role = "roles/iam.workloadIdentityUser" + member = "serviceAccount:${local.project}.svc.id.goog[${local.namespace}/${var.instance_name}]" +} + +# ── Helm: grafana/loki (unified chart) ──────────────────────────────────────── +resource "helm_release" "loki" { + repository = "https://grafana.github.io/helm-charts" + chart = "loki" + name = var.instance_name + version = local.loki_chart_version + namespace = local.namespace + create_namespace = true + cleanup_on_fail = true + timeout = 600 + wait = true + + values = [ + yamlencode(local.default_loki_values), + yamlencode(local.loki_user_values), + ] +} + +# ── Helm: promtail ──────────────────────────────────────────────────────────── +resource "helm_release" "promtail" { + count = local.promtail_enabled ? 1 : 0 + + depends_on = [helm_release.loki] + repository = "https://grafana.github.io/helm-charts" + chart = "promtail" + name = "${var.instance_name}-promtail" + version = local.promtail_chart_version + namespace = local.namespace + create_namespace = true + cleanup_on_fail = true + timeout = 600 + wait = false + + values = [ + yamlencode(local.default_promtail_values), + yamlencode(local.promtail_user_values), + ] +} + +# ── Grafana datasource ConfigMap (picked up by Grafana sidecar) ─────────────── +resource "kubernetes_config_map_v1" "grafana_loki_datasource" { + depends_on = [helm_release.loki] + + metadata { + name = "${var.instance_name}-loki-datasource" + namespace = local.namespace + labels = { + grafana_datasource = "1" + datasource_name = var.instance_name + } + } + + data = { + "datasource-loki-${var.instance_name}.yaml" = yamlencode({ + apiVersion = 1 + datasources = [ + { + name = "Loki ${var.instance_name}" + type = "loki" + url = "http://${local.loki_endpoint}" + access = "proxy" + isDefault = false + jsonData = { + timeout = 300 + } + } + ] + }) + } +} diff --git a/modules/log_collector/loki_gcs/1.0/outputs.tf b/modules/log_collector/loki_gcs/1.0/outputs.tf new file mode 100644 index 00000000..209b5c5d --- /dev/null +++ b/modules/log_collector/loki_gcs/1.0/outputs.tf @@ -0,0 +1,14 @@ +locals { + output_attributes = { + loki_url = "http://${local.loki_endpoint}" + namespace = local.namespace + service_account = google_service_account.loki_gcs.email + bucket_name = local.bucket_name + } + + output_interfaces = { + default = { + url = "http://${local.loki_endpoint}" + } + } +} diff --git a/modules/log_collector/loki_gcs/1.0/outputs.yaml b/modules/log_collector/loki_gcs/1.0/outputs.yaml new file mode 100644 index 00000000..2ca3442d --- /dev/null +++ b/modules/log_collector/loki_gcs/1.0/outputs.yaml @@ -0,0 +1,23 @@ +properties: + type: object + properties: + attributes: + type: object + properties: + loki_url: + type: string + namespace: + type: string + service_account: + type: string + bucket_name: + type: string + interfaces: + type: object + properties: + default: + type: object + properties: + url: + type: string +providers: [] diff --git a/modules/log_collector/loki_gcs/1.0/variables.tf b/modules/log_collector/loki_gcs/1.0/variables.tf new file mode 100644 index 00000000..f73d6d15 --- /dev/null +++ b/modules/log_collector/loki_gcs/1.0/variables.tf @@ -0,0 +1,40 @@ +variable "instance" { + description = "The full Facets resource instance object (kind, flavor, version, spec)" + type = any +} + +variable "instance_name" { + description = "Unique architectural name from the blueprint" + type = string +} + +variable "environment" { + description = "Facets environment object" + type = any +} + +variable "inputs" { + description = "Typed input dependencies from other Facets resources" + type = object({ + kubernetes_details = object({ + attributes = any + }) + cloud_account = object({ + attributes = object({ + project_id = string + }) + }) + storage = object({ + attributes = object({ + bucket_name = string + }) + }) + kubernetes_node_pool_details = optional(object({ + attributes = optional(object({ + taints = optional(list(any), []) + node_selector = optional(map(string), {}) + }), {}) + interfaces = optional(object({}), {}) + })) + }) +} diff --git a/outputs/loki/outputs.yaml b/outputs/loki/outputs.yaml new file mode 100644 index 00000000..78f4517d --- /dev/null +++ b/outputs/loki/outputs.yaml @@ -0,0 +1,24 @@ +name: '@facets/loki' +properties: + type: object + properties: + attributes: + type: object + properties: + loki_url: + type: string + namespace: + type: string + service_account: + type: string + bucket_name: + type: string + interfaces: + type: object + properties: + default: + type: object + properties: + url: + type: string +providers: [] diff --git a/project-type/gcp/project-type.yml b/project-type/gcp/project-type.yml index 80fc0e26..5c4eb9a2 100644 --- a/project-type/gcp/project-type.yml +++ b/project-type/gcp/project-type.yml @@ -99,6 +99,8 @@ modules: flavor: mongo - intent: prometheus flavor: k8s_standard + - intent: log_collector + flavor: loki_gcs - intent: strimzi-operator flavor: helm - intent: vpa