Merge pull request #23 from datafold/gerard-eng-2673-reduce-clickhouse-backup-retention-to-7-days

fix: Update clickhouse backup retention

Author: gtoonstra

README.md

@@ -261,6 +261,7 @@ https://docs.datafold.com/datafold-deployment/dedicated-cloud/azure
 | <a name="input_azure_bastion_subnet_cidrs"></a> [azure\_bastion\_subnet\_cidrs](#input\_azure\_bastion\_subnet\_cidrs) | The CIDR block for the Azure Bastion subnet. If empty it will be calculated from the VPC CIDR and given size. | `list(string)` | `[]` | no |
 | <a name="input_azure_bastion_subnet_name_override"></a> [azure\_bastion\_subnet\_name\_override](#input\_azure\_bastion\_subnet\_name\_override) | Override for the name used in resource.azurerm\_subnet.azure\_bastion\_subnet (modules/networking) | `string` | `""` | no |
 | <a name="input_azure_bastion_subnet_size"></a> [azure\_bastion\_subnet\_size](#input\_azure\_bastion\_subnet\_size) | The size of the Azure Bastion subnet in number of IPs | `number` | `256` | no |
+| <a name="input_backup_lifecycle_expiration_days"></a> [backup\_lifecycle\_expiration\_days](#input\_backup\_lifecycle\_expiration\_days) | Number of days after which clickhouse backup objects will expire and be deleted. | `number` | `6` | no |
 | <a name="input_bastion_host_name_override"></a> [bastion\_host\_name\_override](#input\_bastion\_host\_name\_override) | Override for the name used in resource.azurerm\_bastion\_host.bastion (modules/networking) | `string` | `""` | no |
 | <a name="input_bastion_public_ip_name_override"></a> [bastion\_public\_ip\_name\_override](#input\_bastion\_public\_ip\_name\_override) | Override for the name used in resource.azurerm\_public\_ip.ip\_bastion\_host (modules/networking) | `string` | `""` | no |
 | <a name="input_ch_data_disk_iops"></a> [ch\_data\_disk\_iops](#input\_ch\_data\_disk\_iops) | IOPS of volume | `number` | `3000` | no |

main.tf

@@ -233,11 +233,12 @@ module "clickhouse_backup" {
   identity                        = module.identity.identity
 
   # Resource name overrides
-  storage_account_name_override           = var.storage_account_name_override
+  storage_account_name_override             = var.storage_account_name_override
   clickhouse_backup_container_name_override = var.clickhouse_backup_container_name_override
-  storage_private_dns_zone_name_override = var.storage_private_dns_zone_name_override
-  storage_private_endpoint_name_override  = var.storage_private_endpoint_name_override
-  storage_dns_link_name_override          = var.storage_dns_link_name_override
+  storage_private_dns_zone_name_override    = var.storage_private_dns_zone_name_override
+  storage_private_endpoint_name_override    = var.storage_private_endpoint_name_override
+  storage_dns_link_name_override            = var.storage_dns_link_name_override
+  backup_lifecycle_expiration_days          = var.backup_lifecycle_expiration_days
 }
 
 module "data_lake" {

modules/clickhouse_backup/main.tf

@@ -26,6 +26,24 @@ resource "azurerm_storage_container" "clickhouse_backup" {
   container_access_type = "private"
 }
 
+resource "azurerm_storage_management_policy" "clickhouse_backup" {
+  storage_account_id = azurerm_storage_account.storage.id
+
+  rule {
+    name    = "backup_retention"
+    enabled = true
+    filters {
+      blob_types   = ["blockBlob"]
+      prefix_match = [local.clickhouse_backup_container_name]
+    }
+    actions {
+      base_blob {
+        delete_after_days_since_modification_greater_than = var.backup_lifecycle_expiration_days
+      }
+    }
+  }
+}
+
 # ============PrivateLink for Storage Account====================
 resource "azurerm_private_dns_zone" "storage_account_dns" {
   name                = local.storage_private_dns_zone_name

modules/clickhouse_backup/variables.tf

@@ -66,4 +66,10 @@ variable "storage_dns_link_name_override" {
   description = "Override for the name used in resource.azurerm_private_dns_zone_virtual_network_link.storage_account_link"
   type        = string
   default     = ""
+}
+
+variable "backup_lifecycle_expiration_days" {
+  type        = number
+  default     = 6
+  description = "Number of days after which clickhouse backup objects will expire and be deleted."
 }

variables.tf

@@ -579,6 +579,12 @@ variable "storage_dns_link_name_override" {
   default     = ""
 }
 
+variable "backup_lifecycle_expiration_days" {
+  type        = number
+  default     = 6
+  description = "Number of days after which clickhouse backup objects will expire and be deleted."
+}
+
 # Key Vault Module Overrides
 variable "key_vault_name_override" {
   description = "Override for the name used in resource.azurerm_key_vault.default (modules/key_vault)"