Adding Azure blob support

Author: ptasker

README.md

@@ -112,7 +112,66 @@ To view a dump of the LMDB contents:
 You can browse the snapshots in MinIO at <http://localhost:4731/buckets/lightningstream/browse>
 (login with minioadmin / minioadmin).
 
+## Using Azure Blob storage
 
+Lightning Stream supports Azure Blob Storage as a backend for storing snapshots. You can configure it to use either static credentials (account name and key) or Azure service principal authentication.
+
+### Basic configuration with static credentials
+
+```yaml
+storage:
+  type: azure
+  options:
+    account_name: myaccountname
+    account_key: myaccountkey
+    container: lightningstream
+    create_container: true
+```
+
+### Configuration with Azure service principal (recommended for production)
+
+When using service principal authentication, set `use_env_creds: true` and ensure the following environment variables are set:
+- `AZURE_CLIENT_ID`
+- `AZURE_TENANT_ID`
+- `AZURE_CLIENT_SECRET`
+
+```yaml
+storage:
+  type: azure
+  options:
+    use_env_creds: true
+    container: lightningstream
+    endpoint_url: https://myaccount.blob.core.windows.net/
+    create_container: true
+```
+
+### Available options
+
+| Option | Type | Summary |
+|--------|------|---------|
+| account_name | string | Azure storage account name (required if not using `use_env_creds`) |
+| account_key | string | Azure storage account key (required if not using `use_env_creds`) |
+| use_env_creds | bool | Use Azure service principal authentication via environment variables |
+| container | string | Azure blob container name (required) |
+| create_container | bool | Create container if it does not exist |
+| endpoint_url | string | Custom endpoint URL (defaults to `https://<account_name>.blob.core.windows.net/`) |
+| global_prefix | string | Transparently apply a global prefix to all blob names |
+| disable_send_content_md5 | bool | Disable sending the Content-MD5 header |
+| tls | [tlsconfig.Config](https://github.com/PowerDNS/go-tlsconfig) | TLS configuration |
+| init_timeout | duration | Time allowed for initialisation (default: "20s") |
+| use_update_marker | bool | Reduce LIST commands by using an update marker (see below) |
+| update_marker_force_list_interval | duration | Force full LIST sync at this interval (default: "5m") |
+| concurrency | int | Max number of concurrent uploads (default: 1) |
+
+The `use_update_marker` option can reduce Azure costs by replacing LIST operations (which are more expensive) with GET operations. However, it cannot be used if the container itself is replicated in an active-active fashion between data centers.
+
+You can see a working example in the docker-compose setup, which uses [Azurite](https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite) (Azure Storage Emulator):
+
+```bash
+docker-compose up
+```
+
+For all available Azure backend options with full descriptions, see [Simpleblob's Azure backend Options struct](https://github.com/PowerDNS/simpleblob/blob/main/backends/azure/azure.go).
 
 ## Open Source
 
@@ -123,5 +182,3 @@ For more information on how we provide support for Open Source products, please
 PowerDNS also offers an Enterprise edition of Lightning Stream that includes professional support, advanced features, deployment
 tooling for large deployments, Kubernetes integration, and more.
 
-
-

cmd/lightningstream/main.go

@@ -4,6 +4,7 @@ import (
 	"github.com/PowerDNS/lightningstream/cmd/lightningstream/commands"
 
 	// Register storage backends
+	_ "github.com/PowerDNS/simpleblob/backends/azure"
 	_ "github.com/PowerDNS/simpleblob/backends/fs"
 	_ "github.com/PowerDNS/simpleblob/backends/memory"
 	_ "github.com/PowerDNS/simpleblob/backends/s3"

config/config.go

@@ -302,7 +302,7 @@ type DBIOptions struct {
 }
 
 type Storage struct {
-	Type    string                 `yaml:"type"`    // "fs", "s3", "memory"
+	Type    string                 `yaml:"type"`    // "fs", "s3", "memory", "azure"
 	Options map[string]interface{} `yaml:"options"` // backend specific
 
 	// FIXME: Configure per LMDB instead, since we run a cleaner per LMDB?

docker-compose.yml

@@ -6,6 +6,7 @@ version: '2.4'
 volumes:
     lmdb:
         driver: local
+    azurite_data:
     minio:
         driver: local
     #snapshots:
@@ -21,7 +22,22 @@ services:
         command: server /data --console-address :9001
         volumes:
             - "minio:/data" 
-
+    # https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite
+    azurite:
+        image: mcr.microsoft.com/azure-storage/azurite
+        container_name: azurite
+        hostname: azurite
+        restart: always
+        ports:
+            - "10000:10000"
+            - "10001:10001"
+            - "10002:10002" 
+        command: azurite --blobHost 0.0.0.0  --queueHost 0.0.0.0 --tableHost 0.0.0.0
+        volumes:    
+        - azurite_data:/data
+        - ./azure/certs:/certs
+        environment:
+         - AZURITE_ACCOUNTS=devstoreaccount1:key1
     auth1:
         image: powerdns/pdns-auth-49
         environment:
@@ -78,7 +94,7 @@ services:
             dockerfile: Dockerfile
             context: .
         environment:
-            instance: 1  # used in config file
+            - instance=1  # used in config file
         ports:
             - "${DEVENV_BIND_IP:-127.0.0.1}:${PORT_PREFIX:-47}91:8500"
         volumes:
@@ -94,7 +110,7 @@ services:
             dockerfile: Dockerfile
             context: .
         environment:
-            instance: 2  # used in config file
+            - instance=2  # used in config file
         ports:
             - "${DEVENV_BIND_IP:-127.0.0.1}:${PORT_PREFIX:-47}92:8500"
         volumes:
@@ -110,13 +126,12 @@ services:
             dockerfile: Dockerfile
             context: .
         environment:
-            instance: 3  # used in config file
+            - instance=3  # used in config file
         ports:
             - "${DEVENV_BIND_IP:-127.0.0.1}:${PORT_PREFIX:-47}93:8500"
         volumes:
             - "lmdb:/lmdb" 
             - "./docker/pdns/lightningstream.yaml:/lightningstream.yaml:ro"
-            #- "snapshots:/snapshots" 
         working_dir: /
         user: "953"  # pdns
         command: --minimum-pid 50 receive

docker/pdns/lightningstream.yaml

@@ -40,6 +40,19 @@ storage:
     interval: 15m
     must_keep_interval: 24h
     remove_old_instances_interval: 168h
+  # type: azure
+  # options:
+  #   account_name: devstoreaccount1
+  #   account_key: key1
+  #   container: lightningstreamcontainer
+  #   endpoint_url: http://azurite:10000/devstoreaccount1
+  #   create_container: true
+  #   use_update_marker: false
+  # cleanup:
+  #   enabled: true
+  #   interval: 2m
+  #   must_keep_interval: 24h
+  #   remove_old_instances_interval: 168h
 
 http:
   address: ":8500"

go.mod

@@ -1,29 +1,36 @@
 module github.com/PowerDNS/lightningstream
 
-go 1.24
+go 1.24.0
+
+toolchain go1.24.5
 
 require (
 	github.com/CrowdStrike/csproto v0.23.1
 	github.com/PowerDNS/lmdb-go v1.9.3
-	github.com/PowerDNS/simpleblob v0.3.0
+	github.com/PowerDNS/simpleblob v0.4.0
 	github.com/bufbuild/buf v0.56.0
 	github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
 	github.com/gogo/protobuf v1.3.2
-	github.com/klauspost/compress v1.17.11
+	github.com/klauspost/compress v1.18.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.20.4
+	github.com/prometheus/client_golang v1.23.0
 	github.com/samber/lo v1.37.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.9.1
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.11.1
 	github.com/wojas/go-healthz v0.2.0
 	go.uber.org/atomic v1.10.0
 	golang.org/x/exp v0.0.0-20230111222715-75897c7a292a
-	golang.org/x/sync v0.15.0
+	golang.org/x/sync v0.16.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 // indirect
 	github.com/PowerDNS/go-tlsconfig v0.0.0-20221101135152-0956853b28df // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
@@ -32,39 +39,45 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
-	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gofrs/uuid v4.0.0+incompatible // indirect
+	github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jhump/protoreflect v1.9.1-0.20210817181203-db1a327a393e // indirect
-	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.11 // indirect
 	github.com/klauspost/pgzip v1.2.5 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/minio/crc64nvme v1.0.2 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/minio-go/v7 v7.0.78 // indirect
+	github.com/minio/minio-go/v7 v7.0.95 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/philhofer/fwd v1.2.0 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/profile v1.6.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.55.0 // indirect
-	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.65.0 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/rs/xid v1.6.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/tinylib/msgp v1.3.0 // indirect
 	github.com/twitchtv/twirp v8.1.0+incompatible // indirect
-	go.opencensus.io v0.23.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.19.0 // indirect
-	golang.org/x/crypto v0.39.0 // indirect
-	golang.org/x/net v0.41.0 // indirect
-	golang.org/x/sys v0.33.0 // indirect
-	golang.org/x/term v0.32.0 // indirect
-	golang.org/x/text v0.26.0 // indirect
+	golang.org/x/crypto v0.41.0 // indirect
+	golang.org/x/net v0.43.0 // indirect
+	golang.org/x/sys v0.35.0 // indirect
+	golang.org/x/term v0.34.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
 	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
 	google.golang.org/grpc v1.43.0 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	google.golang.org/protobuf v1.36.6 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-)
+)