feat: add remote repository to handle DockerHub rate limits (#1266)

Author: dobrac

iac/provider-gcp/Makefile

@@ -62,7 +62,8 @@ tf_vars := 	TF_VAR_environment=$(TERRAFORM_ENVIRONMENT) \
 	$(call tfvar, FILESTORE_MAX_DISK_USAGE_TARGET) \
 	$(call tfvar, BUILD_CLUSTER_CACHE_DISK_TYPE) \
 	$(call tfvar, CLIENT_CLUSTER_CACHE_DISK_TYPE) \
-	$(call tfvar, MIN_CPU_PLATFORM)
+	$(call tfvar, MIN_CPU_PLATFORM) \
+	$(call tfvar, REMOTE_REPOSITORY_ENABLED)
 
 .PHONY: init
 init:

iac/provider-gcp/main.tf

@@ -232,11 +232,12 @@ module "nomad" {
   envd_timeout                = var.envd_timeout
 
   # Template manager
-  builder_node_pool              = var.build_node_pool
-  template_manager_port          = var.template_manager_port
-  template_bucket_name           = module.init.fc_template_bucket_name
-  build_cache_bucket_name        = module.init.fc_build_cache_bucket_name
-  template_manager_machine_count = var.build_cluster_size
+  builder_node_pool               = var.build_node_pool
+  template_manager_port           = var.template_manager_port
+  template_bucket_name            = module.init.fc_template_bucket_name
+  build_cache_bucket_name         = module.init.fc_build_cache_bucket_name
+  template_manager_machine_count  = var.build_cluster_size
+  dockerhub_remote_repository_url = var.remote_repository_enabled ? module.remote_repository[0].dockerhub_remote_repository_url : ""
 
   # Redis
   redis_managed = var.redis_managed
@@ -259,3 +260,16 @@ module "redis" {
 
   prefix = var.prefix
 }
+
+module "remote_repository" {
+  source = "./remote-repository"
+
+  count = var.remote_repository_enabled ? 1 : 0
+
+  prefix = var.prefix
+
+  gcp_project_id = var.gcp_project_id
+  gcp_region     = var.gcp_region
+
+  google_service_account_email = module.init.service_account_email
+}

iac/provider-gcp/nomad/jobs/template-manager.hcl

@@ -76,6 +76,7 @@ job "template-manager-system" {
         ALLOW_SANDBOX_INTERNET        = "${allow_sandbox_internet}"
         SHARED_CHUNK_CACHE_PATH       = "${shared_chunk_cache_path}"
         CLICKHOUSE_CONNECTION_STRING  = "${clickhouse_connection_string}"
+        DOCKERHUB_REMOTE_REPOSITORY_URL  = "${dockerhub_remote_repository_url}"
 %{ if !update_stanza }
         FORCE_STOP                    = "true"
 %{ endif }

iac/provider-gcp/nomad/main.tf

@@ -494,19 +494,20 @@ resource "nomad_job" "template_manager" {
     environment      = var.environment
     consul_acl_token = var.consul_acl_token_secret
 
-    api_secret                   = var.api_secret
-    bucket_name                  = var.fc_env_pipeline_bucket_name
-    docker_registry              = var.custom_envs_repository_name
-    google_service_account_key   = var.google_service_account_key
-    template_manager_checksum    = data.external.template_manager.result.hex
-    otel_tracing_print           = var.otel_tracing_print
-    template_bucket_name         = var.template_bucket_name
-    build_cache_bucket_name      = var.build_cache_bucket_name
-    otel_collector_grpc_endpoint = "localhost:${var.otel_collector_grpc_port}"
-    logs_collector_address       = "http://localhost:${var.logs_proxy_port.port}"
-    orchestrator_services        = "template-manager"
-    allow_sandbox_internet       = var.allow_sandbox_internet
-    clickhouse_connection_string = local.clickhouse_connection_string
+    api_secret                      = var.api_secret
+    bucket_name                     = var.fc_env_pipeline_bucket_name
+    docker_registry                 = var.custom_envs_repository_name
+    google_service_account_key      = var.google_service_account_key
+    template_manager_checksum       = data.external.template_manager.result.hex
+    otel_tracing_print              = var.otel_tracing_print
+    template_bucket_name            = var.template_bucket_name
+    build_cache_bucket_name         = var.build_cache_bucket_name
+    otel_collector_grpc_endpoint    = "localhost:${var.otel_collector_grpc_port}"
+    logs_collector_address          = "http://localhost:${var.logs_proxy_port.port}"
+    orchestrator_services           = "template-manager"
+    allow_sandbox_internet          = var.allow_sandbox_internet
+    clickhouse_connection_string    = local.clickhouse_connection_string
+    dockerhub_remote_repository_url = var.dockerhub_remote_repository_url
 
     # For now we DISABLE the shared chunk cache in the template manager
     shared_chunk_cache_path = ""

iac/provider-gcp/nomad/variables.tf

@@ -340,3 +340,7 @@ variable "filestore_cache_max_disk_usage_target" {
   type        = number
   description = "The maximum disk usage target for the Filestore cache in percent"
 }
+
+variable "dockerhub_remote_repository_url" {
+  type = string
+}

iac/provider-gcp/remote-repository/main.tf

@@ -0,0 +1,27 @@
+resource "google_artifact_registry_repository" "dockerhub_remote_repository" {
+  location      = var.gcp_region
+  repository_id = "${var.prefix}docker-remote-repository"
+  description   = "remote docker repository"
+  format        = "DOCKER"
+  mode          = "REMOTE_REPOSITORY"
+  remote_repository_config {
+    description = "Docker Hub"
+    docker_repository {
+      public_repository = "DOCKER_HUB"
+    }
+  }
+
+  cleanup_policies {
+    id     = "delete-older-than-90-days"
+    action = "DELETE"
+    condition {
+      older_than = "90d"
+    }
+  }
+}
+
+resource "google_artifact_registry_repository_iam_member" "dockerhub_remote_repository_member" {
+  repository = google_artifact_registry_repository.dockerhub_remote_repository.name
+  role       = "roles/artifactregistry.repoAdmin"
+  member     = "serviceAccount:${var.google_service_account_email}"
+}

iac/provider-gcp/remote-repository/outputs.tf

@@ -0,0 +1,3 @@
+output "dockerhub_remote_repository_url" {
+  value = "${var.gcp_region}-docker.pkg.dev/${var.gcp_project_id}/${google_artifact_registry_repository.dockerhub_remote_repository.name}"
+}

iac/provider-gcp/remote-repository/variables.tf

@@ -0,0 +1,15 @@
+variable "prefix" {
+  type = string
+}
+
+variable "gcp_project_id" {
+  type = string
+}
+
+variable "gcp_region" {
+  type = string
+}
+
+variable "google_service_account_email" {
+  type = string
+}

iac/provider-gcp/variables.tf

@@ -452,3 +452,9 @@ variable "orchestrator_base_hugepages_percentage" {
   type        = number
   default     = 80
 }
+
+variable "remote_repository_enabled" {
+  type        = bool
+  description = "Set to true to enable remote repository cache. Can be set via TF_VAR_remote_repository_enabled or REMOTE_REPOSITORY_ENABLED env var."
+  default     = false
+}

packages/orchestrator/cmd/build-template/main.go

@@ -23,6 +23,7 @@ import (
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/config"
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/metrics"
 	artifactsregistry "github.com/e2b-dev/infra/packages/shared/pkg/artifacts-registry"
+	"github.com/e2b-dev/infra/packages/shared/pkg/dockerhub"
 	featureflags "github.com/e2b-dev/infra/packages/shared/pkg/feature-flags"
 	l "github.com/e2b-dev/infra/packages/shared/pkg/logger"
 	sbxlogger "github.com/e2b-dev/infra/packages/shared/pkg/logger/sandbox"
@@ -136,6 +137,17 @@ func buildTemplate(
 		return fmt.Errorf("error getting artifacts registry provider: %w", err)
 	}
 
+	dockerhubRepository, err := dockerhub.GetRemoteRepository(ctx)
+	if err != nil {
+		return fmt.Errorf("error getting dockerhub repository: %w", err)
+	}
+	defer func() {
+		err := dockerhubRepository.Close()
+		if err != nil {
+			logger.Error("error closing dockerhub repository", zap.Error(err))
+		}
+	}()
+
 	blockMetrics, err := blockmetrics.NewMetrics(noop.NewMeterProvider())
 	if err != nil {
 		return fmt.Errorf("error creating metrics: %w", err)
@@ -164,6 +176,7 @@ func buildTemplate(
 		persistenceTemplate,
 		persistenceBuild,
 		artifactRegistry,
+		dockerhubRepository,
 		sandboxProxy,
 		sandboxes,
 		templateCache,