feat: add remote repository to handle DockerHub rate limits

iac/provider-gcp/Makefile

@@ -62,7 +62,8 @@ tf_vars := 	TF_VAR_environment=$(TERRAFORM_ENVIRONMENT) \
 	$(call tfvar, FILESTORE_MAX_DISK_USAGE_TARGET) \
 	$(call tfvar, BUILD_CLUSTER_CACHE_DISK_TYPE) \
 	$(call tfvar, CLIENT_CLUSTER_CACHE_DISK_TYPE) \
-	$(call tfvar, MIN_CPU_PLATFORM)
+	$(call tfvar, MIN_CPU_PLATFORM) \
+	$(call tfvar, REMOTE_REPOSITORY_ENABLED)
 
 .PHONY: init
 init:

iac/provider-gcp/main.tf

@@ -232,11 +232,12 @@ module "nomad" {
   envd_timeout                = var.envd_timeout
 
   # Template manager
-  builder_node_pool              = var.build_node_pool
-  template_manager_port          = var.template_manager_port
-  template_bucket_name           = module.init.fc_template_bucket_name
-  build_cache_bucket_name        = module.init.fc_build_cache_bucket_name
-  template_manager_machine_count = var.build_cluster_size
+  builder_node_pool               = var.build_node_pool
+  template_manager_port           = var.template_manager_port
+  template_bucket_name            = module.init.fc_template_bucket_name
+  build_cache_bucket_name         = module.init.fc_build_cache_bucket_name
+  template_manager_machine_count  = var.build_cluster_size
+  dockerhub_remote_repository_url = var.remote_repository_enabled ? module.remote_repository[0].dockerhub_remote_repository_url : ""
 
   # Redis
   redis_managed = var.redis_managed
@@ -259,3 +260,16 @@ module "redis" {
 
   prefix = var.prefix
 }
+
+module "remote_repository" {
+  source = "./remote-repository"
+
+  count = var.remote_repository_enabled ? 1 : 0
+
+  prefix = var.prefix
+
+  gcp_project_id = var.gcp_project_id
+  gcp_region     = var.gcp_region
+
+  google_service_account_email = module.init.service_account_email
+}

iac/provider-gcp/nomad/jobs/template-manager.hcl

@@ -76,6 +76,7 @@ job "template-manager-system" {
         ALLOW_SANDBOX_INTERNET        = "${allow_sandbox_internet}"
         SHARED_CHUNK_CACHE_PATH       = "${shared_chunk_cache_path}"
         CLICKHOUSE_CONNECTION_STRING  = "${clickhouse_connection_string}"
+        DOCKERHUB_REMOTE_REPOSITORY_URL  = "${dockerhub_remote_repository_url}"
 %{ if !update_stanza }
         FORCE_STOP                    = "true"
 %{ endif }

iac/provider-gcp/nomad/main.tf

@@ -494,19 +494,20 @@ resource "nomad_job" "template_manager" {
     environment      = var.environment
     consul_acl_token = var.consul_acl_token_secret
 
-    api_secret                   = var.api_secret
-    bucket_name                  = var.fc_env_pipeline_bucket_name
-    docker_registry              = var.custom_envs_repository_name
-    google_service_account_key   = var.google_service_account_key
-    template_manager_checksum    = data.external.template_manager.result.hex
-    otel_tracing_print           = var.otel_tracing_print
-    template_bucket_name         = var.template_bucket_name
-    build_cache_bucket_name      = var.build_cache_bucket_name
-    otel_collector_grpc_endpoint = "localhost:${var.otel_collector_grpc_port}"
-    logs_collector_address       = "http://localhost:${var.logs_proxy_port.port}"
-    orchestrator_services        = "template-manager"
-    allow_sandbox_internet       = var.allow_sandbox_internet
-    clickhouse_connection_string = local.clickhouse_connection_string
+    api_secret                      = var.api_secret
+    bucket_name                     = var.fc_env_pipeline_bucket_name
+    docker_registry                 = var.custom_envs_repository_name
+    google_service_account_key      = var.google_service_account_key
+    template_manager_checksum       = data.external.template_manager.result.hex
+    otel_tracing_print              = var.otel_tracing_print
+    template_bucket_name            = var.template_bucket_name
+    build_cache_bucket_name         = var.build_cache_bucket_name
+    otel_collector_grpc_endpoint    = "localhost:${var.otel_collector_grpc_port}"
+    logs_collector_address          = "http://localhost:${var.logs_proxy_port.port}"
+    orchestrator_services           = "template-manager"
+    allow_sandbox_internet          = var.allow_sandbox_internet
+    clickhouse_connection_string    = local.clickhouse_connection_string
+    dockerhub_remote_repository_url = var.dockerhub_remote_repository_url
 
     # For now we DISABLE the shared chunk cache in the template manager
     shared_chunk_cache_path = ""

iac/provider-gcp/nomad/variables.tf

@@ -340,3 +340,7 @@ variable "filestore_cache_max_disk_usage_target" {
   type        = number
   description = "The maximum disk usage target for the Filestore cache in percent"
 }
+
+variable "dockerhub_remote_repository_url" {
+  type = string
+}

iac/provider-gcp/remote-repository/main.tf

@@ -0,0 +1,27 @@
+resource "google_artifact_registry_repository" "dockerhub_remote_repository" {
+  location      = var.gcp_region
+  repository_id = "${var.prefix}docker-remote-repository"
+  description   = "remote docker repository"
+  format        = "DOCKER"
+  mode          = "REMOTE_REPOSITORY"
+  remote_repository_config {
+    description = "Docker Hub"
+    docker_repository {
+      public_repository = "DOCKER_HUB"
+    }
+  }
+
+  cleanup_policies {
+    id     = "delete-older-than-90-days"
+    action = "DELETE"
+    condition {
+      older_than = "90d"
+    }
+  }
+}
+
+resource "google_artifact_registry_repository_iam_member" "dockerhub_remote_repository_member" {
+  repository = google_artifact_registry_repository.dockerhub_remote_repository.name
+  role       = "roles/artifactregistry.repoAdmin"
+  member     = "serviceAccount:${var.google_service_account_email}"
+}

iac/provider-gcp/remote-repository/outputs.tf

@@ -0,0 +1,3 @@
+output "dockerhub_remote_repository_url" {
+  value = "${var.gcp_region}-docker.pkg.dev/${var.gcp_project_id}/${google_artifact_registry_repository.dockerhub_remote_repository.name}"
+}

iac/provider-gcp/remote-repository/variables.tf

@@ -0,0 +1,15 @@
+variable "prefix" {
+  type = string
+}
+
+variable "gcp_project_id" {
+  type = string
+}
+
+variable "gcp_region" {
+  type = string
+}
+
+variable "google_service_account_email" {
+  type = string
+}

iac/provider-gcp/variables.tf

@@ -452,3 +452,9 @@ variable "orchestrator_base_hugepages_percentage" {
   type        = number
   default     = 80
 }
+
+variable "remote_repository_enabled" {
+  type        = bool
+  description = "Set to true to enable remote repository cache. Can be set via TF_VAR_remote_repository_enabled or REMOTE_REPOSITORY_ENABLED env var."
+  default     = false
+}

packages/orchestrator/cmd/build-template/main.go

@@ -23,6 +23,7 @@ import (
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/config"
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/metrics"
 	artifactsregistry "github.com/e2b-dev/infra/packages/shared/pkg/artifacts-registry"
+	"github.com/e2b-dev/infra/packages/shared/pkg/dockerhub"
 	featureflags "github.com/e2b-dev/infra/packages/shared/pkg/feature-flags"
 	l "github.com/e2b-dev/infra/packages/shared/pkg/logger"
 	sbxlogger "github.com/e2b-dev/infra/packages/shared/pkg/logger/sandbox"
@@ -136,6 +137,17 @@ func buildTemplate(
 		return fmt.Errorf("error getting artifacts registry provider: %w", err)
 	}
 
+	dockerhubRepository, err := dockerhub.GetRemoteRepository(ctx)
+	if err != nil {
+		return fmt.Errorf("error getting dockerhub repository: %w", err)
+	}
+	defer func() {
+		err := dockerhubRepository.Close()
+		if err != nil {
+			logger.Error("error closing dockerhub repository", zap.Error(err))
+		}
+	}()
+
 	blockMetrics, err := blockmetrics.NewMetrics(noop.NewMeterProvider())
 	if err != nil {
 		return fmt.Errorf("error creating metrics: %w", err)
@@ -164,6 +176,7 @@ func buildTemplate(
 		persistenceTemplate,
 		persistenceBuild,
 		artifactRegistry,
+		dockerhubRepository,
 		sandboxProxy,
 		sandboxes,
 		templateCache,

packages/orchestrator/internal/template/build/builder.go

@@ -28,6 +28,7 @@ import (
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/writer"
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/template/constants"
 	artifactsregistry "github.com/e2b-dev/infra/packages/shared/pkg/artifacts-registry"
+	"github.com/e2b-dev/infra/packages/shared/pkg/dockerhub"
 	"github.com/e2b-dev/infra/packages/shared/pkg/smap"
 	"github.com/e2b-dev/infra/packages/shared/pkg/storage"
 	"github.com/e2b-dev/infra/packages/shared/pkg/storage/header"
@@ -40,14 +41,15 @@ var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/orchestrator/interna
 type Builder struct {
 	logger *zap.Logger
 
-	sandboxFactory   *sandbox.Factory
-	templateStorage  storage.StorageProvider
-	buildStorage     storage.StorageProvider
-	artifactRegistry artifactsregistry.ArtifactsRegistry
-	proxy            *proxy.SandboxProxy
-	sandboxes        *smap.Map[*sandbox.Sandbox]
-	templateCache    *sbxtemplate.Cache
-	metrics          *metrics.BuildMetrics
+	sandboxFactory      *sandbox.Factory
+	templateStorage     storage.StorageProvider
+	buildStorage        storage.StorageProvider
+	artifactRegistry    artifactsregistry.ArtifactsRegistry
+	dockerhubRepository dockerhub.RemoteRepository
+	proxy               *proxy.SandboxProxy
+	sandboxes           *smap.Map[*sandbox.Sandbox]
+	templateCache       *sbxtemplate.Cache
+	metrics             *metrics.BuildMetrics
 }
 
 func NewBuilder(
@@ -56,21 +58,23 @@ func NewBuilder(
 	templateStorage storage.StorageProvider,
 	buildStorage storage.StorageProvider,
 	artifactRegistry artifactsregistry.ArtifactsRegistry,
+	dockerhubRepository dockerhub.RemoteRepository,
 	proxy *proxy.SandboxProxy,
 	sandboxes *smap.Map[*sandbox.Sandbox],
 	templateCache *sbxtemplate.Cache,
 	buildMetrics *metrics.BuildMetrics,
 ) *Builder {
 	return &Builder{
-		logger:           logger,
-		templateStorage:  templateStorage,
-		buildStorage:     buildStorage,
-		artifactRegistry: artifactRegistry,
-		proxy:            proxy,
-		sandboxes:        sandboxes,
-		templateCache:    templateCache,
-		metrics:          buildMetrics,
-		sandboxFactory:   sandboxFactory,
+		logger:              logger,
+		sandboxFactory:      sandboxFactory,
+		templateStorage:     templateStorage,
+		buildStorage:        buildStorage,
+		artifactRegistry:    artifactRegistry,
+		dockerhubRepository: dockerhubRepository,
+		proxy:               proxy,
+		sandboxes:           sandboxes,
+		templateCache:       templateCache,
+		metrics:             buildMetrics,
 	}
 }
 
@@ -201,6 +205,7 @@ func runBuild(
 		builder.proxy,
 		builder.templateStorage,
 		builder.artifactRegistry,
+		builder.dockerhubRepository,
 		layerExecutor,
 		index,
 		builder.metrics,

packages/orchestrator/internal/template/build/core/oci/oci.go

@@ -21,6 +21,7 @@ import (
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/core/filesystem"
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/core/oci/auth"
 	artifactsregistry "github.com/e2b-dev/infra/packages/shared/pkg/artifacts-registry"
+	"github.com/e2b-dev/infra/packages/shared/pkg/dockerhub"
 	"github.com/e2b-dev/infra/packages/shared/pkg/telemetry"
 	"github.com/e2b-dev/infra/packages/shared/pkg/utils"
 )
@@ -37,21 +38,33 @@ var DefaultPlatform = containerregistry.Platform{
 	Architecture: "amd64",
 }
 
-func GetPublicImage(ctx context.Context, tag string, authProvider auth.RegistryAuthProvider) (containerregistry.Image, error) {
-	childCtx, childSpan := tracer.Start(ctx, "pull-public-docker-image")
-	defer childSpan.End()
+func GetPublicImage(ctx context.Context, dockerhubRepository dockerhub.RemoteRepository, tag string, authProvider auth.RegistryAuthProvider) (containerregistry.Image, error) {
+	ctx, span := tracer.Start(ctx, "pull-public-docker-image")
+	defer span.End()
 
 	ref, err := name.ParseReference(tag)
 	if err != nil {
 		return nil, fmt.Errorf("invalid image reference: %w", err)
 	}
 
-	// Build authentication options
+	// When no auth provider is provided and the image is from the default registry
+	// use docker remote repository proxy with cached images
+	if authProvider == nil && ref.Context().RegistryStr() == name.DefaultRegistry {
+		img, err := dockerhubRepository.GetImage(ctx, tag, DefaultPlatform)
+		if err != nil {
+			return nil, fmt.Errorf("error getting image: %w", err)
+		}
+
+		telemetry.ReportEvent(ctx, "pulled public image")
+		return img, nil
+	}
+
+	// Build options
 	opts := []remote.Option{remote.WithPlatform(DefaultPlatform)}
 
 	// Use the auth provider if provided
 	if authProvider != nil {
-		authOption, err := authProvider.GetAuthOption(childCtx)
+		authOption, err := authProvider.GetAuthOption(ctx)
 		if err != nil {
 			return nil, fmt.Errorf("error getting auth option: %w", err)
 		}
@@ -65,7 +78,7 @@ func GetPublicImage(ctx context.Context, tag string, authProvider auth.RegistryA
 		return nil, fmt.Errorf("error pulling image: %w", err)
 	}
 
-	telemetry.ReportEvent(childCtx, "pulled public image")
+	telemetry.ReportEvent(ctx, "pulled public image")
 	return img, nil
 }
 

packages/orchestrator/internal/template/build/core/oci/oci_test.go

@@ -23,6 +23,7 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/core/oci/auth"
+	"github.com/e2b-dev/infra/packages/shared/pkg/dockerhub"
 	templatemanager "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager"
 )
 
@@ -145,6 +146,14 @@ func TestGetPublicImageWithGeneralAuth(t *testing.T) {
 	testRepository := "test/image"
 	testImageRef := testRepository + ":latest"
 
+	dockerhubRepository := dockerhub.NewNoopRemoteRepository()
+	t.Cleanup(func() {
+		err := dockerhubRepository.Close()
+		if err != nil {
+			t.Errorf("error closing dockerhub repository: %v", err)
+		}
+	})
+
 	t.Run("successful auth and pull", func(t *testing.T) {
 		reg := registry.New()
 
@@ -185,7 +194,7 @@ func TestGetPublicImageWithGeneralAuth(t *testing.T) {
 		require.NotNil(t, authOption)
 
 		// Now test GetPublicImage
-		img, err := GetPublicImage(ctx, imageRef, authProvider)
+		img, err := GetPublicImage(ctx, dockerhubRepository, imageRef, authProvider)
 		require.NoError(t, err)
 		require.NotNil(t, img)
 
@@ -235,7 +244,7 @@ func TestGetPublicImageWithGeneralAuth(t *testing.T) {
 		require.NotNil(t, authOption)
 
 		// Now test GetPublicImage
-		img, err := GetPublicImage(ctx, imageRef, authProvider)
+		img, err := GetPublicImage(ctx, dockerhubRepository, imageRef, authProvider)
 		require.Error(t, err)
 		require.Nil(t, img)
 	})
@@ -260,7 +269,7 @@ func TestGetPublicImageWithGeneralAuth(t *testing.T) {
 		require.NoError(t, err)
 
 		// Get image without auth provider (nil)
-		img, err := GetPublicImage(ctx, imageRef, nil)
+		img, err := GetPublicImage(ctx, dockerhubRepository, imageRef, nil)
 		require.NoError(t, err)
 		require.NotNil(t, img)