Enable enforcing tls traffic over worker proxy

main.tf

@@ -1,7 +1,7 @@
 locals {
   # the list of egress hosts to allow for runner-manager and always needed by runner workers
   manager_egress_allowlist = toset([
-    var.cg_api_wildcard, # cf-cli calls from manager
+    "*.${var.cf_api_base}", # cf-cli calls from manager
     var.ci_server_url
   ])
   technology_allowlist    = flatten([for t in var.program_technologies : local.allowlist_map[t]])
@@ -111,9 +111,10 @@ resource "cloudfoundry_app" "gitlab-runner-manager" {
     OBJECT_STORE_INSTANCE = var.object_store_instance
     PROXY_APP_NAME        = var.egress_app_name
     PROXY_SPACE           = module.egress_space.space_name
+    WORKER_PROXY_MODE     = var.worker_egress_https_mode
     CF_USERNAME           = local.sa_cf_username
     CF_PASSWORD           = local.sa_cf_password
-    CG_SSH_HOST           = var.cg_ssh_host
+    CG_SSH_HOST           = "ssh.${var.cf_api_base}"
     DOCKER_HUB_USER       = var.docker_hub_user
     DOCKER_HUB_TOKEN      = var.docker_hub_token
     # DANGER: Do not set RUNNER_DEBUG to true without reading
@@ -170,7 +171,7 @@ module "egress_proxy" {
       allowlist = local.manager_egress_allowlist
     }
     "wsr-worker" = {
-      ports     = [80, 443]
+      ports     = var.worker_egress_ports
       allowlist = local.worker_egress_allowlist
     }
   }
@@ -184,13 +185,12 @@ resource "cloudfoundry_network_policy" "egress_routing" {
     {
       source_app      = cloudfoundry_app.gitlab-runner-manager.id
       destination_app = module.egress_proxy.app_id
-      port            = "61443"
+      port            = module.egress_proxy.https_port
     },
-
     {
       source_app      = cloudfoundry_app.gitlab-runner-manager.id
       destination_app = module.egress_proxy.app_id
-      port            = "8080"
+      port            = module.egress_proxy.http_port
     }
   ]
 }
@@ -202,7 +202,7 @@ resource "cloudfoundry_service_instance" "manager-egress-credentials" {
   type  = "user-provided"
   credentials = jsonencode({
     https_uri   = module.egress_proxy.https_proxy["wsr-manager"]
-    http_uri    = module.egress_proxy.http_proxy["wsr-manager"]
+    http_uri    = module.egress_proxy.https_proxy["wsr-manager"]
     cred_string = "${module.egress_proxy.username["wsr-manager"]}:${module.egress_proxy.password["wsr-manager"]}"
     domain      = module.egress_proxy.domain
     http_port   = module.egress_proxy.http_port
@@ -221,7 +221,8 @@ resource "cloudfoundry_service_instance" "worker-egress-credentials" {
   space = module.worker_space.space_id
   type  = "user-provided"
   credentials = jsonencode({
-    http_uri = module.egress_proxy.http_proxy["wsr-worker"]
+    http_uri  = (var.worker_egress_https_mode == "https" ? module.egress_proxy.https_proxy["wsr-worker"] : module.egress_proxy.http_proxy["wsr-worker"])
+    https_uri = (var.worker_egress_https_mode == "http" ? module.egress_proxy.http_proxy["wsr-worker"] : module.egress_proxy.https_proxy["wsr-worker"])
   })
   depends_on = [module.worker_space]
 }

runner-manager/cf-driver/prepare.sh

@@ -63,10 +63,16 @@ setup_proxy_access() {
     container_id="$1"
 
     # setup network policy to egress-proxy
-    cf add-network-policy "$container_id" "$PROXY_APP_NAME" -s "$PROXY_SPACE" \
-        --protocol "tcp" --port "61443"
-    cf add-network-policy "$container_id" "$PROXY_APP_NAME" -s "$PROXY_SPACE" \
-        --protocol "tcp" --port "8080"
+    if [[ "$WORKER_PROXY_MODE" != "http" ]]; then
+        # mode must be https or both, open network policy to https port
+        cf add-network-policy "$container_id" "$PROXY_APP_NAME" -s "$PROXY_SPACE" \
+            --protocol "tcp" --port "61443"
+    fi
+    if [[ "$WORKER_PROXY_MODE" != "https" ]]; then
+        # mode must be http or both, open network policy to http port
+        cf add-network-policy "$container_id" "$PROXY_APP_NAME" -s "$PROXY_SPACE" \
+            --protocol "tcp" --port "8080"
+    fi
 }
 
 get_start_command() {

runner-manager/cf-driver/worker-setup/bundle/glrw-profile.d/proxy-setup.sh

@@ -3,5 +3,5 @@
 
 if command -v yq >/dev/null 2>&1; then
     export http_proxy=$(echo "$VCAP_SERVICES" | yq '.[][] | select(.name == "worker-egress-credentials") | .credentials.http_uri')
-    export https_proxy="$http_proxy"
+    export https_proxy=$(echo "$VCAP_SERVICES" | yq '.[][] | select(.name == "worker-egress-credentials") | .credentials.https_uri')
 fi

sandbox-deploy/main.tf

@@ -12,21 +12,22 @@ provider "cloudfoundry" {}
 module "sandbox-runner" {
   source = "../"
 
-  cg_api_wildcard         = "*.fr-stage.cloud.gov"
-  cg_ssh_host             = "ssh.fr-stage.cloud.gov"
-  cf_org_name             = "cloud-gov-devtools-development"
-  cf_org_managers         = [var.cf_org_manager]
-  cf_space_prefix         = var.cf_space_prefix
-  ci_server_token         = var.ci_server_token
-  docker_hub_user         = var.docker_hub_user
-  docker_hub_token        = var.docker_hub_token
-  manager_instances       = 1
-  runner_concurrency      = 10
-  developer_emails        = var.developer_emails
-  worker_disk_size        = var.worker_disk_size
-  program_technologies    = var.program_technologies
-  worker_egress_allowlist = setunion(["*.fr-stage.cloud.gov"], var.worker_egress_allowlist)
-  allow_ssh               = var.allow_ssh
+  cf_api_base              = "fr-stage.cloud.gov"
+  cf_org_name              = "cloud-gov-devtools-development"
+  cf_org_managers          = [var.cf_org_manager]
+  cf_space_prefix          = var.cf_space_prefix
+  ci_server_token          = var.ci_server_token
+  docker_hub_user          = var.docker_hub_user
+  docker_hub_token         = var.docker_hub_token
+  manager_instances        = 1
+  runner_concurrency       = 10
+  developer_emails         = var.developer_emails
+  worker_disk_size         = var.worker_disk_size
+  program_technologies     = var.program_technologies
+  worker_egress_allowlist  = setunion(["*.fr-stage.cloud.gov"], var.worker_egress_allowlist)
+  worker_egress_ports      = var.worker_proxy_ports
+  worker_egress_https_mode = var.worker_proxy_https_mode
+  allow_ssh                = var.allow_ssh
 }
 
 locals {

sandbox-deploy/variables.tf

@@ -29,6 +29,14 @@ variable "program_technologies" {
 variable "worker_egress_allowlist" {
   type = set(string)
 }
+variable "worker_proxy_https_mode" {
+  type    = string
+  default = "http"
+}
+variable "worker_proxy_ports" {
+  type    = list(number)
+  default = [443, 80]
+}
 variable "allow_ssh" {
   type = bool
 }

variables.tf

@@ -15,12 +15,6 @@ variable "auditor_emails" {
   default     = []
 }
 
-variable "cg_ssh_host" {
-  type        = string
-  default     = "ssh.fr.cloud.gov"
-  description = "cloud.gov ssh jumpbox domain name"
-}
-
 variable "cf_org_name" {
   type        = string
   default     = "gsa-tts-devtools-prototyping"
@@ -44,10 +38,10 @@ variable "ci_server_url" {
   description = "Gitlab Dedicated for Government URL"
 }
 
-variable "cg_api_wildcard" {
+variable "cf_api_base" {
   type        = string
-  default     = "*.fr.cloud.gov"
-  description = "Wildcard domain for the various cloud.gov endpoints"
+  default     = "fr.cloud.gov"
+  description = "Hostname under which all api and ssh endpoints fall"
 }
 
 variable "default_job_image" {
@@ -155,6 +149,23 @@ variable "worker_egress_allowlist" {
   description = "A list of external domain names that runner workers must be able to connect to"
 }
 
+variable "worker_egress_ports" {
+  type        = list(number)
+  default     = [443]
+  description = "List of ports that the egress proxy will forward traffic to from the runner workers"
+}
+
+variable "worker_egress_https_mode" {
+  type        = string
+  default     = "https"
+  description = "Which egress proxy protocol to send traffic over. Must be http, https, or both"
+
+  validation {
+    condition     = contains(["http", "https", "both"], var.worker_egress_https_mode)
+    error_message = "worker_egress_https_mode must be one of 'http', 'https', or 'both'"
+  }
+}
+
 variable "allow_ssh" {
   type        = bool
   default     = false