From a21fc972f86209a73d5c9cf938fc4410f75eb930 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Mon, 4 Aug 2025 15:57:39 -0400 Subject: [PATCH 1/3] Always send manager http traffic to https_proxy --- main.tf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/main.tf b/main.tf index 2c66e71..a8e28b5 100644 --- a/main.tf +++ b/main.tf @@ -184,13 +184,12 @@ resource "cloudfoundry_network_policy" "egress_routing" { { source_app = cloudfoundry_app.gitlab-runner-manager.id destination_app = module.egress_proxy.app_id - port = "61443" + port = module.egress_proxy.https_port }, - { source_app = cloudfoundry_app.gitlab-runner-manager.id destination_app = module.egress_proxy.app_id - port = "8080" + port = module.egress_proxy.http_port } ] } @@ -202,7 +201,7 @@ resource "cloudfoundry_service_instance" "manager-egress-credentials" { type = "user-provided" credentials = jsonencode({ https_uri = module.egress_proxy.https_proxy["wsr-manager"] - http_uri = module.egress_proxy.http_proxy["wsr-manager"] + http_uri = module.egress_proxy.https_proxy["wsr-manager"] cred_string = "${module.egress_proxy.username["wsr-manager"]}:${module.egress_proxy.password["wsr-manager"]}" domain = module.egress_proxy.domain http_port = module.egress_proxy.http_port From 84bdf5fae2c6afd940ac02b471b5ae073ab9c0f5 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Mon, 4 Aug 2025 16:06:45 -0400 Subject: [PATCH 2/3] Enable setting worker proxy ports and tls mode --- main.tf | 6 ++-- runner-manager/cf-driver/prepare.sh | 14 +++++--- .../bundle/glrw-profile.d/proxy-setup.sh | 2 +- sandbox-deploy/main.tf | 32 ++++++++++--------- sandbox-deploy/variables.tf | 8 +++++ variables.tf | 17 ++++++++++ 6 files changed, 57 insertions(+), 22 deletions(-) diff --git a/main.tf b/main.tf index a8e28b5..a489494 100644 --- a/main.tf +++ b/main.tf @@ -111,6 +111,7 @@ resource "cloudfoundry_app" "gitlab-runner-manager" { OBJECT_STORE_INSTANCE = var.object_store_instance PROXY_APP_NAME = var.egress_app_name PROXY_SPACE = module.egress_space.space_name + WORKER_PROXY_MODE = var.worker_egress_https_mode CF_USERNAME = local.sa_cf_username CF_PASSWORD = local.sa_cf_password CG_SSH_HOST = var.cg_ssh_host @@ -170,7 +171,7 @@ module "egress_proxy" { allowlist = local.manager_egress_allowlist } "wsr-worker" = { - ports = [80, 443] + ports = var.worker_egress_ports allowlist = local.worker_egress_allowlist } } @@ -220,7 +221,8 @@ resource "cloudfoundry_service_instance" "worker-egress-credentials" { space = module.worker_space.space_id type = "user-provided" credentials = jsonencode({ - http_uri = module.egress_proxy.http_proxy["wsr-worker"] + http_uri = (var.worker_egress_https_mode == "https" ? module.egress_proxy.https_proxy["wsr-worker"] : module.egress_proxy.http_proxy["wsr-worker"]) + https_uri = (var.worker_egress_https_mode == "http" ? module.egress_proxy.http_proxy["wsr-worker"] : module.egress_proxy.https_proxy["wsr-worker"]) }) depends_on = [module.worker_space] } diff --git a/runner-manager/cf-driver/prepare.sh b/runner-manager/cf-driver/prepare.sh index 69ee6e7..2b04862 100755 --- a/runner-manager/cf-driver/prepare.sh +++ b/runner-manager/cf-driver/prepare.sh @@ -63,10 +63,16 @@ setup_proxy_access() { container_id="$1" # setup network policy to egress-proxy - cf add-network-policy "$container_id" "$PROXY_APP_NAME" -s "$PROXY_SPACE" \ - --protocol "tcp" --port "61443" - cf add-network-policy "$container_id" "$PROXY_APP_NAME" -s "$PROXY_SPACE" \ - --protocol "tcp" --port "8080" + if [[ "$WORKER_PROXY_MODE" != "http" ]]; then + # mode must be https or both, open network policy to https port + cf add-network-policy "$container_id" "$PROXY_APP_NAME" -s "$PROXY_SPACE" \ + --protocol "tcp" --port "61443" + fi + if [[ "$WORKER_PROXY_MODE" != "https" ]]; then + # mode must be http or both, open network policy to http port + cf add-network-policy "$container_id" "$PROXY_APP_NAME" -s "$PROXY_SPACE" \ + --protocol "tcp" --port "8080" + fi } get_start_command() { diff --git a/runner-manager/cf-driver/worker-setup/bundle/glrw-profile.d/proxy-setup.sh b/runner-manager/cf-driver/worker-setup/bundle/glrw-profile.d/proxy-setup.sh index 076c049..e5fe49e 100644 --- a/runner-manager/cf-driver/worker-setup/bundle/glrw-profile.d/proxy-setup.sh +++ b/runner-manager/cf-driver/worker-setup/bundle/glrw-profile.d/proxy-setup.sh @@ -3,5 +3,5 @@ if command -v yq >/dev/null 2>&1; then export http_proxy=$(echo "$VCAP_SERVICES" | yq '.[][] | select(.name == "worker-egress-credentials") | .credentials.http_uri') - export https_proxy="$http_proxy" + export https_proxy=$(echo "$VCAP_SERVICES" | yq '.[][] | select(.name == "worker-egress-credentials") | .credentials.https_uri') fi diff --git a/sandbox-deploy/main.tf b/sandbox-deploy/main.tf index 6452157..33df980 100644 --- a/sandbox-deploy/main.tf +++ b/sandbox-deploy/main.tf @@ -12,21 +12,23 @@ provider "cloudfoundry" {} module "sandbox-runner" { source = "../" - cg_api_wildcard = "*.fr-stage.cloud.gov" - cg_ssh_host = "ssh.fr-stage.cloud.gov" - cf_org_name = "cloud-gov-devtools-development" - cf_org_managers = [var.cf_org_manager] - cf_space_prefix = var.cf_space_prefix - ci_server_token = var.ci_server_token - docker_hub_user = var.docker_hub_user - docker_hub_token = var.docker_hub_token - manager_instances = 1 - runner_concurrency = 10 - developer_emails = var.developer_emails - worker_disk_size = var.worker_disk_size - program_technologies = var.program_technologies - worker_egress_allowlist = setunion(["*.fr-stage.cloud.gov"], var.worker_egress_allowlist) - allow_ssh = var.allow_ssh + cg_api_wildcard = "*.fr-stage.cloud.gov" + cg_ssh_host = "ssh.fr-stage.cloud.gov" + cf_org_name = "cloud-gov-devtools-development" + cf_org_managers = [var.cf_org_manager] + cf_space_prefix = var.cf_space_prefix + ci_server_token = var.ci_server_token + docker_hub_user = var.docker_hub_user + docker_hub_token = var.docker_hub_token + manager_instances = 1 + runner_concurrency = 10 + developer_emails = var.developer_emails + worker_disk_size = var.worker_disk_size + program_technologies = var.program_technologies + worker_egress_allowlist = setunion(["*.fr-stage.cloud.gov"], var.worker_egress_allowlist) + worker_egress_ports = var.worker_proxy_ports + worker_egress_https_mode = var.worker_proxy_https_mode + allow_ssh = var.allow_ssh } locals { diff --git a/sandbox-deploy/variables.tf b/sandbox-deploy/variables.tf index bf1f7b3..52d01ce 100644 --- a/sandbox-deploy/variables.tf +++ b/sandbox-deploy/variables.tf @@ -29,6 +29,14 @@ variable "program_technologies" { variable "worker_egress_allowlist" { type = set(string) } +variable "worker_proxy_https_mode" { + type = string + default = "http" +} +variable "worker_proxy_ports" { + type = list(number) + default = [443, 80] +} variable "allow_ssh" { type = bool } diff --git a/variables.tf b/variables.tf index f78b380..5c7f49e 100644 --- a/variables.tf +++ b/variables.tf @@ -155,6 +155,23 @@ variable "worker_egress_allowlist" { description = "A list of external domain names that runner workers must be able to connect to" } +variable "worker_egress_ports" { + type = list(number) + default = [443] + description = "List of ports that the egress proxy will forward traffic to from the runner workers" +} + +variable "worker_egress_https_mode" { + type = string + default = "https" + description = "Which egress proxy protocol to send traffic over. Must be http, https, or both" + + validation { + condition = contains(["http", "https", "both"], var.worker_egress_https_mode) + error_message = "worker_egress_https_mode must be one of 'http', 'https', or 'both'" + } +} + variable "allow_ssh" { type = bool default = false From 0dbb865381cabeb56ce80dbfad29fc701aa3a83f Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Mon, 4 Aug 2025 16:26:09 -0400 Subject: [PATCH 3/3] Refactor how we target the staging cloud.gov foundation --- main.tf | 4 ++-- sandbox-deploy/main.tf | 3 +-- variables.tf | 12 +++--------- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/main.tf b/main.tf index a489494..6792ac6 100644 --- a/main.tf +++ b/main.tf @@ -1,7 +1,7 @@ locals { # the list of egress hosts to allow for runner-manager and always needed by runner workers manager_egress_allowlist = toset([ - var.cg_api_wildcard, # cf-cli calls from manager + "*.${var.cf_api_base}", # cf-cli calls from manager var.ci_server_url ]) technology_allowlist = flatten([for t in var.program_technologies : local.allowlist_map[t]]) @@ -114,7 +114,7 @@ resource "cloudfoundry_app" "gitlab-runner-manager" { WORKER_PROXY_MODE = var.worker_egress_https_mode CF_USERNAME = local.sa_cf_username CF_PASSWORD = local.sa_cf_password - CG_SSH_HOST = var.cg_ssh_host + CG_SSH_HOST = "ssh.${var.cf_api_base}" DOCKER_HUB_USER = var.docker_hub_user DOCKER_HUB_TOKEN = var.docker_hub_token # DANGER: Do not set RUNNER_DEBUG to true without reading diff --git a/sandbox-deploy/main.tf b/sandbox-deploy/main.tf index 33df980..b4fd8a4 100644 --- a/sandbox-deploy/main.tf +++ b/sandbox-deploy/main.tf @@ -12,8 +12,7 @@ provider "cloudfoundry" {} module "sandbox-runner" { source = "../" - cg_api_wildcard = "*.fr-stage.cloud.gov" - cg_ssh_host = "ssh.fr-stage.cloud.gov" + cf_api_base = "fr-stage.cloud.gov" cf_org_name = "cloud-gov-devtools-development" cf_org_managers = [var.cf_org_manager] cf_space_prefix = var.cf_space_prefix diff --git a/variables.tf b/variables.tf index 5c7f49e..ecbda62 100644 --- a/variables.tf +++ b/variables.tf @@ -15,12 +15,6 @@ variable "auditor_emails" { default = [] } -variable "cg_ssh_host" { - type = string - default = "ssh.fr.cloud.gov" - description = "cloud.gov ssh jumpbox domain name" -} - variable "cf_org_name" { type = string default = "gsa-tts-devtools-prototyping" @@ -44,10 +38,10 @@ variable "ci_server_url" { description = "Gitlab Dedicated for Government URL" } -variable "cg_api_wildcard" { +variable "cf_api_base" { type = string - default = "*.fr.cloud.gov" - description = "Wildcard domain for the various cloud.gov endpoints" + default = "fr.cloud.gov" + description = "Hostname under which all api and ssh endpoints fall" } variable "default_job_image" {