diff --git a/.github/workflows/e2e-tests-kind.yml b/.github/workflows/e2e-tests-kind.yml new file mode 100644 index 00000000..4a397d55 --- /dev/null +++ b/.github/workflows/e2e-tests-kind.yml @@ -0,0 +1,76 @@ +name: E2E Tests (kind) + +on: + workflow_call: + inputs: + image: + required: true + type: string + +env: + REGISTRY: quay.io + IMAGE_NAME: rhacs-eng/roxie + +jobs: + e2e-tests-kind: + runs-on: ubuntu-latest + env: + SKIP_OLM_TESTS: "true" + # TODO: Once the config-file-first approach lands, this will be changed from an + # environment variable to configuring a YAML configuration file, which will be + # used by the e2e tests. + ROXIE_RESOURCE_PROFILE: "tiny" + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + + - name: Log in to Quay.io + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Extract roxie binary from image + run: | + docker create --name roxie-extract "${{ inputs.image }}" + docker cp roxie-extract:/usr/local/bin/roxie "$GITHUB_WORKSPACE/roxie" + docker rm roxie-extract + + - name: Install roxie binary + run: | + cp "${GITHUB_WORKSPACE}/roxie" /usr/local/bin/roxie + chmod +x /usr/local/bin/roxie + roxie version + + - name: Install roxctl + env: + ROXCTL_VERSION: "4.10.0" + ROXCTL_SHA256: "5db647b14569465866c0162522e83393ebf02f671f4556b1b3ed551b9f8433bc" + run: | + curl -fsSLo /usr/local/bin/roxctl \ + "https://mirror.openshift.com/pub/rhacs/assets/${ROXCTL_VERSION}/bin/Linux/roxctl" + echo "${ROXCTL_SHA256} /usr/local/bin/roxctl" | sha256sum -c - + chmod +x /usr/local/bin/roxctl + roxctl version + + - name: Create kind cluster + uses: helm/kind-action@v1 + with: + cluster_name: roxie-e2e + + - name: Run e2e tests + env: + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + run: | + make run-test-e2e diff --git a/.github/workflows/main-push.yml b/.github/workflows/main-push.yml index 32b0322d..9beecbab 100644 --- a/.github/workflows/main-push.yml +++ b/.github/workflows/main-push.yml @@ -35,6 +35,13 @@ jobs: image: ${{ needs.build-roxie-image.outputs.image }} secrets: inherit + e2e-tests-kind: + needs: [ build-roxie-image ] + uses: ./.github/workflows/e2e-tests-kind.yml + with: + image: ${{ needs.build-roxie-image.outputs.image }} + secrets: inherit + delete-dev-cluster: if: ${{ always() && needs.create-dev-cluster.result == 'success' }} needs: [ create-dev-cluster, e2e-tests ] diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8da10822..da8b8444 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -71,6 +71,13 @@ jobs: skip-olm-tests: 'false' secrets: inherit + e2e-tests-kind: + needs: [ build-roxie-image ] + uses: ./.github/workflows/e2e-tests-kind.yml + with: + image: ${{ needs.build-roxie-image.outputs.image }} + secrets: inherit + delete-gke-cluster: if: ${{ always() && needs.create-gke-cluster.result == 'success' }} needs: [ create-gke-cluster, e2e-tests ] diff --git a/cmd/deploy.go b/cmd/deploy.go index a5a6131a..a324a5cc 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -76,14 +76,6 @@ func runDeploy(cmd *cobra.Command, args []string) error { return errors.New("running without a controlling terminal requires --envrc to be set") } - if envrc != "" && portForwarding { - return errors.New("cannot use --envrc with --port-forwarding. The --envrc flag is for non-interactive mode with remote cluster access") - } - - if envrc != "" && exposure == "none" { - return errors.New("cannot use --envrc with --exposure=none. The --envrc flag requires a remotely accessible endpoint (e.g., --exposure=loadbalancer)") - } - portForwardEnabledFinal := portForwarding || exposure == "none" if env.RunningInRoxieContainer { diff --git a/internal/deployer/constants.go b/internal/deployer/constants.go index 8d1afbab..0068e06a 100644 --- a/internal/deployer/constants.go +++ b/internal/deployer/constants.go @@ -6,8 +6,20 @@ var ( centralCrName = "stackrox-central-services" securedClusterCrName = "stackrox-secured-cluster-services" + centralDbPVCSizeTiny = "10Gi" centralDbPVCSizeSmall = "30Gi" + centralResourcesTiny = map[string]interface{}{ + "requests": map[string]string{ + "memory": "300Mi", + "cpu": "200m", + }, + "limits": map[string]string{ + "memory": "2Gi", + "cpu": "1", + }, + } + centralResourcesSmall = map[string]interface{}{ "requests": map[string]string{ "memory": "1Gi", @@ -19,6 +31,17 @@ var ( }, } + centralDbResourcesTiny = map[string]interface{}{ + "requests": map[string]string{ + "memory": "400Mi", + "cpu": "200m", + }, + "limits": map[string]string{ + "memory": "2Gi", + "cpu": "1", + }, + } + centralDbResourcesSmall = map[string]interface{}{ "requests": map[string]string{ "memory": "1Gi", @@ -52,6 +75,17 @@ var ( }, } + centralScannerV4DbResourcesTiny = map[string]interface{}{ + "requests": map[string]string{ + "memory": "400Mi", + "cpu": "300m", + }, + "limits": map[string]string{ + "memory": "2000Mi", + "cpu": "1000m", + }, + } + centralScannerV4DbResourcesSmall = map[string]interface{}{ "requests": map[string]string{ "memory": "512Mi", @@ -63,6 +97,17 @@ var ( }, } + centralScannerV4IndexerResourcesTiny = map[string]interface{}{ + "requests": map[string]string{ + "memory": "300Mi", + "cpu": "200m", + }, + "limits": map[string]string{ + "memory": "2Gi", + "cpu": "2000m", + }, + } + centralScannerV4IndexerResourcesSmall = map[string]interface{}{ "requests": map[string]string{ "memory": "512Mi", @@ -74,6 +119,17 @@ var ( }, } + centralScannerV4MatcherResourcesTiny = map[string]interface{}{ + "requests": map[string]string{ + "memory": "300Mi", + "cpu": "200m", + }, + "limits": map[string]string{ + "memory": "2Gi", + "cpu": "1000m", + }, + } + centralScannerV4MatcherResourcesSmall = map[string]interface{}{ "requests": map[string]string{ "memory": "512Mi", @@ -87,6 +143,17 @@ var ( // Secured Cluster + securedClusterSensorResourcesTiny = map[string]interface{}{ + "requests": map[string]string{ + "memory": "300Mi", + "cpu": "200m", + }, + "limits": map[string]string{ + "memory": "2Gi", + "cpu": "1000m", + }, + } + securedClusterSensorResourcesSmall = map[string]interface{}{ "requests": map[string]string{ "memory": "500Mi", diff --git a/internal/deployer/deploy_via_operator.go b/internal/deployer/deploy_via_operator.go index 2bc1ca73..bc0277e8 100644 --- a/internal/deployer/deploy_via_operator.go +++ b/internal/deployer/deploy_via_operator.go @@ -288,6 +288,38 @@ func (d *Deployer) createCentralCR(resources, exposure string) (map[string]inter func (d *Deployer) getCentralResourcesOperator(resourcesName string) map[string]interface{} { switch resourcesName { + case "tiny": + return map[string]interface{}{ + "spec": map[string]interface{}{ + "central": map[string]interface{}{ + "resources": centralResourcesTiny, + "db": map[string]interface{}{ + "resources": centralDbResourcesTiny, + "persistence": map[string]interface{}{ + "persistentVolumeClaim": map[string]interface{}{ + "size": centralDbPVCSizeTiny, + }, + }, + }, + }, + "scanner": map[string]interface{}{ + "scannerComponent": "Disabled", + }, + "scannerV4": map[string]interface{}{ + "db": map[string]interface{}{ + "resources": centralScannerV4DbResourcesTiny, + }, + "indexer": map[string]interface{}{ + "resources": centralScannerV4IndexerResourcesTiny, + "scaling": noScaling, + }, + "matcher": map[string]interface{}{ + "resources": centralScannerV4MatcherResourcesTiny, + "scaling": noScaling, + }, + }, + }, + } case "small": return map[string]interface{}{ "spec": map[string]interface{}{ @@ -607,11 +639,20 @@ func (d *Deployer) configureCentralEndpoint(ctx context.Context, exposure string } } - endpoint, err := d.portForward.Start(d.centralNamespace, serviceName, 443, 8443) - if err != nil { - return fmt.Errorf("failed to start port-forward: %w", err) + if d.envrcFile != "" { + endpoint, pid, err := d.portForward.StartDetached(d.centralNamespace, serviceName, 443, 8443) + if err != nil { + return fmt.Errorf("failed to start detached port-forward: %w", err) + } + d.centralEndpoint = endpoint + d.portForwardPID = pid + } else { + endpoint, err := d.portForward.Start(d.centralNamespace, serviceName, 443, 8443) + if err != nil { + return fmt.Errorf("failed to start port-forward: %w", err) + } + d.centralEndpoint = endpoint } - d.centralEndpoint = endpoint } else if exposure == "loadbalancer" { endpoint, err := d.waitForLoadBalancer(ctx, d.centralNamespace, "central-loadbalancer", 300) if err != nil { @@ -716,6 +757,20 @@ func (d *Deployer) createSecuredClusterCR(resources string) (map[string]interfac func (d *Deployer) getSecuredClusterResourcesOperator(resourcesName string) map[string]interface{} { switch resourcesName { + case "tiny": + return map[string]interface{}{ + "spec": map[string]interface{}{ + "sensor": map[string]interface{}{ + "resources": securedClusterSensorResourcesTiny, + }, + "scanner": map[string]interface{}{ + "scannerComponent": "Disabled", + }, + "scannerV4": map[string]interface{}{ + "scannerComponent": "Disabled", + }, + }, + } case "small": return map[string]interface{}{ "spec": map[string]interface{}{ diff --git a/internal/deployer/deployer.go b/internal/deployer/deployer.go index 69b77c19..a30372a0 100644 --- a/internal/deployer/deployer.go +++ b/internal/deployer/deployer.go @@ -7,8 +7,10 @@ import ( "fmt" "os" "os/exec" + "strconv" "strings" "sync" + "syscall" "time" "github.com/fatih/color" @@ -64,6 +66,7 @@ type Deployer struct { securedClusterOverrides map[string]interface{} featureFlagOverrides map[string]interface{} envrcFile string + portForwardPID int useOLM bool useKonflux bool shouldDeployOperator bool @@ -434,6 +437,12 @@ func New(log *logger.Logger) (*Deployer, error) { d.roxCACertFile = caCert } + if pidStr := os.Getenv("ROXIE_PORT_FORWARD_PID"); pidStr != "" { + if pid, err := strconv.Atoi(pidStr); err == nil { + d.portForwardPID = pid + } + } + d.kubeContext = env.GetCurrentContext() clusterResourceKinds, err := d.getClusterResourceKinds() @@ -480,6 +489,22 @@ func (d *Deployer) Cleanup() { } } +func (d *Deployer) stopDetachedPortForward() { + if d.portForwardPID == 0 { + return + } + proc, err := os.FindProcess(d.portForwardPID) + if err != nil { + return + } + if err := proc.Signal(syscall.SIGKILL); err != nil { + d.logger.Dimf("Detached port-forward (pid %d) already gone", d.portForwardPID) + return + } + d.logger.Dimf("Stopped detached port-forward (pid %d)", d.portForwardPID) + d.portForwardPID = 0 +} + // Deploy deploys the specified components to the cluster. func (d *Deployer) Deploy(ctx context.Context, components component.Component, resources, exposure string) error { adjustedResources, adjustedExposure, adjustedPortForward := d.clusterDefaults.ApplyConvenienceDefaults( @@ -559,7 +584,6 @@ func (d *Deployer) deployCentral(ctx context.Context, resources, exposure string return err } - // envrc may be used from different processes, so use actual endpoint not port-forward if d.envrcFile != "" { d.logger.Dimf("Writing environment variables to %s", d.envrcFile) if err := d.writeEnvrcFile(ctx, exposure, portForwardWanted); err != nil { @@ -636,6 +660,7 @@ func (d *Deployer) teardownCentral(ctx context.Context) error { } d.portForward.Stop() + d.stopDetachedPortForward() // Add pause-reconcile annotation to not have the operator interfere during resource deletion. if d.doesResourceExist(ctx, "central", "stackrox-central-services", d.centralNamespace) { @@ -998,6 +1023,9 @@ func (d *Deployer) writeEnvrcFile(ctx context.Context, exposure string, portForw fmt.Fprintf(&content, "export ROX_USERNAME=%q\n", AdminUsername) fmt.Fprintf(&content, "export ROX_ADMIN_PASSWORD=%q\n", d.centralPassword) fmt.Fprintf(&content, "export ROX_CA_CERT_FILE=%q\n", d.roxCACertFile) + if d.portForwardPID != 0 { + fmt.Fprintf(&content, "export ROXIE_PORT_FORWARD_PID=%d\n", d.portForwardPID) + } if err := os.WriteFile(d.envrcFile, []byte(content.String()), 0600); err != nil { return fmt.Errorf("failed to write envrc file: %w", err) diff --git a/internal/portforward/portforward.go b/internal/portforward/portforward.go index 4dcbd0cf..b62f69cb 100644 --- a/internal/portforward/portforward.go +++ b/internal/portforward/portforward.go @@ -120,6 +120,51 @@ func (m *Manager) Start(namespace, serviceName string, remotePort, preferredLoca return endpoint, nil } +// StartDetached starts port-forward as a detached process that survives the +// parent process exiting. Returns the endpoint and the PID of the subprocess. +// The caller is responsible for killing the process when done. +func (m *Manager) StartDetached(namespace, serviceName string, remotePort, preferredLocalPort int) (string, int, error) { + localPort, err := m.findFreeLocalPort(preferredLocalPort) + if err != nil { + return "", 0, fmt.Errorf("failed to find free port: %w", err) + } + + cmd := exec.Command( + m.kubectl, + "-n", namespace, + "port-forward", + fmt.Sprintf("svc/%s", serviceName), + fmt.Sprintf("%d:%d", localPort, remotePort), + "--address", "127.0.0.1", + ) + + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setsid: true, + } + + cmd.Stdout = nil + cmd.Stderr = nil + + if err := cmd.Start(); err != nil { + return "", 0, fmt.Errorf("failed to start port-forward: %w", err) + } + + pid := cmd.Process.Pid + + // Release the process so it won't be waited on by this process. + cmd.Process.Release() + + if !m.waitTCPReady("127.0.0.1", localPort, 20.0) { + syscall.Kill(pid, syscall.SIGTERM) + return "", 0, fmt.Errorf("port-forward did not become ready") + } + + endpoint := fmt.Sprintf("127.0.0.1:%d", localPort) + m.logger.Successf("✓ Detached port-forward active at https://%s (pid %d)", endpoint, pid) + + return endpoint, pid, nil +} + // Stop stops the active port-forward if running func (m *Manager) Stop() { if m.proc == nil || m.proc.Process == nil { diff --git a/tests/e2e/basic_test.go b/tests/e2e/basic_test.go index ac91591f..e4f58ef6 100644 --- a/tests/e2e/basic_test.go +++ b/tests/e2e/basic_test.go @@ -22,7 +22,7 @@ func TestDeployBothSimple(t *testing.T) { envrcFile.Close() t.Log("=== Deploying both components together ===") - args := append([]string{roxieBinary, "deploy", "--early-readiness", "both", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + args := append([]string{roxieBinary, "deploy", "--early-readiness", "both", "--envrc", envrcPath}, commonDeployArgs...) runCommand(t, deployTimeout*2, nil, args...) // Verify namespaces exist and have managed-by labels diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index 425ec7e9..cb42af42 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -50,7 +50,7 @@ func TestDeployBothComponentsTogetherInSingleNamespace(t *testing.T) { envrcFile.Close() t.Log("=== Deploying both components in single namespace ===") - args := append([]string{roxieBinary, "deploy", "both", "--single-namespace", "--early-readiness", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + args := append([]string{roxieBinary, "deploy", "both", "--single-namespace", "--early-readiness", "--envrc", envrcPath}, commonDeployArgs...) runCommand(t, deployTimeout*2, nil, args...) verifyCentralInstalled(t, "stackrox") diff --git a/tests/e2e/helpers.go b/tests/e2e/helpers.go index 14c20fce..e913f7c9 100644 --- a/tests/e2e/helpers.go +++ b/tests/e2e/helpers.go @@ -26,12 +26,19 @@ const ( ) var ( - commonDeployArgs = []string{"--port-forwarding", "--exposure=none", "--resources=small"} - commonDeployArgsNoPortForward = []string{"--exposure=loadbalancer", "--resources=small"} + commonDeployArgs = deployArgsFromEnv() roxieBinary = "roxie" ) +func deployArgsFromEnv() []string { + profile := os.Getenv("ROXIE_RESOURCE_PROFILE") + if profile == "" { + profile = "small" + } + return []string{"--resources=" + profile} +} + func teardownAllDeployments() error { fmt.Println("=== Tearing down all deployments before running tests ===") diff --git a/tests/e2e/olm_switch_test.go b/tests/e2e/olm_switch_test.go index fe9512b0..0c6fa480 100644 --- a/tests/e2e/olm_switch_test.go +++ b/tests/e2e/olm_switch_test.go @@ -81,7 +81,7 @@ func TestOLMToNonOLMSwitch(t *testing.T) { // Step 1: Deploy central with OLM operator t.Log("=== Step 1: Deploy central with OLM operator ===") - args := append([]string{roxieBinary, "deploy", "central", "--olm", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + args := append([]string{roxieBinary, "deploy", "central", "--olm", "--envrc", envrcPath}, commonDeployArgs...) runCommand(t, deployTimeout, nil, args...) // Verify operator is in OLM mode @@ -94,7 +94,7 @@ func TestOLMToNonOLMSwitch(t *testing.T) { // Step 2: Deploy central again without OLM (should switch modes) t.Log("=== Step 2: Redeploy central without OLM (triggering mode switch) ===") - args = append([]string{roxieBinary, "deploy", "central", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + args = append([]string{roxieBinary, "deploy", "central", "--envrc", envrcPath}, commonDeployArgs...) runCommand(t, deployTimeout, nil, args...) // Verify operator switched to non-OLM mode @@ -131,7 +131,7 @@ func TestNonOLMToOLMSwitch(t *testing.T) { // Step 1: Deploy central without OLM (non-OLM operator) t.Log("=== Step 1: Deploy central with non-OLM operator ===") - args := append([]string{roxieBinary, "deploy", "central", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + args := append([]string{roxieBinary, "deploy", "central", "--envrc", envrcPath}, commonDeployArgs...) runCommand(t, deployTimeout, nil, args...) // Verify operator is in non-OLM mode @@ -144,7 +144,7 @@ func TestNonOLMToOLMSwitch(t *testing.T) { // Step 2: Deploy central again with OLM (should switch modes) t.Log("=== Step 2: Redeploy central with OLM (triggering mode switch) ===") - args = append([]string{roxieBinary, "deploy", "central", "--olm", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + args = append([]string{roxieBinary, "deploy", "central", "--olm", "--envrc", envrcPath}, commonDeployArgs...) runCommand(t, deployTimeout, nil, args...) // Verify operator switched to OLM mode @@ -185,7 +185,7 @@ func TestOLMOperatorVersionUpgrade(t *testing.T) { // Step 1: Deploy central with OLM operator t.Log("=== Step 1: Deploy central with OLM operator ===") - args := append([]string{roxieBinary, "deploy", "central", "--olm", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + args := append([]string{roxieBinary, "deploy", "central", "--olm", "--envrc", envrcPath}, commonDeployArgs...) runCommand(t, deployTimeout, nil, args...) // Verify operator is in OLM mode @@ -207,7 +207,7 @@ func TestOLMOperatorVersionUpgrade(t *testing.T) { // Step 2: Redeploy with same version (should skip if version matches) t.Log("=== Step 2: Redeploy with same version (should detect correct version) ===") - args = append([]string{roxieBinary, "deploy", "central", "--olm", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + args = append([]string{roxieBinary, "deploy", "central", "--olm", "--envrc", envrcPath}, commonDeployArgs...) runCommand(t, deployTimeout, nil, args...) // Verify operator is still in OLM mode and deployment exists @@ -245,7 +245,7 @@ func TestSecuredClusterWithOLMSwitch(t *testing.T) { // Step 1: Deploy central with OLM t.Log("=== Step 1: Deploy central with OLM ===") - args := append([]string{roxieBinary, "deploy", "--early-readiness", "central", "--olm", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) + args := append([]string{roxieBinary, "deploy", "--early-readiness", "central", "--olm", "--envrc", envrcPath}, commonDeployArgs...) runCommand(t, deployTimeout, nil, args...) verifyOperatorMode(t, true) @@ -259,7 +259,7 @@ func TestSecuredClusterWithOLMSwitch(t *testing.T) { // Step 2: Deploy secured-cluster (should reuse OLM operator) t.Log("=== Step 2: Deploy secured-cluster (should reuse OLM operator) ===") - args = append([]string{roxieBinary, "deploy", "--early-readiness", "secured-cluster", "--olm"}, commonDeployArgsNoPortForward...) + args = append([]string{roxieBinary, "deploy", "--early-readiness", "secured-cluster", "--olm"}, commonDeployArgs...) runCommand(t, deployTimeout, envrcEnv, args...) // Verify operator is still in OLM mode @@ -268,7 +268,7 @@ func TestSecuredClusterWithOLMSwitch(t *testing.T) { // Step 3: Switch to non-OLM by redeploying secured-cluster without --olm t.Log("=== Step 3: Redeploy secured-cluster without OLM (triggering mode switch) ===") - args = append([]string{roxieBinary, "deploy", "--early-readiness", "secured-cluster"}, commonDeployArgsNoPortForward...) + args = append([]string{roxieBinary, "deploy", "--early-readiness", "secured-cluster"}, commonDeployArgs...) runCommand(t, deployTimeout, envrcEnv, args...) // Verify operator switched to non-OLM mode