diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c9c0f751..977fc87f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ ### Fixed * Fix Tailor deployment drifts for D, Q envs ([#1055](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1055)) + +## [4.13.0] - 2026-06-30 +### Changed +* Push container images to AWS ECR independently of EKS, while optionally triggering EKS deployments when needed ([#1267](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1267)) +* Updated Java version in sonarqube execution to Java 21 ([#1269](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1269)) + +### Fixed * Fix branch mapping logic to ensure correct environment assignment ([#1263](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1263)) * Fix Nexus not accepting npm artifacts when uploaded through odsComponentStageUploadToNexus ([#1268](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1268)) * Aqua Stage is being skipped when branches are not eligable ([#1170](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1170)) @@ -46,6 +53,7 @@ ### Fixed * Fix SonarQube run enable/disable logic ([#1259](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1259)) + ## [4.11.1] - 2025-12-19 ### Fixed diff --git a/docs/modules/jenkins-shared-library/partials/odsComponentStageRolloutOpenShiftDeployment.adoc b/docs/modules/jenkins-shared-library/partials/odsComponentStageRolloutOpenShiftDeployment.adoc index 28686ee80..8a7c2d56e 100644 --- a/docs/modules/jenkins-shared-library/partials/odsComponentStageRolloutOpenShiftDeployment.adoc +++ b/docs/modules/jenkins-shared-library/partials/odsComponentStageRolloutOpenShiftDeployment.adoc @@ -61,6 +61,19 @@ In addition to the configuration options below, one can use e.g. a `Tailorfile` | Option | Description +| *awsEnvPath* + +_String_ +|Path to AWS environment configuration env files (defaults to `./environments`). + Files (dev.yml, test.yaml, prod.yaml) must contain the following keys: + account: "..." + eksCluster: "..." + region: ... + credentials: + key: ... (e.g. projectName-cd-aws-us-access-key-id-dev) + secret: ... (e.g. projectName-cd-aws-us-secret-access-key-dev) + Only relevant for AWS-based deployments (odsComponentStageRolloutAWSDeployment) + + | *branch* + _String_ |Branch to run stage for. @@ -165,6 +178,14 @@ _List_ directory referenced by `chartDir` exists. +| *helmWithOnlyECR* + +_Boolean_ +|Whether to use Helm for deployment or only sync with ECR (defaults to `false`) + If `true`, the process will only use the ECR repository to store the image. So, no Helm deployment will be done. + If `false`, the image will be stored in ECR and the helm deployment will be executed. + Only relevant for AWS-based deployments (odsComponentStageRolloutAWSDeployment) + + | *imageTag* + _String_ |Image tag on which to apply the `latest` tag (defaults to `context.shortGitCommit`). diff --git a/src/org/ods/component/AbstractDeploymentStrategy.groovy b/src/org/ods/component/AbstractDeploymentStrategy.groovy index b7c3f6486..d282f8d85 100644 --- a/src/org/ods/component/AbstractDeploymentStrategy.groovy +++ b/src/org/ods/component/AbstractDeploymentStrategy.groovy @@ -37,23 +37,6 @@ abstract class AbstractDeploymentStrategy implements IDeploymentStrategy { originalVersions } - protected findOrCreateImageStream(String targetProject, String image) { - try { - openShift.findOrCreateImageStream(targetProject, image) - } catch (Exception ex) { - steps.error "Could not find/create ImageStream ${image} in ${targetProject}. Error was: ${ex}" - } - } - - protected void retagImages(String targetProject, Set images) { - images.each { image -> - findOrCreateImageStream(targetProject, image) - openShift.importImageTagFromProject( - targetProject, image, context.cdProject, options.imageTag, options.imageTag - ) - } - } - @TypeChecked(TypeCheckingMode.SKIP) protected Set getBuiltImages() { context.buildArtifactURIs.builds.keySet().findAll { it -> diff --git a/src/org/ods/component/Context.groovy b/src/org/ods/component/Context.groovy index b7fb073fb..70407744a 100644 --- a/src/org/ods/component/Context.groovy +++ b/src/org/ods/component/Context.groovy @@ -495,7 +495,7 @@ class Context implements IContext { // https://issues.jenkins-ci.org/browse/JENKINS-27421 and // https://issues.jenkins-ci.org/browse/JENKINS-35191. for (def key : config.branchToEnvironmentMapping.keySet()) { - if (config.gitBranch.startsWith(key)) { + if (config.gitBranch.startsWith(key) && key.endsWith('/')) { setMostSpecificEnvironment( config.branchToEnvironmentMapping[key], config.gitBranch.replace(key, '') diff --git a/src/org/ods/component/ECROnlyDeploymentStrategy.groovy b/src/org/ods/component/ECROnlyDeploymentStrategy.groovy new file mode 100644 index 000000000..a1eb7062a --- /dev/null +++ b/src/org/ods/component/ECROnlyDeploymentStrategy.groovy @@ -0,0 +1,43 @@ +package org.ods.component + +import org.ods.util.ILogger +import org.ods.util.PodData + +/** + * Deployment strategy that only pushes images to ECR without performing a + * Helm upgrade or any OpenShift/EKS rollout. Use this when images must be + * synchronised to ECR but the actual cluster deployment is handled elsewhere. + */ +class ECROnlyDeploymentStrategy extends AbstractDeploymentStrategy { + + private final IContext context + private final ILogger logger + private final IImageRepository imageRepository + private final RolloutOpenShiftDeploymentOptions options + + ECROnlyDeploymentStrategy( + IContext context, + Map config, + IImageRepository imageRepository, + ILogger logger + ) { + this.context = context + this.logger = logger + this.imageRepository = imageRepository + this.options = new RolloutOpenShiftDeploymentOptions(config) + } + + @Override + Map> deploy() { + def targetProject = context.targetProject + if (options.helmValues && options.helmValues['namespaceOverride']) { + targetProject = options.helmValues['namespaceOverride'] + logger.info("Override namespace deployment to ${targetProject}") + } + logger.info("Retagging images for ${targetProject}") + imageRepository.retagImages(targetProject, getBuiltImages(), options.imageTag, options.imageTag) + logger.info("ECR-only deployment configured, skipping Helm/OpenShift rollout.") + return [:] + } + +} diff --git a/src/org/ods/component/HelmDeploymentConfig.groovy b/src/org/ods/component/HelmDeploymentConfig.groovy new file mode 100644 index 000000000..bb78e50ea --- /dev/null +++ b/src/org/ods/component/HelmDeploymentConfig.groovy @@ -0,0 +1,63 @@ +package org.ods.component + +import com.cloudbees.groovy.cps.NonCPS +import groovy.transform.TypeChecked +import groovy.transform.TypeCheckingMode + +/** + * Centralises the default configuration values shared by Helm-based deployment + * stages and strategies. Call {@link #applyDefaults} once with the pipeline + * context and the user-supplied config map; values that were not explicitly set + * by the user will be filled in with sensible defaults. + */ +@TypeChecked +class HelmDeploymentConfig { + + @NonCPS + @TypeChecked(TypeCheckingMode.SKIP) + static void applyDefaults(IContext context, Map config) { + if (!config.selector) { + config.selector = context.selector + } + if (!config.imageTag) { + config.imageTag = context.shortGitCommit + } + if (!config.deployTimeoutMinutes) { + config.deployTimeoutMinutes = context.openshiftRolloutTimeout ?: 15 + } + if (!config.deployTimeoutRetries) { + config.deployTimeoutRetries = context.openshiftRolloutTimeoutRetries ?: 5 + } + if (!config.chartDir) { + config.chartDir = 'chart' + } + if (!config.containsKey('helmReleaseName')) { + config.helmReleaseName = context.componentId + } + if (!config.containsKey('helmValues')) { + config.helmValues = [:] + } + if (!config.containsKey('helmValuesFiles')) { + config.helmValuesFiles = ['values.yaml'] + } + if (!config.containsKey('helmEnvBasedValuesFiles')) { + config.helmEnvBasedValuesFiles = [] + } + if (!config.containsKey('helmDefaultFlags')) { + config.helmDefaultFlags = ['--install', '--atomic'] + } + if (!config.containsKey('helmAdditionalFlags')) { + config.helmAdditionalFlags = [] + } + if (!config.containsKey('helmDiff')) { + config.helmDiff = true + } + if (!config.helmPrivateKeyCredentialsId) { + config.helmPrivateKeyCredentialsId = "${context.cdProject}-helm-private-key" + } + if (!config.containsKey('helmReleasesHistoryLimit')) { + config.helmReleasesHistoryLimit = 5 + } + } + +} diff --git a/src/org/ods/component/HelmDeploymentStrategy.groovy b/src/org/ods/component/HelmDeploymentStrategy.groovy index 3e28d7bb1..076d5448a 100644 --- a/src/org/ods/component/HelmDeploymentStrategy.groovy +++ b/src/org/ods/component/HelmDeploymentStrategy.groovy @@ -2,6 +2,7 @@ package org.ods.component import groovy.transform.TypeChecked import groovy.transform.TypeCheckingMode +import org.ods.orchestration.util.MROPipelineUtil import org.ods.services.JenkinsService import org.ods.services.OpenShiftService import org.ods.util.HelmStatus @@ -17,6 +18,8 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy { private final JenkinsService jenkins private final ILogger logger private final IPipelineSteps steps + private final IImageRepository imageRepository + // assigned in constructor private final RolloutOpenShiftDeploymentOptions options @@ -27,57 +30,17 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy { Map config, OpenShiftService openShift, JenkinsService jenkins, + IImageRepository imageRepository, ILogger logger ) { - if (!config.selector) { - config.selector = context.selector - } - if (!config.imageTag) { - config.imageTag = context.shortGitCommit - } - if (!config.deployTimeoutMinutes) { - config.deployTimeoutMinutes = context.openshiftRolloutTimeout ?: 15 - } - if (!config.deployTimeoutRetries) { - config.deployTimeoutRetries = context.openshiftRolloutTimeoutRetries ?: 5 - } - // Helm options - if (!config.chartDir) { - config.chartDir = 'chart' - } - if (!config.containsKey('helmReleaseName')) { - config.helmReleaseName = context.componentId - } - if (!config.containsKey('helmValues')) { - config.helmValues = [:] - } - if (!config.containsKey('helmValuesFiles')) { - config.helmValuesFiles = ['values.yaml'] - } - if (!config.containsKey('helmEnvBasedValuesFiles')) { - config.helmEnvBasedValuesFiles = [] - } - // helmDefaultFlags are always set and cannot be overridden - config.helmDefaultFlags = ['--install', '--atomic'] - if (!config.containsKey('helmAdditionalFlags')) { - config.helmAdditionalFlags = [] - } - if (!config.containsKey('helmDiff')) { - config.helmDiff = true - } - if (!config.helmPrivateKeyCredentialsId) { - config.helmPrivateKeyCredentialsId = "${context.cdProject}-helm-private-key" - } - if (!config.containsKey('helmReleasesHistoryLimit')) { - config.helmReleasesHistoryLimit = 5 - } + HelmDeploymentConfig.applyDefaults(context, config) this.context = context this.logger = logger this.steps = steps - this.options = new RolloutOpenShiftDeploymentOptions(config) this.openShift = openShift this.jenkins = jenkins + this.imageRepository = imageRepository } @Override @@ -86,13 +49,19 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy { logger.warn 'Skipping because of empty (target) environment ...' return [:] } - - // Tag images which have been built in this pipeline from cd project into target project - retagImages(context.targetProject, getBuiltImages()) + // Maybe we need to deploy to another namespace + // (ie we want to deploy a monitoring stack into a specific namespace) + def targetProject = context.targetProject + if (options.helmValues['namespaceOverride']) { + targetProject = options.helmValues['namespaceOverride'] + logger.info("Override namespace deployment to ${targetProject} ") + } + logger.info("Retagging images for ${targetProject} ") + imageRepository.retagImages(targetProject, getBuiltImages(), options.imageTag, options.imageTag) logger.info("Rolling out ${context.componentId} with HELM, selector: ${options.selector}") - helmUpgrade(context.targetProject) - HelmStatus helmStatus = openShift.helmStatus(context.targetProject, options.helmReleaseName) + helmUpgrade(targetProject) + HelmStatus helmStatus = openShift.helmStatus(targetProject, options.helmReleaseName) if (logger.debugMode) { def helmStatusMap = helmStatus.toMap() logger.debug("${this.class.name} -- HELM STATUS: ${helmStatusMap}") @@ -102,8 +71,12 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy { // // we assume that Helm does "Deployment" that should work for most // // cases since they don't have triggers. // metadataSvc.updateMetadata(false, deploymentResources) - def rolloutData = getRolloutData(helmStatus) - return rolloutData + Map> rolloutData = [:] + if (steps.env.repoType == MROPipelineUtil.PipelineConfig.REPO_TYPE_ODS_INFRA) { + return rolloutData + } else { + return getRolloutData(helmStatus, targetProject) + } } private void helmUpgrade(String targetProject) { @@ -167,7 +140,7 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy { // ] @TypeChecked(TypeCheckingMode.SKIP) private Map> getRolloutData( - HelmStatus helmStatus + HelmStatus helmStatus, String targetProject ) { Map> rolloutData = [:] @@ -183,7 +156,7 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy { names.each { name -> // Get container images directly from the resource spec (deployment/statefulset/cronjob) def containers = openShift.getContainerImagesWithNameFromPodSpec( - context.targetProject, kind, name + targetProject, kind, name ) logger.debug("Helm container images for ${kind}/${name}: ${containers}") @@ -201,7 +174,7 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy { def deploymentMean = [ type: 'helm', selector: options.selector, - namespace: context.targetProject, + namespace: targetProject, chartDir: options.chartDir, helmReleaseName: options.helmReleaseName, helmEnvBasedValuesFiles: options.helmEnvBasedValuesFiles, @@ -252,4 +225,4 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy { return rolloutData } -} \ No newline at end of file +} diff --git a/src/org/ods/component/IImageRepository.groovy b/src/org/ods/component/IImageRepository.groovy new file mode 100644 index 000000000..a9992c9b4 --- /dev/null +++ b/src/org/ods/component/IImageRepository.groovy @@ -0,0 +1,7 @@ +package org.ods.component + +interface IImageRepository { + + void retagImages(String targetProject, Set images, String sourceTag, String targetTag) + +} diff --git a/src/org/ods/component/ImageECR.groovy b/src/org/ods/component/ImageECR.groovy new file mode 100644 index 000000000..649bb424a --- /dev/null +++ b/src/org/ods/component/ImageECR.groovy @@ -0,0 +1,92 @@ +package org.ods.component + +import org.ods.util.IPipelineSteps +import org.ods.util.ILogger + +class ImageECR implements IImageRepository { + + // Constructor arguments + private final IPipelineSteps steps + private final IContext context + private final ILogger logger + private final Map awsEnvironmentVars + private String ocToken + + @SuppressWarnings(['AbcMetric', 'CyclomaticComplexity', 'ParameterCount']) + ImageECR( + IPipelineSteps steps, + IContext context, + ILogger logger, + Map awsEnvironmentVars + ) { + this.steps = steps + this.context = context + this.logger = logger + this.awsEnvironmentVars = awsEnvironmentVars + } + + void fetchOCToken() { + this.ocToken = getOCToken() + } + + public void retagImages(String targetProject, Set images, String sourceTag, String targetTag) { + logger.debug("retagImages called with targetProject=${targetProject}, images=${images}") + images.each { image -> + createRepository(image) + copyImage(image, context, sourceTag, targetTag) + } + } + + private void createRepository(String repositoryName) { + def createRepoCmd = "aws ecr create-repository --repository-name" + executeCommand("${createRepoCmd} ${repositoryName} --region ${awsEnvironmentVars.region}", false) + } + + private int copyImage(image, context, sourceTag, targetTag) { + String ocCredentials = "jenkins:${this.ocToken}" + String awsCredentials = "AWS:${getAWSPassword()}" + String dockerSource = "docker://${context.config.dockerRegistry}/${context.cdProject}/${image}:${sourceTag}" + String awsTarget = "docker://${getECRRegistry()}/${image}:${targetTag}" + + return steps.sh( + script: """ + skopeo copy \ + --src-tls-verify=false --src-creds "${ocCredentials}"\ + --dest-tls-verify=false --dest-creds "${awsCredentials}"\ + $dockerSource $awsTarget + """, + returnStatus: true, + label: "Copy image to awsTarget ${awsTarget}" + ) as int + } + + private String getOCToken() { + return steps.sh( + script: "oc whoami -t", + returnStdout: true + ).trim() + } + + private String getAWSPassword() { + return steps.sh( + script: "aws ecr get-login-password --region ${awsEnvironmentVars.region}", + returnStdout: true + ).trim() + } + + private String getECRRegistry() { + return "${awsEnvironmentVars.account}.dkr.ecr.${awsEnvironmentVars.region}.amazonaws.com" + } + + private void executeCommand(String command, boolean showError = true) { + def status = steps.sh( + script: command, + returnStatus: true, + label: "Executing command: ${command}" + ) as int + if (status != 0 && showError) { + steps.error("Error executing ${command}, status ${status}") + } + } + +} diff --git a/src/org/ods/component/ImageRepositoryOpenshift.groovy b/src/org/ods/component/ImageRepositoryOpenshift.groovy new file mode 100644 index 000000000..899e51062 --- /dev/null +++ b/src/org/ods/component/ImageRepositoryOpenshift.groovy @@ -0,0 +1,41 @@ +package org.ods.component + +import org.ods.services.OpenShiftService +import org.ods.util.IPipelineSteps + +class ImageRepositoryOpenshift implements IImageRepository { + + // Constructor arguments + private final IPipelineSteps steps + private final IContext context + private final OpenShiftService openShift + + @SuppressWarnings(['AbcMetric', 'CyclomaticComplexity', 'ParameterCount']) + ImageRepositoryOpenshift( + IPipelineSteps steps, + IContext context, + OpenShiftService openShift + ) { + this.steps = steps + this.context = context + this.openShift = openShift + } + + void retagImages(String targetProject, Set images, String sourceTag, String targetTag) { + images.each { image -> + findOrCreateImageStream(targetProject, image) + openShift.importImageTagFromProject( + targetProject, image, context.cdProject, sourceTag, targetTag + ) + } + } + + private findOrCreateImageStream(String targetProject, String image) { + try { + openShift.findOrCreateImageStream(targetProject, image) + } catch (Exception ex) { + steps.error "Could not find/create ImageStream ${image} in ${targetProject}. Error was: ${ex}" + } + } + +} diff --git a/src/org/ods/component/RolloutAWSDeploymentStage.groovy b/src/org/ods/component/RolloutAWSDeploymentStage.groovy new file mode 100644 index 000000000..f7fc68981 --- /dev/null +++ b/src/org/ods/component/RolloutAWSDeploymentStage.groovy @@ -0,0 +1,66 @@ +package org.ods.component + +import groovy.transform.TypeChecked +import groovy.transform.TypeCheckingMode +import org.ods.services.AWSService +import org.ods.services.JenkinsService +import org.ods.services.OpenShiftService +import org.ods.util.ILogger + +@SuppressWarnings('ParameterCount') +@TypeChecked + class RolloutAWSDeploymentStage extends Stage { + + public final String STAGE_NAME = 'Deploy to OpenShift' + private final OpenShiftService openShift + private final JenkinsService jenkins + private final RolloutOpenShiftDeploymentOptions options + private IDeploymentStrategy depStr + private Map config + private Map awsEnvironmentVars + + @TypeChecked(TypeCheckingMode.SKIP) + RolloutAWSDeploymentStage( + def script, + IContext context, + Map config, + OpenShiftService openShift, + JenkinsService jenkins, + Map awsEnvironmentVars, + ILogger logger) { + super(script, context, logger) + HelmDeploymentConfig.applyDefaults(context, config) + this.config = config + this.options = new RolloutOpenShiftDeploymentOptions(config) + this.openShift = openShift + this.jenkins = jenkins + this.awsEnvironmentVars = awsEnvironmentVars + } + + // This is called from Stage#execute if the branch being built is eligible. + protected run() { + ImageECR imageECR = new ImageECR(steps, context, logger, awsEnvironmentVars) + imageECR.fetchOCToken() + AWSService aWSService = new AWSService(steps, context, awsEnvironmentVars, logger) + aWSService.awsLogin() + if (config.helmWithOnlyECR) { + logger.info('ECR-only deployment configured (no EKS rollout)') + depStr = new ECROnlyDeploymentStrategy(context, config, imageECR, logger) + } else { + logger.info('Full EKS deployment configured') + aWSService.setEKSCluster() + depStr = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins, imageECR, logger) + } + + logger.info("deploymentStrategy: ${depStr} -- ${depStr.class.name}") + return depStr.deploy() + } + + protected String stageLabel() { + if (options.selector != context.selector) { + return "${STAGE_NAME} (${options.selector})" + } + STAGE_NAME + } + +} diff --git a/src/org/ods/component/RolloutOpenShiftDeploymentOptions.groovy b/src/org/ods/component/RolloutOpenShiftDeploymentOptions.groovy index 9f5264429..bc4fb5b43 100644 --- a/src/org/ods/component/RolloutOpenShiftDeploymentOptions.groovy +++ b/src/org/ods/component/RolloutOpenShiftDeploymentOptions.groovy @@ -134,4 +134,25 @@ class RolloutOpenShiftDeploymentOptions extends Options { * relevant if the directory referenced by `openshiftDir` exists. */ List tailorParams + /** + * Path to AWS environment configuration env files (defaults to `./environments`). + * Files (dev.yml, test.yaml, prod.yaml) must contain the following keys: + * account: "..." + * eksCluster: "..." + * region: ... + * credentials: + * key: ... (e.g. projectName-cd-aws-us-access-key-id-dev) + * secret: ... (e.g. projectName-cd-aws-us-secret-access-key-dev) + * Only relevant for AWS-based deployments (odsComponentStageRolloutAWSDeployment) + */ + String awsEnvPath + + /** + * Whether to use Helm for deployment or only sync with ECR (defaults to `false`) + * If `true`, the process will only use the ECR repository to store the image. So, no Helm deployment will be done. + * If `false`, the image will be stored in ECR and the helm deployment will be executed. + * Only relevant for AWS-based deployments (odsComponentStageRolloutAWSDeployment) + */ + Boolean helmWithOnlyECR + } diff --git a/src/org/ods/component/RolloutOpenShiftDeploymentStage.groovy b/src/org/ods/component/RolloutOpenShiftDeploymentStage.groovy index 7f8607811..0d1bcfe68 100644 --- a/src/org/ods/component/RolloutOpenShiftDeploymentStage.groovy +++ b/src/org/ods/component/RolloutOpenShiftDeploymentStage.groovy @@ -29,48 +29,7 @@ class RolloutOpenShiftDeploymentStage extends Stage { JenkinsService jenkins, ILogger logger) { super(script, context, logger) - - if (!config.selector) { - config.selector = context.selector - } - if (!config.imageTag) { - config.imageTag = context.shortGitCommit - } - if (!config.deployTimeoutMinutes) { - config.deployTimeoutMinutes = context.openshiftRolloutTimeout ?: 15 - } - if (!config.deployTimeoutRetries) { - config.deployTimeoutRetries = context.openshiftRolloutTimeoutRetries ?: 5 - } - // Helm options - if (!config.chartDir) { - config.chartDir = 'chart' - } - if (!config.containsKey('helmReleaseName')) { - config.helmReleaseName = context.componentId - } - if (!config.containsKey('helmValues')) { - config.helmValues = [:] - } - if (!config.containsKey('helmValuesFiles')) { - config.helmValuesFiles = [ 'values.yaml' ] - } - if (!config.containsKey('helmEnvBasedValuesFiles')) { - config.helmEnvBasedValuesFiles = [] - } - if (!config.containsKey('helmDefaultFlags')) { - config.helmDefaultFlags = ['--install', '--atomic'] - } - if (!config.containsKey('helmAdditionalFlags')) { - config.helmAdditionalFlags = [] - } - if (!config.containsKey('helmDiff')) { - config.helmDiff = true - } - if (!config.helmPrivateKeyCredentialsId) { - config.helmPrivateKeyCredentialsId = "${context.cdProject}-helm-private-key" - } - // Tailor options + HelmDeploymentConfig.applyDefaults(context, config) if (!config.openshiftDir) { config.openshiftDir = 'openshift' } @@ -121,18 +80,21 @@ class RolloutOpenShiftDeploymentStage extends Stage { throw new IllegalStateException("Must be either a Tailor based deployment or a Helm based deployment") } + IPipelineSteps steps = new PipelineSteps(script) + IImageRepository imgRepo = new ImageRepositoryOpenshift(steps, context, openShift) + // Use tailorDeployment in the following cases: // (1) We have an openshiftDir // (2) We do not have an openshiftDir but neither do we have an indication that it is Helm - IPipelineSteps steps = new PipelineSteps(script) if (isTailorDeployment || (!isHelmDeployment && !isTailorDeployment)) { - deploymentStrategy = new TailorDeploymentStrategy(steps, context, config, openShift, jenkins, logger) + deploymentStrategy = new TailorDeploymentStrategy( + steps, context, config, openShift, jenkins, imgRepo, logger) String resourcePath = 'org/ods/component/RolloutOpenShiftDeploymentStage.deprecate-tailor.GString.txt' def msg = this.steps.libraryResource(resourcePath) logger.warn(msg) } if (isHelmDeployment) { - deploymentStrategy = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins, logger) + deploymentStrategy = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins, imgRepo, logger) } logger.info("deploymentStrategy: ${deploymentStrategy} -- ${deploymentStrategy.class.name}") return deploymentStrategy.deploy() diff --git a/src/org/ods/component/TailorDeploymentStrategy.groovy b/src/org/ods/component/TailorDeploymentStrategy.groovy index 6d8bbece7..88d557f0b 100644 --- a/src/org/ods/component/TailorDeploymentStrategy.groovy +++ b/src/org/ods/component/TailorDeploymentStrategy.groovy @@ -16,6 +16,7 @@ class TailorDeploymentStrategy extends AbstractDeploymentStrategy { private final JenkinsService jenkins private final ILogger logger private final IPipelineSteps steps + private final IImageRepository imageRepository // assigned in constructor private final RolloutOpenShiftDeploymentOptions options @@ -27,6 +28,7 @@ class TailorDeploymentStrategy extends AbstractDeploymentStrategy { Map config, OpenShiftService openShift, JenkinsService jenkins, + IImageRepository imageRepository, ILogger logger ) { if (!config.selector) { @@ -73,6 +75,7 @@ class TailorDeploymentStrategy extends AbstractDeploymentStrategy { this.options = new RolloutOpenShiftDeploymentOptions(config) this.openShift = openShift this.jenkins = jenkins + this.imageRepository = imageRepository } @Override @@ -96,7 +99,7 @@ class TailorDeploymentStrategy extends AbstractDeploymentStrategy { def refreshResources = false // Tag images which have been built in this pipeline from cd project into target project - retagImages(context.targetProject, getBuiltImages()) + imageRepository.retagImages(context.targetProject, getBuiltImages(), options.imageTag, options.imageTag) if (steps.fileExists(options.openshiftDir)) { refreshResources = true diff --git a/src/org/ods/component/UploadToNexusStage.groovy b/src/org/ods/component/UploadToNexusStage.groovy index 2803553f0..2b33a732d 100644 --- a/src/org/ods/component/UploadToNexusStage.groovy +++ b/src/org/ods/component/UploadToNexusStage.groovy @@ -55,6 +55,8 @@ class UploadToNexusStage extends Stage { nexusParams << ['raw.directory': (options.targetDirectory ?: context.projectId)] } else if (options.repositoryType == 'pypi') { nexusParams << ['pypi.asset': options.distributionFile] + } else if (options.repositoryType == 'npm') { + nexusParams << ['npm.asset': options.distributionFile] } def data = steps.readFile(file: options.distributionFile, encoding: 'Base64').toString().getBytes() diff --git a/src/org/ods/services/AWSService.groovy b/src/org/ods/services/AWSService.groovy new file mode 100644 index 000000000..08cbef6b1 --- /dev/null +++ b/src/org/ods/services/AWSService.groovy @@ -0,0 +1,67 @@ +package org.ods.services + +import org.ods.util.IPipelineSteps +import org.ods.component.IContext +import org.ods.util.ILogger + +class AWSService { + + // Constructor arguments + private final IPipelineSteps steps + private final IContext context + private final Map awsEnvironmentVars + private final ILogger logger + + @SuppressWarnings(['AbcMetric', 'CyclomaticComplexity', 'ParameterCount']) + AWSService( + IPipelineSteps steps, + IContext context, + Map awsEnvironmentVars, + ILogger logger + ) { + this.steps = steps + this.context = context + this.awsEnvironmentVars = awsEnvironmentVars + this.logger = logger + } + + void awsLogin() { + withCredentials((awsEnvironmentVars.credentials.key as String).toLowerCase(), + (awsEnvironmentVars.credentials.secret as String).toLowerCase()) { + executeCommand('aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default') + executeCommand('aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default') + executeCommand("aws configure set region ${awsEnvironmentVars.region} --profile default") + } + } + + void setEKSCluster() { + withCredentials((awsEnvironmentVars.credentials.key as String).toLowerCase(), + (awsEnvironmentVars.credentials.secret as String).toLowerCase()) { + executeCommand("aws eks list-clusters") + def updateCommand = "aws eks update-kubeconfig --region" + executeCommand("${updateCommand} ${awsEnvironmentVars.region} --name ${awsEnvironmentVars.eksCluster}") + executeCommand("kubectl create namespace ${context.getProjectId()}-${context.getEnvironment()}", false) + } + } + + private void executeCommand(String command, boolean showError = true) { + def status = steps.sh( + script: command, + returnStatus: true, + label: "Executing command: ${command}" + ) as int + if (status != 0 && showError) { + steps.error("Error executing ${command}, status ${status}") + } + } + + private withCredentials(String awsAccessKeyId, String awsSecretAccessKey, Closure block) { + steps.withCredentials([ + steps.string(credentialsId: awsAccessKeyId, variable: 'AWS_ACCESS_KEY_ID'), + steps.string(credentialsId: awsSecretAccessKey, variable: 'AWS_SECRET_ACCESS_KEY') + ]) { + block() + } + } + +} diff --git a/src/org/ods/services/NexusService.groovy b/src/org/ods/services/NexusService.groovy index cfc0f8805..87c7d3fe3 100644 --- a/src/org/ods/services/NexusService.groovy +++ b/src/org/ods/services/NexusService.groovy @@ -146,8 +146,12 @@ class NexusService { new ByteArrayInputStream(artifact), ContentType.create(contentType), nexusParams['pypi.asset'].substring( nexusParams['pypi.asset'].lastIndexOf("/") + 1 )) - } - else { + } else if (repositoryType == 'npm') { + restCall = restCall.field("npm.asset", + new ByteArrayInputStream(artifact), + ContentType.create(contentType), + nexusParams['npm.asset']) + } else { restCall = restCall.field( repositoryType == 'raw' || repositoryType == 'maven2' ? "${repositoryType}.asset1" : "${repositoryType}.asset", new ByteArrayInputStream(artifact), contentType) diff --git a/src/org/ods/services/OpenShiftService.groovy b/src/org/ods/services/OpenShiftService.groovy index e38347e01..f23687549 100644 --- a/src/org/ods/services/OpenShiftService.groovy +++ b/src/org/ods/services/OpenShiftService.groovy @@ -90,7 +90,7 @@ class OpenShiftService { throw new RuntimeException ("Route does not contain a dot: ${routeUrl}") } - def openShiftPublicHost = routeUrl[routePrefixLength+1..-1] + def openShiftPublicHost = routeUrl[routePrefixLength + 1..-1] return openShiftPublicHost } @@ -866,7 +866,7 @@ class OpenShiftService { * @param patch a Map specifying the patch to apply. * @param path the optional absolute path at which to apply the patch. * @param project the namespace of the resource. Default: null (the current project). - * @return + * @return String */ String patch(String resource, Map patch, String path = null, String project = null) { if (!resource) { @@ -1087,9 +1087,18 @@ class OpenShiftService { @TypeChecked(TypeCheckingMode.SKIP) List parsePodJson(podJson, String resourceName = null) { List pods = [] - if (podJson && podJson.items.collect { it.status?.phase?.toLowerCase() }.every { it == 'running' }) { + if (podJson && + podJson.items.collect { it.status?.phase?.toLowerCase() }.every { it == 'running' || it == 'succeeded' }) { // If we got passed a resourceName we need to collect all the pod data from each pod pods = extractPodData(podJson) + } else { + logger.debug("Not all pods are in 'running' state or succeeded") + podJson.items.collect { pod -> + [name: pod.metadata?.name, phase: pod.status?.phase?.toLowerCase()] + if (pod.status?.phase?.toLowerCase() != 'running') { + logger.debug("Pod ${pod.metadata?.name} is in phase '${pod.status?.phase}'") + } + } } // if we have a resourceName only return the items matching that if (resourceName != null) { @@ -1499,4 +1508,5 @@ class OpenShiftService { ) } } + } diff --git a/src/org/ods/services/SonarQubeService.groovy b/src/org/ods/services/SonarQubeService.groovy index 1d24ce3db..c7de1da30 100644 --- a/src/org/ods/services/SonarQubeService.groovy +++ b/src/org/ods/services/SonarQubeService.groovy @@ -83,8 +83,8 @@ class SonarQubeService { scannerParams << "-Dsonar.branch.name=${properties['sonar.branch.name']}" } script.sh( - label: 'Set Java 17 for SonarQube scan', - script: "source use-j17.sh" + label: 'Set Java 21 for SonarQube scan', + script: "source use-j21.sh" ) script.sh( label: 'Run SonarQube scan', diff --git a/test/groovy/org/ods/component/ECROnlyDeploymentStrategySpec.groovy b/test/groovy/org/ods/component/ECROnlyDeploymentStrategySpec.groovy new file mode 100644 index 000000000..bd2965460 --- /dev/null +++ b/test/groovy/org/ods/component/ECROnlyDeploymentStrategySpec.groovy @@ -0,0 +1,85 @@ +package org.ods.component + +import org.ods.util.ILogger +import org.ods.util.PodData +import spock.lang.Specification + +class ECROnlyDeploymentStrategySpec extends Specification { + + IContext context + IImageRepository imageRepository + ILogger logger + + def setup() { + context = Stub(IContext) { + getTargetProject() >> 'foo-dev' + getComponentId() >> 'component-a' + } + imageRepository = Mock(IImageRepository) + logger = Mock(ILogger) + } + + private ECROnlyDeploymentStrategy createStrategy(Map config = [:]) { + // Apply minimal config defaults so RolloutOpenShiftDeploymentOptions can be constructed + if (!config.containsKey('selector')) { config.selector = 'app=component-a' } + if (!config.containsKey('imageTag')) { config.imageTag = 'abc12345' } + if (!config.containsKey('helmValues')) { config.helmValues = [:] } + if (!config.containsKey('helmValuesFiles')) { config.helmValuesFiles = ['values.yaml'] } + if (!config.containsKey('helmEnvBasedValuesFiles')) { config.helmEnvBasedValuesFiles = [] } + if (!config.containsKey('helmDefaultFlags')) { config.helmDefaultFlags = ['--install', '--atomic'] } + if (!config.containsKey('helmAdditionalFlags')) { config.helmAdditionalFlags = [] } + if (!config.containsKey('helmDiff')) { config.helmDiff = true } + new ECROnlyDeploymentStrategy(context, config, imageRepository, logger) + } + + def "deploy retags images and returns empty rollout data"() { + given: + context.getBuildArtifactURIs() >> [builds: [image1: [:], image2: [:]]] + def strategy = createStrategy() + + when: + Map> result = strategy.deploy() + + then: + 1 * imageRepository.retagImages('foo-dev', { it.containsAll(['image1', 'image2']) && it.size() == 2 }, 'abc12345', 'abc12345') + result == [:] + } + + def "deploy uses namespaceOverride when set"() { + given: + context.getBuildArtifactURIs() >> [builds: [myapp: [:]]] + def strategy = createStrategy([helmValues: [namespaceOverride: 'custom-ns']]) + + when: + Map> result = strategy.deploy() + + then: + 1 * imageRepository.retagImages('custom-ns', _ as Set, 'abc12345', 'abc12345') + result == [:] + } + + def "deploy handles empty builds"() { + given: + context.getBuildArtifactURIs() >> [builds: [:]] + def strategy = createStrategy() + + when: + Map> result = strategy.deploy() + + then: + 1 * imageRepository.retagImages('foo-dev', { it.isEmpty() }, 'abc12345', 'abc12345') + result == [:] + } + + def "deploy filters out imported images"() { + given: + context.getBuildArtifactURIs() >> [builds: ['myapp': [:], 'imported-base': [:]]] + def strategy = createStrategy() + + when: + strategy.deploy() + + then: + 1 * imageRepository.retagImages('foo-dev', { it.contains('myapp') && !it.contains('imported-base') && it.size() == 1 }, _, _) + } +} diff --git a/test/groovy/org/ods/component/HelmDeploymentConfigSpec.groovy b/test/groovy/org/ods/component/HelmDeploymentConfigSpec.groovy new file mode 100644 index 000000000..81ee180b4 --- /dev/null +++ b/test/groovy/org/ods/component/HelmDeploymentConfigSpec.groovy @@ -0,0 +1,98 @@ +package org.ods.component + +import org.ods.util.Logger +import vars.test_helper.PipelineSpockTestBase + +class HelmDeploymentConfigSpec extends PipelineSpockTestBase { + + private static final Map DEFAULT_CONTEXT = [ + componentId: 'component-a', + projectId: 'foo', + cdProject: 'foo-cd', + gitCommit: 'abc12345678', + openshiftRolloutTimeout: 20, + openshiftRolloutTimeoutRetries: 7, + ] + + private IContext createContext(Map overrides = [:]) { + def script = loadScript('vars/withStage.groovy') + def logger = new Logger(script, false) + new Context(script, DEFAULT_CONTEXT + overrides, logger) + } + + def "applyDefaults fills all missing config values from context"() { + given: + def context = createContext() + def config = [:] + + when: + HelmDeploymentConfig.applyDefaults(context, config) + + then: + config.selector == 'app=foo-component-a' + config.imageTag == 'abc12345' + config.deployTimeoutMinutes == 20 + config.deployTimeoutRetries == 7 + config.chartDir == 'chart' + config.helmReleaseName == 'component-a' + config.helmValues == [:] + config.helmValuesFiles == ['values.yaml'] + config.helmEnvBasedValuesFiles == [] + config.helmDefaultFlags == ['--install', '--atomic'] + config.helmAdditionalFlags == [] + config.helmDiff == true + config.helmPrivateKeyCredentialsId == 'foo-cd-helm-private-key' + } + + def "applyDefaults preserves explicitly set config values"() { + given: + def context = createContext() + def config = [ + selector: 'custom=selector', + imageTag: 'tag-1', + deployTimeoutMinutes: 3, + deployTimeoutRetries: 2, + chartDir: 'helm-chart', + helmReleaseName: 'custom-release', + helmValues: [k: 'v'], + helmValuesFiles: ['a.yaml'], + helmEnvBasedValuesFiles: ['b.yaml'], + helmDefaultFlags: ['--wait'], + helmAdditionalFlags: ['--debug'], + helmDiff: false, + helmPrivateKeyCredentialsId: 'custom-cred', + ] + + when: + HelmDeploymentConfig.applyDefaults(context, config) + + then: + config.selector == 'custom=selector' + config.imageTag == 'tag-1' + config.deployTimeoutMinutes == 3 + config.deployTimeoutRetries == 2 + config.chartDir == 'helm-chart' + config.helmReleaseName == 'custom-release' + config.helmValues == [k: 'v'] + config.helmValuesFiles == ['a.yaml'] + config.helmEnvBasedValuesFiles == ['b.yaml'] + config.helmDefaultFlags == ['--wait'] + config.helmAdditionalFlags == ['--debug'] + config.helmDiff == false + config.helmPrivateKeyCredentialsId == 'custom-cred' + } + + def "applyDefaults uses fallback when context timeout values are not set"() { + given: + // Use different timeout values to verify they flow through + def context = createContext(openshiftRolloutTimeout: 30, openshiftRolloutTimeoutRetries: 10) + def config = [:] + + when: + HelmDeploymentConfig.applyDefaults(context, config) + + then: + config.deployTimeoutMinutes == 30 + config.deployTimeoutRetries == 10 + } +} diff --git a/test/groovy/org/ods/component/HelmDeploymentStrategySpec.groovy b/test/groovy/org/ods/component/HelmDeploymentStrategySpec.groovy index c6c2990ab..a52bdc16b 100644 --- a/test/groovy/org/ods/component/HelmDeploymentStrategySpec.groovy +++ b/test/groovy/org/ods/component/HelmDeploymentStrategySpec.groovy @@ -1,5 +1,6 @@ package org.ods.component +import org.ods.orchestration.util.MROPipelineUtil import org.ods.services.JenkinsService import org.ods.services.OpenShiftService import org.ods.util.HelmStatus @@ -94,20 +95,20 @@ class HelmDeploymentStrategySpec extends PipelineSpockTestBase { // of HelmDeploymentStrategy and should also be tested steps.dir(contextData.chartDir, _ as Closure) >> { args -> args[1]() } jenkins.maybeWithPrivateKeyCredentials("${contextData.cdProject}-helm-private-key", _ as Closure) >> { args -> args[1]() } + IImageRepository imgRepo = new ImageRepositoryOpenshift(steps, context, openShift) // Setup stub for getContainerImagesWithNameFromPodSpec openShift.getContainerImagesWithNameFromPodSpec(contextData.targetProject, 'Deployment', 'core') >> coreContainerImages openShift.getContainerImagesWithNameFromPodSpec(contextData.targetProject, 'Deployment', 'standalone-gateway') >> standaloneGatewayContainerImages Map> rolloutData - HelmDeploymentStrategy strategy = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins, logger) + HelmDeploymentStrategy strategy = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins, imgRepo, logger) when: rolloutData = strategy.deploy() then: 1 * openShift.helmStatus(contextData.targetProject, contextData.componentId) >> helmStatus - 1 * openShift.helmUpgrade( contextData.targetProject, contextData.componentId, @@ -142,14 +143,14 @@ class HelmDeploymentStrategySpec extends PipelineSpockTestBase { def "rollout: check deploymentMean with container images from pod spec"() { given: - + def targetProject = 'foo-dev' def expectedDeploymentMeans = [ builds: [:], deployments: [ 'core-deploymentMean': [ type: 'helm', selector: 'app=myproject-core', - namespace: 'foo-dev', + namespace: targetProject, chartDir: 'chart', helmReleaseName: 'core', helmEnvBasedValuesFiles: [], @@ -185,8 +186,7 @@ class HelmDeploymentStrategySpec extends PipelineSpockTestBase { ] def config = [:] - - def ctxData = contextData + [environment: 'dev', targetProject: 'foo-dev', openshiftRolloutTimeoutRetries: 5, chartDir: 'chart'] + def ctxData = contextData + [environment: 'dev', targetProject: targetProject, openshiftRolloutTimeoutRetries: 5, chartDir: 'chart'] IContext context = new Context(null, ctxData, logger) OpenShiftService openShiftService = Mock(OpenShiftService.class) // Now uses getContainerImagesWithNameFromPodSpec instead of checkForPodData @@ -201,12 +201,14 @@ class HelmDeploymentStrategySpec extends PipelineSpockTestBase { JenkinsService jenkinsService = Stub(JenkinsService.class) jenkinsService.maybeWithPrivateKeyCredentials(*_) >> { args -> args[1]('/tmp/file') } ServiceRegistry.instance.add(JenkinsService, jenkinsService) + IPipelineSteps steps = Stub() + IImageRepository imgRepo = new ImageRepositoryOpenshift(steps, context, openShiftService) - HelmDeploymentStrategy strategy = Spy(HelmDeploymentStrategy, constructorArgs: [null, context, config, openShiftService, jenkinsService, logger]) + HelmDeploymentStrategy strategy = Spy(HelmDeploymentStrategy, constructorArgs: [null, context, config, openShiftService, jenkinsService, imgRepo, logger]) when: def deploymentResources = HelmStatus.fromJsonObject(FixtureHelper.createHelmCmdStatusMap()) - def rolloutData = strategy.getRolloutData(deploymentResources) + def rolloutData = strategy.getRolloutData(deploymentResources, targetProject) def actualDeploymentMeans = context.getBuildArtifactURIs() then: @@ -230,7 +232,6 @@ class HelmDeploymentStrategySpec extends PipelineSpockTestBase { def "rollout: check rolloutData with StatefulSet resource"() { given: def helmStatus = HelmStatus.fromJsonObject(FixtureHelper.createHelmCmdStatusMapWithStatefulSet()) - OpenShiftService openShift = Mock() JenkinsService jenkins = Stub() IPipelineSteps steps = Stub() @@ -257,9 +258,9 @@ class HelmDeploymentStrategySpec extends PipelineSpockTestBase { openShift.getContainerImagesWithNameFromPodSpec(contextData.targetProject, 'StatefulSet', 'database') >> [ 'postgres': "${contextData.clusterRegistryAddress}/myproject-dev/postgres@sha256:dbsha123456" ] - + IImageRepository imgRepo = new ImageRepositoryOpenshift(steps, context, openShift) Map> rolloutData - HelmDeploymentStrategy strategy = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins, logger) + HelmDeploymentStrategy strategy = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins, imgRepo, logger) when: rolloutData = strategy.deploy() @@ -305,9 +306,9 @@ class HelmDeploymentStrategySpec extends PipelineSpockTestBase { openShift.getContainerImagesWithNameFromPodSpec(contextData.targetProject, 'CronJob', 'cleanup-job') >> [ 'cleanup': "${contextData.clusterRegistryAddress}/myproject-dev/cleanup@sha256:cronjob12345" ] - + IImageRepository imgRepo = new ImageRepositoryOpenshift(steps, context, openShift) Map> rolloutData - HelmDeploymentStrategy strategy = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins, logger) + HelmDeploymentStrategy strategy = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins, imgRepo, logger) when: rolloutData = strategy.deploy() @@ -354,9 +355,9 @@ class HelmDeploymentStrategySpec extends PipelineSpockTestBase { 'main-app': "${contextData.clusterRegistryAddress}/myproject-dev/main-app@sha256:mainapp12345", 'envoy-sidecar': "${contextData.clusterRegistryAddress}/myproject-dev/envoy@sha256:envoy67890" ] - + IImageRepository imgRepo = new ImageRepositoryOpenshift(steps, context, openShift) Map> rolloutData - HelmDeploymentStrategy strategy = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins, logger) + HelmDeploymentStrategy strategy = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins, imgRepo, logger) when: rolloutData = strategy.deploy() @@ -411,9 +412,9 @@ class HelmDeploymentStrategySpec extends PipelineSpockTestBase { openShift.getContainerImagesWithNameFromPodSpec(contextData.targetProject, 'CronJob', 'cleanup-job') >> [ 'cleanup': "${contextData.clusterRegistryAddress}/myproject-dev/cleanup@sha256:cronjob12345" ] - + IImageRepository imgRepo = new ImageRepositoryOpenshift(steps, context, openShift) Map> rolloutData - HelmDeploymentStrategy strategy = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins, logger) + HelmDeploymentStrategy strategy = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins,imgRepo, logger) when: rolloutData = strategy.deploy() @@ -441,4 +442,39 @@ class HelmDeploymentStrategySpec extends PipelineSpockTestBase { rolloutData['cleanup-job'][0].containers['cleanup'].contains('cleanup@sha256') } + def "rollout: returns empty rolloutData when repoType is ods-infra"() { + given: + def helmStatus = HelmStatus.fromJsonObject(FixtureHelper.createHelmCmdStatusMap()) + + OpenShiftService openShift = Mock() + JenkinsService jenkins = Stub() + IPipelineSteps steps = Stub() + + IContext context = Stub { + getTargetProject() >> contextData.targetProject + getEnvironment() >> contextData.environment + getBuildArtifactURIs() >> contextData.artifactUriStore + getComponentId() >> contextData.componentId + getClusterRegistryAddress() >> contextData.clusterRegistryAddress + getCdProject() >> contextData.cdProject + } + + steps.getEnv() >> [repoType: MROPipelineUtil.PipelineConfig.REPO_TYPE_ODS_INFRA] + steps.dir(contextData.chartDir, _ as Closure) >> { args -> args[1]() } + jenkins.maybeWithPrivateKeyCredentials("${contextData.cdProject}-helm-private-key", _ as Closure) >> { args -> args[1]() } + IImageRepository imgRepo = new ImageRepositoryOpenshift(steps, context, openShift) + + HelmDeploymentStrategy strategy = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins, imgRepo, logger) + + when: + Map> rolloutData = strategy.deploy() + + then: + 1 * openShift.helmStatus(contextData.targetProject, contextData.componentId) >> helmStatus + 1 * openShift.helmUpgrade(*_) + 0 * openShift.getContainerImagesWithNameFromPodSpec(*_) + + rolloutData == [:] + } + } diff --git a/test/groovy/org/ods/component/ImageECRSpec.groovy b/test/groovy/org/ods/component/ImageECRSpec.groovy new file mode 100644 index 000000000..0986220f3 --- /dev/null +++ b/test/groovy/org/ods/component/ImageECRSpec.groovy @@ -0,0 +1,186 @@ +package org.ods.component + +import org.ods.util.IPipelineSteps +import org.ods.util.Logger +import vars.test_helper.PipelineSpockTestBase + +class ImageECRSpec extends PipelineSpockTestBase { + + private static final Map AWS_ENV = [ + account: '123456789012', + region: 'eu-west-1', + ] + + private static final Map DEFAULT_CONTEXT = [ + projectId: 'foo', + cdProject: 'foo-cd', + componentId: 'component-a', + dockerRegistry: 'docker-registry.example.com', + gitCommit: 'abc12345678', + ] + + IPipelineSteps steps + + def setup() { + steps = Mock(IPipelineSteps) + } + + private ImageECR createImageECR(Map awsEnv = AWS_ENV, Map contextOverrides = [:]) { + def script = loadScript('vars/withStage.groovy') + def logger = new Logger(script, false) + IContext context = new Context(script, DEFAULT_CONTEXT + contextOverrides, logger) + // Stub the OC token call made in fetchOCToken + steps.sh([script: 'oc whoami -t', returnStdout: true]) >> 'my-oc-token' + def imageECR = new ImageECR(steps, context, logger, awsEnv) + imageECR.fetchOCToken() + return imageECR + } + + def "constructor does not call shell commands"() { + given: + def script = loadScript('vars/withStage.groovy') + def logger = new Logger(script, false) + IContext context = new Context(script, DEFAULT_CONTEXT, logger) + + when: + def imageECR = new ImageECR(steps, context, logger, AWS_ENV) + + then: + imageECR != null + 0 * steps.sh(_) + } + + def "fetchOCToken retrieves OC token"() { + given: + def script = loadScript('vars/withStage.groovy') + def logger = new Logger(script, false) + IContext context = new Context(script, DEFAULT_CONTEXT, logger) + def imageECR = new ImageECR(steps, context, logger, AWS_ENV) + + when: + imageECR.fetchOCToken() + + then: + 1 * steps.sh({ it.script == 'oc whoami -t' && it.returnStdout == true }) >> 'my-oc-token' + } + + def "retagImages creates repository and copies image for each image"() { + given: + def imageECR = createImageECR() + def images = ['image-a', 'image-b'] as Set + + when: + imageECR.retagImages('target-project', images, 'source-tag', 'target-tag') + + then: + // get AWS password called (at least once) + _ * steps.sh({ it.script?.contains('aws ecr get-login-password') }) >> 'aws-password' + + // create-repository called for each image + 1 * steps.sh({ + it.script?.contains('aws ecr create-repository --repository-name image-a') && + it.script?.contains("--region ${AWS_ENV.region}") && + it.returnStatus == true + }) >> 0 + 1 * steps.sh({ + it.script?.contains('aws ecr create-repository --repository-name image-b') && + it.script?.contains("--region ${AWS_ENV.region}") && + it.returnStatus == true + }) >> 0 + + // skopeo copy called for each image + 1 * steps.sh({ + it.script?.contains('skopeo copy') && + it.script?.contains("foo-cd/image-a:source-tag") && + it.script?.contains("${AWS_ENV.account}.dkr.ecr.${AWS_ENV.region}.amazonaws.com/image-a:target-tag") && + it.returnStatus == true + }) >> 0 + 1 * steps.sh({ + it.script?.contains('skopeo copy') && + it.script?.contains("foo-cd/image-b:source-tag") && + it.script?.contains("${AWS_ENV.account}.dkr.ecr.${AWS_ENV.region}.amazonaws.com/image-b:target-tag") && + it.returnStatus == true + }) >> 0 + } + + def "retagImages creates repository for a single image"() { + given: + def imageECR = createImageECR() + def images = ['my-service'] as Set + + when: + imageECR.retagImages('target-project', images, 'v1.0', 'v1.1') + + then: + _ * steps.sh({ it.script?.contains('aws ecr get-login-password') }) >> 'aws-password' + + 1 * steps.sh({ + it.script?.contains('aws ecr create-repository --repository-name my-service') && + it.returnStatus == true + }) >> 0 + 1 * steps.sh({ + it.script?.contains('skopeo copy') && + it.script?.contains('my-service:v1.0') && + it.script?.contains("${AWS_ENV.account}.dkr.ecr.${AWS_ENV.region}.amazonaws.com/my-service:v1.1") && + it.returnStatus == true + }) >> 0 + } + + def "retagImages handles empty image set"() { + given: + def imageECR = createImageECR() + def images = [] as Set + + when: + imageECR.retagImages('target-project', images, 'tag1', 'tag2') + + then: + // No shell commands for create-repository or skopeo copy should be issued + 0 * steps.sh({ it.script?.contains('aws ecr create-repository') }) + 0 * steps.sh({ it.script?.contains('skopeo copy') }) + } + + def "getECRRegistry returns correct URL based on awsEnvironmentVars"() { + given: + def customAwsEnv = [account: '999888777666', region: 'us-east-1'] + def imageECR = createImageECR(customAwsEnv) + def images = ['app'] as Set + + when: + imageECR.retagImages('proj', images, 'src', 'dst') + + then: + _ * steps.sh({ it.script?.contains('aws ecr get-login-password') }) >> 'pwd' + + 1 * steps.sh({ + it.script?.contains('aws ecr create-repository') && + it.returnStatus == true + }) >> 0 + 1 * steps.sh({ + it.script?.contains('999888777666.dkr.ecr.us-east-1.amazonaws.com/app:dst') && + it.returnStatus == true + }) >> 0 + } + + def "copyImage uses correct docker source with registry and cdProject"() { + given: + def imageECR = createImageECR() + def images = ['my-app'] as Set + + when: + imageECR.retagImages('proj', images, 'abc123', 'def456') + + then: + _ * steps.sh({ it.script?.contains('aws ecr get-login-password') }) >> 'aws-pwd' + + 1 * steps.sh({ + it.script?.contains('aws ecr create-repository') && + it.returnStatus == true + }) >> 0 + 1 * steps.sh({ + it.script?.contains("docker://docker-registry.example.com/foo-cd/my-app:abc123") && + it.script?.contains("docker://123456789012.dkr.ecr.eu-west-1.amazonaws.com/my-app:def456") && + it.returnStatus == true + }) >> 0 + } +} diff --git a/test/groovy/org/ods/component/RolloutAWSDeploymentStageSpec.groovy b/test/groovy/org/ods/component/RolloutAWSDeploymentStageSpec.groovy new file mode 100644 index 000000000..1ecb19e38 --- /dev/null +++ b/test/groovy/org/ods/component/RolloutAWSDeploymentStageSpec.groovy @@ -0,0 +1,105 @@ +package org.ods.component + +import org.ods.services.JenkinsService +import org.ods.services.OpenShiftService +import org.ods.util.Logger +import vars.test_helper.PipelineSpockTestBase + +class RolloutAWSDeploymentStageSpec extends PipelineSpockTestBase { + + private static final Map DEFAULT_CONTEXT = [ + componentId: 'component-a', + projectId: 'foo', + cdProject: 'foo-cd', + selector: 'app=component-a', + gitCommit: 'abc12345678', + openshiftRolloutTimeout: 20, + openshiftRolloutTimeoutRetries: 7 + ] + + private RolloutAWSDeploymentStage createStage(Map config = [:], Map contextValues = [:]) { + def script = loadScript('vars/withStage.groovy') + def logger = Spy(new Logger(script, false)) + IContext context = new Context(script, DEFAULT_CONTEXT + contextValues, logger) + OpenShiftService openShift = Mock(OpenShiftService) + JenkinsService jenkins = Mock(JenkinsService) + new RolloutAWSDeploymentStage(script, context, config, openShift, jenkins, [:], logger) + } + + def "constructor sets default rollout and helm options from context"() { + given: + def config = [:] + + when: + def stage = createStage(config) + + then: + config.selector == 'app=foo-component-a' + config.imageTag == 'abc12345' + config.deployTimeoutMinutes == 20 + config.deployTimeoutRetries == 7 + config.chartDir == 'chart' + config.helmReleaseName == 'component-a' + config.helmValues == [:] + config.helmValuesFiles == ['values.yaml'] + config.helmEnvBasedValuesFiles == [] + config.helmDefaultFlags == ['--install', '--atomic'] + config.helmAdditionalFlags == [] + config.helmDiff == true + config.helmPrivateKeyCredentialsId == 'foo-cd-helm-private-key' + stage != null + } + + def "constructor keeps explicit options unchanged"() { + given: + def config = [ + selector: 'custom=selector', + imageTag: 'tag-1', + deployTimeoutMinutes: 3, + deployTimeoutRetries: 2, + chartDir: 'helm-chart', + helmReleaseName: 'custom-release', + helmValues: [k: 'v'], + helmValuesFiles: ['a.yaml'], + helmEnvBasedValuesFiles: ['b.yaml'], + helmDefaultFlags: ['--wait'], + helmAdditionalFlags: ['--debug'], + helmDiff: false, + helmPrivateKeyCredentialsId: 'custom-cred' + ] + + when: + createStage(config) + + then: + config.selector == 'custom=selector' + config.imageTag == 'tag-1' + config.deployTimeoutMinutes == 3 + config.deployTimeoutRetries == 2 + config.chartDir == 'helm-chart' + config.helmReleaseName == 'custom-release' + config.helmValues == [k: 'v'] + config.helmValuesFiles == ['a.yaml'] + config.helmEnvBasedValuesFiles == ['b.yaml'] + config.helmDefaultFlags == ['--wait'] + config.helmAdditionalFlags == ['--debug'] + config.helmDiff == false + config.helmPrivateKeyCredentialsId == 'custom-cred' + } + + def "stage label includes selector when different from context"() { + when: + def stage = createStage([selector: 'other=selector']) + + then: + stage.stageLabel() == 'Deploy to OpenShift (other=selector)' + } + + def "stage label falls back to stage name when selector matches context"() { + when: + def stage = createStage([selector: 'app=foo-component-a']) + + then: + stage.stageLabel() == 'Deploy to OpenShift' + } +} diff --git a/test/groovy/org/ods/services/AWSServiceSpec.groovy b/test/groovy/org/ods/services/AWSServiceSpec.groovy new file mode 100644 index 000000000..b615467a3 --- /dev/null +++ b/test/groovy/org/ods/services/AWSServiceSpec.groovy @@ -0,0 +1,121 @@ +package org.ods.services + +import org.ods.component.IContext +import org.ods.util.Logger +import vars.test_helper.PipelineSpockTestBase + +class AWSServiceSpec extends PipelineSpockTestBase { + + private Map createAwsEnvironmentVars() { + return [ + credentials: [ + key: 'AWS_ACCESS_KEY_ID', + secret: 'AWS_SECRET_ACCESS_KEY' + ], + region: 'eu-west-1', + eksCluster: 'my-eks-cluster' + ] + } + + def "awsLogin configures AWS credentials and region"() { + given: + def steps = Spy(util.PipelineSteps) + steps.withCredentials(_, _) >> { args -> args[1].call() } + def context = Mock(IContext) + def awsEnvVars = createAwsEnvironmentVars() + def service = new AWSService(steps, context, awsEnvVars, new Logger(steps, false)) + + when: + service.awsLogin() + + then: + 1 * steps.sh({ it.script == 'aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default' && it.returnStatus == true }) >> 0 + 1 * steps.sh({ it.script == 'aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default' && it.returnStatus == true }) >> 0 + 1 * steps.sh({ it.script == 'aws configure set region eu-west-1 --profile default' && it.returnStatus == true }) >> 0 + } + + def "awsLogin reports error when command fails"() { + given: + def steps = Spy(util.PipelineSteps) + steps.withCredentials(_, _) >> { args -> args[1].call() } + steps.error(_) >> { throw new RuntimeException(it[0]) } + def context = Mock(IContext) + def awsEnvVars = createAwsEnvironmentVars() + def service = new AWSService(steps, context, awsEnvVars, new Logger(steps, false)) + + when: + service.awsLogin() + + then: + thrown(RuntimeException) + 1 * steps.sh({ it.script == 'aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default' && it.returnStatus == true }) >> 1 + } + + def "setEKSCluster updates kubeconfig and creates namespace"() { + given: + def steps = Spy(util.PipelineSteps) + steps.withCredentials(_, _) >> { args -> args[1].call() } + def context = Mock(IContext) { + getProjectId() >> 'myproject' + getEnvironment() >> 'dev' + } + def awsEnvVars = createAwsEnvironmentVars() + def service = new AWSService(steps, context, awsEnvVars, new Logger(steps, false)) + + when: + service.setEKSCluster() + + then: + 1 * steps.sh({ it.script == 'aws eks list-clusters' && it.returnStatus == true }) >> 0 + 1 * steps.sh({ it.script == 'aws eks update-kubeconfig --region eu-west-1 --name my-eks-cluster' && it.returnStatus == true }) >> 0 + 1 * steps.sh({ it.script == 'kubectl create namespace myproject-dev' && it.returnStatus == true }) >> 0 + } + + def "setEKSCluster does not report error when namespace already exists"() { + given: + def steps = Spy(util.PipelineSteps) + steps.withCredentials(_, _) >> { args -> args[1].call() } + def context = Mock(IContext) { + getProjectId() >> 'myproject' + getEnvironment() >> 'dev' + } + def awsEnvVars = createAwsEnvironmentVars() + def service = new AWSService(steps, context, awsEnvVars, new Logger(steps, false)) + + when: + service.setEKSCluster() + + then: + 1 * steps.sh({ it.script == 'aws eks list-clusters' && it.returnStatus == true }) >> 0 + 1 * steps.sh({ it.script == 'aws eks update-kubeconfig --region eu-west-1 --name my-eks-cluster' && it.returnStatus == true }) >> 0 + // namespace creation fails (already exists) - should NOT call error because showError=false + 1 * steps.sh({ it.script == 'kubectl create namespace myproject-dev' && it.returnStatus == true }) >> 1 + 0 * steps.error(_) + } + + def "credentials keys are lowercased in withCredentials call"() { + given: + def steps = Spy(util.PipelineSteps) + steps.withCredentials(_, _) >> { args -> args[1].call() } + steps.sh(_) >> 0 + def context = Mock(IContext) + def awsEnvVars = [ + credentials: [ + key: 'MY_CUSTOM_KEY_ID', + secret: 'MY_CUSTOM_SECRET_KEY' + ], + region: 'us-east-1', + eksCluster: 'cluster-1' + ] + def service = new AWSService(steps, context, awsEnvVars, new Logger(steps, false)) + + when: + service.awsLogin() + + then: + 1 * steps.string(credentialsId: 'my_custom_key_id', variable: 'AWS_ACCESS_KEY_ID') + 1 * steps.string(credentialsId: 'my_custom_secret_key', variable: 'AWS_SECRET_ACCESS_KEY') + } + +} + diff --git a/test/groovy/org/ods/services/NexusServiceSpec.groovy b/test/groovy/org/ods/services/NexusServiceSpec.groovy index 50b6cce09..1c17fbec0 100644 --- a/test/groovy/org/ods/services/NexusServiceSpec.groovy +++ b/test/groovy/org/ods/services/NexusServiceSpec.groovy @@ -128,6 +128,54 @@ class NexusServiceSpec extends SpecHelper { stopServer(server) } + Map storeNpmArtifactRequestData(Map mixins = [:]) { + def result = [ + data: [ + artifact: [0] as byte[], + contentType: "application/octet-stream", + name: "package.tgz", + repository: "npm-hosted", + ], + password: "password", + path: "/service/rest/v1/components", + username: "username" + ] + + result.multipartRequestBody = [ + "npm.asset": result.data.artifact + ] + + result.queryParams = [ + "repository": result.data.repository + ] + + return result << mixins + } + + def "store npm artifact"() { + given: + def request = storeNpmArtifactRequestData() + def response = storeArtifactResponseData() + + def server = createServer(WireMock.&post, request, response) + def service = createService(server.port(), "foo-cd-cd-user-with-password") + + when: + def result = service.storeComplextArtifact( + request.data.repository, + request.data.artifact, + request.data.contentType, + "npm", + ["npm.asset": request.data.name] + ) + + then: + result == new URIBuilder("http://localhost:${server.port()}/repository/${request.data.repository}").build() + + cleanup: + stopServer(server) + } + def "store artifact with HTTP 404 failure"() { given: def request = storeArtifactRequestData() diff --git a/test/groovy/org/ods/services/SonarQubeServiceSpec.groovy b/test/groovy/org/ods/services/SonarQubeServiceSpec.groovy index 06905b6ef..2bd68545c 100644 --- a/test/groovy/org/ods/services/SonarQubeServiceSpec.groovy +++ b/test/groovy/org/ods/services/SonarQubeServiceSpec.groovy @@ -108,7 +108,7 @@ class SonarQubeServiceSpec extends PipelineSpockTestBase { service.scan(optionsWithoutPrivateToken) then: - 2 * steps.sh(label: 'Set Java 17 for SonarQube scan', script: "source use-j17.sh") + 2 * steps.sh(label: 'Set Java 21 for SonarQube scan', script: "source use-j21.sh") 1 * steps.sh(label: 'Run SonarQube scan', script: { it.contains("/opt/sonar-scanner/bin/sonar-scanner") && it.contains("-Dsonar.projectKey=key") && it.contains("-Dsonar.projectName=name") && it.contains("-Dsonar.filesize.limit=2") && it.contains("-Dsonar.exclusions=test/**") && it.contains("-Dsonar.auth.token=my-private-token") }) 1 * steps.sh(label: 'Run SonarQube scan', script: { it.contains("/opt/sonar-scanner/bin/sonar-scanner") && it.contains("-Dsonar.projectKey=key") && it.contains("-Dsonar.projectName=name") && it.contains("-Dsonar.filesize.limit=2") && it.contains("-Dsonar.exclusions=test/**") && it.contains("-Dsonar.auth.token=public-token") }) } diff --git a/test/groovy/vars/OdsComponentStageScanWithAquaSpec.groovy b/test/groovy/vars/OdsComponentStageScanWithAquaSpec.groovy index 73801d9ba..3e2a68c60 100644 --- a/test/groovy/vars/OdsComponentStageScanWithAquaSpec.groovy +++ b/test/groovy/vars/OdsComponentStageScanWithAquaSpec.groovy @@ -310,4 +310,57 @@ class OdsComponentStageScanWithAquaSpec extends PipelineSpockTestBase { assertJobStatusSuccess() } + def "forwards branches config into ScanWithAquaStage"() { + given: + def c = config + [environment: 'dev'] + IContext context = new Context(null, c, logger) + def aquaConfigMapName = ScanWithAquaStage.AQUA_CONFIG_MAP_NAME + + OpenShiftService openShiftService = Stub(OpenShiftService.class) + openShiftService.getConfigMapData('ods', aquaConfigMapName) >> [ + enabled: true, alertEmails: "mail1@mail.com", url: "http://aqua", registry: "internal" + ] + openShiftService.getConfigMapData("foo", aquaConfigMapName) >> [enabled: true] + ServiceRegistry.instance.add(OpenShiftService, openShiftService) + + NexusService nexusService = Stub(NexusService.class) + ServiceRegistry.instance.add(NexusService, nexusService) + + and: "mock the stage constructor and validate inherited config" + GroovyMock(ScanWithAquaStage, global: true) + ScanWithAquaStage.getAQUA_CONFIG_MAP_NAME() >> aquaConfigMapName + + when: + def script = loadScript('vars/odsComponentStageScanWithAqua.groovy') + script.call( + context, + [ + imageLabels: [JENKINS_MASTER_OPENSHIFT_BUILD_NAMESPACE: 'ods'], + resourceName: 'my-image', + branches: ['master', 'release/'] + ] + ) + + then: + 1 * new ScanWithAquaStage( + _, + _, + { + it.resourceName == 'my-image' && + it.branches == ['master', 'release/'] + }, + _, + _, + _, + _, + _, + _, + _ + ) >> Stub(ScanWithAquaStage) { + execute() >> null + } + printCallStack() + assertJobStatusSuccess() + } + } diff --git a/vars/odsComponentStageRolloutAWSDeployment.groovy b/vars/odsComponentStageRolloutAWSDeployment.groovy new file mode 100644 index 000000000..d0c2385f8 --- /dev/null +++ b/vars/odsComponentStageRolloutAWSDeployment.groovy @@ -0,0 +1,42 @@ +import org.ods.component.RolloutAWSDeploymentStage +import org.ods.component.IContext +import org.ods.orchestration.util.MROPipelineUtil +import org.ods.services.OpenShiftService +import org.ods.services.JenkinsService +import org.ods.services.ServiceRegistry +import org.ods.util.Logger +import org.ods.util.ILogger + +def call(IContext context, Map config = [:]) { + ILogger logger = ServiceRegistry.instance.get(Logger) + if (!logger) { + logger = new Logger (this, !!env.DEBUG) + } + if (!context.environment) { + logger.warn 'Skipping because of empty (target) environment ...' + return + } + if (!config.awsEnvPath) { + config.awsEnvPath = "./environments" + } + + Map awsEnvironmentVars = readYaml(file: "${config.awsEnvPath}/${context.environment}.yml") + def metadata = readYaml(file: 'metadata.yml') + def repoType = metadata.type + if (repoType != MROPipelineUtil.PipelineConfig.REPO_TYPE_ODS_INFRA) { + logger.warn "Skipping because of unsupported repository type '${repoType}' ..." + logger.warn "Update repo type in metadata.yml to ods-infra." + return + } + env.repoType = repoType + return new RolloutAWSDeploymentStage( + this, + context, + config, + ServiceRegistry.instance.get(OpenShiftService), + ServiceRegistry.instance.get(JenkinsService), + awsEnvironmentVars, + logger + ).execute() +} +return this diff --git a/vars/odsComponentStageScanWithAqua.groovy b/vars/odsComponentStageScanWithAqua.groovy index e0bf98e38..8b428ea28 100644 --- a/vars/odsComponentStageScanWithAqua.groovy +++ b/vars/odsComponentStageScanWithAqua.groovy @@ -82,6 +82,9 @@ def call(IContext context, Map config = [:]) { if (config.resourceName) { inheritedConfig.resourceName = config.resourceName } + if (config.branches) { + inheritedConfig.branches = config.branches + } if (enabledInCluster && enabledInProject) { new ScanWithAquaStage(this, context,