Skip to content

Commit f16c1aa

Browse files
vaibhav-sivaibhav-si
authored
feat: [CDS-30595]: migration mapping scenarios handling (#32511)
* [CDS-30595]: migration mapping scenarios handling * minor changes
1 parent 09d2ab0 commit f16c1aa

25 files changed

Lines changed: 467 additions & 123 deletions

‎100-migrator/src/main/java/io/harness/ngmigration/api/NgMigrationResource.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,13 @@
1717
import io.harness.ngmigration.beans.DiscoveryInput;
1818
import io.harness.ngmigration.beans.MigrationInputDTO;
1919
import io.harness.ngmigration.beans.MigrationInputResult;
20+
import io.harness.ngmigration.beans.NGYamlFile;
2021
import io.harness.ngmigration.service.DiscoveryService;
2122
import io.harness.ngmigration.utils.NGMigrationConstants;
2223
import io.harness.rest.RestResponse;
2324

2425
import software.wings.ngmigration.DiscoveryResult;
2526
import software.wings.ngmigration.NGMigrationEntityType;
26-
import software.wings.ngmigration.NGYamlFile;
2727
import software.wings.security.PermissionAttribute.ResourceType;
2828
import software.wings.security.annotations.Scope;
2929

957-cg-beans/src/main/java/software/wings/ngmigration/NGYamlFile.java renamed to 100-migrator/src/main/java/io/harness/ngmigration/beans/NGYamlFile.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,15 @@
55
* https://polyformproject.org/wp-content/uploads/2020/06/PolyForm-Shield-1.0.0.txt.
66
*/
77

8-
package software.wings.ngmigration;
8+
package io.harness.ngmigration.beans;
99

1010
import io.harness.annotations.dev.HarnessTeam;
1111
import io.harness.annotations.dev.OwnedBy;
1212
import io.harness.gitsync.beans.YamlDTO;
1313

14+
import software.wings.ngmigration.CgBasicInfo;
15+
import software.wings.ngmigration.NGMigrationEntityType;
16+
1417
import lombok.AllArgsConstructor;
1518
import lombok.Builder;
1619
import lombok.Data;
@@ -27,4 +30,5 @@ public class NGYamlFile {
2730
private YamlDTO yaml;
2831
private boolean exists;
2932
private CgBasicInfo cgBasicInfo;
33+
private NgEntityDetail ngEntityDetail;
3034
}

‎100-migrator/src/main/java/io/harness/ngmigration/service/DiscoveryService.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import io.harness.ngmigration.beans.DiscoveryInput;
2323
import io.harness.ngmigration.beans.MigrationInputDTO;
2424
import io.harness.ngmigration.beans.MigrationInputResult;
25+
import io.harness.ngmigration.beans.NGYamlFile;
2526
import io.harness.ngmigration.beans.NgEntityDetail;
2627
import io.harness.ngmigration.client.NGClient;
2728
import io.harness.ngmigration.client.PmsClient;
@@ -34,7 +35,6 @@
3435
import software.wings.ngmigration.DiscoveryResult;
3536
import software.wings.ngmigration.NGMigrationEntity;
3637
import software.wings.ngmigration.NGMigrationEntityType;
37-
import software.wings.ngmigration.NGYamlFile;
3838

3939
import com.google.common.collect.Sets;
4040
import com.google.inject.Inject;
@@ -289,6 +289,10 @@ private void exportZip(List<NGYamlFile> ngYamlFiles, String dirName) {
289289
zipFile.getParentFile().mkdirs();
290290
try (ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile))) {
291291
for (NGYamlFile file : ngYamlFiles) {
292+
if (file.isExists()) {
293+
// TODO: @vaibhav.si Add the mapping to the response
294+
continue;
295+
}
292296
ZipEntry e = new ZipEntry(file.getFilename());
293297
out.putNextEntry(e);
294298
byte[] data = NGYamlUtils.getYamlString(file.getYaml()).getBytes();
@@ -339,7 +343,7 @@ private List<NGYamlFile> getAllYamlFiles(MigrationInputDTO inputDTO, Map<CgEntit
339343
List<CgEntityId> leafNodes = getLeafNodes(leafTracker);
340344
for (CgEntityId entry : leafNodes) {
341345
List<NGYamlFile> currentEntity =
342-
migrationFactory.getMethod(entry.getType()).getYamls(inputDTO, entities, graph, entry, migratedEntities);
346+
migrationFactory.getMethod(entry.getType()).getYaml(inputDTO, entities, graph, entry, migratedEntities);
343347
if (isNotEmpty(currentEntity)) {
344348
files.addAll(currentEntity);
345349
}

‎100-migrator/src/main/java/io/harness/ngmigration/service/MigratorMappingService.java‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,11 @@
1313
import io.harness.beans.MigratedEntityMapping.MigratedEntityMappingKeys;
1414
import io.harness.data.structure.EmptyPredicate;
1515
import io.harness.encryption.Scope;
16+
import io.harness.ngmigration.beans.NGYamlFile;
17+
import io.harness.ngmigration.beans.NgEntityDetail;
1618
import io.harness.persistence.HPersistence;
1719

1820
import software.wings.ngmigration.CgBasicInfo;
19-
import software.wings.ngmigration.NGYamlFile;
2021

2122
import com.google.inject.Inject;
2223
import java.util.List;
@@ -54,25 +55,38 @@ public static Scope getScope(String org, String project) {
5455

5556
public void mapCgNgEntity(NGYamlFile yamlFile) {
5657
MigratedEntityMapping mapping = ngMigrationFactory.getMethod(yamlFile.getType()).generateMappingEntity(yamlFile);
58+
if (mapping == null) {
59+
// Cases like Secret Manager where we are not saving the entity right now
60+
return;
61+
}
5762
if (!doesMappingExist(yamlFile)) {
5863
hPersistence.save(mapping);
5964
}
6065
}
6166

6267
public boolean doesMappingExist(NGYamlFile yamlFile) {
63-
CgBasicInfo basicInfo = yamlFile.getCgBasicInfo();
68+
CgBasicInfo cgBasicInfo = yamlFile.getCgBasicInfo();
6469
MigratedEntityMapping mapping = ngMigrationFactory.getMethod(yamlFile.getType()).generateMappingEntity(yamlFile);
70+
NgEntityDetail ngEntityDetail = NgEntityDetail.builder()
71+
.identifier(mapping.getIdentifier())
72+
.projectIdentifier(mapping.getProjectIdentifier())
73+
.orgIdentifier(mapping.getOrgIdentifier())
74+
.build();
75+
return doesMappingExist(cgBasicInfo, ngEntityDetail);
76+
}
77+
78+
public boolean doesMappingExist(CgBasicInfo cgBasicInfo, NgEntityDetail ngEntityDetail) {
6579
Query<MigratedEntityMapping> query =
6680
hPersistence.createQuery(MigratedEntityMapping.class)
67-
.filter(MigratedEntityMappingKeys.accountId, basicInfo.getAccountId())
68-
.filter(MigratedEntityMappingKeys.cgEntityId, basicInfo.getId())
69-
.filter(MigratedEntityMappingKeys.entityType, basicInfo.getType().name())
70-
.filter(MigratedEntityMappingKeys.accountIdentifier, basicInfo.getAccountId())
71-
.filter(MigratedEntityMappingKeys.identifier, mapping.getIdentifier())
72-
.filter(
73-
MigratedEntityMappingKeys.scope, getScope(mapping.getOrgIdentifier(), mapping.getProjectIdentifier()));
74-
if (StringUtils.isNotBlank(basicInfo.getAppId())) {
75-
query.filter(MigratedEntityMappingKeys.appId, basicInfo.getAppId());
81+
.filter(MigratedEntityMappingKeys.accountId, cgBasicInfo.getAccountId())
82+
.filter(MigratedEntityMappingKeys.cgEntityId, cgBasicInfo.getId())
83+
.filter(MigratedEntityMappingKeys.entityType, cgBasicInfo.getType().name())
84+
.filter(MigratedEntityMappingKeys.accountIdentifier, cgBasicInfo.getAccountId())
85+
.filter(MigratedEntityMappingKeys.identifier, ngEntityDetail.getIdentifier())
86+
.filter(MigratedEntityMappingKeys.scope,
87+
getScope(ngEntityDetail.getOrgIdentifier(), ngEntityDetail.getProjectIdentifier()));
88+
if (StringUtils.isNotBlank(cgBasicInfo.getAppId())) {
89+
query.filter(MigratedEntityMappingKeys.appId, cgBasicInfo.getAppId());
7690
}
7791
return EmptyPredicate.isNotEmpty(query.asList());
7892
}

‎100-migrator/src/main/java/io/harness/ngmigration/service/MigratorUtility.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,11 @@
1010
import io.harness.encryption.Scope;
1111
import io.harness.exception.InvalidArgumentsException;
1212
import io.harness.ngmigration.beans.InputDefaults;
13+
import io.harness.ngmigration.beans.NGYamlFile;
1314
import io.harness.ngmigration.beans.NgEntityDetail;
1415
import io.harness.pms.yaml.ParameterField;
1516

1617
import software.wings.ngmigration.NGMigrationEntityType;
17-
import software.wings.ngmigration.NGYamlFile;
1818

1919
import java.util.Comparator;
2020
import java.util.List;

‎100-migrator/src/main/java/io/harness/ngmigration/service/NgMigrationFactory.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,19 @@
77

88
package io.harness.ngmigration.service;
99

10+
import io.harness.ngmigration.service.entity.AppMigrationService;
11+
import io.harness.ngmigration.service.entity.ArtifactStreamMigrationService;
12+
import io.harness.ngmigration.service.entity.ConnectorMigrationService;
13+
import io.harness.ngmigration.service.entity.DummyMigrationService;
14+
import io.harness.ngmigration.service.entity.EnvironmentMigrationService;
15+
import io.harness.ngmigration.service.entity.InfraMigrationService;
16+
import io.harness.ngmigration.service.entity.ManifestMigrationService;
17+
import io.harness.ngmigration.service.entity.PipelineMigrationService;
18+
import io.harness.ngmigration.service.entity.SecretManagerMigrationService;
19+
import io.harness.ngmigration.service.entity.SecretMigrationService;
20+
import io.harness.ngmigration.service.entity.ServiceMigrationService;
21+
import io.harness.ngmigration.service.entity.WorkflowMigrationService;
22+
1023
import software.wings.ngmigration.NGMigrationEntityType;
1124

1225
import com.google.inject.Inject;

‎100-migrator/src/main/java/io/harness/ngmigration/service/NgMigrationService.java‎

Lines changed: 105 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,40 +8,134 @@
88
package io.harness.ngmigration.service;
99

1010
import io.harness.beans.MigratedEntityMapping;
11+
import io.harness.encryption.Scope;
12+
import io.harness.gitsync.beans.YamlDTO;
1113
import io.harness.ngmigration.beans.BaseEntityInput;
14+
import io.harness.ngmigration.beans.BaseProvidedInput;
1215
import io.harness.ngmigration.beans.MigrationInputDTO;
16+
import io.harness.ngmigration.beans.NGYamlFile;
1317
import io.harness.ngmigration.beans.NgEntityDetail;
1418
import io.harness.ngmigration.client.NGClient;
1519
import io.harness.ngmigration.client.PmsClient;
20+
import io.harness.persistence.NameAccess;
1621

22+
import software.wings.ngmigration.CgBasicInfo;
1723
import software.wings.ngmigration.CgEntityId;
1824
import software.wings.ngmigration.CgEntityNode;
1925
import software.wings.ngmigration.DiscoveryNode;
2026
import software.wings.ngmigration.NGMigrationEntity;
27+
import software.wings.ngmigration.NGMigrationEntityType;
2128
import software.wings.ngmigration.NGMigrationStatus;
22-
import software.wings.ngmigration.NGYamlFile;
2329

30+
import com.google.inject.Inject;
2431
import java.io.IOException;
32+
import java.util.ArrayList;
33+
import java.util.Arrays;
2534
import java.util.List;
2635
import java.util.Map;
2736
import java.util.Set;
37+
import lombok.extern.slf4j.Slf4j;
38+
import org.apache.commons.lang3.StringUtils;
2839

29-
public interface NgMigrationService {
30-
MigratedEntityMapping generateMappingEntity(NGYamlFile yamlFile);
40+
@Slf4j
41+
public abstract class NgMigrationService {
42+
@Inject MigratorMappingService migratorMappingService;
3143

32-
DiscoveryNode discover(NGMigrationEntity entity);
44+
public abstract MigratedEntityMapping generateMappingEntity(NGYamlFile yamlFile);
3345

34-
DiscoveryNode discover(String accountId, String appId, String entityId);
46+
public abstract DiscoveryNode discover(NGMigrationEntity entity);
3547

36-
NGMigrationStatus canMigrate(
48+
public abstract DiscoveryNode discover(String accountId, String appId, String entityId);
49+
50+
public abstract NGMigrationStatus canMigrate(
3751
Map<CgEntityId, CgEntityNode> entities, Map<CgEntityId, Set<CgEntityId>> graph, CgEntityId entityId);
3852

39-
void migrate(String auth, NGClient ngClient, PmsClient pmsClient, MigrationInputDTO inputDTO, NGYamlFile yamlFile)
40-
throws IOException;
53+
public abstract void migrate(String auth, NGClient ngClient, PmsClient pmsClient, MigrationInputDTO inputDTO,
54+
NGYamlFile yamlFile) throws IOException;
55+
56+
public abstract List<NGYamlFile> generateYaml(MigrationInputDTO inputDTO, Map<CgEntityId, CgEntityNode> entities,
57+
Map<CgEntityId, Set<CgEntityId>> graph, CgEntityId entityId, Map<CgEntityId, NgEntityDetail> migratedEntities,
58+
NgEntityDetail ngEntityDetail);
59+
60+
public List<NGYamlFile> getYaml(MigrationInputDTO inputDTO, Map<CgEntityId, CgEntityNode> entities,
61+
Map<CgEntityId, Set<CgEntityId>> graph, CgEntityId entityId, Map<CgEntityId, NgEntityDetail> migratedEntities) {
62+
if (!isNGEntityExists()) {
63+
return new ArrayList<>();
64+
}
65+
CgEntityNode cgEntityNode = entities.get(entityId);
66+
// TODO: CG Basic Info should be part of CGEntityNode
67+
CgBasicInfo cgBasicInfo = CgBasicInfo.builder()
68+
.accountId(inputDTO.getAccountIdentifier())
69+
.appId(cgEntityNode.getAppId())
70+
.id(entityId.getId())
71+
.type(entityId.getType())
72+
.build();
73+
NgEntityDetail ngEntityDetail = getNGEntityDetail(inputDTO, entities, graph, entityId, migratedEntities);
74+
boolean mappingExist = migratorMappingService.doesMappingExist(cgBasicInfo, ngEntityDetail);
75+
NGYamlFile ngYamlFile =
76+
NGYamlFile.builder()
77+
.ngEntityDetail(ngEntityDetail)
78+
.cgBasicInfo(cgBasicInfo)
79+
.filename(entityId.getType().getYamlFolderName() + "/" + ngEntityDetail.getIdentifier() + ".yaml")
80+
.type(entityId.getType())
81+
.build();
82+
if (mappingExist) {
83+
YamlDTO yamlDTO = null;
84+
try {
85+
yamlDTO = getNGEntity(ngEntityDetail, inputDTO.getAccountIdentifier());
86+
} catch (Exception ex) {
87+
log.error("Failed to retrieve NG Entity. ", ex);
88+
}
89+
if (yamlDTO == null) {
90+
// Deleted
91+
return generateYaml(inputDTO, entities, graph, entityId, migratedEntities, ngEntityDetail);
92+
} else {
93+
ngYamlFile.setExists(true);
94+
ngYamlFile.setYaml(yamlDTO);
95+
migratedEntities.put(entityId, ngEntityDetail);
96+
return Arrays.asList(ngYamlFile);
97+
}
98+
} else {
99+
return generateYaml(inputDTO, entities, graph, entityId, migratedEntities, ngEntityDetail);
100+
}
101+
}
102+
103+
private NgEntityDetail getNGEntityDetail(MigrationInputDTO inputDTO, Map<CgEntityId, CgEntityNode> entities,
104+
Map<CgEntityId, Set<CgEntityId>> graph, CgEntityId entityId, Map<CgEntityId, NgEntityDetail> migratedEntities) {
105+
CgEntityNode cgEntityNode = entities.get(entityId);
106+
String identifier = null;
107+
if (cgEntityNode.getEntity() instanceof NameAccess) {
108+
identifier = MigratorUtility.generateIdentifier(((NameAccess) cgEntityNode.getEntity()).getName());
109+
}
110+
String projectIdentifier = null;
111+
String orgIdentifier = null;
112+
Scope scope =
113+
MigratorUtility.getDefaultScope(inputDTO.getDefaults(), NGMigrationEntityType.CONNECTOR, Scope.PROJECT);
114+
if (inputDTO.getInputs() != null && inputDTO.getInputs().containsKey(entityId)) {
115+
BaseProvidedInput input = inputDTO.getInputs().get(entityId);
116+
identifier = StringUtils.isNotBlank(input.getIdentifier()) ? input.getIdentifier() : identifier;
117+
if (input.getScope() != null) {
118+
scope = input.getScope();
119+
}
120+
}
121+
if (Scope.PROJECT.equals(scope)) {
122+
projectIdentifier = inputDTO.getProjectIdentifier();
123+
orgIdentifier = inputDTO.getOrgIdentifier();
124+
}
125+
if (Scope.ORG.equals(scope)) {
126+
orgIdentifier = inputDTO.getOrgIdentifier();
127+
}
128+
return NgEntityDetail.builder()
129+
.identifier(identifier)
130+
.projectIdentifier(projectIdentifier)
131+
.orgIdentifier(orgIdentifier)
132+
.build();
133+
}
134+
135+
protected abstract YamlDTO getNGEntity(NgEntityDetail ngEntityDetail, String accountIdentifier);
41136

42-
List<NGYamlFile> getYamls(MigrationInputDTO inputDTO, Map<CgEntityId, CgEntityNode> entities,
43-
Map<CgEntityId, Set<CgEntityId>> graph, CgEntityId entityId, Map<CgEntityId, NgEntityDetail> migratedEntities);
137+
protected abstract boolean isNGEntityExists();
44138

45-
BaseEntityInput generateInput(
139+
public abstract BaseEntityInput generateInput(
46140
Map<CgEntityId, CgEntityNode> entities, Map<CgEntityId, Set<CgEntityId>> graph, CgEntityId entityId);
47141
}

100-migrator/src/main/java/io/harness/ngmigration/service/AppMigrationService.java renamed to 100-migrator/src/main/java/io/harness/ngmigration/service/entity/AppMigrationService.java

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,17 @@
55
* https://polyformproject.org/wp-content/uploads/2020/05/PolyForm-Free-Trial-1.0.0.txt.
66
*/
77

8-
package io.harness.ngmigration.service;
8+
package io.harness.ngmigration.service.entity;
99

1010
import io.harness.beans.MigratedEntityMapping;
11+
import io.harness.gitsync.beans.YamlDTO;
1112
import io.harness.ngmigration.beans.BaseEntityInput;
1213
import io.harness.ngmigration.beans.MigrationInputDTO;
14+
import io.harness.ngmigration.beans.NGYamlFile;
1315
import io.harness.ngmigration.beans.NgEntityDetail;
1416
import io.harness.ngmigration.client.NGClient;
1517
import io.harness.ngmigration.client.PmsClient;
18+
import io.harness.ngmigration.service.NgMigrationService;
1619
import io.harness.persistence.HPersistence;
1720

1821
import software.wings.beans.Application;
@@ -24,7 +27,6 @@
2427
import software.wings.ngmigration.NGMigrationEntity;
2528
import software.wings.ngmigration.NGMigrationEntityType;
2629
import software.wings.ngmigration.NGMigrationStatus;
27-
import software.wings.ngmigration.NGYamlFile;
2830
import software.wings.service.intfc.AppService;
2931

3032
import com.google.inject.Inject;
@@ -35,7 +37,7 @@
3537
import java.util.Set;
3638
import java.util.stream.Collectors;
3739

38-
public class AppMigrationService implements NgMigrationService {
40+
public class AppMigrationService extends NgMigrationService {
3941
@Inject private AppService appService;
4042
@Inject private HPersistence hPersistence;
4143

@@ -87,11 +89,22 @@ public void migrate(String auth, NGClient ngClient, PmsClient pmsClient, Migrati
8789
}
8890

8991
@Override
90-
public List<NGYamlFile> getYamls(MigrationInputDTO inputDTO, Map<CgEntityId, CgEntityNode> entities,
91-
Map<CgEntityId, Set<CgEntityId>> graph, CgEntityId entityId, Map<CgEntityId, NgEntityDetail> migratedEntities) {
92+
public List<NGYamlFile> generateYaml(MigrationInputDTO inputDTO, Map<CgEntityId, CgEntityNode> entities,
93+
Map<CgEntityId, Set<CgEntityId>> graph, CgEntityId entityId, Map<CgEntityId, NgEntityDetail> migratedEntities,
94+
NgEntityDetail ngEntityDetail) {
9295
return new ArrayList<>();
9396
}
9497

98+
@Override
99+
protected YamlDTO getNGEntity(NgEntityDetail ngEntityDetail, String accountIdentifier) {
100+
return null;
101+
}
102+
103+
@Override
104+
protected boolean isNGEntityExists() {
105+
return false;
106+
}
107+
95108
@Override
96109
public BaseEntityInput generateInput(
97110
Map<CgEntityId, CgEntityNode> entities, Map<CgEntityId, Set<CgEntityId>> graph, CgEntityId entityId) {

0 commit comments

Comments
 (0)