Skip to content

Commit 8418a5e

Browse files
mbenguigmbenguig
authored
GCE integration (not finalized)
* GCE integration * GCE integration * GCE integration * GCE integration
1 parent ddb3468 commit 8418a5e

10 files changed

Lines changed: 258 additions & 99 deletions

File tree

‎sal-common/src/main/java/org/ow2/proactive/sal/model/CloudProviderType.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ public enum CloudProviderType {
1616

1717
AWS_EC2("aws-ec2"),
1818
AZURE("azure"),
19-
GCE("gce"),
19+
GCE("google-compute-engine"),
2020
OPENSTACK("openstack"), //openstack-nova
2121

2222
EDGE("EDGE"),

‎sal-common/src/main/java/org/ow2/proactive/sal/model/Credential.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ public class Credential implements Serializable {
2828
// JSON field constants
2929
public static final String JSON_USER = "user";
3030

31+
public static final String JSON_PROJECT_ID = "projectId";
32+
3133
public static final String JSON_SECRET = "secret";
3234

3335
public static final String JSON_DOMAIN = "domain";
@@ -37,6 +39,9 @@ public class Credential implements Serializable {
3739
@JsonProperty(JSON_USER)
3840
private String user = null;
3941

42+
@JsonProperty(JSON_PROJECT_ID)
43+
private String projectId = null;
44+
4045
@JsonProperty(JSON_SECRET)
4146
private String secret = null;
4247

‎sal-common/src/main/java/org/ow2/proactive/sal/model/Credentials.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ public class Credentials implements Serializable {
3131

3232
public static final String JSON_USER_NAME = "userName";
3333

34+
public static final String JSON_PROJECT_ID = "projectId";
35+
3436
public static final String JSON_PASSWORD = "password";
3537

3638
public static final String JSON_PRIVATE_KEY = "privateKey";
@@ -51,6 +53,10 @@ public class Credentials implements Serializable {
5153
@JsonProperty(JSON_USER_NAME)
5254
private String userName;
5355

56+
@Column(name = "PROJECT_ID")
57+
@JsonProperty(JSON_PROJECT_ID)
58+
private String projectId;
59+
5460
@Column(name = "PASSWORD")
5561
@JsonProperty(JSON_PASSWORD)
5662
private String password;

‎sal-common/src/main/java/org/ow2/proactive/sal/model/EmsDeploymentRequest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public enum TargetProvider {
5959
// Azure VM
6060
AZUREVM("azure"),
6161
// Google CLoud Engine
62-
GCE("gce"),
62+
GCE("google-compute-engine"),
6363
// OpenStack NOVA
6464
OPENSTACKNOVA("openstack"),
6565
// BYON, to be used for on-premise baremetal

‎sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java‎

Lines changed: 64 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import javax.annotation.PostConstruct;
1919

2020
import org.apache.commons.lang3.StringUtils;
21+
import org.javatuples.Pair;
2122
import org.javatuples.Quartet;
2223
import org.json.JSONArray;
2324
import org.json.JSONObject;
@@ -363,6 +364,22 @@ private static JSONObject convertObjectToJson(Object object) {
363364
return myJson;
364365
}
365366

367+
private String getOsAccordingToCloudProvider(CloudProviderType cloudProviderType, String os)
368+
throws IllegalArgumentException {
369+
switch (cloudProviderType) {
370+
case AWS_EC2:
371+
return "Linux";
372+
case OPENSTACK:
373+
return os;
374+
case AZURE:
375+
return os;
376+
case GCE:
377+
return os;
378+
default:
379+
throw new IllegalArgumentException("The infrastructure " + cloudProviderType + " is not handled yet.");
380+
}
381+
}
382+
366383
public void saveNodeCandidates(List<String> newCloudIds) {
367384
newCloudIds.forEach(newCloudId -> {
368385
PACloud paCloud = repositoryService.getPACloud(newCloudId);
@@ -378,65 +395,60 @@ public void saveNodeCandidates(List<String> newCloudIds) {
378395
return;
379396
}
380397
LOGGER.info("Returned images: {}", images);
381-
List<JSONObject> consolidatedImages = images.toList()
382-
.parallelStream()
383-
.map(NodeCandidateUtils::convertObjectToJson)
384-
.filter(record -> !blacklistedRegions.contains(record.get("location")))
385-
.collect(Collectors.toList());
386-
LOGGER.info("Consolidated images: {}", consolidatedImages);
387-
388-
//TODO: (Optimization) An images per region map structure <region,[image1,image2]> could be the best here.
389-
// It can reduce the getNodeCandidates calls to PA.
390-
List<String> entries = new LinkedList<>();
391-
List<String> openstackOsList = Arrays.asList("Ubuntu", "Fedora", "Centos", "Debian");
392-
consolidatedImages.forEach(image -> {
393-
String region = image.optString("location");
394-
String imageReq;
395-
String os = (String) ((JSONObject) image.get("operatingSystem")).get("family");
396-
os = os.substring(0, 1).toUpperCase() + os.substring(1);
397-
String pair = os + ":" + region;
398-
399-
switch (paCloud.getCloudProvider()) {
400-
case AWS_EC2:
401-
imageReq = "Linux";
402-
break;
403-
case OPENSTACK:
404-
imageReq = os;
405-
break;
406-
case AZURE:
407-
imageReq = os;
408-
break;
409-
default:
410-
throw new IllegalArgumentException("The infrastructure " + paCloud.getCloudType() +
411-
" is not handled yet.");
412-
}
413-
414-
if (paCloud.getCloudProvider() == OPENSTACK) {
415-
entries.add(pair);
398+
Map<Pair<String, String>, List<JSONObject>> consolidatedImagesGrouped = images.toList()
399+
.parallelStream()
400+
.map(NodeCandidateUtils::convertObjectToJson)
401+
.filter(record -> record.get("location")
402+
.toString()
403+
.isEmpty() ||
404+
!blacklistedRegions.contains(record.get("location")))
405+
.collect(Collectors.groupingBy(image -> {
406+
// Retrieve the region
407+
String region = image.optString("location");
408+
// Retrieve the imageReq
409+
String os = (String) ((JSONObject) image.get("operatingSystem")).get("family");
410+
os = os.substring(0, 1)
411+
.toUpperCase() +
412+
os.substring(1);
413+
String imageReq = getOsAccordingToCloudProvider(paCloud.getCloudProvider(),
414+
os);
415+
416+
return new Pair<>(region,
417+
imageReq);
418+
}));
419+
420+
consolidatedImagesGrouped.entrySet().parallelStream().forEach(entry -> {
421+
String region = entry.getKey().getValue0();
422+
String imageReq = entry.getKey().getValue1();
423+
List<JSONObject> imageList = entry.getValue();
424+
425+
try {
426+
JSONArray nodeCandidates = nodeCandidatesCache.get(Quartet.with(paCloud, region, imageReq, ""));
427+
428+
imageList.forEach(image -> createAndStoreIaasNodesAndNodeCandidates(nodeCandidates,
429+
paCloud,
430+
image));
431+
} catch (ExecutionException e) {
432+
LOGGER.error("Could not get node candidates from cache: ", e);
416433
}
417-
populateNodeCandidatesFromCache(paCloud, region, imageReq, image);
418434
});
435+
419436
});
420437

421438
repositoryService.flush();
422439
}
423440

424-
private void populateNodeCandidatesFromCache(PACloud paCloud, String region, String imageReq, JSONObject image) {
425-
try {
426-
JSONArray nodeCandidates = nodeCandidatesCache.get(Quartet.with(paCloud, region, imageReq, ""));
427-
nodeCandidates.forEach(nc -> {
428-
JSONObject nodeCandidate = (JSONObject) nc;
429-
createLocation(nodeCandidate, paCloud);
430-
NodeCandidate newNodeCandidate = createNodeCandidate(nodeCandidate, image, paCloud);
431-
repositoryService.saveNodeCandidate(newNodeCandidate);
432-
IaasNode newIaasNode = new IaasNode(newNodeCandidate);
433-
repositoryService.saveIaasNode(newIaasNode);
434-
newNodeCandidate.setNodeId(newIaasNode.getId());
435-
repositoryService.saveNodeCandidate(newNodeCandidate);
436-
});
437-
} catch (ExecutionException ee) {
438-
LOGGER.error("Could not get node candidates from cache: ", ee);
439-
}
441+
private void createAndStoreIaasNodesAndNodeCandidates(JSONArray nodeCandidates, PACloud paCloud, JSONObject image) {
442+
nodeCandidates.forEach(nc -> {
443+
JSONObject nodeCandidate = (JSONObject) nc;
444+
createLocation(nodeCandidate, paCloud);
445+
NodeCandidate newNodeCandidate = createNodeCandidate(nodeCandidate, image, paCloud);
446+
repositoryService.saveNodeCandidate(newNodeCandidate);
447+
IaasNode newIaasNode = new IaasNode(newNodeCandidate);
448+
repositoryService.saveIaasNode(newIaasNode);
449+
newNodeCandidate.setNodeId(newIaasNode.getId());
450+
repositoryService.saveNodeCandidate(newNodeCandidate);
451+
});
440452
}
441453

442454
private JSONArray getAllPagedNodeCandidates(PACloud paCloud, String region, String imageReq, String token) {

‎sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ public Integer addClouds(String sessionId, List<CloudDefinition> clouds) throws
9797

9898
Credentials credentials = new Credentials();
9999
credentials.setUserName(cloud.getCredentials().getUser());
100+
credentials.setProjectId(cloud.getCredentials().getProjectId());
100101
credentials.setPrivateKey(cloud.getCredentials().getSecret());
101102
credentials.setDomain(cloud.getCredentials().getDomain());
102103
credentials.setSubscriptionId(cloud.getCredentials().getSubscriptionId());
@@ -467,6 +468,9 @@ private Credentials hideCredentials(Credentials creds) {
467468
if (creds.getUserName() != null) {
468469
newCreds.setUserName(hideString(creds.getUserName(), 5));
469470
}
471+
if (creds.getProjectId() != null) {
472+
newCreds.setProjectId(hideString(creds.getProjectId(), 5));
473+
}
470474
}
471475
return newCreds;
472476
}

‎sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,17 @@ public Deployment addNode(IaasDefinition node, Job job) {
167167
return newDeployment;
168168
}
169169

170+
private String getZoneFromGceHardwareLocation(String location) {
171+
String[] locationAsArray = location.split("/");
172+
List<String> locationAsList = Arrays.asList(locationAsArray);
173+
int indexOfZones = locationAsList.indexOf("zones");
174+
if (indexOfZones == -1) {
175+
return "null";
176+
} else {
177+
return locationAsArray[indexOfZones + 1];
178+
}
179+
}
180+
170181
/**
171182
* Define a node source in PA server related to a deployment information
172183
* @param nodeSourceName A valid and unique node source name
@@ -251,6 +262,25 @@ private void defineNSWithDeploymentInfo(String nodeSourceName, PACloud cloud, De
251262
variables.put("vmPublicKey", cloud.getSshCredentials().getPublicKey());
252263
variables.put("region", deployment.getNode().getNodeCandidate().getLocation().getName());
253264
break;
265+
case GCE:
266+
filename = File.separator + "Define_NS_GCE.xml";
267+
variables.put("user", cloud.getCredentials().getUserName());
268+
variables.put("projectId", cloud.getCredentials().getProjectId());
269+
variables.put("secret", cloud.getCredentials().getPrivateKey());
270+
variables.put("vmUsername", cloud.getSshCredentials().getUsername());
271+
variables.put("vmPublicKey", cloud.getSshCredentials().getPublicKey());
272+
variables.put("vmPrivateKey", cloud.getSshCredentials().getPrivateKey());
273+
variables.put("image", deployment.getNode().getNodeCandidate().getImage().getName());
274+
// To avoid: "Machine type specified xxx is in a different scope than the instance"
275+
variables.put("region",
276+
getZoneFromGceHardwareLocation(deployment.getNode()
277+
.getNodeCandidate()
278+
.getHardware()
279+
.getId()));
280+
variables.put("machineType", deployment.getNode().getNodeCandidate().getHardware().getProviderId());
281+
variables.put("ram", deployment.getNode().getNodeCandidate().getHardware().getRam().toString());
282+
variables.put("cores", deployment.getNode().getNodeCandidate().getHardware().getCores().toString());
283+
break;
254284
default:
255285
throw new IllegalArgumentException("Unhandled cloud provider: " + cloud.getCloudProvider());
256286
}

‎sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java‎

Lines changed: 13 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -343,53 +343,19 @@ private Map<String, TaskVariable> createVariablesMapForAcquiringIAASNode(Task ta
343343
return (variablesMap);
344344
}
345345

346-
private ScriptTask createInfraIAASTaskForAWS(Task task, Deployment deployment, String taskNameSuffix,
347-
String nodeToken) {
348-
LOGGER.debug("Acquiring node AWS script file: " +
349-
getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString());
350-
ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireAWSNode_" + task.getName() +
351-
taskNameSuffix, ACQUIRE_NODE_SCRIPT);
352-
353-
deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy"));
354-
355-
Map<String, TaskVariable> variablesMap = createVariablesMapForAcquiringIAASNode(task, deployment, nodeToken);
356-
LOGGER.debug("Variables to be added to the task acquiring AWS IAAS node: " + variablesMap.toString());
357-
deployNodeTask.setVariables(variablesMap);
358-
359-
addLocalDefaultNSRegexSelectionScript(deployNodeTask);
360-
361-
return deployNodeTask;
362-
}
363-
364-
private ScriptTask createInfraIAASTaskForOS(Task task, Deployment deployment, String taskNameSuffix,
365-
String nodeToken) {
366-
LOGGER.debug("Acquiring node OS script file: " +
346+
private ScriptTask createInfraIAASTaskForCloudProvider(Task task, Deployment deployment, String taskNameSuffix,
347+
String nodeToken, String cloudProvider) {
348+
LOGGER.debug("Acquiring node " + cloudProvider + " script file: " +
367349
getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString());
368-
ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireOSNode_" + task.getName() +
369-
taskNameSuffix, ACQUIRE_NODE_SCRIPT);
370-
371-
deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy"));
372-
373-
Map<String, TaskVariable> variablesMap = createVariablesMapForAcquiringIAASNode(task, deployment, nodeToken);
374-
LOGGER.debug("Variables to be added to the task acquiring OS IAAS node: " + variablesMap.toString());
375-
deployNodeTask.setVariables(variablesMap);
376-
377-
addLocalDefaultNSRegexSelectionScript(deployNodeTask);
378-
379-
return deployNodeTask;
380-
}
381-
382-
private ScriptTask createInfraIAASTaskForAzure(Task task, Deployment deployment, String taskNameSuffix,
383-
String nodeToken) {
384-
LOGGER.debug("Acquiring node Azure script file: " +
385-
getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString());
386-
ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireAzureNode_" + task.getName() +
387-
taskNameSuffix, ACQUIRE_NODE_SCRIPT);
350+
ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquire" + cloudProvider + "Node_" +
351+
task.getName() + taskNameSuffix,
352+
ACQUIRE_NODE_SCRIPT);
388353

389354
deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy"));
390355

391356
Map<String, TaskVariable> variablesMap = createVariablesMapForAcquiringIAASNode(task, deployment, nodeToken);
392-
LOGGER.debug("Variables to be added to the task acquiring Azure IAAS node: " + variablesMap.toString());
357+
LOGGER.debug("Variables to be added to the task acquiring " + cloudProvider + " IAAS node: " +
358+
variablesMap.toString());
393359
deployNodeTask.setVariables(variablesMap);
394360

395361
addLocalDefaultNSRegexSelectionScript(deployNodeTask);
@@ -400,11 +366,13 @@ private ScriptTask createInfraIAASTaskForAzure(Task task, Deployment deployment,
400366
private ScriptTask createInfraIAASTask(Task task, Deployment deployment, String taskNameSuffix, String nodeToken) {
401367
switch (deployment.getPaCloud().getCloudProvider()) {
402368
case AWS_EC2:
403-
return createInfraIAASTaskForAWS(task, deployment, taskNameSuffix, nodeToken);
369+
return createInfraIAASTaskForCloudProvider(task, deployment, taskNameSuffix, nodeToken, "AWS");
404370
case OPENSTACK:
405-
return createInfraIAASTaskForOS(task, deployment, taskNameSuffix, nodeToken);
371+
return createInfraIAASTaskForCloudProvider(task, deployment, taskNameSuffix, nodeToken, "OS");
406372
case AZURE:
407-
return createInfraIAASTaskForAzure(task, deployment, taskNameSuffix, nodeToken);
373+
return createInfraIAASTaskForCloudProvider(task, deployment, taskNameSuffix, nodeToken, "Azure");
374+
case GCE:
375+
return createInfraIAASTaskForCloudProvider(task, deployment, taskNameSuffix, nodeToken, "GCE");
408376
default:
409377
return new ScriptTask();
410378
}

‎sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAConnectorIaasGateway.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,13 @@ public void defineInfrastructure(String infrastructureName, PACloud cloud, Strin
152152
cloud.getCredentials().getDomain() + "\", \"subscriptionId\": \"" +
153153
cloud.getCredentials().getSubscriptionId() + "\"}}";
154154
break;
155+
case GCE:
156+
jsonOutputString = "{\"id\": \"" + infrastructureName + "\"," + "\"type\": \"" +
157+
cloud.getCloudProvider() + "\"," + "\"credentials\": {\"username\": \"" +
158+
cloud.getCredentials().getUserName() + "\", \"projectId\": \"" +
159+
cloud.getCredentials().getProjectId() + "\", \"password\": \"" +
160+
cloud.getCredentials().getPrivateKey() + "\"}, \"region\": \"" + region + "\"}";
161+
break;
155162
default:
156163
throw new IllegalArgumentException("The infrastructure " + infrastructureName + " is not handled yet.");
157164
}

0 commit comments

Comments
 (0)