Skip to content

Commit 6f9f6f6

Browse files
vberaudivberaudi
authored
Merge pull request #19 from IBMDecisionOptimization/vberaudi-custom-tags
Support custom params in WML-DO + path in COS storage.
2 parents 255d435 + ff684a4 commit 6f9f6f6

6 files changed

Lines changed: 94 additions & 36 deletions

File tree

‎src/main/java/com/ibm/ml/ilog/COSConnector.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33
import com.ibm.json.java.JSONObject;
44
import ilog.concert.IloException;
55

6-
import java.io.IOException;
7-
86
/*
97
A Simple connector to IBM Cloud Object Storage
108
*/
@@ -26,9 +24,13 @@ static COSConnector getConnector(Credentials creds) throws IloException {
2624
/*
2725
Method to create a connection asset from COS credentials.
2826
*/
29-
String getConnection() throws IloException;
27+
String getConnectionId() throws IloException;
28+
29+
void deleteConnection();
3030
/*
3131
Download the content of a COS file as a string.
3232
*/
3333
String getFile(String fileName) throws IloException;
34+
35+
void setBucketPath(String path) throws IloException;
3436
}

‎src/main/java/com/ibm/ml/ilog/Connector.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import com.ibm.json.java.JSONObject;
77

88
import java.io.IOException;
9+
import java.util.HashMap;
910

1011

1112
/* A WML interface */
@@ -145,7 +146,7 @@ public String toString() {
145146

146147
String getOrMakeDeployment(String name, boolean isCplex) throws IloException;
147148

148-
String createNewModel(String modelName, Runtime runtime, ModelType type, String modelAssetFilePath) throws IloException;
149+
String createNewModel(String modelName, Runtime runtime, ModelType type, String modelAssetFilePath, HashMap<String,String> custom) throws IloException;
149150

150151
String deployModel(String deployName, String model_id, TShirtSize size, int nodes) throws IloException;
151152

@@ -159,14 +160,16 @@ Job createJob(String deployment_id,
159160
JSONArray input_data,
160161
JSONArray input_data_references,
161162
JSONArray output_data,
162-
JSONArray output_data_references) throws IloException;
163+
JSONArray output_data_references, HashMap<String,String> custom) throws IloException;
163164

164165
Job createAndRunJob(String deployment_id,
165166
JSONArray input_data,
166167
JSONArray input_data_references,
167168
JSONArray output_data,
168-
JSONArray output_data_references) throws IloException;
169-
169+
JSONArray output_data_references, HashMap<String,String> custom) throws IloException;
170+
Job createAndRunJob(String deployment_id,
171+
JSONArray input_data_references,
172+
JSONArray output_data_references, HashMap<String,String> custom) throws IloException;
170173
JSONObject getDeployments() throws IloException;
171174

172175
JSONObject getModels() throws IloException;

‎src/main/java/com/ibm/ml/ilog/utils/HttpUtils.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -366,6 +366,8 @@ protected String doCall(String host, String url, Map<String, String> params, Map
366366
logger.warn("Calling " + url + " with empty token!");
367367
}
368368
}
369+
if (body != null && method.equals(PUT) == false)
370+
curl.append(new String(body));
369371
logger.info("Curl info: " + curl);
370372

371373
try {
@@ -415,7 +417,8 @@ public String doPost(String host, String targetUrl, Map<String, String> params,
415417
}
416418

417419
public String doGet(String host, String targetUrl, Map<String, String> params, Map<String, String> headers) throws IloException {
418-
return doCall(host, targetUrl, params, headers, null, GET);
420+
String ret = doCall(host, targetUrl, params, headers, null, GET);
421+
return ret;
419422
}
420423

421424
public String doDelete(String host, String targetUrl, Map<String, String> params, Map<String, String> headers) throws IloException {

‎src/main/java/com/ibm/ml/ilog/v4/COSConnector.java‎

Lines changed: 48 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,25 @@
11
package com.ibm.ml.ilog.v4;
22

3-
import com.ibm.json.java.JSON;
43
import com.ibm.json.java.JSONObject;
54
import com.ibm.ml.ilog.Credentials;
65
import com.ibm.ml.ilog.utils.HttpUtils;
76
import ilog.concert.IloException;
87
import org.apache.logging.log4j.LogManager;
98
import org.apache.logging.log4j.Logger;
109

11-
import java.io.IOException;
1210
import java.util.Date;
13-
import java.util.HashMap;
14-
import java.util.LinkedHashMap;
1511
import java.util.Map;
1612

1713
public class COSConnector extends HttpUtils implements com.ibm.ml.ilog.COSConnector {
1814

1915
private String _connectionId = null;
16+
private String _bucketPath = null;
2017
private static final Logger logger = LogManager.getLogger();
18+
@Override
19+
public void setBucketPath(String path) throws IloException {
20+
if (_bucketPath != null) throw new IloException("Path in bucket already exist");
21+
_bucketPath = path;
22+
}
2123

2224
@Override
2325
public void end() {
@@ -27,9 +29,8 @@ public void end() {
2729
Map<String, String> headers = getPlatformHeaders();
2830

2931
long t1 = new Date().getTime();
30-
String res = null;
3132
try {
32-
res = doDelete(
33+
doDelete(
3334
wml_credentials.get(Credentials.PLATFORM_HOST),
3435
V2_CONNECTIONS+"/"+_connectionId,
3536
params, headers);
@@ -44,6 +45,29 @@ public void end() {
4445
super.end();
4546
}
4647

48+
public void deleteConnection() {
49+
if (_connectionId != null) {
50+
Map<String, String> params = getWMLParams();
51+
52+
Map<String, String> headers = getPlatformHeaders();
53+
54+
long t1 = new Date().getTime();
55+
String res = null;
56+
try {
57+
res = doDelete(
58+
wml_credentials.get(Credentials.PLATFORM_HOST),
59+
V2_CONNECTIONS + "/" + _connectionId,
60+
params, headers);
61+
} catch (IloException e) {
62+
logger.error(e.getMessage());
63+
}
64+
long t2 = new Date().getTime();
65+
logger.info("Connection Id = " + _connectionId);
66+
logger.info("Deleting the connection took " + (t2 - t1) / 1000 + " seconds.");
67+
_connectionId = null;
68+
}
69+
}
70+
4771
private void checkCredentials() throws IloException {
4872
for (String k : Credentials.COSFields) {
4973
if (!wml_credentials.containsKey(k)) {
@@ -66,25 +90,27 @@ public JSONObject getDataReferences(String id) throws IloException {
6690

6791
String cos_bucket = wml_credentials.get(Credentials.COS_BUCKET);
6892

93+
String filename = "\"file_name\": \"";
94+
if (_bucketPath != null)
95+
filename = filename + _bucketPath + "/";
96+
filename = filename + id + "\"\n";
6997
String data = "{\n" +
7098
//"\"id\": \"" + id + "\",\n" +
7199
"\"type\": \"connection_asset\",\n" +
72100
"\"id\": \"" + id + "\",\n" +
73101
"\"connection\": {\n" +
74-
"\"id\": \"" + getConnection() + "\"\n" +
102+
"\"id\": \"" + getConnectionId() + "\"\n" +
75103
"},\n" +
76104
"\"location\": {\n" +
77105
"\"bucket\": \"" + cos_bucket + "\",\n" +
78-
"\"file_name\": \"" + id + "\"\n" +
106+
filename +
79107
"},\n" +
80108
"}\n";
81109
return parseJson(data);
82110
}
83111

84112
@Override
85113
public String putFile(String fileName, String filePath) throws IloException {
86-
String cos_bucket = wml_credentials.get(Credentials.COS_BUCKET);
87-
88114
byte[] bytes = getFileContent(filePath);
89115

90116
Map<String, String> params = getPlatformParams();
@@ -96,13 +122,22 @@ public String putFile(String fileName, String filePath) throws IloException {
96122
long t1 = new Date().getTime();
97123
String ret = doPut(
98124
wml_credentials.get(Credentials.COS_ENDPOINT),
99-
"/" + cos_bucket + "/" + fileName,
125+
getTargetUrl(fileName),
100126
params, headers, bytes);
101127
long t2 = new Date().getTime();
102128
logger.info("Uploading in COS took " + (t2 - t1) / 1000 + " seconds.");
103129
return ret;
104130
}
105131

132+
private String getTargetUrl(String fileName){
133+
String cos_bucket = wml_credentials.get(Credentials.COS_BUCKET);
134+
String targetUrl = "/" + cos_bucket + "/";
135+
if (_bucketPath != null)
136+
targetUrl = targetUrl + _bucketPath + "/";
137+
targetUrl = targetUrl + fileName;
138+
return targetUrl;
139+
}
140+
106141
private String getS3Id() throws IloException{
107142
Map<String, String> params = getWMLParams();
108143
Map<String, String> headers = getPlatformHeaders();
@@ -117,7 +152,7 @@ private String getS3Id() throws IloException{
117152

118153

119154
@Override
120-
public String getConnection() throws IloException {
155+
public String getConnectionId() throws IloException {
121156
if (_connectionId != null)
122157
return _connectionId;
123158
long t1 = new Date().getTime();
@@ -158,14 +193,12 @@ public String getConnection() throws IloException {
158193

159194
@Override
160195
public String getFile(String fileName) throws IloException {
161-
String cos_bucket = wml_credentials.get(Credentials.COS_BUCKET);
162-
163196
Map<String, String> params = getPlatformParams();
164197
Map<String, String> headers = getPlatformHeaders();
165198

166199
String res = doGet(
167200
wml_credentials.get(Credentials.COS_ENDPOINT),
168-
"/" + cos_bucket + "/" + fileName,
201+
getTargetUrl(fileName),
169202
params,
170203
headers);
171204

‎src/main/java/com/ibm/ml/ilog/v4/Connector.java‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -422,13 +422,23 @@ public byte[] buildPayload(String deployment_id, String sav, String savFileName,
422422
}
423423
return ret;
424424
}
425+
@Override
426+
public Job createAndRunJob(String deployment_id,
427+
JSONArray input_data_references,
428+
JSONArray output_data_references, HashMap<String, String> custom) throws IloException {
429+
return createAndRunJob(deployment_id,
430+
null,
431+
input_data_references,
432+
null,
433+
output_data_references, custom);
434+
}
425435

426436
@Override
427437
public Job createJob(String deployment_id,
428438
JSONArray input_data,
429439
JSONArray input_data_references,
430440
JSONArray output_data,
431-
JSONArray output_data_references) throws IloException {
441+
JSONArray output_data_references, HashMap<String,String> custom) throws IloException {
432442
logger.info("Create job");
433443
JSONObject payload = new JSONObject();
434444

@@ -446,6 +456,11 @@ public Job createJob(String deployment_id,
446456
solve_parameters.put("oaas.logTailEnabled", "true");
447457
else
448458
solve_parameters.put("oaas.logTailEnabled", "false");
459+
if (custom != null) {
460+
for (Map.Entry<String, String> set : ((HashMap<String, String>)custom).entrySet()) {
461+
solve_parameters.put(set.getKey(), set.getValue());
462+
}
463+
}
449464
solve_parameters.put("oaas.includeInputData", "false");
450465
solve_parameters.put("oaas.resultsFormat", resultFormat);
451466
solve_parameters.put("oaas.engineLogLevel", engineLogLevel);
@@ -615,8 +630,8 @@ public Job createAndRunJob(String deployment_id,
615630
JSONArray input_data,
616631
JSONArray input_data_references,
617632
JSONArray output_data,
618-
JSONArray output_data_references) throws IloException {
619-
Job job = createJob(deployment_id, input_data, input_data_references, output_data, output_data_references);
633+
JSONArray output_data_references, HashMap<String,String> custom) throws IloException {
634+
Job job = createJob(deployment_id, input_data, input_data_references, output_data, output_data_references, custom);
620635

621636
String state = waitCompletion(job);
622637

@@ -631,7 +646,7 @@ public Job createAndRunJob(String deployment_id,
631646

632647

633648
@Override
634-
public String createNewModel(String modelName, Runtime runtime, ModelType type, String modelAssetFilePath) throws IloException {
649+
public String createNewModel(String modelName, Runtime runtime, ModelType type, String modelAssetFilePath, HashMap<String,String> custom) throws IloException {
635650
String modelId = null;
636651
{
637652
Map<String, String> headers = getWMLHeaders();
@@ -644,6 +659,15 @@ public String createNewModel(String modelName, Runtime runtime, ModelType type,
644659
JSONObject soft = new JSONObject();
645660
soft.put(NAME, runtime.getShortName());
646661
payload.put("software_spec", soft);
662+
if (custom != null){
663+
JSONObject jcustom = new JSONObject();
664+
JSONObject decision_optimization = new JSONObject();
665+
for (Map.Entry<String, String> set : ((HashMap<String, String>)custom).entrySet()) {
666+
decision_optimization.put(set.getKey(), set.getValue());
667+
}
668+
jcustom.put("decision_optimization", decision_optimization);
669+
payload.put("custom", jcustom);
670+
}
647671
payload.put(SPACE_ID, wml_credentials.get(Credentials.WML_SPACE_ID));
648672

649673
String res = doPost(
@@ -842,7 +866,7 @@ public String getOrMakeDeployment(String name, boolean isCplex) throws IloExcept
842866
else
843867
type = com.ibm.ml.ilog.Connector.getCPOModelType(wml_runtime);
844868

845-
String model_id = this.createNewModel(name, wml_runtime, type, null);
869+
String model_id = this.createNewModel(name, wml_runtime, type, null, null);
846870
logger.info("model_id = " + model_id);
847871

848872
deployment_id = this.deployModel(name, model_id, wml_size, wml_nodes);

‎src/main/java/ilog/cplex/WmlCplex.java‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -49,14 +49,7 @@ public WmlCplex(Credentials credentials, Connector.Runtime runtime, Connector.TS
4949
@Override
5050
public void end(){
5151
logger.info("Cleaning the CPLEX job");
52-
if (job != null){
53-
logger.info("Cleaning remaining job "+job.getId());
54-
try {
55-
wmlConnector.deleteJob(job.getId());
56-
} catch (IloException e) {
57-
logger.warn("Ignoring error "+e.getMessage());
58-
}
59-
}
52+
6053
wmlConnector.end();
6154
super.end();
6255
}

0 commit comments

Comments
 (0)