Skip to content

Commit dd65d79

Browse files
HenryGeorgistHenryGeorgist
authored
Merge pull request #16 from USACE/kansascity
Kansascity
2 parents 16cd96b + f1783d6 commit dd65d79

34 files changed

Lines changed: 955 additions & 403 deletions

‎.github/workflows/CI.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@ jobs:
2323

2424
steps:
2525
- uses: actions/checkout@v3
26-
- name: Set up JDK 11
26+
- name: Set up JDK 17
2727
uses: actions/setup-java@v3
2828
with:
29-
java-version: '11'
29+
java-version: '17'
3030
distribution: 'temurin'
3131

3232
- name: Validate Gradle wrapper

‎Dockerfile‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
1-
FROM mcr.microsoft.com/openjdk/jdk:17-ubuntu as dev
2-
ENV TZ=America/New_York
3-
RUN apt update
4-
RUN apt -y install wget
5-
RUN apt -y install git
6-
#FROM ubuntu:20.04 as prod
1+
FROM --platform=linux/amd64 ubuntu:24.04
2+
3+
ENV TZ=America/New_York
4+
5+
RUN echo 'Acquire::http::Pipeline-Depth 0;\nAcquire::http::No-Cache true; \nAcquire::BrokenProxy true;\n' > /etc/apt/apt.conf.d/99fixbadproxy
6+
7+
RUN apt update &&\
8+
apt -y install wget &&\
9+
apt -y install git &&\
10+
apt -y install curl &&\
11+
apt -y install vim &&\
12+
apt -y install unzip &&\
13+
apt -y install openjdk-17-jdk

‎build.gradle‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ dependencies {
1818
implementation 'com.amazonaws:aws-java-sdk-s3'
1919
}
2020
jar {
21-
sourceCompatibility = JavaVersion.VERSION_11
22-
targetCompatibility = JavaVersion.VERSION_11
21+
sourceCompatibility = JavaVersion.VERSION_17
22+
targetCompatibility = JavaVersion.VERSION_17
2323
}
2424
publishing {
2525
repositories {
@@ -51,5 +51,5 @@ publishing {
5151
}
5252

5353
group 'mil.army.usace.hec'
54-
version '0.0.56'
54+
version '0.0.59'
5555
}

‎build/tmp/compileJava/previous-compilation-data.bin‎

-44.6 KB
Binary file not shown.

‎build/tmp/jar/MANIFEST.MF‎

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎src/main/java/usace/cc/plugin/AWSConfig.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,30 @@
11
package usace.cc.plugin;
2+
23
import com.fasterxml.jackson.annotation.JsonProperty;
4+
35
public class AWSConfig {
46
@JsonProperty
57
public String aws_config_name;
8+
69
@JsonProperty
710
public String aws_access_key_id;
11+
812
@JsonProperty
913
public String aws_secret_access_key_id;
14+
1015
@JsonProperty
1116
public String aws_region;
17+
1218
@JsonProperty
1319
public String aws_bucket;
14-
@JsonProperty
15-
public Boolean aws_mock;
20+
1621
@JsonProperty
1722
public String aws_endpoint;
23+
1824
@JsonProperty
1925
public Boolean aws_disable_ssl;
26+
2027
@JsonProperty
2128
public Boolean aws_force_path_style;
29+
2230
}

‎src/main/java/usace/cc/plugin/Action.java‎

Lines changed: 9 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,21 @@
11
package usace.cc.plugin;
22

3-
import java.util.Map;
4-
53
import com.fasterxml.jackson.annotation.JsonProperty;
64

7-
public class Action {
8-
@JsonProperty
9-
private String name;
5+
public class Action extends IOManager{
6+
107
@JsonProperty
8+
private String type;
9+
10+
@JsonProperty("description")
1111
private String desc;
12-
@JsonProperty
13-
private Map<String,DataSource> params;
14-
public String getName(){
15-
return name;
16-
}
1712

13+
public String getType(){
14+
return type;
15+
}
16+
1817
public String getDescription(){
1918
return desc;
2019
}
21-
22-
public Map<String,DataSource> getParameters(){
23-
return params;
24-
}
25-
public void UpdateActionPaths(){
26-
PluginManager pm = PluginManager.getInstance();
27-
this.name = pm.SubstitutePath(this.name);
28-
this.desc = pm.SubstitutePath(this.desc);
29-
if(params!=null){
30-
for(Map.Entry<String, DataSource> apb : params.entrySet()){
31-
params.replace(apb.getKey(),apb.getValue().UpdatePaths());
32-
}
33-
}
3420

35-
}
3621
}

‎src/main/java/usace/cc/plugin/CcStore.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
package usace.cc.plugin;
22

33
public interface CcStore {
4-
public boolean PutObject(PutObjectInput input);
5-
public boolean PullObject(PullObjectInput input);
6-
public byte[] GetObject(GetObjectInput input) throws Exception;
7-
public Payload GetPayload() throws Exception;
4+
public boolean putObject(PutObjectInput input);
5+
public boolean pullObject(PullObjectInput input);
6+
public byte[] getObject(GetObjectInput input) throws Exception;
7+
public Payload getPayload() throws Exception;
88
//public void SetPayload(Payload payload); only used in the go sdk to support cloudcompute which is written in go.
9-
public String RootPath();
10-
public boolean HandlesDataStoreType(StoreType datastoretype);
9+
public String rootPath();
10+
public boolean handlesDataStoreType(StoreType datastoretype);
1111
}

‎src/main/java/usace/cc/plugin/CcStoreS3.java‎

Lines changed: 47 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -34,37 +34,27 @@ public class CcStoreS3 implements CcStore {
3434
String bucket;
3535
String root;
3636
String manifestId;
37+
String payloadId;
3738
StoreType storeType;
3839
AmazonS3 awsS3;
3940
AWSConfig config;
40-
@Override
41-
public String RootPath() {
42-
return bucket;
43-
}
41+
4442
public CcStoreS3(){
4543
AWSConfig acfg = new AWSConfig();
4644
acfg.aws_access_key_id = System.getenv(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_ACCESS_KEY_ID);
4745
acfg.aws_secret_access_key_id = System.getenv(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_SECRET_ACCESS_KEY);
4846
acfg.aws_region = System.getenv(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_DEFAULT_REGION);
4947
acfg.aws_bucket = System.getenv(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_S3_BUCKET);
50-
acfg.aws_mock = Boolean.parseBoolean(System.getenv(EnvironmentVariables.CC_PROFILE + "_" +"S3_MOCK"));//convert to boolean;//stringformat
51-
acfg.aws_endpoint = System.getenv(EnvironmentVariables.CC_PROFILE + "_" +"S3_ENDPOINT");
52-
acfg.aws_disable_ssl = Boolean.parseBoolean(System.getenv(EnvironmentVariables.CC_PROFILE + "_" +"S3_DISABLE_SSL"));//convert to bool?
53-
acfg.aws_force_path_style = Boolean.parseBoolean(System.getenv(EnvironmentVariables.CC_PROFILE + "_" +"S3_FORCE_PATH_STYLE"));//convert to bool
48+
acfg.aws_endpoint = System.getenv(EnvironmentVariables.CC_PROFILE + "_" +"AWS_ENDPOINT");
5449
config = acfg;
55-
//System.out.println(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_DEFAULT_REGION+"::"+config.aws_region);
56-
//System.out.println(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_ACCESS_KEY_ID+"::"+config.aws_access_key_id);
57-
//System.out.println(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_SECRET_ACCESS_KEY+"::"+config.aws_secret_access_key_id);
58-
//System.out.println(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_S3_BUCKET+"::"+config.aws_bucket);
50+
5951
Region clientRegion = RegionUtils.getRegion(config.aws_region);//.toUpperCase().replace("-", "_"));//Regions.valueOf(config.aws_region.toUpperCase().replace("-", "_"));
6052
try {
6153
AmazonS3 s3Client = null;
62-
if(config.aws_mock){
63-
System.out.println("mocking s3 with minio");
64-
//System.out.println(EnvironmentVariables.CC_PROFILE + "_S3_MOCK::"+config.aws_mock);
65-
//System.out.println(EnvironmentVariables.CC_PROFILE + "_S3_ENDPOINT::"+config.aws_endpoint);
66-
//System.out.println(EnvironmentVariables.CC_PROFILE + "_S3_DISABLE_SSL::"+config.aws_disable_ssl);
67-
//System.out.println(EnvironmentVariables.CC_PROFILE + "_S3_FORCE_PATH_STYLE::"+config.aws_force_path_style);
54+
if(!config.aws_endpoint.equals("")){
55+
System.out.println(String.format("Using alt endpoint: %s",config.aws_endpoint));
56+
config.aws_force_path_style=true;
57+
config.aws_disable_ssl=true;
6858
AWSCredentials credentials = new BasicAWSCredentials(config.aws_access_key_id, config.aws_secret_access_key_id);
6959
ClientConfiguration clientConfiguration = new ClientConfiguration();
7060
clientConfiguration.setSignerOverride("AWSS3V4SignerType");
@@ -97,20 +87,28 @@ public CcStoreS3(){
9787
}
9888
storeType = StoreType.S3;
9989
manifestId = System.getenv(EnvironmentVariables.CC_MANIFEST_ID);
100-
localRootPath = Constants.LocalRootPath;
90+
payloadId = System.getenv(EnvironmentVariables.CC_PAYLOAD_ID);
91+
localRootPath = Constants.LOCAL_ROOT_PATH;
10192
bucket = config.aws_bucket;// + Constants.RemoteRootPath;
10293
root = System.getenv(EnvironmentVariables.CC_ROOT);
10394
}
95+
10496
@Override
105-
public boolean HandlesDataStoreType(StoreType storeType){
97+
public String rootPath() {
98+
return bucket;
99+
}
100+
101+
@Override
102+
public boolean handlesDataStoreType(StoreType storeType){
106103
return this.storeType == storeType;
107104
}
105+
108106
@Override
109-
public boolean PutObject(PutObjectInput input) {
107+
public boolean putObject(PutObjectInput input) {
110108
String path = root + "/" + manifestId + "/" + input.getFileName() + "." + input.getFileExtension();
111109
byte[] data;
112110
switch(input.getObjectState()){
113-
case LocalDisk:
111+
case LOCAL_DISK:
114112
//read from local
115113
File file = new File(path);
116114
data = new byte[(int) file.length()];
@@ -120,26 +118,27 @@ public boolean PutObject(PutObjectInput input) {
120118
catch(Exception e){
121119
//@TODOprint?
122120
}
123-
UploadToS3(config.aws_bucket, path, data);
121+
uploadToS3(config.aws_bucket, path, data);
124122
break;
125-
case Memory:
123+
case MEMORY:
126124
data = input.getData();
127-
UploadToS3(config.aws_bucket, path, data);
125+
uploadToS3(config.aws_bucket, path, data);
128126
break;
129127
default:
130128
return false;
131129
}
132130

133131
return true;
134132
}
133+
135134
@Override
136-
public boolean PullObject(PullObjectInput input) {
135+
public boolean pullObject(PullObjectInput input) {
137136
String path = root + "/" + manifestId + "/" + input.getFileName() + "." + input.getFileExtension();
138137
byte[] data;
139138
String localPath = input.getDestRootPath() + "/" + input.getFileName() + "." + input.getFileExtension();
140139
try {
141140
//get the object from s3
142-
data = DownloadBytesFromS3(path);
141+
data = downloadBytesFromS3(path);
143142
//create localpath writer
144143
InputStream stream = new ByteArrayInputStream(data);
145144
//write it.
@@ -149,38 +148,45 @@ public boolean PullObject(PullObjectInput input) {
149148
}
150149
return false;
151150
}
151+
152152
private void writeInputStreamToDisk(InputStream input, String outputDestination) throws IOException {
153153
String directory = new File(outputDestination).getParent();
154154
File f = new File(directory);
155155
if(!f.exists()){
156156
f.mkdirs();
157157
}
158158
byte[] bytes = input.readAllBytes();
159-
OutputStream os = new FileOutputStream(new File(outputDestination));
160-
os.write(bytes);
159+
try (OutputStream os = new FileOutputStream(new File(outputDestination))) {
160+
os.write(bytes);
161+
} catch (Exception e) {
162+
e.printStackTrace();
163+
};
161164
}
165+
162166
@Override
163-
public byte[] GetObject(GetObjectInput input) throws AmazonS3Exception {
164-
String path = root + "/" + manifestId + "/" + input.getFileName() + "." + input.getFileExtension();
167+
public byte[] getObject(GetObjectInput input) throws AmazonS3Exception {
168+
String path = root + "/" + payloadId + "/" + input.getFileName() + "." + input.getFileExtension();
165169
byte[] data;
166170
try {
167-
data = DownloadBytesFromS3(path);
171+
data = downloadBytesFromS3(path);
168172
} catch (Exception e) {
169173
throw new AmazonS3Exception(e.toString());
170174
}
171175
return data;
172176
}
177+
173178
@Override
174-
public Payload GetPayload() throws AmazonS3Exception {
175-
String filepath = root + "/" + manifestId + "/" + Constants.PayloadFileName;
179+
public Payload getPayload() throws AmazonS3Exception {
180+
String filepath = root + "/" + payloadId + "/" + Constants.PAYLOAD_FILE_NAME;
176181
try{
177-
byte[] body = DownloadBytesFromS3(filepath);
178-
return ReadJsonModelPayloadFromBytes(body);
182+
byte[] body = downloadBytesFromS3(filepath);
183+
return readJsonModelPayloadFromBytes(body);
179184
} catch (Exception e){
180185
throw new AmazonS3Exception(e.toString());
181186
}
182187
}
183-
private byte[] DownloadBytesFromS3(String key) throws Exception{
188+
189+
private byte[] downloadBytesFromS3(String key) throws Exception{
184190
S3Object fullObject = null;
185191
boolean spaces = key.contains(" ");
186192
if(spaces){
@@ -205,15 +211,17 @@ private byte[] DownloadBytesFromS3(String key) throws Exception{
205211
}
206212
}
207213
}
208-
private Payload ReadJsonModelPayloadFromBytes(byte[] bytes) throws Exception {
214+
215+
private Payload readJsonModelPayloadFromBytes(byte[] bytes) throws Exception {
209216
final ObjectMapper mapper = new ObjectMapper(); // jackson databind
210217
try {
211218
return mapper.readValue(bytes, Payload.class);
212219
} catch (Exception e) {
213220
throw e;
214221
}
215222
}
216-
private void UploadToS3(String bucketName, String objectKey, byte[] fileBytes) {
223+
224+
private void uploadToS3(String bucketName, String objectKey, byte[] fileBytes) {
217225
try {
218226
//File file = new File(objectPath);
219227
InputStream stream = new ByteArrayInputStream(fileBytes);

‎src/main/java/usace/cc/plugin/Config.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
package usace.cc.plugin;
2+
23
import com.fasterxml.jackson.annotation.JsonProperty;
4+
35
public class Config {
6+
47
@JsonProperty
58
public AWSConfig[] aws_configs;
9+
610
}

0 commit comments

Comments
 (0)