Skip to content

Commit 8987d47

Browse files
danishjamal104danishjamal104
authored
PHEE-415 Implemented the batch REST Api and Optimised imports and updated the batch API (#105)
* Implemented the batch REST Api and Optimised imports and updated the batch API * Header parsing bug fixed in camel * Reverted the codestyle changes * Reverted the codestyle changes * Reverted the codestyle changes * Removed code Formatting * Bug fixed in converting the batchRequest to txn object * Added header validation * Wired raw request with csv request
1 parent 975624b commit 8987d47

22 files changed

Lines changed: 443 additions & 149 deletions

‎.circleci/config.yml‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ jobs:
1212
executor: docker-executor
1313
environment:
1414
JVM_OPTS: -Xmx512m
15-
TERM: dumb
15+
TERM: dumb
1616
GITHUB_TOKEN: ${GITHUB_TOKEN} # Add the GitHub token as an environment variable
1717

1818
steps:
@@ -40,7 +40,7 @@ jobs:
4040
docker push "openmf/ph-ee-bulk-processor:$IMAGE_TAG"
4141
4242
# when: always # The job will be executed even if there's no match for the tag filter
43-
43+
4444
build_and_push_latest_image:
4545
executor: docker-executor
4646
environment:
@@ -68,7 +68,7 @@ jobs:
6868
# Push the Docker image to DockerHub
6969
- run:
7070
name: Push Docker image to DockerHub
71-
command: docker push openmf/ph-ee-bulk-processor:latest
71+
command: docker push openmf/ph-ee-bulk-processor:latest
7272

7373
workflows:
7474
version: 2
@@ -79,8 +79,7 @@ workflows:
7979
tags:
8080
only: /^v\d+\.\d+\.\d+$/ # Match tags in the format v1.2.3
8181
context:
82-
- DOCKER
82+
- DOCKER
8383
- build_and_push_latest_image:
8484
context:
85-
- DOCKER
86-
85+
- DOCKER

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ server:
1111
key-store-password: "<replace-with-password>"
1212
port: 8443
1313
```
14-
#### NOTE: For disabling TLS, change the port to "8080" and add null values for all the "ssl" related fields.
14+
#### NOTE: For disabling TLS, change the port to "8080" and add null values for all the "ssl" related fields.
1515
1616
## Checkstyle
1717
Use below command to execute the checkstyle test.

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ dependencies {
108108
// miscellaneous dependency
109109
implementation 'com.google.code.gson:gson:2.8.9'
110110
implementation 'org.json:json:20210307'
111-
implementation 'org.mifos:ph-ee-connector-common:1.6.0-rc.1'
111+
implementation 'org.mifos:ph-ee-connector-common:1.6.1-rc.1'
112112
implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0'
113113
implementation 'org.apache.camel:camel-undertow:3.4.0'
114114
implementation 'org.springframework.boot:spring-boot-starter:2.5.2'

‎src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,28 @@
88
import org.springframework.web.multipart.MultipartFile;
99

1010
import javax.mail.Multipart;
11+
import javax.servlet.http.HttpServletRequest;
1112
import javax.servlet.http.HttpServletResponse;
1213

1314
import java.io.IOException;
1415

1516
import static org.mifos.processor.bulk.camel.config.CamelProperties.*;
17+
import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_CLIENT_CORRELATION_ID;
18+
import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID;
1619
import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*;
1720

1821
public interface BatchTransactions {
1922

20-
@PostMapping(value = "/batchtransactions", produces="application/json")
21-
String batchTransactions(HttpServletResponse httpServletResponse, @RequestHeader(value = "X-CorrelationID") String requestId,
22-
@RequestParam("data") MultipartFile file,
23-
@RequestHeader(value = FILE_NAME) String fileName,
24-
@RequestHeader(value = PURPOSE) String purpose,
25-
@RequestHeader(value = "Type") String type,
26-
@RequestHeader(value = HEADER_PLATFORM_TENANT_ID) String tenant,
27-
@RequestHeader(value = HEADER_REGISTERING_INSTITUTE_ID, required = false) String registeringInstitutionId,
28-
@RequestHeader(value = HEADER_PROGRAM_ID, required = false) String programId) throws IOException;
23+
@PostMapping(value = "/batchtransactions", produces = "application/json")
24+
String batchTransactions(
25+
HttpServletRequest httpServletRequest,
26+
HttpServletResponse httpServletResponse,
27+
@RequestHeader(value = HEADER_CLIENT_CORRELATION_ID) String requestId,
28+
@RequestHeader(value = FILE_NAME, required = false) String fileName,
29+
@RequestHeader(value = PURPOSE) String purpose,
30+
@RequestHeader(value = HEADER_TYPE) String type,
31+
@RequestHeader(value = HEADER_PLATFORM_TENANT_ID) String tenant,
32+
@RequestHeader(value = HEADER_REGISTERING_INSTITUTE_ID, required = false) String registeringInstitutionId,
33+
@RequestHeader(value = HEADER_PROGRAM_ID, required = false) String programId) throws IOException;
34+
2935
}

‎src/main/java/org/mifos/processor/bulk/API/definition/BulkTransfer.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,11 @@
1111
import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE;
1212

1313
public interface BulkTransfer {
14+
15+
@Deprecated
1416
@PostMapping(value = "/bulk/transfer/{requestId}/{fileName}", produces = "application/json")
1517
String bulkTransfer(@RequestHeader(value = "X-CorrelationID", required = false) String requestId,
16-
@RequestParam("data") MultipartFile file,
17-
@RequestHeader(value = FILE_NAME, required = false) String fileName,
18-
@RequestHeader(value = PURPOSE, required = false) String purpose,
19-
@RequestHeader(value = "Type", required = false) String type,
20-
@RequestHeader(value = "Platform-TenantId") String tenant) throws IOException;
18+
@RequestParam("data") MultipartFile file, @RequestHeader(value = FILE_NAME, required = false) String fileName,
19+
@RequestHeader(value = PURPOSE, required = false) String purpose, @RequestHeader(value = "Type", required = false) String type,
20+
@RequestHeader(value = "Platform-TenantId") String tenant) throws IOException;
2121
}

‎src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java‎

Lines changed: 123 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,46 @@
11
package org.mifos.processor.bulk.api.implementation;
22

3+
import com.fasterxml.jackson.core.type.TypeReference;
34
import com.fasterxml.jackson.databind.ObjectMapper;
5+
import com.fasterxml.jackson.dataformat.csv.CsvMapper;
46
import lombok.SneakyThrows;
57
import lombok.extern.slf4j.Slf4j;
68
import org.apache.camel.Exchange;
79
import org.apache.camel.ProducerTemplate;
10+
import org.apache.commons.io.IOUtils;
811
import org.json.JSONObject;
12+
import org.mifos.connector.common.interceptor.JWSUtil;
913
import org.mifos.processor.bulk.api.definition.BatchTransactions;
1014
import org.mifos.processor.bulk.file.FileStorageService;
15+
import org.mifos.processor.bulk.format.RestRequestConvertor;
16+
import org.mifos.processor.bulk.schema.BatchRequestDTO;
17+
import org.mifos.processor.bulk.schema.CamelApiResponse;
18+
import org.mifos.processor.bulk.schema.Transaction;
19+
import org.mifos.processor.bulk.utility.CsvWriter;
1120
import org.mifos.processor.bulk.utility.Headers;
1221
import org.mifos.processor.bulk.utility.SpringWrapperUtil;
1322
import org.springframework.beans.factory.annotation.Autowired;
23+
import org.springframework.beans.factory.annotation.Value;
1424
import org.springframework.web.bind.annotation.ExceptionHandler;
1525
import org.springframework.web.bind.annotation.RequestHeader;
1626
import org.springframework.web.bind.annotation.RestController;
1727
import org.springframework.web.multipart.MultipartException;
18-
import org.springframework.web.multipart.MultipartFile;
28+
import javax.servlet.http.HttpServletRequest;
1929
import javax.servlet.http.HttpServletResponse;
2030
import java.io.IOException;
21-
2231
import static org.mifos.processor.bulk.camel.config.CamelProperties.*;
2332
import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE;
33+
import java.nio.charset.Charset;
34+
import java.util.List;
35+
import java.util.Optional;
36+
import java.util.UUID;
37+
import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PROGRAM_ID;
38+
import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID;
2439
import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME;
40+
import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_CLIENT_CORRELATION_ID;
41+
import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_PLATFORM_TENANT_ID;
42+
import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_TYPE;
43+
import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE;
2544

2645
@Slf4j
2746
@RestController
@@ -36,41 +55,118 @@ public class BatchTransactionsController implements BatchTransactions {
3655
@Autowired
3756
FileStorageService fileStorageService;
3857

58+
@Autowired
59+
RestRequestConvertor restRequestConvertor;
60+
61+
@Value("#{'${tenants}'.split(',')}")
62+
protected List<String> tenants;
63+
@Autowired
64+
private CsvMapper csvMapper;
65+
3966
@SneakyThrows
4067
@Override
41-
public String batchTransactions(HttpServletResponse httpServletResponse,
42-
String requestId, MultipartFile file, String fileName,
43-
String purpose, String type, String tenant,
44-
String registeringInstitutionId, String programId) throws IOException {
45-
log.debug("Inside api logic");
46-
String localFileName = fileStorageService.save(file);
47-
Headers headers = new Headers.HeaderBuilder()
68+
public String batchTransactions(
69+
HttpServletRequest httpServletRequest,
70+
HttpServletResponse httpServletResponse,
71+
String requestId,
72+
String fileName,
73+
String purpose,
74+
String type,
75+
String tenant,
76+
String registeringInstitutionId,
77+
String programId) {
78+
79+
log.info("Inside api logic");
80+
Headers.HeaderBuilder headerBuilder = new Headers.HeaderBuilder()
4881
.addHeader(HEADER_CLIENT_CORRELATION_ID, requestId)
49-
.addHeader(PURPOSE,purpose)
50-
.addHeader(FILE_NAME,localFileName)
51-
.addHeader("Type",type)
52-
.addHeader(HEADER_PLATFORM_TENANT_ID,tenant)
82+
.addHeader(PURPOSE, purpose)
83+
.addHeader(HEADER_TYPE, type)
84+
.addHeader(HEADER_PLATFORM_TENANT_ID, tenant)
5385
.addHeader(HEADER_REGISTERING_INSTITUTE_ID, registeringInstitutionId)
54-
.addHeader(HEADER_PROGRAM_ID, programId)
55-
.build();
56-
log.debug("Headers passed: {}", headers);
57-
Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(),
58-
headers);
59-
log.debug("Header in exchange: {}", exchange.getIn().getHeaders());
86+
.addHeader(HEADER_PROGRAM_ID, programId);
87+
88+
Optional<String> validationResponse = isValidRequest(httpServletRequest, fileName, type);
89+
if (validationResponse.isPresent()) {
90+
httpServletResponse.setStatus(httpServletResponse.SC_BAD_REQUEST);
91+
return validationResponse.get();
92+
}
93+
94+
if (JWSUtil.isMultipartRequest(httpServletRequest)) {
95+
log.info("This is file based request");
96+
String localFileName = fileStorageService.save(JWSUtil.parseFormData(httpServletRequest), fileName);
97+
Headers headers = headerBuilder.addHeader(FILE_NAME, localFileName).build();
98+
log.info("Headers passed: {}", headers.getHeaders());
99+
100+
CamelApiResponse response = sendRequestToCamel(headers);
101+
httpServletResponse.setStatus(response.getStatus());
102+
return response.getBody();
103+
} else {
104+
log.info("This is json based request");
105+
String jsonString = IOUtils.toString(httpServletRequest.getInputStream(), Charset.defaultCharset());
106+
List<BatchRequestDTO> batchRequestDTOList = objectMapper.readValue(jsonString, new TypeReference<>() {});
107+
List<Transaction> transactionList = restRequestConvertor.convertListFrom(batchRequestDTOList);
108+
109+
String localFileName = UUID.randomUUID() + ".csv";
110+
CsvWriter.writeToCsv(transactionList, Transaction.class, csvMapper, true, localFileName);
111+
Headers headers = headerBuilder
112+
.addHeader(HEADER_TYPE, "csv")
113+
.addHeader(FILE_NAME, localFileName).build();
114+
115+
CamelApiResponse response = sendRequestToCamel(headers);
116+
httpServletResponse.setStatus(response.getStatus());
117+
return response.getBody();
118+
}
119+
}
120+
121+
@ExceptionHandler({ MultipartException.class })
122+
public String handleMultipartException(HttpServletResponse httpServletResponse) {
123+
httpServletResponse.setStatus(httpServletResponse.SC_BAD_REQUEST);
124+
return getErrorResponse("File not uploaded", "There was no fie uploaded with the request. " +
125+
"Please upload a file and try again.", 400);
126+
}
127+
128+
private CamelApiResponse sendRequestToCamel(Headers headers) {
129+
Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(), headers);
60130
exchange = producerTemplate.send("direct:post-batch-transactions", exchange);
61131
int statusCode = exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE, Integer.class);
62-
httpServletResponse.setStatus(statusCode);
63-
return exchange.getIn().getBody(String.class);
132+
String body = exchange.getIn().getBody(String.class);
133+
return new CamelApiResponse(body, statusCode);
64134
}
65135

66-
@ExceptionHandler({MultipartException.class})
67-
public String handleMultipartException(HttpServletResponse httpServletResponse) {
136+
private String getErrorResponse(String information, String description, int code) {
68137
JSONObject json = new JSONObject();
69-
json.put("Error Information: ", "File not uploaded");
70-
json.put("Error Description : ", "There was no fie uploaded with the request. " +
71-
"Please upload a file and try again.");
72-
httpServletResponse.setStatus(httpServletResponse.SC_BAD_REQUEST);
138+
json.put("errorInformation", "File not uploaded");
139+
json.put("errorDescription", "There was no fie uploaded with the request. " + "Please upload a file and try again.");
140+
json.put("errorCode", code);
73141
return json.toString();
74142
}
75143

144+
// validates the request header, and return errorJson string if the request is invalid else an empty optional
145+
private Optional<String> isValidRequest(HttpServletRequest httpServletRequest,
146+
String fileName,
147+
String type) {
148+
149+
Optional<String> response = Optional.empty();
150+
if ((JWSUtil.isMultipartRequest(httpServletRequest) && !type.equalsIgnoreCase("csv")) ||
151+
(!JWSUtil.isMultipartRequest(httpServletRequest) && !type.equalsIgnoreCase("raw"))) {
152+
String errorJson = getErrorResponse("Type mismatch",
153+
"The value of the header \"" + HEADER_TYPE +
154+
"\" doesn't match with the request content-type", 400);
155+
response = Optional.of(errorJson);
156+
157+
}
158+
if (JWSUtil.isMultipartRequest(httpServletRequest) && fileName.isEmpty()) {
159+
String errorJson = getErrorResponse("Header can't be empty",
160+
"If the request is of type csv, the header \"" +
161+
FILE_NAME + "\"can't be empty", 400);
162+
response = Optional.of(errorJson);
163+
}
164+
if (!type.equalsIgnoreCase("raw") && !type.equalsIgnoreCase("csv")) {
165+
String errorJson = getErrorResponse("Invalid TYPE header value passed",
166+
"The value of the header \"" + HEADER_TYPE +
167+
"\" can be \"[raw,csv]\" but is " + type, 400);
168+
response = Optional.of(errorJson);
169+
}
170+
return response;
171+
}
76172
}

‎src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,16 @@
1010
import org.springframework.beans.factory.annotation.Autowired;
1111
import org.springframework.web.bind.annotation.RestController;
1212
import org.springframework.web.multipart.MultipartFile;
13-
1413
import java.io.IOException;
15-
1614
import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID;
1715
import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME;
16+
import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_CLIENT_CORRELATION_ID;
17+
import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_TYPE;
1818
import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE;
1919

2020
@RestController
2121
public class BulkTransferController implements BulkTransfer {
22+
2223
@Autowired
2324
private ProducerTemplate producerTemplate;
2425

@@ -29,16 +30,11 @@ public class BulkTransferController implements BulkTransfer {
2930
FileStorageService fileStorageService;
3031

3132
@Override
32-
public String bulkTransfer(String requestId, MultipartFile file, String fileName, String purpose, String type, String tenant) throws IOException {
33-
Headers headers = new Headers.HeaderBuilder()
34-
.addHeader("X-CorrelationID", requestId)
35-
.addHeader(PURPOSE,purpose)
36-
.addHeader(FILE_NAME,fileName)
37-
.addHeader("Type",type)
38-
.addHeader(HEADER_PLATFORM_TENANT_ID,tenant)
39-
.build();
40-
Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(),
41-
headers);
33+
public String bulkTransfer(String requestId, MultipartFile file, String fileName, String purpose, String type, String tenant)
34+
throws IOException {
35+
Headers headers = new Headers.HeaderBuilder().addHeader(HEADER_CLIENT_CORRELATION_ID, requestId).addHeader(PURPOSE, purpose)
36+
.addHeader(FILE_NAME, fileName).addHeader(HEADER_TYPE, type).addHeader(HEADER_PLATFORM_TENANT_ID, tenant).build();
37+
Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(), headers);
4238
fileStorageService.save(file);
4339
producerTemplate.send("direct:post-bulk-transfer", exchange);
4440
return exchange.getIn().getBody(String.class);

‎src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55

66
import org.apache.camel.LoggingLevel;
77
import org.apache.camel.builder.RouteBuilder;
8-
import org.springframework.stereotype.Component;
98

109
//@Component
1110
public class ExternalApiCallRoute extends RouteBuilder {

‎src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import java.util.List;
2121
import org.mifos.processor.bulk.schema.Transaction;
2222
import org.mifos.processor.bulk.schema.TransactionResult;
23+
import org.mifos.processor.bulk.utility.CsvWriter;
2324
import org.springframework.beans.factory.annotation.Autowired;
2425
import org.springframework.stereotype.Component;
2526

@@ -77,7 +78,7 @@ public void configure() {
7778
// getting header
7879
Boolean overrideHeader = exchange.getProperty(OVERRIDE_HEADER, Boolean.class);
7980

80-
csvWriter(transactionList, TransactionResult.class, csvMapper, overrideHeader, filepath);
81+
CsvWriter.writeToCsv(transactionList, TransactionResult.class, csvMapper, overrideHeader, filepath);
8182
}).log("Update complete");
8283

8384
/**
@@ -113,20 +114,4 @@ public void configure() {
113114
}
114115
});
115116
}
116-
117-
private <T> void csvWriter(List<T> data, Class<T> tClass, CsvMapper csvMapper, boolean overrideHeader, String filepath)
118-
throws IOException {
119-
CsvSchema csvSchema = csvMapper.schemaFor(tClass);
120-
if (overrideHeader) {
121-
csvSchema = csvSchema.withHeader();
122-
} else {
123-
csvSchema = csvSchema.withoutHeader();
124-
}
125-
126-
File file = new File(filepath);
127-
SequenceWriter writer = csvMapper.writerWithSchemaFor(tClass).with(csvSchema).writeValues(file);
128-
for (T object : data) {
129-
writer.write(object);
130-
}
131-
}
132117
}

0 commit comments

Comments
 (0)