Skip to content

Commit fe24390

Browse files
committed
Fix s3 folder scan depth for RDS source when partition prefix is not set
1 parent 506a04f commit fe24390

1 file changed

Lines changed: 3 additions & 1 deletion

File tree

data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/transformer/DynamicConfigTransformer.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -433,7 +433,9 @@ public String calculateDepth(String s3Prefix) {
433433
* @return s3 folder scan depth
434434
*/
435435
public String calculateDepthForRdsSource(String s3Prefix) {
436-
return Integer.toString(getDepth(s3Prefix, 3));
436+
String envSourceCoordinationIdentifier = System.getenv(SOURCE_COORDINATION_IDENTIFIER_ENVIRONMENT_VARIABLE);
437+
int baseDepth = envSourceCoordinationIdentifier != null ? 3 : 2;
438+
return Integer.toString(getDepth(s3Prefix, baseDepth));
437439
}
438440

439441
private int getDepth(String s3Prefix, int baseDepth) {

0 commit comments

Comments
 (0)