Skip to content

Commit 4275426

Browse files
vaijoshvaijosh
committed
feat(store): implement pluggable cloud storage for HStore
#3081 -added multipart upload for the S3 storage provider
1 parent 2bb8e9f commit 4275426

20 files changed

Lines changed: 2734 additions & 347 deletions

File tree

‎hugegraph-store/docs/pluggable-cloud-storage-architecture.md‎

Lines changed: 148 additions & 32 deletions
Large diffs are not rendered by default.

‎hugegraph-store/hg-store-cloud-s3/pom.xml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,5 +78,39 @@
7878
<artifactId>lombok</artifactId>
7979
<scope>provided</scope>
8080
</dependency>
81+
82+
<!-- Test dependencies -->
83+
<dependency>
84+
<groupId>junit</groupId>
85+
<artifactId>junit</artifactId>
86+
<version>4.13.2</version>
87+
<scope>test</scope>
88+
</dependency>
89+
<dependency>
90+
<groupId>org.slf4j</groupId>
91+
<artifactId>slf4j-simple</artifactId>
92+
<version>1.7.36</version>
93+
<scope>test</scope>
94+
</dependency>
8195
</dependencies>
96+
97+
<build>
98+
<plugins>
99+
<!-- Allow long-running benchmarks with sufficient heap -->
100+
<plugin>
101+
<groupId>org.apache.maven.plugins</groupId>
102+
<artifactId>maven-surefire-plugin</artifactId>
103+
<configuration>
104+
<!-- Default: skip; benchmarks must be run explicitly with -Dtest=S3PerformanceBenchmark -->
105+
<excludes>
106+
<exclude>**/*Benchmark.java</exclude>
107+
</excludes>
108+
<!-- When running explicitly, allow plenty of memory for generating 20 GB of files -->
109+
<argLine>-Xmx512m</argLine>
110+
<!-- Generous timeout for large data volumes (6 hours) -->
111+
<forkedProcessTimeoutInSeconds>21600</forkedProcessTimeoutInSeconds>
112+
</configuration>
113+
</plugin>
114+
</plugins>
115+
</build>
82116
</project>

‎hugegraph-store/hg-store-cloud-s3/src/main/java/org/apache/hugegraph/store/cloud/s3/S3CloudStorageConfig.java‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.hugegraph.store.cloud.s3;
19+
20+
import lombok.Data;
21+
22+
/**
23+
* Amazon S3 / S3-compatible provider configuration, bound from
24+
* {@code cloud.storage.s3.*} in application.yml.
25+
*
26+
* <p>This is a plain Java POJO (no Spring annotations) so it can be used
27+
* both as a Spring {@code @ConfigurationProperties} target (via standard setters)
28+
* and directly in tests without a Spring context.
29+
*
30+
* <pre>
31+
* cloud:
32+
* storage:
33+
* s3:
34+
* bucket: hugegraph-store
35+
* region: us-east-1
36+
* endpoint: # optional custom endpoint (MinIO, Ceph…)
37+
* access-key: AKIAIOSFODNN7EXAMPLE # omit to use AWS default credentials chain
38+
* secret-key: wJalrXUtnFEMI/… # omit to use AWS default credentials chain
39+
* multipart-part-retry-max-attempts: 3
40+
* multipart-part-retry-base-backoff-ms: 1000
41+
* multipart-exhausted-direct-dlq: false
42+
* </pre>
43+
*/
44+
@Data
45+
public class S3CloudStorageConfig {
46+
47+
public static final int DEFAULT_MULTIPART_PART_RETRY_MAX_ATTEMPTS = 3;
48+
public static final long DEFAULT_MULTIPART_PART_RETRY_BASE_BACKOFF_MS = 1000L;
49+
50+
public static final String KEY_BUCKET = "bucket";
51+
public static final String KEY_REGION = "region";
52+
public static final String KEY_ENDPOINT = "endpoint";
53+
public static final String KEY_ACCESS_KEY = "access-key";
54+
public static final String KEY_SECRET_KEY = "secret-key";
55+
public static final String KEY_MULTIPART_RETRY_MAX_ATTEMPTS =
56+
"multipart-part-retry-max-attempts";
57+
public static final String KEY_MULTIPART_RETRY_BASE_BACKOFF_MS =
58+
"multipart-part-retry-base-backoff-ms";
59+
public static final String KEY_MULTIPART_EXHAUSTED_DIRECT_DLQ =
60+
"multipart-exhausted-direct-dlq";
61+
62+
/**
63+
* S3 bucket name.
64+
*/
65+
private String bucket;
66+
67+
/**
68+
* AWS region (e.g. "us-east-1").
69+
*/
70+
private String region;
71+
72+
/**
73+
* Optional custom endpoint URL for S3-compatible stores (e.g. MinIO, Ceph).
74+
* Leave empty to use the default AWS endpoint.
75+
*/
76+
private String endpoint;
77+
78+
/**
79+
* AWS access key ID. Omit to use the default AWS credentials chain
80+
* (env vars, instance profile, ~/.aws/credentials, etc.).
81+
*/
82+
private String accessKey;
83+
84+
/**
85+
* AWS secret access key. Omit to use the default AWS credentials chain.
86+
*/
87+
private String secretKey;
88+
89+
/**
90+
* Maximum retry attempts for a single multipart chunk upload (part-level retry).
91+
* Default: {@value DEFAULT_MULTIPART_PART_RETRY_MAX_ATTEMPTS}.
92+
*/
93+
private int multipartPartRetryMaxAttempts =
94+
DEFAULT_MULTIPART_PART_RETRY_MAX_ATTEMPTS;
95+
96+
/**
97+
* Base backoff in milliseconds for multipart chunk retry (1x/2x/4x…).
98+
* Default: {@value DEFAULT_MULTIPART_PART_RETRY_BASE_BACKOFF_MS} ms.
99+
*/
100+
private long multipartPartRetryBaseBackoffMs =
101+
DEFAULT_MULTIPART_PART_RETRY_BASE_BACKOFF_MS;
102+
103+
/**
104+
* If true, multipart chunk retry exhaustion is treated as non-retryable so outer
105+
* SST retry can move directly to DLQ without further whole-file attempts.
106+
*/
107+
private boolean multipartExhaustedDirectDlq = false;
108+
109+
}
110+

0 commit comments

Comments
 (0)