Skip to content

Commit 9635e0c

Browse files
zhchbinzhchbin
committed
feat: support s3
1 parent 6048237 commit 9635e0c

7 files changed

Lines changed: 155 additions & 6 deletions

File tree

‎.env.example‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,33 @@
6868
# RELAY_TIMEOUT=0
6969
# 流模式无响应超时时间，单位秒，如果出现空补全可以尝试改为更大值
7070
# STREAMING_TIMEOUT=300
71+
# 中继归档
72+
# RELAY_ARCHIVE_ENABLED=false
73+
# local / aliyun_oss / s3
74+
# RELAY_ARCHIVE_STORAGE_TYPE=local
75+
# 本地归档目录（storage_type=local 时生效）
76+
# RELAY_ARCHIVE_DIR=relay-archive
77+
# 归档路径前缀，会追加到日期目录前，例如 prod/audit/20260413/xxxx.jsonl.gz
78+
# RELAY_ARCHIVE_PREFIX=prod/audit
79+
# 归档定时刷盘间隔（秒）
80+
# RELAY_ARCHIVE_FLUSH_INTERVAL=600
81+
# 归档内存队列阈值（字节）
82+
# RELAY_ARCHIVE_MEMORY_SIZE=67108864
83+
# 阿里云 OSS 配置（storage_type=aliyun_oss 时生效）
84+
# RELAY_ARCHIVE_OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com
85+
# RELAY_ARCHIVE_OSS_BUCKET=your-bucket
86+
# RELAY_ARCHIVE_OSS_ACCESS_KEY_ID=your-access-key-id
87+
# RELAY_ARCHIVE_OSS_ACCESS_KEY_SECRET=your-access-key-secret
88+
# S3 配置（storage_type=s3 时生效；兼容 AWS S3 / MinIO / R2 等 S3 API）
89+
# AWS S3 建议填写；MinIO / 部分 S3 兼容服务可留空，程序会默认使用 us-east-1
90+
# RELAY_ARCHIVE_S3_REGION=ap-southeast-1
91+
# RELAY_ARCHIVE_S3_BUCKET=your-bucket
92+
# RELAY_ARCHIVE_S3_ACCESS_KEY_ID=your-access-key-id
93+
# RELAY_ARCHIVE_S3_SECRET_ACCESS_KEY=your-secret-access-key
94+
# 自定义 endpoint，可用于 MinIO / R2 / 其他兼容服务；AWS 官方 S3 可留空
95+
# RELAY_ARCHIVE_S3_ENDPOINT=https://s3.ap-southeast-1.amazonaws.com
96+
# 某些兼容服务需要 path-style 访问，例如 MinIO
97+
# RELAY_ARCHIVE_S3_USE_PATH_STYLE=false
7198

7299
# TLS / HTTP 跳过验证设置
73100
# TLS_INSECURE_SKIP_VERIFY=false

‎common/constants.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,10 +83,17 @@ var RelayArchiveStorageType = "local"
8383
var RelayArchiveFlushInterval = 600
8484
var RelayArchiveMemorySize = 64 * 1024 * 1024
8585
var RelayArchiveDir = "relay-archive"
86+
var RelayArchivePrefix = ""
8687
var RelayArchiveOSSEndpoint = ""
8788
var RelayArchiveOSSBucket = ""
8889
var RelayArchiveOSSAccessKeyID = ""
8990
var RelayArchiveOSSAccessKeySecret = ""
91+
var RelayArchiveS3Endpoint = ""
92+
var RelayArchiveS3Region = ""
93+
var RelayArchiveS3Bucket = ""
94+
var RelayArchiveS3AccessKeyID = ""
95+
var RelayArchiveS3SecretAccessKey = ""
96+
var RelayArchiveS3UsePathStyle = false
9097

9198
var TLSInsecureSkipVerify bool
9299
var InsecureTLSConfig = &tls.Config{InsecureSkipVerify: true}

‎common/init.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,10 +108,17 @@ func InitEnv() {
108108
RelayArchiveFlushInterval = GetEnvOrDefault("RELAY_ARCHIVE_FLUSH_INTERVAL", 600)
109109
RelayArchiveMemorySize = GetEnvOrDefault("RELAY_ARCHIVE_MEMORY_SIZE", 64*1024*1024)
110110
RelayArchiveDir = GetEnvOrDefaultString("RELAY_ARCHIVE_DIR", "relay-archive")
111+
RelayArchivePrefix = GetEnvOrDefaultString("RELAY_ARCHIVE_PREFIX", "")
111112
RelayArchiveOSSEndpoint = GetEnvOrDefaultString("RELAY_ARCHIVE_OSS_ENDPOINT", "")
112113
RelayArchiveOSSBucket = GetEnvOrDefaultString("RELAY_ARCHIVE_OSS_BUCKET", "")
113114
RelayArchiveOSSAccessKeyID = GetEnvOrDefaultString("RELAY_ARCHIVE_OSS_ACCESS_KEY_ID", "")
114115
RelayArchiveOSSAccessKeySecret = GetEnvOrDefaultString("RELAY_ARCHIVE_OSS_ACCESS_KEY_SECRET", "")
116+
RelayArchiveS3Endpoint = GetEnvOrDefaultString("RELAY_ARCHIVE_S3_ENDPOINT", "")
117+
RelayArchiveS3Region = GetEnvOrDefaultString("RELAY_ARCHIVE_S3_REGION", "")
118+
RelayArchiveS3Bucket = GetEnvOrDefaultString("RELAY_ARCHIVE_S3_BUCKET", "")
119+
RelayArchiveS3AccessKeyID = GetEnvOrDefaultString("RELAY_ARCHIVE_S3_ACCESS_KEY_ID", "")
120+
RelayArchiveS3SecretAccessKey = GetEnvOrDefaultString("RELAY_ARCHIVE_S3_SECRET_ACCESS_KEY", "")
121+
RelayArchiveS3UsePathStyle = GetEnvOrDefaultBool("RELAY_ARCHIVE_S3_USE_PATH_STYLE", false)
115122
TrustedProxies = parseEnvStringList(GetEnvOrDefaultString("TRUSTED_PROXIES", "172.16.0.0/12"))
116123

117124
// Initialize string variables with GetEnvOrDefaultString

‎go.mod‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,11 @@ require (
88
github.com/abema/go-mp4 v1.4.1
99
github.com/andybalholm/brotli v1.1.1
1010
github.com/anknown/ahocorasick v0.0.0-20190904063843-d75dbd5169c0
11-
github.com/aws/aws-sdk-go-v2 v1.41.2
12-
github.com/aws/aws-sdk-go-v2/credentials v1.19.10
11+
github.com/aws/aws-sdk-go-v2 v1.41.5
12+
github.com/aws/aws-sdk-go-v2/config v1.32.14
13+
github.com/aws/aws-sdk-go-v2/credentials v1.19.14
1314
github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.50.0
15+
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0
1416
github.com/aws/smithy-go v1.24.2
1517
github.com/bytedance/gopkg v0.1.3
1618
github.com/gin-contrib/cors v1.7.2
@@ -64,9 +66,20 @@ require (
6466
github.com/alicebob/miniredis/v2 v2.37.0 // indirect
6567
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect
6668
github.com/anknown/darts v0.0.0-20151216065714-83ff685239e6 // indirect
67-
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect
68-
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect
69-
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect
69+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
70+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
71+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
72+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
73+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
74+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
75+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
76+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
77+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
78+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
79+
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
80+
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
81+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
82+
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect
7083
github.com/beorn7/perks v1.0.1 // indirect
7184
github.com/boombuler/barcode v1.1.0 // indirect
7285
github.com/bytedance/sonic v1.14.1 // indirect

‎go.sum‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,26 +18,62 @@ github.com/aws/aws-sdk-go-v2 v1.37.2 h1:xkW1iMYawzcmYFYEV0UCMxc8gSsjCGEhBXQkdQyw
1818
github.com/aws/aws-sdk-go-v2 v1.37.2/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg=
1919
github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls=
2020
github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4=
21+
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
22+
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
2123
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg=
2224
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0/go.mod h1:/mXlTIVG9jbxkqDnr5UQNQxW1HRYxeGklkM9vAFeabg=
2325
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q=
2426
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c=
27+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
28+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
29+
github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI=
30+
github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo=
2531
github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
2632
github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
2733
github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iKMdkkYFTRpZXNw7m8=
2834
github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE=
35+
github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
36+
github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
37+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
38+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
2939
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.2 h1:sPiRHLVUIIQcoVZTNwqQcdtjkqkPopyYmIX0M5ElRf4=
3040
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.2/go.mod h1:ik86P3sgV+Bk7c1tBFCwI3VxMoSEwl4YkRB9xn1s340=
3141
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM=
3242
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug=
43+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
44+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
3345
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.2 h1:ZdzDAg075H6stMZtbD2o+PyB933M/f20e9WmCBC17wA=
3446
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.2/go.mod h1:eE1IIzXG9sdZCB0pNNpMpsYTLl4YdOQD3njiVN1e/E4=
3547
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ=
3648
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k=
49+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
50+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
51+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
52+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
53+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
54+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
3755
github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.33.0 h1:JzidOz4Hcn2RbP5fvIS1iAP+DcRv5VJtgixbEYDsI5g=
3856
github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.33.0/go.mod h1:9A4/PJYlWjvjEzzoOLGQjkLt4bYK9fRWi7uz1GSsAcA=
3957
github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.50.0 h1:TDKR8ACRw7G+GFaQlhoy6biu+8q6ZtSddQCy9avMdMI=
4058
github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.50.0/go.mod h1:XlhOh5Ax/lesqN4aZCUgj9vVJed5VoXYHHFYGAlJEwU=
59+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
60+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
61+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
62+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
63+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
64+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
65+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
66+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
67+
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk=
68+
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
69+
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
70+
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
71+
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
72+
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
73+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
74+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
75+
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
76+
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
4177
github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw=
4278
github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
4379
github.com/aws/smithy-go v1.24.1 h1:VbyeNfmYkWoxMVpGUAbQumkODcYmfMRfZ8yQiH30SK0=

‎model/option.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ func InitOptionMap() {
135135
common.OptionMap["RelayArchiveFlushInterval"] = strconv.Itoa(common.RelayArchiveFlushInterval)
136136
common.OptionMap["RelayArchiveMemorySize"] = strconv.Itoa(common.RelayArchiveMemorySize)
137137
common.OptionMap["RelayArchiveDir"] = common.RelayArchiveDir
138+
common.OptionMap["RelayArchivePrefix"] = common.RelayArchivePrefix
138139
common.OptionMap["DataExportInterval"] = strconv.Itoa(common.DataExportInterval)
139140
common.OptionMap["DataExportDefaultTime"] = common.DataExportDefaultTime
140141
common.OptionMap["DefaultCollapseSidebar"] = strconv.FormatBool(common.DefaultCollapseSidebar)
@@ -351,6 +352,8 @@ func updateOptionMap(key string, value string) (err error) {
351352
common.RelayArchiveMemorySize, _ = strconv.Atoi(value)
352353
case "RelayArchiveDir":
353354
common.RelayArchiveDir = value
355+
case "RelayArchivePrefix":
356+
common.RelayArchivePrefix = value
354357
case "USDExchangeRate":
355358
operation_setting.USDExchangeRate, _ = strconv.ParseFloat(value, 64)
356359
case "MinTopUp":

‎model/relay_archive.go‎

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"bufio"
55
"bytes"
66
"compress/gzip"
7+
"context"
78
cryptorand "crypto/rand"
89
"fmt"
910
"io"
@@ -16,6 +17,10 @@ import (
1617

1718
"github.com/QuantumNous/new-api/common"
1819
"github.com/aliyun/aliyun-oss-go-sdk/oss"
20+
"github.com/aws/aws-sdk-go-v2/aws"
21+
awsconfig "github.com/aws/aws-sdk-go-v2/config"
22+
"github.com/aws/aws-sdk-go-v2/credentials"
23+
"github.com/aws/aws-sdk-go-v2/service/s3"
1924
)
2025

2126
type RelayArchiveRecord struct {
@@ -172,7 +177,14 @@ func randomRelayArchiveSuffix(n int) string {
172177

173178
func buildRelayArchiveObjectName() string {
174179
now := time.Now()
175-
return now.Format("20060102") + "/" + now.Format("150405") + "-" + randomRelayArchiveSuffix(6) + ".jsonl.gz"
180+
objectName := now.Format("20060102") + "/" + now.Format("150405") + "-" + randomRelayArchiveSuffix(6) + ".jsonl.gz"
181+
prefix := strings.TrimSpace(common.RelayArchivePrefix)
182+
prefix = strings.ReplaceAll(prefix, "\\", "/")
183+
prefix = strings.Trim(prefix, "/")
184+
if prefix == "" {
185+
return objectName
186+
}
187+
return prefix + "/" + objectName
176188
}
177189

178190
func buildRelayArchivePayload(records [][]byte) ([]byte, error) {
@@ -204,6 +216,8 @@ func writeRelayArchivePayload(payload []byte) (string, error) {
204216
return writeRelayArchiveToLocal(objectName, payload)
205217
case "aliyun_oss", "aliyun-oss", "oss":
206218
return writeRelayArchiveToOSS(objectName, payload)
219+
case "s3":
220+
return writeRelayArchiveToS3(objectName, payload)
207221
default:
208222
return "", fmt.Errorf("unsupported relay archive storage type: %s", common.RelayArchiveStorageType)
209223
}
@@ -244,6 +258,48 @@ func writeRelayArchiveToOSS(objectName string, payload []byte) (string, error) {
244258
return "oss://" + common.RelayArchiveOSSBucket + "/" + objectName, nil
245259
}
246260

261+
func writeRelayArchiveToS3(objectName string, payload []byte) (string, error) {
262+
if common.RelayArchiveS3Bucket == "" || common.RelayArchiveS3AccessKeyID == "" || common.RelayArchiveS3SecretAccessKey == "" {
263+
return "", fmt.Errorf("s3 relay archive config is incomplete")
264+
}
265+
region := strings.TrimSpace(common.RelayArchiveS3Region)
266+
if region == "" {
267+
region = "us-east-1"
268+
}
269+
270+
options := make([]func(*awsconfig.LoadOptions) error, 0, 3)
271+
options = append(options,
272+
awsconfig.WithRegion(region),
273+
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
274+
common.RelayArchiveS3AccessKeyID,
275+
common.RelayArchiveS3SecretAccessKey,
276+
"",
277+
)),
278+
)
279+
if common.RelayArchiveS3Endpoint != "" {
280+
options = append(options, awsconfig.WithBaseEndpoint(common.RelayArchiveS3Endpoint))
281+
}
282+
283+
cfg, err := awsconfig.LoadDefaultConfig(context.Background(), options...)
284+
if err != nil {
285+
return "", err
286+
}
287+
288+
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
289+
o.UsePathStyle = common.RelayArchiveS3UsePathStyle
290+
})
291+
_, err = client.PutObject(context.Background(), &s3.PutObjectInput{
292+
Bucket: aws.String(common.RelayArchiveS3Bucket),
293+
Key: aws.String(objectName),
294+
Body: bytes.NewReader(payload),
295+
ContentType: aws.String("application/gzip"),
296+
})
297+
if err != nil {
298+
return "", err
299+
}
300+
return "s3://" + common.RelayArchiveS3Bucket + "/" + objectName, nil
301+
}
302+
247303
func RelayArchiveBody(body []byte) (string, int) {
248304
if len(body) == 0 {
249305
return "", 0

0 commit comments

Comments
 (0)