From f390f755a078663b57c6c75a4bfd3c50b719e0a5 Mon Sep 17 00:00:00 2001 From: maybaby Date: Thu, 19 Aug 2021 17:32:11 +0800 Subject: [PATCH 1/5] Add a sqlite implement bucket ak manager and memory cache implement. To achieve that we can use different accesskey and secret key for per bucket. --- pom.xml | 5 + src/main/java/org/gaul/s3proxy/Main.java | 31 +++- .../java/org/gaul/s3proxy/S3ProxyHandler.java | 1 + .../s3proxy/extend/AbStractAKSKManager.java | 59 +++++++ .../s3proxy/extend/AccessSecretManager.java | 24 +++ .../org/gaul/s3proxy/extend/AkSkPair.java | 29 ++++ .../org/gaul/s3proxy/extend/CacheManager.java | 27 ++++ .../org/gaul/s3proxy/extend/MemoryCache.java | 33 ++++ .../s3proxy/extend/SqliteAKSKManager.java | 144 ++++++++++++++++++ 9 files changed, 348 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/gaul/s3proxy/extend/AbStractAKSKManager.java create mode 100644 src/main/java/org/gaul/s3proxy/extend/AccessSecretManager.java create mode 100644 src/main/java/org/gaul/s3proxy/extend/AkSkPair.java create mode 100644 src/main/java/org/gaul/s3proxy/extend/CacheManager.java create mode 100644 src/main/java/org/gaul/s3proxy/extend/MemoryCache.java create mode 100644 src/main/java/org/gaul/s3proxy/extend/SqliteAKSKManager.java diff --git a/pom.xml b/pom.xml index 18704d525..4c54d64b7 100644 --- a/pom.xml +++ b/pom.xml @@ -452,5 +452,10 @@ javax.annotation-api 1.3.2 + + org.xerial + sqlite-jdbc + 3.36.0.1 + diff --git a/src/main/java/org/gaul/s3proxy/Main.java b/src/main/java/org/gaul/s3proxy/Main.java index 8d86f3f0e..693246ef0 100644 --- a/src/main/java/org/gaul/s3proxy/Main.java +++ b/src/main/java/org/gaul/s3proxy/Main.java @@ -23,10 +23,7 @@ import java.io.InputStream; import java.io.PrintStream; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; @@ -40,6 +37,9 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Module; +import org.gaul.s3proxy.extend.AccessSecretManager; +import org.gaul.s3proxy.extend.AkSkPair; +import org.gaul.s3proxy.extend.SqliteAKSKManager; import org.jclouds.Constants; import org.jclouds.ContextBuilder; import org.jclouds.JcloudsVersion; @@ -102,6 +102,10 @@ public static void main(String[] args) throws Exception { 1, 20, 60 * 1000, factory); ImmutableMap.Builder> locators = ImmutableMap.builder(); + // 加载用户数据库 + AccessSecretManager accessSecretManager = new SqliteAKSKManager(); + Map accessCache= accessSecretManager.getBucketAkSkList(); + for (File propertiesFile : options.propertiesFiles) { Properties properties = new Properties(); try (InputStream is = new FileInputStream(propertiesFile)) { @@ -124,6 +128,13 @@ public static void main(String[] args) throws Exception { S3ProxyConstants.PROPERTY_CREDENTIAL); locators.put(localIdentity, Maps.immutableEntry( localCredential, blobStore)); + for (Map.Entry entry : accessCache.entrySet()) { + // 增加 sqlite中access key的 locators + locators.put(entry.getValue().getAccess_key(), Maps.immutableEntry( + entry.getValue().getSecret_key(), blobStore)); + } + + } S3Proxy.Builder s3ProxyBuilder2 = S3Proxy.Builder @@ -139,6 +150,7 @@ public static void main(String[] args) throws Exception { s3ProxyBuilder = s3ProxyBuilder2; } + S3Proxy s3Proxy; try { s3Proxy = s3ProxyBuilder.build(); @@ -164,7 +176,16 @@ public Map.Entry locateBlobStore( "cannot use anonymous access with multiple" + " backends"); } - return locator.get(identity); + // verify key + Map.Entry provider = locator.get(identity); + String bucket = accessSecretManager.getBucketFromAccessKey(identity); + if ( !bucket.equalsIgnoreCase("")) { + // if bucket and access_key not match. [1:1 in db] return null + if (!container.equalsIgnoreCase(bucket)){ + return null; + } + } + return provider; } }); } diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java index 1760c4c37..84729cd37 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java @@ -608,6 +608,7 @@ public final void doHandle(HttpServletRequest baseRequest, if (!constantTimeEquals(expectedSignature, authHeader.getSignature())) { + // 验证verify access key check throw new S3Exception(S3ErrorCode.SIGNATURE_DOES_NOT_MATCH); } } diff --git a/src/main/java/org/gaul/s3proxy/extend/AbStractAKSKManager.java b/src/main/java/org/gaul/s3proxy/extend/AbStractAKSKManager.java new file mode 100644 index 000000000..d58ee0818 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/extend/AbStractAKSKManager.java @@ -0,0 +1,59 @@ +package org.gaul.s3proxy.extend; + +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +import java.util.List; +import java.util.Map; + +/** + * @author yangyanbo + */ +public class AbStractAKSKManager implements AccessSecretManager { + + public String cacheType = "memory"; + + + public CacheManager cacheManager; + + public AbStractAKSKManager() { + cacheManager = createCacheManager(); + } + + @Override + public CacheManager createCacheManager() { + switch (cacheType) { + case "memory": + cacheManager = new MemoryCache(); + break; + default: + cacheManager = new MemoryCache(); + } + return cacheManager; + } + + @Override + public void createAKSKForBucket(String bucket, String access_key, String secret_key) { + throw new NotImplementedException(); + } + + @Override + public void loads2Cache() { + throw new NotImplementedException(); + + } + + @Override + public Map getBucketAkSkList() throws Exception { + throw new NotImplementedException(); + } + + @Override + public String getBucketFromAccessKey(String ak) { + throw new NotImplementedException(); + } + + public CacheManager getCacheManager() { + return cacheManager; + } + +} diff --git a/src/main/java/org/gaul/s3proxy/extend/AccessSecretManager.java b/src/main/java/org/gaul/s3proxy/extend/AccessSecretManager.java new file mode 100644 index 000000000..7b8d23971 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/extend/AccessSecretManager.java @@ -0,0 +1,24 @@ +package org.gaul.s3proxy.extend; + +import java.util.List; +import java.util.Map; + +/** + * @author yangyanbo + */ +public interface AccessSecretManager { + + + CacheManager createCacheManager() throws Exception; + + void createAKSKForBucket(String bucket, String access_key, String secret_key) throws Exception; + + void loads2Cache() throws Exception ; + + Map getBucketAkSkList() throws Exception ; + + String getBucketFromAccessKey(String ak); + + + +} diff --git a/src/main/java/org/gaul/s3proxy/extend/AkSkPair.java b/src/main/java/org/gaul/s3proxy/extend/AkSkPair.java new file mode 100644 index 000000000..1d0668d31 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/extend/AkSkPair.java @@ -0,0 +1,29 @@ +package org.gaul.s3proxy.extend; + +/** + * @author yangyanbo + */ +public class AkSkPair { + String access_key; + String secret_key; + + public String getSecret_key() { + return secret_key; + } + + public void setSecret_key(String secret_key) { + this.secret_key = secret_key; + } + + public AkSkPair(String ak, String sk){ + this.access_key = ak; + this.secret_key = sk; + } + public String getAccess_key() { + return access_key; + } + + public void setAccess_key(String access_key) { + this.access_key = access_key; + } +} diff --git a/src/main/java/org/gaul/s3proxy/extend/CacheManager.java b/src/main/java/org/gaul/s3proxy/extend/CacheManager.java new file mode 100644 index 000000000..bcea6d898 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/extend/CacheManager.java @@ -0,0 +1,27 @@ +package org.gaul.s3proxy.extend; + +import java.util.List; +import java.util.Map; + +/** + * @author yangyanbo + */ +public interface CacheManager { + + /** + * + * @param key + * @param o + * @return int + */ + int setKey(String key, Object o) throws Exception; + + /** + * + * @param key + * @return + */ + Object getKey(String key) throws Exception; + + Map getAll() throws Exception; +} diff --git a/src/main/java/org/gaul/s3proxy/extend/MemoryCache.java b/src/main/java/org/gaul/s3proxy/extend/MemoryCache.java new file mode 100644 index 000000000..796ea63cc --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/extend/MemoryCache.java @@ -0,0 +1,33 @@ +package org.gaul.s3proxy.extend; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author yangyanbo + */ +public class MemoryCache implements CacheManager{ + public static Map cache = new HashMap<>(); + + @Override + public int setKey(String key, Object o) { + int ret = 0; + try { + cache.put(key, o); + return ret; + }catch (Exception e){ + System.out.println(e.getMessage()); + return -1; + } + } + + @Override + public Object getKey(String key) { + return cache.get(key); + } + + @Override + public Map getAll() throws Exception { + return cache; + } +} diff --git a/src/main/java/org/gaul/s3proxy/extend/SqliteAKSKManager.java b/src/main/java/org/gaul/s3proxy/extend/SqliteAKSKManager.java new file mode 100644 index 000000000..9d435d845 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/extend/SqliteAKSKManager.java @@ -0,0 +1,144 @@ +package org.gaul.s3proxy.extend; + +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +import java.sql.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author yangyanbo + */ +public class SqliteAKSKManager extends AbStractAKSKManager { + String confPath = "src/main/resources/config.db"; + Connection conn = null; + + public SqliteAKSKManager() throws SQLException { + super(); + String dbUrl = "jdbc:sqlite:" + confPath; + conn = DriverManager.getConnection(dbUrl); + if (conn != null) { + DatabaseMetaData meta = conn.getMetaData(); + System.out.println("The driver name is " + meta.getDriverName()); + System.out.println("A new database has been created."); + } + this.createTable(); + this.insertSomeUsers(); + this.loads2Cache(); + } + + @Override + public void createAKSKForBucket(String bucket, String access_key, String secret_key) { + try { + String sql = "INSERT INTO bucket_access_manage(bucket, access_key, secret_key) VALUES(?,?, ?)"; + PreparedStatement pstmt = conn.prepareStatement(sql); + pstmt.setString(1, bucket); + pstmt.setString(2, access_key); + pstmt.setString(3, secret_key); + pstmt.executeUpdate(); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + @Override + public void loads2Cache() { + try { + List> bucketAkList = this.getBucketAkSKsFromDB(); + for (Map m :bucketAkList ){ + for (String key :m.keySet()){ + this.getCacheManager().setKey(key, m.get(key)); + } + } + }catch (Exception e) { + System.out.println("Take record from db to cache fail."); + } + + } + + @Override + public Map getBucketAkSkList() throws Exception { + // first take from cache. if not take from db + Map map = new HashMap<>(); + Map m = this.getCacheManager().getAll(); + for (String key: m.keySet()){ + map.put(key, (AkSkPair) m.get(key)); + } + return map; + } + + @Override + public String getBucketFromAccessKey(String ak) { + // ugly here but work + String bucket = ""; + try{ + Map m = this.getCacheManager().getAll(); + for (String k : m.keySet()){ + AkSkPair akSkPair = (AkSkPair) m.get(k); + if (akSkPair.getAccess_key().equalsIgnoreCase(ak)){ + bucket = k; + break; + } + } + + }catch (Exception e){ + e.printStackTrace(); + return bucket; + }finally { + return bucket; + } + } + + public void createTable() { + try { + // SQL statement for creating a new table + String sql = "CREATE TABLE IF NOT EXISTS bucket_access_manage (\n" + + " id integer PRIMARY KEY,\n" + + " bucket text UNIQUE NOT NULL,\n" + + " access_key text NOT NULL,\n" + + " secret_key text NOT NULL,\n" + + " enable integer\n" + + ");"; + Statement stmt = conn.createStatement(); + stmt.execute(sql); + }catch (Exception e){ + e.printStackTrace(); + } + + } + + public List> getBucketAkSKsFromDB() throws Exception { + String sql = "SELECT id, bucket, access_key,secret_key FROM bucket_access_manage"; + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(sql); + List> rsl = new ArrayList<>(); + // loop through the result set + while (rs.next()) { + System.out.println(rs.getInt("id") + "\t" + + rs.getString("bucket") + "\t" + + rs.getString("access_key") + "\t" + + rs.getString("secret_key")); + + Map m = new HashMap<>(); + AkSkPair akSkPair = new AkSkPair(rs.getString("access_key"), + rs.getString("secret_key")); + + m.put(rs.getString("bucket"), akSkPair); + rsl.add(m); + } + return rsl; + } + + public void insertSomeUsers() { + // SQL statement for creating a new table + this.createAKSKForBucket("ai-ctr", "helloworld1", "1234512345"); + this.createAKSKForBucket("jianjiang", "helloworld2", "1234512345"); + this.createAKSKForBucket("lhli3", "helloworld3", "1234512345"); + + } + + +} From 3ac7ca81c340ad0f7e45c9e8664e46ef65b36408 Mon Sep 17 00:00:00 2001 From: maybaby Date: Thu, 19 Aug 2021 17:40:26 +0800 Subject: [PATCH 2/5] Add a sqlite implement bucket ak manager and memory cache implement. To achieve that we can use different accesskey and secret key for per bucket. --- src/main/java/org/gaul/s3proxy/Main.java | 4 ++-- src/main/java/org/gaul/s3proxy/S3ProxyHandler.java | 2 +- .../java/org/gaul/s3proxy/extend/SqliteAKSKManager.java | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/gaul/s3proxy/Main.java b/src/main/java/org/gaul/s3proxy/Main.java index 693246ef0..cae32e00f 100644 --- a/src/main/java/org/gaul/s3proxy/Main.java +++ b/src/main/java/org/gaul/s3proxy/Main.java @@ -102,7 +102,7 @@ public static void main(String[] args) throws Exception { 1, 20, 60 * 1000, factory); ImmutableMap.Builder> locators = ImmutableMap.builder(); - // 加载用户数据库 + // load bucket access db AccessSecretManager accessSecretManager = new SqliteAKSKManager(); Map accessCache= accessSecretManager.getBucketAkSkList(); @@ -129,7 +129,7 @@ public static void main(String[] args) throws Exception { locators.put(localIdentity, Maps.immutableEntry( localCredential, blobStore)); for (Map.Entry entry : accessCache.entrySet()) { - // 增加 sqlite中access key的 locators + // add sqlite access key to locators locators.put(entry.getValue().getAccess_key(), Maps.immutableEntry( entry.getValue().getSecret_key(), blobStore)); } diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java index 84729cd37..0bac1d0a6 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java @@ -608,7 +608,7 @@ public final void doHandle(HttpServletRequest baseRequest, if (!constantTimeEquals(expectedSignature, authHeader.getSignature())) { - // 验证verify access key check + // verify access key check throw new S3Exception(S3ErrorCode.SIGNATURE_DOES_NOT_MATCH); } } diff --git a/src/main/java/org/gaul/s3proxy/extend/SqliteAKSKManager.java b/src/main/java/org/gaul/s3proxy/extend/SqliteAKSKManager.java index 9d435d845..63e8b8255 100644 --- a/src/main/java/org/gaul/s3proxy/extend/SqliteAKSKManager.java +++ b/src/main/java/org/gaul/s3proxy/extend/SqliteAKSKManager.java @@ -134,9 +134,9 @@ public List> getBucketAkSKsFromDB() throws Exception { public void insertSomeUsers() { // SQL statement for creating a new table - this.createAKSKForBucket("ai-ctr", "helloworld1", "1234512345"); - this.createAKSKForBucket("jianjiang", "helloworld2", "1234512345"); - this.createAKSKForBucket("lhli3", "helloworld3", "1234512345"); + this.createAKSKForBucket("test1", "helloworld1", "1234512345"); + this.createAKSKForBucket("test2", "helloworld2", "1234512345"); + this.createAKSKForBucket("test3", "helloworld3", "1234512345"); } From 725166ce740669e89283cba9c970b7559afff4bd Mon Sep 17 00:00:00 2001 From: maybaby Date: Mon, 23 Aug 2021 09:23:09 +0800 Subject: [PATCH 3/5] Add a yaml file config to support different bucket ak/sk --- pom.xml | 7 + src/main/java/org/gaul/s3proxy/Main.java | 121 ++++++++++------ .../s3proxy/extend/AbStractAKSKManager.java | 35 ++++- .../s3proxy/extend/AccessSecretManager.java | 7 +- .../s3proxy/extend/SqliteAKSKManager.java | 62 +++++---- .../gaul/s3proxy/extend/YamlAKSKManager.java | 131 ++++++++++++++++++ .../extend/YamlConfigReloadAdaptor.java | 49 +++++++ src/main/resources/bucket_access.yml | 15 ++ 8 files changed, 359 insertions(+), 68 deletions(-) create mode 100644 src/main/java/org/gaul/s3proxy/extend/YamlAKSKManager.java create mode 100644 src/main/java/org/gaul/s3proxy/extend/YamlConfigReloadAdaptor.java create mode 100644 src/main/resources/bucket_access.yml diff --git a/pom.xml b/pom.xml index 4c54d64b7..6bb351e10 100644 --- a/pom.xml +++ b/pom.xml @@ -457,5 +457,12 @@ sqlite-jdbc 3.36.0.1 + + + org.yaml + snakeyaml + 1.29 + + diff --git a/src/main/java/org/gaul/s3proxy/Main.java b/src/main/java/org/gaul/s3proxy/Main.java index cae32e00f..f7f8dee82 100644 --- a/src/main/java/org/gaul/s3proxy/Main.java +++ b/src/main/java/org/gaul/s3proxy/Main.java @@ -16,19 +16,6 @@ package org.gaul.s3proxy; -import java.io.Console; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintStream; -import java.nio.charset.StandardCharsets; -import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; - import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -36,10 +23,10 @@ import com.google.common.io.Files; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Module; - -import org.gaul.s3proxy.extend.AccessSecretManager; +import org.gaul.s3proxy.extend.AbStractAKSKManager; import org.gaul.s3proxy.extend.AkSkPair; import org.gaul.s3proxy.extend.SqliteAKSKManager; +import org.gaul.s3proxy.extend.YamlAKSKManager; import org.jclouds.Constants; import org.jclouds.ContextBuilder; import org.jclouds.JcloudsVersion; @@ -56,8 +43,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + public final class Main { private static final Logger logger = LoggerFactory.getLogger(Main.class); + private Main() { throw new AssertionError("intentionally not implemented"); } @@ -69,8 +65,27 @@ private static final class Options { @Option(name = "--version", usage = "display version") private boolean version; + + @Option(name = "--enable_bucket_ak_sk", + usage = "experimental feature to provide different ak/sk for per bucket" + ) + private boolean enableBucketAkSk; + + @Option(name = "--config_store", + usage = "yaml,sqlite.. etc." + ) + private String configStoreType="yaml"; + + + @Option(name = "--config_path", + usage = "config file path. for sqlite is db file path, and for yaml is local yml file absolute path." + ) + private String configPath=""; + ; + } + public static void main(String[] args) throws Exception { Console console = System.console(); if (console == null) { @@ -100,11 +115,25 @@ public static void main(String[] args) throws Exception { .build(); ExecutorService executorService = DynamicExecutors.newScalingThreadPool( 1, 20, 60 * 1000, factory); - ImmutableMap.Builder> locators = - ImmutableMap.builder(); + Map> locators = + new LinkedHashMap<>(); // load bucket access db - AccessSecretManager accessSecretManager = new SqliteAKSKManager(); - Map accessCache= accessSecretManager.getBucketAkSkList(); + AbStractAKSKManager accessSecretManager = null; + Map accessCache = null; + if (options.enableBucketAkSk) { + switch (options.configStoreType) { + case "sqlite": + accessSecretManager = new SqliteAKSKManager(options.configPath); + break; + case "yaml": + accessSecretManager = new YamlAKSKManager(options.configPath); + break; + default: + accessSecretManager = new YamlAKSKManager(options.configPath); + } + accessCache = accessSecretManager.getBucketAkSkList(); + + } for (File propertiesFile : options.propertiesFiles) { Properties properties = new Properties(); @@ -115,6 +144,8 @@ public static void main(String[] args) throws Exception { BlobStore blobStore = createBlobStore(properties, executorService); + + blobStore = parseMiddlewareProperties(blobStore, executorService, properties); @@ -126,15 +157,20 @@ public static void main(String[] args) throws Exception { S3ProxyConstants.PROPERTY_IDENTITY); String localCredential = properties.getProperty( S3ProxyConstants.PROPERTY_CREDENTIAL); + Map> save = new HashMap<>(); locators.put(localIdentity, Maps.immutableEntry( localCredential, blobStore)); - for (Map.Entry entry : accessCache.entrySet()) { - // add sqlite access key to locators - locators.put(entry.getValue().getAccess_key(), Maps.immutableEntry( - entry.getValue().getSecret_key(), blobStore)); + // register blobStore to access manager + if (accessSecretManager!=null) { + accessSecretManager.registerBlobStore(blobStore); + } + if (accessCache != null){ + for (Map.Entry entry : accessCache.entrySet()) { + // add sqlite access key to locators + locators.put(entry.getValue().getAccess_key(), Maps.immutableEntry( + entry.getValue().getSecret_key(), blobStore)); + } } - - } S3Proxy.Builder s3ProxyBuilder2 = S3Proxy.Builder @@ -161,8 +197,12 @@ public static void main(String[] args) throws Exception { } final Map> locator = - locators.build(); + locators; + if (accessSecretManager!=null){ + accessSecretManager.setLocator(locator); + } if (!locator.isEmpty()) { + AbStractAKSKManager finalAbStractAKSKManager = accessSecretManager; s3Proxy.setBlobStoreLocator(new BlobStoreLocator() { @Override public Map.Entry locateBlobStore( @@ -173,18 +213,21 @@ public Map.Entry locateBlobStore( .getValue(); } throw new IllegalArgumentException( - "cannot use anonymous access with multiple" + - " backends"); + "cannot use anonymous access with multiple" + + " backends"); } // verify key - Map.Entry provider = locator.get(identity); - String bucket = accessSecretManager.getBucketFromAccessKey(identity); - if ( !bucket.equalsIgnoreCase("")) { - // if bucket and access_key not match. [1:1 in db] return null - if (!container.equalsIgnoreCase(bucket)){ - return null; + Map.Entry provider = locator.get(identity); + if (finalAbStractAKSKManager != null){ + String bucket = finalAbStractAKSKManager.getBucketFromAccessKey(identity); + if (!bucket.equalsIgnoreCase("")) { + // if bucket and access_key not match. [1:1 in db] return null + if (!container.equalsIgnoreCase(bucket)) { + return null; + } } } + return provider; } }); @@ -199,7 +242,7 @@ public Map.Entry locateBlobStore( } private static BlobStore parseMiddlewareProperties(BlobStore blobStore, - ExecutorService executorService, Properties properties) + ExecutorService executorService, Properties properties) throws IOException { Properties altProperties = new Properties(); for (Map.Entry entry : properties.entrySet()) { @@ -287,7 +330,7 @@ public void write(byte[] buf, int off, int len) { } private static BlobStore createBlobStore(Properties properties, - ExecutorService executorService) throws IOException { + ExecutorService executorService) throws IOException { String provider = properties.getProperty(Constants.PROPERTY_PROVIDER); String identity = properties.getProperty(Constants.PROPERTY_IDENTITY); String credential = properties.getProperty( @@ -300,7 +343,7 @@ private static BlobStore createBlobStore(Properties properties, if (provider == null) { System.err.println( "Properties file must contain: " + - Constants.PROPERTY_PROVIDER); + Constants.PROPERTY_PROVIDER); System.exit(1); } @@ -320,8 +363,8 @@ private static BlobStore createBlobStore(Properties properties, if (identity == null || credential == null) { System.err.println( "Properties file must contain: " + - Constants.PROPERTY_IDENTITY + " and " + - Constants.PROPERTY_CREDENTIAL); + Constants.PROPERTY_IDENTITY + " and " + + Constants.PROPERTY_CREDENTIAL); System.exit(1); } diff --git a/src/main/java/org/gaul/s3proxy/extend/AbStractAKSKManager.java b/src/main/java/org/gaul/s3proxy/extend/AbStractAKSKManager.java index d58ee0818..26aa38b2e 100644 --- a/src/main/java/org/gaul/s3proxy/extend/AbStractAKSKManager.java +++ b/src/main/java/org/gaul/s3proxy/extend/AbStractAKSKManager.java @@ -1,7 +1,9 @@ package org.gaul.s3proxy.extend; +import org.jclouds.blobstore.BlobStore; import sun.reflect.generics.reflectiveObjects.NotImplementedException; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -12,12 +14,26 @@ public class AbStractAKSKManager implements AccessSecretManager { public String cacheType = "memory"; + static public List blobStores = new ArrayList<>(); public CacheManager cacheManager; + Map> locator; + + /** static variable save last config + * + */ + public static Map save; public AbStractAKSKManager() { cacheManager = createCacheManager(); } + public Map> getLocator() { + return locator; + } + + public void setLocator(Map> locator) { + this.locator = locator; + } @Override public CacheManager createCacheManager() { @@ -37,7 +53,7 @@ public void createAKSKForBucket(String bucket, String access_key, String secret_ } @Override - public void loads2Cache() { + public Map loads2Cache() { throw new NotImplementedException(); } @@ -52,8 +68,25 @@ public String getBucketFromAccessKey(String ak) { throw new NotImplementedException(); } + @Override + public void registerBlobStore(BlobStore blobStore) { + blobStores.add(blobStore); + } + + @Override + public List listBlobStores() { + return blobStores; + } + public CacheManager getCacheManager() { return cacheManager; } + public static Map getSave() { + return save; + } + + public static void setSave(Map save) { + AbStractAKSKManager.save = save; + } } diff --git a/src/main/java/org/gaul/s3proxy/extend/AccessSecretManager.java b/src/main/java/org/gaul/s3proxy/extend/AccessSecretManager.java index 7b8d23971..acfb95c30 100644 --- a/src/main/java/org/gaul/s3proxy/extend/AccessSecretManager.java +++ b/src/main/java/org/gaul/s3proxy/extend/AccessSecretManager.java @@ -1,5 +1,7 @@ package org.gaul.s3proxy.extend; +import org.jclouds.blobstore.BlobStore; + import java.util.List; import java.util.Map; @@ -13,12 +15,15 @@ public interface AccessSecretManager { void createAKSKForBucket(String bucket, String access_key, String secret_key) throws Exception; - void loads2Cache() throws Exception ; + Map loads2Cache() throws Exception ; Map getBucketAkSkList() throws Exception ; String getBucketFromAccessKey(String ak); + void registerBlobStore(BlobStore blobStore); + + List listBlobStores(); } diff --git a/src/main/java/org/gaul/s3proxy/extend/SqliteAKSKManager.java b/src/main/java/org/gaul/s3proxy/extend/SqliteAKSKManager.java index 63e8b8255..af336bcd5 100644 --- a/src/main/java/org/gaul/s3proxy/extend/SqliteAKSKManager.java +++ b/src/main/java/org/gaul/s3proxy/extend/SqliteAKSKManager.java @@ -1,6 +1,8 @@ package org.gaul.s3proxy.extend; -import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +import com.google.common.collect.ImmutableMap; +import org.jclouds.blobstore.BlobStore; import java.sql.*; import java.util.ArrayList; @@ -12,12 +14,15 @@ * @author yangyanbo */ public class SqliteAKSKManager extends AbStractAKSKManager { - String confPath = "src/main/resources/config.db"; + String defaultConf = "src/main/resources/config.db"; Connection conn = null; - public SqliteAKSKManager() throws SQLException { + public SqliteAKSKManager(String configPath) throws SQLException { super(); - String dbUrl = "jdbc:sqlite:" + confPath; + if (configPath.equalsIgnoreCase("")) { + configPath = defaultConf; + } + String dbUrl = "jdbc:sqlite:" + configPath; conn = DriverManager.getConnection(dbUrl); if (conn != null) { DatabaseMetaData meta = conn.getMetaData(); @@ -45,16 +50,16 @@ public void createAKSKForBucket(String bucket, String access_key, String secret_ } @Override - public void loads2Cache() { + public Map loads2Cache() { try { - List> bucketAkList = this.getBucketAkSKsFromDB(); - for (Map m :bucketAkList ){ - for (String key :m.keySet()){ - this.getCacheManager().setKey(key, m.get(key)); - } + Map bucketAkList = this.getBucketAkSKsFromDB(); + for (String key : bucketAkList.keySet()) { + this.getCacheManager().setKey(key, bucketAkList.get(key)); } - }catch (Exception e) { + return bucketAkList; + } catch (Exception e) { System.out.println("Take record from db to cache fail."); + return null; } } @@ -64,7 +69,7 @@ public Map getBucketAkSkList() throws Exception { // first take from cache. if not take from db Map map = new HashMap<>(); Map m = this.getCacheManager().getAll(); - for (String key: m.keySet()){ + for (String key : m.keySet()) { map.put(key, (AkSkPair) m.get(key)); } return map; @@ -74,20 +79,20 @@ public Map getBucketAkSkList() throws Exception { public String getBucketFromAccessKey(String ak) { // ugly here but work String bucket = ""; - try{ + try { Map m = this.getCacheManager().getAll(); - for (String k : m.keySet()){ + for (String k : m.keySet()) { AkSkPair akSkPair = (AkSkPair) m.get(k); - if (akSkPair.getAccess_key().equalsIgnoreCase(ak)){ + if (akSkPair.getAccess_key().equalsIgnoreCase(ak)) { bucket = k; break; } } - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); return bucket; - }finally { + } finally { return bucket; } } @@ -98,23 +103,23 @@ public void createTable() { String sql = "CREATE TABLE IF NOT EXISTS bucket_access_manage (\n" + " id integer PRIMARY KEY,\n" + " bucket text UNIQUE NOT NULL,\n" - + " access_key text NOT NULL,\n" + + " access_key text UNIQUE NOT NULL,\n" + " secret_key text NOT NULL,\n" + " enable integer\n" + ");"; Statement stmt = conn.createStatement(); stmt.execute(sql); - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } } - public List> getBucketAkSKsFromDB() throws Exception { + public Map getBucketAkSKsFromDB() throws Exception { String sql = "SELECT id, bucket, access_key,secret_key FROM bucket_access_manage"; Statement stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(sql); - List> rsl = new ArrayList<>(); + Map rsl = new HashMap<>(); // loop through the result set while (rs.next()) { System.out.println(rs.getInt("id") + "\t" + @@ -122,21 +127,24 @@ public List> getBucketAkSKsFromDB() throws Exception { rs.getString("access_key") + "\t" + rs.getString("secret_key")); - Map m = new HashMap<>(); AkSkPair akSkPair = new AkSkPair(rs.getString("access_key"), rs.getString("secret_key")); - m.put(rs.getString("bucket"), akSkPair); - rsl.add(m); + rsl.put(rs.getString("bucket"), akSkPair); } return rsl; } public void insertSomeUsers() { // SQL statement for creating a new table - this.createAKSKForBucket("test1", "helloworld1", "1234512345"); - this.createAKSKForBucket("test2", "helloworld2", "1234512345"); - this.createAKSKForBucket("test3", "helloworld3", "1234512345"); + try { + this.createAKSKForBucket("test1", "cccccc", "ZDU3NzI3M2ZmODg1YzNmODRkYWRiODU3OGJiNDEzOTkgIC0K"); + this.createAKSKForBucket("test", "test", "NzIzZWU2ZmUyNjlkYjAzZDQ5YTllMjA0MTYwZGYzNGQgIC0K"); + this.createAKSKForBucket("test3", "helloworld3", "1234512345"); + } catch (Exception e) { + e.printStackTrace(); + } + } diff --git a/src/main/java/org/gaul/s3proxy/extend/YamlAKSKManager.java b/src/main/java/org/gaul/s3proxy/extend/YamlAKSKManager.java new file mode 100644 index 000000000..0d2535431 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/extend/YamlAKSKManager.java @@ -0,0 +1,131 @@ +package org.gaul.s3proxy.extend; + +import com.google.common.collect.ImmutableMap; +import org.apache.commons.io.filefilter.FileFilterUtils; +import org.apache.commons.io.monitor.FileAlterationListenerAdaptor; +import org.apache.commons.io.monitor.FileAlterationMonitor; +import org.apache.commons.io.monitor.FileAlterationObserver; +import org.jclouds.blobstore.BlobStore; +import org.yaml.snakeyaml.Yaml; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +/** + * @author yangyanbo + */ +public class YamlAKSKManager extends AbStractAKSKManager { + + String defaultBucketAccessYamlPath = "src/main/resources/bucket_access.yml"; + String confPath; + private FileAlterationListenerAdaptor listener; + ImmutableMap.Builder> locators; + + public YamlAKSKManager(String configPath) { + super(); + if (configPath.equalsIgnoreCase("")) { + this.confPath = defaultBucketAccessYamlPath; + } + + this.loads2Cache(); + this.startReloadMonitor(locators); + + } + + @Override + public void createAKSKForBucket(String bucket, String access_key, String secret_key) { + try { + + } catch (Exception e) { + e.printStackTrace(); + } + + } + + @Override + public Map loads2Cache() { + try { + Map bucketAkList = this.getBucketAkSkList(); + for (String key : bucketAkList.keySet()) { + this.getCacheManager().setKey(key, bucketAkList.get(key)); + } + // save + setSave(bucketAkList); + + return bucketAkList; + } catch (Exception e) { + System.out.println("Take record from db to cache fail."); + return null; + } + + + } + + @Override + public Map getBucketAkSkList() throws Exception { + InputStream inputStream = null; + try { + inputStream = new FileInputStream(this.confPath); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + Yaml yaml = new Yaml(); + Map properties = yaml.loadAs(inputStream, Map.class); + Map m = new HashMap<>(); + for (String k : properties.keySet()) { + Map mm = (Map) properties.get(k); + AkSkPair akSkPair = new AkSkPair((String) mm.get("access_key"), (String) mm.get("secret_key")); + m.put(k, akSkPair); + } + return m; + } + + @Override + public String getBucketFromAccessKey(String ak) { + // ugly here but work + String bucket = ""; + try { + Map m = this.getCacheManager().getAll(); + for (String k : m.keySet()) { + AkSkPair akSkPair = (AkSkPair) m.get(k); + if (akSkPair.getAccess_key().equalsIgnoreCase(ak)) { + bucket = k; + break; + } + } + + } catch (Exception e) { + e.printStackTrace(); + return bucket; + } finally { + return bucket; + } + } + + public void startReloadMonitor(ImmutableMap.Builder> locators) { + // set monitor monitor the yml file change + File conf = new File(this.confPath); + FileAlterationObserver observer = new FileAlterationObserver(conf.getParentFile().getAbsolutePath(), + FileFilterUtils.suffixFileFilter(".yml")); + // add event + listener = new YamlConfigReloadAdaptor(this); + observer.addListener(listener); + + // 5s + FileAlterationMonitor monitor = new FileAlterationMonitor(5); + monitor.addObserver(observer); + + try { + monitor.start(); + } catch (Exception e) { + e.printStackTrace(); + } + + + } + +} diff --git a/src/main/java/org/gaul/s3proxy/extend/YamlConfigReloadAdaptor.java b/src/main/java/org/gaul/s3proxy/extend/YamlConfigReloadAdaptor.java new file mode 100644 index 000000000..4d35d008e --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/extend/YamlConfigReloadAdaptor.java @@ -0,0 +1,49 @@ +package org.gaul.s3proxy.extend; + +import com.google.common.collect.Maps; +import org.apache.commons.io.monitor.FileAlterationListenerAdaptor; +import org.jclouds.blobstore.BlobStore; + +import java.io.File; +import java.util.Map; + +/** + * @author yangyanbo + */ +public class YamlConfigReloadAdaptor extends FileAlterationListenerAdaptor { + + private YamlAKSKManager accessSecretManager; + + public YamlConfigReloadAdaptor(YamlAKSKManager manager) { + this.accessSecretManager = manager; + } + + @Override + public void onFileChange(File file) { + System.out.println("Yaml Config changed... config.....reload....."); + super.onFileChange(file); + Map last = AbStractAKSKManager.getSave(); + Map modify = this.accessSecretManager.loads2Cache(); + // compaire two map to find out which is update or delete. + // two loop ugly. + for (BlobStore blobStore : this.accessSecretManager.listBlobStores()) { + if (this.accessSecretManager.getCacheManager() != null) { + for (Map.Entry entry : modify.entrySet()) { + // add access key to locators + this.accessSecretManager.getLocator().put(entry.getValue().getAccess_key(), Maps.immutableEntry( + entry.getValue().getSecret_key(), blobStore)); + + } + // delete this time yaml delete + for (Map.Entry entry : last.entrySet()) { + if (!modify.containsKey(entry.getKey())){ + this.accessSecretManager.getLocator().remove(entry.getValue().access_key); + } + + } + } + } + + } + +} diff --git a/src/main/resources/bucket_access.yml b/src/main/resources/bucket_access.yml new file mode 100644 index 000000000..5e8575638 --- /dev/null +++ b/src/main/resources/bucket_access.yml @@ -0,0 +1,15 @@ +ai_ctr: + access_key: "helloworld1" + secret_key: "cccccccccdddddddddddd" +hellworld: + access_key: "helloworld2" + secret_key: "ccccccdcssdddddddddddd" +hellworld2: + access_key: "hellowo2rld2" + secret_key: "ccccccdc2ssdddddddddddd" +test3: + access_key: "helloworld3" + secret_key: "12345122345" +test4: + access_key: "hello11world3" + secret_key: "1234511122345" From 7a31d52bb860fc18080eb434bf405c2ba9216092 Mon Sep 17 00:00:00 2001 From: maybaby Date: Mon, 23 Aug 2021 09:34:47 +0800 Subject: [PATCH 4/5] Reformat code --- src/main/java/org/gaul/s3proxy/Main.java | 13 ++++++------- .../gaul/s3proxy/extend/AbStractAKSKManager.java | 7 ++++--- .../gaul/s3proxy/extend/AccessSecretManager.java | 14 +++++++------- .../java/org/gaul/s3proxy/extend/AkSkPair.java | 3 ++- .../java/org/gaul/s3proxy/extend/CacheManager.java | 9 +++------ .../java/org/gaul/s3proxy/extend/MemoryCache.java | 6 +++--- .../s3proxy/extend/YamlConfigReloadAdaptor.java | 4 ++-- src/main/resources/s3proxy-fs.conf | 6 ++++++ src/test/resources/run-s3-tests.sh | 4 ++-- 9 files changed, 35 insertions(+), 31 deletions(-) create mode 100644 src/main/resources/s3proxy-fs.conf diff --git a/src/main/java/org/gaul/s3proxy/Main.java b/src/main/java/org/gaul/s3proxy/Main.java index f7f8dee82..7e74b413d 100644 --- a/src/main/java/org/gaul/s3proxy/Main.java +++ b/src/main/java/org/gaul/s3proxy/Main.java @@ -74,13 +74,13 @@ private static final class Options { @Option(name = "--config_store", usage = "yaml,sqlite.. etc." ) - private String configStoreType="yaml"; + private String configStoreType = "yaml"; @Option(name = "--config_path", usage = "config file path. for sqlite is db file path, and for yaml is local yml file absolute path." ) - private String configPath=""; + private String configPath = ""; ; } @@ -145,7 +145,6 @@ public static void main(String[] args) throws Exception { BlobStore blobStore = createBlobStore(properties, executorService); - blobStore = parseMiddlewareProperties(blobStore, executorService, properties); @@ -161,10 +160,10 @@ public static void main(String[] args) throws Exception { locators.put(localIdentity, Maps.immutableEntry( localCredential, blobStore)); // register blobStore to access manager - if (accessSecretManager!=null) { + if (accessSecretManager != null) { accessSecretManager.registerBlobStore(blobStore); } - if (accessCache != null){ + if (accessCache != null) { for (Map.Entry entry : accessCache.entrySet()) { // add sqlite access key to locators locators.put(entry.getValue().getAccess_key(), Maps.immutableEntry( @@ -198,7 +197,7 @@ public static void main(String[] args) throws Exception { final Map> locator = locators; - if (accessSecretManager!=null){ + if (accessSecretManager != null) { accessSecretManager.setLocator(locator); } if (!locator.isEmpty()) { @@ -218,7 +217,7 @@ public Map.Entry locateBlobStore( } // verify key Map.Entry provider = locator.get(identity); - if (finalAbStractAKSKManager != null){ + if (finalAbStractAKSKManager != null) { String bucket = finalAbStractAKSKManager.getBucketFromAccessKey(identity); if (!bucket.equalsIgnoreCase("")) { // if bucket and access_key not match. [1:1 in db] return null diff --git a/src/main/java/org/gaul/s3proxy/extend/AbStractAKSKManager.java b/src/main/java/org/gaul/s3proxy/extend/AbStractAKSKManager.java index 26aa38b2e..7bb983c92 100644 --- a/src/main/java/org/gaul/s3proxy/extend/AbStractAKSKManager.java +++ b/src/main/java/org/gaul/s3proxy/extend/AbStractAKSKManager.java @@ -17,16 +17,17 @@ public class AbStractAKSKManager implements AccessSecretManager { static public List blobStores = new ArrayList<>(); public CacheManager cacheManager; - Map> locator; + Map> locator; - /** static variable save last config - * + /** + * static variable save last config */ public static Map save; public AbStractAKSKManager() { cacheManager = createCacheManager(); } + public Map> getLocator() { return locator; } diff --git a/src/main/java/org/gaul/s3proxy/extend/AccessSecretManager.java b/src/main/java/org/gaul/s3proxy/extend/AccessSecretManager.java index acfb95c30..377633a78 100644 --- a/src/main/java/org/gaul/s3proxy/extend/AccessSecretManager.java +++ b/src/main/java/org/gaul/s3proxy/extend/AccessSecretManager.java @@ -11,19 +11,19 @@ public interface AccessSecretManager { - CacheManager createCacheManager() throws Exception; + CacheManager createCacheManager() throws Exception; - void createAKSKForBucket(String bucket, String access_key, String secret_key) throws Exception; + void createAKSKForBucket(String bucket, String access_key, String secret_key) throws Exception; - Map loads2Cache() throws Exception ; + Map loads2Cache() throws Exception; - Map getBucketAkSkList() throws Exception ; + Map getBucketAkSkList() throws Exception; - String getBucketFromAccessKey(String ak); + String getBucketFromAccessKey(String ak); - void registerBlobStore(BlobStore blobStore); + void registerBlobStore(BlobStore blobStore); - List listBlobStores(); + List listBlobStores(); } diff --git a/src/main/java/org/gaul/s3proxy/extend/AkSkPair.java b/src/main/java/org/gaul/s3proxy/extend/AkSkPair.java index 1d0668d31..a1d5ad5f0 100644 --- a/src/main/java/org/gaul/s3proxy/extend/AkSkPair.java +++ b/src/main/java/org/gaul/s3proxy/extend/AkSkPair.java @@ -15,10 +15,11 @@ public void setSecret_key(String secret_key) { this.secret_key = secret_key; } - public AkSkPair(String ak, String sk){ + public AkSkPair(String ak, String sk) { this.access_key = ak; this.secret_key = sk; } + public String getAccess_key() { return access_key; } diff --git a/src/main/java/org/gaul/s3proxy/extend/CacheManager.java b/src/main/java/org/gaul/s3proxy/extend/CacheManager.java index bcea6d898..5e26e0de4 100644 --- a/src/main/java/org/gaul/s3proxy/extend/CacheManager.java +++ b/src/main/java/org/gaul/s3proxy/extend/CacheManager.java @@ -1,6 +1,5 @@ package org.gaul.s3proxy.extend; -import java.util.List; import java.util.Map; /** @@ -9,19 +8,17 @@ public interface CacheManager { /** - * * @param key * @param o * @return int */ - int setKey(String key, Object o) throws Exception; + int setKey(String key, Object o) throws Exception; /** - * * @param key * @return */ - Object getKey(String key) throws Exception; + Object getKey(String key) throws Exception; - Map getAll() throws Exception; + Map getAll() throws Exception; } diff --git a/src/main/java/org/gaul/s3proxy/extend/MemoryCache.java b/src/main/java/org/gaul/s3proxy/extend/MemoryCache.java index 796ea63cc..11a0dba3d 100644 --- a/src/main/java/org/gaul/s3proxy/extend/MemoryCache.java +++ b/src/main/java/org/gaul/s3proxy/extend/MemoryCache.java @@ -6,8 +6,8 @@ /** * @author yangyanbo */ -public class MemoryCache implements CacheManager{ - public static Map cache = new HashMap<>(); +public class MemoryCache implements CacheManager { + public static Map cache = new HashMap<>(); @Override public int setKey(String key, Object o) { @@ -15,7 +15,7 @@ public int setKey(String key, Object o) { try { cache.put(key, o); return ret; - }catch (Exception e){ + } catch (Exception e) { System.out.println(e.getMessage()); return -1; } diff --git a/src/main/java/org/gaul/s3proxy/extend/YamlConfigReloadAdaptor.java b/src/main/java/org/gaul/s3proxy/extend/YamlConfigReloadAdaptor.java index 4d35d008e..555d2726d 100644 --- a/src/main/java/org/gaul/s3proxy/extend/YamlConfigReloadAdaptor.java +++ b/src/main/java/org/gaul/s3proxy/extend/YamlConfigReloadAdaptor.java @@ -22,7 +22,7 @@ public YamlConfigReloadAdaptor(YamlAKSKManager manager) { public void onFileChange(File file) { System.out.println("Yaml Config changed... config.....reload....."); super.onFileChange(file); - Map last = AbStractAKSKManager.getSave(); + Map last = AbStractAKSKManager.getSave(); Map modify = this.accessSecretManager.loads2Cache(); // compaire two map to find out which is update or delete. // two loop ugly. @@ -36,7 +36,7 @@ public void onFileChange(File file) { } // delete this time yaml delete for (Map.Entry entry : last.entrySet()) { - if (!modify.containsKey(entry.getKey())){ + if (!modify.containsKey(entry.getKey())) { this.accessSecretManager.getLocator().remove(entry.getValue().access_key); } diff --git a/src/main/resources/s3proxy-fs.conf b/src/main/resources/s3proxy-fs.conf new file mode 100644 index 000000000..721c68b14 --- /dev/null +++ b/src/main/resources/s3proxy-fs.conf @@ -0,0 +1,6 @@ +s3proxy.authorization=aws-v2-or-v4 +s3proxy.endpoint=http://127.0.0.1:8083 +s3proxy.identity=wocao +s3proxy.credential=1234cccc +jclouds.provider=filesystem +jclouds.filesystem.basedir=/Users/yangyanbo/projects/cetc-cloud/code/s3proxy/tmp \ No newline at end of file diff --git a/src/test/resources/run-s3-tests.sh b/src/test/resources/run-s3-tests.sh index 37fecbaf9..b17bd026c 100755 --- a/src/test/resources/run-s3-tests.sh +++ b/src/test/resources/run-s3-tests.sh @@ -14,8 +14,8 @@ popd # launch S3Proxy using HTTP and a fixed port sed "s,^\(s3proxy.endpoint\)=.*,\1=http://127.0.0.1:${S3PROXY_PORT}," \ - < src/test/resources/s3proxy.conf | grep -v secure-endpoint > target/s3proxy.conf -$S3PROXY_BIN --properties target/s3proxy.conf & + < src/test/resources/s3proxy-fs.conf | grep -v secure-endpoint > target/s3proxy-fs.conf +$S3PROXY_BIN --properties target/s3proxy-fs.conf & S3PROXY_PID=$! function finish { From 885f9585a82942f6a80ba8f8a95ef9e3b128b1f2 Mon Sep 17 00:00:00 2001 From: maybaby Date: Fri, 27 Aug 2021 16:03:05 +0800 Subject: [PATCH 5/5] bugfix --- src/main/java/org/gaul/s3proxy/extend/YamlAKSKManager.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/gaul/s3proxy/extend/YamlAKSKManager.java b/src/main/java/org/gaul/s3proxy/extend/YamlAKSKManager.java index 0d2535431..90b4a066c 100644 --- a/src/main/java/org/gaul/s3proxy/extend/YamlAKSKManager.java +++ b/src/main/java/org/gaul/s3proxy/extend/YamlAKSKManager.java @@ -29,6 +29,8 @@ public YamlAKSKManager(String configPath) { super(); if (configPath.equalsIgnoreCase("")) { this.confPath = defaultBucketAccessYamlPath; + }else{ + this.confPath = configPath; } this.loads2Cache();