Implement cache for helm packages (#603)

Author: olevitt

README.md

@@ -80,13 +80,16 @@ Configurable properties :
 | `security.cors.allowed_origins` | | To indicate which origins are allowed by [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS) |
 
 ### HTTP configuration  
-| Key | Default | Description |
-| --------------------- | ------- | ------------------------------------------------------------------ |
-| `http.proxyHost` | | Proxy hostname (e.g : proxy.example.com) |
-| `http.proxyPort` | 80 for `HTTP`, 443 for `HTTPS` | Proxy port |
-| `http.noProxy` | | Hosts that should not use the proxy (e.g : `localhost,host.example.com`) |
-| `http.proxyUsername` | | Username if the proxy requires authentication |
-| `http.proxyPassword` | | Password if the proxy requires authentication |
+| Key | Default                                                            | Description                                                                                        |
+| --------------------- |--------------------------------------------------------------------|----------------------------------------------------------------------------------------------------|
+| `http.proxyHost` |                                                                    | Proxy hostname (e.g : proxy.example.com)                                                           |
+| `http.proxyPort` | 80 for `HTTP`, 443 for `HTTPS`                                     | Proxy port                                                                                         |
+| `http.noProxy` |                                                                    | Hosts that should not use the proxy (e.g : `localhost,host.example.com`)                           |
+| `http.proxyUsername` |                                                                    | Username if the proxy requires authentication                                                      |
+| `http.proxyPassword` |                                                                    | Password if the proxy requires authentication                                                      |
+| `http.cacheEnabled` | true                                                               | Enable cache for charts retrieval.                                                                 |
+| `http.cacheMaxSizeMB` | 500                                                                | Maximum size (in MB) used by the cache (cleanup will be done automatically if space is not enough) |
+| `http.overrideCacheLocation` | `${java.io.tmpdir}/http_cache` (=`/tmp/http_cache` on most setups) | Specify where to store the cache                                                                   |
 
 ### Events
 | Key                      | Default | Description                                                        |

onyxia-api/src/main/java/fr/insee/onyxia/api/configuration/HttpClientProvider.java

@@ -1,23 +1,60 @@
 package fr.insee.onyxia.api.configuration;
 
 import fr.insee.onyxia.model.region.Region;
+import io.micrometer.common.util.StringUtils;
+import java.io.File;
 import java.io.IOException;
 import okhttp3.*;
 import okhttp3.logging.HttpLoggingInterceptor;
 import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
 @Configuration
 public class HttpClientProvider {
 
+    @Value("${http.cacheEnabled}")
+    private boolean cacheEnabled;
+
+    @Value("${http.cacheMaxSizeMB}")
+    private Integer cacheMaxSizeMB;
+
+    @Value("${http.overrideCacheLocation}")
+    private String overrideCacheLocation;
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(HttpClientProvider.class);
+
     @Bean
     public OkHttpClient httpClient() {
         OkHttpClient.Builder builder = new OkHttpClient.Builder();
         enableDebugFeatureIfEnabled(builder);
         return builder.build();
     }
 
+    @Bean
+    @Qualifier("withCache")
+    public OkHttpClient httpClientWithCache() {
+        OkHttpClient.Builder builder = new OkHttpClient.Builder();
+        enableDebugFeatureIfEnabled(builder);
+        if (cacheEnabled) {
+            File httpCacheDirectory;
+            if (StringUtils.isNotBlank(overrideCacheLocation)) {
+                httpCacheDirectory = new File(overrideCacheLocation);
+            } else {
+                httpCacheDirectory = new File(System.getProperty("java.io.tmpdir"), "http_cache");
+            }
+            LOGGER.info("Caching helm packages at {}", httpCacheDirectory.getAbsolutePath());
+            long cacheSize = cacheMaxSizeMB * 1024L * 1024L;
+            Cache cache = new Cache(httpCacheDirectory, cacheSize);
+            builder.cache(cache);
+        }
+        return builder.build();
+    }
+
     public OkHttpClient getClientForRegion(Region region) {
         OkHttpClient.Builder builder =
                 new OkHttpClient.Builder()
@@ -71,4 +108,28 @@ private void enableDebugFeatureIfEnabled(OkHttpClient.Builder builder) {
             builder.addInterceptor(logging);
         }
     }
+
+    public void setCacheEnabled(boolean cacheEnabled) {
+        this.cacheEnabled = cacheEnabled;
+    }
+
+    public boolean isCacheEnabled() {
+        return cacheEnabled;
+    }
+
+    public void setCacheMaxSizeMB(Integer cacheMaxSizeMB) {
+        this.cacheMaxSizeMB = cacheMaxSizeMB;
+    }
+
+    public Integer getCacheMaxSizeMB() {
+        return cacheMaxSizeMB;
+    }
+
+    public void setOverrideCacheLocation(String overrideCacheLocation) {
+        this.overrideCacheLocation = overrideCacheLocation;
+    }
+
+    public String getOverrideCacheLocation() {
+        return overrideCacheLocation;
+    }
 }

onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java

@@ -18,9 +18,7 @@
 import java.io.InputStream;
 import java.util.*;
 import java.util.stream.Collectors;
-import okhttp3.Credentials;
-import okhttp3.OkHttpClient;
-import okhttp3.Request;
+import okhttp3.*;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
@@ -49,7 +47,7 @@ public class CatalogLoader {
     public CatalogLoader(
             ResourceLoader resourceLoader,
             @Qualifier("helm") ObjectMapper mapperHelm,
-            OkHttpClient httpClient) {
+            @Qualifier("withCache") OkHttpClient httpClient) {
         this.resourceLoader = resourceLoader;
         this.mapperHelm = mapperHelm;
         this.httpClient = httpClient;
@@ -69,7 +67,10 @@ private void updateHelmRepository(CatalogWrapper cw) {
 
         try (InputStream stream =
                 fetchResource(
-                        cw.getLocation() + "/index.yaml", cw.getUsername(), cw.getPassword())) {
+                        cw.getLocation() + "/index.yaml",
+                        cw.getUsername(),
+                        cw.getPassword(),
+                        true)) {
             Repository repository = mapperHelm.readValue(stream, Repository.class);
 
             repository.setEntries(
@@ -128,11 +129,19 @@ private void updateHelmRepository(CatalogWrapper cw) {
 
     private InputStream fetchResource(String url, String username, String password)
             throws IOException {
+        return fetchResource(url, username, password, false);
+    }
+
+    private InputStream fetchResource(
+            String url, String username, String password, boolean skipCache) throws IOException {
         if (url.startsWith("http")) {
             Request.Builder builder = new Request.Builder().url(url);
             if (username != null && password != null) {
                 builder = builder.addHeader("Authorization", Credentials.basic(username, password));
             }
+            if (skipCache) {
+                builder.cacheControl(new CacheControl.Builder().noCache().build());
+            }
             return httpClient.newCall(builder.build()).execute().body().byteStream();
         } else {
             return resourceLoader.getResource(url).getInputStream();

onyxia-api/src/main/resources/application.properties

@@ -22,6 +22,9 @@ http.proxyPort=
 http.noProxy=
 http.proxyUsername=
 http.proxyPassword=
+http.cacheEnabled=true
+http.cacheMaxSizeMB=500
+http.overrideCacheLocation=
 # Enable compression by default for responses > 1k
 server.compression.enabled=true
 server.compression.min-response-size=1024