Add mirror support for registry fs

Author: adam-rl

README.md

@@ -191,13 +191,16 @@ Default configure file `overlaybd.json` is installed to `/etc/overlaybd/`.
 | credentialFilePath(legacy)  | The credential used for fetching images on registry. `/opt/overlaybd/cred.json` is the default value. |
 | credentialConfig.mode       | Authentication mode for lazy-loading. <br> - `file` means reading credential from `credentialConfig.path`.  <br> - `http` means sending an http request to `credentialConfig.path` |
 | credentialConfig.path       | credential file path or url which is determined by `mode`                                     |
-| download.enable     | Whether background downloading is enabled or not.                                                     |
-| download.delay      | The seconds waiting to start downloading task after the overlaybd device launched.                    |
-| download.delayExtra | A random extra delay is attached to delay, avoiding too many tasks started at the same time.          |
-| download.maxMBps    | The speed limit in MB/s for a downloading task.                                                       |
-| download.blockSize  | The download block size from source, in byte. `262144` is default (256 KB).                           |
-| p2pConfig.enable    | Whether p2p proxy is enabled or not.                                                                  |
-| p2pConfig.address   | The proxy for p2p download, the format is `localhost:<P2PConfig.Port>/<P2PConfig.APIKey>`, depending on dadip2p.yaml |
+| download.enable      | Whether background downloading is enabled or not.                                                     |
+| download.delay       | The seconds waiting to start downloading task after the overlaybd device launched.                    |
+| download.delayExtra  | A random extra delay is attached to delay, avoiding too many tasks started at the same time.          |
+| download.maxMBps     | The speed limit in MB/s for a downloading task.                                                       |
+| download.blockSize   | The download block size from source, in byte. `262144` is default (256 KB).                           |
+| p2pConfig.enable     | Whether p2p proxy is enabled or not.                                                                  |
+| p2pConfig.address    | The proxy for p2p download, the format is `localhost:<P2PConfig.Port>/<P2PConfig.APIKey>`, depending on dadip2p.yaml |
+| mirrors.[].host      | The host name of the registry to mirror. Mirrors are only supported for `registryFsVersion` 'v2'.       |
+| mirrors.[].mirror    | The mirror URL to use to download a blob inplace of the given host. Includes the scheme (e.g http://localhost:8080) |
+| mirrors.[].timeoutMs | The timeout to get a response from the mirror, in milliseconds.                                      |
 | exporterConfig.enable         | whether or not create a server to show Prometheus metrics.                                  |
 | exporterConfig.uriPrefix      | URI prefix for export metrics.                                                              |
 | exporterConfig.port           | port for http server to show metrics.                                                       |

src/config.h

@@ -74,6 +74,14 @@ struct P2PConfig : public ConfigUtils::Config {
     APPCFG_PARA(address, std::string, "http://localhost:9731/accelerator");
 };
 
+struct MirrorConfig : public ConfigUtils::Config {
+    APPCFG_CLASS
+
+    APPCFG_PARA(host, std::string);
+    APPCFG_PARA(mirror, std::string);
+    APPCFG_PARA(timeoutMs, uint64_t, 200);
+};
+
 struct GzipCacheConfig : public ConfigUtils::Config {
     APPCFG_CLASS
 
@@ -146,6 +154,7 @@ struct GlobalConfig : public ConfigUtils::Config {
     APPCFG_PARA(enableAudit, bool, true);
     APPCFG_PARA(enableThread, bool, false);
     APPCFG_PARA(p2pConfig, P2PConfig);
+    APPCFG_PARA(mirrors, std::vector<MirrorConfig>);
     APPCFG_PARA(exporterConfig, ExporterConfig);
     APPCFG_PARA(auditPath, std::string, "/var/log/overlaybd-audit.log");
     APPCFG_PARA(registryFsVersion, std::string, "v2");

src/image_service.cpp

@@ -348,7 +348,7 @@ int ImageService::init() {
             registryfs_creator = new_registryfs_v2;
 
         global_fs.underlay_registryfs = registryfs_creator(
-            {this, &ImageService::reload_auth}, cafile, 30UL * 1000000,
+            {this, &ImageService::reload_auth}, global_conf.mirrors(), cafile, 30UL * 1000000,
             global_conf.certConfig().certFile().c_str(), global_conf.certConfig().keyFile().c_str(), global_conf.userAgent().c_str());
         if (global_fs.underlay_registryfs == nullptr) {
             LOG_ERROR_RETURN(0, -1, "create registryfs failed.");

src/overlaybd/registryfs/registryfs.cpp

@@ -527,7 +527,7 @@ inline IFile *RegistryFSImpl::open(const char *pathname, int) {
     return file;
 }
 
-IFileSystem *new_registryfs_v1(PasswordCB callback, const char *caFile, uint64_t timeout,
+IFileSystem *new_registryfs_v1(PasswordCB callback, std::vector<ImageConfigNS::MirrorConfig> /*mirrors*/, const char *caFile, uint64_t timeout,
                                const char *cert_file, const char *key_file, const char *__) {
     if (!callback)
         LOG_ERROR_RETURN(EINVAL, nullptr, "password callback not set");

src/overlaybd/registryfs/registryfs.h

@@ -15,6 +15,9 @@
 */
 
 #pragma once
+
+#include "../../config.h"
+
 #include <stdint.h>
 #include <string>
 #include <photon/common/callback.h>
@@ -29,13 +32,15 @@ using PasswordCB = Delegate<std::pair<std::string, std::string>, const char *>;
 
 extern "C" {
 photon::fs::IFileSystem *new_registryfs_v1(PasswordCB callback,
+                                           std::vector<ImageConfigNS::MirrorConfig> mirrors,
                                            const char *caFile = nullptr,
                                            uint64_t timeout = -1,
                                            const char *cert_file = nullptr,
                                            const char *key_file = nullptr,
                                            const char *__ = nullptr);
 
 photon::fs::IFileSystem *new_registryfs_v2(PasswordCB callback,
+                                           std::vector<ImageConfigNS::MirrorConfig> mirrors,
                                            const char *caFile = nullptr,
                                            uint64_t timeout = -1,
                                            const char *cert_file = nullptr,

src/overlaybd/registryfs/registryfs_v2.cpp

@@ -39,6 +39,7 @@
 #include <photon/fs/virtual-file.h>
 #include <photon/fs/localfs.h>
 #include <photon/net/http/client.h>
+#include <photon/net/http/url.h>
 #include <photon/net/utils.h>
 #include <photon/net/security-context/tls-stream.h>
 #include <rapidjson/document.h>
@@ -126,9 +127,9 @@ class RegistryFSImpl_v2 : public RegistryFS {
         return open(pathname, flags); // ignore mode
     }
 
-    RegistryFSImpl_v2(PasswordCB callback, const char *caFile, uint64_t timeout,
+    RegistryFSImpl_v2(PasswordCB callback, std::vector<ImageConfigNS::MirrorConfig> mirrors, const char *caFile, uint64_t timeout,
                       photon::net::TLSContext *ctx, const char *ua)
-        : m_callback(callback), m_caFile(caFile), m_timeout(timeout), m_tls_ctx(ctx),
+        : m_callback(callback), m_mirrors(std::move(mirrors)), m_caFile(caFile), m_timeout(timeout), m_tls_ctx(ctx),
           m_meta_size(kMinimalMetaLife), m_scope_token(kMinimalTokenLife),
           m_url_info(kMinimalAUrlLife) {
 
@@ -209,8 +210,35 @@ class RegistryFSImpl_v2 : public RegistryFS {
     }
 
     UrlInfo* get_actual_url(const estring &url, uint64_t timeout, long &code) {
-
         Timeout tmo(timeout);
+        if (m_accelerate.empty() && !m_mirrors.empty()) {
+            // If p2p acceleration is not enabled, and mirrors are configured,
+            // check each mirror whose host matches, in order, until one of them succeeds.
+            URL parsed(url);
+            if (!parsed.empty()) {
+                // The URL was parsed successfully.
+                for (ImageConfigNS::MirrorConfig& mirror : m_mirrors) {
+                    if (mirror.host() == parsed.host_port()) {
+                        long mirror_code = 0;
+                        estring mirror_url = mirror.mirror() + std::string(parsed.path());
+                        Timeout mirrorTimeout(std::min(tmo.timeout(), mirror.timeoutMs() * 1000));
+                        LOG_INFO("trying mirror `", mirror_url, "` for original url `", url, "` with timeout=", mirrorTimeout.timeout_ms(), "ms");
+                        UrlInfo* info = get_actual_url_impl(mirror_url, mirrorTimeout, mirror_code);
+                        if (info != nullptr) {
+                            code = mirror_code;
+                            return info;
+                        }
+                    }
+                }
+                LOG_INFO("falling back to original url `", url, "`");
+            } else {
+                LOG_WARN("failed to parse url `", url, "`");
+            }
+        }
+        return get_actual_url_impl(url, tmo.timeout(), code);
+    }
+
+    UrlInfo* get_actual_url_impl(const estring &url, Timeout tmo, long &code) {
         estring authurl, scope;
         estring *token = nullptr;
 
@@ -317,6 +345,7 @@ class RegistryFSImpl_v2 : public RegistryFS {
 
 protected:
     PasswordCB m_callback;
+    std::vector<ImageConfigNS::MirrorConfig> m_mirrors;
     estring m_accelerate;
     estring m_caFile;
     estring m_useragent;
@@ -533,15 +562,15 @@ inline IFile *RegistryFSImpl_v2::open(const char *pathname, int) {
     return file;
 }
 
-IFileSystem *new_registryfs_v2(PasswordCB callback, const char *caFile, uint64_t timeout,
+IFileSystem *new_registryfs_v2(PasswordCB callback, std::vector<ImageConfigNS::MirrorConfig> mirrors, const char *caFile, uint64_t timeout,
                                const char *cert_file, const char *key_file, const char *customized_ua) {
     if (!callback)
         LOG_ERROR_RETURN(EINVAL, nullptr, "password callback not set");
     auto ctx = new_tls_context_from_file(cert_file, key_file);
     if (!ctx) {
         LOG_ERRNO_RETURN(0, nullptr, "failed to new tls context");
     }
-    return new RegistryFSImpl_v2(callback, caFile ? caFile : "", timeout, ctx, customized_ua);
+    return new RegistryFSImpl_v2(callback, std::move(mirrors), caFile ? caFile : "", timeout, ctx, customized_ua);
 }
 
 class RegistryUploader : public VirtualFile {
@@ -738,7 +767,7 @@ class RegistryUploader : public VirtualFile {
         photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE);
         DEFER(photon::fini());
         m_upload_fs = new RegistryFSImpl_v2({this, &RegistryUploader::load_auth},
-                                            "", m_timeout, m_tls_ctx, nullptr);
+                                            {}, "", m_timeout, m_tls_ctx, nullptr);
         DEFER({ delete m_upload_fs; });
         m_http_client_ts = photon::now;
         ::posix_memalign(&m_upload_buf, 4096, 1024 * 1024);