diff --git a/README.md b/README.md index 8b1adab2..8fac36f3 100644 --- a/README.md +++ b/README.md @@ -191,13 +191,16 @@ Default configure file `overlaybd.json` is installed to `/etc/overlaybd/`. | credentialFilePath(legacy) | The credential used for fetching images on registry. `/opt/overlaybd/cred.json` is the default value. | | credentialConfig.mode | Authentication mode for lazy-loading.
- `file` means reading credential from `credentialConfig.path`.
- `http` means sending an http request to `credentialConfig.path` | | credentialConfig.path | credential file path or url which is determined by `mode` | -| download.enable | Whether background downloading is enabled or not. | -| download.delay | The seconds waiting to start downloading task after the overlaybd device launched. | -| download.delayExtra | A random extra delay is attached to delay, avoiding too many tasks started at the same time. | -| download.maxMBps | The speed limit in MB/s for a downloading task. | -| download.blockSize | The download block size from source, in byte. `262144` is default (256 KB). | -| p2pConfig.enable | Whether p2p proxy is enabled or not. | -| p2pConfig.address | The proxy for p2p download, the format is `localhost:/`, depending on dadip2p.yaml | +| download.enable | Whether background downloading is enabled or not. | +| download.delay | The seconds waiting to start downloading task after the overlaybd device launched. | +| download.delayExtra | A random extra delay is attached to delay, avoiding too many tasks started at the same time. | +| download.maxMBps | The speed limit in MB/s for a downloading task. | +| download.blockSize | The download block size from source, in byte. `262144` is default (256 KB). | +| p2pConfig.enable | Whether p2p proxy is enabled or not. | +| p2pConfig.address | The proxy for p2p download, the format is `localhost:/`, depending on dadip2p.yaml | +| mirrors.[].host | The host name of the registry to mirror. Mirrors are only supported for `registryFsVersion` 'v2'. | +| mirrors.[].mirror | The mirror URL to use to download a blob inplace of the given host. Includes the scheme (e.g http://localhost:8080) | +| mirrors.[].timeoutMs | The timeout to get a response from the mirror, in milliseconds. | | exporterConfig.enable | whether or not create a server to show Prometheus metrics. | | exporterConfig.uriPrefix | URI prefix for export metrics. | | exporterConfig.port | port for http server to show metrics. | diff --git a/src/config.h b/src/config.h index 33ce1d80..8df7f62e 100644 --- a/src/config.h +++ b/src/config.h @@ -74,6 +74,14 @@ struct P2PConfig : public ConfigUtils::Config { APPCFG_PARA(address, std::string, "http://localhost:9731/accelerator"); }; +struct MirrorConfig : public ConfigUtils::Config { + APPCFG_CLASS + + APPCFG_PARA(host, std::string); + APPCFG_PARA(mirror, std::string); + APPCFG_PARA(timeoutMs, uint64_t, 200); +}; + struct GzipCacheConfig : public ConfigUtils::Config { APPCFG_CLASS @@ -146,6 +154,7 @@ struct GlobalConfig : public ConfigUtils::Config { APPCFG_PARA(enableAudit, bool, true); APPCFG_PARA(enableThread, bool, false); APPCFG_PARA(p2pConfig, P2PConfig); + APPCFG_PARA(mirrors, std::vector); APPCFG_PARA(exporterConfig, ExporterConfig); APPCFG_PARA(auditPath, std::string, "/var/log/overlaybd-audit.log"); APPCFG_PARA(registryFsVersion, std::string, "v2"); diff --git a/src/image_service.cpp b/src/image_service.cpp index a5e2cda5..4487a9e0 100644 --- a/src/image_service.cpp +++ b/src/image_service.cpp @@ -348,7 +348,7 @@ int ImageService::init() { registryfs_creator = new_registryfs_v2; global_fs.underlay_registryfs = registryfs_creator( - {this, &ImageService::reload_auth}, cafile, 30UL * 1000000, + {this, &ImageService::reload_auth}, global_conf.mirrors(), cafile, 30UL * 1000000, global_conf.certConfig().certFile().c_str(), global_conf.certConfig().keyFile().c_str(), global_conf.userAgent().c_str()); if (global_fs.underlay_registryfs == nullptr) { LOG_ERROR_RETURN(0, -1, "create registryfs failed."); diff --git a/src/overlaybd/registryfs/registryfs.cpp b/src/overlaybd/registryfs/registryfs.cpp index 27b52954..6282b8d1 100644 --- a/src/overlaybd/registryfs/registryfs.cpp +++ b/src/overlaybd/registryfs/registryfs.cpp @@ -527,7 +527,7 @@ inline IFile *RegistryFSImpl::open(const char *pathname, int) { return file; } -IFileSystem *new_registryfs_v1(PasswordCB callback, const char *caFile, uint64_t timeout, +IFileSystem *new_registryfs_v1(PasswordCB callback, std::vector /*mirrors*/, const char *caFile, uint64_t timeout, const char *cert_file, const char *key_file, const char *__) { if (!callback) LOG_ERROR_RETURN(EINVAL, nullptr, "password callback not set"); diff --git a/src/overlaybd/registryfs/registryfs.h b/src/overlaybd/registryfs/registryfs.h index ce90df01..f18c2d58 100644 --- a/src/overlaybd/registryfs/registryfs.h +++ b/src/overlaybd/registryfs/registryfs.h @@ -15,6 +15,9 @@ */ #pragma once + +#include "../../config.h" + #include #include #include @@ -29,6 +32,7 @@ using PasswordCB = Delegate, const char *>; extern "C" { photon::fs::IFileSystem *new_registryfs_v1(PasswordCB callback, + std::vector mirrors, const char *caFile = nullptr, uint64_t timeout = -1, const char *cert_file = nullptr, @@ -36,6 +40,7 @@ photon::fs::IFileSystem *new_registryfs_v1(PasswordCB callback, const char *__ = nullptr); photon::fs::IFileSystem *new_registryfs_v2(PasswordCB callback, + std::vector mirrors, const char *caFile = nullptr, uint64_t timeout = -1, const char *cert_file = nullptr, diff --git a/src/overlaybd/registryfs/registryfs_v2.cpp b/src/overlaybd/registryfs/registryfs_v2.cpp index 92416bc3..a9d59a05 100644 --- a/src/overlaybd/registryfs/registryfs_v2.cpp +++ b/src/overlaybd/registryfs/registryfs_v2.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -126,9 +127,9 @@ class RegistryFSImpl_v2 : public RegistryFS { return open(pathname, flags); // ignore mode } - RegistryFSImpl_v2(PasswordCB callback, const char *caFile, uint64_t timeout, + RegistryFSImpl_v2(PasswordCB callback, std::vector mirrors, const char *caFile, uint64_t timeout, photon::net::TLSContext *ctx, const char *ua) - : m_callback(callback), m_caFile(caFile), m_timeout(timeout), m_tls_ctx(ctx), + : m_callback(callback), m_mirrors(std::move(mirrors)), m_caFile(caFile), m_timeout(timeout), m_tls_ctx(ctx), m_meta_size(kMinimalMetaLife), m_scope_token(kMinimalTokenLife), m_url_info(kMinimalAUrlLife) { @@ -209,8 +210,35 @@ class RegistryFSImpl_v2 : public RegistryFS { } UrlInfo* get_actual_url(const estring &url, uint64_t timeout, long &code) { - Timeout tmo(timeout); + if (m_accelerate.empty() && !m_mirrors.empty()) { + // If p2p acceleration is not enabled, and mirrors are configured, + // check each mirror whose host matches, in order, until one of them succeeds. + URL parsed(url); + if (!parsed.empty()) { + // The URL was parsed successfully. + for (ImageConfigNS::MirrorConfig& mirror : m_mirrors) { + if (mirror.host() == parsed.host_port()) { + long mirror_code = 0; + estring mirror_url = mirror.mirror() + std::string(parsed.path()); + Timeout mirrorTimeout(std::min(tmo.timeout(), mirror.timeoutMs() * 1000)); + LOG_INFO("trying mirror `", mirror_url, "` for original url `", url, "` with timeout=", mirrorTimeout.timeout_ms(), "ms"); + UrlInfo* info = get_actual_url_impl(mirror_url, mirrorTimeout, mirror_code); + if (info != nullptr) { + code = mirror_code; + return info; + } + } + } + LOG_INFO("falling back to original url `", url, "`"); + } else { + LOG_WARN("failed to parse url `", url, "`"); + } + } + return get_actual_url_impl(url, tmo.timeout(), code); + } + + UrlInfo* get_actual_url_impl(const estring &url, Timeout tmo, long &code) { estring authurl, scope; estring *token = nullptr; @@ -317,6 +345,7 @@ class RegistryFSImpl_v2 : public RegistryFS { protected: PasswordCB m_callback; + std::vector m_mirrors; estring m_accelerate; estring m_caFile; estring m_useragent; @@ -533,7 +562,7 @@ inline IFile *RegistryFSImpl_v2::open(const char *pathname, int) { return file; } -IFileSystem *new_registryfs_v2(PasswordCB callback, const char *caFile, uint64_t timeout, +IFileSystem *new_registryfs_v2(PasswordCB callback, std::vector mirrors, const char *caFile, uint64_t timeout, const char *cert_file, const char *key_file, const char *customized_ua) { if (!callback) LOG_ERROR_RETURN(EINVAL, nullptr, "password callback not set"); @@ -541,7 +570,7 @@ IFileSystem *new_registryfs_v2(PasswordCB callback, const char *caFile, uint64_t if (!ctx) { LOG_ERRNO_RETURN(0, nullptr, "failed to new tls context"); } - return new RegistryFSImpl_v2(callback, caFile ? caFile : "", timeout, ctx, customized_ua); + return new RegistryFSImpl_v2(callback, std::move(mirrors), caFile ? caFile : "", timeout, ctx, customized_ua); } class RegistryUploader : public VirtualFile { @@ -738,7 +767,7 @@ class RegistryUploader : public VirtualFile { photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE); DEFER(photon::fini()); m_upload_fs = new RegistryFSImpl_v2({this, &RegistryUploader::load_auth}, - "", m_timeout, m_tls_ctx, nullptr); + {}, "", m_timeout, m_tls_ctx, nullptr); DEFER({ delete m_upload_fs; }); m_http_client_ts = photon::now; ::posix_memalign(&m_upload_buf, 4096, 1024 * 1024);