extensions: add sync, download functionality

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

Author: shwstppr

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -546,6 +546,7 @@ public class ApiConstants {
 
     public static final String SOURCE_CIDR_LIST = "sourcecidrlist";
     public static final String SOURCE_ZONE_ID = "sourcezoneid";
+    public static final String SOURCE_MANAGEMENT_SERVER_ID = "sourcemanagementserverid";
     public static final String SSL_VERIFICATION = "sslverification";
     public static final String START_ASN = "startasn";
     public static final String START_DATE = "startdate";
@@ -566,6 +567,7 @@ public class ApiConstants {
     public static final String SWAP_OWNER = "swapowner";
     public static final String SYSTEM_VM_TYPE = "systemvmtype";
     public static final String TAGS = "tags";
+    public static final String TARGET_MANAGEMENT_SERVER_IDS = "targetmanagementserverids";
     public static final String STORAGE_TAGS = "storagetags";
     public static final String STORAGE_ACCESS_GROUPS = "storageaccessgroups";
     public static final String STORAGE_ACCESS_GROUP = "storageaccessgroup";

client/conf/server.properties.in

@@ -62,3 +62,18 @@ extensions.deployment.mode=@EXTENSIONSDEPLOYMENTMODE@
 # Thread pool configuration
 #threads.min=10
 #threads.max=500
+
+# These properties configure the share endpoint, which enables controlled file sharing through the management server.
+# They allow administrators to enable or disable sharing, set the base directory for shared files, define cache
+# behavior, restrict access to specific directories, and secure access with a secret key. This ensures flexible and
+# secure file sharing for different modules such as extensions, etc.
+# Enable or disable file sharing feature (true/false). Default is true
+# share.enabled=true
+# The base directory from which files can be shared. Default is <HOME_DIRECTORY_OF_CLOUD_USER>/share
+# share.base.dir=
+# The cache control header value to be used for shared files. Default is public,max-age=86400,immutable
+# share.cache.control=public,max-age=86400,immutable
+# Allow or disallow directory listing when accessing a directory. Default is false
+# share.dir.allowed=false
+# Secret key for securing links using HMAC signature. Default is empty (no security)
+# share.secret=

client/src/main/java/org/apache/cloudstack/ServerDaemon.java

@@ -24,15 +24,25 @@
 import java.io.InputStream;
 import java.lang.management.ManagementFactory;
 import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.List;
 import java.util.Properties;
 
-import com.cloud.api.ApiServer;
+import javax.servlet.DispatcherType;
+
+import org.apache.cloudstack.utils.server.ServerPropertiesUtil;
 import org.apache.commons.daemon.Daemon;
 import org.apache.commons.daemon.DaemonContext;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.eclipse.jetty.jmx.MBeanContainer;
 import org.eclipse.jetty.server.ForwardedRequestCustomizer;
+import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.RequestLog;
@@ -46,14 +56,18 @@
 import org.eclipse.jetty.server.handler.RequestLogHandler;
 import org.eclipse.jetty.server.handler.gzip.GzipHandler;
 import org.eclipse.jetty.server.session.SessionHandler;
+import org.eclipse.jetty.servlet.DefaultServlet;
+import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.resource.Resource;
 import org.eclipse.jetty.util.ssl.KeyStoreScanner;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
 import org.eclipse.jetty.webapp.WebAppContext;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
 
+import com.cloud.api.ApiServer;
 import com.cloud.utils.Pair;
 import com.cloud.utils.PropertiesUtil;
 import com.cloud.utils.server.ServerProperties;
@@ -89,6 +103,11 @@ public class ServerDaemon implements Daemon {
     private static final String THREADS_MIN = "threads.min";
     private static final String THREADS_MAX = "threads.max";
 
+    public static final String SHARE_ENABLED = "share.enabled";
+    public static final String SHARE_CACHE_CONTROL = "share.cache.control";
+    public static final String SHARE_DIR_ALLOWED = "share.dir.allowed";
+    public static final String SHARE_SECRET = "share.secret";
+
     ////////////////////////////////////////////////////////
     /////////////// Server Configuration ///////////////////
     ////////////////////////////////////////////////////////
@@ -111,6 +130,12 @@ public class ServerDaemon implements Daemon {
     private int minThreads;
     private int maxThreads;
 
+    private boolean shareEnabled = false;
+    private String shareBaseDir;
+    private String shareCacheCtl;
+    private boolean shareDirList = false;
+    private String shareSecret;
+
     //////////////////////////////////////////////////
     /////////////// Public methods ///////////////////
     //////////////////////////////////////////////////
@@ -121,6 +146,14 @@ public static void main(final String... anArgs) throws Exception {
         daemon.start();
     }
 
+    protected void initShareConfigFromProperties(Properties properties) {
+        setShareEnabled(Boolean.parseBoolean(properties.getProperty(SHARE_ENABLED, "true")));
+        setShareBaseDir(ServerPropertiesUtil.getShareBaseDirectory());
+        setShareCacheCtl(properties.getProperty(SHARE_CACHE_CONTROL, "public,max-age=86400,immutable"));
+        setShareDirList(Boolean.parseBoolean(properties.getProperty(SHARE_DIR_ALLOWED, "false")));
+        setShareSecret(properties.getProperty(SHARE_SECRET, ""));
+    }
+
     @Override
     public void init(final DaemonContext context) {
         final File confFile = PropertiesUtil.findConfigFile("server.properties");
@@ -153,6 +186,7 @@ public void init(final DaemonContext context) {
             setMaxFormKeys(Integer.valueOf(properties.getProperty(REQUEST_MAX_FORM_KEYS_KEY, String.valueOf(DEFAULT_REQUEST_MAX_FORM_KEYS))));
             setMinThreads(Integer.valueOf(properties.getProperty(THREADS_MIN, "10")));
             setMaxThreads(Integer.valueOf(properties.getProperty(THREADS_MAX, "500")));
+            initShareConfigFromProperties(properties);
         } catch (final IOException e) {
             logger.warn("Failed to read configuration from server.properties file", e);
         } finally {
@@ -288,6 +322,52 @@ private void createHttpsConnector(final HttpConfiguration httpConfig) {
         }
     }
 
+    /**
+     * Creates a Jetty context at /share to serve static files for modules (e.g. Extensions Framework).
+     * Controlled via server properties
+     *
+     * @return a configured Handler or null if disabled.
+     */
+    private Handler createShareContextHandler() throws IOException {
+        if (!shareEnabled) {
+            logger.info("{}=false; /{} context not mounted", SHARE_ENABLED, ServerPropertiesUtil.SHARE_DIR);
+            return null;
+        }
+
+        final Path base = Paths.get(shareBaseDir);
+        Files.createDirectories(base);
+
+        final ServletContextHandler shareCtx = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
+        shareCtx.setContextPath("/" + ServerPropertiesUtil.SHARE_DIR);
+        shareCtx.setBaseResource(Resource.newResource(base.toAbsolutePath().toUri()));
+
+        // Efficient static file serving
+        ServletHolder def = shareCtx.addServlet(DefaultServlet.class, "/*");
+        def.setInitParameter("dirAllowed", Boolean.toString(shareDirList));
+        def.setInitParameter("etags", "true");
+        def.setInitParameter("cacheControl", shareCacheCtl);
+        def.setInitParameter("useFileMappedBuffer", "true");
+        def.setInitParameter("acceptRanges", "true");
+
+        // Gzip using modern Jetty handler
+        org.eclipse.jetty.server.handler.gzip.GzipHandler gzipHandler =
+                new org.eclipse.jetty.server.handler.gzip.GzipHandler();
+        gzipHandler.setMinGzipSize(1024);
+        gzipHandler.setIncludedMimeTypes(
+                "text/html", "text/plain", "text/css", "text/javascript",
+                "application/javascript", "application/json", "application/xml");
+        gzipHandler.setHandler(shareCtx);
+
+        // Optional signed-URL guard (path + "|" + exp => HMAC-SHA256, base64url)
+        if (StringUtils.isNotBlank(shareSecret)) {
+            shareCtx.addFilter(new FilterHolder(new ShareSignedUrlFilter(true, shareSecret)),
+                    "/*", EnumSet.of(DispatcherType.REQUEST));
+        }
+
+        logger.info("Mounted /{} static context at baseDir={}", ServerPropertiesUtil.SHARE_DIR, base);
+        return shareCtx;
+    }
+
     private Pair<SessionHandler,HandlerCollection> createHandlers() {
         final WebAppContext webApp = new WebAppContext();
         webApp.setContextPath(contextPath);
@@ -318,8 +398,23 @@ private Pair<SessionHandler,HandlerCollection> createHandlers() {
         rootRedirect.setNewContextURL(contextPath);
         rootRedirect.setPermanent(true);
 
+        // Optional /share handler (served by createShareContextHandler)
+        Handler shareHandler = null;
+        try {
+            shareHandler = createShareContextHandler();
+        } catch (IOException e) {
+            logger.error("Failed to initialize /share context", e);
+        }
+
+        List<Handler> handlers = new java.util.ArrayList<>();
+        handlers.add(log);
+        handlers.add(gzipHandler);
+        if (shareHandler != null) {
+            handlers.add(shareHandler);
+        }
         // Put rootRedirect at the end!
-        return new Pair<>(webApp.getSessionHandler(), new HandlerCollection(log, gzipHandler, rootRedirect));
+        handlers.add(rootRedirect);
+        return new Pair<>(webApp.getSessionHandler(), new HandlerCollection(handlers.toArray(new Handler[0])));
     }
 
     private RequestLog createRequestLog() {
@@ -408,4 +503,24 @@ public void setMinThreads(int minThreads) {
     public void setMaxThreads(int maxThreads) {
         this.maxThreads = maxThreads;
     }
+
+    public void setShareEnabled(boolean shareEnabled) {
+        this.shareEnabled = shareEnabled;
+    }
+
+    public void setShareBaseDir(String shareBaseDir) {
+        this.shareBaseDir = shareBaseDir;
+    }
+
+    public void setShareCacheCtl(String shareCacheCtl) {
+        this.shareCacheCtl = shareCacheCtl;
+    }
+
+    public void setShareDirList(boolean shareDirList) {
+        this.shareDirList = shareDirList;
+    }
+
+    public void setShareSecret(String shareSecret) {
+        this.shareSecret = shareSecret;
+    }
 }

client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java

@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack;
+
+import java.io.IOException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.time.Instant;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.cloudstack.utils.security.HMACSignUtil;
+import org.apache.commons.codec.DecoderException;
+
+/**
+ * Optional HMAC token check: /share/...?...&exp=1699999999&sig=BASE64URL(HMACSHA256(path|exp))
+ * If no token params are present, request is allowed (set requireToken=true to enforce).
+ */
+public class ShareSignedUrlFilter implements Filter {
+    private final boolean requireToken;
+    private final String secret;
+
+    public ShareSignedUrlFilter(boolean requireToken, String secret) {
+        this.requireToken = requireToken;
+        this.secret = secret;
+    }
+
+    @Override public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain)
+            throws IOException, ServletException {
+        HttpServletRequest r = (HttpServletRequest) req;
+        HttpServletResponse w = (HttpServletResponse) res;
+
+        String expStr = r.getParameter("exp");
+        String sig = r.getParameter("sig");
+
+        if (!requireToken && (expStr == null || sig == null)) {
+            chain.doFilter(req, res);
+            return;
+        }
+        if (expStr == null || sig == null) {
+            w.sendError(HttpServletResponse.SC_FORBIDDEN, "Missing token");
+            return;
+        }
+        long exp;
+        try {
+            exp = Long.parseLong(expStr);
+        } catch (NumberFormatException e) {
+            w.sendError(HttpServletResponse.SC_FORBIDDEN, "Bad exp");
+            return;
+        }
+        if (Instant.now().getEpochSecond() > exp) {
+            w.sendError(HttpServletResponse.SC_FORBIDDEN, "Token expired");
+            return;
+        }
+        String want = "";
+        try {
+            String data = r.getRequestURI() + "|" + expStr;
+            want = HMACSignUtil.generateSignature(data, secret);
+        } catch (InvalidKeyException | NoSuchAlgorithmException | DecoderException e) {
+            w.sendError(HttpServletResponse.SC_FORBIDDEN, "Auth error");
+            return;
+        }
+        if (!want.equals(sig)) {
+            w.sendError(HttpServletResponse.SC_FORBIDDEN, "Bad signature");
+            return;
+        }
+        chain.doFilter(req, res);
+    }
+}
+

engine/components-api/src/main/java/com/cloud/hypervisor/ExternalProvisioner.java

@@ -35,18 +35,6 @@
 
 public interface ExternalProvisioner extends Manager {
 
-    String getExtensionsPath();
-
-    String getExtensionPath(String relativePath);
-
-    String getChecksumForExtensionPath(String extensionName, String relativePath);
-
-    void prepareExtensionPath(String extensionName, boolean userDefined, String extensionRelativePath);
-
-    void cleanupExtensionPath(String extensionName, String extensionRelativePath);
-
-    void cleanupExtensionData(String extensionName, int olderThanDays, boolean cleanupDirectory);
-
     PrepareExternalProvisioningAnswer prepareExternalProvisioning(String hostGuid, String extensionName, String extensionRelativePath, PrepareExternalProvisioningCommand cmd);
 
     StartAnswer startInstance(String hostGuid, String extensionName, String extensionRelativePath, StartCommand cmd);

framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java

@@ -468,6 +468,8 @@ public String execute(final String strPeer, final long agentId, final String cmd
         return null;
     }
 
+
+
     @Override
     public ManagementServerHostVO getPeer(final String mgmtServerId) {
         return _mshostDao.findByMsid(Long.parseLong(mgmtServerId));