Feature/cstackex 22: Shared NFS pool and volume creation - Approach 1

plugins/storage/volume/ontap/pom.xml

@@ -24,14 +24,13 @@
     <parent>
         <groupId>org.apache.cloudstack</groupId>
         <artifactId>cloudstack-plugins</artifactId>
-        <version>4.22.0.0-SNAPSHOT</version>
+        <version>4.23.0.0-SNAPSHOT</version>
         <relativePath>../../../pom.xml</relativePath>
     </parent>
     <properties>
         <spring-cloud.version>2021.0.7</spring-cloud.version>
         <openfeign.version>11.0</openfeign.version>
         <json.version>20230227</json.version>
-        <jackson-databind.version>2.15.2</jackson-databind.version>
         <httpclient.version>4.5.14</httpclient.version>
         <swagger-annotations.version>1.6.2</swagger-annotations.version>
         <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
@@ -77,7 +76,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>${jackson-databind.version}</version>
+            <version>2.13.4</version>
         </dependency>
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>

plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/driver/OntapPrimaryDatastoreDriver.java

@@ -64,13 +64,15 @@ public class OntapPrimaryDatastoreDriver implements PrimaryDataStoreDriver {
 
     @Inject private StoragePoolDetailsDao storagePoolDetailsDao;
     @Inject private PrimaryDataStoreDao storagePoolDao;
+
     @Override
     public Map<String, String> getCapabilities() {
         s_logger.trace("OntapPrimaryDatastoreDriver: getCapabilities: Called");
         Map<String, String> mapCapabilities = new HashMap<>();
-
-        mapCapabilities.put(DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT.toString(), Boolean.TRUE.toString());
-        mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_SNAPSHOT.toString(), Boolean.TRUE.toString());
+        // RAW managed initial implementation: snapshot features not yet supported
+        // TODO Set it to false once we start supporting snapshot feature
+        mapCapabilities.put(DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT.toString(), Boolean.FALSE.toString());
+        mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_SNAPSHOT.toString(), Boolean.FALSE.toString());
 
         return mapCapabilities;
     }
@@ -81,9 +83,7 @@ public DataTO getTO(DataObject data) {
     }
 
     @Override
-    public DataStoreTO getStoreTO(DataStore store) {
-        return null;
-    }
+    public DataStoreTO getStoreTO(DataStore store) { return null; }
 
     @Override
     public void createAsync(DataStore dataStore, DataObject dataObject, AsyncCompletionCallback<CreateCmdResult> callback) {
@@ -103,7 +103,7 @@ public void createAsync(DataStore dataStore, DataObject dataObject, AsyncComplet
             s_logger.info("createAsync: Started for data store [{}] and data object [{}] of type [{}]",
                     dataStore, dataObject, dataObject.getType());
             if (dataObject.getType() == DataObjectType.VOLUME) {
-                path = createCloudStackVolumeForTypeVolume(dataStore, dataObject);
+                path = createCloudStackVolumeForTypeVolume(dataStore, (VolumeInfo)dataObject);
                 createCmdResult = new CreateCmdResult(path, new Answer(null, true, null));
             } else {
                 errMsg = "Invalid DataObjectType (" + dataObject.getType() + ") passed to createAsync";
@@ -116,11 +116,14 @@ public void createAsync(DataStore dataStore, DataObject dataObject, AsyncComplet
             createCmdResult = new CreateCmdResult(null, new Answer(null, false, errMsg));
             createCmdResult.setResult(e.toString());
         } finally {
+            if (createCmdResult != null && createCmdResult.isSuccess()) {
+                s_logger.info("createAsync: Volume created successfully. Path: {}", path);
+            }
             callback.complete(createCmdResult);
         }
     }
 
-    private String createCloudStackVolumeForTypeVolume(DataStore dataStore, DataObject dataObject) {
+    private String createCloudStackVolumeForTypeVolume(DataStore dataStore, VolumeInfo volumeObject) {
         StoragePoolVO storagePool = storagePoolDao.findById(dataStore.getId());
         if(storagePool == null) {
             s_logger.error("createCloudStackVolume : Storage Pool not found for id: " + dataStore.getId());
@@ -129,20 +132,43 @@ private String createCloudStackVolumeForTypeVolume(DataStore dataStore, DataObje
         Map<String, String> details = storagePoolDetailsDao.listDetailsKeyPairs(dataStore.getId());
         StorageStrategy storageStrategy = getStrategyByStoragePoolDetails(details);
         s_logger.info("createCloudStackVolumeForTypeVolume: Connection to Ontap SVM [{}] successful, preparing CloudStackVolumeRequest", details.get(Constants.SVM_NAME));
-        CloudStackVolume cloudStackVolumeRequest = Utility.createCloudStackVolumeRequestByProtocol(storagePool, details, dataObject);
+        CloudStackVolume cloudStackVolumeRequest = Utility.createCloudStackVolumeRequestByProtocol(storagePool, details, volumeObject);
         CloudStackVolume cloudStackVolume = storageStrategy.createCloudStackVolume(cloudStackVolumeRequest);
         if (ProtocolType.ISCSI.name().equalsIgnoreCase(details.get(Constants.PROTOCOL)) && cloudStackVolume.getLun() != null && cloudStackVolume.getLun().getName() != null) {
             return cloudStackVolume.getLun().getName();
+        } else if (ProtocolType.NFS.name().equalsIgnoreCase(details.get(Constants.PROTOCOL))) {
+            return volumeObject.getUuid(); // return the volume UUID for agent as path for mounting
         } else {
-            String errMsg = "createCloudStackVolumeForTypeVolume: Volume creation failed. Lun or Lun Path is null for dataObject: " + dataObject;
+            String errMsg = "createCloudStackVolumeForTypeVolume: Volume creation failed. Lun or Lun Path is null for dataObject: " + volumeObject;
             s_logger.error(errMsg);
             throw new CloudRuntimeException(errMsg);
         }
     }
 
     @Override
     public void deleteAsync(DataStore store, DataObject data, AsyncCompletionCallback<CommandResult> callback) {
-
+        CommandResult commandResult = new CommandResult();
+        try {
+            if (store == null || data == null) {
+                throw new CloudRuntimeException("deleteAsync: store or data is null");
+            }
+            if (data.getType() == DataObjectType.VOLUME) {
+                StoragePoolVO storagePool = storagePoolDao.findById(store.getId());
+                if(storagePool == null) {
+                    s_logger.error("deleteAsync : Storage Pool not found for id: " + store.getId());
+                    throw new CloudRuntimeException("deleteAsync : Storage Pool not found for id: " + store.getId());
+                }
+                Map<String, String> details = storagePoolDetailsDao.listDetailsKeyPairs(store.getId());
+                if (ProtocolType.NFS.name().equalsIgnoreCase(details.get(Constants.PROTOCOL))) {
+                    // ManagedNFS qcow2 backing file deletion handled by KVM host/libvirt; nothing to do via ONTAP REST.
+                    s_logger.info("deleteAsync: ManagedNFS volume {} no-op ONTAP deletion", data.getId());
+                }
+            }
+        } catch (Exception e) {
+            commandResult.setResult(e.getMessage());
+        } finally {
+            callback.complete(commandResult);
+        }
     }
 
     @Override
@@ -177,7 +203,6 @@ public boolean grantAccess(DataObject dataObject, Host host, DataStore dataStore
 
     @Override
     public void revokeAccess(DataObject dataObject, Host host, DataStore dataStore) {
-
     }
 
     @Override
@@ -217,7 +242,7 @@ public void handleQualityOfServiceForVolumeMigration(VolumeInfo volumeInfo, Qual
 
     @Override
     public boolean canProvideStorageStats() {
-        return true;
+        return false;
     }
 
     @Override
@@ -227,7 +252,7 @@ public Pair<Long, Long> getStorageStats(StoragePool storagePool) {
 
     @Override
     public boolean canProvideVolumeStats() {
-        return true;
+        return false; // Not yet implemented for RAW managed NFS
     }
 
     @Override

plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/FeignConfiguration.java

@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package org.apache.cloudstack.storage.feign;
 
 import feign.RequestInterceptor;
@@ -11,7 +30,7 @@
 import feign.codec.EncodeException;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.json.JsonMapper;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.http.conn.ConnectionKeepAliveStrategy;
 import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
@@ -36,13 +55,11 @@ public class FeignConfiguration {
     private final int retryMaxInterval = 5;
     private final String ontapFeignMaxConnection = "80";
     private final String ontapFeignMaxConnectionPerRoute = "20";
-    private final JsonMapper jsonMapper;
+    private final ObjectMapper jsonMapper;
 
     public FeignConfiguration() {
-        this.jsonMapper = JsonMapper.builder()
-                .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
-                .findAndAddModules()
-                .build();
+        this.jsonMapper = new ObjectMapper();
+        this.jsonMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
     }
 
     public Client createClient() {
@@ -120,16 +137,36 @@ public Decoder createDecoder() {
             @Override
             public Object decode(Response response, Type type) throws IOException, DecodeException {
                 if (response.body() == null) {
+                    logger.debug("Response body is null, returning null");
                     return null;
                 }
                 String json = null;
                 try (InputStream bodyStream = response.body().asInputStream()) {
                     json = new String(bodyStream.readAllBytes(), StandardCharsets.UTF_8);
                     logger.debug("Decoding JSON response: {}", json);
-                    return jsonMapper.readValue(json, jsonMapper.getTypeFactory().constructType(type));
+                    Object result = null;
+                    try {
+                        var javaType = jsonMapper.getTypeFactory().constructType(type);
+                        result = jsonMapper.readValue(json, javaType);
+                        logger.debug("jsonMapper.readValue() completed successfully");
+                    } catch (Throwable ex) {
+                        logger.error("EXCEPTION in jsonMapper.readValue()! Type: {}, Message: {}", ex.getClass().getName(), ex.getMessage(), ex);
+                        throw ex;
+                    }
+
+                    if (result == null) {
+                        logger.warn("Decoded result is null!");
+                    } else {
+                        logger.debug("Successfully decoded to object of type: {}", result.getClass().getName());
+                    }
+                    logger.debug("Returning result from decoder");
+                    return result;
                 } catch (IOException e) {
-                    logger.error("Error decoding JSON response. Status: {}, Raw body: {}", response.status(), json, e);
+                    logger.error("IOException during decoding. Status: {}, Raw body: {}", response.status(), json, e);
                     throw new DecodeException(response.status(), "Error decoding JSON response", response.request(), e);
+                } catch (Exception e) {
+                    logger.error("Unexpected error during decoding. Status: {}, Type: {}, Raw body: {}", response.status(), type, json, e);
+                    throw new DecodeException(response.status(), "Unexpected error during decoding", response.request(), e);
                 }
             }
         };

plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/NASFeignClient.java

@@ -19,65 +19,68 @@
 
 package org.apache.cloudstack.storage.feign.client;
 
+import feign.QueryMap;
 import org.apache.cloudstack.storage.feign.model.ExportPolicy;
 import org.apache.cloudstack.storage.feign.model.FileInfo;
 import org.apache.cloudstack.storage.feign.model.response.OntapResponse;
 import feign.Headers;
 import feign.Param;
 import feign.RequestLine;
 
-//TODO: Proper URLs should be added in the RequestLine annotations below
+import java.util.Map;
+
 public interface NASFeignClient {
 
     // File Operations
-    @RequestLine("GET /{volumeUuid}/files/{path}")
+    @RequestLine("GET /api/storage/volumes/{volumeUuid}/files/{path}")
     @Headers({"Authorization: {authHeader}"})
     OntapResponse<FileInfo> getFileResponse(@Param("authHeader") String authHeader,
-                                          @Param("volumeUuid") String volumeUUID,
-                                          @Param("path") String filePath);
+                                            @Param("volumeUuid") String volumeUUID,
+                                            @Param("path") String filePath);
 
-    @RequestLine("DELETE /{volumeUuid}/files/{path}")
+    @RequestLine("DELETE /api/storage/volumes/{volumeUuid}/files/{path}")
     @Headers({"Authorization: {authHeader}"})
     void deleteFile(@Param("authHeader") String authHeader,
-                   @Param("volumeUuid") String volumeUUID,
-                   @Param("path") String filePath);
+                    @Param("volumeUuid") String volumeUUID,
+                    @Param("path") String filePath);
 
-    @RequestLine("PATCH /{volumeUuid}/files/{path}")
+    @RequestLine("PATCH /api/storage/volumes/{volumeUuid}/files/{path}")
     @Headers({"Authorization: {authHeader}"})
     void updateFile(@Param("authHeader") String authHeader,
-                   @Param("volumeUuid") String volumeUUID,
-                   @Param("path") String filePath, FileInfo fileInfo);
+                    @Param("volumeUuid") String volumeUUID,
+                    @Param("path") String filePath,
+                    FileInfo fileInfo);
 
-    @RequestLine("POST /{volumeUuid}/files/{path}")
+    @RequestLine("POST /api/storage/volumes/{volumeUuid}/files/{path}")
     @Headers({"Authorization: {authHeader}"})
     void createFile(@Param("authHeader") String authHeader,
-                   @Param("volumeUuid") String volumeUUID,
-                   @Param("path") String filePath, FileInfo file);
+                    @Param("volumeUuid") String volumeUUID,
+                    @Param("path") String filePath,
+                    FileInfo file);
 
     // Export Policy Operations
-    @RequestLine("POST /")
-    @Headers({"Authorization: {authHeader}", "return_records: {returnRecords}"})
-    ExportPolicy createExportPolicy(@Param("authHeader") String authHeader,
-                                   @Param("returnRecords") boolean returnRecords,
+    @RequestLine("POST /api/protocols/nfs/export-policies")
+    @Headers({"Authorization: {authHeader}"})
+    void createExportPolicy(@Param("authHeader") String authHeader,
                                     ExportPolicy exportPolicy);
 
-    @RequestLine("GET /")
+    @RequestLine("GET /api/protocols/nfs/export-policies")
     @Headers({"Authorization: {authHeader}"})
-    OntapResponse<ExportPolicy> getExportPolicyResponse(@Param("authHeader") String authHeader);
+    OntapResponse<ExportPolicy> getExportPolicyResponse(@Param("authHeader") String authHeader, @QueryMap Map<String, Object> queryMap);
 
-    @RequestLine("GET /{id}")
+    @RequestLine("GET /api/protocols/nfs/export-policies/{id}")
     @Headers({"Authorization: {authHeader}"})
-    OntapResponse<ExportPolicy> getExportPolicyById(@Param("authHeader") String authHeader,
-                                                   @Param("id") String id);
+    ExportPolicy getExportPolicyById(@Param("authHeader") String authHeader,
+                                                    @Param("id") String id);
 
-    @RequestLine("DELETE /{id}")
+    @RequestLine("DELETE /api/protocols/nfs/export-policies/{id}")
     @Headers({"Authorization: {authHeader}"})
     void deleteExportPolicyById(@Param("authHeader") String authHeader,
-                               @Param("id") String id);
+                                @Param("id") String id);
 
-    @RequestLine("PATCH /{id}")
+    @RequestLine("PATCH /api/protocols/nfs/export-policies/{id}")
     @Headers({"Authorization: {authHeader}"})
     OntapResponse<ExportPolicy> updateExportPolicy(@Param("authHeader") String authHeader,
-                                                  @Param("id") String id,
-                                                  ExportPolicy request);
+                                                   @Param("id") String id,
+                                                   ExportPolicy request);
 }

plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/VolumeFeignClient.java

@@ -18,11 +18,15 @@
  */
 package org.apache.cloudstack.storage.feign.client;
 
+import feign.QueryMap;
 import org.apache.cloudstack.storage.feign.model.Volume;
 import org.apache.cloudstack.storage.feign.model.response.JobResponse;
 import feign.Headers;
 import feign.Param;
 import feign.RequestLine;
+import org.apache.cloudstack.storage.feign.model.response.OntapResponse;
+
+import java.util.Map;
 
 public interface VolumeFeignClient {
 
@@ -38,8 +42,11 @@ public interface VolumeFeignClient {
     @Headers({"Authorization: {authHeader}"})
     Volume getVolumeByUUID(@Param("authHeader") String authHeader, @Param("uuid") String uuid);
 
+    @RequestLine("GET /api/storage/volumes")
+    @Headers({"Authorization: {authHeader}"})
+    OntapResponse<Volume> getVolume(@Param("authHeader") String authHeader, @QueryMap Map<String, Object> queryMap);
+
     @RequestLine("PATCH /api/storage/volumes/{uuid}")
-    @Headers({"Accept: {acceptHeader}", "Authorization: {authHeader}"})
-    JobResponse updateVolumeRebalancing(@Param("acceptHeader") String acceptHeader, @Param("uuid") String uuid, Volume volumeRequest);
+    @Headers({ "Authorization: {authHeader}"})
+    JobResponse updateVolumeRebalancing(@Param("authHeader") String authHeader, @Param("uuid") String uuid, Volume volumeRequest);
 }
-