CLOUDSTACK-10303 : test data to nuage_test_data.py + run all tests against simulator

engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java

@@ -128,14 +128,14 @@ protected EndPoint findEndPointInScope(Scope scope, String sqlBase, Long poolId)
         String sql = sbuilder.toString();
         HostVO host = null;
         TransactionLegacy txn = TransactionLegacy.currentTxn();
-        try(PreparedStatement pstmt = txn.prepareStatement(sql);) {
+        try (PreparedStatement pstmt = txn.prepareStatement(sql)) {
             pstmt.setLong(1, poolId);
             try(ResultSet rs = pstmt.executeQuery();) {
                 while (rs.next()) {
                     long id = rs.getLong(1);
                     host = hostDao.findById(id);
                 }
-            }catch (SQLException e) {
+            } catch (SQLException e) {
                 s_logger.warn("can't find endpoint", e);
             }
         } catch (SQLException e) {

plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockStorageManager.java

@@ -33,6 +33,9 @@
 import com.cloud.agent.api.DeleteStoragePoolCommand;
 import com.cloud.agent.api.GetStorageStatsAnswer;
 import com.cloud.agent.api.GetStorageStatsCommand;
+import com.cloud.agent.api.GetVolumeStatsAnswer;
+import com.cloud.agent.api.GetVolumeStatsCommand;
+import com.cloud.agent.api.HandleConfigDriveIsoCommand;
 import com.cloud.agent.api.ManageSnapshotCommand;
 import com.cloud.agent.api.ModifyStoragePoolCommand;
 import com.cloud.agent.api.SecStorageSetupCommand;
@@ -77,6 +80,8 @@ public interface MockStorageManager extends Manager {
 
     public Answer DownloadProcess(DownloadProgressCommand cmd);
 
+    GetVolumeStatsAnswer getVolumeStats(GetVolumeStatsCommand cmd);
+
     public GetStorageStatsAnswer GetStorageStats(GetStorageStatsCommand cmd);
 
     public Answer ManageSnapshot(ManageSnapshotCommand cmd);
@@ -107,4 +112,5 @@ public interface MockStorageManager extends Manager {
 
     public UploadStatusAnswer getUploadStatus(UploadStatusCommand cmd);
 
+    Answer handleConfigDriveIso(HandleConfigDriveIsoCommand cmd);
 }

plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockStorageManagerImpl.java

@@ -26,6 +26,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
@@ -38,6 +40,7 @@
 import org.apache.cloudstack.storage.command.UploadStatusAnswer;
 import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
 import org.apache.cloudstack.storage.command.UploadStatusCommand;
+import org.apache.cloudstack.storage.to.VolumeObjectTO;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.AttachIsoCommand;
@@ -52,6 +55,9 @@
 import com.cloud.agent.api.DeleteStoragePoolCommand;
 import com.cloud.agent.api.GetStorageStatsAnswer;
 import com.cloud.agent.api.GetStorageStatsCommand;
+import com.cloud.agent.api.GetVolumeStatsAnswer;
+import com.cloud.agent.api.GetVolumeStatsCommand;
+import com.cloud.agent.api.HandleConfigDriveIsoCommand;
 import com.cloud.agent.api.ManageSnapshotAnswer;
 import com.cloud.agent.api.ManageSnapshotCommand;
 import com.cloud.agent.api.ModifyStoragePoolAnswer;
@@ -60,6 +66,7 @@
 import com.cloud.agent.api.SecStorageSetupCommand;
 import com.cloud.agent.api.SecStorageVMSetupCommand;
 import com.cloud.agent.api.StoragePoolInfo;
+import com.cloud.agent.api.VolumeStatsEntry;
 import com.cloud.agent.api.storage.CopyVolumeAnswer;
 import com.cloud.agent.api.storage.CopyVolumeCommand;
 import com.cloud.agent.api.storage.CreateAnswer;
@@ -578,6 +585,37 @@ public DownloadAnswer DownloadProcess(DownloadProgressCommand cmd) {
         }
     }
 
+    @Override
+    public GetVolumeStatsAnswer getVolumeStats(final GetVolumeStatsCommand cmd) {
+        HashMap<String, VolumeStatsEntry> volumeStats =
+                cmd.getVolumeUuids()
+                .stream()
+                .collect(Collectors.toMap(Function.identity(),
+                                          this::getVolumeStat,
+                                          (v1, v2) -> v1, HashMap::new));
+
+        return new GetVolumeStatsAnswer(cmd, "", volumeStats);
+    }
+
+    private VolumeStatsEntry getVolumeStat(final String volumeUuid)  {
+        TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.SIMULATOR_DB);
+
+        try {
+            txn.start();
+            MockVolumeVO volume  = _mockVolumeDao.findByUuid(volumeUuid);
+            txn.commit();
+            return new VolumeStatsEntry(volumeUuid, volume.getSize(), volume.getSize());
+        } catch (Exception ex) {
+            txn.rollback();
+            throw new CloudRuntimeException("Error when finding volume " + volumeUuid, ex);
+        } finally {
+            txn.close();
+            txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB);
+            txn.close();
+        }
+
+    }
+
     @Override
     public GetStorageStatsAnswer GetStorageStats(GetStorageStatsCommand cmd) {
         String uuid = cmd.getStorageId();
@@ -786,9 +824,13 @@ public Answer Delete(DeleteCommand cmd) {
             txn.start();
             MockVolumeVO template = _mockVolumeDao.findByStoragePathAndType(cmd.getData().getPath());
             if (template == null) {
-                return new Answer(cmd, false, "can't find object to delete:" + cmd.getData().getPath());
+                if(!((VolumeObjectTO)cmd.getData()).getName().startsWith("ROOT-")) {
+                    return new Answer(cmd, false, "can't find object to delete:" + cmd.getData()
+                                                                                      .getPath());
+                }
+            } else {
+                _mockVolumeDao.remove(template.getId());
             }
-            _mockVolumeDao.remove(template.getId());
             txn.commit();
         } catch (Exception ex) {
             txn.rollback();
@@ -1228,4 +1270,49 @@ public CopyVolumeAnswer CopyVolume(CopyVolumeCommand cmd) {
     public UploadStatusAnswer getUploadStatus(UploadStatusCommand cmd) {
         return new UploadStatusAnswer(cmd, UploadStatus.COMPLETED);
     }
+
+    @Override public Answer handleConfigDriveIso(HandleConfigDriveIsoCommand cmd) {
+        TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.SIMULATOR_DB);
+        MockSecStorageVO sec;
+        try {
+            txn.start();
+            sec = _mockSecStorageDao.findByUrl(cmd.getDestStore().getUrl());
+            if (sec == null) {
+                return new Answer(cmd, false, "can't find secondary storage");
+            }
+
+            txn.commit();
+        } catch (Exception ex) {
+            txn.rollback();
+            throw new CloudRuntimeException("Error when creating config drive.");
+        } finally {
+            txn.close();
+            txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB);
+            txn.close();
+        }
+
+        MockVolumeVO template = new MockVolumeVO();
+        String uuid = UUID.randomUUID().toString();
+        template.setName(uuid);
+        template.setPath(sec.getMountPoint() + cmd.getIsoFile());
+        template.setPoolId(sec.getId());
+        template.setSize((long)(Math.random() * 200L) + 200L);
+        template.setStatus(Status.DOWNLOADED);
+        template.setType(MockVolumeType.ISO);
+        txn = TransactionLegacy.open(TransactionLegacy.SIMULATOR_DB);
+        try {
+            txn.start();
+            template = _mockVolumeDao.persist(template);
+            txn.commit();
+        } catch (Exception ex) {
+            txn.rollback();
+            throw new CloudRuntimeException("Encountered " + ex.getMessage() + " when persisting config drive " + template.getName(), ex);
+        } finally {
+            txn.close();
+            txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB);
+            txn.close();
+        }
+
+        return new Answer(cmd);
+    }
 }

plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java

@@ -64,6 +64,8 @@
 import com.cloud.agent.api.GetStorageStatsCommand;
 import com.cloud.agent.api.GetVmStatsCommand;
 import com.cloud.agent.api.GetVncPortCommand;
+import com.cloud.agent.api.GetVolumeStatsCommand;
+import com.cloud.agent.api.HandleConfigDriveIsoCommand;
 import com.cloud.agent.api.MaintainCommand;
 import com.cloud.agent.api.ManageSnapshotCommand;
 import com.cloud.agent.api.MigrateCommand;
@@ -206,6 +208,7 @@ public List<Class<?>> getCommands() {
     @DB
     @Override
     public Answer simulate(final Command cmd, final String hostGuid) {
+        s_logger.debug("Simulate command " + cmd);
         Answer answer = null;
         Exception exception = null;
         TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.SIMULATOR_DB);
@@ -363,6 +366,8 @@ public Answer simulate(final Command cmd, final String hostGuid) {
                     answer = _mockStorageMgr.Download((DownloadCommand)cmd);
                 } else if (cmd instanceof GetStorageStatsCommand) {
                     answer = _mockStorageMgr.GetStorageStats((GetStorageStatsCommand)cmd);
+                } else if (cmd instanceof GetVolumeStatsCommand) {
+                    answer = _mockStorageMgr.getVolumeStats((GetVolumeStatsCommand)cmd);
                 } else if (cmd instanceof ManageSnapshotCommand) {
                     answer = _mockStorageMgr.ManageSnapshot((ManageSnapshotCommand)cmd);
                 } else if (cmd instanceof BackupSnapshotCommand) {
@@ -431,8 +436,14 @@ public Answer simulate(final Command cmd, final String hostGuid) {
                     answer = storageHandler.handleStorageCommands((StorageSubSystemCommand)cmd);
                 } else if (cmd instanceof FenceCommand) {
                     answer = _mockVmMgr.fence((FenceCommand)cmd);
-                } else if (cmd instanceof GetRouterAlertsCommand || cmd instanceof VpnUsersCfgCommand || cmd instanceof RemoteAccessVpnCfgCommand || cmd instanceof SetMonitorServiceCommand || cmd instanceof AggregationControlCommand ||
-                        cmd instanceof SecStorageFirewallCfgCommand) {
+                } else if (cmd instanceof HandleConfigDriveIsoCommand) {
+                    answer = _mockStorageMgr.handleConfigDriveIso((HandleConfigDriveIsoCommand)cmd);
+                } else if (cmd instanceof GetRouterAlertsCommand
+                        || cmd instanceof VpnUsersCfgCommand
+                        || cmd instanceof RemoteAccessVpnCfgCommand
+                        || cmd instanceof SetMonitorServiceCommand
+                        || cmd instanceof AggregationControlCommand
+                        || cmd instanceof SecStorageFirewallCfgCommand) {
                     answer = new Answer(cmd);
                 } else {
                     s_logger.error("Simulator does not implement command of type " + cmd.toString());
@@ -447,6 +458,8 @@ public Answer simulate(final Command cmd, final String hostGuid) {
                 }
             }
 
+            s_logger.debug("Finished simulate command " + cmd);
+
             return answer;
         } catch (final Exception e) {
             s_logger.error("Failed execute cmd: ", e);

plugins/hypervisors/simulator/src/com/cloud/resource/SimulatorStorageProcessor.java

@@ -85,7 +85,7 @@ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd)
     public Answer copyTemplateToPrimaryStorage(CopyCommand cmd) {
         TemplateObjectTO template = new TemplateObjectTO();
         template.setPath(UUID.randomUUID().toString());
-        template.setSize(new Long(100));
+        template.setSize(100L);
         template.setFormat(Storage.ImageFormat.RAW);
         return new CopyCmdAnswer(template);
     }

server/src/com/cloud/network/element/ConfigDriveNetworkElement.java

@@ -118,9 +118,9 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
     @Inject
     VolumeOrchestrationService _volumeMgr;
 
-    public final static String CONFIGDRIVEFILENAME = "configdrive.iso";
-    public final static String CONFIGDRIVEDIR= "ConfigDrive";
-    public final static Integer CONFIGDRIVEDISKSEQ= new Integer(4);
+    private final static String CONFIGDRIVEFILENAME = "configdrive.iso";
+    private final static String CONFIGDRIVEDIR = "ConfigDrive";
+    private final static Integer CONFIGDRIVEDISKSEQ = 4;
 
     private boolean canHandle(TrafficType trafficType) {
         return trafficType.equals(TrafficType.Guest);
@@ -320,9 +320,10 @@ private boolean updateConfigDriveIso(Network network, VirtualMachineProfile prof
         s_logger.debug(String.format("%s config drive ISO for  vm %s in host %s",
                 (update?"update":"create"), profile.getInstanceName(), _hostDao.findById(hostId).getName()));
         EndPoint endpoint = _ep.select(secondaryStore);
-        if (endpoint == null )
-            throw new ResourceUnavailableException(String.format("%s failed, secondary store not available",
-                    (update?"Update":"Create")),secondaryStore.getClass(),secondaryStore.getId());
+        if (endpoint == null) {
+            throw new ResourceUnavailableException(String.format("%s failed, secondary store not available", (update ? "Update" : "Create")), secondaryStore.getClass(),
+                                                   secondaryStore.getId());
+        }
         String isoPath = CONFIGDRIVEDIR + "/" + profile.getInstanceName() + "/"  + CONFIGDRIVEFILENAME;
         HandleConfigDriveIsoCommand configDriveIsoCommand = new HandleConfigDriveIsoCommand(profile.getVmData(),
                 profile.getConfigDriveLabel(), secondaryStore.getTO(), isoPath, true, update);

test/integration/plugins/nuagevsp/libVSD/__init__.py

@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from .client import ApiClient
+from .helpers import VSDHelpers
+
+__version__ = "1.0"
+__all__ = ['ApiClient', 'VSDHelpers']