Merge branch '4.22' into forwarded-ip-accesslog

Author: abh1sar

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -55,6 +55,9 @@ public interface VmDetailConstants {
     String NIC_MULTIQUEUE_NUMBER = "nic.multiqueue.number";
     String NIC_PACKED_VIRTQUEUES_ENABLED = "nic.packed.virtqueues.enabled";
 
+    // KVM specific, disk controllers
+    String KVM_SKIP_FORCE_DISK_CONTROLLER = "skip.force.disk.controller";
+
     // Mac OSX guest specific (internal)
     String SMC_PRESENT = "smc.present";
     String FIRMWARE = "firmware";

api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java

@@ -26,7 +26,6 @@
 import org.apache.cloudstack.api.BaseCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.ServerApiException;
-import org.apache.cloudstack.api.response.BackupResponse;
 import org.apache.cloudstack.api.response.BackupScheduleResponse;
 import org.apache.cloudstack.api.response.UserVmResponse;
 import org.apache.cloudstack.backup.BackupManager;
@@ -38,7 +37,7 @@
 
 @APICommand(name = "createBackupSchedule",
         description = "Creates a User-defined Instance backup schedule",
-        responseObject = BackupResponse.class, since = "4.14.0",
+        responseObject = BackupScheduleResponse.class, since = "4.14.0",
         authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User})
 public class CreateBackupScheduleCmd extends BaseCmd {
 

api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java

@@ -17,6 +17,7 @@
 
 package org.apache.cloudstack.api.command.user.backup.repository;
 
+import com.cloud.utils.StringUtils;
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
@@ -100,7 +101,7 @@ public String getProvider() {
     }
 
     public String getMountOptions() {
-        return mountOptions == null ? "" : mountOptions;
+        return StringUtils.isBlank(mountOptions) ? "" : mountOptions;
     }
 
     public Long getZoneId() {

api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java

@@ -57,6 +57,10 @@ public class BackupRepositoryResponse extends BaseResponse {
     @Param(description = "backup type")
     private String type;
 
+    @SerializedName(ApiConstants.MOUNT_OPTIONS)
+    @Param(description = "mount options", since = "4.22.1")
+    private String mountOptions;
+
     @SerializedName(ApiConstants.CAPACITY_BYTES)
     @Param(description = "capacity of the backup repository")
     private Long capacityBytes;
@@ -128,6 +132,14 @@ public void setType(String type) {
         this.type = type;
     }
 
+    public String getMountOptions() {
+        return mountOptions;
+    }
+
+    public void setMountOptions(String mountOptions) {
+        this.mountOptions = mountOptions;
+    }
+
     public Long getCapacityBytes() {
         return capacityBytes;
     }

api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java

@@ -32,5 +32,4 @@ public interface BackupRepositoryService {
     BackupRepository updateBackupRepository(UpdateBackupRepositoryCmd cmd);
     boolean deleteBackupRepository(DeleteBackupRepositoryCmd cmd);
     Pair<List<BackupRepository>, Integer> listBackupRepositories(ListBackupRepositoriesCmd cmd);
-
 }

core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java

@@ -151,7 +151,7 @@ private void checkCredentials(String user, String password) {
                 client.getParams().setAuthenticationPreemptive(true);
                 Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
                 client.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
-                logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second());
+                logger.info("Added username={}, password=****** for host {}:{}", user, hostAndPort.first(), hostAndPort.second());
             } else {
                 logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second());
             }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectInDataStoreStateMachine.java

@@ -35,9 +35,9 @@ enum State {
         Failed("Failed to download Template"),
         Hidden("The object is hidden from the user");
 
-        String _description;
+        final String _description;
 
-        private State(String description) {
+        State(String description) {
             _description = description;
         }
 
@@ -50,7 +50,7 @@ enum Event {
         CreateRequested,
         CreateOnlyRequested,
         DestroyRequested,
-        OperationSuccessed,
+        OperationSucceeded,
         OperationFailed,
         CopyRequested,
         CopyingRequested,

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -3053,7 +3053,7 @@ private void orchestrateMigrate(final String vmUuid, final long srcHostId, final
     }
 
     protected void migrate(final VMInstanceVO vm, final long srcHostId, final DeployDestination dest) throws ResourceUnavailableException, ConcurrentOperationException {
-        logger.info("Migrating {} to {}", vm, dest);
+        logger.info("Start preparing migration of the VM: {} to {}", vm, dest);
         final long dstHostId = dest.getHost().getId();
         final Host fromHost = _hostDao.findById(srcHostId);
         if (fromHost == null) {
@@ -3118,9 +3118,11 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
             if (pfma == null || !pfma.getResult()) {
                 final String details = pfma != null ? pfma.getDetails() : "null answer returned";
                 final String msg = "Unable to prepare for migration due to " + details;
+                logger.error("Failed to prepare destination host {} for migration of VM {} : {}", dstHostId, vm.getInstanceName(), details);
                 pfma = null;
                 throw new AgentUnavailableException(msg, dstHostId);
             }
+            logger.debug("Successfully prepared destination host {} for migration of VM {} ", dstHostId, vm.getInstanceName());
         } catch (final OperationTimedoutException e1) {
             throw new AgentUnavailableException("Operation timed out", dstHostId);
         } finally {
@@ -3141,18 +3143,23 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
                     volumeMgr.release(vm.getId(), dstHostId);
                 }
 
-                logger.info("Migration cancelled because state has changed: {}", vm);
-                throw new ConcurrentOperationException("Migration cancelled because state has changed: " + vm);
+                String msg = "Migration cancelled because state has changed: " + vm;
+                logger.warn(msg);
+                throw new ConcurrentOperationException(msg);
             }
         } catch (final NoTransitionException e1) {
             _networkMgr.rollbackNicForMigration(vmSrc, profile);
             volumeMgr.release(vm.getId(), dstHostId);
-            logger.info("Migration cancelled because {}", e1.getMessage());
+            String msg = String.format("Migration cancelled for VM %s due to state transition failure: %s",
+                    vm.getInstanceName(), e1.getMessage());
+            logger.warn(msg, e1);
             throw new ConcurrentOperationException("Migration cancelled because " + e1.getMessage());
         } catch (final CloudRuntimeException e2) {
             _networkMgr.rollbackNicForMigration(vmSrc, profile);
             volumeMgr.release(vm.getId(), dstHostId);
-            logger.info("Migration cancelled because {}", e2.getMessage());
+            String msg = String.format("Migration cancelled for VM %s due to runtime exception: %s",
+                    vm.getInstanceName(), e2.getMessage());
+            logger.error(msg, e2);
             work.setStep(Step.Done);
             _workDao.update(work.getId(), work);
             try {
@@ -3172,8 +3179,12 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
                 final Answer ma = _agentMgr.send(vm.getLastHostId(), mc);
                 if (ma == null || !ma.getResult()) {
                     final String details = ma != null ? ma.getDetails() : "null answer returned";
+                    String msg = String.format("Migration command failed for VM %s on source host id=%s to destination host %s: %s",
+                            vm.getInstanceName(), vm.getLastHostId(), dstHostId, details);
+                    logger.error(msg);
                     throw new CloudRuntimeException(details);
                 }
+                logger.info("Migration command successful for VM {}", vm.getInstanceName());
             } catch (final OperationTimedoutException e) {
                 boolean success = false;
                 if (HypervisorType.KVM.equals(vm.getHypervisorType())) {
@@ -3210,7 +3221,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
 
             try {
                 if (!checkVmOnHost(vm, dstHostId)) {
-                    logger.error("Unable to complete migration for {}", vm);
+                    logger.error("Migration verification failed for VM {} : VM not found on destination host {} ", vm.getInstanceName(), dstHostId);
                     try {
                         _agentMgr.send(srcHostId, new Commands(cleanup(vm, dpdkInterfaceMapping)), null);
                     } catch (final AgentUnavailableException e) {
@@ -3225,7 +3236,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
             migrated = true;
         } finally {
             if (!migrated) {
-                logger.info("Migration was unsuccessful.  Cleaning up: {}", vm);
+                logger.info("Migration was unsuccessful. Cleaning up: {}", vm);
                 _networkMgr.rollbackNicForMigration(vmSrc, profile);
                 volumeMgr.release(vm.getId(), dstHostId);
                 // deallocate GPU devices for the VM on the destination host
@@ -3237,7 +3248,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
                 try {
                     _agentMgr.send(dstHostId, new Commands(cleanup(vm, dpdkInterfaceMapping)), null);
                 } catch (final AgentUnavailableException ae) {
-                    logger.warn("Looks like the destination Host is unavailable for cleanup", ae);
+                    logger.warn("Destination host {} unavailable for cleanup after failed migration of VM {}", dstHostId, vm.getInstanceName(), ae);
                 }
                 _networkMgr.setHypervisorHostname(profile, dest, false);
                 try {
@@ -3246,6 +3257,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
                     logger.warn(e.getMessage());
                 }
             } else {
+                logger.info("Migration completed successfully for VM %s" + vm);
                 _networkMgr.commitNicForMigration(vmSrc, profile);
                 volumeMgr.release(vm.getId(), srcHostId);
                 // deallocate GPU devices for the VM on the src host after migration is complete
@@ -3276,6 +3288,7 @@ protected MigrateCommand buildMigrateCommand(VMInstanceVO vmInstance, VirtualMac
             migrateCommand.setVlanToPersistenceMap(vlanToPersistenceMap);
         }
 
+        logger.debug("Setting auto convergence to: {}", StorageManager.KvmAutoConvergence.value());
         migrateCommand.setAutoConvergence(StorageManager.KvmAutoConvergence.value());
         migrateCommand.setHostGuid(destination.getHost().getGuid());
 

engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java

@@ -213,6 +213,8 @@ public class CapacityDaoImpl extends GenericDaoBase<CapacityVO, Long> implements
 
     private static final String LEFT_JOIN_VM_TEMPLATE = "LEFT JOIN vm_template ON vm_template.id = vi.vm_template_id ";
 
+    private static final String STORAGE_POOLS_WITH_CHILDREN = "SELECT DISTINCT parent FROM storage_pool WHERE parent != 0 AND removed IS NULL";
+
     public CapacityDaoImpl() {
         _hostIdTypeSearch = createSearchBuilder();
         _hostIdTypeSearch.and("hostId", _hostIdTypeSearch.entity().getHostOrPoolId(), SearchCriteria.Op.EQ);
@@ -379,6 +381,11 @@ public List<SummedCapacity> listCapacitiesGroupedByLevelAndType(Integer capacity
             finalQuery.append(" AND capacity_type = ?");
             resourceIdList.add(capacityType.longValue());
         }
+
+        // Exclude storage pools with children from capacity calculations to avoid double counting
+        finalQuery.append(" AND NOT (capacity.capacity_type = ").append(Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED)
+                .append(" AND capacity.host_id IN (").append(STORAGE_POOLS_WITH_CHILDREN).append("))");
+
         if (CollectionUtils.isNotEmpty(hostIds)) {
             finalQuery.append(String.format(" AND capacity.host_id IN (%s)", StringUtils.join(hostIds, ",")));
             if (capacityType == null) {
@@ -541,6 +548,10 @@ public List<SummedCapacity> findFilteredCapacityBy(Integer capacityType, Long zo
         StringBuilder sql = new StringBuilder(LIST_CAPACITY_GROUP_BY_CAPACITY_PART1);
         List<Long> resourceIdList = new ArrayList<Long>();
 
+        // Exclude storage pools with children from capacity calculations to avoid double counting
+        sql.append(" AND NOT (capacity.capacity_type = ").append(Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED)
+                .append(" AND capacity.host_id IN (").append(STORAGE_POOLS_WITH_CHILDREN).append("))");
+
         if (zoneId != null) {
             sql.append(" AND capacity.data_center_id = ?");
             resourceIdList.add(zoneId);

engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java

@@ -163,5 +163,7 @@ public interface VolumeDao extends GenericDao<VolumeVO, Long>, StateDao<Volume.S
 
     VolumeVO findOneByIScsiName(String iScsiName);
 
+    int getVolumeCountByOfferingId(long diskOfferingId);
+
     VolumeVO findByLastIdAndState(long lastVolumeId, Volume.State...states);
 }