Merge remote-tracking branch 'apache/4.20' into 4.22

Author: shwstppr

plugins/hypervisors/kvm/pom.xml

@@ -67,6 +67,31 @@
             <artifactId>java-linstor</artifactId>
             <version>${cs.java-linstor.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+            <version>${cs.jackson.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+            <version>${cs.jackson.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>${cs.jackson.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-jsr310</artifactId>
+            <version>${cs.jackson.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.module</groupId>
+            <artifactId>jackson-module-jaxb-annotations</artifactId>
+            <version>${cs.jackson.version}</version>
+        </dependency>
         <dependency>
             <groupId>net.java.dev.jna</groupId>
             <artifactId>jna</artifactId>

plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java

@@ -350,7 +350,7 @@ protected AmazonS3 getS3Client(String url, String accessKey, String secretKey) {
                         new AWSStaticCredentialsProvider(
                                 new BasicAWSCredentials(accessKey, secretKey)))
                 .withEndpointConfiguration(
-                        new AwsClientBuilder.EndpointConfiguration(url, null))
+                        new AwsClientBuilder.EndpointConfiguration(url, "us-east-1"))
                 .build();
 
         if (client == null) {

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java

@@ -63,6 +63,8 @@
 import com.cloud.api.storage.LinstorRevertBackupSnapshotCommand;
 import com.cloud.configuration.Config;
 import com.cloud.host.Host;
+import com.cloud.host.HostVO;
+import com.cloud.host.Status;
 import com.cloud.host.dao.HostDao;
 import com.cloud.resource.ResourceState;
 import com.cloud.storage.DataStoreRole;
@@ -922,9 +924,10 @@ private String revertSnapshotFromImageStore(
             _backupsnapshotwait,
             VirtualMachineManager.ExecuteInSequence.value());
 
-        Optional<RemoteHostEndPoint> optEP = getDiskfullEP(linstorApi, rscName);
+        final StoragePool pool = (StoragePool) volumeInfo.getDataStore();
+        Optional<RemoteHostEndPoint> optEP = getDiskfullEP(linstorApi, pool, rscName);
         if (optEP.isEmpty()) {
-            optEP = getLinstorEP(linstorApi, rscName);
+            optEP = getLinstorEP(linstorApi, pool, rscName);
         }
 
         if (optEP.isPresent()) {
@@ -1064,13 +1067,29 @@ public void copyAsync(DataObject srcData, DataObject dstData, AsyncCompletionCal
             Answer answer = copyVolume(srcData, dstData);
             res = new CopyCommandResult(null, answer);
         } else {
-            Answer answer = new Answer(null, false, "noimpl");
-            res = new CopyCommandResult(null, answer);
-            res.setResult("Not implemented yet");
+            throw new CloudRuntimeException("Not implemented for Linstor primary storage.");
         }
         callback.complete(res);
     }
 
+    private Host getEnabledClusterHost(StoragePool storagePool, List<String> linstorNodeNames) {
+        List<HostVO> csHosts;
+        if (storagePool.getClusterId() != null) {
+            csHosts = _hostDao.findByClusterId(storagePool.getClusterId());
+        } else {
+            csHosts = _hostDao.findByDataCenterId(storagePool.getDataCenterId());
+        }
+        Collections.shuffle(csHosts); // so we do not always pick the same host for operations
+        for (HostVO host : csHosts) {
+            if (host.getResourceState() == ResourceState.Enabled &&
+                    host.getStatus() == Status.Up &&
+                    linstorNodeNames.contains(host.getName())) {
+                return host;
+            }
+        }
+        return null;
+    }
+
     /**
      * Tries to get a Linstor cloudstack end point, that is at least diskless.
      *
@@ -1079,47 +1098,37 @@ public void copyAsync(DataObject srcData, DataObject dstData, AsyncCompletionCal
      * @return Optional RemoteHostEndPoint if one could get found.
      * @throws ApiException
      */
-    private Optional<RemoteHostEndPoint> getLinstorEP(DevelopersApi api, String rscName) throws ApiException {
+    private Optional<RemoteHostEndPoint> getLinstorEP(DevelopersApi api, StoragePool storagePool, String rscName)
+            throws ApiException {
         List<String> linstorNodeNames = LinstorUtil.getLinstorNodeNames(api);
-        Collections.shuffle(linstorNodeNames);  // do not always pick the first linstor node
-
-        Host host = null;
-        for (String nodeName : linstorNodeNames) {
-            host = _hostDao.findByName(nodeName);
-            if (host != null && host.getResourceState() == ResourceState.Enabled) {
-                logger.info(String.format("Linstor: Make resource %s available on node %s ...", rscName, nodeName));
-                ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, nodeName, new ResourceMakeAvailable());
-                if (!answers.hasError()) {
-                    break; // found working host
-                } else {
-                    logger.error(
-                        String.format("Linstor: Unable to make resource %s on node %s available: %s",
-                            rscName,
-                            nodeName,
-                            LinstorUtil.getBestErrorMessage(answers)));
-                }
+        Host host = getEnabledClusterHost(storagePool, linstorNodeNames);
+        if (host != null) {
+            logger.info("Linstor: Make resource {} available on node {} ...", rscName, host.getName());
+            ApiCallRcList answers = api.resourceMakeAvailableOnNode(
+                    rscName, host.getName(), new ResourceMakeAvailable());
+            if (answers.hasError()) {
+                logger.error("Linstor: Unable to make resource {} on node {} available: {}",
+                        rscName, host.getName(), LinstorUtil.getBestErrorMessage(answers));
+                return Optional.empty();
+            } else {
+                return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host));
             }
         }
 
-        if (host == null)
-        {
-            logger.error("Linstor: Couldn't create a resource on any cloudstack host.");
-            return Optional.empty();
-        }
-        else
-        {
-            return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host));
-        }
+        logger.error("Linstor: Couldn't create a resource on any cloudstack host.");
+        return Optional.empty();
     }
 
-    private Optional<RemoteHostEndPoint> getDiskfullEP(DevelopersApi api, String rscName) throws ApiException {
+    private Optional<RemoteHostEndPoint> getDiskfullEP(DevelopersApi api, StoragePool storagePool, String rscName)
+            throws ApiException {
         List<com.linbit.linstor.api.model.StoragePool> linSPs = LinstorUtil.getDiskfulStoragePools(api, rscName);
         if (linSPs != null) {
-            for (com.linbit.linstor.api.model.StoragePool sp : linSPs) {
-                Host host = _hostDao.findByName(sp.getNodeName());
-                if (host != null && host.getResourceState() == ResourceState.Enabled) {
-                    return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host));
-                }
+            List<String> linstorNodeNames = linSPs.stream()
+                    .map(com.linbit.linstor.api.model.StoragePool::getNodeName)
+                    .collect(Collectors.toList());
+            Host host = getEnabledClusterHost(storagePool, linstorNodeNames);
+            if (host != null) {
+                return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host));
             }
         }
         logger.error("Linstor: No diskfull host found.");
@@ -1200,12 +1209,12 @@ private Answer copyTemplate(DataObject srcData, DataObject dstData) {
                     VirtualMachineManager.ExecuteInSequence.value());
 
             try {
-                Optional<RemoteHostEndPoint> optEP = getLinstorEP(api, rscName);
+                Optional<RemoteHostEndPoint> optEP = getLinstorEP(api, pool, rscName);
                 if (optEP.isPresent()) {
                     answer = optEP.get().sendMessage(cmd);
                 } else {
-                    answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint.");
                     deleteResourceDefinition(pool, rscName);
+                    throw new CloudRuntimeException("Unable to get matching Linstor endpoint.");
                 }
             } catch (ApiException exc) {
                 logger.error("copy template failed: ", exc);
@@ -1242,12 +1251,12 @@ private Answer copyVolume(DataObject srcData, DataObject dstData) {
         Answer answer;
 
         try {
-            Optional<RemoteHostEndPoint> optEP = getLinstorEP(api, rscName);
+            Optional<RemoteHostEndPoint> optEP = getLinstorEP(api, pool, rscName);
             if (optEP.isPresent()) {
                 answer = optEP.get().sendMessage(cmd);
             }
             else {
-                answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint.");
+                throw new CloudRuntimeException("Unable to get matching Linstor endpoint.");
             }
         } catch (ApiException exc) {
             logger.error("copy volume failed: ", exc);
@@ -1280,14 +1289,14 @@ private Answer copyFromTemporaryResource(
         try {
             String devName = restoreResourceFromSnapshot(api, pool, rscName, snapshotName, restoreName);
 
-            Optional<RemoteHostEndPoint> optEPAny = getLinstorEP(api, restoreName);
+            Optional<RemoteHostEndPoint> optEPAny = getLinstorEP(api, pool, restoreName);
             if (optEPAny.isPresent()) {
                 // patch the src device path to the temporary linstor resource
                 snapshotObject.setPath(devName);
                 origCmd.setSrcTO(snapshotObject.getTO());
                 answer = optEPAny.get().sendMessage(origCmd);
-            } else{
-                answer = new Answer(origCmd, false, "Unable to get matching Linstor endpoint.");
+            } else {
+                throw new CloudRuntimeException("Unable to get matching Linstor endpoint.");
             }
         } finally {
             // delete the temporary resource, noop if already gone
@@ -1349,7 +1358,7 @@ protected Answer copySnapshot(DataObject srcData, DataObject destData) {
                 VirtualMachineManager.ExecuteInSequence.value());
             cmd.setOptions(options);
 
-            Optional<RemoteHostEndPoint> optEP = getDiskfullEP(api, rscName);
+            Optional<RemoteHostEndPoint> optEP = getDiskfullEP(api, pool, rscName);
             Answer answer;
             if (optEP.isPresent()) {
                 answer = optEP.get().sendMessage(cmd);

pom.xml

@@ -188,6 +188,7 @@
         <org.springframework.version>5.3.26</org.springframework.version>
         <cs.ini.version>0.5.4</cs.ini.version>
         <cs.caffeine.version>3.1.7</cs.caffeine.version>
+        <cs.protobuf.version>3.25.5</cs.protobuf.version>
     </properties>
 
     <distributionManagement>
@@ -730,6 +731,17 @@
                 <artifactId>xml-apis</artifactId>
                 <version>2.0.2</version>
             </dependency>
+            <!-- enforced protobuf version here as mysql-connector-java is pulling older version (3.19.3) -->
+            <dependency>
+                <groupId>com.google.protobuf</groupId>
+                <artifactId>protobuf-java</artifactId>
+                <version>${cs.protobuf.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.google.protobuf</groupId>
+                <artifactId>protobuf-java-util</artifactId>
+                <version>${cs.protobuf.version}</version>
+            </dependency>
             <dependency>
                 <groupId>com.linbit.linstor.api</groupId>
                 <artifactId>java-linstor</artifactId>

ui/public/locales/en.json

@@ -403,6 +403,7 @@
 "label.app.name": "CloudStack",
 "label.application.policy.set": "Application Policy Set",
 "label.apply": "Apply",
+"label.apply.to.all": "Apply to all",
 "label.apply.tungsten.firewall.policy": "Apply Firewall Policy",
 "label.apply.tungsten.network.policy": "Apply Network Policy",
 "label.apply.tungsten.tag": "Apply tag",
@@ -4034,6 +4035,7 @@
 "message.vnf.no.credentials": "No credentials found for the VNF appliance.",
 "message.vnf.select.networks": "Please select the relevant network for each VNF NIC.",
 "message.volume.desc": "Volume to use as a ROOT disk",
+"message.volume.pool.apply.to.all": "Selected storage pool will be applied to all existing volumes of the instance.",
 "message.volume.state.allocated": "The volume is allocated but has not been created yet.",
 "message.volume.state.attaching": "The volume is attaching to a volume from Ready state.",
 "message.volume.state.copying": "The volume is being copied from the image store to primary storage, in case it's an uploaded volume.",

ui/src/components/view/InfoCard.vue

@@ -709,7 +709,7 @@
               <div class="resource-detail-item__label">{{ $t('label.storagepool') }}</div>
               <div class="resource-detail-item__details">
                 <database-outlined />
-                <router-link v-if="!isStatic && $router.resolve('/storagepool/' + resource.storageid).matched[0].redirect !== '/exception/404'" :to="{ path: '/storagepool/' + resource.storageid }">{{ resource.storage || resource.storageid }} </router-link>
+                <router-link v-if="!isStatic && $router.resolve('/storagepool/' + encodeURIComponent(resource.storageid)).matched[0].redirect !== '/exception/404'" :to="{ path: '/storagepool/' + encodeURIComponent(resource.storageid) }">{{ resource.storage || resource.storageid }} </router-link>
                 <span v-else>{{ resource.storage || resource.storageid }}</span>
                 <a-tag style="margin-left: 5px;" v-if="resource.storagetype">
                   {{ resource.storagetype }}

ui/src/components/view/InstanceVolumesStoragePoolSelectListView.vue

@@ -206,13 +206,19 @@ export default {
     closeVolumeStoragePoolSelector () {
       this.selectedVolumeForStoragePoolSelection = {}
     },
-    handleVolumeStoragePoolSelection (volumeId, storagePool) {
+    handleVolumeStoragePoolSelection (volumeId, storagePool, applyToAll) {
       for (const volume of this.volumes) {
-        if (volume.id === volumeId) {
+        if (applyToAll) {
           volume.selectedstorageid = storagePool.id
           volume.selectedstoragename = storagePool.name
           volume.selectedstorageclusterid = storagePool.clusterid
-          break
+        } else {
+          if (volume.id === volumeId) {
+            volume.selectedstorageid = storagePool.id
+            volume.selectedstoragename = storagePool.name
+            volume.selectedstorageclusterid = storagePool.clusterid
+            break
+          }
         }
       }
       this.updateVolumeToStoragePoolSelection()

ui/src/components/view/ListView.vue

@@ -161,17 +161,9 @@
             >{{ $t(text.toLowerCase()) }}</router-link>
           </span>
           <span v-else>
-            <router-link
-              :to="{ path: $route.path + '/' + record.id }"
-              v-if="record.id"
-            >{{ text }}</router-link>
-            <router-link
-              :to="{ path: $route.path + '/' + record.name }"
-              v-else
-            >{{ text }}</router-link>
-            <span
-              v-if="['guestnetwork','vpc'].includes($route.path.split('/')[1]) && record.restartrequired && !record.vpcid"
-            >
+            <router-link :to="{ path: $route.path + '/' + encodeURIComponent(record.id) }" v-if="record.id">{{ text }}</router-link>
+            <router-link :to="{ path: $route.path + '/' + record.name }" v-else>{{ text }}</router-link>
+            <span v-if="['guestnetwork','vpc'].includes($route.path.split('/')[1]) && record.restartrequired && !record.vpcid">
               &nbsp;
               <a-tooltip>
                 <template #title>{{ $t('label.restartrequired') }}</template>
@@ -607,10 +599,7 @@
         <span v-else>{{ text }}</span>
       </template>
       <template v-if="column.key === 'storage'">
-        <router-link
-          v-if="record.storageid"
-          :to="{ path: '/storagepool/' + record.storageid }"
-        >{{ text }}</router-link>
+        <router-link v-if="record.storageid" :to="{ path: '/storagepool/' + encodeURIComponent(record.storageid) }">{{ text }}</router-link>
         <span v-else>{{ text }}</span>
       </template>
       <template

ui/src/components/view/VolumeStoragePoolSelectForm.vue

@@ -25,6 +25,15 @@
       :autoAssignAllowed="autoAssignAllowed"
       @select="handleSelect" />
 
+    <a-form-item
+      class="top-spaced">
+      <template #label>
+        <tooltip-label :title="$t('label.apply.to.all')" :tooltip="$t('message.volume.pool.apply.to.all')"/>
+      </template>
+      <a-switch
+        v-model:checked="applyToAll" />
+    </a-form-item>
+
     <a-divider />
 
     <div class="actions">
@@ -36,11 +45,13 @@
 </template>
 
 <script>
+import TooltipLabel from '@/components/widgets/TooltipLabel'
 import StoragePoolSelectView from '@/components/view/StoragePoolSelectView'
 
 export default {
   name: 'VolumeStoragePoolSelectionForm',
   components: {
+    TooltipLabel,
     StoragePoolSelectView
   },
   props: {
@@ -70,7 +81,8 @@ export default {
   },
   data () {
     return {
-      selectedStoragePool: null
+      selectedStoragePool: null,
+      applyToAll: false
     }
   },
   watch: {
@@ -95,7 +107,7 @@ export default {
       }
     },
     submitForm () {
-      this.$emit('select', this.resource.id, this.selectedStoragePool)
+      this.$emit('select', this.resource.id, this.selectedStoragePool, this.applyToAll)
       this.closeModal()
     }
   }

ui/src/components/view/VolumesTab.vue

@@ -41,7 +41,7 @@
         {{ parseFloat(record.size / (1024.0 * 1024.0 * 1024.0)).toFixed(2) }} GB
       </template>
       <template v-if="column.key === 'storage'">
-        <router-link v-if="record.storageid" :to="{ path: '/storagepool/' + record.storageid }">{{ text }}</router-link>
+        <router-link v-if="record.storageid" :to="{ path: '/storagepool/' + encodeURIComponent(record.storageid) }">{{ text }}</router-link>
         <span v-else>{{ text }}</span>
       </template>
     </template>