Skip to content

Commit 2ef5d1f

Browse files
taogan21taogan21
committed
<feature>[vm-metadata]: data model, DTO and serialization
Resolves: ZSV-10000
1 parent e3362fe commit 2ef5d1f

File tree

100 files changed

+5172
-85
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

100 files changed

+5172
-85
lines changed

‎compute/src/main/java/org/zstack/compute/vm/AbstractVmInstance.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,12 @@ public abstract class AbstractVmInstance implements VmInstance {
205205
APIDestroyVmInstanceMsg.class.getName(),
206206
DestroyVmInstanceMsg.class.getName());
207207

208+
// Registering state: only metadata-related reads, destroy (for cleanup/rollback),
209+
// and ChangeVmMetaDataMsg (for state transitions during registration) are allowed.
210+
allowedOperations.addState(VmInstanceState.Registering,
211+
ChangeVmMetaDataMsg.class.getName(),
212+
APIDestroyVmInstanceMsg.class.getName(),
213+
DestroyVmInstanceMsg.class.getName());
208214

209215
stateChangeChecker.addState(VmInstanceStateEvent.unknown.toString(),
210216
VmInstanceState.Created.toString(),

‎compute/src/main/java/org/zstack/compute/vm/VmExpungeMetadataFlow.java‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
package org.zstack.compute.vm;
2+
3+
import org.springframework.beans.factory.annotation.Autowire;
4+
import org.springframework.beans.factory.annotation.Autowired;
5+
import org.springframework.beans.factory.annotation.Configurable;
6+
import org.zstack.core.cloudbus.CloudBus;
7+
import org.zstack.core.cloudbus.CloudBusCallBack;
8+
import org.zstack.core.db.Q;
9+
import org.zstack.header.core.workflow.FlowTrigger;
10+
import org.zstack.header.core.workflow.NoRollbackFlow;
11+
import org.zstack.header.message.MessageReply;
12+
import org.zstack.header.storage.primary.CleanupVmInstanceMetadataOnPrimaryStorageMsg;
13+
import org.zstack.header.storage.primary.PrimaryStorageConstant;
14+
import org.zstack.header.vm.VmInstanceConstant;
15+
import org.zstack.header.vm.VmInstanceSpec;
16+
import org.zstack.header.volume.VolumeVO;
17+
import org.zstack.header.volume.VolumeVO_;
18+
import org.zstack.utils.Utils;
19+
import org.zstack.utils.logging.CLogger;
20+
21+
import java.util.Map;
22+
23+
@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE)
24+
public class VmExpungeMetadataFlow extends NoRollbackFlow {
25+
private static final CLogger logger = Utils.getLogger(VmExpungeMetadataFlow.class);
26+
27+
@Autowired
28+
private CloudBus bus;
29+
30+
@Override
31+
public void run(FlowTrigger trigger, Map data) {
32+
final VmInstanceSpec spec = (VmInstanceSpec) data.get(VmInstanceConstant.Params.VmInstanceSpec.toString());
33+
final String vmUuid = spec.getVmInventory().getUuid();
34+
35+
if (!VmGlobalConfig.VM_METADATA.value(Boolean.class)) {
36+
trigger.next();
37+
return;
38+
}
39+
40+
String rootVolumeUuid = spec.getVmInventory().getRootVolumeUuid();
41+
if (rootVolumeUuid == null) {
42+
logger.debug(String.format("[MetadataExpunge] vm[uuid:%s] has no root volume, skipping metadata cleanup", vmUuid));
43+
trigger.next();
44+
return;
45+
}
46+
47+
String psUuid = Q.New(VolumeVO.class)
48+
.eq(VolumeVO_.uuid, rootVolumeUuid)
49+
.select(VolumeVO_.primaryStorageUuid)
50+
.findValue();
51+
52+
if (psUuid == null) {
53+
logger.debug(String.format("[MetadataExpunge] vm[uuid:%s] root volume[uuid:%s] has no primaryStorageUuid, " +
54+
"skipping metadata cleanup", vmUuid, rootVolumeUuid));
55+
trigger.next();
56+
return;
57+
}
58+
59+
CleanupVmInstanceMetadataOnPrimaryStorageMsg cmsg = new CleanupVmInstanceMetadataOnPrimaryStorageMsg();
60+
cmsg.setPrimaryStorageUuid(psUuid);
61+
cmsg.setVmUuid(vmUuid);
62+
bus.makeTargetServiceIdByResourceUuid(cmsg, PrimaryStorageConstant.SERVICE_ID, psUuid);
63+
64+
bus.send(cmsg, new CloudBusCallBack(trigger) {
65+
@Override
66+
public void run(MessageReply reply) {
67+
if (reply.isSuccess()) {
68+
logger.info(String.format("[MetadataExpunge] successfully deleted metadata for vm[uuid:%s] on ps[uuid:%s]",
69+
vmUuid, psUuid));
70+
} else {
71+
// best-effort: do not fail the expunge flow, MetadataStorageOrphanDetector will clean up later
72+
logger.warn(String.format("[MetadataExpunge] failed to delete metadata for vm[uuid:%s] on ps[uuid:%s]: %s",
73+
vmUuid, psUuid, reply.getError()));
74+
}
75+
trigger.next();
76+
}
77+
});
78+
}
79+
}

‎compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,4 +133,90 @@ public class VmGlobalConfig {
133133
@GlobalConfigValidation(validValues = {"None", "AuthenticAMD"})
134134
@BindResourceConfig(value = {VmInstanceVO.class})
135135
public static GlobalConfig VM_CPUID_VENDOR = new GlobalConfig(CATEGORY, "vm.cpuid.vendor");
136+
137+
138+
139+
140+
@GlobalConfigValidation(validValues = {"true", "false"})
141+
public static GlobalConfig VM_METADATA = new GlobalConfig(CATEGORY, "vm.metadata");
142+
143+
@GlobalConfigValidation(numberGreaterThan = 0)
144+
public static GlobalConfig VM_METADATA_PS_MAX_CONCURRENT = new GlobalConfig(CATEGORY, "vm.metadata.ps.maxConcurrent");
145+
146+
@GlobalConfigValidation(numberGreaterThan = 0)
147+
public static GlobalConfig VM_METADATA_GLOBAL_MAX_CONCURRENT = new GlobalConfig(CATEGORY, "vm.metadata.global.maxConcurrent");
148+
149+
@GlobalConfigValidation(numberGreaterThan = 0)
150+
public static GlobalConfig VM_METADATA_GC_INITIAL_DELAY_SEC = new GlobalConfig(CATEGORY, "vm.metadata.gc.initialDelaySec");
151+
152+
@GlobalConfigValidation(numberGreaterThan = 0)
153+
public static GlobalConfig VM_METADATA_MAX_RETRY = new GlobalConfig(CATEGORY, "vm.metadata.maxRetry");
154+
155+
@GlobalConfigValidation(numberGreaterThan = 0)
156+
public static GlobalConfig VM_METADATA_DIRTY_POLL_INTERVAL = new GlobalConfig(CATEGORY, "vm.metadata.dirty.pollIntervalSec");
157+
158+
@GlobalConfigValidation(numberGreaterThan = 0)
159+
public static GlobalConfig VM_METADATA_DIRTY_BATCH_SIZE = new GlobalConfig(CATEGORY, "vm.metadata.dirty.batchSize");
160+
161+
@GlobalConfigValidation(numberGreaterThan = 0)
162+
public static GlobalConfig VM_METADATA_UPGRADE_REFRESH_DELAY = new GlobalConfig(CATEGORY, "vm.metadata.upgrade.refreshDelaySec");
163+
164+
@GlobalConfigValidation(numberGreaterThan = 0)
165+
public static GlobalConfig VM_METADATA_UPGRADE_REFRESH_BATCH_SIZE = new GlobalConfig(CATEGORY, "vm.metadata.upgrade.refreshBatchSize");
166+
167+
@GlobalConfigValidation(numberGreaterThan = 0)
168+
public static GlobalConfig VM_METADATA_NODE_LEFT_DELAY = new GlobalConfig(CATEGORY, "vm.metadata.nodeLeft.delaySec");
169+
170+
@GlobalConfigValidation(numberGreaterThan = 0)
171+
public static GlobalConfig VM_METADATA_STALE_RECOVERY_INTERVAL = new GlobalConfig(CATEGORY, "vm.metadata.staleRecovery.intervalSec");
172+
173+
@GlobalConfigValidation(numberGreaterThan = 0)
174+
public static GlobalConfig VM_METADATA_STALE_RECOVERY_BATCH_SIZE = new GlobalConfig(CATEGORY, "vm.metadata.staleRecovery.batchSize");
175+
176+
@GlobalConfigValidation(numberGreaterThan = 0)
177+
public static GlobalConfig VM_METADATA_STALE_RECOVERY_MAX_CYCLES = new GlobalConfig(CATEGORY, "vm.metadata.staleRecovery.maxCycles");
178+
179+
@GlobalConfigValidation(numberGreaterThan = 0)
180+
public static GlobalConfig VM_METADATA_PENDING_API_TIMEOUT = new GlobalConfig(CATEGORY, "vm.metadata.pendingApi.timeoutMinutes");
181+
182+
@GlobalConfigValidation(numberGreaterThan = 0)
183+
public static GlobalConfig VM_METADATA_RETRY_BASE_DELAY = new GlobalConfig(CATEGORY, "vm.metadata.retry.baseDelaySeconds");
184+
185+
@GlobalConfigValidation(numberGreaterThan = 0)
186+
public static GlobalConfig VM_METADATA_RETRY_MAX_EXPONENT = new GlobalConfig(CATEGORY, "vm.metadata.retry.maxExponent");
187+
188+
@GlobalConfigValidation(numberGreaterThan = 0)
189+
public static GlobalConfig VM_METADATA_INIT_BATCH_SIZE = new GlobalConfig(CATEGORY, "vm.metadata.init.batchSize");
190+
191+
@GlobalConfigValidation(numberGreaterThan = 0)
192+
public static GlobalConfig VM_METADATA_INIT_BATCH_DELAY = new GlobalConfig(CATEGORY, "vm.metadata.init.batchDelaySec");
193+
194+
@GlobalConfigValidation(numberGreaterThan = 0)
195+
public static GlobalConfig VM_METADATA_ORPHAN_CHECK_INTERVAL = new GlobalConfig(CATEGORY, "vm.metadata.orphanCheck.intervalSec");
196+
197+
@GlobalConfigValidation(numberGreaterThan = 0)
198+
public static GlobalConfig VM_METADATA_ZOMBIE_CLAIM_THRESHOLD = new GlobalConfig(CATEGORY, "vm.metadata.zombieClaim.thresholdMinutes");
199+
200+
@GlobalConfigValidation(numberGreaterThan = 0)
201+
public static GlobalConfig VM_METADATA_STALE_CLAIM_THRESHOLD = new GlobalConfig(CATEGORY, "vm.metadata.staleClaim.thresholdMinutes");
202+
203+
@GlobalConfigValidation(numberGreaterThan = 0)
204+
public static GlobalConfig VM_METADATA_TRIGGER_FLUSH_STALE = new GlobalConfig(CATEGORY, "vm.metadata.triggerFlush.staleMinutes");
205+
206+
@GlobalConfigValidation(numberGreaterThan = 0)
207+
public static GlobalConfig VM_METADATA_DELETE_MAX_RETRY = new GlobalConfig(CATEGORY, "vm.metadata.delete.maxRetry");
208+
209+
@GlobalConfigValidation(numberGreaterThan = 0)
210+
public static GlobalConfig VM_METADATA_DELETE_BASE_DELAY = new GlobalConfig(CATEGORY, "vm.metadata.delete.baseDelaySec");
211+
212+
public static GlobalConfig VM_METADATA_LAST_REFRESH_VERSION = new GlobalConfig(CATEGORY, "vm.metadata.lastRefreshVersion");
213+
214+
@GlobalConfigValidation(numberGreaterThan = 0)
215+
public static GlobalConfig VM_METADATA_CONTENT_CHECK_INTERVAL = new GlobalConfig(CATEGORY, "vm.metadata.contentCheck.intervalSec");
216+
217+
@GlobalConfigValidation(numberGreaterThan = 0)
218+
public static GlobalConfig VM_METADATA_CONTENT_CHECK_BATCH_SIZE = new GlobalConfig(CATEGORY, "vm.metadata.contentCheck.batchSize");
219+
220+
@GlobalConfigValidation(numberGreaterThan = 0)
221+
public static GlobalConfig VM_METADATA_STORAGE_ORPHAN_CHECK_INTERVAL = new GlobalConfig(CATEGORY, "vm.metadata.storageOrphanCheck.intervalSec");
136222
}

‎compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9370,4 +9370,3 @@ public void run(MessageReply reply) {
93709370
});
93719371
}
93729372
}
9373-

‎compute/src/main/java/org/zstack/compute/vm/VmInstanceManagerImpl.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import org.springframework.transaction.annotation.Transactional;
77
import org.zstack.compute.allocator.HostAllocatorManager;
88
import org.zstack.compute.vm.quota.*;
9+
import org.zstack.header.vm.metadata.VmMetadataDirtyService;
910
import org.zstack.core.Platform;
1011
import org.zstack.core.asyncbatch.While;
1112
import org.zstack.core.cloudbus.*;
@@ -63,6 +64,10 @@
6364
import org.zstack.header.vm.VmInstanceDeletionPolicyManager.VmInstanceDeletionPolicy;
6465
import org.zstack.header.vm.cdrom.VmCdRomVO;
6566
import org.zstack.header.vm.cdrom.VmCdRomVO_;
67+
import org.zstack.header.vm.metadata.APICleanupVmInstanceMetadataEvent;
68+
import org.zstack.header.vm.metadata.APICleanupVmInstanceMetadataMsg;
69+
import org.zstack.header.vm.metadata.APIUpdateVmInstanceMetadataMsg;
70+
import org.zstack.header.vm.metadata.APIUpdateVmInstanceMetadataEvent;
6671
import org.zstack.header.volume.*;
6772
import org.zstack.header.zone.ZoneInventory;
6873
import org.zstack.header.zone.ZoneVO;
@@ -168,6 +173,8 @@ public class VmInstanceManagerImpl extends AbstractService implements
168173
private VmFactoryManager vmFactoryManager;
169174
@Autowired
170175
protected EventFacade evtf;
176+
@Autowired(required = false)
177+
private VmMetadataDirtyService vmMetadataDirtyMarker;
171178

172179
private List<VmInstanceExtensionManager> vmExtensionManagers = new ArrayList<>();
173180
private final static VmConfigSyncHelper vmConfigSyncHelper = new VmConfigSyncHelper();
@@ -239,6 +246,10 @@ private void handleApiMessage(APIMessage msg) {
239246
handle((APIGetSpiceCertificatesMsg) msg);
240247
} else if (msg instanceof APIGetVmsCapabilitiesMsg) {
241248
handle((APIGetVmsCapabilitiesMsg) msg);
249+
} else if (msg instanceof APICleanupVmInstanceMetadataMsg) {
250+
handle((APICleanupVmInstanceMetadataMsg) msg);
251+
} else if (msg instanceof APIUpdateVmInstanceMetadataMsg) {
252+
handle((APIUpdateVmInstanceMetadataMsg) msg);
242253
} else if (msg instanceof VmInstanceMessage) {
243254
passThrough((VmInstanceMessage) msg);
244255
} else {
@@ -2860,4 +2871,67 @@ protected void run(Map tokens, Object data) {
28602871
}
28612872
});
28622873
}
2874+
2875+
private void handle(APICleanupVmInstanceMetadataMsg msg) {
2876+
APICleanupVmInstanceMetadataEvent evt = new APICleanupVmInstanceMetadataEvent(msg.getId());
2877+
2878+
List<String> vmUuids = msg.getVmUuids();
2879+
List<CleanupVmInstanceMetadataOnPrimaryStorageMsg> msgs = new ArrayList<>();
2880+
for (String vmUuid : vmUuids) {
2881+
String psUuid = Q.New(VolumeVO.class)
2882+
.eq(VolumeVO_.vmInstanceUuid, vmUuid)
2883+
.eq(VolumeVO_.type, VolumeType.Root)
2884+
.select(VolumeVO_.primaryStorageUuid)
2885+
.findValue();
2886+
if (psUuid == null) {
2887+
continue;
2888+
}
2889+
2890+
CleanupVmInstanceMetadataOnPrimaryStorageMsg cmsg = new CleanupVmInstanceMetadataOnPrimaryStorageMsg();
2891+
cmsg.setPrimaryStorageUuid(psUuid);
2892+
cmsg.setVmUuid(vmUuid);
2893+
bus.makeTargetServiceIdByResourceUuid(cmsg, PrimaryStorageConstant.SERVICE_ID, psUuid);
2894+
msgs.add(cmsg);
2895+
}
2896+
2897+
if (msgs.isEmpty()) {
2898+
evt.setTotalCleaned(0);
2899+
evt.setTotalFailed(0);
2900+
evt.setFailedVmUuids(new ArrayList<>());
2901+
bus.publish(evt);
2902+
return;
2903+
}
2904+
2905+
bus.send(msgs, new CloudBusListCallBack(msg) {
2906+
@Override
2907+
public void run(List<MessageReply> replies) {
2908+
int totalCleaned = 0;
2909+
int totalFailed = 0;
2910+
List<String> failedVmUuids = new ArrayList<>();
2911+
for (int i = 0; i < replies.size(); i++) {
2912+
MessageReply r = replies.get(i);
2913+
if (r.isSuccess()) {
2914+
totalCleaned++;
2915+
} else {
2916+
totalFailed++;
2917+
failedVmUuids.add(msgs.get(i).getVmUuid());
2918+
}
2919+
}
2920+
evt.setTotalCleaned(totalCleaned);
2921+
evt.setTotalFailed(totalFailed);
2922+
evt.setFailedVmUuids(failedVmUuids);
2923+
bus.publish(evt);
2924+
}
2925+
});
2926+
}
2927+
2928+
private void handle(APIUpdateVmInstanceMetadataMsg msg) {
2929+
APIUpdateVmInstanceMetadataEvent event = new APIUpdateVmInstanceMetadataEvent(msg.getId());
2930+
if (vmMetadataDirtyMarker != null) {
2931+
for (String vmUuid : msg.getVmUuids()) {
2932+
vmMetadataDirtyMarker.markDirty(vmUuid, true);
2933+
}
2934+
}
2935+
bus.publish(event);
2936+
}
28632937
}

‎conf/db/upgrade/V5.0.0__schema.sql‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
CREATE TABLE IF NOT EXISTS `zstack`.`VmMetadataDirtyVO` (
2+
`vmInstanceUuid` VARCHAR(32) NOT NULL,
3+
`managementNodeUuid` VARCHAR(32) DEFAULT NULL,
4+
`dirtyVersion` BIGINT NOT NULL DEFAULT 1,
5+
`lastClaimTime` TIMESTAMP NULL DEFAULT NULL,
6+
`storageStructureChange` TINYINT(1) NOT NULL DEFAULT 0,
7+
`retryCount` INT NOT NULL DEFAULT 0,
8+
`nextRetryTime` TIMESTAMP NULL DEFAULT NULL,
9+
`lastOpDate` timestamp on update CURRENT_TIMESTAMP,
10+
`createDate` timestamp,
11+
PRIMARY KEY (`vmInstanceUuid`),
12+
CONSTRAINT `fkVmMetadataDirtyVOVmInstanceEO` FOREIGN KEY (`vmInstanceUuid`) REFERENCES `VmInstanceEO` (`uuid`) ON DELETE CASCADE,
13+
CONSTRAINT `fkVmMetadataDirtyVOManagementNodeVO` FOREIGN KEY (`managementNodeUuid`) REFERENCES `ManagementNodeVO` (`uuid`) ON DELETE SET NULL
14+
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
15+
16+
CREATE TABLE IF NOT EXISTS `zstack`.`VmMetadataFingerprintVO` (
17+
`vmInstanceUuid` VARCHAR(32) NOT NULL,
18+
`metadataSnapshot` LONGTEXT,
19+
`lastFlushTime` TIMESTAMP NULL DEFAULT NULL,
20+
`lastFlushFailed` TINYINT(1) NOT NULL DEFAULT 0,
21+
`staleRecoveryCount` INT NOT NULL DEFAULT 0,
22+
PRIMARY KEY (`vmInstanceUuid`),
23+
CONSTRAINT `fkVmMetadataFingerprintVOVmInstanceEO` FOREIGN KEY (`vmInstanceUuid`) REFERENCES `VmInstanceEO` (`uuid`) ON DELETE CASCADE
24+
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

0 commit comments

Comments
 (0)