Skip to content

<feature>[vm_local_volume_cache]: vm local volume cache imp #3406

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zstack-robot-2 wants to merge 1 commit into
base: feature-5.5.6-local-cache
Choose a base branch
from
Open

<feature>[vm_local_volume_cache]: vm local volume cache imp #3406

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions compute/src/main/java/org/zstack/compute/host/HostBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,8 @@ protected void handleApiMessage(APIMessage msg) {
handle((APIPowerResetHostMsg) msg);
} else if (msg instanceof APIGetHostPowerStatusMsg) {
handle((APIGetHostPowerStatusMsg) msg);
} else if (msg instanceof APIGetBlockDevicesMsg) {
handle((APIGetBlockDevicesMsg) msg);
} else {
bus.dealWithUnknownMessage(msg);
}
Expand All @@ -214,6 +216,26 @@ public void run(MessageReply reply) {
});
}

private void handle(APIGetBlockDevicesMsg msg) {
APIGetBlockDevicesEvent event = new APIGetBlockDevicesEvent(msg.getId());
GetBlockDevicesOnHostMsg gmsg = new GetBlockDevicesOnHostMsg();
gmsg.setHostUuid(msg.getHostUuid());
bus.makeTargetServiceIdByResourceUuid(gmsg, HostConstant.SERVICE_ID, msg.getHostUuid());
bus.send(gmsg, new CloudBusCallBack(msg) {
@Override
public void run(MessageReply reply) {
if (!reply.isSuccess()) {
event.setSuccess(false);
event.setError(reply.getError());
} else {
GetBlockDevicesOnHostReply r = reply.castReply();
event.setBlockDevices(r.getBlockDevices());
}
bus.publish(event);
}
});
}

private void handle(APIPowerResetHostMsg msg) {
final APIPowerResetHostEvent event = new APIPowerResetHostEvent(msg.getId());
RebootHostMsg rebootHostMsg = new RebootHostMsg();
Expand Down
55 changes: 55 additions & 0 deletions compute/src/main/java/org/zstack/compute/vm/InstantiateVmFromNewCreatedStruct.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ public class InstantiateVmFromNewCreatedStruct {
private Map<String, List<String>> dataVolumeSystemTagsOnIndex;
private List<String> disableL3Networks;
private List<String> sshKeyPairUuids;
private Boolean enableRootVolumeCache;
private String cacheMode;
private String rootVolumeCachePoolUuid;
private String rootVolumeCacheMode;
private Map<Integer, APICreateVmInstanceMsg.VolumeCacheConfig> dataDiskCacheConfigOnIndex;
private final List<String> candidatePrimaryStorageUuidsForRootVolume = new ArrayList<>();
private final List<String> candidatePrimaryStorageUuidsForDataVolume = new ArrayList<>();

Expand Down Expand Up @@ -142,6 +147,11 @@ public static InstantiateVmFromNewCreatedStruct fromMessage(InstantiateNewCreate
struct.setDataVolumeSystemTagsOnIndex(msg.getDataVolumeSystemTagsOnIndex());
struct.setDisableL3Networks(msg.getDisableL3Networks());
struct.setDiskAOs(msg.getDiskAOs());
struct.setEnableRootVolumeCache(msg.getEnableRootVolumeCache());
struct.setCacheMode(msg.getCacheMode());
struct.setRootVolumeCachePoolUuid(msg.getRootVolumeCachePoolUuid());
struct.setRootVolumeCacheMode(msg.getRootVolumeCacheMode());
struct.setDataDiskCacheConfigOnIndex(msg.getDataDiskCacheConfigOnIndex());
return struct;
}

Expand All @@ -161,6 +171,11 @@ public static InstantiateVmFromNewCreatedStruct fromMessage(CreateVmInstanceMsg
struct.setDataVolumeSystemTagsOnIndex(msg.getDataVolumeSystemTagsOnIndex());
struct.setDisableL3Networks(msg.getDisableL3Networks());
struct.setDiskAOs(msg.getDiskAOs());
struct.setEnableRootVolumeCache(msg.getEnableRootVolumeCache());
struct.setCacheMode(msg.getCacheMode());
struct.setRootVolumeCachePoolUuid(msg.getRootVolumeCachePoolUuid());
struct.setRootVolumeCacheMode(msg.getRootVolumeCacheMode());
struct.setDataDiskCacheConfigOnIndex(msg.getDataDiskCacheConfigOnIndex());
return struct;
}

Expand Down Expand Up @@ -243,4 +258,44 @@ public List<String> getSshKeyPairUuids() {
public void setSshKeyPairUuids(List<String> sshKeyPairUuids) {
this.sshKeyPairUuids = sshKeyPairUuids;
}

public Boolean getEnableRootVolumeCache() {
return enableRootVolumeCache;
}

public void setEnableRootVolumeCache(Boolean enableRootVolumeCache) {
this.enableRootVolumeCache = enableRootVolumeCache;
}

public String getCacheMode() {
return cacheMode;
}

public void setCacheMode(String cacheMode) {
this.cacheMode = cacheMode;
}

public String getRootVolumeCachePoolUuid() {
return rootVolumeCachePoolUuid;
}

public void setRootVolumeCachePoolUuid(String rootVolumeCachePoolUuid) {
this.rootVolumeCachePoolUuid = rootVolumeCachePoolUuid;
}

public String getRootVolumeCacheMode() {
return rootVolumeCacheMode;
}

public void setRootVolumeCacheMode(String rootVolumeCacheMode) {
this.rootVolumeCacheMode = rootVolumeCacheMode;
}

public Map<Integer, APICreateVmInstanceMsg.VolumeCacheConfig> getDataDiskCacheConfigOnIndex() {
return dataDiskCacheConfigOnIndex;
}

public void setDataDiskCacheConfigOnIndex(Map<Integer, APICreateVmInstanceMsg.VolumeCacheConfig> dataDiskCacheConfigOnIndex) {
this.dataDiskCacheConfigOnIndex = dataDiskCacheConfigOnIndex;
}
}
34 changes: 34 additions & 0 deletions compute/src/main/java/org/zstack/compute/vm/VmAllocateHostFlow.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,43 @@ public String call(L3NetworkInventory arg) {
msg.getRequiredPrimaryStorageUuids().addAll(spec.getDiskAOs().stream()
.map(APICreateVmInstanceMsg.DiskAO::getPrimaryStorageUuid).filter(Objects::nonNull).collect(Collectors.toList()));
}

// Add cache pool requirements as system tags for host allocation filtering
addCacheSystemTags(spec, msg);

return msg;
}

private void addCacheSystemTags(VmInstanceSpec spec, AllocateHostMsg msg) {
boolean enableRoot = Boolean.TRUE.equals(spec.getEnableRootVolumeCache());
boolean enableData = spec.getDataDiskCacheConfigOnIndex() != null
&& !spec.getDataDiskCacheConfigOnIndex().isEmpty();
if (!enableRoot && !enableData) {
return;
}

long totalCacheSize = 0;
if (enableRoot) {
// Root volume size: from image size or root disk offering
ImageInventory image = spec.getImageSpec().getInventory();
if (image != null && image.getSize() != 0) {
totalCacheSize += image.getSize();
} else if (spec.getRootDiskOffering() != null) {
totalCacheSize += spec.getRootDiskOffering().getDiskSize();
}
Comment on lines +155 to +162
Copy link

@coderabbitai coderabbitai bot Mar 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

根卷缓存大小计算缺少 ISO 分支，可能导致主机分配误判。

当前只要 image 存在就优先用 image.getSize()。但 ISO 场景根卷大小应来自 root disk offering（与本文件 prepareMsg 的根盘计算逻辑不一致），否则会把 ISO 体积当作缓存需求，放大 requiredSize 约束。

🔧 建议修改 
 if (enableRoot) {
-    // Root volume size: from image size or root disk offering
-    ImageInventory image = spec.getImageSpec().getInventory();
-    if (image != null && image.getSize() != 0) {
-        totalCacheSize += image.getSize();
-    } else if (spec.getRootDiskOffering() != null) {
-        totalCacheSize += spec.getRootDiskOffering().getDiskSize();
-    }
+    // Root volume size: ISO场景取root disk offering，其它镜像优先取image size
+    ImageInventory image = spec.getImageSpec() == null ? null : spec.getImageSpec().getInventory();
+    boolean nonIsoImage = image != null && !ImageMediaType.ISO.toString().equals(image.getMediaType());
+    if (nonIsoImage && image.getSize() > 0) {
+        totalCacheSize += image.getSize();
+    } else if (spec.getRootDiskOffering() != null) {
+        totalCacheSize += spec.getRootDiskOffering().getDiskSize();
+    }
 }
🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@compute/src/main/java/org/zstack/compute/vm/VmAllocateHostFlow.java` around
lines 154 - 161, The root-cache-size calculation currently uses image.getSize()
whenever an ImageInventory exists, which treats ISOs as having cache size and
mismatches the prepareMsg logic; update the block in VmAllocateHostFlow where
enableRoot is handled so that you only use image.getSize() when the image is not
an ISO and size != 0, otherwise fall back to
spec.getRootDiskOffering().getDiskSize() (i.e., change the condition around
ImageInventory image = spec.getImageSpec().getInventory() to check image media
type/ISO status before using image.getSize()); keep totalCacheSize accumulation
and ensure this logic mirrors the root-size calculation in prepareMsg.

}
if (enableData) {
totalCacheSize += getTotalDataDiskSize(spec);
}

if (totalCacheSize > 0) {
msg.addSystemTag("volumeCache::requiredSize::" + totalCacheSize);
}
if (spec.getRootVolumeCachePoolUuid() != null) {
msg.addSystemTag("volumeCache::poolUuid::" + spec.getRootVolumeCachePoolUuid());
}
}

@Override
public void run(final FlowTrigger chain, Map data) {
taskProgress("allocate candidate hosts");
Expand Down
5 changes: 5 additions & 0 deletions compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7355,6 +7355,11 @@ public DiskOfferingVO call(DiskOfferingVO arg) {
}

spec.setDiskAOs(struct.getDiskAOs());
spec.setEnableRootVolumeCache(struct.getEnableRootVolumeCache());
spec.setCacheMode(struct.getCacheMode());
spec.setRootVolumeCachePoolUuid(struct.getRootVolumeCachePoolUuid());
spec.setRootVolumeCacheMode(struct.getRootVolumeCacheMode());
spec.setDataDiskCacheConfigOnIndex(struct.getDataDiskCacheConfigOnIndex());

List<CdRomSpec> cdRomSpecs = buildVmCdRomSpecsForNewCreated(spec);
spec.setCdRomSpecs(cdRomSpecs);
Expand Down
5 changes: 5 additions & 0 deletions compute/src/main/java/org/zstack/compute/vm/VmInstanceManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1323,6 +1323,11 @@ public void run(FlowTrigger trigger, Map data) {
smsg.setDataVolumeSystemTags(msg.getDataVolumeSystemTags());
smsg.setDataVolumeSystemTagsOnIndex(msg.getDataVolumeSystemTagsOnIndex());
smsg.setDiskAOs(msg.getDiskAOs());
smsg.setEnableRootVolumeCache(msg.getEnableRootVolumeCache());
smsg.setCacheMode(msg.getCacheMode());
smsg.setRootVolumeCachePoolUuid(msg.getRootVolumeCachePoolUuid());
smsg.setRootVolumeCacheMode(msg.getRootVolumeCacheMode());
smsg.setDataDiskCacheConfigOnIndex(msg.getDataDiskCacheConfigOnIndex());
bus.makeTargetServiceIdByResourceUuid(smsg, VmInstanceConstant.SERVICE_ID, finalVo.getUuid());
bus.send(smsg, new CloudBusCallBack(smsg) {
@Override
Expand Down
5 changes: 5 additions & 0 deletions compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,11 @@ public static CreateVmInstanceMsg fromAPICreateVmInstanceMsg(APICreateVmInstance
cmsg.setStrategy(msg.getStrategy());

cmsg.setDiskAOs(msg.getDiskAOs());
cmsg.setEnableRootVolumeCache(msg.getEnableRootVolumeCache());
cmsg.setCacheMode(msg.getCacheMode());
cmsg.setRootVolumeCachePoolUuid(msg.getRootVolumeCachePoolUuid());
cmsg.setRootVolumeCacheMode(msg.getRootVolumeCacheMode());
cmsg.setDataDiskCacheConfigOnIndex(msg.getDataDiskCacheConfigOnIndex());

if (CollectionUtils.isNotEmpty(msg.getDataDiskOfferingUuids()) || CollectionUtils.isNotEmpty(msg.getDataDiskSizes())) {
cmsg.setPrimaryStorageUuidForDataVolume(getPSUuidForDataVolume(msg.getSystemTags()));
Expand Down
34 changes: 34 additions & 0 deletions compute/src/main/java/org/zstack/compute/vm/VmInstantiateOtherDiskFlow.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@
import org.zstack.header.storage.primary.*;
import org.zstack.header.vm.*;
import org.zstack.header.volume.*;
import org.zstack.header.localVolumeCache.EnableVolumeCacheMsg;
import org.zstack.header.localVolumeCache.EnableVolumeCacheReply;
import org.zstack.header.localVolumeCache.VmLocalVolumeCacheConstant;
import org.zstack.identity.AccountManager;
import org.zstack.utils.CollectionUtils;
import org.zstack.utils.Utils;
Expand Down Expand Up @@ -398,6 +401,37 @@ public void run(MessageReply reply) {
});
}
});

// Enable volume cache if requested for this DiskAO
if (Boolean.TRUE.equals(diskAO.getEnableCache())) {
flow(new NoRollbackFlow() {
String __name__ = String.format("enable-cache-for-diskAO-volume-on-vm-%s", vmUuid);

@Override
public void run(final FlowTrigger innerTrigger, Map data) {
if (volumeInventory == null) {
innerTrigger.next();
return;
}
EnableVolumeCacheMsg emsg = new EnableVolumeCacheMsg();
emsg.setVolumeUuid(volumeInventory.getUuid());
emsg.setPoolUuid(diskAO.getCachePoolUuid());
emsg.setCacheMode(diskAO.getCacheMode());
bus.makeLocalServiceId(emsg, VmLocalVolumeCacheConstant.CACHE_SERVICE_ID);
bus.send(emsg, new CloudBusCallBack(innerTrigger) {
@Override
public void run(MessageReply reply) {
if (!reply.isSuccess()) {
logger.warn(String.format("failed to enable cache for DiskAO volume[uuid:%s]: %s",
volumeInventory.getUuid(), reply.getError()));
}
// Don't fail the whole flow if cache enablement fails
innerTrigger.next();
}
});
}
});
}
}

private void setupAttachOtherDiskFlows() {
Expand Down
35 changes: 35 additions & 0 deletions conf/db/upgrade/V5.5.12__schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
CREATE TABLE IF NOT EXISTS `VmLocalVolumeCachePoolVO` (
`uuid` VARCHAR(32) NOT NULL,
`hostUuid` VARCHAR(32) NOT NULL,
`name` VARCHAR(255) DEFAULT NULL,
`description` VARCHAR(2048) DEFAULT NULL,
`metadata` VARCHAR(2048) DEFAULT NULL,
`totalCapacity` BIGINT NOT NULL DEFAULT 0,
`availableCapacity` BIGINT NOT NULL DEFAULT 0,
`state` VARCHAR(32) NOT NULL,
`status` VARCHAR(32) NOT NULL,
`createDate` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
`lastOpDate` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`uuid`),
CONSTRAINT `fkVmLocalVolumeCachePoolVOHostEO`
FOREIGN KEY (`hostUuid`) REFERENCES `HostEO` (`uuid`)
ON DELETE CASCADE
) ENGINE = InnoDB DEFAULT CHARSET = utf8;

CREATE TABLE IF NOT EXISTS `VmLocalVolumeCacheVO` (
`uuid` VARCHAR(32) NOT NULL,
`volumeUuid` VARCHAR(32) NOT NULL,
`poolUuid` VARCHAR(32) DEFAULT NULL,
`cacheMode` VARCHAR(32) NOT NULL,
`state` VARCHAR(32) NOT NULL,
`createDate` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
`lastOpDate` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`uuid`),
UNIQUE KEY `uniVmLocalVolumeCacheVOVolumeUuid` (`volumeUuid`),
CONSTRAINT `fkVmLocalVolumeCacheVOVolumeEO`
FOREIGN KEY (`volumeUuid`) REFERENCES `VolumeEO` (`uuid`)
ON DELETE CASCADE,
CONSTRAINT `fkVmLocalVolumeCacheVOPoolUuid`
FOREIGN KEY (`poolUuid`) REFERENCES `VmLocalVolumeCachePoolVO` (`uuid`)
ON DELETE SET NULL
) ENGINE = InnoDB DEFAULT CHARSET = utf8;
Comment on lines +19 to +35
Copy link

@coderabbitai coderabbitai bot Mar 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

缺少 installPath 列会导致表结构与 Java VO 不一致。

当前 VmLocalVolumeCacheVO 未包含 installPath，这会在实体映射/运行期触发错误（尤其在严格校验环境下）。建议在本次升级脚本中补齐该列。

🛠 建议修复 
 CREATE TABLE IF NOT EXISTS `VmLocalVolumeCacheVO` (
     `uuid`        VARCHAR(32)   NOT NULL,
     `volumeUuid`  VARCHAR(32)   NOT NULL,
     `poolUuid`    VARCHAR(32)   DEFAULT NULL,
+    `installPath` VARCHAR(2048) DEFAULT NULL,
     `cacheMode`   VARCHAR(32)   NOT NULL,
     `state`       VARCHAR(32)   NOT NULL,
     `createDate`  TIMESTAMP     NOT NULL DEFAULT CURRENT_TIMESTAMP,
     `lastOpDate`  TIMESTAMP     NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@conf/db/upgrade/V5.5.12__schema.sql` around lines 19 - 35, The CREATE TABLE
for VmLocalVolumeCacheVO is missing the installPath column required by the Java
VO; update the schema in V5.5.12__schema.sql to add an installPath column (e.g.
a VARCHAR with the same length/nullability as the VO expects) to the
VmLocalVolumeCacheVO definition and ensure any ALTER/upgrade path adds the same
installPath column for existing DBs so the table structure matches the Java
class VmLocalVolumeCacheVO and avoids mapping/runtime errors.

24 changes: 24 additions & 0 deletions conf/globalConfig/kvm.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,4 +273,28 @@
<defaultValue>10737418240</defaultValue>
<type>java.lang.Long</type>
</config>

<config>
<category>kvm</category>
<name>localVolumeCache.pool.capacitySync.interval</name>
<description>Interval in seconds between periodic capacity sync tasks for local volume cache pools.</description>
<defaultValue>300</defaultValue>
<type>java.lang.Long</type>
</config>

<config>
<category>kvm</category>
<name>localVolumeCache.pool.healthCheck.interval</name>
<description>Interval in seconds between periodic health check tasks for local volume cache pools.</description>
<defaultValue>60</defaultValue>
<type>java.lang.Long</type>
</config>

<config>
<category>kvm</category>
<name>localVolumeCache.pool.gc.interval</name>
<description>Interval in seconds between periodic garbage collection tasks for local volume cache pools.</description>
<defaultValue>3600</defaultValue>
<type>java.lang.Long</type>
</config>
</globalConfig>
2 changes: 2 additions & 0 deletions conf/persistence.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,8 @@
<class>org.zstack.network.hostNetworkInterface.lldp.entity.HostNetworkInterfaceLldpRefVO</class>
<class>org.zstack.header.volume.block.BlockVolumeVO</class>
<class>org.zstack.header.host.HostHwMonitorStatusVO</class>
<class>org.zstack.header.localVolumeCache.VmLocalVolumeCachePoolVO</class>
<class>org.zstack.header.localVolumeCache.VmLocalVolumeCacheVO</class>
<class>org.zstack.kvm.xmlhook.XmlHookVO</class>
<class>org.zstack.kvm.xmlhook.XmlHookVmInstanceRefVO</class>
<class>org.zstack.log.server.LogServerVO</class>
Expand Down
12 changes: 12 additions & 0 deletions conf/serviceConfig/vmLocalVolumeCache.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<service xmlns="http://zstack.org/schema/zstack">
<id>vmLocalVolumeCache</id>

<message>
<name>org.zstack.header.localVolumeCache.APIEnableVolumeCacheMsg</name>
</message>

<message>
<name>org.zstack.header.localVolumeCache.APIDisableVolumeCacheMsg</name>
</message>
</service>
37 changes: 37 additions & 0 deletions conf/serviceConfig/vmLocalVolumeCachePool.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
<?xml version="1.0" encoding="UTF-8"?>
<service xmlns="http://zstack.org/schema/zstack">
<id>vmLocalVolumeCachePool</id>

<message>
<name>org.zstack.header.localVolumeCache.APICreateVmLocalVolumeCachePoolMsg</name>
</message>

<message>
<name>org.zstack.header.localVolumeCache.APIDeleteVmLocalVolumeCachePoolMsg</name>
</message>

<message>
<name>org.zstack.header.localVolumeCache.APIUpdateVmLocalVolumeCachePoolMsg</name>
</message>

<message>
<name>org.zstack.header.localVolumeCache.APIChangeVmLocalVolumeCachePoolStateMsg</name>
</message>

<message>
<name>org.zstack.header.localVolumeCache.APIReconnectVmLocalVolumeCachePoolMsg</name>
</message>

<message>
<name>org.zstack.header.localVolumeCache.APIExtendVmLocalVolumeCachePoolMsg</name>
</message>

<message>
<name>org.zstack.header.localVolumeCache.APISyncVmLocalVolumeCachePoolCapacityMsg</name>
</message>

<message>
<name>org.zstack.header.localVolumeCache.APIQueryVmLocalVolumeCachePoolMsg</name>
<serviceId>query</serviceId>
</message>
</service>
Loading