diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 00383c249f88..20397c1f2fcc 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -73,6 +73,7 @@ import com.cloud.storage.Volume; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotDetailsDao; import com.cloud.storage.dao.SnapshotDetailsVO; @@ -80,6 +81,7 @@ import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.user.ResourceLimitService; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; @@ -133,6 +135,8 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver @Inject private HostDao _hostDao; @Inject private VMTemplateDao _vmTemplateDao; + @Inject private DiskOfferingDao _diskOfferingDao; + @Inject private ResourceLimitService _resourceLimitMgr; private final Map volumeStatsLastUpdate = new HashMap<>(); private final Map> volumeStats = new HashMap<>(); @@ -354,6 +358,33 @@ private void resizeResource(DevelopersApi api, String resourceName, long sizeByt } } + /** + * Records the size of the volume's Linstor device as the size of the volume, if it is larger. Linstor rounds the + * storage of a volume up and the VM sees all of it, so copies of the volume (snapshot backups, templates, + * downloads) have to cover all of it too. + * @param adjustResourceCount whether to count the difference against the account's primary storage + * @return the size of the volume + */ + protected long recordDeviceSize(DevelopersApi api, VolumeVO volume, String rscName, boolean adjustResourceCount) { + final long size = volume.getSize() != null ? volume.getSize() : 0; + try { + final long deviceSize = LinstorUtil.getVolumeDeviceSize(api, rscName); + if (deviceSize > size) { + logger.info("Linstor: the device of {} has {} bytes, recording that instead of {}", volume, deviceSize, size); + volume.setSize(deviceSize); + _volumeDao.update(volume.getId(), volume); + if (adjustResourceCount) { + _resourceLimitMgr.incrementVolumePrimaryStorageResourceCount(volume.getAccountId(), + volume.isDisplayVolume(), deviceSize - size, _diskOfferingDao.findById(volume.getDiskOfferingId())); + } + return deviceSize; + } + } catch (ApiException apiEx) { + logger.warn("Linstor: unable to get the device size of {}, keeping {} bytes: {}", rscName, size, apiEx.getBestMessage()); + } + return size; + } + /** * Update resource-definitions resource-group to the correct one if it isn't already the intended. * @param api Linstor api @@ -614,6 +645,7 @@ public void createAsync(DataStore dataStore, DataObject vol, AsyncCompletionCall volume.setPath(vol.getUuid()); _volumeDao.update(volume.getId(), volume); + recordDeviceSize(getLinstorAPI(storagePool), volume, LinstorUtil.RSC_PREFIX + volume.getPath(), true); break; case SNAPSHOT: logger.debug("createAsync - SNAPSHOT"); @@ -975,8 +1007,11 @@ private Answer copyVolume(DataObject srcData, DataObject dstData) { final StoragePoolVO pool = _storagePoolDao.findById(srcVolInfo.getDataStore().getId()); final DevelopersApi api = getLinstorAPI(pool); final String rscName = LinstorUtil.RSC_PREFIX + srcVolInfo.getPath(); + // volumes created before their device size was recorded + final long size = recordDeviceSize(api, _volumeDao.findById(srcVolInfo.getId()), rscName, true); VolumeObjectTO to = (VolumeObjectTO) srcVolInfo.getTO(); + to.setSize(size); // patch source format // Linstor volumes are stored as RAW, but we can't set the correct format as RAW (we use QCOW2) // otherwise create template from snapshot won't work, because this operation @@ -1205,12 +1240,13 @@ public void copyAsync(DataObject srcData, DataObject destData, Host destHost, As private CreateCmdResult notifyResize( VolumeObject vol, long oldSize, + long newSize, ResizeVolumePayload resizeParameter) { StoragePool pool = (StoragePool) vol.getDataStore(); ResizeVolumeCommand resizeCmd = - new ResizeVolumeCommand(vol.getPath(), new StorageFilerTO(pool), oldSize, resizeParameter.newSize, resizeParameter.shrinkOk, + new ResizeVolumeCommand(vol.getPath(), new StorageFilerTO(pool), oldSize, newSize, resizeParameter.shrinkOk, resizeParameter.instanceName, null); CreateCmdResult result = new CreateCmdResult(null, null); try { @@ -1244,6 +1280,7 @@ public void resize(DataObject data, AsyncCompletionCallback cal final long oldSize = vol.getSize(); String errMsg = null; + long newSize = resizeParameter.newSize; VolumeDefinitionModify dfm = new VolumeDefinitionModify(); dfm.setSizeKib(resizeParameter.newSize / 1024); try @@ -1257,6 +1294,8 @@ public void resize(DataObject data, AsyncCompletionCallback cal volume.setMaxIops(resizeParameter.newMaxIops); volume.setSize(resizeParameter.newSize); _volumeDao.update(volume.getId(), volume); + // the resize counts the recorded size against the account's primary storage + newSize = recordDeviceSize(api, volume, rscName, false); } } catch (ApiException apiExc) { @@ -1270,7 +1309,7 @@ public void resize(DataObject data, AsyncCompletionCallback cal result.setResult(errMsg); } else { // notify guests - result = notifyResize(vol, oldSize, resizeParameter); + result = notifyResize(vol, oldSize, newSize, resizeParameter); } callback.complete(result); @@ -1313,6 +1352,8 @@ public void takeSnapshot(SnapshotInfo snapshotInfo, AsyncCompletionCallback rscDfns = api.resourceDefinitionList( + Collections.singletonList(rscName), true, null, null, null); + if (rscDfns.isEmpty() || CollectionUtils.isEmpty(rscDfns.get(0).getVolumeDefinitions())) { + return 0; + } + ResourceDefinition rscDfn = rscDfns.get(0); + final long definedSize = rscDfn.getVolumeDefinitions().get(0).getSizeKib() * 1024; + if (rscDfn.getProps() != null && Boolean.parseBoolean(rscDfn.getProps().get(PROP_DRBD_EXACT_SIZE))) { + return definedSize; + } + + List resources = api.viewResources( + Collections.emptyList(), + Collections.singletonList(rscName), + Collections.emptyList(), + null, + null, + null); + Long deviceKib = null; + for (ResourceWithVolumes rsc : resources) { + if (!isRscDiskless(rsc) && !rsc.getVolumes().isEmpty()) { + Long usableKib = getTopLayerUsableSizeKib(rsc.getVolumes().get(0)); + if (usableKib != null && usableKib > 0 && (deviceKib == null || usableKib < deviceKib)) { + deviceKib = usableKib; + } + } + } + return deviceKib != null ? Math.max(definedSize, deviceKib * 1024) : definedSize; + } + /** * Checks if all diskful resource are on a zeroed block device. * @param pool Linstor pool to use diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java index 69a6e90128f7..30f5d9209d42 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java @@ -19,8 +19,15 @@ import com.linbit.linstor.api.ApiException; import com.linbit.linstor.api.DevelopersApi; import com.linbit.linstor.api.model.AutoSelectFilter; +import com.linbit.linstor.api.model.DrbdVolume; import com.linbit.linstor.api.model.LayerType; +import com.linbit.linstor.api.model.Properties; +import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceGroup; +import com.linbit.linstor.api.model.ResourceWithVolumes; +import com.linbit.linstor.api.model.Volume; +import com.linbit.linstor.api.model.VolumeDefinition; +import com.linbit.linstor.api.model.VolumeLayer; import java.util.Arrays; import java.util.Collections; @@ -42,6 +49,11 @@ import org.mockito.junit.MockitoJUnitRunner; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.ResourceLimitService; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -54,6 +66,15 @@ public class LinstorPrimaryDataStoreDriverImplTest { @Mock private SnapshotDataStoreDao snapshotStoreDao; + @Mock + private VolumeDao volumeDao; + + @Mock + private DiskOfferingDao diskOfferingDao; + + @Mock + private ResourceLimitService resourceLimitMgr; + @InjectMocks private LinstorPrimaryDataStoreDriverImpl linstorPrimaryDataStoreDriver; @@ -178,4 +199,87 @@ public void testGetEncryptedLayerList() throws ApiException { layers = LinstorUtil.getEncryptedLayerList(api, "EncryptedGrp"); Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers); } + + /** + * A 512 MiB volume definition whose DRBD device has 540909568 bytes (LVM-thin rounds the storage up). + */ + private void mockLinstorVolume(String rscName) throws ApiException { + VolumeDefinition vlmDfn = new VolumeDefinition(); + vlmDfn.setSizeKib(524288L); + ResourceDefinition rscDfn = new ResourceDefinition(); + rscDfn.setName(rscName); + rscDfn.addVolumeDefinitionsItem(vlmDfn); + rscDfn.setProps(new Properties()); + when(api.resourceDefinitionList(Collections.singletonList(rscName), true, null, null, null)) + .thenReturn(Collections.singletonList(rscDfn)); + + DrbdVolume drbdVolume = new DrbdVolume(); + drbdVolume.setUsableSizeKib(528232L); + VolumeLayer layer = new VolumeLayer(); + layer.setType(LayerType.DRBD); + layer.setData(drbdVolume); + Volume volume = new Volume(); + volume.addLayerDataListItem(layer); + ResourceWithVolumes rsc = new ResourceWithVolumes(); + rsc.setNodeName("nodeA"); + rsc.setVolumes(Collections.singletonList(volume)); + when(api.viewResources(Collections.emptyList(), Collections.singletonList(rscName), Collections.emptyList(), + null, null, null)).thenReturn(Collections.singletonList(rsc)); + } + + private VolumeVO mockVolume(long size) { + VolumeVO volume = mock(VolumeVO.class); + Mockito.lenient().when(volume.getId()).thenReturn(7L); + Mockito.lenient().when(volume.getAccountId()).thenReturn(2L); + Mockito.lenient().when(volume.isDisplayVolume()).thenReturn(true); + Mockito.lenient().when(volume.getDiskOfferingId()).thenReturn(3L); + when(volume.getSize()).thenReturn(size); + return volume; + } + + @Test + public void testRecordDeviceSizeRecordsTheLargerDevice() throws ApiException { + mockLinstorVolume("cs-vol"); + VolumeVO volume = mockVolume(536870912L); + DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); + when(diskOfferingDao.findById(3L)).thenReturn(diskOffering); + + long size = linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", true); + + Assert.assertEquals(540909568L, size); + Mockito.verify(volume).setSize(540909568L); + Mockito.verify(volumeDao).update(7L, volume); + Mockito.verify(resourceLimitMgr).incrementVolumePrimaryStorageResourceCount(2L, true, 4038656L, diskOffering); + } + + @Test + public void testRecordDeviceSizeWithoutResourceCount() throws ApiException { + // a resize counts the recorded size itself + mockLinstorVolume("cs-vol"); + VolumeVO volume = mockVolume(536870912L); + + Assert.assertEquals(540909568L, linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", false)); + Mockito.verify(volumeDao).update(7L, volume); + Mockito.verifyNoInteractions(resourceLimitMgr); + } + + @Test + public void testRecordDeviceSizeKeepsRecordedDeviceSize() throws ApiException { + mockLinstorVolume("cs-vol"); + VolumeVO volume = mockVolume(540909568L); + + Assert.assertEquals(540909568L, linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", true)); + Mockito.verify(volume, Mockito.never()).setSize(Mockito.anyLong()); + Mockito.verifyNoInteractions(volumeDao, resourceLimitMgr); + } + + @Test + public void testRecordDeviceSizeKeepsSizeIfLinstorFails() throws ApiException { + when(api.resourceDefinitionList(Collections.singletonList("cs-vol"), true, null, null, null)) + .thenThrow(new ApiException(500, "controller unreachable")); + VolumeVO volume = mockVolume(536870912L); + + Assert.assertEquals(536870912L, linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", true)); + Mockito.verifyNoInteractions(volumeDao, resourceLimitMgr); + } } diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java index fc4153e33644..69cc74c1a961 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java @@ -20,14 +20,20 @@ import com.linbit.linstor.api.ApiException; import com.linbit.linstor.api.DevelopersApi; import com.linbit.linstor.api.model.AutoSelectFilter; +import com.linbit.linstor.api.model.DrbdVolume; +import com.linbit.linstor.api.model.LayerType; import com.linbit.linstor.api.model.Node; import com.linbit.linstor.api.model.Properties; import com.linbit.linstor.api.model.ProviderKind; +import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceGroup; import com.linbit.linstor.api.model.ResourceState; import com.linbit.linstor.api.model.ResourceWithVolumes; import com.linbit.linstor.api.model.StoragePool; +import com.linbit.linstor.api.model.StorageVolume; import com.linbit.linstor.api.model.Volume; +import com.linbit.linstor.api.model.VolumeDefinition; +import com.linbit.linstor.api.model.VolumeLayer; import java.util.Arrays; import java.util.Collections; @@ -38,8 +44,10 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -199,4 +207,88 @@ public void testGetRscGroupStoragePools() throws ApiException { .collect(Collectors.toList()); Assert.assertEquals(names, Arrays.asList("nodeA::thinpool", "nodeB::thinpool", "nodeC::thinpool")); } + + private void mockResourceDefinition(String rscName, long sizeKib, Properties props) throws ApiException { + VolumeDefinition vlmDfn = new VolumeDefinition(); + vlmDfn.setSizeKib(sizeKib); + ResourceDefinition rscDfn = new ResourceDefinition(); + rscDfn.setName(rscName); + rscDfn.addVolumeDefinitionsItem(vlmDfn); + rscDfn.setProps(props); + when(api.resourceDefinitionList(Collections.singletonList(rscName), true, null, null, null)) + .thenReturn(Collections.singletonList(rscDfn)); + } + + private ResourceWithVolumes mockResource(String node, boolean diskless, LayerType topLayer, long usableSizeKib) { + VolumeLayer layer = new VolumeLayer(); + layer.setType(topLayer); + if (topLayer == LayerType.DRBD) { + DrbdVolume drbdVolume = new DrbdVolume(); + drbdVolume.setUsableSizeKib(usableSizeKib); + layer.setData(drbdVolume); + } else { + StorageVolume storageVolume = new StorageVolume(); + storageVolume.setUsableSizeKib(usableSizeKib); + layer.setData(storageVolume); + } + Volume volume = new Volume(); + volume.addLayerDataListItem(layer); + ResourceWithVolumes rsc = new ResourceWithVolumes(); + rsc.setNodeName(node); + rsc.setFlags(diskless ? Collections.singletonList(ApiConsts.FLAG_DISKLESS) : Collections.emptyList()); + rsc.setVolumes(Collections.singletonList(volume)); + return rsc; + } + + private void mockResources(String rscName, ResourceWithVolumes... resources) throws ApiException { + when(api.viewResources(Collections.emptyList(), Collections.singletonList(rscName), Collections.emptyList(), + null, null, null)).thenReturn(Arrays.asList(resources)); + } + + @Test + public void testGetVolumeDeviceSizeIsTheSmallestDiskfulDrbdDevice() throws ApiException { + // a 512 MiB volume definition on LVM-thin, the diskless resource only reports the volume definition's size + mockResourceDefinition("cs-vol", 524288L, new Properties()); + mockResources("cs-vol", + mockResource("nodeA", false, LayerType.DRBD, 528232L), + mockResource("nodeB", true, LayerType.DRBD, 524288L), + mockResource("nodeC", false, LayerType.DRBD, 528236L)); + + Assert.assertEquals(528232L * 1024, LinstorUtil.getVolumeDeviceSize(api, "cs-vol")); + } + + @Test + public void testGetVolumeDeviceSizeWithoutDrbd() throws ApiException { + mockResourceDefinition("cs-vol", 524288L, new Properties()); + mockResources("cs-vol", mockResource("nodeA", false, LayerType.STORAGE, 528384L)); + + Assert.assertEquals(528384L * 1024, LinstorUtil.getVolumeDeviceSize(api, "cs-vol")); + } + + @Test + public void testGetVolumeDeviceSizeWithExactSize() throws ApiException { + Properties props = new Properties(); + props.put("DrbdOptions/ExactSize", "True"); + mockResourceDefinition("cs-vol", 524288L, props); + + Assert.assertEquals(524288L * 1024, LinstorUtil.getVolumeDeviceSize(api, "cs-vol")); + Mockito.verify(api, Mockito.never()).viewResources(any(), any(), any(), any(), any(), any()); + } + + @Test + public void testGetVolumeDeviceSizeFallsBackToVolumeDefinition() throws ApiException { + // no diskful resource reports a size, e.g. while the resources are being created + mockResourceDefinition("cs-vol", 524288L, new Properties()); + mockResources("cs-vol", mockResource("nodeB", true, LayerType.DRBD, 524288L)); + + Assert.assertEquals(524288L * 1024, LinstorUtil.getVolumeDeviceSize(api, "cs-vol")); + } + + @Test + public void testGetVolumeDeviceSizeOfMissingResourceDefinition() throws ApiException { + when(api.resourceDefinitionList(Collections.singletonList("cs-gone"), true, null, null, null)) + .thenReturn(Collections.emptyList()); + + Assert.assertEquals(0L, LinstorUtil.getVolumeDeviceSize(api, "cs-gone")); + } }