Skip to content

linstor: record the size of the DRBD device as the volume size #14658

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rp- wants to merge 1 commit into
base: 4.22
Choose a base branch
from
Open

linstor: record the size of the DRBD device as the volume size #14658

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 43 additions & 2 deletions ...ava/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,15 @@
import com.cloud.storage.Volume;
import com.cloud.storage.VolumeDetailVO;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.DiskOfferingDao;
import com.cloud.storage.dao.SnapshotDao;
import com.cloud.storage.dao.SnapshotDetailsDao;
import com.cloud.storage.dao.SnapshotDetailsVO;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplatePoolDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.dao.VolumeDetailsDao;
import com.cloud.user.ResourceLimitService;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
Expand Down Expand Up @@ -133,6 +135,8 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
@Inject
private HostDao _hostDao;
@Inject private VMTemplateDao _vmTemplateDao;
@Inject private DiskOfferingDao _diskOfferingDao;
@Inject private ResourceLimitService _resourceLimitMgr;

private final Map<String, Long> volumeStatsLastUpdate = new HashMap<>();
private final Map<String, Pair<Long, Long>> volumeStats = new HashMap<>();
Expand Down Expand Up @@ -354,6 +358,33 @@ private void resizeResource(DevelopersApi api, String resourceName, long sizeByt
}
}

/**
* Records the size of the volume's Linstor device as the size of the volume, if it is larger. Linstor rounds the
* storage of a volume up and the VM sees all of it, so copies of the volume (snapshot backups, templates,
* downloads) have to cover all of it too.
* @param adjustResourceCount whether to count the difference against the account's primary storage
* @return the size of the volume
*/
protected long recordDeviceSize(DevelopersApi api, VolumeVO volume, String rscName, boolean adjustResourceCount) {
final long size = volume.getSize() != null ? volume.getSize() : 0;
try {
final long deviceSize = LinstorUtil.getVolumeDeviceSize(api, rscName);
if (deviceSize > size) {
logger.info("Linstor: the device of {} has {} bytes, recording that instead of {}", volume, deviceSize, size);
volume.setSize(deviceSize);
_volumeDao.update(volume.getId(), volume);
if (adjustResourceCount) {
_resourceLimitMgr.incrementVolumePrimaryStorageResourceCount(volume.getAccountId(),
volume.isDisplayVolume(), deviceSize - size, _diskOfferingDao.findById(volume.getDiskOfferingId()));
}
return deviceSize;
}
} catch (ApiException apiEx) {
logger.warn("Linstor: unable to get the device size of {}, keeping {} bytes: {}", rscName, size, apiEx.getBestMessage());
}
return size;
}

/**
* Update resource-definitions resource-group to the correct one if it isn't already the intended.
* @param api Linstor api
Expand Down Expand Up @@ -614,6 +645,7 @@ public void createAsync(DataStore dataStore, DataObject vol, AsyncCompletionCall
volume.setPath(vol.getUuid());

_volumeDao.update(volume.getId(), volume);
recordDeviceSize(getLinstorAPI(storagePool), volume, LinstorUtil.RSC_PREFIX + volume.getPath(), true);
break;
case SNAPSHOT:
logger.debug("createAsync - SNAPSHOT");
Expand Down Expand Up @@ -975,8 +1007,11 @@ private Answer copyVolume(DataObject srcData, DataObject dstData) {
final StoragePoolVO pool = _storagePoolDao.findById(srcVolInfo.getDataStore().getId());
final DevelopersApi api = getLinstorAPI(pool);
final String rscName = LinstorUtil.RSC_PREFIX + srcVolInfo.getPath();
// volumes created before their device size was recorded
final long size = recordDeviceSize(api, _volumeDao.findById(srcVolInfo.getId()), rscName, true);

VolumeObjectTO to = (VolumeObjectTO) srcVolInfo.getTO();
to.setSize(size);
// patch source format
// Linstor volumes are stored as RAW, but we can't set the correct format as RAW (we use QCOW2)
// otherwise create template from snapshot won't work, because this operation
Expand Down Expand Up @@ -1205,12 +1240,13 @@ public void copyAsync(DataObject srcData, DataObject destData, Host destHost, As
private CreateCmdResult notifyResize(
VolumeObject vol,
long oldSize,
long newSize,
ResizeVolumePayload resizeParameter)
{
StoragePool pool = (StoragePool) vol.getDataStore();

ResizeVolumeCommand resizeCmd =
new ResizeVolumeCommand(vol.getPath(), new StorageFilerTO(pool), oldSize, resizeParameter.newSize, resizeParameter.shrinkOk,
new ResizeVolumeCommand(vol.getPath(), new StorageFilerTO(pool), oldSize, newSize, resizeParameter.shrinkOk,
resizeParameter.instanceName, null);
CreateCmdResult result = new CreateCmdResult(null, null);
try {
Expand Down Expand Up @@ -1244,6 +1280,7 @@ public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> cal
final long oldSize = vol.getSize();

String errMsg = null;
long newSize = resizeParameter.newSize;
VolumeDefinitionModify dfm = new VolumeDefinitionModify();
dfm.setSizeKib(resizeParameter.newSize / 1024);
try
Expand All @@ -1257,6 +1294,8 @@ public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> cal
volume.setMaxIops(resizeParameter.newMaxIops);
volume.setSize(resizeParameter.newSize);
_volumeDao.update(volume.getId(), volume);
// the resize counts the recorded size against the account's primary storage
newSize = recordDeviceSize(api, volume, rscName, false);
}
} catch (ApiException apiExc)
{
Expand All @@ -1270,7 +1309,7 @@ public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> cal
result.setResult(errMsg);
} else {
// notify guests
result = notifyResize(vol, oldSize, resizeParameter);
result = notifyResize(vol, oldSize, newSize, resizeParameter);
}

callback.complete(result);
Expand Down Expand Up @@ -1313,6 +1352,8 @@ public void takeSnapshot(SnapshotInfo snapshotInfo, AsyncCompletionCallback<Crea
final StoragePoolVO storagePool = _storagePoolDao.findById(storagePoolId);
final DevelopersApi api = getLinstorAPI(storagePool);
final String rscName = LinstorUtil.RSC_PREFIX + volumeVO.getPath();
// volumes created before their device size was recorded, a backup of the snapshot covers the whole device
recordDeviceSize(api, volumeVO, rscName, true);

Snapshot snapshot = new Snapshot();
snapshot.setName(getSnapshotName(snapshotInfo.getUuid()));
Expand Down
63 changes: 63 additions & 0 deletions ...olume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,11 @@
import com.linbit.linstor.api.model.ApiCallRc;
import com.linbit.linstor.api.model.ApiCallRcList;
import com.linbit.linstor.api.model.AutoSelectFilter;
import com.linbit.linstor.api.model.DrbdVolume;
import com.linbit.linstor.api.model.LUKSVolume;
import com.linbit.linstor.api.model.LayerType;
import com.linbit.linstor.api.model.Node;
import com.linbit.linstor.api.model.OneOfVolumeLayerData;
import com.linbit.linstor.api.model.Properties;
import com.linbit.linstor.api.model.ProviderKind;
import com.linbit.linstor.api.model.Resource;
Expand All @@ -34,6 +37,7 @@
import com.linbit.linstor.api.model.ResourceGroupSpawn;
import com.linbit.linstor.api.model.ResourceWithVolumes;
import com.linbit.linstor.api.model.StoragePool;
import com.linbit.linstor.api.model.StorageVolume;
import com.linbit.linstor.api.model.Volume;
import com.linbit.linstor.api.model.VolumeDefinition;
import com.linbit.linstor.api.model.VolumeDefinitionModify;
Expand Down Expand Up @@ -72,6 +76,8 @@ public class LinstorUtil {
public static final String RSC_PREFIX = "cs-";
public static final String RSC_GROUP = "resourceGroup";
public static final String CS_TEMPLATE_FOR_PREFIX = "_cs-template-for-";
// the DRBD device has exactly the size of the volume definition
public static final String PROP_DRBD_EXACT_SIZE = ApiConsts.NAMESPC_DRBD_OPTIONS + "/ExactSize";

public static final String LIN_PROP_DRBDOPT_EXACT_SIZE = "DrbdOptions/ExactSize";

Expand Down Expand Up @@ -545,6 +551,63 @@ public static boolean isRscDiskless(ResourceWithVolumes rsc) {
return rsc.getFlags() != null && rsc.getFlags().contains(ApiConsts.FLAG_DISKLESS);
}

/**
* The usable size of the topmost layer of a volume, the size of its block device on that node.
*/
private static Long getTopLayerUsableSizeKib(Volume vlm) {
if (CollectionUtils.isNotEmpty(vlm.getLayerDataList())) {
OneOfVolumeLayerData data = vlm.getLayerDataList().get(0).getData();
if (data instanceof DrbdVolume) {
return ((DrbdVolume) data).getUsableSizeKib();
} else if (data instanceof LUKSVolume) {
return ((LUKSVolume) data).getUsableSizeKib();
} else if (data instanceof StorageVolume) {
return ((StorageVolume) data).getUsableSizeKib();
}
}
return vlm.getUsableSizeKib();
}

/**
* The size of the block device of a resource's (only) volume in bytes, the size a VM sees. Linstor rounds the
* storage of a volume up, e.g. to the LVM extent size, and DRBD makes all of it usable, so the device is usually
* a few MiB larger than the volume definition. A DRBD device has the smallest usable size of its diskful
* resources; diskless resources only report the size of the volume definition. With the property
* DrbdOptions/ExactSize the device has exactly the size of the volume definition.
* @return the device size, never less than the size of the volume definition, 0 if the resource definition
* has no volume
*/
public static long getVolumeDeviceSize(DevelopersApi api, String rscName) throws ApiException {
List<ResourceDefinition> rscDfns = api.resourceDefinitionList(
Collections.singletonList(rscName), true, null, null, null);
if (rscDfns.isEmpty() || CollectionUtils.isEmpty(rscDfns.get(0).getVolumeDefinitions())) {
return 0;
}
ResourceDefinition rscDfn = rscDfns.get(0);
final long definedSize = rscDfn.getVolumeDefinitions().get(0).getSizeKib() * 1024;
if (rscDfn.getProps() != null && Boolean.parseBoolean(rscDfn.getProps().get(PROP_DRBD_EXACT_SIZE))) {
return definedSize;
}

List<ResourceWithVolumes> resources = api.viewResources(
Collections.emptyList(),
Collections.singletonList(rscName),
Collections.emptyList(),
null,
null,
null);
Long deviceKib = null;
for (ResourceWithVolumes rsc : resources) {
if (!isRscDiskless(rsc) && !rsc.getVolumes().isEmpty()) {
Long usableKib = getTopLayerUsableSizeKib(rsc.getVolumes().get(0));
if (usableKib != null && usableKib > 0 && (deviceKib == null || usableKib < deviceKib)) {
deviceKib = usableKib;
}
}
}
return deviceKib != null ? Math.max(definedSize, deviceKib * 1024) : definedSize;
}

/**
* Checks if all diskful resource are on a zeroed block device.
* @param pool Linstor pool to use
Expand Down
104 changes: 104 additions & 0 deletions ...org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,15 @@
import com.linbit.linstor.api.ApiException;
import com.linbit.linstor.api.DevelopersApi;
import com.linbit.linstor.api.model.AutoSelectFilter;
import com.linbit.linstor.api.model.DrbdVolume;
import com.linbit.linstor.api.model.LayerType;
import com.linbit.linstor.api.model.Properties;
import com.linbit.linstor.api.model.ResourceDefinition;
import com.linbit.linstor.api.model.ResourceGroup;
import com.linbit.linstor.api.model.ResourceWithVolumes;
import com.linbit.linstor.api.model.Volume;
import com.linbit.linstor.api.model.VolumeDefinition;
import com.linbit.linstor.api.model.VolumeLayer;

import java.util.Arrays;
import java.util.Collections;
Expand All @@ -42,6 +49,11 @@
import org.mockito.junit.MockitoJUnitRunner;

import com.cloud.storage.DataStoreRole;
import com.cloud.storage.DiskOfferingVO;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.DiskOfferingDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.user.ResourceLimitService;

import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
Expand All @@ -54,6 +66,15 @@ public class LinstorPrimaryDataStoreDriverImplTest {
@Mock
private SnapshotDataStoreDao snapshotStoreDao;

@Mock
private VolumeDao volumeDao;

@Mock
private DiskOfferingDao diskOfferingDao;

@Mock
private ResourceLimitService resourceLimitMgr;

@InjectMocks
private LinstorPrimaryDataStoreDriverImpl linstorPrimaryDataStoreDriver;

Expand Down Expand Up @@ -178,4 +199,87 @@ public void testGetEncryptedLayerList() throws ApiException {
layers = LinstorUtil.getEncryptedLayerList(api, "EncryptedGrp");
Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers);
}

/**
* A 512 MiB volume definition whose DRBD device has 540909568 bytes (LVM-thin rounds the storage up).
*/
private void mockLinstorVolume(String rscName) throws ApiException {
VolumeDefinition vlmDfn = new VolumeDefinition();
vlmDfn.setSizeKib(524288L);
ResourceDefinition rscDfn = new ResourceDefinition();
rscDfn.setName(rscName);
rscDfn.addVolumeDefinitionsItem(vlmDfn);
rscDfn.setProps(new Properties());
when(api.resourceDefinitionList(Collections.singletonList(rscName), true, null, null, null))
.thenReturn(Collections.singletonList(rscDfn));

DrbdVolume drbdVolume = new DrbdVolume();
drbdVolume.setUsableSizeKib(528232L);
VolumeLayer layer = new VolumeLayer();
layer.setType(LayerType.DRBD);
layer.setData(drbdVolume);
Volume volume = new Volume();
volume.addLayerDataListItem(layer);
ResourceWithVolumes rsc = new ResourceWithVolumes();
rsc.setNodeName("nodeA");
rsc.setVolumes(Collections.singletonList(volume));
when(api.viewResources(Collections.emptyList(), Collections.singletonList(rscName), Collections.emptyList(),
null, null, null)).thenReturn(Collections.singletonList(rsc));
}

private VolumeVO mockVolume(long size) {
VolumeVO volume = mock(VolumeVO.class);
Mockito.lenient().when(volume.getId()).thenReturn(7L);
Mockito.lenient().when(volume.getAccountId()).thenReturn(2L);
Mockito.lenient().when(volume.isDisplayVolume()).thenReturn(true);
Mockito.lenient().when(volume.getDiskOfferingId()).thenReturn(3L);
when(volume.getSize()).thenReturn(size);
return volume;
}

@Test
public void testRecordDeviceSizeRecordsTheLargerDevice() throws ApiException {
mockLinstorVolume("cs-vol");
VolumeVO volume = mockVolume(536870912L);
DiskOfferingVO diskOffering = mock(DiskOfferingVO.class);
when(diskOfferingDao.findById(3L)).thenReturn(diskOffering);

long size = linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", true);

Assert.assertEquals(540909568L, size);
Mockito.verify(volume).setSize(540909568L);
Mockito.verify(volumeDao).update(7L, volume);
Mockito.verify(resourceLimitMgr).incrementVolumePrimaryStorageResourceCount(2L, true, 4038656L, diskOffering);
}

@Test
public void testRecordDeviceSizeWithoutResourceCount() throws ApiException {
// a resize counts the recorded size itself
mockLinstorVolume("cs-vol");
VolumeVO volume = mockVolume(536870912L);

Assert.assertEquals(540909568L, linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", false));
Mockito.verify(volumeDao).update(7L, volume);
Mockito.verifyNoInteractions(resourceLimitMgr);
}

@Test
public void testRecordDeviceSizeKeepsRecordedDeviceSize() throws ApiException {
mockLinstorVolume("cs-vol");
VolumeVO volume = mockVolume(540909568L);

Assert.assertEquals(540909568L, linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", true));
Mockito.verify(volume, Mockito.never()).setSize(Mockito.anyLong());
Mockito.verifyNoInteractions(volumeDao, resourceLimitMgr);
}

@Test
public void testRecordDeviceSizeKeepsSizeIfLinstorFails() throws ApiException {
when(api.resourceDefinitionList(Collections.singletonList("cs-vol"), true, null, null, null))
.thenThrow(new ApiException(500, "controller unreachable"));
VolumeVO volume = mockVolume(536870912L);

Assert.assertEquals(536870912L, linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", true));
Mockito.verifyNoInteractions(volumeDao, resourceLimitMgr);
}
}
Loading
Loading