From a2b1681d548a780eeec1ad210cc209c12d3630b7 Mon Sep 17 00:00:00 2001 From: tyderian1978 Date: Tue, 17 Nov 2026 22:46:28 +1100 Subject: [PATCH 1/2] vmware: reduce repeated requests for the same vCenter data Every read of a property of a pyVmomi managed object is a SOAP round trip which returns the whole top level property. The VMware source read e.g. a VM's 'config' about 9 times, its 'guest' once per NIC, the ~1 MB 'config' of a host 6 times per host, and walked the same folders and resource pools again for every VM. * read 'config', 'runtime', 'guest' of a VM and 'summary', 'config' of a host once per object * skip VMs already parsed in the first run over all VMs by their managed object id instead of requesting name and config again * cache names, parents and resource pools of objects for the run; the folder and resource pool walks for VM paths, tenants and resource pool filters use these caches * cache the tags attached to an object for the run (the tags of parent folders, clusters and datacenters are requested for every VM); tag and category definitions are already cached since ec1ba92 * cache custom attribute definitions per object type and only request them if the object has custom values * hand the already retrieved guest info to add_update_interface() through the new SourceBase.get_vmware_guest_info() hook get_parent_object_by_class() kept its depth in an instance attribute which was not reset when a walk ended at the root, so later lookups could give up early. The rewrite walks iteratively and has no such state. Measured against vcsim with the captured inventories, synced inventory unchanged: vchvr: 1469 -> 323 SOAP calls, 20.1 -> 3.6 MB vc001: 1707 -> 355 SOAP calls, 30.1 -> 6.0 MB --- module/sources/common/source_base.py | 25 +- module/sources/vmware/connection.py | 325 +++++++++++++++++++------- tests/test_vmware_vcenter_requests.py | 211 +++++++++++++++++ 3 files changed, 474 insertions(+), 87 deletions(-) create mode 100644 tests/test_vmware_vcenter_requests.py diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index e44de2b5..7ca4aa09 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -353,6 +353,23 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac return return_data + def get_vmware_guest_info(self, vmware_object): + """ + Return the guest info of a VMware VM. Reading it is a round trip to vCenter, sources which already + retrieved it can return their copy. + + Parameters + ---------- + vmware_object: vim.VirtualMachine + the VM + + Returns + ------- + vim.vm.GuestInfo, None: the guest info + """ + + return grab(vmware_object, "guest") + def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None) -> Optional[NBPrefix]: """ This is a lazy approach to find the longest matching prefix to an IP address. @@ -539,10 +556,14 @@ def add_update_interface(self, interface_object, device_object, interface_data, interface_object.update(data={"primary_mac_address": primary_mac_address_object}, source=self) + vmware_guest_info = None + if type(device_object) == NBVM: + vmware_guest_info = self.get_vmware_guest_info(vmware_object) + # skip handling of IPs for VMs with not installed/running guest tools skip_ip_handling = False if type(device_object) == NBVM and vmware_object is not None and \ - grab(vmware_object, 'guest.toolsRunningStatus') != "guestToolsRunning": + grab(vmware_guest_info, 'toolsRunningStatus') != "guestToolsRunning": log.debug(f"VM '{device_object.name}' guest tool status is 'NotRunning', skipping IP handling") skip_ip_handling = True @@ -818,7 +839,7 @@ def add_update_interface(self, interface_object, device_object, interface_data, # (seen on old TMOS releases), not a statement that every address is gone. Keep what is # in NetBox instead of tearing it off on every run. A real removal still reports the # interface, just without an address, so that case is unaffected - reported_interfaces = grab(vmware_object, "guest.net") + reported_interfaces = grab(vmware_guest_info, "net") if type(device_object) == NBVM and isinstance(reported_interfaces, list) and \ len(reported_interfaces) == 0: log.debug(f"VM '{device_object.name}' guest tools report no network interface at all, " diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 123260cb..591f1171 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -109,8 +109,6 @@ class VMWareHandler(SourceBase): vm_name_max_length = 64 vm_description_max_length = 200 - recursion_level = 0 - # internal vars session = None tag_session = None @@ -188,6 +186,7 @@ def __init__(self, name=None): self.processed_host_names = dict() self.processed_vm_names = dict() self.processed_vm_uuid = list() + self.processed_vm_object_keys = set() self.object_cache = dict() self.object_path_cache = dict() self.parsing_vms_the_first_time = True @@ -1318,13 +1317,22 @@ def get_vmware_object_tags(self, obj): tag_list = list() if self.tag_session is not None: - # noinspection PyBroadException - try: - object_tag_ids = self.tag_session.tagging.TagAssociation.list_attached_tags( - DynamicID(type=grab(obj, "_wsdlName"), id=grab(obj, "_moId"))) - except Exception as e: - log.error(f"Unable to retrieve vCenter tags for '{obj.name}': {e}") - return + # the tags of parent objects (folders, clusters, datacenters) are requested for each + # VM and host, ask vCenter only once per object + attached_tags_cache = self.get_run_cache("attached_tags") + object_key = (grab(obj, "_wsdlName"), grab(obj, "_moId")) + + object_tag_ids = attached_tags_cache.get(object_key) + if object_tag_ids is None: + # noinspection PyBroadException + try: + object_tag_ids = self.tag_session.tagging.TagAssociation.list_attached_tags( + DynamicID(type=object_key[0], id=object_key[1])) + except Exception as e: + log.error(f"Unable to retrieve vCenter tags for '{obj.name}': {e}") + return + + attached_tags_cache[object_key] = object_tag_ids # vCenter tags and categories by id: most objects share a few hundred tags, so each # is fetched once per run (per source) instead of once per object it's attached to @@ -1404,7 +1412,7 @@ def collect_object_tags(self, obj): if tag_source is None or self.tag_session is None: return tag_list - log.debug2(f"Collecting tags for {obj.name}") + log.debug2(f"Collecting tags for {self.get_object_name(obj)}") if "object" in tag_source: tag_list.extend(self.get_vmware_object_tags(obj)) @@ -1463,9 +1471,10 @@ def get_object_custom_fields(self, obj): # add basic host data to device if object_type == "dcim.device": - num_cpu_cores = grab(obj, "summary.hardware.numCpuCores") - cpu_model = grab(obj, "summary.hardware.cpuModel") - memory_size = grab(obj, "summary.hardware.memorySize") + host_hardware = grab(obj, "summary.hardware") + num_cpu_cores = grab(host_hardware, "numCpuCores") + cpu_model = grab(host_hardware, "cpuModel") + memory_size = grab(host_hardware, "memorySize") if num_cpu_cores is not None: custom_field = self.add_update_custom_field({ @@ -1514,11 +1523,22 @@ def get_object_custom_fields(self, obj): return_custom_fields[grab(custom_field, "data.name")] = guest_hostname else: - log.debug2(f"VM '{grab(obj, 'name')}' guest hostname not reported by VMware Tools " + log.debug2(f"VM '{self.get_object_name(obj)}' guest hostname not reported by VMware Tools " "(not installed, not running or no data available yet). Keeping current " f"value of custom field '{guest_hostname_field_name}' untouched.") - field_definition = {grab(k, "key"): grab(k, "name") for k in grab(obj, "availableField", fallback=list())} + # the custom attribute definitions are the same for all objects of a type, + # request them only once per type and only if there are custom values to label + field_definition = dict() + if len(custom_value) > 0: + field_definition_cache = self.get_run_cache("custom_field_definitions") + object_wsdl_name = grab(obj, "_wsdlName") + field_definition = field_definition_cache.get(object_wsdl_name) + if field_definition is None: + field_definition = {grab(k, "key"): grab(k, "name") + for k in grab(obj, "availableField", fallback=list())} + if object_wsdl_name is not None: + field_definition_cache[object_wsdl_name] = field_definition for obj_custom_field in custom_value: key = grab(obj_custom_field, "key") @@ -2034,34 +2054,139 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v return device_vm_object, interface_objects + def get_run_cache(self, name): + """ + Return a dict which caches vCenter data for the duration of this run. Reading a property of a + pyVmomi managed object is a round trip to vCenter, the caches avoid asking for the same data + again and again (parents and names of clusters, folders and datacenters, tag definitions, ...). + + The caches are created on first use, handlers which were built without running __init__ + (as the unit tests do) get them as well. + + Parameters + ---------- + name: str + name of the cache + + Returns + ------- + dict: the cache + """ + + caches = self.__dict__.setdefault("_run_caches", dict()) + + return caches.setdefault(name, dict()) + + @staticmethod + def get_vmware_object_key(obj): + """ + Return a key which identifies a vCenter managed object without a round trip to vCenter + + Parameters + ---------- + obj: pyVmomi managed object + + Returns + ------- + tuple, None: type name and managed object id or None if obj is not a managed object + """ + + object_id = getattr(obj, "_moId", None) + + if object_id is None: + return None + + return obj.__class__.__name__, object_id + + def get_object_name(self, obj): + """ + Return the name of a vCenter managed object. The name is only requested once per object and run. + + Parameters + ---------- + obj: pyVmomi managed object + + Returns + ------- + str, None: name of the object + """ + + return self.get_cached_property(obj, "name") + + def get_cached_property(self, obj, property_name): + """ + Return a top level property of a vCenter managed object which doesn't change during a run, like + its name, parent or resource pool. It is only requested once per object and run. + + Parameters + ---------- + obj: pyVmomi managed object + property_name: str + name of the property + + Returns + ------- + the value of the property, None if it can't be read + """ + + object_key = self.get_vmware_object_key(obj) + + if object_key is None: + return grab(obj, property_name) + + property_cache = self.get_run_cache(f"property:{property_name}") + + if object_key not in property_cache: + property_cache[object_key] = grab(obj, property_name) + + return property_cache[object_key] + + def get_vmware_guest_info(self, vmware_object): + """ + Return the guest info of the VM which is currently parsed by add_virtual_machine() without + asking vCenter again, other objects are passed on to SourceBase.get_vmware_guest_info(). + """ + + object_key = self.get_vmware_object_key(vmware_object) + guest_info_cache = self.get_run_cache("vm_guest") + + if object_key is not None and object_key in guest_info_cache: + return guest_info_cache[object_key] + + return super().get_vmware_guest_info(vmware_object) + def get_parent_object_by_class(self, obj, object_class_to_find): + """ + Walk up the vCenter inventory tree until an object of the requested class is found. + The parent of each object is only requested once per run. + + Parameters + ---------- + obj: pyVmomi managed object + object to start with, returned itself if it is of the requested class + object_class_to_find: type + the pyVmomi class to find + + Returns + ------- + pyVmomi managed object, None: the found object or None if there is none + """ if obj is None or object_class_to_find is None: return - if isinstance(obj, object_class_to_find): - self.recursion_level = 0 - return obj - max_recursion = 20 - while True: - if self.recursion_level >= max_recursion: - self.recursion_level = 0 - return None + for _ in range(max_recursion + 1): - # noinspection PyBroadException - try: - parent = obj.parent - except Exception: - self.recursion_level = 0 - return None + if isinstance(obj, object_class_to_find): + return obj + + obj = self.get_cached_property(obj, "parent") - if isinstance(parent, object_class_to_find): - self.recursion_level = 0 - return parent + if obj is None: + return None - self.recursion_level += 1 - return self.get_parent_object_by_class(parent, object_class_to_find) + return None def add_object_to_cache(self, vm_object, netbox_object): @@ -2189,7 +2314,7 @@ def get_resource_pool_path(self, vm): otherwise None (i.e. for template VMs) """ - if grab(vm, "resourcePool") is None: + if self.get_cached_property(vm, "resourcePool") is None: return None cached_path = self.get_object_path_from_cache(vm, "resource_pool_path") @@ -2197,28 +2322,28 @@ def get_resource_pool_path(self, vm): return cached_path path_parts = list() - pool = grab(vm, "resourcePool") + pool = self.get_cached_property(vm, "resourcePool") # walk up the pool hierarchy, the cluster root pool ends the walk: it is the pool whose # parent is the cluster itself rather than another pool (vCenter names it "Resources", # but a child pool may carry that name too) while pool is not None and len(path_parts) < self.object_path_max_depth: - if not isinstance(grab(pool, "parent"), vim.ResourcePool): + if not isinstance(self.get_cached_property(pool, "parent"), vim.ResourcePool): break - pool_name = get_string_or_none(grab(pool, "name")) + pool_name = get_string_or_none(self.get_object_name(pool)) if pool_name is None: break path_parts.append(pool_name) - pool = grab(pool, "parent") + pool = self.get_cached_property(pool, "parent") - if pool is not None and isinstance(grab(pool, "parent"), vim.ResourcePool): - log.warning(f"Resource pool hierarchy of VM '{get_string_or_none(grab(vm, 'name'))}' is deeper " + if pool is not None and isinstance(self.get_cached_property(pool, "parent"), vim.ResourcePool): + log.warning(f"Resource pool hierarchy of VM '{get_string_or_none(self.get_object_name(vm))}' is deeper " f"than {self.object_path_max_depth} levels, the pool path is cut off at the top") path = "/".join(reversed(path_parts)) - log.debug2(f"VM '{get_string_or_none(grab(vm, 'name'))}' has resource pool path '{path}'") + log.debug2(f"VM '{get_string_or_none(self.get_object_name(vm))}' has resource pool path '{path}'") self.add_object_paths_to_cache(vm, {"resource_pool_path": path}) @@ -2242,7 +2367,7 @@ def get_vm_folder_path(self, vm): str, None: VM folder path if the VM is organized in a folder, otherwise None """ - if grab(vm, "parent") is None or not isinstance(grab(vm, "parent"), vim.Folder): + if not isinstance(self.get_cached_property(vm, "parent"), vim.Folder): return None cached_path = self.get_object_path_from_cache(vm, "vm_folder_path") @@ -2250,28 +2375,29 @@ def get_vm_folder_path(self, vm): return cached_path path_parts = list() - folder = grab(vm, "parent") + folder = self.get_cached_property(vm, "parent") # walk up the folder hierarchy, the datacenter's VM folder ends the walk while isinstance(folder, vim.Folder) and len(path_parts) < self.object_path_max_depth: # the folder directly below the datacenter is the datacenter's VM folder - if isinstance(grab(folder, "parent"), vim.Datacenter): + if isinstance(self.get_cached_property(folder, "parent"), vim.Datacenter): break - folder_name = get_string_or_none(grab(folder, "name")) + folder_name = get_string_or_none(self.get_object_name(folder)) if folder_name is None: break path_parts.append(folder_name) - folder = grab(folder, "parent") + folder = self.get_cached_property(folder, "parent") - if isinstance(folder, vim.Folder) and not isinstance(grab(folder, "parent"), vim.Datacenter): - log.warning(f"Folder hierarchy of VM '{get_string_or_none(grab(vm, 'name'))}' is deeper than " + if isinstance(folder, vim.Folder) and not isinstance(self.get_cached_property(folder, "parent"), + vim.Datacenter): + log.warning(f"Folder hierarchy of VM '{get_string_or_none(self.get_object_name(vm))}' is deeper than " f"{self.object_path_max_depth} levels, the folder path is cut off at the top") path = "/".join(reversed(path_parts)) - log.debug2(f"VM '{get_string_or_none(grab(vm, 'name'))}' has VM folder path '{path}'") + log.debug2(f"VM '{get_string_or_none(self.get_object_name(vm))}' has VM folder path '{path}'") self.add_object_paths_to_cache(vm, {"vm_folder_path": path}) @@ -2317,7 +2443,7 @@ def get_vm_tenant_name(self, vm, vm_name, nb_cluster_object, cluster_full_name): tenant_name = self.get_object_relation(resource_pool_path, "vm_tenant_resource_pool_relation") if tenant_name is None: - resource_pool_name = get_string_or_none(grab(vm, "resourcePool.name")) + resource_pool_name = get_string_or_none(self.get_object_name(self.get_cached_property(vm, "resourcePool"))) if resource_pool_name is not None: tenant_name = self.get_object_relation(resource_pool_name, "vm_tenant_resource_pool_relation") @@ -2330,7 +2456,7 @@ def get_vm_tenant_name(self, vm, vm_name, nb_cluster_object, cluster_full_name): tenant_name = self.get_object_relation(vm_folder_path, "vm_tenant_folder_relation") if tenant_name is None: - vm_folder_name = get_string_or_none(grab(vm, "parent.name")) + vm_folder_name = get_string_or_none(self.get_object_name(self.get_cached_property(vm, "parent"))) if vm_folder_name is not None: tenant_name = self.get_object_relation(vm_folder_name, "vm_tenant_folder_relation") @@ -2377,7 +2503,7 @@ def vm_passes_resource_pool_filter(self, vm): return True resource_pool_path = self.get_resource_pool_path(vm) - resource_pool_name = get_string_or_none(grab(vm, "resourcePool.name")) + resource_pool_name = get_string_or_none(self.get_object_name(self.get_cached_property(vm, "resourcePool"))) pool_identifiers = [x for x in dict.fromkeys([resource_pool_path, resource_pool_name]) if x not in [None, ""]] @@ -2769,11 +2895,15 @@ def add_host(self, obj): # Collecting data # + # each read of a property of 'obj' is a round trip to vCenter which returns the whole + # property, so 'summary' and 'config' (which is large) are requested only once + host_summary = grab(obj, "summary") + # collect all necessary data - manufacturer = get_string_or_none(grab(obj, "summary.hardware.vendor")) - model = get_string_or_none(grab(obj, "summary.hardware.model")) - product_name = get_string_or_none(grab(obj, "summary.config.product.name")) - product_version = get_string_or_none(grab(obj, "summary.config.product.version")) + manufacturer = get_string_or_none(grab(host_summary, "hardware.vendor")) + model = get_string_or_none(grab(host_summary, "hardware.model")) + product_name = get_string_or_none(grab(host_summary, "config.product.name")) + product_version = get_string_or_none(grab(host_summary, "config.product.version")) # collect platform platform = f"{product_name} {product_version}" @@ -2788,11 +2918,11 @@ def add_host(self, obj): # get status status = "offline" - if get_string_or_none(grab(obj, "summary.runtime.connectionState")) == "connected": + if get_string_or_none(grab(host_summary, "runtime.connectionState")) == "connected": status = "active" # prepare identifiers to find asset tag and serial number - identifiers = grab(obj, "summary.hardware.otherIdentifyingInfo", fallback=list()) + identifiers = grab(host_summary, "hardware.otherIdentifyingInfo", fallback=list()) identifier_dict = dict() for item in identifiers: value = grab(item, "identifierValue", fallback="") @@ -2858,9 +2988,12 @@ def add_host(self, obj): if len(host_custom_fields) > 0: host_data["custom_fields"] = host_custom_fields + host_config = grab(obj, "config") + host_network = grab(host_config, "network") + # iterate over hosts virtual switches, needed to enrich data on physical interfaces self.network_data["vswitch"][name] = dict() - for vswitch in grab(obj, "config.network.vswitch", fallback=list()): + for vswitch in grab(host_network, "vswitch", fallback=list()): vswitch_name = unquote(grab(vswitch, "name")) @@ -2878,7 +3011,7 @@ def add_host(self, obj): # iterate over hosts proxy switches, needed to enrich data on physical interfaces # also stores data on proxy switch configured mtu which is used for VM interfaces self.network_data["pswitch"][name] = dict() - for pswitch in grab(obj, "config.network.proxySwitch", fallback=list()): + for pswitch in grab(host_network, "proxySwitch", fallback=list()): pswitch_uuid = grab(pswitch, "dvsUuid") pswitch_name = unquote(grab(pswitch, "dvsName")) @@ -2896,7 +3029,7 @@ def add_host(self, obj): # iterate over hosts port groups, needed to enrich data on physical interfaces self.network_data["host_pgroup"][name] = dict() - for pgroup in grab(obj, "config.network.portgroup", fallback=list()): + for pgroup in grab(host_network, "portgroup", fallback=list()): pgroup_name = grab(pgroup, "spec.name") @@ -2928,7 +3061,7 @@ def add_host(self, obj): except Exception as e: log.debug(f"Unable to query network hints (CDP/LLDP) of host '{name}': {e}") - pnic_list = grab(obj, "config.network.pnic", fallback=list()) + pnic_list = grab(host_network, "pnic", fallback=list()) if self.settings.skip_host_nics is True: log.debug(f"Skipping physical interfaces of host '{name}' (skip_host_nics)") pnic_list = list() @@ -3079,7 +3212,7 @@ def add_host(self, obj): fc_hba_data_dict = dict() if self.settings.sync_host_fc_adapters is True: # only Fibre Channel adapters are synced, block and PCIe adapters are ignored - for hba in self.fc_hba_ports(grab(obj, "config.storageDevice.hostBusAdapter", fallback=list()), name): + for hba in self.fc_hba_ports(grab(host_config, "storageDevice.hostBusAdapter", fallback=list()), name): hba_device = get_string_or_none(grab(hba, "device")) if hba_device is None: @@ -3121,7 +3254,7 @@ def add_host(self, obj): # now iterate over all virtual interfaces and collect data vnic_data_dict = dict() vnic_ips = dict() - for vnic in grab(obj, "config.network.vnic", fallback=list()): + for vnic in grab(host_network, "vnic", fallback=list()): vnic_name = grab(vnic, "device") @@ -3357,34 +3490,46 @@ def add_virtual_machine(self, obj): virtual machine object to parse """ - name = get_string_or_none(grab(obj, "name")) + reevaluating_this_vm = self.parsing_objects_to_reevaluate is True and obj in self.objects_to_reevaluate + + # a VM parsed in the first run over all VMs is skipped in the second one by its UUID. Skip it by + # its managed object id before anything is requested from vCenter. + vm_object_key = self.get_vmware_object_key(obj) + if vm_object_key is not None and vm_object_key in self.processed_vm_object_keys and not reevaluating_this_vm: + return + + name = get_string_or_none(self.get_object_name(obj)) name, vm_description = self.get_vm_name_and_description(name) + # each read of a property of 'obj' is a round trip to vCenter which returns the whole property, + # so 'config' and 'runtime' (and 'guest' further down) are requested only once + vm_config = grab(obj, "config") + vm_runtime = grab(obj, "runtime") + # # Filtering # # get VM UUID - vm_uuid = grab(obj, "config.instanceUuid") + vm_uuid = grab(vm_config, "instanceUuid") - if (vm_uuid is None or vm_uuid in self.processed_vm_uuid) and \ - not (self.parsing_objects_to_reevaluate is True and obj in self.objects_to_reevaluate): + if (vm_uuid is None or vm_uuid in self.processed_vm_uuid) and not reevaluating_this_vm: return log.debug(f"Parsing vCenter VM: {name}") # get VM power state - status = "active" if get_string_or_none(grab(obj, "runtime.powerState")) == "poweredOn" else "offline" + status = "active" if get_string_or_none(grab(vm_runtime, "powerState")) == "poweredOn" else "offline" # check if vm is template - template = grab(obj, "config.template") + template = grab(vm_config, "template") if bool(self.settings.skip_vm_templates) is True and template is True: log.debug2(f"VM '{name}' is a template. Skipping") return if bool(self.settings.skip_srm_placeholder_vms) is True \ - and f"{grab(obj, 'config.managedBy.extensionKey')}".startswith("com.vmware.vcDr"): + and f"{grab(vm_config, 'managedBy.extensionKey')}".startswith("com.vmware.vcDr"): log.debug2(f"VM '{name}' is a SRM placeholder VM. Skipping") return @@ -3395,8 +3540,10 @@ def add_virtual_machine(self, obj): # add to processed VMs self.processed_vm_uuid.append(vm_uuid) + if vm_object_key is not None: + self.processed_vm_object_keys.add(vm_object_key) - parent_host = self.get_parent_object_by_class(grab(obj, "runtime.host"), vim.HostSystem) + parent_host = self.get_parent_object_by_class(grab(vm_runtime, "host"), vim.HostSystem) cluster_object = self.get_parent_object_by_class(parent_host, vim.ClusterComputeResource) # get single host 'cluster' if VM runs on one @@ -3419,9 +3566,9 @@ def add_virtual_machine(self, obj): log.debug(f"Virtual machine '{name}' is not part of a permitted cluster. Skipping") return - parent_name = grab(parent_host, "name") + parent_name = self.get_object_name(parent_host) cluster_name = grab(nb_cluster_object, "data.name") - cluster_full_name = f"{group.name}/{cluster_name}" + cluster_full_name = f"{self.get_object_name(group)}/{cluster_name}" if name in self.processed_vm_names.get(cluster_full_name, list()) and obj not in self.objects_to_reevaluate: if self.settings.vm_name_regex is not None: @@ -3455,12 +3602,20 @@ def add_virtual_machine(self, obj): if site_name is None: site_name = self.get_site_name(NBCluster, cluster_full_name) + vm_guest = grab(obj, "guest") + vm_guest_nics = grab(vm_guest, "net", fallback=list()) + + # source_base.add_update_interface() checks the guest tools state for every interface of this VM + self.get_run_cache("vm_guest").clear() + if vm_object_key is not None: + self.get_run_cache("vm_guest")[vm_object_key] = vm_guest + # first check against vm_platform_relation - platform = get_string_or_none(grab(obj, "config.guestFullName")) - platform = get_string_or_none(grab(obj, "guest.guestFullName", fallback=platform)) + platform = get_string_or_none(grab(vm_config, "guestFullName")) + platform = get_string_or_none(grab(vm_guest, "guestFullName", fallback=platform)) # extract prettyName from extraConfig exposed by guest tools - extra_config = {x.key: x.value for x in grab(obj, "config.extraConfig", fallback=[]) + extra_config = {x.key: x.value for x in grab(vm_config, "extraConfig", fallback=[]) if x.key in ["guestOS.detailed.data", "guestInfo.detailed.data"]} # first try 'guestInfo.detailed.data' and then 'guestOS.detailed.data' @@ -3493,11 +3648,11 @@ def add_virtual_machine(self, obj): if platform is not None: platform = self.get_object_relation(platform, "vm_platform_relation", fallback=platform) - hardware_devices = grab(obj, "config.hardware.device", fallback=list()) + hardware_devices = grab(vm_config, "hardware.device", fallback=list()) annotation = None if self.settings.skip_vm_comments is False: - annotation = get_string_or_none(grab(obj, "config.annotation")) + annotation = get_string_or_none(grab(vm_config, "annotation")) # resolve tenant for this VM tenant_name = self.get_vm_tenant_name(obj, name, nb_cluster_object, cluster_full_name) @@ -3519,7 +3674,7 @@ def add_virtual_machine(self, obj): vm_tags.extend(vcenter_tags) # vm memory depending on setting - vm_memory = grab(obj, "config.hardware.memoryMB", fallback=0) + vm_memory = grab(vm_config, "hardware.memoryMB", fallback=0) if self.settings.vm_disk_and_ram_in_decimal is True: vm_memory = int(vm_memory / 1024 * 1000) @@ -3529,7 +3684,7 @@ def add_virtual_machine(self, obj): "cluster": nb_cluster_object, "status": status, "memory": vm_memory, - "vcpus": grab(obj, "config.hardware.numCPU") + "vcpus": grab(vm_config, "hardware.numCPU") } # Add adaption for change in NetBox 3.3.0 VM model @@ -3574,7 +3729,7 @@ def add_virtual_machine(self, obj): vm_default_gateway_ip6 = None # check vm routing to determine which is the default interface for each IP version - for route in grab(obj, "guest.ipStack.0.ipRouteConfig.ipRoute", fallback=list()): + for route in grab(vm_guest, "ipStack.0.ipRouteConfig.ipRoute", fallback=list()): # we found a default route if grab(route, "prefixLength") == 0: @@ -3737,7 +3892,7 @@ def add_virtual_machine(self, obj): int_description = f"{int_description} ({vlan_description})" # find corresponding guest NIC and get IP addresses and connected status - for guest_nic in grab(obj, "guest.net", fallback=list()): + for guest_nic in vm_guest_nics: # get matching guest NIC if int_mac != normalize_mac_address(grab(guest_nic, "macAddress")): @@ -3820,7 +3975,7 @@ def add_virtual_machine(self, obj): dummy_interfaces_filter = self.settings.vm_dummy_interfaces_include_filter if self.settings.sync_vm_dummy_interfaces is True or \ (dummy_interfaces_filter is not None and dummy_interfaces_filter.match(name)): - for guest_nic in grab(obj, "guest.net", fallback=list()): + for guest_nic in vm_guest_nics: # get matching guest NIC MAC guest_nic_mac = normalize_mac_address(grab(guest_nic, "macAddress")) diff --git a/tests/test_vmware_vcenter_requests.py b/tests/test_vmware_vcenter_requests.py new file mode 100644 index 00000000..007f98b5 --- /dev/null +++ b/tests/test_vmware_vcenter_requests.py @@ -0,0 +1,211 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +""" +Reading a property of a pyVmomi managed object is a round trip to vCenter which returns the whole +property ('config' of a host is about 1 MB). The VMware source must not ask vCenter for the same data +over and over again. +""" +import collections +from types import SimpleNamespace + +import pytest + +from module.sources.vmware import connection as vmware_connection +from module.sources.vmware.connection import VMWareHandler + + +@pytest.fixture +def fetch_counter(monkeypatch): + """Counts every property read (SOAP 'Fetch') per managed object and property.""" + from pyVmomi import SoapAdapter + + counts = collections.Counter() + original = SoapAdapter.SoapStubAdapter.InvokeMethod + + def invoke(self, mo, info, args, outerStub=None): + if info.wsdlName == "Fetch": + counts[(mo._wsdlName, mo._moId, args[0])] += 1 + return original(self, mo, info, args, outerStub) + + monkeypatch.setattr(SoapAdapter.SoapStubAdapter, "InvokeMethod", invoke) + return counts + + +def test_host_and_vm_properties_are_not_requested_repeatedly(vmware_source, fetch_counter): + vmware_source.apply() + + assert len(fetch_counter) > 0, "no property reads were counted" + + # an offline VM is looked at in both runs over all VMs, so twice is fine + repeated = {key: count for key, count in fetch_counter.items() + if key[0] in ("VirtualMachine", "HostSystem") and count > 2} + + assert repeated == {}, "properties requested again and again from vCenter" + + +def test_a_vm_parsed_in_the_first_run_is_not_requested_again_in_the_second(vmware_source, fetch_counter, sdk): + from pyVmomi import vim + + view = sdk.viewManager.CreateContainerView(sdk.rootFolder, [vim.VirtualMachine], True) + powered_on = {vm._moId for vm in view.view if vm.runtime.powerState == "poweredOn"} + view.Destroy() + + # the reads above went through the counted stub as well + fetch_counter.clear() + vmware_source.apply() + + reevaluated = {obj._moId for obj in vmware_source.objects_to_reevaluate} + config_reads = [fetch_counter[("VirtualMachine", mo_id, "config")] for mo_id in powered_on - reevaluated] + + assert len(config_reads) > 0 + assert set(config_reads) == {1} + + +# +# unit tests without vcsim +# + +@pytest.fixture(autouse=True) +def dynamic_id(monkeypatch): + """The vSphere automation SDK is optional and absent here, the tag call needs the type.""" + monkeypatch.setattr(vmware_connection, "DynamicID", + lambda **kwargs: SimpleNamespace(**kwargs), raising=False) + + +class _CountingTagging: + """The parts of the vSphere tagging API get_vmware_object_tags() uses, counting each call.""" + + def __init__(self, attached): + self.calls = collections.Counter() + outer = self + + class _Tag: + @staticmethod + def get(tag_id): + outer.calls["Tag.get"] += 1 + return SimpleNamespace(name=f"name-{tag_id}", description="", category_id="cat-1") + + class _Category: + @staticmethod + def get(_category_id): + outer.calls["Category.get"] += 1 + return SimpleNamespace(name="env") + + class _TagAssociation: + @staticmethod + def list_attached_tags(dynamic_id): + outer.calls["list_attached_tags"] += 1 + return attached[dynamic_id.id] + + self.Tag, self.Category, self.TagAssociation = _Tag, _Category, _TagAssociation + + +def _bare_handler(inventory, **settings): + handler = object.__new__(VMWareHandler) + handler.inventory = inventory + handler.name = "test" + handler.source_tag = "Source: test" + handler.settings = SimpleNamespace(**settings) + return handler + + +def test_tag_definitions_and_attached_tags_are_requested_once(inventory): + tagging = _CountingTagging({"vm-1": ["t1", "t2"], "vm-2": ["t1"], "group-v1": ["t2"]}) + handler = _bare_handler(inventory, tag_name_include_category=True) + handler.tag_session = SimpleNamespace(tagging=tagging) + + def vm(mo_id): + return SimpleNamespace(name=mo_id, _wsdlName="VirtualMachine", _moId=mo_id) + + folder = SimpleNamespace(name="folder", _wsdlName="Folder", _moId="group-v1") + + results = list() + for obj in [vm("vm-1"), folder, vm("vm-2"), folder, vm("vm-1")]: + results.append(sorted(t.get_display_name() for t in handler.get_vmware_object_tags(obj))) + + assert results == [["env:name-t1", "env:name-t2"], ["env:name-t2"], ["env:name-t1"], + ["env:name-t2"], ["env:name-t1", "env:name-t2"]] + assert tagging.calls == {"list_attached_tags": 3, "Tag.get": 2, "Category.get": 1} + + +class _Node(SimpleNamespace): + """Fake managed object counting how often its parent is requested.""" + + def __init__(self, mo_id, parent, counter): + super().__init__(_moId=mo_id, _parent=parent, _counter=counter) + + @property + def parent(self): + self._counter[self._moId] += 1 + return self._parent + + +class _Datacenter(_Node): + pass + + +def test_parents_are_requested_once_per_object(): + counter = collections.Counter() + handler = object.__new__(VMWareHandler) + + datacenter = _Datacenter("datacenter-1", None, counter) + folder = _Node("group-h1", datacenter, counter) + cluster = _Node("domain-c1", folder, counter) + + for _ in range(5): + assert handler.get_parent_object_by_class(cluster, _Datacenter) is datacenter + + assert counter == {"domain-c1": 1, "group-h1": 1} + + +def test_unsuccessful_parent_lookups_do_not_break_later_ones(): + # a walk which ended at the root used to leave its depth (11 here) behind, the next walk then + # gave up after 20 - 11 levels and didn't find the datacenter 10 levels up + counter = collections.Counter() + handler = object.__new__(VMWareHandler) + + datacenter = _Datacenter("datacenter-1", None, counter) + chain = datacenter + for level in range(10): + chain = _Node(f"node-{level}", chain, counter) + + class _NotInTree: + pass + + assert handler.get_parent_object_by_class(chain, _NotInTree) is None + + assert handler.get_parent_object_by_class(chain, _Datacenter) is datacenter + + +def test_custom_field_definitions_are_only_requested_for_custom_values(inventory): + handler = _bare_handler(inventory, sync_custom_attributes=True, vm_custom_object_attributes=[], + host_custom_object_attributes=[], custom_attribute_exclude=None, + vm_guest_hostname_custom_field=None) + inventory.netbox_api_version = "4.3.0" + + class _VM(SimpleNamespace): + reads = 0 + + @property + def availableField(self): + _VM.reads += 1 + return [SimpleNamespace(key=1, name="Owner")] + + without_values = _VM(_wsdlName="VirtualMachine", customValue=[]) + with_values = [_VM(_wsdlName="VirtualMachine", customValue=[SimpleNamespace(key=1, value=f"user{i}")]) + for i in range(3)] + + assert handler.get_object_custom_fields(without_values) == dict() + assert _VM.reads == 0 + + results = [handler.get_object_custom_fields(vm) for vm in with_values] + + assert results == [{"vcsa_owner": f"user{i}"} for i in range(3)] + assert _VM.reads == 1 From 6f2f5a7d146083db9a6c194d541611bd0e2e3444 Mon Sep 17 00:00:00 2001 From: tyderian1978 Date: Tue, 17 Nov 2026 22:53:22 +1100 Subject: [PATCH 2/2] vmware: retrieve object properties from vCenter in bulk pyVmomi requests each property read from vCenter separately. Retrieve the properties the handlers need for many objects at once with the PropertyCollector (RetrievePropertiesEx over a ListView, paged with ContinueRetrievePropertiesEx) instead. The new PropertyCache wraps the session's SoapStubAdapter as its 'outerStub', so every managed object returned through it (prefetched objects, their parents, the host and resource pool of a VM, ...) reads prefetched properties from the cache. Properties which were not prefetched are still requested from vCenter: a path missing from the prefetch list costs time but never returns wrong data. The handlers are unchanged apart from reading the nested paths 'config.network', 'config.storageDevice.hostBusAdapter' and custom object attributes through get_prefetched_value(). * objects are prefetched and parsed in chunks of 250, the heavy properties of a chunk are dropped afterwards to bound memory * folders and resource pools are prefetched (name, parent) for the walks up the tree (datacenter, VM folder and resource pool paths) * only the parts of a host's ~1 MB 'config' which are used are requested * the second run over all VMs only requests the VMs not parsed yet * objects to reevaluate are prefetched again in one go * attached vCenter tags are requested per chunk with list_attached_tags_on_objects(), falling back to one request per object * custom object attribute paths vCenter refuses are dropped from the bulk request for the rest of the run and read object by object * if bulk retrieval fails, objects read their properties one by one as before Measured against vcsim with the captured inventories, synced inventory unchanged (relative to the previous commit / to development): vchvr: 323 / 1469 -> 68 SOAP calls, 3.6 / 20.1 -> 1.5 MB vc001: 355 / 1707 -> 66 SOAP calls, 6.0 / 30.1 -> 2.5 MB The number of requests no longer grows with the number of VMs (about 4 per 250 VMs plus one QueryNetworkHint per host). --- module/sources/vmware/connection.py | 282 +++++++++++++++++++++++- module/sources/vmware/property_cache.py | 207 +++++++++++++++++ tests/test_vmware_vcenter_requests.py | 149 ++++++++++++- 3 files changed, 630 insertions(+), 8 deletions(-) create mode 100644 module/sources/vmware/property_cache.py diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 591f1171..e8509961 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -29,6 +29,7 @@ from module.sources.common.source_base import SourceBase from module.sources.vmware.config import VMWareConfig +from module.sources.vmware.property_cache import PropertyCache from module.common.logging import get_logger, DEBUG3 from module.common.misc import grab, dump, get_string_or_none, plural, quoted_split from module.common.support import normalize_mac_address @@ -109,9 +110,14 @@ class VMWareHandler(SourceBase): vm_name_max_length = 64 vm_description_max_length = 200 + # number of objects whose properties are requested from vCenter in one go. Bounds the memory + # the prefetched properties take, a VM 'config' with all its devices can be a few hundred KB. + prefetch_chunk_size = 250 + # internal vars session = None tag_session = None + property_cache = None site_name = None @@ -716,6 +722,17 @@ def apply(self): """ object_mapping = { + # nothing is synced from folders, their names, parents and tags are retrieved in one + # go here instead of one by one while walking up the inventory tree + "folder": { + "view_type": vim.Folder, + "view_handler": None + }, + # the same for resource pools (and vApps), walked for VM resource pool paths + "resource pool": { + "view_type": vim.ResourcePool, + "view_handler": None + }, "datacenter": { "view_type": vim.Datacenter, "view_handler": self.add_datacenter @@ -767,6 +784,9 @@ def apply(self): log.error("Recreating session failed") break + if self.property_cache is None or self.property_cache.stub is not self._sdk_instance._stub: + self.property_cache = PropertyCache.for_stub(self._sdk_instance._stub) + view_data = { "container": self.session.rootFolder, "type": [view_details.get("view_type")], @@ -791,7 +811,17 @@ def apply(self): self.parsing_vms_the_first_time = False log.debug("Iterating over all virtual machines a second time ") - for obj in view_objects: + # VMs parsed in the first run are skipped by add_virtual_machine() anyway, + # don't request their properties again + view_objects = [x for x in view_objects + if self.get_vmware_object_key(x) not in self.processed_vm_object_keys] + + view_handler = view_details.get("view_handler") + + for obj in self.iterate_prefetched(view_objects, view_details.get("view_type")): + + if view_handler is None: + continue if log.level == DEBUG3: try: @@ -800,15 +830,24 @@ def apply(self): log.error(e) # noinspection PyArgumentList - view_details.get("view_handler")(obj) + view_handler(obj) container_view.Destroy() self.parsing_objects_to_reevaluate = True log.info("Parsing objects which were marked to be reevaluated") + # request the properties of all objects to reevaluate in one go as well + prefetched_objects_to_reevaluate = dict() + for object_type in [vim.HostSystem, vim.VirtualMachine]: + objects = [x for x in self.objects_to_reevaluate if isinstance(x, object_type)] + for prefetched_obj in self.prefetch_properties(objects, object_type): + prefetched_objects_to_reevaluate[self.get_vmware_object_key(prefetched_obj)] = prefetched_obj + for obj in self.objects_to_reevaluate: + obj = prefetched_objects_to_reevaluate.get(self.get_vmware_object_key(obj), obj) + if isinstance(obj, vim.HostSystem): self.add_host(obj) elif isinstance(obj, vim.VirtualMachine): @@ -816,6 +855,9 @@ def apply(self): else: log.error(f"Unable to handle reevaluation of {obj} (type: {type(obj)})") + if self.property_cache is not None: + self.property_cache.evict(list(prefetched_objects_to_reevaluate.values())) + self.update_basic_data() @staticmethod @@ -1571,7 +1613,7 @@ def get_object_custom_fields(self, obj): for custom_object_attribute in custom_object_attributes: - attribute_data = grab(obj, custom_object_attribute, fallback="NOT FOUND") + attribute_data = self.get_prefetched_value(obj, custom_object_attribute, fallback="NOT FOUND") if attribute_data == "NOT FOUND": log.warning(f"This object has no attribute '{custom_object_attribute}' or attribute is undefined.") @@ -2141,6 +2183,233 @@ def get_cached_property(self, obj, property_name): return property_cache[object_key] + def get_prefetch_paths(self, object_type): + """ + Return the property paths which are retrieved in bulk for objects of a type. These have to + cover what the add_* handlers read, anything else is requested object by object. + + Parameters + ---------- + object_type: type + pyVmomi type of the objects + + Returns + ------- + tuple: (list, list) the paths the handlers need and the configured custom object attribute paths + """ + + custom_paths = list() + + if object_type is vim.VirtualMachine: + paths = ["name", "parent", "config", "runtime", "guest", "resourcePool"] + custom_paths = self.settings.vm_custom_object_attributes or list() + elif object_type is vim.HostSystem: + # the whole 'config' of a host is about 1 MB, only its network part is needed + paths = ["name", "parent", "summary", "configManager", "config.network"] + if self.settings.sync_host_fc_adapters is True: + paths.append("config.storageDevice.hostBusAdapter") + custom_paths = self.settings.host_custom_object_attributes or list() + elif object_type is vim.dvs.DistributedVirtualPortgroup: + paths = ["key", "name", "config"] + else: + paths = ["name", "parent"] + + if object_type in [vim.VirtualMachine, vim.HostSystem] and self.settings.sync_custom_attributes is True: + paths.append("customValue") + + return paths, [x for x in custom_paths if x not in paths] + + def prefetch_properties(self, objects, object_type): + """ + Retrieve the properties of objects from vCenter in bulk, see get_prefetch_paths() + + Parameters + ---------- + objects: list + managed objects of type 'object_type' + object_type: type + pyVmomi type of the objects + + Returns + ------- + list: the objects in the same order, bound to the property cache. If the properties could not be + retrieved the objects are returned as they are and read their properties one by one. + """ + + if self.property_cache is None or len(objects) == 0: + return objects + + paths, custom_paths = self.get_prefetch_paths(object_type) + + # custom object attribute paths vCenter refused once are read object by object for the rest of the run + refused_custom_paths = self.get_run_cache("refused_custom_paths") + if refused_custom_paths.get(object_type) == custom_paths: + custom_paths = list() + + prefetched_objects = list() + for start in range(0, len(objects), self.prefetch_chunk_size): + chunk = objects[start:start + self.prefetch_chunk_size] + + try: + prefetched_objects.extend(self.property_cache.prefetch(self.session, chunk, object_type, + paths + custom_paths)) + continue + except Exception as e: + if len(custom_paths) == 0: + log.warning(f"Unable to retrieve properties of {len(chunk)} {object_type.__name__} " + f"objects in bulk, requesting them one by one: {e}") + prefetched_objects.extend(chunk) + continue + + # a misspelled custom object attribute is a property path vCenter refuses + log.warning(f"Unable to retrieve properties of {object_type.__name__} objects in bulk including " + f"the custom object attributes {custom_paths}, retrying without them: {e}") + + refused_custom_paths[object_type] = custom_paths + custom_paths = list() + + try: + prefetched_objects.extend(self.property_cache.prefetch(self.session, chunk, object_type, paths)) + except Exception as e: + log.warning(f"Unable to retrieve properties of {len(chunk)} {object_type.__name__} " + f"objects in bulk, requesting them one by one: {e}") + prefetched_objects.extend(chunk) + + return prefetched_objects + + def iterate_prefetched(self, objects, object_type): + """ + Yield the objects in order after their properties (and tags) were retrieved in bulk, a chunk of + prefetch_chunk_size objects at a time. The properties of a chunk are dropped once the next chunk + is requested. + + Parameters + ---------- + objects: list + managed objects of type 'object_type' + object_type: type + pyVmomi type of the objects + """ + + for start in range(0, len(objects), self.prefetch_chunk_size): + chunk = self.prefetch_properties(objects[start:start + self.prefetch_chunk_size], object_type) + + if self.vmware_object_tags_needed(object_type): + self.prefetch_vmware_object_tags(chunk) + + yield from chunk + + if self.property_cache is not None: + self.property_cache.evict(chunk) + + def get_prefetched_value(self, obj, path, fallback=None): + """ + Return a property of an object by its property path. Uses the prefetched value if the path was + retrieved in bulk, otherwise reads the property from vCenter. + + Parameters + ---------- + obj: pyVmomi managed object + object to read the property of + path: str + '.' separated property path + fallback: + returned if the property has no value + + Returns + ------- + the value of the property or fallback + """ + + if self.property_cache is not None: + found, value = self.property_cache.get(obj, path) + if found is True: + return value if value is not None else fallback + + return grab(obj, path, fallback=fallback) + + def vmware_object_tags_needed(self, object_type): + """ + Check if vCenter tags of objects of this type are needed with the configured tag sources + + Parameters + ---------- + object_type: type + pyVmomi type of the objects + + Returns + ------- + bool: True if tags of this object type are synced + """ + + if self.tag_session is None: + return False + + tag_sources = { + "cluster": self.settings.cluster_tag_source or list(), + "host": self.settings.host_tag_source or list(), + "vm": self.settings.vm_tag_source or list() + } + + if object_type is vim.VirtualMachine: + return "object" in tag_sources["vm"] + if object_type is vim.HostSystem: + return "object" in tag_sources["host"] + if object_type in [vim.ClusterComputeResource, vim.ComputeResource]: + return "object" in tag_sources["cluster"] or \ + any("cluster" in x for x in [tag_sources["host"], tag_sources["vm"]]) + if object_type is vim.Datacenter: + return any("datacenter" in x for x in tag_sources.values()) + if object_type is vim.Folder: + return any("parent_folder_1" in x or "parent_folder_2" in x for x in tag_sources.values()) + + return False + + def prefetch_vmware_object_tags(self, objects): + """ + Retrieve the tags attached to a list of objects with one request and store them in the cache + get_vmware_object_tags() reads from. Objects stay uncached if the request fails, their tags are + then requested one by one. + + Parameters + ---------- + objects: list + pyVmomi managed objects + """ + + if self.tag_session is None or len(objects) == 0: + return + + attached_tags_cache = self.get_run_cache("attached_tags") + + object_keys = list() + for obj in objects: + object_key = (grab(obj, "_wsdlName"), grab(obj, "_moId")) + if None not in object_key and object_key not in attached_tags_cache: + object_keys.append(object_key) + + if len(object_keys) == 0: + return + + # noinspection PyBroadException + try: + attached_tags = self.tag_session.tagging.TagAssociation.list_attached_tags_on_objects( + [DynamicID(type=object_type, id=object_id) for object_type, object_id in object_keys]) + except Exception as e: + log.debug(f"Unable to retrieve vCenter tags of {len(object_keys)} objects at once, " + f"requesting them object by object: {e}") + return + + # objects without tags are not part of the answer + requested_tags = {object_key: list() for object_key in object_keys} + + for object_tags in attached_tags or list(): + object_key = (grab(object_tags, "object_id.type"), grab(object_tags, "object_id.id")) + if object_key in requested_tags: + requested_tags[object_key] = list(grab(object_tags, "tag_ids", fallback=list())) + + attached_tags_cache.update(requested_tags) + def get_vmware_guest_info(self, vmware_object): """ Return the guest info of the VM which is currently parsed by add_virtual_machine() without @@ -2988,8 +3257,8 @@ def add_host(self, obj): if len(host_custom_fields) > 0: host_data["custom_fields"] = host_custom_fields - host_config = grab(obj, "config") - host_network = grab(host_config, "network") + # only the parts of the ~1 MB 'config' of a host which are used are retrieved in bulk + host_network = self.get_prefetched_value(obj, "config.network") # iterate over hosts virtual switches, needed to enrich data on physical interfaces self.network_data["vswitch"][name] = dict() @@ -3212,7 +3481,8 @@ def add_host(self, obj): fc_hba_data_dict = dict() if self.settings.sync_host_fc_adapters is True: # only Fibre Channel adapters are synced, block and PCIe adapters are ignored - for hba in self.fc_hba_ports(grab(host_config, "storageDevice.hostBusAdapter", fallback=list()), name): + for hba in self.fc_hba_ports(self.get_prefetched_value(obj, "config.storageDevice.hostBusAdapter", + fallback=list()), name): hba_device = get_string_or_none(grab(hba, "device")) if hba_device is None: diff --git a/module/sources/vmware/property_cache.py b/module/sources/vmware/property_cache.py new file mode 100644 index 00000000..c9043e2e --- /dev/null +++ b/module/sources/vmware/property_cache.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +""" +Bulk retrieval of vCenter managed object properties. + +pyVmomi requests a property of a managed object from vCenter each time it is read, one SOAP round +trip per read. The PropertyCache retrieves the properties of many objects with one PropertyCollector +request and hands out managed objects which read these properties from the cache instead. Properties +which were not retrieved in advance are requested from vCenter as usual, so reading a property which +is missing from the prefetch list costs time, but never returns wrong data. +""" + +from pyVmomi import vim, vmodl, SoapAdapter +from pyVmomi.StubAdapterAccessorImpl import StubAdapterAccessorMixin + +from module.common.logging import get_logger + +log = get_logger() + + +class PropertyCache: + """ + A pyVmomi stub which wraps the stub of the vCenter session. + + Every managed object returned by a request through this stub (the objects of a prefetch, their + parents, the host of a VM, ...) uses this stub as well. Reading a property of such an object is + answered from the cache if the property was prefetched for this object. + + Cached values are stored per managed object id under the property path they were requested with. + pyVmomi reads properties top level only ('config'), a top level path is answered by the stub + directly. Nested paths ('config.network') are read with get(). + """ + + def __init__(self, stub): + self.stub = stub + self.cache = dict() + + @classmethod + def for_stub(cls, stub): + """ + Return a PropertyCache for the stub of a vCenter session or None if the stub can't be wrapped + + Parameters + ---------- + stub: pyVmomi stub + stub of the vCenter session + + Returns + ------- + PropertyCache, None: the PropertyCache + """ + + # handing out objects bound to the wrapper relies on the 'outerStub' of SoapStubAdapter + if not isinstance(stub, SoapAdapter.SoapStubAdapter): + log.debug(f"Unable to retrieve vCenter object properties in bulk with a '{type(stub).__name__}'") + return None + + return cls(stub) + + def __getattr__(self, name): + # everything pyVmomi expects from a stub (version, SupportServerGUIDs, ...) + return getattr(self.stub, name) + + def InvokeMethod(self, mo, info, args): + """pyVmomi stub interface: invoke a method, returned managed objects are bound to this stub""" + + status, result = self.stub.InvokeMethod(mo, info, args, self) + + if status == 200: + return result + + if isinstance(result, Exception): + raise result + + raise Exception(f"vCenter returned status {status} for '{info.wsdlName}': {result}") + + def InvokeAccessor(self, mo, info): + """pyVmomi stub interface: read a property of a managed object""" + + properties = self.cache.get(mo._moId) + + if properties is not None and info.name in properties: + return properties[info.name] + + value = StubAdapterAccessorMixin.InvokeAccessor(self, mo, info) + + # remember properties of prefetched objects which were read in addition + if properties is not None: + properties[info.name] = value + + return value + + def get(self, obj, path): + """ + Return a prefetched property of an object + + Parameters + ---------- + obj: pyVmomi managed object + object to return the property for + path: str + the property path exactly as it was prefetched + + Returns + ------- + tuple: (bool, value) True and the value if the property was prefetched, (False, None) otherwise + """ + + if getattr(obj, "_stub", None) is not self: + return False, None + + properties = self.cache.get(obj._moId) + + if properties is None or path not in properties: + return False, None + + return True, properties[path] + + def prefetch(self, service_content, objects, object_type, paths): + """ + Retrieve properties of a list of objects from vCenter with one PropertyCollector request + + Parameters + ---------- + service_content: vim.ServiceInstanceContent + content of the vCenter session + objects: list + managed objects of type 'object_type' + object_type: type + the pyVmomi type of the objects + paths: list + property paths to retrieve for each object + + Returns + ------- + list: the objects in the same order, bound to this stub + """ + + if len(objects) == 0: + return list() + + retrieved = dict() + + # a ListView drops objects which got deleted in the meantime, explicit object specs would fail + list_view = service_content.viewManager.CreateListView(obj=objects) + + try: + traversal_spec = vmodl.query.PropertyCollector.TraversalSpec( + name="traverseList", path="view", skip=False, type=vim.view.ListView) + + filter_spec = vmodl.query.PropertyCollector.FilterSpec( + objectSet=[vmodl.query.PropertyCollector.ObjectSpec(obj=list_view, skip=True, + selectSet=[traversal_spec])], + propSet=[vmodl.query.PropertyCollector.PropertySpec(type=object_type, pathSet=list(paths))]) + + collector = vmodl.query.PropertyCollector(service_content.propertyCollector._moId, self) + + result = collector.RetrievePropertiesEx([filter_spec], vmodl.query.PropertyCollector.RetrieveOptions()) + + while result is not None: + for object_content in result.objects or list(): + # a property which is unset (or not readable) is not returned, reading it + # from vCenter directly yields no value either + properties = dict.fromkeys(paths) + properties.update({x.name: x.val for x in object_content.propSet or list()}) + retrieved[object_content.obj._moId] = properties + + result = collector.ContinueRetrievePropertiesEx(result.token) if result.token else None + + finally: + list_view.Destroy() + + # only now, a request which failed half way through leaves no partial data behind + self.cache.update(retrieved) + + return [type(x)(x._moId, stub=self, serverGuid=x._serverGuid) for x in objects] + + def evict(self, objects, keep=("name", "parent")): + """ + Drop prefetched properties of objects which are not needed anymore + + Parameters + ---------- + objects: list + managed objects + keep: tuple + properties to keep, small ones which are read again for other objects + """ + + for obj in objects: + properties = self.cache.get(obj._moId) + + if properties is None: + continue + + for path in list(properties.keys()): + if path not in keep: + del properties[path] + +# EOF diff --git a/tests/test_vmware_vcenter_requests.py b/tests/test_vmware_vcenter_requests.py index 007f98b5..73cbaa4f 100644 --- a/tests/test_vmware_vcenter_requests.py +++ b/tests/test_vmware_vcenter_requests.py @@ -23,16 +23,33 @@ @pytest.fixture def fetch_counter(monkeypatch): - """Counts every property read (SOAP 'Fetch') per managed object and property.""" + """ + Counts how often each property of each managed object was transferred from vCenter, read one by + one (SOAP 'Fetch') or in bulk (PropertyCollector). The keys of the bulk reads carry the property + path as requested. The SOAP methods called are counted under the key 'methods'. + """ from pyVmomi import SoapAdapter counts = collections.Counter() + counts.methods = collections.Counter() + counts.single_reads = collections.Counter() original = SoapAdapter.SoapStubAdapter.InvokeMethod def invoke(self, mo, info, args, outerStub=None): + counts.methods[info.wsdlName] += 1 if info.wsdlName == "Fetch": counts[(mo._wsdlName, mo._moId, args[0])] += 1 - return original(self, mo, info, args, outerStub) + counts.single_reads[(mo._wsdlName, mo._moId, args[0])] += 1 + + result = original(self, mo, info, args, outerStub) + + if info.wsdlName in ["RetrievePropertiesEx", "ContinueRetrievePropertiesEx"]: + retrieved = result[1] if isinstance(result, tuple) else result + for object_content in getattr(retrieved, "objects", None) or list(): + for prop in object_content.propSet or list(): + counts[(object_content.obj._wsdlName, object_content.obj._moId, prop.name)] += 1 + + return result monkeypatch.setattr(SoapAdapter.SoapStubAdapter, "InvokeMethod", invoke) return counts @@ -59,6 +76,7 @@ def test_a_vm_parsed_in_the_first_run_is_not_requested_again_in_the_second(vmwar # the reads above went through the counted stub as well fetch_counter.clear() + fetch_counter.single_reads.clear() vmware_source.apply() reevaluated = {obj._moId for obj in vmware_source.objects_to_reevaluate} @@ -68,6 +86,68 @@ def test_a_vm_parsed_in_the_first_run_is_not_requested_again_in_the_second(vmwar assert set(config_reads) == {1} +def test_vm_and_host_properties_are_retrieved_in_bulk(vmware_source, fetch_counter): + vmware_source.apply() + + single_reads = [key for key in fetch_counter.single_reads if key[0] in ("VirtualMachine", "HostSystem")] + + assert fetch_counter.methods["RetrievePropertiesEx"] > 0 + assert single_reads == [], "properties requested object by object" + + +def test_property_cache_serves_prefetched_properties(sdk, fetch_counter): + from pyVmomi import vim + from module.sources.vmware.property_cache import PropertyCache + + view = sdk.viewManager.CreateContainerView(sdk.rootFolder, [vim.VirtualMachine], True) + vms = list(view.view) + view.Destroy() + expected = [(vm.name, vm.runtime.powerState) for vm in vms] + + cache = PropertyCache.for_stub(vms[0]._stub) + prefetched = cache.prefetch(sdk, vms, vim.VirtualMachine, ["name", "runtime.powerState"]) + + # same objects in the same order, now reading through the cache + assert prefetched == vms + assert all(vm._stub is cache for vm in prefetched) + + fetch_counter.single_reads.clear() + assert [(vm.name, cache.get(vm, "runtime.powerState")[1]) for vm in prefetched] == expected + assert fetch_counter.single_reads == {}, "prefetched properties were requested again" + + # a property which was not prefetched is requested once, objects reached through it use the cache too + host = prefetched[0].runtime.host + prefetched[0].runtime + assert fetch_counter.single_reads == {("VirtualMachine", vms[0]._moId, "runtime"): 1} + assert host._stub is cache + + # only small properties survive eviction + cache.evict(prefetched) + assert cache.get(prefetched[0], "name") == (True, expected[0][0]) + assert cache.get(prefetched[0], "runtime.powerState") == (False, None) + + +def test_an_unknown_custom_object_attribute_does_not_break_the_sync(vcsim, inventory, load_config, vmware_settings, + sdk): + from pyVmomi import vim + from module.netbox.object_classes import NBVM + from module.sources import instantiate_sources + + load_config(vmware_settings + "vm_custom_object_attributes = config.doesNotExist, config.uuid\n") + source = instantiate_sources()[0] + inventory.resolve_relations() + source.apply() + + view = sdk.viewManager.CreateContainerView(sdk.rootFolder, [vim.VirtualMachine], True) + uuids = {vm.name: vm.config.uuid for vm in view.view} + view.Destroy() + + synced = {vm.get_display_name(): vm for vm in inventory.get_all_items(NBVM)} + assert set(synced) == set(uuids) + for name, vm in synced.items(): + assert uuids[name] in vm.data.get("custom_fields", dict()).values() + + # # unit tests without vcsim # @@ -209,3 +289,68 @@ def availableField(self): assert results == [{"vcsa_owner": f"user{i}"} for i in range(3)] assert _VM.reads == 1 + + +class _BulkTagging(_CountingTagging): + """Adds the bulk request for attached tags, optionally failing like an old vCenter would.""" + + def __init__(self, attached, bulk_fails=False): + super().__init__(attached) + outer = self + + class _TagAssociation(self.TagAssociation): + @staticmethod + def list_attached_tags_on_objects(dynamic_ids): + outer.calls["list_attached_tags_on_objects"] += 1 + if bulk_fails: + raise RuntimeError("not supported") + # objects without tags are left out, as vCenter does + return [SimpleNamespace(object_id=x, tag_ids=attached[x.id]) for x in dynamic_ids if attached[x.id]] + + self.TagAssociation = _TagAssociation + + +@pytest.mark.parametrize("bulk_fails", [False, True], ids=["bulk", "bulk-fails"]) +def test_attached_tags_are_retrieved_in_bulk(inventory, bulk_fails): + attached = {"vm-1": ["t1", "t2"], "vm-2": [], "vm-3": ["t1"]} + tagging = _BulkTagging(attached, bulk_fails=bulk_fails) + handler = _bare_handler(inventory, tag_name_include_category=False) + handler.tag_session = SimpleNamespace(tagging=tagging) + + vms = [SimpleNamespace(name=x, _wsdlName="VirtualMachine", _moId=x) for x in attached] + + handler.prefetch_vmware_object_tags(vms) + results = [sorted(t.get_display_name() for t in handler.get_vmware_object_tags(vm)) for vm in vms] + + assert results == [["name-t1", "name-t2"], [], ["name-t1"]] + assert tagging.calls["list_attached_tags_on_objects"] == 1 + assert tagging.calls["list_attached_tags"] == (3 if bulk_fails else 0) + + +def test_an_invalid_custom_object_attribute_path_is_retried_without_it(inventory): + # vCenter answers a PropertyCollector request with an invalid property path with an + # InvalidProperty fault for the whole request, vcsim just leaves the path out + from pyVmomi import vim + + class _RefusingCache: + def __init__(self): + self.requests = list() + + def prefetch(self, _session, objects, _object_type, paths): + self.requests.append(list(paths)) + if "config.doesNotExist" in paths: + raise vim.fault.InvalidProperty(name="config.doesNotExist") + return [f"prefetched-{x}" for x in objects] + + handler = _bare_handler(inventory, sync_custom_attributes=False, + vm_custom_object_attributes=["config.doesNotExist", "config.uuid"]) + handler.property_cache = _RefusingCache() + handler.prefetch_chunk_size = 2 + + assert handler.prefetch_properties(["vm-1", "vm-2", "vm-3"], vim.VirtualMachine) == \ + ["prefetched-vm-1", "prefetched-vm-2", "prefetched-vm-3"] + + base_paths = handler.get_prefetch_paths(vim.VirtualMachine)[0] + with_custom = base_paths + ["config.doesNotExist", "config.uuid"] + # the refused paths are not tried again for the next chunk + assert handler.property_cache.requests == [with_custom, base_paths, base_paths]