diff --git a/osism/tasks/conductor/ironic.py b/osism/tasks/conductor/ironic.py index abf36554b..33b8603a7 100644 --- a/osism/tasks/conductor/ironic.py +++ b/osism/tasks/conductor/ironic.py @@ -24,6 +24,7 @@ deep_merge, get_vault, mask_secrets, + VaultDecryptError, ) SUPPORTED_IPA_TYPES = { @@ -197,7 +198,7 @@ def _prepare_node_attributes( and "ironic_parameters" in device.config_context ): config_context_ironic = device.config_context["ironic_parameters"] - deep_decrypt(config_context_ironic, vault) + deep_decrypt(config_context_ironic, vault, "config_context.ironic_parameters") deep_merge(node_attributes, config_context_ironic) # Decrypt and merge ironic_parameters Custom Field if present @@ -206,14 +207,14 @@ def _prepare_node_attributes( and device.custom_fields["ironic_parameters"] ): ironic_parameters_cf = device.custom_fields["ironic_parameters"] - deep_decrypt(ironic_parameters_cf, vault) + deep_decrypt(ironic_parameters_cf, vault, "ironic_parameters") deep_merge(node_attributes, ironic_parameters_cf) # Decrypt secrets Custom Field node_secrets = device.custom_fields.get("secrets", {}) if node_secrets is None: node_secrets = {} - deep_decrypt(node_secrets, vault) + deep_decrypt(node_secrets, vault, "secrets") if ( "driver" in node_attributes @@ -262,7 +263,7 @@ def _prepare_node_attributes( if match and match.group(1) in SUPPORTED_IPA_TYPES: ipa_type = match.group(1) frr = device.custom_fields.get("frr_parameters") or {} - deep_decrypt(frr, vault) + deep_decrypt(frr, vault, "frr_parameters") derived_as = ( _derive_as_from_hostname_yrzn(device.name) if ipa_type == "yrzn001" @@ -326,7 +327,7 @@ def _prepare_node_attributes( and device.custom_fields["frr_parameters"] ): frr_params = device.custom_fields["frr_parameters"] - deep_decrypt(frr_params, vault) + deep_decrypt(frr_params, vault, "frr_parameters") node_attributes["extra"].update({"frr_parameters": json.dumps(frr_params)}) return node_attributes, template_vars @@ -764,6 +765,29 @@ def sync_ironic( osism_utils.finish_task_output(request_id, rc=1) return + # NOTE: Prepare all nodes before changing anything. A secret that cannot + # be decrypted would otherwise be replaced by defaults and written to + # ironic, so abort the whole sync instead. + prepared = [] + for device in devices: + try: + node_attributes, template_vars = _prepare_node_attributes( + device, + get_ironic_parameters, + skip_kernel_params=skip_kernel_params, + extra_kernel_params=extra_kernel_params, + ) + except VaultDecryptError as exc: + osism_utils.push_task_output( + request_id, + f"ERROR: Device {device.name}: {exc}\n" + "Aborting, no changes were made. Check the Ansible Vault " + "password ('osism set vault password').\n", + ) + osism_utils.finish_task_output(request_id, rc=1) + return + prepared.append((device, node_attributes, template_vars)) + # NOTE: Find nodes in Ironic which are no longer present in NetBox and remove them device_names = {dev.name for dev in devices} nodes = openstack.baremetal_node_list() @@ -813,19 +837,12 @@ def sync_ironic( ) # NOTE: Find nodes in NetBox which are not present in Ironic and add them - for device in devices: + for device, node_attributes, template_vars in prepared: osism_utils.push_task_output( request_id, f"Looking for {device.name} in ironic\n" ) node_interfaces = list(netbox.get_interfaces_by_device(device.name)) - - node_attributes, template_vars = _prepare_node_attributes( - device, - get_ironic_parameters, - skip_kernel_params=skip_kernel_params, - extra_kernel_params=extra_kernel_params, - ) ports_attributes = [ dict(address=interface.mac_address) for interface in node_interfaces diff --git a/osism/tasks/conductor/utils.py b/osism/tasks/conductor/utils.py index 1a81e394e..25908397a 100644 --- a/osism/tasks/conductor/utils.py +++ b/osism/tasks/conductor/utils.py @@ -46,27 +46,43 @@ def deep_merge(a, b): deep_merge(a[key], value) -def deep_decrypt(a, vault): - if a is None: - return +class VaultDecryptError(Exception): + """A vault-encrypted value could not be decrypted.""" + + +def deep_decrypt(a, vault, path=""): + """Decrypt all vault-encrypted strings in ``a`` in place. + + Every value that can be decrypted is decrypted. If any value fails, + VaultDecryptError names all failing keys afterwards, so that a caller + never continues with a secret silently missing or still encrypted. + """ + failed = [] + _deep_decrypt(a, vault, path, failed) + if failed: + paths = ", ".join(f"'{key_path}'" for key_path, _ in failed) + first_exc = failed[0][1] + raise VaultDecryptError( + f"Unable to decrypt {paths}: {first_exc}" + ) from first_exc + + +def _deep_decrypt(a, vault, path, failed): if isinstance(a, dict): - for key, value in list(a.items()): - if isinstance(value, (dict, list)): - deep_decrypt(a[key], vault) - elif vault.is_encrypted(value): - try: - a[key] = vault.decrypt(value).decode().strip() - except Exception: - a.pop(key, None) + items = list(a.items()) elif isinstance(a, list): - for i, item in enumerate(a): - if isinstance(item, (dict, list)): - deep_decrypt(item, vault) - elif vault.is_encrypted(item): - try: - a[i] = vault.decrypt(item).decode().strip() - except Exception: - pass + items = list(enumerate(a)) + else: + return + for key, value in items: + key_path = f"{path}.{key}" if path else str(key) + if isinstance(value, (dict, list)): + _deep_decrypt(value, vault, key_path, failed) + elif vault.is_encrypted(value): + try: + a[key] = vault.decrypt(value).decode().strip() + except Exception as exc: + failed.append((key_path, exc)) def get_vault(): @@ -89,7 +105,6 @@ def get_vault(): except Exception as exc: # Handle other errors (file access, decryption, etc.) logger.error(f"Unable to get vault secret: {exc}") - logger.error("Dropping encrypted entries") vault = VaultLib() return vault diff --git a/osism/tasks/openstack.py b/osism/tasks/openstack.py index 32ffc0eff..cf79dc8c3 100644 --- a/osism/tasks/openstack.py +++ b/osism/tasks/openstack.py @@ -226,7 +226,12 @@ def _mask_node_secret_parameters( The ``(kernel_append_params, netplan_parameters, frr_parameters)`` triple with secret values replaced by ``***``. """ - from osism.tasks.conductor.utils import deep_decrypt, get_vault, mask_secrets + from osism.tasks.conductor.utils import ( + VaultDecryptError, + deep_decrypt, + get_vault, + mask_secrets, + ) secret_values = set() if utils.nb and node_name: @@ -243,7 +248,7 @@ def _mask_node_secret_parameters( node_secrets = {} # Collect ironic_osism_* parameter names BEFORE decryption - # so they're available even if vault decryption removes them. + # so they're available even if vault decryption fails. # Convention: ironic_osism_aa -> osism-aa in kernel_append_params secret_param_names = set() for key in node_secrets: @@ -253,8 +258,11 @@ def _mask_node_secret_parameters( ) secret_param_names.add(param_name) - vault = get_vault() - deep_decrypt(node_secrets, vault) + try: + deep_decrypt(node_secrets, get_vault()) + except VaultDecryptError as exc: + # The name-based masking below still applies + logger.warning(f"Could not decrypt secrets of {node_name}: {exc}") # Collect secret values using the same logic as dry-run # (matches _is_secret_key + dry-run secret collection) diff --git a/tests/unit/tasks/conductor/test_ironic_sync.py b/tests/unit/tasks/conductor/test_ironic_sync.py index 52cbb1ef3..8dcbb5ba5 100644 --- a/tests/unit/tasks/conductor/test_ironic_sync.py +++ b/tests/unit/tasks/conductor/test_ironic_sync.py @@ -766,6 +766,45 @@ def test_sync_ironic_stale_node_not_eligible(sync_env): sync_env.openstack.baremetal_node_delete.assert_not_called() +def _fail_on_second_device(sync_env, error): + """Two devices; preparing the first succeeds, the second raises.""" + sync_env.netbox.get_devices.return_value = [ + _make_device(name="node1"), + _make_device(name="node2"), + ] + sync_env.prepare.side_effect = [({"driver": "redfish"}, {}), error] + + +def test_sync_ironic_aborts_before_any_change_on_vault_error(sync_env): + _fail_on_second_device( + sync_env, + ironic.VaultDecryptError( + "Unable to decrypt 'secrets.remote_board_password': no vault secrets" + ), + ) + sync_env.openstack.baremetal_node_list.return_value = [ + _make_node(name="stale", uuid="u-stale", provision_state="available") + ] + + ironic.sync_ironic("req", MagicMock()) + + assert sync_env.prepare.call_count == 2 + assert _pushed(sync_env.utils, "secrets.remote_board_password") + assert _pushed(sync_env.utils, "no changes were made") + sync_env.utils.finish_task_output.assert_called_once_with("req", rc=1) + sync_env.openstack.baremetal_node_delete.assert_not_called() + sync_env.sync_device.assert_not_called() + + +def test_sync_ironic_dry_run_aborts_on_vault_error(sync_env): + _fail_on_second_device(sync_env, ironic.VaultDecryptError("boom")) + + ironic.sync_ironic("req", MagicMock(), dry_run=True) + + sync_env.utils.finish_task_output.assert_called_once_with("req", rc=1) + sync_env.sync_dry.assert_not_called() + + def test_sync_ironic_kernel_params_propagated(sync_env): sync_env.netbox.get_devices.return_value = [_make_device()] diff --git a/tests/unit/tasks/conductor/test_utils.py b/tests/unit/tasks/conductor/test_utils.py index c5260286e..4273fd316 100644 --- a/tests/unit/tasks/conductor/test_utils.py +++ b/tests/unit/tasks/conductor/test_utils.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 +import re from unittest.mock import MagicMock, patch import pytest @@ -10,6 +11,7 @@ DELETE_SENTINEL, _get_conductor_redfish_address, _get_conductor_redfish_credentials, + VaultDecryptError, _is_secret_key, deep_compare, deep_decrypt, @@ -301,30 +303,43 @@ def test_deep_decrypt_top_level_list_replaces_encrypted_strings(): assert data == ["plain-1", "literal", "plain-2"] -def test_deep_decrypt_dict_value_decrypt_failure_drops_key(): +def test_deep_decrypt_dict_value_decrypt_failure_raises_with_path(): vault = _make_vault( - encrypted_values=["broken", "ok"], - decrypt_map={"ok": b"clear"}, + encrypted_values=["broken"], decrypt_errors={"broken": AnsibleError("bad signature")}, ) - data = {"a": "broken", "b": "ok"} + data = {"outer": {"a": "broken"}} - deep_decrypt(data, vault) + with pytest.raises(VaultDecryptError, match=re.escape("'secrets.outer.a'")): + deep_decrypt(data, vault, "secrets") + + +def test_deep_decrypt_list_element_decrypt_failure_raises_with_index(): + vault = _make_vault( + encrypted_values=["broken"], + decrypt_errors={"broken": AnsibleError("bad signature")}, + ) + data = ["ok", "broken"] - assert data == {"b": "clear"} + with pytest.raises(VaultDecryptError, match="'1'.*bad signature"): + deep_decrypt(data, vault) -def test_deep_decrypt_list_element_decrypt_failure_left_in_place(): +def test_deep_decrypt_decrypts_the_rest_before_raising(): vault = _make_vault( - encrypted_values=["broken", "ok"], + encrypted_values=["broken", "ok", "also-broken"], decrypt_map={"ok": b"clear"}, - decrypt_errors={"broken": AnsibleError("bad signature")}, + decrypt_errors={ + "broken": AnsibleError("bad signature"), + "also-broken": AnsibleError("bad signature"), + }, ) - data = ["broken", "ok"] + data = {"a": "broken", "b": "ok", "c": ["also-broken"]} - deep_decrypt(data, vault) + with pytest.raises(VaultDecryptError, match=re.escape("'a', 'c.0': bad signature")): + deep_decrypt(data, vault) - assert data == ["broken", "clear"] + assert data == {"a": "broken", "b": "clear", "c": ["also-broken"]} def test_deep_decrypt_strips_whitespace_after_decode(): diff --git a/tests/unit/tasks/test_openstack_baremetal.py b/tests/unit/tasks/test_openstack_baremetal.py index 13e2ab94b..aba64fc64 100644 --- a/tests/unit/tasks/test_openstack_baremetal.py +++ b/tests/unit/tasks/test_openstack_baremetal.py @@ -29,6 +29,7 @@ import pytest from osism.tasks import openstack as openstack_tasks +from osism.tasks.conductor.utils import VaultDecryptError def _has_log(records, level, substring): @@ -514,11 +515,9 @@ def test_mask_masks_secret_values(node_params_env): def test_mask_ironic_osism_param_by_name(node_params_env): """The ``ironic_osism_aa`` param name is collected before decryption, so the ``osism-aa=...`` argument is masked by the name-based regex even when - vault decryption drops the key and its value cannot be collected.""" - node_params_env.device.custom_fields["secrets"] = {"ironic_osism_aa": "topsecret"} - node_params_env.deep_decrypt.side_effect = lambda secrets, vault: secrets.pop( - "ironic_osism_aa", None - ) + vault decryption fails and its value cannot be collected.""" + node_params_env.device.custom_fields["secrets"] = {"ironic_osism_aa": "$ANSIBLE"} + node_params_env.deep_decrypt.side_effect = VaultDecryptError("no vault secrets") kernel, _, _ = openstack_tasks._mask_node_secret_parameters( "node-1", "osism-aa=topsecret other=x", {}, {} @@ -527,6 +526,23 @@ def test_mask_ironic_osism_param_by_name(node_params_env): assert kernel == "osism-aa=*** other=x" +def test_mask_uses_values_decrypted_before_the_error(node_params_env): + """Values decrypted in place stay collected when another one fails.""" + node_params_env.device.custom_fields["secrets"] = {"os_password": "$ANSIBLE"} + + def decrypt_partially(secrets, vault): + secrets["os_password"] = "hunter2" + raise VaultDecryptError("Unable to decrypt 'other'") + + node_params_env.deep_decrypt.side_effect = decrypt_partially + + kernel, _, _ = openstack_tasks._mask_node_secret_parameters( + "node-1", "pw=hunter2", {}, {} + ) + + assert kernel == "pw=***" + + def test_mask_ignores_non_string_secret_values(node_params_env): node_params_env.device.custom_fields["secrets"] = {"db_password": 42}