From 64f6c85f7eac235f0a1a3b4b176ad79db40260fb Mon Sep 17 00:00:00 2001 From: Roger Luethi Date: Mon, 16 Nov 2026 12:45:25 +0100 Subject: [PATCH] Abort sync ironic on vault decryption failure deep_decrypt() silently dropped every dict value it could not decrypt and left undecryptable list elements as ciphertext. get_vault() only logs when the vault password is missing, so a missing or wrong password went unnoticed: sync ironic carried on, the node secrets lacked remote_board_username/remote_board_password, and _prepare_node_attributes() fell back to the hard-coded "admin" and "password". Because node updates send the full node attributes, those defaults replaced the working BMC credentials in driver_info whenever the node was created, --force was given, or any other attribute differed. deep_decrypt() now decrypts every value it can and then raises VaultDecryptError naming all keys that failed (e.g. 'secrets.remote_board_password'), for dicts and lists alike. Carrying on past a failure matters for callers that catch the error: they still get every value that could be decrypted, e.g. when some entries were encrypted with a different vault password. get_vault() stays lenient, since it is also called for devices without any encrypted values, and those must keep working without a vault password; its "Dropping encrypted entries" message is gone because nothing is dropped anymore. sync_ironic() now prepares the attributes of all devices before it deletes stale nodes or touches any node. On a decryption error it reports the device and key, states that no changes were made, and finishes with rc=1 so the CLI fails rather than printing a warning that scrolls past. Dry runs abort the same way, since their output would otherwise present the fallback credentials as the planned change. The other callers of deep_decrypt() now fail instead of continuing with secrets missing: the SONiC config generator already turns a per-device exception into a failed device and a non-zero rc, and baremetal deploy/dump stop before applying FRR parameters with missing secrets. The secret-masking helper in tasks/openstack.py is the exception: it only redacts kernel parameters for display, and it must keep masking ironic_osism_* parameters by name when their values cannot be decrypted. Its outer exception handler would skip that, so it catches VaultDecryptError around the decryption, logs a warning and carries on with the values that were decrypted. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Roger Luethi --- osism/tasks/conductor/ironic.py | 43 ++++++++++----- osism/tasks/conductor/utils.py | 55 ++++++++++++------- osism/tasks/openstack.py | 16 ++++-- .../unit/tasks/conductor/test_ironic_sync.py | 39 +++++++++++++ tests/unit/tasks/conductor/test_utils.py | 39 +++++++++---- tests/unit/tasks/test_openstack_baremetal.py | 26 +++++++-- 6 files changed, 164 insertions(+), 54 deletions(-) diff --git a/osism/tasks/conductor/ironic.py b/osism/tasks/conductor/ironic.py index abf36554b..33b8603a7 100644 --- a/osism/tasks/conductor/ironic.py +++ b/osism/tasks/conductor/ironic.py @@ -24,6 +24,7 @@ deep_merge, get_vault, mask_secrets, + VaultDecryptError, ) SUPPORTED_IPA_TYPES = { @@ -197,7 +198,7 @@ def _prepare_node_attributes( and "ironic_parameters" in device.config_context ): config_context_ironic = device.config_context["ironic_parameters"] - deep_decrypt(config_context_ironic, vault) + deep_decrypt(config_context_ironic, vault, "config_context.ironic_parameters") deep_merge(node_attributes, config_context_ironic) # Decrypt and merge ironic_parameters Custom Field if present @@ -206,14 +207,14 @@ def _prepare_node_attributes( and device.custom_fields["ironic_parameters"] ): ironic_parameters_cf = device.custom_fields["ironic_parameters"] - deep_decrypt(ironic_parameters_cf, vault) + deep_decrypt(ironic_parameters_cf, vault, "ironic_parameters") deep_merge(node_attributes, ironic_parameters_cf) # Decrypt secrets Custom Field node_secrets = device.custom_fields.get("secrets", {}) if node_secrets is None: node_secrets = {} - deep_decrypt(node_secrets, vault) + deep_decrypt(node_secrets, vault, "secrets") if ( "driver" in node_attributes @@ -262,7 +263,7 @@ def _prepare_node_attributes( if match and match.group(1) in SUPPORTED_IPA_TYPES: ipa_type = match.group(1) frr = device.custom_fields.get("frr_parameters") or {} - deep_decrypt(frr, vault) + deep_decrypt(frr, vault, "frr_parameters") derived_as = ( _derive_as_from_hostname_yrzn(device.name) if ipa_type == "yrzn001" @@ -326,7 +327,7 @@ def _prepare_node_attributes( and device.custom_fields["frr_parameters"] ): frr_params = device.custom_fields["frr_parameters"] - deep_decrypt(frr_params, vault) + deep_decrypt(frr_params, vault, "frr_parameters") node_attributes["extra"].update({"frr_parameters": json.dumps(frr_params)}) return node_attributes, template_vars @@ -764,6 +765,29 @@ def sync_ironic( osism_utils.finish_task_output(request_id, rc=1) return + # NOTE: Prepare all nodes before changing anything. A secret that cannot + # be decrypted would otherwise be replaced by defaults and written to + # ironic, so abort the whole sync instead. + prepared = [] + for device in devices: + try: + node_attributes, template_vars = _prepare_node_attributes( + device, + get_ironic_parameters, + skip_kernel_params=skip_kernel_params, + extra_kernel_params=extra_kernel_params, + ) + except VaultDecryptError as exc: + osism_utils.push_task_output( + request_id, + f"ERROR: Device {device.name}: {exc}\n" + "Aborting, no changes were made. Check the Ansible Vault " + "password ('osism set vault password').\n", + ) + osism_utils.finish_task_output(request_id, rc=1) + return + prepared.append((device, node_attributes, template_vars)) + # NOTE: Find nodes in Ironic which are no longer present in NetBox and remove them device_names = {dev.name for dev in devices} nodes = openstack.baremetal_node_list() @@ -813,19 +837,12 @@ def sync_ironic( ) # NOTE: Find nodes in NetBox which are not present in Ironic and add them - for device in devices: + for device, node_attributes, template_vars in prepared: osism_utils.push_task_output( request_id, f"Looking for {device.name} in ironic\n" ) node_interfaces = list(netbox.get_interfaces_by_device(device.name)) - - node_attributes, template_vars = _prepare_node_attributes( - device, - get_ironic_parameters, - skip_kernel_params=skip_kernel_params, - extra_kernel_params=extra_kernel_params, - ) ports_attributes = [ dict(address=interface.mac_address) for interface in node_interfaces diff --git a/osism/tasks/conductor/utils.py b/osism/tasks/conductor/utils.py index 1a81e394e..25908397a 100644 --- a/osism/tasks/conductor/utils.py +++ b/osism/tasks/conductor/utils.py @@ -46,27 +46,43 @@ def deep_merge(a, b): deep_merge(a[key], value) -def deep_decrypt(a, vault): - if a is None: - return +class VaultDecryptError(Exception): + """A vault-encrypted value could not be decrypted.""" + + +def deep_decrypt(a, vault, path=""): + """Decrypt all vault-encrypted strings in ``a`` in place. + + Every value that can be decrypted is decrypted. If any value fails, + VaultDecryptError names all failing keys afterwards, so that a caller + never continues with a secret silently missing or still encrypted. + """ + failed = [] + _deep_decrypt(a, vault, path, failed) + if failed: + paths = ", ".join(f"'{key_path}'" for key_path, _ in failed) + first_exc = failed[0][1] + raise VaultDecryptError( + f"Unable to decrypt {paths}: {first_exc}" + ) from first_exc + + +def _deep_decrypt(a, vault, path, failed): if isinstance(a, dict): - for key, value in list(a.items()): - if isinstance(value, (dict, list)): - deep_decrypt(a[key], vault) - elif vault.is_encrypted(value): - try: - a[key] = vault.decrypt(value).decode().strip() - except Exception: - a.pop(key, None) + items = list(a.items()) elif isinstance(a, list): - for i, item in enumerate(a): - if isinstance(item, (dict, list)): - deep_decrypt(item, vault) - elif vault.is_encrypted(item): - try: - a[i] = vault.decrypt(item).decode().strip() - except Exception: - pass + items = list(enumerate(a)) + else: + return + for key, value in items: + key_path = f"{path}.{key}" if path else str(key) + if isinstance(value, (dict, list)): + _deep_decrypt(value, vault, key_path, failed) + elif vault.is_encrypted(value): + try: + a[key] = vault.decrypt(value).decode().strip() + except Exception as exc: + failed.append((key_path, exc)) def get_vault(): @@ -89,7 +105,6 @@ def get_vault(): except Exception as exc: # Handle other errors (file access, decryption, etc.) logger.error(f"Unable to get vault secret: {exc}") - logger.error("Dropping encrypted entries") vault = VaultLib() return vault diff --git a/osism/tasks/openstack.py b/osism/tasks/openstack.py index 32ffc0eff..cf79dc8c3 100644 --- a/osism/tasks/openstack.py +++ b/osism/tasks/openstack.py @@ -226,7 +226,12 @@ def _mask_node_secret_parameters( The ``(kernel_append_params, netplan_parameters, frr_parameters)`` triple with secret values replaced by ``***``. """ - from osism.tasks.conductor.utils import deep_decrypt, get_vault, mask_secrets + from osism.tasks.conductor.utils import ( + VaultDecryptError, + deep_decrypt, + get_vault, + mask_secrets, + ) secret_values = set() if utils.nb and node_name: @@ -243,7 +248,7 @@ def _mask_node_secret_parameters( node_secrets = {} # Collect ironic_osism_* parameter names BEFORE decryption - # so they're available even if vault decryption removes them. + # so they're available even if vault decryption fails. # Convention: ironic_osism_aa -> osism-aa in kernel_append_params secret_param_names = set() for key in node_secrets: @@ -253,8 +258,11 @@ def _mask_node_secret_parameters( ) secret_param_names.add(param_name) - vault = get_vault() - deep_decrypt(node_secrets, vault) + try: + deep_decrypt(node_secrets, get_vault()) + except VaultDecryptError as exc: + # The name-based masking below still applies + logger.warning(f"Could not decrypt secrets of {node_name}: {exc}") # Collect secret values using the same logic as dry-run # (matches _is_secret_key + dry-run secret collection) diff --git a/tests/unit/tasks/conductor/test_ironic_sync.py b/tests/unit/tasks/conductor/test_ironic_sync.py index 52cbb1ef3..8dcbb5ba5 100644 --- a/tests/unit/tasks/conductor/test_ironic_sync.py +++ b/tests/unit/tasks/conductor/test_ironic_sync.py @@ -766,6 +766,45 @@ def test_sync_ironic_stale_node_not_eligible(sync_env): sync_env.openstack.baremetal_node_delete.assert_not_called() +def _fail_on_second_device(sync_env, error): + """Two devices; preparing the first succeeds, the second raises.""" + sync_env.netbox.get_devices.return_value = [ + _make_device(name="node1"), + _make_device(name="node2"), + ] + sync_env.prepare.side_effect = [({"driver": "redfish"}, {}), error] + + +def test_sync_ironic_aborts_before_any_change_on_vault_error(sync_env): + _fail_on_second_device( + sync_env, + ironic.VaultDecryptError( + "Unable to decrypt 'secrets.remote_board_password': no vault secrets" + ), + ) + sync_env.openstack.baremetal_node_list.return_value = [ + _make_node(name="stale", uuid="u-stale", provision_state="available") + ] + + ironic.sync_ironic("req", MagicMock()) + + assert sync_env.prepare.call_count == 2 + assert _pushed(sync_env.utils, "secrets.remote_board_password") + assert _pushed(sync_env.utils, "no changes were made") + sync_env.utils.finish_task_output.assert_called_once_with("req", rc=1) + sync_env.openstack.baremetal_node_delete.assert_not_called() + sync_env.sync_device.assert_not_called() + + +def test_sync_ironic_dry_run_aborts_on_vault_error(sync_env): + _fail_on_second_device(sync_env, ironic.VaultDecryptError("boom")) + + ironic.sync_ironic("req", MagicMock(), dry_run=True) + + sync_env.utils.finish_task_output.assert_called_once_with("req", rc=1) + sync_env.sync_dry.assert_not_called() + + def test_sync_ironic_kernel_params_propagated(sync_env): sync_env.netbox.get_devices.return_value = [_make_device()] diff --git a/tests/unit/tasks/conductor/test_utils.py b/tests/unit/tasks/conductor/test_utils.py index c5260286e..4273fd316 100644 --- a/tests/unit/tasks/conductor/test_utils.py +++ b/tests/unit/tasks/conductor/test_utils.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 +import re from unittest.mock import MagicMock, patch import pytest @@ -10,6 +11,7 @@ DELETE_SENTINEL, _get_conductor_redfish_address, _get_conductor_redfish_credentials, + VaultDecryptError, _is_secret_key, deep_compare, deep_decrypt, @@ -301,30 +303,43 @@ def test_deep_decrypt_top_level_list_replaces_encrypted_strings(): assert data == ["plain-1", "literal", "plain-2"] -def test_deep_decrypt_dict_value_decrypt_failure_drops_key(): +def test_deep_decrypt_dict_value_decrypt_failure_raises_with_path(): vault = _make_vault( - encrypted_values=["broken", "ok"], - decrypt_map={"ok": b"clear"}, + encrypted_values=["broken"], decrypt_errors={"broken": AnsibleError("bad signature")}, ) - data = {"a": "broken", "b": "ok"} + data = {"outer": {"a": "broken"}} - deep_decrypt(data, vault) + with pytest.raises(VaultDecryptError, match=re.escape("'secrets.outer.a'")): + deep_decrypt(data, vault, "secrets") + + +def test_deep_decrypt_list_element_decrypt_failure_raises_with_index(): + vault = _make_vault( + encrypted_values=["broken"], + decrypt_errors={"broken": AnsibleError("bad signature")}, + ) + data = ["ok", "broken"] - assert data == {"b": "clear"} + with pytest.raises(VaultDecryptError, match="'1'.*bad signature"): + deep_decrypt(data, vault) -def test_deep_decrypt_list_element_decrypt_failure_left_in_place(): +def test_deep_decrypt_decrypts_the_rest_before_raising(): vault = _make_vault( - encrypted_values=["broken", "ok"], + encrypted_values=["broken", "ok", "also-broken"], decrypt_map={"ok": b"clear"}, - decrypt_errors={"broken": AnsibleError("bad signature")}, + decrypt_errors={ + "broken": AnsibleError("bad signature"), + "also-broken": AnsibleError("bad signature"), + }, ) - data = ["broken", "ok"] + data = {"a": "broken", "b": "ok", "c": ["also-broken"]} - deep_decrypt(data, vault) + with pytest.raises(VaultDecryptError, match=re.escape("'a', 'c.0': bad signature")): + deep_decrypt(data, vault) - assert data == ["broken", "clear"] + assert data == {"a": "broken", "b": "clear", "c": ["also-broken"]} def test_deep_decrypt_strips_whitespace_after_decode(): diff --git a/tests/unit/tasks/test_openstack_baremetal.py b/tests/unit/tasks/test_openstack_baremetal.py index 13e2ab94b..aba64fc64 100644 --- a/tests/unit/tasks/test_openstack_baremetal.py +++ b/tests/unit/tasks/test_openstack_baremetal.py @@ -29,6 +29,7 @@ import pytest from osism.tasks import openstack as openstack_tasks +from osism.tasks.conductor.utils import VaultDecryptError def _has_log(records, level, substring): @@ -514,11 +515,9 @@ def test_mask_masks_secret_values(node_params_env): def test_mask_ironic_osism_param_by_name(node_params_env): """The ``ironic_osism_aa`` param name is collected before decryption, so the ``osism-aa=...`` argument is masked by the name-based regex even when - vault decryption drops the key and its value cannot be collected.""" - node_params_env.device.custom_fields["secrets"] = {"ironic_osism_aa": "topsecret"} - node_params_env.deep_decrypt.side_effect = lambda secrets, vault: secrets.pop( - "ironic_osism_aa", None - ) + vault decryption fails and its value cannot be collected.""" + node_params_env.device.custom_fields["secrets"] = {"ironic_osism_aa": "$ANSIBLE"} + node_params_env.deep_decrypt.side_effect = VaultDecryptError("no vault secrets") kernel, _, _ = openstack_tasks._mask_node_secret_parameters( "node-1", "osism-aa=topsecret other=x", {}, {} @@ -527,6 +526,23 @@ def test_mask_ironic_osism_param_by_name(node_params_env): assert kernel == "osism-aa=*** other=x" +def test_mask_uses_values_decrypted_before_the_error(node_params_env): + """Values decrypted in place stay collected when another one fails.""" + node_params_env.device.custom_fields["secrets"] = {"os_password": "$ANSIBLE"} + + def decrypt_partially(secrets, vault): + secrets["os_password"] = "hunter2" + raise VaultDecryptError("Unable to decrypt 'other'") + + node_params_env.deep_decrypt.side_effect = decrypt_partially + + kernel, _, _ = openstack_tasks._mask_node_secret_parameters( + "node-1", "pw=hunter2", {}, {} + ) + + assert kernel == "pw=***" + + def test_mask_ignores_non_string_secret_values(node_params_env): node_params_env.device.custom_fields["secrets"] = {"db_password": 42}