Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 30 additions & 13 deletions osism/tasks/conductor/ironic.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
deep_merge,
get_vault,
mask_secrets,
VaultDecryptError,
)

SUPPORTED_IPA_TYPES = {
Expand Down Expand Up @@ -197,7 +198,7 @@ def _prepare_node_attributes(
and "ironic_parameters" in device.config_context
):
config_context_ironic = device.config_context["ironic_parameters"]
deep_decrypt(config_context_ironic, vault)
deep_decrypt(config_context_ironic, vault, "config_context.ironic_parameters")
deep_merge(node_attributes, config_context_ironic)

# Decrypt and merge ironic_parameters Custom Field if present
Expand All @@ -206,14 +207,14 @@ def _prepare_node_attributes(
and device.custom_fields["ironic_parameters"]
):
ironic_parameters_cf = device.custom_fields["ironic_parameters"]
deep_decrypt(ironic_parameters_cf, vault)
deep_decrypt(ironic_parameters_cf, vault, "ironic_parameters")
deep_merge(node_attributes, ironic_parameters_cf)

# Decrypt secrets Custom Field
node_secrets = device.custom_fields.get("secrets", {})
if node_secrets is None:
node_secrets = {}
deep_decrypt(node_secrets, vault)
deep_decrypt(node_secrets, vault, "secrets")

if (
"driver" in node_attributes
Expand Down Expand Up @@ -262,7 +263,7 @@ def _prepare_node_attributes(
if match and match.group(1) in SUPPORTED_IPA_TYPES:
ipa_type = match.group(1)
frr = device.custom_fields.get("frr_parameters") or {}
deep_decrypt(frr, vault)
deep_decrypt(frr, vault, "frr_parameters")
derived_as = (
_derive_as_from_hostname_yrzn(device.name)
if ipa_type == "yrzn001"
Expand Down Expand Up @@ -326,7 +327,7 @@ def _prepare_node_attributes(
and device.custom_fields["frr_parameters"]
):
frr_params = device.custom_fields["frr_parameters"]
deep_decrypt(frr_params, vault)
deep_decrypt(frr_params, vault, "frr_parameters")
node_attributes["extra"].update({"frr_parameters": json.dumps(frr_params)})

return node_attributes, template_vars
Expand Down Expand Up @@ -764,6 +765,29 @@ def sync_ironic(
osism_utils.finish_task_output(request_id, rc=1)
return

# NOTE: Prepare all nodes before changing anything. A secret that cannot
# be decrypted would otherwise be replaced by defaults and written to
# ironic, so abort the whole sync instead.
prepared = []
for device in devices:
try:
node_attributes, template_vars = _prepare_node_attributes(
device,
get_ironic_parameters,
skip_kernel_params=skip_kernel_params,
extra_kernel_params=extra_kernel_params,
)
except VaultDecryptError as exc:
osism_utils.push_task_output(
request_id,
f"ERROR: Device {device.name}: {exc}\n"
"Aborting, no changes were made. Check the Ansible Vault "
"password ('osism set vault password').\n",
)
osism_utils.finish_task_output(request_id, rc=1)
return
prepared.append((device, node_attributes, template_vars))

# NOTE: Find nodes in Ironic which are no longer present in NetBox and remove them
device_names = {dev.name for dev in devices}
nodes = openstack.baremetal_node_list()
Expand Down Expand Up @@ -813,19 +837,12 @@ def sync_ironic(
)

# NOTE: Find nodes in NetBox which are not present in Ironic and add them
for device in devices:
for device, node_attributes, template_vars in prepared:
osism_utils.push_task_output(
request_id, f"Looking for {device.name} in ironic\n"
)

node_interfaces = list(netbox.get_interfaces_by_device(device.name))

node_attributes, template_vars = _prepare_node_attributes(
device,
get_ironic_parameters,
skip_kernel_params=skip_kernel_params,
extra_kernel_params=extra_kernel_params,
)
ports_attributes = [
dict(address=interface.mac_address)
for interface in node_interfaces
Expand Down
55 changes: 35 additions & 20 deletions osism/tasks/conductor/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,27 +46,43 @@ def deep_merge(a, b):
deep_merge(a[key], value)


def deep_decrypt(a, vault):
if a is None:
return
class VaultDecryptError(Exception):
"""A vault-encrypted value could not be decrypted."""


def deep_decrypt(a, vault, path=""):
"""Decrypt all vault-encrypted strings in ``a`` in place.

Every value that can be decrypted is decrypted. If any value fails,
VaultDecryptError names all failing keys afterwards, so that a caller
never continues with a secret silently missing or still encrypted.
"""
failed = []
_deep_decrypt(a, vault, path, failed)
if failed:
paths = ", ".join(f"'{key_path}'" for key_path, _ in failed)
first_exc = failed[0][1]
raise VaultDecryptError(
f"Unable to decrypt {paths}: {first_exc}"
) from first_exc


def _deep_decrypt(a, vault, path, failed):
if isinstance(a, dict):
for key, value in list(a.items()):
if isinstance(value, (dict, list)):
deep_decrypt(a[key], vault)
elif vault.is_encrypted(value):
try:
a[key] = vault.decrypt(value).decode().strip()
except Exception:
a.pop(key, None)
items = list(a.items())
elif isinstance(a, list):
for i, item in enumerate(a):
if isinstance(item, (dict, list)):
deep_decrypt(item, vault)
elif vault.is_encrypted(item):
try:
a[i] = vault.decrypt(item).decode().strip()
except Exception:
pass
items = list(enumerate(a))
else:
return
for key, value in items:
key_path = f"{path}.{key}" if path else str(key)
if isinstance(value, (dict, list)):
_deep_decrypt(value, vault, key_path, failed)
elif vault.is_encrypted(value):
try:
a[key] = vault.decrypt(value).decode().strip()
except Exception as exc:
failed.append((key_path, exc))


def get_vault():
Expand All @@ -89,7 +105,6 @@ def get_vault():
except Exception as exc:
# Handle other errors (file access, decryption, etc.)
logger.error(f"Unable to get vault secret: {exc}")
logger.error("Dropping encrypted entries")
vault = VaultLib()
return vault

Expand Down
16 changes: 12 additions & 4 deletions osism/tasks/openstack.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,12 @@ def _mask_node_secret_parameters(
The ``(kernel_append_params, netplan_parameters, frr_parameters)``
triple with secret values replaced by ``***``.
"""
from osism.tasks.conductor.utils import deep_decrypt, get_vault, mask_secrets
from osism.tasks.conductor.utils import (
VaultDecryptError,
deep_decrypt,
get_vault,
mask_secrets,
)

secret_values = set()
if utils.nb and node_name:
Expand All @@ -243,7 +248,7 @@ def _mask_node_secret_parameters(
node_secrets = {}

# Collect ironic_osism_* parameter names BEFORE decryption
# so they're available even if vault decryption removes them.
# so they're available even if vault decryption fails.
# Convention: ironic_osism_aa -> osism-aa in kernel_append_params
secret_param_names = set()
for key in node_secrets:
Expand All @@ -253,8 +258,11 @@ def _mask_node_secret_parameters(
)
secret_param_names.add(param_name)

vault = get_vault()
deep_decrypt(node_secrets, vault)
try:
deep_decrypt(node_secrets, get_vault())
except VaultDecryptError as exc:
Comment thread
sourcery-ai[bot] marked this conversation as resolved.
Resolved
# The name-based masking below still applies
logger.warning(f"Could not decrypt secrets of {node_name}: {exc}")

# Collect secret values using the same logic as dry-run
# (matches _is_secret_key + dry-run secret collection)
Expand Down
39 changes: 39 additions & 0 deletions tests/unit/tasks/conductor/test_ironic_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -766,6 +766,45 @@ def test_sync_ironic_stale_node_not_eligible(sync_env):
sync_env.openstack.baremetal_node_delete.assert_not_called()


def _fail_on_second_device(sync_env, error):
"""Two devices; preparing the first succeeds, the second raises."""
sync_env.netbox.get_devices.return_value = [
_make_device(name="node1"),
_make_device(name="node2"),
]
sync_env.prepare.side_effect = [({"driver": "redfish"}, {}), error]


def test_sync_ironic_aborts_before_any_change_on_vault_error(sync_env):
_fail_on_second_device(
sync_env,
ironic.VaultDecryptError(
"Unable to decrypt 'secrets.remote_board_password': no vault secrets"
),
)
sync_env.openstack.baremetal_node_list.return_value = [
_make_node(name="stale", uuid="u-stale", provision_state="available")
]

ironic.sync_ironic("req", MagicMock())

assert sync_env.prepare.call_count == 2
assert _pushed(sync_env.utils, "secrets.remote_board_password")
assert _pushed(sync_env.utils, "no changes were made")
sync_env.utils.finish_task_output.assert_called_once_with("req", rc=1)
sync_env.openstack.baremetal_node_delete.assert_not_called()
sync_env.sync_device.assert_not_called()


def test_sync_ironic_dry_run_aborts_on_vault_error(sync_env):
_fail_on_second_device(sync_env, ironic.VaultDecryptError("boom"))

ironic.sync_ironic("req", MagicMock(), dry_run=True)

sync_env.utils.finish_task_output.assert_called_once_with("req", rc=1)
sync_env.sync_dry.assert_not_called()


def test_sync_ironic_kernel_params_propagated(sync_env):
sync_env.netbox.get_devices.return_value = [_make_device()]

Expand Down
39 changes: 27 additions & 12 deletions tests/unit/tasks/conductor/test_utils.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0

import re
from unittest.mock import MagicMock, patch

import pytest
Expand All @@ -10,6 +11,7 @@
DELETE_SENTINEL,
_get_conductor_redfish_address,
_get_conductor_redfish_credentials,
VaultDecryptError,
_is_secret_key,
deep_compare,
deep_decrypt,
Expand Down Expand Up @@ -301,30 +303,43 @@ def test_deep_decrypt_top_level_list_replaces_encrypted_strings():
assert data == ["plain-1", "literal", "plain-2"]


def test_deep_decrypt_dict_value_decrypt_failure_drops_key():
def test_deep_decrypt_dict_value_decrypt_failure_raises_with_path():
vault = _make_vault(
encrypted_values=["broken", "ok"],
decrypt_map={"ok": b"clear"},
encrypted_values=["broken"],
decrypt_errors={"broken": AnsibleError("bad signature")},
)
data = {"a": "broken", "b": "ok"}
data = {"outer": {"a": "broken"}}

deep_decrypt(data, vault)
with pytest.raises(VaultDecryptError, match=re.escape("'secrets.outer.a'")):
deep_decrypt(data, vault, "secrets")


def test_deep_decrypt_list_element_decrypt_failure_raises_with_index():
vault = _make_vault(
encrypted_values=["broken"],
decrypt_errors={"broken": AnsibleError("bad signature")},
)
data = ["ok", "broken"]

assert data == {"b": "clear"}
with pytest.raises(VaultDecryptError, match="'1'.*bad signature"):
deep_decrypt(data, vault)


def test_deep_decrypt_list_element_decrypt_failure_left_in_place():
def test_deep_decrypt_decrypts_the_rest_before_raising():
vault = _make_vault(
encrypted_values=["broken", "ok"],
encrypted_values=["broken", "ok", "also-broken"],
decrypt_map={"ok": b"clear"},
decrypt_errors={"broken": AnsibleError("bad signature")},
decrypt_errors={
"broken": AnsibleError("bad signature"),
"also-broken": AnsibleError("bad signature"),
},
)
data = ["broken", "ok"]
data = {"a": "broken", "b": "ok", "c": ["also-broken"]}

deep_decrypt(data, vault)
with pytest.raises(VaultDecryptError, match=re.escape("'a', 'c.0': bad signature")):
deep_decrypt(data, vault)

assert data == ["broken", "clear"]
assert data == {"a": "broken", "b": "clear", "c": ["also-broken"]}


def test_deep_decrypt_strips_whitespace_after_decode():
Expand Down
26 changes: 21 additions & 5 deletions tests/unit/tasks/test_openstack_baremetal.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import pytest

from osism.tasks import openstack as openstack_tasks
from osism.tasks.conductor.utils import VaultDecryptError


def _has_log(records, level, substring):
Expand Down Expand Up @@ -514,11 +515,9 @@ def test_mask_masks_secret_values(node_params_env):
def test_mask_ironic_osism_param_by_name(node_params_env):
"""The ``ironic_osism_aa`` param name is collected before decryption, so
the ``osism-aa=...`` argument is masked by the name-based regex even when
vault decryption drops the key and its value cannot be collected."""
node_params_env.device.custom_fields["secrets"] = {"ironic_osism_aa": "topsecret"}
node_params_env.deep_decrypt.side_effect = lambda secrets, vault: secrets.pop(
"ironic_osism_aa", None
)
vault decryption fails and its value cannot be collected."""
node_params_env.device.custom_fields["secrets"] = {"ironic_osism_aa": "$ANSIBLE"}
node_params_env.deep_decrypt.side_effect = VaultDecryptError("no vault secrets")

kernel, _, _ = openstack_tasks._mask_node_secret_parameters(
"node-1", "osism-aa=topsecret other=x", {}, {}
Expand All @@ -527,6 +526,23 @@ def test_mask_ironic_osism_param_by_name(node_params_env):
assert kernel == "osism-aa=*** other=x"


def test_mask_uses_values_decrypted_before_the_error(node_params_env):
"""Values decrypted in place stay collected when another one fails."""
node_params_env.device.custom_fields["secrets"] = {"os_password": "$ANSIBLE"}

def decrypt_partially(secrets, vault):
secrets["os_password"] = "hunter2"
raise VaultDecryptError("Unable to decrypt 'other'")

node_params_env.deep_decrypt.side_effect = decrypt_partially

kernel, _, _ = openstack_tasks._mask_node_secret_parameters(
"node-1", "pw=hunter2", {}, {}
)

assert kernel == "pw=***"


def test_mask_ignores_non_string_secret_values(node_params_env):
node_params_env.device.custom_fields["secrets"] = {"db_password": 42}

Expand Down