Skip to content
Draft
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 110 additions & 3 deletions test_blockserver.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,25 @@
import json
import os
import socket
import time
import unittest
import zlib
from itertools import product

import requests
from genie_python import genie as g
from genie_python.channel_access_exceptions import UnableToConnectToPVException
from genie_python.utilities import compress_and_hex
from genie_python.genie_cachannel_wrapper import CaChannelWrapper, AlarmCondition
from genie_python.utilities import compress_and_hex, dehex_and_decompress
from hamcrest import assert_that, contains_string, ends_with, is_
from parameterized import parameterized

from utilities import utilities
from utilities.utilities import assert_with_timeout, parameterized_list

Check failure on line 18 in test_blockserver.py

View workflow job for this annotation

GitHub Actions / call-workflow / ruff

ruff (I001)

test_blockserver.py:1:1: I001 Import block is un-sorted or un-formatted help: Organize imports

SECONDS_TO_WAIT_FOR_IOC_STARTS = 120

SERVICE_RESTART_TIMEOUT = 180
NO_RESTART_GRACE_PERIOD = 15

def wait_for_server():
status_was_busy = False
Expand Down Expand Up @@ -141,7 +145,7 @@
"CAENV895 ioc should have restarted",
)
break
except (Exception, AssertionError) as e:
except AssertionError as e:
err = e
time.sleep(1)
else:
Expand Down Expand Up @@ -516,3 +520,106 @@
test_pv_with_macro_value(expected_1, "SIMPLE:MACROTEST1", self)
test_pv_with_macro_value(expected_2, "SIMPLE:MACROTEST2", self)
test_pv_with_macro_value(expected_3, "SIMPLE:MACROTEST3", self)

@parameterized.expand(
parameterized_list(
list(
product(
[
# (PV to monitor, service IOC to restart, values which indicate a restart)
# A CA disconnect on the monitored PV always counts as a restart.
("CS:PS:RUNCTRL_01:STATUS", "RUNCTRL_01", ("Shutdown",)),
# The blockserver serves this PV itself, so it disconnects on restart
("CS:BLOCKSERVER:SERVER_STATUS", "BLOCKSVR", ()),
("CS:PS:ISISDAE_01:STATUS", "ISISDAE_01", ("Shutdown",)),
("CS:PS:INSTETC_01:STATUS", "INSTETC_01", ("Shutdown",)),
("CS:PS:ALARM:STATUS", "ALARM", ("Shutdown",)),
("CS:PS:ARINST:STATUS", "ARINST", ("Shutdown",)),
("CS:PS:ARBLOCK:STATUS", "ARBLOCK", ("Shutdown",)),
# No blockgateway as this is not restarted in the typical sense, just "reloaded"
],
[True, False],
)
)
)
)
def test_WHEN_restart_service_ioc_pv_written_THEN_restarts_service(
self, _, pv_ioc_markers, manager
):
restarted = "RESTARTED"
events = []
pv, ioc, restart_markers = pv_ioc_markers
full_pv = g.prefix_pv_name(pv)

def _on_change(value, severity, status):
if status == AlarmCondition.Link:
events.append(restarted)
return
text = str(value)
try:
text = dehex_and_decompress(text)
except (ValueError, zlib.error):
pass
events.append(restarted if text in restart_markers else text)

def _restarted_and_came_back() -> bool:
# Need a value before the restart and at least one after it (i.e. it came back)
return restarted in events and 0 < events.index(restarted) < len(events) - 1

# Make sure a previous test's restart has fully finished before we start monitoring
self._wait_for_service_ready(ioc)

unsubscribe = CaChannelWrapper.add_monitor(
full_pv, _on_change, to_string=True, use_numpy=False
)
self.addCleanup(unsubscribe)
self._set_manager_mode(manager)

g.set_pv(
"CS:BLOCKSERVER:RESTART_SERVICE_IOC",
compress_and_hex(json.dumps([ioc])),
is_local=True,
)

if manager:
end_time = time.time() + SERVICE_RESTART_TIMEOUT
while time.time() < end_time and not _restarted_and_came_back():
time.sleep(0.5)
self.assertTrue(
_restarted_and_came_back(),
f"{ioc} did not restart and come back, events seen on {pv}: {events}",
)
# procServ reports "Running" as soon as the process is launched, but the service
# may not be serving PVs yet (e.g. INSTETC serves CS:MANAGER, which cleanup resets)
self._wait_for_service_ready(ioc)
else:
time.sleep(NO_RESTART_GRACE_PERIOD)
self.assertNotIn(
restarted, events, f"{ioc} restarted without manager mode, events: {events}"
)

@staticmethod
def _service_is_ready(ioc: str) -> bool:
try:
if ioc == "BLOCKSVR":
return utilities.get_server_status() == ""
if ioc in ("RUNCTRL_01", "ISISDAE_01", "INSTETC_01"):
# EPICS IOCs: heartbeat only exists once iocInit has completed
return utilities.is_ioc_up(ioc)
# Non-EPICS services (java processes) have no heartbeat, procServ status is all we have
return g.get_pv(f"CS:PS:{ioc}:STATUS", is_local=True) == "Running"
except UnableToConnectToPVException:
return False

def _wait_for_service_ready(self, ioc: str) -> None:
end_time = time.time() + SERVICE_RESTART_TIMEOUT
while time.time() < end_time:
if self._service_is_ready(ioc):
return
time.sleep(1)
self.fail(f"{ioc} was not ready after {SERVICE_RESTART_TIMEOUT}s")

def _set_manager_mode(self, manager: bool) -> None:
previous = g.get_pv("CS:MANAGER", is_local=True)
self.addCleanup(g.set_pv, "CS:MANAGER", previous, wait=True, is_local=True)
g.set_pv("CS:MANAGER", "Yes" if manager else "No", wait=True, is_local=True)
Loading