From 75e1bf3760659cdd5f4b1c4fa935836ca0b313c6 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Thu, 5 Nov 2026 15:18:46 -0800 Subject: [PATCH] Add characterization tests for legacy network sync Pins what a full push and a full pull send and store when the server lacks ASYNC_OPERATIONS, so the refactors of the LegacyNetwork* operations can show they change neither. The tests run against a live server that emulates a morango version before 0.6.0: the MORANGO_TEST_LEGACY_SERVER variable turns off both ASYNC_OPERATIONS and FSIC_V2_FORMAT on the server, and LiveServer now takes extra environment variables for that. The client keeps its full set of capabilities, as a current Kolibri does when it syncs with an older one. Reducing only the client capabilities, as #364 first proposed, fails today on the server for both push and pull (#416). The expected lists show that the client creates the transfer session and sends no stage updates for serializing, queuing, dequeuing, or deserializing. The legacy pull moves 11 records instead of 10: the client creates the transfer session before it serializes, so it sends an empty client_fsic, and the server queues the user from the initial pull again. Closes #364 Co-Authored-By: Claude Opus 5.5 --- tests/testapp/testapp/server2_settings.py | 7 + tests/testapp/tests/integration/helpers.py | 7 +- .../test_network_characterization.py | 138 +++++++++++++++--- 3 files changed, 126 insertions(+), 26 deletions(-) diff --git a/tests/testapp/testapp/server2_settings.py b/tests/testapp/testapp/server2_settings.py index c7473347..6e43c953 100644 --- a/tests/testapp/testapp/server2_settings.py +++ b/tests/testapp/testapp/server2_settings.py @@ -1,5 +1,12 @@ +import os + from .settings import * # noqa DATABASES = { "default": DATABASES["default2"], # noqa } + +# emulates a server on a morango version before 0.6.0, which lacks both of these capabilities +if os.environ.get("MORANGO_TEST_LEGACY_SERVER"): + MORANGO_DISALLOW_ASYNC_OPERATIONS = True + MORANGO_DISABLE_FSIC_V2_FORMAT = True diff --git a/tests/testapp/tests/integration/helpers.py b/tests/testapp/tests/integration/helpers.py index 62611886..11c72f79 100644 --- a/tests/testapp/tests/integration/helpers.py +++ b/tests/testapp/tests/integration/helpers.py @@ -39,9 +39,10 @@ def second_environment(): class LiveServer: - def __init__(self): + def __init__(self, env=None): self.env = os.environ.copy() self.env["MORANGO_SYSTEM_ID"] = SECOND_SYSTEM_ID + self.env.update(env or {}) self.port = get_free_tcp_port() self.host = "127.0.0.1" self.start() @@ -94,11 +95,13 @@ class BasePushPullClientTestCase(TransactionTestCase): profile = "facilitydata" databases = ["default", SECOND_TEST_DATABASE] + # extra environment variables for the live server process + server_env = {} @classmethod def setUpClass(cls): super(TransactionTestCase, cls).setUpClass() - cls.server = LiveServer() + cls.server = LiveServer(env=cls.server_env) @classmethod def tearDownClass(cls): diff --git a/tests/testapp/tests/integration/test_network_characterization.py b/tests/testapp/tests/integration/test_network_characterization.py index 04dd326f..56fc2fd7 100644 --- a/tests/testapp/tests/integration/test_network_characterization.py +++ b/tests/testapp/tests/integration/test_network_characterization.py @@ -1,6 +1,7 @@ """ -Characterization tests that pin the HTTP requests and the final state of a network push and pull -that use `ASYNC_OPERATIONS`. Refactors of the sync layer must pass these tests without edits. +Characterization tests that pin the HTTP requests and the final state of a network push and pull, +both with `ASYNC_OPERATIONS` and with the legacy flow of a server that lacks it. Refactors of the +sync layer must pass these tests without edits. """ import gzip @@ -14,6 +15,7 @@ from morango.constants import transfer_stages from morango.constants import transfer_statuses +from morango.constants.capabilities import ASYNC_OPERATIONS from morango.models.core import Buffer from morango.models.core import Store from morango.models.core import TransferSession @@ -283,9 +285,55 @@ def pull_buffers(offset): ] -class NetworkCharacterizationTestCase(BasePushPullClientTestCase): +# The legacy server serializes when the client creates the transfer session, and dequeues and +# deserializes when the client closes it, so the client sends no stage updates for those stages. +# The client tells the server how many records it queued, since the server did not queue them. +LEGACY_PUSH_REQUESTS = [ + # initializing + create_transfer_session(push=True), + # queuing + update_transfer_session(("records_total",), records_total=10), + # transferring, where the server advances its transfer session as it receives each chunk + push_buffers(3), + push_buffers(3), + push_buffers(3), + push_buffers(1), + # cleanup + close_transfer_session(), +] + +# The legacy server serializes and queues when the client creates the transfer session, and returns +# the number of records it queued in that response. The client creates the transfer session before +# it serializes, so it sends an empty `client_fsic` and the server queues the user from the initial +# pull in `setUp` again, for 11 records in chunks of 3. +LEGACY_PULL_REQUESTS = [ + # initializing + create_transfer_session(push=False), + # transferring, where the client reports its progress after each chunk + pull_buffers(offset=0), + update_transfer_session( + PULL_TRANSFER_KEYS, records_transferred=3, transfer_stage=transfer_stages.TRANSFERRING + ), + pull_buffers(offset=3), + update_transfer_session( + PULL_TRANSFER_KEYS, records_transferred=6, transfer_stage=transfer_stages.TRANSFERRING + ), + pull_buffers(offset=6), + update_transfer_session( + PULL_TRANSFER_KEYS, records_transferred=9, transfer_stage=transfer_stages.TRANSFERRING + ), + pull_buffers(offset=9), + update_transfer_session( + PULL_TRANSFER_KEYS, records_transferred=11, transfer_stage=transfer_stages.TRANSFERRING + ), + # cleanup + close_transfer_session(), +] + + +class BaseNetworkCharacterizationTestCase(BasePushPullClientTestCase): def setUp(self): - super(NetworkCharacterizationTestCase, self).setUp() + super(BaseNetworkCharacterizationTestCase, self).setUp() # the expected requests depend on the chunk size, so pin it here rather than rely on # the harness self.conn.chunk_size = CHUNK_SIZE @@ -316,15 +364,7 @@ def assertTransferSessionFinalState(self, transfer_session, records_total): self.assertFalse(transfer_session.active) self.assertEqual(0, Buffer.objects.filter(transfer_session=transfer_session).count()) - def test_async_push(self): - for _ in range(5): - SummaryLog.objects.create(user=self.local_user) - InteractionLog.objects.create(user=self.local_user) - - client = self.client.get_push_client() - requests = self._sync(client) - self.assertEqual(PUSH_REQUESTS, requests) - + def _assertPushFinalState(self, client): transfer_session = client.context.transfer_session self.assertTransferSessionFinalState(transfer_session, records_total=10) @@ -339,18 +379,9 @@ def test_async_push(self): self.assertEqual(5, SummaryLog.objects.filter(user=self.remote_user).count()) self.assertEqual(5, InteractionLog.objects.filter(user=self.remote_user).count()) - def test_async_pull(self): - with second_environment(): - for _ in range(5): - SummaryLog.objects.create(user=self.remote_user) - InteractionLog.objects.create(user=self.remote_user) - - client = self.client.get_pull_client() - requests = self._sync(client) - self.assertEqual(PULL_REQUESTS, requests) - + def _assertPullFinalState(self, client, records_total=10): transfer_session = client.context.transfer_session - self.assertTransferSessionFinalState(transfer_session, records_total=10) + self.assertTransferSessionFinalState(transfer_session, records_total=records_total) # the client reports its byte counts to the server, flipped to the server's point of view remote_transfer_session = self._remote_transfer_session(transfer_session) @@ -361,3 +392,62 @@ def test_async_pull(self): self.assertEqual(11, self._store_count()) self.assertEqual(5, SummaryLog.objects.filter(user=self.local_user).count()) self.assertEqual(5, InteractionLog.objects.filter(user=self.local_user).count()) + + def _create_local_logs(self): + for _ in range(5): + SummaryLog.objects.create(user=self.local_user) + InteractionLog.objects.create(user=self.local_user) + + def _create_remote_logs(self): + with second_environment(): + for _ in range(5): + SummaryLog.objects.create(user=self.remote_user) + InteractionLog.objects.create(user=self.remote_user) + + +class NetworkCharacterizationTestCase(BaseNetworkCharacterizationTestCase): + def test_async_push(self): + self._create_local_logs() + + client = self.client.get_push_client() + self.assertIn(ASYNC_OPERATIONS, client.context.capabilities) + requests = self._sync(client) + self.assertEqual(PUSH_REQUESTS, requests) + self._assertPushFinalState(client) + + def test_async_pull(self): + self._create_remote_logs() + + client = self.client.get_pull_client() + self.assertIn(ASYNC_OPERATIONS, client.context.capabilities) + requests = self._sync(client) + self.assertEqual(PULL_REQUESTS, requests) + self._assertPullFinalState(client) + + +class LegacyNetworkCharacterizationTestCase(BaseNetworkCharacterizationTestCase): + """ + Syncs with a server that emulates a morango version before 0.6.0, so the server lacks + `ASYNC_OPERATIONS` and `FSIC_V2_FORMAT`. The client keeps its full set of capabilities. + """ + + server_env = {"MORANGO_TEST_LEGACY_SERVER": "1"} + + def test_legacy_push(self): + self._create_local_logs() + + client = self.client.get_push_client() + self.assertNotIn(ASYNC_OPERATIONS, client.context.capabilities) + requests = self._sync(client) + self.assertEqual(LEGACY_PUSH_REQUESTS, requests) + self._assertPushFinalState(client) + + def test_legacy_pull(self): + self._create_remote_logs() + + client = self.client.get_pull_client() + self.assertNotIn(ASYNC_OPERATIONS, client.context.capabilities) + requests = self._sync(client) + self.assertEqual(LEGACY_PULL_REQUESTS, requests) + # the server sent the user from the initial pull again, which the client already had + self._assertPullFinalState(client, records_total=11)