From c2baafa48c5f9c0f9719793ded3cab30970a00d7 Mon Sep 17 00:00:00 2001 From: Caglar Pir Date: Mon, 16 Nov 2026 16:00:23 +0100 Subject: [PATCH] Measure the remaining upload bytes once, not before every chunk Since #833, chunkize_byte_stream seeks to the end of the stream and back before reading each chunk, to cap the read to the remaining bytes. A video is uploaded as a stream chained from one slice per sample, and seeking it walks the slices, so every chunk walked them all and uploading a large video became quadratic. For a 21.5 GB GoPro HERO12 video (95,249 samples, 10,190 chunks of 2 MB), a chunk took 40 ms instead of 0.5 ms, and more further into the file. Measure the remaining bytes once, when the first chunk is read, and count them down by the bytes read. Reads are still capped as in #833, and chunking that video takes 5 s again, as before #833. --- mapillary_tools/upload_api_v4.py | 21 ++++++++---------- tests/unit/test_upload_api_v4.py | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/mapillary_tools/upload_api_v4.py b/mapillary_tools/upload_api_v4.py index 5f8f7c42c..5c0ba6215 100644 --- a/mapillary_tools/upload_api_v4.py +++ b/mapillary_tools/upload_api_v4.py @@ -70,25 +70,22 @@ def chunkize_byte_stream( if chunk_size <= 0: raise ValueError("Expect positive chunk size") + # Cap read() to the remaining bytes so FileIO does not allocate + # chunk_size (e.g. 1 GiB). Measure them once: seeking a video stream + # rebuilt from many samples walks all of them + remaining = cls._remaining_bytes(stream) + while True: - to_read = cls._bounded_read_size(stream, chunk_size) - if to_read == 0: + to_read = chunk_size if remaining is None else min(chunk_size, remaining) + if to_read <= 0: break data = stream.read(to_read) if not data: break + if remaining is not None: + remaining -= len(data) yield data - @classmethod - def _bounded_read_size(cls, stream: T.IO[bytes], chunk_size: int) -> int: - """Cap read() to remaining bytes so FileIO does not allocate chunk_size (e.g. 1 GiB).""" - remaining = cls._remaining_bytes(stream) - if remaining is None: - return chunk_size - if remaining <= 0: - return 0 - return min(chunk_size, remaining) - @classmethod def _remaining_bytes(cls, stream: T.IO[bytes]) -> int | None: try: diff --git a/tests/unit/test_upload_api_v4.py b/tests/unit/test_upload_api_v4.py index b6473f97a..01a9668a7 100644 --- a/tests/unit/test_upload_api_v4.py +++ b/tests/unit/test_upload_api_v4.py @@ -92,3 +92,41 @@ def test_chunkize_caps_read_to_remaining_bytes(): ) assert b"".join(chunks) == b"hello world" assert spy.read_sizes == [len(b"hello world")] + + +class _SeekSpy(_ReadSpy): + def __init__(self, data: bytes): + super().__init__(data) + self.seek_count = 0 + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + self.seek_count += 1 + return super().seek(offset, whence) + + +def test_chunkize_measures_remaining_bytes_once(): + # Seeking a video stream rebuilt from many samples walks all of them, + # so seeking for every chunk made large video uploads quadratic + spy = _SeekSpy(b"hello world") + spy.seek(6) + spy.seek_count = 0 + chunks = list(upload_api_v4.UploadService.chunkize_byte_stream(spy, 2)) + assert chunks == [b"wo", b"rl", b"d"] + assert spy.read_sizes == [2, 2, 1] + assert spy.seek_count == 2 + + +class _UnseekableStream: + def __init__(self, data: bytes): + self._stream = io.BytesIO(data) + + def read(self, size: int = -1) -> bytes: + return self._stream.read(size) + + +def test_chunkize_unseekable_stream(): + stream = _UnseekableStream(b"hello world") + chunks = list( + upload_api_v4.UploadService.chunkize_byte_stream(stream, 4) # type: ignore[arg-type] + ) + assert chunks == [b"hell", b"o wo", b"rld"]