diff --git a/mapillary_tools/upload_api_v4.py b/mapillary_tools/upload_api_v4.py index 5f8f7c42..5c0ba621 100644 --- a/mapillary_tools/upload_api_v4.py +++ b/mapillary_tools/upload_api_v4.py @@ -70,25 +70,22 @@ def chunkize_byte_stream( if chunk_size <= 0: raise ValueError("Expect positive chunk size") + # Cap read() to the remaining bytes so FileIO does not allocate + # chunk_size (e.g. 1 GiB). Measure them once: seeking a video stream + # rebuilt from many samples walks all of them + remaining = cls._remaining_bytes(stream) + while True: - to_read = cls._bounded_read_size(stream, chunk_size) - if to_read == 0: + to_read = chunk_size if remaining is None else min(chunk_size, remaining) + if to_read <= 0: break data = stream.read(to_read) if not data: break + if remaining is not None: + remaining -= len(data) yield data - @classmethod - def _bounded_read_size(cls, stream: T.IO[bytes], chunk_size: int) -> int: - """Cap read() to remaining bytes so FileIO does not allocate chunk_size (e.g. 1 GiB).""" - remaining = cls._remaining_bytes(stream) - if remaining is None: - return chunk_size - if remaining <= 0: - return 0 - return min(chunk_size, remaining) - @classmethod def _remaining_bytes(cls, stream: T.IO[bytes]) -> int | None: try: diff --git a/tests/unit/test_upload_api_v4.py b/tests/unit/test_upload_api_v4.py index b6473f97..01a9668a 100644 --- a/tests/unit/test_upload_api_v4.py +++ b/tests/unit/test_upload_api_v4.py @@ -92,3 +92,41 @@ def test_chunkize_caps_read_to_remaining_bytes(): ) assert b"".join(chunks) == b"hello world" assert spy.read_sizes == [len(b"hello world")] + + +class _SeekSpy(_ReadSpy): + def __init__(self, data: bytes): + super().__init__(data) + self.seek_count = 0 + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + self.seek_count += 1 + return super().seek(offset, whence) + + +def test_chunkize_measures_remaining_bytes_once(): + # Seeking a video stream rebuilt from many samples walks all of them, + # so seeking for every chunk made large video uploads quadratic + spy = _SeekSpy(b"hello world") + spy.seek(6) + spy.seek_count = 0 + chunks = list(upload_api_v4.UploadService.chunkize_byte_stream(spy, 2)) + assert chunks == [b"wo", b"rl", b"d"] + assert spy.read_sizes == [2, 2, 1] + assert spy.seek_count == 2 + + +class _UnseekableStream: + def __init__(self, data: bytes): + self._stream = io.BytesIO(data) + + def read(self, size: int = -1) -> bytes: + return self._stream.read(size) + + +def test_chunkize_unseekable_stream(): + stream = _UnseekableStream(b"hello world") + chunks = list( + upload_api_v4.UploadService.chunkize_byte_stream(stream, 4) # type: ignore[arg-type] + ) + assert chunks == [b"hell", b"o wo", b"rld"]