Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 9 additions & 12 deletions mapillary_tools/upload_api_v4.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,25 +70,22 @@ def chunkize_byte_stream(
if chunk_size <= 0:
raise ValueError("Expect positive chunk size")

# Cap read() to the remaining bytes so FileIO does not allocate
# chunk_size (e.g. 1 GiB). Measure them once: seeking a video stream
# rebuilt from many samples walks all of them
remaining = cls._remaining_bytes(stream)

while True:
to_read = cls._bounded_read_size(stream, chunk_size)
if to_read == 0:
to_read = chunk_size if remaining is None else min(chunk_size, remaining)
if to_read <= 0:
break
data = stream.read(to_read)
if not data:
break
if remaining is not None:
remaining -= len(data)
yield data

@classmethod
def _bounded_read_size(cls, stream: T.IO[bytes], chunk_size: int) -> int:
"""Cap read() to remaining bytes so FileIO does not allocate chunk_size (e.g. 1 GiB)."""
remaining = cls._remaining_bytes(stream)
if remaining is None:
return chunk_size
if remaining <= 0:
return 0
return min(chunk_size, remaining)

@classmethod
def _remaining_bytes(cls, stream: T.IO[bytes]) -> int | None:
try:
Expand Down
38 changes: 38 additions & 0 deletions tests/unit/test_upload_api_v4.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,3 +92,41 @@ def test_chunkize_caps_read_to_remaining_bytes():
)
assert b"".join(chunks) == b"hello world"
assert spy.read_sizes == [len(b"hello world")]


class _SeekSpy(_ReadSpy):
def __init__(self, data: bytes):
super().__init__(data)
self.seek_count = 0

def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
self.seek_count += 1
return super().seek(offset, whence)


def test_chunkize_measures_remaining_bytes_once():
# Seeking a video stream rebuilt from many samples walks all of them,
# so seeking for every chunk made large video uploads quadratic
spy = _SeekSpy(b"hello world")
spy.seek(6)
spy.seek_count = 0
chunks = list(upload_api_v4.UploadService.chunkize_byte_stream(spy, 2))
assert chunks == [b"wo", b"rl", b"d"]
assert spy.read_sizes == [2, 2, 1]
assert spy.seek_count == 2


class _UnseekableStream:
def __init__(self, data: bytes):
self._stream = io.BytesIO(data)

def read(self, size: int = -1) -> bytes:
return self._stream.read(size)


def test_chunkize_unseekable_stream():
stream = _UnseekableStream(b"hello world")
chunks = list(
upload_api_v4.UploadService.chunkize_byte_stream(stream, 4) # type: ignore[arg-type]
)
assert chunks == [b"hell", b"o wo", b"rld"]
Loading