Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 35 additions & 1 deletion mapillary_tools/history.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,29 @@ def history_desc_path(md5sum: str) -> Path:
)


def is_unfinished_upload(record: T.Any) -> bool:
"""
Tell whether an upload history record is of an upload that Mapillary did
not finish, so nothing was uploaded: its summary has the cluster ID null.
upload wrote such records when finish_upload answered without a cluster
ID, as for an organization that the account is not a member of. Only an
explicit null counts: records written before 0.14.0 have no cluster ID.
"""
if not isinstance(record, dict):
return False
summary = record.get("summary")
return (
isinstance(summary, dict)
and "cluster_id" in summary
and summary["cluster_id"] is None
)


def read_history_record(md5sum: str) -> None | T.Dict[str, T.Any]:
"""
Read the upload history record of the checksum, or return None if there
is none, or if it is of an upload that Mapillary did not finish
"""
if not constants.MAPILLARY_UPLOAD_HISTORY_PATH:
return None

Expand All @@ -57,11 +79,20 @@ def read_history_record(md5sum: str) -> None | T.Dict[str, T.Any]:

with path.open("r") as fp:
try:
return json.load(fp)
record = json.load(fp)
except json.JSONDecodeError as ex:
LOG.error(f"Failed to read upload history {path}: {ex}")
return None

if is_unfinished_upload(record):
LOG.info(
"Not taken as uploaded: the upload in the history record %s was not finished",
path,
)
return None

return record


def _normalize_md5sum(value: T.Any) -> str | None:
if not isinstance(value, str) or len(value) != 32:
Expand Down Expand Up @@ -113,6 +144,9 @@ def find_uploaded_image_md5s(md5sums: T.Iterable[str]) -> set[str]:
LOG.warning("Invalid upload history record %s", path)
continue

if is_unfinished_upload(record):
continue

descs = record.get("descs")
if not isinstance(descs, list):
continue
Expand Down
154 changes: 153 additions & 1 deletion tests/unit/test_history.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@

import json
import tempfile
import typing as T
from pathlib import Path
from unittest.mock import patch

import pytest
from mapillary_tools import history, types
from mapillary_tools import constants, history, types, upload


class TestValidateHexdigits:
Expand Down Expand Up @@ -147,3 +148,154 @@ def test_write_history_with_metadatas(self):
data = json.load(fp)
assert "descs" in data
assert len(data["descs"]) == 1


TEST_IMAGE_DIR = Path(__file__).parent.parent.joinpath("data", "images")


def _write_record(md5sum: str, summary: dict, descs: T.Optional[list] = None) -> Path:
path = history.history_desc_path(md5sum)
path.parent.mkdir(parents=True, exist_ok=True)
record: dict = {"params": {}, "summary": summary}
if descs is not None:
record["descs"] = descs
path.write_text(json.dumps(record))
return path


class TestUnfinishedUploads:
# upload wrote a record with the cluster ID null when finish_upload
# answered without one, as for an organization that the account is not a
# member of: nothing was uploaded

@pytest.fixture(autouse=True)
def _history_path(self, tmp_path, monkeypatch):
monkeypatch.setattr(
constants, "MAPILLARY_UPLOAD_HISTORY_PATH", str(tmp_path / "history")
)

def test_unfinished_record_is_not_read(self):
_write_record("a" * 32, {"cluster_id": None, "upload_end_time": 1.0})
assert history.read_history_record("a" * 32) is None

@pytest.mark.parametrize(
"summary",
[
# Written before 0.14.0, which added the cluster ID
{"upload_end_time": 1.0},
{"cluster_id": "123", "upload_end_time": 1.0},
# Uploaded with --dry_run
{"cluster_id": "0", "upload_end_time": 1.0},
],
)
def test_finished_record_is_read(self, summary: dict):
_write_record("a" * 32, summary)
record = history.read_history_record("a" * 32)
assert record is not None
assert record["summary"] == summary

def test_find_uploaded_image_md5s_skips_unfinished_records(self):
unfinished_image, finished_image = "b" * 32, "c" * 32
_write_record(
"a" * 32,
{"cluster_id": None},
[{"filetype": "image", "md5sum": unfinished_image}],
)
_write_record("d" * 32, {}, [{"filetype": "image", "md5sum": finished_image}])
assert history.find_uploaded_image_md5s([unfinished_image, finished_image]) == {
finished_image
}

def test_check_upload_history_ignores_unfinished_records(self, tmp_path):
image_paths = [tmp_path / "first.jpg", tmp_path / "second.jpg"]
metadatas = []
for idx, image_path in enumerate(image_paths):
image_path.write_bytes(f"image {idx}".encode())
metadatas.append(
types.ImageMetadata(
filename=image_path,
lat=1.0,
lon=2.0,
alt=None,
angle=None,
time=float(idx),
MAPSequenceUUID="sequence",
)
)

def check() -> dict:
[result] = upload.check_upload_history(
image_paths, _metadatas_from_process=metadatas
)
return result

sequence_md5sum = check()["sequence_md5sum"]
image_descs = [
{"filetype": "image", "md5sum": metadata.md5sum} for metadata in metadatas
]
# Neither the sequence nor its images, also looked up one by one
_write_record(sequence_md5sum, {"cluster_id": None}, image_descs)
result = check()
assert result["already_uploaded"] is False
assert result["already_uploaded_filenames"] == []

_write_record(sequence_md5sum, {"cluster_id": "123"}, image_descs)
assert check()["already_uploaded"] is True

def test_upload_uploads_unfinished_upload_again(self, tmp_path, monkeypatch):
monkeypatch.setattr(
constants, "MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN", True
)
upload_path = tmp_path / "uploads"
upload_path.mkdir()
monkeypatch.setenv("MAPILLARY_UPLOAD_ENDPOINT", str(upload_path))

image_dir = tmp_path / "images"
image_dir.mkdir()
metadatas: list[types.MetadataOrError] = []
for idx, name in enumerate(["DSC00001.JPG", "DSC00497.JPG"]):
image_path = image_dir / name
image_path.write_bytes(TEST_IMAGE_DIR.joinpath(name).read_bytes())
metadatas.append(
types.ImageMetadata(
filename=image_path,
lat=58.5927694 + idx * 0.0001,
lon=16.1840944,
alt=None,
angle=None,
time=1613222680.0 + idx,
MAPSequenceUUID="sequence",
filesize=image_path.stat().st_size,
)
)

def upload_images() -> None:
upload.upload(
image_dir,
{"user_upload_token": "YOUR_USER_ACCESS_TOKEN"},
num_upload_workers=1,
_metadatas_from_process=metadatas,
dry_run=True,
)

upload_images()
[record_path] = (tmp_path / "history").glob("*/*.json")

def rewrite_summary(**summary) -> None:
record = json.loads(record_path.read_text())
record["summary"] = summary
record_path.write_text(json.dumps(record))

# Not finished: uploaded again, and the record replaced
rewrite_summary(cluster_id=None, upload_end_time=1.0)
upload_images()
summary = json.loads(record_path.read_text())["summary"]
assert summary["cluster_id"] == "0"
assert summary["upload_end_time"] != 1.0

# Written before 0.14.0, without a cluster ID: skipped as uploaded
rewrite_summary(upload_end_time=1.0)
upload_images()
assert json.loads(record_path.read_text())["summary"] == {
"upload_end_time": 1.0
}
Loading