From 880a8f020435214adbc2443450a6c32812593e0c Mon Sep 17 00:00:00 2001 From: Caglar Pir Date: Tue, 17 Nov 2026 14:17:29 +0100 Subject: [PATCH] Don't take unfinished uploads in the upload history as uploaded When finish_upload answered without a cluster ID, as for an organization that the account is not a member of, upload wrote the files to the upload history with "cluster_id": null, although nothing was uploaded. #955 stops writing such records, but the ones already written still made upload skip the files as already uploaded, and check_upload_history report them so. A record whose summary has the cluster ID null is now not taken as uploaded, by the duplicate check of upload, by check_upload_history and by the lookup of images uploaded in other sequences; upload logs that it ignores it. Only an explicit null counts: records written before 0.14.0 have no cluster ID and still count as uploaded. --- mapillary_tools/history.py | 36 ++++++++- tests/unit/test_history.py | 154 ++++++++++++++++++++++++++++++++++++- 2 files changed, 188 insertions(+), 2 deletions(-) diff --git a/mapillary_tools/history.py b/mapillary_tools/history.py index d35310f9e..158044e15 100644 --- a/mapillary_tools/history.py +++ b/mapillary_tools/history.py @@ -46,7 +46,29 @@ def history_desc_path(md5sum: str) -> Path: ) +def is_unfinished_upload(record: T.Any) -> bool: + """ + Tell whether an upload history record is of an upload that Mapillary did + not finish, so nothing was uploaded: its summary has the cluster ID null. + upload wrote such records when finish_upload answered without a cluster + ID, as for an organization that the account is not a member of. Only an + explicit null counts: records written before 0.14.0 have no cluster ID. + """ + if not isinstance(record, dict): + return False + summary = record.get("summary") + return ( + isinstance(summary, dict) + and "cluster_id" in summary + and summary["cluster_id"] is None + ) + + def read_history_record(md5sum: str) -> None | T.Dict[str, T.Any]: + """ + Read the upload history record of the checksum, or return None if there + is none, or if it is of an upload that Mapillary did not finish + """ if not constants.MAPILLARY_UPLOAD_HISTORY_PATH: return None @@ -57,11 +79,20 @@ def read_history_record(md5sum: str) -> None | T.Dict[str, T.Any]: with path.open("r") as fp: try: - return json.load(fp) + record = json.load(fp) except json.JSONDecodeError as ex: LOG.error(f"Failed to read upload history {path}: {ex}") return None + if is_unfinished_upload(record): + LOG.info( + "Not taken as uploaded: the upload in the history record %s was not finished", + path, + ) + return None + + return record + def _normalize_md5sum(value: T.Any) -> str | None: if not isinstance(value, str) or len(value) != 32: @@ -113,6 +144,9 @@ def find_uploaded_image_md5s(md5sums: T.Iterable[str]) -> set[str]: LOG.warning("Invalid upload history record %s", path) continue + if is_unfinished_upload(record): + continue + descs = record.get("descs") if not isinstance(descs, list): continue diff --git a/tests/unit/test_history.py b/tests/unit/test_history.py index dd0783768..da659798b 100644 --- a/tests/unit/test_history.py +++ b/tests/unit/test_history.py @@ -5,11 +5,12 @@ import json import tempfile +import typing as T from pathlib import Path from unittest.mock import patch import pytest -from mapillary_tools import history, types +from mapillary_tools import constants, history, types, upload class TestValidateHexdigits: @@ -147,3 +148,154 @@ def test_write_history_with_metadatas(self): data = json.load(fp) assert "descs" in data assert len(data["descs"]) == 1 + + +TEST_IMAGE_DIR = Path(__file__).parent.parent.joinpath("data", "images") + + +def _write_record(md5sum: str, summary: dict, descs: T.Optional[list] = None) -> Path: + path = history.history_desc_path(md5sum) + path.parent.mkdir(parents=True, exist_ok=True) + record: dict = {"params": {}, "summary": summary} + if descs is not None: + record["descs"] = descs + path.write_text(json.dumps(record)) + return path + + +class TestUnfinishedUploads: + # upload wrote a record with the cluster ID null when finish_upload + # answered without one, as for an organization that the account is not a + # member of: nothing was uploaded + + @pytest.fixture(autouse=True) + def _history_path(self, tmp_path, monkeypatch): + monkeypatch.setattr( + constants, "MAPILLARY_UPLOAD_HISTORY_PATH", str(tmp_path / "history") + ) + + def test_unfinished_record_is_not_read(self): + _write_record("a" * 32, {"cluster_id": None, "upload_end_time": 1.0}) + assert history.read_history_record("a" * 32) is None + + @pytest.mark.parametrize( + "summary", + [ + # Written before 0.14.0, which added the cluster ID + {"upload_end_time": 1.0}, + {"cluster_id": "123", "upload_end_time": 1.0}, + # Uploaded with --dry_run + {"cluster_id": "0", "upload_end_time": 1.0}, + ], + ) + def test_finished_record_is_read(self, summary: dict): + _write_record("a" * 32, summary) + record = history.read_history_record("a" * 32) + assert record is not None + assert record["summary"] == summary + + def test_find_uploaded_image_md5s_skips_unfinished_records(self): + unfinished_image, finished_image = "b" * 32, "c" * 32 + _write_record( + "a" * 32, + {"cluster_id": None}, + [{"filetype": "image", "md5sum": unfinished_image}], + ) + _write_record("d" * 32, {}, [{"filetype": "image", "md5sum": finished_image}]) + assert history.find_uploaded_image_md5s([unfinished_image, finished_image]) == { + finished_image + } + + def test_check_upload_history_ignores_unfinished_records(self, tmp_path): + image_paths = [tmp_path / "first.jpg", tmp_path / "second.jpg"] + metadatas = [] + for idx, image_path in enumerate(image_paths): + image_path.write_bytes(f"image {idx}".encode()) + metadatas.append( + types.ImageMetadata( + filename=image_path, + lat=1.0, + lon=2.0, + alt=None, + angle=None, + time=float(idx), + MAPSequenceUUID="sequence", + ) + ) + + def check() -> dict: + [result] = upload.check_upload_history( + image_paths, _metadatas_from_process=metadatas + ) + return result + + sequence_md5sum = check()["sequence_md5sum"] + image_descs = [ + {"filetype": "image", "md5sum": metadata.md5sum} for metadata in metadatas + ] + # Neither the sequence nor its images, also looked up one by one + _write_record(sequence_md5sum, {"cluster_id": None}, image_descs) + result = check() + assert result["already_uploaded"] is False + assert result["already_uploaded_filenames"] == [] + + _write_record(sequence_md5sum, {"cluster_id": "123"}, image_descs) + assert check()["already_uploaded"] is True + + def test_upload_uploads_unfinished_upload_again(self, tmp_path, monkeypatch): + monkeypatch.setattr( + constants, "MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN", True + ) + upload_path = tmp_path / "uploads" + upload_path.mkdir() + monkeypatch.setenv("MAPILLARY_UPLOAD_ENDPOINT", str(upload_path)) + + image_dir = tmp_path / "images" + image_dir.mkdir() + metadatas: list[types.MetadataOrError] = [] + for idx, name in enumerate(["DSC00001.JPG", "DSC00497.JPG"]): + image_path = image_dir / name + image_path.write_bytes(TEST_IMAGE_DIR.joinpath(name).read_bytes()) + metadatas.append( + types.ImageMetadata( + filename=image_path, + lat=58.5927694 + idx * 0.0001, + lon=16.1840944, + alt=None, + angle=None, + time=1613222680.0 + idx, + MAPSequenceUUID="sequence", + filesize=image_path.stat().st_size, + ) + ) + + def upload_images() -> None: + upload.upload( + image_dir, + {"user_upload_token": "YOUR_USER_ACCESS_TOKEN"}, + num_upload_workers=1, + _metadatas_from_process=metadatas, + dry_run=True, + ) + + upload_images() + [record_path] = (tmp_path / "history").glob("*/*.json") + + def rewrite_summary(**summary) -> None: + record = json.loads(record_path.read_text()) + record["summary"] = summary + record_path.write_text(json.dumps(record)) + + # Not finished: uploaded again, and the record replaced + rewrite_summary(cluster_id=None, upload_end_time=1.0) + upload_images() + summary = json.loads(record_path.read_text())["summary"] + assert summary["cluster_id"] == "0" + assert summary["upload_end_time"] != 1.0 + + # Written before 0.14.0, without a cluster ID: skipped as uploaded + rewrite_summary(upload_end_time=1.0) + upload_images() + assert json.loads(record_path.read_text())["summary"] == { + "upload_end_time": 1.0 + }