From 58a441a836653056284e96614e0cd2fd128c745b Mon Sep 17 00:00:00 2001 From: Caglar Pir Date: Mon, 16 Nov 2026 15:01:35 +0100 Subject: [PATCH] Show progress while upload loads the description file and finds the files Before the first checksum, upload validated every entry of the description file and searched the import paths three times (images, videos, zip files), resolving every path found twice, all without output. With 100,000 images on an SSD that took about 12 seconds, and longer on network drives. - Loading the description file shows a progress bar. - The import paths are searched once for all file types, with a counter of the files looked at. - Paths are resolved with each folder resolved once (utils.PathResolver). The result is the same as Path.resolve(): a file that is a link is still resolved in full, and on Windows every path is, as resolving there also gives file names the case they have on the drive. Errors checking a file for a link are ignored, as Path.resolve() ignores them, such as for a file in a folder that can be listed but not searched. check_upload_history loads and finds files the same way, so it shows the same progress on stderr. Its JSON on stdout is unchanged. On a Mac SSD with 100,000 images, finding the files went from 7.2 s to 2.3 s. Validating the entries still takes about 5 s, now with the progress shown. --- mapillary_tools/upload.py | 104 ++++++++++----- mapillary_tools/utils.py | 25 ++++ tests/unit/test_upload.py | 264 ++++++++++++++++++++++++++++++++++++++ tests/unit/test_utils.py | 51 ++++++++ 4 files changed, 409 insertions(+), 35 deletions(-) create mode 100644 tests/unit/test_upload.py diff --git a/mapillary_tools/upload.py b/mapillary_tools/upload.py index 460d299b..456fd59d 100644 --- a/mapillary_tools/upload.py +++ b/mapillary_tools/upload.py @@ -686,20 +686,20 @@ def _api_logging_failed(payload: dict, exc: Exception, dry_run: bool = False): LOG.warning(f"Error from logging action {action}", exc_info=True) -_M = T.TypeVar("_M", bound=types.Metadata) - - class _UploadCandidates(T.NamedTuple): image_metadatas: list[types.ImageMetadata] video_metadatas: list[types.VideoMetadata] zip_paths: list[Path] -def _find_metadata_with_filename_existed_in( - metadatas: T.Iterable[_M], paths: T.Iterable[Path] -) -> list[_M]: - resolved_image_paths = set(p.resolve() for p in paths) - return [d for d in metadatas if d.filename.resolve() in resolved_image_paths] +def _iterate_import_files( + import_paths: T.Sequence[Path], skip_subfolders: bool +) -> T.Generator[Path, None, None]: + for path in import_paths: + if path.is_dir(): + yield from utils.iterate_files(path, not skip_subfolders) + else: + yield path def _find_upload_candidates( @@ -707,16 +707,40 @@ def _find_upload_candidates( import_paths: T.Sequence[Path], skip_subfolders: bool, ) -> _UploadCandidates: - image_metadatas = _find_metadata_with_filename_existed_in( - (m for m in metadatas if isinstance(m, types.ImageMetadata)), - utils.find_images(import_paths, skip_subfolders=skip_subfolders), - ) - video_metadatas = _find_metadata_with_filename_existed_in( - (m for m in metadatas if isinstance(m, types.VideoMetadata)), - utils.find_videos(import_paths, skip_subfolders=skip_subfolders), - ) + resolver = utils.PathResolver() + found_image_paths: set[Path] = set() + found_video_paths: set[Path] = set() + # Resolved path to the path found, in the order found + found_zip_paths: dict[Path, Path] = {} + + # Look through the import paths once for all file types, and show the + # progress, as it takes long for many files or on network drives + with tqdm( + desc="Finding files", + unit=" files", + disable=LOG.isEnabledFor(logging.DEBUG), + ) as pbar: + for path in _iterate_import_files(import_paths, skip_subfolders): + if utils.is_image_file(path): + found_image_paths.add(resolver.resolve(path)) + elif utils.is_video_file(path): + found_video_paths.add(resolver.resolve(path)) + elif path.suffix.lower() == ".zip": + found_zip_paths.setdefault(resolver.resolve(path), path) + pbar.update(1) + + image_metadatas: list[types.ImageMetadata] = [] + video_metadatas: list[types.VideoMetadata] = [] + for metadata in metadatas: + if isinstance(metadata, types.ImageMetadata): + if resolver.resolve(metadata.filename) in found_image_paths: + image_metadatas.append(metadata) + elif isinstance(metadata, types.VideoMetadata): + if resolver.resolve(metadata.filename) in found_video_paths: + video_metadatas.append(metadata) + zip_paths: list[Path] = [] - for zip_path in utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders): + for zip_path in found_zip_paths.values(): if zip_path.name.startswith("._"): # The AppleDouble file (._NAME) that macOS writes beside a file it # copies to a drive that cannot hold its metadata: named like a @@ -850,26 +874,36 @@ def _load_valid_metadatas_from_desc_path( desc_path = _find_desc_path(import_paths) if desc_path == "-": - try: - metadatas = DescriptionJSONSerializer.deserialize_stream(sys.stdin.buffer) - except json.JSONDecodeError as ex: - raise exceptions.MapillaryInvalidDescriptionFile( - f"Invalid JSON stream from {desc_path}: {ex}" - ) from ex + return _deserialize_desc_stream(sys.stdin.buffer, desc_path) - else: - if not os.path.isfile(desc_path): - raise exceptions.MapillaryFileNotFoundError( - f"Description file not found: {desc_path}" - ) - with open(desc_path, "rb") as fp: - try: - metadatas = DescriptionJSONSerializer.deserialize_stream(fp) - except json.JSONDecodeError as ex: - raise exceptions.MapillaryInvalidDescriptionFile( - f"Invalid JSON stream from {desc_path}: {ex}" - ) from ex + if not os.path.isfile(desc_path): + raise exceptions.MapillaryFileNotFoundError( + f"Description file not found: {desc_path}" + ) + with open(desc_path, "rb") as fp: + return _deserialize_desc_stream(fp, desc_path) + +def _deserialize_desc_stream(fp: T.IO[bytes], desc_path: str) -> list[types.Metadata]: + try: + descs = json.load(fp) + except json.JSONDecodeError as ex: + raise exceptions.MapillaryInvalidDescriptionFile( + f"Invalid JSON stream from {desc_path}: {ex}" + ) from ex + + # Show the progress, as validating every entry takes long for large + # description files + metadatas: list[types.Metadata] = [] + with tqdm( + descs, + desc="Loading the description file", + unit=" entries", + disable=LOG.isEnabledFor(logging.DEBUG), + ) as pbar: + for desc in pbar: + if "error" not in desc: + metadatas.append(DescriptionJSONSerializer.from_desc(desc)) return metadatas diff --git a/mapillary_tools/utils.py b/mapillary_tools/utils.py index e5c28e3e..215f8ebb 100644 --- a/mapillary_tools/utils.py +++ b/mapillary_tools/utils.py @@ -219,6 +219,31 @@ def find_image_samples( return samples_by_video, ambiguous +class PathResolver: + """ + Resolve the paths of many files like Path.resolve(), resolving each folder + only once. Path.resolve() checks every folder on the way to a file, which + takes long for many files, especially on network drives + """ + + def __init__(self) -> None: + self._folders: dict[Path, Path] = {} + + def resolve(self, path: Path) -> Path: + # On Windows, resolving also gives a file name the case it has on the drive. + # os.path.islink() ignores errors as Path.resolve() does, such as for a file + # in a folder that can be listed but not searched, which Path.is_symlink() + # raises before Python 3.14 + if os.name == "nt" or path.name in ("", "..") or os.path.islink(path): + return path.resolve() + folder = path.parent + resolved_folder = self._folders.get(folder) + if resolved_folder is None: + resolved_folder = folder.resolve() + self._folders[folder] = resolved_folder + return resolved_folder.joinpath(path.name) + + def deduplicate_paths(paths: T.Iterable[Path]) -> T.Generator[Path, None, None]: resolved_paths: set[Path] = set() for p in paths: diff --git a/tests/unit/test_upload.py b/tests/unit/test_upload.py new file mode 100644 index 00000000..21c8ebd0 --- /dev/null +++ b/tests/unit/test_upload.py @@ -0,0 +1,264 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the BSD license found in the +# LICENSE file in the root directory of this source tree. + +import io +import json +import os +import sys +import typing as T +from pathlib import Path + +import pytest + +from mapillary_tools import exceptions, types, upload, utils + + +def _image(filename: Path) -> types.ImageMetadata: + return types.ImageMetadata( + filename=filename, + lat=1.0, + lon=2.0, + alt=None, + angle=None, + time=1.0, + MAPSequenceUUID="0", + ) + + +def _video(filename: Path) -> types.VideoMetadata: + return types.VideoMetadata( + filename=filename, filetype=types.FileType.CAMM, points=[] + ) + + +def _find_upload_candidates_by_file_type( + metadatas: T.Sequence[types.Metadata], + import_paths: T.Sequence[Path], + skip_subfolders: bool, +): + """How the candidates were found before: one scan per file type""" + resolved_images = { + p.resolve() + for p in utils.find_images(import_paths, skip_subfolders=skip_subfolders) + } + resolved_videos = { + p.resolve() + for p in utils.find_videos(import_paths, skip_subfolders=skip_subfolders) + } + return upload._UploadCandidates( + [ + m + for m in metadatas + if isinstance(m, types.ImageMetadata) + and m.filename.resolve() in resolved_images + ], + [ + m + for m in metadatas + if isinstance(m, types.VideoMetadata) + and m.filename.resolve() in resolved_videos + ], + [ + p + for p in utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders) + if not p.name.startswith("._") + ], + ) + + +@pytest.fixture +def capture_tree(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + root = tmp_path / "capture" + for name in [ + "a.jpg", + "b.JPG", + "c.mp4", + "d.zip", + "notes.txt", + ".hidden.jpg", + "sub/e.jpg", + "sub/f.mov", + "sub/g.zip", + ".hidden_folder/h.jpg", + ]: + root.joinpath(name).parent.mkdir(parents=True, exist_ok=True) + root.joinpath(name).touch() + (tmp_path / "elsewhere.jpg").touch() + monkeypatch.chdir(tmp_path) + return root + + +def _capture_metadatas(root: Path) -> list[types.Metadata]: + return [ + _image(root / "sub" / "e.jpg"), + _image(root / "b.JPG"), + _video(root / "sub" / "f.mov"), + _image(root / "a.jpg"), + _video(root / "c.mp4"), + # Under the import path, but not found by a scan + _image(root / ".hidden.jpg"), + _image(root / ".hidden_folder" / "h.jpg"), + # Not under the import path + _image(root.parent / "elsewhere.jpg"), + # A video with an image name is not uploaded as a video + _video(root / "a.jpg"), + ] + + +@pytest.mark.parametrize("skip_subfolders", [False, True]) +def test_find_upload_candidates(capture_tree: Path, skip_subfolders: bool): + metadatas = _capture_metadatas(capture_tree) + + candidates = upload._find_upload_candidates( + metadatas, [capture_tree], skip_subfolders=skip_subfolders + ) + + if skip_subfolders: + assert candidates == upload._UploadCandidates( + [metadatas[1], metadatas[3]], + [metadatas[4]], + [capture_tree / "d.zip"], + ) + else: + # In the order of the metadatas + assert candidates == upload._UploadCandidates( + [metadatas[0], metadatas[1], metadatas[3]], + [metadatas[2], metadatas[4]], + [capture_tree / "d.zip", capture_tree / "sub" / "g.zip"], + ) + assert candidates == _find_upload_candidates_by_file_type( + metadatas, [capture_tree], skip_subfolders=skip_subfolders + ) + + +def test_find_upload_candidates_once_from_overlapping_paths(capture_tree: Path): + metadatas = _capture_metadatas(capture_tree) + import_paths = [ + # Relative to the working directory + Path("capture"), + capture_tree / "sub" / ".." / "a.jpg", + capture_tree / "sub", + capture_tree / "sub" / "g.zip", + # Hidden, but named + capture_tree / ".hidden.jpg", + ] + if not sys.platform.startswith("win"): + (capture_tree.parent / "linked_capture").symlink_to(capture_tree) + (capture_tree.parent / "linked_image.jpg").symlink_to( + capture_tree.parent / "elsewhere.jpg" + ) + import_paths += [ + capture_tree.parent / "linked_capture", + capture_tree.parent / "linked_image.jpg", + ] + + candidates = upload._find_upload_candidates( + metadatas, import_paths, skip_subfolders=False + ) + + image_metadatas = [metadatas[0], metadatas[1], metadatas[3], metadatas[5]] + if not sys.platform.startswith("win"): + # Found through the link to it + image_metadatas.append(metadatas[7]) + assert candidates == upload._UploadCandidates( + image_metadatas, + [metadatas[2], metadatas[4]], + [Path("capture") / "d.zip", Path("capture") / "sub" / "g.zip"], + ) + assert candidates == _find_upload_candidates_by_file_type( + metadatas, import_paths, skip_subfolders=False + ) + + +@pytest.mark.skipif( + sys.platform.startswith("win") or (hasattr(os, "geteuid") and os.geteuid() == 0), + reason="Folder permissions do not apply on Windows or to root", +) +def test_find_upload_candidates_in_a_folder_that_cannot_be_searched( + capture_tree: Path, +): + metadatas = _capture_metadatas(capture_tree) + folder = capture_tree / "sub" + # Listed, but its files cannot be looked at + folder.chmod(0o600) + try: + candidates = upload._find_upload_candidates( + metadatas, [capture_tree], skip_subfolders=False + ) + expected = _find_upload_candidates_by_file_type( + metadatas, [capture_tree], skip_subfolders=False + ) + finally: + folder.chmod(0o700) + + assert candidates == expected + assert [m.filename for m in candidates.image_metadatas] == [ + folder / "e.jpg", + capture_tree / "b.JPG", + capture_tree / "a.jpg", + ] + + +def test_find_upload_candidates_skips_apple_double_zip(capture_tree: Path): + apple_double = capture_tree / "._d.zip" + apple_double.touch() + + candidates = upload._find_upload_candidates([], [apple_double], False) + + assert candidates == upload._UploadCandidates([], [], []) + + +def _write_desc_file(path: Path, descs: list) -> str: + path.write_text(json.dumps(descs)) + return str(path) + + +def test_load_descs_from_desc_path(tmp_path: Path): + image = _image(tmp_path / "a.jpg") + video = _video(tmp_path / "c.mp4") + desc_path = _write_desc_file( + tmp_path / "descs.json", + [ + upload.DescriptionJSONSerializer.as_desc(image), + {"error": {"type": "Error", "message": "x"}, "filename": "b.jpg"}, + upload.DescriptionJSONSerializer.as_desc(video), + ], + ) + + metadatas = upload._load_valid_metadatas_from_desc_path([tmp_path], desc_path) + + assert [type(m) for m in metadatas] == [types.ImageMetadata, types.VideoMetadata] + assert [m.filename for m in metadatas] == [ + image.filename.resolve(), + video.filename.resolve(), + ] + + +def test_load_descs_from_stdin(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + image = _image(tmp_path / "a.jpg") + data = json.dumps([upload.DescriptionJSONSerializer.as_desc(image)]).encode() + monkeypatch.setattr(sys, "stdin", io.TextIOWrapper(io.BytesIO(data))) + + metadatas = upload._load_valid_metadatas_from_desc_path([tmp_path], "-") + + assert [m.filename for m in metadatas] == [image.filename.resolve()] + + +def test_load_descs_invalid(tmp_path: Path): + bad_json = tmp_path / "bad.json" + bad_json.write_text("[{") + with pytest.raises(exceptions.MapillaryInvalidDescriptionFile): + upload._load_valid_metadatas_from_desc_path([tmp_path], str(bad_json)) + + desc_path = _write_desc_file( + tmp_path / "descs.json", [{"filename": "a.jpg", "filetype": "image"}] + ) + with pytest.raises(exceptions.MapillaryMetadataValidationError): + upload._load_valid_metadatas_from_desc_path([tmp_path], desc_path) + + with pytest.raises(exceptions.MapillaryFileNotFoundError): + upload._load_valid_metadatas_from_desc_path( + [tmp_path], str(tmp_path / "missing.json") + ) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 4e3a4390..35914bec 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -356,3 +356,54 @@ def test_md5sum_fp_progress(): assert md5.hexdigest() == hashlib.md5(data).hexdigest() assert sum(progress) == len(data) + + +def test_path_resolver_resolves_like_path_resolve(tmp_path: Path, monkeypatch): + folder = tmp_path / "folder" + folder.joinpath("sub").mkdir(parents=True) + image = folder / "image.jpg" + image.touch() + paths = [ + image, + folder / "sub" / ".." / "image.jpg", + folder / "missing.jpg", + folder / "sub" / "..", + Path("image.jpg"), + Path("sub") / "image.jpg", + Path("."), + Path(".."), + ] + if not sys.platform.startswith("win"): + # A link to a folder, a link to a file, and a link that leads nowhere + (tmp_path / "linked_folder").symlink_to(folder) + (folder / "linked_image.jpg").symlink_to(image) + (folder / "broken_link.jpg").symlink_to(tmp_path / "nowhere.jpg") + paths += [ + tmp_path / "linked_folder" / "image.jpg", + tmp_path / "linked_folder" / "linked_image.jpg", + folder / "linked_image.jpg", + folder / "broken_link.jpg", + ] + monkeypatch.chdir(folder) + + resolver = utils.PathResolver() + # Twice, to resolve with the folders resolved before too + for _ in range(2): + assert [resolver.resolve(p) for p in paths] == [p.resolve() for p in paths] + + +@pytest.mark.skipif( + sys.platform.startswith("win") or (hasattr(os, "geteuid") and os.geteuid() == 0), + reason="Folder permissions do not apply on Windows or to root", +) +def test_path_resolver_in_a_folder_that_cannot_be_searched(tmp_path: Path): + folder = tmp_path / "unsearchable" + folder.mkdir() + image = folder / "image.jpg" + image.touch() + # Listed, but its files cannot be looked at + folder.chmod(0o600) + try: + assert utils.PathResolver().resolve(image) == image.resolve() + finally: + folder.chmod(0o700)