diff --git a/mapillary_tools/upload.py b/mapillary_tools/upload.py index 460d299b..456fd59d 100644 --- a/mapillary_tools/upload.py +++ b/mapillary_tools/upload.py @@ -686,20 +686,20 @@ def _api_logging_failed(payload: dict, exc: Exception, dry_run: bool = False): LOG.warning(f"Error from logging action {action}", exc_info=True) -_M = T.TypeVar("_M", bound=types.Metadata) - - class _UploadCandidates(T.NamedTuple): image_metadatas: list[types.ImageMetadata] video_metadatas: list[types.VideoMetadata] zip_paths: list[Path] -def _find_metadata_with_filename_existed_in( - metadatas: T.Iterable[_M], paths: T.Iterable[Path] -) -> list[_M]: - resolved_image_paths = set(p.resolve() for p in paths) - return [d for d in metadatas if d.filename.resolve() in resolved_image_paths] +def _iterate_import_files( + import_paths: T.Sequence[Path], skip_subfolders: bool +) -> T.Generator[Path, None, None]: + for path in import_paths: + if path.is_dir(): + yield from utils.iterate_files(path, not skip_subfolders) + else: + yield path def _find_upload_candidates( @@ -707,16 +707,40 @@ def _find_upload_candidates( import_paths: T.Sequence[Path], skip_subfolders: bool, ) -> _UploadCandidates: - image_metadatas = _find_metadata_with_filename_existed_in( - (m for m in metadatas if isinstance(m, types.ImageMetadata)), - utils.find_images(import_paths, skip_subfolders=skip_subfolders), - ) - video_metadatas = _find_metadata_with_filename_existed_in( - (m for m in metadatas if isinstance(m, types.VideoMetadata)), - utils.find_videos(import_paths, skip_subfolders=skip_subfolders), - ) + resolver = utils.PathResolver() + found_image_paths: set[Path] = set() + found_video_paths: set[Path] = set() + # Resolved path to the path found, in the order found + found_zip_paths: dict[Path, Path] = {} + + # Look through the import paths once for all file types, and show the + # progress, as it takes long for many files or on network drives + with tqdm( + desc="Finding files", + unit=" files", + disable=LOG.isEnabledFor(logging.DEBUG), + ) as pbar: + for path in _iterate_import_files(import_paths, skip_subfolders): + if utils.is_image_file(path): + found_image_paths.add(resolver.resolve(path)) + elif utils.is_video_file(path): + found_video_paths.add(resolver.resolve(path)) + elif path.suffix.lower() == ".zip": + found_zip_paths.setdefault(resolver.resolve(path), path) + pbar.update(1) + + image_metadatas: list[types.ImageMetadata] = [] + video_metadatas: list[types.VideoMetadata] = [] + for metadata in metadatas: + if isinstance(metadata, types.ImageMetadata): + if resolver.resolve(metadata.filename) in found_image_paths: + image_metadatas.append(metadata) + elif isinstance(metadata, types.VideoMetadata): + if resolver.resolve(metadata.filename) in found_video_paths: + video_metadatas.append(metadata) + zip_paths: list[Path] = [] - for zip_path in utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders): + for zip_path in found_zip_paths.values(): if zip_path.name.startswith("._"): # The AppleDouble file (._NAME) that macOS writes beside a file it # copies to a drive that cannot hold its metadata: named like a @@ -850,26 +874,36 @@ def _load_valid_metadatas_from_desc_path( desc_path = _find_desc_path(import_paths) if desc_path == "-": - try: - metadatas = DescriptionJSONSerializer.deserialize_stream(sys.stdin.buffer) - except json.JSONDecodeError as ex: - raise exceptions.MapillaryInvalidDescriptionFile( - f"Invalid JSON stream from {desc_path}: {ex}" - ) from ex + return _deserialize_desc_stream(sys.stdin.buffer, desc_path) - else: - if not os.path.isfile(desc_path): - raise exceptions.MapillaryFileNotFoundError( - f"Description file not found: {desc_path}" - ) - with open(desc_path, "rb") as fp: - try: - metadatas = DescriptionJSONSerializer.deserialize_stream(fp) - except json.JSONDecodeError as ex: - raise exceptions.MapillaryInvalidDescriptionFile( - f"Invalid JSON stream from {desc_path}: {ex}" - ) from ex + if not os.path.isfile(desc_path): + raise exceptions.MapillaryFileNotFoundError( + f"Description file not found: {desc_path}" + ) + with open(desc_path, "rb") as fp: + return _deserialize_desc_stream(fp, desc_path) + +def _deserialize_desc_stream(fp: T.IO[bytes], desc_path: str) -> list[types.Metadata]: + try: + descs = json.load(fp) + except json.JSONDecodeError as ex: + raise exceptions.MapillaryInvalidDescriptionFile( + f"Invalid JSON stream from {desc_path}: {ex}" + ) from ex + + # Show the progress, as validating every entry takes long for large + # description files + metadatas: list[types.Metadata] = [] + with tqdm( + descs, + desc="Loading the description file", + unit=" entries", + disable=LOG.isEnabledFor(logging.DEBUG), + ) as pbar: + for desc in pbar: + if "error" not in desc: + metadatas.append(DescriptionJSONSerializer.from_desc(desc)) return metadatas diff --git a/mapillary_tools/utils.py b/mapillary_tools/utils.py index e5c28e3e..215f8ebb 100644 --- a/mapillary_tools/utils.py +++ b/mapillary_tools/utils.py @@ -219,6 +219,31 @@ def find_image_samples( return samples_by_video, ambiguous +class PathResolver: + """ + Resolve the paths of many files like Path.resolve(), resolving each folder + only once. Path.resolve() checks every folder on the way to a file, which + takes long for many files, especially on network drives + """ + + def __init__(self) -> None: + self._folders: dict[Path, Path] = {} + + def resolve(self, path: Path) -> Path: + # On Windows, resolving also gives a file name the case it has on the drive. + # os.path.islink() ignores errors as Path.resolve() does, such as for a file + # in a folder that can be listed but not searched, which Path.is_symlink() + # raises before Python 3.14 + if os.name == "nt" or path.name in ("", "..") or os.path.islink(path): + return path.resolve() + folder = path.parent + resolved_folder = self._folders.get(folder) + if resolved_folder is None: + resolved_folder = folder.resolve() + self._folders[folder] = resolved_folder + return resolved_folder.joinpath(path.name) + + def deduplicate_paths(paths: T.Iterable[Path]) -> T.Generator[Path, None, None]: resolved_paths: set[Path] = set() for p in paths: diff --git a/tests/unit/test_upload.py b/tests/unit/test_upload.py new file mode 100644 index 00000000..21c8ebd0 --- /dev/null +++ b/tests/unit/test_upload.py @@ -0,0 +1,264 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the BSD license found in the +# LICENSE file in the root directory of this source tree. + +import io +import json +import os +import sys +import typing as T +from pathlib import Path + +import pytest + +from mapillary_tools import exceptions, types, upload, utils + + +def _image(filename: Path) -> types.ImageMetadata: + return types.ImageMetadata( + filename=filename, + lat=1.0, + lon=2.0, + alt=None, + angle=None, + time=1.0, + MAPSequenceUUID="0", + ) + + +def _video(filename: Path) -> types.VideoMetadata: + return types.VideoMetadata( + filename=filename, filetype=types.FileType.CAMM, points=[] + ) + + +def _find_upload_candidates_by_file_type( + metadatas: T.Sequence[types.Metadata], + import_paths: T.Sequence[Path], + skip_subfolders: bool, +): + """How the candidates were found before: one scan per file type""" + resolved_images = { + p.resolve() + for p in utils.find_images(import_paths, skip_subfolders=skip_subfolders) + } + resolved_videos = { + p.resolve() + for p in utils.find_videos(import_paths, skip_subfolders=skip_subfolders) + } + return upload._UploadCandidates( + [ + m + for m in metadatas + if isinstance(m, types.ImageMetadata) + and m.filename.resolve() in resolved_images + ], + [ + m + for m in metadatas + if isinstance(m, types.VideoMetadata) + and m.filename.resolve() in resolved_videos + ], + [ + p + for p in utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders) + if not p.name.startswith("._") + ], + ) + + +@pytest.fixture +def capture_tree(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + root = tmp_path / "capture" + for name in [ + "a.jpg", + "b.JPG", + "c.mp4", + "d.zip", + "notes.txt", + ".hidden.jpg", + "sub/e.jpg", + "sub/f.mov", + "sub/g.zip", + ".hidden_folder/h.jpg", + ]: + root.joinpath(name).parent.mkdir(parents=True, exist_ok=True) + root.joinpath(name).touch() + (tmp_path / "elsewhere.jpg").touch() + monkeypatch.chdir(tmp_path) + return root + + +def _capture_metadatas(root: Path) -> list[types.Metadata]: + return [ + _image(root / "sub" / "e.jpg"), + _image(root / "b.JPG"), + _video(root / "sub" / "f.mov"), + _image(root / "a.jpg"), + _video(root / "c.mp4"), + # Under the import path, but not found by a scan + _image(root / ".hidden.jpg"), + _image(root / ".hidden_folder" / "h.jpg"), + # Not under the import path + _image(root.parent / "elsewhere.jpg"), + # A video with an image name is not uploaded as a video + _video(root / "a.jpg"), + ] + + +@pytest.mark.parametrize("skip_subfolders", [False, True]) +def test_find_upload_candidates(capture_tree: Path, skip_subfolders: bool): + metadatas = _capture_metadatas(capture_tree) + + candidates = upload._find_upload_candidates( + metadatas, [capture_tree], skip_subfolders=skip_subfolders + ) + + if skip_subfolders: + assert candidates == upload._UploadCandidates( + [metadatas[1], metadatas[3]], + [metadatas[4]], + [capture_tree / "d.zip"], + ) + else: + # In the order of the metadatas + assert candidates == upload._UploadCandidates( + [metadatas[0], metadatas[1], metadatas[3]], + [metadatas[2], metadatas[4]], + [capture_tree / "d.zip", capture_tree / "sub" / "g.zip"], + ) + assert candidates == _find_upload_candidates_by_file_type( + metadatas, [capture_tree], skip_subfolders=skip_subfolders + ) + + +def test_find_upload_candidates_once_from_overlapping_paths(capture_tree: Path): + metadatas = _capture_metadatas(capture_tree) + import_paths = [ + # Relative to the working directory + Path("capture"), + capture_tree / "sub" / ".." / "a.jpg", + capture_tree / "sub", + capture_tree / "sub" / "g.zip", + # Hidden, but named + capture_tree / ".hidden.jpg", + ] + if not sys.platform.startswith("win"): + (capture_tree.parent / "linked_capture").symlink_to(capture_tree) + (capture_tree.parent / "linked_image.jpg").symlink_to( + capture_tree.parent / "elsewhere.jpg" + ) + import_paths += [ + capture_tree.parent / "linked_capture", + capture_tree.parent / "linked_image.jpg", + ] + + candidates = upload._find_upload_candidates( + metadatas, import_paths, skip_subfolders=False + ) + + image_metadatas = [metadatas[0], metadatas[1], metadatas[3], metadatas[5]] + if not sys.platform.startswith("win"): + # Found through the link to it + image_metadatas.append(metadatas[7]) + assert candidates == upload._UploadCandidates( + image_metadatas, + [metadatas[2], metadatas[4]], + [Path("capture") / "d.zip", Path("capture") / "sub" / "g.zip"], + ) + assert candidates == _find_upload_candidates_by_file_type( + metadatas, import_paths, skip_subfolders=False + ) + + +@pytest.mark.skipif( + sys.platform.startswith("win") or (hasattr(os, "geteuid") and os.geteuid() == 0), + reason="Folder permissions do not apply on Windows or to root", +) +def test_find_upload_candidates_in_a_folder_that_cannot_be_searched( + capture_tree: Path, +): + metadatas = _capture_metadatas(capture_tree) + folder = capture_tree / "sub" + # Listed, but its files cannot be looked at + folder.chmod(0o600) + try: + candidates = upload._find_upload_candidates( + metadatas, [capture_tree], skip_subfolders=False + ) + expected = _find_upload_candidates_by_file_type( + metadatas, [capture_tree], skip_subfolders=False + ) + finally: + folder.chmod(0o700) + + assert candidates == expected + assert [m.filename for m in candidates.image_metadatas] == [ + folder / "e.jpg", + capture_tree / "b.JPG", + capture_tree / "a.jpg", + ] + + +def test_find_upload_candidates_skips_apple_double_zip(capture_tree: Path): + apple_double = capture_tree / "._d.zip" + apple_double.touch() + + candidates = upload._find_upload_candidates([], [apple_double], False) + + assert candidates == upload._UploadCandidates([], [], []) + + +def _write_desc_file(path: Path, descs: list) -> str: + path.write_text(json.dumps(descs)) + return str(path) + + +def test_load_descs_from_desc_path(tmp_path: Path): + image = _image(tmp_path / "a.jpg") + video = _video(tmp_path / "c.mp4") + desc_path = _write_desc_file( + tmp_path / "descs.json", + [ + upload.DescriptionJSONSerializer.as_desc(image), + {"error": {"type": "Error", "message": "x"}, "filename": "b.jpg"}, + upload.DescriptionJSONSerializer.as_desc(video), + ], + ) + + metadatas = upload._load_valid_metadatas_from_desc_path([tmp_path], desc_path) + + assert [type(m) for m in metadatas] == [types.ImageMetadata, types.VideoMetadata] + assert [m.filename for m in metadatas] == [ + image.filename.resolve(), + video.filename.resolve(), + ] + + +def test_load_descs_from_stdin(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + image = _image(tmp_path / "a.jpg") + data = json.dumps([upload.DescriptionJSONSerializer.as_desc(image)]).encode() + monkeypatch.setattr(sys, "stdin", io.TextIOWrapper(io.BytesIO(data))) + + metadatas = upload._load_valid_metadatas_from_desc_path([tmp_path], "-") + + assert [m.filename for m in metadatas] == [image.filename.resolve()] + + +def test_load_descs_invalid(tmp_path: Path): + bad_json = tmp_path / "bad.json" + bad_json.write_text("[{") + with pytest.raises(exceptions.MapillaryInvalidDescriptionFile): + upload._load_valid_metadatas_from_desc_path([tmp_path], str(bad_json)) + + desc_path = _write_desc_file( + tmp_path / "descs.json", [{"filename": "a.jpg", "filetype": "image"}] + ) + with pytest.raises(exceptions.MapillaryMetadataValidationError): + upload._load_valid_metadatas_from_desc_path([tmp_path], desc_path) + + with pytest.raises(exceptions.MapillaryFileNotFoundError): + upload._load_valid_metadatas_from_desc_path( + [tmp_path], str(tmp_path / "missing.json") + ) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 4e3a4390..35914bec 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -356,3 +356,54 @@ def test_md5sum_fp_progress(): assert md5.hexdigest() == hashlib.md5(data).hexdigest() assert sum(progress) == len(data) + + +def test_path_resolver_resolves_like_path_resolve(tmp_path: Path, monkeypatch): + folder = tmp_path / "folder" + folder.joinpath("sub").mkdir(parents=True) + image = folder / "image.jpg" + image.touch() + paths = [ + image, + folder / "sub" / ".." / "image.jpg", + folder / "missing.jpg", + folder / "sub" / "..", + Path("image.jpg"), + Path("sub") / "image.jpg", + Path("."), + Path(".."), + ] + if not sys.platform.startswith("win"): + # A link to a folder, a link to a file, and a link that leads nowhere + (tmp_path / "linked_folder").symlink_to(folder) + (folder / "linked_image.jpg").symlink_to(image) + (folder / "broken_link.jpg").symlink_to(tmp_path / "nowhere.jpg") + paths += [ + tmp_path / "linked_folder" / "image.jpg", + tmp_path / "linked_folder" / "linked_image.jpg", + folder / "linked_image.jpg", + folder / "broken_link.jpg", + ] + monkeypatch.chdir(folder) + + resolver = utils.PathResolver() + # Twice, to resolve with the folders resolved before too + for _ in range(2): + assert [resolver.resolve(p) for p in paths] == [p.resolve() for p in paths] + + +@pytest.mark.skipif( + sys.platform.startswith("win") or (hasattr(os, "geteuid") and os.geteuid() == 0), + reason="Folder permissions do not apply on Windows or to root", +) +def test_path_resolver_in_a_folder_that_cannot_be_searched(tmp_path: Path): + folder = tmp_path / "unsearchable" + folder.mkdir() + image = folder / "image.jpg" + image.touch() + # Listed, but its files cannot be looked at + folder.chmod(0o600) + try: + assert utils.PathResolver().resolve(image) == image.resolve() + finally: + folder.chmod(0o700)