From 862e5047d7ad68ae57da0571fc6f037e5b515cde Mon Sep 17 00:00:00 2001 From: Caglar Pir Date: Mon, 16 Nov 2026 15:00:44 +0100 Subject: [PATCH] Save organization names with authenticate Saved organization names could be listed and deleted with authenticate, but only saved by upload. authenticate now saves them too: mapillary_tools authenticate --user_name PROFILE \ --organization_key ID --organization_name NAME An existing profile is not logged in again: its token is checked and the organization is looked up, as upload does, and the ID is saved as the server returns it. If the token is rejected, the error says how to log in again. With --jwt, --user_email or --user_password, or for a new profile, authenticate logs in first and then saves the name. If the organization lookup fails after the login, the login is kept, so that a mistyped ID needs no new login. upload --organization_key ID --organization_name NAME still saves the name. --- README.md | 18 +- docs/command_reference.md | 14 +- mapillary_tools/authenticate.py | 114 +++++++++- mapillary_tools/commands/authenticate.py | 8 +- tests/integration/test_organization_name.py | 30 +++ tests/unit/test_authenticate.py | 231 ++++++++++++++++++++ 6 files changed, 394 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index f116345c..7b3d73c2 100644 --- a/README.md +++ b/README.md @@ -263,9 +263,8 @@ To upload to an organization by a name instead of its ID, save a name for it onc The name is saved in the user profile in the config file: ```sh -# Upload to the organization, and save its ID under the name "work" in the profile "my_username" -mapillary_tools upload MY_CAPTURE_DIR \ - --desc_path /tmp/mapillary_image_description.json \ +# Save the ID of the organization under the name "work" in the profile "my_username" +mapillary_tools authenticate \ --user_name "my_username" \ --organization_key "my_organization_id" \ --organization_name "work" @@ -277,6 +276,7 @@ mapillary_tools upload MY_CAPTURE_DIR \ --organization_name "work" ``` +An upload can save the name too: give it `--organization_key` and `--organization_name` together. `--organization_name` works with `upload`, `process_and_upload` and `video_process_and_upload`. A name is 2 to 32 letters, numbers, hyphens and underscores, starting with a letter, and it is case-sensitive. Saving a name that is already saved with another ID replaces the ID. @@ -550,8 +550,16 @@ Authenticate for user `my_username`. If the user is already authenticated, it wi mapillary_tools authenticate --user_name "my_username" ``` -The command lists the existing profiles first, with the organization names saved in each of them -(see [Upload](#upload) to save one). Updating the credentials keeps the saved names. +The command lists the existing profiles first, with the organization names saved in each of them. +Updating the credentials keeps the saved names. + +Save the ID of an organization under the name `work` in the profile `my_username`, to upload to it with +`--organization_name "work"` (see [Upload](#upload)). The profile is not logged in again, unless it is new or +`--jwt`, `--user_email` or `--user_password` is given: + +```sh +mapillary_tools authenticate --user_name "my_username" --organization_key "my_organization_id" --organization_name "work" +``` Delete the organization name `work` saved in the profile `my_username`. The profile and its other saved names are kept: diff --git a/docs/command_reference.md b/docs/command_reference.md index 3b139dc5..3a5a9702 100644 --- a/docs/command_reference.md +++ b/docs/command_reference.md @@ -42,7 +42,7 @@ For the full list of options of the version you have installed, run `mapillary_t | `sample_video` | Extracts frames from videos as geotagged images. Needs FFmpeg. | | `video_process` | `sample_video`, then `process` the frames. | | `video_process_and_upload` | `video_process`, then `upload` the frames. | -| `authenticate` | Adds, updates or deletes user profiles, and deletes saved organization names. | +| `authenticate` | Adds, updates or deletes user profiles, and saves or deletes organization names. | | `check_upload_history` | Tells whether processed files were already uploaded from this computer. | | `zip` | Writes processed image sequences into zip files, to upload later. | @@ -357,7 +357,7 @@ Uploads the images (as sequences), videos and zip files under the import paths, mapillary_tools process MY_CAPTURE_DIR mapillary_tools upload MY_CAPTURE_DIR --user_name my_profile -# Save the organization ID under the name "work" once, then use the name +# Save the organization ID under the name "work" once (or with authenticate), then use the name mapillary_tools upload MY_CAPTURE_DIR --organization_key 1234567890 --organization_name work mapillary_tools upload MY_CAPTURE_DIR --organization_name work ``` @@ -467,7 +467,7 @@ mapillary_tools authenticate [--user_name PROFILE] [OPTIONS] Logs in to Mapillary and saves the credentials as a profile in the config file (a file in `~/.config/mapillary/configs/`, or the file that `MAPILLARY_CONFIG_PATH` names). It lists the existing profiles first, with the -organization names saved in each. +organization names saved in each. It also saves and deletes the organization names of a profile. | Option | Description | | ---------------------------- | ----------------------------------------------------------------------------------- | @@ -475,18 +475,22 @@ organization names saved in each. | `--user_email`, `--user_password` | Log in without prompts. | | `--jwt TOKEN` | Save a user access token instead of logging in. | | `--delete` | Delete the profile, or with `--organization_name NAME`, only that saved name. | +| `--organization_key ID` | With `--organization_name NAME`, save the organization ID under NAME in the profile. | +| `--organization_name NAME` | The organization name to save, or with `--delete`, to delete. | ```sh mapillary_tools authenticate --user_name my_profile mapillary_tools authenticate --user_name my_profile --delete +mapillary_tools authenticate --user_name my_profile --organization_key 1234567890 --organization_name work mapillary_tools authenticate --user_name my_profile --organization_name work --delete ``` Gotchas: - Authenticating an existing profile again replaces its credentials, and keeps its saved organization names. -- Organization names are saved by `upload`, not by `authenticate` - (`--organization_key ID --organization_name NAME`). +- Saving an organization name does not log an existing profile in again, unless `--jwt`, `--user_email` or + `--user_password` is given. It checks the profile's token and looks up the organization ID on Mapillary, so it + needs a connection. A new profile logs in first. - Without a terminal, or with `MAPILLARY_TOOLS_PROMPT_DISABLED=YES`, give everything as options: missing values are an error instead of a prompt. diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index 10d7e285..96d0791f 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -5,6 +5,7 @@ from __future__ import annotations +import contextlib import getpass import logging import re @@ -29,17 +30,41 @@ def authenticate( user_password: str | None = None, jwt: str | None = None, delete: bool = False, + organization_key: str | None = None, organization_name: str | None = None, ): """ - Prompt for authentication information and save it to the config file + Prompt for authentication information and save it to the config file. + + With organization_key and organization_name, save the organization ID under + the name in the profile, logging in only if the profile does not exist yet + or login options are given """ - if organization_name is not None and not delete: - raise exceptions.MapillaryBadParameterError( - "authenticate takes --organization_name only with --delete. To save an " - "organization name, upload with --organization_key ID --organization_name NAME" - ) + if organization_name is not None: + organization_name = organization_name.strip() + if organization_key is not None: + organization_key = organization_key.strip() + + if delete: + if organization_key is not None: + raise exceptions.MapillaryBadParameterError( + "authenticate --delete takes --organization_name NAME, not --organization_key" + ) + elif organization_key is None: + if organization_name is not None: + raise exceptions.MapillaryBadParameterError( + "authenticate takes --organization_name with --organization_key ID " + "to save the name, or with --delete to delete it" + ) + else: + if organization_name is None: + raise exceptions.MapillaryBadParameterError( + "To save an organization, give the name to save its ID under: " + "--organization_key ID --organization_name NAME" + ) + _validate_organization_name(organization_name) + _validate_organization_id(organization_key) # We still have to accept --user_name for the back compatibility profile_name = user_name @@ -65,10 +90,33 @@ def authenticate( assert profile_name is not None, "profile_name should be set" if delete and organization_name is not None: - _delete_organization(profile_name, organization_name.strip(), all_user_items) + _delete_organization(profile_name, organization_name, all_user_items) elif delete: config.remove_config(profile_name) LOG.info('Profile "%s" deleted', profile_name) + elif ( + organization_key is not None + and profile_name in all_user_items + and not jwt + and user_email is None + and user_password is None + ): + # Save the organization in the profile without logging in again, with + # the token saved in it checked + assert organization_name is not None + with _connection_errors_raised_as_mapillary_errors(): + try: + profile_items = _verify_user_auth( + _validate_profile(all_user_items[profile_name]) + ) + except exceptions.MapillaryUploadUnauthorizedError as ex: + raise exceptions.MapillaryUploadUnauthorizedError( + f'{str(ex).rstrip(".")}. To log in to profile "{profile_name}" ' + "again and save the organization, add --jwt TOKEN or --user_email EMAIL" + ) from ex + _fetch_and_save_organization( + profile_name, profile_items, organization_key, organization_name + ) else: if profile_name in all_user_items: LOG.warning( @@ -100,6 +148,47 @@ def authenticate( 'Profile "%s" created: %s', profile_name, http._sanitize(user_items) ) + if organization_key is not None: + assert organization_name is not None + _fetch_and_save_organization( + profile_name, user_items, organization_key, organization_name + ) + + +def _fetch_and_save_organization( + profile_name: str, + user_items: config.UserItem, + organization_key: str, + organization_name: str, +) -> None: + with _connection_errors_raised_as_mapillary_errors(): + data = _fetch_organization(user_items, organization_key) + saved_user_items = config.load_user(profile_name) + _save_organization( + profile_name, + organization_name, + # The ID as the server returns it + str(data.get("id") or organization_key), + data.get("name"), + config.list_organizations(saved_user_items) if saved_user_items else {}, + ) + + +@contextlib.contextmanager +def _connection_errors_raised_as_mapillary_errors() -> T.Iterator[None]: + try: + yield + except requests.Timeout as ex: + raise exceptions.MapillaryUploadTimeoutError(str(ex)) from ex + except requests.ConnectionError as ex: + raise exceptions.MapillaryUploadConnectionError(str(ex)) from ex + + +def _fetch_organization(user_items: config.UserItem, organization_id: str) -> T.Any: + with api_v4.create_user_session(user_items["user_upload_token"]) as session: + resp = api_v4.fetch_organization(session, organization_id) + return api_v4.jsonify_response(resp) + def fetch_user_items( user_name: str | None = None, @@ -189,9 +278,7 @@ def fetch_user_items( ) if organization_id is not None: - with api_v4.create_user_session(user_items["user_upload_token"]) as session: - resp = api_v4.fetch_organization(session, organization_id) - data = api_v4.jsonify_response(resp) + data = _fetch_organization(user_items, organization_id) LOG.info( f"Uploading to organization: {data.get('name')} (ID: {data.get('id')})" ) @@ -362,6 +449,13 @@ def _validate_organization_name(organization_name: str): _validate_name(organization_name, "organization name") +def _validate_organization_id(organization_key: str): + if not _ORGANIZATION_ID_PATTERN.fullmatch(organization_key): + raise exceptions.MapillaryBadParameterError( + f'--organization_key takes an organization ID, a number, not "{organization_key}"' + ) + + def _list_all_profiles(profiles: dict[str, config.UserItem]) -> None: _echo("Existing Mapillary profiles:") diff --git a/mapillary_tools/commands/authenticate.py b/mapillary_tools/commands/authenticate.py index b62a5103..7723c604 100644 --- a/mapillary_tools/commands/authenticate.py +++ b/mapillary_tools/commands/authenticate.py @@ -39,9 +39,15 @@ def add_basic_arguments(self, parser: argparse.ArgumentParser): required=False, action="store_true", ) + parser.add_argument( + "--organization_key", + help="The Mapillary organization ID to save in the user profile under --organization_name NAME. An existing profile is not logged in again, unless login options are given.", + default=None, + required=False, + ) parser.add_argument( "--organization_name", - help="The saved organization name to delete from the user profile, with --delete", + help="With --organization_key ID, save the ID under this name in the user profile, to upload with --organization_name NAME. With --delete, delete the saved name. Names are 2 to 32 letters, numbers, hyphens and underscores, starting with a letter.", default=None, required=False, ) diff --git a/tests/integration/test_organization_name.py b/tests/integration/test_organization_name.py index 5a5dac62..2e0a43f6 100644 --- a/tests/integration/test_organization_name.py +++ b/tests/integration/test_organization_name.py @@ -127,6 +127,36 @@ def test_save_then_use_organization_name( assert graph_api == [ORGANIZATION_ID, ORGANIZATION_ID] +def test_authenticate_saves_organization_name( + setup_config: py.path.local, + setup_data: py.path.local, + setup_upload: py.path.local, + graph_api: list[str], +): + # Saved without logging in again: the token stays + run_command( + [ + *["--user_name", USERNAME], + *["--organization_key", ORGANIZATION_ID], + *["--organization_name", "work"], + ], + command="authenticate", + ) + assert config.load_user(USERNAME, config_path=str(setup_config)) == { + "user_upload_token": "test_user_token", + "MAPOrganization.work": ORGANIZATION_ID, + } + assert graph_api == [ORGANIZATION_ID] + + run_process_and_upload_for_descs( + ["--file_types=image", "--organization_name", "work"] + + [str(setup_data.join("images"))] + ) + keys = _history_organization_keys() + assert keys and set(keys) == {ORGANIZATION_ID} + assert graph_api == [ORGANIZATION_ID, ORGANIZATION_ID] + + def test_video_process_and_upload_by_organization_name( setup_config: py.path.local, setup_data: py.path.local, diff --git a/tests/unit/test_authenticate.py b/tests/unit/test_authenticate.py index 41a65777..76315db9 100644 --- a/tests/unit/test_authenticate.py +++ b/tests/unit/test_authenticate.py @@ -511,6 +511,237 @@ def test_authenticate_takes_organization_name_only_to_delete(config_path: Path): assert config_path.read_text() == before +def _forbid_login(monkeypatch: pytest.MonkeyPatch) -> None: + def _no_login(*args, **kwargs): + raise AssertionError("should not log in") + + monkeypatch.setattr(authenticate, "_prompt_login", _no_login) + + +def test_authenticate_saves_organization_without_logging_in( + config_path: Path, + fetched: list[str], + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +): + _add_profile("me", MAPSettingsUserKey="1001") + _forbid_login(monkeypatch) + verified: list[str] = [] + + def _verify_user_auth(user_items: config.UserItem) -> config.UserItem: + verified.append(user_items["user_upload_token"]) + return {**user_items, "MAPSettingsUsername": "me_on_mapillary"} + + monkeypatch.setattr(authenticate, "_verify_user_auth", _verify_user_auth) + caplog.set_level(logging.INFO) + + authenticate.authenticate( + user_name="me", organization_key=" 01234567890 ", organization_name=" work " + ) + + # The token is checked, and the ID is saved as the server returns it + assert verified == ["me_token"] + assert fetched == ["01234567890"] + assert config.load_user("me") == { + "user_upload_token": "me_token", + "MAPSettingsUserKey": "1001", + "MAPOrganization.work": "1234567890", + } + assert ( + 'Saved organization "work": Acme Mapping (ID: 1234567890) in profile "me"' + in caplog.text + ) + assert "will be overridden" not in caplog.text + + # Then uploads go to it by the name + user_items = authenticate.fetch_user_items(organization_name="work") + assert user_items["MAPOrganizationKey"] == "1234567890" + + # Saving the name with another ID replaces the ID + caplog.clear() + authenticate.authenticate( + user_name="me", organization_key="2222222222", organization_name="work" + ) + assert _saved("me") == {"work": "2222222222"} + assert ( + 'The organization "work" (ID: 1234567890) already exists in profile "me" and will be overridden' + in caplog.text + ) + + +def test_authenticate_saves_organization_in_the_profile_chosen( + config_path: Path, fetched: list[str], monkeypatch: pytest.MonkeyPatch +): + _add_profile("me") + _add_profile("other") + _forbid_login(monkeypatch) + _enable_prompt(monkeypatch, ["2"]) + + authenticate.authenticate(organization_key="1234567890", organization_name="work") + + assert _saved("me") == {} + assert _saved("other") == {"work": "1234567890"} + + +def test_authenticate_logs_in_and_saves_organization( + config_path: Path, fetched: list[str], monkeypatch: pytest.MonkeyPatch +): + _add_profile("me", **{"MAPOrganization.home": "2222222222"}) + verified: list[str] = [] + + def _verify_user_auth(user_items: config.UserItem) -> config.UserItem: + verified.append(user_items["user_upload_token"]) + return user_items + + monkeypatch.setattr(authenticate, "_verify_user_auth", _verify_user_auth) + + # With a login option, an existing profile logs in again first + authenticate.authenticate( + user_name="me", + jwt="new_token", + organization_key="1234567890", + organization_name="work", + ) + assert config.load_user("me") == { + "user_upload_token": "new_token", + "MAPOrganization.home": "2222222222", + "MAPOrganization.work": "1234567890", + } + + # A new profile logs in first + authenticate.authenticate( + user_name="new", + jwt="other_token", + organization_key="1234567890", + organization_name="work", + ) + assert config.load_user("new") == { + "user_upload_token": "other_token", + "MAPOrganization.work": "1234567890", + } + assert fetched == ["1234567890", "1234567890"] + # Each token is checked once, by the login + assert verified == ["new_token", "other_token"] + assert "MAPOrganizationKey" not in config_path.read_text() + + +def test_authenticate_keeps_the_login_if_the_organization_is_rejected( + config_path: Path, fetched: list[str], monkeypatch: pytest.MonkeyPatch +): + _add_profile("me", **{"MAPOrganization.home": "2222222222"}) + + # The login works, the organization ID is wrong + with pytest.raises(requests.HTTPError): + authenticate.authenticate( + user_name="me", + jwt="new_token", + organization_key="999", + organization_name="work", + ) + with pytest.raises(requests.HTTPError): + authenticate.authenticate( + user_name="new", + jwt="other_token", + organization_key="999", + organization_name="work", + ) + + # The logins are kept, so that a mistyped ID needs no login again + assert config.load_user("me") == { + "user_upload_token": "new_token", + "MAPOrganization.home": "2222222222", + } + assert config.load_user("new") == {"user_upload_token": "other_token"} + + # Saving with the right ID then does not log in + _forbid_login(monkeypatch) + authenticate.authenticate( + user_name="new", organization_key="1234567890", organization_name="work" + ) + assert config.load_user("new") == { + "user_upload_token": "other_token", + "MAPOrganization.work": "1234567890", + } + assert fetched == ["999", "999", "1234567890"] + + +def test_authenticate_new_profile_without_login_options_needs_a_login( + config_path: Path, fetched: list[str] +): + _add_profile("me") + before = config_path.read_text() + + # Without prompts, the login of the new profile fails + with pytest.raises(exceptions.MapillaryBadParameterError, match="user_email"): + authenticate.authenticate( + user_name="new", organization_key="1234567890", organization_name="work" + ) + + assert fetched == [] + assert config_path.read_text() == before + + +def test_authenticate_saves_no_organization_the_server_rejects( + config_path: Path, fetched: list[str], monkeypatch: pytest.MonkeyPatch +): + _add_profile("me", **{"MAPOrganization.work": "1234567890"}) + before = config_path.read_text() + + with pytest.raises(requests.HTTPError): + authenticate.authenticate( + user_name="me", organization_key="999", organization_name="work" + ) + assert fetched == ["999"] + assert config_path.read_text() == before + + def _unauthorized(user_items: config.UserItem) -> config.UserItem: + raise exceptions.MapillaryUploadUnauthorizedError("Invalid token.") + + monkeypatch.setattr(authenticate, "_verify_user_auth", _unauthorized) + # With how to log in again, as the profile was not logged in + with pytest.raises( + exceptions.MapillaryUploadUnauthorizedError, + match=r'^Invalid token\. To log in to profile "me" again and save the organization, add --jwt TOKEN or --user_email EMAIL$', + ): + authenticate.authenticate( + user_name="me", organization_key="2222222222", organization_name="work" + ) + assert fetched == ["999"] + assert config_path.read_text() == before + + +@pytest.mark.parametrize( + "kwargs, message", + [ + ({"organization_key": "1234567890"}, "--organization_name NAME"), + ({"organization_name": "work"}, "--delete"), + ({"organization_key": "work", "organization_name": "work"}, "organization ID"), + ({"organization_key": "1234567890", "organization_name": "1work"}, "Invalid"), + ( + {"organization_key": "1234567890", "delete": True}, + "not --organization_key", + ), + ], +) +def test_authenticate_organization_options_fail_before_anything( + config_path: Path, + fetched: list[str], + monkeypatch: pytest.MonkeyPatch, + kwargs: dict, + message: str, +): + _add_profile("me", **{"MAPOrganization.work": "1234567890"}) + before = config_path.read_text() + _forbid_login(monkeypatch) + _enable_prompt(monkeypatch, []) + + with pytest.raises(exceptions.MapillaryBadParameterError, match=message): + authenticate.authenticate(**kwargs) + + assert fetched == [] + assert config_path.read_text() == before + + @pytest.mark.parametrize( "command", ["upload", "process_and_upload", "video_process_and_upload"] )