diff --git a/README.md b/README.md index f116345c..7b3d73c2 100644 --- a/README.md +++ b/README.md @@ -263,9 +263,8 @@ To upload to an organization by a name instead of its ID, save a name for it onc The name is saved in the user profile in the config file: ```sh -# Upload to the organization, and save its ID under the name "work" in the profile "my_username" -mapillary_tools upload MY_CAPTURE_DIR \ - --desc_path /tmp/mapillary_image_description.json \ +# Save the ID of the organization under the name "work" in the profile "my_username" +mapillary_tools authenticate \ --user_name "my_username" \ --organization_key "my_organization_id" \ --organization_name "work" @@ -277,6 +276,7 @@ mapillary_tools upload MY_CAPTURE_DIR \ --organization_name "work" ``` +An upload can save the name too: give it `--organization_key` and `--organization_name` together. `--organization_name` works with `upload`, `process_and_upload` and `video_process_and_upload`. A name is 2 to 32 letters, numbers, hyphens and underscores, starting with a letter, and it is case-sensitive. Saving a name that is already saved with another ID replaces the ID. @@ -550,8 +550,16 @@ Authenticate for user `my_username`. If the user is already authenticated, it wi mapillary_tools authenticate --user_name "my_username" ``` -The command lists the existing profiles first, with the organization names saved in each of them -(see [Upload](#upload) to save one). Updating the credentials keeps the saved names. +The command lists the existing profiles first, with the organization names saved in each of them. +Updating the credentials keeps the saved names. + +Save the ID of an organization under the name `work` in the profile `my_username`, to upload to it with +`--organization_name "work"` (see [Upload](#upload)). The profile is not logged in again, unless it is new or +`--jwt`, `--user_email` or `--user_password` is given: + +```sh +mapillary_tools authenticate --user_name "my_username" --organization_key "my_organization_id" --organization_name "work" +``` Delete the organization name `work` saved in the profile `my_username`. The profile and its other saved names are kept: diff --git a/docs/command_reference.md b/docs/command_reference.md index 3b139dc5..3a5a9702 100644 --- a/docs/command_reference.md +++ b/docs/command_reference.md @@ -42,7 +42,7 @@ For the full list of options of the version you have installed, run `mapillary_t | `sample_video` | Extracts frames from videos as geotagged images. Needs FFmpeg. | | `video_process` | `sample_video`, then `process` the frames. | | `video_process_and_upload` | `video_process`, then `upload` the frames. | -| `authenticate` | Adds, updates or deletes user profiles, and deletes saved organization names. | +| `authenticate` | Adds, updates or deletes user profiles, and saves or deletes organization names. | | `check_upload_history` | Tells whether processed files were already uploaded from this computer. | | `zip` | Writes processed image sequences into zip files, to upload later. | @@ -357,7 +357,7 @@ Uploads the images (as sequences), videos and zip files under the import paths, mapillary_tools process MY_CAPTURE_DIR mapillary_tools upload MY_CAPTURE_DIR --user_name my_profile -# Save the organization ID under the name "work" once, then use the name +# Save the organization ID under the name "work" once (or with authenticate), then use the name mapillary_tools upload MY_CAPTURE_DIR --organization_key 1234567890 --organization_name work mapillary_tools upload MY_CAPTURE_DIR --organization_name work ``` @@ -467,7 +467,7 @@ mapillary_tools authenticate [--user_name PROFILE] [OPTIONS] Logs in to Mapillary and saves the credentials as a profile in the config file (a file in `~/.config/mapillary/configs/`, or the file that `MAPILLARY_CONFIG_PATH` names). It lists the existing profiles first, with the -organization names saved in each. +organization names saved in each. It also saves and deletes the organization names of a profile. | Option | Description | | ---------------------------- | ----------------------------------------------------------------------------------- | @@ -475,18 +475,22 @@ organization names saved in each. | `--user_email`, `--user_password` | Log in without prompts. | | `--jwt TOKEN` | Save a user access token instead of logging in. | | `--delete` | Delete the profile, or with `--organization_name NAME`, only that saved name. | +| `--organization_key ID` | With `--organization_name NAME`, save the organization ID under NAME in the profile. | +| `--organization_name NAME` | The organization name to save, or with `--delete`, to delete. | ```sh mapillary_tools authenticate --user_name my_profile mapillary_tools authenticate --user_name my_profile --delete +mapillary_tools authenticate --user_name my_profile --organization_key 1234567890 --organization_name work mapillary_tools authenticate --user_name my_profile --organization_name work --delete ``` Gotchas: - Authenticating an existing profile again replaces its credentials, and keeps its saved organization names. -- Organization names are saved by `upload`, not by `authenticate` - (`--organization_key ID --organization_name NAME`). +- Saving an organization name does not log an existing profile in again, unless `--jwt`, `--user_email` or + `--user_password` is given. It checks the profile's token and looks up the organization ID on Mapillary, so it + needs a connection. A new profile logs in first. - Without a terminal, or with `MAPILLARY_TOOLS_PROMPT_DISABLED=YES`, give everything as options: missing values are an error instead of a prompt. diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index 10d7e285..96d0791f 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -5,6 +5,7 @@ from __future__ import annotations +import contextlib import getpass import logging import re @@ -29,17 +30,41 @@ def authenticate( user_password: str | None = None, jwt: str | None = None, delete: bool = False, + organization_key: str | None = None, organization_name: str | None = None, ): """ - Prompt for authentication information and save it to the config file + Prompt for authentication information and save it to the config file. + + With organization_key and organization_name, save the organization ID under + the name in the profile, logging in only if the profile does not exist yet + or login options are given """ - if organization_name is not None and not delete: - raise exceptions.MapillaryBadParameterError( - "authenticate takes --organization_name only with --delete. To save an " - "organization name, upload with --organization_key ID --organization_name NAME" - ) + if organization_name is not None: + organization_name = organization_name.strip() + if organization_key is not None: + organization_key = organization_key.strip() + + if delete: + if organization_key is not None: + raise exceptions.MapillaryBadParameterError( + "authenticate --delete takes --organization_name NAME, not --organization_key" + ) + elif organization_key is None: + if organization_name is not None: + raise exceptions.MapillaryBadParameterError( + "authenticate takes --organization_name with --organization_key ID " + "to save the name, or with --delete to delete it" + ) + else: + if organization_name is None: + raise exceptions.MapillaryBadParameterError( + "To save an organization, give the name to save its ID under: " + "--organization_key ID --organization_name NAME" + ) + _validate_organization_name(organization_name) + _validate_organization_id(organization_key) # We still have to accept --user_name for the back compatibility profile_name = user_name @@ -65,10 +90,33 @@ def authenticate( assert profile_name is not None, "profile_name should be set" if delete and organization_name is not None: - _delete_organization(profile_name, organization_name.strip(), all_user_items) + _delete_organization(profile_name, organization_name, all_user_items) elif delete: config.remove_config(profile_name) LOG.info('Profile "%s" deleted', profile_name) + elif ( + organization_key is not None + and profile_name in all_user_items + and not jwt + and user_email is None + and user_password is None + ): + # Save the organization in the profile without logging in again, with + # the token saved in it checked + assert organization_name is not None + with _connection_errors_raised_as_mapillary_errors(): + try: + profile_items = _verify_user_auth( + _validate_profile(all_user_items[profile_name]) + ) + except exceptions.MapillaryUploadUnauthorizedError as ex: + raise exceptions.MapillaryUploadUnauthorizedError( + f'{str(ex).rstrip(".")}. To log in to profile "{profile_name}" ' + "again and save the organization, add --jwt TOKEN or --user_email EMAIL" + ) from ex + _fetch_and_save_organization( + profile_name, profile_items, organization_key, organization_name + ) else: if profile_name in all_user_items: LOG.warning( @@ -100,6 +148,47 @@ def authenticate( 'Profile "%s" created: %s', profile_name, http._sanitize(user_items) ) + if organization_key is not None: + assert organization_name is not None + _fetch_and_save_organization( + profile_name, user_items, organization_key, organization_name + ) + + +def _fetch_and_save_organization( + profile_name: str, + user_items: config.UserItem, + organization_key: str, + organization_name: str, +) -> None: + with _connection_errors_raised_as_mapillary_errors(): + data = _fetch_organization(user_items, organization_key) + saved_user_items = config.load_user(profile_name) + _save_organization( + profile_name, + organization_name, + # The ID as the server returns it + str(data.get("id") or organization_key), + data.get("name"), + config.list_organizations(saved_user_items) if saved_user_items else {}, + ) + + +@contextlib.contextmanager +def _connection_errors_raised_as_mapillary_errors() -> T.Iterator[None]: + try: + yield + except requests.Timeout as ex: + raise exceptions.MapillaryUploadTimeoutError(str(ex)) from ex + except requests.ConnectionError as ex: + raise exceptions.MapillaryUploadConnectionError(str(ex)) from ex + + +def _fetch_organization(user_items: config.UserItem, organization_id: str) -> T.Any: + with api_v4.create_user_session(user_items["user_upload_token"]) as session: + resp = api_v4.fetch_organization(session, organization_id) + return api_v4.jsonify_response(resp) + def fetch_user_items( user_name: str | None = None, @@ -189,9 +278,7 @@ def fetch_user_items( ) if organization_id is not None: - with api_v4.create_user_session(user_items["user_upload_token"]) as session: - resp = api_v4.fetch_organization(session, organization_id) - data = api_v4.jsonify_response(resp) + data = _fetch_organization(user_items, organization_id) LOG.info( f"Uploading to organization: {data.get('name')} (ID: {data.get('id')})" ) @@ -362,6 +449,13 @@ def _validate_organization_name(organization_name: str): _validate_name(organization_name, "organization name") +def _validate_organization_id(organization_key: str): + if not _ORGANIZATION_ID_PATTERN.fullmatch(organization_key): + raise exceptions.MapillaryBadParameterError( + f'--organization_key takes an organization ID, a number, not "{organization_key}"' + ) + + def _list_all_profiles(profiles: dict[str, config.UserItem]) -> None: _echo("Existing Mapillary profiles:") diff --git a/mapillary_tools/commands/authenticate.py b/mapillary_tools/commands/authenticate.py index b62a5103..7723c604 100644 --- a/mapillary_tools/commands/authenticate.py +++ b/mapillary_tools/commands/authenticate.py @@ -39,9 +39,15 @@ def add_basic_arguments(self, parser: argparse.ArgumentParser): required=False, action="store_true", ) + parser.add_argument( + "--organization_key", + help="The Mapillary organization ID to save in the user profile under --organization_name NAME. An existing profile is not logged in again, unless login options are given.", + default=None, + required=False, + ) parser.add_argument( "--organization_name", - help="The saved organization name to delete from the user profile, with --delete", + help="With --organization_key ID, save the ID under this name in the user profile, to upload with --organization_name NAME. With --delete, delete the saved name. Names are 2 to 32 letters, numbers, hyphens and underscores, starting with a letter.", default=None, required=False, ) diff --git a/tests/integration/test_organization_name.py b/tests/integration/test_organization_name.py index 5a5dac62..2e0a43f6 100644 --- a/tests/integration/test_organization_name.py +++ b/tests/integration/test_organization_name.py @@ -127,6 +127,36 @@ def test_save_then_use_organization_name( assert graph_api == [ORGANIZATION_ID, ORGANIZATION_ID] +def test_authenticate_saves_organization_name( + setup_config: py.path.local, + setup_data: py.path.local, + setup_upload: py.path.local, + graph_api: list[str], +): + # Saved without logging in again: the token stays + run_command( + [ + *["--user_name", USERNAME], + *["--organization_key", ORGANIZATION_ID], + *["--organization_name", "work"], + ], + command="authenticate", + ) + assert config.load_user(USERNAME, config_path=str(setup_config)) == { + "user_upload_token": "test_user_token", + "MAPOrganization.work": ORGANIZATION_ID, + } + assert graph_api == [ORGANIZATION_ID] + + run_process_and_upload_for_descs( + ["--file_types=image", "--organization_name", "work"] + + [str(setup_data.join("images"))] + ) + keys = _history_organization_keys() + assert keys and set(keys) == {ORGANIZATION_ID} + assert graph_api == [ORGANIZATION_ID, ORGANIZATION_ID] + + def test_video_process_and_upload_by_organization_name( setup_config: py.path.local, setup_data: py.path.local, diff --git a/tests/unit/test_authenticate.py b/tests/unit/test_authenticate.py index 41a65777..76315db9 100644 --- a/tests/unit/test_authenticate.py +++ b/tests/unit/test_authenticate.py @@ -511,6 +511,237 @@ def test_authenticate_takes_organization_name_only_to_delete(config_path: Path): assert config_path.read_text() == before +def _forbid_login(monkeypatch: pytest.MonkeyPatch) -> None: + def _no_login(*args, **kwargs): + raise AssertionError("should not log in") + + monkeypatch.setattr(authenticate, "_prompt_login", _no_login) + + +def test_authenticate_saves_organization_without_logging_in( + config_path: Path, + fetched: list[str], + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +): + _add_profile("me", MAPSettingsUserKey="1001") + _forbid_login(monkeypatch) + verified: list[str] = [] + + def _verify_user_auth(user_items: config.UserItem) -> config.UserItem: + verified.append(user_items["user_upload_token"]) + return {**user_items, "MAPSettingsUsername": "me_on_mapillary"} + + monkeypatch.setattr(authenticate, "_verify_user_auth", _verify_user_auth) + caplog.set_level(logging.INFO) + + authenticate.authenticate( + user_name="me", organization_key=" 01234567890 ", organization_name=" work " + ) + + # The token is checked, and the ID is saved as the server returns it + assert verified == ["me_token"] + assert fetched == ["01234567890"] + assert config.load_user("me") == { + "user_upload_token": "me_token", + "MAPSettingsUserKey": "1001", + "MAPOrganization.work": "1234567890", + } + assert ( + 'Saved organization "work": Acme Mapping (ID: 1234567890) in profile "me"' + in caplog.text + ) + assert "will be overridden" not in caplog.text + + # Then uploads go to it by the name + user_items = authenticate.fetch_user_items(organization_name="work") + assert user_items["MAPOrganizationKey"] == "1234567890" + + # Saving the name with another ID replaces the ID + caplog.clear() + authenticate.authenticate( + user_name="me", organization_key="2222222222", organization_name="work" + ) + assert _saved("me") == {"work": "2222222222"} + assert ( + 'The organization "work" (ID: 1234567890) already exists in profile "me" and will be overridden' + in caplog.text + ) + + +def test_authenticate_saves_organization_in_the_profile_chosen( + config_path: Path, fetched: list[str], monkeypatch: pytest.MonkeyPatch +): + _add_profile("me") + _add_profile("other") + _forbid_login(monkeypatch) + _enable_prompt(monkeypatch, ["2"]) + + authenticate.authenticate(organization_key="1234567890", organization_name="work") + + assert _saved("me") == {} + assert _saved("other") == {"work": "1234567890"} + + +def test_authenticate_logs_in_and_saves_organization( + config_path: Path, fetched: list[str], monkeypatch: pytest.MonkeyPatch +): + _add_profile("me", **{"MAPOrganization.home": "2222222222"}) + verified: list[str] = [] + + def _verify_user_auth(user_items: config.UserItem) -> config.UserItem: + verified.append(user_items["user_upload_token"]) + return user_items + + monkeypatch.setattr(authenticate, "_verify_user_auth", _verify_user_auth) + + # With a login option, an existing profile logs in again first + authenticate.authenticate( + user_name="me", + jwt="new_token", + organization_key="1234567890", + organization_name="work", + ) + assert config.load_user("me") == { + "user_upload_token": "new_token", + "MAPOrganization.home": "2222222222", + "MAPOrganization.work": "1234567890", + } + + # A new profile logs in first + authenticate.authenticate( + user_name="new", + jwt="other_token", + organization_key="1234567890", + organization_name="work", + ) + assert config.load_user("new") == { + "user_upload_token": "other_token", + "MAPOrganization.work": "1234567890", + } + assert fetched == ["1234567890", "1234567890"] + # Each token is checked once, by the login + assert verified == ["new_token", "other_token"] + assert "MAPOrganizationKey" not in config_path.read_text() + + +def test_authenticate_keeps_the_login_if_the_organization_is_rejected( + config_path: Path, fetched: list[str], monkeypatch: pytest.MonkeyPatch +): + _add_profile("me", **{"MAPOrganization.home": "2222222222"}) + + # The login works, the organization ID is wrong + with pytest.raises(requests.HTTPError): + authenticate.authenticate( + user_name="me", + jwt="new_token", + organization_key="999", + organization_name="work", + ) + with pytest.raises(requests.HTTPError): + authenticate.authenticate( + user_name="new", + jwt="other_token", + organization_key="999", + organization_name="work", + ) + + # The logins are kept, so that a mistyped ID needs no login again + assert config.load_user("me") == { + "user_upload_token": "new_token", + "MAPOrganization.home": "2222222222", + } + assert config.load_user("new") == {"user_upload_token": "other_token"} + + # Saving with the right ID then does not log in + _forbid_login(monkeypatch) + authenticate.authenticate( + user_name="new", organization_key="1234567890", organization_name="work" + ) + assert config.load_user("new") == { + "user_upload_token": "other_token", + "MAPOrganization.work": "1234567890", + } + assert fetched == ["999", "999", "1234567890"] + + +def test_authenticate_new_profile_without_login_options_needs_a_login( + config_path: Path, fetched: list[str] +): + _add_profile("me") + before = config_path.read_text() + + # Without prompts, the login of the new profile fails + with pytest.raises(exceptions.MapillaryBadParameterError, match="user_email"): + authenticate.authenticate( + user_name="new", organization_key="1234567890", organization_name="work" + ) + + assert fetched == [] + assert config_path.read_text() == before + + +def test_authenticate_saves_no_organization_the_server_rejects( + config_path: Path, fetched: list[str], monkeypatch: pytest.MonkeyPatch +): + _add_profile("me", **{"MAPOrganization.work": "1234567890"}) + before = config_path.read_text() + + with pytest.raises(requests.HTTPError): + authenticate.authenticate( + user_name="me", organization_key="999", organization_name="work" + ) + assert fetched == ["999"] + assert config_path.read_text() == before + + def _unauthorized(user_items: config.UserItem) -> config.UserItem: + raise exceptions.MapillaryUploadUnauthorizedError("Invalid token.") + + monkeypatch.setattr(authenticate, "_verify_user_auth", _unauthorized) + # With how to log in again, as the profile was not logged in + with pytest.raises( + exceptions.MapillaryUploadUnauthorizedError, + match=r'^Invalid token\. To log in to profile "me" again and save the organization, add --jwt TOKEN or --user_email EMAIL$', + ): + authenticate.authenticate( + user_name="me", organization_key="2222222222", organization_name="work" + ) + assert fetched == ["999"] + assert config_path.read_text() == before + + +@pytest.mark.parametrize( + "kwargs, message", + [ + ({"organization_key": "1234567890"}, "--organization_name NAME"), + ({"organization_name": "work"}, "--delete"), + ({"organization_key": "work", "organization_name": "work"}, "organization ID"), + ({"organization_key": "1234567890", "organization_name": "1work"}, "Invalid"), + ( + {"organization_key": "1234567890", "delete": True}, + "not --organization_key", + ), + ], +) +def test_authenticate_organization_options_fail_before_anything( + config_path: Path, + fetched: list[str], + monkeypatch: pytest.MonkeyPatch, + kwargs: dict, + message: str, +): + _add_profile("me", **{"MAPOrganization.work": "1234567890"}) + before = config_path.read_text() + _forbid_login(monkeypatch) + _enable_prompt(monkeypatch, []) + + with pytest.raises(exceptions.MapillaryBadParameterError, match=message): + authenticate.authenticate(**kwargs) + + assert fetched == [] + assert config_path.read_text() == before + + @pytest.mark.parametrize( "command", ["upload", "process_and_upload", "video_process_and_upload"] )