Support galaxy v3/autohub API in ansible-galaxy (#60982)

* Add galaxy collections API v3 support

Issue: ansible/galaxy-dev#60

- Determine if server supports v3

Use 'available_versions' from `GET /api`
to determine if 'v3' api is available on
the server.

- Support v3 pagination style

ie, 'limit/offset style', with the paginated
responses based on https://jsonapi.org/format/#fetching-pagination

v2 galaxy uses pagination that is more or less
'django rest framework style' or 'page/page_size style',
based on the default drf pagination described
at https://www.django-rest-framework.org/api-guide/pagination/#pagenumberpagination

- Support galaxy v3 style error response

The error objects returned by the galaxy v3 api are based
on the JSONAPI response/errors format
(https://jsonapi.org/format/#errors).

This handles that style response. At least for publish_collection
for now. Needs extracting/generalizing.

Handle HTTPError in CollectionRequirement.from_name()
with _handle_http_error(). It will raise AnsibleError
based on the json in an error response.

- Update unit tests

update test/unit/galaxy/test_collection*
to paramaterize calls to test against
mocked v2 and v3 servers apis.

Update artifacts_versions_json() to tale an
api version paramater.

Add error_json() for generating v3/v3 style error
responses.

So now, the urls generated and the pagination schema
of the response will use the v3 version if
the passed in GalaxyAPI 'galaxy_api' instance
has 'v3' in it's available_api_versions

* Move checking of server avail versions to collections.py

collections.py needs to know the server api versions
supported before it makes collection related calls,
so the 'lazy' server version check in api.GalaxyAPI
is never called and isn't set, so 'v3' servers weren't
found.

Update unit tests to mock the return value of the
request instead of GalaxyAPI itself.

1 files changed, 216 insertions, 13 deletions

diff --git a/test/units/galaxy/test_collection.py b/test/units/galaxy/test_collection.py
index 8d0aff3c86..6276fb346c 100644
--- a/test/units/galaxy/test_collection.py
+++ b/test/units/galaxy/test_collection.py
@@ -56,6 +56,13 @@ def collection_input(tmp_path_factory):
 
 
 @pytest.fixture()
+def galaxy_api_version(monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
+
+@pytest.fixture()
 def collection_artifact(monkeypatch, tmp_path_factory):
     ''' Creates a temp collection artifact and mocked open_url instance for publishing tests '''
     mock_open = MagicMock()
@@ -421,6 +428,10 @@ def test_publish_not_a_tarball():
 
 
 def test_publish_no_wait(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     mock_display = MagicMock()
     monkeypatch.setattr(Display, 'display', mock_display)
 
@@ -445,12 +456,15 @@ def test_publish_no_wait(galaxy_server, collection_artifact, monkeypatch):
     assert mock_display.mock_calls[0][1][0] == "Publishing collection artifact '%s' to %s %s" \
         % (artifact_path, galaxy_server.name, galaxy_server.api_server)
     assert mock_display.mock_calls[1][1][0] == \
-        "Collection has been pushed to the Galaxy server %s %s, not waiting until import has completed due to " \
-        "--no-wait being set. Import task results can be found at %s"\
-        % (galaxy_server.name, galaxy_server.api_server, fake_import_uri)
+        "Collection has been pushed to the Galaxy server %s %s, not waiting until import has completed due to --no-wait " \
+        "being set. Import task results can be found at %s" % (galaxy_server.name, galaxy_server.api_server, fake_import_uri)
 
 
-def test_publish_dont_validate_cert(galaxy_server, collection_artifact):
+def test_publish_dont_validate_cert(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     galaxy_server.validate_certs = False
     artifact_path, mock_open = collection_artifact
 
@@ -462,29 +476,47 @@ def test_publish_dont_validate_cert(galaxy_server, collection_artifact):
     assert mock_open.mock_calls[0][2]['validate_certs'] is False
 
 
-def test_publish_failure(galaxy_server, collection_artifact):
+def test_publish_failure(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     artifact_path, mock_open = collection_artifact
 
     mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 500, 'msg', {}, StringIO())
 
-    expected = 'Error when publishing collection (HTTP Code: 500, Message: Unknown error returned by Galaxy ' \
+    expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
+               '(HTTP Code: 500, Message: Unknown error returned by Galaxy ' \
                'server. Code: Unknown)'
     with pytest.raises(AnsibleError, match=re.escape(expected)):
         collection.publish_collection(artifact_path, galaxy_server, True, 0)
 
 
-def test_publish_failure_with_json_info(galaxy_server, collection_artifact):
+def test_publish_failure_with_json_info(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     artifact_path, mock_open = collection_artifact
 
     return_content = StringIO(u'{"message":"Galaxy error message","code":"GWE002"}')
     mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 503, 'msg', {}, return_content)
 
-    expected = 'Error when publishing collection (HTTP Code: 503, Message: Galaxy error message Code: GWE002)'
+    expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
+               '(HTTP Code: 503, Message: Galaxy error message Code: GWE002)'
     with pytest.raises(AnsibleError, match=re.escape(expected)):
         collection.publish_collection(artifact_path, galaxy_server, True, 0)
 
 
-def test_publish_with_wait(galaxy_server, collection_artifact, monkeypatch):
+@pytest.mark.parametrize("api_version,token_type", [
+    ('v2', 'Token'),
+    ('v3', 'Bearer')
+])
+def test_publish_with_wait(api_version, token_type, galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {api_version: '/api/%s' % api_version}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     mock_display = MagicMock()
     monkeypatch.setattr(Display, 'display', mock_display)
 
@@ -501,7 +533,7 @@ def test_publish_with_wait(galaxy_server, collection_artifact, monkeypatch):
 
     assert mock_open.call_count == 2
     assert mock_open.mock_calls[1][1][0] == fake_import_uri
-    assert mock_open.mock_calls[1][2]['headers']['Authorization'] == 'Token key'
+    assert mock_open.mock_calls[1][2]['headers']['Authorization'] == '%s key' % token_type
     assert mock_open.mock_calls[1][2]['validate_certs'] is True
     assert mock_open.mock_calls[1][2]['method'] == 'GET'
 
@@ -515,7 +547,15 @@ def test_publish_with_wait(galaxy_server, collection_artifact, monkeypatch):
                                                'Galaxy server %s %s' % (galaxy_server.name, galaxy_server.api_server)
 
 
-def test_publish_with_wait_timeout(galaxy_server, collection_artifact, monkeypatch):
+@pytest.mark.parametrize("api_version,exp_api_url,token_type", [
+    ('v2', '/api/v2/collections/', 'Token'),
+    ('v3', '/api/v3/artifacts/collections/', 'Bearer')
+])
+def test_publish_with_wait_timeout(api_version, exp_api_url, token_type, galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {api_version: '/api/%s' % api_version}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     monkeypatch.setattr(time, 'sleep', MagicMock())
 
     mock_vvv = MagicMock()
@@ -535,11 +575,11 @@ def test_publish_with_wait_timeout(galaxy_server, collection_artifact, monkeypat
 
     assert mock_open.call_count == 3
     assert mock_open.mock_calls[1][1][0] == fake_import_uri
-    assert mock_open.mock_calls[1][2]['headers']['Authorization'] == 'Token key'
+    assert mock_open.mock_calls[1][2]['headers']['Authorization'] == '%s key' % token_type
     assert mock_open.mock_calls[1][2]['validate_certs'] is True
     assert mock_open.mock_calls[1][2]['method'] == 'GET'
     assert mock_open.mock_calls[2][1][0] == fake_import_uri
-    assert mock_open.mock_calls[2][2]['headers']['Authorization'] == 'Token key'
+    assert mock_open.mock_calls[2][2]['headers']['Authorization'] == '%s key' % token_type
     assert mock_open.mock_calls[2][2]['validate_certs'] is True
     assert mock_open.mock_calls[2][2]['method'] == 'GET'
 
@@ -549,6 +589,10 @@ def test_publish_with_wait_timeout(galaxy_server, collection_artifact, monkeypat
 
 
 def test_publish_with_wait_timeout_failure(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     monkeypatch.setattr(time, 'sleep', MagicMock())
 
     mock_vvv = MagicMock()
@@ -588,6 +632,10 @@ def test_publish_with_wait_timeout_failure(galaxy_server, collection_artifact, m
 
 
 def test_publish_with_wait_and_failure(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     mock_display = MagicMock()
     monkeypatch.setattr(Display, 'display', mock_display)
 
@@ -661,6 +709,10 @@ def test_publish_with_wait_and_failure(galaxy_server, collection_artifact, monke
 
 
 def test_publish_with_wait_and_failure_and_no_error(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     mock_display = MagicMock()
     monkeypatch.setattr(Display, 'display', mock_display)
 
@@ -729,6 +781,69 @@ def test_publish_with_wait_and_failure_and_no_error(galaxy_server, collection_ar
     assert mock_err.mock_calls[0][1][0] == 'Galaxy import error message: Some error'
 
 
+def test_publish_failure_v3_with_json_info_409_conflict(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v3': '/api/v3'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
+    artifact_path, mock_open = collection_artifact
+
+    error_response = {
+        "errors": [
+            {
+                "code": "conflict.collection_exists",
+                "detail": 'Collection "testing-ansible_testing_content-4.0.4" already exists.',
+                "title": "Conflict.",
+                "status": "409",
+            },
+        ]
+    }
+
+    return_content = StringIO(to_text(json.dumps(error_response)))
+    mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 409, 'msg', {}, return_content)
+
+    expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
+               '(HTTP Code: 409, Message: Collection "testing-ansible_testing_content-4.0.4"' \
+               ' already exists. Code: conflict.collection_exists)'
+    with pytest.raises(AnsibleError, match=re.escape(expected)):
+        collection.publish_collection(artifact_path, galaxy_server, True, 0)
+
+
+def test_publish_failure_v3_with_json_info_multiple_errors(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v3': '/api/v3'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
+    artifact_path, mock_open = collection_artifact
+
+    error_response = {
+        "errors": [
+            {
+                "code": "conflict.collection_exists",
+                "detail": 'Collection "mynamespace-mycollection-4.1.1" already exists.',
+                "title": "Conflict.",
+                "status": "400",
+            },
+            {
+                "code": "quantum_improbability",
+                "title": "Random(?) quantum improbability.",
+                "source": {"parameter": "the_arrow_of_time"},
+                "meta": {"remediation": "Try again before"}
+            },
+        ]
+    }
+
+    return_content = StringIO(to_text(json.dumps(error_response)))
+    mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 400, 'msg', {}, return_content)
+
+    expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
+               '(HTTP Code: 400, Message: Collection "mynamespace-mycollection-4.1.1"' \
+               ' already exists. Code: conflict.collection_exists),' \
+               ' (HTTP Code: 400, Message: Random(?) quantum improbability. Code: quantum_improbability)'
+    with pytest.raises(AnsibleError, match=re.escape(expected)):
+        collection.publish_collection(artifact_path, galaxy_server, True, 0)
+
+
 def test_find_existing_collections(tmp_path_factory, monkeypatch):
     test_dir = to_text(tmp_path_factory.mktemp('test-ÅÑŚÌβŁÈ Collections'))
     collection1 = os.path.join(test_dir, 'namespace1', 'collection1')
@@ -847,3 +962,91 @@ def test_extract_tar_file_missing_parent_dir(tmp_tarfile):
 
     collection._extract_tar_file(tfile, filename, output_dir, temp_dir, checksum)
     os.path.isfile(output_file)
+
+
+def test_get_available_api_versions_v2_auth_not_required_without_auth(galaxy_server, collection_artifact, monkeypatch):
+    # mock_avail_ver = MagicMock()
+    # mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version}
+    # monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+    response_obj = {
+        "description": "GALAXY REST API",
+        "current_version": "v1",
+        "available_versions": {
+            "v1": "/api/v1/",
+            "v2": "/api/v2/"
+        },
+        "server_version": "3.2.4",
+        "version_name": "Doin' it Right",
+        "team_members": [
+            "chouseknecht",
+            "cutwater",
+            "alikins",
+            "newswangerd",
+            "awcrosby",
+            "tima",
+            "gregdek"
+        ]
+    }
+
+    artifact_path, mock_open = collection_artifact
+
+    return_content = StringIO(to_text(json.dumps(response_obj)))
+    mock_open.return_value = return_content
+    res = collection.get_available_api_versions(galaxy_server)
+
+    assert res == {'v1': '/api/v1/', 'v2': '/api/v2/'}
+
+
+def test_get_available_api_versions_v3_auth_required_without_auth(galaxy_server, collection_artifact, monkeypatch):
+    # mock_avail_ver = MagicMock()
+    # mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version}
+    # monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+    error_response = {'code': 'unauthorized', 'detail': 'The request was not authorized'}
+    artifact_path, mock_open = collection_artifact
+
+    return_content = StringIO(to_text(json.dumps(error_response)))
+    mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {'WWW-Authenticate': 'Bearer'}, return_content)
+    with pytest.raises(AnsibleError):
+        collection.get_available_api_versions(galaxy_server)
+
+
+def test_get_available_api_versions_v3_auth_required_with_auth_on_retry(galaxy_server, collection_artifact, monkeypatch):
+    # mock_avail_ver = MagicMock()
+    # mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version}
+    # monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+    error_obj = {'code': 'unauthorized', 'detail': 'The request was not authorized'}
+    success_obj = {
+        "description": "GALAXY REST API",
+        "current_version": "v1",
+        "available_versions": {
+            "v3": "/api/v3/"
+        },
+        "server_version": "3.2.4",
+        "version_name": "Doin' it Right",
+        "team_members": [
+            "chouseknecht",
+            "cutwater",
+            "alikins",
+            "newswangerd",
+            "awcrosby",
+            "tima",
+            "gregdek"
+        ]
+    }
+
+    artifact_path, mock_open = collection_artifact
+
+    error_response = StringIO(to_text(json.dumps(error_obj)))
+    success_response = StringIO(to_text(json.dumps(success_obj)))
+    mock_open.side_effect = [
+        urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {'WWW-Authenticate': 'Bearer'}, error_response),
+        success_response,
+    ]
+
+    try:
+        res = collection.get_available_api_versions(galaxy_server)
+    except AnsibleError as err:
+        print(err)
+        raise
+
+    assert res == {'v3': '/api/v3/'}