diff options
author | Matt Clay <matt@mystile.com> | 2022-04-29 23:15:21 +0200 |
---|---|---|
committer | Matt Clay <matt@mystile.com> | 2022-04-30 03:20:14 +0200 |
commit | 2cc74b04c49338b48af070ddd811b25b5d801c12 (patch) | |
tree | 708cfc1700a31bba111c9ef5223a29e808519686 /test/lib/ansible_test | |
parent | ansible-test - Remove obsolete code. (diff) | |
download | ansible-2cc74b04c49338b48af070ddd811b25b5d801c12.tar.xz ansible-2cc74b04c49338b48af070ddd811b25b5d801c12.zip |
ansible-test - Add multi-arch remote support.
Diffstat (limited to 'test/lib/ansible_test')
16 files changed, 273 insertions, 113 deletions
diff --git a/test/lib/ansible_test/_data/completion/network.txt b/test/lib/ansible_test/_data/completion/network.txt index 8c6243e9a1..1d6b0c196a 100644 --- a/test/lib/ansible_test/_data/completion/network.txt +++ b/test/lib/ansible_test/_data/completion/network.txt @@ -1,2 +1,2 @@ -ios/csr1000v collection=cisco.ios connection=ansible.netcommon.network_cli provider=aws -vyos/1.1.8 collection=vyos.vyos connection=ansible.netcommon.network_cli provider=aws +ios/csr1000v collection=cisco.ios connection=ansible.netcommon.network_cli provider=aws arch=x86_64 +vyos/1.1.8 collection=vyos.vyos connection=ansible.netcommon.network_cli provider=aws arch=x86_64 diff --git a/test/lib/ansible_test/_data/completion/remote.txt b/test/lib/ansible_test/_data/completion/remote.txt index 68d02e90c7..cffaf8a9b5 100644 --- a/test/lib/ansible_test/_data/completion/remote.txt +++ b/test/lib/ansible_test/_data/completion/remote.txt @@ -1,10 +1,10 @@ -freebsd/12.3 python=3.8 python_dir=/usr/local/bin provider=aws -freebsd/13.0 python=3.7,3.8,3.9 python_dir=/usr/local/bin provider=aws -freebsd python_dir=/usr/local/bin provider=aws -macos/12.0 python=3.10 python_dir=/usr/local/bin provider=parallels -macos python_dir=/usr/local/bin provider=parallels -rhel/7.9 python=2.7 provider=aws -rhel/8.5 python=3.6,3.8,3.9 provider=aws -rhel provider=aws -ubuntu/22.04 python=3.10 provider=aws -ubuntu provider=aws +freebsd/12.3 python=3.8 python_dir=/usr/local/bin provider=aws arch=x86_64 +freebsd/13.0 python=3.7,3.8,3.9 python_dir=/usr/local/bin provider=aws arch=x86_64 +freebsd python_dir=/usr/local/bin provider=aws arch=x86_64 +macos/12.0 python=3.10 python_dir=/usr/local/bin provider=parallels arch=x86_64 +macos python_dir=/usr/local/bin provider=parallels arch=x86_64 +rhel/7.9 python=2.7 provider=aws arch=x86_64 +rhel/8.5 python=3.6,3.8,3.9 provider=aws arch=x86_64 +rhel provider=aws arch=x86_64 +ubuntu/22.04 python=3.10 provider=aws arch=x86_64 +ubuntu provider=aws arch=x86_64 diff --git a/test/lib/ansible_test/_data/completion/windows.txt b/test/lib/ansible_test/_data/completion/windows.txt index 280ad97f13..767c36cbcb 100644 --- a/test/lib/ansible_test/_data/completion/windows.txt +++ b/test/lib/ansible_test/_data/completion/windows.txt @@ -1,6 +1,6 @@ -windows/2012 provider=aws -windows/2012-R2 provider=aws -windows/2016 provider=aws -windows/2019 provider=aws -windows/2022 provider=aws -windows provider=aws +windows/2012 provider=aws arch=x86_64 +windows/2012-R2 provider=aws arch=x86_64 +windows/2016 provider=aws arch=x86_64 +windows/2019 provider=aws arch=x86_64 +windows/2022 provider=aws arch=x86_64 +windows provider=aws arch=x86_64 diff --git a/test/lib/ansible_test/_internal/cli/compat.py b/test/lib/ansible_test/_internal/cli/compat.py index dfa7cfa6d0..ad8a33cb9f 100644 --- a/test/lib/ansible_test/_internal/cli/compat.py +++ b/test/lib/ansible_test/_internal/cli/compat.py @@ -93,6 +93,18 @@ class PythonVersionUnspecifiedError(ApplicationError): super().__init__(f'A Python version was not specified for environment `{context}`. Use the `--python` option to specify a Python version.') +class RemoteProviderUnspecifiedError(ApplicationError): + """A remote provider was not specified for a context which is unknown, thus the remote provider version is unknown.""" + def __init__(self, context): + super().__init__(f'A remote provider was not specified for environment `{context}`. Use the `--remote-provider` option to specify a provider.') + + +class RemoteArchitectureUnspecifiedError(ApplicationError): + """A remote architecture was not specified for a context which is unknown, thus the remote architecture version is unknown.""" + def __init__(self, context): + super().__init__(f'A remote architecture was not specified for environment `{context}`. Use the `--remote-arch` option to specify an architecture.') + + class ControllerNotSupportedError(ApplicationError): """Option(s) were specified which do not provide support for the controller and would be ignored because they are irrelevant for the target.""" def __init__(self, context): @@ -115,6 +127,7 @@ class LegacyHostOptions: venv_system_site_packages: t.Optional[bool] = None remote: t.Optional[str] = None remote_provider: t.Optional[str] = None + remote_arch: t.Optional[str] = None docker: t.Optional[str] = None docker_privileged: t.Optional[bool] = None docker_seccomp: t.Optional[str] = None @@ -371,33 +384,40 @@ def get_legacy_host_config( if remote_config.controller_supported: if controller_python(options.python) or not options.python: - controller = PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider) + controller = PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider, + arch=options.remote_arch) targets = controller_targets(mode, options, controller) else: controller_fallback = f'remote:{options.remote}', f'--remote {options.remote} --python {options.python}', FallbackReason.PYTHON - controller = PosixRemoteConfig(name=options.remote, provider=options.remote_provider) + controller = PosixRemoteConfig(name=options.remote, provider=options.remote_provider, arch=options.remote_arch) targets = controller_targets(mode, options, controller) else: context, reason = f'--remote {options.remote}', FallbackReason.ENVIRONMENT controller = None - targets = [PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider)] + targets = [PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider, arch=options.remote_arch)] elif mode == TargetMode.SHELL and options.remote.startswith('windows/'): if options.python and options.python not in CONTROLLER_PYTHON_VERSIONS: raise ControllerNotSupportedError(f'--python {options.python}') controller = OriginConfig(python=native_python(options)) - targets = [WindowsRemoteConfig(name=options.remote, provider=options.remote_provider)] + targets = [WindowsRemoteConfig(name=options.remote, provider=options.remote_provider, arch=options.remote_arch)] else: if not options.python: raise PythonVersionUnspecifiedError(f'--remote {options.remote}') + if not options.remote_provider: + raise RemoteProviderUnspecifiedError(f'--remote {options.remote}') + + if not options.remote_arch: + raise RemoteArchitectureUnspecifiedError(f'--remote {options.remote}') + if controller_python(options.python): - controller = PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider) + controller = PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider, arch=options.remote_arch) targets = controller_targets(mode, options, controller) else: context, reason = f'--remote {options.remote} --python {options.python}', FallbackReason.PYTHON controller = None - targets = [PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider)] + targets = [PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider, arch=options.remote_arch)] if not controller: if docker_available(): @@ -455,12 +475,13 @@ def handle_non_posix_targets( """Return a list of non-POSIX targets if the target mode is non-POSIX.""" if mode == TargetMode.WINDOWS_INTEGRATION: if options.windows: - targets = [WindowsRemoteConfig(name=f'windows/{version}', provider=options.remote_provider) for version in options.windows] + targets = [WindowsRemoteConfig(name=f'windows/{version}', provider=options.remote_provider, arch=options.remote_arch) + for version in options.windows] else: targets = [WindowsInventoryConfig(path=options.inventory)] elif mode == TargetMode.NETWORK_INTEGRATION: if options.platform: - network_targets = [NetworkRemoteConfig(name=platform, provider=options.remote_provider) for platform in options.platform] + network_targets = [NetworkRemoteConfig(name=platform, provider=options.remote_provider, arch=options.remote_arch) for platform in options.platform] for platform, collection in options.platform_collection or []: for entry in network_targets: diff --git a/test/lib/ansible_test/_internal/cli/environments.py b/test/lib/ansible_test/_internal/cli/environments.py index 5709c7c1ec..e3e759fda5 100644 --- a/test/lib/ansible_test/_internal/cli/environments.py +++ b/test/lib/ansible_test/_internal/cli/environments.py @@ -13,6 +13,10 @@ from ..constants import ( SUPPORTED_PYTHON_VERSIONS, ) +from ..util import ( + REMOTE_ARCHITECTURES, +) + from ..completion import ( docker_completion, network_completion, @@ -532,6 +536,13 @@ def add_environment_remote( help=suppress or 'remote provider to use: %(choices)s', ) + environments_parser.add_argument( + '--remote-arch', + metavar='ARCH', + choices=REMOTE_ARCHITECTURES, + help=suppress or 'remote arch to use: %(choices)s', + ) + def complete_remote_stage(prefix: str, **_) -> t.List[str]: """Return a list of supported stages matching the given prefix.""" diff --git a/test/lib/ansible_test/_internal/cli/parsers/key_value_parsers.py b/test/lib/ansible_test/_internal/cli/parsers/key_value_parsers.py index b22705f731..8f71e7635e 100644 --- a/test/lib/ansible_test/_internal/cli/parsers/key_value_parsers.py +++ b/test/lib/ansible_test/_internal/cli/parsers/key_value_parsers.py @@ -10,6 +10,10 @@ from ...constants import ( SUPPORTED_PYTHON_VERSIONS, ) +from ...util import ( + REMOTE_ARCHITECTURES, +) + from ...host_configs import ( OriginConfig, ) @@ -126,6 +130,7 @@ class PosixRemoteKeyValueParser(KeyValueParser): """Return a dictionary of key names and value parsers.""" return dict( provider=ChoicesParser(REMOTE_PROVIDERS), + arch=ChoicesParser(REMOTE_ARCHITECTURES), python=PythonParser(versions=self.versions, allow_venv=False, allow_default=self.allow_default), ) @@ -137,6 +142,7 @@ class PosixRemoteKeyValueParser(KeyValueParser): state.sections[f'{"controller" if self.controller else "target"} {section_name} (comma separated):'] = '\n'.join([ f' provider={ChoicesParser(REMOTE_PROVIDERS).document(state)}', + f' arch={ChoicesParser(REMOTE_ARCHITECTURES).document(state)}', f' python={python_parser.document(state)}', ]) @@ -149,6 +155,7 @@ class WindowsRemoteKeyValueParser(KeyValueParser): """Return a dictionary of key names and value parsers.""" return dict( provider=ChoicesParser(REMOTE_PROVIDERS), + arch=ChoicesParser(REMOTE_ARCHITECTURES), ) def document(self, state): # type: (DocumentationState) -> t.Optional[str] @@ -157,6 +164,7 @@ class WindowsRemoteKeyValueParser(KeyValueParser): state.sections[f'target {section_name} (comma separated):'] = '\n'.join([ f' provider={ChoicesParser(REMOTE_PROVIDERS).document(state)}', + f' arch={ChoicesParser(REMOTE_ARCHITECTURES).document(state)}', ]) return f'{{{section_name}}}' @@ -168,6 +176,7 @@ class NetworkRemoteKeyValueParser(KeyValueParser): """Return a dictionary of key names and value parsers.""" return dict( provider=ChoicesParser(REMOTE_PROVIDERS), + arch=ChoicesParser(REMOTE_ARCHITECTURES), collection=AnyParser(), connection=AnyParser(), ) @@ -178,7 +187,8 @@ class NetworkRemoteKeyValueParser(KeyValueParser): state.sections[f'target {section_name} (comma separated):'] = '\n'.join([ f' provider={ChoicesParser(REMOTE_PROVIDERS).document(state)}', - ' collection={collecton}', + f' arch={ChoicesParser(REMOTE_ARCHITECTURES).document(state)}', + ' collection={collection}', ' connection={connection}', ]) diff --git a/test/lib/ansible_test/_internal/commands/integration/cloud/aws.py b/test/lib/ansible_test/_internal/commands/integration/cloud/aws.py index b2b02095f3..a67a0f89a9 100644 --- a/test/lib/ansible_test/_internal/commands/integration/cloud/aws.py +++ b/test/lib/ansible_test/_internal/commands/integration/cloud/aws.py @@ -21,6 +21,7 @@ from ....target import ( from ....core_ci import ( AnsibleCoreCI, + CloudResource, ) from ....host_configs import ( @@ -91,7 +92,7 @@ class AwsCloudProvider(CloudProvider): def _create_ansible_core_ci(self): # type: () -> AnsibleCoreCI """Return an AWS instance of AnsibleCoreCI.""" - return AnsibleCoreCI(self.args, 'aws', 'aws', 'aws', persist=False) + return AnsibleCoreCI(self.args, CloudResource(platform='aws')) class AwsCloudEnvironment(CloudEnvironment): diff --git a/test/lib/ansible_test/_internal/commands/integration/cloud/azure.py b/test/lib/ansible_test/_internal/commands/integration/cloud/azure.py index cf16c7f54a..f67d1adf25 100644 --- a/test/lib/ansible_test/_internal/commands/integration/cloud/azure.py +++ b/test/lib/ansible_test/_internal/commands/integration/cloud/azure.py @@ -19,6 +19,7 @@ from ....target import ( from ....core_ci import ( AnsibleCoreCI, + CloudResource, ) from . import ( @@ -97,7 +98,7 @@ class AzureCloudProvider(CloudProvider): def _create_ansible_core_ci(self): # type: () -> AnsibleCoreCI """Return an Azure instance of AnsibleCoreCI.""" - return AnsibleCoreCI(self.args, 'azure', 'azure', 'azure', persist=False) + return AnsibleCoreCI(self.args, CloudResource(platform='azure')) class AzureCloudEnvironment(CloudEnvironment): diff --git a/test/lib/ansible_test/_internal/commands/integration/cloud/hcloud.py b/test/lib/ansible_test/_internal/commands/integration/cloud/hcloud.py index 28b07e7230..6912aff36d 100644 --- a/test/lib/ansible_test/_internal/commands/integration/cloud/hcloud.py +++ b/test/lib/ansible_test/_internal/commands/integration/cloud/hcloud.py @@ -18,6 +18,7 @@ from ....target import ( from ....core_ci import ( AnsibleCoreCI, + CloudResource, ) from . import ( @@ -78,7 +79,7 @@ class HcloudCloudProvider(CloudProvider): def _create_ansible_core_ci(self): # type: () -> AnsibleCoreCI """Return a Heztner instance of AnsibleCoreCI.""" - return AnsibleCoreCI(self.args, 'hetzner', 'hetzner', 'hetzner', persist=False) + return AnsibleCoreCI(self.args, CloudResource(platform='hetzner')) class HcloudCloudEnvironment(CloudEnvironment): diff --git a/test/lib/ansible_test/_internal/commands/integration/filters.py b/test/lib/ansible_test/_internal/commands/integration/filters.py index 0396ce9231..35acae52c8 100644 --- a/test/lib/ansible_test/_internal/commands/integration/filters.py +++ b/test/lib/ansible_test/_internal/commands/integration/filters.py @@ -10,6 +10,7 @@ from ...config import ( from ...util import ( cache, + detect_architecture, display, get_type_map, ) @@ -223,6 +224,14 @@ class NetworkInventoryTargetFilter(TargetFilter[NetworkInventoryConfig]): class OriginTargetFilter(PosixTargetFilter[OriginConfig]): """Target filter for localhost.""" + def filter_targets(self, targets, exclude): # type: (t.List[IntegrationTarget], t.Set[str]) -> None + """Filter the list of targets, adding any which this host profile cannot support to the provided exclude list.""" + super().filter_targets(targets, exclude) + + arch = detect_architecture(self.config.python.path) + + if arch: + self.skip(f'skip/{arch}', f'which are not supported by {arch}', targets, exclude) @cache @@ -247,10 +256,7 @@ def get_target_filter(args, configs, controller): # type: (IntegrationConfig, t def get_remote_skip_aliases(config): # type: (RemoteConfig) -> t.Dict[str, str] """Return a dictionary of skip aliases and the reason why they apply.""" - if isinstance(config, PosixRemoteConfig): - return get_platform_skip_aliases(config.platform, config.version, config.arch) - - return get_platform_skip_aliases(config.platform, config.version, None) + return get_platform_skip_aliases(config.platform, config.version, config.arch) def get_platform_skip_aliases(platform, version, arch): # type: (str, str, t.Optional[str]) -> t.Dict[str, str] diff --git a/test/lib/ansible_test/_internal/completion.py b/test/lib/ansible_test/_internal/completion.py index 46845816d2..73aee2f25f 100644 --- a/test/lib/ansible_test/_internal/completion.py +++ b/test/lib/ansible_test/_internal/completion.py @@ -79,6 +79,7 @@ class PythonCompletionConfig(PosixCompletionConfig, metaclass=abc.ABCMeta): class RemoteCompletionConfig(CompletionConfig): """Base class for completion configuration of remote environments provisioned through Ansible Core CI.""" provider: t.Optional[str] = None + arch: t.Optional[str] = None @property def platform(self): @@ -99,6 +100,9 @@ class RemoteCompletionConfig(CompletionConfig): if not self.provider: raise Exception(f'Remote completion entry "{self.name}" must provide a "provider" setting.') + if not self.arch: + raise Exception(f'Remote completion entry "{self.name}" must provide a "arch" setting.') + @dataclasses.dataclass(frozen=True) class InventoryCompletionConfig(CompletionConfig): @@ -152,6 +156,11 @@ class NetworkRemoteCompletionConfig(RemoteCompletionConfig): """Configuration for remote network platforms.""" collection: str = '' connection: str = '' + placeholder: bool = False + + def __post_init__(self): + if not self.placeholder: + super().__post_init__() @dataclasses.dataclass(frozen=True) @@ -160,7 +169,8 @@ class PosixRemoteCompletionConfig(RemoteCompletionConfig, PythonCompletionConfig placeholder: bool = False def __post_init__(self): - super().__post_init__() + if not self.placeholder: + super().__post_init__() if not self.supported_pythons: if self.version and not self.placeholder: diff --git a/test/lib/ansible_test/_internal/config.py b/test/lib/ansible_test/_internal/config.py index 5da5fc00e4..060962fb2e 100644 --- a/test/lib/ansible_test/_internal/config.py +++ b/test/lib/ansible_test/_internal/config.py @@ -48,29 +48,6 @@ class TerminateMode(enum.Enum): return self.name.lower() -class ParsedRemote: - """A parsed version of a "remote" string.""" - def __init__(self, arch, platform, version): # type: (t.Optional[str], str, str) -> None - self.arch = arch - self.platform = platform - self.version = version - - @staticmethod - def parse(value): # type: (str) -> t.Optional['ParsedRemote'] - """Return a ParsedRemote from the given value or None if the syntax is invalid.""" - parts = value.split('/') - - if len(parts) == 2: - arch = None - platform, version = parts - elif len(parts) == 3: - arch, platform, version = parts - else: - return None - - return ParsedRemote(arch, platform, version) - - class EnvironmentConfig(CommonConfig): """Configuration common to all commands which execute in an environment.""" def __init__(self, args, command): # type: (t.Any, str) -> None diff --git a/test/lib/ansible_test/_internal/core_ci.py b/test/lib/ansible_test/_internal/core_ci.py index 37e4ac061f..62d063b2b7 100644 --- a/test/lib/ansible_test/_internal/core_ci.py +++ b/test/lib/ansible_test/_internal/core_ci.py @@ -1,6 +1,8 @@ """Access Ansible Core CI remote services.""" from __future__ import annotations +import abc +import dataclasses import json import os import re @@ -48,6 +50,65 @@ from .data import ( ) +@dataclasses.dataclass(frozen=True) +class Resource(metaclass=abc.ABCMeta): + """Base class for Ansible Core CI resources.""" + @abc.abstractmethod + def as_tuple(self) -> t.Tuple[str, str, str, str]: + """Return the resource as a tuple of platform, version, architecture and provider.""" + + @abc.abstractmethod + def get_label(self) -> str: + """Return a user-friendly label for this resource.""" + + @property + @abc.abstractmethod + def persist(self) -> bool: + """True if the resource is persistent, otherwise false.""" + + +@dataclasses.dataclass(frozen=True) +class VmResource(Resource): + """Details needed to request a VM from Ansible Core CI.""" + platform: str + version: str + architecture: str + provider: str + tag: str + + def as_tuple(self) -> t.Tuple[str, str, str, str]: + """Return the resource as a tuple of platform, version, architecture and provider.""" + return self.platform, self.version, self.architecture, self.provider + + def get_label(self) -> str: + """Return a user-friendly label for this resource.""" + return f'{self.platform} {self.version} ({self.architecture}) [{self.tag}] @{self.provider}' + + @property + def persist(self) -> bool: + """True if the resource is persistent, otherwise false.""" + return True + + +@dataclasses.dataclass(frozen=True) +class CloudResource(Resource): + """Details needed to request cloud credentials from Ansible Core CI.""" + platform: str + + def as_tuple(self) -> t.Tuple[str, str, str, str]: + """Return the resource as a tuple of platform, version, architecture and provider.""" + return self.platform, '', '', self.platform + + def get_label(self) -> str: + """Return a user-friendly label for this resource.""" + return self.platform + + @property + def persist(self) -> bool: + """True if the resource is persistent, otherwise false.""" + return False + + class AnsibleCoreCI: """Client for Ansible Core CI services.""" DEFAULT_ENDPOINT = 'https://ansible-core-ci.testing.ansible.com' @@ -55,16 +116,12 @@ class AnsibleCoreCI: def __init__( self, args, # type: EnvironmentConfig - platform, # type: str - version, # type: str - provider, # type: str - persist=True, # type: bool + resource, # type: Resource load=True, # type: bool - suffix=None, # type: t.Optional[str] ): # type: (...) -> None self.args = args - self.platform = platform - self.version = version + self.resource = resource + self.platform, self.version, self.arch, self.provider = self.resource.as_tuple() self.stage = args.remote_stage self.client = HttpClient(args) self.connection = None @@ -73,35 +130,33 @@ class AnsibleCoreCI: self.default_endpoint = args.remote_endpoint or self.DEFAULT_ENDPOINT self.retries = 3 self.ci_provider = get_ci_provider() - self.provider = provider - self.name = '%s-%s' % (self.platform, self.version) + self.label = self.resource.get_label() - if suffix: - self.name += '-' + suffix + stripped_label = re.sub('[^A-Za-z0-9_.]+', '-', self.label).strip('-') - self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s-%s' % (self.name, self.provider, self.stage)) + self.name = f"{stripped_label}-{self.stage}" # turn the label into something suitable for use as a filename + + self.path = os.path.expanduser(f'~/.ansible/test/instances/{self.name}') self.ssh_key = SshKey(args) - if persist and load and self._load(): + if self.resource.persist and load and self._load(): try: - display.info('Checking existing %s/%s instance %s.' % (self.platform, self.version, self.instance_id), - verbosity=1) + display.info(f'Checking existing {self.label} instance using: {self._uri}', verbosity=1) self.connection = self.get(always_raise_on=[404]) - display.info('Loaded existing %s/%s from: %s' % (self.platform, self.version, self._uri), verbosity=1) + display.info(f'Loaded existing {self.label} instance.', verbosity=1) except HttpError as ex: if ex.status != 404: raise self._clear() - display.info('Cleared stale %s/%s instance %s.' % (self.platform, self.version, self.instance_id), - verbosity=1) + display.info(f'Cleared stale {self.label} instance.', verbosity=1) self.instance_id = None self.endpoint = None - elif not persist: + elif not self.resource.persist: self.instance_id = None self.endpoint = None self._clear() @@ -126,8 +181,7 @@ class AnsibleCoreCI: def start(self): """Start instance.""" if self.started: - display.info('Skipping started %s/%s instance %s.' % (self.platform, self.version, self.instance_id), - verbosity=1) + display.info(f'Skipping started {self.label} instance.', verbosity=1) return None return self._start(self.ci_provider.prepare_core_ci_auth()) @@ -135,22 +189,19 @@ class AnsibleCoreCI: def stop(self): """Stop instance.""" if not self.started: - display.info('Skipping invalid %s/%s instance %s.' % (self.platform, self.version, self.instance_id), - verbosity=1) + display.info(f'Skipping invalid {self.label} instance.', verbosity=1) return response = self.client.delete(self._uri) if response.status_code == 404: self._clear() - display.info('Cleared invalid %s/%s instance %s.' % (self.platform, self.version, self.instance_id), - verbosity=1) + display.info(f'Cleared invalid {self.label} instance.', verbosity=1) return if response.status_code == 200: self._clear() - display.info('Stopped running %s/%s instance %s.' % (self.platform, self.version, self.instance_id), - verbosity=1) + display.info(f'Stopped running {self.label} instance.', verbosity=1) return raise self._create_http_error(response) @@ -158,8 +209,7 @@ class AnsibleCoreCI: def get(self, tries=3, sleep=15, always_raise_on=None): # type: (int, int, t.Optional[t.List[int]]) -> t.Optional[InstanceConnection] """Get instance connection information.""" if not self.started: - display.info('Skipping invalid %s/%s instance %s.' % (self.platform, self.version, self.instance_id), - verbosity=1) + display.info(f'Skipping invalid {self.label} instance.', verbosity=1) return None if not always_raise_on: @@ -180,7 +230,7 @@ class AnsibleCoreCI: if not tries or response.status_code in always_raise_on: raise error - display.warning('%s. Trying again after %d seconds.' % (error, sleep)) + display.warning(f'{error}. Trying again after {sleep} seconds.') time.sleep(sleep) if self.args.explain: @@ -216,9 +266,7 @@ class AnsibleCoreCI: status = 'running' if self.connection.running else 'starting' - display.info('Status update: %s/%s on instance %s is %s.' % - (self.platform, self.version, self.instance_id, status), - verbosity=1) + display.info(f'The {self.label} instance is {status}.', verbosity=1) return self.connection @@ -229,16 +277,15 @@ class AnsibleCoreCI: return time.sleep(10) - raise ApplicationError('Timeout waiting for %s/%s instance %s.' % - (self.platform, self.version, self.instance_id)) + raise ApplicationError(f'Timeout waiting for {self.label} instance.') @property def _uri(self): - return '%s/%s/%s/%s' % (self.endpoint, self.stage, self.provider, self.instance_id) + return f'{self.endpoint}/{self.stage}/{self.provider}/{self.instance_id}' def _start(self, auth): """Start instance.""" - display.info('Initializing new %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1) + display.info(f'Initializing new {self.label} instance using: {self._uri}', verbosity=1) if self.platform == 'windows': winrm_config = read_text_file(os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'ConfigureRemotingForAnsible.ps1')) @@ -249,6 +296,7 @@ class AnsibleCoreCI: config=dict( platform=self.platform, version=self.version, + architecture=self.arch, public_key=self.ssh_key.pub_contents, query=False, winrm_config=winrm_config, @@ -266,7 +314,7 @@ class AnsibleCoreCI: self.started = True self._save() - display.info('Started %s/%s from: %s' % (self.platform, self.version, self._uri), verbosity=1) + display.info(f'Started {self.label} instance.', verbosity=1) if self.args.explain: return {} @@ -277,8 +325,6 @@ class AnsibleCoreCI: tries = self.retries sleep = 15 - display.info('Trying endpoint: %s' % self.endpoint, verbosity=1) - while True: tries -= 1 response = self.client.put(self._uri, data=json.dumps(data), headers=headers) @@ -294,7 +340,7 @@ class AnsibleCoreCI: if not tries: raise error - display.warning('%s. Trying again after %d seconds.' % (error, sleep)) + display.warning(f'{error}. Trying again after {sleep} seconds.') time.sleep(sleep) def _clear(self): @@ -345,14 +391,14 @@ class AnsibleCoreCI: def save(self): # type: () -> t.Dict[str, str] """Save instance details and return as a dictionary.""" return dict( - platform_version='%s/%s' % (self.platform, self.version), + label=self.resource.get_label(), instance_id=self.instance_id, endpoint=self.endpoint, ) @staticmethod def _create_http_error(response): # type: (HttpResponse) -> ApplicationError - """Return an exception created from the given HTTP resposne.""" + """Return an exception created from the given HTTP response.""" response_json = response.json() stack_trace = '' @@ -369,7 +415,7 @@ class AnsibleCoreCI: traceback_lines = traceback.format_list(traceback_lines) trace = '\n'.join([x.rstrip() for x in traceback_lines]) - stack_trace = ('\nTraceback (from remote server):\n%s' % trace) + stack_trace = f'\nTraceback (from remote server):\n{trace}' else: message = str(response_json) @@ -379,7 +425,7 @@ class AnsibleCoreCI: class CoreHttpError(HttpError): """HTTP response as an error.""" def __init__(self, status, remote_message, remote_stack_trace): # type: (int, str, str) -> None - super().__init__(status, '%s%s' % (remote_message, remote_stack_trace)) + super().__init__(status, f'{remote_message}{remote_stack_trace}') self.remote_message = remote_message self.remote_stack_trace = remote_stack_trace @@ -388,8 +434,8 @@ class CoreHttpError(HttpError): class SshKey: """Container for SSH key used to connect to remote instances.""" KEY_TYPE = 'rsa' # RSA is used to maintain compatibility with paramiko and EC2 - KEY_NAME = 'id_%s' % KEY_TYPE - PUB_NAME = '%s.pub' % KEY_NAME + KEY_NAME = f'id_{KEY_TYPE}' + PUB_NAME = f'{KEY_NAME}.pub' @mutex def __init__(self, args): # type: (EnvironmentConfig) -> None @@ -502,6 +548,6 @@ class InstanceConnection: def __str__(self): if self.password: - return '%s:%s [%s:%s]' % (self.hostname, self.port, self.username, self.password) + return f'{self.hostname}:{self.port} [{self.username}:{self.password}]' - return '%s:%s [%s]' % (self.hostname, self.port, self.username) + return f'{self.hostname}:{self.port} [{self.username}]' diff --git a/test/lib/ansible_test/_internal/host_configs.py b/test/lib/ansible_test/_internal/host_configs.py index fee741e8b9..11a4506465 100644 --- a/test/lib/ansible_test/_internal/host_configs.py +++ b/test/lib/ansible_test/_internal/host_configs.py @@ -39,6 +39,7 @@ from .util import ( get_available_python_versions, str_to_version, version_to_str, + Architecture, ) @@ -206,6 +207,7 @@ class RemoteConfig(HostConfig, metaclass=abc.ABCMeta): """Base class for remote host configuration.""" name: t.Optional[str] = None provider: t.Optional[str] = None + arch: t.Optional[str] = None @property def platform(self): # type: () -> str @@ -227,6 +229,7 @@ class RemoteConfig(HostConfig, metaclass=abc.ABCMeta): self.provider = None self.provider = self.provider or defaults.provider or 'aws' + self.arch = self.arch or defaults.arch or Architecture.X86_64 @property def is_managed(self): # type: () -> bool @@ -330,8 +333,6 @@ class DockerConfig(ControllerHostConfig, PosixConfig): @dataclasses.dataclass class PosixRemoteConfig(RemoteConfig, ControllerHostConfig, PosixConfig): """Configuration for a POSIX remote host.""" - arch: t.Optional[str] = None - def get_defaults(self, context): # type: (HostContext) -> PosixRemoteCompletionConfig """Return the default settings.""" return filter_completion(remote_completion()).get(self.name) or remote_completion().get(self.platform) or PosixRemoteCompletionConfig( @@ -388,6 +389,7 @@ class NetworkRemoteConfig(RemoteConfig, NetworkConfig): """Return the default settings.""" return filter_completion(network_completion()).get(self.name) or NetworkRemoteCompletionConfig( name=self.name, + placeholder=True, ) def apply_defaults(self, context, defaults): # type: (HostContext, CompletionConfig) -> None diff --git a/test/lib/ansible_test/_internal/host_profiles.py b/test/lib/ansible_test/_internal/host_profiles.py index e5718b7ed0..463521f027 100644 --- a/test/lib/ansible_test/_internal/host_profiles.py +++ b/test/lib/ansible_test/_internal/host_profiles.py @@ -40,6 +40,7 @@ from .host_configs import ( from .core_ci import ( AnsibleCoreCI, SshKey, + VmResource, ) from .util import ( @@ -50,6 +51,7 @@ from .util import ( get_type_map, sanitize_host_name, sorted_versions, + InternalError, ) from .util_common import ( @@ -295,12 +297,18 @@ class RemoteProfile(SshTargetHostProfile[TRemoteConfig], metaclass=abc.ABCMeta): def create_core_ci(self, load): # type: (bool) -> AnsibleCoreCI """Create and return an AnsibleCoreCI instance.""" + if not self.config.arch: + raise InternalError(f'No arch specified for config: {self.config}') + return AnsibleCoreCI( args=self.args, - platform=self.config.platform, - version=self.config.version, - provider=self.config.provider, - suffix='controller' if self.controller else 'target', + resource=VmResource( + platform=self.config.platform, + version=self.config.version, + architecture=self.config.arch, + provider=self.config.provider, + tag='controller' if self.controller else 'target', + ), load=load, ) diff --git a/test/lib/ansible_test/_internal/util.py b/test/lib/ansible_test/_internal/util.py index a9929d8a09..e43e3a31f8 100644 --- a/test/lib/ansible_test/_internal/util.py +++ b/test/lib/ansible_test/_internal/util.py @@ -6,8 +6,10 @@ import errno import fcntl import importlib.util import inspect +import json import keyword import os +import platform import pkgutil import random import re @@ -98,6 +100,18 @@ MODE_DIRECTORY = MODE_READ | stat.S_IWUSR | stat.S_IXUSR | stat.S_IXGRP | stat.S MODE_DIRECTORY_WRITE = MODE_DIRECTORY | stat.S_IWGRP | stat.S_IWOTH +class Architecture: + """ + Normalized architecture names. + These are the architectures supported by ansible-test, such as when provisioning remote instances. + """ + X86_64 = 'x86_64' + AARCH64 = 'aarch64' + + +REMOTE_ARCHITECTURES = list(value for key, value in Architecture.__dict__.items() if not key.startswith('__')) + + def is_valid_identifier(value: str) -> bool: """Return True if the given value is a valid non-keyword Python identifier, otherwise return False.""" return value.isidentifier() and not keyword.iskeyword(value) @@ -121,6 +135,58 @@ def cache(func): # type: (t.Callable[[], TValue]) -> t.Callable[[], TValue] return wrapper +@mutex +def detect_architecture(python: str) -> t.Optional[str]: + """Detect the architecture of the specified Python and return a normalized version, or None if it cannot be determined.""" + results: t.Dict[str, t.Optional[str]] + + try: + results = detect_architecture.results # type: ignore[attr-defined] + except AttributeError: + results = detect_architecture.results = {} # type: ignore[attr-defined] + + if python in results: + return results[python] + + if python == sys.executable or os.path.realpath(python) == os.path.realpath(sys.executable): + uname = platform.uname() + else: + data = raw_command([python, '-c', 'import json, platform; print(json.dumps(platform.uname()));'], capture=True)[0] + uname = json.loads(data) + + translation = { + 'x86_64': Architecture.X86_64, # Linux, macOS + 'amd64': Architecture.X86_64, # FreeBSD + 'aarch64': Architecture.AARCH64, # Linux, FreeBSD + 'arm64': Architecture.AARCH64, # FreeBSD + } + + candidates = [] + + if len(uname) >= 5: + candidates.append(uname[4]) + + if len(uname) >= 6: + candidates.append(uname[5]) + + candidates = sorted(set(candidates)) + architectures = sorted(set(arch for arch in [translation.get(candidate) for candidate in candidates] if arch)) + + architecture: t.Optional[str] = None + + if not architectures: + display.warning(f'Unable to determine architecture for Python interpreter "{python}" from: {candidates}') + elif len(architectures) == 1: + architecture = architectures[0] + display.info(f'Detected architecture {architecture} for Python interpreter: {python}', verbosity=1) + else: + display.warning(f'Conflicting architectures detected ({architectures}) for Python interpreter "{python}" from: {candidates}') + + results[python] = architecture + + return architecture + + def filter_args(args, filters): # type: (t.List[str], t.Dict[str, int]) -> t.List[str] """Return a filtered version of the given command line arguments.""" remaining = 0 |