Adding waiter to cluster remove process (#57324)

* Adding waiter to cluster remove process

* blank line contains whitespace

* update aws_eks integration test

* Refactor aws_eks test suite to use pip

* update version testing

* missing parens...

* add changelog fragment

* Add waiter to module_utils, fix exception handling.

* Correct EKS waiter checks

11 files changed, 127 insertions, 45 deletions

diff --git a/test/integration/targets/aws_eks/playbooks/roles/aws_eks/defaults/main.yml b/test/integration/targets/aws_eks/defaults/main.yml
index 214f249611..214f249611 100644
--- a/test/integration/targets/aws_eks/playbooks/roles/aws_eks/defaults/main.yml
+++ b/test/integration/targets/aws_eks/defaults/main.yml
diff --git a/test/integration/targets/aws_eks/playbooks/roles/aws_eks/files/eks-trust-policy.json b/test/integration/targets/aws_eks/files/eks-trust-policy.json
index 85cfb59dd2..85cfb59dd2 100644
--- a/test/integration/targets/aws_eks/playbooks/roles/aws_eks/files/eks-trust-policy.json
+++ b/test/integration/targets/aws_eks/files/eks-trust-policy.json
diff --git a/test/integration/targets/aws_eks/meta/main.yml b/test/integration/targets/aws_eks/meta/main.yml
new file mode 100644
index 0000000000..1810d4bec9
--- /dev/null
+++ b/test/integration/targets/aws_eks/meta/main.yml
@@ -0,0 +1,2 @@
+dependencies:
+  - setup_remote_tmp_dir
diff --git a/test/integration/targets/aws_eks/playbooks/full_test.yml b/test/integration/targets/aws_eks/playbooks/full_test.yml
deleted file mode 100644
index 354dd744d3..0000000000
--- a/test/integration/targets/aws_eks/playbooks/full_test.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-- hosts: localhost
-  connection: local
-  environment: "{{ ansible_test.environment }}"
-
-  roles:
-    - aws_eks
diff --git a/test/integration/targets/aws_eks/playbooks/old_version.yml b/test/integration/targets/aws_eks/playbooks/old_version.yml
deleted file mode 100644
index c055cc68e2..0000000000
--- a/test/integration/targets/aws_eks/playbooks/old_version.yml
+++ /dev/null
@@ -1,17 +0,0 @@
-- hosts: localhost
-  connection: local
-  environment: "{{ ansible_test.environment }}"
-
-  tasks:
-    - name: try and use aws_eks_cluster module
-      aws_eks_cluster:
-        state: absent
-        name: my_cluster
-      ignore_errors: yes
-      register: aws_eks_cluster
-
-    - name: ensure that aws_eks fails with friendly error message
-      assert:
-        that:
-          - '"msg" in aws_eks_cluster'
-          - aws_eks_cluster is failed
diff --git a/test/integration/targets/aws_eks/playbooks/roles/aws_eks/meta/main.yml b/test/integration/targets/aws_eks/playbooks/roles/aws_eks/meta/main.yml
deleted file mode 100644
index 32cf5dda7e..0000000000
--- a/test/integration/targets/aws_eks/playbooks/roles/aws_eks/meta/main.yml
+++ /dev/null
@@ -1 +0,0 @@
-dependencies: []
diff --git a/test/integration/targets/aws_eks/runme.sh b/test/integration/targets/aws_eks/runme.sh
deleted file mode 100755
index d9bb0581ed..0000000000
--- a/test/integration/targets/aws_eks/runme.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-
-set -eux
-
-# Test graceful failure for older versions of botocore
-source virtualenv.sh
-pip install 'botocore<1.10.0' boto3
-ansible-playbook -i ../../inventory -e @../../integration_config.yml -v playbooks/old_version.yml "$@"
-
-# Run full test suite
-source virtualenv.sh
-pip install 'botocore>=1.10.1' boto3
-ansible-playbook -i ../../inventory -e @../../integration_config.yml -v playbooks/full_test.yml "$@"
diff --git a/test/integration/targets/aws_eks/tasks/botocore_lt_1.10.1.yml b/test/integration/targets/aws_eks/tasks/botocore_lt_1.10.1.yml
new file mode 100644
index 0000000000..e4c4b31fe5
--- /dev/null
+++ b/test/integration/targets/aws_eks/tasks/botocore_lt_1.10.1.yml
@@ -0,0 +1,12 @@
+- name: try and use aws_eks_cluster module
+  aws_eks_cluster:
+    state: absent
+    name: my_cluster
+  ignore_errors: yes
+  register: aws_eks_cluster
+
+- name: ensure that aws_eks fails with friendly error message
+  assert:
+    that:
+      - '"msg" in aws_eks_cluster'
+      - aws_eks_cluster is failed
diff --git a/test/integration/targets/aws_eks/tasks/botocore_lt_1.12.38.yml b/test/integration/targets/aws_eks/tasks/botocore_lt_1.12.38.yml
new file mode 100644
index 0000000000..4feb7ab48f
--- /dev/null
+++ b/test/integration/targets/aws_eks/tasks/botocore_lt_1.12.38.yml
@@ -0,0 +1,13 @@
+- name: try using aws_eks_cluster wait with state=absent
+  aws_eks_cluster:
+    state: absent
+    name: my_cluster
+    wait: yes
+  ignore_errors: yes
+  register: aws_eks_cluster
+
+- name: ensure that aws_eks fails with friendly error message
+  assert:
+    that:
+      - '"msg" in aws_eks_cluster'
+      - aws_eks_cluster is failed
diff --git a/test/integration/targets/aws_eks/playbooks/roles/aws_eks/tasks/main.yml b/test/integration/targets/aws_eks/tasks/full_test.yml
index 05ab19986f..a48abd45f3 100644
--- a/test/integration/targets/aws_eks/playbooks/roles/aws_eks/tasks/main.yml
+++ b/test/integration/targets/aws_eks/tasks/full_test.yml
@@ -2,9 +2,7 @@
 # tasks file for aws_eks modules
 
 - block:
-    # FIXME: ap-south-1 only has two AZs, ap-south-1a and ap-south-1b
-    # That makes it my best guess as to it being among the last to support EKS
-    # If it does become supported, change this test to use an unsupported region
+    # If us-west-1 does become supported, change this test to use an unsupported region
     # or if all regions are supported, delete this test
     - name: attempt to use eks in unsupported region
       aws_eks_cluster:
@@ -13,7 +11,7 @@
         aws_access_key: "{{ aws_access_key }}"
         aws_secret_key: "{{ aws_secret_key }}"
         security_token: "{{ security_token }}"
-        region: ap-south-1
+        region: us-west-1
       register: aws_eks_unsupported_region
       ignore_errors: yes
 
@@ -142,16 +140,45 @@
           - eks_create is not changed
           - eks_create.name == eks_cluster_name
 
-    - name: remove EKS cluster
+    - name: remove EKS cluster, waiting until complete
       aws_eks_cluster:
         name: "{{ eks_cluster_name }}"
         state: absent
+        wait: yes
         <<: *aws_connection_info
       register: eks_delete
 
+    - name: check that EKS cluster was removed
+      assert:
+        that:
+          - eks_delete is changed
+
+    - name: create EKS cluster with same details but wait for it to become active
+      aws_eks_cluster:
+        name: "{{ eks_cluster_name }}"
+        security_groups: "{{ eks_security_groups | json_query('[].name') }}"
+        subnets: "{{ setup_subnets.results | json_query('[].subnet.id') }}"
+        role_arn: "{{ iam_role.arn }}"
+        wait: yes
+        <<: *aws_connection_info
+      register: eks_create
+
     - name: check that EKS cluster was created
       assert:
         that:
+          - eks_create is changed
+          - eks_create.name == eks_cluster_name
+
+    - name: remove EKS cluster, without waiting this time
+      aws_eks_cluster:
+        name: "{{ eks_cluster_name }}"
+        state: absent
+        <<: *aws_connection_info
+      register: eks_delete
+
+    - name: check that EKS cluster remove has started
+      assert:
+        that:
           - eks_delete is changed
 
   always:
@@ -163,13 +190,11 @@
       aws_eks_cluster:
         name: "{{ eks_cluster_name }}"
         state: absent
+        wait: yes
         <<: *aws_connection_info
       register: eks_delete
       ignore_errors: yes
 
-    - pause:
-        minutes: 5
-
     - debug:
         msg: "{{ eks_security_groups|reverse|list }}"
 
@@ -190,6 +215,7 @@
         vpc_id: '{{ setup_vpc.vpc.id }}'
         <<: *aws_connection_info
       with_items: "{{ eks_security_groups }}"
+      ignore_errors: yes
 
     - name: remove security groups
       ec2_group:
diff --git a/test/integration/targets/aws_eks/tasks/main.yml b/test/integration/targets/aws_eks/tasks/main.yml
new file mode 100644
index 0000000000..da65e18c66
--- /dev/null
+++ b/test/integration/targets/aws_eks/tasks/main.yml
@@ -0,0 +1,66 @@
+- set_fact:
+    virtualenv: "{{ remote_tmp_dir }}/virtualenv"
+    virtualenv_command: "{{ ansible_python_interpreter }} -m virtualenv"
+
+- set_fact:
+    virtualenv_interpreter: "{{ virtualenv }}/bin/python"
+
+- pip:
+    name: virtualenv
+
+# Test graceful failure for missing kubernetes-validate
+
+- pip:
+    name:
+      - 'botocore<1.10.1'
+      - boto3
+      - coverage
+    virtualenv: "{{ virtualenv }}"
+    virtualenv_command: "{{ virtualenv_command }}"
+    virtualenv_site_packages: no
+
+- include_tasks: botocore_lt_1.10.1.yml
+  vars:
+    ansible_python_interpreter: "{{ virtualenv_interpreter }}"
+
+- file:
+    path: "{{ virtualenv }}"
+    state: absent
+
+# Test graceful failures when botocore<1.12.38
+
+- pip:
+    name:
+      - 'botocore>1.10.1,<1.12.38'
+      - boto3
+      - coverage
+    virtualenv: "{{ virtualenv }}"
+    virtualenv_command: "{{ virtualenv_command }}"
+    virtualenv_site_packages: no
+
+- include_tasks: botocore_lt_1.12.38.yml
+  vars:
+    ansible_python_interpreter: "{{ virtualenv_interpreter }}"
+
+- file:
+    path: "{{ virtualenv }}"
+    state: absent
+
+# Test validate with kubernetes-validate
+
+- pip:
+    name:
+      - 'botocore>=1.10.1'
+      - boto3
+    virtualenv: "{{ virtualenv }}"
+    virtualenv_command: "{{ virtualenv_command }}"
+    virtualenv_site_packages: no
+
+- include_tasks: full_test.yml
+  vars:
+    ansible_python_interpreter: "{{ virtualenv_interpreter }}"
+    playbook_namespace: ansible-test-k8s-validate
+
+- file:
+    path: "{{ virtualenv }}"
+    state: absent