add optional module_utils import support (#73832)

* add optional module_utils import support

Treat core and collections module_utils imports nested within any Python block statement (eg, `try`, `if`) as optional. This allows Ansible modules to implement runtime fallback behavior for missing module_utils (eg from a newer version of ansible-core), where previously, the module payload builder would always fail when unable to locate a module_util (regardless of any runtime behavior the module may implement).

* sanity test fixes

ci_complete

4 files changed, 103 insertions, 8 deletions

diff --git a/test/integration/targets/module_utils/collections/ansible_collections/testns/testcoll/plugins/module_utils/legit.py b/test/integration/targets/module_utils/collections/ansible_collections/testns/testcoll/plugins/module_utils/legit.py
new file mode 100644
index 0000000000..b9d63482a1
--- /dev/null
+++ b/test/integration/targets/module_utils/collections/ansible_collections/testns/testcoll/plugins/module_utils/legit.py
@@ -0,0 +1,6 @@
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+def importme():
+    return "successfully imported from testns.testcoll"
diff --git a/test/integration/targets/module_utils/library/test_failure.py b/test/integration/targets/module_utils/library/test_failure.py
index 258217a74e..efb3ddae78 100644
--- a/test/integration/targets/module_utils/library/test_failure.py
+++ b/test/integration/targets/module_utils/library/test_failure.py
@@ -6,11 +6,9 @@ results = {}
 # Test that we are rooted correctly
 # Following files:
 #   module_utils/yak/zebra/foo.py
-try:
-    from ansible.module_utils.zebra import foo
-    results['zebra'] = foo.data
-except ImportError:
-    results['zebra'] = 'Failed in module as expected but incorrectly did not fail in AnsiballZ construction'
+from ansible.module_utils.zebra import foo
+
+results['zebra'] = foo.data
 
 from ansible.module_utils.basic import AnsibleModule
 AnsibleModule(argument_spec=dict()).exit_json(**results)
diff --git a/test/integration/targets/module_utils/library/test_optional.py b/test/integration/targets/module_utils/library/test_optional.py
new file mode 100644
index 0000000000..4d0225d96f
--- /dev/null
+++ b/test/integration/targets/module_utils/library/test_optional.py
@@ -0,0 +1,84 @@
+#!/usr/bin/python
+# Most of these names are only available via PluginLoader so pylint doesn't
+# know they exist
+# pylint: disable=no-name-in-module
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+from ansible.module_utils.basic import AnsibleModule
+
+# internal constants to keep pylint from griping about constant-valued conditionals
+_private_false = False
+_private_true = True
+
+# module_utils import statements nested below any block are considered optional "best-effort" for AnsiballZ to include.
+# test a number of different import shapes and nesting types to exercise this...
+
+# first, some nested imports that should succeed...
+try:
+    from ansible.module_utils.urls import fetch_url as yep1
+except ImportError:
+    yep1 = None
+
+try:
+    import ansible.module_utils.common.text.converters as yep2
+except ImportError:
+    yep2 = None
+
+try:
+    # optional import from a legit collection
+    from ansible_collections.testns.testcoll.plugins.module_utils.legit import importme as yep3
+except ImportError:
+    yep3 = None
+
+# and a bunch that should fail to be found, but not break the module_utils payload build in the process...
+try:
+    from ansible.module_utils.bogus import fromnope1
+except ImportError:
+    fromnope1 = None
+
+if _private_false:
+    from ansible.module_utils.alsobogus import fromnope2
+else:
+    fromnope2 = None
+
+try:
+    import ansible.module_utils.verybogus
+    nope1 = ansible.module_utils.verybogus
+except ImportError:
+    nope1 = None
+
+# deepish nested with multiple block types- make sure the AST walker made it all the way down
+try:
+    if _private_true:
+        if _private_true:
+            if _private_true:
+                if _private_true:
+                    try:
+                        import ansible.module_utils.stillbogus as nope2
+                    except ImportError:
+                        raise
+except ImportError:
+    nope2 = None
+
+try:
+    # optional import from a valid collection with an invalid package
+    from ansible_collections.testns.testcoll.plugins.module_utils.bogus import collnope1
+except ImportError:
+    collnope1 = None
+
+try:
+    # optional import from a bogus collection
+    from ansible_collections.bogusns.boguscoll.plugins.module_utils.bogus import collnope2
+except ImportError:
+    collnope2 = None
+
+module = AnsibleModule(argument_spec={})
+
+if not all([yep1, yep2, yep3]):
+    module.fail_json(msg='one or more existing optional imports did not resolve')
+
+if any([fromnope1, fromnope2, nope1, nope2, collnope1, collnope2]):
+    module.fail_json(msg='one or more missing optional imports resolved unexpectedly')
+
+module.exit_json(msg='all missing optional imports behaved as expected')
diff --git a/test/integration/targets/module_utils/module_utils_test.yml b/test/integration/targets/module_utils/module_utils_test.yml
index 2149b7e30e..96b2a9e0a6 100644
--- a/test/integration/targets/module_utils/module_utils_test.yml
+++ b/test/integration/targets/module_utils/module_utils_test.yml
@@ -43,7 +43,6 @@
     ignore_errors: True
     register: result
 
-  - debug: var=result
   - name: Make sure we failed in AnsiBallZ
     assert:
       that:
@@ -104,8 +103,16 @@
       datetime: "2020-10-05T10:05:05"
     register: datetimetest
 
-  - name:
-    assert:
+  - assert:
       that:
         - datetimetest.date == '2020-10-05'
         - datetimetest.datetime == '2020-10-05T10:05:05'
+
+  - name: Test that optional imports behave properly
+    test_optional:
+    register: optionaltest
+
+  - assert:
+      that:
+      - optionaltest is success
+      - optionaltest.msg == 'all missing optional imports behaved as expected'