Reduce number of implicit meta tasks (#84007)

This greatly reduces run time on large inventories since meta tasks are
executed in the main process sequentially and just executing them is expensive.

This change avoids running the following implicit meta tasks:
  * ``flush_handlers`` on hosts where no handlers are notified
  * ``noop`` for the linear strategy's lockstep, instead hosts that are
    not executing the current task are just not part of the current host loop

A playbook consiting of two simple plays both running on ~6000 hosts
runs in:
devel: 37s
this PR: 1.3s

Co-authored-by: Sloane Hertel <19572925+s-hertel@users.noreply.github.com>
Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) <wk.cvs.github@sydorenko.org.ua>

1 files changed, 26 insertions, 108 deletions

diff --git a/test/units/plugins/strategy/test_linear.py b/test/units/plugins/strategy/test_linear.py
index 6f4ea92627..6318de33f0 100644
--- a/test/units/plugins/strategy/test_linear.py
+++ b/test/units/plugins/strategy/test_linear.py
@@ -85,16 +85,6 @@ class TestStrategyLinear(unittest.TestCase):
         strategy._hosts_cache = [h.name for h in hosts]
         strategy._hosts_cache_all = [h.name for h in hosts]
 
-        # implicit meta: flush_handlers
-        hosts_left = strategy.get_hosts_left(itr)
-        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
-        host1_task = hosts_tasks[0][1]
-        host2_task = hosts_tasks[1][1]
-        self.assertIsNotNone(host1_task)
-        self.assertIsNotNone(host2_task)
-        self.assertEqual(host1_task.action, 'meta')
-        self.assertEqual(host2_task.action, 'meta')
-
         # debug: task1, debug: task1
         hosts_left = strategy.get_hosts_left(itr)
         hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
@@ -110,69 +100,35 @@ class TestStrategyLinear(unittest.TestCase):
         # mark the second host failed
         itr.mark_host_failed(hosts[1])
 
-        # debug: task2, meta: noop
+        # debug: task2, noop
         hosts_left = strategy.get_hosts_left(itr)
         hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
-        host1_task = hosts_tasks[0][1]
-        host2_task = hosts_tasks[1][1]
-        self.assertIsNotNone(host1_task)
-        self.assertIsNotNone(host2_task)
-        self.assertEqual(host1_task.action, 'debug')
-        self.assertEqual(host2_task.action, 'meta')
-        self.assertEqual(host1_task.name, 'task2')
-        self.assertEqual(host2_task.name, '')
+        self.assertEqual(len(hosts_tasks), 1)
+        host, task = hosts_tasks[0]
+        self.assertEqual(host.name, 'host00')
+        self.assertEqual(task.action, 'debug')
+        self.assertEqual(task.name, 'task2')
 
-        # meta: noop, debug: rescue1
+        # noop, debug: rescue1
         hosts_left = strategy.get_hosts_left(itr)
         hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
-        host1_task = hosts_tasks[0][1]
-        host2_task = hosts_tasks[1][1]
-        self.assertIsNotNone(host1_task)
-        self.assertIsNotNone(host2_task)
-        self.assertEqual(host1_task.action, 'meta')
-        self.assertEqual(host2_task.action, 'debug')
-        self.assertEqual(host1_task.name, '')
-        self.assertEqual(host2_task.name, 'rescue1')
+        self.assertEqual(len(hosts_tasks), 1)
+        host, task = hosts_tasks[0]
+        self.assertEqual(host.name, 'host01')
+        self.assertEqual(task.action, 'debug')
+        self.assertEqual(task.name, 'rescue1')
 
-        # meta: noop, debug: rescue2
+        # noop, debug: rescue2
         hosts_left = strategy.get_hosts_left(itr)
         hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
-        host1_task = hosts_tasks[0][1]
-        host2_task = hosts_tasks[1][1]
-        self.assertIsNotNone(host1_task)
-        self.assertIsNotNone(host2_task)
-        self.assertEqual(host1_task.action, 'meta')
-        self.assertEqual(host2_task.action, 'debug')
-        self.assertEqual(host1_task.name, '')
-        self.assertEqual(host2_task.name, 'rescue2')
-
-        # implicit meta: flush_handlers
-        hosts_left = strategy.get_hosts_left(itr)
-        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
-        host1_task = hosts_tasks[0][1]
-        host2_task = hosts_tasks[1][1]
-        self.assertIsNotNone(host1_task)
-        self.assertIsNotNone(host2_task)
-        self.assertEqual(host1_task.action, 'meta')
-        self.assertEqual(host2_task.action, 'meta')
-
-        # implicit meta: flush_handlers
-        hosts_left = strategy.get_hosts_left(itr)
-        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
-        host1_task = hosts_tasks[0][1]
-        host2_task = hosts_tasks[1][1]
-        self.assertIsNotNone(host1_task)
-        self.assertIsNotNone(host2_task)
-        self.assertEqual(host1_task.action, 'meta')
-        self.assertEqual(host2_task.action, 'meta')
+        self.assertEqual(len(hosts_tasks), 1)
+        host, task = hosts_tasks[0]
+        self.assertEqual(host.name, 'host01')
+        self.assertEqual(task.action, 'debug')
+        self.assertEqual(task.name, 'rescue2')
 
         # end of iteration
-        hosts_left = strategy.get_hosts_left(itr)
-        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
-        host1_task = hosts_tasks[0][1]
-        host2_task = hosts_tasks[1][1]
-        self.assertIsNone(host1_task)
-        self.assertIsNone(host2_task)
+        assert not strategy._get_next_task_lockstep(strategy.get_hosts_left(itr), itr)
 
     def test_noop_64999(self):
         fake_loader = DictDataLoader({
@@ -240,16 +196,6 @@ class TestStrategyLinear(unittest.TestCase):
         strategy._hosts_cache = [h.name for h in hosts]
         strategy._hosts_cache_all = [h.name for h in hosts]
 
-        # implicit meta: flush_handlers
-        hosts_left = strategy.get_hosts_left(itr)
-        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
-        host1_task = hosts_tasks[0][1]
-        host2_task = hosts_tasks[1][1]
-        self.assertIsNotNone(host1_task)
-        self.assertIsNotNone(host2_task)
-        self.assertEqual(host1_task.action, 'meta')
-        self.assertEqual(host2_task.action, 'meta')
-
         # debug: task1, debug: task1
         hosts_left = strategy.get_hosts_left(itr)
         hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
@@ -265,17 +211,14 @@ class TestStrategyLinear(unittest.TestCase):
         # mark the second host failed
         itr.mark_host_failed(hosts[1])
 
-        # meta: noop, debug: rescue1
+        # noop, debug: rescue1
         hosts_left = strategy.get_hosts_left(itr)
         hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
-        host1_task = hosts_tasks[0][1]
-        host2_task = hosts_tasks[1][1]
-        self.assertIsNotNone(host1_task)
-        self.assertIsNotNone(host2_task)
-        self.assertEqual(host1_task.action, 'meta')
-        self.assertEqual(host2_task.action, 'debug')
-        self.assertEqual(host1_task.name, '')
-        self.assertEqual(host2_task.name, 'rescue1')
+        self.assertEqual(len(hosts_tasks), 1)
+        host, task = hosts_tasks[0]
+        self.assertEqual(host.name, 'host01')
+        self.assertEqual(task.action, 'debug')
+        self.assertEqual(task.name, 'rescue1')
 
         # debug: after_rescue1, debug: after_rescue1
         hosts_left = strategy.get_hosts_left(itr)
@@ -289,30 +232,5 @@ class TestStrategyLinear(unittest.TestCase):
         self.assertEqual(host1_task.name, 'after_rescue1')
         self.assertEqual(host2_task.name, 'after_rescue1')
 
-        # implicit meta: flush_handlers
-        hosts_left = strategy.get_hosts_left(itr)
-        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
-        host1_task = hosts_tasks[0][1]
-        host2_task = hosts_tasks[1][1]
-        self.assertIsNotNone(host1_task)
-        self.assertIsNotNone(host2_task)
-        self.assertEqual(host1_task.action, 'meta')
-        self.assertEqual(host2_task.action, 'meta')
-
-        # implicit meta: flush_handlers
-        hosts_left = strategy.get_hosts_left(itr)
-        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
-        host1_task = hosts_tasks[0][1]
-        host2_task = hosts_tasks[1][1]
-        self.assertIsNotNone(host1_task)
-        self.assertIsNotNone(host2_task)
-        self.assertEqual(host1_task.action, 'meta')
-        self.assertEqual(host2_task.action, 'meta')
-
         # end of iteration
-        hosts_left = strategy.get_hosts_left(itr)
-        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
-        host1_task = hosts_tasks[0][1]
-        host2_task = hosts_tasks[1][1]
-        self.assertIsNone(host1_task)
-        self.assertIsNone(host2_task)
+        assert not strategy._get_next_task_lockstep(strategy.get_hosts_left(itr), itr)