tweaks to make data generator not fail on super large job event data sets

refine formatting of job event print statement
update presets.tsv file

2 files changed, 63 insertions, 26 deletions

diff --git a/tools/data_generators/presets.tsv b/tools/data_generators/presets.tsv
index c07e33fd13..ef4a0f8ffa 100644
--- a/tools/data_generators/presets.tsv
+++ b/tools/data_generators/presets.tsv
@@ -1,15 +1,15 @@
-resource	medium	Jan2017	jobs1k	jobs10k	jobs50k	jobs100k
-organizations	500	1	1	1	1	1
-users	5000	3	3	3	3	3
-teams	500	2	2	2	2	2
-projects	1000	30	30	30	30	30
-job_templates	2000	127	127	127	127	127
-credentials	2000	50	50	50	50	50
-inventories	2000	6	6	6	6	6
-inventory_groups	500	15	15	15	15	15
-inventory_hosts	2500	15	15	15	15	15
-wfjts	100	0	0	0	0	0
-nodes	1000	0	0	0	0	0
-labels	1000	0	0	0	0	0
-jobs	1000	157208	1000	10000	50000	100000
-job_events	1000	3370942	20000	200000	1000000	2000000
\ No newline at end of file
+resource	small	medium	Jan2017	jobs1k	jobs10k	jobs50k	jobs100k	jobs200k
+organizations	50	500	1	1	1	1	1	1
+users	500	5000	3	3	3	3	3	3
+teams	200	500	2	2	2	2	2	2
+projects	150	1000	30	30	30	30	30	30
+job_templates	300	2000	127	127	127	127	127	127
+credentials	150	2000	50	50	50	50	50	50
+inventories	150	2000	6	6	6	6	6	6
+inventory_groups	700	500	15	15	15	15	15	15
+inventory_hosts	1100	2500	15	15	15	15	15	15
+wfjts	50	100	0	0	0	0	0	0
+nodes	1000	1000	0	0	0	0	0	0
+labels	1000	1000	0	0	0	0	0	0
+jobs	2000	5000	157208	1000	10000	50000	100000	200000
+job_events	40000	100000	3370942	20000	200000	1000000	2000000	4000000
\ No newline at end of file
diff --git a/tools/data_generators/rbac_dummy_data_generator.py b/tools/data_generators/rbac_dummy_data_generator.py
index 5907e7a34a..b7bc2c7720 100755
--- a/tools/data_generators/rbac_dummy_data_generator.py
+++ b/tools/data_generators/rbac_dummy_data_generator.py
@@ -40,6 +40,11 @@ from django.db import transaction # noqa
 
 # awx
 from awx.main.models import * # noqa
+from awx.main.signals import ( # noqa
+    emit_update_inventory_on_created_or_deleted,
+    emit_update_inventory_computed_fields
+)
+from django.db.models.signals import post_save, post_delete, m2m_changed # noqa
 
 
 option_list = [
@@ -182,9 +187,32 @@ def mock_save(self, *args, **kwargs):
     return super(PrimordialModel, self).save(*args, **kwargs)
 
 
+def mock_computed_fields(self, **kwargs):
+    pass
+
+
 PrimordialModel.save = mock_save
 
 
+sigstat = []
+sigstat.append(post_save.disconnect(emit_update_inventory_on_created_or_deleted, sender=Host))
+sigstat.append(post_delete.disconnect(emit_update_inventory_on_created_or_deleted, sender=Host))
+sigstat.append(post_save.disconnect(emit_update_inventory_on_created_or_deleted, sender=Group))
+sigstat.append(post_delete.disconnect(emit_update_inventory_on_created_or_deleted, sender=Group))
+sigstat.append(m2m_changed.disconnect(emit_update_inventory_computed_fields, sender=Group.hosts.through))
+sigstat.append(m2m_changed.disconnect(emit_update_inventory_computed_fields, sender=Group.parents.through))
+sigstat.append(m2m_changed.disconnect(emit_update_inventory_computed_fields, sender=Host.inventory_sources.through))
+sigstat.append(m2m_changed.disconnect(emit_update_inventory_computed_fields, sender=Group.inventory_sources.through))
+sigstat.append(post_save.disconnect(emit_update_inventory_on_created_or_deleted, sender=InventorySource))
+sigstat.append(post_delete.disconnect(emit_update_inventory_on_created_or_deleted, sender=InventorySource))
+sigstat.append(post_save.disconnect(emit_update_inventory_on_created_or_deleted, sender=Job))
+sigstat.append(post_delete.disconnect(emit_update_inventory_on_created_or_deleted, sender=Job))
+
+print ' status of signal disconnects '
+print ' (True means successful disconnect)'
+print str(sigstat)
+
+
 startTime = datetime.now()
 
 
@@ -594,6 +622,8 @@ try:
             print('# Adding labels to job templates')
             jt_idx = 0
             for n in spread(n_labels * 7, n_job_templates):
+                if n == 0:
+                    continue
                 jt = job_templates[jt_idx]
                 if not jt._is_new:
                     continue
@@ -676,17 +706,24 @@ try:
                 # Check if job already has events, for idempotence
                 if not job._is_new:
                     continue
-                sys.stdout.write('\r   Creating %d job events for job %d' % (n, job.id))
-                sys.stdout.flush()
-                JobEvent.objects.bulk_create([
-                    JobEvent(
-                        created=now(),
-                        modified=now(),
-                        job=job,
-                        event='runner_on_ok'
-                    )
-                    for i in range(n)
-                ])
+                # Bulk create in chunks with maximum chunk size
+                MAX_BULK_CREATE = 100
+                for j in range((n / MAX_BULK_CREATE) + 1):
+                    n_subgroup = MAX_BULK_CREATE
+                    if j == n / MAX_BULK_CREATE:
+                        # on final pass, create the remainder
+                        n_subgroup = n % MAX_BULK_CREATE
+                    sys.stdout.write('\r   Creating %d job events for job %d, subgroup: %d' % (n, job.id, j))
+                    sys.stdout.flush()
+                    JobEvent.objects.bulk_create([
+                        JobEvent(
+                            created=now(),
+                            modified=now(),
+                            job=job,
+                            event='runner_on_ok'
+                        )
+                        for i in range(n_subgroup)
+                    ])
                 job_idx += 1
                 if n:
                     print('')