topotest: improve json_cmp assert output

Create a specialized assert and json_cmp() result to improve the
comparison output. With this we also got a way to display all comparison
failures instead of just the first one.

2 files changed, 44 insertions, 8 deletions

diff --git a/tests/topotests/conftest.py b/tests/topotests/conftest.py
index a8bc53994..cea7ad448 100755
--- a/tests/topotests/conftest.py
+++ b/tests/topotests/conftest.py
@@ -3,6 +3,7 @@ Topotest conftest.py file.
 """
 
 from lib.topogen import get_topogen
+from lib.topotest import json_cmp_result
 import pytest
 
 def pytest_addoption(parser):
@@ -27,3 +28,15 @@ def pytest_runtest_call():
             tgen.mininet_cli()
 
         pytest.exit('the topology executed successfully')
+
+def pytest_assertrepr_compare(op, left, right):
+    """
+    Show proper assertion error message for json_cmp results.
+    """
+    json_result = left
+    if not isinstance(json_result, json_cmp_result):
+        json_result = right
+        if not isinstance(json_result, json_cmp_result):
+            return None
+
+    return json_result.errors
diff --git a/tests/topotests/lib/topotest.py b/tests/topotests/lib/topotest.py
index f48436675..9f540fc86 100644
--- a/tests/topotests/lib/topotest.py
+++ b/tests/topotests/lib/topotest.py
@@ -42,6 +42,20 @@ from mininet.link import Intf
 
 from time import sleep
 
+class json_cmp_result(object):
+    "json_cmp result class for better assertion messages"
+
+    def __init__(self):
+        self.errors = []
+
+    def add_error(self, error):
+        "Append error message to the result"
+        self.errors.append(error)
+
+    def has_errors(self):
+        "Returns True if there were errors, otherwise False."
+        return len(self.errors) > 0
+
 
 def json_cmp(d1, d2, reason=False):
     """
@@ -54,30 +68,39 @@ def json_cmp(d1, d2, reason=False):
 
     Note: key absence can be tested by adding a key with value `None`.
     """
-    squeue = [(d1, d2)]
+    squeue = [(d1, d2, 'json')]
+    result = json_cmp_result()
     for s in squeue:
-        nd1, nd2 = s
+        nd1, nd2, parent = s
         s1, s2 = set(nd1), set(nd2)
 
         # Expect all required fields to exist.
         s2_req = set([key for key in nd2 if nd2[key] is not None])
         diff = s2_req - s1
         if diff != set({}):
-            return 'expected keys "{}" in "{}"'.format(diff, str(nd1))
+            result.add_error('expected key(s) {} in {} (have {})'.format(
+                str(list(diff)), parent, str(list(s1))))
 
         for key in s2.intersection(s1):
             # Test for non existence of key in d2
             if nd2[key] is None:
-                return '"{}" should not exist in "{}"'.format(key, str(nd1))
+                result.add_error('"{}" should not exist in {} (have {})'.format(
+                    key, parent, str(s1)))
+                continue
             # If nd1 key is a dict, we have to recurse in it later.
             if isinstance(nd2[key], type({})):
-                squeue.append((nd1[key], nd2[key]))
+                nparent = '{}["{}"]'.format(parent, key)
+                squeue.append((nd1[key], nd2[key], nparent))
                 continue
             # Compare JSON values
             if nd1[key] != nd2[key]:
-                return '"{}" value is different ("{}" != "{}")'.format(
-                    key, str(nd1[key]), str(nd2[key])
-                )
+                result.add_error(
+                    '{}["{}"] value is different (have "{}", expected "{}")'.format(
+                        parent, key, str(nd1[key]), str(nd2[key])))
+                continue
+
+    if result.has_errors():
+        return result
 
     return None