[#1723] exhonerate existing duplicate messages

2 files changed, 153 insertions, 5 deletions

diff --git a/tools/check-messages.py b/tools/check-messages.py
index c7e4800a9b..c2c07f4946 100755
--- a/tools/check-messages.py
+++ b/tools/check-messages.py
@@ -27,13 +27,24 @@ This script does several verifications regarding logged messages:
 
 
 def check_duplicate_occurences(occurences):
+    exhonerated = {}
+    parent_dir = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
+    with open(f'{parent_dir}/exhonerated-duplicate-messages.txt', 'r', encoding='utf-8') as f:
+        lines = f.read().splitlines()
+        for line in lines:
+            message_id = line.split('%')[1].split(':')[0].strip()
+            max_allowed = line.split(':')[1].strip()
+            exhonerated[message_id] = int(max_allowed)
+
     failure = False
     duplicate_occurences = {k: v for k, v in occurences.items() if v > 1}
-    if len(duplicate_occurences) > 1:
-        print('Duplicate occurences found:')
-        for k, v in duplicate_occurences.items():
-            print(f'    % {k}: {v}')
-        failure = True
+    for k, v in duplicate_occurences.items():
+        if k in exhonerated and v <= exhonerated[k]:
+            continue
+        if not failure:  # in other words: if first
+            print('Duplicate occurences found:')
+            failure = True
+        print(f'    % {k}: {v}')
     return failure
 
 
diff --git a/tools/exhonerated-duplicate-messages.txt b/tools/exhonerated-duplicate-messages.txt
new file mode 100644
index 0000000000..64e230f749
--- /dev/null
+++ b/tools/exhonerated-duplicate-messages.txt
@@ -0,0 +1,137 @@
+    % ALLOC_ENGINE_V6_ALLOC_HR_LEASE_EXISTS: 2
+    % DHCP4_PACKET_DROP_0007: 2
+    % DHCP4_PACKET_DROP_0011: 2
+    % DHCP4_PACKET_DROP_0012: 2
+    % DHCP6_PACKET_DROP_DUPLICATE: 2
+    % DHCP4_CLIENT_HOSTNAME_DATA: 2
+    % DHCP4_CLASSES_ASSIGNED: 3
+    % DHCP4_CLASS_ASSIGNED: 3
+    % DHCP4_PACKET_QUEUE_FULL: 2
+    % DHCP4_CONFIG_RECEIVED: 2
+    % EVAL_RESULT: 15
+    % DHCP6_CLASSES_ASSIGNED: 8
+    % DHCP6_CLASS_ASSIGNED: 3
+    % DHCP6_PACKET_QUEUE_FULL: 2
+    % DHCP6_CONFIG_RECEIVED: 2
+    % DHCPSRV_HOOK_LEASE4_SELECT_SKIP: 2
+    % DHCPSRV_HOOK_LEASE6_SELECT_SKIP: 2
+    % DHCPSRV_MYSQL_TLS_CIPHER: 2
+    % DHCPSRV_MYSQL_DELETE_ADDR: 2
+    % DHCPSRV_PGSQL_DELETE_ADDR: 2
+    % DHCPSRV_MEMFILE_DELETE_ADDR: 2
+    % EVAL_DEBUG_OPTION: 2
+    % msgid: 2
+    % FLEX_OPTION_PROCESS_ADD: 2
+    % FLEX_OPTION_PROCESS_SUPERSEDE: 2
+    % DHCP4_HOOK_BUFFER_SEND_SKIP: 2
+    % DHCP4_HOOK_SUBNET4_SELECT_PARK: 2
+    % DHCP6_HOOK_BUFFER_SEND_SKIP: 2
+    % HOSTS_CFG_CLOSE_HOST_DATA_SOURCE: 2
+    % HOSTS_CFG_GET_ALL_SUBNET_ID4_COUNT: 2
+    % HOSTS_CFG_GET_ALL_SUBNET_ID6_COUNT: 2
+    % HOSTS_CFG_GET_ALL_SUBNET_ID4: 2
+    % HOSTS_CFG_GET_ALL_SUBNET_ID6: 2
+    % HOSTS_CFG_GET_ALL_SUBNET_ID4_HOST: 2
+    % HOSTS_CFG_GET_ALL_SUBNET_ID6_HOST: 2
+    % LOG_BAD_DESTINATION: 6
+    % LOG_BAD_SEVERITY: 7
+    % LOG_READING_LOCAL_FILE: 5
+    % DHCP4_DHCP4O6_BAD_PACKET: 3
+    % HOOKS_CALLOUT_EXCEPTION: 2
+    % DHCP_DDNS_ADD_FAILED: 4
+    % DHCP_DDNS_FORWARD_ADD_BAD_DNSCLIENT_STATUS: 4
+    % DHCP_DDNS_FORWARD_ADD_BUILD_FAILURE: 4
+    % DHCP_DDNS_FORWARD_ADD_IO_ERROR: 4
+    % DHCP_DDNS_FORWARD_ADD_REJECTED: 4
+    % DHCP_DDNS_FORWARD_ADD_RESP_CORRUPT: 4
+    % DHCP_DDNS_FORWARD_ADD_TIMEOUT: 4
+    % DHCP_DDNS_FORWARD_REMOVE_ADDRS_IO_ERROR: 2
+    % DHCP_DDNS_FORWARD_REMOVE_ADDRS_REJECTED: 2
+    % DHCP_DDNS_FORWARD_REMOVE_ADDRS_RESP_CORRUPT: 2
+    % DHCP_DDNS_FORWARD_REMOVE_ADDRS_TIMEOUT: 2
+    % DHCP_DDNS_FORWARD_REMOVE_RRS_IO_ERROR: 4
+    % DHCP_DDNS_FORWARD_REMOVE_RRS_REJECTED: 4
+    % DHCP_DDNS_FORWARD_REMOVE_RRS_RESP_CORRUPT: 4
+    % DHCP_DDNS_FORWARD_REMOVE_RRS_TIMEOUT: 4
+    % DHCP_DDNS_FORWARD_REPLACE_BUILD_FAILURE: 2
+    % DHCP_DDNS_FORWARD_REPLACE_IO_ERROR: 2
+    % DHCP_DDNS_FORWARD_REPLACE_REJECTED: 2
+    % DHCP_DDNS_FORWARD_REPLACE_RESP_CORRUPT: 2
+    % DHCP_DDNS_FORWARD_REPLACE_TIMEOUT: 2
+    % DHCP_DDNS_REMOVE_FAILED: 4
+    % DHCP_DDNS_REVERSE_REMOVE_BUILD_FAILURE: 4
+    % DHCP_DDNS_REVERSE_REMOVE_IO_ERROR: 4
+    % DHCP_DDNS_REVERSE_REMOVE_REJECTED: 4
+    % DHCP_DDNS_REVERSE_REMOVE_RESP_CORRUPT: 4
+    % DHCP_DDNS_REVERSE_REMOVE_TIMEOUT: 4
+    % DHCP_DDNS_REVERSE_REPLACE_BUILD_FAILURE: 4
+    % DHCP_DDNS_REVERSE_REPLACE_IO_ERROR: 4
+    % DHCP_DDNS_REVERSE_REPLACE_REJECTED: 4
+    % DHCP_DDNS_REVERSE_REPLACE_RESP_CORRUPT: 4
+    % DHCP_DDNS_REVERSE_REPLACE_TIMEOUT: 4
+    % DHCP_DDNS_FORWARD_REMOVE_ADDRS_BAD_DNSCLIENT_STATUS: 2
+    % DHCP_DDNS_FORWARD_REMOVE_ADDRS_BUILD_FAILURE: 2
+    % DHCP_DDNS_FORWARD_REMOVE_RRS_BAD_DNSCLIENT_STATUS: 4
+    % DHCP_DDNS_FORWARD_REMOVE_RRS_BUILD_FAILURE: 4
+    % DHCP_DDNS_FORWARD_REPLACE_BAD_DNSCLIENT_STATUS: 2
+    % DHCP_DDNS_REVERSE_REMOVE_BAD_DNSCLIENT_STATUS: 4
+    % DHCP_DDNS_REVERSE_REPLACE_BAD_DNSCLIENT_STATUS: 4
+    % DHCP4_CONFIG_UNSUPPORTED_OBJECT: 2
+    % DHCP4_DB_RECONNECT_NO_DB_CTL: 3
+    % DHCP4_INIT_FAIL: 2
+    % DHCP4_PARSER_COMMIT_EXCEPTION: 3
+    % DHCP4_PARSER_COMMIT_FAIL: 5
+    % DHCP4_PARSER_EXCEPTION: 2
+    % DHCP4_PARSER_FAIL: 2
+    % DHCP6_CONFIG_UNSUPPORTED_OBJECT: 2
+    % DHCP6_DB_RECONNECT_NO_DB_CTL: 3
+    % DHCP6_INIT_FAIL: 2
+    % DHCP6_PARSER_COMMIT_EXCEPTION: 3
+    % DHCP6_PARSER_COMMIT_FAIL: 5
+    % DHCP6_PARSER_EXCEPTION: 2
+    % DHCP6_PARSER_FAIL: 2
+    % DHCP_DDNS_UNCAUGHT_NCR_RECV_HANDLER_ERROR: 2
+    % DHCP_DDNS_UNCAUGHT_NCR_SEND_HANDLER_ERROR: 2
+    % DHCP_DDNS_NCR_FLUSH_IO_ERROR: 2
+    % DHCPSRV_MYSQL_NO_TLS: 2
+    % DHCPSRV_PGSQL_NO_TLS_SUPPORT: 2
+    % DHCPSRV_UNKNOWN_DB: 3
+    % FLEX_OPTION_PROCESS_ERROR: 2
+    % HA_PAUSE_CLIENT_LISTENER_FAILED: 2
+    % LOG_BAD_STREAM: 4
+    % LOG_DUPLICATE_MESSAGE_ID: 4
+    % MYSQL_CB_RECONNECT_ATTEMPT_FAILED4: 2
+    % MYSQL_CB_RECONNECT_ATTEMPT_FAILED6: 2
+    % NETCONF_GET_CONFIG_FAILED: 4
+    % NETCONF_SET_CONFIG_FAILED: 5
+    % NETCONF_UPDATE_CONFIG_FAILED: 5
+    % NETCONF_VALIDATE_CONFIG_FAILED: 4
+    % DHCP4_PACKET_PROCESS_EXCEPTION: 5
+    % DHCP4_PACKET_PROCESS_STD_EXCEPTION: 4
+    % DHCP6_PACKET_PROCESS_EXCEPTION: 4
+    % DHCP6_PACKET_PROCESS_STD_EXCEPTION: 4
+    % PGSQL_CB_RECONNECT_ATTEMPT_FAILED4: 2
+    % PGSQL_CB_RECONNECT_ATTEMPT_FAILED6: 2
+    % DCTL_CONFIG_FILE_LOAD_FAIL: 2
+    % LOG_DUPLICATE_NAMESPACE: 2
+    % LOG_NO_MESSAGE_ID: 2
+    % LOG_NO_SUCH_MESSAGE: 2
+    % ALLOC_ENGINE_V6_HR_ADDR_GRANTED: 2
+    % ALLOC_ENGINE_V6_HR_PREFIX_GRANTED: 2
+    % DHCP_DDNS_ADD_SUCCEEDED: 4
+    % DHCP_DDNS_REMOVE_SUCCEEDED: 4
+    % DCTL_CONFIG_COMPLETE: 2
+    % DHCP4_CONFIG_PACKET_QUEUE: 2
+    % DHCP6_CONFIG_PACKET_QUEUE: 2
+    % DHCPSRV_LEASE_SANITY_FAIL_DISCARD: 2
+    % DHCPSRV_LEASE_SANITY_FIXED: 2
+    % DHCPSRV_PGSQL_TLS_SUPPORT: 2
+    % DHCP6_LEASE_REUSE: 2
+    % DHCP6_PD_LEASE_REUSE: 2
+    % LOG_READ_ERROR: 2
+    % DCTL_UNSUPPORTED_SIGNAL: 2
+    % DHCPSRV_CFGMGR_IP_RESERVATIONS_UNIQUE_DUPLICATES_DETECTED: 2
+    % DHCPSRV_DHCP4O6_RECEIVED_BAD_PACKET: 5
+    % DHCPSRV_LEASE_SANITY_FAIL: 2
+    % LEASE_CMDS_WIPE4_DEPRECATED: 2
+    % LEASE_CMDS_WIPE6_DEPRECATED: 2