zebra: Do not delete nhg's when retain_mode is engaged

When `-r` is specified to zebra, on shutdown we should
not remove any routes from the fib.  This was a problem
with nhg's on shutdown due to their ref-count behavior.

Introduce a methodology where on shutdown we don't mess
with the nexthop groups in the kernel.  That way on
next startup things will be ok.

Signed-off-by: Donald Sharp <sharpd@nvidia.com>

3 files changed, 35 insertions, 2 deletions

diff --git a/zebra/main.c b/zebra/main.c
index 6b6409f84..ced29e1a2 100644
--- a/zebra/main.c
+++ b/zebra/main.c
@@ -170,12 +170,14 @@ static void sigint(void)
 
 	zebra_ptm_finish();
 
-	if (retain_mode)
+	if (retain_mode) {
+		zebra_nhg_mark_keep();
 		RB_FOREACH (vrf, vrf_name_head, &vrfs_by_name) {
 			zvrf = vrf->info;
 			if (zvrf)
 				SET_FLAG(zvrf->flags, ZEBRA_VRF_RETAIN);
 		}
+	}
 	if (zrouter.lsp_process_q)
 		work_queue_free_and_null(&zrouter.lsp_process_q);
 
diff --git a/zebra/zebra_nhg.c b/zebra/zebra_nhg.c
index f1f6e7e2b..fa7bab9d2 100644
--- a/zebra/zebra_nhg.c
+++ b/zebra/zebra_nhg.c
@@ -2711,6 +2711,30 @@ void zebra_nhg_sweep_table(struct hash *hash)
 	hash_iterate(hash, zebra_nhg_sweep_entry, NULL);
 }
 
+static void zebra_nhg_mark_keep_entry(struct hash_bucket *bucket, void *arg)
+{
+	struct nhg_hash_entry *nhe = bucket->data;
+
+	UNSET_FLAG(nhe->flags, NEXTHOP_GROUP_INSTALLED);
+}
+
+/*
+ * When we are shutting down and we have retain mode enabled
+ * in zebra the process is to mark each vrf that it's
+ * routes should not be deleted.  The problem with that
+ * is that shutdown actually free's up memory which
+ * causes the nexthop group's ref counts to go to zero
+ * we need a way to subtly tell the system to not remove
+ * the nexthop groups from the kernel at the same time.
+ * The easiest just looks like that we should not mark
+ * the nhg's as installed any more and when the ref count
+ * goes to zero we'll attempt to delete and do nothing
+ */
+void zebra_nhg_mark_keep(void)
+{
+	hash_iterate(zrouter.nhgs_id, zebra_nhg_mark_keep_entry, NULL);
+}
+
 /* Global control to disable use of kernel nexthops, if available. We can't
  * force the kernel to support nexthop ids, of course, but we can disable
  * zebra's use of them, for testing e.g. By default, if the kernel supports
diff --git a/zebra/zebra_nhg.h b/zebra/zebra_nhg.h
index 052fa65d0..b2ef88bb6 100644
--- a/zebra/zebra_nhg.h
+++ b/zebra/zebra_nhg.h
@@ -324,9 +324,16 @@ struct zebra_dplane_ctx;
 extern void zebra_nhg_dplane_result(struct zebra_dplane_ctx *ctx);
 
 
-/* Sweet the nhg hash tables for old entries on restart */
+/* Sweep the nhg hash tables for old entries on restart */
 extern void zebra_nhg_sweep_table(struct hash *hash);
 
+/*
+ * We are shutting down but the nexthops should be kept
+ * as that -r has been specified and we don't want to delete
+ * the routes unintentionally
+ */
+extern void zebra_nhg_mark_keep(void);
+
 /* Nexthop resolution processing */
 struct route_entry; /* Forward ref to avoid circular includes */
 extern int nexthop_active_update(struct route_node *rn, struct route_entry *re);