pbrd: Properly hook back up when vrf is destroyed than recreated

Currently when a vrf is deleted than added back in PBR was
not going through and touching up all the data structures
that needed to be massaged to allow it to start working again.

This includes:

a) Search through the nexthop groups to find any nexthop
that references the old nexthop id and set it right again.

b) Search through the nexthop cache for nht and reset
those nexthops to the right vrf as well as re-register

Signed-off-by: Stephen Worley <sworley@cumulusnetworks.com>
Signed-off-by: Donald Sharp <sharpd@cumulusnetworks.com>

6 files changed, 135 insertions, 3 deletions

diff --git a/pbrd/pbr_map.c b/pbrd/pbr_map.c
index 10a75a9f5..e37110c1c 100644
--- a/pbrd/pbr_map.c
+++ b/pbrd/pbr_map.c
@@ -771,6 +771,31 @@ void pbr_map_check_nh_group_change(const char *nh_group)
 	}
 }
 
+void pbr_map_check_vrf_nh_group_change(const char *nh_group,
+				       struct pbr_vrf *pbr_vrf,
+				       uint32_t old_vrf_id)
+{
+	struct pbr_map *pbrm;
+	struct pbr_map_sequence *pbrms;
+	struct listnode *node;
+
+	RB_FOREACH (pbrm, pbr_map_entry_head, &pbr_maps) {
+		for (ALL_LIST_ELEMENTS_RO(pbrm->seqnumbers, node, pbrms)) {
+			if (pbrms->nhgrp_name)
+				continue;
+
+			if (pbrms->nhg
+			    && strcmp(nh_group, pbrms->internal_nhg_name))
+				continue;
+
+			if (pbrms->nhg->nexthop->vrf_id != old_vrf_id)
+				continue;
+
+			pbrms->nhg->nexthop->vrf_id = pbr_vrf_id(pbr_vrf);
+		}
+	}
+}
+
 void pbr_map_check(struct pbr_map_sequence *pbrms, bool changed)
 {
 	struct pbr_map *pbrm;
diff --git a/pbrd/pbr_map.h b/pbrd/pbr_map.h
index 64c090d2e..850070911 100644
--- a/pbrd/pbr_map.h
+++ b/pbrd/pbr_map.h
@@ -205,4 +205,8 @@ extern void pbr_map_install(struct pbr_map *pbrm);
 extern void pbr_map_policy_install(const char *name);
 extern void pbr_map_policy_delete(struct pbr_map *pbrm,
 				  struct pbr_map_interface *pmi);
+
+extern void pbr_map_check_vrf_nh_group_change(const char *nh_group,
+					      struct pbr_vrf *pbr_vrf,
+					      uint32_t old_vrf_id);
 #endif
diff --git a/pbrd/pbr_nht.c b/pbrd/pbr_nht.c
index 31da65679..e875ea1ce 100644
--- a/pbrd/pbr_nht.c
+++ b/pbrd/pbr_nht.c
@@ -37,7 +37,7 @@
 
 DEFINE_MTYPE_STATIC(PBRD, PBR_NHG, "PBR Nexthop Groups")
 
-static struct hash *pbr_nhg_hash;
+struct hash *pbr_nhg_hash;
 static struct hash *pbr_nhrc_hash;
 
 static uint32_t pbr_nhg_low_table;
@@ -556,6 +556,13 @@ void pbr_nht_add_individual_nexthop(struct pbr_map_sequence *pbrms,
 	lookup.nexthop = pbrms->nhg->nexthop;
 	pnhc = hash_get(pnhgc->nhh, &lookup, pbr_nh_alloc);
 	pnhc->parent = pnhgc;
+	if (nhop->vrf_id != VRF_DEFAULT) {
+		struct vrf *vrf = vrf_lookup_by_id(nhop->vrf_id);
+
+		if (vrf)
+			strlcpy(pnhc->vrf_name, vrf->name,
+				sizeof(pnhc->vrf_name));
+	}
 	pbr_nht_install_nexthop_group(pnhgc, *pbrms->nhg);
 }
 
@@ -689,8 +696,12 @@ bool pbr_nht_nexthop_group_valid(const char *name)
 struct pbr_nht_individual {
 	struct zapi_route *nhr;
 	struct interface *ifp;
+	struct pbr_vrf *pbr_vrf;
+	struct pbr_nexthop_cache *pnhc;
 
 	uint32_t valid;
+
+	bool nhr_matched;
 };
 
 static bool
@@ -877,6 +888,65 @@ void pbr_nht_nexthop_update(struct zapi_route *nhr)
 	hash_iterate(pbr_nhg_hash, pbr_nht_nexthop_update_lookup, nhr);
 }
 
+static void pbr_nht_individual_nexthop_vrf_handle(struct hash_bucket *b,
+						  void *data)
+{
+	struct pbr_nexthop_cache *pnhc = b->data;
+	struct pbr_nht_individual *pnhi = data;
+
+	if (pnhc->nexthop->vrf_id == VRF_DEFAULT)
+		return;
+
+	if ((strncmp(pnhc->vrf_name, pbr_vrf_name(pnhi->pbr_vrf),
+		     sizeof(pnhc->vrf_name))
+	     == 0)
+	    && pnhc->nexthop->vrf_id != pbr_vrf_id(pnhi->pbr_vrf)) {
+		pnhi->pnhc = pnhc;
+		pnhi->nhr_matched = true;
+	}
+}
+
+static void pbr_nht_nexthop_vrf_handle(struct hash_bucket *b, void *data)
+{
+	struct pbr_nexthop_group_cache *pnhgc = b->data;
+	struct pbr_vrf *pbr_vrf = data;
+	struct pbr_nht_individual pnhi = {};
+	struct nhrc *nhrc;
+	uint32_t old_vrf_id;
+
+	do {
+		memset(&pnhi, 0, sizeof(pnhi));
+		pnhi.pbr_vrf = pbr_vrf;
+		hash_iterate(pnhgc->nhh, pbr_nht_individual_nexthop_vrf_handle,
+			     &pnhi);
+
+		if (!pnhi.pnhc)
+			continue;
+
+		pnhi.pnhc = hash_release(pnhgc->nhh, pnhi.pnhc);
+		old_vrf_id = pnhi.pnhc->nexthop->vrf_id;
+
+		nhrc = hash_lookup(pbr_nhrc_hash, pnhi.pnhc->nexthop);
+		if (nhrc) {
+			hash_release(pbr_nhrc_hash, nhrc);
+			nhrc->nexthop.vrf_id = pbr_vrf_id(pbr_vrf);
+			hash_get(pbr_nhrc_hash, nhrc, hash_alloc_intern);
+			pbr_send_rnh(pnhi.pnhc->nexthop, true);
+		}
+		pnhi.pnhc->nexthop->vrf_id = pbr_vrf_id(pbr_vrf);
+
+		hash_get(pnhgc->nhh, pnhi.pnhc, hash_alloc_intern);
+
+		pbr_map_check_vrf_nh_group_change(pnhgc->name, pbr_vrf,
+						  old_vrf_id);
+	} while (pnhi.pnhc);
+}
+
+void pbr_nht_vrf_update(struct pbr_vrf *pbr_vrf)
+{
+	hash_iterate(pbr_nhg_hash, pbr_nht_nexthop_vrf_handle, pbr_vrf);
+}
+
 static void
 pbr_nht_individual_nexthop_interface_update_lookup(struct hash_bucket *b,
 						   void *data)
diff --git a/pbrd/pbr_nht.h b/pbrd/pbr_nht.h
index cbcf71d2f..d1afbaa73 100644
--- a/pbrd/pbr_nht.h
+++ b/pbrd/pbr_nht.h
@@ -28,6 +28,8 @@
 
 #define PBR_NHC_NAMELEN PBR_MAP_NAMELEN + 10
 
+extern struct hash *pbr_nhg_hash;
+
 struct pbr_nexthop_group_cache {
 	char name[PBR_NHC_NAMELEN];
 
@@ -46,6 +48,8 @@ struct pbr_nexthop_group_cache {
 struct pbr_nexthop_cache {
 	struct pbr_nexthop_group_cache *parent;
 
+	char vrf_name[VRF_NAMSIZ + 1];
+
 	struct nexthop *nexthop;
 
 	bool valid;
@@ -126,4 +130,6 @@ extern void pbr_nht_nexthop_update(struct zapi_route *nhr);
 extern void pbr_nht_nexthop_interface_update(struct interface *ifp);
 
 extern void pbr_nht_init(void);
+
+extern void pbr_nht_vrf_update(struct pbr_vrf *pbr_vrf);
 #endif
diff --git a/pbrd/pbr_vrf.c b/pbrd/pbr_vrf.c
index d5a2bd0fe..389e5e8be 100644
--- a/pbrd/pbr_vrf.c
+++ b/pbrd/pbr_vrf.c
@@ -25,6 +25,7 @@
 #include "pbr_memory.h"
 #include "pbr_map.h"
 #include "pbr_debug.h"
+#include "pbr_nht.h"
 
 DEFINE_MTYPE_STATIC(PBRD, PBR_MAP_VRF, "PBR Map VRF")
 
@@ -59,6 +60,7 @@ static int pbr_vrf_enable(struct vrf *vrf)
 {
 	DEBUGD(&pbr_dbg_event, "%s: %u (%s)", __func__, vrf->vrf_id, vrf->name);
 
+	pbr_nht_vrf_update(vrf->info);
 	pbr_map_vrf_update(vrf->info);
 
 	return 0;
diff --git a/pbrd/pbr_vty.c b/pbrd/pbr_vty.c
index a73d885ea..86d088d44 100644
--- a/pbrd/pbr_vty.c
+++ b/pbrd/pbr_vty.c
@@ -618,6 +618,31 @@ DEFPY (show_pbr,
 	return CMD_SUCCESS;
 }
 
+static void
+pbrms_nexthop_group_write_individual_nexthop(
+	struct vty *vty, const struct pbr_map_sequence *pbrms)
+{
+	struct pbr_nexthop_group_cache find;
+	struct pbr_nexthop_group_cache *pnhgc;
+	struct pbr_nexthop_cache lookup;
+	struct pbr_nexthop_cache *pnhc;
+
+	nexthop_group_write_nexthop_simple(vty, pbrms->nhg->nexthop);
+
+	memset(&find, 0, sizeof(find));
+	strlcpy(find.name, pbrms->internal_nhg_name, sizeof(find.name));
+
+	pnhgc = hash_lookup(pbr_nhg_hash, &find);
+	assert(pnhgc);
+
+	lookup.nexthop = pbrms->nhg->nexthop;
+	pnhc = hash_lookup(pnhgc->nhh, &lookup);
+	if (pnhc->nexthop->vrf_id != VRF_DEFAULT)
+		vty_out(vty, " nexthop-vrf %s", pnhc->vrf_name);
+
+	vty_out(vty, "\n");
+}
+
 static void vty_show_pbrms(struct vty *vty,
 			   const struct pbr_map_sequence *pbrms, bool detail)
 {
@@ -670,7 +695,7 @@ static void vty_show_pbrms(struct vty *vty,
 
 	} else if (pbrms->nhg) {
 		vty_out(vty, "        ");
-		nexthop_group_write_nexthop(vty, pbrms->nhg->nexthop);
+		pbrms_nexthop_group_write_individual_nexthop(vty, pbrms);
 		if (detail)
 			vty_out(vty,
 				"          Installed: %u(%d) Tableid: %d\n",
@@ -1065,7 +1090,7 @@ static int pbr_vty_map_config_write_sequence(struct vty *vty,
 
 	if (pbrms->nhg) {
 		vty_out(vty, " set ");
-		nexthop_group_write_nexthop(vty, pbrms->nhg->nexthop);
+		pbrms_nexthop_group_write_individual_nexthop(vty, pbrms);
 	}
 
 	vty_out(vty, "!\n");