bgpd: Convert from struct bgp_node to struct bgp_dest

This is based on @donaldsharp's work

The current code base is the struct bgp_node data structure.
The problem with this is that it creates a bunch of
extra data per route_node.
The table structure generates ‘holder’ nodes
that are never going to receive bgp routes,
and now the memory of those nodes is allocated
as if they are a full bgp_node.

After splitting up the bgp_node into bgp_dest and route_node,
the memory of ‘holder’ node which does not have any bgp data
will be allocated as the route_node, not the bgp_node,
and the memory usage is reduced.
The memory usage of BGP node will be reduced from 200B to 96B.
The total memory usage optimization of this part is ~16.00%.

Signed-off-by: Donald Sharp <sharpd@nvidia.com>
Signed-off-by: Yuqing Zhao <xiaopanghu99@163.com>

14 files changed, 163 insertions, 132 deletions

diff --git a/bgpd/bgp_bmp.c b/bgpd/bgp_bmp.c
index 04a70aa59..9821dcf7b 100644
--- a/bgpd/bgp_bmp.c
+++ b/bgpd/bgp_bmp.c
@@ -1307,7 +1307,7 @@ static int bmp_process(struct bgp *bgp, afi_t afi, safi_t safi,
 	if (frrtrace_enabled(frr_bgp, bmp_process)) {
 		char pfxprint[PREFIX2STR_BUFFER];
 
-		prefix2str(&bn->p, pfxprint, sizeof(pfxprint));
+		prefix2str(&bn->rn->p, pfxprint, sizeof(pfxprint));
 		frrtrace(5, frr_bgp, bmp_process, peer, pfxprint, afi, safi,
 			 withdraw);
 	}
diff --git a/bgpd/bgp_evpn.c b/bgpd/bgp_evpn.c
index 7457326c1..e5d33e6d5 100644
--- a/bgpd/bgp_evpn.c
+++ b/bgpd/bgp_evpn.c
@@ -1699,7 +1699,7 @@ static void bgp_evpn_get_sync_info(struct bgp *bgp, esi_t *esi,
 	int paths_eq;
 	struct ethaddr *tmp_mac;
 	bool mac_cmp = false;
-	struct prefix_evpn *evp = (struct prefix_evpn *)&dest->p;
+	struct prefix_evpn *evp = (struct prefix_evpn *)&dest->rn->p;
 
 
 	/* mac comparison is not needed for MAC-only routes */
@@ -2361,15 +2361,15 @@ void bgp_evpn_update_type2_route_entry(struct bgp *bgp, struct bgpevpn *vpn,
 	 * VNI table MAC-IP prefixes don't have MAC so make sure it's set from
 	 * path info here.
 	 */
-	if (is_evpn_prefix_ipaddr_none((struct prefix_evpn *)&dest->p)) {
+	if (is_evpn_prefix_ipaddr_none((struct prefix_evpn *)&dest->rn->p)) {
 		/* VNI MAC -> Global */
 		evpn_type2_prefix_global_copy(
-			&evp, (struct prefix_evpn *)&dest->p, NULL /* mac */,
+			&evp, (struct prefix_evpn *)&dest->rn->p, NULL /* mac */,
 			evpn_type2_path_info_get_ip(local_pi));
 	} else {
 		/* VNI IP -> Global */
 		evpn_type2_prefix_global_copy(
-			&evp, (struct prefix_evpn *)&dest->p,
+			&evp, (struct prefix_evpn *)&dest->rn->p,
 			evpn_type2_path_info_get_mac(local_pi), NULL /* ip */);
 	}
 
@@ -3091,7 +3091,7 @@ static int install_evpn_route_entry_in_vni_common(
 
 			if (BGP_DEBUG(evpn_mh, EVPN_MH_RT))
 				zlog_debug("VNI %d path %pFX chg to %s es",
-					   vpn->vni, &pi->net->p,
+					   vpn->vni, &pi->net->rn->p,
 					   new_local_es ? "local"
 							: "non-local");
 			bgp_path_info_set_flag(dest, pi, BGP_PATH_ATTR_CHANGED);
@@ -4174,7 +4174,7 @@ void bgp_evpn_import_type2_route(struct bgp_path_info *pi, int import)
 		return;
 
 	install_uninstall_evpn_route(bgp_evpn, AFI_L2VPN, SAFI_EVPN,
-				     &pi->net->p, pi, import);
+				     &pi->net->rn->p, pi, import);
 }
 
 /*
@@ -7259,7 +7259,7 @@ static void bgp_evpn_remote_ip_hash_add(struct bgpevpn *vpn,
 	    || !CHECK_FLAG(pi->flags, BGP_PATH_VALID))
 		return;
 
-	evp = (struct prefix_evpn *)&pi->net->p;
+	evp = (struct prefix_evpn *)&pi->net->rn->p;
 
 	if (evp->family != AF_EVPN
 	    || evp->prefix.route_type != BGP_EVPN_MAC_IP_ROUTE
@@ -7292,7 +7292,7 @@ static void bgp_evpn_remote_ip_hash_del(struct bgpevpn *vpn,
 	if (!evpn_resolve_overlay_index())
 		return;
 
-	evp = (struct prefix_evpn *)&pi->net->p;
+	evp = (struct prefix_evpn *)&pi->net->rn->p;
 
 	if (evp->family != AF_EVPN
 	    || evp->prefix.route_type != BGP_EVPN_MAC_IP_ROUTE
@@ -7337,7 +7337,7 @@ static void show_remote_ip_entry(struct hash_bucket *bucket, void *args)
 		ipaddr2str(&ip->addr, buf, sizeof(buf)));
 	vty_out(vty, "      Linked MAC/IP routes:\n");
 	for (ALL_LIST_ELEMENTS_RO(ip->macip_path_list, node, pi))
-		vty_out(vty, "        %pFX\n", &pi->net->p);
+		vty_out(vty, "        %pFX\n", &pi->net->rn->p);
 }
 
 void bgp_evpn_show_remote_ip_hash(struct hash_bucket *bucket, void *args)
diff --git a/bgpd/bgp_evpn_mh.c b/bgpd/bgp_evpn_mh.c
index 36bf752d4..91db9a061 100644
--- a/bgpd/bgp_evpn_mh.c
+++ b/bgpd/bgp_evpn_mh.c
@@ -527,7 +527,7 @@ int delete_global_ead_evi_routes(struct bgp *bgp, struct bgpevpn *vpn)
 {
 	afi_t afi;
 	safi_t safi;
-	struct bgp_dest *rdrn, *rn;
+	struct bgp_dest *rdrn, *bd;
 	struct bgp_table *table;
 	struct bgp_path_info *pi;
 
@@ -543,15 +543,15 @@ int delete_global_ead_evi_routes(struct bgp *bgp, struct bgpevpn *vpn)
 		 * Iterate over all the routes in this table and delete EAD/EVI
 		 * routes
 		 */
-		for (rn = bgp_table_top(table); rn; rn = bgp_route_next(rn)) {
-			struct prefix_evpn *evp = (struct prefix_evpn *)&rn->p;
+		for (bd = bgp_table_top(table); bd; bd = bgp_route_next(bd)) {
+			struct prefix_evpn *evp = (struct prefix_evpn *)&bd->rn->p;
 
 			if (evp->prefix.route_type != BGP_EVPN_AD_ROUTE)
 				continue;
 
-			delete_evpn_route_entry(bgp, afi, safi, rn, &pi);
+			delete_evpn_route_entry(bgp, afi, safi, bd, &pi);
 			if (pi)
-				bgp_process(bgp, rn, afi, safi);
+				bgp_process(bgp, bd, afi, safi);
 		}
 	}
 
@@ -1583,7 +1583,7 @@ static void bgp_evpn_path_es_unlink(struct bgp_path_es_info *es_info)
 	pi = es_info->pi;
 	if (BGP_DEBUG(evpn_mh, EVPN_MH_RT))
 		zlog_debug("vni %u path %pFX unlinked from es %s", es_info->vni,
-			   &pi->net->p, es->esi_str);
+			   &pi->net->rn->p, es->esi_str);
 
 	if (es_info->vni)
 		list_delete_node(es->macip_evi_path_list,
@@ -1641,7 +1641,7 @@ void bgp_evpn_path_es_link(struct bgp_path_info *pi, vni_t vni, esi_t *esi)
 	bgp_evpn_path_es_unlink(es_info);
 
 	if (BGP_DEBUG(evpn_mh, EVPN_MH_RT))
-		zlog_debug("vni %u path %pFX linked to es %s", vni, &pi->net->p,
+		zlog_debug("vni %u path %pFX linked to es %s", vni, &pi->net->rn->p,
 			   es->esi_str);
 
 	/* link mac-ip path to the new destination ES */
@@ -1661,7 +1661,7 @@ static bool bgp_evpn_is_macip_path(struct bgp_path_info *pi)
 	 * skipped) as these lists are maintained for managing
 	 * host routes in the tenant VRF
 	 */
-	evp = (struct prefix_evpn *)&pi->net->p;
+	evp = (struct prefix_evpn *)&pi->net->rn->p;
 	return is_evpn_prefix_ipaddr_v4(evp) || is_evpn_prefix_ipaddr_v6(evp);
 }
 
@@ -1697,7 +1697,7 @@ bgp_evpn_es_path_update_on_es_vrf_chg(struct bgp_evpn_es_vrf *es_vrf,
 		if (BGP_DEBUG(evpn_mh, EVPN_MH_RT))
 			zlog_debug(
 				"update path %pFX linked to es %s on vrf chg",
-				&pi->net->p, es->esi_str);
+				&pi->net->rn->p, es->esi_str);
 		bgp_evpn_route_entry_install_if_vrf_match(es_vrf->bgp_vrf, pi,
 							  1);
 	}
@@ -2086,7 +2086,7 @@ static void bgp_evpn_mac_update_on_es_oper_chg(struct bgp_evpn_es *es)
 		if (BGP_DEBUG(evpn_mh, EVPN_MH_RT))
 			zlog_debug(
 				"update path %d %pFX linked to es %s on oper chg",
-				es_info->vni, &pi->net->p, es->esi_str);
+				es_info->vni, &pi->net->rn->p, es->esi_str);
 
 		bgp_evpn_update_type2_route_entry(bgp, vpn, pi->net, pi,
 						  __func__);
@@ -2135,7 +2135,7 @@ static void bgp_evpn_mac_update_on_es_local_chg(struct bgp_evpn_es *es,
 		if (BGP_DEBUG(evpn_mh, EVPN_MH_RT))
 			zlog_debug(
 				"update path %pFX linked to es %s on chg to %s",
-				&pi->net->p, es->esi_str,
+				&pi->net->rn->p, es->esi_str,
 				is_local ? "local" : "non-local");
 
 		attr_tmp = *pi->attr;
@@ -3160,7 +3160,7 @@ bool bgp_evpn_path_es_use_nhg(struct bgp *bgp_vrf, struct bgp_path_info *pi,
 	esi_t *esi;
 	struct bgp_evpn_es_vrf *es_vrf = NULL;
 	struct bgp_path_info *parent_pi;
-	struct bgp_node *rn;
+	struct bgp_dest *bd;
 	struct prefix_evpn *evp;
 	struct bgp_path_info *mpinfo;
 	bool use_l3nhg = false;
@@ -3176,11 +3176,11 @@ bool bgp_evpn_path_es_use_nhg(struct bgp *bgp_vrf, struct bgp_path_info *pi,
 	if (!parent_pi)
 		return false;
 
-	rn = parent_pi->net;
-	if (!rn)
+	bd = parent_pi->net;
+	if (!bd)
 		return false;
 
-	evp = (struct prefix_evpn *)&rn->p;
+	evp = (struct prefix_evpn *)&bd->rn->p;
 	if (evp->prefix.route_type != BGP_EVPN_MAC_IP_ROUTE)
 		return false;
 
@@ -4706,7 +4706,7 @@ static void bgp_evpn_path_nh_unlink(struct bgp_path_evpn_nh_info *nh_info)
 	pi = nh_info->pi;
 	if (BGP_DEBUG(evpn_mh, EVPN_MH_RT))
 		zlog_debug("path %s unlinked from nh %s %s",
-			   pi->net ? prefix2str(&pi->net->p, prefix_buf,
+			   pi->net ? prefix2str(&pi->net->rn->p, prefix_buf,
 						sizeof(prefix_buf))
 				   : "",
 			   nh->bgp_vrf->name_pretty, nh->nh_str);
@@ -4741,7 +4741,7 @@ static void bgp_evpn_path_nh_link(struct bgp *bgp_vrf, struct bgp_path_info *pi)
 	if (!bgp_vrf->evpn_nh_table) {
 		if (BGP_DEBUG(evpn_mh, EVPN_MH_RT))
 			zlog_debug("path %pFX linked to vrf %s failed",
-				   &pi->net->p, bgp_vrf->name_pretty);
+				   &pi->net->rn->p, bgp_vrf->name_pretty);
 		return;
 	}
 
@@ -4764,7 +4764,7 @@ static void bgp_evpn_path_nh_link(struct bgp *bgp_vrf, struct bgp_path_info *pi)
 
 	/* find-create nh */
 	memset(&ip, 0, sizeof(ip));
-	if (pi->net->p.family == AF_INET6) {
+	if (pi->net->rn->p.family == AF_INET6) {
 		SET_IPADDR_V6(&ip);
 		memcpy(&ip.ipaddr_v6, &pi->attr->mp_nexthop_global,
 		       sizeof(ip.ipaddr_v6));
@@ -4788,7 +4788,7 @@ static void bgp_evpn_path_nh_link(struct bgp *bgp_vrf, struct bgp_path_info *pi)
 	bgp_evpn_path_nh_unlink(nh_info);
 
 	if (BGP_DEBUG(evpn_mh, EVPN_MH_RT))
-		zlog_debug("path %pFX linked to nh %s %s", &pi->net->p,
+		zlog_debug("path %pFX linked to nh %s %s", &pi->net->rn->p,
 			   nh->bgp_vrf->name_pretty, nh->nh_str);
 
 	/* link mac-ip path to the new nh */
@@ -4803,7 +4803,7 @@ static void bgp_evpn_path_nh_link(struct bgp *bgp_vrf, struct bgp_path_info *pi)
 		if (!nh->ref_pi)
 			zlog_debug(
 				"path %pFX linked to nh %s %s with no valid pi",
-				&pi->net->p, nh->bgp_vrf->name_pretty,
+				&pi->net->rn->p, nh->bgp_vrf->name_pretty,
 				nh->nh_str);
 	}
 }
@@ -4840,7 +4840,7 @@ static void bgp_evpn_nh_show_entry(struct bgp_evpn_nh *nh, struct vty *vty,
 
 	prefix_mac2str(&nh->rmac, mac_buf, sizeof(mac_buf));
 	if (nh->ref_pi && nh->ref_pi->net)
-		prefix2str(&nh->ref_pi->net->p, prefix_buf, sizeof(prefix_buf));
+		prefix2str(&nh->ref_pi->net->rn->p, prefix_buf, sizeof(prefix_buf));
 	else
 		prefix_buf[0] = '\0';
 	if (json) {
diff --git a/bgpd/bgp_evpn_private.h b/bgpd/bgp_evpn_private.h
index f141ddd6e..5af99afa3 100644
--- a/bgpd/bgp_evpn_private.h
+++ b/bgpd/bgp_evpn_private.h
@@ -750,7 +750,7 @@ bgp_evpn_vni_node_lookup(const struct bgpevpn *vpn, const struct prefix_evpn *p,
 extern void bgp_evpn_import_route_in_vrfs(struct bgp_path_info *pi, int import);
 extern void bgp_evpn_update_type2_route_entry(struct bgp *bgp,
 					      struct bgpevpn *vpn,
-					      struct bgp_node *rn,
+					      struct bgp_dest *rn,
 					      struct bgp_path_info *local_pi,
 					      const char *caller);
 extern int bgp_evpn_route_entry_install_if_vrf_match(struct bgp *bgp_vrf,
diff --git a/bgpd/bgp_evpn_vty.c b/bgpd/bgp_evpn_vty.c
index 2f158ab1b..e4c7fdb12 100644
--- a/bgpd/bgp_evpn_vty.c
+++ b/bgpd/bgp_evpn_vty.c
@@ -714,7 +714,7 @@ static void bgp_evpn_show_routes_mac_ip_es(struct vty *vty, esi_t *esi,
 					   json_object *json, int detail,
 					   bool global_table)
 {
-	struct bgp_node *rn;
+	struct bgp_dest *bd;
 	struct bgp_path_info *pi;
 	int header = detail ? 0 : 1;
 	uint32_t path_cnt;
@@ -747,7 +747,7 @@ static void bgp_evpn_show_routes_mac_ip_es(struct vty *vty, esi_t *esi,
 			json_object *json_path = NULL;
 
 			pi = es_info->pi;
-			rn = pi->net;
+			bd = pi->net;
 
 			if (!CHECK_FLAG(pi->flags, BGP_PATH_VALID))
 				continue;
@@ -765,11 +765,11 @@ static void bgp_evpn_show_routes_mac_ip_es(struct vty *vty, esi_t *esi,
 
 			if (detail)
 				route_vty_out_detail(
-					vty, bgp, rn, bgp_dest_get_prefix(rn),
+					vty, bgp, bd, bgp_dest_get_prefix(bd),
 					pi, AFI_L2VPN, SAFI_EVPN,
 					RPKI_NOT_BEING_USED, json_path);
 			else
-				route_vty_out(vty, &rn->p, pi, 0, SAFI_EVPN,
+				route_vty_out(vty, &bd->rn->p, pi, 0, SAFI_EVPN,
 					      json_path, false);
 
 			if (json)
diff --git a/bgpd/bgp_fsm.c b/bgpd/bgp_fsm.c
index 63e9fa7bc..8241a9f82 100644
--- a/bgpd/bgp_fsm.c
+++ b/bgpd/bgp_fsm.c
@@ -723,7 +723,7 @@ static void bgp_set_llgr_stale(struct peer *peer, afi_t afi, safi_t safi)
 					if (bgp_debug_neighbor_events(peer))
 						zlog_debug(
 							"%pBP Long-lived set stale community (LLGR_STALE) for: %pFX",
-							peer, &dest->p);
+							peer, &dest->rn->p);
 
 					attr = *pi->attr;
 					bgp_attr_add_llgr_community(&attr);
@@ -751,7 +751,7 @@ static void bgp_set_llgr_stale(struct peer *peer, afi_t afi, safi_t safi)
 				if (bgp_debug_neighbor_events(peer))
 					zlog_debug(
 						"%pBP Long-lived set stale community (LLGR_STALE) for: %pFX",
-						peer, &dest->p);
+						peer, &dest->rn->p);
 
 				attr = *pi->attr;
 				bgp_attr_add_llgr_community(&attr);
diff --git a/bgpd/bgp_route.c b/bgpd/bgp_route.c
index a91292d97..d074792aa 100644
--- a/bgpd/bgp_route.c
+++ b/bgpd/bgp_route.c
@@ -77,7 +77,7 @@
 #include "bgpd/bgp_route_clippy.c"
 
 DEFINE_HOOK(bgp_snmp_update_stats,
-	    (struct bgp_node *rn, struct bgp_path_info *pi, bool added),
+	    (struct bgp_dest *rn, struct bgp_path_info *pi, bool added),
 	    (rn, pi, added));
 
 DEFINE_HOOK(bgp_rpki_prefix_status,
@@ -265,7 +265,7 @@ struct bgp_path_info_extra *bgp_path_info_extra_get(struct bgp_path_info *pi)
 {
 	if (!pi->extra)
 		pi->extra = bgp_path_info_extra_new();
-	if (!pi->extra->evpn && pi->net && pi->net->p.family == AF_EVPN)
+	if (!pi->extra->evpn && pi->net && pi->net->rn->p.family == AF_EVPN)
 		pi->extra->evpn =
 			XCALLOC(MTYPE_BGP_ROUTE_EXTRA_EVPN,
 				sizeof(struct bgp_path_info_extra_evpn));
diff --git a/bgpd/bgp_route.h b/bgpd/bgp_route.h
index f424e6d28..e001bf4f0 100644
--- a/bgpd/bgp_route.h
+++ b/bgpd/bgp_route.h
@@ -670,7 +670,7 @@ static inline bool bgp_check_withdrawal(struct bgp *bgp, struct bgp_dest *dest,
 
 /* called before bgp_process() */
 DECLARE_HOOK(bgp_process,
-	     (struct bgp * bgp, afi_t afi, safi_t safi, struct bgp_dest *bn,
+	     (struct bgp *bgp, afi_t afi, safi_t safi, struct bgp_dest *bn,
 	      struct peer *peer, bool withdraw),
 	     (bgp, afi, safi, bn, peer, withdraw));
 
diff --git a/bgpd/bgp_table.c b/bgpd/bgp_table.c
index cb658c030..149a5b914 100644
--- a/bgpd/bgp_table.c
+++ b/bgpd/bgp_table.c
@@ -47,16 +47,6 @@ void bgp_table_finish(struct bgp_table **rt)
 }
 
 /*
- * bgp_dest_unlock_node
- */
-void bgp_dest_unlock_node(struct bgp_dest *dest)
-{
-	frrtrace(1, frr_bgp, bgp_dest_unlock, dest);
-	bgp_delete_listnode(dest);
-	route_unlock_node(bgp_dest_to_rnode(dest));
-}
-
-/*
  * bgp_dest_lock_node
  */
 struct bgp_dest *bgp_dest_lock_node(struct bgp_dest *dest)
@@ -83,44 +73,56 @@ const char *bgp_dest_get_prefix_str(struct bgp_dest *dest)
 }
 
 /*
- * bgp_node_create
+ * bgp_dest_unlock_node
  */
-static struct route_node *bgp_node_create(route_table_delegate_t *delegate,
-					  struct route_table *table)
+inline void bgp_dest_unlock_node(struct bgp_dest *dest)
 {
-	struct bgp_node *node;
-	node = XCALLOC(MTYPE_BGP_NODE, sizeof(struct bgp_node));
-
-	RB_INIT(bgp_adj_out_rb, &node->adj_out);
-	return bgp_dest_to_rnode(node);
+	frrtrace(1, frr_bgp, bgp_dest_unlock, dest);
+	bgp_delete_listnode(dest);
+	struct route_node *rn = bgp_dest_to_rnode(dest);
+
+	if (rn->lock == 1) {
+		struct bgp_table *rt = bgp_dest_table(dest);
+		if (rt->bgp) {
+			bgp_addpath_free_node_data(&rt->bgp->tx_addpath,
+						   &dest->tx_addpath, rt->afi,
+						   rt->safi);
+		}
+		XFREE(MTYPE_BGP_NODE, dest);
+		rn->info = NULL;
+	}
+	route_unlock_node(rn);
 }
 
 /*
  * bgp_node_destroy
  */
 static void bgp_node_destroy(route_table_delegate_t *delegate,
-			     struct route_table *table, struct route_node *node)
+							struct route_table *table, struct route_node *node)
 {
-	struct bgp_node *bgp_node;
+	struct bgp_dest *dest;
 	struct bgp_table *rt;
-	bgp_node = bgp_dest_from_rnode(node);
+	dest = bgp_dest_from_rnode(node);
 	rt = table->info;
-
-	if (rt->bgp) {
-		bgp_addpath_free_node_data(&rt->bgp->tx_addpath,
-					 &bgp_node->tx_addpath,
-					 rt->afi, rt->safi);
+	if (dest) {
+		if (rt->bgp) {
+			bgp_addpath_free_node_data(&rt->bgp->tx_addpath,
+										&dest->tx_addpath,
+										rt->afi, rt->safi);
+		}
+		XFREE(MTYPE_BGP_NODE, dest);
+		node->info = NULL;
 	}
 
-	XFREE(MTYPE_BGP_NODE, bgp_node);
+	XFREE(MTYPE_ROUTE_NODE, node);
 }
 
 /*
  * Function vector to customize the behavior of the route table
  * library for BGP route tables.
  */
-route_table_delegate_t bgp_table_delegate = {.create_node = bgp_node_create,
-					     .destroy_node = bgp_node_destroy};
+route_table_delegate_t bgp_table_delegate = { .create_node = route_node_create,
+					      .destroy_node = bgp_node_destroy };
 
 /*
  * bgp_table_init
@@ -151,9 +153,9 @@ struct bgp_table *bgp_table_init(struct bgp *bgp, afi_t afi, safi_t safi)
 }
 
 /* Delete the route node from the selection deferral route list */
-void bgp_delete_listnode(struct bgp_node *node)
+void bgp_delete_listnode(struct bgp_dest *dest)
 {
-	struct route_node *rn = NULL;
+	const struct route_node *rn = NULL;
 	struct bgp_table *table = NULL;
 	struct bgp *bgp = NULL;
 	afi_t afi;
@@ -162,8 +164,8 @@ void bgp_delete_listnode(struct bgp_node *node)
 	/* If the route to be deleted is selection pending, update the
 	 * route node in gr_info
 	 */
-	if (CHECK_FLAG(node->flags, BGP_NODE_SELECT_DEFER)) {
-		table = bgp_dest_table(node);
+	if (CHECK_FLAG(dest->flags, BGP_NODE_SELECT_DEFER)) {
+		table = bgp_dest_table(dest);
 
 		if (table) {
 			bgp = table->bgp;
@@ -172,47 +174,48 @@ void bgp_delete_listnode(struct bgp_node *node)
 		} else
 			return;
 
-		rn = bgp_dest_to_rnode(node);
+		rn = bgp_dest_to_rnode(dest);
 
 		if (bgp && rn && rn->lock == 1) {
 			/* Delete the route from the selection pending list */
 			bgp->gr_info[afi][safi].gr_deferred--;
-			UNSET_FLAG(node->flags, BGP_NODE_SELECT_DEFER);
+			UNSET_FLAG(dest->flags, BGP_NODE_SELECT_DEFER);
 		}
 	}
 }
 
-struct bgp_node *bgp_table_subtree_lookup(const struct bgp_table *table,
+struct bgp_dest *bgp_table_subtree_lookup(const struct bgp_table *table,
 					  const struct prefix *p)
 {
-	struct bgp_node *node = bgp_dest_from_rnode(table->route_table->top);
-	struct bgp_node *matched = NULL;
+	struct bgp_dest *dest = bgp_dest_from_rnode(table->route_table->top);
+	struct bgp_dest *matched = NULL;
 
-	if (node == NULL)
+	if (dest == NULL)
 		return NULL;
 
 
-	while (node) {
-		const struct prefix *node_p = bgp_dest_get_prefix(node);
+	while (dest) {
+		const struct prefix *dest_p = bgp_dest_get_prefix(dest);
+		struct route_node *node = dest->rn;
 
-		if (node_p->prefixlen >= p->prefixlen) {
-			if (!prefix_match(p, node_p))
+		if (dest_p->prefixlen >= p->prefixlen) {
+			if (!prefix_match(p, dest_p))
 				return NULL;
 
-			matched = node;
+			matched = dest;
 			break;
 		}
 
-		if (!prefix_match(node_p, p))
+		if (!prefix_match(dest_p, p))
 			return NULL;
 
-		if (node_p->prefixlen == p->prefixlen) {
-			matched = node;
+		if (dest_p->prefixlen == p->prefixlen) {
+			matched = dest;
 			break;
 		}
 
-		node = bgp_dest_from_rnode(node->link[prefix_bit(
-			&p->u.prefix, node_p->prefixlen)]);
+		dest = bgp_dest_from_rnode(
+			node->link[prefix_bit(&p->u.prefix, dest_p->prefixlen)]);
 	}
 
 	if (!matched)
diff --git a/bgpd/bgp_table.h b/bgpd/bgp_table.h
index 91941315f..1f28624c1 100644
--- a/bgpd/bgp_table.h
+++ b/bgpd/bgp_table.h
@@ -6,10 +6,6 @@
 #ifndef _QUAGGA_BGP_TABLE_H
 #define _QUAGGA_BGP_TABLE_H
 
-/* XXX BEGIN TEMPORARY COMPAT */
-#define bgp_dest bgp_node
-/* XXX END TEMPORARY COMPAT */
-
 #include "mpls.h"
 #include "table.h"
 #include "queue.h"
@@ -67,16 +63,10 @@ enum bgp_path_selection_reason {
 	bgp_path_selection_default,
 };
 
-struct bgp_node {
-	/*
-	 * CAUTION
-	 *
-	 * These fields must be the very first fields in this structure.
-	 *
-	 * @see bgp_node_to_rnode
-	 * @see bgp_node_from_rnode
-	 */
-	ROUTE_NODE_FIELDS
+struct bgp_dest {
+	struct route_node *rn;
+
+	void *info;
 
 	struct bgp_adj_out_rb adj_out;
 
@@ -134,7 +124,7 @@ extern const char *bgp_dest_get_prefix_str(struct bgp_dest *dest);
  */
 static inline struct bgp_dest *bgp_dest_from_rnode(struct route_node *rnode)
 {
-	return (struct bgp_dest *)rnode;
+	return (rnode && rnode->info) ? (struct bgp_dest *)rnode->info : NULL;
 }
 
 /*
@@ -144,7 +134,7 @@ static inline struct bgp_dest *bgp_dest_from_rnode(struct route_node *rnode)
  */
 static inline struct route_node *bgp_dest_to_rnode(const struct bgp_dest *dest)
 {
-	return (struct route_node *)dest;
+	return dest ? dest->rn : NULL;
 }
 
 /*
@@ -166,6 +156,9 @@ static inline struct bgp_dest *bgp_dest_parent_nolock(struct bgp_dest *dest)
 {
 	struct route_node *rn = bgp_dest_to_rnode(dest)->parent;
 
+	while (rn && !rn->info)
+		rn = rn->parent;
+
 	return bgp_dest_from_rnode(rn);
 }
 
@@ -179,7 +172,17 @@ static inline struct bgp_dest *bgp_dest_parent_nolock(struct bgp_dest *dest)
 static inline struct bgp_dest *
 bgp_table_top_nolock(const struct bgp_table *const table)
 {
-	return bgp_dest_from_rnode(table->route_table->top);
+	struct route_node *top;
+	struct route_node *rn = top = table->route_table->top;
+
+	while (rn && !rn->info) {
+		if (rn == top)
+			route_lock_node(rn);
+		rn = route_next(rn);
+	}
+	if (rn && rn != top)
+		route_unlock_node(rn);
+	return rn ? rn->info : NULL;
 }
 
 /*
@@ -188,7 +191,11 @@ bgp_table_top_nolock(const struct bgp_table *const table)
 static inline struct bgp_dest *
 bgp_table_top(const struct bgp_table *const table)
 {
-	return bgp_dest_from_rnode(route_top(table->route_table));
+	struct route_node *rn = route_top(table->route_table);
+
+	while (rn && !rn->info)
+		rn = route_next(rn);
+	return rn ? rn->info : NULL;
 }
 
 /*
@@ -196,7 +203,11 @@ bgp_table_top(const struct bgp_table *const table)
  */
 static inline struct bgp_dest *bgp_route_next(struct bgp_dest *dest)
 {
-	return bgp_dest_from_rnode(route_next(bgp_dest_to_rnode(dest)));
+	struct route_node *rn = route_next(bgp_dest_to_rnode(dest));
+
+	while (rn && !rn->info)
+		rn = route_next(rn);
+	return bgp_dest_from_rnode(rn);
 }
 
 /*
@@ -210,6 +221,9 @@ static inline struct bgp_dest *bgp_route_next_until(struct bgp_dest *dest,
 	rnode = route_next_until(bgp_dest_to_rnode(dest),
 			bgp_dest_to_rnode(limit));
 
+	while (rnode && !rnode->info)
+		rnode = route_next_until(rnode, bgp_dest_to_rnode(limit));
+
 	return bgp_dest_from_rnode(rnode);
 }
 
@@ -219,7 +233,17 @@ static inline struct bgp_dest *bgp_route_next_until(struct bgp_dest *dest,
 static inline struct bgp_dest *bgp_node_get(struct bgp_table *const table,
 					    const struct prefix *p)
 {
-	return bgp_dest_from_rnode(route_node_get(table->route_table, p));
+	struct route_node *rn = route_node_get(table->route_table, p);
+
+	if (!rn->info) {
+		struct bgp_dest *dest = XCALLOC(MTYPE_BGP_NODE,
+						sizeof(struct bgp_dest));
+
+		RB_INIT(bgp_adj_out_rb, &dest->adj_out);
+		rn->info = dest;
+		dest->rn = rn;
+	}
+	return rn->info;
 }
 
 /*
@@ -255,7 +279,11 @@ static inline unsigned long bgp_table_count(const struct bgp_table *const table)
 static inline struct bgp_dest *bgp_table_get_next(const struct bgp_table *table,
 						  const struct prefix *p)
 {
-	return bgp_dest_from_rnode(route_table_get_next(table->route_table, p));
+	struct route_node *rn = route_table_get_next(table->route_table, p);
+
+	while (rn && !rn->info)
+		rn = route_next(rn);
+	return bgp_dest_from_rnode(rn);
 }
 
 /* This would benefit from a real atomic operation...
@@ -357,7 +385,7 @@ static inline void bgp_dest_set_bgp_path_info(struct bgp_dest *dest,
 static inline struct bgp_table *
 bgp_dest_get_bgp_table_info(struct bgp_dest *dest)
 {
-	return dest->info;
+	return dest ? dest->info : NULL;
 }
 
 static inline void bgp_dest_set_bgp_table_info(struct bgp_dest *dest,
@@ -368,17 +396,17 @@ static inline void bgp_dest_set_bgp_table_info(struct bgp_dest *dest,
 
 static inline bool bgp_dest_has_bgp_path_info_data(struct bgp_dest *dest)
 {
-	return !!dest->info;
+	return dest ? !!dest->info : false;
 }
 
 static inline const struct prefix *bgp_dest_get_prefix(const struct bgp_dest *dest)
 {
-	return &dest->p;
+	return dest ? &dest->rn->p : NULL;
 }
 
 static inline unsigned int bgp_dest_get_lock_count(const struct bgp_dest *dest)
 {
-	return dest->lock;
+	return dest ? dest->rn->lock : 0;
 }
 
 #ifdef _FRR_ATTRIBUTE_PRINTFRR
diff --git a/bgpd/bgp_zebra.c b/bgpd/bgp_zebra.c
index 21ccbebe6..d8204d0f3 100644
--- a/bgpd/bgp_zebra.c
+++ b/bgpd/bgp_zebra.c
@@ -2658,7 +2658,7 @@ static int bgp_zebra_route_notify_owner(int command, struct zclient *zclient,
 		UNSET_FLAG(dest->flags, BGP_NODE_FIB_INSTALL_PENDING);
 		SET_FLAG(dest->flags, BGP_NODE_FIB_INSTALLED);
 		if (BGP_DEBUG(zebra, ZEBRA))
-			zlog_debug("route %pRN : INSTALLED", dest);
+			zlog_debug("route %pRN : INSTALLED", (void *)dest);
 		/* Find the best route */
 		for (pi = dest->info; pi; pi = pi->next) {
 			/* Process aggregate route */
@@ -2671,7 +2671,7 @@ static int bgp_zebra_route_notify_owner(int command, struct zclient *zclient,
 			group_announce_route(bgp, afi, safi, dest, new_select);
 		else {
 			flog_err(EC_BGP_INVALID_ROUTE,
-				 "selected route %pRN not found", dest);
+				 "selected route %pRN not found", (void *)dest);
 
 			bgp_dest_unlock_node(dest);
 			return -1;
@@ -2684,13 +2684,13 @@ static int bgp_zebra_route_notify_owner(int command, struct zclient *zclient,
 		 */
 		UNSET_FLAG(dest->flags, BGP_NODE_FIB_INSTALLED);
 		if (BGP_DEBUG(zebra, ZEBRA))
-			zlog_debug("route %pRN: Removed from Fib", dest);
+			zlog_debug("route %pRN: Removed from Fib", (void *)dest);
 		break;
 	case ZAPI_ROUTE_FAIL_INSTALL:
 		new_select = NULL;
 		if (BGP_DEBUG(zebra, ZEBRA))
 			zlog_debug("route: %pRN Failed to Install into Fib",
-				   dest);
+				   (void *)dest);
 		UNSET_FLAG(dest->flags, BGP_NODE_FIB_INSTALL_PENDING);
 		UNSET_FLAG(dest->flags, BGP_NODE_FIB_INSTALLED);
 		for (pi = bgp_dest_get_bgp_path_info(dest); pi; pi = pi->next) {
@@ -2704,7 +2704,7 @@ static int bgp_zebra_route_notify_owner(int command, struct zclient *zclient,
 	case ZAPI_ROUTE_BETTER_ADMIN_WON:
 		if (BGP_DEBUG(zebra, ZEBRA))
 			zlog_debug("route: %pRN removed due to better admin won",
-				   dest);
+				   (void *)dest);
 		new_select = NULL;
 		UNSET_FLAG(dest->flags, BGP_NODE_FIB_INSTALL_PENDING);
 		UNSET_FLAG(dest->flags, BGP_NODE_FIB_INSTALLED);
@@ -2719,7 +2719,7 @@ static int bgp_zebra_route_notify_owner(int command, struct zclient *zclient,
 		break;
 	case ZAPI_ROUTE_REMOVE_FAIL:
 		zlog_warn("%s: Route %pRN failure to remove",
-			  __func__, dest);
+			  __func__, (void *)dest);
 		break;
 	}
 
diff --git a/bgpd/bgpd.h b/bgpd/bgpd.h
index 51e0cb380..1b05fbfdb 100644
--- a/bgpd/bgpd.h
+++ b/bgpd/bgpd.h
@@ -2692,7 +2692,7 @@ DECLARE_HOOK(peer_status_changed, (struct peer *peer), (peer));
 DECLARE_HOOK(bgp_snmp_init_stats, (struct bgp *bgp), (bgp));
 DECLARE_HOOK(bgp_snmp_update_last_changed, (struct bgp *bgp), (bgp));
 DECLARE_HOOK(bgp_snmp_update_stats,
-	     (struct bgp_node *rn, struct bgp_path_info *pi, bool added),
+	     (struct bgp_dest *rn, struct bgp_path_info *pi, bool added),
 	     (rn, pi, added));
 DECLARE_HOOK(bgp_rpki_prefix_status,
 	     (struct peer * peer, struct attr *attr,
diff --git a/lib/table.c b/lib/table.c
index 380852b54..3bf93894e 100644
--- a/lib/table.c
+++ b/lib/table.c
@@ -334,7 +334,6 @@ void route_node_delete(struct route_node *node)
 	struct route_node *parent;
 
 	assert(node->lock == 0);
-	assert(node->info == NULL);
 
 	if (node->l_left && node->l_right)
 		return;
diff --git a/tests/bgpd/test_mpath.c b/tests/bgpd/test_mpath.c
index 0124ad9b2..bc2c1e8a4 100644
--- a/tests/bgpd/test_mpath.c
+++ b/tests/bgpd/test_mpath.c
@@ -244,6 +244,7 @@ static int run_bgp_mp_list(testcase_t *t)
 	for (i = 0, mp_node = mp_list.head; i < test_mp_list_info_count;
 	     i++, mp_node = listnextnode(mp_node)) {
 		info = listgetdata(mp_node);
+		info->lock++;
 		EXPECT_TRUE(info == &test_mp_list_info[i], test_result);
 	}
 
@@ -274,14 +275,14 @@ testcase_t test_bgp_mp_list = {
  * Testcase for bgp_path_info_mpath_update
  */
 
-struct bgp_node test_rn;
+static struct bgp_dest *dest;
 
 static int setup_bgp_path_info_mpath_update(testcase_t *t)
 {
 	int i;
 	struct bgp *bgp;
 	struct bgp_table *rt;
-	struct route_node *rt_node;
+	struct prefix p;
 	as_t asn = 1;
 
 	t->tmp_data = bgp_create_fake(&asn, NULL);
@@ -294,13 +295,12 @@ static int setup_bgp_path_info_mpath_update(testcase_t *t)
 	if (!rt)
 		return -1;
 
-	str2prefix("42.1.1.0/24", &test_rn.p);
-	rt_node = bgp_dest_to_rnode(&test_rn);
-	memcpy((struct route_table *)&rt_node->table, &rt->route_table,
-	       sizeof(struct route_table));
+	str2prefix("42.1.1.0/24", &p);
+	dest = bgp_node_get(rt, &p);
+
 	setup_bgp_mp_list(t);
 	for (i = 0; i < test_mp_list_info_count; i++)
-		bgp_path_info_add(&test_rn, &test_mp_list_info[i]);
+		bgp_path_info_add(dest, &test_mp_list_info[i]);
 	return 0;
 }
 
@@ -309,6 +309,7 @@ static int run_bgp_path_info_mpath_update(testcase_t *t)
 	struct bgp_path_info *new_best, *old_best, *mpath;
 	struct list mp_list;
 	struct bgp_maxpaths_cfg mp_cfg = {3, 3};
+
 	int test_result = TEST_PASSED;
 	bgp_mp_list_init(&mp_list);
 	bgp_mp_list_add(&mp_list, &test_mp_list_info[4]);
@@ -317,7 +318,7 @@ static int run_bgp_path_info_mpath_update(testcase_t *t)
 	bgp_mp_list_add(&mp_list, &test_mp_list_info[1]);
 	new_best = &test_mp_list_info[3];
 	old_best = NULL;
-	bgp_path_info_mpath_update(NULL, &test_rn, new_best, old_best, &mp_list,
+	bgp_path_info_mpath_update(NULL, dest, new_best, old_best, &mp_list,
 				   &mp_cfg);
 	bgp_mp_list_clear(&mp_list);
 	EXPECT_TRUE(bgp_path_info_mpath_count(new_best) == 2, test_result);
@@ -332,7 +333,7 @@ static int run_bgp_path_info_mpath_update(testcase_t *t)
 	bgp_mp_list_add(&mp_list, &test_mp_list_info[1]);
 	new_best = &test_mp_list_info[0];
 	old_best = &test_mp_list_info[3];
-	bgp_path_info_mpath_update(NULL, &test_rn, new_best, old_best, &mp_list,
+	bgp_path_info_mpath_update(NULL, dest, new_best, old_best, &mp_list,
 				   &mp_cfg);
 	bgp_mp_list_clear(&mp_list);
 	EXPECT_TRUE(bgp_path_info_mpath_count(new_best) == 1, test_result);