zebra: Allow GR to run per AFI as they are reported

The GR code in FRR used to wait till all AFI's were complete
before cleaning up the routes from the upper level protocol.
This of course can lead to some weird situations where say
ipv4 finishes and then v6 is stuck waiting for a peer to come
up and never finishes.  v4 when it finishes signals zebra that
it is done but no action is taken at that moment.

Modify the code to allow the zebra_gr.c code to handle a per
afi removal, instead of doing it all at the end.

Signed-off-by: Donald Sharp <sharpd@nvidia.com>

1 files changed, 100 insertions, 54 deletions

diff --git a/zebra/zebra_gr.c b/zebra/zebra_gr.c
index a209eb8c3..fcf391a21 100644
--- a/zebra/zebra_gr.c
+++ b/zebra/zebra_gr.c
@@ -43,7 +43,7 @@ static void zebra_gr_route_stale_delete_timer_expiry(struct event *thread);
 static int32_t zebra_gr_delete_stale_routes(struct client_gr_info *info);
 static void zebra_gr_process_client_stale_routes(struct zserv *client,
 						 struct client_gr_info *info);
-
+static void zebra_gr_delete_stale_route_table_afi(struct event *event);
 /*
  * Debug macros.
  */
@@ -101,6 +101,8 @@ static struct client_gr_info *zebra_gr_client_info_create(struct zserv *client)
 
 	info = XCALLOC(MTYPE_ZEBRA_GR, sizeof(struct client_gr_info));
 
+	info->stale_client_ptr = client;
+
 	TAILQ_INSERT_TAIL(&(client->gr_info_queue), info, gr_info);
 	return info;
 }
@@ -108,8 +110,8 @@ static struct client_gr_info *zebra_gr_client_info_create(struct zserv *client)
 /*
  * A helper function to delete and destroy client info.
  */
-static void zebra_gr_client_info_delte(struct zserv *client,
-				       struct client_gr_info *info)
+static void zebra_gr_client_info_delete(struct zserv *client,
+					struct client_gr_info *info)
 {
 	struct vrf *vrf = vrf_lookup_by_id(info->vrf_id);
 
@@ -285,6 +287,15 @@ void zebra_gr_client_reconnect(struct zserv *client)
 	zserv_client_delete(old_client);
 }
 
+struct zebra_gr_afi_clean {
+	struct client_gr_info *info;
+	afi_t afi;
+	uint8_t proto;
+	uint8_t instance;
+
+	struct event *t_gac;
+};
+
 /*
  * Functions to deal with capabilities
  */
@@ -346,7 +357,7 @@ void zread_client_capabilities(ZAPI_HANDLER_ARGS)
 		if ((info->gr_enable) && (client->gr_instance_count > 0))
 			client->gr_instance_count--;
 
-		zebra_gr_client_info_delte(client, info);
+		zebra_gr_client_info_delete(client, info);
 		break;
 	case ZEBRA_CLIENT_GR_CAPABILITIES:
 		/* Allocate bgp info */
@@ -386,15 +397,29 @@ void zread_client_capabilities(ZAPI_HANDLER_ARGS)
 		break;
 	case ZEBRA_CLIENT_ROUTE_UPDATE_COMPLETE:
 		if (!info) {
-			LOG_GR("%s: Client %s route update complete for AFI %d, SAFI %d",
+			LOG_GR("%s: Client %s route update complete for AFI %d, SAFI %d, no Graceful Restart communication, returning",
 			       __func__, zebra_route_string(client->proto),
 			       api.afi, api.safi);
+			return;
 		} else {
+			struct zebra_gr_afi_clean *gac;
+
 			LOG_GR("%s: Client %s vrf %s(%u) route update complete for AFI %d, SAFI %d",
 			       __func__, zebra_route_string(client->proto),
 			       VRF_LOGNAME(vrf), info->vrf_id, api.afi,
 			       api.safi);
 			info->route_sync[api.afi] = true;
+
+			gac = XCALLOC(MTYPE_ZEBRA_GR, sizeof(*gac));
+
+			gac->info = info;
+			gac->afi = api.afi;
+			gac->proto = client->proto;
+			gac->instance = client->instance;
+
+			event_add_event(zrouter.master,
+					zebra_gr_delete_stale_route_table_afi,
+					gac, 0, &gac->t_gac);
 		}
 		zebra_gr_process_client_stale_routes(client, info);
 		break;
@@ -480,6 +505,58 @@ static bool zebra_gr_process_route_entry(struct zserv *client,
 	return false;
 }
 
+static void zebra_gr_delete_stale_route_table_afi(struct event *event)
+{
+	struct zebra_gr_afi_clean *gac = EVENT_ARG(event);
+	struct route_table *table;
+	struct route_node *rn;
+	struct route_entry *re, *next;
+	struct zebra_vrf *zvrf = zebra_vrf_lookup_by_id(gac->info->vrf_id);
+	int32_t n = 0;
+
+	if (!zvrf)
+		goto done;
+
+	table = zvrf->table[gac->afi][SAFI_UNICAST];
+	if (!table)
+		goto done;
+
+	for (rn = route_top(table); rn; rn = srcdest_route_next(rn)) {
+		RNODE_FOREACH_RE_SAFE (rn, re, next) {
+			if (CHECK_FLAG(re->status, ROUTE_ENTRY_REMOVED))
+				continue;
+
+			/* If the route refresh is received
+			 * after restart then do not delete
+			 * the route
+			 */
+
+			if (re->type == gac->proto &&
+			    re->instance == gac->instance &&
+			    zebra_gr_process_route_entry(
+				    gac->info->stale_client_ptr, rn, re))
+				n++;
+
+			/* If the max route count is reached
+			 * then timer thread will be restarted
+			 * Store the current prefix and afi
+			 */
+			if ((n >= ZEBRA_MAX_STALE_ROUTE_COUNT) &&
+			    (gac->info->do_delete == false)) {
+				event_add_timer(
+					zrouter.master,
+					zebra_gr_delete_stale_route_table_afi,
+					gac, ZEBRA_DEFAULT_STALE_UPDATE_DELAY,
+					&gac->t_gac);
+			}
+		}
+	}
+
+done:
+	XFREE(MTYPE_ZEBRA_GR, gac);
+	return;
+}
+
 /*
  * This function walks through the route table for all vrf and deletes
  * the stale routes for the restarted client specified by the protocol
@@ -488,11 +565,6 @@ static bool zebra_gr_process_route_entry(struct zserv *client,
 static int32_t zebra_gr_delete_stale_route(struct client_gr_info *info,
 					   struct zebra_vrf *zvrf)
 {
-	struct route_node *rn;
-	struct route_entry *re;
-	struct route_entry *next;
-	struct route_table *table;
-	int32_t n = 0;
 	afi_t afi, curr_afi;
 	uint8_t proto;
 	uint16_t instance;
@@ -514,36 +586,22 @@ static int32_t zebra_gr_delete_stale_route(struct client_gr_info *info,
 
 	/* Process routes for all AFI */
 	for (afi = curr_afi; afi < AFI_MAX; afi++) {
-		table = zvrf->table[afi][SAFI_UNICAST];
-
-		if (!table)
-			continue;
-
-		for (rn = route_top(table); rn; rn = srcdest_route_next(rn)) {
-			RNODE_FOREACH_RE_SAFE (rn, re, next) {
-				if (CHECK_FLAG(re->status, ROUTE_ENTRY_REMOVED))
-					continue;
-				/* If the route refresh is received
-				 * after restart then do not delete
-				 * the route
-				 */
-				if (re->type == proto &&
-				    re->instance == instance &&
-				    zebra_gr_process_route_entry(s_client, rn,
-								 re))
-					n++;
-
-				/* If the max route count is reached
-				 * then timer thread will be restarted
-				 * Store the current prefix and afi
-				 */
-				if ((n >= ZEBRA_MAX_STALE_ROUTE_COUNT) &&
-				    (info->do_delete == false)) {
-					info->current_afi = afi;
-					return n;
-				}
-			}
-		}
+		struct zebra_gr_afi_clean *gac =
+			XCALLOC(MTYPE_ZEBRA_GR, sizeof(*gac));
+
+		gac->info = info;
+		gac->afi = afi;
+		gac->proto = proto;
+		gac->instance = instance;
+
+		if (info->do_delete)
+			event_execute(zrouter.master,
+				      zebra_gr_delete_stale_route_table_afi,
+				      gac, 0);
+		else
+			event_add_event(zrouter.master,
+					zebra_gr_delete_stale_route_table_afi,
+					gac, 0, &gac->t_gac);
 	}
 	return 0;
 }
@@ -554,21 +612,13 @@ static int32_t zebra_gr_delete_stale_route(struct client_gr_info *info,
  */
 static int32_t zebra_gr_delete_stale_routes(struct client_gr_info *info)
 {
-	struct vrf *vrf;
 	struct zebra_vrf *zvrf;
 	uint64_t cnt = 0;
 
 	if (info == NULL)
 		return -1;
 
-	/* Get the current VRF */
-	vrf = vrf_lookup_by_id(info->vrf_id);
-	if (vrf == NULL) {
-		LOG_GR("%s: Invalid VRF specified %u", __func__, info->vrf_id);
-		return -1;
-	}
-
-	zvrf = vrf->info;
+	zvrf = zebra_vrf_lookup_by_id(info->vrf_id);
 	if (zvrf == NULL) {
 		LOG_GR("%s: Invalid VRF entry %u", __func__, info->vrf_id);
 		return -1;
@@ -604,7 +654,7 @@ static void zebra_gr_process_client_stale_routes(struct zserv *client,
 
 	/*
 	 * Route update completed for all AFI, SAFI
-	 * Cancel the stale timer and process the routes
+	 * Cancel the stale timer, routes are already being processed
 	 */
 	if (info->t_stale_removal) {
 		struct vrf *vrf = vrf_lookup_by_id(info->vrf_id);
@@ -613,9 +663,5 @@ static void zebra_gr_process_client_stale_routes(struct zserv *client,
 		       __func__, zebra_route_string(client->proto),
 		       VRF_LOGNAME(vrf), info->vrf_id);
 		EVENT_OFF(info->t_stale_removal);
-		info->do_delete = false;
-		event_execute(zrouter.master,
-			      zebra_gr_route_stale_delete_timer_expiry, info,
-			      0);
 	}
 }