zebra: Capabality and stale route handling for GR client.

Handling capability received from client. It may contain
GR enable/disable, Stale time changes, RIB update complete
for given AFi, ASAFI and instance. It also has changes for
stale route handling.

Signed-off-by: Santosh P K <sapk@vmware.com>

4 files changed, 512 insertions, 3 deletions

diff --git a/zebra/main.c b/zebra/main.c
index 75f825e50..f23702d87 100644
--- a/zebra/main.c
+++ b/zebra/main.c
@@ -151,6 +151,10 @@ static void sigint(void)
 
 	zebra_dplane_pre_finish();
 
+	/* Clean up GR related info. */
+	zebra_gr_stale_client_cleanup(zrouter.stale_client_list);
+	list_delete_all_node(zrouter.stale_client_list);
+
 	for (ALL_LIST_ELEMENTS(zrouter.client_list, ln, nn, client))
 		zserv_close_client(client);
 
diff --git a/zebra/zapi_msg.c b/zebra/zapi_msg.c
index 72629318e..3f52a9def 100644
--- a/zebra/zapi_msg.c
+++ b/zebra/zapi_msg.c
@@ -2668,6 +2668,7 @@ void (*const zserv_handlers[])(ZAPI_HANDLER_ARGS) = {
 	[ZEBRA_MLAG_CLIENT_REGISTER] = zebra_mlag_client_register,
 	[ZEBRA_MLAG_CLIENT_UNREGISTER] = zebra_mlag_client_unregister,
 	[ZEBRA_MLAG_FORWARD_MSG] = zebra_mlag_forward_client_msg,
+	[ZEBRA_CLIENT_CAPABILITIES] = zread_client_capabilities
 };
 
 #if defined(HANDLE_ZAPI_FUZZING)
diff --git a/zebra/zebra_gr.c b/zebra/zebra_gr.c
index 86669df51..e8c7304f4 100644
--- a/zebra/zebra_gr.c
+++ b/zebra/zebra_gr.c
@@ -51,7 +51,10 @@
  * Forward declaration.
  */
 static struct zserv *zebra_gr_find_stale_client(struct zserv *client);
-static int zebra_gr_route_stale_delete_timer_expiry(struct thread *thread);
+static int32_t zebra_gr_route_stale_delete_timer_expiry(struct thread *thread);
+static int32_t zebra_gr_delete_stale_routes(struct client_gr_info *info);
+static void zebra_gr_process_client_stale_routes(struct zserv *client,
+						 vrf_id_t vrf_id);
 
 /*
  * Debug macros.
@@ -68,9 +71,79 @@ static int zebra_gr_route_stale_delete_timer_expiry(struct thread *thread);
  */
 
 /*
+ * Function to clean all the stale clients,
+ * function will also clean up all per instance
+ * capabilities that are exchanged.
+ */
+void zebra_gr_stale_client_cleanup(struct list *client_list)
+{
+	struct listnode *node, *nnode;
+	struct zserv *s_client = NULL;
+	struct client_gr_info *info, *ninfo;
+
+	/* Find the stale client */
+	for (ALL_LIST_ELEMENTS(client_list, node, nnode, s_client)) {
+
+		LOG_GR("%s: Stale client %s is being deleted", __func__,
+		       zebra_route_string(s_client->proto));
+
+		TAILQ_FOREACH_SAFE (info, &s_client->gr_info_queue, gr_info,
+				    ninfo) {
+
+			/* Cancel the stale timer */
+			if (info->t_stale_removal != NULL) {
+				THREAD_OFF(info->t_stale_removal);
+				info->t_stale_removal = NULL;
+				/* Process the stale routes */
+				thread_execute(
+				    zrouter.master,
+				    zebra_gr_route_stale_delete_timer_expiry,
+				    info, 1);
+			}
+		}
+	}
+}
+
+/*
+ * A helper function to create client info.
+ */
+static struct client_gr_info *zebra_gr_client_info_create(struct zserv *client)
+{
+	struct client_gr_info *info;
+
+	info = XCALLOC(MTYPE_TMP, sizeof(struct client_gr_info));
+
+	TAILQ_INSERT_TAIL(&(client->gr_info_queue), info, gr_info);
+	return info;
+}
+
+/*
+ * A helper function to delte and destory client info.
+ */
+static void zebra_gr_client_info_delte(struct zserv *client,
+				       struct client_gr_info *info)
+{
+	TAILQ_REMOVE(&(client->gr_info_queue), info, gr_info);
+
+	THREAD_OFF(info->t_stale_removal);
+
+	if (info->current_prefix)
+		XFREE(MTYPE_TMP, info->current_prefix);
+
+	LOG_GR("%s: Instance info is being deleted for client %s", __func__,
+	       zebra_route_string(client->proto));
+
+	/* Delete all the stale routes. */
+	info->delete = true;
+	zebra_gr_delete_stale_routes(info);
+
+	XFREE(MTYPE_TMP, info);
+}
+
+/*
  * Function to handle client when it disconnect.
  */
-int zebra_gr_client_disconnect(struct zserv *client)
+int32_t zebra_gr_client_disconnect(struct zserv *client)
 {
 	struct zserv *stale_client;
 	struct timeval tv;
@@ -114,6 +187,47 @@ int zebra_gr_client_disconnect(struct zserv *client)
 }
 
 /*
+ * Function to delete stale client
+ */
+static void zebra_gr_delete_stale_client(struct client_gr_info *info)
+{
+	struct client_gr_info *bgp_info;
+	struct zserv *s_client = NULL;
+
+	s_client = info->stale_client_ptr;
+
+	if (!s_client || !info->stale_client)
+		return;
+
+	/*
+	 * If there are bgp instances with the stale delete timer pending
+	 * then stale client is not deleted
+	 */
+	if ((s_client->gr_instance_count > 0) && info->gr_enable)
+		s_client->gr_instance_count--;
+
+	TAILQ_REMOVE(&(s_client->gr_info_queue), info, gr_info);
+
+	LOG_GR("%s: Client %s gr count %d", __func__,
+	       zebra_route_string(s_client->proto),
+	       s_client->gr_instance_count);
+
+	TAILQ_FOREACH (bgp_info, &s_client->gr_info_queue, gr_info) {
+		if (bgp_info->t_stale_removal != NULL)
+			return;
+	}
+
+	LOG_GR("%s: Client %s is being deleted", __func__,
+	       zebra_route_string(s_client->proto));
+
+	TAILQ_INIT(&(s_client->gr_info_queue));
+	listnode_delete(zrouter.stale_client_list, s_client);
+	if (info->stale_client)
+		XFREE(MTYPE_TMP, s_client);
+	XFREE(MTYPE_TMP, info);
+}
+
+/*
  * Function to find stale client.
  */
 static struct zserv *zebra_gr_find_stale_client(struct zserv *client)
@@ -177,8 +291,394 @@ void zebra_gr_client_reconnect(struct zserv *client)
 	}
 }
 
+/*
+ * Functions to deal with capabilities
+ */
+
+/*
+ * Update the graceful restart information
+ * for the client instance.
+ * This function handles all the capabilties that are received.
+ */
+static void zebra_client_update_info(struct zserv *client, struct zapi_cap *api)
+{
+	struct client_gr_info *info = NULL;
+
+	/* Find the bgp information for the specified vrf id */
+	TAILQ_FOREACH (info, &client->gr_info_queue, gr_info) {
+		if (info->vrf_id == api->vrf_id)
+			break;
+	}
+
+
+	/*
+	 * If the command is delete, then cancel the stale timer and
+	 * delete the bgp info
+	 */
+	switch (api->cap) {
+	case ZEBRA_CLIENT_GR_DISABLE:
+		if (!info)
+			return;
+
+		LOG_GR("%s: Client %s instance GR disabled count %d", __func__,
+		       zebra_route_string(client->proto),
+		       client->gr_instance_count);
+
+		if ((info->gr_enable) && (client->gr_instance_count > 0))
+			client->gr_instance_count--;
+
+		zebra_gr_client_info_delte(client, info);
+		break;
+	case ZEBRA_CLIENT_GR_CAPABILITIES:
+		/* Allocate bgp info */
+		if (!info)
+			info = zebra_gr_client_info_create(client);
+
+		/* Udpate other parameters */
+		if (!info->gr_enable) {
+			client->gr_instance_count++;
+
+			LOG_GR("%s: Cient %s GR enabled count %d", __func__,
+			       zebra_route_string(client->proto),
+			       client->gr_instance_count);
+
+			info->capabilities = api->cap;
+			info->stale_removal_time = api->stale_removal_time;
+			info->vrf_id = api->vrf_id;
+			info->gr_enable = true;
+		}
+		break;
+	case ZEBRA_CLIENT_RIB_STALE_TIME:
+		LOG_GR("%s: Client %s stale time update event", __func__,
+		       zebra_route_string(client->proto));
+
+		/* Update the stale removal timer */
+		if (info && info->t_stale_removal == NULL) {
+
+			LOG_GR("%s: Stale time: %d is now update to: %d",
+			       __func__, info->stale_removal_time,
+			       api->stale_removal_time);
+
+			info->stale_removal_time = api->stale_removal_time;
+		}
+
+		break;
+	case ZEBRA_CLIENT_ROUTE_UPDATE_COMPLETE:
+		LOG_GR(
+		   "%s: Client %s route update complete for AFI %d, SAFI %d",
+		   __func__, zebra_route_string(client->proto), api->afi,
+		   api->safi);
+		if (info)
+			info->route_sync[api->afi][api->safi] = true;
+		break;
+	case ZEBRA_CLIENT_ROUTE_UPDATE_PENDING:
+		LOG_GR("%s: Client %s route update pending for AFI %d, SAFI %d",
+		       __func__, zebra_route_string(client->proto), api->afi,
+		       api->safi);
+		if (info)
+			info->af_enabled[api->afi][api->safi] = true;
+		break;
+	}
+}
+
+/*
+ * Handler for capabilities that are received from client.
+ */
+static void zebra_client_capabilities_handler(struct zserv *client,
+					      struct zapi_cap *api)
+{
+	switch (api->cap) {
+	case ZEBRA_CLIENT_GR_CAPABILITIES:
+	case ZEBRA_CLIENT_ROUTE_UPDATE_PENDING:
+	case ZEBRA_CLIENT_GR_DISABLE:
+	case ZEBRA_CLIENT_RIB_STALE_TIME:
+		/*
+		 * For all the cases we need to update the client info.
+		 */
+		zebra_client_update_info(client, api);
+		break;
+	case ZEBRA_CLIENT_ROUTE_UPDATE_COMPLETE:
+		/*
+		 * After client info has been updated delete all
+		 * stale routes
+		 */
+		zebra_client_update_info(client, api);
+		zebra_gr_process_client_stale_routes(client, api->vrf_id);
+		break;
+	}
+}
+
+/*
+ * Function to decode and call appropriate functions
+ * to handle client capabilities.
+ */
+void zread_client_capabilities(ZAPI_HANDLER_ARGS)
+{
+	struct zapi_cap api;
+	struct stream *s;
+
+	s = msg;
+
+	if (zapi_capabilities_decode(s, &api)) {
+		LOG_GR("%s: Error in reading capabilities for client %s",
+		       __func__, zebra_route_string(client->proto));
+		return;
+	}
+
+	/* Call the capabilities handler */
+	zebra_client_capabilities_handler(client, &api);
+}
+
+
+/*
+ * Stale route handling
+ */
+
+/*
+ * Delete all the stale routes that have not been refreshed
+ * post restart.
+ */
+static int32_t zebra_gr_route_stale_delete_timer_expiry(struct thread *thread)
+{
+	struct client_gr_info *info;
+	int32_t cnt = 0;
+	struct zserv *client;
+
+	info = THREAD_ARG(thread);
+	info->t_stale_removal = NULL;
+	client = (struct zserv *)info->stale_client_ptr;
+
+	/* Set the flag to indicate all stale route deletion */
+	if (thread->u.val == 1)
+		info->delete = true;
+
+	cnt = zebra_gr_delete_stale_routes(info);
+
+	/* Retsart the timer */
+	if (cnt > 0) {
+		LOG_GR("%s: Client %s processed %d routes. Start timer again",
+		       __func__, zebra_route_string(client->proto), cnt);
+
+		thread_add_timer(zrouter.master,
+				 zebra_gr_route_stale_delete_timer_expiry, info,
+				 ZEBRA_DEFAULT_STALE_UPDATE_DELAY,
+				 &info->t_stale_removal);
+	} else {
+		/* No routes to delete for the VRF */
+		LOG_GR("%s: Client %s all starle routes processed", __func__,
+		       zebra_route_string(client->proto));
+
+		if (info->current_prefix != NULL)
+			XFREE(MTYPE_TMP, info->current_prefix);
+		info->current_prefix = NULL;
+		info->current_afi = 0;
+		zebra_gr_delete_stale_client(info);
+	}
+	return 0;
+}
+
+
+/*
+ * Function to process to check if route entry is stale
+ * or has been updated.
+ */
+static void zebra_gr_process_route_entry(struct zserv *client,
+					 struct route_node *rn,
+					 struct route_entry *re)
+{
+	char buf[PREFIX2STR_BUFFER];
+
+	if ((client == NULL) || (rn == NULL) || (re == NULL))
+		return;
+
+	/* If the route is not refreshed after restart, delete the entry */
+	if (re->uptime < client->restart_time) {
+		if (IS_ZEBRA_DEBUG_RIB) {
+			prefix2str(&rn->p, buf, sizeof(buf));
+			zlog_debug("%s: Client %s stale route %s is deleted",
+				   __func__, zebra_route_string(client->proto),
+				   buf);
+		}
+		rib_delnode(rn, re);
+	}
+}
 
-static int zebra_gr_route_stale_delete_timer_expiry(struct thread *thread)
+/*
+ * This function walks through the route table for all vrf and deletes
+ * the stale routes for the restarted client specified by the protocol
+ * type
+ */
+static int32_t zebra_gr_delete_stale_route(struct client_gr_info *info,
+					   struct zebra_vrf *zvrf)
 {
+	struct route_node *rn, *curr;
+	struct route_entry *re;
+	struct route_entry *next;
+	struct route_table *table;
+	int32_t n = 0;
+	struct prefix *p;
+	afi_t afi, curr_afi;
+	uint8_t proto;
+	uint16_t instance;
+	struct zserv *s_client;
+
+	if ((info == NULL) || (zvrf == NULL))
+		return -1;
+
+	s_client = info->stale_client_ptr;
+	if (s_client == NULL) {
+		LOG_GR("%s: Stale client not present", __func__);
+		return -1;
+	}
+
+	proto = s_client->proto;
+	instance = s_client->instance;
+	curr_afi = info->current_afi;
+
+	LOG_GR("%s: Client %s stale routes are being deleted", __func__,
+	       zebra_route_string(proto));
+
+	/* Process routes for all AFI */
+	for (afi = curr_afi; afi < AFI_MAX; afi++) {
+		table = zvrf->table[afi][SAFI_UNICAST];
+		p = info->current_prefix;
+
+		if (table) {
+			/*
+			 * If the current prefix is NULL then get the first
+			 * route entry in the table
+			 */
+			if (p == NULL) {
+				rn = route_top(table);
+				if (rn == NULL)
+					continue;
+				p = XCALLOC(MTYPE_TMP, sizeof(struct prefix));
+				if (p == NULL)
+					return -1;
+				curr = rn;
+				prefix_copy(p, &rn->p);
+			} else
+				/* Get the next route entry */
+				curr = route_table_get_next(table, p);
+
+			for (rn = curr; rn; rn = srcdest_route_next(rn)) {
+				RNODE_FOREACH_RE_SAFE (rn, re, next) {
+					if (CHECK_FLAG(re->status,
+						       ROUTE_ENTRY_REMOVED))
+						continue;
+					/* If the route refresh is received
+					 * after restart then do not delete
+					 * the route
+					 */
+					if (re->type == proto
+					    && re->instance == instance) {
+						zebra_gr_process_route_entry(
+							s_client, rn, re);
+						n++;
+					}
+
+					/* If the max route count is reached
+					 * then timer thread will be restarted
+					 * Store the current prefix and afi
+					 */
+					if ((n >= ZEBRA_MAX_STALE_ROUTE_COUNT)
+					    && (info->delete == false)) {
+						prefix_copy(p, &rn->p);
+						info->current_afi = afi;
+						info->current_prefix = p;
+						return n;
+					}
+				}
+			}
+		}
+		/*
+		 * Reset the current prefix to indicate processing completion
+		 * of the current AFI
+		 */
+		if (info->current_prefix) {
+			XFREE(MTYPE_TMP, info->current_prefix);
+			info->current_prefix = NULL;
+		}
+		continue;
+	}
 	return 0;
 }
+
+/*
+ * Delete the stale routes when client is restarted and routes are not
+ * refreshed within the stale timeout
+ */
+static int32_t zebra_gr_delete_stale_routes(struct client_gr_info *info)
+{
+	struct vrf *vrf;
+	struct zebra_vrf *zvrf;
+	uint64_t cnt = 0;
+
+	if (info == NULL)
+		return -1;
+
+	/* Get the current VRF */
+	vrf = vrf_lookup_by_id(info->vrf_id);
+	if (vrf == NULL) {
+		LOG_GR("%s: Invalid VRF %d", __func__, info->vrf_id);
+		return -1;
+	}
+
+	zvrf = vrf->info;
+	if (zvrf == NULL) {
+		LOG_GR("%s: Invalid VRF entry %d", __func__, info->vrf_id);
+		return -1;
+	}
+
+	cnt = zebra_gr_delete_stale_route(info, zvrf);
+	return cnt;
+}
+
+/*
+ * This function checks if route update for all AFI, SAFI is completed
+ * and cancels the stale timer
+ */
+static void zebra_gr_process_client_stale_routes(struct zserv *client,
+						 vrf_id_t vrf_id)
+{
+	struct client_gr_info *info = NULL;
+	afi_t afi;
+	safi_t safi;
+
+	TAILQ_FOREACH (info, &client->gr_info_queue, gr_info) {
+		if (info->vrf_id == vrf_id)
+			break;
+	}
+
+	if (info == NULL)
+		return;
+
+	/* Check if route update completed for all AFI, SAFI */
+	for (afi = AFI_IP; afi < AFI_MAX; afi++)
+		for (safi = SAFI_UNICAST; safi <= SAFI_MPLS_VPN; safi++) {
+			if (info->af_enabled[afi][safi]) {
+				if (!info->route_sync[afi][safi]) {
+					LOG_GR(
+					   "%s: Client %s route update not completed for AFI %d, SAFI %d",
+					   __func__, zebra_route_string(
+							    client->proto),
+					   afi, safi);
+					return;
+				}
+			}
+		}
+
+	/*
+	 * Route update completed for all AFI, SAFI
+	 * Cancel the stale timer and process the routes
+	 */
+	if (info->t_stale_removal) {
+		LOG_GR("%s: Client %s cancled stale delete timer vrf %d",
+		       __func__, zebra_route_string(client->proto),
+		       info->vrf_id);
+		THREAD_OFF(info->t_stale_removal);
+		thread_execute(zrouter.master,
+			       zebra_gr_route_stale_delete_timer_expiry, info,
+			       0);
+	}
+}
diff --git a/zebra/zserv.h b/zebra/zserv.h
index 877f3a8e3..77ea19202 100644
--- a/zebra/zserv.h
+++ b/zebra/zserv.h
@@ -317,6 +317,10 @@ int zebra_finalize(struct thread *event);
  */
 extern int zebra_gr_client_disconnect(struct zserv *client);
 extern void zebra_gr_client_reconnect(struct zserv *client);
+extern void zebra_gr_stale_client_cleanup(struct list *client_list);
+extern void zread_client_capabilities(struct zserv *client, struct zmsghdr *hdr,
+				      struct stream *msg,
+				      struct zebra_vrf *zvrf);
 
 #ifdef __cplusplus
 }