bgpd: Adding Selection Deferral Timer handler changes.

* Selection Deferral Timer for Graceful Restart.
* Added selection deferral timer handling function.
* Route marking as selection defer when update message is received.
* Staggered processing of routes which are pending best selection.
* Fix for multi-path test case.

Signed-off-by: Biswajit Sadhu <sadhub@vmware.com>

10 files changed, 498 insertions, 14 deletions

diff --git a/bgpd/bgp_fsm.c b/bgpd/bgp_fsm.c
index 24b54d353..bafba6202 100644
--- a/bgpd/bgp_fsm.c
+++ b/bgpd/bgp_fsm.c
@@ -58,7 +58,8 @@
 
 DEFINE_HOOK(peer_backward_transition, (struct peer * peer), (peer))
 DEFINE_HOOK(peer_status_changed, (struct peer * peer), (peer))
-
+extern const char *get_afi_safi_str(afi_t afi,
+			safi_t safi, bool for_json);
 /* Definition of display strings corresponding to FSM events. This should be
  * kept consistent with the events defined in bgpd.h
  */
@@ -613,6 +614,33 @@ static int bgp_graceful_stale_timer_expire(struct thread *thread)
 	return 0;
 }
 
+/* Selection deferral timer processing function */
+static int bgp_graceful_deferral_timer_expire(struct thread *thread)
+{
+	struct afi_safi_info *info;
+	afi_t afi;
+	safi_t safi;
+	struct bgp *bgp;
+
+	info = THREAD_ARG(thread);
+	afi = info->afi;
+	safi = info->safi;
+	bgp = info->bgp;
+
+	if (BGP_DEBUG(update, UPDATE_OUT))
+		zlog_debug("afi %d, safi %d : graceful restart deferral timer expired",
+				afi, safi);
+
+	bgp->gr_info[afi][safi].t_select_deferral = NULL;
+
+	bgp->gr_info[afi][safi].eor_required = 0;
+	bgp->gr_info[afi][safi].eor_received = 0;
+	XFREE(MTYPE_TMP, info);
+
+	/* Best path selection */
+	return bgp_best_path_select_defer(bgp, afi, safi);
+}
+
 static int bgp_update_delay_applicable(struct bgp *bgp)
 {
 	/* update_delay_over flag should be reset (set to 0) for any new
@@ -1077,6 +1105,8 @@ int bgp_stop(struct peer *peer)
 	char orf_name[BUFSIZ];
 	int ret = 0;
 	peer->nsf_af_count = 0;
+	struct bgp *bgp = peer->bgp;
+	struct graceful_restart_info *gr_info = NULL;
 
 	if (peer_dynamic_neighbor(peer)
 	    && !(CHECK_FLAG(peer->flags, PEER_FLAG_DELETE))) {
@@ -1143,6 +1173,33 @@ int bgp_stop(struct peer *peer)
 					peer->nsf[afi][safi] = 0;
 		}
 
+		/* If peer reset before receiving EOR, decrement EOR count and
+		 * cancel the selection deferral timer if there are no
+		 * pending EOR messages to be received
+		 */
+		if (BGP_PEER_GRACEFUL_RESTART_CAPABLE(peer)) {
+			FOREACH_AFI_SAFI (afi, safi) {
+				if (peer->afc_nego[afi][safi] &&
+				    !CHECK_FLAG(peer->af_sflags[afi][safi],
+						PEER_STATUS_EOR_RECEIVED)) {
+					gr_info = &bgp->gr_info[afi][safi];
+					if (gr_info && gr_info->eor_required)
+						gr_info->eor_required--;
+					if (BGP_DEBUG(update, UPDATE_OUT))
+						zlog_debug("peer %s, EOR %d",
+							peer->host,
+							gr_info->eor_required);
+
+					/* There is no pending EOR message */
+					if (gr_info->eor_required == 0) {
+						BGP_TIMER_OFF(
+						gr_info->t_select_deferral);
+						gr_info->eor_received = 0;
+					}
+				}
+			}
+		}
+
 		/* set last reset time */
 		peer->resettime = peer->uptime = bgp_clock();
 
@@ -1576,6 +1633,85 @@ static int bgp_fsm_holdtime_expire(struct peer *peer)
 	return bgp_stop_with_notify(peer, BGP_NOTIFY_HOLD_ERR, 0);
 }
 
+/* Start the selection deferral timer thread for the specified AFI, SAFI */
+static int bgp_start_deferral_timer(struct bgp *bgp, afi_t afi, safi_t safi,
+		struct graceful_restart_info *gr_info)
+{
+	struct afi_safi_info *thread_info;
+
+	/* If the deferral timer is active, then increment eor count */
+	if (gr_info->t_select_deferral) {
+		gr_info->eor_required++;
+		return 0;
+	}
+
+	/* Start the deferral timer when the first peer enabled for the graceful
+	 * restart is established
+	 */
+	if (gr_info->eor_required == 0) {
+		thread_info = XMALLOC(MTYPE_TMP, sizeof(struct afi_safi_info));
+		if (thread_info == NULL) {
+			if (BGP_DEBUG(update, UPDATE_OUT))
+				zlog_debug("%s : Error allocating thread info",
+						__func__);
+			return -1;
+		}
+
+		thread_info->afi = afi;
+		thread_info->safi = safi;
+		thread_info->bgp = bgp;
+
+		thread_add_timer(bm->master,
+				bgp_graceful_deferral_timer_expire,
+				thread_info, bgp->select_defer_time,
+				&gr_info->t_select_deferral);
+		if (gr_info->t_select_deferral == NULL) {
+			if (BGP_DEBUG(update, UPDATE_OUT))
+				zlog_debug("Error starting deferral timer for %s",
+					get_afi_safi_str(afi, safi, false));
+			return -1;
+		}
+	}
+
+	gr_info->eor_required++;
+	if (BGP_DEBUG(update, UPDATE_OUT))
+		zlog_debug("Started the deferral timer for %s eor_required %d",
+				get_afi_safi_str(afi, safi, false),
+				gr_info->eor_required);
+	return 0;
+}
+
+/* Update the graceful restart information for the specified AFI, SAFI */
+static int bgp_update_gr_info(struct peer *peer, afi_t afi, safi_t safi)
+{
+	struct graceful_restart_info *gr_info;
+	struct bgp *bgp = peer->bgp;
+	int ret = 0;
+
+	if ((afi < AFI_IP) || (afi >= AFI_MAX)) {
+		if (BGP_DEBUG(update, UPDATE_OUT))
+			zlog_debug("%s : invalid afi %d", __func__, afi);
+		return -1;
+	}
+
+	if ((safi < SAFI_UNICAST) || (safi > SAFI_MPLS_VPN)) {
+		if (BGP_DEBUG(update, UPDATE_OUT))
+			zlog_debug("%s : invalid safi %d", __func__, safi);
+		return -1;
+	}
+
+	/* Restarting router */
+	if (BGP_PEER_GRACEFUL_RESTART_CAPABLE(peer) &&
+	    BGP_PEER_RESTARTING_MODE(peer)) {
+		/* Check if the forwarding state is preserved */
+		if (bgp_flag_check(bgp, BGP_FLAG_GR_PRESERVE_FWD)) {
+			gr_info = &(bgp->gr_info[afi][safi]);
+			ret = bgp_start_deferral_timer(bgp, afi, safi, gr_info);
+		}
+	}
+	return (ret);
+}
+
 /**
  * Transition to Established state.
  *
@@ -1589,6 +1725,7 @@ static int bgp_establish(struct peer *peer)
 	int nsf_af_count = 0;
 	int ret = 0;
 	struct peer *other;
+	int status;
 
 	other = peer->doppelganger;
 	peer = peer_xfer_conn(peer);
@@ -1628,6 +1765,14 @@ static int bgp_establish(struct peer *peer)
 
 	/* graceful restart */
 	UNSET_FLAG(peer->sflags, PEER_STATUS_NSF_WAIT);
+	if (bgp_debug_neighbor_events(peer)) {
+		if (BGP_PEER_RESTARTING_MODE(peer))
+			zlog_debug("peer %s BGP_RESTARTING_MODE",
+				   peer->host);
+		else if (BGP_PEER_HELPER_MODE(peer))
+			zlog_debug("peer %s BGP_HELPER_MODE",
+				   peer->host);
+	}
 	for (afi = AFI_IP; afi < AFI_MAX; afi++)
 		for (safi = SAFI_UNICAST; safi <= SAFI_MPLS_VPN; safi++) {
 			if (peer->afc_nego[afi][safi]
@@ -1647,6 +1792,17 @@ static int bgp_establish(struct peer *peer)
 					bgp_clear_stale_route(peer, afi, safi);
 				peer->nsf[afi][safi] = 0;
 			}
+			/* Update the graceful restart information */
+			if (peer->afc_nego[afi][safi]) {
+				if (!BGP_SELECT_DEFER_DISABLE(peer->bgp)) {
+					status = bgp_update_gr_info(peer, afi,
+								    safi);
+					if (status < 0)
+						zlog_debug("Error in updating graceful restart for %s",
+						get_afi_safi_str(afi,
+						safi, false));
+				}
+			}
 		}
 
 	peer->nsf_af_count = nsf_af_count;
diff --git a/bgpd/bgp_fsm.h b/bgpd/bgp_fsm.h
index e69002133..ced4f292a 100644
--- a/bgpd/bgp_fsm.h
+++ b/bgpd/bgp_fsm.h
@@ -94,6 +94,20 @@
 			UNSET_FLAG(peer->peer_gr_new_status_flag, \
 				PEER_GRACEFUL_RESTART_NEW_STATE_INHERIT)
 
+#define BGP_PEER_GRACEFUL_RESTART_CAPABLE(peer)                         \
+	(CHECK_FLAG(peer->cap, PEER_CAP_RESTART_ADV) &&                 \
+	 CHECK_FLAG(peer->cap, PEER_CAP_RESTART_RCV))
+
+#define BGP_PEER_RESTARTING_MODE(peer)\
+	(CHECK_FLAG(peer->flags, PEER_FLAG_GRACEFUL_RESTART) && \
+	 CHECK_FLAG(peer->cap, PEER_CAP_RESTART_BIT_ADV) &&     \
+	 !CHECK_FLAG(peer->cap, PEER_CAP_RESTART_BIT_RCV))
+
+#define BGP_PEER_HELPER_MODE(peer)\
+	(CHECK_FLAG(peer->flags, PEER_FLAG_GRACEFUL_RESTART_HELPER) && \
+	 CHECK_FLAG(peer->cap, PEER_CAP_RESTART_BIT_RCV) &&           \
+	 !CHECK_FLAG(peer->cap, PEER_CAP_RESTART_BIT_ADV))
+
 /* Prototypes. */
 extern void bgp_fsm_event_update(struct peer *peer, int valid);
 extern int bgp_event(struct thread *);
diff --git a/bgpd/bgp_packet.c b/bgpd/bgp_packet.c
index b7e9af002..0c52585a1 100644
--- a/bgpd/bgp_packet.c
+++ b/bgpd/bgp_packet.c
@@ -723,13 +723,17 @@ void bgp_notify_send_with_data(struct peer *peer, uint8_t code,
 				if (first) {
 					snprintf(c, sizeof(c), " %02x",
 						 data[i]);
+
 					strlcat(bgp_notify.data, c,
-						bgp_notify.length);
+						bgp_notify.length * 3);
+
 				} else {
 					first = 1;
 					snprintf(c, sizeof(c), "%02x", data[i]);
+
 					strlcpy(bgp_notify.data, c,
-						bgp_notify.length);
+						bgp_notify.length * 3);
+
 				}
 		}
 		bgp_notify_print(peer, &bgp_notify, "sending");
@@ -1404,6 +1408,7 @@ static int bgp_update_receive(struct peer *peer, bgp_size_t size)
 	bgp_size_t attribute_len;
 	bgp_size_t update_len;
 	bgp_size_t withdraw_len;
+	bool restart = false;
 
 	enum NLRI_TYPES {
 		NLRI_UPDATE,
@@ -1626,6 +1631,12 @@ static int bgp_update_receive(struct peer *peer, bgp_size_t size)
 	    || (attr_parse_ret == BGP_ATTR_PARSE_EOR)) {
 		afi_t afi = 0;
 		safi_t safi;
+		struct graceful_restart_info *gr_info;
+
+		/* Restarting router */
+		if (BGP_PEER_GRACEFUL_RESTART_CAPABLE(peer) &&
+		    BGP_PEER_RESTARTING_MODE(peer))
+			restart = true;
 
 		/* Non-MP IPv4/Unicast is a completely emtpy UPDATE - already
 		 * checked
@@ -1652,6 +1663,31 @@ static int bgp_update_receive(struct peer *peer, bgp_size_t size)
 				SET_FLAG(peer->af_sflags[afi][safi],
 					 PEER_STATUS_EOR_RECEIVED);
 				bgp_update_explicit_eors(peer);
+				/* Update graceful restart information */
+				gr_info = &(peer->bgp->gr_info[afi][safi]);
+				if (restart)
+					gr_info->eor_received++;
+				/* If EOR received from all peers and selection
+				 * deferral timer is running, cancel the timer
+				 * and invoke the best path calculation
+				 */
+				if (gr_info->eor_required ==
+						gr_info->eor_received) {
+					if (bgp_debug_neighbor_events(peer))
+						zlog_debug("%s %d, %s %d",
+							"EOR REQ",
+							gr_info->eor_required,
+							"EOR RCV",
+							gr_info->eor_received);
+					BGP_TIMER_OFF(
+						gr_info->t_select_deferral);
+					gr_info->eor_required = 0;
+					gr_info->eor_received = 0;
+					/* Best path selection */
+					if (bgp_best_path_select_defer(
+						peer->bgp, afi, safi) < 0)
+						return BGP_Stop;
+				}
 			}
 
 			/* NSF delete stale route */
@@ -1723,14 +1759,18 @@ static int bgp_notify_receive(struct peer *peer, bgp_size_t size)
 				if (first) {
 					snprintf(c, sizeof(c), " %02x",
 						stream_getc(peer->curr));
+
 					strlcat(bgp_notify.data, c,
-						bgp_notify.length);
+						bgp_notify.length * 3);
+
 				} else {
 					first = 1;
 					snprintf(c, sizeof(c), "%02x",
 						 stream_getc(peer->curr));
+
 					strlcpy(bgp_notify.data, c,
-						bgp_notify.length);
+						bgp_notify.length * 3);
+
 				}
 			bgp_notify.raw_data = (uint8_t *)peer->notify.data;
 		}
diff --git a/bgpd/bgp_route.c b/bgpd/bgp_route.c
index 5f4486b80..e58158957 100644
--- a/bgpd/bgp_route.c
+++ b/bgpd/bgp_route.c
@@ -89,7 +89,8 @@
 /* Extern from bgp_dump.c */
 extern const char *bgp_origin_str[];
 extern const char *bgp_origin_long_str[];
-
+const char *get_afi_safi_str(afi_t afi,
+		safi_t safi, bool for_json);
 /* PMSI strings. */
 #define PMSI_TNLTYPE_STR_NO_INFO "No info"
 #define PMSI_TNLTYPE_STR_DEFAULT PMSI_TNLTYPE_STR_NO_INFO
@@ -295,6 +296,76 @@ struct bgp_path_info *bgp_path_info_unlock(struct bgp_path_info *path)
 	return path;
 }
 
+/* This function sets flag BGP_NODE_SELECT_DEFER based on condition */
+static int bgp_node_set_defer_flag(struct bgp_node *rn, bool delete)
+{
+	struct peer *peer;
+	struct bgp_path_info *old_pi, *nextpi;
+	bool set_flag = 0;
+	struct bgp *bgp = NULL;
+	struct bgp_table *table = NULL;
+	afi_t afi = 0;
+	safi_t safi = 0;
+	char buf[PREFIX2STR_BUFFER];
+
+	/* If the flag BGP_NODE_SELECT_DEFER is set and new path is added
+	 * then the route selection is deferred
+	 */
+	if (CHECK_FLAG(rn->flags, BGP_NODE_SELECT_DEFER) && (delete == false))
+		return 0;
+
+	table = bgp_node_table(rn);
+	if (table) {
+		bgp = table->bgp;
+		afi = table->afi;
+		safi = table->safi;
+	}
+
+	for (old_pi = bgp_node_get_bgp_path_info(rn);
+	     (old_pi != NULL) && (nextpi = old_pi->next, 1); old_pi = nextpi) {
+		if (CHECK_FLAG(old_pi->flags, BGP_PATH_SELECTED))
+			continue;
+
+		/* Route selection is deferred if there is a stale path which
+		 * which indicates peer is in restart mode
+		 */
+		if (CHECK_FLAG(old_pi->flags, BGP_PATH_STALE) &&
+				(old_pi->sub_type == BGP_ROUTE_NORMAL)) {
+			set_flag = 1;
+		} else {
+			/* If the peer is graceful restart capable and peer is
+			 * restarting mode, set the flag BGP_NODE_SELECT_DEFER
+			 */
+			peer = old_pi->peer;
+			if (BGP_PEER_GRACEFUL_RESTART_CAPABLE(peer) &&
+			    BGP_PEER_RESTARTING_MODE(peer) &&
+			    (old_pi && old_pi->sub_type == BGP_ROUTE_NORMAL)) {
+				set_flag = 1;
+			}
+		}
+		if (set_flag)
+			break;
+	}
+
+	/* Set the flag BGP_NODE_SELECT_DEFER if route selection deferral timer
+	 * is active
+	 */
+	if (set_flag) {
+		if (bgp && (bgp->gr_info[afi][safi].t_select_deferral)) {
+			SET_FLAG(rn->flags, BGP_NODE_SELECT_DEFER);
+			prefix2str(&rn->p, buf, PREFIX2STR_BUFFER);
+			if (rn->rt_node == NULL)
+				rn->rt_node = listnode_add(
+					bgp->gr_info[afi][safi].route_list, rn);
+			if (BGP_DEBUG(update, UPDATE_OUT))
+				zlog_debug("DEFER route %s, rn %p, node %p",
+						buf, rn, rn->rt_node);
+			return 0;
+		}
+	}
+	return -1;
+}
+
 void bgp_path_info_add(struct bgp_node *rn, struct bgp_path_info *pi)
 {
 	struct bgp_path_info *top;
@@ -310,6 +381,7 @@ void bgp_path_info_add(struct bgp_node *rn, struct bgp_path_info *pi)
 	bgp_path_info_lock(pi);
 	bgp_lock_node(rn);
 	peer_lock(pi->peer); /* bgp_path_info peer reference */
+	bgp_node_set_defer_flag(rn, false);
 }
 
 /* Do the actual removal of info from RIB, for use by bgp_process
@@ -1973,6 +2045,30 @@ int subgroup_announce_check(struct bgp_node *rn, struct bgp_path_info *pi,
 	return 1;
 }
 
+static int bgp_route_select_timer_expire(struct thread *thread)
+{
+	struct afi_safi_info *info;
+	afi_t afi;
+	safi_t safi;
+	struct bgp *bgp;
+
+	info = THREAD_ARG(thread);
+	afi = info->afi;
+	safi = info->safi;
+	bgp = info->bgp;
+
+	if (BGP_DEBUG(update, UPDATE_OUT))
+		zlog_debug("afi %d, safi %d : route select timer expired",
+				afi, safi);
+
+	bgp->gr_info[afi][safi].t_route_select = NULL;
+
+	XFREE(MTYPE_TMP, info);
+
+	/* Best path selection */
+	return bgp_best_path_select_defer(bgp, afi, safi);
+}
+
 void bgp_best_selection(struct bgp *bgp, struct bgp_node *rn,
 			struct bgp_maxpaths_cfg *mpath_cfg,
 			struct bgp_path_info_pair *result, afi_t afi,
@@ -2376,6 +2472,15 @@ static void bgp_process_main_one(struct bgp *bgp, struct bgp_node *rn,
 			   afi2str(afi), safi2str(safi));
 	}
 
+	/* The best path calculation for the route is deferred if
+	 * BGP_NODE_SELECT_DEFER is set
+	 */
+	if (CHECK_FLAG(rn->flags, BGP_NODE_SELECT_DEFER)) {
+		if (BGP_DEBUG(update, UPDATE_OUT))
+			zlog_debug("DEFER set for route %p", rn);
+		return;
+	}
+
 	/* Best path selection. */
 	bgp_best_selection(bgp, rn, &bgp->maxpaths[afi][safi], &old_and_new,
 			   afi, safi);
@@ -2603,6 +2708,73 @@ static void bgp_process_main_one(struct bgp *bgp, struct bgp_node *rn,
 	return;
 }
 
+/* Process the routes with the flag BGP_NODE_SELECT_DEFER set */
+int bgp_best_path_select_defer(struct bgp *bgp, afi_t afi, safi_t safi)
+{
+	struct bgp_node *rn;
+	int cnt = 0;
+	struct afi_safi_info *thread_info;
+	struct listnode *node = NULL, *nnode = NULL;
+
+	if (bgp->gr_info[afi][safi].t_route_select)
+		BGP_TIMER_OFF(bgp->gr_info[afi][safi].t_route_select);
+
+	if (BGP_DEBUG(update, UPDATE_OUT)) {
+		zlog_debug("%s: processing route for %s : cnt %d",
+			   __func__, get_afi_safi_str(afi, safi, false),
+				listcount(bgp->gr_info[afi][safi].route_list));
+	}
+
+	/* Process the route list */
+	node = listhead(bgp->gr_info[afi][safi].route_list);
+	node = listhead(bgp->gr_info[afi][safi].route_list);
+	while (node) {
+		rn = listgetdata(node);
+		nnode = node->next;
+		list_delete_node(bgp->gr_info[afi][safi].route_list, node);
+		rn->rt_node = NULL;
+
+		if (CHECK_FLAG(rn->flags, BGP_NODE_SELECT_DEFER)) {
+			UNSET_FLAG(rn->flags, BGP_NODE_SELECT_DEFER);
+			bgp_process_main_one(bgp, rn, afi, safi);
+			cnt++;
+			if (cnt >= BGP_MAX_BEST_ROUTE_SELECT)
+				break;
+		}
+		node = nnode;
+	}
+
+	if (list_isempty(bgp->gr_info[afi][safi].route_list))
+		return 0;
+
+	thread_info = XMALLOC(MTYPE_TMP, sizeof(struct afi_safi_info));
+	if (thread_info == NULL) {
+		if (BGP_DEBUG(update, UPDATE_OUT))
+			zlog_debug("%s : error allocating thread info",
+					__func__);
+		return -1;
+	}
+
+	thread_info->afi = afi;
+	thread_info->safi = safi;
+	thread_info->bgp = bgp;
+
+	/* If there are more routes to be processed, start the
+	 * selection timer
+	 */
+	thread_add_timer(bm->master, bgp_route_select_timer_expire, thread_info,
+			BGP_ROUTE_SELECT_DELAY,
+			&bgp->gr_info[afi][safi].t_route_select);
+	if (bgp->gr_info[afi][safi].t_route_select == NULL) {
+		if (BGP_DEBUG(update, UPDATE_OUT))
+			zlog_debug("%s : error starting selection thread for %s",
+					__func__, get_afi_safi_str(afi,
+					safi, false));
+		return -1;
+	}
+	return 0;
+}
+
 static wq_item_status bgp_process_wq(struct work_queue *wq, void *data)
 {
 	struct bgp_process_queue *pqnode = data;
@@ -2681,6 +2853,16 @@ void bgp_process(struct bgp *bgp, struct bgp_node *rn, afi_t afi, safi_t safi)
 	if (CHECK_FLAG(rn->flags, BGP_NODE_PROCESS_SCHEDULED))
 		return;
 
+	/* If the flag BGP_NODE_SELECT_DEFER is set, do not add route to
+	 * the workqueue
+	 */
+	if (CHECK_FLAG(rn->flags, BGP_NODE_SELECT_DEFER)) {
+		if (BGP_DEBUG(update, UPDATE_OUT))
+			zlog_debug("BGP_NODE_SELECT_DEFER set for route %p",
+				    rn);
+		return;
+	}
+
 	if (wq == NULL)
 		return;
 
@@ -2844,13 +3026,43 @@ int bgp_maximum_prefix_overflow(struct peer *peer, afi_t afi, safi_t safi,
 void bgp_rib_remove(struct bgp_node *rn, struct bgp_path_info *pi,
 		    struct peer *peer, afi_t afi, safi_t safi)
 {
+
+	struct bgp *bgp = NULL;
+	bool delete_route = false;
+
 	bgp_aggregate_decrement(peer->bgp, &rn->p, pi, afi, safi);
 
-	if (!CHECK_FLAG(pi->flags, BGP_PATH_HISTORY))
+	if (!CHECK_FLAG(pi->flags, BGP_PATH_HISTORY)) {
 		bgp_path_info_delete(rn, pi); /* keep historical info */
 
-	hook_call(bgp_process, peer->bgp, afi, safi, rn, peer, true);
+		/* If the selected path is removed, reset BGP_NODE_SELECT_DEFER
+		 * flag
+		 */
+		if (CHECK_FLAG(pi->flags, BGP_PATH_SELECTED))
+			delete_route = true;
+		else
+			if (bgp_node_set_defer_flag(rn, true) < 0)
+				delete_route = true;
+		if (delete_route) {
+			if (CHECK_FLAG(rn->flags, BGP_NODE_SELECT_DEFER)) {
+				UNSET_FLAG(rn->flags, BGP_NODE_SELECT_DEFER);
+				UNSET_FLAG(rn->flags,
+						BGP_NODE_PROCESS_SCHEDULED);
+				bgp = pi->peer->bgp;
+				if ((rn->rt_node) &&
+					(bgp->gr_info[afi][safi]
+					.route_list)) {
+					list_delete_node(
+						bgp->gr_info[afi][safi]
+						.route_list,
+						rn->rt_node);
+					rn->rt_node = NULL;
+				}
+			}
+		}
+	}
 
+	hook_call(bgp_process, peer->bgp, afi, safi, rn, peer, true);
 	bgp_process(peer->bgp, rn, afi, safi);
 }
 
@@ -3302,6 +3514,7 @@ int bgp_update(struct peer *peer, struct prefix *p, uint32_t addpath_id,
 				if (CHECK_FLAG(pi->flags, BGP_PATH_STALE)) {
 					bgp_path_info_unset_flag(
 						rn, pi, BGP_PATH_STALE);
+					bgp_node_set_defer_flag(rn, false);
 					bgp_process(bgp, rn, afi, safi);
 				}
 			}
@@ -3337,8 +3550,10 @@ int bgp_update(struct peer *peer, struct prefix *p, uint32_t addpath_id,
 		}
 
 		/* graceful restart STALE flag unset. */
-		if (CHECK_FLAG(pi->flags, BGP_PATH_STALE))
+		if (CHECK_FLAG(pi->flags, BGP_PATH_STALE)) {
 			bgp_path_info_unset_flag(rn, pi, BGP_PATH_STALE);
+			bgp_node_set_defer_flag(rn, false);
+		}
 
 		/* The attribute is changed. */
 		bgp_path_info_set_flag(rn, pi, BGP_PATH_ATTR_CHANGED);
diff --git a/bgpd/bgp_route.h b/bgpd/bgp_route.h
index b9f3f3f76..e335c39fb 100644
--- a/bgpd/bgp_route.h
+++ b/bgpd/bgp_route.h
@@ -635,4 +635,5 @@ extern int bgp_show_table_rd(struct vty *vty, struct bgp *bgp, safi_t safi,
 			     struct bgp_table *table, struct prefix_rd *prd,
 			     enum bgp_show_type type, void *output_arg,
 			     bool use_json);
+extern int bgp_best_path_select_defer(struct bgp *bgp, afi_t afi, safi_t safi);
 #endif /* _QUAGGA_BGP_ROUTE_H */
diff --git a/bgpd/bgp_table.c b/bgpd/bgp_table.c
index b75246b17..01efab8f2 100644
--- a/bgpd/bgp_table.c
+++ b/bgpd/bgp_table.c
@@ -127,6 +127,40 @@ struct bgp_table *bgp_table_init(struct bgp *bgp, afi_t afi, safi_t safi)
 	return rt;
 }
 
+/* Delete the route node from the selection deferral route list */
+void bgp_delete_listnode(struct bgp_node *node)
+{
+	struct route_node *rn = NULL;
+	struct bgp_table *table = NULL;
+	struct bgp *bgp = NULL;
+	afi_t afi;
+	safi_t safi;
+
+	/* If the route to be deleted is selection pending, update the
+	 * route node in gr_info
+	 */
+	if (CHECK_FLAG(node->flags, BGP_NODE_SELECT_DEFER)) {
+		table = bgp_node_table(node);
+		if (table)
+			bgp = table->bgp;
+		rn = bgp_node_to_rnode(node);
+
+		afi = table->afi;
+		safi = table->safi;
+
+		if (bgp && rn && rn->lock == 1) {
+			/* Delete the route from the selection pending list */
+			if ((node->rt_node) &&
+			    (bgp->gr_info[afi][safi].route_list)) {
+				list_delete_node(
+					bgp->gr_info[afi][safi].route_list,
+						node->rt_node);
+				node->rt_node = NULL;
+			}
+		}
+	}
+}
+
 static struct bgp_node *
 bgp_route_next_until_maxlen(struct bgp_node *node, const struct bgp_node *limit,
 			    const uint8_t maxlen)
diff --git a/bgpd/bgp_table.h b/bgpd/bgp_table.h
index b3542e784..69cca9eee 100644
--- a/bgpd/bgp_table.h
+++ b/bgpd/bgp_table.h
@@ -28,6 +28,8 @@
 #include "bgpd.h"
 #include "bgp_advertise.h"
 
+extern void bgp_delete_listnode(struct bgp_node *node);
+
 struct bgp_table {
 	/* table belongs to this instance */
 	struct bgp *bgp;
@@ -95,7 +97,9 @@ struct bgp_node {
 #define BGP_NODE_USER_CLEAR             (1 << 1)
 #define BGP_NODE_LABEL_CHANGED          (1 << 2)
 #define BGP_NODE_REGISTERED_FOR_LABEL   (1 << 3)
-
+#define BGP_NODE_SELECT_DEFER           (1 << 4)
+	/* list node pointer */
+	struct listnode *rt_node;
 	struct bgp_addpath_node_data tx_addpath;
 
 	enum bgp_path_selection_reason reason;
@@ -162,6 +166,7 @@ static inline struct bgp_node *bgp_node_parent_nolock(struct bgp_node *node)
  */
 static inline void bgp_unlock_node(struct bgp_node *node)
 {
+	bgp_delete_listnode(node);
 	route_unlock_node(bgp_node_to_rnode(node));
 }
 
diff --git a/bgpd/bgp_vty.c b/bgpd/bgp_vty.c
index 2ccc9cfda..49135269c 100644
--- a/bgpd/bgp_vty.c
+++ b/bgpd/bgp_vty.c
@@ -15393,7 +15393,8 @@ void bgp_vty_init(void)
 	install_element(BGP_NODE, &bgp_graceful_restart_restart_time_cmd);
 	install_element(BGP_NODE, &no_bgp_graceful_restart_restart_time_cmd);
 	install_element(BGP_NODE, &bgp_graceful_restart_select_defer_time_cmd);
-	install_element(BGP_NODE, &no_bgp_graceful_restart_select_defer_time_cmd);
+	install_element(BGP_NODE,
+			&no_bgp_graceful_restart_select_defer_time_cmd);
 	install_element(BGP_NODE, &bgp_graceful_restart_preserve_fw_cmd);
 	install_element(BGP_NODE, &no_bgp_graceful_restart_preserve_fw_cmd);
 
diff --git a/bgpd/bgpd.h b/bgpd/bgpd.h
index d1b154f92..390dc8a7f 100644
--- a/bgpd/bgpd.h
+++ b/bgpd/bgpd.h
@@ -650,8 +650,8 @@ DECLARE_HOOK(bgp_inst_config_write,
 	 || (bgp->inst_type == BGP_INSTANCE_TYPE_VRF                           \
 	     && bgp->vrf_id != VRF_UNKNOWN))
 
-#define BGP_SELECT_DEFER_DISABLE(bgp)                                  \
-       (bgp_flag_check(bgp, BGP_FLAG_SELECT_DEFER_DISABLE))
+#define BGP_SELECT_DEFER_DISABLE(bgp)                 \
+	(bgp_flag_check(bgp, BGP_FLAG_SELECT_DEFER_DISABLE))
 
 /* BGP peer-group support. */
 struct peer_group {
diff --git a/tests/bgpd/test_mpath.c b/tests/bgpd/test_mpath.c
index 21f4b3877..a51ce4c6b 100644
--- a/tests/bgpd/test_mpath.c
+++ b/tests/bgpd/test_mpath.c
@@ -297,7 +297,25 @@ struct bgp_node test_rn;
 static int setup_bgp_path_info_mpath_update(testcase_t *t)
 {
 	int i;
+	struct bgp *bgp;
+	struct bgp_table *rt;
+	struct route_node *rt_node;
+	as_t asn = 1;
+
+	t->tmp_data = bgp_create_fake(&asn, NULL);
+	if (!t->tmp_data)
+		return -1;
+
+	bgp = t->tmp_data;
+	rt = bgp->rib[AFI_IP][SAFI_UNICAST];
+
+	if (!rt)
+		return -1;
+
 	str2prefix("42.1.1.0/24", &test_rn.p);
+	rt_node = bgp_node_to_rnode(&test_rn);
+	memcpy((struct route_table *)&rt_node->table, &rt->route_table,
+		sizeof(struct route_table));
 	setup_bgp_mp_list(t);
 	for (i = 0; i < test_mp_list_info_count; i++)
 		bgp_path_info_add(&test_rn, &test_mp_list_info[i]);
@@ -352,7 +370,7 @@ static int cleanup_bgp_path_info_mpath_update(testcase_t *t)
 	for (i = 0; i < test_mp_list_peer_count; i++)
 		sockunion_free(test_mp_list_peer[i].su_remote);
 
-	return 0;
+	return bgp_delete((struct bgp *)t->tmp_data);
 }
 
 testcase_t test_bgp_path_info_mpath_update = {