Merge pull request #2142 from pguibert6WIND/fs_zebra_complement

Flowspec complement : port support and policy routing per interface and plugin wrapper

12 files changed, 665 insertions, 95 deletions

diff --git a/bgpd/bgp_attr.c b/bgpd/bgp_attr.c
index 276a7054e..e714e17bd 100644
--- a/bgpd/bgp_attr.c
+++ b/bgpd/bgp_attr.c
@@ -2276,7 +2276,8 @@ static int bgp_attr_check(struct peer *peer, struct attr *attr)
 	   are present, it should.  Check for any other attribute being present
 	   instead.
 	 */
-	if (attr->flag == ATTR_FLAG_BIT(BGP_ATTR_MP_UNREACH_NLRI))
+	if ((!CHECK_FLAG(attr->flag, ATTR_FLAG_BIT(BGP_ATTR_MP_REACH_NLRI)) &&
+	     CHECK_FLAG(attr->flag, ATTR_FLAG_BIT(BGP_ATTR_MP_UNREACH_NLRI))))
 		return BGP_ATTR_PARSE_PROCEED;
 
 	if (!CHECK_FLAG(attr->flag, ATTR_FLAG_BIT(BGP_ATTR_ORIGIN)))
diff --git a/bgpd/bgp_debug.c b/bgpd/bgp_debug.c
index 3e3fbcbfe..a4ded57c2 100644
--- a/bgpd/bgp_debug.c
+++ b/bgpd/bgp_debug.c
@@ -1661,11 +1661,23 @@ DEFUN (no_debug_bgp_vpn,
 /* debug bgp pbr */
 DEFUN (debug_bgp_pbr,
        debug_bgp_pbr_cmd,
-       "debug bgp pbr",
+       "debug bgp pbr [error]",
        DEBUG_STR
        BGP_STR
-       "BGP policy based routing\n")
+       "BGP policy based routing\n"
+       "BGP PBR error\n")
 {
+	int idx = 3;
+
+	if (argv_find(argv, argc, "error", &idx)) {
+		if (vty->node == CONFIG_NODE)
+			DEBUG_ON(pbr, PBR_ERROR);
+		else {
+			TERM_DEBUG_ON(pbr, PBR_ERROR);
+			vty_out(vty, "BGP policy based routing error is on\n");
+		}
+		return CMD_SUCCESS;
+	}
 	if (vty->node == CONFIG_NODE)
 		DEBUG_ON(pbr, PBR);
 	else {
@@ -1677,12 +1689,24 @@ DEFUN (debug_bgp_pbr,
 
 DEFUN (no_debug_bgp_pbr,
        no_debug_bgp_pbr_cmd,
-       "no debug bgp pbr",
+       "no debug bgp pbr [error]",
        NO_STR
        DEBUG_STR
        BGP_STR
-       "BGP policy based routing\n")
+       "BGP policy based routing\n"
+       "BGP PBR Error\n")
 {
+	int idx = 3;
+
+	if (argv_find(argv, argc, "error", &idx)) {
+		if (vty->node == CONFIG_NODE)
+			DEBUG_OFF(pbr, PBR_ERROR);
+		else {
+			TERM_DEBUG_OFF(pbr, PBR_ERROR);
+			vty_out(vty, "BGP policy based routing error is off\n");
+		}
+		return CMD_SUCCESS;
+	}
 	if (vty->node == CONFIG_NODE)
 		DEBUG_OFF(pbr, PBR);
 	else {
@@ -1769,6 +1793,7 @@ DEFUN (no_debug_bgp,
 	TERM_DEBUG_OFF(flowspec, FLOWSPEC);
 	TERM_DEBUG_OFF(labelpool, LABELPOOL);
 	TERM_DEBUG_OFF(pbr, PBR);
+	TERM_DEBUG_OFF(pbr, PBR_ERROR);
 	vty_out(vty, "All possible debugging has been turned off\n");
 
 	return CMD_SUCCESS;
@@ -1846,6 +1871,8 @@ DEFUN_NOSH (show_debugging_bgp,
 
 	if (BGP_DEBUG(pbr, PBR))
 		vty_out(vty, "  BGP policy based routing debugging is on\n");
+	if (BGP_DEBUG(pbr, PBR_ERROR))
+		vty_out(vty, "  BGP policy based routing error debugging is on\n");
 
 	vty_out(vty, "\n");
 	return CMD_SUCCESS;
@@ -1906,6 +1933,8 @@ int bgp_debug_count(void)
 
 	if (BGP_DEBUG(pbr, PBR))
 		ret++;
+	if (BGP_DEBUG(pbr, PBR_ERROR))
+		ret++;
 
 	return ret;
 }
@@ -2012,6 +2041,10 @@ static int bgp_config_write_debug(struct vty *vty)
 		vty_out(vty, "debug bgp pbr\n");
 		write++;
 	}
+	if (CONF_BGP_DEBUG(pbr, PBR_ERROR)) {
+		vty_out(vty, "debug bgp pbr error\n");
+		write++;
+	}
 	return write;
 }
 
diff --git a/bgpd/bgp_ecommunity.c b/bgpd/bgp_ecommunity.c
index 99fe80f05..20f5e15d6 100644
--- a/bgpd/bgp_ecommunity.c
+++ b/bgpd/bgp_ecommunity.c
@@ -642,9 +642,9 @@ char *ecommunity_ecom2str(struct ecommunity *ecom, int format, int filter)
 {
 	int i;
 	uint8_t *pnt;
-	int type = 0;
-	int sub_type = 0;
-#define ECOMMUNITY_STR_DEFAULT_LEN  27
+	uint8_t type = 0;
+	uint8_t sub_type = 0;
+#define ECOMMUNITY_STR_DEFAULT_LEN  64
 	int str_size;
 	int str_pnt;
 	char *str_buf;
@@ -750,10 +750,25 @@ char *ecommunity_ecom2str(struct ecommunity *ecom, int format, int filter)
 					"FS:redirect IP 0x%x", *(pnt+5));
 			} else
 				unk_ecom = 1;
-		} else if (type == ECOMMUNITY_ENCODE_TRANS_EXP) {
+		} else if (type == ECOMMUNITY_ENCODE_TRANS_EXP ||
+			   type == ECOMMUNITY_EXTENDED_COMMUNITY_PART_2 ||
+			   type == ECOMMUNITY_EXTENDED_COMMUNITY_PART_3) {
 			sub_type = *pnt++;
+			if (sub_type == ECOMMUNITY_REDIRECT_VRF) {
+				char buf[16];
 
-			if (sub_type == ECOMMUNITY_TRAFFIC_ACTION) {
+				memset(buf, 0, sizeof(buf));
+				ecommunity_rt_soo_str(buf, (uint8_t *)pnt,
+						      type &
+						      ~ECOMMUNITY_ENCODE_TRANS_EXP,
+						      ECOMMUNITY_ROUTE_TARGET,
+						      ECOMMUNITY_FORMAT_DISPLAY);
+				len = snprintf(str_buf + str_pnt,
+					       str_size - len,
+					       "FS:redirect VRF %s", buf);
+			} else if (type != ECOMMUNITY_ENCODE_TRANS_EXP)
+				unk_ecom = 1;
+			else if (sub_type == ECOMMUNITY_TRAFFIC_ACTION) {
 				char action[64];
 				char *ptr = action;
 
@@ -782,30 +797,20 @@ char *ecommunity_ecom2str(struct ecommunity *ecom, int format, int filter)
 				len = sprintf(
 					str_buf + str_pnt,
 					"FS:rate %f", data.rate_float);
-			} else if (sub_type == ECOMMUNITY_REDIRECT_VRF) {
-				char buf[16];
-
-				memset(buf, 0, sizeof(buf));
-				ecommunity_rt_soo_str(buf, (uint8_t *)pnt,
-						type &
-						~ECOMMUNITY_ENCODE_TRANS_EXP,
-						ECOMMUNITY_ROUTE_TARGET,
-						ECOMMUNITY_FORMAT_DISPLAY);
-				len = snprintf(
-					str_buf + str_pnt,
-					str_size - len,
-					"FS:redirect VRF %s", buf);
 			} else if (sub_type == ECOMMUNITY_TRAFFIC_MARKING) {
 				len = sprintf(
 					str_buf + str_pnt,
 					"FS:marking %u", *(pnt+5));
 			} else
 				unk_ecom = 1;
-		} else
+		} else {
+			sub_type = *pnt++;
 			unk_ecom = 1;
+		}
 
 		if (unk_ecom)
-			len = sprintf(str_buf + str_pnt, "?");
+			len = sprintf(str_buf + str_pnt, "UNK:%d, %d",
+				      type, sub_type);
 
 		str_pnt += len;
 		first = 0;
diff --git a/bgpd/bgp_flowspec.h b/bgpd/bgp_flowspec.h
index 392b32153..5dd2c3931 100644
--- a/bgpd/bgp_flowspec.h
+++ b/bgpd/bgp_flowspec.h
@@ -47,4 +47,7 @@ extern void bgp_fs_nlri_get_string(unsigned char *nlri_content, size_t len,
 extern void route_vty_out_flowspec(struct vty *vty, struct prefix *p,
 				   struct bgp_info *binfo,
 				   int display, json_object *json_paths);
+extern int bgp_fs_config_write_pbr(struct vty *vty, struct bgp *bgp,
+				   afi_t afi, safi_t safi);
+
 #endif /* _FRR_BGP_FLOWSPEC_H */
diff --git a/bgpd/bgp_flowspec_vty.c b/bgpd/bgp_flowspec_vty.c
index 7bf11f12a..b21e5ae0d 100644
--- a/bgpd/bgp_flowspec_vty.c
+++ b/bgpd/bgp_flowspec_vty.c
@@ -30,6 +30,7 @@
 #include "bgpd/bgp_flowspec_util.h"
 #include "bgpd/bgp_flowspec_private.h"
 #include "bgpd/bgp_debug.h"
+#include "bgpd/bgp_pbr.h"
 
 /* Local Structures and variables declarations
  * This code block hosts the struct declared that host the flowspec rules
@@ -318,16 +319,32 @@ void route_vty_out_flowspec(struct vty *vty, struct prefix *p,
 		XFREE(MTYPE_ECOMMUNITY_STR, s);
 	}
 	peer_uptime(binfo->uptime, timebuf, BGP_UPTIME_LEN, 0, NULL);
-	if (display == NLRI_STRING_FORMAT_LARGE)
-		vty_out(vty, "\tup for %8s\n", timebuf);
-	else if (json_paths) {
+	if (display == NLRI_STRING_FORMAT_LARGE) {
+		vty_out(vty, "\treceived for %8s\n", timebuf);
+	} else if (json_paths) {
 		json_time_path = json_object_new_object();
 		json_object_string_add(json_time_path,
 				       "time", timebuf);
 		if (display == NLRI_STRING_FORMAT_JSON)
 			json_object_array_add(json_paths, json_time_path);
 	}
+	if (display == NLRI_STRING_FORMAT_LARGE) {
+		struct bgp_info_extra *extra = bgp_info_extra_get(binfo);
 
+		if (extra->bgp_fs_pbr) {
+			struct bgp_pbr_match_entry *bpme;
+			struct bgp_pbr_match *bpm;
+
+			bpme = (struct bgp_pbr_match_entry *)extra->bgp_fs_pbr;
+			bpm = bpme->backpointer;
+			vty_out(vty, "\tinstalled in PBR");
+			if (bpm)
+				vty_out(vty, " (%s)\n", bpm->ipset_name);
+			else
+				vty_out(vty, "\n");
+		} else
+			vty_out(vty, "\tnot installed in PBR\n");
+	}
 }
 
 int bgp_show_table_flowspec(struct vty *vty, struct bgp *bgp, afi_t afi,
@@ -408,10 +425,113 @@ DEFUN (no_debug_bgp_flowspec,
 	return CMD_SUCCESS;
 }
 
+int bgp_fs_config_write_pbr(struct vty *vty, struct bgp *bgp,
+			    afi_t afi, safi_t safi)
+{
+	struct bgp_pbr_interface *pbr_if;
+	bool declare_node = false;
+	struct bgp_pbr_config *bgp_pbr_cfg = bgp->bgp_pbr_cfg;
+	struct bgp_pbr_interface_head *head;
+	bool bgp_pbr_interface_any;
+
+	if (!bgp_pbr_cfg || safi != SAFI_FLOWSPEC || afi != AFI_IP)
+		return 0;
+	head = &(bgp_pbr_cfg->ifaces_by_name_ipv4);
+	bgp_pbr_interface_any = bgp_pbr_cfg->pbr_interface_any_ipv4;
+	if (!RB_EMPTY(bgp_pbr_interface_head, head) ||
+	     !bgp_pbr_interface_any)
+		declare_node = true;
+	RB_FOREACH (pbr_if, bgp_pbr_interface_head, head) {
+		vty_out(vty, "  local-install %s\n", pbr_if->name);
+	}
+	if (!bgp_pbr_interface_any)
+		vty_out(vty, "  no local-install any\n");
+	return declare_node ? 1 : 0;
+}
+
+static int bgp_fs_local_install_interface(struct bgp *bgp,
+					  const char *no, const char *ifname)
+{
+	struct bgp_pbr_interface *pbr_if;
+	struct bgp_pbr_config *bgp_pbr_cfg = bgp->bgp_pbr_cfg;
+	struct bgp_pbr_interface_head *head;
+	bool *bgp_pbr_interface_any;
+
+	if (!bgp_pbr_cfg)
+		return CMD_SUCCESS;
+	head = &(bgp_pbr_cfg->ifaces_by_name_ipv4);
+	bgp_pbr_interface_any = &(bgp_pbr_cfg->pbr_interface_any_ipv4);
+	if (no) {
+		if (!ifname) {
+			if (*bgp_pbr_interface_any) {
+				*bgp_pbr_interface_any = false;
+				/* remove all other interface list */
+				bgp_pbr_reset(bgp, AFI_IP);
+			}
+			return CMD_SUCCESS;
+		}
+		pbr_if = bgp_pbr_interface_lookup(ifname, head);
+		if (!pbr_if)
+			return CMD_SUCCESS;
+		RB_REMOVE(bgp_pbr_interface_head, head, pbr_if);
+		return CMD_SUCCESS;
+	}
+	if (ifname) {
+		pbr_if = bgp_pbr_interface_lookup(ifname, head);
+		if (pbr_if)
+			return CMD_SUCCESS;
+		pbr_if = XCALLOC(MTYPE_TMP,
+				 sizeof(struct bgp_pbr_interface));
+		strlcpy(pbr_if->name, ifname, INTERFACE_NAMSIZ);
+		RB_INSERT(bgp_pbr_interface_head, head, pbr_if);
+		*bgp_pbr_interface_any = false;
+	} else {
+		/* set to default */
+		if (!*bgp_pbr_interface_any) {
+			/* remove all other interface list
+			 */
+			bgp_pbr_reset(bgp, AFI_IP);
+			*bgp_pbr_interface_any = true;
+		}
+	}
+	return CMD_SUCCESS;
+}
+
+DEFUN (bgp_fs_local_install_ifname,
+	bgp_fs_local_install_ifname_cmd,
+	"[no] local-install INTERFACE",
+	NO_STR
+	"Apply local policy routing\n"
+	"Interface name\n")
+{
+	struct bgp *bgp = VTY_GET_CONTEXT(bgp);
+	int idx = 0;
+	const char *no = strmatch(argv[0]->text, (char *)"no") ? "no" : NULL;
+	char *ifname = argv_find(argv, argc, "INTERFACE", &idx) ?
+		argv[idx]->arg : NULL;
+
+	return bgp_fs_local_install_interface(bgp, no, ifname);
+}
+
+DEFUN (bgp_fs_local_install_any,
+	bgp_fs_local_install_any_cmd,
+	"[no] local-install any",
+	NO_STR
+	"Apply local policy routing\n"
+	"Any Interface\n")
+{
+	struct bgp *bgp = VTY_GET_CONTEXT(bgp);
+	const char *no = strmatch(argv[0]->text, (char *)"no") ? "no" : NULL;
+
+	return bgp_fs_local_install_interface(bgp, no, NULL);
+}
+
 void bgp_flowspec_vty_init(void)
 {
 	install_element(ENABLE_NODE, &debug_bgp_flowspec_cmd);
 	install_element(CONFIG_NODE, &debug_bgp_flowspec_cmd);
 	install_element(ENABLE_NODE, &no_debug_bgp_flowspec_cmd);
 	install_element(CONFIG_NODE, &no_debug_bgp_flowspec_cmd);
+	install_element(BGP_FLOWSPECV4_NODE, &bgp_fs_local_install_any_cmd);
+	install_element(BGP_FLOWSPECV4_NODE, &bgp_fs_local_install_ifname_cmd);
 }
diff --git a/bgpd/bgp_pbr.c b/bgpd/bgp_pbr.c
index 04d6314fd..1054ea402 100644
--- a/bgpd/bgp_pbr.c
+++ b/bgpd/bgp_pbr.c
@@ -35,6 +35,12 @@
 DEFINE_MTYPE_STATIC(BGPD, PBR_MATCH_ENTRY, "PBR match entry")
 DEFINE_MTYPE_STATIC(BGPD, PBR_MATCH, "PBR match")
 DEFINE_MTYPE_STATIC(BGPD, PBR_ACTION, "PBR action")
+DEFINE_MTYPE_STATIC(BGPD, PBR, "BGP PBR Context")
+
+RB_GENERATE(bgp_pbr_interface_head, bgp_pbr_interface,
+	    id_entry, bgp_pbr_interface_compare);
+struct bgp_pbr_interface_head ifaces_by_name_ipv4 =
+	RB_INITIALIZER(&ifaces_by_name_ipv4);
 
 static int bgp_pbr_match_counter_unique;
 static int bgp_pbr_match_entry_counter_unique;
@@ -169,7 +175,60 @@ static int sprintf_bgp_pbr_match_val(char *str, struct bgp_pbr_match_val *mval,
 		_cnt++; \
 	} while (0)
 
-/* return 1 if OK, 0 if validation should stop) */
+struct bgp_pbr_range_port {
+	uint16_t min_port;
+	uint16_t max_port;
+};
+
+/* return true if extraction ok
+ */
+static bool bgp_pbr_extract(struct bgp_pbr_match_val list[],
+			    int num,
+			    struct bgp_pbr_range_port *range)
+{
+	int i = 0;
+	bool exact_match = false;
+
+	if (range)
+		memset(range, 0, sizeof(struct bgp_pbr_range_port));
+
+	if (num > 2)
+		return false;
+	for (i = 0; i < num; i++) {
+		if (i != 0 && (list[i].compare_operator ==
+			       OPERATOR_COMPARE_EQUAL_TO))
+			return false;
+		if (i == 0 && (list[i].compare_operator ==
+			       OPERATOR_COMPARE_EQUAL_TO)) {
+			if (range)
+				range->min_port = list[i].value;
+			exact_match = true;
+		}
+		if (exact_match == true && i > 0)
+			return false;
+		if (list[i].compare_operator ==
+		    (OPERATOR_COMPARE_GREATER_THAN +
+		     OPERATOR_COMPARE_EQUAL_TO)) {
+			if (range)
+				range->min_port = list[i].value;
+		} else if (list[i].compare_operator ==
+			   (OPERATOR_COMPARE_LESS_THAN +
+			    OPERATOR_COMPARE_EQUAL_TO)) {
+			if (range)
+				range->max_port = list[i].value;
+		} else if (list[i].compare_operator ==
+			   OPERATOR_COMPARE_LESS_THAN) {
+			if (range)
+				range->max_port = list[i].value - 1;
+		} else if (list[i].compare_operator ==
+			   OPERATOR_COMPARE_GREATER_THAN) {
+			if (range)
+				range->min_port = list[i].value + 1;
+		}
+	}
+	return true;
+}
+
 static int bgp_pbr_validate_policy_route(struct bgp_pbr_entry_main *api)
 {
 	/* because bgp pbr entry may contain unsupported
@@ -179,18 +238,63 @@ static int bgp_pbr_validate_policy_route(struct bgp_pbr_entry_main *api)
 	 * - combination src/dst => redirect nexthop [ + rate]
 	 * - combination src/dst => redirect VRF [ + rate]
 	 * - combination src/dst => drop
+	 * - combination srcport + @IP
 	 */
-	if (api->match_src_port_num || api->match_dst_port_num
-	    || api->match_port_num || api->match_protocol_num
-	    || api->match_icmp_type_num || api->match_icmp_type_num
+	if (api->match_icmp_type_num || api->match_icmp_type_num
 	    || api->match_packet_length_num || api->match_dscp_num
 	    || api->match_tcpflags_num) {
 		if (BGP_DEBUG(pbr, PBR)) {
 			bgp_pbr_print_policy_route(api);
 			zlog_debug("BGP: some SET actions not supported by Zebra. ignoring.");
+			zlog_debug("BGP: case icmp or length or dscp or tcp flags");
 		}
 		return 0;
 	}
+
+	if (api->match_protocol_num > 1) {
+		if (BGP_DEBUG(pbr, PBR))
+			zlog_debug("BGP: match protocol operations:"
+				 "multiple protocols ( %d). ignoring.",
+				 api->match_protocol_num);
+		return 0;
+	}
+	if (api->match_protocol_num == 1 &&
+	    api->protocol[0].value != PROTOCOL_UDP &&
+	    api->protocol[0].value != PROTOCOL_TCP) {
+		if (BGP_DEBUG(pbr, PBR))
+			zlog_debug("BGP: match protocol operations:"
+				   "protocol (%d) not supported. ignoring",
+				   api->match_protocol_num);
+		return 0;
+	}
+	if (!bgp_pbr_extract(api->src_port, api->match_src_port_num, NULL)) {
+		if (BGP_DEBUG(pbr, PBR))
+			zlog_debug("BGP: match src port operations:"
+				   "too complex. ignoring.");
+		return 0;
+	}
+	if (!bgp_pbr_extract(api->dst_port, api->match_dst_port_num, NULL)) {
+		if (BGP_DEBUG(pbr, PBR))
+			zlog_debug("BGP: match dst port operations:"
+				   "too complex. ignoring.");
+		return 0;
+	}
+	if (!bgp_pbr_extract(api->port, api->match_port_num, NULL)) {
+		if (BGP_DEBUG(pbr, PBR))
+			zlog_debug("BGP: match port operations:"
+				 "too complex. ignoring.");
+		return 0;
+	}
+	/* no combinations with both src_port and dst_port
+	 * or port with src_port and dst_port
+	 */
+	if (api->match_src_port_num + api->match_dst_port_num +
+	    api->match_port_num > 3) {
+		if (BGP_DEBUG(pbr, PBR))
+			zlog_debug("BGP: match multiple port operations:"
+				 " too complex. ignoring.");
+		return 0;
+	}
 	if (!(api->match_bitmask & PREFIX_SRC_PRESENT) &&
 	    !(api->match_bitmask & PREFIX_DST_PRESENT)) {
 		if (BGP_DEBUG(pbr, PBR)) {
@@ -228,15 +332,15 @@ static int bgp_pbr_build_and_validate_entry(struct prefix *p,
 		ecom = info->attr->ecommunity;
 		for (i = 0; i < ecom->size; i++) {
 			ecom_eval = (struct ecommunity_val *)
-				ecom->val + (i * ECOMMUNITY_SIZE);
-
+				(ecom->val + (i * ECOMMUNITY_SIZE));
+			action_count++;
 			if (action_count > ACTIONS_MAX_NUM) {
 				if (BGP_DEBUG(pbr, PBR_ERROR))
 					zlog_err("%s: flowspec actions exceeds limit (max %u)",
 						 __func__, action_count);
 				break;
 			}
-			api_action = &api->actions[action_count];
+			api_action = &api->actions[action_count - 1];
 
 			if ((ecom_eval->val[1] ==
 			     (char)ECOMMUNITY_REDIRECT_VRF) &&
@@ -376,6 +480,7 @@ static void bgp_pbr_action_free(void *arg)
 						   AFI_IP,
 						   bpa->table_id,
 						   false);
+			bpa->installed = false;
 		}
 	}
 	XFREE(MTYPE_PBR_ACTION, bpa);
@@ -447,6 +552,11 @@ uint32_t bgp_pbr_match_entry_hash_key(void *arg)
 	pbme = (struct bgp_pbr_match_entry *)arg;
 	key = prefix_hash_key(&pbme->src);
 	key = jhash_1word(prefix_hash_key(&pbme->dst), key);
+	key = jhash(&pbme->dst_port_min, 2, key);
+	key = jhash(&pbme->src_port_min, 2, key);
+	key = jhash(&pbme->dst_port_max, 2, key);
+	key = jhash(&pbme->src_port_max, 2, key);
+	key = jhash(&pbme->proto, 1, key);
 
 	return key;
 }
@@ -474,6 +584,21 @@ int bgp_pbr_match_entry_hash_equal(const void *arg1, const void *arg2)
 	if (!prefix_same(&r1->dst, &r2->dst))
 		return 0;
 
+	if (r1->src_port_min != r2->src_port_min)
+		return 0;
+
+	if (r1->dst_port_min != r2->dst_port_min)
+		return 0;
+
+	if (r1->src_port_max != r2->src_port_max)
+		return 0;
+
+	if (r1->dst_port_max != r2->dst_port_max)
+		return 0;
+
+	if (r1->proto != r2->proto)
+		return 0;
+
 	return 1;
 }
 
@@ -497,10 +622,8 @@ int bgp_pbr_action_hash_equal(const void *arg1, const void *arg2)
 
 	/* unique value is self calculated
 	 * table and fwmark is self calculated
+	 * rate is ignored
 	 */
-	if (r1->rate != r2->rate)
-		return 0;
-
 	if (r1->vrf_id != r2->vrf_id)
 		return 0;
 
@@ -587,6 +710,11 @@ void bgp_pbr_cleanup(struct bgp *bgp)
 		hash_free(bgp->pbr_action_hash);
 		bgp->pbr_action_hash = NULL;
 	}
+	if (bgp->bgp_pbr_cfg == NULL)
+		return;
+	bgp_pbr_reset(bgp, AFI_IP);
+	XFREE(MTYPE_PBR, bgp->bgp_pbr_cfg);
+	bgp->bgp_pbr_cfg = NULL;
 }
 
 void bgp_pbr_init(struct bgp *bgp)
@@ -599,6 +727,9 @@ void bgp_pbr_init(struct bgp *bgp)
 		hash_create_size(8, bgp_pbr_action_hash_key,
 				 bgp_pbr_action_hash_equal,
 				 "Match Hash Entry");
+
+	bgp->bgp_pbr_cfg = XCALLOC(MTYPE_PBR, sizeof(struct bgp_pbr_config));
+	bgp->bgp_pbr_cfg->pbr_interface_any_ipv4 = true;
 }
 
 void bgp_pbr_print_policy_route(struct bgp_pbr_entry_main *api)
@@ -749,6 +880,16 @@ static void bgp_pbr_flush_entry(struct bgp *bgp, struct bgp_pbr_action *bpa,
 		bgp_send_pbr_ipset_entry_match(bpme, false);
 		bpme->installed = false;
 		bpme->backpointer = NULL;
+		if (bpme->bgp_info) {
+			struct bgp_info *bgp_info;
+			struct bgp_info_extra *extra;
+
+			/* unlink bgp_info to bpme */
+			bgp_info = (struct bgp_info *)bpme->bgp_info;
+			extra = bgp_info_extra_get(bgp_info);
+			extra->bgp_fs_pbr = NULL;
+			bpme->bgp_info = NULL;
+		}
 	}
 	hash_release(bpm->entry_hash, bpme);
 	if (hashcount(bpm->entry_hash) == 0) {
@@ -777,6 +918,7 @@ static void bgp_pbr_flush_entry(struct bgp *bgp, struct bgp_pbr_action *bpa,
 						   AFI_IP,
 						   bpa->table_id,
 						   false);
+			bpa->installed = false;
 		}
 	}
 }
@@ -813,10 +955,13 @@ static int bgp_pbr_get_remaining_entry(struct hash_backet *backet, void *arg)
 }
 
 static void bgp_pbr_policyroute_remove_from_zebra(struct bgp *bgp,
-						  struct bgp_info *binfo,
-						  vrf_id_t vrf_id,
-						  struct prefix *src,
-						  struct prefix *dst)
+					  struct bgp_info *binfo,
+					  vrf_id_t vrf_id,
+					  struct prefix *src,
+					  struct prefix *dst,
+					  uint8_t protocol,
+					  struct bgp_pbr_range_port *src_port,
+					  struct bgp_pbr_range_port *dst_port)
 {
 	struct bgp_pbr_match temp;
 	struct bgp_pbr_match_entry temp2;
@@ -840,11 +985,35 @@ static void bgp_pbr_policyroute_remove_from_zebra(struct bgp *bgp,
 		prefix_copy(&temp2.dst, dst);
 	} else
 		temp2.dst.family = AF_INET;
-
-	if (src == NULL || dst == NULL)
-		temp.type = IPSET_NET;
-	else
-		temp.type = IPSET_NET_NET;
+	if (src_port) {
+		temp.flags |= MATCH_PORT_SRC_SET;
+		temp2.src_port_min = src_port->min_port;
+		if (src_port->max_port) {
+			temp.flags |= MATCH_PORT_SRC_RANGE_SET;
+			temp2.src_port_max = src_port->max_port;
+		}
+	}
+	if (dst_port) {
+		temp.flags |= MATCH_PORT_DST_SET;
+		temp2.dst_port_min = dst_port->min_port;
+		if (dst_port->max_port) {
+			temp.flags |= MATCH_PORT_DST_RANGE_SET;
+			temp2.dst_port_max = dst_port->max_port;
+		}
+	}
+	temp2.proto = protocol;
+
+	if (src == NULL || dst == NULL) {
+		if (temp.flags & (MATCH_PORT_DST_SET | MATCH_PORT_SRC_SET))
+			temp.type = IPSET_NET_PORT;
+		else
+			temp.type = IPSET_NET;
+	} else {
+		if (temp.flags & (MATCH_PORT_DST_SET | MATCH_PORT_SRC_SET))
+			temp.type = IPSET_NET_PORT_NET;
+		else
+			temp.type = IPSET_NET_NET;
+	}
 	if (vrf_id == VRF_UNKNOWN) /* XXX case BGP destroy */
 		temp.vrf_id = 0;
 	else
@@ -870,12 +1039,15 @@ static void bgp_pbr_policyroute_remove_from_zebra(struct bgp *bgp,
 }
 
 static void bgp_pbr_policyroute_add_to_zebra(struct bgp *bgp,
-					     struct bgp_info *binfo,
-					     vrf_id_t vrf_id,
-					     struct prefix *src,
-					     struct prefix *dst,
-					     struct nexthop *nh,
-					     float *rate)
+				     struct bgp_info *binfo,
+				     vrf_id_t vrf_id,
+				     struct prefix *src,
+				     struct prefix *dst,
+				     struct nexthop *nh,
+				     float *rate,
+				     uint8_t protocol,
+				     struct bgp_pbr_range_port *src_port,
+				     struct bgp_pbr_range_port *dst_port)
 {
 	struct bgp_pbr_match temp;
 	struct bgp_pbr_match_entry temp2;
@@ -913,15 +1085,33 @@ static void bgp_pbr_policyroute_add_to_zebra(struct bgp *bgp,
 
 	/* then look for bpm */
 	memset(&temp, 0, sizeof(temp));
-	if (src == NULL || dst == NULL)
-		temp.type = IPSET_NET;
-	else
-		temp.type = IPSET_NET_NET;
+	if (src == NULL || dst == NULL) {
+		if ((src_port && src_port->min_port) ||
+		    (dst_port && dst_port->min_port))
+			temp.type = IPSET_NET_PORT;
+		else
+			temp.type = IPSET_NET;
+	} else {
+		if ((src_port && src_port->min_port) ||
+		    (dst_port && dst_port->min_port))
+			temp.type = IPSET_NET_PORT_NET;
+		else
+			temp.type = IPSET_NET_NET;
+	}
 	temp.vrf_id = vrf_id;
 	if (src)
 		temp.flags |= MATCH_IP_SRC_SET;
 	if (dst)
 		temp.flags |= MATCH_IP_DST_SET;
+
+	if (src_port && src_port->min_port)
+		temp.flags |= MATCH_PORT_SRC_SET;
+	if (dst_port && dst_port->min_port)
+		temp.flags |= MATCH_PORT_DST_SET;
+	if (src_port && src_port->max_port)
+		temp.flags |= MATCH_PORT_SRC_RANGE_SET;
+	if (dst_port && dst_port->max_port)
+		temp.flags |= MATCH_PORT_DST_RANGE_SET;
 	temp.action = bpa;
 	bpm = hash_get(bgp->pbr_match_hash, &temp,
 		       bgp_pbr_match_alloc_intern);
@@ -953,15 +1143,22 @@ static void bgp_pbr_policyroute_add_to_zebra(struct bgp *bgp,
 		prefix_copy(&temp2.dst, dst);
 	else
 		temp2.dst.family = AF_INET;
+	temp2.src_port_min = src_port ? src_port->min_port : 0;
+	temp2.dst_port_min = dst_port ? dst_port->min_port : 0;
+	temp2.src_port_max = src_port ? src_port->max_port : 0;
+	temp2.dst_port_max = dst_port ? dst_port->max_port : 0;
+	temp2.proto = protocol;
 	if (bpm)
 		bpme = hash_get(bpm->entry_hash, &temp2,
-			bgp_pbr_match_entry_alloc_intern);
+				bgp_pbr_match_entry_alloc_intern);
 	if (bpme && bpme->unique == 0) {
 		bpme->unique = ++bgp_pbr_match_entry_counter_unique;
 		/* 0 value is forbidden */
 		bpme->backpointer = bpm;
 		bpme->installed = false;
 		bpme->install_in_progress = false;
+		/* link bgp info to bpme */
+		bpme->bgp_info = (void *)binfo;
 	}
 
 	/* BGP FS: append entry to zebra
@@ -1021,17 +1218,44 @@ static void bgp_pbr_handle_entry(struct bgp *bgp,
 	int continue_loop = 1;
 	float rate = 0;
 	struct prefix *src = NULL, *dst = NULL;
+	uint8_t proto = 0;
+	struct bgp_pbr_range_port *srcp = NULL, *dstp = NULL;
+	struct bgp_pbr_range_port range;
 
+	memset(&nh, 0, sizeof(struct nexthop));
 	if (api->match_bitmask & PREFIX_SRC_PRESENT)
 		src = &api->src_prefix;
 	if (api->match_bitmask & PREFIX_DST_PRESENT)
 		dst = &api->dst_prefix;
 	memset(&nh, 0, sizeof(struct nexthop));
 	nh.vrf_id = VRF_UNKNOWN;
-
+	if (api->match_protocol_num)
+		proto = (uint8_t)api->protocol[0].value;
+	/* if match_port is selected, then either src or dst port will be parsed
+	 * but not both at the same time
+	 */
+	if (api->match_port_num >= 1) {
+		bgp_pbr_extract(api->port,
+				api->match_port_num,
+				&range);
+		srcp = dstp = &range;
+	} else if (api->match_src_port_num >= 1) {
+		bgp_pbr_extract(api->src_port,
+				api->match_src_port_num,
+				&range);
+		srcp = &range;
+		dstp = NULL;
+	} else if (api->match_dst_port_num >= 1) {
+		bgp_pbr_extract(api->dst_port,
+				api->match_dst_port_num,
+				&range);
+		dstp = &range;
+		srcp = NULL;
+	}
 	if (!add)
 		return bgp_pbr_policyroute_remove_from_zebra(bgp, binfo,
-					     api->vrf_id, src, dst);
+					     api->vrf_id, src, dst,
+					     proto, srcp, dstp);
 	/* no action for add = true */
 	for (i = 0; i < api->action_num; i++) {
 		switch (api->actions[i].action) {
@@ -1042,7 +1266,8 @@ static void bgp_pbr_handle_entry(struct bgp *bgp,
 				nh.type = NEXTHOP_TYPE_BLACKHOLE;
 				bgp_pbr_policyroute_add_to_zebra(bgp, binfo,
 						    api->vrf_id, src, dst,
-						    &nh, &rate);
+						    &nh, &rate, proto,
+						    srcp, dstp);
 			} else {
 				/* update rate. can be reentrant */
 				rate = api->actions[i].u.r.rate;
@@ -1085,7 +1310,8 @@ static void bgp_pbr_handle_entry(struct bgp *bgp,
 			bgp_pbr_policyroute_add_to_zebra(bgp, binfo,
 							    api->vrf_id,
 							    src, dst,
-							    &nh, &rate);
+							    &nh, &rate, proto,
+							    srcp, dstp);
 			/* XXX combination with REDIRECT_VRF
 			 * + REDIRECT_NH_IP not done
 			 */
@@ -1097,7 +1323,8 @@ static void bgp_pbr_handle_entry(struct bgp *bgp,
 			bgp_pbr_policyroute_add_to_zebra(bgp, binfo,
 							 api->vrf_id,
 							 src, dst,
-							 &nh, &rate);
+							 &nh, &rate, proto,
+							 srcp, dstp);
 			continue_loop = 0;
 			break;
 		case ACTION_MARKING:
@@ -1120,6 +1347,7 @@ void bgp_pbr_update_entry(struct bgp *bgp, struct prefix *p,
 			 bool nlri_update)
 {
 	struct bgp_pbr_entry_main api;
+	struct bgp_info_extra *extra = bgp_info_extra_get(info);
 
 	if (afi == AFI_IP6)
 		return; /* IPv6 not supported */
@@ -1130,11 +1358,61 @@ void bgp_pbr_update_entry(struct bgp *bgp, struct prefix *p,
 	api.vrf_id = bgp->vrf_id;
 	api.afi = afi;
 
-	if (bgp_pbr_build_and_validate_entry(p, info, &api) < 0) {
+	if (!bgp_zebra_tm_chunk_obtained()) {
 		if (BGP_DEBUG(pbr, PBR_ERROR))
-			zlog_err("%s: cancel updating entry in bgp pbr",
+			zlog_err("%s: table chunk not obtained yet",
 				 __func__);
 		return;
 	}
+	/* already installed */
+	if (nlri_update && extra->bgp_fs_pbr) {
+		if (BGP_DEBUG(pbr, PBR_ERROR))
+			zlog_err("%s: entry %p already installed in bgp pbr",
+				 __func__, info);
+		return;
+	}
+
+	if (bgp_pbr_build_and_validate_entry(p, info, &api) < 0) {
+		if (BGP_DEBUG(pbr, PBR_ERROR))
+			zlog_err("%s: cancel updating entry %p in bgp pbr",
+				 __func__, info);
+		return;
+	}
 	bgp_pbr_handle_entry(bgp, info, &api, nlri_update);
 }
+
+int bgp_pbr_interface_compare(const struct bgp_pbr_interface *a,
+			  const struct bgp_pbr_interface *b)
+{
+	return strcmp(a->name, b->name);
+}
+
+struct bgp_pbr_interface *bgp_pbr_interface_lookup(const char *name,
+					   struct bgp_pbr_interface_head *head)
+{
+	struct bgp_pbr_interface pbr_if;
+
+	strlcpy(pbr_if.name, name, sizeof(pbr_if.name));
+	return (RB_FIND(bgp_pbr_interface_head,
+			head, &pbr_if));
+}
+
+/* this function resets to the default policy routing
+ * go back to default status
+ */
+void bgp_pbr_reset(struct bgp *bgp, afi_t afi)
+{
+	struct bgp_pbr_config *bgp_pbr_cfg = bgp->bgp_pbr_cfg;
+	struct bgp_pbr_interface_head *head;
+	struct bgp_pbr_interface *pbr_if;
+
+	if (!bgp_pbr_cfg || afi != AFI_IP)
+		return;
+	head = &(bgp_pbr_cfg->ifaces_by_name_ipv4);
+
+	while (!RB_EMPTY(bgp_pbr_interface_head, head)) {
+		pbr_if = RB_ROOT(bgp_pbr_interface_head, head);
+		RB_REMOVE(bgp_pbr_interface_head, head, pbr_if);
+		XFREE(MTYPE_TMP, pbr_if);
+	}
+}
diff --git a/bgpd/bgp_pbr.h b/bgpd/bgp_pbr.h
index 5129ada37..20edaf30b 100644
--- a/bgpd/bgp_pbr.h
+++ b/bgpd/bgp_pbr.h
@@ -123,6 +123,8 @@ struct bgp_pbr_entry_main {
 	struct prefix src_prefix;
 	struct prefix dst_prefix;
 
+#define PROTOCOL_UDP 17
+#define PROTOCOL_TCP 6
 	struct bgp_pbr_match_val protocol[BGP_PBR_MATCH_VAL_MAX];
 	struct bgp_pbr_match_val src_port[BGP_PBR_MATCH_VAL_MAX];
 	struct bgp_pbr_match_val dst_port[BGP_PBR_MATCH_VAL_MAX];
@@ -148,6 +150,25 @@ struct bgp_pbr_entry_main {
 	vrf_id_t vrf_id;
 };
 
+struct bgp_pbr_interface {
+	RB_ENTRY(bgp_pbr_interface) id_entry;
+	char name[INTERFACE_NAMSIZ];
+};
+
+RB_HEAD(bgp_pbr_interface_head, bgp_pbr_interface);
+RB_PROTOTYPE(bgp_pbr_interface_head, bgp_pbr_interface, id_entry,
+	     bgp_pbr_interface_compare);
+
+extern int bgp_pbr_interface_compare(const struct bgp_pbr_interface *a,
+				     const struct bgp_pbr_interface *b);
+
+struct bgp_pbr_config {
+	struct bgp_pbr_interface_head ifaces_by_name_ipv4;
+	bool pbr_interface_any_ipv4;
+};
+
+extern struct bgp_pbr_config *bgp_pbr_cfg;
+
 struct bgp_pbr_match {
 	char ipset_name[ZEBRA_IPSET_NAME_SIZE];
 
@@ -157,6 +178,10 @@ struct bgp_pbr_match {
 
 #define MATCH_IP_SRC_SET		(1 << 0)
 #define MATCH_IP_DST_SET		(1 << 1)
+#define MATCH_PORT_SRC_SET		(1 << 2)
+#define MATCH_PORT_DST_SET		(1 << 3)
+#define MATCH_PORT_SRC_RANGE_SET	(1 << 4)
+#define MATCH_PORT_DST_RANGE_SET	(1 << 5)
 	uint32_t flags;
 
 	vrf_id_t vrf_id;
@@ -189,6 +214,14 @@ struct bgp_pbr_match_entry {
 	struct prefix src;
 	struct prefix dst;
 
+	uint16_t src_port_min;
+	uint16_t src_port_max;
+	uint16_t dst_port_min;
+	uint16_t dst_port_max;
+	uint8_t proto;
+
+	void *bgp_info;
+
 	bool installed;
 	bool install_in_progress;
 };
@@ -253,4 +286,10 @@ extern void bgp_pbr_update_entry(struct bgp *bgp, struct prefix *p,
 				afi_t afi, safi_t safi,
 				bool nlri_update);
 
+/* bgp pbr utilities */
+extern struct bgp_pbr_interface *pbr_interface_lookup(const char *name);
+extern void bgp_pbr_reset(struct bgp *bgp, afi_t afi);
+extern struct bgp_pbr_interface *bgp_pbr_interface_lookup(const char *name,
+				   struct bgp_pbr_interface_head *head);
+
 #endif /* __BGP_PBR_H__ */
diff --git a/bgpd/bgp_route.h b/bgpd/bgp_route.h
index 288b66fb8..72923901b 100644
--- a/bgpd/bgp_route.h
+++ b/bgpd/bgp_route.h
@@ -146,6 +146,8 @@ struct bgp_info_extra {
 	 * Set nexthop_orig.family to 0 if not valid.
 	 */
 	struct prefix nexthop_orig;
+	/* presence of FS pbr entry */
+	void *bgp_fs_pbr;
 };
 
 struct bgp_info {
diff --git a/bgpd/bgp_zebra.c b/bgpd/bgp_zebra.c
index 390eb44eb..7a6b80f3a 100644
--- a/bgpd/bgp_zebra.c
+++ b/bgpd/bgp_zebra.c
@@ -1001,6 +1001,7 @@ static bool bgp_tm_status_connected;
 static bool bgp_tm_chunk_obtained;
 #define BGP_FLOWSPEC_TABLE_CHUNK 100000
 static uint32_t bgp_tm_min, bgp_tm_max, bgp_tm_chunk_size;
+struct bgp *bgp_tm_bgp;
 
 static int bgp_zebra_tm_connect(struct thread *t)
 {
@@ -1024,8 +1025,11 @@ static int bgp_zebra_tm_connect(struct thread *t)
 		if (!bgp_tm_chunk_obtained) {
 			if (bgp_zebra_get_table_range(bgp_tm_chunk_size,
 						      &bgp_tm_min,
-						      &bgp_tm_max) >= 0)
+						      &bgp_tm_max) >= 0) {
 				bgp_tm_chunk_obtained = true;
+				/* parse non installed entries */
+				bgp_zebra_announce_table(bgp_tm_bgp, AFI_IP, SAFI_FLOWSPEC);
+			}
 		}
 	}
 	thread_add_timer(bm->master, bgp_zebra_tm_connect, zclient, delay,
@@ -1033,6 +1037,11 @@ static int bgp_zebra_tm_connect(struct thread *t)
 	return 0;
 }
 
+bool bgp_zebra_tm_chunk_obtained(void)
+{
+	return bgp_tm_chunk_obtained;
+}
+
 uint32_t bgp_zebra_tm_get_id(void)
 {
 	static int table_id;
@@ -1042,7 +1051,7 @@ uint32_t bgp_zebra_tm_get_id(void)
 	return bgp_tm_min++;
 }
 
-void bgp_zebra_init_tm_connect(void)
+void bgp_zebra_init_tm_connect(struct bgp *bgp)
 {
 	int delay = 1;
 
@@ -1054,6 +1063,7 @@ void bgp_zebra_init_tm_connect(void)
 	bgp_tm_chunk_obtained = false;
 	bgp_tm_min = bgp_tm_max = 0;
 	bgp_tm_chunk_size = BGP_FLOWSPEC_TABLE_CHUNK;
+	bgp_tm_bgp = bgp;
 	thread_add_timer(bm->master, bgp_zebra_tm_connect, zclient, delay,
 			 &bgp_tm_thread_connect);
 }
@@ -1962,6 +1972,7 @@ static int rule_notify_owner(int command, struct zclient *zclient,
 			zlog_debug("%s: Received RULE_INSTALLED",
 				   __PRETTY_FUNCTION__);
 		break;
+	case ZAPI_RULE_FAIL_REMOVE:
 	case ZAPI_RULE_REMOVED:
 		if (BGP_DEBUG(zebra, ZEBRA))
 			zlog_debug("%s: Received RULE REMOVED",
@@ -1987,8 +1998,8 @@ static int ipset_notify_owner(int command, struct zclient *zclient,
 	bgp_pbim = bgp_pbr_match_ipset_lookup(vrf_id, unique);
 	if (!bgp_pbim) {
 		if (BGP_DEBUG(zebra, ZEBRA))
-			zlog_debug("%s: Fail to look BGP match (%u)",
-				   __PRETTY_FUNCTION__, unique);
+			zlog_debug("%s: Fail to look BGP match ( %u %u)",
+				   __PRETTY_FUNCTION__, note, unique);
 		return 0;
 	}
 
@@ -2007,6 +2018,7 @@ static int ipset_notify_owner(int command, struct zclient *zclient,
 			zlog_debug("%s: Received IPSET_INSTALLED",
 				   __PRETTY_FUNCTION__);
 		break;
+	case ZAPI_IPSET_FAIL_REMOVE:
 	case ZAPI_IPSET_REMOVED:
 		if (BGP_DEBUG(zebra, ZEBRA))
 			zlog_debug("%s: Received IPSET REMOVED",
@@ -2036,8 +2048,8 @@ static int ipset_entry_notify_owner(int command, struct zclient *zclient,
 						     unique);
 	if (!bgp_pbime) {
 		if (BGP_DEBUG(zebra, ZEBRA))
-			zlog_debug("%s: Fail to look BGP match entry (%u)",
-				   __PRETTY_FUNCTION__, unique);
+			zlog_debug("%s: Fail to look BGP match entry (%u %u)",
+				   __PRETTY_FUNCTION__, note, unique);
 		return 0;
 	}
 
@@ -2050,12 +2062,22 @@ static int ipset_entry_notify_owner(int command, struct zclient *zclient,
 		bgp_pbime->install_in_progress = false;
 		break;
 	case ZAPI_IPSET_ENTRY_INSTALLED:
-		bgp_pbime->installed = true;
-		bgp_pbime->install_in_progress = false;
-		if (BGP_DEBUG(zebra, ZEBRA))
-			zlog_debug("%s: Received IPSET_ENTRY_INSTALLED",
-				   __PRETTY_FUNCTION__);
+		{
+			struct bgp_info *bgp_info;
+			struct bgp_info_extra *extra;
+
+			bgp_pbime->installed = true;
+			bgp_pbime->install_in_progress = false;
+			if (BGP_DEBUG(zebra, ZEBRA))
+				zlog_debug("%s: Received IPSET_ENTRY_INSTALLED",
+					   __PRETTY_FUNCTION__);
+			/* link bgp_info to bpme */
+			bgp_info = (struct bgp_info *)bgp_pbime->bgp_info;
+			extra = bgp_info_extra_get(bgp_info);
+			extra->bgp_fs_pbr = (void *)bgp_pbime;
+		}
 		break;
+	case ZAPI_IPSET_ENTRY_FAIL_REMOVE:
 	case ZAPI_IPSET_ENTRY_REMOVED:
 		if (BGP_DEBUG(zebra, ZEBRA))
 			zlog_debug("%s: Received IPSET_ENTRY_REMOVED",
@@ -2080,8 +2102,8 @@ static int iptable_notify_owner(int command, struct zclient *zclient,
 	bgpm = bgp_pbr_match_iptable_lookup(vrf_id, unique);
 	if (!bgpm) {
 		if (BGP_DEBUG(zebra, ZEBRA))
-			zlog_debug("%s: Fail to look BGP iptable (%u)",
-				   __PRETTY_FUNCTION__, unique);
+			zlog_debug("%s: Fail to look BGP iptable (%u %u)",
+				   __PRETTY_FUNCTION__, note, unique);
 		return 0;
 	}
 	switch (note) {
@@ -2100,6 +2122,7 @@ static int iptable_notify_owner(int command, struct zclient *zclient,
 				   __PRETTY_FUNCTION__);
 		bgpm->action->refcnt++;
 		break;
+	case ZAPI_IPTABLE_FAIL_REMOVE:
 	case ZAPI_IPTABLE_REMOVED:
 		if (BGP_DEBUG(zebra, ZEBRA))
 			zlog_debug("%s: Received IPTABLE REMOVED",
@@ -2167,6 +2190,12 @@ static void bgp_encode_pbr_ipset_entry_match(struct stream *s,
 	stream_putc(s, pbime->dst.family);
 	stream_putc(s, pbime->dst.prefixlen);
 	stream_put(s, &pbime->dst.u.prefix, prefix_blen(&pbime->dst));
+
+	stream_putw(s, pbime->src_port_min);
+	stream_putw(s, pbime->src_port_max);
+	stream_putw(s, pbime->dst_port_min);
+	stream_putw(s, pbime->dst_port_max);
+	stream_putc(s, pbime->proto);
 }
 
 static void bgp_encode_pbr_iptable_match(struct stream *s,
@@ -2490,8 +2519,10 @@ void bgp_send_pbr_rule_action(struct bgp_pbr_action *pbra, bool install)
 
 	if (pbra->install_in_progress)
 		return;
-	zlog_debug("%s: table %d fwmark %d %d", __PRETTY_FUNCTION__,
-		   pbra->table_id, pbra->fwmark, install);
+	if (BGP_DEBUG(zebra, ZEBRA))
+		zlog_debug("%s: table %d fwmark %d %d",
+			   __PRETTY_FUNCTION__,
+			   pbra->table_id, pbra->fwmark, install);
 	s = zclient->obuf;
 	stream_reset(s);
 
@@ -2513,8 +2544,10 @@ void bgp_send_pbr_ipset_match(struct bgp_pbr_match *pbrim, bool install)
 
 	if (pbrim->install_in_progress)
 		return;
-	zlog_debug("%s: name %s type %d %d", __PRETTY_FUNCTION__,
-		   pbrim->ipset_name, pbrim->type, install);
+	if (BGP_DEBUG(zebra, ZEBRA))
+		zlog_debug("%s: name %s type %d %d",
+			   __PRETTY_FUNCTION__,
+			   pbrim->ipset_name, pbrim->type, install);
 	s = zclient->obuf;
 	stream_reset(s);
 
@@ -2539,9 +2572,10 @@ void bgp_send_pbr_ipset_entry_match(struct bgp_pbr_match_entry *pbrime,
 
 	if (pbrime->install_in_progress)
 		return;
-	zlog_debug("%s: name %s %d %d", __PRETTY_FUNCTION__,
-		   pbrime->backpointer->ipset_name,
-		   pbrime->unique, install);
+	if (BGP_DEBUG(zebra, ZEBRA))
+		zlog_debug("%s: name %s %d %d", __PRETTY_FUNCTION__,
+			   pbrime->backpointer->ipset_name,
+			   pbrime->unique, install);
 	s = zclient->obuf;
 	stream_reset(s);
 
@@ -2559,16 +2593,56 @@ void bgp_send_pbr_ipset_entry_match(struct bgp_pbr_match_entry *pbrime,
 		pbrime->install_in_progress = true;
 }
 
+static void bgp_encode_pbr_interface_list(struct bgp *bgp, struct stream *s)
+{
+	struct bgp_pbr_config *bgp_pbr_cfg = bgp->bgp_pbr_cfg;
+	struct bgp_pbr_interface_head *head;
+	struct bgp_pbr_interface *pbr_if;
+	struct interface *ifp;
+
+	if (!bgp_pbr_cfg)
+		return;
+	head = &(bgp_pbr_cfg->ifaces_by_name_ipv4);
+
+	RB_FOREACH (pbr_if, bgp_pbr_interface_head, head) {
+		ifp = if_lookup_by_name(pbr_if->name, bgp->vrf_id);
+		if (ifp)
+			stream_putl(s, ifp->ifindex);
+	}
+}
+
+static int bgp_pbr_get_ifnumber(struct bgp *bgp)
+{
+	struct bgp_pbr_config *bgp_pbr_cfg = bgp->bgp_pbr_cfg;
+	struct bgp_pbr_interface_head *head;
+	struct bgp_pbr_interface *pbr_if;
+	int cnt = 0;
+
+	if (!bgp_pbr_cfg)
+		return 0;
+	head = &(bgp_pbr_cfg->ifaces_by_name_ipv4);
+
+	RB_FOREACH (pbr_if, bgp_pbr_interface_head, head) {
+		if (if_lookup_by_name(pbr_if->name, bgp->vrf_id))
+			cnt++;
+	}
+	return cnt;
+}
+
 void bgp_send_pbr_iptable(struct bgp_pbr_action *pba,
 			  struct bgp_pbr_match *pbm,
 			  bool install)
 {
 	struct stream *s;
+	int ret = 0;
+	int nb_interface;
 
 	if (pbm->install_iptable_in_progress)
 		return;
-	zlog_debug("%s: name %s type %d mark %d %d", __PRETTY_FUNCTION__,
-		   pbm->ipset_name, pbm->type, pba->fwmark, install);
+	if (BGP_DEBUG(zebra, ZEBRA))
+		zlog_debug("%s: name %s type %d mark %d %d",
+			   __PRETTY_FUNCTION__, pbm->ipset_name,
+			   pbm->type, pba->fwmark, install);
 	s = zclient->obuf;
 	stream_reset(s);
 
@@ -2578,11 +2652,17 @@ void bgp_send_pbr_iptable(struct bgp_pbr_action *pba,
 			      VRF_DEFAULT);
 
 	bgp_encode_pbr_iptable_match(s, pba, pbm);
-
+	nb_interface = bgp_pbr_get_ifnumber(pba->bgp);
+	stream_putl(s, nb_interface);
+	if (nb_interface)
+		bgp_encode_pbr_interface_list(pba->bgp, s);
 	stream_putw_at(s, 0, stream_get_endp(s));
-	if (!zclient_send_message(zclient) && install) {
-		pbm->install_iptable_in_progress = true;
-		pba->refcnt++;
+	ret = zclient_send_message(zclient);
+	if (install) {
+		if (ret)
+			pba->refcnt++;
+		else
+			pbm->install_iptable_in_progress = true;
 	}
 }
 
@@ -2626,7 +2706,8 @@ void bgp_zebra_announce_default(struct bgp *bgp, struct nexthop *nh,
 
 		inet_ntop(AF_INET, &(nh->gate.ipv4), buff, INET_ADDRSTRLEN);
 		if (BGP_DEBUG(zebra, ZEBRA))
-			zlog_info("BGP: sending default route to %s table %d (redirect IP)",
+			zlog_info("BGP: %s default route to %s table %d (redirect IP)",
+				  announce ? "adding" : "withdrawing",
 				  buff, table_id);
 		zclient_route_send(announce ? ZEBRA_ROUTE_ADD
 				   : ZEBRA_ROUTE_DELETE,
@@ -2648,7 +2729,8 @@ void bgp_zebra_announce_default(struct bgp *bgp, struct nexthop *nh,
 		api_nh->type = NEXTHOP_TYPE_IFINDEX;
 		api_nh->ifindex = ifp->ifindex;
 		if (BGP_DEBUG(zebra, ZEBRA))
-			zlog_info("BGP: sending default route to %s table %d (redirect VRF)",
+			zlog_info("BGP: %s default route to %s table %d (redirect VRF)",
+				  announce ? "adding" : "withdrawing",
 				  vrf->name, table_id);
 		zclient_route_send(announce ? ZEBRA_ROUTE_ADD
 				   : ZEBRA_ROUTE_DELETE,
diff --git a/bgpd/bgp_zebra.h b/bgpd/bgp_zebra.h
index 7ac40fecf..e3c88b9db 100644
--- a/bgpd/bgp_zebra.h
+++ b/bgpd/bgp_zebra.h
@@ -24,8 +24,9 @@
 #include "vxlan.h"
 
 extern void bgp_zebra_init(struct thread_master *master);
-extern void bgp_zebra_init_tm_connect(void);
+extern void bgp_zebra_init_tm_connect(struct bgp *bgp);
 extern uint32_t bgp_zebra_tm_get_id(void);
+extern bool bgp_zebra_tm_chunk_obtained(void);
 extern void bgp_zebra_destroy(void);
 extern int bgp_zebra_get_table_range(uint32_t chunk_size,
 				     uint32_t *start, uint32_t *end);
diff --git a/bgpd/bgpd.c b/bgpd/bgpd.c
index 71707b6af..69297cd3e 100644
--- a/bgpd/bgpd.c
+++ b/bgpd/bgpd.c
@@ -1988,7 +1988,7 @@ int peer_activate(struct peer *peer, afi_t afi, safi_t safi)
 
 	if (safi == SAFI_FLOWSPEC) {
 		/* connect to table manager */
-		bgp_zebra_init_tm_connect();
+		bgp_zebra_init_tm_connect(bgp);
 	}
 	return ret;
 }
@@ -7266,6 +7266,9 @@ static void bgp_config_write_family(struct vty *vty, struct bgp *bgp, afi_t afi,
 	if (safi == SAFI_EVPN)
 		bgp_config_write_evpn_info(vty, bgp, afi, safi);
 
+	if (safi == SAFI_FLOWSPEC)
+		bgp_fs_config_write_pbr(vty, bgp, afi, safi);
+
 	if (safi == SAFI_UNICAST) {
 		bgp_vpn_policy_config_write_afi(vty, bgp, afi);
 		if (CHECK_FLAG(bgp->af_flags[afi][safi],
diff --git a/bgpd/bgpd.h b/bgpd/bgpd.h
index 340851e8d..f663df162 100644
--- a/bgpd/bgpd.h
+++ b/bgpd/bgpd.h
@@ -47,6 +47,7 @@
 
 struct update_subgroup;
 struct bpacket;
+struct bgp_pbr_config;
 
 /*
  * Allow the neighbor XXXX remote-as to take internal or external
@@ -531,6 +532,8 @@ struct bgp {
 
 	struct vpn_policy vpn_policy[AFI_MAX];
 
+	struct bgp_pbr_config *bgp_pbr_cfg;
+
 	QOBJ_FIELDS
 };
 DECLARE_QOBJ_TYPE(bgp)