zebra: uplink tracking and startup delay for EVPN-MH

Local ethernet segments are held in a protodown or error-disabled state
if access to the VxLAN overlay is not ready -
1. When FRR comes up the local-ESs/access-port are kept protodown
for the startup-delay duration. During this time the underlay and
EVPN routes via it are expected to converge.
2. When all the uplinks/core-links attached to the underlay go down
the access-ports are similarly protodowned.

The ES-bond protodown state is propagated to each ES-bond member
and programmed in the dataplane/kernel (per-bond-member).

Configuring uplinks -
vtysh -c "conf t" vtysh -c "interface swp4" vtysh -c "evpn mh uplink"

Configuring startup delay -
vtysh -c "conf t" vtysh -c "evpn mh startup-delay 100"

>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
EVPN protodown display -
========================
root@torm-11:mgmt:~# vtysh -c "show evpn"
L2 VNIs: 10
L3 VNIs: 3
Advertise gateway mac-ip: No
Advertise svi mac-ip: No
Duplicate address detection: Disable
  Detection max-moves 5, time 180
EVPN MH:
  mac-holdtime: 60s, neigh-holdtime: 60s
  startup-delay: 180s, start-delay-timer: 00:01:14 <<<<<<<<<<<<
  uplink-cfg-cnt: 4, uplink-active-cnt: 4
  protodown: startup-delay <<<<<<<<<<<<<<<<<<<<<<<
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
ES-bond protodown display -
===========================
root@torm-11:mgmt:~# vtysh -c "show interface hostbond1"
Interface hostbond1 is up, line protocol is down
  Link ups:       0    last: (never)
  Link downs:     1    last: 2020/04/26 20:38:03.53
  PTM status: disabled
  vrf: default
  OS Description: Local Node/s torm-11 and Ports swp5 <==> Remote  Node/s hostd-11 and Ports swp1
  index 58 metric 0 mtu 9152 speed 4294967295
  flags: <UP,BROADCAST,MULTICAST>
  Type: Ethernet
  HWaddr: 00:02:00:00:00:35
  Interface Type bond
  Master interface: bridge
  EVPN-MH: ES id 1 ES sysmac 00:00:00:00:01:11
  protodown: off rc: startup-delay  <<<<<<<<<<<<<<<<<
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
ES-bond member protodown display -
==================================
root@torm-11:mgmt:~# vtysh -c "show interface swp5"
Interface swp5 is up, line protocol is down
  Link ups:       0    last: (never)
  Link downs:     3    last: 2020/04/26 20:38:03.52
  PTM status: disabled
  vrf: default
  index 7 metric 0 mtu 9152 speed 10000
  flags: <UP,BROADCAST,MULTICAST>
  Type: Ethernet
  HWaddr: 00:02:00:00:00:35
  Interface Type Other
  Master interface: hostbond1
  protodown: on rc: startup-delay <<<<<<<<<<<<<<<<
root@torm-11:mgmt:~#
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

Signed-off-by: Anuradha Karuppiah <anuradhak@cumulusnetworks.com>

10 files changed, 744 insertions, 40 deletions

diff --git a/zebra/if_netlink.c b/zebra/if_netlink.c
index 90a08bbd6..a68873882 100644
--- a/zebra/if_netlink.c
+++ b/zebra/if_netlink.c
@@ -691,6 +691,36 @@ static int netlink_bridge_interface(struct nlmsghdr *h, int len, ns_id_t ns_id,
 	return 0;
 }
 
+/* If the interface is and es bond member then it must follow EVPN's
+ * protodown setting
+ */
+static void netlink_proc_dplane_if_protodown(struct zebra_if *zif,
+					     bool protodown)
+{
+	bool zif_protodown;
+
+	zif_protodown = !!(zif->flags & ZIF_FLAG_PROTODOWN);
+	if (protodown == zif_protodown)
+		return;
+
+	if (IS_ZEBRA_DEBUG_EVPN_MH_ES || IS_ZEBRA_DEBUG_KERNEL)
+		zlog_debug("interface %s dplane change, protdown %s",
+			   zif->ifp->name, protodown ? "on" : "off");
+
+	if (zebra_evpn_is_es_bond_member(zif->ifp)) {
+		if (IS_ZEBRA_DEBUG_EVPN_MH_ES || IS_ZEBRA_DEBUG_KERNEL)
+			zlog_debug(
+				"bond mbr %s re-instate protdown %s in the dplane",
+				zif->ifp->name, zif_protodown ? "on" : "off");
+		netlink_protodown(zif->ifp, zif_protodown);
+	} else {
+		if (protodown)
+			zif->flags |= ZIF_FLAG_PROTODOWN;
+		else
+			zif->flags &= ~ZIF_FLAG_PROTODOWN;
+	}
+}
+
 /*
  * Called from interface_lookup_netlink().  This function is only used
  * during bootstrap.
@@ -849,11 +879,20 @@ static int netlink_interface(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 	 */
 	netlink_interface_update_l2info(ifp, linkinfo[IFLA_INFO_DATA],
 					1, link_nsid);
+	if (IS_ZEBRA_IF_BOND(ifp))
+		zebra_l2if_update_bond(ifp, true);
 	if (IS_ZEBRA_IF_BRIDGE_SLAVE(ifp))
 		zebra_l2if_update_bridge_slave(ifp, bridge_ifindex, ns_id);
 	else if (IS_ZEBRA_IF_BOND_SLAVE(ifp))
 		zebra_l2if_update_bond_slave(ifp, bond_ifindex);
 
+	if (tb[IFLA_PROTO_DOWN]) {
+		uint8_t protodown;
+
+		protodown = *(uint8_t *)RTA_DATA(tb[IFLA_PROTO_DOWN]);
+		netlink_proc_dplane_if_protodown(zif, !!protodown);
+	}
+
 	return 0;
 }
 
@@ -1284,6 +1323,7 @@ int netlink_link_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 	uint8_t old_hw_addr[INTERFACE_HWADDR_MAX];
 	struct zebra_if *zif;
 	ns_id_t link_nsid = ns_id;
+	ifindex_t master_infindex = IFINDEX_INTERNAL;
 
 	zns = zebra_ns_lookup(ns_id);
 	ifi = NLMSG_DATA(h);
@@ -1373,16 +1413,17 @@ int netlink_link_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 			if (slave_kind && (strcmp(slave_kind, "vrf") == 0)
 			    && !vrf_is_backend_netns()) {
 				zif_slave_type = ZEBRA_IF_SLAVE_VRF;
-				vrf_id = *(uint32_t *)RTA_DATA(tb[IFLA_MASTER]);
+				master_infindex = vrf_id =
+					*(uint32_t *)RTA_DATA(tb[IFLA_MASTER]);
 			} else if (slave_kind
 				   && (strcmp(slave_kind, "bridge") == 0)) {
 				zif_slave_type = ZEBRA_IF_SLAVE_BRIDGE;
-				bridge_ifindex =
+				master_infindex = bridge_ifindex =
 					*(ifindex_t *)RTA_DATA(tb[IFLA_MASTER]);
 			} else if (slave_kind
 				   && (strcmp(slave_kind, "bond") == 0)) {
 				zif_slave_type = ZEBRA_IF_SLAVE_BOND;
-				bond_ifindex =
+				master_infindex = bond_ifindex =
 					*(ifindex_t *)RTA_DATA(tb[IFLA_MASTER]);
 			} else
 				zif_slave_type = ZEBRA_IF_SLAVE_OTHER;
@@ -1396,7 +1437,7 @@ int netlink_link_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 				zlog_debug(
 					"RTM_NEWLINK ADD for %s(%u) vrf_id %u type %d sl_type %d master %u flags 0x%x",
 					name, ifi->ifi_index, vrf_id, zif_type,
-					zif_slave_type, bridge_ifindex,
+					zif_slave_type, master_infindex,
 					ifi->ifi_flags);
 
 			if (ifp == NULL) {
@@ -1446,6 +1487,15 @@ int netlink_link_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 							       ns_id);
 			else if (IS_ZEBRA_IF_BOND_SLAVE(ifp))
 				zebra_l2if_update_bond_slave(ifp, bond_ifindex);
+
+			if (tb[IFLA_PROTO_DOWN]) {
+				uint8_t protodown;
+
+				protodown = *(uint8_t *)RTA_DATA(
+					tb[IFLA_PROTO_DOWN]);
+				netlink_proc_dplane_if_protodown(ifp->info,
+								 !!protodown);
+			}
 		} else if (ifp->vrf_id != vrf_id) {
 			/* VRF change for an interface. */
 			if (IS_ZEBRA_DEBUG_KERNEL)
@@ -1463,7 +1513,7 @@ int netlink_link_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 				zlog_debug(
 					"RTM_NEWLINK update for %s(%u) sl_type %d master %u flags 0x%x",
 					name, ifp->ifindex, zif_slave_type,
-					bridge_ifindex, ifi->ifi_flags);
+					master_infindex, ifi->ifi_flags);
 
 			set_ifindex(ifp, ifi->ifi_index, zns);
 			if (!tb[IFLA_MTU]) {
@@ -1542,12 +1592,23 @@ int netlink_link_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 			netlink_interface_update_l2info(
 				ifp, linkinfo[IFLA_INFO_DATA],
 				0, link_nsid);
+			if (IS_ZEBRA_IF_BOND(ifp))
+				zebra_l2if_update_bond(ifp, true);
 			if (IS_ZEBRA_IF_BRIDGE_SLAVE(ifp) || was_bridge_slave)
 				zebra_l2if_update_bridge_slave(ifp,
 							       bridge_ifindex,
 							       ns_id);
 			else if (IS_ZEBRA_IF_BOND_SLAVE(ifp) || was_bond_slave)
 				zebra_l2if_update_bond_slave(ifp, bond_ifindex);
+
+			if (tb[IFLA_PROTO_DOWN]) {
+				uint8_t protodown;
+
+				protodown = *(uint8_t *)RTA_DATA(
+					tb[IFLA_PROTO_DOWN]);
+				netlink_proc_dplane_if_protodown(ifp->info,
+								 !!protodown);
+			}
 		}
 
 		zif = ifp->info;
@@ -1572,6 +1633,8 @@ int netlink_link_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 
 		UNSET_FLAG(ifp->status, ZEBRA_INTERFACE_VRF_LOOPBACK);
 
+		if (IS_ZEBRA_IF_BOND(ifp))
+			zebra_l2if_update_bond(ifp, false);
 		/* Special handling for bridge or VxLAN interfaces. */
 		if (IS_ZEBRA_IF_BRIDGE(ifp))
 			zebra_l2_bridge_del(ifp);
diff --git a/zebra/interface.c b/zebra/interface.c
index 3b7d6f243..67be78193 100644
--- a/zebra/interface.c
+++ b/zebra/interface.c
@@ -1069,6 +1069,9 @@ void if_up(struct interface *ifp)
 
 	if (zif->es_info.es)
 		zebra_evpn_es_if_oper_state_change(zif, true /*up*/);
+
+	if (zif->flags & ZIF_FLAG_EVPN_MH_UPLINK)
+		zebra_evpn_mh_uplink_oper_update(zif);
 }
 
 /* Interface goes down.  We have to manage different behavior of based
@@ -1106,6 +1109,9 @@ void if_down(struct interface *ifp)
 	if (zif->es_info.es)
 		zebra_evpn_es_if_oper_state_change(zif, false /*up*/);
 
+	if (zif->flags & ZIF_FLAG_EVPN_MH_UPLINK)
+		zebra_evpn_mh_uplink_oper_update(zif);
+
 	/* Notify to the protocol daemons. */
 	zebra_interface_down_update(ifp);
 
@@ -1156,6 +1162,18 @@ void zebra_if_update_all_links(void)
 		if (!ifp)
 			continue;
 		zif = ifp->info;
+		/* update bond-member to bond linkages */
+		if ((IS_ZEBRA_IF_BOND_SLAVE(ifp))
+		    && (zif->bondslave_info.bond_ifindex != IFINDEX_INTERNAL)
+		    && !zif->bondslave_info.bond_if) {
+			if (IS_ZEBRA_DEBUG_EVPN_MH_ES || IS_ZEBRA_DEBUG_KERNEL)
+				zlog_debug("bond mbr %s map to bond %d",
+					   zif->ifp->name,
+					   zif->bondslave_info.bond_ifindex);
+			zebra_l2_map_slave_to_bond(zif, ifp->vrf_id);
+		}
+
+		/* update SVI linkages */
 		if ((zif->link_ifindex != IFINDEX_INTERNAL) && !zif->link) {
 			zif->link = if_lookup_by_index_per_ns(ns,
 							 zif->link_ifindex);
@@ -1382,6 +1400,40 @@ static void ifs_dump_brief_vty(struct vty *vty, struct vrf *vrf)
 	vty_out(vty, "\n");
 }
 
+char *zebra_protodown_rc_str(enum protodown_reasons protodown_rc, char *pd_buf,
+			     uint32_t pd_buf_len)
+{
+	bool first = true;
+
+	pd_buf[0] = '\0';
+
+	snprintf(pd_buf + strlen(pd_buf), pd_buf_len - strlen(pd_buf), "(");
+
+	if (protodown_rc & ZEBRA_PROTODOWN_EVPN_STARTUP_DELAY) {
+		if (first)
+			first = false;
+		else
+			snprintf(pd_buf + strlen(pd_buf),
+				 pd_buf_len - strlen(pd_buf), ",");
+		snprintf(pd_buf + strlen(pd_buf), pd_buf_len - strlen(pd_buf),
+			 "startup-delay");
+	}
+
+	if (protodown_rc & ZEBRA_PROTODOWN_EVPN_UPLINK_DOWN) {
+		if (first)
+			first = false;
+		else
+			snprintf(pd_buf + strlen(pd_buf),
+				 pd_buf_len - strlen(pd_buf), ",");
+		snprintf(pd_buf + strlen(pd_buf), pd_buf_len - strlen(pd_buf),
+			 "uplinks-down");
+	}
+
+	snprintf(pd_buf + strlen(pd_buf), pd_buf_len - strlen(pd_buf), ")");
+
+	return pd_buf;
+}
+
 /* Interface's information print out to vty interface. */
 static void if_dump_vty(struct vty *vty, struct interface *ifp)
 {
@@ -1391,6 +1443,7 @@ static void if_dump_vty(struct vty *vty, struct interface *ifp)
 	struct route_node *rn;
 	struct zebra_if *zebra_if;
 	struct vrf *vrf;
+	char pd_buf[ZEBRA_PROTODOWN_RC_STR_LEN];
 
 	zebra_if = ifp->info;
 
@@ -1545,6 +1598,14 @@ static void if_dump_vty(struct vty *vty, struct interface *ifp)
 	}
 
 	zebra_evpn_if_es_print(vty, zebra_if);
+	vty_out(vty, "  protodown: %s",
+		(zebra_if->flags & ZIF_FLAG_PROTODOWN) ? "on" : "off");
+	if (zebra_if->protodown_rc)
+		vty_out(vty, " rc: %s\n",
+			zebra_protodown_rc_str(zebra_if->protodown_rc, pd_buf,
+					       sizeof(pd_buf)));
+	else
+		vty_out(vty, "\n");
 
 	if (zebra_if->link_ifindex != IFINDEX_INTERNAL) {
 		if (zebra_if->link)
diff --git a/zebra/interface.h b/zebra/interface.h
index 2791edf2f..a925dcc96 100644
--- a/zebra/interface.h
+++ b/zebra/interface.h
@@ -29,6 +29,7 @@
 
 #include "zebra/zebra_l2.h"
 #include "zebra/zebra_nhg_private.h"
+#include "zebra/zebra_router.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -284,11 +285,24 @@ struct zebra_es_if_info {
 	struct zebra_evpn_es *es; /* local ES */
 };
 
+enum zebra_if_flags {
+	/* device has been configured as an uplink for
+	 * EVPN multihoming
+	 */
+	ZIF_FLAG_EVPN_MH_UPLINK = (1 << 0),
+	ZIF_FLAG_EVPN_MH_UPLINK_OPER_UP = (1 << 1),
+
+	/* Dataplane protodown-on */
+	ZIF_FLAG_PROTODOWN = (1 << 2)
+};
+
 /* `zebra' daemon local interface structure. */
 struct zebra_if {
 	/* back pointer to the interface */
 	struct interface *ifp;
 
+	enum zebra_if_flags flags;
+
 	/* Shutdown configuration. */
 	uint8_t shutdown;
 
@@ -352,6 +366,7 @@ struct zebra_if {
 	struct zebra_l2info_brslave brslave_info;
 
 	struct zebra_l2info_bondslave bondslave_info;
+	struct zebra_l2info_bond bond_info;
 
 	/* ethernet segment */
 	struct zebra_es_if_info es_info;
@@ -359,6 +374,14 @@ struct zebra_if {
 	/* bitmap of vlans associated with this interface */
 	bitfield_t vlan_bitmap;
 
+	/* An interface can be error-disabled if a protocol (such as EVPN or
+	 * VRRP) detects a problem with keeping it operationally-up.
+	 * If any of the protodown bits are set protodown-on is programmed
+	 * in the dataplane. This results in a carrier/L1 down on the
+	 * physical device.
+	 */
+	enum protodown_reasons protodown_rc;
+
 	/* Link fields - for sub-interfaces. */
 	ifindex_t link_ifindex;
 	struct interface *link;
@@ -400,6 +423,9 @@ DECLARE_HOOK(zebra_if_config_wr, (struct vty * vty, struct interface *ifp),
 #define IS_ZEBRA_IF_VETH(ifp)                                               \
 	(((struct zebra_if *)(ifp->info))->zif_type == ZEBRA_IF_VETH)
 
+#define IS_ZEBRA_IF_BOND(ifp)                                                  \
+	(((struct zebra_if *)(ifp->info))->zif_type == ZEBRA_IF_BOND)
+
 #define IS_ZEBRA_IF_BRIDGE_SLAVE(ifp)					\
 	(((struct zebra_if *)(ifp->info))->zif_slave_type                      \
 	 == ZEBRA_IF_SLAVE_BRIDGE)
@@ -463,6 +489,10 @@ extern unsigned int if_nhg_dependents_count(const struct interface *ifp);
 extern bool if_nhg_dependents_is_empty(const struct interface *ifp);
 
 extern void vrf_add_update(struct vrf *vrfp);
+extern void zebra_l2_map_slave_to_bond(struct zebra_if *zif, vrf_id_t vrf);
+extern void zebra_l2_unmap_slave_from_bond(struct zebra_if *zif);
+extern char *zebra_protodown_rc_str(enum protodown_reasons protodown_rc,
+				    char *pd_buf, uint32_t pd_buf_len);
 
 #ifdef HAVE_PROC_NET_DEV
 extern void ifstat_update_proc(void);
diff --git a/zebra/zebra_evpn_mh.c b/zebra/zebra_evpn_mh.c
index 7a383ef8c..692dfca5d 100644
--- a/zebra/zebra_evpn_mh.c
+++ b/zebra/zebra_evpn_mh.c
@@ -38,6 +38,7 @@
 #include "zebra/rib.h"
 #include "zebra/rt.h"
 #include "zebra/rt_netlink.h"
+#include "zebra/if_netlink.h"
 #include "zebra/zebra_errors.h"
 #include "zebra/zebra_l2.h"
 #include "zebra/zebra_memory.h"
@@ -65,6 +66,9 @@ static int zebra_evpn_local_es_update(struct zebra_if *zif, uint32_t lid,
 		struct ethaddr *sysmac);
 static bool zebra_evpn_es_br_port_dplane_update(struct zebra_evpn_es *es,
 						const char *caller);
+static void zebra_evpn_mh_uplink_cfg_update(struct zebra_if *zif, bool set);
+static void zebra_evpn_mh_update_protodown_es(struct zebra_evpn_es *es);
+static void zebra_evpn_mh_clear_protodown_es(struct zebra_evpn_es *es);
 
 esi_t zero_esi_buf, *zero_esi = &zero_esi_buf;
 
@@ -1713,6 +1717,9 @@ static void zebra_evpn_es_local_info_set(struct zebra_evpn_es *es,
 	 */
 	zebra_evpn_es_local_mac_update(es,
 			false /* force_clear_static */);
+
+	/* inherit EVPN protodown flags on the access port */
+	zebra_evpn_mh_update_protodown_es(es);
 }
 
 static void zebra_evpn_es_local_info_clear(struct zebra_evpn_es **esp)
@@ -1729,6 +1736,9 @@ static void zebra_evpn_es_local_info_clear(struct zebra_evpn_es **esp)
 	/* remove the DF filter */
 	dplane_updated = zebra_evpn_es_run_df_election(es, __func__);
 
+	/* clear EVPN protodown flags on the access port */
+	zebra_evpn_mh_clear_protodown_es(es);
+
 	/* if there any local macs referring to the ES as dest we
 	 * need to clear the static reference on them
 	 */
@@ -2144,12 +2154,34 @@ bool zebra_evpn_is_if_es_capable(struct zebra_if *zif)
 void zebra_evpn_if_es_print(struct vty *vty, struct zebra_if *zif)
 {
 	char buf[ETHER_ADDR_STRLEN];
+	char mh_buf[80];
+	bool vty_print = false;
+
+	mh_buf[0] = '\0';
+	snprintf(mh_buf + strlen(mh_buf), sizeof(mh_buf) - strlen(mh_buf),
+		 "  EVPN-MH:");
+	if (zif->es_info.lid || !is_zero_mac(&zif->es_info.sysmac)) {
+		vty_print = true;
+		snprintf(
+			mh_buf + strlen(mh_buf),
+			sizeof(mh_buf) - strlen(mh_buf),
+			" ES id %u ES sysmac %s", zif->es_info.lid,
+			prefix_mac2str(&zif->es_info.sysmac, buf, sizeof(buf)));
+	}
+
+	if (zif->flags & ZIF_FLAG_EVPN_MH_UPLINK) {
+		vty_print = true;
+		if (zif->flags & ZIF_FLAG_EVPN_MH_UPLINK_OPER_UP)
+			snprintf(mh_buf + strlen(mh_buf),
+				 sizeof(mh_buf) - strlen(mh_buf), " uplink-up");
+		else
+			snprintf(mh_buf + strlen(mh_buf),
+				 sizeof(mh_buf) - strlen(mh_buf),
+				 " uplink-down");
+	}
 
-	if (zif->es_info.lid || !is_zero_mac(&zif->es_info.sysmac))
-		vty_out(vty, "  EVPN MH: ES id %u ES sysmac %s\n",
-				zif->es_info.lid,
-				prefix_mac2str(&zif->es_info.sysmac,
-					buf, sizeof(buf)));
+	if (vty_print)
+		vty_out(vty, "%s\n", mh_buf);
 }
 
 void zebra_evpn_es_if_oper_state_change(struct zebra_if *zif, bool up)
@@ -2478,6 +2510,9 @@ int zebra_evpn_mh_if_write(struct vty *vty, struct interface *ifp)
 	if (zif->es_info.df_pref)
 		vty_out(vty, " evpn mh es-df-pref %u\n", zif->es_info.df_pref);
 
+	if (zif->flags & ZIF_FLAG_EVPN_MH_UPLINK)
+		vty_out(vty, " evpn mh uplink\n");
+
 	return 0;
 }
 
@@ -2600,6 +2635,70 @@ DEFPY(zebra_evpn_es_id,
 	return CMD_SUCCESS;
 }
 
+/* CLI for tagging an interface as an uplink */
+DEFPY(zebra_evpn_mh_uplink, zebra_evpn_mh_uplink_cmd, "[no] evpn mh uplink",
+      NO_STR "EVPN\n" EVPN_MH_VTY_STR "uplink to the VxLAN core\n")
+{
+	VTY_DECLVAR_CONTEXT(interface, ifp);
+	struct zebra_if *zif;
+
+	zif = ifp->info;
+	zebra_evpn_mh_uplink_cfg_update(zif, no ? false : true);
+
+	return CMD_SUCCESS;
+}
+
+void zebra_evpn_mh_json(json_object *json)
+{
+	json_object *json_array;
+	char thread_buf[THREAD_TIMER_STRLEN];
+
+	json_object_int_add(json, "macHoldtime", zmh_info->mac_hold_time);
+	json_object_int_add(json, "neighHoldtime", zmh_info->neigh_hold_time);
+	json_object_int_add(json, "startupDelay", zmh_info->startup_delay_time);
+	json_object_string_add(
+		json, "startupDelayTimer",
+		thread_timer_to_hhmmss(thread_buf, sizeof(thread_buf),
+				       zmh_info->startup_delay_timer));
+	json_object_int_add(json, "uplinkConfigCount",
+			    zmh_info->uplink_cfg_cnt);
+	json_object_int_add(json, "uplinkActiveCount",
+			    zmh_info->uplink_oper_up_cnt);
+
+	if (zmh_info->protodown_rc) {
+		json_array = json_object_new_array();
+		if (zmh_info->protodown_rc & ZEBRA_PROTODOWN_EVPN_STARTUP_DELAY)
+			json_object_array_add(
+				json_array,
+				json_object_new_string("startupDelay"));
+		if (zmh_info->protodown_rc & ZEBRA_PROTODOWN_EVPN_UPLINK_DOWN)
+			json_object_array_add(
+				json_array,
+				json_object_new_string("uplinkDown"));
+		json_object_object_add(json, "protodownReasons", json_array);
+	}
+}
+
+void zebra_evpn_mh_print(struct vty *vty)
+{
+	char pd_buf[ZEBRA_PROTODOWN_RC_STR_LEN];
+	char thread_buf[THREAD_TIMER_STRLEN];
+
+	vty_out(vty, "EVPN MH:\n");
+	vty_out(vty, "  mac-holdtime: %ds, neigh-holdtime: %ds\n",
+		zmh_info->mac_hold_time, zmh_info->neigh_hold_time);
+	vty_out(vty, "  startup-delay: %ds, start-delay-timer: %s\n",
+		zmh_info->startup_delay_time,
+		thread_timer_to_hhmmss(thread_buf, sizeof(thread_buf),
+				       zmh_info->startup_delay_timer));
+	vty_out(vty, "  uplink-cfg-cnt: %u, uplink-active-cnt: %u\n",
+		zmh_info->uplink_cfg_cnt, zmh_info->uplink_oper_up_cnt);
+	if (zmh_info->protodown_rc)
+		vty_out(vty, "  protodown: %s\n",
+			zebra_protodown_rc_str(zmh_info->protodown_rc, pd_buf,
+					       sizeof(pd_buf)));
+}
+
 /*****************************************************************************/
 /* A base L2-VNI is maintained to derive parameters such as ES originator-IP.
  * XXX: once single vxlan device model becomes available this will not be
@@ -2705,23 +2804,316 @@ static void zebra_evpn_es_get_one_base_evpn(void)
 	hash_walk(zvrf->evpn_table, zebra_evpn_es_get_one_base_evpn_cb, NULL);
 }
 
+/*****************************************************************************
+ * local ethernet segments can be error-disabled if the switch is not
+ * ready to start transmitting traffic via the VxLAN overlay
+ */
+bool zebra_evpn_is_es_bond(struct interface *ifp)
+{
+	struct zebra_if *zif = ifp->info;
+
+	return !!(struct zebra_if *)zif->es_info.es;
+}
+
+bool zebra_evpn_is_es_bond_member(struct interface *ifp)
+{
+	struct zebra_if *zif = ifp->info;
+
+	return IS_ZEBRA_IF_BOND_SLAVE(zif->ifp) && zif->bondslave_info.bond_if
+	       && ((struct zebra_if *)zif->bondslave_info.bond_if->info)
+			  ->es_info.es;
+}
+
+void zebra_evpn_mh_update_protodown_bond_mbr(struct zebra_if *zif, bool clear,
+					     const char *caller)
+{
+	bool old_protodown;
+	bool new_protodown;
+	enum protodown_reasons old_protodown_rc = 0;
+	enum protodown_reasons protodown_rc = 0;
+
+	if (!clear) {
+		struct zebra_if *bond_zif;
+
+		bond_zif = zif->bondslave_info.bond_if->info;
+		protodown_rc = bond_zif->protodown_rc;
+	}
+
+	if (zif->protodown_rc == protodown_rc)
+		return;
+
+	old_protodown = !!(zif->flags & ZIF_FLAG_PROTODOWN);
+	old_protodown_rc = zif->protodown_rc;
+	zif->protodown_rc &= ~ZEBRA_PROTODOWN_EVPN_ALL;
+	zif->protodown_rc |= (protodown_rc & ZEBRA_PROTODOWN_EVPN_ALL);
+	new_protodown = !!zif->protodown_rc;
+
+	if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
+		zlog_debug(
+			"%s bond mbr %s protodown_rc changed; old 0x%x new 0x%x",
+			caller, zif->ifp->name, old_protodown_rc,
+			zif->protodown_rc);
+
+	if (old_protodown == new_protodown)
+		return;
+
+	if (new_protodown)
+		zif->flags |= ZIF_FLAG_PROTODOWN;
+	else
+		zif->flags &= ~ZIF_FLAG_PROTODOWN;
+
+	if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
+		zlog_debug("%s protodown %s", zif->ifp->name,
+			   new_protodown ? "on" : "off");
+
+	zebra_if_set_protodown(zif->ifp, new_protodown);
+}
+
+/* The bond members inherit the protodown reason code from the bond */
+static void zebra_evpn_mh_update_protodown_bond(struct zebra_if *bond_zif)
+{
+	struct zebra_if *zif;
+	struct listnode *node;
+
+	if (!bond_zif->bond_info.mbr_zifs)
+		return;
+
+	for (ALL_LIST_ELEMENTS_RO(bond_zif->bond_info.mbr_zifs, node, zif)) {
+		zebra_evpn_mh_update_protodown_bond_mbr(zif, false /*clear*/,
+							__func__);
+	}
+}
+
+/* The global EVPN MH protodown rc is applied to all local ESs */
+static void zebra_evpn_mh_update_protodown_es(struct zebra_evpn_es *es)
+{
+	struct zebra_if *zif;
+	enum protodown_reasons old_protodown_rc;
+
+	zif = es->zif;
+	if ((zif->protodown_rc & ZEBRA_PROTODOWN_EVPN_ALL)
+	    == (zmh_info->protodown_rc & ZEBRA_PROTODOWN_EVPN_ALL))
+		return;
+
+	old_protodown_rc = zif->protodown_rc;
+	zif->protodown_rc &= ~ZEBRA_PROTODOWN_EVPN_ALL;
+	zif->protodown_rc |=
+		(zmh_info->protodown_rc & ZEBRA_PROTODOWN_EVPN_ALL);
+
+	if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
+		zlog_debug(
+			"es %s ifp %s protodown_rc changed; old 0x%x new 0x%x",
+			es->esi_str, zif->ifp->name, old_protodown_rc,
+			zif->protodown_rc);
+
+	/* update dataplane with the new protodown setting */
+	zebra_evpn_mh_update_protodown_bond(zif);
+}
+
+static void zebra_evpn_mh_clear_protodown_es(struct zebra_evpn_es *es)
+{
+	struct zebra_if *zif;
+	enum protodown_reasons old_protodown_rc;
+
+	zif = es->zif;
+	if (!(zif->protodown_rc & ZEBRA_PROTODOWN_EVPN_ALL))
+		return;
+
+	old_protodown_rc = zif->protodown_rc;
+	zif->protodown_rc &= ~ZEBRA_PROTODOWN_EVPN_ALL;
+
+	if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
+		zlog_debug(
+			"clear: es %s ifp %s protodown_rc cleared; old 0x%x new 0x%x",
+			es->esi_str, zif->ifp->name, old_protodown_rc,
+			zif->protodown_rc);
+
+	/* update dataplane with the new protodown setting */
+	zebra_evpn_mh_update_protodown_bond(zif);
+}
+
+static void zebra_evpn_mh_update_protodown_es_all(void)
+{
+	struct listnode *node;
+	struct zebra_evpn_es *es;
+
+	for (ALL_LIST_ELEMENTS_RO(zmh_info->local_es_list, node, es))
+		zebra_evpn_mh_update_protodown_es(es);
+}
+
+static void zebra_evpn_mh_update_protodown(enum protodown_reasons protodown_rc,
+					   bool set)
+{
+	enum protodown_reasons old_protodown_rc = zmh_info->protodown_rc;
+
+	if (set) {
+		if ((protodown_rc & zmh_info->protodown_rc) == protodown_rc)
+			return;
+
+		zmh_info->protodown_rc |= protodown_rc;
+	} else {
+		if (!(protodown_rc & zmh_info->protodown_rc))
+			return;
+		zmh_info->protodown_rc &= ~protodown_rc;
+	}
+
+	if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
+		zlog_debug("mh protodown_rc changed; old 0x%x new 0x%x",
+			   old_protodown_rc, zmh_info->protodown_rc);
+	zebra_evpn_mh_update_protodown_es_all();
+}
+
+static inline bool zebra_evpn_mh_is_all_uplinks_down(void)
+{
+	return zmh_info->uplink_cfg_cnt && !zmh_info->uplink_oper_up_cnt;
+}
+
+static void zebra_evpn_mh_uplink_oper_flags_update(struct zebra_if *zif,
+						   bool set)
+{
+	if (set) {
+		if (if_is_operative(zif->ifp)) {
+			if (!(zif->flags & ZIF_FLAG_EVPN_MH_UPLINK_OPER_UP)) {
+				zif->flags |= ZIF_FLAG_EVPN_MH_UPLINK_OPER_UP;
+				++zmh_info->uplink_oper_up_cnt;
+			}
+		} else {
+			if (zif->flags & ZIF_FLAG_EVPN_MH_UPLINK_OPER_UP) {
+				zif->flags &= ~ZIF_FLAG_EVPN_MH_UPLINK_OPER_UP;
+				if (zmh_info->uplink_oper_up_cnt)
+					--zmh_info->uplink_oper_up_cnt;
+			}
+		}
+	} else {
+		if (zif->flags & ZIF_FLAG_EVPN_MH_UPLINK_OPER_UP) {
+			zif->flags &= ~ZIF_FLAG_EVPN_MH_UPLINK_OPER_UP;
+			if (zmh_info->uplink_oper_up_cnt)
+				--zmh_info->uplink_oper_up_cnt;
+		}
+	}
+}
+
+static void zebra_evpn_mh_uplink_cfg_update(struct zebra_if *zif, bool set)
+{
+	bool old_protodown = zebra_evpn_mh_is_all_uplinks_down();
+	bool new_protodown;
+
+	if (set) {
+		if (zif->flags & ZIF_FLAG_EVPN_MH_UPLINK)
+			return;
+
+		zif->flags |= ZIF_FLAG_EVPN_MH_UPLINK;
+		++zmh_info->uplink_cfg_cnt;
+	} else {
+		if (!(zif->flags & ZIF_FLAG_EVPN_MH_UPLINK))
+			return;
+
+		zif->flags &= ~ZIF_FLAG_EVPN_MH_UPLINK;
+		if (zmh_info->uplink_cfg_cnt)
+			--zmh_info->uplink_cfg_cnt;
+	}
+
+	zebra_evpn_mh_uplink_oper_flags_update(zif, set);
+	new_protodown = zebra_evpn_mh_is_all_uplinks_down();
+	if (old_protodown == new_protodown)
+		return;
+
+	if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
+		zlog_debug(
+			"mh-uplink-cfg-chg on if %s/%d %s uplinks cfg %u up %u",
+			zif->ifp->name, zif->ifp->ifindex, set ? "set" : "down",
+			zmh_info->uplink_cfg_cnt, zmh_info->uplink_oper_up_cnt);
+
+	zebra_evpn_mh_update_protodown(ZEBRA_PROTODOWN_EVPN_UPLINK_DOWN,
+				       new_protodown);
+}
+
+void zebra_evpn_mh_uplink_oper_update(struct zebra_if *zif)
+{
+	bool old_protodown = zebra_evpn_mh_is_all_uplinks_down();
+	bool new_protodown;
+
+	zebra_evpn_mh_uplink_oper_flags_update(zif, true /*set*/);
+
+	if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
+		zlog_debug(
+			"mh-uplink-oper-chg on if %s/%d %s; uplinks cfg %u up %u",
+			zif->ifp->name, zif->ifp->ifindex,
+			if_is_operative(zif->ifp) ? "up" : "down",
+			zmh_info->uplink_cfg_cnt, zmh_info->uplink_oper_up_cnt);
+
+	new_protodown = zebra_evpn_mh_is_all_uplinks_down();
+	if (old_protodown == new_protodown)
+		return;
+
+	zebra_evpn_mh_update_protodown(ZEBRA_PROTODOWN_EVPN_UPLINK_DOWN,
+				       new_protodown);
+}
+
+static int zebra_evpn_mh_startup_delay_exp_cb(struct thread *t)
+{
+	if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
+		zlog_debug("startup-delay expired");
+
+	zebra_evpn_mh_update_protodown(ZEBRA_PROTODOWN_EVPN_STARTUP_DELAY,
+				       false /* set */);
+
+	return 0;
+}
+
+static void zebra_evpn_mh_startup_delay_timer_start(bool init)
+{
+	/* 1. This timer can be started during init.
+	 * 2. It can also be restarted if it is alreay running and the
+	 * admin wants to increase or decrease its value
+	 */
+	if (!init && !zmh_info->startup_delay_timer)
+		return;
+
+	if (zmh_info->startup_delay_timer) {
+		if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
+			zlog_debug("startup-delay timer cancelled");
+		thread_cancel(&zmh_info->startup_delay_timer);
+		zmh_info->startup_delay_timer = NULL;
+	}
+
+	if (zmh_info->startup_delay_time) {
+		if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
+			zlog_debug("startup-delay timer started for %d sec",
+				   zmh_info->startup_delay_time);
+		thread_add_timer(zrouter.master,
+				 zebra_evpn_mh_startup_delay_exp_cb, NULL,
+				 zmh_info->startup_delay_time,
+				 &zmh_info->startup_delay_timer);
+		zebra_evpn_mh_update_protodown(
+			ZEBRA_PROTODOWN_EVPN_STARTUP_DELAY, true /* set */);
+	} else {
+		zebra_evpn_mh_update_protodown(
+			ZEBRA_PROTODOWN_EVPN_STARTUP_DELAY, false /* set */);
+	}
+}
+
 /*****************************************************************************/
 void zebra_evpn_mh_config_write(struct vty *vty)
 {
-	if (zmh_info->mac_hold_time != EVPN_MH_MAC_HOLD_TIME_DEF)
-		vty_out(vty, "evpn mh mac-holdtime %ld\n",
+	if (zmh_info->mac_hold_time != ZEBRA_EVPN_MH_MAC_HOLD_TIME_DEF)
+		vty_out(vty, "evpn mh mac-holdtime %d\n",
 			zmh_info->mac_hold_time);
 
-	if (zmh_info->neigh_hold_time != EVPN_MH_NEIGH_HOLD_TIME_DEF)
-		vty_out(vty, "evpn mh neigh-holdtime %ld\n",
+	if (zmh_info->neigh_hold_time != ZEBRA_EVPN_MH_NEIGH_HOLD_TIME_DEF)
+		vty_out(vty, "evpn mh neigh-holdtime %d\n",
 			zmh_info->neigh_hold_time);
+
+	if (zmh_info->startup_delay_time != ZEBRA_EVPN_MH_STARTUP_DELAY_DEF)
+		vty_out(vty, "evpn mh startup-delay %d\n",
+			zmh_info->startup_delay_time);
 }
 
 int zebra_evpn_mh_neigh_holdtime_update(struct vty *vty,
 		uint32_t duration, bool set_default)
 {
 	if (set_default)
-		duration = EVPN_MH_NEIGH_HOLD_TIME_DEF;
+		duration = ZEBRA_EVPN_MH_NEIGH_HOLD_TIME_DEF;
 
 	zmh_info->neigh_hold_time = duration;
 
@@ -2732,26 +3124,39 @@ int zebra_evpn_mh_mac_holdtime_update(struct vty *vty,
 		uint32_t duration, bool set_default)
 {
 	if (set_default)
-		duration = EVPN_MH_MAC_HOLD_TIME_DEF;
+		duration = ZEBRA_EVPN_MH_MAC_HOLD_TIME_DEF;
 
 	zmh_info->mac_hold_time = duration;
 
 	return 0;
 }
 
+int zebra_evpn_mh_startup_delay_update(struct vty *vty, uint32_t duration,
+				       bool set_default)
+{
+	if (set_default)
+		duration = ZEBRA_EVPN_MH_STARTUP_DELAY_DEF;
+
+	zmh_info->startup_delay_time = duration;
+	zebra_evpn_mh_startup_delay_timer_start(false /* init */);
+
+	return 0;
+}
+
 void zebra_evpn_interface_init(void)
 {
 	install_element(INTERFACE_NODE, &zebra_evpn_es_id_cmd);
 	install_element(INTERFACE_NODE, &zebra_evpn_es_sys_mac_cmd);
 	install_element(INTERFACE_NODE, &zebra_evpn_es_pref_cmd);
+	install_element(INTERFACE_NODE, &zebra_evpn_mh_uplink_cmd);
 }
 
 void zebra_evpn_mh_init(void)
 {
 	zrouter.mh_info = XCALLOC(MTYPE_ZMH_INFO, sizeof(*zrouter.mh_info));
 
-	zmh_info->mac_hold_time = EVPN_MH_MAC_HOLD_TIME_DEF;
-	zmh_info->neigh_hold_time = EVPN_MH_NEIGH_HOLD_TIME_DEF;
+	zmh_info->mac_hold_time = ZEBRA_EVPN_MH_MAC_HOLD_TIME_DEF;
+	zmh_info->neigh_hold_time = ZEBRA_EVPN_MH_NEIGH_HOLD_TIME_DEF;
 	/* setup ES tables */
 	RB_INIT(zebra_es_rb_head, &zmh_info->es_rb_tree);
 	zmh_info->local_es_list = list_new();
@@ -2763,6 +3168,9 @@ void zebra_evpn_mh_init(void)
 	/* setup broadcast domain tables */
 	zmh_info->evpn_vlan_table = hash_create(zebra_evpn_acc_vl_hash_keymake,
 			zebra_evpn_acc_vl_cmp, "access VLAN hash table");
+
+	zmh_info->startup_delay_time = ZEBRA_EVPN_MH_STARTUP_DELAY_DEF;
+	zebra_evpn_mh_startup_delay_timer_start(true /*init*/);
 }
 
 void zebra_evpn_mh_terminate(void)
diff --git a/zebra/zebra_evpn_mh.h b/zebra/zebra_evpn_mh.h
index ee162ad48..09af26a3a 100644
--- a/zebra/zebra_evpn_mh.h
+++ b/zebra/zebra_evpn_mh.h
@@ -195,10 +195,25 @@ struct zebra_evpn_mh_info {
 #define EVPN_NHG_ID_TYPE_BIT (2 << EVPN_NH_ID_TYPE_POS)
 
 	/* XXX - re-visit the default hold timer value */
-#define EVPN_MH_MAC_HOLD_TIME_DEF (18 * 60)
-	long mac_hold_time;
-#define EVPN_MH_NEIGH_HOLD_TIME_DEF (18 * 60)
-	long neigh_hold_time;
+	int mac_hold_time;
+#define ZEBRA_EVPN_MH_MAC_HOLD_TIME_DEF (18 * 60)
+	int neigh_hold_time;
+#define ZEBRA_EVPN_MH_NEIGH_HOLD_TIME_DEF (18 * 60)
+
+	/* During this period access ports will be held in a protodown
+	 * state
+	 */
+	int startup_delay_time; /* seconds */
+#define ZEBRA_EVPN_MH_STARTUP_DELAY_DEF (3 * 60)
+	struct thread *startup_delay_timer;
+
+	/* Number of configured uplinks */
+	uint32_t uplink_cfg_cnt;
+	/* Number of operationally-up uplinks */
+	uint32_t uplink_oper_up_cnt;
+
+	/* These protodown bits are inherited by all ES bonds */
+	enum protodown_reasons protodown_rc;
 };
 
 static inline bool zebra_evpn_mac_is_es_local(zebra_mac_t *mac)
@@ -258,5 +273,16 @@ void zebra_evpn_mh_config_write(struct vty *vty);
 int zebra_evpn_mh_neigh_holdtime_update(struct vty *vty,
 		uint32_t duration, bool set_default);
 void zebra_evpn_es_local_br_port_update(struct zebra_if *zif);
+extern int zebra_evpn_mh_startup_delay_update(struct vty *vty,
+					      uint32_t duration,
+					      bool set_default);
+extern void zebra_evpn_mh_uplink_oper_update(struct zebra_if *zif);
+extern void zebra_evpn_mh_update_protodown_bond_mbr(struct zebra_if *zif,
+						    bool clear,
+						    const char *caller);
+extern bool zebra_evpn_is_es_bond(struct interface *ifp);
+extern bool zebra_evpn_is_es_bond_member(struct interface *ifp);
+extern void zebra_evpn_mh_print(struct vty *vty);
+extern void zebra_evpn_mh_json(json_object *json);
 
 #endif /* _ZEBRA_EVPN_MH_H */
diff --git a/zebra/zebra_l2.c b/zebra/zebra_l2.c
index 19d8bfd73..c1ad91c8c 100644
--- a/zebra/zebra_l2.c
+++ b/zebra/zebra_l2.c
@@ -41,6 +41,7 @@
 #include "zebra/zebra_memory.h"
 #include "zebra/zebra_vrf.h"
 #include "zebra/rt_netlink.h"
+#include "zebra/interface.h"
 #include "zebra/zebra_l2.h"
 #include "zebra/zebra_vxlan.h"
 #include "zebra/zebra_evpn_mh.h"
@@ -109,24 +110,99 @@ void zebra_l2_unmap_slave_from_bridge(struct zebra_l2info_brslave *br_slave)
 	br_slave->br_if = NULL;
 }
 
-void zebra_l2_map_slave_to_bond(struct zebra_l2info_bondslave *bond_slave,
-				vrf_id_t vrf_id)
+void zebra_l2_map_slave_to_bond(struct zebra_if *zif, vrf_id_t vrf_id)
 {
 	struct interface *bond_if;
+	struct zebra_if *bond_zif;
+	struct zebra_l2info_bondslave *bond_slave = &zif->bondslave_info;
 
-	/* TODO: Handle change of master */
 	bond_if = if_lookup_by_index_all_vrf(bond_slave->bond_ifindex);
-	if (bond_if)
-		bond_slave->bond_if = bond_if;
-	else
-		bond_slave->bond_if = if_create_ifindex(bond_slave->bond_ifindex,
-							vrf_id);
+	if (bond_if == bond_slave->bond_if)
+		return;
+
+	/* unlink the slave from the old master */
+	zebra_l2_unmap_slave_from_bond(zif);
+
+	/* If the bond is present and ready link the bond-member
+	 * to it
+	 */
+	if (bond_if && (bond_zif = bond_if->info)) {
+		if (bond_zif->bond_info.mbr_zifs) {
+			if (IS_ZEBRA_DEBUG_EVPN_MH_ES || IS_ZEBRA_DEBUG_EVENT)
+				zlog_debug("bond mbr %s linked to %s",
+					   zif->ifp->name, bond_if->name);
+			bond_slave->bond_if = bond_if;
+			/* link the slave to the new bond master */
+			listnode_add(bond_zif->bond_info.mbr_zifs, zif);
+			/* inherit protodown flags from the es-bond */
+			if (zebra_evpn_is_es_bond(bond_if))
+				zebra_evpn_mh_update_protodown_bond_mbr(
+					zif, false /*clear*/, __func__);
+		}
+	} else {
+		if (IS_ZEBRA_DEBUG_EVPN_MH_ES || IS_ZEBRA_DEBUG_EVENT)
+			zlog_debug("bond mbr %s link to bond skipped",
+				   zif->ifp->name);
+	}
+}
+
+void zebra_l2_unmap_slave_from_bond(struct zebra_if *zif)
+{
+	struct zebra_l2info_bondslave *bond_slave = &zif->bondslave_info;
+	struct zebra_if *bond_zif;
+
+	if (!bond_slave->bond_if) {
+		if (IS_ZEBRA_DEBUG_EVPN_MH_ES || IS_ZEBRA_DEBUG_EVENT)
+			zlog_debug("bond mbr %s unlink from bond skipped",
+				   zif->ifp->name);
+		return;
+	}
+
+	if (IS_ZEBRA_DEBUG_EVPN_MH_ES || IS_ZEBRA_DEBUG_EVENT)
+		zlog_debug("bond mbr %s un-linked from %s", zif->ifp->name,
+			   bond_slave->bond_if->name);
+
+	/* unlink the slave from the bond master */
+	bond_zif = bond_slave->bond_if->info;
+	/* clear protodown flags */
+	if (zebra_evpn_is_es_bond(bond_zif->ifp))
+		zebra_evpn_mh_update_protodown_bond_mbr(zif, true /*clear*/,
+							__func__);
+	listnode_delete(bond_zif->bond_info.mbr_zifs, zif);
+	bond_slave->bond_if = NULL;
 }
 
-void zebra_l2_unmap_slave_from_bond(struct zebra_l2info_bondslave *bond_slave)
+void zebra_l2if_update_bond(struct interface *ifp, bool add)
 {
-	if (bond_slave != NULL)
-		bond_slave->bond_if = NULL;
+	struct zebra_if *zif;
+	struct zebra_l2info_bond *bond;
+
+	zif = ifp->info;
+	assert(zif);
+	bond = &zif->bond_info;
+
+	if (add) {
+		if (!bond->mbr_zifs) {
+			if (IS_ZEBRA_DEBUG_EVPN_MH_ES || IS_ZEBRA_DEBUG_EVENT)
+				zlog_debug("bond %s mbr list create",
+					   ifp->name);
+			bond->mbr_zifs = list_new();
+		}
+	} else {
+		struct listnode *node;
+		struct listnode *nnode;
+		struct zebra_if *bond_mbr;
+
+		if (!bond->mbr_zifs)
+			return;
+
+		if (IS_ZEBRA_DEBUG_EVPN_MH_ES || IS_ZEBRA_DEBUG_EVENT)
+			zlog_debug("bond %s mbr list delete", ifp->name);
+		for (ALL_LIST_ELEMENTS(bond->mbr_zifs, node, nnode, bond_mbr))
+			zebra_l2_unmap_slave_from_bond(bond_mbr);
+
+		list_delete(&bond->mbr_zifs);
+	}
 }
 
 /*
@@ -318,9 +394,9 @@ void zebra_l2if_update_bond_slave(struct interface *ifp, ifindex_t bond_ifindex)
 
 	/* Set up or remove link with master */
 	if (bond_ifindex != IFINDEX_INTERNAL)
-		zebra_l2_map_slave_to_bond(&zif->bondslave_info, ifp->vrf_id);
+		zebra_l2_map_slave_to_bond(zif, ifp->vrf_id);
 	else if (old_bond_ifindex != IFINDEX_INTERNAL)
-		zebra_l2_unmap_slave_from_bond(&zif->bondslave_info);
+		zebra_l2_unmap_slave_from_bond(zif);
 }
 
 void zebra_vlan_bitmap_compute(struct interface *ifp,
diff --git a/zebra/zebra_l2.h b/zebra/zebra_l2.h
index f3b15c777..4b84eb071 100644
--- a/zebra/zebra_l2.h
+++ b/zebra/zebra_l2.h
@@ -40,6 +40,10 @@ struct zebra_l2info_brslave {
 	ns_id_t ns_id; /* network namespace where bridge is */
 };
 
+struct zebra_l2info_bond {
+	struct list *mbr_zifs; /* slaves using this bond as a master */
+};
+
 /* zebra L2 interface information - bridge interface */
 struct zebra_l2info_bridge {
 	uint8_t vlan_aware; /* VLAN-aware bridge? */
@@ -86,10 +90,6 @@ extern void zebra_l2_map_slave_to_bridge(struct zebra_l2info_brslave *br_slave,
 					 struct zebra_ns *zns);
 extern void
 zebra_l2_unmap_slave_from_bridge(struct zebra_l2info_brslave *br_slave);
-extern void
-zebra_l2_map_slave_to_bond(struct zebra_l2info_bondslave *bond_slave, vrf_id_t);
-extern void
-zebra_l2_unmap_slave_from_bond(struct zebra_l2info_bondslave *bond_slave);
 extern void zebra_l2_bridge_add_update(struct interface *ifp,
 				       struct zebra_l2info_bridge *bridge_info,
 				       int add);
@@ -112,6 +112,7 @@ extern void zebra_vlan_bitmap_compute(struct interface *ifp,
 		uint32_t vid_start, uint16_t vid_end);
 extern void zebra_vlan_mbr_re_eval(struct interface *ifp,
 		bitfield_t vlan_bitmap);
+extern void zebra_l2if_update_bond(struct interface *ifp, bool add);
 
 #ifdef __cplusplus
 }
diff --git a/zebra/zebra_router.h b/zebra/zebra_router.h
index 67f94bfcf..8651a01e9 100644
--- a/zebra/zebra_router.h
+++ b/zebra/zebra_router.h
@@ -61,6 +61,29 @@ enum multicast_mode {
 			      /* on equal value, MRIB wins for last 2 */
 };
 
+/* An interface can be error-disabled if a protocol (such as EVPN or
+ * VRRP) detects a problem with keeping it operationally-up.
+ * If any of the protodown bits are set protodown-on is programmed
+ * in the dataplane. This results in a carrier/L1 down on the
+ * physical device.
+ */
+enum protodown_reasons {
+	/* On startup local ESs are held down for some time to
+	 * allow the underlay to converge and EVPN routes to
+	 * get learnt
+	 */
+	ZEBRA_PROTODOWN_EVPN_STARTUP_DELAY = (1 << 0),
+	/* If all the uplinks are down the switch has lost access
+	 * to the VxLAN overlay and must shut down the access
+	 * ports to allow servers to re-direct their traffic to
+	 * other switches on the Ethernet Segment
+	 */
+	ZEBRA_PROTODOWN_EVPN_UPLINK_DOWN = (1 << 1),
+	ZEBRA_PROTODOWN_EVPN_ALL = (ZEBRA_PROTODOWN_EVPN_UPLINK_DOWN
+				    | ZEBRA_PROTODOWN_EVPN_STARTUP_DELAY)
+};
+#define ZEBRA_PROTODOWN_RC_STR_LEN 80
+
 struct zebra_mlag_info {
 	/* Role this zebra router is playing */
 	enum mlag_role role;
diff --git a/zebra/zebra_vty.c b/zebra/zebra_vty.c
index ed235ba79..df0e22b40 100644
--- a/zebra/zebra_vty.c
+++ b/zebra/zebra_vty.c
@@ -2433,6 +2433,20 @@ DEFPY (evpn_mh_neigh_holdtime,
 			no ? true : false);
 }
 
+DEFPY (evpn_mh_startup_delay,
+       evpn_mh_startup_delay_cmd,
+       "[no] evpn mh startup-delay(0-3600)$duration",
+       NO_STR
+       "EVPN\n"
+       "Multihoming\n"
+       "Startup delay\n"
+       "duration in seconds\n")
+{
+
+	return zebra_evpn_mh_startup_delay_update(vty, duration,
+			no ? true : false);
+}
+
 DEFUN (default_vrf_vni_mapping,
        default_vrf_vni_mapping_cmd,
        "vni " CMD_VNI_RANGE "[prefix-routes-only]",
@@ -3990,6 +4004,7 @@ void zebra_vty_init(void)
 
 	install_element(CONFIG_NODE, &evpn_mh_mac_holdtime_cmd);
 	install_element(CONFIG_NODE, &evpn_mh_neigh_holdtime_cmd);
+	install_element(CONFIG_NODE, &evpn_mh_startup_delay_cmd);
 	install_element(CONFIG_NODE, &default_vrf_vni_mapping_cmd);
 	install_element(CONFIG_NODE, &no_default_vrf_vni_mapping_cmd);
 	install_element(VRF_NODE, &vrf_vni_mapping_cmd);
diff --git a/zebra/zebra_vxlan.c b/zebra/zebra_vxlan.c
index cca768095..4b3b142d4 100644
--- a/zebra/zebra_vxlan.c
+++ b/zebra/zebra_vxlan.c
@@ -3427,7 +3427,7 @@ void zebra_vxlan_print_evpn(struct vty *vty, bool uj)
 		json_object_int_add(json, "detectionTime", zvrf->dad_time);
 		json_object_int_add(json, "detectionFreezeTime",
 				    zvrf->dad_freeze_time);
-
+		zebra_evpn_mh_json(json);
 	} else {
 		vty_out(vty, "L2 VNIs: %u\n", num_l2vnis);
 		vty_out(vty, "L3 VNIs: %u\n", num_l3vnis);
@@ -3447,6 +3447,7 @@ void zebra_vxlan_print_evpn(struct vty *vty, bool uj)
 				vty_out(vty, "  Detection freeze %s\n",
 					"permanent");
 		}
+		zebra_evpn_mh_print(vty);
 	}
 
 	if (uj) {