bgpd, zebra: Support for sticky MACs

Implement support for sticky (static) MACs. This includes the following:

- Recognize MAC is static (using NUD_NOARP flag) and inform BGP
- Construct MAC mobility extended community for sticky MACs as per
RFC 7432 section 15.2
- Inform to zebra that remote MAC is sticky, where appropriate
- Install sticky MACs into the kernel with the right flag
- Appropriate handling in route selection

Signed-off-by: Daniel Walton <dwalton@cumulusnetworks.com>
Reviewed-by:   Vivek Venkatraman <vivek@cumulusnetworks.com>
Reviewed-by:   Donald Sharp <sharpd@cumulusnetworks.com>

13 files changed, 186 insertions, 58 deletions

diff --git a/bgpd/bgp_attr.c b/bgpd/bgp_attr.c
index e5bebfff7..e62d1bb0a 100644
--- a/bgpd/bgp_attr.c
+++ b/bgpd/bgp_attr.c
@@ -2115,6 +2115,7 @@ bgp_attr_ext_communities (struct bgp_attr_parser_args *args)
   struct peer *const peer = args->peer;  
   struct attr *const attr = args->attr;  
   const bgp_size_t length = args->length;
+  u_char sticky = 0;
   
   if (length == 0)
     {
@@ -2137,7 +2138,8 @@ bgp_attr_ext_communities (struct bgp_attr_parser_args *args)
   attr->flag |= ATTR_FLAG_BIT (BGP_ATTR_EXT_COMMUNITIES);
 
   /* Extract MAC mobility sequence number, if any. */
-  attr->extra->mm_seqnum = bgp_attr_mac_mobility_seqnum (attr);
+  attr->extra->mm_seqnum = bgp_attr_mac_mobility_seqnum (attr, &sticky);
+  attr->extra->sticky = sticky;
 
   return BGP_ATTR_PARSE_PROCEED;
 }
diff --git a/bgpd/bgp_attr_evpn.c b/bgpd/bgp_attr_evpn.c
index 740f517a9..e66fdde5f 100644
--- a/bgpd/bgp_attr_evpn.c
+++ b/bgpd/bgp_attr_evpn.c
@@ -113,10 +113,11 @@ char *ecom_mac2str(char *ecom_mac)
  * community, if present, else 0.
  */
 u_int32_t
-bgp_attr_mac_mobility_seqnum (struct attr *attr)
+bgp_attr_mac_mobility_seqnum (struct attr *attr, u_char *sticky)
 {
   struct ecommunity *ecom;
   int i;
+  u_char flags = 0;
 
   ecom = attr->extra->ecommunity;
   if (!ecom || !ecom->size)
@@ -140,7 +141,13 @@ bgp_attr_mac_mobility_seqnum (struct attr *attr)
       if (!(type == ECOMMUNITY_ENCODE_EVPN &&
             sub_type == ECOMMUNITY_EVPN_SUBTYPE_MACMOBILITY))
         continue;
-      pnt++;
+      flags = *pnt++;
+
+      if (flags & ECOMMUNITY_EVPN_SUBTYPE_MACMOBILITY_FLAG_STICKY)
+        *sticky = 1;
+      else
+        *sticky = 0;
+
       pnt++;
       seq_num = (*pnt++ << 24);
       seq_num |= (*pnt++ << 16);
diff --git a/bgpd/bgp_attr_evpn.h b/bgpd/bgp_attr_evpn.h
index 8978731d5..26650ef8b 100644
--- a/bgpd/bgp_attr_evpn.h
+++ b/bgpd/bgp_attr_evpn.h
@@ -64,6 +64,6 @@ extern int bgp_build_evpn_prefix(int type, uint32_t eth_tag,
          struct prefix *dst);
 
 extern u_int32_t
-bgp_attr_mac_mobility_seqnum (struct attr *attr);
+bgp_attr_mac_mobility_seqnum (struct attr *attr, u_char *sticky);
 
 #endif        /* _QUAGGA_BGP_ATTR_EVPN_H */
diff --git a/bgpd/bgp_evpn.c b/bgpd/bgp_evpn.c
index d021def1b..130017562 100644
--- a/bgpd/bgp_evpn.c
+++ b/bgpd/bgp_evpn.c
@@ -349,7 +349,7 @@ static int
 bgp_zebra_send_remote_macip (struct bgp *bgp, struct bgpevpn *vpn,
                              struct prefix_evpn *p,
                              struct in_addr remote_vtep_ip,
-                             int add)
+                             int add, u_char sticky)
 {
   struct stream *s;
   int ipa_len;
@@ -384,11 +384,16 @@ bgp_zebra_send_remote_macip (struct bgp *bgp, struct bgpevpn *vpn,
     }
   stream_put_in_addr(s, &remote_vtep_ip);
 
+  /* TX MAC sticky status */
+  if (add)
+    stream_putc (s, sticky);
+
   stream_putw_at (s, 0, stream_get_endp (s));
 
   if (bgp_debug_zebra (NULL))
-    zlog_debug("Tx %s MACIP, VNI %u MAC %s IP %s remote VTEP %s",
+    zlog_debug("Tx %s MACIP, VNI %u %sMAC %s IP %s remote VTEP %s",
                add ? "ADD" : "DEL", vpn->vni,
+               sticky ? "sticky " : "",
                prefix_mac2str (&p->prefix.mac, buf1, sizeof(buf1)),
                ipaddr2str (&p->prefix.ip, buf3, sizeof(buf3)),
                inet_ntop(AF_INET, &remote_vtep_ip, buf2, sizeof(buf2)));
@@ -447,10 +452,13 @@ build_evpn_route_extcomm (struct bgpevpn *vpn, struct attr *attr)
 {
   struct attr_extra *attre;
   struct ecommunity ecom_encap;
+  struct ecommunity ecom_sticky;
   struct ecommunity_val eval;
+  struct ecommunity_val eval_sticky;
   bgp_encap_types tnl_type;
   struct listnode *node, *nnode;
   struct ecommunity *ecom;
+  u_int32_t seqnum;
 
   attre = bgp_attr_extra_get (attr);
 
@@ -468,6 +476,16 @@ build_evpn_route_extcomm (struct bgpevpn *vpn, struct attr *attr)
   for (ALL_LIST_ELEMENTS (vpn->export_rtl, node, nnode, ecom))
     attre->ecommunity = ecommunity_merge (attre->ecommunity, ecom);
 
+  if (attre->sticky)
+    {
+      seqnum = 0;
+      memset (&ecom_sticky, 0, sizeof (ecom_sticky));
+      encode_mac_mobility_extcomm(1, seqnum, &eval_sticky);
+      ecom_sticky.size = 1;
+      ecom_sticky.val = (u_int8_t *)eval_sticky.val;
+      attre->ecommunity = ecommunity_merge (attre->ecommunity, &ecom_sticky);
+    }
+
   attr->flag |= ATTR_FLAG_BIT (BGP_ATTR_EXT_COMMUNITIES);
 }
 
@@ -515,7 +533,6 @@ add_mac_mobility_to_attr (u_int32_t seq_num, struct attr *attr)
     {
       memcpy(ecom_mm->val, eval.val, sizeof(char) * ECOMMUNITY_SIZE);
     }
-
   /* Add MM to existing */
   else
     {
@@ -530,12 +547,13 @@ add_mac_mobility_to_attr (u_int32_t seq_num, struct attr *attr)
 /* Install EVPN route into zebra. */
 static int
 evpn_zebra_install (struct bgp *bgp, struct bgpevpn *vpn,
-                    struct prefix_evpn *p, struct in_addr remote_vtep_ip)
+                    struct prefix_evpn *p, struct in_addr remote_vtep_ip,
+                    u_char sticky)
 {
   int ret;
 
   if (p->prefix.route_type == BGP_EVPN_MAC_IP_ROUTE)
-    ret = bgp_zebra_send_remote_macip (bgp, vpn, p, remote_vtep_ip, 1);
+    ret = bgp_zebra_send_remote_macip (bgp, vpn, p, remote_vtep_ip, 1, sticky);
   else
     ret = bgp_zebra_send_remote_vtep (bgp, vpn, p, 1);
 
@@ -550,7 +568,7 @@ evpn_zebra_uninstall (struct bgp *bgp, struct bgpevpn *vpn,
   int ret;
 
   if (p->prefix.route_type == BGP_EVPN_MAC_IP_ROUTE)
-    ret = bgp_zebra_send_remote_macip (bgp, vpn, p, remote_vtep_ip, 0);
+    ret = bgp_zebra_send_remote_macip (bgp, vpn, p, remote_vtep_ip, 0, 0);
   else
     ret = bgp_zebra_send_remote_vtep (bgp, vpn, p, 0);
 
@@ -626,7 +644,8 @@ evpn_route_select_install (struct bgp *bgp, struct bgpevpn *vpn,
     {
       if (bgp_zebra_has_route_changed (rn, old_select))
         ret = evpn_zebra_install (bgp, vpn, (struct prefix_evpn *)&rn->p,
-                                  old_select->attr->nexthop);
+                                  old_select->attr->nexthop,
+                                  old_select->attr->extra->sticky);
       UNSET_FLAG (old_select->flags, BGP_INFO_MULTIPATH_CHG);
       bgp_zebra_clear_route_change_flags (rn);
       return ret;
@@ -655,7 +674,8 @@ evpn_route_select_install (struct bgp *bgp, struct bgpevpn *vpn,
       && new_select->sub_type == BGP_ROUTE_NORMAL)
     {
       ret = evpn_zebra_install (bgp, vpn, (struct prefix_evpn *) &rn->p,
-                                new_select->attr->nexthop);
+                                new_select->attr->nexthop,
+                                new_select->attr->extra->sticky);
       /* If an old best existed and it was a "local" route, the only reason
        * it would be supplanted is due to MAC mobility procedures. So, we
        * need to do an implicit delete and withdraw that route from peers.
@@ -685,6 +705,31 @@ evpn_route_select_install (struct bgp *bgp, struct bgpevpn *vpn,
   return ret;
 }
 
+
+/*
+ * Return true if the local ri for this rn has sticky set
+ */
+static int
+evpn_route_is_sticky (struct bgp *bgp, struct bgp_node *rn)
+{
+  struct bgp_info *tmp_ri;
+  struct bgp_info *local_ri;
+
+  local_ri = NULL;
+  for (tmp_ri = rn->info; tmp_ri; tmp_ri = tmp_ri->next)
+    {
+      if (tmp_ri->peer == bgp->peer_self
+          && tmp_ri->type == ZEBRA_ROUTE_BGP
+          && tmp_ri->sub_type == BGP_ROUTE_STATIC)
+        local_ri = tmp_ri;
+    }
+
+  if (!local_ri)
+    return 0;
+
+  return local_ri->attr->extra->sticky;
+}
+
 /*
  * Create or update EVPN route entry. This could be in the VNI route table
  * or the global route table.
@@ -699,6 +744,7 @@ update_evpn_route_entry (struct bgp *bgp, struct bgpevpn *vpn, afi_t afi,
   struct attr *attr_new;
   mpls_label_t label = MPLS_INVALID_LABEL;
   int route_change = 1;
+  u_char sticky = 0;
 
   *ri = NULL;
 
@@ -755,7 +801,8 @@ update_evpn_route_entry (struct bgp *bgp, struct bgpevpn *vpn, afi_t afi,
       attr_new = bgp_attr_intern (attr);
 
       /* Extract MAC mobility sequence number, if any. */
-      attr_new->extra->mm_seqnum = bgp_attr_mac_mobility_seqnum (attr_new);
+      attr_new->extra->mm_seqnum = bgp_attr_mac_mobility_seqnum (attr_new, &sticky);
+      attr_new->extra->sticky = sticky;
 
       /* Create new route with its attribute. */
       tmp_ri = info_make (ZEBRA_ROUTE_BGP, BGP_ROUTE_STATIC, 0,
@@ -804,7 +851,7 @@ update_evpn_route_entry (struct bgp *bgp, struct bgpevpn *vpn, afi_t afi,
  */
 static int
 update_evpn_route (struct bgp *bgp, struct bgpevpn *vpn,
-                   struct prefix_evpn *p)
+                   struct prefix_evpn *p, u_char sticky)
 {
   struct bgp_node *rn;
   struct attr attr;
@@ -821,6 +868,7 @@ update_evpn_route (struct bgp *bgp, struct bgpevpn *vpn,
   attr.nexthop = vpn->originator_ip;
   attr.extra->mp_nexthop_global_in = vpn->originator_ip;
   attr.extra->mp_nexthop_len = BGP_ATTR_NHLEN_IPV4;
+  attr.extra->sticky = sticky;
 
   /* Set up RT and ENCAP extended community. */
   build_evpn_route_extcomm (vpn, &attr);
@@ -955,22 +1003,30 @@ update_all_type2_routes (struct bgp *bgp, struct bgpevpn *vpn)
   struct bgp_node *rn;
   struct bgp_info *ri;
   struct attr attr;
+  struct attr attr_sticky;
   struct attr *attr_new;
 
   afi = AFI_L2VPN;
   safi = SAFI_EVPN;
   memset (&attr, 0, sizeof (struct attr));
+  memset (&attr_sticky, 0, sizeof (struct attr));
 
   /* Build path-attribute - all type-2 routes for this VNI will share the
    * same path attribute.
    */
   bgp_attr_default_set (&attr, BGP_ORIGIN_IGP);
+  bgp_attr_default_set (&attr_sticky, BGP_ORIGIN_IGP);
   attr.nexthop = vpn->originator_ip;
   attr.extra->mp_nexthop_global_in = vpn->originator_ip;
   attr.extra->mp_nexthop_len = BGP_ATTR_NHLEN_IPV4;
+  attr_sticky.nexthop = vpn->originator_ip;
+  attr_sticky.extra->mp_nexthop_global_in = vpn->originator_ip;
+  attr_sticky.extra->mp_nexthop_len = BGP_ATTR_NHLEN_IPV4;
+  attr_sticky.extra->sticky = 1;
 
-  /* Set up RT, ENCAP extended community. */
+  /* Set up RT, ENCAP and sticky MAC extended community. */
   build_evpn_route_extcomm (vpn, &attr);
+  build_evpn_route_extcomm (vpn, &attr_sticky);
 
   /* Walk this VNI's route table and update local type-2 routes. For any
    * routes updated, update corresponding entry in the global table too.
@@ -984,7 +1040,10 @@ update_all_type2_routes (struct bgp *bgp, struct bgpevpn *vpn)
       if (evp->prefix.route_type != BGP_EVPN_MAC_IP_ROUTE)
         continue;
 
-      update_evpn_route_entry (bgp, vpn, afi, safi, rn, &attr, 0, 1, &ri);
+      if (evpn_route_is_sticky (bgp, rn))
+        update_evpn_route_entry (bgp, vpn, afi, safi, rn, &attr_sticky, 0, 1, &ri);
+      else
+        update_evpn_route_entry (bgp, vpn, afi, safi, rn, &attr, 0, 1, &ri);
 
       /* If a local route exists for this prefix, we need to update
        * the global routing table too.
@@ -1013,7 +1072,9 @@ update_all_type2_routes (struct bgp *bgp, struct bgpevpn *vpn)
 
   /* Unintern temporary. */
   aspath_unintern (&attr.aspath);
+  aspath_unintern (&attr_sticky.aspath);
   bgp_attr_extra_free (&attr);
+  bgp_attr_extra_free (&attr_sticky);
 
   return 0;
 }
@@ -1134,7 +1195,7 @@ update_routes_for_vni (struct bgp *bgp, struct bgpevpn *vpn)
    * locally learnt type-2 routes (MACIP) - for this VNI.
    */
   build_evpn_type3_prefix (&p, vpn->originator_ip);
-  ret = update_evpn_route (bgp, vpn, &p);
+  ret = update_evpn_route (bgp, vpn, &p, 0);
   if (ret)
     return ret;
 
@@ -2299,7 +2360,8 @@ bgp_evpn_local_macip_del (struct bgp *bgp, vni_t vni,
  */
 int
 bgp_evpn_local_macip_add (struct bgp *bgp, vni_t vni,
-                          struct ethaddr *mac, struct ipaddr *ip)
+                          struct ethaddr *mac, struct ipaddr *ip,
+                          u_char sticky)
 {
   struct bgpevpn *vpn;
   struct prefix_evpn p;
@@ -2321,13 +2383,14 @@ bgp_evpn_local_macip_add (struct bgp *bgp, vni_t vni,
 
   /* Create EVPN type-2 route and schedule for processing. */
   build_evpn_type2_prefix (&p, mac, ip);
-  if (update_evpn_route (bgp, vpn, &p))
+  if (update_evpn_route (bgp, vpn, &p, sticky))
     {
       char buf[ETHER_ADDR_STRLEN];
       char buf2[INET6_ADDRSTRLEN];
 
-      zlog_err ("%u:Failed to create Type-2 route, VNI %u MAC %s IP %s",
+      zlog_err ("%u:Failed to create Type-2 route, VNI %u %sMAC %s IP %s",
                 bgp->vrf_id, vpn->vni,
+                sticky ? "sticky" : "",
                 prefix_mac2str (mac, buf, sizeof (buf)),
                 ipaddr2str (ip, buf2, sizeof(buf2)));
       return -1;
@@ -2417,7 +2480,7 @@ bgp_evpn_local_vni_add (struct bgp *bgp, vni_t vni, struct in_addr originator_ip
 
   /* Create EVPN type-3 route and schedule for processing. */
   build_evpn_type3_prefix (&p, vpn->originator_ip);
-  if (update_evpn_route (bgp, vpn, &p))
+  if (update_evpn_route (bgp, vpn, &p, 0))
     {
       zlog_err ("%u: Type3 route creation failure for VNI %u",
                 bgp->vrf_id, vni);
diff --git a/bgpd/bgp_evpn.h b/bgpd/bgp_evpn.h
index 40b61bb00..ed76bc3a4 100644
--- a/bgpd/bgp_evpn.h
+++ b/bgpd/bgp_evpn.h
@@ -41,7 +41,8 @@ bgp_evpn_local_macip_del (struct bgp *bgp, vni_t vni,
                           struct ethaddr *mac, struct ipaddr *ip);
 extern int
 bgp_evpn_local_macip_add (struct bgp *bgp, vni_t vni,
-                          struct ethaddr *mac, struct ipaddr *ip);
+                          struct ethaddr *mac, struct ipaddr *ip,
+                          u_char sticky);
 extern int
 bgp_evpn_local_vni_del (struct bgp *bgp, vni_t vni);
 extern int
diff --git a/bgpd/bgp_zebra.c b/bgpd/bgp_zebra.c
index a085fecc7..91c155b3f 100644
--- a/bgpd/bgp_zebra.c
+++ b/bgpd/bgp_zebra.c
@@ -2180,6 +2180,7 @@ bgp_zebra_process_local_macip (int command, struct zclient *zclient,
   int ipa_len;
   char buf[ETHER_ADDR_STRLEN];
   char buf1[INET6_ADDRSTRLEN];
+  u_char sticky;
 
   memset (&ip, 0, sizeof (ip));
   s = zclient->ibuf;
@@ -2201,19 +2202,21 @@ bgp_zebra_process_local_macip (int command, struct zclient *zclient,
       ip.ipa_type = (ipa_len == IPV4_MAX_BYTELEN) ? IPADDR_V4: IPADDR_V6;
       stream_get (&ip.ip.addr, s, ipa_len);
     }
+  sticky = stream_getc (s);
 
   bgp = bgp_lookup_by_vrf_id (vrf_id);
   if (!bgp)
     return 0;
 
   if (BGP_DEBUG (zebra, ZEBRA))
-    zlog_debug ("%u:Recv MACIP %s MAC %s IP %s VNI %u",
+    zlog_debug ("%u:Recv MACIP %s %sMAC %s IP %s VNI %u",
                 vrf_id, (command == ZEBRA_MACIP_ADD) ? "Add" : "Del",
+                sticky ? "sticky " : "",
                 prefix_mac2str (&mac, buf, sizeof (buf)),
                 ipaddr2str (&ip, buf1, sizeof(buf1)), vni);
 
   if (command == ZEBRA_MACIP_ADD)
-    return bgp_evpn_local_macip_add (bgp, vni, &mac, &ip);
+    return bgp_evpn_local_macip_add (bgp, vni, &mac, &ip, sticky);
   else
     return bgp_evpn_local_macip_del (bgp, vni, &mac, &ip);
 }
diff --git a/zebra/rt.h b/zebra/rt.h
index ee85eceec..90654fb3e 100644
--- a/zebra/rt.h
+++ b/zebra/rt.h
@@ -48,7 +48,8 @@ extern int kernel_add_vtep (vni_t vni, struct interface *ifp,
 extern int kernel_del_vtep (vni_t vni, struct interface *ifp,
                             struct in_addr *vtep_ip);
 extern int kernel_add_mac (struct interface *ifp, vlanid_t vid,
-                           struct ethaddr *mac, struct in_addr vtep_ip);
+                           struct ethaddr *mac, struct in_addr vtep_ip,
+                           u_char sticky);
 extern int kernel_del_mac (struct interface *ifp, vlanid_t vid,
                            struct ethaddr *mac, struct in_addr vtep_ip, int local);
 
diff --git a/zebra/rt_netlink.c b/zebra/rt_netlink.c
index 4be1e96ce..d676fc829 100644
--- a/zebra/rt_netlink.c
+++ b/zebra/rt_netlink.c
@@ -1658,6 +1658,7 @@ netlink_macfdb_change (struct sockaddr_nl *snl, struct nlmsghdr *h, int len)
   char buf[ETHER_ADDR_STRLEN];
   char vid_buf[20];
   char dst_buf[30];
+  u_char sticky = 0;
 
   ndm = NLMSG_DATA (h);
 
@@ -1738,12 +1739,15 @@ netlink_macfdb_change (struct sockaddr_nl *snl, struct nlmsghdr *h, int len)
       sprintf (dst_buf, " dst %s", inet_ntoa (vtep_ip.u.prefix4));
     }
 
+  sticky = (ndm->ndm_state & NUD_NOARP) ? 1 : 0;
+
   if (IS_ZEBRA_DEBUG_KERNEL)
-    zlog_debug ("Rx %s family %s IF %s(%u)%s MAC %s%s",
+    zlog_debug ("Rx %s family %s IF %s(%u)%s %sMAC %s%s",
                 nl_msg_type_to_str (h->nlmsg_type),
                 nl_family_to_str (ndm->ndm_family),
                 ifp->name, ndm->ndm_ifindex,
                 vid_present ? vid_buf : "",
+                sticky ? "sticky " : "",
                 prefix_mac2str (&mac, buf, sizeof (buf)),
                 dst_present ? dst_buf: "");
 
@@ -1763,7 +1767,7 @@ netlink_macfdb_change (struct sockaddr_nl *snl, struct nlmsghdr *h, int len)
       if (IS_ZEBRA_IF_VXLAN(ifp))
         return zebra_vxlan_check_del_local_mac (ifp, br_if, &mac, vid);
 
-      return zebra_vxlan_local_mac_add_update (ifp, br_if, &mac, vid);
+      return zebra_vxlan_local_mac_add_update (ifp, br_if, &mac, vid, sticky);
     }
 
   /* This is a delete notification.
@@ -1884,7 +1888,8 @@ static int
 netlink_macfdb_update (struct interface *ifp, vlanid_t vid,
                        struct ethaddr *mac,
                        struct in_addr vtep_ip,
-                       int local, int cmd)
+                       int local, int cmd,
+                       u_char sticky)
 {
   struct zebra_ns *zns = zebra_ns_lookup (NS_DEFAULT);
   struct
@@ -1923,7 +1928,10 @@ netlink_macfdb_update (struct interface *ifp, vlanid_t vid,
   req.ndm.ndm_flags |= NTF_SELF | NTF_MASTER;
   req.ndm.ndm_state = NUD_REACHABLE;
 
-  req.ndm.ndm_flags |= NTF_EXT_LEARNED;
+  if (sticky)
+    req.ndm.ndm_state |= NUD_NOARP;
+  else
+    req.ndm.ndm_flags |= NTF_EXT_LEARNED;
 
   addattr_l (&req.n, sizeof (req), NDA_LLADDR, mac, 6);
   req.ndm.ndm_ifindex = ifp->ifindex;
@@ -1944,11 +1952,12 @@ netlink_macfdb_update (struct interface *ifp, vlanid_t vid,
   addattr32 (&req.n, sizeof (req), NDA_MASTER, br_if->ifindex);
 
   if (IS_ZEBRA_DEBUG_KERNEL)
-    zlog_debug ("Tx %s family %s IF %s(%u)%s MAC %s%s",
+    zlog_debug ("Tx %s family %s IF %s(%u)%s %sMAC %s%s",
                 nl_msg_type_to_str (cmd),
                 nl_family_to_str (req.ndm.ndm_family),
                 ifp->name, ifp->ifindex,
                 vid_present ? vid_buf : "",
+                sticky ? "sticky " : "",
                 prefix_mac2str (mac, buf, sizeof (buf)),
                 dst_present ? dst_buf : "");
 
@@ -2243,16 +2252,17 @@ netlink_neigh_update2 (struct interface *ifp, struct ipaddr *ip,
 
 int
 kernel_add_mac (struct interface *ifp, vlanid_t vid,
-                struct ethaddr *mac, struct in_addr vtep_ip)
+                struct ethaddr *mac, struct in_addr vtep_ip,
+                u_char sticky)
 {
- return netlink_macfdb_update (ifp, vid, mac, vtep_ip, 0, RTM_NEWNEIGH);
+ return netlink_macfdb_update (ifp, vid, mac, vtep_ip, 0, RTM_NEWNEIGH, sticky);
 }
 
 int
 kernel_del_mac (struct interface *ifp, vlanid_t vid,
                 struct ethaddr *mac, struct in_addr vtep_ip, int local)
 {
- return netlink_macfdb_update (ifp, vid, mac, vtep_ip, local, RTM_DELNEIGH);
+ return netlink_macfdb_update (ifp, vid, mac, vtep_ip, local, RTM_DELNEIGH, 0);
 }
 
 int kernel_add_neigh (struct interface *ifp, struct ipaddr *ip,
diff --git a/zebra/rt_socket.c b/zebra/rt_socket.c
index c03ed27c6..93adf0229 100644
--- a/zebra/rt_socket.c
+++ b/zebra/rt_socket.c
@@ -444,7 +444,8 @@ kernel_del_vtep (vni_t vni, struct interface *ifp, struct in_addr *vtep_ip)
 
 int
 kernel_add_mac (struct interface *ifp, vlanid_t vid,
-                struct ethaddr *mac, struct in_addr vtep_ip)
+                struct ethaddr *mac, struct in_addr vtep_ip,
+                u_char sticky)
 {
   return 0;
 }
diff --git a/zebra/zebra_vxlan.c b/zebra/zebra_vxlan.c
index 633083f42..5eca8dc5d 100644
--- a/zebra/zebra_vxlan.c
+++ b/zebra/zebra_vxlan.c
@@ -76,7 +76,7 @@ zvni_print_hash (struct hash_backet *backet, void *ctxt);
 static int
 zvni_macip_send_msg_to_client (struct zebra_vrf *zvrf, vni_t vni,
                                struct ethaddr *macaddr,
-                               struct ipaddr *ip,
+                               struct ipaddr *ip, u_char sticky,
                                u_int16_t cmd);
 static unsigned int
 neigh_hash_keymake (void *p);
@@ -136,10 +136,10 @@ static zebra_mac_t *
 zvni_mac_lookup (zebra_vni_t *zvni, struct ethaddr *macaddr);
 static int
 zvni_mac_send_add_to_client (struct zebra_vrf *zvrf, vni_t vni,
-                             struct ethaddr *macaddr);
+                             struct ethaddr *macaddr, u_char sticky);
 static int
 zvni_mac_send_del_to_client (struct zebra_vrf *zvrf, vni_t vni,
-                             struct ethaddr *macaddr);
+                             struct ethaddr *macaddr, u_char sticky);
 static zebra_vni_t *
 zvni_map_vlan (struct interface *ifp, struct interface *br_if, vlanid_t vid);
 static int
@@ -542,7 +542,7 @@ zvni_print_hash (struct hash_backet *backet, void *ctxt)
 static int
 zvni_macip_send_msg_to_client (struct zebra_vrf *zvrf, vni_t vni,
                                struct ethaddr *macaddr,
-                               struct ipaddr *ip,
+                               struct ipaddr *ip, u_char sticky,
                                u_int16_t cmd)
 {
   struct zserv *client;
@@ -577,12 +577,15 @@ zvni_macip_send_msg_to_client (struct zebra_vrf *zvrf, vni_t vni,
   else
     stream_putl (s, 0); /* Just MAC. */
 
+  stream_putc (s, sticky);  /* Sticky MAC? */
+
   /* Write packet size. */
   stream_putw_at (s, 0, stream_get_endp (s));
 
   if (IS_ZEBRA_DEBUG_VXLAN)
-    zlog_debug ("%u:Send MACIP %s MAC %s IP %s VNI %u to %s",
+    zlog_debug ("%u:Send MACIP %s %sMAC %s IP %s VNI %u to %s",
                 zvrf_id (zvrf), (cmd == ZEBRA_MACIP_ADD) ? "Add" : "Del",
+                sticky ? "sticky " : "",
                 prefix_mac2str (macaddr, buf, sizeof (buf)),
                 ipaddr2str (ip, buf2, sizeof(buf2)), vni,
                 zebra_route_string (client->proto));
@@ -776,7 +779,7 @@ static int
 zvni_neigh_send_add_to_client (struct zebra_vrf *zvrf, vni_t vni,
                                struct ipaddr *ip, struct ethaddr *macaddr)
 {
-  return zvni_macip_send_msg_to_client (zvrf, vni, macaddr, ip,
+  return zvni_macip_send_msg_to_client (zvrf, vni, macaddr, ip, 0,
                                         ZEBRA_MACIP_ADD);
 }
 
@@ -787,7 +790,7 @@ static int
 zvni_neigh_send_del_to_client (struct zebra_vrf *zvrf, vni_t vni,
                                struct ipaddr *ip, struct ethaddr *macaddr)
 {
-  return zvni_macip_send_msg_to_client (zvrf, vni, macaddr, ip,
+  return zvni_macip_send_msg_to_client (zvrf, vni, macaddr, ip, 0,
                                         ZEBRA_MACIP_DEL);
 }
 
@@ -967,6 +970,7 @@ zvni_mac_del_hash_entry (struct hash_backet *backet, void *arg)
 {
   struct mac_walk_ctx *wctx = arg;
   zebra_mac_t *mac = backet->data;
+  u_char sticky = 0;
 
   if (((wctx->flags & DEL_LOCAL_MAC) && (mac->flags & ZEBRA_MAC_LOCAL)) ||
       ((wctx->flags & DEL_REMOTE_MAC) && (mac->flags & ZEBRA_MAC_REMOTE)) ||
@@ -977,8 +981,9 @@ zvni_mac_del_hash_entry (struct hash_backet *backet, void *arg)
     {
       if (wctx->upd_client && (mac->flags & ZEBRA_MAC_LOCAL))
         {
+          sticky = CHECK_FLAG (mac->flags, ZEBRA_MAC_STICKY) ? 1: 0;
           zvni_mac_send_del_to_client (wctx->zvrf, wctx->zvni->vni,
-                                       &mac->macaddr);
+                                       &mac->macaddr, sticky);
         }
 
       if (wctx->uninstall)
@@ -1058,10 +1063,10 @@ zvni_mac_lookup (zebra_vni_t *zvni, struct ethaddr *mac)
  */
 static int
 zvni_mac_send_add_to_client (struct zebra_vrf *zvrf, vni_t vni,
-                             struct ethaddr *macaddr)
+                             struct ethaddr *macaddr, u_char sticky)
 {
   return zvni_macip_send_msg_to_client (zvrf, vni, macaddr, NULL,
-                                        ZEBRA_MACIP_ADD);
+                                        sticky, ZEBRA_MACIP_ADD);
 }
 
 /*
@@ -1069,10 +1074,10 @@ zvni_mac_send_add_to_client (struct zebra_vrf *zvrf, vni_t vni,
  */
 static int
 zvni_mac_send_del_to_client (struct zebra_vrf *zvrf, vni_t vni,
-                             struct ethaddr *macaddr)
+                             struct ethaddr *macaddr, u_char sticky)
 {
   return zvni_macip_send_msg_to_client (zvrf, vni, macaddr, NULL,
-                                        ZEBRA_MACIP_DEL);
+                                        sticky, ZEBRA_MACIP_DEL);
 }
 
 /*
@@ -1259,6 +1264,7 @@ zvni_mac_install (zebra_vni_t *zvni, zebra_mac_t *mac)
 {
   struct zebra_if *zif;
   struct zebra_l2info_vxlan *vxl;
+  u_char sticky;
 
   if (!(mac->flags & ZEBRA_MAC_REMOTE))
     return 0;
@@ -1268,8 +1274,10 @@ zvni_mac_install (zebra_vni_t *zvni, zebra_mac_t *mac)
     return -1;
   vxl = &zif->l2info.vxl;
 
+  sticky = CHECK_FLAG (mac->flags, ZEBRA_MAC_STICKY) ? 1: 0;
+
   return kernel_add_mac (zvni->vxlan_if, vxl->access_vlan,
-                         &mac->macaddr, mac->fwd_info.r_vtep_ip);
+                         &mac->macaddr, mac->fwd_info.r_vtep_ip, sticky);
 }
 
 /*
@@ -2353,6 +2361,7 @@ zebra_vxlan_remote_macip_add (struct zserv *client, int sock,
   int update_mac = 0, update_neigh = 0;
   char buf[ETHER_ADDR_STRLEN];
   char buf1[INET6_ADDRSTRLEN];
+  u_char sticky;
 
   assert (EVPN_ENABLED (zvrf));
 
@@ -2380,9 +2389,14 @@ zebra_vxlan_remote_macip_add (struct zserv *client, int sock,
       vtep_ip.s_addr = stream_get_ipv4 (s);
       l += IPV4_MAX_BYTELEN;
 
+      /* Get 'sticky' flag. */
+      sticky = stream_getc(s);
+      l++;
+
       if (IS_ZEBRA_DEBUG_VXLAN)
-        zlog_debug ("%u:Recv MACIP Add MAC %s IP %s VNI %u Remote VTEP %s from %s",
+        zlog_debug ("%u:Recv MACIP Add %sMAC %s IP %s VNI %u Remote VTEP %s from %s",
                     zvrf_id (zvrf),
+                    sticky ? "sticky " : "",
                     prefix_mac2str (&macaddr, buf, sizeof (buf)),
                     ipaddr2str (&ip, buf1, sizeof (buf1)),
                     vni, inet_ntoa (vtep_ip),
@@ -2431,6 +2445,7 @@ zebra_vxlan_remote_macip_add (struct zserv *client, int sock,
        */
       mac = zvni_mac_lookup (zvni, &macaddr);
       if (!mac || !CHECK_FLAG (mac->flags, ZEBRA_MAC_REMOTE) ||
+          (CHECK_FLAG (mac->flags, ZEBRA_MAC_STICKY) ? 1: 0) != sticky ||
           !IPV4_ADDR_SAME(&mac->fwd_info.r_vtep_ip, &vtep_ip))
         update_mac = 1;
 
@@ -2464,6 +2479,11 @@ zebra_vxlan_remote_macip_add (struct zserv *client, int sock,
           SET_FLAG (mac->flags, ZEBRA_MAC_REMOTE);
           mac->fwd_info.r_vtep_ip = vtep_ip;
 
+          if (sticky)
+            SET_FLAG (mac->flags, ZEBRA_MAC_STICKY);
+          else
+            UNSET_FLAG (mac->flags, ZEBRA_MAC_STICKY);
+
           /* Install the entry. */
           zvni_mac_install (zvni, mac);
         }
@@ -2543,6 +2563,7 @@ zebra_vxlan_check_del_local_mac (struct interface *ifp,
   zebra_vni_t *zvni;
   zebra_mac_t *mac;
   char buf[ETHER_ADDR_STRLEN];
+  u_char sticky;
 
   zif = ifp->info;
   assert(zif);
@@ -2577,7 +2598,8 @@ zebra_vxlan_check_del_local_mac (struct interface *ifp,
                 ifp->name, ifp->ifindex, vni);
 
   /* Remove MAC from BGP. */
-  zvni_mac_send_del_to_client (zvrf, zvni->vni, macaddr);
+  sticky = CHECK_FLAG (mac->flags, ZEBRA_MAC_STICKY) ? 1: 0;
+  zvni_mac_send_del_to_client (zvrf, zvni->vni, macaddr, sticky);
 
   /* Delete this MAC entry. */
   zvni_mac_del (zvni, mac);
@@ -2651,6 +2673,7 @@ zebra_vxlan_local_mac_del (struct interface *ifp, struct interface *br_if,
   zebra_mac_t *mac;
   struct zebra_vrf *zvrf;
   char buf[ETHER_ADDR_STRLEN];
+  u_char sticky;
 
   /* We are interested in MACs only on ports or (port, VLAN) that
    * map to a VNI.
@@ -2684,7 +2707,8 @@ zebra_vxlan_local_mac_del (struct interface *ifp, struct interface *br_if,
   assert(zvrf);
 
   /* Remove MAC from BGP. */
-  zvni_mac_send_del_to_client (zvrf, zvni->vni, macaddr);
+  sticky = CHECK_FLAG (mac->flags, ZEBRA_MAC_STICKY) ? 1: 0;
+  zvni_mac_send_del_to_client (zvrf, zvni->vni, macaddr, sticky);
 
   /* Delete this MAC entry. */
   zvni_mac_del (zvni, mac);
@@ -2697,13 +2721,15 @@ zebra_vxlan_local_mac_del (struct interface *ifp, struct interface *br_if,
  */
 int
 zebra_vxlan_local_mac_add_update (struct interface *ifp, struct interface *br_if,
-                                  struct ethaddr *macaddr, vlanid_t vid)
+                                  struct ethaddr *macaddr, vlanid_t vid,
+                                  u_char sticky)
 {
   zebra_vni_t *zvni;
   zebra_mac_t *mac;
   struct zebra_vrf *zvrf;
   char buf[ETHER_ADDR_STRLEN];
   int add = 1;
+  u_char mac_sticky;
 
   /* We are interested in MACs only on ports or (port, VLAN) that
    * map to a VNI.
@@ -2712,8 +2738,9 @@ zebra_vxlan_local_mac_add_update (struct interface *ifp, struct interface *br_if
   if (!zvni)
     {
       if (IS_ZEBRA_DEBUG_VXLAN)
-        zlog_debug ("%u:Add/Update MAC %s intf %s(%u) VID %u, could not find VNI",
+        zlog_debug ("%u:Add/Update %sMAC %s intf %s(%u) VID %u, could not find VNI",
                     ifp->vrf_id,
+                    sticky ? "sticky " : "",
                     prefix_mac2str (macaddr, buf, sizeof (buf)),
                     ifp->name, ifp->ifindex, vid);
       return 0;
@@ -2727,8 +2754,9 @@ zebra_vxlan_local_mac_add_update (struct interface *ifp, struct interface *br_if
     }
 
   if (IS_ZEBRA_DEBUG_VXLAN)
-    zlog_debug ("%u:Add/Update MAC %s intf %s(%u) VID %u -> VNI %u",
+    zlog_debug ("%u:Add/Update %sMAC %s intf %s(%u) VID %u -> VNI %u",
                 ifp->vrf_id,
+                sticky ? "sticky " : "",
                 prefix_mac2str (macaddr, buf, sizeof (buf)),
                 ifp->name, ifp->ifindex, vid, zvni->vni);
 
@@ -2738,13 +2766,17 @@ zebra_vxlan_local_mac_add_update (struct interface *ifp, struct interface *br_if
     {
       if (CHECK_FLAG (mac->flags, ZEBRA_MAC_LOCAL))
         {
-          if (mac->fwd_info.local.ifindex == ifp->ifindex &&
+          mac_sticky = CHECK_FLAG (mac->flags, ZEBRA_MAC_STICKY) ? 1: 0;
+
+          if (mac_sticky == sticky &&
+              mac->fwd_info.local.ifindex == ifp->ifindex &&
               mac->fwd_info.local.vid == vid)
             {
               if (IS_ZEBRA_DEBUG_VXLAN)
-                zlog_debug ("%u:Add/Update MAC %s intf %s(%u) VID %u -> VNI %u, "
+                zlog_debug ("%u:Add/Update %sMAC %s intf %s(%u) VID %u -> VNI %u, "
                             "entry exists and has not changed ",
                             ifp->vrf_id,
+                            sticky ? "sticky " : "",
                             prefix_mac2str (macaddr, buf, sizeof (buf)),
                             ifp->name, ifp->ifindex, vid, zvni->vni);
               return 0;
@@ -2777,9 +2809,14 @@ zebra_vxlan_local_mac_add_update (struct interface *ifp, struct interface *br_if
   mac->fwd_info.local.ifindex = ifp->ifindex;
   mac->fwd_info.local.vid = vid;
 
+  if (sticky)
+    SET_FLAG (mac->flags, ZEBRA_MAC_STICKY);
+  else
+    UNSET_FLAG (mac->flags, ZEBRA_MAC_STICKY);
+
   /* Inform BGP if required. */
   if (add)
-    return zvni_mac_send_add_to_client (zvrf, zvni->vni, macaddr);
+    return zvni_mac_send_add_to_client (zvrf, zvni->vni, macaddr, sticky);
 
   return 0;
 }
diff --git a/zebra/zebra_vxlan.h b/zebra/zebra_vxlan.h
index 37af55e1e..e5f0bd2f6 100644
--- a/zebra/zebra_vxlan.h
+++ b/zebra/zebra_vxlan.h
@@ -80,7 +80,8 @@ extern int zebra_vxlan_remote_macip_del (struct zserv *client, int sock,
                                          u_short length, struct zebra_vrf *zvrf);
 extern int zebra_vxlan_local_mac_add_update (struct interface *ifp,
                                              struct interface *br_if,
-                                             struct ethaddr *mac, vlanid_t vid);
+                                             struct ethaddr *mac, vlanid_t vid,
+                                             u_char sticky);
 extern int zebra_vxlan_local_mac_del (struct interface *ifp,
                                       struct interface *br_if,
                                       struct ethaddr *mac, vlanid_t vid);
diff --git a/zebra/zebra_vxlan_null.c b/zebra/zebra_vxlan_null.c
index 75aab4087..b0f09930b 100644
--- a/zebra/zebra_vxlan_null.c
+++ b/zebra/zebra_vxlan_null.c
@@ -111,7 +111,8 @@ int zebra_vxlan_remote_macip_del (struct zserv *client, int sock,
 
 int zebra_vxlan_local_mac_add_update (struct interface *ifp,
                                       struct interface *br_if,
-                                      struct ethaddr *mac, vlanid_t vid)
+                                      struct ethaddr *mac, vlanid_t vid,
+                                      u_char sticky)
 {
   return 0;
 }
diff --git a/zebra/zebra_vxlan_private.h b/zebra/zebra_vxlan_private.h
index 6d72edcb0..0de86c3bc 100644
--- a/zebra/zebra_vxlan_private.h
+++ b/zebra/zebra_vxlan_private.h
@@ -102,6 +102,7 @@ struct zebra_mac_t_
 #define ZEBRA_MAC_LOCAL   0x01
 #define ZEBRA_MAC_REMOTE  0x02
 #define ZEBRA_MAC_AUTO    0x04  /* Auto created for neighbor. */
+#define ZEBRA_MAC_STICKY  0x08  /* Static MAC */
 
   /* Local or remote info. */
   union