pimd: Nexthop tracking support

Add pim Nexthop tracking feature 1st part where, specific RP or Source address (unicast address)
register with Zebra. Once nexthop update received from Zebra for a given address, scan RP or upstream
entries impacted by the change in nexthop.

Reviewed By: CCR-5761, Donald Sharp <sharpd@cumulusnetworks.com>
Testing Done: Tested with multiple RPs and multiple *,G entries at LHR.
Add new Nexthop or remove one of the link towards RP and verify RP and upstream nexthop update.
similar test done at RP with multiple S,G entries to reach source.

Signed-off-by: Chirag Shah <chirag@cumulusnetworks.com>

17 files changed, 1020 insertions, 154 deletions

diff --git a/pimd/Makefile.am b/pimd/Makefile.am
index 59abd1aa3..7e1b451ea 100644
--- a/pimd/Makefile.am
+++ b/pimd/Makefile.am
@@ -53,7 +53,7 @@ libpim_a_SOURCES = \
 	pim_ssmpingd.c pim_int.c pim_rp.c \
 	pim_static.c pim_br.c pim_register.c pim_routemap.c \
 	pim_msdp.c pim_msdp_socket.c pim_msdp_packet.c \
-	pim_jp_agg.c
+	pim_jp_agg.c pim_nht.c
 
 noinst_HEADERS = \
 	pim_memory.h \
@@ -65,7 +65,7 @@ noinst_HEADERS = \
 	pim_msg.h pim_upstream.h pim_rpf.h pim_macro.h \
 	pim_igmp_join.h pim_ssmpingd.h pim_int.h pim_rp.h \
 	pim_static.h pim_br.h pim_register.h \
-	pim_msdp.h pim_msdp_socket.h pim_msdp_packet.h \
+	pim_msdp.h pim_msdp_socket.h pim_msdp_packet.h pim_nht.h \
 	pim_jp_agg.h
 
 pimd_SOURCES = \
diff --git a/pimd/pim_main.c b/pimd/pim_main.c
index a0e42aab5..07f281272 100644
--- a/pimd/pim_main.c
+++ b/pimd/pim_main.c
@@ -89,6 +89,7 @@ FRR_DAEMON_INFO(pimd, PIM,
 	.privs = &pimd_privs,
 )
 
+
 int main(int argc, char** argv, char** envp) {
   frr_preinit(&pimd_di, argc, argv);
   frr_opt_add("", longopts, "");
@@ -116,7 +117,7 @@ int main(int argc, char** argv, char** envp) {
   /* 
    * Initializations
    */
-  vrf_init ();
+  pim_vrf_init ();
   access_list_init();
   prefix_list_init ();
   prefix_list_add_hook (pim_rp_prefix_list_update);
diff --git a/pimd/pim_memory.c b/pimd/pim_memory.c
index 5af2a8203..f46cf193b 100644
--- a/pimd/pim_memory.c
+++ b/pimd/pim_memory.c
@@ -49,3 +49,5 @@ DEFINE_MTYPE(PIMD, PIM_MSDP_MG_MBR,       "PIM MSDP mesh group mbr")
 DEFINE_MTYPE(PIMD, PIM_SEC_ADDR,          "PIM secondary address")
 DEFINE_MTYPE(PIMD, PIM_JP_AGG_GROUP,      "PIM JP AGG Group")
 DEFINE_MTYPE(PIMD, PIM_JP_AGG_SOURCE,     "PIM JP AGG Source")
+DEFINE_MTYPE(PIMD, PIM_PIM_INSTANCE,      "PIM global state")
+DEFINE_MTYPE(PIMD, PIM_NEXTHOP_CACHE,     "PIM nexthop cache state")
diff --git a/pimd/pim_memory.h b/pimd/pim_memory.h
index 0d5f131a4..bd9e12f2f 100644
--- a/pimd/pim_memory.h
+++ b/pimd/pim_memory.h
@@ -48,4 +48,7 @@ DECLARE_MTYPE(PIM_MSDP_MG_MBR)
 DECLARE_MTYPE(PIM_SEC_ADDR)
 DECLARE_MTYPE(PIM_JP_AGG_GROUP)
 DECLARE_MTYPE(PIM_JP_AGG_SOURCE)
+DECLARE_MTYPE(PIM_PIM_INSTANCE)
+DECLARE_MTYPE(PIM_NEXTHOP_CACHE)
+
 #endif /* _QUAGGA_PIM_MEMORY_H */
diff --git a/pimd/pim_mroute.c b/pimd/pim_mroute.c
index 940147581..a244534ab 100644
--- a/pimd/pim_mroute.c
+++ b/pimd/pim_mroute.c
@@ -759,7 +759,7 @@ int pim_mroute_add(struct channel_oil *c_oil, const char *name)
   ++qpim_mroute_add_events;
 
   /* Do not install route if incoming interface is undefined. */
-  if (c_oil->oil.mfcc_parent == MAXVIFS)
+  if (c_oil->oil.mfcc_parent >= MAXVIFS)
     {
       if (PIM_DEBUG_MROUTE)
         {
diff --git a/pimd/pim_nht.c b/pimd/pim_nht.c
new file mode 100644
index 000000000..c8cc18730
--- /dev/null
+++ b/pimd/pim_nht.c
@@ -0,0 +1,647 @@
+/*
+ * PIM for Quagga
+ * Copyright (C) 2017 Cumulus Networks, Inc.
+ * Chirag Shah
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ */
+#include <zebra.h>
+#include "network.h"
+#include "zclient.h"
+#include "stream.h"
+#include "nexthop.h"
+#include "if.h"
+#include "hash.h"
+#include "jhash.h"
+
+#include "pimd.h"
+#include "pimd/pim_nht.h"
+#include "log.h"
+#include "pim_time.h"
+#include "pim_oil.h"
+#include "pim_ifchannel.h"
+#include "pim_mroute.h"
+#include "pim_zebra.h"
+#include "pim_upstream.h"
+#include "pim_join.h"
+#include "pim_jp_agg.h"
+#include "pim_zebra.h"
+
+/**
+ * pim_sendmsg_zebra_rnh -- Format and send a nexthop register/Unregister
+ *   command to Zebra.
+ */
+static void
+pim_sendmsg_zebra_rnh (struct zclient *zclient, struct pim_nexthop_cache *pnc,
+                       int command)
+{
+  struct stream *s;
+  struct prefix *p;
+  int ret;
+
+  /* Check socket. */
+  if (!zclient || zclient->sock < 0)
+    return;
+
+  p = &(pnc->rpf.rpf_addr);
+  s = zclient->obuf;
+  stream_reset (s);
+  zclient_create_header (s, command, VRF_DEFAULT);
+  /* get update for all routes for a prefix */
+  stream_putc (s, 0);
+
+  stream_putw (s, PREFIX_FAMILY (p));
+  stream_putc (s, p->prefixlen);
+  switch (PREFIX_FAMILY (p))
+    {
+    case AF_INET:
+      stream_put_in_addr (s, &p->u.prefix4);
+      break;
+    case AF_INET6:
+      stream_put (s, &(p->u.prefix6), 16);
+      break;
+    default:
+      break;
+    }
+  stream_putw_at (s, 0, stream_get_endp (s));
+
+  ret = zclient_send_message (zclient);
+  if (ret < 0)
+    zlog_warn ("sendmsg_nexthop: zclient_send_message() failed");
+
+
+  if (PIM_DEBUG_TRACE)
+    {
+      char buf[PREFIX2STR_BUFFER];
+      prefix2str (p, buf, sizeof (buf));
+      zlog_debug ("%s: NHT Addr %s %sregistered with Zebra ret:%d ",
+                  __PRETTY_FUNCTION__, buf,
+                  (command == ZEBRA_NEXTHOP_REGISTER) ? " " : "de", ret);
+    }
+
+  return;
+}
+
+struct pim_nexthop_cache *
+pim_nexthop_cache_find (struct pim_rpf *rpf)
+{
+  struct pim_nexthop_cache *pnc = NULL;
+  struct pim_nexthop_cache lookup;
+
+  lookup.rpf.rpf_addr.family = rpf->rpf_addr.family;
+  lookup.rpf.rpf_addr.prefixlen = rpf->rpf_addr.prefixlen;
+  lookup.rpf.rpf_addr.u.prefix4.s_addr = rpf->rpf_addr.u.prefix4.s_addr;
+
+  pnc = hash_lookup (pimg->rpf_hash, &lookup);
+
+  return pnc;
+
+}
+
+static int
+pim_rp_list_cmp (void *v1, void *v2)
+{
+  struct rp_info *rp1 = (struct rp_info *) v1;
+  struct rp_info *rp2 = (struct rp_info *) v2;
+
+  if (rp1 == rp2)
+    return 0;
+
+  if (!rp1 && rp2)
+    return -1;
+
+  if (rp1 && !rp2)
+    return 1;
+
+  /*
+   * Sort by RP IP address
+   */
+  if (rp1->rp.rpf_addr.u.prefix4.s_addr < rp2->rp.rpf_addr.u.prefix4.s_addr)
+    return -1;
+
+  if (rp1->rp.rpf_addr.u.prefix4.s_addr > rp2->rp.rpf_addr.u.prefix4.s_addr)
+    return 1;
+
+  /*
+   * Sort by group IP address
+   */
+  if (rp1->group.u.prefix4.s_addr < rp2->group.u.prefix4.s_addr)
+    return -1;
+
+  if (rp1->group.u.prefix4.s_addr > rp2->group.u.prefix4.s_addr)
+    return 1;
+
+  return -1;
+}
+
+struct pim_nexthop_cache *
+pim_nexthop_cache_add (struct pim_rpf *rpf_addr)
+{
+  struct pim_nexthop_cache *pnc;
+
+  pnc = XCALLOC (MTYPE_PIM_NEXTHOP_CACHE, sizeof (struct pim_nexthop_cache));
+  if (!pnc)
+    {
+      zlog_err ("%s: NHT PIM XCALLOC failure ", __PRETTY_FUNCTION__);
+      return NULL;
+    }
+  pnc->rpf.rpf_addr.family = rpf_addr->rpf_addr.family;
+  pnc->rpf.rpf_addr.prefixlen = rpf_addr->rpf_addr.prefixlen;
+  pnc->rpf.rpf_addr.u.prefix4.s_addr = rpf_addr->rpf_addr.u.prefix4.s_addr;
+
+  pnc = hash_get (pimg->rpf_hash, pnc, hash_alloc_intern);
+
+  pnc->rp_list = list_new ();
+  pnc->rp_list->cmp = pim_rp_list_cmp;
+
+  pnc->upstream_list = list_new ();
+  pnc->upstream_list->cmp = pim_upstream_compare;
+
+  if (PIM_DEBUG_ZEBRA)
+    {
+      char rpf_str[PREFIX_STRLEN];
+      pim_addr_dump ("<nht?>", &rpf_addr->rpf_addr, rpf_str,
+                     sizeof (rpf_str));
+      zlog_debug ("%s: NHT hash node, RP and UP lists allocated for %s ",
+                  __PRETTY_FUNCTION__, rpf_str);
+    }
+
+  return pnc;
+}
+
+/* This API is used to Register an address with Zebra */
+int
+pim_find_or_track_nexthop (struct prefix *addr, struct pim_upstream *up,
+                           struct rp_info *rp)
+{
+  struct pim_nexthop_cache *pnc = NULL;
+  struct pim_rpf rpf;
+  struct listnode *ch_node = NULL;
+  struct zclient *zclient = NULL;
+
+  zclient = pim_zebra_zclient_get ();
+  memset (&rpf, 0, sizeof (struct pim_rpf));
+  rpf.rpf_addr.family = addr->family;
+  rpf.rpf_addr.prefixlen = addr->prefixlen;
+  rpf.rpf_addr.u.prefix4 = addr->u.prefix4;
+
+  pnc = pim_nexthop_cache_find (&rpf);
+  if (!pnc)
+    {
+      if (PIM_DEBUG_ZEBRA)
+        {
+          char buf[PREFIX2STR_BUFFER];
+          prefix2str (&rpf.rpf_addr, buf, sizeof (buf));
+          zlog_debug ("%s: NHT New PNC allocated for addr %s ",
+                      __PRETTY_FUNCTION__, buf);
+        }
+      pnc = pim_nexthop_cache_add (&rpf);
+      if (pnc)
+        pim_sendmsg_zebra_rnh (zclient, pnc,
+                               ZEBRA_NEXTHOP_REGISTER);
+      else
+        {
+          zlog_warn ("%s: pnc node allocation failed. ", __PRETTY_FUNCTION__);
+        }
+    }
+
+  if (rp != NULL)
+    {
+      ch_node = listnode_lookup (pnc->rp_list, rp);
+      if (ch_node == NULL)
+        {
+          if (PIM_DEBUG_ZEBRA)
+            {
+              char rp_str[PREFIX_STRLEN];
+              pim_addr_dump ("<rp?>", &rp->rp.rpf_addr, rp_str,
+                             sizeof (rp_str));
+              zlog_debug ("%s: NHT add RP %s node to cached list",
+                          __PRETTY_FUNCTION__, rp_str);
+            }
+          listnode_add_sort (pnc->rp_list, rp);
+        }
+    }
+
+  if (up != NULL)
+    {
+      ch_node = listnode_lookup (pnc->upstream_list, up);
+      if (ch_node == NULL)
+        {
+          if (PIM_DEBUG_ZEBRA)
+            {
+              char buf[PREFIX2STR_BUFFER];
+              prefix2str (addr, buf, sizeof (buf));
+              zlog_debug
+                ("%s: NHT add upstream %s node to cached list, rpf %s",
+                 __PRETTY_FUNCTION__, up->sg_str, buf);
+            }
+          listnode_add_sort (pnc->upstream_list, up);
+        }
+    }
+
+  if (CHECK_FLAG (pnc->flags, PIM_NEXTHOP_VALID))
+    return 1;
+
+  return 0;
+}
+
+void
+pim_delete_tracked_nexthop (struct prefix *addr, struct pim_upstream *up,
+                            struct rp_info *rp)
+{
+  struct pim_nexthop_cache *pnc = NULL;
+  struct pim_nexthop_cache lookup;
+  struct zclient *zclient = NULL;
+
+  zclient = pim_zebra_zclient_get ();
+
+  /* Remove from RPF hash if it is the last entry */
+  lookup.rpf.rpf_addr = *addr;
+  pnc = hash_lookup (pimg->rpf_hash, &lookup);
+  if (pnc)
+    {
+      if (rp)
+        listnode_delete (pnc->rp_list, rp);
+      if (up)
+        listnode_delete (pnc->upstream_list, up);
+
+      if (PIM_DEBUG_ZEBRA)
+        zlog_debug ("%s: NHT rp_list count:%d upstream_list count:%d ",
+                    __PRETTY_FUNCTION__, pnc->rp_list->count,
+                    pnc->upstream_list->count);
+
+      if (pnc->rp_list->count == 0 && pnc->upstream_list->count == 0)
+        {
+          pim_sendmsg_zebra_rnh (zclient, pnc,
+                                 ZEBRA_NEXTHOP_UNREGISTER);
+
+          list_delete (pnc->rp_list);
+          list_delete (pnc->upstream_list);
+
+          hash_release (pimg->rpf_hash, pnc);
+          if (pnc->nexthop)
+            nexthops_free (pnc->nexthop);
+          XFREE (MTYPE_PIM_NEXTHOP_CACHE, pnc);
+        }
+    }
+}
+
+/* Update RP nexthop info based on Nexthop update received from Zebra.*/
+static int
+pim_update_rp_nh (struct pim_nexthop_cache *pnc)
+{
+  struct listnode *node = NULL;
+  struct rp_info *rp_info = NULL;
+  int ret = 0;
+
+  /*Traverse RP list and update each RP Nexthop info */
+  for (ALL_LIST_ELEMENTS_RO (pnc->rp_list, node, rp_info))
+    {
+      if (rp_info->rp.rpf_addr.u.prefix4.s_addr == INADDR_NONE)
+        continue;
+
+      if (pim_nexthop_lookup (&rp_info->rp.source_nexthop,
+                              rp_info->rp.rpf_addr.u.prefix4, 1) != 0)
+        {
+          if (PIM_DEBUG_PIM_TRACE)
+            zlog_debug ("Unable to lookup nexthop for rp specified");
+          ret++;
+          continue;
+        }
+
+      if (PIM_DEBUG_TRACE)
+        {
+          char rp_str[PREFIX_STRLEN];
+          pim_addr_dump ("<rp?>", &rp_info->rp.rpf_addr, rp_str,
+                         sizeof (rp_str));
+          zlog_debug ("%s: NHT update nexthop for RP %s to interface %s ",
+                      __PRETTY_FUNCTION__, rp_str,
+                      rp_info->rp.source_nexthop.interface->name);
+        }
+    }
+
+  if (ret)
+    return 0;
+
+  return 1;
+
+}
+
+/* Update Upstream nexthop info based on Nexthop update received from Zebra.*/
+static int
+pim_update_upstream_nh (struct pim_nexthop_cache *pnc)
+{
+  struct listnode     *up_node;
+  struct listnode     *ifnode;
+  struct listnode     *up_nextnode;
+  struct listnode     *node;
+  struct pim_upstream *up;
+  struct interface    *ifp;
+  int                 vif_index = 0;
+
+  for (ALL_LIST_ELEMENTS (pnc->upstream_list, up_node, up_nextnode, up))
+    {
+      enum pim_rpf_result rpf_result;
+      struct pim_rpf old;
+
+      if (up == NULL)
+        {
+          zlog_debug ("%s: Upstream node is NULL ", __PRETTY_FUNCTION__);
+          continue;
+        }
+
+      old.source_nexthop.interface = up->rpf.source_nexthop.interface;
+      rpf_result = pim_rpf_update (up, &old, 0);
+      if (rpf_result == PIM_RPF_FAILURE)
+        continue;
+
+      if (rpf_result == PIM_RPF_CHANGED)
+        {
+
+          /*
+           * We have detected a case where we might need to rescan
+           * the inherited o_list so do it.
+           */
+          if (up->channel_oil && up->channel_oil->oil_inherited_rescan)
+            {
+              pim_upstream_inherited_olist_decide (up);
+              up->channel_oil->oil_inherited_rescan = 0;
+            }
+
+          if (up->join_state == PIM_UPSTREAM_JOINED)
+            {
+              /*
+               * If we come up real fast we can be here
+               * where the mroute has not been installed
+               * so install it.
+               */
+              if (up->channel_oil && !up->channel_oil->installed)
+                pim_mroute_add (up->channel_oil, __PRETTY_FUNCTION__);
+
+              /*
+                 RFC 4601: 4.5.7.  Sending (S,G) Join/Prune Messages
+
+                 Transitions from Joined State
+
+                 RPF'(S,G) changes not due to an Assert
+
+                 The upstream (S,G) state machine remains in Joined
+                 state. Send Join(S,G) to the new upstream neighbor, which is
+                 the new value of RPF'(S,G).  Send Prune(S,G) to the old
+                 upstream neighbor, which is the old value of RPF'(S,G).  Set
+                 the Join Timer (JT) to expire after t_periodic seconds.
+               */
+              pim_jp_agg_switch_interface (&old, &up->rpf, up);
+
+              pim_upstream_join_timer_restart (up, &old);
+            }                   /* up->join_state == PIM_UPSTREAM_JOINED */
+
+          /* FIXME can join_desired actually be changed by pim_rpf_update()
+             returning PIM_RPF_CHANGED ? */
+          pim_upstream_update_join_desired (up);
+
+        } /* PIM_RPF_CHANGED */
+
+      if (PIM_DEBUG_TRACE)
+        {
+          zlog_debug ("%s: NHT upstream %s old ifp %s new ifp %s",
+                      __PRETTY_FUNCTION__, up->sg_str,
+                      old.source_nexthop.interface->name,
+                      up->rpf.source_nexthop.interface->name);
+        }
+      /* update kernel multicast forwarding cache (MFC) */
+      if (up->channel_oil)
+        {
+          vif_index =
+            pim_if_find_vifindex_by_ifindex (up->rpf.
+                                             source_nexthop.interface->
+                                             ifindex);
+          /* Pass Current selected NH vif index to mroute download */
+          if (vif_index)
+            pim_scan_individual_oil (up->channel_oil, vif_index);
+          else
+            {
+              if (PIM_DEBUG_ZEBRA)
+                zlog_debug ("%s: NHT upstream %s channel_oil IIF %s vif_index is not valid",
+                      __PRETTY_FUNCTION__, up->sg_str,
+                      up->rpf.source_nexthop.interface->name);
+            }
+        }
+
+    } /* for (pnc->upstream_list) */
+
+  for (ALL_LIST_ELEMENTS_RO (vrf_iflist (VRF_DEFAULT), ifnode, ifp))
+    if (ifp->info)
+      {
+        struct pim_interface *pim_ifp = ifp->info;
+        struct pim_iface_upstream_switch *us;
+
+        for (ALL_LIST_ELEMENTS_RO (pim_ifp->upstream_switch_list, node, us))
+          {
+            struct pim_rpf rpf;
+            rpf.source_nexthop.interface = ifp;
+            rpf.rpf_addr.u.prefix4 = us->address;
+            pim_joinprune_send (&rpf, us->us);
+            pim_jp_agg_clear_group (us->us);
+          }
+      }
+
+  return 0;
+}
+
+/* This API is used to parse Registered address nexthop update coming from Zebra */
+void
+pim_parse_nexthop_update (struct zclient *zclient, int command,
+                          vrf_id_t vrf_id)
+{
+  struct stream *s;
+  struct prefix p;
+  struct nexthop *nexthop;
+  struct nexthop *oldnh;
+  struct nexthop *nhlist_head = NULL;
+  struct nexthop *nhlist_tail = NULL;
+  uint32_t metric, distance;
+  u_char nexthop_num = 0;
+  int i;
+  struct pim_rpf rpf;
+  struct pim_nexthop_cache *pnc = NULL;
+  struct pim_neighbor *nbr = NULL;
+  struct interface *ifp = NULL;
+
+  s = zclient->ibuf;
+  memset (&p, 0, sizeof (struct prefix));
+  p.family = stream_getw (s);
+  p.prefixlen = stream_getc (s);
+  switch (p.family)
+    {
+    case AF_INET:
+      p.u.prefix4.s_addr = stream_get_ipv4 (s);
+      break;
+    case AF_INET6:
+      stream_get (&p.u.prefix6, s, 16);
+      break;
+    default:
+      break;
+    }
+
+  if (command == ZEBRA_NEXTHOP_UPDATE)
+    {
+      rpf.rpf_addr.family = p.family;
+      rpf.rpf_addr.prefixlen = p.prefixlen;
+      rpf.rpf_addr.u.prefix4.s_addr = p.u.prefix4.s_addr;
+      pnc = pim_nexthop_cache_find (&rpf);
+      if (!pnc)
+        {
+          if (PIM_DEBUG_TRACE)
+            {
+              char buf[PREFIX2STR_BUFFER];
+              prefix2str (&rpf.rpf_addr, buf, sizeof (buf));
+              zlog_debug ("%s: NHT addr %s is not in local cached DB.",
+                          __PRETTY_FUNCTION__, buf);
+            }
+          return;
+        }
+    }
+
+  pnc->last_update = pim_time_monotonic_sec ();
+  distance = stream_getc (s);
+  metric = stream_getl (s);
+  nexthop_num = stream_getc (s);
+
+  if (PIM_DEBUG_TRACE)
+    {
+      char buf[PREFIX2STR_BUFFER];
+      prefix2str (&p, buf, sizeof (buf));
+      zlog_debug ("%s: NHT Update for %s nexthop_num %d vrf:%d upcount %d rpcount %d",
+                  __PRETTY_FUNCTION__, buf, nexthop_num, vrf_id,
+                  listcount (pnc->upstream_list), listcount (pnc->rp_list));
+    }
+
+  if (nexthop_num)
+    {
+      pnc->flags |= PIM_NEXTHOP_VALID;
+      pnc->distance = distance;
+      pnc->metric = metric;
+      pnc->nexthop_num = nexthop_num;
+
+      for (i = 0; i < nexthop_num; i++)
+        {
+          nexthop = nexthop_new ();
+          nexthop->type = stream_getc (s);
+          switch (nexthop->type)
+            {
+            case NEXTHOP_TYPE_IPV4:
+              nexthop->gate.ipv4.s_addr = stream_get_ipv4 (s);
+              nexthop->ifindex = stream_getl (s);
+              break;
+            case NEXTHOP_TYPE_IFINDEX:
+              nexthop->ifindex = stream_getl (s);
+              break;
+            case NEXTHOP_TYPE_IPV4_IFINDEX:
+              nexthop->gate.ipv4.s_addr = stream_get_ipv4 (s);
+              nexthop->ifindex = stream_getl (s);
+              break;
+            case NEXTHOP_TYPE_IPV6:
+              stream_get (&nexthop->gate.ipv6, s, 16);
+              break;
+            case NEXTHOP_TYPE_IPV6_IFINDEX:
+              stream_get (&nexthop->gate.ipv6, s, 16);
+              nexthop->ifindex = stream_getl (s);
+              nbr =
+                pim_neighbor_find_if (if_lookup_by_index_vrf
+                                      (nexthop->ifindex, VRF_DEFAULT));
+              /* Overwrite with Nbr address as NH addr */
+              if (nbr)
+                nexthop->gate.ipv4 = nbr->source_addr;
+
+              break;
+            default:
+              /* do nothing */
+              break;
+            }
+
+          if (PIM_DEBUG_TRACE)
+            {
+              char p_str[PREFIX2STR_BUFFER];
+              prefix2str (&p, p_str, sizeof (p_str));
+              zlog_debug ("%s: NHT addr %s %d-nhop via %s type %d",
+                          __PRETTY_FUNCTION__, p_str, i + 1,
+                          inet_ntoa (nexthop->gate.ipv4), nexthop->type);
+            }
+
+          ifp = if_lookup_by_index (nexthop->ifindex);
+          if (!ifp)
+            {
+              if (PIM_DEBUG_ZEBRA)
+                {
+                  char buf[NEXTHOP_STRLEN];
+                  zlog_debug("%s: could not find interface for ifindex %d (addr %s)",
+                         __PRETTY_FUNCTION__,
+                         nexthop->ifindex, nexthop2str (nexthop, buf, sizeof (buf)));
+                }
+              nexthop_free (nexthop);
+              continue;
+            }
+
+          if (!ifp->info)
+            {
+              if (PIM_DEBUG_ZEBRA)
+                {
+                  char buf[NEXTHOP_STRLEN];
+                  zlog_debug
+                    ("%s: multicast not enabled on input interface %s (ifindex=%d, addr %s)",
+                     __PRETTY_FUNCTION__, ifp->name, nexthop->ifindex,
+                     nexthop2str (nexthop, buf, sizeof (buf)));
+                }
+              nexthop_free (nexthop);
+              continue;
+            }
+
+          if (nhlist_tail)
+            {
+              nhlist_tail->next = nexthop;
+              nhlist_tail = nexthop;
+            }
+          else
+            {
+              nhlist_tail = nexthop;
+              nhlist_head = nexthop;
+            }
+
+          for (oldnh = pnc->nexthop; oldnh; oldnh = oldnh->next)
+            if (nexthop_same_no_recurse (oldnh, nexthop))
+              break;
+        }
+      /* Reset existing pnc->nexthop before assigning new list */
+      nexthops_free (pnc->nexthop);
+      pnc->nexthop = nhlist_head;
+    }
+  else
+    {
+      pnc->flags &= ~PIM_NEXTHOP_VALID;
+      pnc->nexthop_num = nexthop_num;
+      nexthops_free (pnc->nexthop);
+      pnc->nexthop = NULL;
+    }
+
+  pim_rpf_set_refresh_time ();
+
+  if (listcount (pnc->rp_list))
+    pim_update_rp_nh (pnc);
+  if (listcount (pnc->upstream_list))
+    pim_update_upstream_nh (pnc);
+
+}
diff --git a/pimd/pim_nht.h b/pimd/pim_nht.h
new file mode 100644
index 000000000..6e6602b3b
--- /dev/null
+++ b/pimd/pim_nht.h
@@ -0,0 +1,60 @@
+/*
+ * PIM for Quagga
+ * Copyright (C) 2017 Cumulus Networks, Inc.
+ * Chirag Shah
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
+ * MA 02110-1301 USA
+ */
+#ifndef PIM_NHT_H
+#define PIM_NHT_H
+
+#include "prefix.h"
+#include <zebra.h>
+#include "zclient.h"
+#include "vrf.h"
+
+#include "pimd.h"
+#include "pim_rp.h"
+#include "pim_rpf.h"
+
+/* PIM nexthop cache value structure. */
+struct pim_nexthop_cache
+{
+  struct pim_rpf rpf;
+  /* IGP route's metric. */
+  u_int32_t metric;
+  uint32_t  distance;
+  /* Nexthop number and nexthop linked list. */
+  u_char nexthop_num;
+  struct nexthop *nexthop;
+  int64_t last_update;
+  u_int16_t flags;
+#define PIM_NEXTHOP_VALID             (1 << 0)
+
+  struct list *rp_list;
+  struct list *upstream_list;
+};
+
+void pim_parse_nexthop_update (struct zclient *zclient, int command,
+                               vrf_id_t vrf_id);
+int pim_find_or_track_nexthop (struct prefix *addr, struct pim_upstream *up,
+                               struct rp_info *rp);
+void pim_delete_tracked_nexthop (struct prefix *addr, struct pim_upstream *up,
+                                 struct rp_info *rp);
+struct pim_nexthop_cache *pim_nexthop_cache_add (struct pim_rpf *rpf_addr);
+struct pim_nexthop_cache *pim_nexthop_cache_find (struct pim_rpf *rpf);
+
+#endif
diff --git a/pimd/pim_rp.c b/pimd/pim_rp.c
index dc19002a4..de30116ed 100644
--- a/pimd/pim_rp.c
+++ b/pimd/pim_rp.c
@@ -42,14 +42,8 @@
 #include "pim_memory.h"
 #include "pim_iface.h"
 #include "pim_msdp.h"
+#include "pim_nht.h"
 
-struct rp_info
-{
-  struct prefix group;
-  struct pim_rpf rp;
-  int i_am_rp;
-  char *plist;
-};
 
 static struct list *qpim_rp_list = NULL;
 static struct rp_info *tail = NULL;
@@ -305,6 +299,7 @@ pim_rp_new (const char *rp, const char *group_range, const char *plist)
   struct listnode *node, *nnode;
   struct rp_info *tmp_rp_info;
   char buffer[BUFSIZ];
+  struct prefix nht_p;
 
   rp_info = XCALLOC (MTYPE_PIM_RP, sizeof (*rp_info));
   if (!rp_info)
@@ -401,6 +396,19 @@ pim_rp_new (const char *rp, const char *group_range, const char *plist)
           rp_all->rp.rpf_addr = rp_info->rp.rpf_addr;
           XFREE (MTYPE_PIM_RP, rp_info);
 
+          /* Register addr with Zebra NHT */
+          nht_p.family = AF_INET;
+          nht_p.prefixlen = IPV4_MAX_BITLEN;
+          nht_p.u.prefix4 = rp_all->rp.rpf_addr.u.prefix4;
+          if (PIM_DEBUG_PIM_TRACE)
+            {
+              char buf[PREFIX2STR_BUFFER];
+              prefix2str (&nht_p, buf, sizeof (buf));
+              zlog_debug ("%s: NHT Register rp_all addr %s with NHT ",
+                        __PRETTY_FUNCTION__, buf);
+            }
+          pim_find_or_track_nexthop (&nht_p, NULL, rp_all);
+
           if (pim_nexthop_lookup (&rp_all->rp.source_nexthop, rp_all->rp.rpf_addr.u.prefix4, 1) != 0)
             return PIM_RP_NO_PATH;
 
@@ -448,11 +456,23 @@ pim_rp_new (const char *rp, const char *group_range, const char *plist)
 
   listnode_add_sort (qpim_rp_list, rp_info);
 
+  /* Register addr with Zebra NHT */
+  nht_p.family = AF_INET;
+  nht_p.prefixlen = IPV4_MAX_BITLEN;
+  nht_p.u.prefix4 = rp_info->rp.rpf_addr.u.prefix4;
+  if (PIM_DEBUG_PIM_TRACE)
+    {
+      char buf[PREFIX2STR_BUFFER];
+      prefix2str (&nht_p, buf, sizeof (buf));
+      zlog_debug ("%s: NHT Register RP addr %s with Zebra ", __PRETTY_FUNCTION__, buf);
+    }
+  pim_find_or_track_nexthop (&nht_p, NULL, rp_info);
+
   if (pim_nexthop_lookup (&rp_info->rp.source_nexthop, rp_info->rp.rpf_addr.u.prefix4, 1) != 0)
     return PIM_RP_NO_PATH;
 
   pim_rp_check_interfaces (rp_info);
-  pim_rp_refresh_group_to_rp_mapping();
+  pim_rp_refresh_group_to_rp_mapping ();
   return PIM_SUCCESS;
 }
 
@@ -465,6 +485,7 @@ pim_rp_del (const char *rp, const char *group_range, const char *plist)
   struct rp_info *rp_info;
   struct rp_info *rp_all;
   int result;
+  struct prefix nht_p;
 
   if (group_range == NULL)
     result = str2prefix ("224.0.0.0/4", &group);
@@ -492,6 +513,18 @@ pim_rp_del (const char *rp, const char *group_range, const char *plist)
       rp_info->plist = NULL;
     }
 
+  /* Deregister addr with Zebra NHT */
+  nht_p.family = AF_INET;
+  nht_p.prefixlen = IPV4_MAX_BITLEN;
+  nht_p.u.prefix4 = rp_info->rp.rpf_addr.u.prefix4;
+  if (PIM_DEBUG_PIM_TRACE)
+    {
+      char buf[PREFIX2STR_BUFFER];
+      prefix2str (&nht_p, buf, sizeof (buf));
+      zlog_debug ("%s: Deregister RP addr %s with NHT ", __PRETTY_FUNCTION__, buf);
+    }
+  pim_delete_tracked_nexthop (&nht_p, NULL, rp_info);
+
   str2prefix ("224.0.0.0/4", &g_all);
   rp_all = pim_rp_find_match_group (&g_all);
 
@@ -504,7 +537,7 @@ pim_rp_del (const char *rp, const char *group_range, const char *plist)
     }
 
   listnode_delete (qpim_rp_list, rp_info);
-  pim_rp_refresh_group_to_rp_mapping();
+  pim_rp_refresh_group_to_rp_mapping ();
   return PIM_SUCCESS;
 }
 
@@ -754,7 +787,7 @@ pim_rp_check_is_my_ip_address (struct in_addr group, struct in_addr dest_addr)
 
   if (if_lookup_exact_address (&dest_addr, AF_INET))
     return 1;
-    
+
   return 0;
 }
 
diff --git a/pimd/pim_rp.h b/pimd/pim_rp.h
index b32228ed4..e5580cfa6 100644
--- a/pimd/pim_rp.h
+++ b/pimd/pim_rp.h
@@ -21,6 +21,21 @@
 #ifndef PIM_RP_H
 #define PIM_RP_H
 
+#include <zebra.h>
+#include "prefix.h"
+#include "vty.h"
+#include "plist.h"
+#include "pim_iface.h"
+#include "pim_rpf.h"
+
+struct rp_info
+{
+  struct prefix group;
+  struct pim_rpf rp;
+  int i_am_rp;
+  char *plist;
+};
+
 void pim_rp_init (void);
 void pim_rp_free (void);
 
diff --git a/pimd/pim_rpf.c b/pimd/pim_rpf.c
index ff8a6054c..3d278f12a 100644
--- a/pimd/pim_rpf.c
+++ b/pimd/pim_rpf.c
@@ -34,11 +34,12 @@
 #include "pim_zlookup.h"
 #include "pim_ifchannel.h"
 #include "pim_time.h"
+#include "pim_nht.h"
 
 static long long last_route_change_time = -1;
 long long nexthop_lookups_avoided = 0;
 
-static struct in_addr pim_rpf_find_rpf_addr(struct pim_upstream *up);
+static struct in_addr pim_rpf_find_rpf_addr (struct pim_upstream *up);
 
 void
 pim_rpf_set_refresh_time (void)
@@ -184,14 +185,31 @@ static int nexthop_mismatch(const struct pim_nexthop *nh1,
     (nh1->mrib_route_metric != nh2->mrib_route_metric);
 }
 
-enum pim_rpf_result pim_rpf_update(struct pim_upstream *up, struct pim_rpf *old)
+enum pim_rpf_result pim_rpf_update(struct pim_upstream *up, struct pim_rpf *old, uint8_t is_new)
 {
   struct pim_rpf     *rpf = &up->rpf;
   struct pim_rpf     saved;
+  struct prefix     nht_p;
 
   saved.source_nexthop = rpf->source_nexthop;
   saved.rpf_addr = rpf->rpf_addr;
 
+  if (is_new)
+    {
+      if (PIM_DEBUG_ZEBRA)
+        {
+          char source_str[INET_ADDRSTRLEN];
+          pim_inet4_dump("<source?>", up->upstream_addr, source_str, sizeof(source_str));
+          zlog_debug ("%s: NHT Register upstream %s addr %s with Zebra.",
+                __PRETTY_FUNCTION__, up->sg_str, source_str);
+        }
+      /* Register addr with Zebra NHT */
+      nht_p.family = AF_INET;
+      nht_p.prefixlen = IPV4_MAX_BITLEN;
+      nht_p.u.prefix4.s_addr = up->upstream_addr.s_addr;
+      pim_find_or_track_nexthop (&nht_p, up, NULL);
+    }
+
   if (pim_nexthop_lookup(&rpf->source_nexthop,
                          up->upstream_addr,
                          !PIM_UPSTREAM_FLAG_TEST_FHR (up->flags) && 
diff --git a/pimd/pim_rpf.h b/pimd/pim_rpf.h
index f4a987793..85fb1ed89 100644
--- a/pimd/pim_rpf.h
+++ b/pimd/pim_rpf.h
@@ -64,7 +64,7 @@ struct pim_upstream;
 extern long long nexthop_lookups_avoided;
 
 int pim_nexthop_lookup(struct pim_nexthop *nexthop, struct in_addr addr, int neighbor_needed);
-enum pim_rpf_result pim_rpf_update(struct pim_upstream *up, struct pim_rpf *old);
+enum pim_rpf_result pim_rpf_update(struct pim_upstream *up, struct pim_rpf *old, uint8_t is_new);
 
 int pim_rpf_addr_is_inaddr_none (struct pim_rpf *rpf);
 int pim_rpf_addr_is_inaddr_any (struct pim_rpf *rpf);
diff --git a/pimd/pim_upstream.c b/pimd/pim_upstream.c
index 19f7d3336..183db532b 100644
--- a/pimd/pim_upstream.c
+++ b/pimd/pim_upstream.c
@@ -52,6 +52,7 @@
 #include "pim_register.h"
 #include "pim_msdp.h"
 #include "pim_jp_agg.h"
+#include "pim_nht.h"
 
 struct hash *pim_upstream_hash = NULL;
 struct list *pim_upstream_list = NULL;
@@ -142,6 +143,7 @@ pim_upstream_find_parent (struct pim_upstream *child)
 void pim_upstream_free(struct pim_upstream *up)
 {
   XFREE(MTYPE_PIM_UPSTREAM, up);
+  up = NULL;
 }
 
 static void upstream_channel_oil_detach(struct pim_upstream *up)
@@ -156,6 +158,7 @@ void
 pim_upstream_del(struct pim_upstream *up, const char *name)
 {
   bool notify_msdp = false;
+  struct prefix nht_p;
 
   if (PIM_DEBUG_TRACE)
     zlog_debug ("%s(%s): Delete %s ref count: %d",
@@ -209,10 +212,25 @@ pim_upstream_del(struct pim_upstream *up, const char *name)
   listnode_delete (pim_upstream_list, up);
   hash_release (pim_upstream_hash, up);
 
-  if (notify_msdp) {
-    pim_msdp_up_del(&up->sg);
-  }
-  pim_upstream_free(up);
+  if (notify_msdp)
+    {
+      pim_msdp_up_del (&up->sg);
+    }
+
+  /* Deregister addr with Zebra NHT */
+  nht_p.family = AF_INET;
+  nht_p.prefixlen = IPV4_MAX_BITLEN;
+  nht_p.u.prefix4 = up->upstream_addr;
+  if (PIM_DEBUG_TRACE)
+    {
+      char buf[PREFIX2STR_BUFFER];
+      prefix2str (&nht_p, buf, sizeof (buf));
+      zlog_debug ("%s: Deregister upstream %s upstream addr %s with NHT ",
+                __PRETTY_FUNCTION__, up->sg_str, buf);
+    }
+  pim_delete_tracked_nexthop (&nht_p, up, NULL);
+
+  pim_upstream_free (up);
 }
 
 void
@@ -517,7 +535,7 @@ pim_upstream_switch(struct pim_upstream *up,
   }
 }
 
-static int
+int
 pim_upstream_compare (void *arg1, void *arg2)
 {
   const struct pim_upstream *up1 = (const struct pim_upstream *)arg1;
@@ -548,11 +566,12 @@ pim_upstream_new (struct prefix_sg *sg,
   struct pim_upstream *up;
 
   up = XCALLOC(MTYPE_PIM_UPSTREAM, sizeof(*up));
-  if (!up) {
-    zlog_err("%s: PIM XCALLOC(%zu) failure",
+  if (!up)
+    {
+      zlog_err("%s: PIM XCALLOC(%zu) failure",
 	     __PRETTY_FUNCTION__, sizeof(*up));
-    return NULL;
-  }
+      return NULL;
+    }
   
   up->sg                          = *sg;
   pim_str_sg_set (sg, up->sg_str);
@@ -600,7 +619,7 @@ pim_upstream_new (struct prefix_sg *sg,
   if (up->sg.src.s_addr != INADDR_ANY)
     wheel_add_item (pim_upstream_sg_wheel, up);
 
-  rpf_result = pim_rpf_update(up, NULL);
+  rpf_result = pim_rpf_update(up, NULL, 1);
   if (rpf_result == PIM_RPF_FAILURE) {
     if (PIM_DEBUG_TRACE)
       zlog_debug ("%s: Attempting to create upstream(%s), Unable to RPF for source", __PRETTY_FUNCTION__,
@@ -631,7 +650,11 @@ pim_upstream_new (struct prefix_sg *sg,
   listnode_add_sort(pim_upstream_list, up);
 
   if (PIM_DEBUG_TRACE)
-    zlog_debug ("%s: Created Upstream %s", __PRETTY_FUNCTION__, up->sg_str);
+    {
+      zlog_debug ("%s: Created Upstream %s upstream_addr %s",
+            __PRETTY_FUNCTION__, up->sg_str,
+            inet_ntoa (up->upstream_addr));
+    }
 
   return up;
 }
@@ -1403,7 +1426,7 @@ pim_upstream_find_new_rpf (void)
 	  if (PIM_DEBUG_TRACE)
 	    zlog_debug ("Upstream %s without a path to send join, checking",
 			up->sg_str);
-	  pim_rpf_update (up, NULL);
+	  pim_rpf_update (up, NULL, 1);
 	}
     }
 }
diff --git a/pimd/pim_upstream.h b/pimd/pim_upstream.h
index b191fe940..0bc4c4410 100644
--- a/pimd/pim_upstream.h
+++ b/pimd/pim_upstream.h
@@ -183,4 +183,5 @@ void pim_upstream_init (void);
 void pim_upstream_terminate (void);
 
 void join_timer_start (struct pim_upstream *up);
+int pim_upstream_compare (void *arg1, void *arg2);
 #endif /* PIM_UPSTREAM_H */
diff --git a/pimd/pim_zebra.c b/pimd/pim_zebra.c
index 1db6616c5..ad4ef64ed 100644
--- a/pimd/pim_zebra.c
+++ b/pimd/pim_zebra.c
@@ -45,6 +45,7 @@
 #include "pim_rp.h"
 #include "pim_igmpv3.h"
 #include "pim_jp_agg.h"
+#include "pim_nht.h"
 
 #undef PIM_DEBUG_IFADDR_DUMP
 #define PIM_DEBUG_IFADDR_DUMP
@@ -375,8 +376,8 @@ static void scan_upstream_rpf_cache()
 
     old.source_nexthop.interface = up->rpf.source_nexthop.interface;
     old.source_nexthop.nbr       = up->rpf.source_nexthop.nbr;
-    rpf_result = pim_rpf_update(up, &old);
-    zlog_debug ("Looking at upstream: %s %d", up->sg_str, rpf_result);
+    rpf_result = pim_rpf_update(up, &old, 0);
+
     if (rpf_result == PIM_RPF_FAILURE)
       continue;
 
@@ -444,7 +445,7 @@ static void scan_upstream_rpf_cache()
 }
 
 void
-pim_scan_individual_oil (struct channel_oil *c_oil)
+pim_scan_individual_oil (struct channel_oil *c_oil, int in_vif_index)
 {
   struct in_addr vif_source;
   int input_iface_vif_index;
@@ -453,7 +454,10 @@ pim_scan_individual_oil (struct channel_oil *c_oil)
   if (!pim_rp_set_upstream_addr (&vif_source, c_oil->oil.mfcc_origin, c_oil->oil.mfcc_mcastgrp))
     return;
 
-  input_iface_vif_index = fib_lookup_if_vif_index (vif_source);
+  if (in_vif_index)
+    input_iface_vif_index = in_vif_index;
+  else
+    input_iface_vif_index = fib_lookup_if_vif_index (vif_source);
   if (input_iface_vif_index < 1)
     {
       if (PIM_DEBUG_ZEBRA)
@@ -548,7 +552,7 @@ void pim_scan_oil()
   ++qpim_scan_oil_events;
 
   for (ALL_LIST_ELEMENTS(pim_channel_oil_list, node, nextnode, c_oil))
-    pim_scan_individual_oil (c_oil);
+    pim_scan_individual_oil (c_oil, 0);
 }
 
 static int on_rpf_cache_refresh(struct thread *t)
@@ -594,124 +598,11 @@ void sched_rpf_cache_refresh(void)
                        0, qpim_rpf_cache_refresh_delay_msec);
 }
 
-static int redist_read_ipv4_route(int command, struct zclient *zclient,
-				  zebra_size_t length, vrf_id_t vrf_id)
+static int
+pim_zebra_nexthop_update (int command, struct zclient *zclient,
+                          zebra_size_t length, vrf_id_t vrf_id)
 {
-  struct stream *s;
-  struct zapi_ipv4 api;
-  ifindex_t ifindex;
-  struct in_addr nexthop;
-  struct prefix_ipv4 p;
-  int min_len = 4;
-
-  if (length < min_len) {
-    zlog_warn("%s %s: short buffer: length=%d min=%d",
-	      __FILE__, __PRETTY_FUNCTION__,
-	      length, min_len);
-    return -1;
-  }
-
-  s = zclient->ibuf;
-  ifindex = 0;
-  nexthop.s_addr = 0;
-
-  /* Type, flags, message. */
-  api.type = stream_getc(s);
-  api.instance = stream_getw (s);
-  api.flags = stream_getl(s);
-  api.message = stream_getc(s);
-
-  /* IPv4 prefix length. */
-  memset(&p, 0, sizeof(struct prefix_ipv4));
-  p.family = AF_INET;
-  p.prefixlen = stream_getc(s);
-
-  min_len +=
-    PSIZE(p.prefixlen) +
-    CHECK_FLAG(api.message, ZAPI_MESSAGE_NEXTHOP) ? 5 : 0 +
-    CHECK_FLAG(api.message, ZAPI_MESSAGE_IFINDEX) ? 5 : 0 +
-    CHECK_FLAG(api.message, ZAPI_MESSAGE_DISTANCE) ? 1 : 0 +
-    CHECK_FLAG(api.message, ZAPI_MESSAGE_METRIC) ? 4 : 0;
-
-  if (PIM_DEBUG_ZEBRA) {
-    zlog_debug("%s %s: length=%d min_len=%d flags=%s%s%s%s",
-	       __FILE__, __PRETTY_FUNCTION__,
-	       length, min_len,
-	       CHECK_FLAG(api.message, ZAPI_MESSAGE_NEXTHOP) ? "nh" : "",
-	       CHECK_FLAG(api.message, ZAPI_MESSAGE_IFINDEX) ? " ifi" : "",
-	       CHECK_FLAG(api.message, ZAPI_MESSAGE_DISTANCE) ? " dist" : "",
-	       CHECK_FLAG(api.message, ZAPI_MESSAGE_METRIC) ? " metr" : "");
-  }
-
-  /* IPv4 prefix. */
-  stream_get(&p.prefix, s, PSIZE(p.prefixlen));
-
-  /* Nexthop, ifindex, distance, metric. */
-  if (CHECK_FLAG(api.message, ZAPI_MESSAGE_NEXTHOP)) {
-    api.nexthop_num = stream_getc(s);
-    nexthop.s_addr = stream_get_ipv4(s);
-  }
-  if (CHECK_FLAG(api.message, ZAPI_MESSAGE_IFINDEX)) {
-    api.ifindex_num = stream_getc(s);
-    ifindex = stream_getl(s);
-  }
-
-  api.distance = CHECK_FLAG(api.message, ZAPI_MESSAGE_DISTANCE) ?
-    stream_getc(s) :
-    0;
-
-  api.metric = CHECK_FLAG(api.message, ZAPI_MESSAGE_METRIC) ?
-    stream_getl(s) :
-    0;
-
-  if (CHECK_FLAG (api.message, ZAPI_MESSAGE_TAG))
-    api.tag = stream_getl (s);
-  else
-    api.tag = 0;
-
-  switch (command) {
-  case ZEBRA_REDISTRIBUTE_IPV4_ADD:
-    if (PIM_DEBUG_ZEBRA) {
-      char buf[2][INET_ADDRSTRLEN];
-      zlog_debug("%s: add %s %s/%d "
-		 "nexthop %s ifindex %d metric%s %u distance%s %u",
-		 __PRETTY_FUNCTION__,
-		 zebra_route_string(api.type),
-		 inet_ntop(AF_INET, &p.prefix, buf[0], sizeof(buf[0])),
-		 p.prefixlen,
-		 inet_ntop(AF_INET, &nexthop, buf[1], sizeof(buf[1])),
-		 ifindex,
-		 CHECK_FLAG(api.message, ZAPI_MESSAGE_METRIC) ? "-recv" : "-miss",
-		 api.metric,
-		 CHECK_FLAG(api.message, ZAPI_MESSAGE_DISTANCE) ? "-recv" : "-miss",
-		 api.distance);
-    }
-    break;
-  case ZEBRA_REDISTRIBUTE_IPV4_DEL:
-    if (PIM_DEBUG_ZEBRA) {
-      char buf[2][INET_ADDRSTRLEN];
-      zlog_debug("%s: delete %s %s/%d "
-		 "nexthop %s ifindex %d metric%s %u distance%s %u",
-		 __PRETTY_FUNCTION__,
-		 zebra_route_string(api.type),
-		 inet_ntop(AF_INET, &p.prefix, buf[0], sizeof(buf[0])),
-		 p.prefixlen,
-		 inet_ntop(AF_INET, &nexthop, buf[1], sizeof(buf[1])),
-		 ifindex,
-		 CHECK_FLAG(api.message, ZAPI_MESSAGE_METRIC) ? "-recv" : "-miss",
-		 api.metric,
-		 CHECK_FLAG(api.message, ZAPI_MESSAGE_DISTANCE) ? "-recv" : "-miss",
-		 api.distance);
-    }
-    break;
-  default:
-    zlog_warn("%s: unknown command=%d", __PRETTY_FUNCTION__, command);
-    return -1;
-  }
-
-  sched_rpf_cache_refresh();
-
-  pim_rp_setup ();
+  pim_parse_nexthop_update (zclient, command, vrf_id);
   return 0;
 }
 
@@ -742,8 +633,7 @@ void pim_zebra_init(void)
   zclient->interface_down           = pim_zebra_if_state_down;
   zclient->interface_address_add    = pim_zebra_if_address_add;
   zclient->interface_address_delete = pim_zebra_if_address_del;
-  zclient->redistribute_route_ipv4_add    = redist_read_ipv4_route;
-  zclient->redistribute_route_ipv4_del    = redist_read_ipv4_route;
+  zclient->nexthop_update           = pim_zebra_nexthop_update;
 
   zclient_init(zclient, ZEBRA_ROUTE_PIM, 0);
   if (PIM_DEBUG_PIM_TRACE) {
@@ -1262,3 +1152,11 @@ pim_zebra_zclient_update (struct vty *vty)
     vty_out(vty, "<null zclient>%s", VTY_NEWLINE);
   }
 }
+
+struct zclient *pim_zebra_zclient_get (void)
+{
+  if (zclient)
+    return zclient;
+  else
+    return NULL;
+}
diff --git a/pimd/pim_zebra.h b/pimd/pim_zebra.h
index 751a7be25..5dc06a4a1 100644
--- a/pimd/pim_zebra.h
+++ b/pimd/pim_zebra.h
@@ -21,13 +21,16 @@
 #ifndef PIM_ZEBRA_H
 #define PIM_ZEBRA_H
 
+#include <zebra.h>
+#include "zclient.h"
+
 #include "pim_igmp.h"
 #include "pim_ifchannel.h"
 
 void pim_zebra_init(void);
 void pim_zebra_zclient_update (struct vty *vty);
 
-void pim_scan_individual_oil (struct channel_oil *c_oil);
+void pim_scan_individual_oil (struct channel_oil *c_oil, int in_vif_index);
 void pim_scan_oil(void);
 
 void igmp_anysource_forward_start(struct igmp_group *group);
@@ -40,4 +43,5 @@ void pim_forward_start(struct pim_ifchannel *ch);
 void pim_forward_stop(struct pim_ifchannel *ch);
 
 void sched_rpf_cache_refresh(void);
+struct zclient *pim_zebra_zclient_get (void);
 #endif /* PIM_ZEBRA_H */
diff --git a/pimd/pimd.c b/pimd/pimd.c
index c8a0efc40..b67544b28 100644
--- a/pimd/pimd.c
+++ b/pimd/pimd.c
@@ -26,6 +26,8 @@
 #include "prefix.h"
 #include "vty.h"
 #include "plist.h"
+#include "hash.h"
+#include "jhash.h"
 
 #include "pimd.h"
 #include "pim_cmd.h"
@@ -40,6 +42,7 @@
 #include "pim_static.h"
 #include "pim_rp.h"
 #include "pim_zlookup.h"
+#include "pim_nht.h"
 
 const char *const PIM_ALL_SYSTEMS      = MCAST_ALL_SYSTEMS;
 const char *const PIM_ALL_ROUTERS      = MCAST_ALL_ROUTERS;
@@ -71,10 +74,127 @@ unsigned int              qpim_keep_alive_time = PIM_KEEPALIVE_PERIOD;
 signed int                qpim_rp_keep_alive_time = 0;
 int64_t                   qpim_nexthop_lookups = 0;
 int                       qpim_packet_process = PIM_DEFAULT_PACKET_PROCESS;
+struct pim_instance          *pimg = NULL;
 
 int32_t qpim_register_suppress_time = PIM_REGISTER_SUPPRESSION_TIME_DEFAULT;
 int32_t qpim_register_probe_time = PIM_REGISTER_PROBE_TIME_DEFAULT;
 
+static struct pim_instance *pim_instance_init (vrf_id_t vrf_id, afi_t afi);
+static void pim_instance_terminate (void);
+
+static int
+pim_vrf_new (struct vrf *vrf)
+{
+  zlog_debug ("VRF Created: %s(%d)", vrf->name, vrf->vrf_id);
+  return 0;
+}
+
+static int
+pim_vrf_delete (struct vrf *vrf)
+{
+  zlog_debug ("VRF Deletion: %s(%d)", vrf->name, vrf->vrf_id);
+  return 0;
+}
+
+static int
+pim_vrf_enable (struct vrf *vrf)
+{
+
+  if (!vrf) // unexpected
+    return -1;
+
+  if (vrf->vrf_id == VRF_DEFAULT)
+    {
+      pimg = pim_instance_init (VRF_DEFAULT, AFI_IP);
+      if (pimg == NULL)
+        zlog_err ("%s %s: pim class init failure ", __FILE__,
+              __PRETTY_FUNCTION__);
+    }
+  return 0;
+}
+
+static int
+pim_vrf_disable (struct vrf *vrf)
+{
+  if (vrf->vrf_id == VRF_DEFAULT)
+    return 0;
+
+  if (vrf->vrf_id == VRF_DEFAULT)
+    pim_instance_terminate ();
+
+  /* Note: This is a callback, the VRF will be deleted by the caller. */
+  return 0;
+}
+
+void
+pim_vrf_init (void)
+{
+  vrf_add_hook (VRF_NEW_HOOK, pim_vrf_new);
+  vrf_add_hook (VRF_ENABLE_HOOK, pim_vrf_enable);
+  vrf_add_hook (VRF_DISABLE_HOOK, pim_vrf_disable);
+  vrf_add_hook (VRF_DELETE_HOOK, pim_vrf_delete);
+
+  vrf_init ();
+}
+
+static void
+pim_vrf_terminate (void)
+{
+  vrf_add_hook (VRF_NEW_HOOK, NULL);
+  vrf_add_hook (VRF_ENABLE_HOOK, NULL);
+  vrf_add_hook (VRF_DISABLE_HOOK, NULL);
+  vrf_add_hook (VRF_DELETE_HOOK, NULL);
+
+  vrf_terminate ();
+}
+
+/* Key generate for pim->rpf_hash */
+static unsigned int
+pim_rpf_hash_key (void *arg)
+{
+  struct pim_nexthop_cache *r = (struct pim_nexthop_cache *) arg;
+
+  return jhash_1word (r->rpf.rpf_addr.u.prefix4.s_addr, 0);
+}
+
+/* Compare pim->rpf_hash node data */
+static int
+pim_rpf_equal (const void *arg1, const void *arg2)
+{
+  const struct pim_nexthop_cache *r1 =
+    (const struct pim_nexthop_cache *) arg1;
+  const struct pim_nexthop_cache *r2 =
+    (const struct pim_nexthop_cache *) arg2;
+
+  return prefix_same (&r1->rpf.rpf_addr, &r2->rpf.rpf_addr);
+}
+
+/* Cleanup pim->rpf_hash each node data */
+static void
+pim_rp_list_hash_clean (void *data)
+{
+  struct pim_nexthop_cache *pnc;
+
+  pnc = (struct pim_nexthop_cache *) data;
+  if (pnc->rp_list->count)
+    list_delete_all_node (pnc->rp_list);
+  if (pnc->upstream_list->count)
+    list_delete_all_node (pnc->upstream_list);
+}
+
+static void
+pim_instance_terminate (void)
+{
+  /* Traverse and cleanup rpf_hash */
+  if (pimg && pimg->rpf_hash)
+    {
+      hash_clean (pimg->rpf_hash, (void *) pim_rp_list_hash_clean);
+      hash_free (pimg->rpf_hash);
+    }
+
+  XFREE (MTYPE_PIM_PIM_INSTANCE, pimg);
+}
+
 static void pim_free()
 {
   pim_ssmpingd_destroy();
@@ -96,6 +216,27 @@ static void pim_free()
   zprivs_terminate(&pimd_privs);
 }
 
+static struct pim_instance *
+pim_instance_init (vrf_id_t vrf_id, afi_t afi)
+{
+  struct pim_instance *pim;
+
+  pim = XCALLOC (MTYPE_PIM_PIM_INSTANCE, sizeof (struct pim_instance));
+  if (!pim)
+    return NULL;
+
+  pim->vrf_id = vrf_id;
+  pim->afi = afi;
+
+  pim->rpf_hash = hash_create_size (256, pim_rpf_hash_key, pim_rpf_equal);
+
+  if (PIM_DEBUG_ZEBRA)
+    zlog_debug ("%s: NHT rpf hash init ", __PRETTY_FUNCTION__);
+
+
+  return pim;
+}
+
 void pim_init()
 {
   qpim_rp_keep_alive_time = PIM_RP_KEEPALIVE_PERIOD;
@@ -147,4 +288,11 @@ void pim_init()
 void pim_terminate()
 {
   pim_free();
+
+  /* reverse prefix_list_init */
+  prefix_list_add_hook (NULL);
+  prefix_list_delete_hook (NULL);
+  prefix_list_reset ();
+
+  pim_vrf_terminate ();
 }
diff --git a/pimd/pimd.h b/pimd/pimd.h
index 090601630..b3bdd9e24 100644
--- a/pimd/pimd.h
+++ b/pimd/pimd.h
@@ -22,6 +22,8 @@
 #define PIMD_H
 
 #include <stdint.h>
+#include "zebra.h"
+#include "libfrr.h"
 
 #include "pim_str.h"
 #include "pim_memory.h"
@@ -232,10 +234,21 @@ extern int32_t qpim_register_probe_time;
 #define PIM_DONT_DEBUG_MSDP_PACKETS        (qpim_debugs &= ~PIM_MASK_MSDP_PACKETS)
 #define PIM_DONT_DEBUG_MSDP_INTERNAL       (qpim_debugs &= ~PIM_MASK_MSDP_INTERNAL)
 
+/* Per VRF PIM DB */
+struct pim_instance
+{
+  afi_t afi;
+  vrf_id_t vrf_id;
+  struct hash *rpf_hash;
+};
+
+extern struct pim_instance *pimg; //Pim Global Instance
+
 void pim_init(void);
 void pim_terminate(void);
 
 extern void pim_route_map_init (void);
 extern void pim_route_map_terminate(void);
+void pim_vrf_init (void);
 
 #endif /* PIMD_H */