zebra: support FIB override routes

FIB override routes are for routing protocols that establish
shortcut routes, or establish point-to-point routes that should
not be redistributed. Namely this is useful NHRP daemon to come.

Zebra is extended to select two entries from RIB the "best" entry
from routing protocols, and the FIB entry to install to kernel.
FIB override routes are never selected as best entry, and thus
are never adverticed to other routing daemons. The best FIB
override, or if it does not exist the otherwise best RIB is
selected as FIB entry to be installed.

Signed-off-by: Timo Teräs <timo.teras@iki.fi>
Acked-by: Donald Sharp <sharpd@cumulusnetworks.com>
[CF: Massage to fit cumulus tree]
Signed-off-by: Christian Franke <chris@opensourcerouting.org>

7 files changed, 214 insertions, 142 deletions

diff --git a/lib/zebra.h b/lib/zebra.h
index 8763abd76..599daa350 100644
--- a/lib/zebra.h
+++ b/lib/zebra.h
@@ -474,6 +474,7 @@ extern const char *zserv_command_string (unsigned int command);
 #define ZEBRA_FLAG_STATIC             0x40
 #define ZEBRA_FLAG_REJECT             0x80
 #define ZEBRA_FLAG_SCOPE_LINK         0x100
+#define ZEBRA_FLAG_FIB_OVERRIDE       0x200
 
 #ifndef INADDR_LOOPBACK
 #define	INADDR_LOOPBACK	0x7f000001	/* Internet address 127.0.0.1.  */
diff --git a/zebra/rib.h b/zebra/rib.h
index 285166f06..96301a8af 100644
--- a/zebra/rib.h
+++ b/zebra/rib.h
@@ -87,6 +87,7 @@ struct rib
   /* to simplify NHT logic when NHs change, instead of doing a NH by NH cmp */
 #define RIB_ENTRY_NEXTHOPS_CHANGED 0x2
 #define RIB_ENTRY_CHANGED          0x4
+#define RIB_ENTRY_SELECTED_FIB     0x8
 
   /* Nexthop information. */
   u_char nexthop_num;
diff --git a/zebra/zebra_fpm.c b/zebra/zebra_fpm.c
index 5a68bcbfa..a5a953c51 100644
--- a/zebra/zebra_fpm.c
+++ b/zebra/zebra_fpm.c
@@ -928,7 +928,7 @@ zfpm_route_for_update (rib_dest_t *dest)
 
   RIB_DEST_FOREACH_ROUTE (dest, rib)
     {
-      if (!CHECK_FLAG (rib->flags, ZEBRA_FLAG_SELECTED))
+      if (!CHECK_FLAG (rib->status, RIB_ENTRY_SELECTED_FIB))
 	continue;
 
       return rib;
diff --git a/zebra/zebra_rib.c b/zebra/zebra_rib.c
index f57c0b5d6..b7d12a5cb 100644
--- a/zebra/zebra_rib.c
+++ b/zebra/zebra_rib.c
@@ -432,7 +432,7 @@ nexthop_active_ipv4 (struct rib *rib, struct nexthop *nexthop, int set,
           /* if the next hop is imported from another table, skip it */
           if (match->type == ZEBRA_ROUTE_TABLE)
             continue;
-	  if (CHECK_FLAG (match->flags, ZEBRA_FLAG_SELECTED))
+	  if (CHECK_FLAG (match->status, RIB_ENTRY_SELECTED_FIB))
 	    break;
 	}
 
@@ -639,7 +639,7 @@ nexthop_active_ipv6 (struct rib *rib, struct nexthop *nexthop, int set,
 	{
 	  if (CHECK_FLAG (match->status, RIB_ENTRY_REMOVED))
 	    continue;
-	  if (CHECK_FLAG (match->flags, ZEBRA_FLAG_SELECTED))
+	  if (CHECK_FLAG (match->status, RIB_ENTRY_SELECTED_FIB))
 	    break;
 	}
 
@@ -804,7 +804,7 @@ rib_match (afi_t afi, safi_t safi, vrf_id_t vrf_id,
 	{
 	  if (CHECK_FLAG (match->status, RIB_ENTRY_REMOVED))
 	    continue;
-	  if (CHECK_FLAG (match->flags, ZEBRA_FLAG_SELECTED))
+	  if (CHECK_FLAG (match->status, RIB_ENTRY_SELECTED_FIB))
 	    break;
 	}
 
@@ -940,7 +940,7 @@ rib_lookup_ipv4 (struct prefix_ipv4 *p, vrf_id_t vrf_id)
     {
       if (CHECK_FLAG (match->status, RIB_ENTRY_REMOVED))
 	continue;
-      if (CHECK_FLAG (match->flags, ZEBRA_FLAG_SELECTED))
+      if (CHECK_FLAG (match->status, RIB_ENTRY_SELECTED_FIB))
 	break;
     }
 
@@ -960,7 +960,7 @@ rib_lookup_ipv4 (struct prefix_ipv4 *p, vrf_id_t vrf_id)
 /*
  * This clone function, unlike its original rib_lookup_ipv4(), checks
  * if specified IPv4 route record (prefix/mask -> gate) exists in
- * the whole RIB and has ZEBRA_FLAG_SELECTED set.
+ * the whole RIB and has RIB_ENTRY_SELECTED_FIB set.
  *
  * Return values:
  * -1: error
@@ -1000,7 +1000,7 @@ rib_lookup_ipv4_route (struct prefix_ipv4 *p, union sockunion * qgate,
     {
       if (CHECK_FLAG (match->status, RIB_ENTRY_REMOVED))
 	continue;
-      if (CHECK_FLAG (match->flags, ZEBRA_FLAG_SELECTED))
+      if (CHECK_FLAG (match->status, RIB_ENTRY_SELECTED_FIB))
 	break;
     }
 
@@ -1310,14 +1310,19 @@ rib_uninstall (struct route_node *rn, struct rib *rib)
 {
   rib_table_info_t *info = rn->table->info;
 
-  if (CHECK_FLAG (rib->flags, ZEBRA_FLAG_SELECTED))
+  if (CHECK_FLAG (rib->status, RIB_ENTRY_SELECTED_FIB))
     {
       if (info->safi == SAFI_UNICAST)
         zfpm_trigger_update (rn, "rib_uninstall");
 
-      redistribute_delete (&rn->p, rib);
       if (! RIB_SYSTEM_ROUTE (rib))
 	rib_uninstall_kernel (rn, rib);
+      UNSET_FLAG (rib->status, RIB_ENTRY_SELECTED_FIB);
+    }
+
+  if (CHECK_FLAG (rib->flags, ZEBRA_FLAG_SELECTED))
+    {
+      redistribute_delete (&rn->p, rib);
       UNSET_FLAG (rib->flags, ZEBRA_FLAG_SELECTED);
     }
 }
@@ -1385,76 +1390,70 @@ rib_gc_dest (struct route_node *rn)
 }
 
 static void
-rib_process_add_route (struct zebra_vrf *zvrf, struct route_node *rn,
-                       struct rib *select)
+rib_process_add_fib(struct zebra_vrf *zvrf, struct route_node *rn,
+                    struct rib *new)
 {
   char buf[INET6_ADDRSTRLEN];
-  int installed = 1;
 
   zfpm_trigger_update (rn, "new route selected");
 
   /* Update real nexthop. This may actually determine if nexthop is active or not. */
-  if (!nexthop_active_update (rn, select, 1))
+  if (!nexthop_active_update (rn, new, 1))
     {
-      UNSET_FLAG(select->status, RIB_ENTRY_CHANGED);
+      UNSET_FLAG(new->status, RIB_ENTRY_CHANGED);
       return;
     }
 
-  SET_FLAG (select->flags, ZEBRA_FLAG_SELECTED);
+  SET_FLAG (new->status, RIB_ENTRY_SELECTED_FIB);
   if (IS_ZEBRA_DEBUG_RIB)
     {
       inet_ntop (rn->p.family, &rn->p.u.prefix, buf, INET6_ADDRSTRLEN);
       zlog_debug ("%u:%s/%d: Adding route rn %p, rib %p (type %d)",
-                   zvrf->vrf_id, buf, rn->p.prefixlen, rn, select, select->type);
+                   zvrf->vrf_id, buf, rn->p.prefixlen, rn, new, new->type);
     }
 
-  if (!RIB_SYSTEM_ROUTE (select))
+  if (!RIB_SYSTEM_ROUTE (new))
     {
-      if (rib_install_kernel (rn, select, 0))
+      if (rib_install_kernel (rn, new, 0))
         {
-          installed = 0;
           inet_ntop (rn->p.family, &rn->p.u.prefix, buf, INET6_ADDRSTRLEN);
           zlog_warn ("%u:%s/%d: Route install failed",
                      zvrf->vrf_id, buf, rn->p.prefixlen);
         }
     }
 
-  /* Update for redistribution. */
-  if (installed)
-    redistribute_update (&rn->p, select, NULL);
-  UNSET_FLAG(select->status, RIB_ENTRY_CHANGED);
+  UNSET_FLAG(new->status, RIB_ENTRY_CHANGED);
 }
 
 static void
-rib_process_del_route (struct zebra_vrf *zvrf, struct route_node *rn,
-                       struct rib *fib)
+rib_process_del_fib(struct zebra_vrf *zvrf, struct route_node *rn,
+                    struct rib *old)
 {
   char buf[INET6_ADDRSTRLEN];
 
   zfpm_trigger_update (rn, "removing existing route");
 
-  /* Withdraw redistribute and uninstall from kernel. */
+  /* Uninstall from kernel. */
   if (IS_ZEBRA_DEBUG_RIB)
     {
       inet_ntop (rn->p.family, &rn->p.u.prefix, buf, INET6_ADDRSTRLEN);
       zlog_debug ("%u:%s/%d: Deleting route rn %p, rib %p (type %d)",
-                  zvrf->vrf_id, buf, rn->p.prefixlen, rn, fib, fib->type);
+                  zvrf->vrf_id, buf, rn->p.prefixlen, rn, old, old->type);
     }
 
-  redistribute_delete(&rn->p, fib);
-  if (!RIB_SYSTEM_ROUTE (fib))
-    rib_uninstall_kernel (rn, fib);
+  if (!RIB_SYSTEM_ROUTE (old))
+    rib_uninstall_kernel (rn, old);
 
-  UNSET_FLAG (fib->flags, ZEBRA_FLAG_SELECTED);
+  UNSET_FLAG (old->status, RIB_ENTRY_SELECTED_FIB);
 
   /* Update nexthop for route, reset changed flag. */
-  nexthop_active_update (rn, fib, 1);
-  UNSET_FLAG(fib->status, RIB_ENTRY_CHANGED);
+  nexthop_active_update (rn, old, 1);
+  UNSET_FLAG(old->status, RIB_ENTRY_CHANGED);
 }
 
 static void
-rib_process_update_route (struct zebra_vrf *zvrf, struct route_node *rn,
-                          struct rib *select, struct rib *fib)
+rib_process_update_fib (struct zebra_vrf *zvrf, struct route_node *rn,
+                        struct rib *old, struct rib *new)
 {
   char buf[INET6_ADDRSTRLEN];
   struct nexthop *nexthop = NULL, *tnexthop;
@@ -1469,13 +1468,13 @@ rib_process_update_route (struct zebra_vrf *zvrf, struct route_node *rn,
    * We have to install or update if a new route has been selected or
    * something has changed.
    */
-  if (select != fib ||
-      CHECK_FLAG (select->status, RIB_ENTRY_CHANGED))
+  if (new != old ||
+      CHECK_FLAG (new->status, RIB_ENTRY_CHANGED))
     {
       zfpm_trigger_update (rn, "updating existing route");
 
       /* Update the nexthop; we could determine here that nexthop is inactive. */
-      if (nexthop_active_update (rn, select, 1))
+      if (nexthop_active_update (rn, new, 1))
         nh_active = 1;
 
       /* If nexthop is active, install the selected route, if appropriate. If
@@ -1486,18 +1485,18 @@ rib_process_update_route (struct zebra_vrf *zvrf, struct route_node *rn,
         {
           if (IS_ZEBRA_DEBUG_RIB)
             {
-              if (select != fib)
+              if (new != old)
                 zlog_debug ("%u:%s/%d: Updating route rn %p, rib %p (type %d) "
                             "old %p (type %d)", zvrf->vrf_id, buf, rn->p.prefixlen,
-                            rn, select, select->type, fib, fib->type);
+                            rn, new, new->type, old, old->type);
               else
                 zlog_debug ("%u:%s/%d: Updating route rn %p, rib %p (type %d)",
-                        zvrf->vrf_id, buf, rn->p.prefixlen, rn, select, select->type);
+                        zvrf->vrf_id, buf, rn->p.prefixlen, rn, new, new->type);
             }
           /* Non-system route should be installed. */
-          if (!RIB_SYSTEM_ROUTE (select))
+          if (!RIB_SYSTEM_ROUTE (new))
             {
-              if (rib_install_kernel (rn, select, 1))
+              if (rib_install_kernel (rn, new, 1))
                 {
                   installed = 0;
                   inet_ntop (rn->p.family, &rn->p.u.prefix, buf, INET6_ADDRSTRLEN);
@@ -1507,26 +1506,23 @@ rib_process_update_route (struct zebra_vrf *zvrf, struct route_node *rn,
             }
 
           /* If install succeeded or system route, cleanup flags for prior route. */
-          if (installed && select != fib)
+          if (installed && new != old)
             {
-              if (RIB_SYSTEM_ROUTE(select))
+              if (RIB_SYSTEM_ROUTE(new))
                 {
-                  if (!RIB_SYSTEM_ROUTE (fib))
-                    rib_uninstall_kernel (rn, fib);
+                  if (!RIB_SYSTEM_ROUTE (old))
+                    rib_uninstall_kernel (rn, old);
                 }
               else
                 {
-                  for (nexthop = fib->nexthop; nexthop; nexthop = nexthop->next)
+                  for (nexthop = old->nexthop; nexthop; nexthop = nexthop->next)
                     UNSET_FLAG (nexthop->flags, NEXTHOP_FLAG_FIB);
                 }
             }
 
           /* Update for redistribution. */
           if (installed)
-            {
-              SET_FLAG (select->flags, ZEBRA_FLAG_SELECTED);
-              redistribute_update (&rn->p, select, (select == fib) ? NULL : fib);
-            }
+            SET_FLAG (new->status, RIB_ENTRY_SELECTED_FIB);
         }
 
       /*
@@ -1535,28 +1531,22 @@ rib_process_update_route (struct zebra_vrf *zvrf, struct route_node *rn,
        */
       if (!nh_active || !installed)
         {
-          struct rib *del;
-
           if (IS_ZEBRA_DEBUG_RIB)
             {
-              if (select != fib)
+              if (new != old)
                 zlog_debug ("%u:%s/%d: Deleting route rn %p, rib %p (type %d) "
                             "old %p (type %d) - %s", zvrf->vrf_id, buf, rn->p.prefixlen,
-                            rn, select, select->type, fib, fib->type,
+                            rn, new, new->type, old, old->type,
                             nh_active ? "install failed" : "nexthop inactive");
               else
                 zlog_debug ("%u:%s/%d: Deleting route rn %p, rib %p (type %d) - %s",
-                            zvrf->vrf_id, buf, rn->p.prefixlen, rn, select, select->type,
+                            zvrf->vrf_id, buf, rn->p.prefixlen, rn, new, new->type,
                             nh_active ? "install failed" : "nexthop inactive");
             }
 
-          del = (select == fib) ? select : fib;
-
-          redistribute_delete(&rn->p, del);
-
-          if (!RIB_SYSTEM_ROUTE (del))
-            rib_uninstall_kernel (rn, del);
-          UNSET_FLAG (select->flags, ZEBRA_FLAG_SELECTED);
+          if (!RIB_SYSTEM_ROUTE (old))
+            rib_uninstall_kernel (rn, old);
+          UNSET_FLAG (new->status, RIB_ENTRY_SELECTED_FIB);
         }
     }
   else
@@ -1567,33 +1557,33 @@ rib_process_update_route (struct zebra_vrf *zvrf, struct route_node *rn,
        * netlink reporting interface up before IPv4 or IPv6 protocol is ready
        * to add routes.
        */
-      if (!RIB_SYSTEM_ROUTE (select))
+      if (!RIB_SYSTEM_ROUTE (new))
         {
           int in_fib = 0;
 
-          for (ALL_NEXTHOPS_RO(select->nexthop, nexthop, tnexthop, recursing))
+          for (ALL_NEXTHOPS_RO(new->nexthop, nexthop, tnexthop, recursing))
             if (CHECK_FLAG (nexthop->flags, NEXTHOP_FLAG_FIB))
               {
                 in_fib = 1;
                 break;
               }
           if (!in_fib)
-            rib_install_kernel (rn, select, 0);
+            rib_install_kernel (rn, new, 0);
         }
     }
 
   /* Update prior route. */
-  if (select != fib)
+  if (new != old)
     {
-      UNSET_FLAG (fib->flags, ZEBRA_FLAG_SELECTED);
+      UNSET_FLAG (old->status, RIB_ENTRY_SELECTED_FIB);
 
       /* Set real nexthop. */
-      nexthop_active_update (rn, fib, 1);
-      UNSET_FLAG(fib->status, RIB_ENTRY_CHANGED);
+      nexthop_active_update (rn, old, 1);
+      UNSET_FLAG(old->status, RIB_ENTRY_CHANGED);
     }
 
   /* Clear changed flag. */
-  UNSET_FLAG(select->status, RIB_ENTRY_CHANGED);
+  UNSET_FLAG(new->status, RIB_ENTRY_CHANGED);
 }
 
 /* Check if 'alternate' RIB entry is better than 'current'. */
@@ -1644,33 +1634,32 @@ rib_process (struct route_node *rn)
 {
   struct rib *rib;
   struct rib *next;
-  struct rib *fib = NULL;
-  struct rib *select = NULL;
-  struct rib *del = NULL;
+  struct rib *old_selected = NULL;
+  struct rib *new_selected = NULL;
+  struct rib *old_fib = NULL;
+  struct rib *new_fib = NULL;
   struct rib *best = NULL;
   char buf[INET6_ADDRSTRLEN];
   rib_dest_t *dest;
   struct zebra_vrf *zvrf = NULL;
   vrf_id_t vrf_id = VRF_UNKNOWN;
-  rib_table_info_t *info;
 
   assert (rn);
-  info = rn->table->info;
-  
+
   dest = rib_dest_from_rnode (rn);
   if (dest)
     {
       zvrf = rib_dest_vrf (dest);
       vrf_id = zvrf->vrf_id;
     }
-  
+
   if (IS_ZEBRA_DEBUG_RIB)
     inet_ntop (rn->p.family, &rn->p.u.prefix, buf, INET6_ADDRSTRLEN);
 
   if (IS_ZEBRA_DEBUG_RIB_DETAILED)
     zlog_debug ("%u:%s/%d: Processing rn %p", vrf_id, buf, rn->p.prefixlen, rn);
 
-  RNODE_FOREACH_RIB_SAFE (rn, rib, next)
+  RNODE_FOREACH_RIB (rn, rib)
     {
       if (IS_ZEBRA_DEBUG_RIB_DETAILED)
         zlog_debug ("%u:%s/%d: Examine rib %p (type %d) status %x flags %x "
@@ -1680,31 +1669,23 @@ rib_process (struct route_node *rn)
 
       UNSET_FLAG(rib->status, RIB_ENTRY_NEXTHOPS_CHANGED);
 
-      /* Currently installed rib. */
+      /* Currently selected rib. */
       if (CHECK_FLAG (rib->flags, ZEBRA_FLAG_SELECTED))
         {
-          assert (fib == NULL);
-          fib = rib;
+          assert (old_selected == NULL);
+          old_selected = rib;
         }
-
-      /* Unlock removed routes, so they'll be freed, bar the FIB entry,
-       * which we need to do do further work with below.
-       */
-      if (CHECK_FLAG (rib->status, RIB_ENTRY_REMOVED))
+      /* Currently in fib */
+      if (CHECK_FLAG (rib->status, RIB_ENTRY_SELECTED_FIB))
         {
-          if (rib != fib)
-            {
-              if (IS_ZEBRA_DEBUG_RIB)
-		rnode_debug (rn, vrf_id, "rn %p, removing rib %p",
-			     (void *)rn, (void *)rib);
-	      rib_unlink (rn, rib);
-            }
-          else
-            del = rib;
-          
-          continue;
+          assert (old_fib == NULL);
+          old_fib = rib;
         }
-      
+
+      /* Skip deleted entries from selection */
+      if (CHECK_FLAG (rib->status, RIB_ENTRY_REMOVED))
+        continue;
+
       /* Skip unreachable nexthop. */
       /* This first call to nexthop_active_update is merely to determine if
        * there's any change to nexthops associated with this RIB entry. Now,
@@ -1721,9 +1702,15 @@ rib_process (struct route_node *rn)
         {
           if (rib->type == ZEBRA_ROUTE_TABLE)
             {
+              /* XXX: HERE BE DRAGONS!!!!!
+	       * In all honesty, I have not yet figured out what this part
+	       * does or why the RIB_ENTRY_CHANGED test above is correct
+	       * or why we need to delete a route here, and also not whether
+	       * this concerns both selected and fib route, or only selected
+	       * or only fib */
               /* This entry was denied by the 'ip protocol table' route-map, we
                * need to delete it */
-	      if (rib != fib)
+	      if (rib != old_selected)
 		{
 		  if (IS_ZEBRA_DEBUG_RIB)
 		    zlog_debug ("%s: %s/%d: imported via import-table but denied "
@@ -1732,15 +1719,12 @@ rib_process (struct route_node *rn)
 		  rib_unlink (rn, rib);
 		}
 	      else
-		del = rib;
+		SET_FLAG (rib->status, RIB_ENTRY_REMOVED);
             }
 
           continue;
         }
 
-      if (info->safi == SAFI_MULTICAST)
-	continue;
-
       /* Infinite distance. */
       if (rib->distance == DISTANCE_INFINITY)
         {
@@ -1748,33 +1732,101 @@ rib_process (struct route_node *rn)
           continue;
         }
 
-      best = rib_choose_best(select, rib);
-      if (select && best != select)
-        UNSET_FLAG (select->status, RIB_ENTRY_CHANGED);
+      if (CHECK_FLAG (rib->flags, ZEBRA_FLAG_FIB_OVERRIDE))
+        {
+          best = rib_choose_best(new_fib, rib);
+          if (new_fib && best != new_fib)
+            UNSET_FLAG (new_fib->status, RIB_ENTRY_CHANGED);
+         new_fib = best;
+        }
+      else
+        {
+          best = rib_choose_best(new_selected, rib);
+          if (new_selected && best != new_selected)
+            UNSET_FLAG (new_selected->status, RIB_ENTRY_CHANGED);
+          new_selected = best;
+        }
       if (best != rib)
         UNSET_FLAG (rib->status, RIB_ENTRY_CHANGED);
-      select = best;
-    } /* RNODE_FOREACH_RIB_SAFE */
+    } /* RNODE_FOREACH_RIB */
+
+  /* If no FIB override route, use the selected route also for FIB */
+  if (new_fib == NULL)
+    new_fib = new_selected;
 
   /* After the cycle is finished, the following pointers will be set:
-   * select --- the winner RIB entry, if any was found, otherwise NULL
-   * fib    --- the SELECTED RIB entry, if any, otherwise NULL
-   * del    --- equal to fib, if fib is queued for deletion, NULL otherwise
-   * rib    --- NULL
+   * old_selected --- RIB entry currently having SELECTED
+   * new_selected --- RIB entry that is newly SELECTED
+   * old_fib      --- RIB entry currently in kernel FIB
+   * new_fib      --- RIB entry that is newly to be in kernel FIB
+   *
+   * new_selected will get SELECTED flag, and is going to be redistributed
+   * the zclients. new_fib (which can be new_selected) will be installed in kernel.
    */
+
   if (IS_ZEBRA_DEBUG_RIB_DETAILED)
-    zlog_debug ("%u:%s/%d: After processing: select %p fib %p del %p",
-                vrf_id, buf, rn->p.prefixlen, select, fib, del);
+    {
+    zlog_debug ("%u:%s/%d: After processing: old_selected %p new_selected %p old_fib %p new_fib %p",
+                vrf_id, buf, rn->p.prefixlen,
+                (void *)old_selected,
+                (void *)new_selected,
+                (void *)old_fib,
+                (void *)new_fib);
+    }
+
+  /* Buffer RIB_ENTRY_CHANGED here, because it will get cleared if
+   * fib == selected */
+  bool selected_changed = new_selected && CHECK_FLAG(new_selected->status,
+                                                     RIB_ENTRY_CHANGED);
+
+  /* Update fib according to selection results */
+  if (new_fib && old_fib)
+    rib_process_update_fib (zvrf, rn, old_fib, new_fib);
+  else if (new_fib)
+    rib_process_add_fib (zvrf, rn, new_fib);
+  else if (old_fib)
+    rib_process_del_fib (zvrf, rn, old_fib);
+
+  /* Redistribute SELECTED entry */
+  if (old_selected != new_selected || selected_changed)
+    {
+      struct nexthop *nexthop, *tnexthop;
+      int recursing;
+
+      /* Check if we have a FIB route for the destination, otherwise,
+       * don't redistribute it */
+      for (ALL_NEXTHOPS_RO(new_fib ? new_fib->nexthop : NULL, nexthop,
+                           tnexthop, recursing))
+        {
+          if (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB))
+            {
+              break;
+            }
+        }
+      if (!nexthop)
+        new_selected = NULL;
 
-  /* Same RIB entry is selected. Update FIB and finish. */
-  if (select && select == fib)
-    rib_process_update_route (zvrf, rn, select, select);
-  else if (select && fib)
-    rib_process_update_route (zvrf, rn, select, fib);
-  else if (select)
-    rib_process_add_route (zvrf, rn, select);
-  else if (fib)
-    rib_process_del_route (zvrf, rn, fib);
+      if (new_selected && new_selected != new_fib)
+        {
+          nexthop_active_update(rn, new_selected, 1);
+          UNSET_FLAG(new_selected->status, RIB_ENTRY_CHANGED);
+        }
+
+      if (old_selected)
+        {
+          if (!new_selected)
+            redistribute_delete(&rn->p, old_selected);
+          if (old_selected != new_selected)
+            UNSET_FLAG (old_selected->flags, ZEBRA_FLAG_SELECTED);
+        }
+
+      if (new_selected)
+        {
+          /* Install new or replace existing redistributed entry */
+          SET_FLAG (new_selected->flags, ZEBRA_FLAG_SELECTED);
+          redistribute_update (&rn->p, new_selected, old_selected);
+        }
+    }
 
 #if 0
   if (select && select == fib)
@@ -1951,12 +2003,18 @@ rib_process (struct route_node *rn)
     }
 #endif
 
-  /* FIB route was removed, should be deleted */
-  if (del)
+  /* Remove all RIB entries queued for removal */
+  RNODE_FOREACH_RIB_SAFE (rn, rib, next)
     {
-      if (IS_ZEBRA_DEBUG_RIB)
-	rnode_debug (rn, vrf_id, "Deleting fib %p, rn %p", (void *)del, (void *)rn);
-      rib_unlink (rn, del);
+      if (CHECK_FLAG (rib->status, RIB_ENTRY_REMOVED))
+        {
+          if (IS_ZEBRA_DEBUG_RIB)
+            {
+              rnode_debug (rn, vrf_id, "rn %p, removing rib %p",
+                           (void *)rn, (void *)rib);
+            }
+          rib_unlink(rn, rib);
+        }
     }
 
   /*
@@ -2523,7 +2581,7 @@ void rib_lookup_and_pushup (struct prefix_ipv4 * p, vrf_id_t vrf_id)
    */
   RNODE_FOREACH_RIB (rn, rib)
   {
-    if (CHECK_FLAG (rib->flags, ZEBRA_FLAG_SELECTED) &&
+    if (CHECK_FLAG (rib->status, RIB_ENTRY_SELECTED_FIB) &&
       ! RIB_SYSTEM_ROUTE (rib))
     {
       changed = 1;
@@ -2669,7 +2727,7 @@ rib_delete (afi_t afi, safi_t safi, vrf_id_t vrf_id, int type, u_short instance,
       if (CHECK_FLAG (rib->status, RIB_ENTRY_REMOVED))
         continue;
 
-      if (CHECK_FLAG (rib->flags, ZEBRA_FLAG_SELECTED))
+      if (CHECK_FLAG (rib->status, RIB_ENTRY_SELECTED_FIB))
 	fib = rib;
 
       if (rib->type != type)
@@ -2730,7 +2788,7 @@ rib_delete (afi_t afi, safi_t safi, vrf_id_t vrf_id, int type, u_short instance,
 	      for (nexthop = fib->nexthop; nexthop; nexthop = nexthop->next)
 		UNSET_FLAG (nexthop->flags, NEXTHOP_FLAG_FIB);
 
-	      UNSET_FLAG (fib->flags, ZEBRA_FLAG_SELECTED);
+	      UNSET_FLAG (fib->status, RIB_ENTRY_SELECTED_FIB);
 	    }
 	  else
 	    {
@@ -3107,7 +3165,7 @@ rib_close_table (struct route_table *table)
     for (rn = route_top (table); rn; rn = route_next (rn))
       RNODE_FOREACH_RIB (rn, rib)
         {
-          if (!CHECK_FLAG (rib->flags, ZEBRA_FLAG_SELECTED))
+          if (!CHECK_FLAG (rib->status, RIB_ENTRY_SELECTED_FIB))
 	    continue;
 
           if (info->safi == SAFI_UNICAST)
diff --git a/zebra/zebra_static.c b/zebra/zebra_static.c
index f2362e687..1dc54e171 100644
--- a/zebra/zebra_static.c
+++ b/zebra/zebra_static.c
@@ -320,13 +320,21 @@ static_uninstall_route (afi_t afi, safi_t safi, struct prefix *p, struct static_
           /* If there are other active nexthops, do an update. */
           if (rib->nexthop_active_num > 1)
             {
-              rib_install_kernel (rn, rib, 1);
-              redistribute_update (&rn->p, rib, NULL);
+              /* Update route in kernel if it's in fib */
+              if (CHECK_FLAG(rib->status, RIB_ENTRY_SELECTED_FIB))
+                rib_install_kernel (rn, rib, 1);
+              /* Update redistribution if it's selected */
+              if (CHECK_FLAG(rib->flags, ZEBRA_FLAG_SELECTED))
+                redistribute_update (&rn->p, rib, NULL);
             }
           else
             {
-              redistribute_delete (&rn->p, rib);
-              rib_uninstall_kernel (rn, rib);
+              /* Remove from redistribute if selected route becomes inactive */
+              if (CHECK_FLAG(rib->flags, ZEBRA_FLAG_SELECTED))
+                redistribute_delete (&rn->p, rib);
+              /* Remove from kernel if fib route becomes inactive */
+              if (CHECK_FLAG(rib->status, RIB_ENTRY_SELECTED_FIB))
+                rib_uninstall_kernel (rn, rib);
             }
         }
 
diff --git a/zebra/zebra_vty.c b/zebra/zebra_vty.c
index 1a9fc2294..054979fd0 100644
--- a/zebra/zebra_vty.c
+++ b/zebra/zebra_vty.c
@@ -2023,6 +2023,10 @@ vty_show_ip_route_detail (struct vty *vty, struct route_node *rn, int mcast)
         }
       if (CHECK_FLAG (rib->flags, ZEBRA_FLAG_SELECTED))
 	vty_out (vty, ", best");
+      else if (CHECK_FLAG (rib->status, RIB_ENTRY_SELECTED_FIB))
+        vty_out (vty, ", fib");
+      if (CHECK_FLAG (rib->flags, ZEBRA_FLAG_FIB_OVERRIDE))
+        vty_out (vty, ", fib-override");
       if (rib->refcnt)
 	vty_out (vty, ", refcnt %ld", rib->refcnt);
       if (CHECK_FLAG (rib->flags, ZEBRA_FLAG_BLACKHOLE))
@@ -2299,7 +2303,8 @@ vty_show_ip_route (struct vty *vty, struct route_node *rn, struct rib *rib,
 	    len += vty_out (vty, "[%d]", rib->instance);
           len += vty_out (vty, "%c%c %s",
 			  CHECK_FLAG (rib->flags, ZEBRA_FLAG_SELECTED)
-			  ? '>' : ' ',
+			  ? '>' : CHECK_FLAG (rib->status, RIB_ENTRY_SELECTED_FIB)
+			  ? '!' : ' ',
 			  CHECK_FLAG (nexthop->flags, NEXTHOP_FLAG_FIB)
 			  ? '*' : ' ',
 			  prefix2str (&rn->p, buf, sizeof buf));
diff --git a/zebra/zserv.c b/zebra/zserv.c
index 32c5ea714..25f028af6 100644
--- a/zebra/zserv.c
+++ b/zebra/zserv.c
@@ -669,8 +669,7 @@ zsend_redistribute_route (int cmd, struct zserv *client, struct prefix *p,
 	  break;
 	}
 
-      if (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB)
-          || nexthop_has_fib_child(nexthop))
+      if (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_ACTIVE))
         {
           SET_FLAG (zapi_flags, ZAPI_MESSAGE_NEXTHOP);
           SET_FLAG (zapi_flags, ZAPI_MESSAGE_IFINDEX);
@@ -928,7 +927,7 @@ zsend_ipv4_nexthop_lookup_mrib (struct zserv *client, struct in_addr addr, struc
        * are looking up. Therefore, we will just iterate over the top
        * chain of nexthops. */
       for (nexthop = rib->nexthop; nexthop; nexthop = nexthop->next)
-	if (CHECK_FLAG (nexthop->flags, NEXTHOP_FLAG_FIB))
+	if (CHECK_FLAG (nexthop->flags, NEXTHOP_FLAG_ACTIVE))
 	    num += zsend_write_nexthop (s, nexthop);
     
       stream_putc_at (s, nump, num); /* store nexthop_num */