network: introduce ip nexthop routing

Used to manipulate entries in the kernel's nexthop tables.
Example:
```
[NextHop]
Id=3
Gateway=192.168.5.1
```

11 files changed, 788 insertions, 4 deletions

diff --git a/src/network/meson.build b/src/network/meson.build
index 6bed37a170..c16e095c2c 100644
--- a/src/network/meson.build
+++ b/src/network/meson.build
@@ -93,6 +93,8 @@ sources = files('''
         networkd-network-bus.h
         networkd-network.c
         networkd-network.h
+        networkd-nexthop.c
+        networkd-nexthop.h
         networkd-route.c
         networkd-route.h
         networkd-routing-policy-rule.c
diff --git a/src/network/networkd-link.c b/src/network/networkd-link.c
index a23bddde9b..897a2b2fc9 100644
--- a/src/network/networkd-link.c
+++ b/src/network/networkd-link.c
@@ -672,6 +672,9 @@ static Link *link_free(Link *link) {
         link->routes = set_free_with_destructor(link->routes, route_free);
         link->routes_foreign = set_free_with_destructor(link->routes_foreign, route_free);
 
+        link->nexthops = set_free_with_destructor(link->nexthops, nexthop_free);
+        link->nexthops_foreign = set_free_with_destructor(link->nexthops_foreign, nexthop_free);
+
         link->neighbors = set_free_with_destructor(link->neighbors, neighbor_free);
         link->neighbors_foreign = set_free_with_destructor(link->neighbors_foreign, neighbor_free);
 
@@ -903,6 +906,58 @@ static int link_request_set_routing_policy_rule(Link *link) {
         return 0;
 }
 
+static int nexthop_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *link) {
+        int r;
+
+        assert(link);
+        assert(link->nexthop_messages > 0);
+        assert(IN_SET(link->state, LINK_STATE_CONFIGURING,
+                      LINK_STATE_FAILED, LINK_STATE_LINGER));
+
+        link->nexthop_messages--;
+
+        if (IN_SET(link->state, LINK_STATE_FAILED, LINK_STATE_LINGER))
+                return 1;
+
+        r = sd_netlink_message_get_errno(m);
+        if (r < 0 && r != -EEXIST) {
+                log_link_warning_errno(link, r, "Could not set nexthop: %m");
+                link_enter_failed(link);
+                return 1;
+        }
+
+        if (link->nexthop_messages == 0) {
+                log_link_debug(link, "Nexthop set");
+                link->static_nexthops_configured = true;
+                link_check_ready(link);
+        }
+
+        return 1;
+}
+
+int link_request_set_nexthop(Link *link) {
+        NextHop *nh;
+        int r;
+
+        LIST_FOREACH(nexthops, nh, link->network->static_nexthops) {
+                r = nexthop_configure(nh, link, nexthop_handler);
+                if (r < 0)
+                        return log_link_warning_errno(link, r, "Could not set nexthop: %m");
+                if (r > 0)
+                        link->nexthop_messages++;
+        }
+
+        if (link->nexthop_messages == 0) {
+                link->static_nexthops_configured = true;
+                link_check_ready(link);
+        } else {
+                log_link_debug(link, "Setting nexthop");
+                link_set_state(link, LINK_STATE_CONFIGURING);
+        }
+
+        return 1;
+}
+
 static int route_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *link) {
         int r;
 
@@ -948,6 +1003,7 @@ int link_request_set_routes(Link *link) {
         assert(link->state != _LINK_STATE_INVALID);
 
         link->static_routes_configured = false;
+        link->static_routes_ready = false;
 
         if (!link_has_carrier(link) && !link->network->configure_without_carrier)
                 /* During configuring addresses, the link lost its carrier. As networkd is dropping
@@ -1017,6 +1073,17 @@ void link_check_ready(Link *link) {
         if (!link->static_routes_configured)
                 return;
 
+        if (!link->static_routes_ready) {
+                link->static_routes_ready = true;
+                r = link_request_set_nexthop(link);
+                if (r < 0)
+                        link_enter_failed(link);
+                return;
+        }
+
+        if (!link->static_nexthops_configured)
+                return;
+
         if (!link->routing_policy_rules_configured)
                 return;
 
@@ -1134,6 +1201,8 @@ static int link_request_set_addresses(Link *link) {
         link->addresses_ready = false;
         link->neighbors_configured = false;
         link->static_routes_configured = false;
+        link->static_routes_ready = false;
+        link->static_nexthops_configured = false;
         link->routing_policy_rules_configured = false;
 
         r = link_set_bridge_fdb(link);
diff --git a/src/network/networkd-link.h b/src/network/networkd-link.h
index 446c042fb9..d6604c9120 100644
--- a/src/network/networkd-link.h
+++ b/src/network/networkd-link.h
@@ -69,6 +69,7 @@ typedef struct Link {
         unsigned address_label_messages;
         unsigned neighbor_messages;
         unsigned route_messages;
+        unsigned nexthop_messages;
         unsigned routing_policy_rule_messages;
         unsigned routing_policy_rule_remove_messages;
         unsigned enslaving;
@@ -79,9 +80,8 @@ typedef struct Link {
         Set *neighbors_foreign;
         Set *routes;
         Set *routes_foreign;
-
-        bool addresses_configured;
-        bool addresses_ready;
+        Set *nexthops;
+        Set *nexthops_foreign;
 
         sd_dhcp_client *dhcp_client;
         sd_dhcp_lease *dhcp_lease, *dhcp_lease_old;
@@ -100,8 +100,12 @@ typedef struct Link {
         sd_ipv4ll *ipv4ll;
         bool ipv4ll_address:1;
 
+        bool addresses_configured:1;
+        bool addresses_ready:1;
         bool neighbors_configured:1;
         bool static_routes_configured:1;
+        bool static_routes_ready:1;
+        bool static_nexthops_configured:1;
         bool routing_policy_rules_configured:1;
         bool setting_mtu:1;
 
@@ -198,6 +202,7 @@ uint32_t link_get_vrf_table(Link *link);
 uint32_t link_get_dhcp_route_table(Link *link);
 uint32_t link_get_ipv6_accept_ra_route_table(Link *link);
 int link_request_set_routes(Link *link);
+int link_request_set_nexthop(Link *link);
 
 #define ADDRESS_FMT_VAL(address)                   \
         be32toh((address).s_addr) >> 24,           \
diff --git a/src/network/networkd-manager.c b/src/network/networkd-manager.c
index 546bb2375c..c70194f1f8 100644
--- a/src/network/networkd-manager.c
+++ b/src/network/networkd-manager.c
@@ -5,6 +5,7 @@
 #include <unistd.h>
 #include <linux/if.h>
 #include <linux/fib_rules.h>
+#include <linux/nexthop.h>
 
 #include "sd-daemon.h"
 #include "sd-netlink.h"
@@ -1153,6 +1154,118 @@ int manager_rtnl_process_rule(sd_netlink *rtnl, sd_netlink_message *message, voi
         return 1;
 }
 
+int manager_rtnl_process_nexthop(sd_netlink *rtnl, sd_netlink_message *message, void *userdata) {
+        _cleanup_(nexthop_freep) NextHop *tmp = NULL;
+        _cleanup_free_ char *gateway = NULL;
+        NextHop *nexthop = NULL;
+        Manager *m = userdata;
+        Link *link = NULL;
+        uint16_t type;
+        int r;
+
+        assert(rtnl);
+        assert(message);
+        assert(m);
+
+        if (sd_netlink_message_is_error(message)) {
+                r = sd_netlink_message_get_errno(message);
+                if (r < 0)
+                        log_warning_errno(r, "rtnl: failed to receive rule message, ignoring: %m");
+
+                return 0;
+        }
+
+        r = sd_netlink_message_get_type(message, &type);
+        if (r < 0) {
+                log_warning_errno(r, "rtnl: could not get message type, ignoring: %m");
+                return 0;
+        } else if (!IN_SET(type, RTM_NEWNEXTHOP, RTM_DELNEXTHOP)) {
+                log_warning("rtnl: received unexpected message type %u when processing nexthop, ignoring.", type);
+                return 0;
+        }
+
+        r = nexthop_new(&tmp);
+        if (r < 0)
+                return log_oom();
+
+        r = sd_rtnl_message_get_family(message, &tmp->family);
+        if (r < 0) {
+                log_warning_errno(r, "rtnl: could not get nexthop family, ignoring: %m");
+                return 0;
+        } else if (!IN_SET(tmp->family, AF_INET, AF_INET6)) {
+                log_debug("rtnl: received nexthop message with invalid family %d, ignoring.", tmp->family);
+                return 0;
+        }
+
+        switch (tmp->family) {
+        case AF_INET:
+                r = sd_netlink_message_read_in_addr(message, NHA_GATEWAY, &tmp->gw.in);
+                if (r < 0 && r != -ENODATA) {
+                        log_warning_errno(r, "rtnl: could not get NHA_GATEWAY attribute, ignoring: %m");
+                        return 0;
+                }
+                break;
+
+        case AF_INET6:
+                r = sd_netlink_message_read_in6_addr(message, NHA_GATEWAY, &tmp->gw.in6);
+                if (r < 0 && r != -ENODATA) {
+                        log_warning_errno(r, "rtnl: could not get NHA_GATEWAY attribute, ignoring: %m");
+                        return 0;
+                }
+                break;
+
+        default:
+                assert_not_reached("Received rule message with unsupported address family");
+        }
+
+        r = sd_netlink_message_read_u32(message, NHA_ID, &tmp->id);
+        if (r < 0 && r != -ENODATA) {
+                log_warning_errno(r, "rtnl: could not get NHA_ID attribute, ignoring: %m");
+                return 0;
+        }
+
+        r = sd_netlink_message_read_u32(message, NHA_OIF, &tmp->oif);
+        if (r < 0 && r != -ENODATA) {
+                log_warning_errno(r, "rtnl: could not get NHA_OIF attribute, ignoring: %m");
+                return 0;
+        }
+
+        r = link_get(m, tmp->oif, &link);
+        if (r < 0 || !link) {
+                if (!m->enumerating)
+                        log_warning("rtnl: received nexthop message for link (%d) we do not know about, ignoring", tmp->oif);
+                return 0;
+        }
+
+        (void) nexthop_get(link, tmp, &nexthop);
+
+        if (DEBUG_LOGGING)
+                (void) in_addr_to_string(tmp->family, &tmp->gw, &gateway);
+
+        switch (type) {
+        case RTM_NEWNEXTHOP:
+                if (!nexthop) {
+                        log_debug("Remembering foreign nexthop: %s, oif: %d, id: %d", gateway, tmp->oif, tmp->id);
+                        r = nexthop_add_foreign(link, tmp, &nexthop);
+                        if (r < 0) {
+                                log_warning_errno(r, "Could not remember foreign nexthop, ignoring: %m");
+                                return 0;
+                        }
+                }
+                break;
+        case RTM_DELNEXTHOP:
+                log_debug("Forgetting foreign nexthop: %s, oif: %d, id: %d", gateway, tmp->oif, tmp->id);
+                nexthop_free(nexthop);
+
+                break;
+
+        default:
+                assert_not_reached("Received invalid RTNL message type");
+        }
+
+        return 1;
+}
+
 static int systemd_netlink_fd(void) {
         int n, fd, rtnl_fd = -EINVAL;
 
@@ -1253,6 +1366,14 @@ static int manager_connect_rtnl(Manager *m) {
         if (r < 0)
                 return r;
 
+        r = sd_netlink_add_match(m->rtnl, NULL, RTM_NEWNEXTHOP, &manager_rtnl_process_nexthop, NULL, m, "network-rtnl_process_nexthop");
+        if (r < 0)
+                return r;
+
+        r = sd_netlink_add_match(m->rtnl, NULL, RTM_DELNEXTHOP, &manager_rtnl_process_nexthop, NULL, m, "network-rtnl_process_nexthop");
+        if (r < 0)
+                return r;
+
         return 0;
 }
 
@@ -1931,6 +2052,47 @@ int manager_rtnl_enumerate_rules(Manager *m) {
         return r;
 }
 
+int manager_rtnl_enumerate_nexthop(Manager *m) {
+        _cleanup_(sd_netlink_message_unrefp) sd_netlink_message *req = NULL, *reply = NULL;
+        sd_netlink_message *nexthop;
+        int r;
+
+        assert(m);
+        assert(m->rtnl);
+
+        r = sd_rtnl_message_new_nexthop(m->rtnl, &req, RTM_GETNEXTHOP, 0, 0);
+        if (r < 0)
+                return r;
+
+        r = sd_netlink_message_request_dump(req, true);
+        if (r < 0)
+                return r;
+
+        r = sd_netlink_call(m->rtnl, req, 0, &reply);
+        if (r < 0) {
+                if (r == -EOPNOTSUPP) {
+                        log_debug("Nexthop are not supported by the kernel. Ignoring.");
+                        return 0;
+                }
+
+                return r;
+        }
+
+        for (nexthop = reply; nexthop; nexthop = sd_netlink_message_next(nexthop)) {
+                int k;
+
+                m->enumerating = true;
+
+                k = manager_rtnl_process_nexthop(m->rtnl, nexthop, m);
+                if (k < 0)
+                        r = k;
+
+                m->enumerating = false;
+        }
+
+        return r;
+}
+
 int manager_address_pool_acquire(Manager *m, int family, unsigned prefixlen, union in_addr_union *found) {
         AddressPool *p;
         int r;
diff --git a/src/network/networkd-manager.h b/src/network/networkd-manager.h
index d5049df868..f2f309ffb0 100644
--- a/src/network/networkd-manager.h
+++ b/src/network/networkd-manager.h
@@ -83,11 +83,13 @@ int manager_rtnl_enumerate_addresses(Manager *m);
 int manager_rtnl_enumerate_neighbors(Manager *m);
 int manager_rtnl_enumerate_routes(Manager *m);
 int manager_rtnl_enumerate_rules(Manager *m);
+int manager_rtnl_enumerate_nexthop(Manager *m);
 
 int manager_rtnl_process_address(sd_netlink *nl, sd_netlink_message *message, void *userdata);
 int manager_rtnl_process_neighbor(sd_netlink *nl, sd_netlink_message *message, void *userdata);
 int manager_rtnl_process_route(sd_netlink *nl, sd_netlink_message *message, void *userdata);
 int manager_rtnl_process_rule(sd_netlink *nl, sd_netlink_message *message, void *userdata);
+int manager_rtnl_process_nexthop(sd_netlink *nl, sd_netlink_message *message, void *userdata);
 
 void manager_dirty(Manager *m);
 
diff --git a/src/network/networkd-network-gperf.gperf b/src/network/networkd-network-gperf.gperf
index 689b1a123e..490a0a38a3 100644
--- a/src/network/networkd-network-gperf.gperf
+++ b/src/network/networkd-network-gperf.gperf
@@ -141,6 +141,8 @@ Route.InitialAdvertisedReceiveWindow,   config_parse_tcp_window,
 Route.QuickAck,                         config_parse_quickack,                           0,                             0
 Route.FastOpenNoCookie,                 config_parse_fast_open_no_cookie,                0,                             0
 Route.TTLPropagate,                     config_parse_route_ttl_propagate,                0,                             0
+NextHop.Id,                             config_parse_nexthop_id,                         0,                             0
+NextHop.Gateway,                        config_parse_nexthop_gateway,                    0,                             0
 DHCPv4.ClientIdentifier,                config_parse_dhcp_client_identifier,             0,                             offsetof(Network, dhcp_client_identifier)
 DHCPv4.UseDNS,                          config_parse_bool,                               0,                             offsetof(Network, dhcp_use_dns)
 DHCPv4.RoutesToDNS,                     config_parse_bool,                               0,                             offsetof(Network, dhcp_routes_to_dns)
diff --git a/src/network/networkd-network.c b/src/network/networkd-network.c
index 3ea76a034a..d949cc0f93 100644
--- a/src/network/networkd-network.c
+++ b/src/network/networkd-network.c
@@ -151,6 +151,7 @@ int network_verify(Network *network) {
         AddressLabel *label, *label_next;
         Prefix *prefix, *prefix_next;
         RoutingPolicyRule *rule, *rule_next;
+        NextHop *nexthop, *nextnop_next;
 
         assert(network);
         assert(network->filename);
@@ -282,6 +283,10 @@ int network_verify(Network *network) {
                 if (route_section_verify(route, network) < 0)
                         route_free(route);
 
+        LIST_FOREACH_SAFE(nexthops, nexthop, nextnop_next, network->static_nexthops)
+                if (nexthop_section_verify(nexthop) < 0)
+                        nexthop_free(nexthop);
+
         LIST_FOREACH_SAFE(static_fdb_entries, fdb, fdb_next, network->static_fdb_entries)
                 if (section_is_invalid(fdb->section))
                         fdb_entry_free(fdb);
@@ -453,6 +458,7 @@ int network_load_one(Manager *manager, const char *filename) {
                               "IPv6AddressLabel\0"
                               "RoutingPolicyRule\0"
                               "Route\0"
+                              "NextHop\0"
                               "DHCP\0"
                               "DHCPv4\0" /* compat */
                               "DHCPv6\0"
@@ -525,8 +531,9 @@ static Network *network_free(Network *network) {
         FdbEntry *fdb_entry;
         Neighbor *neighbor;
         AddressLabel *label;
-        Prefix *prefix;
         Address *address;
+        NextHop *nexthop;
+        Prefix *prefix;
         Route *route;
 
         if (!network)
@@ -573,6 +580,9 @@ static Network *network_free(Network *network) {
         while ((route = network->static_routes))
                 route_free(route);
 
+        while ((nexthop = network->static_nexthops))
+                nexthop_free(nexthop);
+
         while ((address = network->static_addresses))
                 address_free(address);
 
@@ -596,6 +606,7 @@ static Network *network_free(Network *network) {
 
         hashmap_free(network->addresses_by_section);
         hashmap_free(network->routes_by_section);
+        hashmap_free(network->nexthops_by_section);
         hashmap_free(network->fdb_entries_by_section);
         hashmap_free(network->neighbors_by_section);
         hashmap_free(network->address_labels_by_section);
diff --git a/src/network/networkd-network.h b/src/network/networkd-network.h
index 35469c05ed..668cc0d348 100644
--- a/src/network/networkd-network.h
+++ b/src/network/networkd-network.h
@@ -19,6 +19,7 @@
 #include "networkd-lldp-rx.h"
 #include "networkd-lldp-tx.h"
 #include "networkd-neighbor.h"
+#include "networkd-nexthop.h"
 #include "networkd-radv.h"
 #include "networkd-route.h"
 #include "networkd-routing-policy-rule.h"
@@ -228,6 +229,7 @@ struct Network {
 
         LIST_HEAD(Address, static_addresses);
         LIST_HEAD(Route, static_routes);
+        LIST_HEAD(NextHop, static_nexthops);
         LIST_HEAD(FdbEntry, static_fdb_entries);
         LIST_HEAD(IPv6ProxyNDPAddress, ipv6_proxy_ndp_addresses);
         LIST_HEAD(Neighbor, neighbors);
@@ -238,6 +240,7 @@ struct Network {
 
         unsigned n_static_addresses;
         unsigned n_static_routes;
+        unsigned n_static_nexthops;
         unsigned n_static_fdb_entries;
         unsigned n_ipv6_proxy_ndp_addresses;
         unsigned n_neighbors;
@@ -248,6 +251,7 @@ struct Network {
 
         Hashmap *addresses_by_section;
         Hashmap *routes_by_section;
+        Hashmap *nexthops_by_section;
         Hashmap *fdb_entries_by_section;
         Hashmap *neighbors_by_section;
         Hashmap *address_labels_by_section;
diff --git a/src/network/networkd-nexthop.c b/src/network/networkd-nexthop.c
new file mode 100644
index 0000000000..9658fe30c0
--- /dev/null
+++ b/src/network/networkd-nexthop.c
@@ -0,0 +1,473 @@
+/* SPDX-License-Identifier: LGPL-2.1+
+ * Copyright © 2019 VMware, Inc.
+ */
+
+#include <linux/nexthop.h>
+
+#include "alloc-util.h"
+#include "conf-parser.h"
+#include "in-addr-util.h"
+#include "netlink-util.h"
+#include "networkd-manager.h"
+#include "networkd-nexthop.h"
+#include "parse-util.h"
+#include "set.h"
+#include "string-util.h"
+#include "util.h"
+
+int nexthop_new(NextHop **ret) {
+        _cleanup_(nexthop_freep) NextHop *nexthop = NULL;
+
+        nexthop = new(NextHop, 1);
+        if (!nexthop)
+                return -ENOMEM;
+
+        *nexthop = (NextHop) {
+                .family = AF_UNSPEC,
+        };
+
+        *ret = TAKE_PTR(nexthop);
+
+        return 0;
+}
+
+static int nexthop_new_static(Network *network, const char *filename, unsigned section_line, NextHop **ret) {
+        _cleanup_(network_config_section_freep) NetworkConfigSection *n = NULL;
+        _cleanup_(nexthop_freep) NextHop *nexthop = NULL;
+        int r;
+
+        assert(network);
+        assert(ret);
+        assert(!!filename == (section_line > 0));
+
+        if (filename) {
+                r = network_config_section_new(filename, section_line, &n);
+                if (r < 0)
+                        return r;
+
+                nexthop = hashmap_get(network->nexthops_by_section, n);
+                if (nexthop) {
+                        *ret = TAKE_PTR(nexthop);
+
+                        return 0;
+                }
+        }
+
+        r = nexthop_new(&nexthop);
+        if (r < 0)
+                return r;
+
+        nexthop->protocol = RTPROT_STATIC;
+        nexthop->network = network;
+        LIST_PREPEND(nexthops, network->static_nexthops, nexthop);
+        network->n_static_nexthops++;
+
+        if (filename) {
+                nexthop->section = TAKE_PTR(n);
+
+                r = hashmap_ensure_allocated(&network->nexthops_by_section, &network_config_hash_ops);
+                if (r < 0)
+                        return r;
+
+                r = hashmap_put(network->nexthops_by_section, nexthop->section, nexthop);
+                if (r < 0)
+                        return r;
+        }
+
+        *ret = TAKE_PTR(nexthop);
+
+        return 0;
+}
+
+void nexthop_free(NextHop *nexthop) {
+        if (!nexthop)
+                return;
+
+        if (nexthop->network) {
+                LIST_REMOVE(nexthops, nexthop->network->static_nexthops, nexthop);
+
+                assert(nexthop->network->n_static_nexthops > 0);
+                nexthop->network->n_static_nexthops--;
+
+                if (nexthop->section)
+                        hashmap_remove(nexthop->network->nexthops_by_section, nexthop->section);
+        }
+
+        network_config_section_free(nexthop->section);
+
+        if (nexthop->link) {
+                set_remove(nexthop->link->nexthops, nexthop);
+                set_remove(nexthop->link->nexthops_foreign, nexthop);
+        }
+
+        free(nexthop);
+}
+
+static void nexthop_hash_func(const NextHop *nexthop, struct siphash *state) {
+        assert(nexthop);
+
+        siphash24_compress(&nexthop->id, sizeof(nexthop->id), state);
+        siphash24_compress(&nexthop->oif, sizeof(nexthop->oif), state);
+        siphash24_compress(&nexthop->family, sizeof(nexthop->family), state);
+
+        switch (nexthop->family) {
+        case AF_INET:
+        case AF_INET6:
+                siphash24_compress(&nexthop->gw, FAMILY_ADDRESS_SIZE(nexthop->family), state);
+
+                break;
+        default:
+                /* treat any other address family as AF_UNSPEC */
+                break;
+        }
+}
+
+static int nexthop_compare_func(const NextHop *a, const NextHop *b) {
+        int r;
+
+        r = CMP(a->id, b->id);
+        if (r != 0)
+                return r;
+
+        r = CMP(a->oif, b->oif);
+        if (r != 0)
+                return r;
+
+        r = CMP(a->family, b->family);
+        if (r != 0)
+                return r;
+
+        switch (a->family) {
+        case AF_INET:
+        case AF_INET6:
+
+                r = memcmp(&a->gw, &b->gw, FAMILY_ADDRESS_SIZE(a->family));
+                if (r != 0)
+                        return r;
+
+                return 0;
+        default:
+                /* treat any other address family as AF_UNSPEC */
+                return 0;
+        }
+}
+
+DEFINE_HASH_OPS_WITH_KEY_DESTRUCTOR(
+                nexthop_hash_ops,
+                NextHop,
+                nexthop_hash_func,
+                nexthop_compare_func,
+                nexthop_free);
+
+bool nexthop_equal(NextHop *r1, NextHop *r2) {
+        if (r1 == r2)
+                return true;
+
+        if (!r1 || !r2)
+                return false;
+
+        return nexthop_compare_func(r1, r2) == 0;
+}
+
+int nexthop_get(Link *link, NextHop *in, NextHop **ret) {
+        NextHop *existing;
+
+        assert(link);
+        assert(in);
+
+        existing = set_get(link->nexthops, in);
+        if (existing) {
+                if (ret)
+                        *ret = existing;
+                return 1;
+        }
+
+        existing = set_get(link->nexthops_foreign, in);
+        if (existing) {
+                if (ret)
+                        *ret = existing;
+                return 0;
+        }
+
+        return -ENOENT;
+}
+
+static int nexthop_add_internal(Link *link, Set **nexthops, NextHop *in, NextHop **ret) {
+        _cleanup_(nexthop_freep) NextHop *nexthop = NULL;
+        int r;
+
+        assert(link);
+        assert(nexthops);
+        assert(in);
+
+        r = nexthop_new(&nexthop);
+        if (r < 0)
+                return r;
+
+        nexthop->id = in->id;
+        nexthop->oif = in->oif;
+        nexthop->family = in->family;
+        nexthop->gw = in->gw;
+
+        r = set_ensure_allocated(nexthops, &nexthop_hash_ops);
+        if (r < 0)
+                return r;
+
+        r = set_put(*nexthops, nexthop);
+        if (r < 0)
+                return r;
+        if (r == 0)
+                return -EEXIST;
+
+        nexthop->link = link;
+
+        if (ret)
+                *ret = nexthop;
+
+        nexthop = NULL;
+
+        return 0;
+}
+
+int nexthop_add_foreign(Link *link, NextHop *in, NextHop **ret) {
+        return nexthop_add_internal(link, &link->nexthops_foreign, in, ret);
+}
+
+int nexthop_add(Link *link, NextHop *in, NextHop **ret) {
+        NextHop *nexthop;
+        int r;
+
+        r = nexthop_get(link, in, &nexthop);
+        if (r == -ENOENT) {
+                /* NextHop does not exist, create a new one */
+                r = nexthop_add_internal(link, &link->nexthops, in, &nexthop);
+                if (r < 0)
+                        return r;
+        } else if (r == 0) {
+                /* Take over a foreign nexthop */
+                r = set_ensure_allocated(&link->nexthops, &nexthop_hash_ops);
+                if (r < 0)
+                        return r;
+
+                r = set_put(link->nexthops, nexthop);
+                if (r < 0)
+                        return r;
+
+                set_remove(link->nexthops_foreign, nexthop);
+        } else if (r == 1) {
+                /* NextHop exists, do nothing */
+                ;
+        } else
+                return r;
+
+        if (ret)
+                *ret = nexthop;
+
+        return 0;
+}
+
+static int nexthop_remove_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *link) {
+        int r;
+
+        assert(m);
+        assert(link);
+        assert(link->ifname);
+
+        if (IN_SET(link->state, LINK_STATE_FAILED, LINK_STATE_LINGER))
+                return 1;
+
+        r = sd_netlink_message_get_errno(m);
+        if (r < 0 && r != -ESRCH)
+                log_link_warning_errno(link, r, "Could not drop nexthop: %m");
+
+        return 1;
+}
+
+int nexthop_remove(NextHop *nexthop, Link *link,
+                   link_netlink_message_handler_t callback) {
+
+        _cleanup_(sd_netlink_message_unrefp) sd_netlink_message *req = NULL;
+        int r;
+
+        assert(link);
+        assert(link->manager);
+        assert(link->manager->rtnl);
+        assert(link->ifindex > 0);
+        assert(IN_SET(nexthop->family, AF_INET, AF_INET6));
+
+        r = sd_rtnl_message_new_nexthop(link->manager->rtnl, &req,
+                                      RTM_DELNEXTHOP, nexthop->family,
+                                      nexthop->protocol);
+        if (r < 0)
+                return log_link_error_errno(link, r, "Could not create RTM_DELNEXTHOP message: %m");
+
+        if (DEBUG_LOGGING) {
+                _cleanup_free_ char *gw = NULL;
+
+                if (!in_addr_is_null(nexthop->family, &nexthop->gw))
+                        (void) in_addr_to_string(nexthop->family, &nexthop->gw, &gw);
+
+                log_link_debug(link, "Removing nexthop: gw: %s", strna(gw));
+        }
+
+        if (in_addr_is_null(nexthop->family, &nexthop->gw) == 0) {
+                r = netlink_message_append_in_addr_union(req, RTA_GATEWAY, nexthop->family, &nexthop->gw);
+                if (r < 0)
+                        return log_link_error_errno(link, r, "Could not append RTA_GATEWAY attribute: %m");
+        }
+
+        r = netlink_call_async(link->manager->rtnl, NULL, req,
+                               callback ?: nexthop_remove_handler,
+                               link_netlink_destroy_callback, link);
+        if (r < 0)
+                return log_link_error_errno(link, r, "Could not send rtnetlink message: %m");
+
+        link_ref(link);
+
+        return 0;
+}
+
+int nexthop_configure(
+                NextHop *nexthop,
+                Link *link,
+                link_netlink_message_handler_t callback) {
+        _cleanup_(sd_netlink_message_unrefp) sd_netlink_message *req = NULL;
+        int r;
+
+        assert(link);
+        assert(link->manager);
+        assert(link->manager->rtnl);
+        assert(link->ifindex > 0);
+        assert(IN_SET(nexthop->family, AF_INET, AF_INET6));
+        assert(callback);
+
+        if (DEBUG_LOGGING) {
+                _cleanup_free_ char *gw = NULL;
+
+                if (!in_addr_is_null(nexthop->family, &nexthop->gw))
+                        (void) in_addr_to_string(nexthop->family, &nexthop->gw, &gw);
+
+                log_link_debug(link, "Configuring nexthop: gw: %s", strna(gw));
+        }
+
+        r = sd_rtnl_message_new_nexthop(link->manager->rtnl, &req,
+                                        RTM_NEWNEXTHOP, nexthop->family,
+                                        nexthop->protocol);
+        if (r < 0)
+                return log_link_error_errno(link, r, "Could not create RTM_NEWNEXTHOP message: %m");
+
+        r = sd_netlink_message_append_u32(req, NHA_ID, nexthop->id);
+        if (r < 0)
+                return log_link_error_errno(link, r, "Could not append NHA_ID attribute: %m");
+
+        r = sd_netlink_message_append_u32(req, NHA_OIF, link->ifindex);
+        if (r < 0)
+                return log_link_error_errno(link, r, "Could not append NHA_OIF attribute: %m");
+
+        if (in_addr_is_null(nexthop->family, &nexthop->gw) == 0) {
+                r = netlink_message_append_in_addr_union(req, NHA_GATEWAY, nexthop->family, &nexthop->gw);
+                if (r < 0)
+                        return log_link_error_errno(link, r, "Could not append NHA_GATEWAY attribute: %m");
+
+                r = sd_rtnl_message_nexthop_set_family(req, nexthop->family);
+                if (r < 0)
+                        return log_link_error_errno(link, r, "Could not set nexthop family: %m");
+        }
+
+        r = netlink_call_async(link->manager->rtnl, NULL, req, callback,
+                               link_netlink_destroy_callback, link);
+        if (r < 0)
+                return log_link_error_errno(link, r, "Could not send rtnetlink message: %m");
+
+        link_ref(link);
+
+        r = nexthop_add(link, nexthop, &nexthop);
+        if (r < 0)
+                return log_link_error_errno(link, r, "Could not add nexthop: %m");
+
+        return 1;
+}
+
+int nexthop_section_verify(NextHop *nh) {
+        if (section_is_invalid(nh->section))
+                return -EINVAL;
+
+        if (in_addr_is_null(nh->family, &nh->gw) < 0)
+                return -EINVAL;
+
+        return 0;
+}
+
+int config_parse_nexthop_id(
+                const char *unit,
+                const char *filename,
+                unsigned line,
+                const char *section,
+                unsigned section_line,
+                const char *lvalue,
+                int ltype,
+                const char *rvalue,
+                void *data,
+                void *userdata) {
+
+        _cleanup_(nexthop_free_or_set_invalidp) NextHop *n = NULL;
+        Network *network = userdata;
+        int r;
+
+        assert(filename);
+        assert(section);
+        assert(lvalue);
+        assert(rvalue);
+        assert(data);
+
+        r = nexthop_new_static(network, filename, section_line, &n);
+        if (r < 0)
+                return r;
+
+        r = safe_atou32(rvalue, &n->id);
+        if (r < 0) {
+                log_syntax(unit, LOG_ERR, filename, line, r,
+                           "Could not parse nexthop id \"%s\", ignoring assignment: %m", rvalue);
+                return 0;
+        }
+
+        TAKE_PTR(n);
+        return 0;
+}
+
+int config_parse_nexthop_gateway(
+                const char *unit,
+                const char *filename,
+                unsigned line,
+                const char *section,
+                unsigned section_line,
+                const char *lvalue,
+                int ltype,
+                const char *rvalue,
+                void *data,
+                void *userdata) {
+
+        _cleanup_(nexthop_free_or_set_invalidp) NextHop *n = NULL;
+        Network *network = userdata;
+        int r;
+
+        assert(filename);
+        assert(section);
+        assert(lvalue);
+        assert(rvalue);
+        assert(data);
+
+        r = nexthop_new_static(network, filename, section_line, &n);
+        if (r < 0)
+                return r;
+
+        r = in_addr_from_string_auto(rvalue, &n->family, &n->gw);
+        if (r < 0) {
+                log_syntax(unit, LOG_ERR, filename, line, r,
+                           "Invalid %s='%s', ignoring assignment: %m", lvalue, rvalue);
+                return 0;
+        }
+
+        TAKE_PTR(n);
+        return 0;
+}
diff --git a/src/network/networkd-nexthop.h b/src/network/networkd-nexthop.h
new file mode 100644
index 0000000000..28cbdad738
--- /dev/null
+++ b/src/network/networkd-nexthop.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: LGPL-2.1+
+ * Copyright © 2019 VMware, Inc.
+ */
+
+#pragma once
+
+#include "conf-parser.h"
+#include "macro.h"
+
+typedef struct NextHop NextHop;
+typedef struct NetworkConfigSection NetworkConfigSection;
+
+#include "networkd-network.h"
+#include "networkd-util.h"
+
+struct NextHop {
+        Network *network;
+        NetworkConfigSection *section;
+
+        Link *link;
+
+        unsigned char protocol;
+
+        int family;
+        uint32_t oif;
+        uint32_t id;
+
+        union in_addr_union gw;
+
+        LIST_FIELDS(NextHop, nexthops);
+};
+
+extern const struct hash_ops nexthop_hash_ops;
+
+int nexthop_new(NextHop **ret);
+void nexthop_free(NextHop *nexthop);
+int nexthop_configure(NextHop *nexthop, Link *link, link_netlink_message_handler_t callback);
+int nexthop_remove(NextHop *nexthop, Link *link, link_netlink_message_handler_t callback);
+
+int nexthop_get(Link *link, NextHop *in, NextHop **ret);
+int nexthop_add(Link *link, NextHop *in, NextHop **ret);
+int nexthop_add_foreign(Link *link, NextHop *in, NextHop **ret);
+bool nexthop_equal(NextHop *r1, NextHop *r2);
+
+int nexthop_section_verify(NextHop *nexthop);
+
+DEFINE_NETWORK_SECTION_FUNCTIONS(NextHop, nexthop_free);
+
+CONFIG_PARSER_PROTOTYPE(config_parse_nexthop_id);
+CONFIG_PARSER_PROTOTYPE(config_parse_nexthop_gateway);
diff --git a/src/network/networkd.c b/src/network/networkd.c
index 38bd9ff1ff..c7ce64b90b 100644
--- a/src/network/networkd.c
+++ b/src/network/networkd.c
@@ -107,6 +107,10 @@ static int run(int argc, char *argv[]) {
         if (r < 0)
                 return log_error_errno(r, "Could not enumerate rules: %m");
 
+        r = manager_rtnl_enumerate_nexthop(m);
+        if (r < 0)
+                return log_error_errno(r, "Could not enumerate nexthop: %m");
+
         r = manager_start(m);
         if (r < 0)
                 return log_error_errno(r, "Could not start manager: %m");