zebra: convert PW updates to async dataplane

Add accessors for pw attributes; init pw attributes;
replace 'hook' calls for pw install/uninstall with dplane
apis.

Signed-off-by: Mark Stapp <mjs@voltanet.io>

8 files changed, 194 insertions, 32 deletions

diff --git a/zebra/rt.h b/zebra/rt.h
index 0b14a3ef3..80b0b502b 100644
--- a/zebra/rt.h
+++ b/zebra/rt.h
@@ -32,7 +32,7 @@
 #include "zebra/zebra_dplane.h"
 
 /*
- * Update or delete a route or LSP from the kernel,
+ * Update or delete a route, LSP, or pseudowire from the kernel,
  * using info from a dataplane context.
  */
 extern enum zebra_dplane_result kernel_route_update(
@@ -41,6 +41,8 @@ extern enum zebra_dplane_result kernel_route_update(
 extern enum zebra_dplane_result kernel_lsp_update(
 	struct zebra_dplane_ctx *ctx);
 
+enum zebra_dplane_result kernel_pw_update(struct zebra_dplane_ctx *ctx);
+
 extern int kernel_address_add_ipv4(struct interface *, struct connected *);
 extern int kernel_address_delete_ipv4(struct interface *, struct connected *);
 extern int kernel_address_add_ipv6(struct interface *, struct connected *);
diff --git a/zebra/zebra_dplane.c b/zebra/zebra_dplane.c
index c29a49d1e..4020cbeb4 100644
--- a/zebra/zebra_dplane.c
+++ b/zebra/zebra_dplane.c
@@ -255,10 +255,11 @@ static struct zebra_dplane_globals {
 	_Atomic uint32_t dg_other_errors;
 
 	_Atomic uint32_t dg_lsps_in;
-	_Atomic uint32_t dg_lsps_queued;
-	_Atomic uint32_t dg_lsps_queued_max;
 	_Atomic uint32_t dg_lsp_errors;
 
+	_Atomic uint32_t dg_pws_in;
+	_Atomic uint32_t dg_pw_errors;
+
 	_Atomic uint32_t dg_update_yields;
 
 	/* Dataplane pthread */
@@ -294,6 +295,8 @@ static void dplane_info_from_zns(struct zebra_dplane_info *ns_info,
 				 struct zebra_ns *zns);
 static enum zebra_dplane_result lsp_update_internal(zebra_lsp_t *lsp,
 						    enum dplane_op_e op);
+static enum zebra_dplane_result pw_update_internal(struct zebra_pw *pw,
+						   enum dplane_op_e op);
 
 /*
  * Public APIs
@@ -381,6 +384,8 @@ static void dplane_ctx_free(struct zebra_dplane_ctx **pctx)
 		break;
 	}
 
+	case DPLANE_OP_PW_INSTALL:
+	case DPLANE_OP_PW_UNINSTALL:
 	case DPLANE_OP_NONE:
 		break;
 	}
@@ -508,6 +513,13 @@ const char *dplane_op2str(enum dplane_op_e op)
 		ret = "LSP_DELETE";
 		break;
 
+	case DPLANE_OP_PW_INSTALL:
+		ret = "PW_INSTALL";
+		break;
+	case DPLANE_OP_PW_UNINSTALL:
+		ret = "PW_UNINSTALL";
+		break;
+
 	};
 
 	return ret;
@@ -1021,6 +1033,46 @@ static int dplane_ctx_lsp_init(struct zebra_dplane_ctx *ctx,
 }
 
 /*
+ * Capture information for an LSP update in a dplane context.
+ */
+static int dplane_ctx_pw_init(struct zebra_dplane_ctx *ctx,
+			      enum dplane_op_e op,
+			      struct zebra_pw *pw)
+{
+	int ret = AOK;
+
+	if (IS_ZEBRA_DEBUG_DPLANE_DETAIL)
+		zlog_debug("init dplane ctx %s: pw '%s', loc %u, rem %u",
+			   dplane_op2str(op), pw->ifname, pw->local_label,
+			   pw->remote_label);
+
+	ctx->zd_op = op;
+	ctx->zd_status = ZEBRA_DPLANE_REQUEST_SUCCESS;
+
+	/* Capture namespace info: no netlink support as of 12/18,
+	 * but just in case...
+	 */
+	dplane_ctx_ns_init(ctx, zebra_ns_lookup(NS_DEFAULT), false);
+
+	memset(&ctx->u.pw, 0, sizeof(ctx->u.pw));
+
+	/* This name appears to be c-string, so we use string copy. */
+	strlcpy(ctx->u.pw.ifname, pw->ifname, sizeof(ctx->u.pw.ifname));
+	ctx->u.pw.ifindex = pw->ifindex;
+	ctx->u.pw.type = pw->type;
+	ctx->u.pw.af = pw->af;
+	ctx->u.pw.local_label = pw->local_label;
+	ctx->u.pw.remote_label = pw->remote_label;
+	ctx->u.pw.flags = pw->flags;
+
+	ctx->u.pw.nexthop = pw->nexthop;
+
+	ctx->u.pw.fields = pw->data;
+
+	return ret;
+}
+
+/*
  * Enqueue a new route update,
  * and ensure an event is active for the dataplane pthread.
  */
@@ -1224,6 +1276,22 @@ enum zebra_dplane_result dplane_lsp_delete(zebra_lsp_t *lsp)
 }
 
 /*
+ * Enqueue pseudowire install for the dataplane.
+ */
+enum zebra_dplane_result dplane_pw_install(struct zebra_pw *pw)
+{
+	return pw_update_internal(pw, DPLANE_OP_PW_INSTALL);
+}
+
+/*
+ * Enqueue pseudowire un-install for the dataplane.
+ */
+enum zebra_dplane_result dplane_pw_uninstall(struct zebra_pw *pw)
+{
+	return pw_update_internal(pw, DPLANE_OP_PW_UNINSTALL);
+}
+
+/*
  * Common internal LSP update utility
  */
 static enum zebra_dplane_result lsp_update_internal(zebra_lsp_t *lsp,
@@ -1264,6 +1332,45 @@ done:
 }
 
 /*
+ * Internal, common handler for pseudowire updates.
+ */
+static enum zebra_dplane_result pw_update_internal(struct zebra_pw *pw,
+						   enum dplane_op_e op)
+{
+	enum zebra_dplane_result result = ZEBRA_DPLANE_REQUEST_FAILURE;
+	int ret;
+	struct zebra_dplane_ctx *ctx = NULL;
+
+	ctx = dplane_ctx_alloc();
+	if (ctx == NULL) {
+		ret = ENOMEM;
+		goto done;
+	}
+
+	ret = dplane_ctx_pw_init(ctx, op, pw);
+	if (ret != AOK)
+		goto done;
+
+	ret = dplane_route_enqueue(ctx);
+
+done:
+	/* Update counter */
+	atomic_fetch_add_explicit(&zdplane_info.dg_pws_in, 1,
+				  memory_order_relaxed);
+
+	if (ret == AOK)
+		result = ZEBRA_DPLANE_REQUEST_QUEUED;
+	else {
+		atomic_fetch_add_explicit(&zdplane_info.dg_pw_errors, 1,
+					  memory_order_relaxed);
+		if (ctx)
+			dplane_ctx_free(&ctx);
+	}
+
+	return result;
+}
+
+/*
  * Handler for 'show dplane'
  */
 int dplane_show_helper(struct vty *vty, bool detailed)
diff --git a/zebra/zebra_dplane.h b/zebra/zebra_dplane.h
index 8efbd275a..81226961e 100644
--- a/zebra/zebra_dplane.h
+++ b/zebra/zebra_dplane.h
@@ -105,7 +105,11 @@ enum dplane_op_e {
 	/* LSP update */
 	DPLANE_OP_LSP_INSTALL,
 	DPLANE_OP_LSP_UPDATE,
-	DPLANE_OP_LSP_DELETE
+	DPLANE_OP_LSP_DELETE,
+
+	/* Pseudowire update */
+	DPLANE_OP_PW_INSTALL,
+	DPLANE_OP_PW_UNINSTALL,
 };
 
 /*
@@ -245,6 +249,12 @@ enum zebra_dplane_result dplane_lsp_add(zebra_lsp_t *lsp);
 enum zebra_dplane_result dplane_lsp_update(zebra_lsp_t *lsp);
 enum zebra_dplane_result dplane_lsp_delete(zebra_lsp_t *lsp);
 
+/*
+ * Enqueue pseudowire operations for the dataplane.
+ */
+enum zebra_dplane_result dplane_pw_install(struct zebra_pw *pw);
+enum zebra_dplane_result dplane_pw_uninstall(struct zebra_pw *pw);
+
 /* Retrieve the limit on the number of pending, unprocessed updates. */
 uint32_t dplane_get_in_queue_limit(void);
 
diff --git a/zebra/zebra_mpls_netlink.c b/zebra/zebra_mpls_netlink.c
index d8b5ef4ce..a9233530d 100644
--- a/zebra/zebra_mpls_netlink.c
+++ b/zebra/zebra_mpls_netlink.c
@@ -61,6 +61,16 @@ done:
 		ZEBRA_DPLANE_REQUEST_SUCCESS : ZEBRA_DPLANE_REQUEST_FAILURE);
 }
 
+/*
+ * Pseudowire update api - not supported by netlink as of 12/18,
+ * but note that the default has been to report 'success' for pw updates
+ * on unsupported platforms.
+ */
+enum zebra_dplane_result kernel_pw_update(struct zebra_dplane_ctx *ctx)
+{
+	return ZEBRA_DPLANE_REQUEST_SUCCESS;
+}
+
 int mpls_kernel_init(void)
 {
 	struct stat st;
diff --git a/zebra/zebra_mpls_null.c b/zebra/zebra_mpls_null.c
index 409432c4f..2cc3f3b69 100644
--- a/zebra/zebra_mpls_null.c
+++ b/zebra/zebra_mpls_null.c
@@ -29,6 +29,15 @@ int mpls_kernel_init(void)
 	return -1;
 };
 
+/*
+ * Pseudowire update api - note that the default has been
+ * to report 'success' for pw updates on unsupported platforms.
+ */
+enum zebra_dplane_result kernel_pw_update(struct zebra_dplane_ctx *ctx)
+{
+	return ZEBRA_DPLANE_REQUEST_SUCCESS;
+}
+
 enum zebra_dplane_result kernel_lsp_update(struct zebra_dplane_ctx *ctx)
 {
 	return ZEBRA_DPLANE_REQUEST_FAILURE;
diff --git a/zebra/zebra_mpls_openbsd.c b/zebra/zebra_mpls_openbsd.c
index da76c6ebf..72c8f7352 100644
--- a/zebra/zebra_mpls_openbsd.c
+++ b/zebra/zebra_mpls_openbsd.c
@@ -314,16 +314,17 @@ enum zebra_dplane_result kernel_lsp_update(struct zebra_dplane_ctx *ctx)
 		ZEBRA_DPLANE_REQUEST_SUCCESS : ZEBRA_DPLANE_REQUEST_FAILURE);
 }
 
-static int kmpw_install(struct zebra_pw *pw)
+static enum zebra_dplane_result kmpw_install(struct zebra_dplane_ctx *ctx)
 {
 	struct ifreq ifr;
 	struct ifmpwreq imr;
 	struct sockaddr_storage ss;
 	struct sockaddr_in *sa_in = (struct sockaddr_in *)&ss;
 	struct sockaddr_in6 *sa_in6 = (struct sockaddr_in6 *)&ss;
+	const union g_addr *gaddr;
 
 	memset(&imr, 0, sizeof(imr));
-	switch (pw->type) {
+	switch (dplane_ctx_get_pw_type(ctx)) {
 	case PW_TYPE_ETHERNET:
 		imr.imr_type = IMR_TYPE_ETHERNET;
 		break;
@@ -332,67 +333,91 @@ static int kmpw_install(struct zebra_pw *pw)
 		break;
 	default:
 		zlog_debug("%s: unhandled pseudowire type (%#X)", __func__,
-			   pw->type);
-		return -1;
+			   dplane_ctx_get_pw_type(ctx));
+		return ZEBRA_DPLANE_REQUEST_FAILURE;
 	}
 
-	if (pw->flags & F_PSEUDOWIRE_CWORD)
+	if (dplane_ctx_get_pw_flags(ctx) & F_PSEUDOWIRE_CWORD)
 		imr.imr_flags |= IMR_FLAG_CONTROLWORD;
 
 	/* pseudowire nexthop */
 	memset(&ss, 0, sizeof(ss));
-	switch (pw->af) {
+	gaddr = dplane_ctx_get_pw_nexthop(ctx);
+	switch (dplane_ctx_get_pw_af(ctx)) {
 	case AF_INET:
 		sa_in->sin_family = AF_INET;
 		sa_in->sin_len = sizeof(struct sockaddr_in);
-		sa_in->sin_addr = pw->nexthop.ipv4;
+		sa_in->sin_addr = gaddr->ipv4;
 		break;
 	case AF_INET6:
 		sa_in6->sin6_family = AF_INET6;
 		sa_in6->sin6_len = sizeof(struct sockaddr_in6);
-		sa_in6->sin6_addr = pw->nexthop.ipv6;
+		sa_in6->sin6_addr = gaddr->ipv6;
 		break;
 	default:
 		zlog_debug("%s: unhandled pseudowire address-family (%u)",
-			   __func__, pw->af);
-		return -1;
+			   __func__, dplane_ctx_get_pw_af(ctx));
+		return ZEBRA_DPLANE_REQUEST_FAILURE;
 	}
 	memcpy(&imr.imr_nexthop, (struct sockaddr *)&ss,
 	       sizeof(imr.imr_nexthop));
 
 	/* pseudowire local/remote labels */
-	imr.imr_lshim.shim_label = pw->local_label;
-	imr.imr_rshim.shim_label = pw->remote_label;
+	imr.imr_lshim.shim_label = dplane_ctx_get_pw_local_label(ctx);
+	imr.imr_rshim.shim_label = dplane_ctx_get_pw_remote_label(ctx);
 
 	/* ioctl */
 	memset(&ifr, 0, sizeof(ifr));
-	strlcpy(ifr.ifr_name, pw->ifname, sizeof(ifr.ifr_name));
+	strlcpy(ifr.ifr_name, dplane_ctx_get_pw_ifname(ctx),
+		sizeof(ifr.ifr_name));
 	ifr.ifr_data = (caddr_t)&imr;
 	if (ioctl(kr_state.ioctl_fd, SIOCSETMPWCFG, &ifr) == -1) {
 		flog_err_sys(EC_LIB_SYSTEM_CALL, "ioctl SIOCSETMPWCFG: %s",
 			     safe_strerror(errno));
-		return -1;
+		return ZEBRA_DPLANE_REQUEST_FAILURE;
 	}
 
-	return 0;
+	return ZEBRA_DPLANE_REQUEST_SUCCESS;
 }
 
-static int kmpw_uninstall(struct zebra_pw *pw)
+static enum zebra_dplane_result kmpw_uninstall(struct zebra_dplane_ctx *ctx)
 {
 	struct ifreq ifr;
 	struct ifmpwreq imr;
 
 	memset(&ifr, 0, sizeof(ifr));
 	memset(&imr, 0, sizeof(imr));
-	strlcpy(ifr.ifr_name, pw->ifname, sizeof(ifr.ifr_name));
+	strlcpy(ifr.ifr_name, dplane_ctx_get_pw_ifname(ctx),
+		sizeof(ifr.ifr_name));
 	ifr.ifr_data = (caddr_t)&imr;
 	if (ioctl(kr_state.ioctl_fd, SIOCSETMPWCFG, &ifr) == -1) {
 		flog_err_sys(EC_LIB_SYSTEM_CALL, "ioctl SIOCSETMPWCFG: %s",
 			     safe_strerror(errno));
-		return -1;
+		return ZEBRA_DPLANE_REQUEST_FAILURE;
 	}
 
-	return 0;
+	return ZEBRA_DPLANE_REQUEST_SUCCESS;
+}
+
+/*
+ * Pseudowire update api for openbsd.
+ */
+enum zebra_dplane_result kernel_pw_update(struct zebra_dplane_ctx *ctx)
+{
+	enum zebra_dplane_result result = ZEBRA_DPLANE_REQUEST_FAILURE;
+
+	switch (dplane_ctx_get_op(ctx)) {
+	case DPLANE_OP_PW_INSTALL:
+		result = kmpw_install(ctx);
+		break;
+	case DPLANE_OP_PW_UNINSTALL:
+		result = kmpw_uninstall(ctx);
+		break;
+	default:
+		break;
+	};
+
+	return result;
 }
 
 #define MAX_RTSOCK_BUF	128 * 1024
@@ -431,10 +456,6 @@ int mpls_kernel_init(void)
 
 	kr_state.rtseq = 1;
 
-	/* register hook to install/uninstall pseudowires */
-	hook_register(pw_install, kmpw_install);
-	hook_register(pw_uninstall, kmpw_uninstall);
-
 	return 0;
 }
 
diff --git a/zebra/zebra_pw.c b/zebra/zebra_pw.c
index fb9a40fe3..42a451124 100644
--- a/zebra/zebra_pw.c
+++ b/zebra/zebra_pw.c
@@ -98,9 +98,10 @@ void zebra_pw_del(struct zebra_vrf *zvrf, struct zebra_pw *pw)
 	zebra_deregister_rnh_pseudowire(pw->vrf_id, pw);
 
 	/* uninstall */
-	if (pw->status == PW_STATUS_UP)
+	if (pw->status == PW_STATUS_UP) {
 		hook_call(pw_uninstall, pw);
-	else if (pw->install_retry_timer)
+		dplane_pw_uninstall(pw);
+	} else if (pw->install_retry_timer)
 		THREAD_TIMER_OFF(pw->install_retry_timer);
 
 	/* unlink and release memory */
@@ -171,7 +172,8 @@ static void zebra_pw_install(struct zebra_pw *pw)
 			   pw->vrf_id, pw->ifname,
 			   zebra_route_string(pw->protocol));
 
-	if (hook_call(pw_install, pw)) {
+	hook_call(pw_install, pw);
+	if (dplane_pw_install(pw) == ZEBRA_DPLANE_REQUEST_FAILURE) {
 		zebra_pw_install_failure(pw);
 		return;
 	}
@@ -192,6 +194,7 @@ static void zebra_pw_uninstall(struct zebra_pw *pw)
 
 	/* ignore any possible error */
 	hook_call(pw_uninstall, pw);
+	dplane_pw_uninstall(pw);
 
 	if (zebra_pw_enabled(pw))
 		zebra_pw_update_status(pw, PW_STATUS_DOWN);
diff --git a/zebra/zebra_pw.h b/zebra/zebra_pw.h
index e6e0a22c2..9692fb4d4 100644
--- a/zebra/zebra_pw.h
+++ b/zebra/zebra_pw.h
@@ -62,8 +62,8 @@ RB_PROTOTYPE(zebra_static_pw_head, zebra_pw, static_pw_entry, zebra_pw_compare);
 DECLARE_HOOK(pw_install, (struct zebra_pw * pw), (pw))
 DECLARE_HOOK(pw_uninstall, (struct zebra_pw * pw), (pw))
 
-struct zebra_pw *zebra_pw_add(struct zebra_vrf *, const char *, uint8_t,
-			      struct zserv *);
+struct zebra_pw *zebra_pw_add(struct zebra_vrf *zvrf, const char *ifname,
+			      uint8_t protocol, struct zserv *client);
 void zebra_pw_del(struct zebra_vrf *, struct zebra_pw *);
 void zebra_pw_change(struct zebra_pw *, ifindex_t, int, int, union g_addr *,
 		     uint32_t, uint32_t, uint8_t, union pw_protocol_fields *);