From dab4a02630d5cb7628bed7f2a58354aca053eda7 Mon Sep 17 00:00:00 2001 From: Yangbo Lu Date: Mon, 25 Jun 2018 20:37:10 +0800 Subject: powerpc/mpc85xx: move ptp timer out of fman in dts This patch is to move ptp timer node out of fman. Because ptp timer will be probed by ptp_qoriq driver, it should be an independent device in case of conflict memory mapping. Signed-off-by: Yangbo Lu Acked-by: Richard Cochran Acked-by: Madalin Bucur Signed-off-by: David S. Miller --- arch/powerpc/boot/dts/fsl/qoriq-fman-0.dtsi | 14 ++++++++------ arch/powerpc/boot/dts/fsl/qoriq-fman-1.dtsi | 14 ++++++++------ arch/powerpc/boot/dts/fsl/qoriq-fman3-0.dtsi | 14 ++++++++------ arch/powerpc/boot/dts/fsl/qoriq-fman3-1.dtsi | 14 ++++++++------ arch/powerpc/boot/dts/fsl/qoriq-fman3l-0.dtsi | 14 ++++++++------ 5 files changed, 40 insertions(+), 30 deletions(-) (limited to 'arch') diff --git a/arch/powerpc/boot/dts/fsl/qoriq-fman-0.dtsi b/arch/powerpc/boot/dts/fsl/qoriq-fman-0.dtsi index abd01d466de4..6b124f73f67a 100644 --- a/arch/powerpc/boot/dts/fsl/qoriq-fman-0.dtsi +++ b/arch/powerpc/boot/dts/fsl/qoriq-fman-0.dtsi @@ -37,12 +37,13 @@ fman0: fman@400000 { #size-cells = <1>; cell-index = <0>; compatible = "fsl,fman"; - ranges = <0 0x400000 0x100000>; - reg = <0x400000 0x100000>; + ranges = <0 0x400000 0xfe000>; + reg = <0x400000 0xfe000>; interrupts = <96 2 0 0>, <16 2 1 1>; clocks = <&clockgen 3 0>; clock-names = "fmanclk"; fsl,qman-channel-range = <0x40 0xc>; + ptimer-handle = <&ptp_timer0>; muram@0 { compatible = "fsl,fman-muram"; @@ -93,9 +94,10 @@ fman0: fman@400000 { reg = <0x87000 0x1000>; status = "disabled"; }; +}; - ptp_timer0: ptp-timer@fe000 { - compatible = "fsl,fman-ptp-timer"; - reg = <0xfe000 0x1000>; - }; +ptp_timer0: ptp-timer@4fe000 { + compatible = "fsl,fman-ptp-timer"; + reg = <0x4fe000 0x1000>; + interrupts = <96 2 0 0>; }; diff --git a/arch/powerpc/boot/dts/fsl/qoriq-fman-1.dtsi b/arch/powerpc/boot/dts/fsl/qoriq-fman-1.dtsi index debea75fd3f0..b80aaf5f00a1 100644 --- a/arch/powerpc/boot/dts/fsl/qoriq-fman-1.dtsi +++ b/arch/powerpc/boot/dts/fsl/qoriq-fman-1.dtsi @@ -37,12 +37,13 @@ fman1: fman@500000 { #size-cells = <1>; cell-index = <1>; compatible = "fsl,fman"; - ranges = <0 0x500000 0x100000>; - reg = <0x500000 0x100000>; + ranges = <0 0x500000 0xfe000>; + reg = <0x500000 0xfe000>; interrupts = <97 2 0 0>, <16 2 1 0>; clocks = <&clockgen 3 1>; clock-names = "fmanclk"; fsl,qman-channel-range = <0x60 0xc>; + ptimer-handle = <&ptp_timer1>; muram@0 { compatible = "fsl,fman-muram"; @@ -93,9 +94,10 @@ fman1: fman@500000 { reg = <0x87000 0x1000>; status = "disabled"; }; +}; - ptp_timer1: ptp-timer@fe000 { - compatible = "fsl,fman-ptp-timer"; - reg = <0xfe000 0x1000>; - }; +ptp_timer1: ptp-timer@5fe000 { + compatible = "fsl,fman-ptp-timer"; + reg = <0x5fe000 0x1000>; + interrupts = <97 2 0 0>; }; diff --git a/arch/powerpc/boot/dts/fsl/qoriq-fman3-0.dtsi b/arch/powerpc/boot/dts/fsl/qoriq-fman3-0.dtsi index 3a20e0d1a6d2..d3720fdde26c 100644 --- a/arch/powerpc/boot/dts/fsl/qoriq-fman3-0.dtsi +++ b/arch/powerpc/boot/dts/fsl/qoriq-fman3-0.dtsi @@ -37,12 +37,13 @@ fman0: fman@400000 { #size-cells = <1>; cell-index = <0>; compatible = "fsl,fman"; - ranges = <0 0x400000 0x100000>; - reg = <0x400000 0x100000>; + ranges = <0 0x400000 0xfe000>; + reg = <0x400000 0xfe000>; interrupts = <96 2 0 0>, <16 2 1 1>; clocks = <&clockgen 3 0>; clock-names = "fmanclk"; fsl,qman-channel-range = <0x800 0x10>; + ptimer-handle = <&ptp_timer0>; muram@0 { compatible = "fsl,fman-muram"; @@ -98,9 +99,10 @@ fman0: fman@400000 { compatible = "fsl,fman-memac-mdio", "fsl,fman-xmdio"; reg = <0xfd000 0x1000>; }; +}; - ptp_timer0: ptp-timer@fe000 { - compatible = "fsl,fman-ptp-timer"; - reg = <0xfe000 0x1000>; - }; +ptp_timer0: ptp-timer@4fe000 { + compatible = "fsl,fman-ptp-timer"; + reg = <0x4fe000 0x1000>; + interrupts = <96 2 0 0>; }; diff --git a/arch/powerpc/boot/dts/fsl/qoriq-fman3-1.dtsi b/arch/powerpc/boot/dts/fsl/qoriq-fman3-1.dtsi index 82750ac944c7..ae34c204a5bc 100644 --- a/arch/powerpc/boot/dts/fsl/qoriq-fman3-1.dtsi +++ b/arch/powerpc/boot/dts/fsl/qoriq-fman3-1.dtsi @@ -37,12 +37,13 @@ fman1: fman@500000 { #size-cells = <1>; cell-index = <1>; compatible = "fsl,fman"; - ranges = <0 0x500000 0x100000>; - reg = <0x500000 0x100000>; + ranges = <0 0x500000 0xfe000>; + reg = <0x500000 0xfe000>; interrupts = <97 2 0 0>, <16 2 1 0>; clocks = <&clockgen 3 1>; clock-names = "fmanclk"; fsl,qman-channel-range = <0x820 0x10>; + ptimer-handle = <&ptp_timer1>; muram@0 { compatible = "fsl,fman-muram"; @@ -98,9 +99,10 @@ fman1: fman@500000 { compatible = "fsl,fman-memac-mdio", "fsl,fman-xmdio"; reg = <0xfd000 0x1000>; }; +}; - ptp_timer1: ptp-timer@fe000 { - compatible = "fsl,fman-ptp-timer"; - reg = <0xfe000 0x1000>; - }; +ptp_timer1: ptp-timer@5fe000 { + compatible = "fsl,fman-ptp-timer"; + reg = <0x5fe000 0x1000>; + interrupts = <97 2 0 0>; }; diff --git a/arch/powerpc/boot/dts/fsl/qoriq-fman3l-0.dtsi b/arch/powerpc/boot/dts/fsl/qoriq-fman3l-0.dtsi index 7f60b6060176..02f2755842cc 100644 --- a/arch/powerpc/boot/dts/fsl/qoriq-fman3l-0.dtsi +++ b/arch/powerpc/boot/dts/fsl/qoriq-fman3l-0.dtsi @@ -37,12 +37,13 @@ fman0: fman@400000 { #size-cells = <1>; cell-index = <0>; compatible = "fsl,fman"; - ranges = <0 0x400000 0x100000>; - reg = <0x400000 0x100000>; + ranges = <0 0x400000 0xfe000>; + reg = <0x400000 0xfe000>; interrupts = <96 2 0 0>, <16 2 1 1>; clocks = <&clockgen 3 0>; clock-names = "fmanclk"; fsl,qman-channel-range = <0x800 0x10>; + ptimer-handle = <&ptp_timer0>; muram@0 { compatible = "fsl,fman-muram"; @@ -86,9 +87,10 @@ fman0: fman@400000 { compatible = "fsl,fman-memac-mdio", "fsl,fman-xmdio"; reg = <0xfd000 0x1000>; }; +}; - ptp_timer0: ptp-timer@fe000 { - compatible = "fsl,fman-ptp-timer"; - reg = <0xfe000 0x1000>; - }; +ptp_timer0: ptp-timer@4fe000 { + compatible = "fsl,fman-ptp-timer"; + reg = <0x4fe000 0x1000>; + interrupts = <96 2 0 0>; }; -- cgit v1.2.3 From 9cd19b52e1c4ba3f097a6ccab9b7595e6b9fbd3c Mon Sep 17 00:00:00 2001 From: Yangbo Lu Date: Mon, 25 Jun 2018 20:37:11 +0800 Subject: arm64: dts: fsl: move ptp timer out of fman This patch is to move ptp timer node out of fman. Because ptp timer will be probed by ptp_qoriq driver, it should be an independent device in case of conflict memory mapping. Signed-off-by: Yangbo Lu Acked-by: Richard Cochran Acked-by: Madalin Bucur Signed-off-by: David S. Miller --- arch/arm64/boot/dts/freescale/qoriq-fman3-0.dtsi | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'arch') diff --git a/arch/arm64/boot/dts/freescale/qoriq-fman3-0.dtsi b/arch/arm64/boot/dts/freescale/qoriq-fman3-0.dtsi index 4dd06767f839..a56a408e9bf7 100644 --- a/arch/arm64/boot/dts/freescale/qoriq-fman3-0.dtsi +++ b/arch/arm64/boot/dts/freescale/qoriq-fman3-0.dtsi @@ -11,13 +11,14 @@ fman0: fman@1a00000 { #size-cells = <1>; cell-index = <0>; compatible = "fsl,fman"; - ranges = <0x0 0x0 0x1a00000 0x100000>; - reg = <0x0 0x1a00000 0x0 0x100000>; + ranges = <0x0 0x0 0x1a00000 0xfe000>; + reg = <0x0 0x1a00000 0x0 0xfe000>; interrupts = , ; clocks = <&clockgen 3 0>; clock-names = "fmanclk"; fsl,qman-channel-range = <0x800 0x10>; + ptimer-handle = <&ptp_timer0>; muram@0 { compatible = "fsl,fman-muram"; @@ -73,9 +74,10 @@ fman0: fman@1a00000 { compatible = "fsl,fman-memac-mdio", "fsl,fman-xmdio"; reg = <0xfd000 0x1000>; }; +}; - ptp_timer0: ptp-timer@fe000 { - compatible = "fsl,fman-ptp-timer"; - reg = <0xfe000 0x1000>; - }; +ptp_timer0: ptp-timer@1afe000 { + compatible = "fsl,fman-ptp-timer"; + reg = <0x0 0x1afe000 0x0 0x1000>; + interrupts = ; }; -- cgit v1.2.3 From 80b14dee2bea128928537d61c333f24cb8cbb62f Mon Sep 17 00:00:00 2001 From: Richard Cochran Date: Tue, 3 Jul 2018 15:42:48 -0700 Subject: net: Add a new socket option for a future transmit time. This patch introduces SO_TXTIME. User space enables this option in order to pass a desired future transmit time in a CMSG when calling sendmsg(2). The argument to this socket option is a 8-bytes long struct provided by the uapi header net_tstamp.h defined as: struct sock_txtime { clockid_t clockid; u32 flags; }; Note that new fields were added to struct sock by filling a 2-bytes hole found in the struct. For that reason, neither the struct size or number of cachelines were altered. Signed-off-by: Richard Cochran Signed-off-by: Jesus Sanchez-Palencia Signed-off-by: David S. Miller --- arch/alpha/include/uapi/asm/socket.h | 3 +++ arch/ia64/include/uapi/asm/socket.h | 3 +++ arch/mips/include/uapi/asm/socket.h | 3 +++ arch/parisc/include/uapi/asm/socket.h | 3 +++ arch/s390/include/uapi/asm/socket.h | 3 +++ arch/sparc/include/uapi/asm/socket.h | 3 +++ arch/xtensa/include/uapi/asm/socket.h | 3 +++ include/net/sock.h | 10 ++++++++++ include/uapi/asm-generic/socket.h | 3 +++ include/uapi/linux/net_tstamp.h | 15 +++++++++++++++ net/core/sock.c | 35 +++++++++++++++++++++++++++++++++++ 11 files changed, 84 insertions(+) (limited to 'arch') diff --git a/arch/alpha/include/uapi/asm/socket.h b/arch/alpha/include/uapi/asm/socket.h index be14f16149d5..065fb372e355 100644 --- a/arch/alpha/include/uapi/asm/socket.h +++ b/arch/alpha/include/uapi/asm/socket.h @@ -112,4 +112,7 @@ #define SO_ZEROCOPY 60 +#define SO_TXTIME 61 +#define SCM_TXTIME SO_TXTIME + #endif /* _UAPI_ASM_SOCKET_H */ diff --git a/arch/ia64/include/uapi/asm/socket.h b/arch/ia64/include/uapi/asm/socket.h index 3efba40adc54..c872c4e6bafb 100644 --- a/arch/ia64/include/uapi/asm/socket.h +++ b/arch/ia64/include/uapi/asm/socket.h @@ -114,4 +114,7 @@ #define SO_ZEROCOPY 60 +#define SO_TXTIME 61 +#define SCM_TXTIME SO_TXTIME + #endif /* _ASM_IA64_SOCKET_H */ diff --git a/arch/mips/include/uapi/asm/socket.h b/arch/mips/include/uapi/asm/socket.h index 49c3d4795963..71370fb3ceef 100644 --- a/arch/mips/include/uapi/asm/socket.h +++ b/arch/mips/include/uapi/asm/socket.h @@ -123,4 +123,7 @@ #define SO_ZEROCOPY 60 +#define SO_TXTIME 61 +#define SCM_TXTIME SO_TXTIME + #endif /* _UAPI_ASM_SOCKET_H */ diff --git a/arch/parisc/include/uapi/asm/socket.h b/arch/parisc/include/uapi/asm/socket.h index 1d0fdc3b5d22..061b9cf2a779 100644 --- a/arch/parisc/include/uapi/asm/socket.h +++ b/arch/parisc/include/uapi/asm/socket.h @@ -104,4 +104,7 @@ #define SO_ZEROCOPY 0x4035 +#define SO_TXTIME 0x4036 +#define SCM_TXTIME SO_TXTIME + #endif /* _UAPI_ASM_SOCKET_H */ diff --git a/arch/s390/include/uapi/asm/socket.h b/arch/s390/include/uapi/asm/socket.h index 3510c0fd06f4..39d901476ee5 100644 --- a/arch/s390/include/uapi/asm/socket.h +++ b/arch/s390/include/uapi/asm/socket.h @@ -111,4 +111,7 @@ #define SO_ZEROCOPY 60 +#define SO_TXTIME 61 +#define SCM_TXTIME SO_TXTIME + #endif /* _ASM_SOCKET_H */ diff --git a/arch/sparc/include/uapi/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h index d58520c2e6ff..7ea35e5601b6 100644 --- a/arch/sparc/include/uapi/asm/socket.h +++ b/arch/sparc/include/uapi/asm/socket.h @@ -101,6 +101,9 @@ #define SO_ZEROCOPY 0x003e +#define SO_TXTIME 0x003f +#define SCM_TXTIME SO_TXTIME + /* Security levels - as per NRL IPv6 - don't actually do anything */ #define SO_SECURITY_AUTHENTICATION 0x5001 #define SO_SECURITY_ENCRYPTION_TRANSPORT 0x5002 diff --git a/arch/xtensa/include/uapi/asm/socket.h b/arch/xtensa/include/uapi/asm/socket.h index 75a07b8119a9..1de07a7f7680 100644 --- a/arch/xtensa/include/uapi/asm/socket.h +++ b/arch/xtensa/include/uapi/asm/socket.h @@ -116,4 +116,7 @@ #define SO_ZEROCOPY 60 +#define SO_TXTIME 61 +#define SCM_TXTIME SO_TXTIME + #endif /* _XTENSA_SOCKET_H */ diff --git a/include/net/sock.h b/include/net/sock.h index 2ed99bfa4595..68347b9821c6 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -319,6 +319,9 @@ struct sock_common { * @sk_destruct: called at sock freeing time, i.e. when all refcnt == 0 * @sk_reuseport_cb: reuseport group container * @sk_rcu: used during RCU grace period + * @sk_clockid: clockid used by time-based scheduling (SO_TXTIME) + * @sk_txtime_deadline_mode: set deadline mode for SO_TXTIME + * @sk_txtime_unused: unused txtime flags */ struct sock { /* @@ -475,6 +478,11 @@ struct sock { u8 sk_shutdown; u32 sk_tskey; atomic_t sk_zckey; + + u8 sk_clockid; + u8 sk_txtime_deadline_mode : 1, + sk_txtime_unused : 7; + struct socket *sk_socket; void *sk_user_data; #ifdef CONFIG_SECURITY @@ -790,6 +798,7 @@ enum sock_flags { SOCK_FILTER_LOCKED, /* Filter cannot be changed anymore */ SOCK_SELECT_ERR_QUEUE, /* Wake select on error queue */ SOCK_RCU_FREE, /* wait rcu grace period in sk_destruct() */ + SOCK_TXTIME, }; #define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)) @@ -1585,6 +1594,7 @@ void sock_kzfree_s(struct sock *sk, void *mem, int size); void sk_send_sigurg(struct sock *sk); struct sockcm_cookie { + u64 transmit_time; u32 mark; u16 tsflags; }; diff --git a/include/uapi/asm-generic/socket.h b/include/uapi/asm-generic/socket.h index 0ae758c90e54..a12692e5f7a8 100644 --- a/include/uapi/asm-generic/socket.h +++ b/include/uapi/asm-generic/socket.h @@ -107,4 +107,7 @@ #define SO_ZEROCOPY 60 +#define SO_TXTIME 61 +#define SCM_TXTIME SO_TXTIME + #endif /* __ASM_GENERIC_SOCKET_H */ diff --git a/include/uapi/linux/net_tstamp.h b/include/uapi/linux/net_tstamp.h index 4fe104b2411f..c9a77c353b98 100644 --- a/include/uapi/linux/net_tstamp.h +++ b/include/uapi/linux/net_tstamp.h @@ -141,4 +141,19 @@ struct scm_ts_pktinfo { __u32 reserved[2]; }; +/* + * SO_TXTIME gets a struct sock_txtime with flags being an integer bit + * field comprised of these values. + */ +enum txtime_flags { + SOF_TXTIME_DEADLINE_MODE = (1 << 0), + + SOF_TXTIME_FLAGS_MASK = (SOF_TXTIME_DEADLINE_MODE) +}; + +struct sock_txtime { + clockid_t clockid; /* reference clockid */ + u32 flags; /* flags defined by enum txtime_flags */ +}; + #endif /* _NET_TIMESTAMPING_H */ diff --git a/net/core/sock.c b/net/core/sock.c index 6429982eb976..fe64b839f1b2 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -91,6 +91,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -697,6 +698,7 @@ EXPORT_SYMBOL(sk_mc_loop); int sock_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen) { + struct sock_txtime sk_txtime; struct sock *sk = sock->sk; int val; int valbool; @@ -1070,6 +1072,24 @@ set_rcvbuf: } break; + case SO_TXTIME: + if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { + ret = -EPERM; + } else if (optlen != sizeof(struct sock_txtime)) { + ret = -EINVAL; + } else if (copy_from_user(&sk_txtime, optval, + sizeof(struct sock_txtime))) { + ret = -EFAULT; + } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) { + ret = -EINVAL; + } else { + sock_valbool_flag(sk, SOCK_TXTIME, true); + sk->sk_clockid = sk_txtime.clockid; + sk->sk_txtime_deadline_mode = + !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE); + } + break; + default: ret = -ENOPROTOOPT; break; @@ -1115,6 +1135,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname, u64 val64; struct linger ling; struct timeval tm; + struct sock_txtime txtime; } v; int lv = sizeof(int); @@ -1403,6 +1424,13 @@ int sock_getsockopt(struct socket *sock, int level, int optname, v.val = sock_flag(sk, SOCK_ZEROCOPY); break; + case SO_TXTIME: + lv = sizeof(v.txtime); + v.txtime.clockid = sk->sk_clockid; + v.txtime.flags |= sk->sk_txtime_deadline_mode ? + SOF_TXTIME_DEADLINE_MODE : 0; + break; + default: /* We implement the SO_SNDLOWAT etc to not be settable * (1003.1g 7). @@ -2137,6 +2165,13 @@ int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg, sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK; sockc->tsflags |= tsflags; break; + case SCM_TXTIME: + if (!sock_flag(sk, SOCK_TXTIME)) + return -EINVAL; + if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64))) + return -EINVAL; + sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg)); + break; /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */ case SCM_RIGHTS: case SCM_CREDENTIALS: -- cgit v1.2.3 From d449ceb11b3884770c06e71cf15edc9f3b4c9b05 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 11 Jul 2018 10:31:31 +0100 Subject: ARM: net: bpf: enumerate the JIT scratch stack layout Enumerate the contents of the JIT scratch stack layout used for storing some of the JITs 64-bit registers, tail call counter and AX register. Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 59 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 17 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index f6a62ae44a65..f2e6ffe57788 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -72,7 +72,38 @@ #define CALLEE_PUSH_MASK (CALLEE_MASK | 1 << ARM_LR) #define CALLEE_POP_MASK (CALLEE_MASK | 1 << ARM_PC) -#define STACK_OFFSET(k) (k) +enum { + /* Stack layout - these are offsets from (top of stack - 4) */ + BPF_R2_HI, + BPF_R2_LO, + BPF_R3_HI, + BPF_R3_LO, + BPF_R4_HI, + BPF_R4_LO, + BPF_R5_HI, + BPF_R5_LO, + BPF_R7_HI, + BPF_R7_LO, + BPF_R8_HI, + BPF_R8_LO, + BPF_R9_HI, + BPF_R9_LO, + BPF_FP_HI, + BPF_FP_LO, + BPF_TC_HI, + BPF_TC_LO, + BPF_AX_HI, + BPF_AX_LO, + /* Stack space for BPF_REG_2, BPF_REG_3, BPF_REG_4, + * BPF_REG_5, BPF_REG_7, BPF_REG_8, BPF_REG_9, + * BPF_REG_FP and Tail call counts. + */ + BPF_JIT_SCRATCH_REGS, +}; + +#define STACK_OFFSET(k) ((k) * 4) +#define SCRATCH_SIZE (BPF_JIT_SCRATCH_REGS * 4) + #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) /* TEMP Register 1 */ #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) /* TEMP Register 2 */ #define TCALL_CNT (MAX_BPF_JIT_REG + 2) /* Tail Call Count */ @@ -100,29 +131,29 @@ static const u8 bpf2a32[][2] = { /* arguments from eBPF program to in-kernel function */ [BPF_REG_1] = {ARM_R3, ARM_R2}, /* Stored on stack scratch space */ - [BPF_REG_2] = {STACK_OFFSET(0), STACK_OFFSET(4)}, - [BPF_REG_3] = {STACK_OFFSET(8), STACK_OFFSET(12)}, - [BPF_REG_4] = {STACK_OFFSET(16), STACK_OFFSET(20)}, - [BPF_REG_5] = {STACK_OFFSET(24), STACK_OFFSET(28)}, + [BPF_REG_2] = {STACK_OFFSET(BPF_R2_HI), STACK_OFFSET(BPF_R2_LO)}, + [BPF_REG_3] = {STACK_OFFSET(BPF_R3_HI), STACK_OFFSET(BPF_R3_LO)}, + [BPF_REG_4] = {STACK_OFFSET(BPF_R4_HI), STACK_OFFSET(BPF_R4_LO)}, + [BPF_REG_5] = {STACK_OFFSET(BPF_R5_HI), STACK_OFFSET(BPF_R5_LO)}, /* callee saved registers that in-kernel function will preserve */ [BPF_REG_6] = {ARM_R5, ARM_R4}, /* Stored on stack scratch space */ - [BPF_REG_7] = {STACK_OFFSET(32), STACK_OFFSET(36)}, - [BPF_REG_8] = {STACK_OFFSET(40), STACK_OFFSET(44)}, - [BPF_REG_9] = {STACK_OFFSET(48), STACK_OFFSET(52)}, + [BPF_REG_7] = {STACK_OFFSET(BPF_R7_HI), STACK_OFFSET(BPF_R7_LO)}, + [BPF_REG_8] = {STACK_OFFSET(BPF_R8_HI), STACK_OFFSET(BPF_R8_LO)}, + [BPF_REG_9] = {STACK_OFFSET(BPF_R9_HI), STACK_OFFSET(BPF_R9_LO)}, /* Read only Frame Pointer to access Stack */ - [BPF_REG_FP] = {STACK_OFFSET(56), STACK_OFFSET(60)}, + [BPF_REG_FP] = {STACK_OFFSET(BPF_FP_HI), STACK_OFFSET(BPF_FP_LO)}, /* Temporary Register for internal BPF JIT, can be used * for constant blindings and others. */ [TMP_REG_1] = {ARM_R7, ARM_R6}, [TMP_REG_2] = {ARM_R10, ARM_R8}, /* Tail call count. Stored on stack scratch space. */ - [TCALL_CNT] = {STACK_OFFSET(64), STACK_OFFSET(68)}, + [TCALL_CNT] = {STACK_OFFSET(BPF_TC_HI), STACK_OFFSET(BPF_TC_LO)}, /* temporary register for blinding constants. * Stored on stack scratch space. */ - [BPF_REG_AX] = {STACK_OFFSET(72), STACK_OFFSET(76)}, + [BPF_REG_AX] = {STACK_OFFSET(BPF_AX_HI), STACK_OFFSET(BPF_AX_LO)}, }; #define dst_lo dst[1] @@ -227,12 +258,6 @@ static void jit_fill_hole(void *area, unsigned int size) #define STACK_ALIGNMENT 4 #endif -/* Stack space for BPF_REG_2, BPF_REG_3, BPF_REG_4, - * BPF_REG_5, BPF_REG_7, BPF_REG_8, BPF_REG_9, - * BPF_REG_FP and Tail call counts. - */ -#define SCRATCH_SIZE 80 - /* total stack size used in JITed code */ #define _STACK_SIZE (ctx->prog->aux->stack_depth + SCRATCH_SIZE) #define STACK_SIZE ALIGN(_STACK_SIZE, STACK_ALIGNMENT) -- cgit v1.2.3 From a8ef95a034233190b1dd73ff03472ff0f7f4fbdf Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 11 Jul 2018 10:31:36 +0100 Subject: ARM: net: bpf: provide load/store ops with negative immediates Provide a set of load/store opcode generators that work with negative immediates as well as positive ones. Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 28 ++++++++++++++++++++++++++++ arch/arm/net/bpf_jit_32.h | 35 +++++++++++++---------------------- 2 files changed, 41 insertions(+), 22 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index f2e6ffe57788..c81da1a50834 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -239,6 +239,34 @@ static int16_t imm8m(u32 x) return -1; } +static u32 arm_bpf_ldst_imm12(u32 op, u8 rt, u8 rn, s16 imm12) +{ + op |= rt << 12 | rn << 16; + if (imm12 >= 0) + op |= ARM_INST_LDST__U; + else + imm12 = -imm12; + return op | (imm12 & 0xfff); +} + +static u32 arm_bpf_ldst_imm8(u32 op, u8 rt, u8 rn, s16 imm8) +{ + op |= rt << 12 | rn << 16; + if (imm8 >= 0) + op |= ARM_INST_LDST__U; + else + imm8 = -imm8; + return op | (imm8 & 0xf0) << 4 | (imm8 & 0x0f); +} + +#define ARM_LDR_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_LDR_I, rt, rn, off) +#define ARM_LDRB_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_LDRB_I, rt, rn, off) +#define ARM_LDRH_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRH_I, rt, rn, off) + +#define ARM_STR_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_STR_I, rt, rn, off) +#define ARM_STRB_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_STRB_I, rt, rn, off) +#define ARM_STRH_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_STRH_I, rt, rn, off) + /* * Initializes the JIT space with undefined instructions. */ diff --git a/arch/arm/net/bpf_jit_32.h b/arch/arm/net/bpf_jit_32.h index d5cf5f6208aa..c55bc39d3e22 100644 --- a/arch/arm/net/bpf_jit_32.h +++ b/arch/arm/net/bpf_jit_32.h @@ -77,11 +77,12 @@ #define ARM_INST_EOR_R 0x00200000 #define ARM_INST_EOR_I 0x02200000 -#define ARM_INST_LDRB_I 0x05d00000 +#define ARM_INST_LDST__U 0x00800000 +#define ARM_INST_LDRB_I 0x05500000 #define ARM_INST_LDRB_R 0x07d00000 -#define ARM_INST_LDRH_I 0x01d000b0 +#define ARM_INST_LDRH_I 0x015000b0 #define ARM_INST_LDRH_R 0x019000b0 -#define ARM_INST_LDR_I 0x05900000 +#define ARM_INST_LDR_I 0x05100000 #define ARM_INST_LDR_R 0x07900000 #define ARM_INST_LDM 0x08900000 @@ -124,9 +125,9 @@ #define ARM_INST_SBC_R 0x00c00000 #define ARM_INST_SBCS_R 0x00d00000 -#define ARM_INST_STR_I 0x05800000 -#define ARM_INST_STRB_I 0x05c00000 -#define ARM_INST_STRH_I 0x01c000b0 +#define ARM_INST_STR_I 0x05000000 +#define ARM_INST_STRB_I 0x05400000 +#define ARM_INST_STRH_I 0x014000b0 #define ARM_INST_TST_R 0x01100000 #define ARM_INST_TST_I 0x03100000 @@ -183,17 +184,14 @@ #define ARM_EOR_R(rd, rn, rm) _AL3_R(ARM_INST_EOR, rd, rn, rm) #define ARM_EOR_I(rd, rn, imm) _AL3_I(ARM_INST_EOR, rd, rn, imm) -#define ARM_LDR_I(rt, rn, off) (ARM_INST_LDR_I | (rt) << 12 | (rn) << 16 \ - | ((off) & 0xfff)) -#define ARM_LDR_R(rt, rn, rm) (ARM_INST_LDR_R | (rt) << 12 | (rn) << 16 \ +#define ARM_LDR_R(rt, rn, rm) (ARM_INST_LDR_R | ARM_INST_LDST__U \ + | (rt) << 12 | (rn) << 16 \ | (rm)) -#define ARM_LDRB_I(rt, rn, off) (ARM_INST_LDRB_I | (rt) << 12 | (rn) << 16 \ - | (off)) -#define ARM_LDRB_R(rt, rn, rm) (ARM_INST_LDRB_R | (rt) << 12 | (rn) << 16 \ +#define ARM_LDRB_R(rt, rn, rm) (ARM_INST_LDRB_R | ARM_INST_LDST__U \ + | (rt) << 12 | (rn) << 16 \ | (rm)) -#define ARM_LDRH_I(rt, rn, off) (ARM_INST_LDRH_I | (rt) << 12 | (rn) << 16 \ - | (((off) & 0xf0) << 4) | ((off) & 0xf)) -#define ARM_LDRH_R(rt, rn, rm) (ARM_INST_LDRH_R | (rt) << 12 | (rn) << 16 \ +#define ARM_LDRH_R(rt, rn, rm) (ARM_INST_LDRH_R | ARM_INST_LDST__U \ + | (rt) << 12 | (rn) << 16 \ | (rm)) #define ARM_LDM(rn, regs) (ARM_INST_LDM | (rn) << 16 | (regs)) @@ -254,13 +252,6 @@ #define ARM_SUBS_I(rd, rn, imm) _AL3_I(ARM_INST_SUBS, rd, rn, imm) #define ARM_SBC_I(rd, rn, imm) _AL3_I(ARM_INST_SBC, rd, rn, imm) -#define ARM_STR_I(rt, rn, off) (ARM_INST_STR_I | (rt) << 12 | (rn) << 16 \ - | ((off) & 0xfff)) -#define ARM_STRH_I(rt, rn, off) (ARM_INST_STRH_I | (rt) << 12 | (rn) << 16 \ - | (((off) & 0xf0) << 4) | ((off) & 0xf)) -#define ARM_STRB_I(rt, rn, off) (ARM_INST_STRB_I | (rt) << 12 | (rn) << 16 \ - | (((off) & 0xf0) << 4) | ((off) & 0xf)) - #define ARM_TST_R(rn, rm) _AL3_R(ARM_INST_TST, 0, rn, rm) #define ARM_TST_I(rn, imm) _AL3_I(ARM_INST_TST, 0, rn, imm) -- cgit v1.2.3 From 1c35ba122d4a4eb32c3f8d63a445c1ebfd66d7bc Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 11 Jul 2018 10:31:41 +0100 Subject: ARM: net: bpf: use negative numbers for stacked registers Use negative numbers for eBPF registers that live on the stack. Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 200 +++++++++++++++++++++++----------------------- 1 file changed, 102 insertions(+), 98 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index c81da1a50834..69bf7ab18bf9 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -101,7 +101,11 @@ enum { BPF_JIT_SCRATCH_REGS, }; -#define STACK_OFFSET(k) ((k) * 4) +/* + * Negative "register" values indicate the register is stored on the stack + * and are the offset from the top of the eBPF JIT scratch space. + */ +#define STACK_OFFSET(k) (-4 - (k) * 4) #define SCRATCH_SIZE (BPF_JIT_SCRATCH_REGS * 4) #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) /* TEMP Register 1 */ @@ -125,7 +129,7 @@ enum { * scratch memory space and we have to build eBPF 64 bit register from those. * */ -static const u8 bpf2a32[][2] = { +static const s8 bpf2a32[][2] = { /* return value from in-kernel function, and exit value from eBPF */ [BPF_REG_0] = {ARM_R1, ARM_R0}, /* arguments from eBPF program to in-kernel function */ @@ -291,7 +295,7 @@ static void jit_fill_hole(void *area, unsigned int size) #define STACK_SIZE ALIGN(_STACK_SIZE, STACK_ALIGNMENT) /* Get the offset of eBPF REGISTERs stored on scratch space. */ -#define STACK_VAR(off) (STACK_SIZE - off) +#define STACK_VAR(off) (STACK_SIZE + (off)) #if __LINUX_ARM_ARCH__ < 7 @@ -408,7 +412,7 @@ static inline int epilogue_offset(const struct jit_ctx *ctx) static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op) { - const u8 *tmp = bpf2a32[TMP_REG_1]; + const s8 *tmp = bpf2a32[TMP_REG_1]; #if __LINUX_ARM_ARCH__ == 7 if (elf_hwcap & HWCAP_IDIVA) { @@ -470,10 +474,10 @@ static inline bool is_on_stack(u8 bpf_reg) return false; } -static inline void emit_a32_mov_i(const u8 dst, const u32 val, +static inline void emit_a32_mov_i(const s8 dst, const u32 val, bool dstk, struct jit_ctx *ctx) { - const u8 *tmp = bpf2a32[TMP_REG_1]; + const s8 *tmp = bpf2a32[TMP_REG_1]; if (dstk) { emit_mov_i(tmp[1], val, ctx); @@ -484,7 +488,7 @@ static inline void emit_a32_mov_i(const u8 dst, const u32 val, } /* Sign extended move */ -static inline void emit_a32_mov_i64(const bool is64, const u8 dst[], +static inline void emit_a32_mov_i64(const bool is64, const s8 dst[], const u32 val, bool dstk, struct jit_ctx *ctx) { u32 hi = 0; @@ -574,12 +578,12 @@ static inline void emit_alu_r(const u8 dst, const u8 src, const bool is64, /* ALU operation (32 bit) * dst = dst (op) src */ -static inline void emit_a32_alu_r(const u8 dst, const u8 src, +static inline void emit_a32_alu_r(const s8 dst, const s8 src, bool dstk, bool sstk, struct jit_ctx *ctx, const bool is64, const bool hi, const u8 op) { - const u8 *tmp = bpf2a32[TMP_REG_1]; - u8 rn = sstk ? tmp[1] : src; + const s8 *tmp = bpf2a32[TMP_REG_1]; + s8 rn = sstk ? tmp[1] : src; if (sstk) emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src)), ctx); @@ -595,8 +599,8 @@ static inline void emit_a32_alu_r(const u8 dst, const u8 src, } /* ALU operation (64 bit) */ -static inline void emit_a32_alu_r64(const bool is64, const u8 dst[], - const u8 src[], bool dstk, +static inline void emit_a32_alu_r64(const bool is64, const s8 dst[], + const s8 src[], bool dstk, bool sstk, struct jit_ctx *ctx, const u8 op) { emit_a32_alu_r(dst_lo, src_lo, dstk, sstk, ctx, is64, false, op); @@ -607,11 +611,11 @@ static inline void emit_a32_alu_r64(const bool is64, const u8 dst[], } /* dst = imm (4 bytes)*/ -static inline void emit_a32_mov_r(const u8 dst, const u8 src, +static inline void emit_a32_mov_r(const s8 dst, const s8 src, bool dstk, bool sstk, struct jit_ctx *ctx) { - const u8 *tmp = bpf2a32[TMP_REG_1]; - u8 rt = sstk ? tmp[0] : src; + const s8 *tmp = bpf2a32[TMP_REG_1]; + s8 rt = sstk ? tmp[0] : src; if (sstk) emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(src)), ctx); @@ -622,8 +626,8 @@ static inline void emit_a32_mov_r(const u8 dst, const u8 src, } /* dst = src */ -static inline void emit_a32_mov_r64(const bool is64, const u8 dst[], - const u8 src[], bool dstk, +static inline void emit_a32_mov_r64(const bool is64, const s8 dst[], + const s8 src[], bool dstk, bool sstk, struct jit_ctx *ctx) { emit_a32_mov_r(dst_lo, src_lo, dstk, sstk, ctx); if (is64) { @@ -636,10 +640,10 @@ static inline void emit_a32_mov_r64(const bool is64, const u8 dst[], } /* Shift operations */ -static inline void emit_a32_alu_i(const u8 dst, const u32 val, bool dstk, +static inline void emit_a32_alu_i(const s8 dst, const u32 val, bool dstk, struct jit_ctx *ctx, const u8 op) { - const u8 *tmp = bpf2a32[TMP_REG_1]; - u8 rd = dstk ? tmp[0] : dst; + const s8 *tmp = bpf2a32[TMP_REG_1]; + s8 rd = dstk ? tmp[0] : dst; if (dstk) emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx); @@ -662,11 +666,11 @@ static inline void emit_a32_alu_i(const u8 dst, const u32 val, bool dstk, } /* dst = ~dst (64 bit) */ -static inline void emit_a32_neg64(const u8 dst[], bool dstk, +static inline void emit_a32_neg64(const s8 dst[], bool dstk, struct jit_ctx *ctx){ - const u8 *tmp = bpf2a32[TMP_REG_1]; - u8 rd = dstk ? tmp[1] : dst[1]; - u8 rm = dstk ? tmp[0] : dst[0]; + const s8 *tmp = bpf2a32[TMP_REG_1]; + s8 rd = dstk ? tmp[1] : dst[1]; + s8 rm = dstk ? tmp[0] : dst[0]; /* Setup Operand */ if (dstk) { @@ -685,15 +689,15 @@ static inline void emit_a32_neg64(const u8 dst[], bool dstk, } /* dst = dst << src */ -static inline void emit_a32_lsh_r64(const u8 dst[], const u8 src[], bool dstk, +static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[], bool dstk, bool sstk, struct jit_ctx *ctx) { - const u8 *tmp = bpf2a32[TMP_REG_1]; - const u8 *tmp2 = bpf2a32[TMP_REG_2]; + const s8 *tmp = bpf2a32[TMP_REG_1]; + const s8 *tmp2 = bpf2a32[TMP_REG_2]; /* Setup Operands */ - u8 rt = sstk ? tmp2[1] : src_lo; - u8 rd = dstk ? tmp[1] : dst_lo; - u8 rm = dstk ? tmp[0] : dst_hi; + s8 rt = sstk ? tmp2[1] : src_lo; + s8 rd = dstk ? tmp[1] : dst_lo; + s8 rm = dstk ? tmp[0] : dst_hi; if (sstk) emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx); @@ -720,14 +724,14 @@ static inline void emit_a32_lsh_r64(const u8 dst[], const u8 src[], bool dstk, } /* dst = dst >> src (signed)*/ -static inline void emit_a32_arsh_r64(const u8 dst[], const u8 src[], bool dstk, +static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[], bool dstk, bool sstk, struct jit_ctx *ctx) { - const u8 *tmp = bpf2a32[TMP_REG_1]; - const u8 *tmp2 = bpf2a32[TMP_REG_2]; + const s8 *tmp = bpf2a32[TMP_REG_1]; + const s8 *tmp2 = bpf2a32[TMP_REG_2]; /* Setup Operands */ - u8 rt = sstk ? tmp2[1] : src_lo; - u8 rd = dstk ? tmp[1] : dst_lo; - u8 rm = dstk ? tmp[0] : dst_hi; + s8 rt = sstk ? tmp2[1] : src_lo; + s8 rd = dstk ? tmp[1] : dst_lo; + s8 rm = dstk ? tmp[0] : dst_hi; if (sstk) emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx); @@ -754,14 +758,14 @@ static inline void emit_a32_arsh_r64(const u8 dst[], const u8 src[], bool dstk, } /* dst = dst >> src */ -static inline void emit_a32_rsh_r64(const u8 dst[], const u8 src[], bool dstk, +static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[], bool dstk, bool sstk, struct jit_ctx *ctx) { - const u8 *tmp = bpf2a32[TMP_REG_1]; - const u8 *tmp2 = bpf2a32[TMP_REG_2]; + const s8 *tmp = bpf2a32[TMP_REG_1]; + const s8 *tmp2 = bpf2a32[TMP_REG_2]; /* Setup Operands */ - u8 rt = sstk ? tmp2[1] : src_lo; - u8 rd = dstk ? tmp[1] : dst_lo; - u8 rm = dstk ? tmp[0] : dst_hi; + s8 rt = sstk ? tmp2[1] : src_lo; + s8 rd = dstk ? tmp[1] : dst_lo; + s8 rm = dstk ? tmp[0] : dst_hi; if (sstk) emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx); @@ -787,13 +791,13 @@ static inline void emit_a32_rsh_r64(const u8 dst[], const u8 src[], bool dstk, } /* dst = dst << val */ -static inline void emit_a32_lsh_i64(const u8 dst[], bool dstk, +static inline void emit_a32_lsh_i64(const s8 dst[], bool dstk, const u32 val, struct jit_ctx *ctx){ - const u8 *tmp = bpf2a32[TMP_REG_1]; - const u8 *tmp2 = bpf2a32[TMP_REG_2]; + const s8 *tmp = bpf2a32[TMP_REG_1]; + const s8 *tmp2 = bpf2a32[TMP_REG_2]; /* Setup operands */ - u8 rd = dstk ? tmp[1] : dst_lo; - u8 rm = dstk ? tmp[0] : dst_hi; + s8 rd = dstk ? tmp[1] : dst_lo; + s8 rm = dstk ? tmp[0] : dst_hi; if (dstk) { emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); @@ -820,13 +824,13 @@ static inline void emit_a32_lsh_i64(const u8 dst[], bool dstk, } /* dst = dst >> val */ -static inline void emit_a32_rsh_i64(const u8 dst[], bool dstk, +static inline void emit_a32_rsh_i64(const s8 dst[], bool dstk, const u32 val, struct jit_ctx *ctx) { - const u8 *tmp = bpf2a32[TMP_REG_1]; - const u8 *tmp2 = bpf2a32[TMP_REG_2]; + const s8 *tmp = bpf2a32[TMP_REG_1]; + const s8 *tmp2 = bpf2a32[TMP_REG_2]; /* Setup operands */ - u8 rd = dstk ? tmp[1] : dst_lo; - u8 rm = dstk ? tmp[0] : dst_hi; + s8 rd = dstk ? tmp[1] : dst_lo; + s8 rm = dstk ? tmp[0] : dst_hi; if (dstk) { emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); @@ -853,13 +857,13 @@ static inline void emit_a32_rsh_i64(const u8 dst[], bool dstk, } /* dst = dst >> val (signed) */ -static inline void emit_a32_arsh_i64(const u8 dst[], bool dstk, +static inline void emit_a32_arsh_i64(const s8 dst[], bool dstk, const u32 val, struct jit_ctx *ctx){ - const u8 *tmp = bpf2a32[TMP_REG_1]; - const u8 *tmp2 = bpf2a32[TMP_REG_2]; + const s8 *tmp = bpf2a32[TMP_REG_1]; + const s8 *tmp2 = bpf2a32[TMP_REG_2]; /* Setup operands */ - u8 rd = dstk ? tmp[1] : dst_lo; - u8 rm = dstk ? tmp[0] : dst_hi; + s8 rd = dstk ? tmp[1] : dst_lo; + s8 rm = dstk ? tmp[0] : dst_hi; if (dstk) { emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); @@ -885,15 +889,15 @@ static inline void emit_a32_arsh_i64(const u8 dst[], bool dstk, } } -static inline void emit_a32_mul_r64(const u8 dst[], const u8 src[], bool dstk, +static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[], bool dstk, bool sstk, struct jit_ctx *ctx) { - const u8 *tmp = bpf2a32[TMP_REG_1]; - const u8 *tmp2 = bpf2a32[TMP_REG_2]; + const s8 *tmp = bpf2a32[TMP_REG_1]; + const s8 *tmp2 = bpf2a32[TMP_REG_2]; /* Setup operands for multiplication */ - u8 rd = dstk ? tmp[1] : dst_lo; - u8 rm = dstk ? tmp[0] : dst_hi; - u8 rt = sstk ? tmp2[1] : src_lo; - u8 rn = sstk ? tmp2[0] : src_hi; + s8 rd = dstk ? tmp[1] : dst_lo; + s8 rm = dstk ? tmp[0] : dst_hi; + s8 rt = sstk ? tmp2[1] : src_lo; + s8 rn = sstk ? tmp2[0] : src_hi; if (dstk) { emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); @@ -920,10 +924,10 @@ static inline void emit_a32_mul_r64(const u8 dst[], const u8 src[], bool dstk, } /* *(size *)(dst + off) = src */ -static inline void emit_str_r(const u8 dst, const u8 src, bool dstk, +static inline void emit_str_r(const s8 dst, const s8 src, bool dstk, const s32 off, struct jit_ctx *ctx, const u8 sz){ - const u8 *tmp = bpf2a32[TMP_REG_1]; - u8 rd = dstk ? tmp[1] : dst; + const s8 *tmp = bpf2a32[TMP_REG_1]; + s8 rd = dstk ? tmp[1] : dst; if (dstk) emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx); @@ -949,11 +953,11 @@ static inline void emit_str_r(const u8 dst, const u8 src, bool dstk, } /* dst = *(size*)(src + off) */ -static inline void emit_ldx_r(const u8 dst[], const u8 src, bool dstk, +static inline void emit_ldx_r(const s8 dst[], const s8 src, bool dstk, s32 off, struct jit_ctx *ctx, const u8 sz){ - const u8 *tmp = bpf2a32[TMP_REG_1]; - const u8 *rd = dstk ? tmp : dst; - u8 rm = src; + const s8 *tmp = bpf2a32[TMP_REG_1]; + const s8 *rd = dstk ? tmp : dst; + s8 rm = src; s32 off_max; if (sz == BPF_H) @@ -1034,11 +1038,11 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) { /* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */ - const u8 *r2 = bpf2a32[BPF_REG_2]; - const u8 *r3 = bpf2a32[BPF_REG_3]; - const u8 *tmp = bpf2a32[TMP_REG_1]; - const u8 *tmp2 = bpf2a32[TMP_REG_2]; - const u8 *tcc = bpf2a32[TCALL_CNT]; + const s8 *r2 = bpf2a32[BPF_REG_2]; + const s8 *r3 = bpf2a32[BPF_REG_3]; + const s8 *tmp = bpf2a32[TMP_REG_1]; + const s8 *tmp2 = bpf2a32[TMP_REG_2]; + const s8 *tcc = bpf2a32[TCALL_CNT]; const int idx0 = ctx->idx; #define cur_offset (ctx->idx - idx0) #define jmp_offset (out_offset - (cur_offset) - 2) @@ -1112,7 +1116,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) static inline void emit_rev16(const u8 rd, const u8 rn, struct jit_ctx *ctx) { #if __LINUX_ARM_ARCH__ < 6 - const u8 *tmp2 = bpf2a32[TMP_REG_2]; + const s8 *tmp2 = bpf2a32[TMP_REG_2]; emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx); emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 8), ctx); @@ -1127,7 +1131,7 @@ static inline void emit_rev16(const u8 rd, const u8 rn, struct jit_ctx *ctx) static inline void emit_rev32(const u8 rd, const u8 rn, struct jit_ctx *ctx) { #if __LINUX_ARM_ARCH__ < 6 - const u8 *tmp2 = bpf2a32[TMP_REG_2]; + const s8 *tmp2 = bpf2a32[TMP_REG_2]; emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx); emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 24), ctx); @@ -1147,10 +1151,10 @@ static inline void emit_rev32(const u8 rd, const u8 rn, struct jit_ctx *ctx) } // push the scratch stack register on top of the stack -static inline void emit_push_r64(const u8 src[], const u8 shift, +static inline void emit_push_r64(const s8 src[], const u8 shift, struct jit_ctx *ctx) { - const u8 *tmp2 = bpf2a32[TMP_REG_2]; + const s8 *tmp2 = bpf2a32[TMP_REG_2]; u16 reg_set = 0; emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(src[1]+shift)), ctx); @@ -1162,13 +1166,13 @@ static inline void emit_push_r64(const u8 src[], const u8 shift, static void build_prologue(struct jit_ctx *ctx) { - const u8 r0 = bpf2a32[BPF_REG_0][1]; - const u8 r2 = bpf2a32[BPF_REG_1][1]; - const u8 r3 = bpf2a32[BPF_REG_1][0]; - const u8 r4 = bpf2a32[BPF_REG_6][1]; - const u8 fplo = bpf2a32[BPF_REG_FP][1]; - const u8 fphi = bpf2a32[BPF_REG_FP][0]; - const u8 *tcc = bpf2a32[TCALL_CNT]; + const s8 r0 = bpf2a32[BPF_REG_0][1]; + const s8 r2 = bpf2a32[BPF_REG_1][1]; + const s8 r3 = bpf2a32[BPF_REG_1][0]; + const s8 r4 = bpf2a32[BPF_REG_6][1]; + const s8 fplo = bpf2a32[BPF_REG_FP][1]; + const s8 fphi = bpf2a32[BPF_REG_FP][0]; + const s8 *tcc = bpf2a32[TCALL_CNT]; /* Save callee saved registers. */ #ifdef CONFIG_FRAME_POINTER @@ -1231,17 +1235,17 @@ static void build_epilogue(struct jit_ctx *ctx) static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) { const u8 code = insn->code; - const u8 *dst = bpf2a32[insn->dst_reg]; - const u8 *src = bpf2a32[insn->src_reg]; - const u8 *tmp = bpf2a32[TMP_REG_1]; - const u8 *tmp2 = bpf2a32[TMP_REG_2]; + const s8 *dst = bpf2a32[insn->dst_reg]; + const s8 *src = bpf2a32[insn->src_reg]; + const s8 *tmp = bpf2a32[TMP_REG_1]; + const s8 *tmp2 = bpf2a32[TMP_REG_2]; const s16 off = insn->off; const s32 imm = insn->imm; const int i = insn - ctx->prog->insnsi; const bool is64 = BPF_CLASS(code) == BPF_ALU64; const bool dstk = is_on_stack(insn->dst_reg); const bool sstk = is_on_stack(insn->src_reg); - u8 rd, rt, rm, rn; + s8 rd, rt, rm, rn; s32 jmp_offset; #define check_imm(bits, imm) do { \ @@ -1672,12 +1676,12 @@ go_jmp: /* function call */ case BPF_JMP | BPF_CALL: { - const u8 *r0 = bpf2a32[BPF_REG_0]; - const u8 *r1 = bpf2a32[BPF_REG_1]; - const u8 *r2 = bpf2a32[BPF_REG_2]; - const u8 *r3 = bpf2a32[BPF_REG_3]; - const u8 *r4 = bpf2a32[BPF_REG_4]; - const u8 *r5 = bpf2a32[BPF_REG_5]; + const s8 *r0 = bpf2a32[BPF_REG_0]; + const s8 *r1 = bpf2a32[BPF_REG_1]; + const s8 *r2 = bpf2a32[BPF_REG_2]; + const s8 *r3 = bpf2a32[BPF_REG_3]; + const s8 *r4 = bpf2a32[BPF_REG_4]; + const s8 *r5 = bpf2a32[BPF_REG_5]; const u32 func = (u32)__bpf_call_base + (u32)imm; emit_a32_mov_r64(true, r0, r1, false, false, ctx); -- cgit v1.2.3 From 47b9c3bf416d80515901469f05aef2870b37c010 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 11 Jul 2018 10:31:47 +0100 Subject: ARM: net: bpf: remove is_on_stack() and sstk/dstk The decision about whether a BPF register is on the stack or in a CPU register is detected at the top BPF insn processing level, and then percolated throughout the remainder of the code. Since we now use negative register values to represent stacked registers, we can detect where a BPF register is stored without restoring to carrying this additional metadata through all code paths. Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 336 ++++++++++++++++++++++------------------------ 1 file changed, 160 insertions(+), 176 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 69bf7ab18bf9..e81401aca2df 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -459,27 +459,18 @@ static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op) emit(ARM_MOV_R(ARM_R0, tmp[1]), ctx); } -/* Checks whether BPF register is on scratch stack space or not. */ -static inline bool is_on_stack(u8 bpf_reg) +/* Is the translated BPF register on stack? */ +static bool is_stacked(s8 reg) { - static u8 stack_regs[] = {BPF_REG_AX, BPF_REG_3, BPF_REG_4, BPF_REG_5, - BPF_REG_7, BPF_REG_8, BPF_REG_9, TCALL_CNT, - BPF_REG_2, BPF_REG_FP}; - int i, reg_len = sizeof(stack_regs); - - for (i = 0 ; i < reg_len ; i++) { - if (bpf_reg == stack_regs[i]) - return true; - } - return false; + return reg < 0; } static inline void emit_a32_mov_i(const s8 dst, const u32 val, - bool dstk, struct jit_ctx *ctx) + struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; - if (dstk) { + if (is_stacked(dst)) { emit_mov_i(tmp[1], val, ctx); emit(ARM_STR_I(tmp[1], ARM_SP, STACK_VAR(dst)), ctx); } else { @@ -489,14 +480,13 @@ static inline void emit_a32_mov_i(const s8 dst, const u32 val, /* Sign extended move */ static inline void emit_a32_mov_i64(const bool is64, const s8 dst[], - const u32 val, bool dstk, - struct jit_ctx *ctx) { + const u32 val, struct jit_ctx *ctx) { u32 hi = 0; if (is64 && (val & (1<<31))) hi = (u32)~0; - emit_a32_mov_i(dst_lo, val, dstk, ctx); - emit_a32_mov_i(dst_hi, hi, dstk, ctx); + emit_a32_mov_i(dst_lo, val, ctx); + emit_a32_mov_i(dst_hi, hi, ctx); } static inline void emit_a32_add_r(const u8 dst, const u8 src, @@ -579,17 +569,16 @@ static inline void emit_alu_r(const u8 dst, const u8 src, const bool is64, * dst = dst (op) src */ static inline void emit_a32_alu_r(const s8 dst, const s8 src, - bool dstk, bool sstk, struct jit_ctx *ctx, const bool is64, const bool hi, const u8 op) { const s8 *tmp = bpf2a32[TMP_REG_1]; - s8 rn = sstk ? tmp[1] : src; + s8 rn = is_stacked(src) ? tmp[1] : src; - if (sstk) + if (is_stacked(src)) emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src)), ctx); /* ALU operation */ - if (dstk) { + if (is_stacked(dst)) { emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(dst)), ctx); emit_alu_r(tmp[0], rn, is64, hi, op, ctx); emit(ARM_STR_I(tmp[0], ARM_SP, STACK_VAR(dst)), ctx); @@ -600,26 +589,24 @@ static inline void emit_a32_alu_r(const s8 dst, const s8 src, /* ALU operation (64 bit) */ static inline void emit_a32_alu_r64(const bool is64, const s8 dst[], - const s8 src[], bool dstk, - bool sstk, struct jit_ctx *ctx, + const s8 src[], struct jit_ctx *ctx, const u8 op) { - emit_a32_alu_r(dst_lo, src_lo, dstk, sstk, ctx, is64, false, op); + emit_a32_alu_r(dst_lo, src_lo, ctx, is64, false, op); if (is64) - emit_a32_alu_r(dst_hi, src_hi, dstk, sstk, ctx, is64, true, op); + emit_a32_alu_r(dst_hi, src_hi, ctx, is64, true, op); else - emit_a32_mov_i(dst_hi, 0, dstk, ctx); + emit_a32_mov_i(dst_hi, 0, ctx); } /* dst = imm (4 bytes)*/ static inline void emit_a32_mov_r(const s8 dst, const s8 src, - bool dstk, bool sstk, struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; - s8 rt = sstk ? tmp[0] : src; + s8 rt = is_stacked(src) ? tmp[0] : src; - if (sstk) + if (is_stacked(src)) emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(src)), ctx); - if (dstk) + if (is_stacked(dst)) emit(ARM_STR_I(rt, ARM_SP, STACK_VAR(dst)), ctx); else emit(ARM_MOV_R(dst, rt), ctx); @@ -627,25 +614,25 @@ static inline void emit_a32_mov_r(const s8 dst, const s8 src, /* dst = src */ static inline void emit_a32_mov_r64(const bool is64, const s8 dst[], - const s8 src[], bool dstk, - bool sstk, struct jit_ctx *ctx) { - emit_a32_mov_r(dst_lo, src_lo, dstk, sstk, ctx); + const s8 src[], + struct jit_ctx *ctx) { + emit_a32_mov_r(dst_lo, src_lo, ctx); if (is64) { /* complete 8 byte move */ - emit_a32_mov_r(dst_hi, src_hi, dstk, sstk, ctx); + emit_a32_mov_r(dst_hi, src_hi, ctx); } else { /* Zero out high 4 bytes */ - emit_a32_mov_i(dst_hi, 0, dstk, ctx); + emit_a32_mov_i(dst_hi, 0, ctx); } } /* Shift operations */ -static inline void emit_a32_alu_i(const s8 dst, const u32 val, bool dstk, +static inline void emit_a32_alu_i(const s8 dst, const u32 val, struct jit_ctx *ctx, const u8 op) { const s8 *tmp = bpf2a32[TMP_REG_1]; - s8 rd = dstk ? tmp[0] : dst; + s8 rd = is_stacked(dst) ? tmp[0] : dst; - if (dstk) + if (is_stacked(dst)) emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx); /* Do shift operation */ @@ -661,19 +648,19 @@ static inline void emit_a32_alu_i(const s8 dst, const u32 val, bool dstk, break; } - if (dstk) + if (is_stacked(dst)) emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst)), ctx); } /* dst = ~dst (64 bit) */ -static inline void emit_a32_neg64(const s8 dst[], bool dstk, +static inline void emit_a32_neg64(const s8 dst[], struct jit_ctx *ctx){ const s8 *tmp = bpf2a32[TMP_REG_1]; - s8 rd = dstk ? tmp[1] : dst[1]; - s8 rm = dstk ? tmp[0] : dst[0]; + s8 rd = is_stacked(dst_lo) ? tmp[1] : dst[1]; + s8 rm = is_stacked(dst_lo) ? tmp[0] : dst[0]; /* Setup Operand */ - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); } @@ -682,26 +669,26 @@ static inline void emit_a32_neg64(const s8 dst[], bool dstk, emit(ARM_RSBS_I(rd, rd, 0), ctx); emit(ARM_RSC_I(rm, rm, 0), ctx); - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); } } /* dst = dst << src */ -static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[], bool dstk, - bool sstk, struct jit_ctx *ctx) { +static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[], + struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; /* Setup Operands */ - s8 rt = sstk ? tmp2[1] : src_lo; - s8 rd = dstk ? tmp[1] : dst_lo; - s8 rm = dstk ? tmp[0] : dst_hi; + s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo; + s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo; + s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi; - if (sstk) + if (is_stacked(src_lo)) emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx); - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); } @@ -714,7 +701,7 @@ static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[], bool dstk, emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd, SRTYPE_LSR, tmp2[0]), ctx); emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_ASL, rt), ctx); - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx); } else { @@ -724,18 +711,18 @@ static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[], bool dstk, } /* dst = dst >> src (signed)*/ -static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[], bool dstk, - bool sstk, struct jit_ctx *ctx) { +static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[], + struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; /* Setup Operands */ - s8 rt = sstk ? tmp2[1] : src_lo; - s8 rd = dstk ? tmp[1] : dst_lo; - s8 rm = dstk ? tmp[0] : dst_hi; + s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo; + s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo; + s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi; - if (sstk) + if (is_stacked(src_lo)) emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx); - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); } @@ -748,7 +735,7 @@ static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[], bool dstk, _emit(ARM_COND_MI, ARM_B(0), ctx); emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASR, tmp2[0]), ctx); emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_ASR, rt), ctx); - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx); } else { @@ -758,18 +745,18 @@ static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[], bool dstk, } /* dst = dst >> src */ -static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[], bool dstk, - bool sstk, struct jit_ctx *ctx) { +static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[], + struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; /* Setup Operands */ - s8 rt = sstk ? tmp2[1] : src_lo; - s8 rd = dstk ? tmp[1] : dst_lo; - s8 rm = dstk ? tmp[0] : dst_hi; + s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo; + s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo; + s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi; - if (sstk) + if (is_stacked(src_lo)) emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx); - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); } @@ -781,7 +768,7 @@ static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[], bool dstk, emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASL, ARM_IP), ctx); emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_LSR, tmp2[0]), ctx); emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_LSR, rt), ctx); - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx); } else { @@ -791,15 +778,15 @@ static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[], bool dstk, } /* dst = dst << val */ -static inline void emit_a32_lsh_i64(const s8 dst[], bool dstk, - const u32 val, struct jit_ctx *ctx){ +static inline void emit_a32_lsh_i64(const s8 dst[], + const u32 val, struct jit_ctx *ctx){ const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; /* Setup operands */ - s8 rd = dstk ? tmp[1] : dst_lo; - s8 rm = dstk ? tmp[0] : dst_hi; + s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo; + s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi; - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); } @@ -817,22 +804,22 @@ static inline void emit_a32_lsh_i64(const s8 dst[], bool dstk, emit(ARM_EOR_R(rd, rd, rd), ctx); } - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); } } /* dst = dst >> val */ -static inline void emit_a32_rsh_i64(const s8 dst[], bool dstk, +static inline void emit_a32_rsh_i64(const s8 dst[], const u32 val, struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; /* Setup operands */ - s8 rd = dstk ? tmp[1] : dst_lo; - s8 rm = dstk ? tmp[0] : dst_hi; + s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo; + s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi; - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); } @@ -850,22 +837,22 @@ static inline void emit_a32_rsh_i64(const s8 dst[], bool dstk, emit(ARM_MOV_I(rm, 0), ctx); } - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); } } /* dst = dst >> val (signed) */ -static inline void emit_a32_arsh_i64(const s8 dst[], bool dstk, +static inline void emit_a32_arsh_i64(const s8 dst[], const u32 val, struct jit_ctx *ctx){ const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; /* Setup operands */ - s8 rd = dstk ? tmp[1] : dst_lo; - s8 rm = dstk ? tmp[0] : dst_hi; + s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo; + s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi; - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); } @@ -883,27 +870,27 @@ static inline void emit_a32_arsh_i64(const s8 dst[], bool dstk, emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, 31), ctx); } - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); } } -static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[], bool dstk, - bool sstk, struct jit_ctx *ctx) { +static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[], + struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; /* Setup operands for multiplication */ - s8 rd = dstk ? tmp[1] : dst_lo; - s8 rm = dstk ? tmp[0] : dst_hi; - s8 rt = sstk ? tmp2[1] : src_lo; - s8 rn = sstk ? tmp2[0] : src_hi; + s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo; + s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi; + s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo; + s8 rn = is_stacked(src_lo) ? tmp2[0] : src_hi; - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); } - if (sstk) { + if (is_stacked(src_lo)) { emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx); emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_hi)), ctx); } @@ -915,7 +902,7 @@ static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[], bool dstk, emit(ARM_UMULL(ARM_IP, rm, rd, rt), ctx); emit(ARM_ADD_R(rm, ARM_LR, rm), ctx); - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); } else { @@ -924,15 +911,15 @@ static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[], bool dstk, } /* *(size *)(dst + off) = src */ -static inline void emit_str_r(const s8 dst, const s8 src, bool dstk, +static inline void emit_str_r(const s8 dst, const s8 src, const s32 off, struct jit_ctx *ctx, const u8 sz){ const s8 *tmp = bpf2a32[TMP_REG_1]; - s8 rd = dstk ? tmp[1] : dst; + s8 rd = is_stacked(dst) ? tmp[1] : dst; - if (dstk) + if (is_stacked(dst)) emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx); if (off) { - emit_a32_mov_i(tmp[0], off, false, ctx); + emit_a32_mov_i(tmp[0], off, ctx); emit(ARM_ADD_R(tmp[0], rd, tmp[0]), ctx); rd = tmp[0]; } @@ -953,10 +940,10 @@ static inline void emit_str_r(const s8 dst, const s8 src, bool dstk, } /* dst = *(size*)(src + off) */ -static inline void emit_ldx_r(const s8 dst[], const s8 src, bool dstk, +static inline void emit_ldx_r(const s8 dst[], const s8 src, s32 off, struct jit_ctx *ctx, const u8 sz){ const s8 *tmp = bpf2a32[TMP_REG_1]; - const s8 *rd = dstk ? tmp : dst; + const s8 *rd = is_stacked(dst_lo) ? tmp : dst; s8 rm = src; s32 off_max; @@ -966,7 +953,7 @@ static inline void emit_ldx_r(const s8 dst[], const s8 src, bool dstk, off_max = 0xfff; if (off < 0 || off > off_max) { - emit_a32_mov_i(tmp[0], off, false, ctx); + emit_a32_mov_i(tmp[0], off, ctx); emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx); rm = tmp[0]; off = 0; @@ -978,17 +965,17 @@ static inline void emit_ldx_r(const s8 dst[], const s8 src, bool dstk, case BPF_B: /* Load a Byte */ emit(ARM_LDRB_I(rd[1], rm, off), ctx); - emit_a32_mov_i(dst[0], 0, dstk, ctx); + emit_a32_mov_i(dst[0], 0, ctx); break; case BPF_H: /* Load a HalfWord */ emit(ARM_LDRH_I(rd[1], rm, off), ctx); - emit_a32_mov_i(dst[0], 0, dstk, ctx); + emit_a32_mov_i(dst[0], 0, ctx); break; case BPF_W: /* Load a Word */ emit(ARM_LDR_I(rd[1], rm, off), ctx); - emit_a32_mov_i(dst[0], 0, dstk, ctx); + emit_a32_mov_i(dst[0], 0, ctx); break; case BPF_DW: /* Load a Double Word */ @@ -996,10 +983,10 @@ static inline void emit_ldx_r(const s8 dst[], const s8 src, bool dstk, emit(ARM_LDR_I(rd[0], rm, off + 4), ctx); break; } - if (dstk) - emit(ARM_STR_I(rd[1], ARM_SP, STACK_VAR(dst[1])), ctx); - if (dstk && sz == BPF_DW) - emit(ARM_STR_I(rd[0], ARM_SP, STACK_VAR(dst[0])), ctx); + if (is_stacked(dst_lo)) + emit(ARM_STR_I(rd[1], ARM_SP, STACK_VAR(dst_lo)), ctx); + if (is_stacked(dst_lo) && sz == BPF_DW) + emit(ARM_STR_I(rd[0], ARM_SP, STACK_VAR(dst_hi)), ctx); } /* Arithmatic Operation */ @@ -1053,7 +1040,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) */ off = offsetof(struct bpf_array, map.max_entries); /* array->map.max_entries */ - emit_a32_mov_i(tmp[1], off, false, ctx); + emit_a32_mov_i(tmp[1], off, ctx); emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r2[1])), ctx); emit(ARM_LDR_R(tmp[1], tmp2[1], tmp[1]), ctx); /* index is 32-bit for arrays */ @@ -1083,7 +1070,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) * goto out; */ off = offsetof(struct bpf_array, ptrs); - emit_a32_mov_i(tmp[1], off, false, ctx); + emit_a32_mov_i(tmp[1], off, ctx); emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r2[1])), ctx); emit(ARM_ADD_R(tmp[1], tmp2[1], tmp[1]), ctx); emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r3[1])), ctx); @@ -1094,7 +1081,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) /* goto *(prog->bpf_func + prologue_size); */ off = offsetof(struct bpf_prog, bpf_func); - emit_a32_mov_i(tmp2[1], off, false, ctx); + emit_a32_mov_i(tmp2[1], off, ctx); emit(ARM_LDR_R(tmp[1], tmp[1], tmp2[1]), ctx); emit(ARM_ADD_I(tmp[1], tmp[1], ctx->prologue_bytes), ctx); emit_bx_r(tmp[1], ctx); @@ -1193,8 +1180,8 @@ static void build_prologue(struct jit_ctx *ctx) emit(ARM_SUB_I(ARM_SP, ARM_SP, ctx->stack_size), ctx); /* Set up BPF prog stack base register */ - emit_a32_mov_r(fplo, ARM_IP, true, false, ctx); - emit_a32_mov_i(fphi, 0, true, ctx); + emit_a32_mov_r(fplo, ARM_IP, ctx); + emit_a32_mov_i(fphi, 0, ctx); /* mov r4, 0 */ emit(ARM_MOV_I(r4, 0), ctx); @@ -1243,8 +1230,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) const s32 imm = insn->imm; const int i = insn - ctx->prog->insnsi; const bool is64 = BPF_CLASS(code) == BPF_ALU64; - const bool dstk = is_on_stack(insn->dst_reg); - const bool sstk = is_on_stack(insn->src_reg); s8 rd, rt, rm, rn; s32 jmp_offset; @@ -1268,11 +1253,11 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) case BPF_ALU64 | BPF_MOV | BPF_X: switch (BPF_SRC(code)) { case BPF_X: - emit_a32_mov_r64(is64, dst, src, dstk, sstk, ctx); + emit_a32_mov_r64(is64, dst, src, ctx); break; case BPF_K: /* Sign-extend immediate value to destination reg */ - emit_a32_mov_i64(is64, dst, imm, dstk, ctx); + emit_a32_mov_i64(is64, dst, imm, ctx); break; } break; @@ -1312,8 +1297,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) case BPF_ALU64 | BPF_XOR | BPF_X: switch (BPF_SRC(code)) { case BPF_X: - emit_a32_alu_r64(is64, dst, src, dstk, sstk, - ctx, BPF_OP(code)); + emit_a32_alu_r64(is64, dst, src, ctx, BPF_OP(code)); break; case BPF_K: /* Move immediate value to the temporary register @@ -1322,9 +1306,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) * value into temporary reg and then it would be * safe to do the operation on it. */ - emit_a32_mov_i64(is64, tmp2, imm, false, ctx); - emit_a32_alu_r64(is64, dst, tmp2, dstk, false, - ctx, BPF_OP(code)); + emit_a32_mov_i64(is64, tmp2, imm, ctx); + emit_a32_alu_r64(is64, dst, tmp2, ctx, BPF_OP(code)); break; } break; @@ -1334,26 +1317,28 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) case BPF_ALU | BPF_DIV | BPF_X: case BPF_ALU | BPF_MOD | BPF_K: case BPF_ALU | BPF_MOD | BPF_X: - rt = src_lo; - rd = dstk ? tmp2[1] : dst_lo; - if (dstk) + rd = is_stacked(dst_lo) ? tmp2[1] : dst_lo; + if (is_stacked(dst_lo)) emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); switch (BPF_SRC(code)) { case BPF_X: - rt = sstk ? tmp2[0] : rt; - if (sstk) + rt = is_stacked(rt) ? tmp2[0] : src_lo; + if (is_stacked(src_lo)) emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx); break; case BPF_K: rt = tmp2[0]; - emit_a32_mov_i(rt, imm, false, ctx); + emit_a32_mov_i(rt, imm, ctx); + break; + default: + rt = src_lo; break; } emit_udivmod(rd, rd, rt, ctx, BPF_OP(code)); - if (dstk) + if (is_stacked(dst_lo)) emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit_a32_mov_i(dst_hi, 0, dstk, ctx); + emit_a32_mov_i(dst_hi, 0, ctx); break; case BPF_ALU64 | BPF_DIV | BPF_K: case BPF_ALU64 | BPF_DIV | BPF_X: @@ -1367,54 +1352,54 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) if (unlikely(imm > 31)) return -EINVAL; if (imm) - emit_a32_alu_i(dst_lo, imm, dstk, ctx, BPF_OP(code)); - emit_a32_mov_i(dst_hi, 0, dstk, ctx); + emit_a32_alu_i(dst_lo, imm, ctx, BPF_OP(code)); + emit_a32_mov_i(dst_hi, 0, ctx); break; /* dst = dst << imm */ case BPF_ALU64 | BPF_LSH | BPF_K: if (unlikely(imm > 63)) return -EINVAL; - emit_a32_lsh_i64(dst, dstk, imm, ctx); + emit_a32_lsh_i64(dst, imm, ctx); break; /* dst = dst >> imm */ case BPF_ALU64 | BPF_RSH | BPF_K: if (unlikely(imm > 63)) return -EINVAL; - emit_a32_rsh_i64(dst, dstk, imm, ctx); + emit_a32_rsh_i64(dst, imm, ctx); break; /* dst = dst << src */ case BPF_ALU64 | BPF_LSH | BPF_X: - emit_a32_lsh_r64(dst, src, dstk, sstk, ctx); + emit_a32_lsh_r64(dst, src, ctx); break; /* dst = dst >> src */ case BPF_ALU64 | BPF_RSH | BPF_X: - emit_a32_rsh_r64(dst, src, dstk, sstk, ctx); + emit_a32_rsh_r64(dst, src, ctx); break; /* dst = dst >> src (signed) */ case BPF_ALU64 | BPF_ARSH | BPF_X: - emit_a32_arsh_r64(dst, src, dstk, sstk, ctx); + emit_a32_arsh_r64(dst, src, ctx); break; /* dst = dst >> imm (signed) */ case BPF_ALU64 | BPF_ARSH | BPF_K: if (unlikely(imm > 63)) return -EINVAL; - emit_a32_arsh_i64(dst, dstk, imm, ctx); + emit_a32_arsh_i64(dst, imm, ctx); break; /* dst = ~dst */ case BPF_ALU | BPF_NEG: - emit_a32_alu_i(dst_lo, 0, dstk, ctx, BPF_OP(code)); - emit_a32_mov_i(dst_hi, 0, dstk, ctx); + emit_a32_alu_i(dst_lo, 0, ctx, BPF_OP(code)); + emit_a32_mov_i(dst_hi, 0, ctx); break; /* dst = ~dst (64 bit) */ case BPF_ALU64 | BPF_NEG: - emit_a32_neg64(dst, dstk, ctx); + emit_a32_neg64(dst, ctx); break; /* dst = dst * src/imm */ case BPF_ALU64 | BPF_MUL | BPF_X: case BPF_ALU64 | BPF_MUL | BPF_K: switch (BPF_SRC(code)) { case BPF_X: - emit_a32_mul_r64(dst, src, dstk, sstk, ctx); + emit_a32_mul_r64(dst, src, ctx); break; case BPF_K: /* Move immediate value to the temporary register @@ -1423,8 +1408,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) * reg then it would be safe to do the operation * on it. */ - emit_a32_mov_i64(is64, tmp2, imm, false, ctx); - emit_a32_mul_r64(dst, tmp2, dstk, false, ctx); + emit_a32_mov_i64(is64, tmp2, imm, ctx); + emit_a32_mul_r64(dst, tmp2, ctx); break; } break; @@ -1432,9 +1417,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) /* dst = htobe(dst) */ case BPF_ALU | BPF_END | BPF_FROM_LE: case BPF_ALU | BPF_END | BPF_FROM_BE: - rd = dstk ? tmp[0] : dst_hi; - rt = dstk ? tmp[1] : dst_lo; - if (dstk) { + rd = is_stacked(dst_lo) ? tmp[0] : dst_hi; + rt = is_stacked(dst_lo) ? tmp[1] : dst_lo; + if (is_stacked(dst_lo)) { emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx); } @@ -1459,7 +1444,7 @@ emit_bswap_uxt: case 16: /* zero-extend 16 bits into 64 bits */ #if __LINUX_ARM_ARCH__ < 6 - emit_a32_mov_i(tmp2[1], 0xffff, false, ctx); + emit_a32_mov_i(tmp2[1], 0xffff, ctx); emit(ARM_AND_R(rt, rt, tmp2[1]), ctx); #else /* ARMv6+ */ emit(ARM_UXTH(rt, rt), ctx); @@ -1475,7 +1460,7 @@ emit_bswap_uxt: break; } exit: - if (dstk) { + if (is_stacked(dst_lo)) { emit(ARM_STR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx); } @@ -1487,8 +1472,8 @@ exit: u32 hi, lo = imm; hi = insn1.imm; - emit_a32_mov_i(dst_lo, lo, dstk, ctx); - emit_a32_mov_i(dst_hi, hi, dstk, ctx); + emit_a32_mov_i(dst_lo, lo, ctx); + emit_a32_mov_i(dst_hi, hi, ctx); return 1; } @@ -1497,10 +1482,10 @@ exit: case BPF_LDX | BPF_MEM | BPF_H: case BPF_LDX | BPF_MEM | BPF_B: case BPF_LDX | BPF_MEM | BPF_DW: - rn = sstk ? tmp2[1] : src_lo; - if (sstk) + rn = is_stacked(src_lo) ? tmp2[1] : src_lo; + if (is_stacked(src_lo)) emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx); - emit_ldx_r(dst, rn, dstk, off, ctx, BPF_SIZE(code)); + emit_ldx_r(dst, rn, off, ctx, BPF_SIZE(code)); break; /* ST: *(size *)(dst + off) = imm */ case BPF_ST | BPF_MEM | BPF_W: @@ -1510,16 +1495,15 @@ exit: switch (BPF_SIZE(code)) { case BPF_DW: /* Sign-extend immediate value into temp reg */ - emit_a32_mov_i64(true, tmp2, imm, false, ctx); - emit_str_r(dst_lo, tmp2[1], dstk, off, ctx, BPF_W); - emit_str_r(dst_lo, tmp2[0], dstk, off+4, ctx, BPF_W); + emit_a32_mov_i64(true, tmp2, imm, ctx); + emit_str_r(dst_lo, tmp2[1], off, ctx, BPF_W); + emit_str_r(dst_lo, tmp2[0], off+4, ctx, BPF_W); break; case BPF_W: case BPF_H: case BPF_B: - emit_a32_mov_i(tmp2[1], imm, false, ctx); - emit_str_r(dst_lo, tmp2[1], dstk, off, ctx, - BPF_SIZE(code)); + emit_a32_mov_i(tmp2[1], imm, ctx); + emit_str_r(dst_lo, tmp2[1], off, ctx, BPF_SIZE(code)); break; } break; @@ -1536,19 +1520,19 @@ exit: { u8 sz = BPF_SIZE(code); - rn = sstk ? tmp2[1] : src_lo; - rm = sstk ? tmp2[0] : src_hi; - if (sstk) { + rn = is_stacked(src_lo) ? tmp2[1] : src_lo; + rm = is_stacked(src_lo) ? tmp2[0] : src_hi; + if (is_stacked(src_lo)) { emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx); emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(src_hi)), ctx); } /* Store the value */ if (BPF_SIZE(code) == BPF_DW) { - emit_str_r(dst_lo, rn, dstk, off, ctx, BPF_W); - emit_str_r(dst_lo, rm, dstk, off+4, ctx, BPF_W); + emit_str_r(dst_lo, rn, off, ctx, BPF_W); + emit_str_r(dst_lo, rm, off+4, ctx, BPF_W); } else { - emit_str_r(dst_lo, rn, dstk, off, ctx, sz); + emit_str_r(dst_lo, rn, off, ctx, sz); } break; } @@ -1575,9 +1559,9 @@ exit: case BPF_JMP | BPF_JSLT | BPF_X: case BPF_JMP | BPF_JSLE | BPF_X: /* Setup source registers */ - rm = sstk ? tmp2[0] : src_hi; - rn = sstk ? tmp2[1] : src_lo; - if (sstk) { + rm = is_stacked(src_lo) ? tmp2[0] : src_hi; + rn = is_stacked(src_lo) ? tmp2[1] : src_lo; + if (is_stacked(src_lo)) { emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx); emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(src_hi)), ctx); } @@ -1609,12 +1593,12 @@ exit: rm = tmp2[0]; rn = tmp2[1]; /* Sign-extend immediate value */ - emit_a32_mov_i64(true, tmp2, imm, false, ctx); + emit_a32_mov_i64(true, tmp2, imm, ctx); go_jmp: /* Setup destination register */ - rd = dstk ? tmp[0] : dst_hi; - rt = dstk ? tmp[1] : dst_lo; - if (dstk) { + rd = is_stacked(dst_lo) ? tmp[0] : dst_hi; + rt = is_stacked(dst_lo) ? tmp[1] : dst_lo; + if (is_stacked(dst_lo)) { emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx); emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx); } @@ -1684,13 +1668,13 @@ go_jmp: const s8 *r5 = bpf2a32[BPF_REG_5]; const u32 func = (u32)__bpf_call_base + (u32)imm; - emit_a32_mov_r64(true, r0, r1, false, false, ctx); - emit_a32_mov_r64(true, r1, r2, false, true, ctx); + emit_a32_mov_r64(true, r0, r1, ctx); + emit_a32_mov_r64(true, r1, r2, ctx); emit_push_r64(r5, 0, ctx); emit_push_r64(r4, 8, ctx); emit_push_r64(r3, 16, ctx); - emit_a32_mov_i(tmp[1], func, false, ctx); + emit_a32_mov_i(tmp[1], func, ctx); emit_blx_r(tmp[1], ctx); emit(ARM_ADD_I(ARM_SP, ARM_SP, imm8m(24)), ctx); // callee clean -- cgit v1.2.3 From 7a9870256361d4a36cb42e0301540256bb4b864e Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 11 Jul 2018 10:31:52 +0100 Subject: ARM: net: bpf: provide accessor functions for BPF registers Many of the code paths need to have knowledge about whether a register is stacked or in a CPU register. Move this decision making to a pair of helper functions instead of having it scattered throughout the code. Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 329 ++++++++++++++++++---------------------------- 1 file changed, 128 insertions(+), 201 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index e81401aca2df..08fb4eb285a2 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -465,6 +465,31 @@ static bool is_stacked(s8 reg) return reg < 0; } +/* If a BPF register is on the stack (stk is true), load it to the + * supplied temporary register and return the temporary register + * for subsequent operations, otherwise just use the CPU register. + */ +static s8 arm_bpf_get_reg32(s8 reg, s8 tmp, struct jit_ctx *ctx) +{ + if (is_stacked(reg)) { + emit(ARM_LDR_I(tmp, ARM_SP, STACK_VAR(reg)), ctx); + reg = tmp; + } + return reg; +} + +/* If a BPF register is on the stack (stk is true), save the register + * back to the stack. If the source register is not the same, then + * move it into the correct register. + */ +static void arm_bpf_put_reg32(s8 reg, s8 src, struct jit_ctx *ctx) +{ + if (is_stacked(reg)) + emit(ARM_STR_I(src, ARM_SP, STACK_VAR(reg)), ctx); + else if (reg != src) + emit(ARM_MOV_R(reg, src), ctx); +} + static inline void emit_a32_mov_i(const s8 dst, const u32 val, struct jit_ctx *ctx) { @@ -472,7 +497,7 @@ static inline void emit_a32_mov_i(const s8 dst, const u32 val, if (is_stacked(dst)) { emit_mov_i(tmp[1], val, ctx); - emit(ARM_STR_I(tmp[1], ARM_SP, STACK_VAR(dst)), ctx); + arm_bpf_put_reg32(dst, tmp[1], ctx); } else { emit_mov_i(dst, val, ctx); } @@ -572,19 +597,13 @@ static inline void emit_a32_alu_r(const s8 dst, const s8 src, struct jit_ctx *ctx, const bool is64, const bool hi, const u8 op) { const s8 *tmp = bpf2a32[TMP_REG_1]; - s8 rn = is_stacked(src) ? tmp[1] : src; - - if (is_stacked(src)) - emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src)), ctx); + s8 rn, rd; + rn = arm_bpf_get_reg32(src, tmp[1], ctx); + rd = arm_bpf_get_reg32(dst, tmp[0], ctx); /* ALU operation */ - if (is_stacked(dst)) { - emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(dst)), ctx); - emit_alu_r(tmp[0], rn, is64, hi, op, ctx); - emit(ARM_STR_I(tmp[0], ARM_SP, STACK_VAR(dst)), ctx); - } else { - emit_alu_r(dst, rn, is64, hi, op, ctx); - } + emit_alu_r(rd, rn, is64, hi, op, ctx); + arm_bpf_put_reg32(dst, rd, ctx); } /* ALU operation (64 bit) */ @@ -598,18 +617,14 @@ static inline void emit_a32_alu_r64(const bool is64, const s8 dst[], emit_a32_mov_i(dst_hi, 0, ctx); } -/* dst = imm (4 bytes)*/ +/* dst = src (4 bytes)*/ static inline void emit_a32_mov_r(const s8 dst, const s8 src, struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; - s8 rt = is_stacked(src) ? tmp[0] : src; + s8 rt; - if (is_stacked(src)) - emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(src)), ctx); - if (is_stacked(dst)) - emit(ARM_STR_I(rt, ARM_SP, STACK_VAR(dst)), ctx); - else - emit(ARM_MOV_R(dst, rt), ctx); + rt = arm_bpf_get_reg32(src, tmp[0], ctx); + arm_bpf_put_reg32(dst, rt, ctx); } /* dst = src */ @@ -630,10 +645,9 @@ static inline void emit_a32_mov_r64(const bool is64, const s8 dst[], static inline void emit_a32_alu_i(const s8 dst, const u32 val, struct jit_ctx *ctx, const u8 op) { const s8 *tmp = bpf2a32[TMP_REG_1]; - s8 rd = is_stacked(dst) ? tmp[0] : dst; + s8 rd; - if (is_stacked(dst)) - emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx); + rd = arm_bpf_get_reg32(dst, tmp[0], ctx); /* Do shift operation */ switch (op) { @@ -648,31 +662,25 @@ static inline void emit_a32_alu_i(const s8 dst, const u32 val, break; } - if (is_stacked(dst)) - emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst)), ctx); + arm_bpf_put_reg32(dst, rd, ctx); } /* dst = ~dst (64 bit) */ static inline void emit_a32_neg64(const s8 dst[], struct jit_ctx *ctx){ const s8 *tmp = bpf2a32[TMP_REG_1]; - s8 rd = is_stacked(dst_lo) ? tmp[1] : dst[1]; - s8 rm = is_stacked(dst_lo) ? tmp[0] : dst[0]; + s8 rd, rm; /* Setup Operand */ - if (is_stacked(dst_lo)) { - emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); - } + rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); + rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); /* Do Negate Operation */ emit(ARM_RSBS_I(rd, rd, 0), ctx); emit(ARM_RSC_I(rm, rm, 0), ctx); - if (is_stacked(dst_lo)) { - emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); - } + arm_bpf_put_reg32(dst_lo, rd, ctx); + arm_bpf_put_reg32(dst_hi, rm, ctx); } /* dst = dst << src */ @@ -680,18 +688,12 @@ static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[], struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; + s8 rt, rd, rm; /* Setup Operands */ - s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo; - s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo; - s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi; - - if (is_stacked(src_lo)) - emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx); - if (is_stacked(dst_lo)) { - emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); - } + rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); + rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); + rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); /* Do LSH operation */ emit(ARM_SUB_I(ARM_IP, rt, 32), ctx); @@ -701,13 +703,8 @@ static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[], emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd, SRTYPE_LSR, tmp2[0]), ctx); emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_ASL, rt), ctx); - if (is_stacked(dst_lo)) { - emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx); - } else { - emit(ARM_MOV_R(rd, ARM_LR), ctx); - emit(ARM_MOV_R(rm, ARM_IP), ctx); - } + arm_bpf_put_reg32(dst_lo, ARM_LR, ctx); + arm_bpf_put_reg32(dst_hi, ARM_IP, ctx); } /* dst = dst >> src (signed)*/ @@ -715,17 +712,12 @@ static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[], struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; + s8 rt, rd, rm; + /* Setup Operands */ - s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo; - s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo; - s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi; - - if (is_stacked(src_lo)) - emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx); - if (is_stacked(dst_lo)) { - emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); - } + rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); + rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); + rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); /* Do the ARSH operation */ emit(ARM_RSB_I(ARM_IP, rt, 32), ctx); @@ -735,13 +727,9 @@ static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[], _emit(ARM_COND_MI, ARM_B(0), ctx); emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASR, tmp2[0]), ctx); emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_ASR, rt), ctx); - if (is_stacked(dst_lo)) { - emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx); - } else { - emit(ARM_MOV_R(rd, ARM_LR), ctx); - emit(ARM_MOV_R(rm, ARM_IP), ctx); - } + + arm_bpf_put_reg32(dst_lo, ARM_LR, ctx); + arm_bpf_put_reg32(dst_hi, ARM_IP, ctx); } /* dst = dst >> src */ @@ -749,17 +737,12 @@ static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[], struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; + s8 rt, rd, rm; + /* Setup Operands */ - s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo; - s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo; - s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi; - - if (is_stacked(src_lo)) - emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx); - if (is_stacked(dst_lo)) { - emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); - } + rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); + rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); + rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); /* Do RSH operation */ emit(ARM_RSB_I(ARM_IP, rt, 32), ctx); @@ -768,13 +751,9 @@ static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[], emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASL, ARM_IP), ctx); emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_LSR, tmp2[0]), ctx); emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_LSR, rt), ctx); - if (is_stacked(dst_lo)) { - emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx); - } else { - emit(ARM_MOV_R(rd, ARM_LR), ctx); - emit(ARM_MOV_R(rm, ARM_IP), ctx); - } + + arm_bpf_put_reg32(dst_lo, ARM_LR, ctx); + arm_bpf_put_reg32(dst_hi, ARM_IP, ctx); } /* dst = dst << val */ @@ -782,14 +761,11 @@ static inline void emit_a32_lsh_i64(const s8 dst[], const u32 val, struct jit_ctx *ctx){ const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; - /* Setup operands */ - s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo; - s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi; + s8 rd, rm; - if (is_stacked(dst_lo)) { - emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); - } + /* Setup operands */ + rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); + rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); /* Do LSH operation */ if (val < 32) { @@ -804,10 +780,8 @@ static inline void emit_a32_lsh_i64(const s8 dst[], emit(ARM_EOR_R(rd, rd, rd), ctx); } - if (is_stacked(dst_lo)) { - emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); - } + arm_bpf_put_reg32(dst_lo, rd, ctx); + arm_bpf_put_reg32(dst_hi, rm, ctx); } /* dst = dst >> val */ @@ -815,14 +789,11 @@ static inline void emit_a32_rsh_i64(const s8 dst[], const u32 val, struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; - /* Setup operands */ - s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo; - s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi; + s8 rd, rm; - if (is_stacked(dst_lo)) { - emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); - } + /* Setup operands */ + rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); + rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); /* Do LSR operation */ if (val < 32) { @@ -837,10 +808,8 @@ static inline void emit_a32_rsh_i64(const s8 dst[], emit(ARM_MOV_I(rm, 0), ctx); } - if (is_stacked(dst_lo)) { - emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); - } + arm_bpf_put_reg32(dst_lo, rd, ctx); + arm_bpf_put_reg32(dst_hi, rm, ctx); } /* dst = dst >> val (signed) */ @@ -848,14 +817,11 @@ static inline void emit_a32_arsh_i64(const s8 dst[], const u32 val, struct jit_ctx *ctx){ const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; - /* Setup operands */ - s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo; - s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi; + s8 rd, rm; - if (is_stacked(dst_lo)) { - emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); - } + /* Setup operands */ + rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); + rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); /* Do ARSH operation */ if (val < 32) { @@ -870,30 +836,21 @@ static inline void emit_a32_arsh_i64(const s8 dst[], emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, 31), ctx); } - if (is_stacked(dst_lo)) { - emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); - } + arm_bpf_put_reg32(dst_lo, rd, ctx); + arm_bpf_put_reg32(dst_hi, rm, ctx); } static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[], struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; + s8 rd, rm, rt, rn; + /* Setup operands for multiplication */ - s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo; - s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi; - s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo; - s8 rn = is_stacked(src_lo) ? tmp2[0] : src_hi; - - if (is_stacked(dst_lo)) { - emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); - } - if (is_stacked(src_lo)) { - emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx); - emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_hi)), ctx); - } + rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); + rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); + rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); + rn = arm_bpf_get_reg32(src_hi, tmp2[0], ctx); /* Do Multiplication */ emit(ARM_MUL(ARM_IP, rd, rn), ctx); @@ -902,22 +859,18 @@ static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[], emit(ARM_UMULL(ARM_IP, rm, rd, rt), ctx); emit(ARM_ADD_R(rm, ARM_LR, rm), ctx); - if (is_stacked(dst_lo)) { - emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx); - } else { - emit(ARM_MOV_R(rd, ARM_IP), ctx); - } + + arm_bpf_put_reg32(dst_lo, ARM_IP, ctx); + arm_bpf_put_reg32(dst_hi, rm, ctx); } /* *(size *)(dst + off) = src */ static inline void emit_str_r(const s8 dst, const s8 src, const s32 off, struct jit_ctx *ctx, const u8 sz){ const s8 *tmp = bpf2a32[TMP_REG_1]; - s8 rd = is_stacked(dst) ? tmp[1] : dst; + s8 rd; - if (is_stacked(dst)) - emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx); + rd = arm_bpf_get_reg32(dst, tmp[1], ctx); if (off) { emit_a32_mov_i(tmp[0], off, ctx); emit(ARM_ADD_R(tmp[0], rd, tmp[0]), ctx); @@ -983,10 +936,9 @@ static inline void emit_ldx_r(const s8 dst[], const s8 src, emit(ARM_LDR_I(rd[0], rm, off + 4), ctx); break; } - if (is_stacked(dst_lo)) - emit(ARM_STR_I(rd[1], ARM_SP, STACK_VAR(dst_lo)), ctx); - if (is_stacked(dst_lo) && sz == BPF_DW) - emit(ARM_STR_I(rd[0], ARM_SP, STACK_VAR(dst_hi)), ctx); + arm_bpf_put_reg32(dst[1], rd[1], ctx); + if (sz == BPF_DW) + arm_bpf_put_reg32(dst[0], rd[0], ctx); } /* Arithmatic Operation */ @@ -1034,6 +986,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) #define cur_offset (ctx->idx - idx0) #define jmp_offset (out_offset - (cur_offset) - 2) u32 off, lo, hi; + s8 r_array, r_index, r_tc_lo, r_tc_hi; /* if (index >= array->map.max_entries) * goto out; @@ -1041,12 +994,12 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) off = offsetof(struct bpf_array, map.max_entries); /* array->map.max_entries */ emit_a32_mov_i(tmp[1], off, ctx); - emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r2[1])), ctx); - emit(ARM_LDR_R(tmp[1], tmp2[1], tmp[1]), ctx); + r_array = arm_bpf_get_reg32(r2[1], tmp2[1], ctx); + emit(ARM_LDR_R(tmp[1], r_array, tmp[1]), ctx); /* index is 32-bit for arrays */ - emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r3[1])), ctx); + r_index = arm_bpf_get_reg32(r3[1], tmp2[1], ctx); /* index >= array->map.max_entries */ - emit(ARM_CMP_R(tmp2[1], tmp[1]), ctx); + emit(ARM_CMP_R(r_index, tmp[1]), ctx); _emit(ARM_COND_CS, ARM_B(jmp_offset), ctx); /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) @@ -1055,15 +1008,15 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) */ lo = (u32)MAX_TAIL_CALL_CNT; hi = (u32)((u64)MAX_TAIL_CALL_CNT >> 32); - emit(ARM_LDR_I(tmp[1], ARM_SP, STACK_VAR(tcc[1])), ctx); - emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(tcc[0])), ctx); - emit(ARM_CMP_I(tmp[0], hi), ctx); - _emit(ARM_COND_EQ, ARM_CMP_I(tmp[1], lo), ctx); + r_tc_lo = arm_bpf_get_reg32(tcc[1], tmp[1], ctx); + r_tc_hi = arm_bpf_get_reg32(tcc[0], tmp[0], ctx); + emit(ARM_CMP_I(r_tc_hi, hi), ctx); + _emit(ARM_COND_EQ, ARM_CMP_I(r_tc_lo, lo), ctx); _emit(ARM_COND_HI, ARM_B(jmp_offset), ctx); - emit(ARM_ADDS_I(tmp[1], tmp[1], 1), ctx); - emit(ARM_ADC_I(tmp[0], tmp[0], 0), ctx); - emit(ARM_STR_I(tmp[1], ARM_SP, STACK_VAR(tcc[1])), ctx); - emit(ARM_STR_I(tmp[0], ARM_SP, STACK_VAR(tcc[0])), ctx); + emit(ARM_ADDS_I(r_tc_lo, r_tc_lo, 1), ctx); + emit(ARM_ADC_I(r_tc_hi, r_tc_hi, 0), ctx); + arm_bpf_put_reg32(tcc[1], r_tc_lo, ctx); + arm_bpf_put_reg32(tcc[0], r_tc_hi, ctx); /* prog = array->ptrs[index] * if (prog == NULL) @@ -1071,10 +1024,10 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) */ off = offsetof(struct bpf_array, ptrs); emit_a32_mov_i(tmp[1], off, ctx); - emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r2[1])), ctx); - emit(ARM_ADD_R(tmp[1], tmp2[1], tmp[1]), ctx); - emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r3[1])), ctx); - emit(ARM_MOV_SI(tmp[0], tmp2[1], SRTYPE_ASL, 2), ctx); + r_array = arm_bpf_get_reg32(r2[1], tmp2[1], ctx); + emit(ARM_ADD_R(tmp[1], r_array, tmp[1]), ctx); + r_index = arm_bpf_get_reg32(r3[1], tmp2[1], ctx); + emit(ARM_MOV_SI(tmp[0], r_index, SRTYPE_ASL, 2), ctx); emit(ARM_LDR_R(tmp[1], tmp[1], tmp[0]), ctx); emit(ARM_CMP_I(tmp[1], 0), ctx); _emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx); @@ -1317,15 +1270,10 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) case BPF_ALU | BPF_DIV | BPF_X: case BPF_ALU | BPF_MOD | BPF_K: case BPF_ALU | BPF_MOD | BPF_X: - rd = is_stacked(dst_lo) ? tmp2[1] : dst_lo; - if (is_stacked(dst_lo)) - emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); + rd = arm_bpf_get_reg32(dst_lo, tmp2[1], ctx); switch (BPF_SRC(code)) { case BPF_X: - rt = is_stacked(rt) ? tmp2[0] : src_lo; - if (is_stacked(src_lo)) - emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), - ctx); + rt = arm_bpf_get_reg32(src_lo, tmp2[0], ctx); break; case BPF_K: rt = tmp2[0]; @@ -1336,8 +1284,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) break; } emit_udivmod(rd, rd, rt, ctx, BPF_OP(code)); - if (is_stacked(dst_lo)) - emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx); + arm_bpf_put_reg32(dst_lo, rd, ctx); emit_a32_mov_i(dst_hi, 0, ctx); break; case BPF_ALU64 | BPF_DIV | BPF_K: @@ -1417,12 +1364,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) /* dst = htobe(dst) */ case BPF_ALU | BPF_END | BPF_FROM_LE: case BPF_ALU | BPF_END | BPF_FROM_BE: - rd = is_stacked(dst_lo) ? tmp[0] : dst_hi; - rt = is_stacked(dst_lo) ? tmp[1] : dst_lo; - if (is_stacked(dst_lo)) { - emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx); - } + rt = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); + rd = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); if (BPF_SRC(code) == BPF_FROM_LE) goto emit_bswap_uxt; switch (imm) { @@ -1460,10 +1403,8 @@ emit_bswap_uxt: break; } exit: - if (is_stacked(dst_lo)) { - emit(ARM_STR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx); - } + arm_bpf_put_reg32(dst_lo, rt, ctx); + arm_bpf_put_reg32(dst_hi, rd, ctx); break; /* dst = imm64 */ case BPF_LD | BPF_IMM | BPF_DW: @@ -1482,9 +1423,7 @@ exit: case BPF_LDX | BPF_MEM | BPF_H: case BPF_LDX | BPF_MEM | BPF_B: case BPF_LDX | BPF_MEM | BPF_DW: - rn = is_stacked(src_lo) ? tmp2[1] : src_lo; - if (is_stacked(src_lo)) - emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx); + rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); emit_ldx_r(dst, rn, off, ctx, BPF_SIZE(code)); break; /* ST: *(size *)(dst + off) = imm */ @@ -1520,12 +1459,8 @@ exit: { u8 sz = BPF_SIZE(code); - rn = is_stacked(src_lo) ? tmp2[1] : src_lo; - rm = is_stacked(src_lo) ? tmp2[0] : src_hi; - if (is_stacked(src_lo)) { - emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx); - emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(src_hi)), ctx); - } + rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); + rm = arm_bpf_get_reg32(src_hi, tmp2[0], ctx); /* Store the value */ if (BPF_SIZE(code) == BPF_DW) { @@ -1559,12 +1494,8 @@ exit: case BPF_JMP | BPF_JSLT | BPF_X: case BPF_JMP | BPF_JSLE | BPF_X: /* Setup source registers */ - rm = is_stacked(src_lo) ? tmp2[0] : src_hi; - rn = is_stacked(src_lo) ? tmp2[1] : src_lo; - if (is_stacked(src_lo)) { - emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx); - emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(src_hi)), ctx); - } + rm = arm_bpf_get_reg32(src_hi, tmp2[0], ctx); + rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); goto go_jmp; /* PC += off if dst == imm */ /* PC += off if dst > imm */ @@ -1596,12 +1527,8 @@ exit: emit_a32_mov_i64(true, tmp2, imm, ctx); go_jmp: /* Setup destination register */ - rd = is_stacked(dst_lo) ? tmp[0] : dst_hi; - rt = is_stacked(dst_lo) ? tmp[1] : dst_lo; - if (is_stacked(dst_lo)) { - emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx); - emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx); - } + rt = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); + rd = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); /* Check for the condition */ emit_ar_r(rd, rt, rm, rn, ctx, BPF_OP(code)); -- cgit v1.2.3 From a6eccac507e5e4aed63fb23320fcadeb253c2af6 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 11 Jul 2018 10:31:57 +0100 Subject: ARM: net: bpf: 64-bit accessor functions for BPF registers Provide a couple of 64-bit register accessors, and use them where appropriate Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 235 ++++++++++++++++++++++++---------------------- 1 file changed, 122 insertions(+), 113 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 08fb4eb285a2..45a3599e94a4 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -478,6 +478,17 @@ static s8 arm_bpf_get_reg32(s8 reg, s8 tmp, struct jit_ctx *ctx) return reg; } +static const s8 *arm_bpf_get_reg64(const s8 *reg, const s8 *tmp, + struct jit_ctx *ctx) +{ + if (is_stacked(reg[1])) { + emit(ARM_LDR_I(tmp[1], ARM_SP, STACK_VAR(reg[1])), ctx); + emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(reg[0])), ctx); + reg = tmp; + } + return reg; +} + /* If a BPF register is on the stack (stk is true), save the register * back to the stack. If the source register is not the same, then * move it into the correct register. @@ -490,6 +501,20 @@ static void arm_bpf_put_reg32(s8 reg, s8 src, struct jit_ctx *ctx) emit(ARM_MOV_R(reg, src), ctx); } +static void arm_bpf_put_reg64(const s8 *reg, const s8 *src, + struct jit_ctx *ctx) +{ + if (is_stacked(reg[1])) { + emit(ARM_STR_I(src[1], ARM_SP, STACK_VAR(reg[1])), ctx); + emit(ARM_STR_I(src[0], ARM_SP, STACK_VAR(reg[0])), ctx); + } else { + if (reg[1] != src[1]) + emit(ARM_MOV_R(reg[1], src[1]), ctx); + if (reg[0] != src[0]) + emit(ARM_MOV_R(reg[0], src[0]), ctx); + } +} + static inline void emit_a32_mov_i(const s8 dst, const u32 val, struct jit_ctx *ctx) { @@ -669,18 +694,16 @@ static inline void emit_a32_alu_i(const s8 dst, const u32 val, static inline void emit_a32_neg64(const s8 dst[], struct jit_ctx *ctx){ const s8 *tmp = bpf2a32[TMP_REG_1]; - s8 rd, rm; + const s8 *rd; /* Setup Operand */ - rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); - rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); + rd = arm_bpf_get_reg64(dst, tmp, ctx); /* Do Negate Operation */ - emit(ARM_RSBS_I(rd, rd, 0), ctx); - emit(ARM_RSC_I(rm, rm, 0), ctx); + emit(ARM_RSBS_I(rd[1], rd[1], 0), ctx); + emit(ARM_RSC_I(rd[0], rd[0], 0), ctx); - arm_bpf_put_reg32(dst_lo, rd, ctx); - arm_bpf_put_reg32(dst_hi, rm, ctx); + arm_bpf_put_reg64(dst, rd, ctx); } /* dst = dst << src */ @@ -688,20 +711,20 @@ static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[], struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; - s8 rt, rd, rm; + const s8 *rd; + s8 rt; /* Setup Operands */ rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); - rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); - rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); + rd = arm_bpf_get_reg64(dst, tmp, ctx); /* Do LSH operation */ emit(ARM_SUB_I(ARM_IP, rt, 32), ctx); emit(ARM_RSB_I(tmp2[0], rt, 32), ctx); - emit(ARM_MOV_SR(ARM_LR, rm, SRTYPE_ASL, rt), ctx); - emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd, SRTYPE_ASL, ARM_IP), ctx); - emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd, SRTYPE_LSR, tmp2[0]), ctx); - emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_ASL, rt), ctx); + emit(ARM_MOV_SR(ARM_LR, rd[0], SRTYPE_ASL, rt), ctx); + emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[1], SRTYPE_ASL, ARM_IP), ctx); + emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd[1], SRTYPE_LSR, tmp2[0]), ctx); + emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_ASL, rt), ctx); arm_bpf_put_reg32(dst_lo, ARM_LR, ctx); arm_bpf_put_reg32(dst_hi, ARM_IP, ctx); @@ -712,21 +735,21 @@ static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[], struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; - s8 rt, rd, rm; + const s8 *rd; + s8 rt; /* Setup Operands */ rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); - rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); - rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); + rd = arm_bpf_get_reg64(dst, tmp, ctx); /* Do the ARSH operation */ emit(ARM_RSB_I(ARM_IP, rt, 32), ctx); emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx); - emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_LSR, rt), ctx); - emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASL, ARM_IP), ctx); + emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx); + emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx); _emit(ARM_COND_MI, ARM_B(0), ctx); - emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASR, tmp2[0]), ctx); - emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_ASR, rt), ctx); + emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASR, tmp2[0]), ctx); + emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_ASR, rt), ctx); arm_bpf_put_reg32(dst_lo, ARM_LR, ctx); arm_bpf_put_reg32(dst_hi, ARM_IP, ctx); @@ -737,20 +760,20 @@ static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[], struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; - s8 rt, rd, rm; + const s8 *rd; + s8 rt; /* Setup Operands */ rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); - rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); - rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); + rd = arm_bpf_get_reg64(dst, tmp, ctx); /* Do RSH operation */ emit(ARM_RSB_I(ARM_IP, rt, 32), ctx); emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx); - emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_LSR, rt), ctx); - emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASL, ARM_IP), ctx); - emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_LSR, tmp2[0]), ctx); - emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_LSR, rt), ctx); + emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx); + emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx); + emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_LSR, tmp2[0]), ctx); + emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_LSR, rt), ctx); arm_bpf_put_reg32(dst_lo, ARM_LR, ctx); arm_bpf_put_reg32(dst_hi, ARM_IP, ctx); @@ -761,27 +784,25 @@ static inline void emit_a32_lsh_i64(const s8 dst[], const u32 val, struct jit_ctx *ctx){ const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; - s8 rd, rm; + const s8 *rd; /* Setup operands */ - rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); - rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); + rd = arm_bpf_get_reg64(dst, tmp, ctx); /* Do LSH operation */ if (val < 32) { - emit(ARM_MOV_SI(tmp2[0], rm, SRTYPE_ASL, val), ctx); - emit(ARM_ORR_SI(rm, tmp2[0], rd, SRTYPE_LSR, 32 - val), ctx); - emit(ARM_MOV_SI(rd, rd, SRTYPE_ASL, val), ctx); + emit(ARM_MOV_SI(tmp2[0], rd[0], SRTYPE_ASL, val), ctx); + emit(ARM_ORR_SI(rd[0], tmp2[0], rd[1], SRTYPE_LSR, 32 - val), ctx); + emit(ARM_MOV_SI(rd[1], rd[1], SRTYPE_ASL, val), ctx); } else { if (val == 32) - emit(ARM_MOV_R(rm, rd), ctx); + emit(ARM_MOV_R(rd[0], rd[1]), ctx); else - emit(ARM_MOV_SI(rm, rd, SRTYPE_ASL, val - 32), ctx); - emit(ARM_EOR_R(rd, rd, rd), ctx); + emit(ARM_MOV_SI(rd[0], rd[1], SRTYPE_ASL, val - 32), ctx); + emit(ARM_EOR_R(rd[1], rd[1], rd[1]), ctx); } - arm_bpf_put_reg32(dst_lo, rd, ctx); - arm_bpf_put_reg32(dst_hi, rm, ctx); + arm_bpf_put_reg64(dst, rd, ctx); } /* dst = dst >> val */ @@ -789,27 +810,25 @@ static inline void emit_a32_rsh_i64(const s8 dst[], const u32 val, struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; - s8 rd, rm; + const s8 *rd; /* Setup operands */ - rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); - rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); + rd = arm_bpf_get_reg64(dst, tmp, ctx); /* Do LSR operation */ if (val < 32) { - emit(ARM_MOV_SI(tmp2[1], rd, SRTYPE_LSR, val), ctx); - emit(ARM_ORR_SI(rd, tmp2[1], rm, SRTYPE_ASL, 32 - val), ctx); - emit(ARM_MOV_SI(rm, rm, SRTYPE_LSR, val), ctx); + emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx); + emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx); + emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_LSR, val), ctx); } else if (val == 32) { - emit(ARM_MOV_R(rd, rm), ctx); - emit(ARM_MOV_I(rm, 0), ctx); + emit(ARM_MOV_R(rd[1], rd[0]), ctx); + emit(ARM_MOV_I(rd[0], 0), ctx); } else { - emit(ARM_MOV_SI(rd, rm, SRTYPE_LSR, val - 32), ctx); - emit(ARM_MOV_I(rm, 0), ctx); + emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_LSR, val - 32), ctx); + emit(ARM_MOV_I(rd[0], 0), ctx); } - arm_bpf_put_reg32(dst_lo, rd, ctx); - arm_bpf_put_reg32(dst_hi, rm, ctx); + arm_bpf_put_reg64(dst, rd, ctx); } /* dst = dst >> val (signed) */ @@ -817,51 +836,47 @@ static inline void emit_a32_arsh_i64(const s8 dst[], const u32 val, struct jit_ctx *ctx){ const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; - s8 rd, rm; + const s8 *rd; /* Setup operands */ - rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); - rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); + rd = arm_bpf_get_reg64(dst, tmp, ctx); /* Do ARSH operation */ if (val < 32) { - emit(ARM_MOV_SI(tmp2[1], rd, SRTYPE_LSR, val), ctx); - emit(ARM_ORR_SI(rd, tmp2[1], rm, SRTYPE_ASL, 32 - val), ctx); - emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, val), ctx); + emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx); + emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx); + emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, val), ctx); } else if (val == 32) { - emit(ARM_MOV_R(rd, rm), ctx); - emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, 31), ctx); + emit(ARM_MOV_R(rd[1], rd[0]), ctx); + emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx); } else { - emit(ARM_MOV_SI(rd, rm, SRTYPE_ASR, val - 32), ctx); - emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, 31), ctx); + emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_ASR, val - 32), ctx); + emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx); } - arm_bpf_put_reg32(dst_lo, rd, ctx); - arm_bpf_put_reg32(dst_hi, rm, ctx); + arm_bpf_put_reg64(dst, rd, ctx); } static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[], struct jit_ctx *ctx) { const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; - s8 rd, rm, rt, rn; + const s8 *rd, *rt; /* Setup operands for multiplication */ - rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); - rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); - rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); - rn = arm_bpf_get_reg32(src_hi, tmp2[0], ctx); + rd = arm_bpf_get_reg64(dst, tmp, ctx); + rt = arm_bpf_get_reg64(src, tmp2, ctx); /* Do Multiplication */ - emit(ARM_MUL(ARM_IP, rd, rn), ctx); - emit(ARM_MUL(ARM_LR, rm, rt), ctx); + emit(ARM_MUL(ARM_IP, rd[1], rt[0]), ctx); + emit(ARM_MUL(ARM_LR, rd[0], rt[1]), ctx); emit(ARM_ADD_R(ARM_LR, ARM_IP, ARM_LR), ctx); - emit(ARM_UMULL(ARM_IP, rm, rd, rt), ctx); - emit(ARM_ADD_R(rm, ARM_LR, rm), ctx); + emit(ARM_UMULL(ARM_IP, rd[0], rd[1], rt[1]), ctx); + emit(ARM_ADD_R(rd[0], ARM_LR, rd[0]), ctx); arm_bpf_put_reg32(dst_lo, ARM_IP, ctx); - arm_bpf_put_reg32(dst_hi, rm, ctx); + arm_bpf_put_reg32(dst_hi, rd[0], ctx); } /* *(size *)(dst + off) = src */ @@ -918,17 +933,17 @@ static inline void emit_ldx_r(const s8 dst[], const s8 src, case BPF_B: /* Load a Byte */ emit(ARM_LDRB_I(rd[1], rm, off), ctx); - emit_a32_mov_i(dst[0], 0, ctx); + emit_a32_mov_i(rd[0], 0, ctx); break; case BPF_H: /* Load a HalfWord */ emit(ARM_LDRH_I(rd[1], rm, off), ctx); - emit_a32_mov_i(dst[0], 0, ctx); + emit_a32_mov_i(rd[0], 0, ctx); break; case BPF_W: /* Load a Word */ emit(ARM_LDR_I(rd[1], rm, off), ctx); - emit_a32_mov_i(dst[0], 0, ctx); + emit_a32_mov_i(rd[0], 0, ctx); break; case BPF_DW: /* Load a Double Word */ @@ -936,9 +951,7 @@ static inline void emit_ldx_r(const s8 dst[], const s8 src, emit(ARM_LDR_I(rd[0], rm, off + 4), ctx); break; } - arm_bpf_put_reg32(dst[1], rd[1], ctx); - if (sz == BPF_DW) - arm_bpf_put_reg32(dst[0], rd[0], ctx); + arm_bpf_put_reg64(dst, rd, ctx); } /* Arithmatic Operation */ @@ -982,11 +995,12 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) const s8 *tmp = bpf2a32[TMP_REG_1]; const s8 *tmp2 = bpf2a32[TMP_REG_2]; const s8 *tcc = bpf2a32[TCALL_CNT]; + const s8 *tc; const int idx0 = ctx->idx; #define cur_offset (ctx->idx - idx0) #define jmp_offset (out_offset - (cur_offset) - 2) u32 off, lo, hi; - s8 r_array, r_index, r_tc_lo, r_tc_hi; + s8 r_array, r_index; /* if (index >= array->map.max_entries) * goto out; @@ -1008,15 +1022,13 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) */ lo = (u32)MAX_TAIL_CALL_CNT; hi = (u32)((u64)MAX_TAIL_CALL_CNT >> 32); - r_tc_lo = arm_bpf_get_reg32(tcc[1], tmp[1], ctx); - r_tc_hi = arm_bpf_get_reg32(tcc[0], tmp[0], ctx); - emit(ARM_CMP_I(r_tc_hi, hi), ctx); - _emit(ARM_COND_EQ, ARM_CMP_I(r_tc_lo, lo), ctx); + tc = arm_bpf_get_reg64(tcc, tmp, ctx); + emit(ARM_CMP_I(tc[0], hi), ctx); + _emit(ARM_COND_EQ, ARM_CMP_I(tc[1], lo), ctx); _emit(ARM_COND_HI, ARM_B(jmp_offset), ctx); - emit(ARM_ADDS_I(r_tc_lo, r_tc_lo, 1), ctx); - emit(ARM_ADC_I(r_tc_hi, r_tc_hi, 0), ctx); - arm_bpf_put_reg32(tcc[1], r_tc_lo, ctx); - arm_bpf_put_reg32(tcc[0], r_tc_hi, ctx); + emit(ARM_ADDS_I(tc[1], tc[1], 1), ctx); + emit(ARM_ADC_I(tc[0], tc[0], 0), ctx); + arm_bpf_put_reg64(tcc, tmp, ctx); /* prog = array->ptrs[index] * if (prog == NULL) @@ -1183,7 +1195,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) const s32 imm = insn->imm; const int i = insn - ctx->prog->insnsi; const bool is64 = BPF_CLASS(code) == BPF_ALU64; - s8 rd, rt, rm, rn; + const s8 *rd, *rs; + s8 rd_lo, rt, rm, rn; s32 jmp_offset; #define check_imm(bits, imm) do { \ @@ -1270,7 +1283,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) case BPF_ALU | BPF_DIV | BPF_X: case BPF_ALU | BPF_MOD | BPF_K: case BPF_ALU | BPF_MOD | BPF_X: - rd = arm_bpf_get_reg32(dst_lo, tmp2[1], ctx); + rd_lo = arm_bpf_get_reg32(dst_lo, tmp2[1], ctx); switch (BPF_SRC(code)) { case BPF_X: rt = arm_bpf_get_reg32(src_lo, tmp2[0], ctx); @@ -1283,8 +1296,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) rt = src_lo; break; } - emit_udivmod(rd, rd, rt, ctx, BPF_OP(code)); - arm_bpf_put_reg32(dst_lo, rd, ctx); + emit_udivmod(rd_lo, rd_lo, rt, ctx, BPF_OP(code)); + arm_bpf_put_reg32(dst_lo, rd_lo, ctx); emit_a32_mov_i(dst_hi, 0, ctx); break; case BPF_ALU64 | BPF_DIV | BPF_K: @@ -1364,21 +1377,20 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) /* dst = htobe(dst) */ case BPF_ALU | BPF_END | BPF_FROM_LE: case BPF_ALU | BPF_END | BPF_FROM_BE: - rt = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); - rd = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); + rd = arm_bpf_get_reg64(dst, tmp, ctx); if (BPF_SRC(code) == BPF_FROM_LE) goto emit_bswap_uxt; switch (imm) { case 16: - emit_rev16(rt, rt, ctx); + emit_rev16(rd[1], rd[1], ctx); goto emit_bswap_uxt; case 32: - emit_rev32(rt, rt, ctx); + emit_rev32(rd[1], rd[1], ctx); goto emit_bswap_uxt; case 64: - emit_rev32(ARM_LR, rt, ctx); - emit_rev32(rt, rd, ctx); - emit(ARM_MOV_R(rd, ARM_LR), ctx); + emit_rev32(ARM_LR, rd[1], ctx); + emit_rev32(rd[1], rd[0], ctx); + emit(ARM_MOV_R(rd[0], ARM_LR), ctx); break; } goto exit; @@ -1388,23 +1400,22 @@ emit_bswap_uxt: /* zero-extend 16 bits into 64 bits */ #if __LINUX_ARM_ARCH__ < 6 emit_a32_mov_i(tmp2[1], 0xffff, ctx); - emit(ARM_AND_R(rt, rt, tmp2[1]), ctx); + emit(ARM_AND_R(rd[1], rd[1], tmp2[1]), ctx); #else /* ARMv6+ */ - emit(ARM_UXTH(rt, rt), ctx); + emit(ARM_UXTH(rd[1], rd[1]), ctx); #endif - emit(ARM_EOR_R(rd, rd, rd), ctx); + emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx); break; case 32: /* zero-extend 32 bits into 64 bits */ - emit(ARM_EOR_R(rd, rd, rd), ctx); + emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx); break; case 64: /* nop */ break; } exit: - arm_bpf_put_reg32(dst_lo, rt, ctx); - arm_bpf_put_reg32(dst_hi, rd, ctx); + arm_bpf_put_reg64(dst, rd, ctx); break; /* dst = imm64 */ case BPF_LD | BPF_IMM | BPF_DW: @@ -1459,15 +1470,14 @@ exit: { u8 sz = BPF_SIZE(code); - rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); - rm = arm_bpf_get_reg32(src_hi, tmp2[0], ctx); + rs = arm_bpf_get_reg64(src, tmp2, ctx); /* Store the value */ if (BPF_SIZE(code) == BPF_DW) { - emit_str_r(dst_lo, rn, off, ctx, BPF_W); - emit_str_r(dst_lo, rm, off+4, ctx, BPF_W); + emit_str_r(dst_lo, rs[1], off, ctx, BPF_W); + emit_str_r(dst_lo, rs[0], off+4, ctx, BPF_W); } else { - emit_str_r(dst_lo, rn, off, ctx, sz); + emit_str_r(dst_lo, rs[1], off, ctx, sz); } break; } @@ -1527,11 +1537,10 @@ exit: emit_a32_mov_i64(true, tmp2, imm, ctx); go_jmp: /* Setup destination register */ - rt = arm_bpf_get_reg32(dst_lo, tmp[1], ctx); - rd = arm_bpf_get_reg32(dst_hi, tmp[0], ctx); + rd = arm_bpf_get_reg64(dst, tmp, ctx); /* Check for the condition */ - emit_ar_r(rd, rt, rm, rn, ctx, BPF_OP(code)); + emit_ar_r(rd[0], rd[1], rm, rn, ctx, BPF_OP(code)); /* Setup JUMP instruction */ jmp_offset = bpf2a32_offset(i+off, i, ctx); -- cgit v1.2.3 From 96cced4e774a2728710c8f8f48441fc7b29d6177 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 11 Jul 2018 10:32:02 +0100 Subject: ARM: net: bpf: access eBPF scratch space using ARM FP register Access the eBPF scratch space using the frame pointer rather than our stack pointer, as the offsets from the ARM frame pointer are constant across all eBPF programs. Since we no longer reference the scratch space registers from the stack pointer, this simplifies emit_push_r64() as it no longer needs to know how many words are pushed onto the stack. Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 45a3599e94a4..753b5b2b2e3d 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -108,6 +108,12 @@ enum { #define STACK_OFFSET(k) (-4 - (k) * 4) #define SCRATCH_SIZE (BPF_JIT_SCRATCH_REGS * 4) +#ifdef CONFIG_FRAME_POINTER +#define EBPF_SCRATCH_TO_ARM_FP(x) ((x) - 4 * hweight16(CALLEE_PUSH_MASK) - 4) +#else +#define EBPF_SCRATCH_TO_ARM_FP(x) (x) +#endif + #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) /* TEMP Register 1 */ #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) /* TEMP Register 2 */ #define TCALL_CNT (MAX_BPF_JIT_REG + 2) /* Tail Call Count */ @@ -294,9 +300,6 @@ static void jit_fill_hole(void *area, unsigned int size) #define _STACK_SIZE (ctx->prog->aux->stack_depth + SCRATCH_SIZE) #define STACK_SIZE ALIGN(_STACK_SIZE, STACK_ALIGNMENT) -/* Get the offset of eBPF REGISTERs stored on scratch space. */ -#define STACK_VAR(off) (STACK_SIZE + (off)) - #if __LINUX_ARM_ARCH__ < 7 static u16 imm_offset(u32 k, struct jit_ctx *ctx) @@ -472,7 +475,7 @@ static bool is_stacked(s8 reg) static s8 arm_bpf_get_reg32(s8 reg, s8 tmp, struct jit_ctx *ctx) { if (is_stacked(reg)) { - emit(ARM_LDR_I(tmp, ARM_SP, STACK_VAR(reg)), ctx); + emit(ARM_LDR_I(tmp, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx); reg = tmp; } return reg; @@ -482,8 +485,10 @@ static const s8 *arm_bpf_get_reg64(const s8 *reg, const s8 *tmp, struct jit_ctx *ctx) { if (is_stacked(reg[1])) { - emit(ARM_LDR_I(tmp[1], ARM_SP, STACK_VAR(reg[1])), ctx); - emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(reg[0])), ctx); + emit(ARM_LDR_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg[1])), + ctx); + emit(ARM_LDR_I(tmp[0], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg[0])), + ctx); reg = tmp; } return reg; @@ -496,7 +501,7 @@ static const s8 *arm_bpf_get_reg64(const s8 *reg, const s8 *tmp, static void arm_bpf_put_reg32(s8 reg, s8 src, struct jit_ctx *ctx) { if (is_stacked(reg)) - emit(ARM_STR_I(src, ARM_SP, STACK_VAR(reg)), ctx); + emit(ARM_STR_I(src, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx); else if (reg != src) emit(ARM_MOV_R(reg, src), ctx); } @@ -505,8 +510,10 @@ static void arm_bpf_put_reg64(const s8 *reg, const s8 *src, struct jit_ctx *ctx) { if (is_stacked(reg[1])) { - emit(ARM_STR_I(src[1], ARM_SP, STACK_VAR(reg[1])), ctx); - emit(ARM_STR_I(src[0], ARM_SP, STACK_VAR(reg[0])), ctx); + emit(ARM_STR_I(src[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg[1])), + ctx); + emit(ARM_STR_I(src[0], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg[0])), + ctx); } else { if (reg[1] != src[1]) emit(ARM_MOV_R(reg[1], src[1]), ctx); @@ -1103,16 +1110,15 @@ static inline void emit_rev32(const u8 rd, const u8 rn, struct jit_ctx *ctx) } // push the scratch stack register on top of the stack -static inline void emit_push_r64(const s8 src[], const u8 shift, - struct jit_ctx *ctx) +static inline void emit_push_r64(const s8 src[], struct jit_ctx *ctx) { const s8 *tmp2 = bpf2a32[TMP_REG_2]; + const s8 *rt; u16 reg_set = 0; - emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(src[1]+shift)), ctx); - emit(ARM_LDR_I(tmp2[0], ARM_SP, STACK_VAR(src[0]+shift)), ctx); + rt = arm_bpf_get_reg64(src, tmp2, ctx); - reg_set = (1 << tmp2[1]) | (1 << tmp2[0]); + reg_set = (1 << rt[1]) | (1 << rt[0]); emit(ARM_PUSH(reg_set), ctx); } @@ -1155,8 +1161,8 @@ static void build_prologue(struct jit_ctx *ctx) emit(ARM_MOV_R(r3, r4), ctx); emit(ARM_MOV_R(r2, r0), ctx); /* Initialize Tail Count */ - emit(ARM_STR_I(r4, ARM_SP, STACK_VAR(tcc[0])), ctx); - emit(ARM_STR_I(r4, ARM_SP, STACK_VAR(tcc[1])), ctx); + emit(ARM_STR_I(r4, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(tcc[0])), ctx); + emit(ARM_STR_I(r4, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(tcc[1])), ctx); /* end of prologue */ } @@ -1606,9 +1612,9 @@ go_jmp: emit_a32_mov_r64(true, r0, r1, ctx); emit_a32_mov_r64(true, r1, r2, ctx); - emit_push_r64(r5, 0, ctx); - emit_push_r64(r4, 8, ctx); - emit_push_r64(r3, 16, ctx); + emit_push_r64(r5, ctx); + emit_push_r64(r4, ctx); + emit_push_r64(r3, ctx); emit_a32_mov_i(tmp[1], func, ctx); emit_blx_r(tmp[1], ctx); -- cgit v1.2.3 From 1ca3b17b777c4136f9dba0195e13502c445f4ade Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 11 Jul 2018 10:32:07 +0100 Subject: ARM: net: bpf: imm12 constant conversion Provide a version of the imm8m() function that the compiler can optimise when used with a constant expression. Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 753b5b2b2e3d..2cc66aa44dfe 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -236,10 +236,56 @@ static inline void emit(u32 inst, struct jit_ctx *ctx) _emit(ARM_COND_AL, inst, ctx); } +/* + * This is rather horrid, but necessary to convert an integer constant + * to an immediate operand for the opcodes, and be able to detect at + * build time whether the constant can't be converted (iow, usable in + * BUILD_BUG_ON()). + */ +#define imm12val(v, s) (rol32(v, (s)) | (s) << 7) +#define const_imm8m(x) \ + ({ int r; \ + u32 v = (x); \ + if (!(v & ~0x000000ff)) \ + r = imm12val(v, 0); \ + else if (!(v & ~0xc000003f)) \ + r = imm12val(v, 2); \ + else if (!(v & ~0xf000000f)) \ + r = imm12val(v, 4); \ + else if (!(v & ~0xfc000003)) \ + r = imm12val(v, 6); \ + else if (!(v & ~0xff000000)) \ + r = imm12val(v, 8); \ + else if (!(v & ~0x3fc00000)) \ + r = imm12val(v, 10); \ + else if (!(v & ~0x0ff00000)) \ + r = imm12val(v, 12); \ + else if (!(v & ~0x03fc0000)) \ + r = imm12val(v, 14); \ + else if (!(v & ~0x00ff0000)) \ + r = imm12val(v, 16); \ + else if (!(v & ~0x003fc000)) \ + r = imm12val(v, 18); \ + else if (!(v & ~0x000ff000)) \ + r = imm12val(v, 20); \ + else if (!(v & ~0x0003fc00)) \ + r = imm12val(v, 22); \ + else if (!(v & ~0x0000ff00)) \ + r = imm12val(v, 24); \ + else if (!(v & ~0x00003fc0)) \ + r = imm12val(v, 26); \ + else if (!(v & ~0x00000ff0)) \ + r = imm12val(v, 28); \ + else if (!(v & ~0x000003fc)) \ + r = imm12val(v, 30); \ + else \ + r = -1; \ + r; }) + /* * Checks if immediate value can be converted to imm12(12 bits) value. */ -static int16_t imm8m(u32 x) +static int imm8m(u32 x) { u32 rot; @@ -249,6 +295,8 @@ static int16_t imm8m(u32 x) return -1; } +#define imm8m(x) (__builtin_constant_p(x) ? const_imm8m(x) : imm8m(x)) + static u32 arm_bpf_ldst_imm12(u32 op, u8 rt, u8 rn, s16 imm12) { op |= rt << 12 | rn << 16; -- cgit v1.2.3 From 828e2b90e8e9b5bd844a25f22ceeb8df4dd18b07 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 11 Jul 2018 10:32:12 +0100 Subject: ARM: net: bpf: use immediate forms of instructions where possible Rather than moving constants to a register and then using them in a subsequent instruction, use them directly in the desired instruction cutting out the "middle" register. This removes two instructions from the tail call code path. Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 21 ++++++++++++--------- arch/arm/net/bpf_jit_32.h | 1 + 2 files changed, 13 insertions(+), 9 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 2cc66aa44dfe..645653e1931e 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -304,7 +304,7 @@ static u32 arm_bpf_ldst_imm12(u32 op, u8 rt, u8 rn, s16 imm12) op |= ARM_INST_LDST__U; else imm12 = -imm12; - return op | (imm12 & 0xfff); + return op | (imm12 & ARM_INST_LDST__IMM12); } static u32 arm_bpf_ldst_imm8(u32 op, u8 rt, u8 rn, s16 imm8) @@ -1054,17 +1054,19 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) const int idx0 = ctx->idx; #define cur_offset (ctx->idx - idx0) #define jmp_offset (out_offset - (cur_offset) - 2) - u32 off, lo, hi; + u32 lo, hi; s8 r_array, r_index; + int off; /* if (index >= array->map.max_entries) * goto out; */ + BUILD_BUG_ON(offsetof(struct bpf_array, map.max_entries) > + ARM_INST_LDST__IMM12); off = offsetof(struct bpf_array, map.max_entries); /* array->map.max_entries */ - emit_a32_mov_i(tmp[1], off, ctx); r_array = arm_bpf_get_reg32(r2[1], tmp2[1], ctx); - emit(ARM_LDR_R(tmp[1], r_array, tmp[1]), ctx); + emit(ARM_LDR_I(tmp[1], r_array, off), ctx); /* index is 32-bit for arrays */ r_index = arm_bpf_get_reg32(r3[1], tmp2[1], ctx); /* index >= array->map.max_entries */ @@ -1089,10 +1091,10 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) * if (prog == NULL) * goto out; */ - off = offsetof(struct bpf_array, ptrs); - emit_a32_mov_i(tmp[1], off, ctx); + BUILD_BUG_ON(imm8m(offsetof(struct bpf_array, ptrs)) < 0); + off = imm8m(offsetof(struct bpf_array, ptrs)); r_array = arm_bpf_get_reg32(r2[1], tmp2[1], ctx); - emit(ARM_ADD_R(tmp[1], r_array, tmp[1]), ctx); + emit(ARM_ADD_I(tmp[1], r_array, off), ctx); r_index = arm_bpf_get_reg32(r3[1], tmp2[1], ctx); emit(ARM_MOV_SI(tmp[0], r_index, SRTYPE_ASL, 2), ctx); emit(ARM_LDR_R(tmp[1], tmp[1], tmp[0]), ctx); @@ -1100,9 +1102,10 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) _emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx); /* goto *(prog->bpf_func + prologue_size); */ + BUILD_BUG_ON(offsetof(struct bpf_prog, bpf_func) > + ARM_INST_LDST__IMM12); off = offsetof(struct bpf_prog, bpf_func); - emit_a32_mov_i(tmp2[1], off, ctx); - emit(ARM_LDR_R(tmp[1], tmp[1], tmp2[1]), ctx); + emit(ARM_LDR_I(tmp[1], tmp[1], off), ctx); emit(ARM_ADD_I(tmp[1], tmp[1], ctx->prologue_bytes), ctx); emit_bx_r(tmp[1], ctx); diff --git a/arch/arm/net/bpf_jit_32.h b/arch/arm/net/bpf_jit_32.h index c55bc39d3e22..dee8a76fb0bc 100644 --- a/arch/arm/net/bpf_jit_32.h +++ b/arch/arm/net/bpf_jit_32.h @@ -78,6 +78,7 @@ #define ARM_INST_EOR_I 0x02200000 #define ARM_INST_LDST__U 0x00800000 +#define ARM_INST_LDST__IMM12 0x00000fff #define ARM_INST_LDRB_I 0x05500000 #define ARM_INST_LDRB_R 0x07d00000 #define ARM_INST_LDRH_I 0x015000b0 -- cgit v1.2.3 From 2b6958ef1151452cb2160fde75a5c5382b512c34 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 11 Jul 2018 10:32:17 +0100 Subject: ARM: net: bpf: use ldr instructions with shifted rm register Rather than pre-shifting the rm register for the ldr in the tail call, shift it in the load instruction. This eliminates one unnecessary instruction. Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 3 +-- arch/arm/net/bpf_jit_32.h | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 645653e1931e..e22dc828420c 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -1096,8 +1096,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) r_array = arm_bpf_get_reg32(r2[1], tmp2[1], ctx); emit(ARM_ADD_I(tmp[1], r_array, off), ctx); r_index = arm_bpf_get_reg32(r3[1], tmp2[1], ctx); - emit(ARM_MOV_SI(tmp[0], r_index, SRTYPE_ASL, 2), ctx); - emit(ARM_LDR_R(tmp[1], tmp[1], tmp[0]), ctx); + emit(ARM_LDR_R_SI(tmp[1], tmp[1], r_index, SRTYPE_ASL, 2), ctx); emit(ARM_CMP_I(tmp[1], 0), ctx); _emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx); diff --git a/arch/arm/net/bpf_jit_32.h b/arch/arm/net/bpf_jit_32.h index dee8a76fb0bc..e541a7a6139a 100644 --- a/arch/arm/net/bpf_jit_32.h +++ b/arch/arm/net/bpf_jit_32.h @@ -188,6 +188,10 @@ #define ARM_LDR_R(rt, rn, rm) (ARM_INST_LDR_R | ARM_INST_LDST__U \ | (rt) << 12 | (rn) << 16 \ | (rm)) +#define ARM_LDR_R_SI(rt, rn, rm, type, imm) \ + (ARM_INST_LDR_R | ARM_INST_LDST__U \ + | (rt) << 12 | (rn) << 16 \ + | (imm) << 7 | (type) << 5 | (rm)) #define ARM_LDRB_R(rt, rn, rm) (ARM_INST_LDRB_R | ARM_INST_LDST__U \ | (rt) << 12 | (rn) << 16 \ | (rm)) -- cgit v1.2.3 From aaffd2f5c3d58f154ca7b3d104a2ee6b6e40bc6b Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 11 Jul 2018 10:32:22 +0100 Subject: ARM: net: bpf: avoid reloading 'index' Avoid reloading 'index' after we have validated it - it remains in tmp2[1] up to the point that we begin the code to index the pointer array, so with a little rearrangement of the registers, we can use the already loaded value. Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index e22dc828420c..0a8b3d0903c4 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -1073,6 +1073,8 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) emit(ARM_CMP_R(r_index, tmp[1]), ctx); _emit(ARM_COND_CS, ARM_B(jmp_offset), ctx); + /* tmp2[1] = index */ + /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) * goto out; * tail_call_cnt++; @@ -1093,9 +1095,8 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) */ BUILD_BUG_ON(imm8m(offsetof(struct bpf_array, ptrs)) < 0); off = imm8m(offsetof(struct bpf_array, ptrs)); - r_array = arm_bpf_get_reg32(r2[1], tmp2[1], ctx); + r_array = arm_bpf_get_reg32(r2[1], tmp2[0], ctx); emit(ARM_ADD_I(tmp[1], r_array, off), ctx); - r_index = arm_bpf_get_reg32(r3[1], tmp2[1], ctx); emit(ARM_LDR_R_SI(tmp[1], tmp[1], r_index, SRTYPE_ASL, 2), ctx); emit(ARM_CMP_I(tmp[1], 0), ctx); _emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx); -- cgit v1.2.3 From b50452299864fbc00a576241e1490541c8754d50 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 11 Jul 2018 10:32:28 +0100 Subject: ARM: net: bpf: avoid reloading 'array' Rearranging the order of the initial tail call code a little allows is to avoid reloading the 'array' pointer. Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 0a8b3d0903c4..f0cad9692952 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -1064,16 +1064,16 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) BUILD_BUG_ON(offsetof(struct bpf_array, map.max_entries) > ARM_INST_LDST__IMM12); off = offsetof(struct bpf_array, map.max_entries); - /* array->map.max_entries */ - r_array = arm_bpf_get_reg32(r2[1], tmp2[1], ctx); - emit(ARM_LDR_I(tmp[1], r_array, off), ctx); + r_array = arm_bpf_get_reg32(r2[1], tmp2[0], ctx); /* index is 32-bit for arrays */ r_index = arm_bpf_get_reg32(r3[1], tmp2[1], ctx); + /* array->map.max_entries */ + emit(ARM_LDR_I(tmp[1], r_array, off), ctx); /* index >= array->map.max_entries */ emit(ARM_CMP_R(r_index, tmp[1]), ctx); _emit(ARM_COND_CS, ARM_B(jmp_offset), ctx); - /* tmp2[1] = index */ + /* tmp2[0] = array, tmp2[1] = index */ /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) * goto out; @@ -1095,7 +1095,6 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) */ BUILD_BUG_ON(imm8m(offsetof(struct bpf_array, ptrs)) < 0); off = imm8m(offsetof(struct bpf_array, ptrs)); - r_array = arm_bpf_get_reg32(r2[1], tmp2[0], ctx); emit(ARM_ADD_I(tmp[1], r_array, off), ctx); emit(ARM_LDR_R_SI(tmp[1], tmp[1], r_index, SRTYPE_ASL, 2), ctx); emit(ARM_CMP_I(tmp[1], 0), ctx); -- cgit v1.2.3 From bef8968df8a6e3eb91081d68affc64b8d87d5721 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 11 Jul 2018 10:32:33 +0100 Subject: ARM: net: bpf: always use odd/even register pair Always use an odd/even register pair for our 64-bit registers, so that we're able to use the double-word load/store instructions in the future. Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index f0cad9692952..006ff9615850 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -47,27 +47,27 @@ * The callee saved registers depends on whether frame pointers are enabled. * With frame pointers (to be compliant with the ABI): * - * high - * original ARM_SP => +------------------+ \ - * | pc | | - * current ARM_FP => +------------------+ } callee saved registers - * |r4-r8,r10,fp,ip,lr| | - * +------------------+ / - * low + * high + * original ARM_SP => +--------------+ \ + * | pc | | + * current ARM_FP => +--------------+ } callee saved registers + * |r4-r9,fp,ip,lr| | + * +--------------+ / + * low * * Without frame pointers: * - * high - * original ARM_SP => +------------------+ - * | r4-r8,r10,fp,lr | callee saved registers - * current ARM_FP => +------------------+ - * low + * high + * original ARM_SP => +--------------+ + * | r4-r9,fp,lr | callee saved registers + * current ARM_FP => +--------------+ + * low * * When popping registers off the stack at the end of a BPF function, we * reference them via the current ARM_FP register. */ #define CALLEE_MASK (1 << ARM_R4 | 1 << ARM_R5 | 1 << ARM_R6 | \ - 1 << ARM_R7 | 1 << ARM_R8 | 1 << ARM_R10 | \ + 1 << ARM_R7 | 1 << ARM_R8 | 1 << ARM_R9 | \ 1 << ARM_FP) #define CALLEE_PUSH_MASK (CALLEE_MASK | 1 << ARM_LR) #define CALLEE_POP_MASK (CALLEE_MASK | 1 << ARM_PC) @@ -157,7 +157,7 @@ static const s8 bpf2a32[][2] = { * for constant blindings and others. */ [TMP_REG_1] = {ARM_R7, ARM_R6}, - [TMP_REG_2] = {ARM_R10, ARM_R8}, + [TMP_REG_2] = {ARM_R9, ARM_R8}, /* Tail call count. Stored on stack scratch space. */ [TCALL_CNT] = {STACK_OFFSET(BPF_TC_HI), STACK_OFFSET(BPF_TC_LO)}, /* temporary register for blinding constants. -- cgit v1.2.3 From 8c9602d38c7262664c31332101f540c1e179797d Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 11 Jul 2018 10:32:38 +0100 Subject: ARM: net: bpf: use double-word load/stores where available Use double-word load and stores where support for this instruction is supported by the CPU architecture. Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 55 ++++++++++++++++++++++++++++++++++++----------- arch/arm/net/bpf_jit_32.h | 2 ++ 2 files changed, 45 insertions(+), 12 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 006ff9615850..a9f68a924800 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "bpf_jit_32.h" @@ -192,6 +193,7 @@ struct jit_ctx { unsigned int idx; unsigned int prologue_bytes; unsigned int epilogue_offset; + unsigned int cpu_architecture; u32 flags; u32 *offsets; u32 *target; @@ -319,10 +321,12 @@ static u32 arm_bpf_ldst_imm8(u32 op, u8 rt, u8 rn, s16 imm8) #define ARM_LDR_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_LDR_I, rt, rn, off) #define ARM_LDRB_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_LDRB_I, rt, rn, off) +#define ARM_LDRD_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRD_I, rt, rn, off) #define ARM_LDRH_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRH_I, rt, rn, off) #define ARM_STR_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_STR_I, rt, rn, off) #define ARM_STRB_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_STRB_I, rt, rn, off) +#define ARM_STRD_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_STRD_I, rt, rn, off) #define ARM_STRH_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_STRH_I, rt, rn, off) /* @@ -533,10 +537,16 @@ static const s8 *arm_bpf_get_reg64(const s8 *reg, const s8 *tmp, struct jit_ctx *ctx) { if (is_stacked(reg[1])) { - emit(ARM_LDR_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg[1])), - ctx); - emit(ARM_LDR_I(tmp[0], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg[0])), - ctx); + if (__LINUX_ARM_ARCH__ >= 6 || + ctx->cpu_architecture >= CPU_ARCH_ARMv5TE) { + emit(ARM_LDRD_I(tmp[1], ARM_FP, + EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); + } else { + emit(ARM_LDR_I(tmp[1], ARM_FP, + EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); + emit(ARM_LDR_I(tmp[0], ARM_FP, + EBPF_SCRATCH_TO_ARM_FP(reg[0])), ctx); + } reg = tmp; } return reg; @@ -558,10 +568,16 @@ static void arm_bpf_put_reg64(const s8 *reg, const s8 *src, struct jit_ctx *ctx) { if (is_stacked(reg[1])) { - emit(ARM_STR_I(src[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg[1])), - ctx); - emit(ARM_STR_I(src[0], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg[0])), - ctx); + if (__LINUX_ARM_ARCH__ >= 6 || + ctx->cpu_architecture >= CPU_ARCH_ARMv5TE) { + emit(ARM_STRD_I(src[1], ARM_FP, + EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); + } else { + emit(ARM_STR_I(src[1], ARM_FP, + EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); + emit(ARM_STR_I(src[0], ARM_FP, + EBPF_SCRATCH_TO_ARM_FP(reg[0])), ctx); + } } else { if (reg[1] != src[1]) emit(ARM_MOV_R(reg[1], src[1]), ctx); @@ -711,13 +727,27 @@ static inline void emit_a32_mov_r(const s8 dst, const s8 src, static inline void emit_a32_mov_r64(const bool is64, const s8 dst[], const s8 src[], struct jit_ctx *ctx) { - emit_a32_mov_r(dst_lo, src_lo, ctx); - if (is64) { + if (!is64) { + emit_a32_mov_r(dst_lo, src_lo, ctx); + /* Zero out high 4 bytes */ + emit_a32_mov_i(dst_hi, 0, ctx); + } else if (__LINUX_ARM_ARCH__ < 6 && + ctx->cpu_architecture < CPU_ARCH_ARMv5TE) { /* complete 8 byte move */ + emit_a32_mov_r(dst_lo, src_lo, ctx); emit_a32_mov_r(dst_hi, src_hi, ctx); + } else if (is_stacked(src_lo) && is_stacked(dst_lo)) { + const u8 *tmp = bpf2a32[TMP_REG_1]; + + emit(ARM_LDRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx); + emit(ARM_STRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx); + } else if (is_stacked(src_lo)) { + emit(ARM_LDRD_I(dst[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx); + } else if (is_stacked(dst_lo)) { + emit(ARM_STRD_I(src[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx); } else { - /* Zero out high 4 bytes */ - emit_a32_mov_i(dst_hi, 0, ctx); + emit(ARM_MOV_R(dst[0], src[0]), ctx); + emit(ARM_MOV_R(dst[1], src[1]), ctx); } } @@ -1778,6 +1808,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) memset(&ctx, 0, sizeof(ctx)); ctx.prog = prog; + ctx.cpu_architecture = cpu_architecture(); /* Not able to allocate memory for offsets[] , then * we must fall back to the interpreter diff --git a/arch/arm/net/bpf_jit_32.h b/arch/arm/net/bpf_jit_32.h index e541a7a6139a..f4e58bcdaa43 100644 --- a/arch/arm/net/bpf_jit_32.h +++ b/arch/arm/net/bpf_jit_32.h @@ -81,6 +81,7 @@ #define ARM_INST_LDST__IMM12 0x00000fff #define ARM_INST_LDRB_I 0x05500000 #define ARM_INST_LDRB_R 0x07d00000 +#define ARM_INST_LDRD_I 0x014000d0 #define ARM_INST_LDRH_I 0x015000b0 #define ARM_INST_LDRH_R 0x019000b0 #define ARM_INST_LDR_I 0x05100000 @@ -128,6 +129,7 @@ #define ARM_INST_STR_I 0x05000000 #define ARM_INST_STRB_I 0x05400000 +#define ARM_INST_STRD_I 0x014000f0 #define ARM_INST_STRH_I 0x014000b0 #define ARM_INST_TST_R 0x01100000 -- cgit v1.2.3 From f9ff5018c13b4918c98178ee81d41372b455201e Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 12 Jul 2018 21:50:41 +0100 Subject: ARM: net: bpf: improve 64-bit load immediate implementation Rather than writing each 32-bit half of the 64-bit immediate value separately when the register is on the stack: movw r6, #45056 ; 0xb000 movt r6, #60979 ; 0xee33 str r6, [fp, #-44] ; 0xffffffd4 mov r6, #0 str r6, [fp, #-40] ; 0xffffffd8 arrange to use the double-word store when available instead: movw r6, #45056 ; 0xb000 movt r6, #60979 ; 0xee33 mov r7, #0 strd r6, [fp, #-44] ; 0xffffffd4 Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index a9f68a924800..6558bd73bbb9 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -599,9 +599,20 @@ static inline void emit_a32_mov_i(const s8 dst, const u32 val, } } +static void emit_a32_mov_i64(const s8 dst[], u64 val, struct jit_ctx *ctx) +{ + const s8 *tmp = bpf2a32[TMP_REG_1]; + const s8 *rd = is_stacked(dst_lo) ? tmp : dst; + + emit_mov_i(rd[1], (u32)val, ctx); + emit_mov_i(rd[0], val >> 32, ctx); + + arm_bpf_put_reg64(dst, rd, ctx); +} + /* Sign extended move */ -static inline void emit_a32_mov_i64(const bool is64, const s8 dst[], - const u32 val, struct jit_ctx *ctx) { +static inline void emit_a32_mov_se_i64(const bool is64, const s8 dst[], + const u32 val, struct jit_ctx *ctx) { u32 hi = 0; if (is64 && (val & (1<<31))) @@ -1309,7 +1320,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) break; case BPF_K: /* Sign-extend immediate value to destination reg */ - emit_a32_mov_i64(is64, dst, imm, ctx); + emit_a32_mov_se_i64(is64, dst, imm, ctx); break; } break; @@ -1358,7 +1369,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) * value into temporary reg and then it would be * safe to do the operation on it. */ - emit_a32_mov_i64(is64, tmp2, imm, ctx); + emit_a32_mov_se_i64(is64, tmp2, imm, ctx); emit_a32_alu_r64(is64, dst, tmp2, ctx, BPF_OP(code)); break; } @@ -1454,7 +1465,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) * reg then it would be safe to do the operation * on it. */ - emit_a32_mov_i64(is64, tmp2, imm, ctx); + emit_a32_mov_se_i64(is64, tmp2, imm, ctx); emit_a32_mul_r64(dst, tmp2, ctx); break; } @@ -1506,12 +1517,9 @@ exit: /* dst = imm64 */ case BPF_LD | BPF_IMM | BPF_DW: { - const struct bpf_insn insn1 = insn[1]; - u32 hi, lo = imm; + u64 val = (u32)imm | (u64)insn[1].imm << 32; - hi = insn1.imm; - emit_a32_mov_i(dst_lo, lo, ctx); - emit_a32_mov_i(dst_hi, hi, ctx); + emit_a32_mov_i64(dst, val, ctx); return 1; } @@ -1531,7 +1539,7 @@ exit: switch (BPF_SIZE(code)) { case BPF_DW: /* Sign-extend immediate value into temp reg */ - emit_a32_mov_i64(true, tmp2, imm, ctx); + emit_a32_mov_se_i64(true, tmp2, imm, ctx); emit_str_r(dst_lo, tmp2[1], off, ctx, BPF_W); emit_str_r(dst_lo, tmp2[0], off+4, ctx, BPF_W); break; @@ -1620,7 +1628,7 @@ exit: rm = tmp2[0]; rn = tmp2[1]; /* Sign-extend immediate value */ - emit_a32_mov_i64(true, tmp2, imm, ctx); + emit_a32_mov_se_i64(true, tmp2, imm, ctx); go_jmp: /* Setup destination register */ rd = arm_bpf_get_reg64(dst, tmp, ctx); -- cgit v1.2.3 From 077513b89424ba814432b1cd7d7f793958059ed8 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 12 Jul 2018 21:50:46 +0100 Subject: ARM: net: bpf: improve 64-bit sign-extended immediate load Improve the 64-bit sign-extended immediate from: mov r6, #1 str r6, [fp, #-52] ; 0xffffffcc mov r6, #0 str r6, [fp, #-48] ; 0xffffffd0 to: mov r6, #1 mov r7, #0 strd r6, [fp, #-52] ; 0xffffffcc Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 6558bd73bbb9..3a182e618441 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -613,12 +613,11 @@ static void emit_a32_mov_i64(const s8 dst[], u64 val, struct jit_ctx *ctx) /* Sign extended move */ static inline void emit_a32_mov_se_i64(const bool is64, const s8 dst[], const u32 val, struct jit_ctx *ctx) { - u32 hi = 0; + u64 val64 = val; if (is64 && (val & (1<<31))) - hi = (u32)~0; - emit_a32_mov_i(dst_lo, val, ctx); - emit_a32_mov_i(dst_hi, hi, ctx); + val64 |= 0xffffffff00000000ULL; + emit_a32_mov_i64(dst, val64, ctx); } static inline void emit_a32_add_r(const u8 dst, const u8 src, -- cgit v1.2.3 From c5eae692571d6c44efeb27fe78a5ed8b5ded9b4a Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 12 Jul 2018 21:50:51 +0100 Subject: ARM: net: bpf: improve 64-bit store implementation Improve the 64-bit store implementation from: ldr r6, [fp, #-8] str r8, [r6] ldr r6, [fp, #-8] mov r7, #4 add r7, r6, r7 str r9, [r7] to: ldr r6, [fp, #-8] str r8, [r6] str r9, [r6, #4] We leave the store as two separate STR instructions rather than using STRD as the store may not be aligned, and STR can handle misalignment. Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 52 +++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 26 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 3a182e618441..026612ee8151 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -975,29 +975,42 @@ static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[], } /* *(size *)(dst + off) = src */ -static inline void emit_str_r(const s8 dst, const s8 src, - const s32 off, struct jit_ctx *ctx, const u8 sz){ +static inline void emit_str_r(const s8 dst, const s8 src[], + s32 off, struct jit_ctx *ctx, const u8 sz){ const s8 *tmp = bpf2a32[TMP_REG_1]; + s32 off_max; s8 rd; rd = arm_bpf_get_reg32(dst, tmp[1], ctx); - if (off) { + + if (sz == BPF_H) + off_max = 0xff; + else + off_max = 0xfff; + + if (off < 0 || off > off_max) { emit_a32_mov_i(tmp[0], off, ctx); - emit(ARM_ADD_R(tmp[0], rd, tmp[0]), ctx); + emit(ARM_ADD_R(tmp[0], tmp[0], rd), ctx); rd = tmp[0]; + off = 0; } switch (sz) { - case BPF_W: - /* Store a Word */ - emit(ARM_STR_I(src, rd, 0), ctx); + case BPF_B: + /* Store a Byte */ + emit(ARM_STRB_I(src_lo, rd, off), ctx); break; case BPF_H: /* Store a HalfWord */ - emit(ARM_STRH_I(src, rd, 0), ctx); + emit(ARM_STRH_I(src_lo, rd, off), ctx); break; - case BPF_B: - /* Store a Byte */ - emit(ARM_STRB_I(src, rd, 0), ctx); + case BPF_W: + /* Store a Word */ + emit(ARM_STR_I(src_lo, rd, off), ctx); + break; + case BPF_DW: + /* Store a Double Word */ + emit(ARM_STR_I(src_lo, rd, off), ctx); + emit(ARM_STR_I(src_hi, rd, off + 4), ctx); break; } } @@ -1539,16 +1552,14 @@ exit: case BPF_DW: /* Sign-extend immediate value into temp reg */ emit_a32_mov_se_i64(true, tmp2, imm, ctx); - emit_str_r(dst_lo, tmp2[1], off, ctx, BPF_W); - emit_str_r(dst_lo, tmp2[0], off+4, ctx, BPF_W); break; case BPF_W: case BPF_H: case BPF_B: emit_a32_mov_i(tmp2[1], imm, ctx); - emit_str_r(dst_lo, tmp2[1], off, ctx, BPF_SIZE(code)); break; } + emit_str_r(dst_lo, tmp2, off, ctx, BPF_SIZE(code)); break; /* STX XADD: lock *(u32 *)(dst + off) += src */ case BPF_STX | BPF_XADD | BPF_W: @@ -1560,20 +1571,9 @@ exit: case BPF_STX | BPF_MEM | BPF_H: case BPF_STX | BPF_MEM | BPF_B: case BPF_STX | BPF_MEM | BPF_DW: - { - u8 sz = BPF_SIZE(code); - rs = arm_bpf_get_reg64(src, tmp2, ctx); - - /* Store the value */ - if (BPF_SIZE(code) == BPF_DW) { - emit_str_r(dst_lo, rs[1], off, ctx, BPF_W); - emit_str_r(dst_lo, rs[0], off+4, ctx, BPF_W); - } else { - emit_str_r(dst_lo, rs[1], off, ctx, sz); - } + emit_str_r(dst_lo, rs, off, ctx, BPF_SIZE(code)); break; - } /* PC += off if dst == src */ /* PC += off if dst > src */ /* PC += off if dst >= src */ -- cgit v1.2.3 From b18bea2a45b136c4da7ff75305b54d66cf57de81 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 12 Jul 2018 21:50:56 +0100 Subject: ARM: net: bpf: improve 64-bit ALU implementation Improbe the 64-bit ALU implementation from: movw r8, #65532 movt r8, #65535 movw r9, #65535 movt r9, #65535 ldr r7, [fp, #-44] adds r7, r7, r8 str r7, [fp, #-44] ldr r7, [fp, #-40] adc r7, r7, r9 str r7, [fp, #-40] to: movw r8, #65532 movt r8, #65535 movw r9, #65535 movt r9, #65535 ldrd r6, [fp, #-44] adds r6, r6, r8 adc r7, r7, r9 strd r6, [fp, #-44] Signed-off-by: Russell King Signed-off-by: Daniel Borkmann --- arch/arm/net/bpf_jit_32.c | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 026612ee8151..25b3ee85066e 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -716,11 +716,30 @@ static inline void emit_a32_alu_r(const s8 dst, const s8 src, static inline void emit_a32_alu_r64(const bool is64, const s8 dst[], const s8 src[], struct jit_ctx *ctx, const u8 op) { - emit_a32_alu_r(dst_lo, src_lo, ctx, is64, false, op); - if (is64) - emit_a32_alu_r(dst_hi, src_hi, ctx, is64, true, op); - else - emit_a32_mov_i(dst_hi, 0, ctx); + const s8 *tmp = bpf2a32[TMP_REG_1]; + const s8 *tmp2 = bpf2a32[TMP_REG_2]; + const s8 *rd; + + rd = arm_bpf_get_reg64(dst, tmp, ctx); + if (is64) { + const s8 *rs; + + rs = arm_bpf_get_reg64(src, tmp2, ctx); + + /* ALU operation */ + emit_alu_r(rd[1], rs[1], true, false, op, ctx); + emit_alu_r(rd[0], rs[0], true, true, op, ctx); + } else { + s8 rs; + + rs = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); + + /* ALU operation */ + emit_alu_r(rd[1], rs, true, false, op, ctx); + emit_a32_mov_i(rd[0], 0, ctx); + } + + arm_bpf_put_reg64(dst, rd, ctx); } /* dst = src (4 bytes)*/ -- cgit v1.2.3 From 22a001e8544fa9d89a1d3d7491ca1d8f9466b69d Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sat, 14 Jul 2018 11:45:56 +0200 Subject: ARM: dts: Add ethernet and switch to D-Link DIR-685 This adds the Ethernet and Realtek switch device to the D-Link DIR-685 Gemini-based device. Signed-off-by: Linus Walleij Signed-off-by: David S. Miller --- arch/arm/boot/dts/gemini-dlink-dir-685.dts | 140 ++++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/arm/boot/dts/gemini-dlink-dir-685.dts b/arch/arm/boot/dts/gemini-dlink-dir-685.dts index fb5c954ab95a..6f258b50eb44 100644 --- a/arch/arm/boot/dts/gemini-dlink-dir-685.dts +++ b/arch/arm/boot/dts/gemini-dlink-dir-685.dts @@ -156,6 +156,100 @@ }; }; + /* This is a RealTek RTL8366RB switch and PHY using SMI over GPIO */ + switch { + compatible = "realtek,rtl8366rb"; + /* 22 = MDIO (has input reads), 21 = MDC (clock, output only) */ + mdc-gpios = <&gpio0 21 GPIO_ACTIVE_HIGH>; + mdio-gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio0 14 GPIO_ACTIVE_LOW>; + realtek,disable-leds; + + switch_intc: interrupt-controller { + /* GPIO 15 provides the interrupt */ + interrupt-parent = <&gpio0>; + interrupts = <15 IRQ_TYPE_LEVEL_LOW>; + interrupt-controller; + #address-cells = <0>; + #interrupt-cells = <1>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + label = "lan0"; + phy-handle = <&phy0>; + }; + port@1 { + reg = <1>; + label = "lan1"; + phy-handle = <&phy1>; + }; + port@2 { + reg = <2>; + label = "lan2"; + phy-handle = <&phy2>; + }; + port@3 { + reg = <3>; + label = "lan3"; + phy-handle = <&phy3>; + }; + port@4 { + reg = <4>; + label = "wan"; + phy-handle = <&phy4>; + }; + rtl8366rb_cpu_port: port@5 { + reg = <5>; + label = "cpu"; + ethernet = <&gmac0>; + phy-mode = "rgmii"; + fixed-link { + speed = <1000>; + full-duplex; + pause; + }; + }; + + }; + + mdio { + compatible = "realtek,smi-mdio"; + #address-cells = <1>; + #size-cells = <0>; + + phy0: phy@0 { + reg = <0>; + interrupt-parent = <&switch_intc>; + interrupts = <0>; + }; + phy1: phy@1 { + reg = <1>; + interrupt-parent = <&switch_intc>; + interrupts = <1>; + }; + phy2: phy@2 { + reg = <2>; + interrupt-parent = <&switch_intc>; + interrupts = <2>; + }; + phy3: phy@3 { + reg = <3>; + interrupt-parent = <&switch_intc>; + interrupts = <3>; + }; + phy4: phy@4 { + reg = <4>; + interrupt-parent = <&switch_intc>; + interrupts = <12>; + }; + }; + }; + soc { flash@30000000 { /* @@ -223,10 +317,12 @@ * gpio0bgrp cover line 7 used by WPS LED * gpio0cgrp cover line 8, 13 used by keys * and 11, 12 used by the HD LEDs + * and line 14, 15 used by RTL8366 + * RESET and phy ready * gpio0egrp cover line 16 used by VDISP * gpio0fgrp cover line 17 used by TK IRQ * gpio0ggrp cover line 20 used by panel CS - * gpio0hgrp cover line 21,22 used by RTL8366RB + * gpio0hgrp cover line 21,22 used by RTL8366RB MDIO */ gpio0_default_pins: pinctrl-gpio0 { mux { @@ -250,6 +346,32 @@ groups = "gpio1bgrp"; }; }; + pinctrl-gmii { + mux { + function = "gmii"; + groups = "gmii_gmac0_grp"; + }; + conf0 { + pins = "V8 GMAC0 RXDV", "T10 GMAC1 RXDV", + "Y7 GMAC0 RXC", "Y11 GMAC1 RXC", + "T8 GMAC0 TXEN", "W11 GMAC1 TXEN", + "U8 GMAC0 TXC", "V11 GMAC1 TXC", + "W8 GMAC0 RXD0", "V9 GMAC0 RXD1", + "Y8 GMAC0 RXD2", "U9 GMAC0 RXD3", + "T7 GMAC0 TXD0", "U6 GMAC0 TXD1", + "V7 GMAC0 TXD2", "U7 GMAC0 TXD3", + "Y12 GMAC1 RXD0", "V12 GMAC1 RXD1", + "T11 GMAC1 RXD2", "W12 GMAC1 RXD3", + "U10 GMAC1 TXD0", "Y10 GMAC1 TXD1", + "W10 GMAC1 TXD2", "T9 GMAC1 TXD3"; + skew-delay = <7>; + }; + /* Set up drive strength on GMAC0 to 16 mA */ + conf1 { + groups = "gmii_gmac0_grp"; + drive-strength = <16>; + }; + }; }; }; @@ -291,6 +413,22 @@ <0x6000 0 0 4 &pci_intc 2>; }; + ethernet@60000000 { + status = "okay"; + + ethernet-port@0 { + phy-mode = "rgmii"; + fixed-link { + speed = <1000>; + full-duplex; + pause; + }; + }; + ethernet-port@1 { + /* Not used in this platform */ + }; + }; + ata@63000000 { status = "okay"; }; -- cgit v1.2.3 From 488dee96bb62f0b3d9e678cf42574034d5b033a5 Mon Sep 17 00:00:00 2001 From: Dmitry Torokhov Date: Fri, 20 Jul 2018 21:56:47 +0000 Subject: kernfs: allow creating kernfs objects with arbitrary uid/gid This change allows creating kernfs files and directories with arbitrary uid/gid instead of always using GLOBAL_ROOT_UID/GID by extending kernfs_create_dir_ns() and kernfs_create_file_ns() with uid/gid arguments. The "simple" kernfs_create_file() and kernfs_create_dir() are left alone and always create objects belonging to the global root. When creating symlinks ownership (uid/gid) is taken from the target kernfs object. Co-Developed-by: Tyler Hicks Signed-off-by: Dmitry Torokhov Signed-off-by: Tyler Hicks Signed-off-by: David S. Miller --- arch/x86/kernel/cpu/intel_rdt_rdtgroup.c | 4 +++- fs/kernfs/dir.c | 29 ++++++++++++++++++++++++++--- fs/kernfs/file.c | 8 ++++++-- fs/kernfs/inode.c | 2 +- fs/kernfs/kernfs-internal.h | 2 ++ fs/kernfs/symlink.c | 11 ++++++++++- fs/sysfs/dir.c | 4 +++- fs/sysfs/file.c | 5 +++-- include/linux/kernfs.h | 28 +++++++++++++++++++--------- kernel/cgroup/cgroup.c | 4 +++- 10 files changed, 76 insertions(+), 21 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c index 749856a2e736..9af1a21265d3 100644 --- a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c +++ b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c @@ -146,6 +146,7 @@ static int rdtgroup_add_file(struct kernfs_node *parent_kn, struct rftype *rft) int ret; kn = __kernfs_create_file(parent_kn, rft->name, rft->mode, + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0, rft->kf_ops, rft, NULL, NULL); if (IS_ERR(kn)) return PTR_ERR(kn); @@ -1503,7 +1504,8 @@ static int mon_addfile(struct kernfs_node *parent_kn, const char *name, struct kernfs_node *kn; int ret = 0; - kn = __kernfs_create_file(parent_kn, name, 0444, 0, + kn = __kernfs_create_file(parent_kn, name, 0444, + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0, &kf_mondata_ops, priv, NULL, NULL); if (IS_ERR(kn)) return PTR_ERR(kn); diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c index d66cc0777303..4ca0b5c18192 100644 --- a/fs/kernfs/dir.c +++ b/fs/kernfs/dir.c @@ -619,6 +619,7 @@ struct kernfs_node *kernfs_node_from_dentry(struct dentry *dentry) static struct kernfs_node *__kernfs_new_node(struct kernfs_root *root, const char *name, umode_t mode, + kuid_t uid, kgid_t gid, unsigned flags) { struct kernfs_node *kn; @@ -661,8 +662,22 @@ static struct kernfs_node *__kernfs_new_node(struct kernfs_root *root, kn->mode = mode; kn->flags = flags; + if (!uid_eq(uid, GLOBAL_ROOT_UID) || !gid_eq(gid, GLOBAL_ROOT_GID)) { + struct iattr iattr = { + .ia_valid = ATTR_UID | ATTR_GID, + .ia_uid = uid, + .ia_gid = gid, + }; + + ret = __kernfs_setattr(kn, &iattr); + if (ret < 0) + goto err_out3; + } + return kn; + err_out3: + idr_remove(&root->ino_idr, kn->id.ino); err_out2: kmem_cache_free(kernfs_node_cache, kn); err_out1: @@ -672,11 +687,13 @@ static struct kernfs_node *__kernfs_new_node(struct kernfs_root *root, struct kernfs_node *kernfs_new_node(struct kernfs_node *parent, const char *name, umode_t mode, + kuid_t uid, kgid_t gid, unsigned flags) { struct kernfs_node *kn; - kn = __kernfs_new_node(kernfs_root(parent), name, mode, flags); + kn = __kernfs_new_node(kernfs_root(parent), + name, mode, uid, gid, flags); if (kn) { kernfs_get(parent); kn->parent = parent; @@ -946,6 +963,7 @@ struct kernfs_root *kernfs_create_root(struct kernfs_syscall_ops *scops, root->next_generation = 1; kn = __kernfs_new_node(root, "", S_IFDIR | S_IRUGO | S_IXUGO, + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, KERNFS_DIR); if (!kn) { idr_destroy(&root->ino_idr); @@ -984,6 +1002,8 @@ void kernfs_destroy_root(struct kernfs_root *root) * @parent: parent in which to create a new directory * @name: name of the new directory * @mode: mode of the new directory + * @uid: uid of the new directory + * @gid: gid of the new directory * @priv: opaque data associated with the new directory * @ns: optional namespace tag of the directory * @@ -991,13 +1011,15 @@ void kernfs_destroy_root(struct kernfs_root *root) */ struct kernfs_node *kernfs_create_dir_ns(struct kernfs_node *parent, const char *name, umode_t mode, + kuid_t uid, kgid_t gid, void *priv, const void *ns) { struct kernfs_node *kn; int rc; /* allocate */ - kn = kernfs_new_node(parent, name, mode | S_IFDIR, KERNFS_DIR); + kn = kernfs_new_node(parent, name, mode | S_IFDIR, + uid, gid, KERNFS_DIR); if (!kn) return ERR_PTR(-ENOMEM); @@ -1028,7 +1050,8 @@ struct kernfs_node *kernfs_create_empty_dir(struct kernfs_node *parent, int rc; /* allocate */ - kn = kernfs_new_node(parent, name, S_IRUGO|S_IXUGO|S_IFDIR, KERNFS_DIR); + kn = kernfs_new_node(parent, name, S_IRUGO|S_IXUGO|S_IFDIR, + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, KERNFS_DIR); if (!kn) return ERR_PTR(-ENOMEM); diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c index 2015d8c45e4a..dbf5bc250bfd 100644 --- a/fs/kernfs/file.c +++ b/fs/kernfs/file.c @@ -965,6 +965,8 @@ const struct file_operations kernfs_file_fops = { * @parent: directory to create the file in * @name: name of the file * @mode: mode of the file + * @uid: uid of the file + * @gid: gid of the file * @size: size of the file * @ops: kernfs operations for the file * @priv: private data for the file @@ -975,7 +977,8 @@ const struct file_operations kernfs_file_fops = { */ struct kernfs_node *__kernfs_create_file(struct kernfs_node *parent, const char *name, - umode_t mode, loff_t size, + umode_t mode, kuid_t uid, kgid_t gid, + loff_t size, const struct kernfs_ops *ops, void *priv, const void *ns, struct lock_class_key *key) @@ -986,7 +989,8 @@ struct kernfs_node *__kernfs_create_file(struct kernfs_node *parent, flags = KERNFS_FILE; - kn = kernfs_new_node(parent, name, (mode & S_IALLUGO) | S_IFREG, flags); + kn = kernfs_new_node(parent, name, (mode & S_IALLUGO) | S_IFREG, + uid, gid, flags); if (!kn) return ERR_PTR(-ENOMEM); diff --git a/fs/kernfs/inode.c b/fs/kernfs/inode.c index 3d73fe9d56e2..80cebcd94c90 100644 --- a/fs/kernfs/inode.c +++ b/fs/kernfs/inode.c @@ -63,7 +63,7 @@ out_unlock: return ret; } -static int __kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr) +int __kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr) { struct kernfs_iattrs *attrs; struct iattr *iattrs; diff --git a/fs/kernfs/kernfs-internal.h b/fs/kernfs/kernfs-internal.h index 0f260dcca177..3d83b114bb08 100644 --- a/fs/kernfs/kernfs-internal.h +++ b/fs/kernfs/kernfs-internal.h @@ -90,6 +90,7 @@ int kernfs_iop_setattr(struct dentry *dentry, struct iattr *iattr); int kernfs_iop_getattr(const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags); ssize_t kernfs_iop_listxattr(struct dentry *dentry, char *buf, size_t size); +int __kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr); /* * dir.c @@ -104,6 +105,7 @@ void kernfs_put_active(struct kernfs_node *kn); int kernfs_add_one(struct kernfs_node *kn); struct kernfs_node *kernfs_new_node(struct kernfs_node *parent, const char *name, umode_t mode, + kuid_t uid, kgid_t gid, unsigned flags); struct kernfs_node *kernfs_find_and_get_node_by_ino(struct kernfs_root *root, unsigned int ino); diff --git a/fs/kernfs/symlink.c b/fs/kernfs/symlink.c index 08ccabd7047f..5ffed48f3d0e 100644 --- a/fs/kernfs/symlink.c +++ b/fs/kernfs/symlink.c @@ -21,6 +21,7 @@ * @target: target node for the symlink to point to * * Returns the created node on success, ERR_PTR() value on error. + * Ownership of the link matches ownership of the target. */ struct kernfs_node *kernfs_create_link(struct kernfs_node *parent, const char *name, @@ -28,8 +29,16 @@ struct kernfs_node *kernfs_create_link(struct kernfs_node *parent, { struct kernfs_node *kn; int error; + kuid_t uid = GLOBAL_ROOT_UID; + kgid_t gid = GLOBAL_ROOT_GID; - kn = kernfs_new_node(parent, name, S_IFLNK|S_IRWXUGO, KERNFS_LINK); + if (target->iattr) { + uid = target->iattr->ia_iattr.ia_uid; + gid = target->iattr->ia_iattr.ia_gid; + } + + kn = kernfs_new_node(parent, name, S_IFLNK|S_IRWXUGO, uid, gid, + KERNFS_LINK); if (!kn) return ERR_PTR(-ENOMEM); diff --git a/fs/sysfs/dir.c b/fs/sysfs/dir.c index 58eba92a0e41..e39b884f0867 100644 --- a/fs/sysfs/dir.c +++ b/fs/sysfs/dir.c @@ -52,7 +52,9 @@ int sysfs_create_dir_ns(struct kobject *kobj, const void *ns) return -ENOENT; kn = kernfs_create_dir_ns(parent, kobject_name(kobj), - S_IRWXU | S_IRUGO | S_IXUGO, kobj, ns); + S_IRWXU | S_IRUGO | S_IXUGO, + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + kobj, ns); if (IS_ERR(kn)) { if (PTR_ERR(kn) == -EEXIST) sysfs_warn_dup(parent, kobject_name(kobj)); diff --git a/fs/sysfs/file.c b/fs/sysfs/file.c index 5c13f29bfcdb..513fa691ecbd 100644 --- a/fs/sysfs/file.c +++ b/fs/sysfs/file.c @@ -302,8 +302,9 @@ int sysfs_add_file_mode_ns(struct kernfs_node *parent, if (!attr->ignore_lockdep) key = attr->key ?: (struct lock_class_key *)&attr->skey; #endif - kn = __kernfs_create_file(parent, attr->name, mode & 0777, size, ops, - (void *)attr, ns, key); + kn = __kernfs_create_file(parent, attr->name, + mode & 0777, GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + size, ops, (void *)attr, ns, key); if (IS_ERR(kn)) { if (PTR_ERR(kn) == -EEXIST) sysfs_warn_dup(parent, attr->name); diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index ab25c8b6d9e3..814643f7ee52 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -15,6 +15,7 @@ #include #include #include +#include #include struct file; @@ -325,12 +326,14 @@ void kernfs_destroy_root(struct kernfs_root *root); struct kernfs_node *kernfs_create_dir_ns(struct kernfs_node *parent, const char *name, umode_t mode, + kuid_t uid, kgid_t gid, void *priv, const void *ns); struct kernfs_node *kernfs_create_empty_dir(struct kernfs_node *parent, const char *name); struct kernfs_node *__kernfs_create_file(struct kernfs_node *parent, - const char *name, - umode_t mode, loff_t size, + const char *name, umode_t mode, + kuid_t uid, kgid_t gid, + loff_t size, const struct kernfs_ops *ops, void *priv, const void *ns, struct lock_class_key *key); @@ -415,12 +418,14 @@ static inline void kernfs_destroy_root(struct kernfs_root *root) { } static inline struct kernfs_node * kernfs_create_dir_ns(struct kernfs_node *parent, const char *name, - umode_t mode, void *priv, const void *ns) + umode_t mode, kuid_t uid, kgid_t gid, + void *priv, const void *ns) { return ERR_PTR(-ENOSYS); } static inline struct kernfs_node * __kernfs_create_file(struct kernfs_node *parent, const char *name, - umode_t mode, loff_t size, const struct kernfs_ops *ops, + umode_t mode, kuid_t uid, kgid_t gid, + loff_t size, const struct kernfs_ops *ops, void *priv, const void *ns, struct lock_class_key *key) { return ERR_PTR(-ENOSYS); } @@ -498,12 +503,15 @@ static inline struct kernfs_node * kernfs_create_dir(struct kernfs_node *parent, const char *name, umode_t mode, void *priv) { - return kernfs_create_dir_ns(parent, name, mode, priv, NULL); + return kernfs_create_dir_ns(parent, name, mode, + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + priv, NULL); } static inline struct kernfs_node * kernfs_create_file_ns(struct kernfs_node *parent, const char *name, - umode_t mode, loff_t size, const struct kernfs_ops *ops, + umode_t mode, kuid_t uid, kgid_t gid, + loff_t size, const struct kernfs_ops *ops, void *priv, const void *ns) { struct lock_class_key *key = NULL; @@ -511,15 +519,17 @@ kernfs_create_file_ns(struct kernfs_node *parent, const char *name, #ifdef CONFIG_DEBUG_LOCK_ALLOC key = (struct lock_class_key *)&ops->lockdep_key; #endif - return __kernfs_create_file(parent, name, mode, size, ops, priv, ns, - key); + return __kernfs_create_file(parent, name, mode, uid, gid, + size, ops, priv, ns, key); } static inline struct kernfs_node * kernfs_create_file(struct kernfs_node *parent, const char *name, umode_t mode, loff_t size, const struct kernfs_ops *ops, void *priv) { - return kernfs_create_file_ns(parent, name, mode, size, ops, priv, NULL); + return kernfs_create_file_ns(parent, name, mode, + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + size, ops, priv, NULL); } static inline int kernfs_remove_by_name(struct kernfs_node *parent, diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 077370bf8964..35cf3d71f8aa 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -3557,7 +3557,9 @@ static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp, key = &cft->lockdep_key; #endif kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name), - cgroup_file_mode(cft), 0, cft->kf_ops, cft, + cgroup_file_mode(cft), + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + 0, cft->kf_ops, cft, NULL, key); if (IS_ERR(kn)) return PTR_ERR(kn); -- cgit v1.2.3 From 18b872d854dc7256863755ece26828aa1856c13f Mon Sep 17 00:00:00 2001 From: Arun Parameswaran Date: Wed, 1 Aug 2018 17:53:48 -0700 Subject: arm64: dts: Fix the base address of the Broadcom iProc mdio mux Modify the base address of the mdio mux driver to point to the start of the mdio mux block's register address space. Signed-off-by: Arun Parameswaran Reviewed-by: Andrew Lunn Reviewed-by: Florian Fainelli Signed-off-by: David S. Miller --- arch/arm64/boot/dts/broadcom/northstar2/ns2.dtsi | 4 ++-- arch/arm64/boot/dts/broadcom/stingray/stingray.dtsi | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/arm64/boot/dts/broadcom/northstar2/ns2.dtsi b/arch/arm64/boot/dts/broadcom/northstar2/ns2.dtsi index 4057197048dc..1a406a76c86a 100644 --- a/arch/arm64/boot/dts/broadcom/northstar2/ns2.dtsi +++ b/arch/arm64/boot/dts/broadcom/northstar2/ns2.dtsi @@ -482,9 +482,9 @@ status = "disabled"; }; - mdio_mux_iproc: mdio-mux@6602023c { + mdio_mux_iproc: mdio-mux@66020000 { compatible = "brcm,mdio-mux-iproc"; - reg = <0x6602023c 0x14>; + reg = <0x66020000 0x250>; #address-cells = <1>; #size-cells = <0>; diff --git a/arch/arm64/boot/dts/broadcom/stingray/stingray.dtsi b/arch/arm64/boot/dts/broadcom/stingray/stingray.dtsi index b203152ad67c..a70e8ddbd66f 100644 --- a/arch/arm64/boot/dts/broadcom/stingray/stingray.dtsi +++ b/arch/arm64/boot/dts/broadcom/stingray/stingray.dtsi @@ -278,9 +278,9 @@ #include "stingray-pinctrl.dtsi" - mdio_mux_iproc: mdio-mux@2023c { + mdio_mux_iproc: mdio-mux@20000 { compatible = "brcm,mdio-mux-iproc"; - reg = <0x0002023c 0x14>; + reg = <0x00020000 0x250>; #address-cells = <1>; #size-cells = <0>; -- cgit v1.2.3 From f63421a70f9e7d6dac6ec06b4d9f7cf516f8a885 Mon Sep 17 00:00:00 2001 From: Yangbo Lu Date: Wed, 1 Aug 2018 18:05:52 +0800 Subject: arm64: dts: fsl: add clocks property for fman ptp timer node This patch is to add clocks property for fman ptp timer node. Signed-off-by: Yangbo Lu Signed-off-by: David S. Miller --- arch/arm64/boot/dts/freescale/qoriq-fman3-0.dtsi | 1 + 1 file changed, 1 insertion(+) (limited to 'arch') diff --git a/arch/arm64/boot/dts/freescale/qoriq-fman3-0.dtsi b/arch/arm64/boot/dts/freescale/qoriq-fman3-0.dtsi index a56a408e9bf7..4664c33e0763 100644 --- a/arch/arm64/boot/dts/freescale/qoriq-fman3-0.dtsi +++ b/arch/arm64/boot/dts/freescale/qoriq-fman3-0.dtsi @@ -80,4 +80,5 @@ ptp_timer0: ptp-timer@1afe000 { compatible = "fsl,fman-ptp-timer"; reg = <0x0 0x1afe000 0x0 0x1000>; interrupts = ; + clocks = <&clockgen 3 0>; }; -- cgit v1.2.3 From a16b5da54d1f3a06df642b866779358fc64d00e2 Mon Sep 17 00:00:00 2001 From: Yangbo Lu Date: Wed, 1 Aug 2018 18:05:53 +0800 Subject: powerpc/mpc85xx: add clocks property for fman ptp timer node This patch is to add clocks property for fman ptp timer node. Signed-off-by: Yangbo Lu Signed-off-by: David S. Miller --- arch/powerpc/boot/dts/fsl/qoriq-fman-0.dtsi | 1 + arch/powerpc/boot/dts/fsl/qoriq-fman-1.dtsi | 1 + arch/powerpc/boot/dts/fsl/qoriq-fman3-0.dtsi | 1 + arch/powerpc/boot/dts/fsl/qoriq-fman3-1.dtsi | 1 + arch/powerpc/boot/dts/fsl/qoriq-fman3l-0.dtsi | 1 + 5 files changed, 5 insertions(+) (limited to 'arch') diff --git a/arch/powerpc/boot/dts/fsl/qoriq-fman-0.dtsi b/arch/powerpc/boot/dts/fsl/qoriq-fman-0.dtsi index 6b124f73f67a..9b6cf9149937 100644 --- a/arch/powerpc/boot/dts/fsl/qoriq-fman-0.dtsi +++ b/arch/powerpc/boot/dts/fsl/qoriq-fman-0.dtsi @@ -100,4 +100,5 @@ ptp_timer0: ptp-timer@4fe000 { compatible = "fsl,fman-ptp-timer"; reg = <0x4fe000 0x1000>; interrupts = <96 2 0 0>; + clocks = <&clockgen 3 0>; }; diff --git a/arch/powerpc/boot/dts/fsl/qoriq-fman-1.dtsi b/arch/powerpc/boot/dts/fsl/qoriq-fman-1.dtsi index b80aaf5f00a1..e95c11ff0417 100644 --- a/arch/powerpc/boot/dts/fsl/qoriq-fman-1.dtsi +++ b/arch/powerpc/boot/dts/fsl/qoriq-fman-1.dtsi @@ -100,4 +100,5 @@ ptp_timer1: ptp-timer@5fe000 { compatible = "fsl,fman-ptp-timer"; reg = <0x5fe000 0x1000>; interrupts = <97 2 0 0>; + clocks = <&clockgen 3 1>; }; diff --git a/arch/powerpc/boot/dts/fsl/qoriq-fman3-0.dtsi b/arch/powerpc/boot/dts/fsl/qoriq-fman3-0.dtsi index d3720fdde26c..d62b36c5a329 100644 --- a/arch/powerpc/boot/dts/fsl/qoriq-fman3-0.dtsi +++ b/arch/powerpc/boot/dts/fsl/qoriq-fman3-0.dtsi @@ -105,4 +105,5 @@ ptp_timer0: ptp-timer@4fe000 { compatible = "fsl,fman-ptp-timer"; reg = <0x4fe000 0x1000>; interrupts = <96 2 0 0>; + clocks = <&clockgen 3 0>; }; diff --git a/arch/powerpc/boot/dts/fsl/qoriq-fman3-1.dtsi b/arch/powerpc/boot/dts/fsl/qoriq-fman3-1.dtsi index ae34c204a5bc..310232460500 100644 --- a/arch/powerpc/boot/dts/fsl/qoriq-fman3-1.dtsi +++ b/arch/powerpc/boot/dts/fsl/qoriq-fman3-1.dtsi @@ -105,4 +105,5 @@ ptp_timer1: ptp-timer@5fe000 { compatible = "fsl,fman-ptp-timer"; reg = <0x5fe000 0x1000>; interrupts = <97 2 0 0>; + clocks = <&clockgen 3 1>; }; diff --git a/arch/powerpc/boot/dts/fsl/qoriq-fman3l-0.dtsi b/arch/powerpc/boot/dts/fsl/qoriq-fman3l-0.dtsi index 02f2755842cc..c90702b04a53 100644 --- a/arch/powerpc/boot/dts/fsl/qoriq-fman3l-0.dtsi +++ b/arch/powerpc/boot/dts/fsl/qoriq-fman3l-0.dtsi @@ -93,4 +93,5 @@ ptp_timer0: ptp-timer@4fe000 { compatible = "fsl,fman-ptp-timer"; reg = <0x4fe000 0x1000>; interrupts = <96 2 0 0>; + clocks = <&clockgen 3 0>; }; -- cgit v1.2.3 From 209b43759d65b2cc99ce7757249aacc82b03c4e2 Mon Sep 17 00:00:00 2001 From: Michael Büsch Date: Tue, 31 Jul 2018 22:15:09 +0200 Subject: ssb: Remove SSB_WARN_ON, SSB_BUG_ON and SSB_DEBUG Use the standard WARN_ON instead. If a small kernel is desired, WARN_ON can be disabled globally. Also remove SSB_DEBUG. Besides WARN_ON it only adds a tiny debug check. Include this check unconditionally. Signed-off-by: Michael Buesch Signed-off-by: Kalle Valo --- arch/mips/configs/bcm47xx_defconfig | 1 - arch/powerpc/configs/wii_defconfig | 1 - drivers/ssb/Kconfig | 9 --------- drivers/ssb/driver_chipcommon.c | 8 ++++---- drivers/ssb/driver_chipcommon_pmu.c | 10 +++++----- drivers/ssb/driver_gpio.c | 4 ++-- drivers/ssb/driver_pcicore.c | 6 +++--- drivers/ssb/embedded.c | 10 +++++----- drivers/ssb/host_soc.c | 12 ++++++------ drivers/ssb/main.c | 38 +++++++++++++++++-------------------- drivers/ssb/pci.c | 19 ++++++------------- drivers/ssb/pcmcia.c | 14 +++++++------- drivers/ssb/scan.c | 4 ++-- drivers/ssb/sdio.c | 12 ++++++------ drivers/ssb/ssb_private.h | 9 --------- include/linux/ssb/ssb.h | 2 -- 16 files changed, 63 insertions(+), 96 deletions(-) (limited to 'arch') diff --git a/arch/mips/configs/bcm47xx_defconfig b/arch/mips/configs/bcm47xx_defconfig index fad8e964f14c..ba800a892384 100644 --- a/arch/mips/configs/bcm47xx_defconfig +++ b/arch/mips/configs/bcm47xx_defconfig @@ -66,7 +66,6 @@ CONFIG_HW_RANDOM=y CONFIG_GPIO_SYSFS=y CONFIG_WATCHDOG=y CONFIG_BCM47XX_WDT=y -CONFIG_SSB_DEBUG=y CONFIG_SSB_DRIVER_GIGE=y CONFIG_BCMA_DRIVER_GMAC_CMN=y CONFIG_USB=y diff --git a/arch/powerpc/configs/wii_defconfig b/arch/powerpc/configs/wii_defconfig index 10940533da71..f5c366b02828 100644 --- a/arch/powerpc/configs/wii_defconfig +++ b/arch/powerpc/configs/wii_defconfig @@ -78,7 +78,6 @@ CONFIG_GPIO_HLWD=y CONFIG_POWER_RESET=y CONFIG_POWER_RESET_GPIO=y # CONFIG_HWMON is not set -CONFIG_SSB_DEBUG=y CONFIG_FB=y # CONFIG_VGA_CONSOLE is not set CONFIG_FRAMEBUFFER_CONSOLE=y diff --git a/drivers/ssb/Kconfig b/drivers/ssb/Kconfig index 6c438c819eb9..df30e1323252 100644 --- a/drivers/ssb/Kconfig +++ b/drivers/ssb/Kconfig @@ -89,15 +89,6 @@ config SSB_HOST_SOC If unsure, say N -config SSB_DEBUG - bool "SSB debugging" - depends on SSB - help - This turns on additional runtime checks and debugging - messages. Turn this on for SSB troubleshooting. - - If unsure, say N - config SSB_SERIAL bool depends on SSB diff --git a/drivers/ssb/driver_chipcommon.c b/drivers/ssb/driver_chipcommon.c index 48050c6fd847..99a4656d113d 100644 --- a/drivers/ssb/driver_chipcommon.c +++ b/drivers/ssb/driver_chipcommon.c @@ -56,7 +56,7 @@ void ssb_chipco_set_clockmode(struct ssb_chipcommon *cc, if (cc->capabilities & SSB_CHIPCO_CAP_PMU) return; /* PMU controls clockmode, separated function needed */ - SSB_WARN_ON(ccdev->id.revision >= 20); + WARN_ON(ccdev->id.revision >= 20); /* chipcommon cores prior to rev6 don't support dynamic clock control */ if (ccdev->id.revision < 6) @@ -111,7 +111,7 @@ void ssb_chipco_set_clockmode(struct ssb_chipcommon *cc, } break; default: - SSB_WARN_ON(1); + WARN_ON(1); } } @@ -164,7 +164,7 @@ static int chipco_pctl_clockfreqlimit(struct ssb_chipcommon *cc, int get_max) divisor = 32; break; default: - SSB_WARN_ON(1); + WARN_ON(1); } } else if (cc->dev->id.revision < 10) { switch (clocksrc) { @@ -277,7 +277,7 @@ static void calc_fast_powerup_delay(struct ssb_chipcommon *cc) minfreq = chipco_pctl_clockfreqlimit(cc, 0); pll_on_delay = chipco_read32(cc, SSB_CHIPCO_PLLONDELAY); tmp = (((pll_on_delay + 2) * 1000000) + (minfreq - 1)) / minfreq; - SSB_WARN_ON(tmp & ~0xFFFF); + WARN_ON(tmp & ~0xFFFF); cc->fast_pwrup_delay = tmp; } diff --git a/drivers/ssb/driver_chipcommon_pmu.c b/drivers/ssb/driver_chipcommon_pmu.c index e28682a53cdf..0f60e90ded26 100644 --- a/drivers/ssb/driver_chipcommon_pmu.c +++ b/drivers/ssb/driver_chipcommon_pmu.c @@ -128,7 +128,7 @@ static void ssb_pmu0_pllinit_r0(struct ssb_chipcommon *cc, ~(1 << SSB_PMURES_5354_BB_PLL_PU)); break; default: - SSB_WARN_ON(1); + WARN_ON(1); } for (i = 1500; i; i--) { tmp = chipco_read32(cc, SSB_CHIPCO_CLKCTLST); @@ -265,7 +265,7 @@ static void ssb_pmu1_pllinit_r0(struct ssb_chipcommon *cc, buffer_strength = 0x222222; break; default: - SSB_WARN_ON(1); + WARN_ON(1); } for (i = 1500; i; i--) { tmp = chipco_read32(cc, SSB_CHIPCO_CLKCTLST); @@ -501,7 +501,7 @@ static void ssb_pmu_resources_init(struct ssb_chipcommon *cc) ~(depend_tab[i].depend)); break; default: - SSB_WARN_ON(1); + WARN_ON(1); } } } @@ -568,12 +568,12 @@ void ssb_pmu_set_ldo_voltage(struct ssb_chipcommon *cc, mask = 0x3F; break; default: - SSB_WARN_ON(1); + WARN_ON(1); return; } break; case 0x4312: - if (SSB_WARN_ON(id != LDO_PAREF)) + if (WARN_ON(id != LDO_PAREF)) return; addr = 0; shift = 21; diff --git a/drivers/ssb/driver_gpio.c b/drivers/ssb/driver_gpio.c index 6ce4abf7d473..e809dae4c470 100644 --- a/drivers/ssb/driver_gpio.c +++ b/drivers/ssb/driver_gpio.c @@ -461,7 +461,7 @@ int ssb_gpio_init(struct ssb_bus *bus) else if (ssb_extif_available(&bus->extif)) return ssb_gpio_extif_init(bus); else - SSB_WARN_ON(1); + WARN_ON(1); return -1; } @@ -473,7 +473,7 @@ int ssb_gpio_unregister(struct ssb_bus *bus) gpiochip_remove(&bus->gpio); return 0; } else { - SSB_WARN_ON(1); + WARN_ON(1); } return -1; diff --git a/drivers/ssb/driver_pcicore.c b/drivers/ssb/driver_pcicore.c index ae80b3171523..6a5622e0ded5 100644 --- a/drivers/ssb/driver_pcicore.c +++ b/drivers/ssb/driver_pcicore.c @@ -115,7 +115,7 @@ static int ssb_extpci_read_config(struct ssb_pcicore *pc, u32 addr, val; void __iomem *mmio; - SSB_WARN_ON(!pc->hostmode); + WARN_ON(!pc->hostmode); if (unlikely(len != 1 && len != 2 && len != 4)) goto out; addr = get_cfgspace_addr(pc, bus, dev, func, off); @@ -161,7 +161,7 @@ static int ssb_extpci_write_config(struct ssb_pcicore *pc, u32 addr, val = 0; void __iomem *mmio; - SSB_WARN_ON(!pc->hostmode); + WARN_ON(!pc->hostmode); if (unlikely(len != 1 && len != 2 && len != 4)) goto out; addr = get_cfgspace_addr(pc, bus, dev, func, off); @@ -702,7 +702,7 @@ int ssb_pcicore_dev_irqvecs_enable(struct ssb_pcicore *pc, /* Calculate the "coremask" for the device. */ coremask = (1 << dev->core_index); - SSB_WARN_ON(bus->bustype != SSB_BUSTYPE_PCI); + WARN_ON(bus->bustype != SSB_BUSTYPE_PCI); err = pci_read_config_dword(bus->host_pci, SSB_PCI_IRQMASK, &tmp); if (err) goto out; diff --git a/drivers/ssb/embedded.c b/drivers/ssb/embedded.c index c39ddf8988c1..8254ed25e063 100644 --- a/drivers/ssb/embedded.c +++ b/drivers/ssb/embedded.c @@ -77,7 +77,7 @@ u32 ssb_gpio_in(struct ssb_bus *bus, u32 mask) else if (ssb_extif_available(&bus->extif)) res = ssb_extif_gpio_in(&bus->extif, mask); else - SSB_WARN_ON(1); + WARN_ON(1); spin_unlock_irqrestore(&bus->gpio_lock, flags); return res; @@ -95,7 +95,7 @@ u32 ssb_gpio_out(struct ssb_bus *bus, u32 mask, u32 value) else if (ssb_extif_available(&bus->extif)) res = ssb_extif_gpio_out(&bus->extif, mask, value); else - SSB_WARN_ON(1); + WARN_ON(1); spin_unlock_irqrestore(&bus->gpio_lock, flags); return res; @@ -113,7 +113,7 @@ u32 ssb_gpio_outen(struct ssb_bus *bus, u32 mask, u32 value) else if (ssb_extif_available(&bus->extif)) res = ssb_extif_gpio_outen(&bus->extif, mask, value); else - SSB_WARN_ON(1); + WARN_ON(1); spin_unlock_irqrestore(&bus->gpio_lock, flags); return res; @@ -145,7 +145,7 @@ u32 ssb_gpio_intmask(struct ssb_bus *bus, u32 mask, u32 value) else if (ssb_extif_available(&bus->extif)) res = ssb_extif_gpio_intmask(&bus->extif, mask, value); else - SSB_WARN_ON(1); + WARN_ON(1); spin_unlock_irqrestore(&bus->gpio_lock, flags); return res; @@ -163,7 +163,7 @@ u32 ssb_gpio_polarity(struct ssb_bus *bus, u32 mask, u32 value) else if (ssb_extif_available(&bus->extif)) res = ssb_extif_gpio_polarity(&bus->extif, mask, value); else - SSB_WARN_ON(1); + WARN_ON(1); spin_unlock_irqrestore(&bus->gpio_lock, flags); return res; diff --git a/drivers/ssb/host_soc.c b/drivers/ssb/host_soc.c index eadaedf466f6..3b438480515c 100644 --- a/drivers/ssb/host_soc.c +++ b/drivers/ssb/host_soc.c @@ -61,7 +61,7 @@ static void ssb_host_soc_block_read(struct ssb_device *dev, void *buffer, case sizeof(u16): { __le16 *buf = buffer; - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); while (count) { *buf = (__force __le16)__raw_readw(addr); buf++; @@ -72,7 +72,7 @@ static void ssb_host_soc_block_read(struct ssb_device *dev, void *buffer, case sizeof(u32): { __le32 *buf = buffer; - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); while (count) { *buf = (__force __le32)__raw_readl(addr); buf++; @@ -81,7 +81,7 @@ static void ssb_host_soc_block_read(struct ssb_device *dev, void *buffer, break; } default: - SSB_WARN_ON(1); + WARN_ON(1); } } #endif /* CONFIG_SSB_BLOCKIO */ @@ -134,7 +134,7 @@ static void ssb_host_soc_block_write(struct ssb_device *dev, const void *buffer, case sizeof(u16): { const __le16 *buf = buffer; - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); while (count) { __raw_writew((__force u16)(*buf), addr); buf++; @@ -145,7 +145,7 @@ static void ssb_host_soc_block_write(struct ssb_device *dev, const void *buffer, case sizeof(u32): { const __le32 *buf = buffer; - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); while (count) { __raw_writel((__force u32)(*buf), addr); buf++; @@ -154,7 +154,7 @@ static void ssb_host_soc_block_write(struct ssb_device *dev, const void *buffer, break; } default: - SSB_WARN_ON(1); + WARN_ON(1); } } #endif /* CONFIG_SSB_BLOCKIO */ diff --git a/drivers/ssb/main.c b/drivers/ssb/main.c index 9da56d2fdbd3..0a26984acb2c 100644 --- a/drivers/ssb/main.c +++ b/drivers/ssb/main.c @@ -209,7 +209,7 @@ int ssb_devices_freeze(struct ssb_bus *bus, struct ssb_freeze_context *ctx) memset(ctx, 0, sizeof(*ctx)); ctx->bus = bus; - SSB_WARN_ON(bus->nr_devices > ARRAY_SIZE(ctx->device_frozen)); + WARN_ON(bus->nr_devices > ARRAY_SIZE(ctx->device_frozen)); for (i = 0; i < bus->nr_devices; i++) { sdev = ssb_device_get(&bus->devices[i]); @@ -220,7 +220,7 @@ int ssb_devices_freeze(struct ssb_bus *bus, struct ssb_freeze_context *ctx) continue; } sdrv = drv_to_ssb_drv(sdev->dev->driver); - if (SSB_WARN_ON(!sdrv->remove)) + if (WARN_ON(!sdrv->remove)) continue; sdrv->remove(sdev); ctx->device_frozen[i] = 1; @@ -248,10 +248,10 @@ int ssb_devices_thaw(struct ssb_freeze_context *ctx) continue; sdev = &bus->devices[i]; - if (SSB_WARN_ON(!sdev->dev || !sdev->dev->driver)) + if (WARN_ON(!sdev->dev || !sdev->dev->driver)) continue; sdrv = drv_to_ssb_drv(sdev->dev->driver); - if (SSB_WARN_ON(!sdrv || !sdrv->probe)) + if (WARN_ON(!sdrv || !sdrv->probe)) continue; err = sdrv->probe(sdev, &sdev->id); @@ -861,13 +861,13 @@ u32 ssb_calc_clock_rate(u32 plltype, u32 n, u32 m) case SSB_PLLTYPE_2: /* 48Mhz, 4 dividers */ n1 += SSB_CHIPCO_CLK_T2_BIAS; n2 += SSB_CHIPCO_CLK_T2_BIAS; - SSB_WARN_ON(!((n1 >= 2) && (n1 <= 7))); - SSB_WARN_ON(!((n2 >= 5) && (n2 <= 23))); + WARN_ON(!((n1 >= 2) && (n1 <= 7))); + WARN_ON(!((n2 >= 5) && (n2 <= 23))); break; case SSB_PLLTYPE_5: /* 25Mhz, 4 dividers */ return 100000000; default: - SSB_WARN_ON(1); + WARN_ON(1); } switch (plltype) { @@ -916,9 +916,9 @@ u32 ssb_calc_clock_rate(u32 plltype, u32 n, u32 m) m1 += SSB_CHIPCO_CLK_T2_BIAS; m2 += SSB_CHIPCO_CLK_T2M2_BIAS; m3 += SSB_CHIPCO_CLK_T2_BIAS; - SSB_WARN_ON(!((m1 >= 2) && (m1 <= 7))); - SSB_WARN_ON(!((m2 >= 3) && (m2 <= 10))); - SSB_WARN_ON(!((m3 >= 2) && (m3 <= 7))); + WARN_ON(!((m1 >= 2) && (m1 <= 7))); + WARN_ON(!((m2 >= 3) && (m2 <= 10))); + WARN_ON(!((m3 >= 2) && (m3 <= 7))); if (!(mc & SSB_CHIPCO_CLK_T2MC_M1BYP)) clock /= m1; @@ -928,7 +928,7 @@ u32 ssb_calc_clock_rate(u32 plltype, u32 n, u32 m) clock /= m3; return clock; default: - SSB_WARN_ON(1); + WARN_ON(1); } return 0; } @@ -1169,9 +1169,7 @@ int ssb_bus_may_powerdown(struct ssb_bus *bus) if (err) goto error; out: -#ifdef CONFIG_SSB_DEBUG bus->powered_up = 0; -#endif return err; error: pr_err("Bus powerdown failed\n"); @@ -1188,9 +1186,7 @@ int ssb_bus_powerup(struct ssb_bus *bus, bool dynamic_pctl) if (err) goto error; -#ifdef CONFIG_SSB_DEBUG bus->powered_up = 1; -#endif mode = dynamic_pctl ? SSB_CLKMODE_DYNAMIC : SSB_CLKMODE_FAST; ssb_chipco_set_clockmode(&bus->chipco, mode); @@ -1242,15 +1238,15 @@ u32 ssb_admatch_base(u32 adm) base = (adm & SSB_ADM_BASE0); break; case SSB_ADM_TYPE1: - SSB_WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ + WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ base = (adm & SSB_ADM_BASE1); break; case SSB_ADM_TYPE2: - SSB_WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ + WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ base = (adm & SSB_ADM_BASE2); break; default: - SSB_WARN_ON(1); + WARN_ON(1); } return base; @@ -1266,15 +1262,15 @@ u32 ssb_admatch_size(u32 adm) size = ((adm & SSB_ADM_SZ0) >> SSB_ADM_SZ0_SHIFT); break; case SSB_ADM_TYPE1: - SSB_WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ + WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ size = ((adm & SSB_ADM_SZ1) >> SSB_ADM_SZ1_SHIFT); break; case SSB_ADM_TYPE2: - SSB_WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ + WARN_ON(adm & SSB_ADM_NEG); /* unsupported */ size = ((adm & SSB_ADM_SZ2) >> SSB_ADM_SZ2_SHIFT); break; default: - SSB_WARN_ON(1); + WARN_ON(1); } size = (1 << (size + 1)); diff --git a/drivers/ssb/pci.c b/drivers/ssb/pci.c index ad4308529eba..84807a9b4b13 100644 --- a/drivers/ssb/pci.c +++ b/drivers/ssb/pci.c @@ -946,7 +946,6 @@ out: return err; } -#ifdef CONFIG_SSB_DEBUG static int ssb_pci_assert_buspower(struct ssb_bus *bus) { if (likely(bus->powered_up)) @@ -960,12 +959,6 @@ static int ssb_pci_assert_buspower(struct ssb_bus *bus) return -ENODEV; } -#else /* DEBUG */ -static inline int ssb_pci_assert_buspower(struct ssb_bus *bus) -{ - return 0; -} -#endif /* DEBUG */ static u8 ssb_pci_read8(struct ssb_device *dev, u16 offset) { @@ -1024,15 +1017,15 @@ static void ssb_pci_block_read(struct ssb_device *dev, void *buffer, ioread8_rep(addr, buffer, count); break; case sizeof(u16): - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); ioread16_rep(addr, buffer, count >> 1); break; case sizeof(u32): - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); ioread32_rep(addr, buffer, count >> 2); break; default: - SSB_WARN_ON(1); + WARN_ON(1); } return; @@ -1098,15 +1091,15 @@ static void ssb_pci_block_write(struct ssb_device *dev, const void *buffer, iowrite8_rep(addr, buffer, count); break; case sizeof(u16): - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); iowrite16_rep(addr, buffer, count >> 1); break; case sizeof(u32): - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); iowrite32_rep(addr, buffer, count >> 2); break; default: - SSB_WARN_ON(1); + WARN_ON(1); } } #endif /* CONFIG_SSB_BLOCKIO */ diff --git a/drivers/ssb/pcmcia.c b/drivers/ssb/pcmcia.c index 20f63cc88e70..567013f8a8be 100644 --- a/drivers/ssb/pcmcia.c +++ b/drivers/ssb/pcmcia.c @@ -169,7 +169,7 @@ int ssb_pcmcia_switch_segment(struct ssb_bus *bus, u8 seg) int err; u8 val; - SSB_WARN_ON((seg != 0) && (seg != 1)); + WARN_ON((seg != 0) && (seg != 1)); while (1) { err = ssb_pcmcia_cfg_write(bus, SSB_PCMCIA_MEMSEG, seg); if (err) @@ -299,7 +299,7 @@ static void ssb_pcmcia_block_read(struct ssb_device *dev, void *buffer, case sizeof(u16): { __le16 *buf = buffer; - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); while (count) { *buf = (__force __le16)__raw_readw(addr); buf++; @@ -310,7 +310,7 @@ static void ssb_pcmcia_block_read(struct ssb_device *dev, void *buffer, case sizeof(u32): { __le16 *buf = buffer; - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); while (count) { *buf = (__force __le16)__raw_readw(addr); buf++; @@ -321,7 +321,7 @@ static void ssb_pcmcia_block_read(struct ssb_device *dev, void *buffer, break; } default: - SSB_WARN_ON(1); + WARN_ON(1); } unlock: spin_unlock_irqrestore(&bus->bar_lock, flags); @@ -399,7 +399,7 @@ static void ssb_pcmcia_block_write(struct ssb_device *dev, const void *buffer, case sizeof(u16): { const __le16 *buf = buffer; - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); while (count) { __raw_writew((__force u16)(*buf), addr); buf++; @@ -410,7 +410,7 @@ static void ssb_pcmcia_block_write(struct ssb_device *dev, const void *buffer, case sizeof(u32): { const __le16 *buf = buffer; - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); while (count) { __raw_writew((__force u16)(*buf), addr); buf++; @@ -421,7 +421,7 @@ static void ssb_pcmcia_block_write(struct ssb_device *dev, const void *buffer, break; } default: - SSB_WARN_ON(1); + WARN_ON(1); } unlock: mmiowb(); diff --git a/drivers/ssb/scan.c b/drivers/ssb/scan.c index 71fbb4b3eb6a..6ceee98ed6ff 100644 --- a/drivers/ssb/scan.c +++ b/drivers/ssb/scan.c @@ -210,7 +210,7 @@ void ssb_iounmap(struct ssb_bus *bus) #ifdef CONFIG_SSB_PCIHOST pci_iounmap(bus->host_pci, bus->mmio); #else - SSB_BUG_ON(1); /* Can't reach this code. */ + WARN_ON(1); /* Can't reach this code. */ #endif break; case SSB_BUSTYPE_SDIO: @@ -236,7 +236,7 @@ static void __iomem *ssb_ioremap(struct ssb_bus *bus, #ifdef CONFIG_SSB_PCIHOST mmio = pci_iomap(bus->host_pci, 0, ~0UL); #else - SSB_BUG_ON(1); /* Can't reach this code. */ + WARN_ON(1); /* Can't reach this code. */ #endif break; case SSB_BUSTYPE_SDIO: diff --git a/drivers/ssb/sdio.c b/drivers/ssb/sdio.c index 1aedc5fbb62f..7fe0afb42234 100644 --- a/drivers/ssb/sdio.c +++ b/drivers/ssb/sdio.c @@ -316,18 +316,18 @@ static void ssb_sdio_block_read(struct ssb_device *dev, void *buffer, break; } case sizeof(u16): { - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); error = sdio_readsb(bus->host_sdio, buffer, offset, count); break; } case sizeof(u32): { - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); offset |= SBSDIO_SB_ACCESS_2_4B_FLAG; /* 32 bit data access */ error = sdio_readsb(bus->host_sdio, buffer, offset, count); break; } default: - SSB_WARN_ON(1); + WARN_ON(1); } if (!error) goto out; @@ -423,18 +423,18 @@ static void ssb_sdio_block_write(struct ssb_device *dev, const void *buffer, (void *)buffer, count); break; case sizeof(u16): - SSB_WARN_ON(count & 1); + WARN_ON(count & 1); error = sdio_writesb(bus->host_sdio, offset, (void *)buffer, count); break; case sizeof(u32): - SSB_WARN_ON(count & 3); + WARN_ON(count & 3); offset |= SBSDIO_SB_ACCESS_2_4B_FLAG; /* 32 bit data access */ error = sdio_writesb(bus->host_sdio, offset, (void *)buffer, count); break; default: - SSB_WARN_ON(1); + WARN_ON(1); } if (!error) goto out; diff --git a/drivers/ssb/ssb_private.h b/drivers/ssb/ssb_private.h index 885e278c4487..5f31bdfbe77f 100644 --- a/drivers/ssb/ssb_private.h +++ b/drivers/ssb/ssb_private.h @@ -9,15 +9,6 @@ #include #include -#ifdef CONFIG_SSB_DEBUG -# define SSB_WARN_ON(x) WARN_ON(x) -# define SSB_BUG_ON(x) BUG_ON(x) -#else -static inline int __ssb_do_nothing(int x) { return x; } -# define SSB_WARN_ON(x) __ssb_do_nothing(unlikely(!!(x))) -# define SSB_BUG_ON(x) __ssb_do_nothing(unlikely(!!(x))) -#endif - /* pci.c */ #ifdef CONFIG_SSB_PCIHOST diff --git a/include/linux/ssb/ssb.h b/include/linux/ssb/ssb.h index 3b43655cabe6..0d5a2691e7e9 100644 --- a/include/linux/ssb/ssb.h +++ b/include/linux/ssb/ssb.h @@ -499,11 +499,9 @@ struct ssb_bus { /* Internal-only stuff follows. Do not touch. */ struct list_head list; -#ifdef CONFIG_SSB_DEBUG /* Is the bus already powered up? */ bool powered_up; int power_warn_count; -#endif /* DEBUG */ }; enum ssb_quirks { -- cgit v1.2.3