From 81d2c6f81996e01fbcd2b5aeefbb519e21c806e9 Mon Sep 17 00:00:00 2001 From: Daniel Axtens Date: Tue, 20 Aug 2019 12:49:40 +1000 Subject: kasan: support instrumented bitops combined with generic bitops Currently bitops-instrumented.h assumes that the architecture provides atomic, non-atomic and locking bitops (e.g. both set_bit and __set_bit). This is true on x86 and s390, but is not always true: there is a generic bitops/non-atomic.h header that provides generic non-atomic operations, and also a generic bitops/lock.h for locking operations. powerpc uses the generic non-atomic version, so it does not have it's own e.g. __set_bit that could be renamed arch___set_bit. Split up bitops-instrumented.h to mirror the atomic/non-atomic/lock split. This allows arches to only include the headers where they have arch-specific versions to rename. Update x86 and s390. (The generic operations are automatically instrumented because they're written in C, not asm.) Suggested-by: Christophe Leroy Reviewed-by: Christophe Leroy Signed-off-by: Daniel Axtens Acked-by: Marco Elver Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20190820024941.12640-1-dja@axtens.net --- Documentation/core-api/kernel-api.rst | 17 +- arch/s390/include/asm/bitops.h | 4 +- arch/x86/include/asm/bitops.h | 4 +- include/asm-generic/bitops-instrumented.h | 263 --------------------- include/asm-generic/bitops/instrumented-atomic.h | 100 ++++++++ include/asm-generic/bitops/instrumented-lock.h | 81 +++++++ .../asm-generic/bitops/instrumented-non-atomic.h | 114 +++++++++ 7 files changed, 317 insertions(+), 266 deletions(-) delete mode 100644 include/asm-generic/bitops-instrumented.h create mode 100644 include/asm-generic/bitops/instrumented-atomic.h create mode 100644 include/asm-generic/bitops/instrumented-lock.h create mode 100644 include/asm-generic/bitops/instrumented-non-atomic.h diff --git a/Documentation/core-api/kernel-api.rst b/Documentation/core-api/kernel-api.rst index f77de49b1d51..2caaeb55e8dd 100644 --- a/Documentation/core-api/kernel-api.rst +++ b/Documentation/core-api/kernel-api.rst @@ -57,7 +57,22 @@ The Linux kernel provides more basic utility functions. Bit Operations -------------- -.. kernel-doc:: include/asm-generic/bitops-instrumented.h +Atomic Operations +~~~~~~~~~~~~~~~~~ + +.. kernel-doc:: include/asm-generic/bitops/instrumented-atomic.h + :internal: + +Non-atomic Operations +~~~~~~~~~~~~~~~~~~~~~ + +.. kernel-doc:: include/asm-generic/bitops/instrumented-non-atomic.h + :internal: + +Locking Operations +~~~~~~~~~~~~~~~~~~ + +.. kernel-doc:: include/asm-generic/bitops/instrumented-lock.h :internal: Bitmap Operations diff --git a/arch/s390/include/asm/bitops.h b/arch/s390/include/asm/bitops.h index eb7eed43e780..431e208a5ea4 100644 --- a/arch/s390/include/asm/bitops.h +++ b/arch/s390/include/asm/bitops.h @@ -241,7 +241,9 @@ static inline void arch___clear_bit_unlock(unsigned long nr, arch___clear_bit(nr, ptr); } -#include +#include +#include +#include /* * Functions which use MSB0 bit numbering. diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h index 7d1f6a49bfae..062cdecb2f24 100644 --- a/arch/x86/include/asm/bitops.h +++ b/arch/x86/include/asm/bitops.h @@ -388,7 +388,9 @@ static __always_inline int fls64(__u64 x) #include -#include +#include +#include +#include #include diff --git a/include/asm-generic/bitops-instrumented.h b/include/asm-generic/bitops-instrumented.h deleted file mode 100644 index ddd1c6d9d8db..000000000000 --- a/include/asm-generic/bitops-instrumented.h +++ /dev/null @@ -1,263 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -/* - * This file provides wrappers with sanitizer instrumentation for bit - * operations. - * - * To use this functionality, an arch's bitops.h file needs to define each of - * the below bit operations with an arch_ prefix (e.g. arch_set_bit(), - * arch___set_bit(), etc.). - */ -#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_H -#define _ASM_GENERIC_BITOPS_INSTRUMENTED_H - -#include - -/** - * set_bit - Atomically set a bit in memory - * @nr: the bit to set - * @addr: the address to start counting from - * - * This is a relaxed atomic operation (no implied memory barriers). - * - * Note that @nr may be almost arbitrarily large; this function is not - * restricted to acting on a single-word quantity. - */ -static inline void set_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch_set_bit(nr, addr); -} - -/** - * __set_bit - Set a bit in memory - * @nr: the bit to set - * @addr: the address to start counting from - * - * Unlike set_bit(), this function is non-atomic. If it is called on the same - * region of memory concurrently, the effect may be that only one operation - * succeeds. - */ -static inline void __set_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch___set_bit(nr, addr); -} - -/** - * clear_bit - Clears a bit in memory - * @nr: Bit to clear - * @addr: Address to start counting from - * - * This is a relaxed atomic operation (no implied memory barriers). - */ -static inline void clear_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch_clear_bit(nr, addr); -} - -/** - * __clear_bit - Clears a bit in memory - * @nr: the bit to clear - * @addr: the address to start counting from - * - * Unlike clear_bit(), this function is non-atomic. If it is called on the same - * region of memory concurrently, the effect may be that only one operation - * succeeds. - */ -static inline void __clear_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch___clear_bit(nr, addr); -} - -/** - * clear_bit_unlock - Clear a bit in memory, for unlock - * @nr: the bit to set - * @addr: the address to start counting from - * - * This operation is atomic and provides release barrier semantics. - */ -static inline void clear_bit_unlock(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch_clear_bit_unlock(nr, addr); -} - -/** - * __clear_bit_unlock - Clears a bit in memory - * @nr: Bit to clear - * @addr: Address to start counting from - * - * This is a non-atomic operation but implies a release barrier before the - * memory operation. It can be used for an unlock if no other CPUs can - * concurrently modify other bits in the word. - */ -static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch___clear_bit_unlock(nr, addr); -} - -/** - * change_bit - Toggle a bit in memory - * @nr: Bit to change - * @addr: Address to start counting from - * - * This is a relaxed atomic operation (no implied memory barriers). - * - * Note that @nr may be almost arbitrarily large; this function is not - * restricted to acting on a single-word quantity. - */ -static inline void change_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch_change_bit(nr, addr); -} - -/** - * __change_bit - Toggle a bit in memory - * @nr: the bit to change - * @addr: the address to start counting from - * - * Unlike change_bit(), this function is non-atomic. If it is called on the same - * region of memory concurrently, the effect may be that only one operation - * succeeds. - */ -static inline void __change_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch___change_bit(nr, addr); -} - -/** - * test_and_set_bit - Set a bit and return its old value - * @nr: Bit to set - * @addr: Address to count from - * - * This is an atomic fully-ordered operation (implied full memory barrier). - */ -static inline bool test_and_set_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch_test_and_set_bit(nr, addr); -} - -/** - * __test_and_set_bit - Set a bit and return its old value - * @nr: Bit to set - * @addr: Address to count from - * - * This operation is non-atomic. If two instances of this operation race, one - * can appear to succeed but actually fail. - */ -static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch___test_and_set_bit(nr, addr); -} - -/** - * test_and_set_bit_lock - Set a bit and return its old value, for lock - * @nr: Bit to set - * @addr: Address to count from - * - * This operation is atomic and provides acquire barrier semantics if - * the returned value is 0. - * It can be used to implement bit locks. - */ -static inline bool test_and_set_bit_lock(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch_test_and_set_bit_lock(nr, addr); -} - -/** - * test_and_clear_bit - Clear a bit and return its old value - * @nr: Bit to clear - * @addr: Address to count from - * - * This is an atomic fully-ordered operation (implied full memory barrier). - */ -static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch_test_and_clear_bit(nr, addr); -} - -/** - * __test_and_clear_bit - Clear a bit and return its old value - * @nr: Bit to clear - * @addr: Address to count from - * - * This operation is non-atomic. If two instances of this operation race, one - * can appear to succeed but actually fail. - */ -static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch___test_and_clear_bit(nr, addr); -} - -/** - * test_and_change_bit - Change a bit and return its old value - * @nr: Bit to change - * @addr: Address to count from - * - * This is an atomic fully-ordered operation (implied full memory barrier). - */ -static inline bool test_and_change_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch_test_and_change_bit(nr, addr); -} - -/** - * __test_and_change_bit - Change a bit and return its old value - * @nr: Bit to change - * @addr: Address to count from - * - * This operation is non-atomic. If two instances of this operation race, one - * can appear to succeed but actually fail. - */ -static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch___test_and_change_bit(nr, addr); -} - -/** - * test_bit - Determine whether a bit is set - * @nr: bit number to test - * @addr: Address to start counting from - */ -static inline bool test_bit(long nr, const volatile unsigned long *addr) -{ - kasan_check_read(addr + BIT_WORD(nr), sizeof(long)); - return arch_test_bit(nr, addr); -} - -#if defined(arch_clear_bit_unlock_is_negative_byte) -/** - * clear_bit_unlock_is_negative_byte - Clear a bit in memory and test if bottom - * byte is negative, for unlock. - * @nr: the bit to clear - * @addr: the address to start counting from - * - * This operation is atomic and provides release barrier semantics. - * - * This is a bit of a one-trick-pony for the filemap code, which clears - * PG_locked and tests PG_waiters, - */ -static inline bool -clear_bit_unlock_is_negative_byte(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch_clear_bit_unlock_is_negative_byte(nr, addr); -} -/* Let everybody know we have it. */ -#define clear_bit_unlock_is_negative_byte clear_bit_unlock_is_negative_byte -#endif - -#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_H */ diff --git a/include/asm-generic/bitops/instrumented-atomic.h b/include/asm-generic/bitops/instrumented-atomic.h new file mode 100644 index 000000000000..18ce3c9e8eec --- /dev/null +++ b/include/asm-generic/bitops/instrumented-atomic.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * This file provides wrappers with sanitizer instrumentation for atomic bit + * operations. + * + * To use this functionality, an arch's bitops.h file needs to define each of + * the below bit operations with an arch_ prefix (e.g. arch_set_bit(), + * arch___set_bit(), etc.). + */ +#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H +#define _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H + +#include + +/** + * set_bit - Atomically set a bit in memory + * @nr: the bit to set + * @addr: the address to start counting from + * + * This is a relaxed atomic operation (no implied memory barriers). + * + * Note that @nr may be almost arbitrarily large; this function is not + * restricted to acting on a single-word quantity. + */ +static inline void set_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch_set_bit(nr, addr); +} + +/** + * clear_bit - Clears a bit in memory + * @nr: Bit to clear + * @addr: Address to start counting from + * + * This is a relaxed atomic operation (no implied memory barriers). + */ +static inline void clear_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch_clear_bit(nr, addr); +} + +/** + * change_bit - Toggle a bit in memory + * @nr: Bit to change + * @addr: Address to start counting from + * + * This is a relaxed atomic operation (no implied memory barriers). + * + * Note that @nr may be almost arbitrarily large; this function is not + * restricted to acting on a single-word quantity. + */ +static inline void change_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch_change_bit(nr, addr); +} + +/** + * test_and_set_bit - Set a bit and return its old value + * @nr: Bit to set + * @addr: Address to count from + * + * This is an atomic fully-ordered operation (implied full memory barrier). + */ +static inline bool test_and_set_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_and_set_bit(nr, addr); +} + +/** + * test_and_clear_bit - Clear a bit and return its old value + * @nr: Bit to clear + * @addr: Address to count from + * + * This is an atomic fully-ordered operation (implied full memory barrier). + */ +static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_and_clear_bit(nr, addr); +} + +/** + * test_and_change_bit - Change a bit and return its old value + * @nr: Bit to change + * @addr: Address to count from + * + * This is an atomic fully-ordered operation (implied full memory barrier). + */ +static inline bool test_and_change_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_and_change_bit(nr, addr); +} + +#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */ diff --git a/include/asm-generic/bitops/instrumented-lock.h b/include/asm-generic/bitops/instrumented-lock.h new file mode 100644 index 000000000000..ec53fdeea9ec --- /dev/null +++ b/include/asm-generic/bitops/instrumented-lock.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * This file provides wrappers with sanitizer instrumentation for bit + * locking operations. + * + * To use this functionality, an arch's bitops.h file needs to define each of + * the below bit operations with an arch_ prefix (e.g. arch_set_bit(), + * arch___set_bit(), etc.). + */ +#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H +#define _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H + +#include + +/** + * clear_bit_unlock - Clear a bit in memory, for unlock + * @nr: the bit to set + * @addr: the address to start counting from + * + * This operation is atomic and provides release barrier semantics. + */ +static inline void clear_bit_unlock(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch_clear_bit_unlock(nr, addr); +} + +/** + * __clear_bit_unlock - Clears a bit in memory + * @nr: Bit to clear + * @addr: Address to start counting from + * + * This is a non-atomic operation but implies a release barrier before the + * memory operation. It can be used for an unlock if no other CPUs can + * concurrently modify other bits in the word. + */ +static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch___clear_bit_unlock(nr, addr); +} + +/** + * test_and_set_bit_lock - Set a bit and return its old value, for lock + * @nr: Bit to set + * @addr: Address to count from + * + * This operation is atomic and provides acquire barrier semantics if + * the returned value is 0. + * It can be used to implement bit locks. + */ +static inline bool test_and_set_bit_lock(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_and_set_bit_lock(nr, addr); +} + +#if defined(arch_clear_bit_unlock_is_negative_byte) +/** + * clear_bit_unlock_is_negative_byte - Clear a bit in memory and test if bottom + * byte is negative, for unlock. + * @nr: the bit to clear + * @addr: the address to start counting from + * + * This operation is atomic and provides release barrier semantics. + * + * This is a bit of a one-trick-pony for the filemap code, which clears + * PG_locked and tests PG_waiters, + */ +static inline bool +clear_bit_unlock_is_negative_byte(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch_clear_bit_unlock_is_negative_byte(nr, addr); +} +/* Let everybody know we have it. */ +#define clear_bit_unlock_is_negative_byte clear_bit_unlock_is_negative_byte +#endif + +#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H */ diff --git a/include/asm-generic/bitops/instrumented-non-atomic.h b/include/asm-generic/bitops/instrumented-non-atomic.h new file mode 100644 index 000000000000..95ff28d128a1 --- /dev/null +++ b/include/asm-generic/bitops/instrumented-non-atomic.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * This file provides wrappers with sanitizer instrumentation for non-atomic + * bit operations. + * + * To use this functionality, an arch's bitops.h file needs to define each of + * the below bit operations with an arch_ prefix (e.g. arch_set_bit(), + * arch___set_bit(), etc.). + */ +#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H +#define _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H + +#include + +/** + * __set_bit - Set a bit in memory + * @nr: the bit to set + * @addr: the address to start counting from + * + * Unlike set_bit(), this function is non-atomic. If it is called on the same + * region of memory concurrently, the effect may be that only one operation + * succeeds. + */ +static inline void __set_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch___set_bit(nr, addr); +} + +/** + * __clear_bit - Clears a bit in memory + * @nr: the bit to clear + * @addr: the address to start counting from + * + * Unlike clear_bit(), this function is non-atomic. If it is called on the same + * region of memory concurrently, the effect may be that only one operation + * succeeds. + */ +static inline void __clear_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch___clear_bit(nr, addr); +} + +/** + * __change_bit - Toggle a bit in memory + * @nr: the bit to change + * @addr: the address to start counting from + * + * Unlike change_bit(), this function is non-atomic. If it is called on the same + * region of memory concurrently, the effect may be that only one operation + * succeeds. + */ +static inline void __change_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch___change_bit(nr, addr); +} + +/** + * __test_and_set_bit - Set a bit and return its old value + * @nr: Bit to set + * @addr: Address to count from + * + * This operation is non-atomic. If two instances of this operation race, one + * can appear to succeed but actually fail. + */ +static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch___test_and_set_bit(nr, addr); +} + +/** + * __test_and_clear_bit - Clear a bit and return its old value + * @nr: Bit to clear + * @addr: Address to count from + * + * This operation is non-atomic. If two instances of this operation race, one + * can appear to succeed but actually fail. + */ +static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch___test_and_clear_bit(nr, addr); +} + +/** + * __test_and_change_bit - Change a bit and return its old value + * @nr: Bit to change + * @addr: Address to count from + * + * This operation is non-atomic. If two instances of this operation race, one + * can appear to succeed but actually fail. + */ +static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch___test_and_change_bit(nr, addr); +} + +/** + * test_bit - Determine whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static inline bool test_bit(long nr, const volatile unsigned long *addr) +{ + kasan_check_read(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_bit(nr, addr); +} + +#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */ -- cgit v1.2.3 From 5bece3d66153d78f1fd62108a1553c3f15e71412 Mon Sep 17 00:00:00 2001 From: Daniel Axtens Date: Tue, 20 Aug 2019 12:49:41 +1000 Subject: powerpc: support KASAN instrumentation of bitops The powerpc-specific bitops are not being picked up by the KASAN test suite. Instrumentation is done via the bitops/instrumented-{atomic,lock}.h headers. They require that arch-specific versions of bitop functions are renamed to arch_*. Do this renaming. For clear_bit_unlock_is_negative_byte, the current implementation uses the PG_waiters constant. This works because it's a preprocessor macro - so it's only actually evaluated in contexts where PG_waiters is defined. With instrumentation however, it becomes a static inline function, and all of a sudden we need the actual value of PG_waiters. Because of the order of header includes, it's not available and we fail to compile. Instead, manually specify that we care about bit 7. This is still correct: bit 7 is the bit that would mark a negative byte. While we're at it, replace __inline__ with inline across the file. Reviewed-by: Christophe Leroy Signed-off-by: Daniel Axtens Tested-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20190820024941.12640-2-dja@axtens.net --- arch/powerpc/include/asm/bitops.h | 51 ++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/arch/powerpc/include/asm/bitops.h b/arch/powerpc/include/asm/bitops.h index 603aed229af7..28dcf8222943 100644 --- a/arch/powerpc/include/asm/bitops.h +++ b/arch/powerpc/include/asm/bitops.h @@ -64,7 +64,7 @@ /* Macro for generating the ***_bits() functions */ #define DEFINE_BITOP(fn, op, prefix) \ -static __inline__ void fn(unsigned long mask, \ +static inline void fn(unsigned long mask, \ volatile unsigned long *_p) \ { \ unsigned long old; \ @@ -86,22 +86,22 @@ DEFINE_BITOP(clear_bits, andc, "") DEFINE_BITOP(clear_bits_unlock, andc, PPC_RELEASE_BARRIER) DEFINE_BITOP(change_bits, xor, "") -static __inline__ void set_bit(int nr, volatile unsigned long *addr) +static inline void arch_set_bit(int nr, volatile unsigned long *addr) { set_bits(BIT_MASK(nr), addr + BIT_WORD(nr)); } -static __inline__ void clear_bit(int nr, volatile unsigned long *addr) +static inline void arch_clear_bit(int nr, volatile unsigned long *addr) { clear_bits(BIT_MASK(nr), addr + BIT_WORD(nr)); } -static __inline__ void clear_bit_unlock(int nr, volatile unsigned long *addr) +static inline void arch_clear_bit_unlock(int nr, volatile unsigned long *addr) { clear_bits_unlock(BIT_MASK(nr), addr + BIT_WORD(nr)); } -static __inline__ void change_bit(int nr, volatile unsigned long *addr) +static inline void arch_change_bit(int nr, volatile unsigned long *addr) { change_bits(BIT_MASK(nr), addr + BIT_WORD(nr)); } @@ -109,7 +109,7 @@ static __inline__ void change_bit(int nr, volatile unsigned long *addr) /* Like DEFINE_BITOP(), with changes to the arguments to 'op' and the output * operands. */ #define DEFINE_TESTOP(fn, op, prefix, postfix, eh) \ -static __inline__ unsigned long fn( \ +static inline unsigned long fn( \ unsigned long mask, \ volatile unsigned long *_p) \ { \ @@ -138,34 +138,34 @@ DEFINE_TESTOP(test_and_clear_bits, andc, PPC_ATOMIC_ENTRY_BARRIER, DEFINE_TESTOP(test_and_change_bits, xor, PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, 0) -static __inline__ int test_and_set_bit(unsigned long nr, - volatile unsigned long *addr) +static inline int arch_test_and_set_bit(unsigned long nr, + volatile unsigned long *addr) { return test_and_set_bits(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0; } -static __inline__ int test_and_set_bit_lock(unsigned long nr, - volatile unsigned long *addr) +static inline int arch_test_and_set_bit_lock(unsigned long nr, + volatile unsigned long *addr) { return test_and_set_bits_lock(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0; } -static __inline__ int test_and_clear_bit(unsigned long nr, - volatile unsigned long *addr) +static inline int arch_test_and_clear_bit(unsigned long nr, + volatile unsigned long *addr) { return test_and_clear_bits(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0; } -static __inline__ int test_and_change_bit(unsigned long nr, - volatile unsigned long *addr) +static inline int arch_test_and_change_bit(unsigned long nr, + volatile unsigned long *addr) { return test_and_change_bits(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0; } #ifdef CONFIG_PPC64 -static __inline__ unsigned long clear_bit_unlock_return_word(int nr, - volatile unsigned long *addr) +static inline unsigned long +clear_bit_unlock_return_word(int nr, volatile unsigned long *addr) { unsigned long old, t; unsigned long *p = (unsigned long *)addr + BIT_WORD(nr); @@ -185,15 +185,18 @@ static __inline__ unsigned long clear_bit_unlock_return_word(int nr, return old; } -/* This is a special function for mm/filemap.c */ -#define clear_bit_unlock_is_negative_byte(nr, addr) \ - (clear_bit_unlock_return_word(nr, addr) & BIT_MASK(PG_waiters)) +/* + * This is a special function for mm/filemap.c + * Bit 7 corresponds to PG_waiters. + */ +#define arch_clear_bit_unlock_is_negative_byte(nr, addr) \ + (clear_bit_unlock_return_word(nr, addr) & BIT_MASK(7)) #endif /* CONFIG_PPC64 */ #include -static __inline__ void __clear_bit_unlock(int nr, volatile unsigned long *addr) +static inline void arch___clear_bit_unlock(int nr, volatile unsigned long *addr) { __asm__ __volatile__(PPC_RELEASE_BARRIER "" ::: "memory"); __clear_bit(nr, addr); @@ -215,14 +218,14 @@ static __inline__ void __clear_bit_unlock(int nr, volatile unsigned long *addr) * fls: find last (most-significant) bit set. * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32. */ -static __inline__ int fls(unsigned int x) +static inline int fls(unsigned int x) { return 32 - __builtin_clz(x); } #include -static __inline__ int fls64(__u64 x) +static inline int fls64(__u64 x) { return 64 - __builtin_clzll(x); } @@ -239,6 +242,10 @@ unsigned long __arch_hweight64(__u64 w); #include +/* wrappers that deal with KASAN instrumentation */ +#include +#include + /* Little-endian versions */ #include -- cgit v1.2.3 From 4f4afc2c9599520300b3f2b3666d2034fca03df3 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 4 Dec 2019 21:20:28 +1100 Subject: docs/core-api: Remove possibly confusing sub-headings from Bit Operations The recent commit 81d2c6f81996 ("kasan: support instrumented bitops combined with generic bitops"), split the KASAN instrumented bitops into separate headers for atomic, non-atomic and locking operations. This was done to allow arches to include just the instrumented bitops they need, while also using some of the generic bitops in asm-generic/bitops (which are automatically instrumented). The generic bitops are already split into atomic, non-atomic and locking headers. This split required an update to kernel-api.rst because it included include/asm-generic/bitops-instrumented.h, which no longer exists. So now kernel-api.rst includes all three instrumented headers to get the definitions for all the bitops. When adding the three headers it seemed sensible to add sub-headings for each, ie. "Atomic", "Non-atomic" and "Locking". The confusion is that test_bit() is (and always has been) in non-atomic.h, but is documented elsewhere (atomic_bitops.txt) as being atomic. So having it appear under the "Non-atomic" heading is possibly confusing. Probably test_bit() should move from bitops/non-atomic.h to atomic.h, but that has flow on effects. For now just remove the newly added sub-headings in the documentation, so we at least aren't adding to the confusion about whether test_bit() is atomic or not. Signed-off-by: Michael Ellerman --- Documentation/core-api/kernel-api.rst | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Documentation/core-api/kernel-api.rst b/Documentation/core-api/kernel-api.rst index 2caaeb55e8dd..4ac53a1363f6 100644 --- a/Documentation/core-api/kernel-api.rst +++ b/Documentation/core-api/kernel-api.rst @@ -57,21 +57,12 @@ The Linux kernel provides more basic utility functions. Bit Operations -------------- -Atomic Operations -~~~~~~~~~~~~~~~~~ - .. kernel-doc:: include/asm-generic/bitops/instrumented-atomic.h :internal: -Non-atomic Operations -~~~~~~~~~~~~~~~~~~~~~ - .. kernel-doc:: include/asm-generic/bitops/instrumented-non-atomic.h :internal: -Locking Operations -~~~~~~~~~~~~~~~~~~ - .. kernel-doc:: include/asm-generic/bitops/instrumented-lock.h :internal: -- cgit v1.2.3