diff options
| -rw-r--r-- | include/linux/jump_label.h | 46 | ||||
| -rw-r--r-- | kernel/jump_label.c | 53 |
2 files changed, 76 insertions, 23 deletions
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 68904469fba1..661af564fae8 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -76,7 +76,6 @@ #include <linux/types.h> #include <linux/compiler.h> -#include <linux/bug.h> extern bool static_key_initialized; @@ -115,20 +114,8 @@ enum jump_label_type { struct module; -#include <linux/atomic.h> - #ifdef HAVE_JUMP_LABEL -static inline int static_key_count(struct static_key *key) -{ - /* - * -1 means the first static_key_slow_inc() is in progress. - * static_key_enabled() must return true, so return 1 here. - */ - int n = atomic_read(&key->enabled); - return n >= 0 ? n : 1; -} - #define JUMP_TYPE_FALSE 0UL #define JUMP_TYPE_TRUE 1UL #define JUMP_TYPE_MASK 1UL @@ -157,16 +144,29 @@ extern int jump_label_text_reserved(void *start, void *end); extern void static_key_slow_inc(struct static_key *key); extern void static_key_slow_dec(struct static_key *key); extern void jump_label_apply_nops(struct module *mod); +extern int static_key_count(struct static_key *key); +extern void static_key_enable(struct static_key *key); +extern void static_key_disable(struct static_key *key); +/* + * We should be using ATOMIC_INIT() for initializing .enabled, but + * the inclusion of atomic.h is problematic for inclusion of jump_label.h + * in 'low-level' headers. Thus, we are initializing .enabled with a + * raw value, but have added a BUILD_BUG_ON() to catch any issues in + * jump_label_init() see: kernel/jump_label.c. + */ #define STATIC_KEY_INIT_TRUE \ - { .enabled = ATOMIC_INIT(1), \ + { .enabled = { 1 }, \ .entries = (void *)JUMP_TYPE_TRUE } #define STATIC_KEY_INIT_FALSE \ - { .enabled = ATOMIC_INIT(0), \ + { .enabled = { 0 }, \ .entries = (void *)JUMP_TYPE_FALSE } #else /* !HAVE_JUMP_LABEL */ +#include <linux/atomic.h> +#include <linux/bug.h> + static inline int static_key_count(struct static_key *key) { return atomic_read(&key->enabled); @@ -216,14 +216,6 @@ static inline int jump_label_apply_nops(struct module *mod) return 0; } -#define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) } -#define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) } - -#endif /* HAVE_JUMP_LABEL */ - -#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE -#define jump_label_enabled static_key_enabled - static inline void static_key_enable(struct static_key *key) { int count = static_key_count(key); @@ -244,6 +236,14 @@ static inline void static_key_disable(struct static_key *key) static_key_slow_dec(key); } +#define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) } +#define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) } + +#endif /* HAVE_JUMP_LABEL */ + +#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE +#define jump_label_enabled static_key_enabled + /* -------------------------------------------------------------------------- */ /* diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 0dbea887d625..f19aa02a8f48 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -14,6 +14,7 @@ #include <linux/err.h> #include <linux/static_key.h> #include <linux/jump_label_ratelimit.h> +#include <linux/bug.h> #ifdef HAVE_JUMP_LABEL @@ -56,6 +57,49 @@ jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) static void jump_label_update(struct static_key *key); +/* + * There are similar definitions for the !HAVE_JUMP_LABEL case in jump_label.h. + * The use of 'atomic_read()' requires atomic.h and its problematic for some + * kernel headers such as kernel.h and others. Since static_key_count() is not + * used in the branch statements as it is for the !HAVE_JUMP_LABEL case its ok + * to have it be a function here. Similarly, for 'static_key_enable()' and + * 'static_key_disable()', which require bug.h. This should allow jump_label.h + * to be included from most/all places for HAVE_JUMP_LABEL. + */ +int static_key_count(struct static_key *key) +{ + /* + * -1 means the first static_key_slow_inc() is in progress. + * static_key_enabled() must return true, so return 1 here. + */ + int n = atomic_read(&key->enabled); + + return n >= 0 ? n : 1; +} +EXPORT_SYMBOL_GPL(static_key_count); + +void static_key_enable(struct static_key *key) +{ + int count = static_key_count(key); + + WARN_ON_ONCE(count < 0 || count > 1); + + if (!count) + static_key_slow_inc(key); +} +EXPORT_SYMBOL_GPL(static_key_enable); + +void static_key_disable(struct static_key *key) +{ + int count = static_key_count(key); + + WARN_ON_ONCE(count < 0 || count > 1); + + if (count) + static_key_slow_dec(key); +} +EXPORT_SYMBOL_GPL(static_key_disable); + void static_key_slow_inc(struct static_key *key) { int v, v1; @@ -235,6 +279,15 @@ void __init jump_label_init(void) struct static_key *key = NULL; struct jump_entry *iter; + /* + * Since we are initializing the static_key.enabled field with + * with the 'raw' int values (to avoid pulling in atomic.h) in + * jump_label.h, let's make sure that is safe. There are only two + * cases to check since we initialize to 0 or 1. + */ + BUILD_BUG_ON((int)ATOMIC_INIT(0) != 0); + BUILD_BUG_ON((int)ATOMIC_INIT(1) != 1); + jump_label_lock(); jump_label_sort_entries(iter_start, iter_stop); |