diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/Kconfig | 7 | ||||
| -rw-r--r-- | lib/debugobjects.c | 55 | ||||
| -rw-r--r-- | lib/iov_iter.c | 48 | ||||
| -rw-r--r-- | lib/percpu_counter.c | 4 |
4 files changed, 71 insertions, 43 deletions
diff --git a/lib/Kconfig b/lib/Kconfig index b4b98a03ff98..b46a9fd122c8 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -635,7 +635,12 @@ config UACCESS_MEMCPY config ARCH_HAS_UACCESS_FLUSHCACHE bool -config ARCH_HAS_UACCESS_MCSAFE +# arch has a concept of a recoverable synchronous exception due to a +# memory-read error like x86 machine-check or ARM data-abort, and +# implements copy_mc_to_{user,kernel} to abort and report +# 'bytes-transferred' if that exception fires when accessing the source +# buffer. +config ARCH_HAS_COPY_MC bool # Temporary. Goes away when all archs are cleaned up diff --git a/lib/debugobjects.c b/lib/debugobjects.c index fe4557955d97..9e14ae02306b 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -19,6 +19,7 @@ #include <linux/slab.h> #include <linux/hash.h> #include <linux/kmemleak.h> +#include <linux/cpu.h> #define ODEBUG_HASH_BITS 14 #define ODEBUG_HASH_SIZE (1 << ODEBUG_HASH_BITS) @@ -90,7 +91,7 @@ static int debug_objects_pool_size __read_mostly = ODEBUG_POOL_SIZE; static int debug_objects_pool_min_level __read_mostly = ODEBUG_POOL_MIN_LEVEL; -static struct debug_obj_descr *descr_test __read_mostly; +static const struct debug_obj_descr *descr_test __read_mostly; static struct kmem_cache *obj_cache __read_mostly; /* @@ -223,7 +224,7 @@ static struct debug_obj *__alloc_object(struct hlist_head *list) * Must be called with interrupts disabled. */ static struct debug_obj * -alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr) +alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr) { struct debug_percpu_free *percpu_pool = this_cpu_ptr(&percpu_obj_pool); struct debug_obj *obj; @@ -433,6 +434,25 @@ static void free_object(struct debug_obj *obj) } } +#ifdef CONFIG_HOTPLUG_CPU +static int object_cpu_offline(unsigned int cpu) +{ + struct debug_percpu_free *percpu_pool; + struct hlist_node *tmp; + struct debug_obj *obj; + + /* Remote access is safe as the CPU is dead already */ + percpu_pool = per_cpu_ptr(&percpu_obj_pool, cpu); + hlist_for_each_entry_safe(obj, tmp, &percpu_pool->free_objs, node) { + hlist_del(&obj->node); + kmem_cache_free(obj_cache, obj); + } + percpu_pool->obj_free = 0; + + return 0; +} +#endif + /* * We run out of memory. That means we probably have tons of objects * allocated. @@ -475,7 +495,7 @@ static struct debug_bucket *get_bucket(unsigned long addr) static void debug_print_object(struct debug_obj *obj, char *msg) { - struct debug_obj_descr *descr = obj->descr; + const struct debug_obj_descr *descr = obj->descr; static int limit; if (limit < 5 && descr != descr_test) { @@ -529,7 +549,7 @@ static void debug_object_is_on_stack(void *addr, int onstack) } static void -__debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack) +__debug_object_init(void *addr, const struct debug_obj_descr *descr, int onstack) { enum debug_obj_state state; bool check_stack = false; @@ -587,7 +607,7 @@ __debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack) * @addr: address of the object * @descr: pointer to an object specific debug description structure */ -void debug_object_init(void *addr, struct debug_obj_descr *descr) +void debug_object_init(void *addr, const struct debug_obj_descr *descr) { if (!debug_objects_enabled) return; @@ -602,7 +622,7 @@ EXPORT_SYMBOL_GPL(debug_object_init); * @addr: address of the object * @descr: pointer to an object specific debug description structure */ -void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr) +void debug_object_init_on_stack(void *addr, const struct debug_obj_descr *descr) { if (!debug_objects_enabled) return; @@ -617,7 +637,7 @@ EXPORT_SYMBOL_GPL(debug_object_init_on_stack); * @descr: pointer to an object specific debug description structure * Returns 0 for success, -EINVAL for check failed. */ -int debug_object_activate(void *addr, struct debug_obj_descr *descr) +int debug_object_activate(void *addr, const struct debug_obj_descr *descr) { enum debug_obj_state state; struct debug_bucket *db; @@ -695,7 +715,7 @@ EXPORT_SYMBOL_GPL(debug_object_activate); * @addr: address of the object * @descr: pointer to an object specific debug description structure */ -void debug_object_deactivate(void *addr, struct debug_obj_descr *descr) +void debug_object_deactivate(void *addr, const struct debug_obj_descr *descr) { struct debug_bucket *db; struct debug_obj *obj; @@ -747,7 +767,7 @@ EXPORT_SYMBOL_GPL(debug_object_deactivate); * @addr: address of the object * @descr: pointer to an object specific debug description structure */ -void debug_object_destroy(void *addr, struct debug_obj_descr *descr) +void debug_object_destroy(void *addr, const struct debug_obj_descr *descr) { enum debug_obj_state state; struct debug_bucket *db; @@ -797,7 +817,7 @@ EXPORT_SYMBOL_GPL(debug_object_destroy); * @addr: address of the object * @descr: pointer to an object specific debug description structure */ -void debug_object_free(void *addr, struct debug_obj_descr *descr) +void debug_object_free(void *addr, const struct debug_obj_descr *descr) { enum debug_obj_state state; struct debug_bucket *db; @@ -838,7 +858,7 @@ EXPORT_SYMBOL_GPL(debug_object_free); * @addr: address of the object * @descr: pointer to an object specific debug description structure */ -void debug_object_assert_init(void *addr, struct debug_obj_descr *descr) +void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr) { struct debug_bucket *db; struct debug_obj *obj; @@ -886,7 +906,7 @@ EXPORT_SYMBOL_GPL(debug_object_assert_init); * @next: state to move to if expected state is found */ void -debug_object_active_state(void *addr, struct debug_obj_descr *descr, +debug_object_active_state(void *addr, const struct debug_obj_descr *descr, unsigned int expect, unsigned int next) { struct debug_bucket *db; @@ -934,7 +954,7 @@ EXPORT_SYMBOL_GPL(debug_object_active_state); static void __debug_check_no_obj_freed(const void *address, unsigned long size) { unsigned long flags, oaddr, saddr, eaddr, paddr, chunks; - struct debug_obj_descr *descr; + const struct debug_obj_descr *descr; enum debug_obj_state state; struct debug_bucket *db; struct hlist_node *tmp; @@ -1052,7 +1072,7 @@ struct self_test { unsigned long dummy2[3]; }; -static __initdata struct debug_obj_descr descr_type_test; +static __initconst const struct debug_obj_descr descr_type_test; static bool __init is_static_object(void *addr) { @@ -1177,7 +1197,7 @@ out: return res; } -static __initdata struct debug_obj_descr descr_type_test = { +static __initconst const struct debug_obj_descr descr_type_test = { .name = "selftest", .is_static_object = is_static_object, .fixup_init = fixup_init, @@ -1367,6 +1387,11 @@ void __init debug_objects_mem_init(void) } else debug_objects_selftest(); +#ifdef CONFIG_HOTPLUG_CPU + cpuhp_setup_state_nocalls(CPUHP_DEBUG_OBJ_DEAD, "object:offline", NULL, + object_cpu_offline); +#endif + /* * Increase the thresholds for allocating and freeing objects * according to the number of possible CPUs available in the system. diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 5e40786c8f12..d13304a034f5 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -637,30 +637,30 @@ size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) } EXPORT_SYMBOL(_copy_to_iter); -#ifdef CONFIG_ARCH_HAS_UACCESS_MCSAFE -static int copyout_mcsafe(void __user *to, const void *from, size_t n) +#ifdef CONFIG_ARCH_HAS_COPY_MC +static int copyout_mc(void __user *to, const void *from, size_t n) { if (access_ok(to, n)) { instrument_copy_to_user(to, from, n); - n = copy_to_user_mcsafe((__force void *) to, from, n); + n = copy_mc_to_user((__force void *) to, from, n); } return n; } -static unsigned long memcpy_mcsafe_to_page(struct page *page, size_t offset, +static unsigned long copy_mc_to_page(struct page *page, size_t offset, const char *from, size_t len) { unsigned long ret; char *to; to = kmap_atomic(page); - ret = memcpy_mcsafe(to + offset, from, len); + ret = copy_mc_to_kernel(to + offset, from, len); kunmap_atomic(to); return ret; } -static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes, +static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { struct pipe_inode_info *pipe = i->pipe; @@ -678,7 +678,7 @@ static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes, size_t chunk = min_t(size_t, n, PAGE_SIZE - off); unsigned long rem; - rem = memcpy_mcsafe_to_page(pipe->bufs[i_head & p_mask].page, + rem = copy_mc_to_page(pipe->bufs[i_head & p_mask].page, off, addr, chunk); i->head = i_head; i->iov_offset = off + chunk - rem; @@ -695,18 +695,17 @@ static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes, } /** - * _copy_to_iter_mcsafe - copy to user with source-read error exception handling + * _copy_mc_to_iter - copy to iter with source memory error exception handling * @addr: source kernel address * @bytes: total transfer length * @iter: destination iterator * - * The pmem driver arranges for filesystem-dax to use this facility via - * dax_copy_to_iter() for protecting read/write to persistent memory. - * Unless / until an architecture can guarantee identical performance - * between _copy_to_iter_mcsafe() and _copy_to_iter() it would be a - * performance regression to switch more users to the mcsafe version. + * The pmem driver deploys this for the dax operation + * (dax_copy_to_iter()) for dax reads (bypass page-cache and the + * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes + * successfully copied. * - * Otherwise, the main differences between this and typical _copy_to_iter(). + * The main differences between this and typical _copy_to_iter(). * * * Typical tail/residue handling after a fault retries the copy * byte-by-byte until the fault happens again. Re-triggering machine @@ -717,23 +716,22 @@ static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes, * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies. * Compare to copy_to_iter() where only ITER_IOVEC attempts might return * a short copy. - * - * See MCSAFE_TEST for self-test. */ -size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i) +size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { const char *from = addr; unsigned long rem, curr_addr, s_addr = (unsigned long) addr; if (unlikely(iov_iter_is_pipe(i))) - return copy_pipe_to_iter_mcsafe(addr, bytes, i); + return copy_mc_pipe_to_iter(addr, bytes, i); if (iter_is_iovec(i)) might_fault(); iterate_and_advance(i, bytes, v, - copyout_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), + copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len, + v.iov_len), ({ - rem = memcpy_mcsafe_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len); + rem = copy_mc_to_page(v.bv_page, v.bv_offset, + (from += v.bv_len) - v.bv_len, v.bv_len); if (rem) { curr_addr = (unsigned long) from; bytes = curr_addr - s_addr - rem; @@ -741,8 +739,8 @@ size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i) } }), ({ - rem = memcpy_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len, - v.iov_len); + rem = copy_mc_to_kernel(v.iov_base, (from += v.iov_len) + - v.iov_len, v.iov_len); if (rem) { curr_addr = (unsigned long) from; bytes = curr_addr - s_addr - rem; @@ -753,8 +751,8 @@ size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i) return bytes; } -EXPORT_SYMBOL_GPL(_copy_to_iter_mcsafe); -#endif /* CONFIG_ARCH_HAS_UACCESS_MCSAFE */ +EXPORT_SYMBOL_GPL(_copy_mc_to_iter); +#endif /* CONFIG_ARCH_HAS_COPY_MC */ size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) { diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c index a2345de90e93..f61689a96e85 100644 --- a/lib/percpu_counter.c +++ b/lib/percpu_counter.c @@ -17,7 +17,7 @@ static DEFINE_SPINLOCK(percpu_counters_lock); #ifdef CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER -static struct debug_obj_descr percpu_counter_debug_descr; +static const struct debug_obj_descr percpu_counter_debug_descr; static bool percpu_counter_fixup_free(void *addr, enum debug_obj_state state) { @@ -33,7 +33,7 @@ static bool percpu_counter_fixup_free(void *addr, enum debug_obj_state state) } } -static struct debug_obj_descr percpu_counter_debug_descr = { +static const struct debug_obj_descr percpu_counter_debug_descr = { .name = "percpu_counter", .fixup_free = percpu_counter_fixup_free, }; |