From 226ab561075f6f8f3cd5f7b3b7544f3997aab51f Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 13 Jul 2018 21:49:34 -0700 Subject: device-dax: Convert to vmf_insert_mixed and vm_fault_t Use new return type vm_fault_t for fault and huge_fault handler. For now, this is just documenting that the function returns a VM_FAULT value rather than an errno. Once all instances are converted, vm_fault_t will become a distinct type. Commit 1c8f422059ae ("mm: change return type to vm_fault_t") Previously vm_insert_mixed() returned an error code which driver mapped into VM_FAULT_* type. The new function vmf_insert_mixed() will replace this inefficiency by returning VM_FAULT_* type. Signed-off-by: Souptick Joarder Reviewed-by: Matthew Wilcox Reviewed-by: Ross Zwisler Signed-off-by: Dan Williams Signed-off-by: Dave Jiang --- mm/huge_memory.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mm') diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 1cd7c1a57a14..feba371169ca 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -752,7 +752,7 @@ static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, spin_unlock(ptl); } -int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, +vm_fault_t vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd, pfn_t pfn, bool write) { pgprot_t pgprot = vma->vm_page_prot; @@ -812,7 +812,7 @@ static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr, spin_unlock(ptl); } -int vmf_insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr, +vm_fault_t vmf_insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr, pud_t *pud, pfn_t pfn, bool write) { pgprot_t pgprot = vma->vm_page_prot; -- cgit v1.2.3 From 86a66810baa87e5d1c93a51f71d36c7f71fe3113 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 13 Jul 2018 21:49:56 -0700 Subject: mm, madvise_inject_error: Disable MADV_SOFT_OFFLINE for ZONE_DEVICE pages Given that dax / device-mapped pages are never subject to page allocations remove them from consideration by the soft-offline mechanism. Reported-by: Naoya Horiguchi Signed-off-by: Dan Williams Acked-by: Naoya Horiguchi Signed-off-by: Dave Jiang --- mm/memory-failure.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'mm') diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 9d142b9b86dc..988f977db3d2 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1751,6 +1751,14 @@ int soft_offline_page(struct page *page, int flags) int ret; unsigned long pfn = page_to_pfn(page); + if (is_zone_device_page(page)) { + pr_debug_ratelimited("soft_offline: %#lx page is device page\n", + pfn); + if (flags & MF_COUNT_INCREASED) + put_page(page); + return -EIO; + } + if (PageHWPoison(page)) { pr_info("soft offline: %#lx page already poisoned\n", pfn); if (flags & MF_COUNT_INCREASED) -- cgit v1.2.3 From 2fa147bdbf672c53386a8f5f2c7fe358004c3ef8 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 13 Jul 2018 21:50:01 -0700 Subject: mm, dev_pagemap: Do not clear ->mapping on final put MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEMORY_DEVICE_FS_DAX relies on typical page semantics whereby ->mapping is only ever cleared by truncation, not final put. Without this fix dax pages may forget their mapping association at the end of every page pin event. Move this atypical behavior that HMM wants into the HMM ->page_free() callback. Cc: Cc: Jan Kara Cc: Jérôme Glisse Cc: Andrew Morton Cc: Ross Zwisler Fixes: d2c997c0f145 ("fs, dax: use page->mapping...") Signed-off-by: Dan Williams Acked-by: Jérôme Glisse Signed-off-by: Dave Jiang --- kernel/memremap.c | 1 - mm/hmm.c | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'mm') diff --git a/kernel/memremap.c b/kernel/memremap.c index 5857267a4af5..62603634a1d2 100644 --- a/kernel/memremap.c +++ b/kernel/memremap.c @@ -339,7 +339,6 @@ void __put_devmap_managed_page(struct page *page) __ClearPageActive(page); __ClearPageWaiters(page); - page->mapping = NULL; mem_cgroup_uncharge(page); page->pgmap->page_free(page, page->pgmap->data); diff --git a/mm/hmm.c b/mm/hmm.c index de7b6bf77201..f9d1d89dec4d 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -963,6 +963,8 @@ static void hmm_devmem_free(struct page *page, void *data) { struct hmm_devmem *devmem = data; + page->mapping = NULL; + devmem->ops->free(devmem, page); } -- cgit v1.2.3 From 23e7b5c2e2715947cf1ff57124dd3e96caf48521 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 13 Jul 2018 21:50:06 -0700 Subject: mm, madvise_inject_error: Let memory_failure() optionally take a page reference The madvise_inject_error() routine uses get_user_pages() to lookup the pfn and other information for injected error, but it does not release that pin. The assumption is that failed pages should be taken out of circulation. However, for dax mappings it is not possible to take pages out of circulation since they are 1:1 physically mapped as filesystem blocks, or device-dax capacity. They also typically represent persistent memory which has an error clearing capability. In preparation for adding a special handler for dax mappings, shift the responsibility of taking the page reference to memory_failure(). I.e. drop the page reference and do not specify MF_COUNT_INCREASED to memory_failure(). Cc: Michal Hocko Cc: Naoya Horiguchi Signed-off-by: Dan Williams Acked-by: Naoya Horiguchi Signed-off-by: Dave Jiang --- mm/madvise.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'mm') diff --git a/mm/madvise.c b/mm/madvise.c index 4d3c922ea1a1..972a9eaa898b 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -631,11 +631,13 @@ static int madvise_inject_error(int behavior, for (; start < end; start += PAGE_SIZE << order) { + unsigned long pfn; int ret; ret = get_user_pages_fast(start, 1, 0, &page); if (ret != 1) return ret; + pfn = page_to_pfn(page); /* * When soft offlining hugepages, after migrating the page @@ -651,17 +653,25 @@ static int madvise_inject_error(int behavior, if (behavior == MADV_SOFT_OFFLINE) { pr_info("Soft offlining pfn %#lx at process virtual address %#lx\n", - page_to_pfn(page), start); + pfn, start); ret = soft_offline_page(page, MF_COUNT_INCREASED); if (ret) return ret; continue; } + pr_info("Injecting memory failure for pfn %#lx at process virtual address %#lx\n", - page_to_pfn(page), start); + pfn, start); - ret = memory_failure(page_to_pfn(page), MF_COUNT_INCREASED); + /* + * Drop the page reference taken by get_user_pages_fast(). In + * the absence of MF_COUNT_INCREASED the memory_failure() + * routine is responsible for pinning the page to prevent it + * from being released back to the page allocator. + */ + put_page(page); + ret = memory_failure(pfn, 0); if (ret) return ret; } -- cgit v1.2.3 From ae1139ece126b8eb6d0770094fbac43ea928d9d9 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 13 Jul 2018 21:50:11 -0700 Subject: mm, memory_failure: Collect mapping size in collect_procs() In preparation for supporting memory_failure() for dax mappings, teach collect_procs() to also determine the mapping size. Unlike typical mappings the dax mapping size is determined by walking page-table entries rather than using the compound-page accounting for THP pages. Acked-by: Naoya Horiguchi Signed-off-by: Dan Williams Signed-off-by: Dave Jiang --- mm/memory-failure.c | 81 ++++++++++++++++++++++++++--------------------------- 1 file changed, 40 insertions(+), 41 deletions(-) (limited to 'mm') diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 988f977db3d2..8a81680d00dd 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -173,23 +173,52 @@ int hwpoison_filter(struct page *p) EXPORT_SYMBOL_GPL(hwpoison_filter); +/* + * Kill all processes that have a poisoned page mapped and then isolate + * the page. + * + * General strategy: + * Find all processes having the page mapped and kill them. + * But we keep a page reference around so that the page is not + * actually freed yet. + * Then stash the page away + * + * There's no convenient way to get back to mapped processes + * from the VMAs. So do a brute-force search over all + * running processes. + * + * Remember that machine checks are not common (or rather + * if they are common you have other problems), so this shouldn't + * be a performance issue. + * + * Also there are some races possible while we get from the + * error detection to actually handle it. + */ + +struct to_kill { + struct list_head nd; + struct task_struct *tsk; + unsigned long addr; + short size_shift; + char addr_valid; +}; + /* * Send all the processes who have the page mapped a signal. * ``action optional'' if they are not immediately affected by the error * ``action required'' if error happened in current execution context */ -static int kill_proc(struct task_struct *t, unsigned long addr, - unsigned long pfn, struct page *page, int flags) +static int kill_proc(struct to_kill *tk, unsigned long pfn, int flags) { - short addr_lsb; + struct task_struct *t = tk->tsk; + short addr_lsb = tk->size_shift; int ret; pr_err("Memory failure: %#lx: Killing %s:%d due to hardware memory corruption\n", pfn, t->comm, t->pid); - addr_lsb = compound_order(compound_head(page)) + PAGE_SHIFT; if ((flags & MF_ACTION_REQUIRED) && t->mm == current->mm) { - ret = force_sig_mceerr(BUS_MCEERR_AR, (void __user *)addr, + ret = force_sig_mceerr(BUS_MCEERR_AR, (void __user *)tk->addr, addr_lsb, current); } else { /* @@ -198,7 +227,7 @@ static int kill_proc(struct task_struct *t, unsigned long addr, * This could cause a loop when the user sets SIGBUS * to SIG_IGN, but hopefully no one will do that? */ - ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)addr, + ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)tk->addr, addr_lsb, t); /* synchronous? */ } if (ret < 0) @@ -234,35 +263,6 @@ void shake_page(struct page *p, int access) } EXPORT_SYMBOL_GPL(shake_page); -/* - * Kill all processes that have a poisoned page mapped and then isolate - * the page. - * - * General strategy: - * Find all processes having the page mapped and kill them. - * But we keep a page reference around so that the page is not - * actually freed yet. - * Then stash the page away - * - * There's no convenient way to get back to mapped processes - * from the VMAs. So do a brute-force search over all - * running processes. - * - * Remember that machine checks are not common (or rather - * if they are common you have other problems), so this shouldn't - * be a performance issue. - * - * Also there are some races possible while we get from the - * error detection to actually handle it. - */ - -struct to_kill { - struct list_head nd; - struct task_struct *tsk; - unsigned long addr; - char addr_valid; -}; - /* * Failure handling: if we can't find or can't kill a process there's * not much we can do. We just print a message and ignore otherwise. @@ -292,6 +292,7 @@ static void add_to_kill(struct task_struct *tsk, struct page *p, } tk->addr = page_address_in_vma(p, vma); tk->addr_valid = 1; + tk->size_shift = compound_order(compound_head(p)) + PAGE_SHIFT; /* * In theory we don't have to kill when the page was @@ -317,9 +318,8 @@ static void add_to_kill(struct task_struct *tsk, struct page *p, * Also when FAIL is set do a force kill because something went * wrong earlier. */ -static void kill_procs(struct list_head *to_kill, int forcekill, - bool fail, struct page *page, unsigned long pfn, - int flags) +static void kill_procs(struct list_head *to_kill, int forcekill, bool fail, + unsigned long pfn, int flags) { struct to_kill *tk, *next; @@ -342,8 +342,7 @@ static void kill_procs(struct list_head *to_kill, int forcekill, * check for that, but we need to tell the * process anyways. */ - else if (kill_proc(tk->tsk, tk->addr, - pfn, page, flags) < 0) + else if (kill_proc(tk, pfn, flags) < 0) pr_err("Memory failure: %#lx: Cannot send advisory machine check signal to %s:%d\n", pfn, tk->tsk->comm, tk->tsk->pid); } @@ -1012,7 +1011,7 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, * any accesses to the poisoned memory. */ forcekill = PageDirty(hpage) || (flags & MF_MUST_KILL); - kill_procs(&tokill, forcekill, !unmap_success, p, pfn, flags); + kill_procs(&tokill, forcekill, !unmap_success, pfn, flags); return unmap_success; } -- cgit v1.2.3 From 6100e34b2526e1dc3dbcc47fea2677974d6aaea5 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 13 Jul 2018 21:50:21 -0700 Subject: mm, memory_failure: Teach memory_failure() about dev_pagemap pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mce: Uncorrected hardware memory error in user-access at af34214200 {1}[Hardware Error]: It has been corrected by h/w and requires no further action mce: [Hardware Error]: Machine check events logged {1}[Hardware Error]: event severity: corrected Memory failure: 0xaf34214: reserved kernel page still referenced by 1 users [..] Memory failure: 0xaf34214: recovery action for reserved kernel page: Failed mce: Memory error not recovered In contrast to typical memory, dev_pagemap pages may be dax mapped. With dax there is no possibility to map in another page dynamically since dax establishes 1:1 physical address to file offset associations. Also dev_pagemap pages associated with NVDIMM / persistent memory devices can internal remap/repair addresses with poison. While memory_failure() assumes that it can discard typical poisoned pages and keep them unmapped indefinitely, dev_pagemap pages may be returned to service after the error is cleared. Teach memory_failure() to detect and handle MEMORY_DEVICE_HOST dev_pagemap pages that have poison consumed by userspace. Mark the memory as UC instead of unmapping it completely to allow ongoing access via the device driver (nd_pmem). Later, nd_pmem will grow support for marking the page back to WB when the error is cleared. Cc: Jan Kara Cc: Christoph Hellwig Cc: Jérôme Glisse Cc: Matthew Wilcox Cc: Naoya Horiguchi Cc: Ross Zwisler Signed-off-by: Dan Williams Signed-off-by: Dave Jiang --- include/linux/mm.h | 1 + mm/memory-failure.c | 125 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 124 insertions(+), 2 deletions(-) (limited to 'mm') diff --git a/include/linux/mm.h b/include/linux/mm.h index a0fbb9ffe380..374e5e9284f7 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2725,6 +2725,7 @@ enum mf_action_page_type { MF_MSG_TRUNCATED_LRU, MF_MSG_BUDDY, MF_MSG_BUDDY_2ND, + MF_MSG_DAX, MF_MSG_UNKNOWN, }; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 8a81680d00dd..32a644d9c2ee 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -55,6 +55,7 @@ #include #include #include +#include #include #include #include "internal.h" @@ -263,6 +264,40 @@ void shake_page(struct page *p, int access) } EXPORT_SYMBOL_GPL(shake_page); +static unsigned long dev_pagemap_mapping_shift(struct page *page, + struct vm_area_struct *vma) +{ + unsigned long address = vma_address(page, vma); + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + pgd = pgd_offset(vma->vm_mm, address); + if (!pgd_present(*pgd)) + return 0; + p4d = p4d_offset(pgd, address); + if (!p4d_present(*p4d)) + return 0; + pud = pud_offset(p4d, address); + if (!pud_present(*pud)) + return 0; + if (pud_devmap(*pud)) + return PUD_SHIFT; + pmd = pmd_offset(pud, address); + if (!pmd_present(*pmd)) + return 0; + if (pmd_devmap(*pmd)) + return PMD_SHIFT; + pte = pte_offset_map(pmd, address); + if (!pte_present(*pte)) + return 0; + if (pte_devmap(*pte)) + return PAGE_SHIFT; + return 0; +} + /* * Failure handling: if we can't find or can't kill a process there's * not much we can do. We just print a message and ignore otherwise. @@ -292,7 +327,10 @@ static void add_to_kill(struct task_struct *tsk, struct page *p, } tk->addr = page_address_in_vma(p, vma); tk->addr_valid = 1; - tk->size_shift = compound_order(compound_head(p)) + PAGE_SHIFT; + if (is_zone_device_page(p)) + tk->size_shift = dev_pagemap_mapping_shift(p, vma); + else + tk->size_shift = compound_order(compound_head(p)) + PAGE_SHIFT; /* * In theory we don't have to kill when the page was @@ -300,7 +338,7 @@ static void add_to_kill(struct task_struct *tsk, struct page *p, * likely very rare kill anyways just out of paranoia, but use * a SIGKILL because the error is not contained anymore. */ - if (tk->addr == -EFAULT) { + if (tk->addr == -EFAULT || tk->size_shift == 0) { pr_info("Memory failure: Unable to find user space address %lx in %s\n", page_to_pfn(p), tsk->comm); tk->addr_valid = 0; @@ -514,6 +552,7 @@ static const char * const action_page_types[] = { [MF_MSG_TRUNCATED_LRU] = "already truncated LRU page", [MF_MSG_BUDDY] = "free buddy page", [MF_MSG_BUDDY_2ND] = "free buddy page (2nd try)", + [MF_MSG_DAX] = "dax page", [MF_MSG_UNKNOWN] = "unknown page", }; @@ -1111,6 +1150,83 @@ out: return res; } +static int memory_failure_dev_pagemap(unsigned long pfn, int flags, + struct dev_pagemap *pgmap) +{ + struct page *page = pfn_to_page(pfn); + const bool unmap_success = true; + unsigned long size = 0; + struct to_kill *tk; + LIST_HEAD(tokill); + int rc = -EBUSY; + loff_t start; + + /* + * Prevent the inode from being freed while we are interrogating + * the address_space, typically this would be handled by + * lock_page(), but dax pages do not use the page lock. This + * also prevents changes to the mapping of this pfn until + * poison signaling is complete. + */ + if (!dax_lock_mapping_entry(page)) + goto out; + + if (hwpoison_filter(page)) { + rc = 0; + goto unlock; + } + + switch (pgmap->type) { + case MEMORY_DEVICE_PRIVATE: + case MEMORY_DEVICE_PUBLIC: + /* + * TODO: Handle HMM pages which may need coordination + * with device-side memory. + */ + goto unlock; + default: + break; + } + + /* + * Use this flag as an indication that the dax page has been + * remapped UC to prevent speculative consumption of poison. + */ + SetPageHWPoison(page); + + /* + * Unlike System-RAM there is no possibility to swap in a + * different physical page at a given virtual address, so all + * userspace consumption of ZONE_DEVICE memory necessitates + * SIGBUS (i.e. MF_MUST_KILL) + */ + flags |= MF_ACTION_REQUIRED | MF_MUST_KILL; + collect_procs(page, &tokill, flags & MF_ACTION_REQUIRED); + + list_for_each_entry(tk, &tokill, nd) + if (tk->size_shift) + size = max(size, 1UL << tk->size_shift); + if (size) { + /* + * Unmap the largest mapping to avoid breaking up + * device-dax mappings which are constant size. The + * actual size of the mapping being torn down is + * communicated in siginfo, see kill_proc() + */ + start = (page->index << PAGE_SHIFT) & ~(size - 1); + unmap_mapping_range(page->mapping, start, start + size, 0); + } + kill_procs(&tokill, flags & MF_MUST_KILL, !unmap_success, pfn, flags); + rc = 0; +unlock: + dax_unlock_mapping_entry(page); +out: + /* drop pgmap ref acquired in caller */ + put_dev_pagemap(pgmap); + action_result(pfn, MF_MSG_DAX, rc ? MF_FAILED : MF_RECOVERED); + return rc; +} + /** * memory_failure - Handle memory failure of a page. * @pfn: Page Number of the corrupted page @@ -1133,6 +1249,7 @@ int memory_failure(unsigned long pfn, int flags) struct page *p; struct page *hpage; struct page *orig_head; + struct dev_pagemap *pgmap; int res; unsigned long page_flags; @@ -1145,6 +1262,10 @@ int memory_failure(unsigned long pfn, int flags) return -ENXIO; } + pgmap = get_dev_pagemap(pfn, NULL); + if (pgmap) + return memory_failure_dev_pagemap(pfn, flags, pgmap); + p = pfn_to_page(pfn); if (PageHuge(p)) return memory_failure_hugetlb(pfn, flags); -- cgit v1.2.3