diff options
Diffstat (limited to 'drivers')
| -rw-r--r-- | drivers/dax/device.c | 75 | ||||
| -rw-r--r-- | drivers/nvdimm/pmem.c | 26 | ||||
| -rw-r--r-- | drivers/nvdimm/pmem.h | 13 |
3 files changed, 87 insertions, 27 deletions
diff --git a/drivers/dax/device.c b/drivers/dax/device.c index 0a2acd7993f0..6fd46083e629 100644 --- a/drivers/dax/device.c +++ b/drivers/dax/device.c @@ -248,13 +248,12 @@ __weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff, return -1; } -static int __dev_dax_pte_fault(struct dev_dax *dev_dax, struct vm_fault *vmf) +static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax, + struct vm_fault *vmf, pfn_t *pfn) { struct device *dev = &dev_dax->dev; struct dax_region *dax_region; - int rc = VM_FAULT_SIGBUS; phys_addr_t phys; - pfn_t pfn; unsigned int fault_size = PAGE_SIZE; if (check_vma(dev_dax, vmf->vma, __func__)) @@ -276,26 +275,19 @@ static int __dev_dax_pte_fault(struct dev_dax *dev_dax, struct vm_fault *vmf) return VM_FAULT_SIGBUS; } - pfn = phys_to_pfn_t(phys, dax_region->pfn_flags); - - rc = vm_insert_mixed(vmf->vma, vmf->address, pfn); - - if (rc == -ENOMEM) - return VM_FAULT_OOM; - if (rc < 0 && rc != -EBUSY) - return VM_FAULT_SIGBUS; + *pfn = phys_to_pfn_t(phys, dax_region->pfn_flags); - return VM_FAULT_NOPAGE; + return vmf_insert_mixed(vmf->vma, vmf->address, *pfn); } -static int __dev_dax_pmd_fault(struct dev_dax *dev_dax, struct vm_fault *vmf) +static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax, + struct vm_fault *vmf, pfn_t *pfn) { unsigned long pmd_addr = vmf->address & PMD_MASK; struct device *dev = &dev_dax->dev; struct dax_region *dax_region; phys_addr_t phys; pgoff_t pgoff; - pfn_t pfn; unsigned int fault_size = PMD_SIZE; if (check_vma(dev_dax, vmf->vma, __func__)) @@ -331,21 +323,21 @@ static int __dev_dax_pmd_fault(struct dev_dax *dev_dax, struct vm_fault *vmf) return VM_FAULT_SIGBUS; } - pfn = phys_to_pfn_t(phys, dax_region->pfn_flags); + *pfn = phys_to_pfn_t(phys, dax_region->pfn_flags); - return vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd, pfn, + return vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd, *pfn, vmf->flags & FAULT_FLAG_WRITE); } #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD -static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf) +static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax, + struct vm_fault *vmf, pfn_t *pfn) { unsigned long pud_addr = vmf->address & PUD_MASK; struct device *dev = &dev_dax->dev; struct dax_region *dax_region; phys_addr_t phys; pgoff_t pgoff; - pfn_t pfn; unsigned int fault_size = PUD_SIZE; @@ -382,23 +374,26 @@ static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf) return VM_FAULT_SIGBUS; } - pfn = phys_to_pfn_t(phys, dax_region->pfn_flags); + *pfn = phys_to_pfn_t(phys, dax_region->pfn_flags); - return vmf_insert_pfn_pud(vmf->vma, vmf->address, vmf->pud, pfn, + return vmf_insert_pfn_pud(vmf->vma, vmf->address, vmf->pud, *pfn, vmf->flags & FAULT_FLAG_WRITE); } #else -static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf) +static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax, + struct vm_fault *vmf, pfn_t *pfn) { return VM_FAULT_FALLBACK; } #endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ -static int dev_dax_huge_fault(struct vm_fault *vmf, +static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf, enum page_entry_size pe_size) { - int rc, id; struct file *filp = vmf->vma->vm_file; + unsigned long fault_size; + int rc, id; + pfn_t pfn; struct dev_dax *dev_dax = filp->private_data; dev_dbg(&dev_dax->dev, "%s: %s (%#lx - %#lx) size = %d\n", current->comm, @@ -408,23 +403,49 @@ static int dev_dax_huge_fault(struct vm_fault *vmf, id = dax_read_lock(); switch (pe_size) { case PE_SIZE_PTE: - rc = __dev_dax_pte_fault(dev_dax, vmf); + fault_size = PAGE_SIZE; + rc = __dev_dax_pte_fault(dev_dax, vmf, &pfn); break; case PE_SIZE_PMD: - rc = __dev_dax_pmd_fault(dev_dax, vmf); + fault_size = PMD_SIZE; + rc = __dev_dax_pmd_fault(dev_dax, vmf, &pfn); break; case PE_SIZE_PUD: - rc = __dev_dax_pud_fault(dev_dax, vmf); + fault_size = PUD_SIZE; + rc = __dev_dax_pud_fault(dev_dax, vmf, &pfn); break; default: rc = VM_FAULT_SIGBUS; } + + if (rc == VM_FAULT_NOPAGE) { + unsigned long i; + pgoff_t pgoff; + + /* + * In the device-dax case the only possibility for a + * VM_FAULT_NOPAGE result is when device-dax capacity is + * mapped. No need to consider the zero page, or racing + * conflicting mappings. + */ + pgoff = linear_page_index(vmf->vma, vmf->address + & ~(fault_size - 1)); + for (i = 0; i < fault_size / PAGE_SIZE; i++) { + struct page *page; + + page = pfn_to_page(pfn_t_to_pfn(pfn) + i); + if (page->mapping) + continue; + page->mapping = filp->f_mapping; + page->index = pgoff + i; + } + } dax_read_unlock(id); return rc; } -static int dev_dax_fault(struct vm_fault *vmf) +static vm_fault_t dev_dax_fault(struct vm_fault *vmf) { return dev_dax_huge_fault(vmf, PE_SIZE_PTE); } diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index c23649867696..6071e2942053 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -20,6 +20,7 @@ #include <linux/hdreg.h> #include <linux/init.h> #include <linux/platform_device.h> +#include <linux/set_memory.h> #include <linux/module.h> #include <linux/moduleparam.h> #include <linux/badblocks.h> @@ -51,6 +52,30 @@ static struct nd_region *to_region(struct pmem_device *pmem) return to_nd_region(to_dev(pmem)->parent); } +static void hwpoison_clear(struct pmem_device *pmem, + phys_addr_t phys, unsigned int len) +{ + unsigned long pfn_start, pfn_end, pfn; + + /* only pmem in the linear map supports HWPoison */ + if (is_vmalloc_addr(pmem->virt_addr)) + return; + + pfn_start = PHYS_PFN(phys); + pfn_end = pfn_start + PHYS_PFN(len); + for (pfn = pfn_start; pfn < pfn_end; pfn++) { + struct page *page = pfn_to_page(pfn); + + /* + * Note, no need to hold a get_dev_pagemap() reference + * here since we're in the driver I/O path and + * outstanding I/O requests pin the dev_pagemap. + */ + if (test_and_clear_pmem_poison(page)) + clear_mce_nospec(pfn); + } +} + static blk_status_t pmem_clear_poison(struct pmem_device *pmem, phys_addr_t offset, unsigned int len) { @@ -65,6 +90,7 @@ static blk_status_t pmem_clear_poison(struct pmem_device *pmem, if (cleared < len) rc = BLK_STS_IOERR; if (cleared > 0 && cleared / 512) { + hwpoison_clear(pmem, pmem->phys_addr + offset, cleared); cleared /= 512; dev_dbg(dev, "%#llx clear %ld sector%s\n", (unsigned long long) sector, cleared, diff --git a/drivers/nvdimm/pmem.h b/drivers/nvdimm/pmem.h index a64ebc78b5df..59cfe13ea8a8 100644 --- a/drivers/nvdimm/pmem.h +++ b/drivers/nvdimm/pmem.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NVDIMM_PMEM_H__ #define __NVDIMM_PMEM_H__ +#include <linux/page-flags.h> #include <linux/badblocks.h> #include <linux/types.h> #include <linux/pfn_t.h> @@ -27,4 +28,16 @@ struct pmem_device { long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff, long nr_pages, void **kaddr, pfn_t *pfn); + +#ifdef CONFIG_MEMORY_FAILURE +static inline bool test_and_clear_pmem_poison(struct page *page) +{ + return TestClearPageHWPoison(page); +} +#else +static inline bool test_and_clear_pmem_poison(struct page *page) +{ + return false; +} +#endif #endif /* __NVDIMM_PMEM_H__ */ |