Merge tag 'libnvdimm-for-4.19_dax-memory-failure' of gitolite.kernel.org:pub/scm/linux/kernel/git/nvdimm/nvdimm

Pull libnvdimm memory-failure update from Dave Jiang:
 "As it stands, memory_failure() gets thoroughly confused by dev_pagemap
  backed mappings. The recovery code has specific enabling for several
  possible page states and needs new enabling to handle poison in dax
  mappings.

  In order to support reliable reverse mapping of user space addresses:

   1/ Add new locking in the memory_failure() rmap path to prevent races
      that would typically be handled by the page lock.

   2/ Since dev_pagemap pages are hidden from the page allocator and the
      "compound page" accounting machinery, add a mechanism to determine
      the size of the mapping that encompasses a given poisoned pfn.

   3/ Given pmem errors can be repaired, change the speculatively
      accessed poison protection, mce_unmap_kpfn(), to be reversible and
      otherwise allow ongoing access from the kernel.

  A side effect of this enabling is that MADV_HWPOISON becomes usable
  for dax mappings, however the primary motivation is to allow the
  system to survive userspace consumption of hardware-poison via dax.
  Specifically the current behavior is:

     mce: Uncorrected hardware memory error in user-access at af34214200
     {1}[Hardware Error]: It has been corrected by h/w and requires no further action
     mce: [Hardware Error]: Machine check events logged
     {1}[Hardware Error]: event severity: corrected
     Memory failure: 0xaf34214: reserved kernel page still referenced by 1 users
     [..]
     Memory failure: 0xaf34214: recovery action for reserved kernel page: Failed
     mce: Memory error not recovered
     <reboot>

  ...and with these changes:

     Injecting memory failure for pfn 0x20cb00 at process virtual address 0x7f763dd00000
     Memory failure: 0x20cb00: Killing dax-pmd:5421 due to hardware memory corruption
     Memory failure: 0x20cb00: recovery action for dax page: Recovered

  Given all the cross dependencies I propose taking this through
  nvdimm.git with acks from Naoya, x86/core, x86/RAS, and of course dax
  folks"

* tag 'libnvdimm-for-4.19_dax-memory-failure' of gitolite.kernel.org:pub/scm/linux/kernel/git/nvdimm/nvdimm:
  libnvdimm, pmem: Restore page attributes when clearing errors
  x86/memory_failure: Introduce {set, clear}_mce_nospec()
  x86/mm/pat: Prepare {reserve, free}_memtype() for "decoy" addresses
  mm, memory_failure: Teach memory_failure() about dev_pagemap pages
  filesystem-dax: Introduce dax_lock_mapping_entry()
  mm, memory_failure: Collect mapping size in collect_procs()
  mm, madvise_inject_error: Let memory_failure() optionally take a page reference
  mm, dev_pagemap: Do not clear ->mapping on final put
  mm, madvise_inject_error: Disable MADV_SOFT_OFFLINE for ZONE_DEVICE pages
  filesystem-dax: Set page->index
  device-dax: Set page->index
  device-dax: Enable page_mapping()
  device-dax: Convert to vmf_insert_mixed and vm_fault_t

1 files changed, 48 insertions, 27 deletions

diff --git a/drivers/dax/device.c b/drivers/dax/device.c
index 0a2acd7993f0..6fd46083e629 100644
--- a/drivers/dax/device.c
+++ b/drivers/dax/device.c
@@ -248,13 +248,12 @@ __weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
         return -1;
 }
 
-static int __dev_dax_pte_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
+static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax,
+                                struct vm_fault *vmf, pfn_t *pfn)
 {
         struct device *dev = &dev_dax->dev;
         struct dax_region *dax_region;
-        int rc = VM_FAULT_SIGBUS;
         phys_addr_t phys;
-        pfn_t pfn;
         unsigned int fault_size = PAGE_SIZE;
 
         if (check_vma(dev_dax, vmf->vma, __func__))
@@ -276,26 +275,19 @@ static int __dev_dax_pte_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
                 return VM_FAULT_SIGBUS;
         }
 
-        pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
-
-        rc = vm_insert_mixed(vmf->vma, vmf->address, pfn);
-
-        if (rc == -ENOMEM)
-                return VM_FAULT_OOM;
-        if (rc < 0 && rc != -EBUSY)
-                return VM_FAULT_SIGBUS;
+        *pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
 
-        return VM_FAULT_NOPAGE;
+        return vmf_insert_mixed(vmf->vma, vmf->address, *pfn);
 }
 
-static int __dev_dax_pmd_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
+static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax,
+                                struct vm_fault *vmf, pfn_t *pfn)
 {
         unsigned long pmd_addr = vmf->address & PMD_MASK;
         struct device *dev = &dev_dax->dev;
         struct dax_region *dax_region;
         phys_addr_t phys;
         pgoff_t pgoff;
-        pfn_t pfn;
         unsigned int fault_size = PMD_SIZE;
 
         if (check_vma(dev_dax, vmf->vma, __func__))
@@ -331,21 +323,21 @@ static int __dev_dax_pmd_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
                 return VM_FAULT_SIGBUS;
         }
 
-        pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
+        *pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
 
-        return vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd, pfn,
+        return vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd, *pfn,
                         vmf->flags & FAULT_FLAG_WRITE);
 }
 
 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
-static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
+static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
+                                struct vm_fault *vmf, pfn_t *pfn)
 {
         unsigned long pud_addr = vmf->address & PUD_MASK;
         struct device *dev = &dev_dax->dev;
         struct dax_region *dax_region;
         phys_addr_t phys;
         pgoff_t pgoff;
-        pfn_t pfn;
         unsigned int fault_size = PUD_SIZE;
 
 
@@ -382,23 +374,26 @@ static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
                 return VM_FAULT_SIGBUS;
         }
 
-        pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
+        *pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
 
-        return vmf_insert_pfn_pud(vmf->vma, vmf->address, vmf->pud, pfn,
+        return vmf_insert_pfn_pud(vmf->vma, vmf->address, vmf->pud, *pfn,
                         vmf->flags & FAULT_FLAG_WRITE);
 }
 #else
-static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
+static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
+                                struct vm_fault *vmf, pfn_t *pfn)
 {
         return VM_FAULT_FALLBACK;
 }
 #endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
 
-static int dev_dax_huge_fault(struct vm_fault *vmf,
+static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf,
                 enum page_entry_size pe_size)
 {
-        int rc, id;
         struct file *filp = vmf->vma->vm_file;
+        unsigned long fault_size;
+        int rc, id;
+        pfn_t pfn;
         struct dev_dax *dev_dax = filp->private_data;
 
         dev_dbg(&dev_dax->dev, "%s: %s (%#lx - %#lx) size = %d\n", current->comm,
@@ -408,23 +403,49 @@ static int dev_dax_huge_fault(struct vm_fault *vmf,
         id = dax_read_lock();
         switch (pe_size) {
         case PE_SIZE_PTE:
-                rc = __dev_dax_pte_fault(dev_dax, vmf);
+                fault_size = PAGE_SIZE;
+                rc = __dev_dax_pte_fault(dev_dax, vmf, &pfn);
                 break;
         case PE_SIZE_PMD:
-                rc = __dev_dax_pmd_fault(dev_dax, vmf);
+                fault_size = PMD_SIZE;
+                rc = __dev_dax_pmd_fault(dev_dax, vmf, &pfn);
                 break;
         case PE_SIZE_PUD:
-                rc = __dev_dax_pud_fault(dev_dax, vmf);
+                fault_size = PUD_SIZE;
+                rc = __dev_dax_pud_fault(dev_dax, vmf, &pfn);
                 break;
         default:
                 rc = VM_FAULT_SIGBUS;
         }
+
+        if (rc == VM_FAULT_NOPAGE) {
+                unsigned long i;
+                pgoff_t pgoff;
+
+                /*
+                 * In the device-dax case the only possibility for a
+                 * VM_FAULT_NOPAGE result is when device-dax capacity is
+                 * mapped. No need to consider the zero page, or racing
+                 * conflicting mappings.
+                 */
+                pgoff = linear_page_index(vmf->vma, vmf->address
+                                & ~(fault_size - 1));
+                for (i = 0; i < fault_size / PAGE_SIZE; i++) {
+                        struct page *page;
+
+                        page = pfn_to_page(pfn_t_to_pfn(pfn) + i);
+                        if (page->mapping)
+                                continue;
+                        page->mapping = filp->f_mapping;
+                        page->index = pgoff + i;
+                }
+        }
         dax_read_unlock(id);
 
         return rc;
 }
 
-static int dev_dax_fault(struct vm_fault *vmf)
+static vm_fault_t dev_dax_fault(struct vm_fault *vmf)
 {
         return dev_dax_huge_fault(vmf, PE_SIZE_PTE);
 }