63 files changed, 1807 insertions, 944 deletions

diff --git a/arch/powerpc/platforms/Kconfig b/arch/powerpc/platforms/Kconfig
index 7e3a2ebba29b..33244e3d9375 100644
--- a/arch/powerpc/platforms/Kconfig
+++ b/arch/powerpc/platforms/Kconfig
@@ -284,6 +284,7 @@ config CPM2
 config AXON_RAM
         tristate "Axon DDR2 memory device driver"
         depends on PPC_IBM_CELL_BLADE && BLOCK
+        select DAX
         default m
         help
           It registers one block device per Axon's DDR2 memory bank found
diff --git a/arch/powerpc/sysdev/axonram.c b/arch/powerpc/sysdev/axonram.c
index f523ac883150..a7fe5fee744f 100644
--- a/arch/powerpc/sysdev/axonram.c
+++ b/arch/powerpc/sysdev/axonram.c
@@ -25,6 +25,7 @@
 
 #include <linux/bio.h>
 #include <linux/blkdev.h>
+#include <linux/dax.h>
 #include <linux/device.h>
 #include <linux/errno.h>
 #include <linux/fs.h>
@@ -62,6 +63,7 @@ static int azfs_major, azfs_minor;
 struct axon_ram_bank {
         struct platform_device  *device;
         struct gendisk          *disk;
+        struct dax_device       *dax_dev;
         unsigned int            irq_id;
         unsigned long           ph_addr;
         unsigned long           io_addr;
@@ -137,25 +139,32 @@ axon_ram_make_request(struct request_queue *queue, struct bio *bio)
         return BLK_QC_T_NONE;
 }
 
-/**
- * axon_ram_direct_access - direct_access() method for block device
- * @device, @sector, @data: see block_device_operations method
- */
+static const struct block_device_operations axon_ram_devops = {
+        .owner          = THIS_MODULE,
+};
+
 static long
-axon_ram_direct_access(struct block_device *device, sector_t sector,
-                       void **kaddr, pfn_t *pfn, long size)
+__axon_ram_direct_access(struct axon_ram_bank *bank, pgoff_t pgoff, long nr_pages,
+                       void **kaddr, pfn_t *pfn)
 {
-        struct axon_ram_bank *bank = device->bd_disk->private_data;
-        loff_t offset = (loff_t)sector << AXON_RAM_SECTOR_SHIFT;
+        resource_size_t offset = pgoff * PAGE_SIZE;
 
         *kaddr = (void *) bank->io_addr + offset;
         *pfn = phys_to_pfn_t(bank->ph_addr + offset, PFN_DEV);
-        return bank->size - offset;
+        return (bank->size - offset) / PAGE_SIZE;
 }
 
-static const struct block_device_operations axon_ram_devops = {
-        .owner          = THIS_MODULE,
-        .direct_access  = axon_ram_direct_access
+static long
+axon_ram_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
+                       void **kaddr, pfn_t *pfn)
+{
+        struct axon_ram_bank *bank = dax_get_private(dax_dev);
+
+        return __axon_ram_direct_access(bank, pgoff, nr_pages, kaddr, pfn);
+}
+
+static const struct dax_operations axon_ram_dax_ops = {
+        .direct_access = axon_ram_dax_direct_access,
 };
 
 /**
@@ -219,6 +228,7 @@ static int axon_ram_probe(struct platform_device *device)
                 goto failed;
         }
 
+
         bank->disk->major = azfs_major;
         bank->disk->first_minor = azfs_minor;
         bank->disk->fops = &axon_ram_devops;
@@ -227,6 +237,13 @@ static int axon_ram_probe(struct platform_device *device)
         sprintf(bank->disk->disk_name, "%s%d",
                         AXON_RAM_DEVICE_NAME, axon_ram_bank_id);
 
+        bank->dax_dev = alloc_dax(bank, bank->disk->disk_name,
+                        &axon_ram_dax_ops);
+        if (!bank->dax_dev) {
+                rc = -ENOMEM;
+                goto failed;
+        }
+
         bank->disk->queue = blk_alloc_queue(GFP_KERNEL);
         if (bank->disk->queue == NULL) {
                 dev_err(&device->dev, "Cannot register disk queue\n");
@@ -278,6 +295,8 @@ failed:
                                 del_gendisk(bank->disk);
                         put_disk(bank->disk);
                 }
+                kill_dax(bank->dax_dev);
+                put_dax(bank->dax_dev);
                 device->dev.platform_data = NULL;
                 if (bank->io_addr != 0)
                         iounmap((void __iomem *) bank->io_addr);
@@ -300,6 +319,8 @@ axon_ram_remove(struct platform_device *device)
 
         device_remove_file(&device->dev, &dev_attr_ecc);
         free_irq(bank->irq_id, device);
+        kill_dax(bank->dax_dev);
+        put_dax(bank->dax_dev);
         del_gendisk(bank->disk);
         put_disk(bank->disk);
         iounmap((void __iomem *) bank->io_addr);
diff --git a/arch/x86/include/asm/pmem.h b/arch/x86/include/asm/pmem.h
index 529bb4a6487a..d5a22bac9988 100644
--- a/arch/x86/include/asm/pmem.h
+++ b/arch/x86/include/asm/pmem.h
@@ -44,11 +44,6 @@ static inline void arch_memcpy_to_pmem(void *dst, const void *src, size_t n)
                 BUG();
 }
 
-static inline int arch_memcpy_from_pmem(void *dst, const void *src, size_t n)
-{
-        return memcpy_mcsafe(dst, src, n);
-}
-
 /**
  * arch_wb_cache_pmem - write back a cache range with CLWB
  * @vaddr:      virtual start address
diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h
index a164862d77e3..733bae07fb29 100644
--- a/arch/x86/include/asm/string_64.h
+++ b/arch/x86/include/asm/string_64.h
@@ -79,6 +79,7 @@ int strcmp(const char *cs, const char *ct);
 #define memset(s, c, n) __memset(s, c, n)
 #endif
 
+#define __HAVE_ARCH_MEMCPY_MCSAFE 1
 __must_check int memcpy_mcsafe_unrolled(void *dst, const void *src, size_t cnt);
 DECLARE_STATIC_KEY_FALSE(mcsafe_key);
 
diff --git a/block/Kconfig b/block/Kconfig
index 89cd28f8d051..a8ad7e77db28 100644
--- a/block/Kconfig
+++ b/block/Kconfig
@@ -6,6 +6,7 @@ menuconfig BLOCK
        default y
        select SBITMAP
        select SRCU
+       select DAX
        help
          Provide block layer support for the kernel.
 
diff --git a/block/partition-generic.c b/block/partition-generic.c
index 0171a2faad68..ff07b9143ca4 100644
--- a/block/partition-generic.c
+++ b/block/partition-generic.c
@@ -16,7 +16,6 @@
 #include <linux/kmod.h>
 #include <linux/ctype.h>
 #include <linux/genhd.h>
-#include <linux/dax.h>
 #include <linux/blktrace_api.h>
 
 #include "partitions/check.h"
@@ -630,24 +629,12 @@ int invalidate_partitions(struct gendisk *disk, struct block_device *bdev)
         return 0;
 }
 
-static struct page *read_pagecache_sector(struct block_device *bdev, sector_t n)
-{
-        struct address_space *mapping = bdev->bd_inode->i_mapping;
-
-        return read_mapping_page(mapping, (pgoff_t)(n >> (PAGE_SHIFT-9)),
-                                 NULL);
-}
-
 unsigned char *read_dev_sector(struct block_device *bdev, sector_t n, Sector *p)
 {
+        struct address_space *mapping = bdev->bd_inode->i_mapping;
         struct page *page;
 
-        /* don't populate page cache for dax capable devices */
-        if (IS_DAX(bdev->bd_inode))
-                page = read_dax_sector(bdev, n);
-        else
-                page = read_pagecache_sector(bdev, n);
-
+        page = read_mapping_page(mapping, (pgoff_t)(n >> (PAGE_SHIFT-9)), NULL);
         if (!IS_ERR(page)) {
                 if (PageError(page))
                         goto fail;
diff --git a/drivers/Makefile b/drivers/Makefile
index 8f8bdc9e3d29..903b19199b69 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -71,7 +71,7 @@ obj-$(CONFIG_PARPORT)		+= parport/
 obj-$(CONFIG_NVM)               += lightnvm/
 obj-y                           += base/ block/ misc/ mfd/ nfc/
 obj-$(CONFIG_LIBNVDIMM)         += nvdimm/
-obj-$(CONFIG_DEV_DAX)           += dax/
+obj-$(CONFIG_DAX)               += dax/
 obj-$(CONFIG_DMA_SHARED_BUFFER) += dma-buf/
 obj-$(CONFIG_NUBUS)             += nubus/
 obj-y                           += macintosh/
diff --git a/drivers/acpi/nfit/Kconfig b/drivers/acpi/nfit/Kconfig
index dd0d53c52552..6d3351452ea2 100644
--- a/drivers/acpi/nfit/Kconfig
+++ b/drivers/acpi/nfit/Kconfig
@@ -12,15 +12,3 @@ config ACPI_NFIT
 
           To compile this driver as a module, choose M here:
           the module will be called nfit.
-
-config ACPI_NFIT_DEBUG
-        bool "NFIT DSM debug"
-        depends on ACPI_NFIT
-        depends on DYNAMIC_DEBUG
-        default n
-        help
-          Enabling this option causes the nfit driver to dump the
-          input and output buffers of _DSM operations on the ACPI0012
-          device and its children.  This can be very verbose, so leave
-          it disabled unless you are debugging a hardware / firmware
-          issue.
diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c
index c8ea9d698cd0..656acb5d7166 100644
--- a/drivers/acpi/nfit/core.c
+++ b/drivers/acpi/nfit/core.c
@@ -49,7 +49,16 @@ MODULE_PARM_DESC(scrub_overflow_abort,
 static bool disable_vendor_specific;
 module_param(disable_vendor_specific, bool, S_IRUGO);
 MODULE_PARM_DESC(disable_vendor_specific,
-                "Limit commands to the publicly specified set\n");
+                "Limit commands to the publicly specified set");
+
+static unsigned long override_dsm_mask;
+module_param(override_dsm_mask, ulong, S_IRUGO);
+MODULE_PARM_DESC(override_dsm_mask, "Bitmask of allowed NVDIMM DSM functions");
+
+static int default_dsm_family = -1;
+module_param(default_dsm_family, int, S_IRUGO);
+MODULE_PARM_DESC(default_dsm_family,
+                "Try this DSM type first when identifying NVDIMM family");
 
 LIST_HEAD(acpi_descs);
 DEFINE_MUTEX(acpi_desc_lock);
@@ -175,14 +184,29 @@ static int xlat_bus_status(void *buf, unsigned int cmd, u32 status)
         return 0;
 }
 
+static int xlat_nvdimm_status(void *buf, unsigned int cmd, u32 status)
+{
+        switch (cmd) {
+        case ND_CMD_GET_CONFIG_SIZE:
+                if (status >> 16 & ND_CONFIG_LOCKED)
+                        return -EACCES;
+                break;
+        default:
+                break;
+        }
+
+        /* all other non-zero status results in an error */
+        if (status)
+                return -EIO;
+        return 0;
+}
+
 static int xlat_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd,
                 u32 status)
 {
         if (!nvdimm)
                 return xlat_bus_status(buf, cmd, status);
-        if (status)
-                return -EIO;
-        return 0;
+        return xlat_nvdimm_status(buf, cmd, status);
 }
 
 int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm,
@@ -259,14 +283,11 @@ int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm,
                 in_buf.buffer.length = call_pkg->nd_size_in;
         }
 
-        if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) {
-                dev_dbg(dev, "%s:%s cmd: %d: func: %d input length: %d\n",
-                                __func__, dimm_name, cmd, func,
-                                in_buf.buffer.length);
-                print_hex_dump_debug("nvdimm in  ", DUMP_PREFIX_OFFSET, 4, 4,
+        dev_dbg(dev, "%s:%s cmd: %d: func: %d input length: %d\n",
+                        __func__, dimm_name, cmd, func, in_buf.buffer.length);
+        print_hex_dump_debug("nvdimm in  ", DUMP_PREFIX_OFFSET, 4, 4,
                         in_buf.buffer.pointer,
                         min_t(u32, 256, in_buf.buffer.length), true);
-        }
 
         out_obj = acpi_evaluate_dsm(handle, uuid, 1, func, &in_obj);
         if (!out_obj) {
@@ -298,13 +319,11 @@ int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm,
                 goto out;
         }
 
-        if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) {
-                dev_dbg(dev, "%s:%s cmd: %s output length: %d\n", __func__,
-                                dimm_name, cmd_name, out_obj->buffer.length);
-                print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4,
-                                4, out_obj->buffer.pointer, min_t(u32, 128,
-                                        out_obj->buffer.length), true);
-        }
+        dev_dbg(dev, "%s:%s cmd: %s output length: %d\n", __func__, dimm_name,
+                        cmd_name, out_obj->buffer.length);
+        print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4, 4,
+                        out_obj->buffer.pointer,
+                        min_t(u32, 128, out_obj->buffer.length), true);
 
         for (i = 0, offset = 0; i < desc->out_num; i++) {
                 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf,
@@ -448,9 +467,9 @@ static bool add_memdev(struct acpi_nfit_desc *acpi_desc,
         INIT_LIST_HEAD(&nfit_memdev->list);
         memcpy(nfit_memdev->memdev, memdev, sizeof(*memdev));
         list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs);
-        dev_dbg(dev, "%s: memdev handle: %#x spa: %d dcr: %d\n",
+        dev_dbg(dev, "%s: memdev handle: %#x spa: %d dcr: %d flags: %#x\n",
                         __func__, memdev->device_handle, memdev->range_index,
-                        memdev->region_index);
+                        memdev->region_index, memdev->flags);
         return true;
 }
 
@@ -729,28 +748,38 @@ static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc,
         }
 }
 
-static int nfit_mem_dcr_init(struct acpi_nfit_desc *acpi_desc,
+static int __nfit_mem_init(struct acpi_nfit_desc *acpi_desc,
                 struct acpi_nfit_system_address *spa)
 {
         struct nfit_mem *nfit_mem, *found;
         struct nfit_memdev *nfit_memdev;
-        int type = nfit_spa_type(spa);
+        int type = spa ? nfit_spa_type(spa) : 0;
 
         switch (type) {
         case NFIT_SPA_DCR:
         case NFIT_SPA_PM:
                 break;
         default:
-                return 0;
+                if (spa)
+                        return 0;
         }
 
+        /*
+         * This loop runs in two modes, when a dimm is mapped the loop
+         * adds memdev associations to an existing dimm, or creates a
+         * dimm. In the unmapped dimm case this loop sweeps for memdev
+         * instances with an invalid / zero range_index and adds those
+         * dimms without spa associations.
+         */
         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
                 struct nfit_flush *nfit_flush;
                 struct nfit_dcr *nfit_dcr;
                 u32 device_handle;
                 u16 dcr;
 
-                if (nfit_memdev->memdev->range_index != spa->range_index)
+                if (spa && nfit_memdev->memdev->range_index != spa->range_index)
+                        continue;
+                if (!spa && nfit_memdev->memdev->range_index)
                         continue;
                 found = NULL;
                 dcr = nfit_memdev->memdev->region_index;
@@ -835,14 +864,15 @@ static int nfit_mem_dcr_init(struct acpi_nfit_desc *acpi_desc,
                                 break;
                         }
                         nfit_mem_init_bdw(acpi_desc, nfit_mem, spa);
-                } else {
+                } else if (type == NFIT_SPA_PM) {
                         /*
                          * A single dimm may belong to multiple SPA-PM
                          * ranges, record at least one in addition to
                          * any SPA-DCR range.
                          */
                         nfit_mem->memdev_pmem = nfit_memdev->memdev;
-                }
+                } else
+                        nfit_mem->memdev_dcr = nfit_memdev->memdev;
         }
 
         return 0;
@@ -866,6 +896,8 @@ static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b)
 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc)
 {
         struct nfit_spa *nfit_spa;
+        int rc;
+
 
         /*
          * For each SPA-DCR or SPA-PMEM address range find its
@@ -876,13 +908,20 @@ static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc)
          * BDWs are optional.
          */
         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
-                int rc;
-
-                rc = nfit_mem_dcr_init(acpi_desc, nfit_spa->spa);
+                rc = __nfit_mem_init(acpi_desc, nfit_spa->spa);
                 if (rc)
                         return rc;
         }
 
+        /*
+         * If a DIMM has failed to be mapped into SPA there will be no
+         * SPA entries above. Find and register all the unmapped DIMMs
+         * for reporting and recovery purposes.
+         */
+        rc = __nfit_mem_init(acpi_desc, NULL);
+        if (rc)
+                return rc;
+
         list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp);
 
         return 0;
@@ -1237,12 +1276,14 @@ static ssize_t flags_show(struct device *dev,
 {
         u16 flags = to_nfit_memdev(dev)->flags;
 
-        return sprintf(buf, "%s%s%s%s%s\n",
+        return sprintf(buf, "%s%s%s%s%s%s%s\n",
                 flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "",
                 flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "",
                 flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "",
                 flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "",
-                flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "");
+                flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "",
+                flags & ACPI_NFIT_MEM_MAP_FAILED ? "map_fail " : "",
+                flags & ACPI_NFIT_MEM_HEALTH_ENABLED ? "smart_notify " : "");
 }
 static DEVICE_ATTR_RO(flags);
 
@@ -1290,8 +1331,16 @@ static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj,
         struct device *dev = container_of(kobj, struct device, kobj);
         struct nvdimm *nvdimm = to_nvdimm(dev);
 
-        if (!to_nfit_dcr(dev))
+        if (!to_nfit_dcr(dev)) {
+                /* Without a dcr only the memdev attributes can be surfaced */
+                if (a == &dev_attr_handle.attr || a == &dev_attr_phys_id.attr
+                                || a == &dev_attr_flags.attr
+                                || a == &dev_attr_family.attr
+                                || a == &dev_attr_dsm_mask.attr)
+                        return a->mode;
                 return 0;
+        }
+
         if (a == &dev_attr_format1.attr && num_nvdimm_formats(nvdimm) <= 1)
                 return 0;
         return a->mode;
@@ -1368,6 +1417,7 @@ static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
         unsigned long dsm_mask;
         const u8 *uuid;
         int i;
+        int family = -1;
 
         /* nfit test assumes 1:1 relationship between commands and dsms */
         nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en;
@@ -1398,11 +1448,14 @@ static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
          */
         for (i = NVDIMM_FAMILY_INTEL; i <= NVDIMM_FAMILY_MSFT; i++)
                 if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1))
-                        break;
+                        if (family < 0 || i == default_dsm_family)
+                                family = i;
 
         /* limit the supported commands to those that are publicly documented */
-        nfit_mem->family = i;
-        if (nfit_mem->family == NVDIMM_FAMILY_INTEL) {
+        nfit_mem->family = family;
+        if (override_dsm_mask && !disable_vendor_specific)
+                dsm_mask = override_dsm_mask;
+        else if (nfit_mem->family == NVDIMM_FAMILY_INTEL) {
                 dsm_mask = 0x3fe;
                 if (disable_vendor_specific)
                         dsm_mask &= ~(1 << ND_CMD_VENDOR);
@@ -1462,6 +1515,7 @@ static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
         list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
                 struct acpi_nfit_flush_address *flush;
                 unsigned long flags = 0, cmd_mask;
+                struct nfit_memdev *nfit_memdev;
                 u32 device_handle;
                 u16 mem_flags;
 
@@ -1473,11 +1527,22 @@ static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
                 }
 
                 if (nfit_mem->bdw && nfit_mem->memdev_pmem)
-                        flags |= NDD_ALIASING;
+                        set_bit(NDD_ALIASING, &flags);
+
+                /* collate flags across all memdevs for this dimm */
+                list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
+                        struct acpi_nfit_memory_map *dimm_memdev;
+
+                        dimm_memdev = __to_nfit_memdev(nfit_mem);
+                        if (dimm_memdev->device_handle
+                                        != nfit_memdev->memdev->device_handle)
+                                continue;
+                        dimm_memdev->flags |= nfit_memdev->memdev->flags;
+                }
 
                 mem_flags = __to_nfit_memdev(nfit_mem)->flags;
                 if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED)
-                        flags |= NDD_UNARMED;
+                        set_bit(NDD_UNARMED, &flags);
 
                 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle);
                 if (rc)
@@ -1507,12 +1572,13 @@ static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
                 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0)
                         continue;
 
-                dev_info(acpi_desc->dev, "%s flags:%s%s%s%s\n",
+                dev_info(acpi_desc->dev, "%s flags:%s%s%s%s%s\n",
                                 nvdimm_name(nvdimm),
                   mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "",
                   mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"",
                   mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "",
-                  mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "");
+                  mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "",
+                  mem_flags & ACPI_NFIT_MEM_MAP_FAILED ? " map_fail" : "");
 
         }
 
@@ -1783,8 +1849,7 @@ static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,
                                 mmio_flush_range((void __force *)
                                         mmio->addr.aperture + offset, c);
 
-                        memcpy_from_pmem(iobuf + copied,
-                                        mmio->addr.aperture + offset, c);
+                        memcpy(iobuf + copied, mmio->addr.aperture + offset, c);
                 }
 
                 copied += c;
@@ -2525,6 +2590,7 @@ static void acpi_nfit_scrub(struct work_struct *work)
                         acpi_nfit_register_region(acpi_desc, nfit_spa);
                 }
         }
+        acpi_desc->init_complete = 1;
 
         list_for_each_entry(nfit_spa, &acpi_desc->spas, list)
                 acpi_nfit_async_scrub(acpi_desc, nfit_spa);
@@ -2547,7 +2613,8 @@ static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc)
                                 return rc;
                 }
 
-        queue_work(nfit_wq, &acpi_desc->work);
+        if (!acpi_desc->cancel)
+                queue_work(nfit_wq, &acpi_desc->work);
         return 0;
 }
 
@@ -2593,32 +2660,11 @@ static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc)
         return 0;
 }
 
-static void acpi_nfit_destruct(void *data)
+static void acpi_nfit_unregister(void *data)
 {
         struct acpi_nfit_desc *acpi_desc = data;
-        struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus);
-
-        /*
-         * Destruct under acpi_desc_lock so that nfit_handle_mce does not
-         * race teardown
-         */
-        mutex_lock(&acpi_desc_lock);
-        acpi_desc->cancel = 1;
-        /*
-         * Bounce the nvdimm bus lock to make sure any in-flight
-         * acpi_nfit_ars_rescan() submissions have had a chance to
-         * either submit or see ->cancel set.
-         */
-        device_lock(bus_dev);
-        device_unlock(bus_dev);
 
-        flush_workqueue(nfit_wq);
-        if (acpi_desc->scrub_count_state)
-                sysfs_put(acpi_desc->scrub_count_state);
         nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
-        acpi_desc->nvdimm_bus = NULL;
-        list_del(&acpi_desc->list);
-        mutex_unlock(&acpi_desc_lock);
 }
 
 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz)
@@ -2636,7 +2682,7 @@ int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz)
                 if (!acpi_desc->nvdimm_bus)
                         return -ENOMEM;
 
-                rc = devm_add_action_or_reset(dev, acpi_nfit_destruct,
+                rc = devm_add_action_or_reset(dev, acpi_nfit_unregister,
                                 acpi_desc);
                 if (rc)
                         return rc;
@@ -2728,6 +2774,13 @@ static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc)
         device_lock(dev);
         device_unlock(dev);
 
+        /* bounce the init_mutex to make init_complete valid */
+        mutex_lock(&acpi_desc->init_mutex);
+        if (acpi_desc->cancel || acpi_desc->init_complete) {
+                mutex_unlock(&acpi_desc->init_mutex);
+                return 0;
+        }
+
         /*
          * Scrub work could take 10s of seconds, userspace may give up so we
          * need to be interruptible while waiting.
@@ -2735,6 +2788,7 @@ static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc)
         INIT_WORK_ONSTACK(&flush.work, flush_probe);
         COMPLETION_INITIALIZER_ONSTACK(flush.cmp);
         queue_work(nfit_wq, &flush.work);
+        mutex_unlock(&acpi_desc->init_mutex);
 
         rc = wait_for_completion_interruptible(&flush.cmp);
         cancel_work_sync(&flush.work);
@@ -2771,10 +2825,12 @@ int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc)
         if (work_busy(&acpi_desc->work))
                 return -EBUSY;
 
-        if (acpi_desc->cancel)
+        mutex_lock(&acpi_desc->init_mutex);
+        if (acpi_desc->cancel) {
+                mutex_unlock(&acpi_desc->init_mutex);
                 return 0;
+        }
 
-        mutex_lock(&acpi_desc->init_mutex);
         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
                 struct acpi_nfit_system_address *spa = nfit_spa->spa;
 
@@ -2818,6 +2874,40 @@ void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev)
 }
 EXPORT_SYMBOL_GPL(acpi_nfit_desc_init);
 
+static void acpi_nfit_put_table(void *table)
+{
+        acpi_put_table(table);
+}
+
+void acpi_nfit_shutdown(void *data)
+{
+        struct acpi_nfit_desc *acpi_desc = data;
+        struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus);
+
+        /*
+         * Destruct under acpi_desc_lock so that nfit_handle_mce does not
+         * race teardown
+         */
+        mutex_lock(&acpi_desc_lock);
+        list_del(&acpi_desc->list);
+        mutex_unlock(&acpi_desc_lock);
+
+        mutex_lock(&acpi_desc->init_mutex);
+        acpi_desc->cancel = 1;
+        mutex_unlock(&acpi_desc->init_mutex);
+
+        /*
+         * Bounce the nvdimm bus lock to make sure any in-flight
+         * acpi_nfit_ars_rescan() submissions have had a chance to
+         * either submit or see ->cancel set.
+         */
+        device_lock(bus_dev);
+        device_unlock(bus_dev);
+
+        flush_workqueue(nfit_wq);
+}
+EXPORT_SYMBOL_GPL(acpi_nfit_shutdown);
+
 static int acpi_nfit_add(struct acpi_device *adev)
 {
         struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
@@ -2834,6 +2924,10 @@ static int acpi_nfit_add(struct acpi_device *adev)
                 dev_dbg(dev, "failed to find NFIT at startup\n");
                 return 0;
         }
+
+        rc = devm_add_action_or_reset(dev, acpi_nfit_put_table, tbl);
+        if (rc)
+                return rc;
         sz = tbl->length;
 
         acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
@@ -2861,12 +2955,15 @@ static int acpi_nfit_add(struct acpi_device *adev)
                 rc = acpi_nfit_init(acpi_desc, (void *) tbl
                                 + sizeof(struct acpi_table_nfit),
                                 sz - sizeof(struct acpi_table_nfit));
-        return rc;
+
+        if (rc)
+                return rc;
+        return devm_add_action_or_reset(dev, acpi_nfit_shutdown, acpi_desc);
 }
 
 static int acpi_nfit_remove(struct acpi_device *adev)
 {
-        /* see acpi_nfit_destruct */
+        /* see acpi_nfit_unregister */
         return 0;
 }
 
diff --git a/drivers/acpi/nfit/nfit.h b/drivers/acpi/nfit/nfit.h
index fc29c2e9832e..58fb7d68e04a 100644
--- a/drivers/acpi/nfit/nfit.h
+++ b/drivers/acpi/nfit/nfit.h
@@ -37,7 +37,7 @@
 
 #define ACPI_NFIT_MEM_FAILED_MASK (ACPI_NFIT_MEM_SAVE_FAILED \
                 | ACPI_NFIT_MEM_RESTORE_FAILED | ACPI_NFIT_MEM_FLUSH_FAILED \
-                | ACPI_NFIT_MEM_NOT_ARMED)
+                | ACPI_NFIT_MEM_NOT_ARMED | ACPI_NFIT_MEM_MAP_FAILED)
 
 enum nfit_uuids {
         /* for simplicity alias the uuid index with the family id */
@@ -163,6 +163,7 @@ struct acpi_nfit_desc {
         unsigned int scrub_count;
         unsigned int scrub_mode;
         unsigned int cancel:1;
+        unsigned int init_complete:1;
         unsigned long dimm_cmd_force_en;
         unsigned long bus_cmd_force_en;
         int (*blk_do_io)(struct nd_blk_region *ndbr, resource_size_t dpa,
@@ -238,6 +239,7 @@ static inline struct acpi_nfit_desc *to_acpi_desc(
 
 const u8 *to_nfit_uuid(enum nfit_uuids id);
 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *nfit, acpi_size sz);
+void acpi_nfit_shutdown(void *data);
 void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event);
 void __acpi_nvdimm_notify(struct device *dev, u32 event);
 int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm,
diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig
index d545abbd5378..8ddc98279c8f 100644
--- a/drivers/block/Kconfig
+++ b/drivers/block/Kconfig
@@ -323,6 +323,7 @@ config BLK_DEV_SX8
 
 config BLK_DEV_RAM
         tristate "RAM block device support"
+        select DAX if BLK_DEV_RAM_DAX
         ---help---
           Saying Y here will allow you to use a portion of your RAM memory as
           a block device, so that you can make file systems on it, read and
diff --git a/drivers/block/brd.c b/drivers/block/brd.c
index 4ec84d504780..57b574f2f66a 100644
--- a/drivers/block/brd.c
+++ b/drivers/block/brd.c
@@ -21,6 +21,7 @@
 #include <linux/slab.h>
 #ifdef CONFIG_BLK_DEV_RAM_DAX
 #include <linux/pfn_t.h>
+#include <linux/dax.h>
 #endif
 
 #include <linux/uaccess.h>
@@ -41,6 +42,9 @@ struct brd_device {
 
         struct request_queue    *brd_queue;
         struct gendisk          *brd_disk;
+#ifdef CONFIG_BLK_DEV_RAM_DAX
+        struct dax_device       *dax_dev;
+#endif
         struct list_head        brd_list;
 
         /*
@@ -326,30 +330,38 @@ static int brd_rw_page(struct block_device *bdev, sector_t sector,
 }
 
 #ifdef CONFIG_BLK_DEV_RAM_DAX
-static long brd_direct_access(struct block_device *bdev, sector_t sector,
-                        void **kaddr, pfn_t *pfn, long size)
+static long __brd_direct_access(struct brd_device *brd, pgoff_t pgoff,
+                long nr_pages, void **kaddr, pfn_t *pfn)
 {
-        struct brd_device *brd = bdev->bd_disk->private_data;
         struct page *page;
 
         if (!brd)
                 return -ENODEV;
-        page = brd_insert_page(brd, sector);
+        page = brd_insert_page(brd, PFN_PHYS(pgoff) / 512);
         if (!page)
                 return -ENOSPC;
         *kaddr = page_address(page);
         *pfn = page_to_pfn_t(page);
 
-        return PAGE_SIZE;
+        return 1;
 }
-#else
-#define brd_direct_access NULL
+
+static long brd_dax_direct_access(struct dax_device *dax_dev,
+                pgoff_t pgoff, long nr_pages, void **kaddr, pfn_t *pfn)
+{
+        struct brd_device *brd = dax_get_private(dax_dev);
+
+        return __brd_direct_access(brd, pgoff, nr_pages, kaddr, pfn);
+}
+
+static const struct dax_operations brd_dax_ops = {
+        .direct_access = brd_dax_direct_access,
+};
 #endif
 
 static const struct block_device_operations brd_fops = {
         .owner =                THIS_MODULE,
         .rw_page =              brd_rw_page,
-        .direct_access =        brd_direct_access,
 };
 
 /*
@@ -415,9 +427,6 @@ static struct brd_device *brd_alloc(int i)
          *  is harmless)
          */
         blk_queue_physical_block_size(brd->brd_queue, PAGE_SIZE);
-#ifdef CONFIG_BLK_DEV_RAM_DAX
-        queue_flag_set_unlocked(QUEUE_FLAG_DAX, brd->brd_queue);
-#endif
         disk = brd->brd_disk = alloc_disk(max_part);
         if (!disk)
                 goto out_free_queue;
@@ -430,8 +439,21 @@ static struct brd_device *brd_alloc(int i)
         sprintf(disk->disk_name, "ram%d", i);
         set_capacity(disk, rd_size * 2);
 
+#ifdef CONFIG_BLK_DEV_RAM_DAX
+        queue_flag_set_unlocked(QUEUE_FLAG_DAX, brd->brd_queue);
+        brd->dax_dev = alloc_dax(brd, disk->disk_name, &brd_dax_ops);
+        if (!brd->dax_dev)
+                goto out_free_inode;
+#endif
+
+
         return brd;
 
+#ifdef CONFIG_BLK_DEV_RAM_DAX
+out_free_inode:
+        kill_dax(brd->dax_dev);
+        put_dax(brd->dax_dev);
+#endif
 out_free_queue:
         blk_cleanup_queue(brd->brd_queue);
 out_free_dev:
@@ -471,6 +493,10 @@ out:
 static void brd_del_one(struct brd_device *brd)
 {
         list_del(&brd->brd_list);
+#ifdef CONFIG_BLK_DEV_RAM_DAX
+        kill_dax(brd->dax_dev);
+        put_dax(brd->dax_dev);
+#endif
         del_gendisk(brd->brd_disk);
         brd_free(brd);
 }
diff --git a/drivers/dax/Kconfig b/drivers/dax/Kconfig
index 9e95bf94eb13..b7053eafd88e 100644
--- a/drivers/dax/Kconfig
+++ b/drivers/dax/Kconfig
@@ -1,8 +1,13 @@
-menuconfig DEV_DAX
+menuconfig DAX
         tristate "DAX: direct access to differentiated memory"
+        select SRCU
         default m if NVDIMM_DAX
+
+if DAX
+
+config DEV_DAX
+        tristate "Device DAX: direct access mapping device"
         depends on TRANSPARENT_HUGEPAGE
-        select SRCU
         help
           Support raw access to differentiated (persistence, bandwidth,
           latency...) memory via an mmap(2) capable character
@@ -11,7 +16,6 @@ menuconfig DEV_DAX
           baseline memory pool.  Mappings of a /dev/daxX.Y device impose
           restrictions that make the mapping behavior deterministic.
 
-if DEV_DAX
 
 config DEV_DAX_PMEM
         tristate "PMEM DAX: direct access to persistent memory"
diff --git a/drivers/dax/Makefile b/drivers/dax/Makefile
index 27c54e38478a..dc7422530462 100644
--- a/drivers/dax/Makefile
+++ b/drivers/dax/Makefile
@@ -1,4 +1,7 @@
-obj-$(CONFIG_DEV_DAX) += dax.o
+obj-$(CONFIG_DAX) += dax.o
+obj-$(CONFIG_DEV_DAX) += device_dax.o
 obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem.o
 
+dax-y := super.o
 dax_pmem-y := pmem.o
+device_dax-y := device.o
diff --git a/drivers/dax/dax-private.h b/drivers/dax/dax-private.h
new file mode 100644
index 000000000000..b6fc4f04636d
--- /dev/null
+++ b/drivers/dax/dax-private.h
@@ -0,0 +1,57 @@
+/*
+ * Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+#ifndef __DAX_PRIVATE_H__
+#define __DAX_PRIVATE_H__
+
+#include <linux/device.h>
+#include <linux/cdev.h>
+
+/**
+ * struct dax_region - mapping infrastructure for dax devices
+ * @id: kernel-wide unique region for a memory range
+ * @base: linear address corresponding to @res
+ * @kref: to pin while other agents have a need to do lookups
+ * @dev: parent device backing this region
+ * @align: allocation and mapping alignment for child dax devices
+ * @res: physical address range of the region
+ * @pfn_flags: identify whether the pfns are paged back or not
+ */
+struct dax_region {
+        int id;
+        struct ida ida;
+        void *base;
+        struct kref kref;
+        struct device *dev;
+        unsigned int align;
+        struct resource res;
+        unsigned long pfn_flags;
+};
+
+/**
+ * struct dev_dax - instance data for a subdivision of a dax region
+ * @region - parent region
+ * @dax_dev - core dax functionality
+ * @dev - device core
+ * @id - child id in the region
+ * @num_resources - number of physical address extents in this device
+ * @res - array of physical address ranges
+ */
+struct dev_dax {
+        struct dax_region *region;
+        struct dax_device *dax_dev;
+        struct device dev;
+        int id;
+        int num_resources;
+        struct resource res[0];
+};
+#endif
diff --git a/drivers/dax/dax.h b/drivers/dax/dax.h
index ddd829ab58c0..f9e5feea742c 100644
--- a/drivers/dax/dax.h
+++ b/drivers/dax/dax.h
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2016 Intel Corporation. All rights reserved.
+ * Copyright(c) 2016 - 2017 Intel Corporation. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -12,14 +12,7 @@
  */
 #ifndef __DAX_H__
 #define __DAX_H__
-struct device;
-struct dax_dev;
-struct resource;
-struct dax_region;
-void dax_region_put(struct dax_region *dax_region);
-struct dax_region *alloc_dax_region(struct device *parent,
-                int region_id, struct resource *res, unsigned int align,
-                void *addr, unsigned long flags);
-struct dax_dev *devm_create_dax_dev(struct dax_region *dax_region,
-                struct resource *res, int count);
+struct dax_device;
+struct dax_device *inode_dax(struct inode *inode);
+struct inode *dax_inode(struct dax_device *dax_dev);
 #endif /* __DAX_H__ */
diff --git a/drivers/dax/device-dax.h b/drivers/dax/device-dax.h
new file mode 100644
index 000000000000..fdcd9769ffde
--- /dev/null
+++ b/drivers/dax/device-dax.h
@@ -0,0 +1,25 @@
+/*
+ * Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+#ifndef __DEVICE_DAX_H__
+#define __DEVICE_DAX_H__
+struct device;
+struct dev_dax;
+struct resource;
+struct dax_region;
+void dax_region_put(struct dax_region *dax_region);
+struct dax_region *alloc_dax_region(struct device *parent,
+                int region_id, struct resource *res, unsigned int align,
+                void *addr, unsigned long flags);
+struct dev_dax *devm_create_dev_dax(struct dax_region *dax_region,
+                struct resource *res, int count);
+#endif /* __DEVICE_DAX_H__ */
diff --git a/drivers/dax/dax.c b/drivers/dax/device.c
index 19795eb35579..006e657dfcb9 100644
--- a/drivers/dax/dax.c
+++ b/drivers/dax/device.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2016 Intel Corporation. All rights reserved.
+ * Copyright(c) 2016 - 2017 Intel Corporation. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -13,100 +13,38 @@
 #include <linux/pagemap.h>
 #include <linux/module.h>
 #include <linux/device.h>
-#include <linux/magic.h>
-#include <linux/mount.h>
 #include <linux/pfn_t.h>
-#include <linux/hash.h>
 #include <linux/cdev.h>
 #include <linux/slab.h>
 #include <linux/dax.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
+#include "dax-private.h"
 #include "dax.h"
 
-static dev_t dax_devt;
-DEFINE_STATIC_SRCU(dax_srcu);
 static struct class *dax_class;
-static DEFINE_IDA(dax_minor_ida);
-static int nr_dax = CONFIG_NR_DEV_DAX;
-module_param(nr_dax, int, S_IRUGO);
-static struct vfsmount *dax_mnt;
-static struct kmem_cache *dax_cache __read_mostly;
-static struct super_block *dax_superblock __read_mostly;
-MODULE_PARM_DESC(nr_dax, "max number of device-dax instances");
-
-/**
- * struct dax_region - mapping infrastructure for dax devices
- * @id: kernel-wide unique region for a memory range
- * @base: linear address corresponding to @res
- * @kref: to pin while other agents have a need to do lookups
- * @dev: parent device backing this region
- * @align: allocation and mapping alignment for child dax devices
- * @res: physical address range of the region
- * @pfn_flags: identify whether the pfns are paged back or not
- */
-struct dax_region {
-        int id;
-        struct ida ida;
-        void *base;
-        struct kref kref;
-        struct device *dev;
-        unsigned int align;
-        struct resource res;
-        unsigned long pfn_flags;
-};
 
-/**
- * struct dax_dev - subdivision of a dax region
- * @region - parent region
- * @dev - device backing the character device
- * @cdev - core chardev data
- * @alive - !alive + srcu grace period == no new mappings can be established
- * @id - child id in the region
- * @num_resources - number of physical address extents in this device
- * @res - array of physical address ranges
+/*
+ * Rely on the fact that drvdata is set before the attributes are
+ * registered, and that the attributes are unregistered before drvdata
+ * is cleared to assume that drvdata is always valid.
  */
-struct dax_dev {
-        struct dax_region *region;
-        struct inode *inode;
-        struct device dev;
-        struct cdev cdev;
-        bool alive;
-        int id;
-        int num_resources;
-        struct resource res[0];
-};
-
 static ssize_t id_show(struct device *dev,
                 struct device_attribute *attr, char *buf)
 {
-        struct dax_region *dax_region;
-        ssize_t rc = -ENXIO;
-
-        device_lock(dev);
-        dax_region = dev_get_drvdata(dev);
-        if (dax_region)
-                rc = sprintf(buf, "%d\n", dax_region->id);
-        device_unlock(dev);
+        struct dax_region *dax_region = dev_get_drvdata(dev);
 
-        return rc;
+        return sprintf(buf, "%d\n", dax_region->id);
 }
 static DEVICE_ATTR_RO(id);
 
 static ssize_t region_size_show(struct device *dev,
                 struct device_attribute *attr, char *buf)
 {
-        struct dax_region *dax_region;
-        ssize_t rc = -ENXIO;
-
-        device_lock(dev);
-        dax_region = dev_get_drvdata(dev);
-        if (dax_region)
-                rc = sprintf(buf, "%llu\n", (unsigned long long)
-                                resource_size(&dax_region->res));
-        device_unlock(dev);
+        struct dax_region *dax_region = dev_get_drvdata(dev);
 
-        return rc;
+        return sprintf(buf, "%llu\n", (unsigned long long)
+                        resource_size(&dax_region->res));
 }
 static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
                 region_size_show, NULL);
@@ -114,16 +52,9 @@ static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
 static ssize_t align_show(struct device *dev,
                 struct device_attribute *attr, char *buf)
 {
-        struct dax_region *dax_region;
-        ssize_t rc = -ENXIO;
-
-        device_lock(dev);
-        dax_region = dev_get_drvdata(dev);
-        if (dax_region)
-                rc = sprintf(buf, "%u\n", dax_region->align);
-        device_unlock(dev);
+        struct dax_region *dax_region = dev_get_drvdata(dev);
 
-        return rc;
+        return sprintf(buf, "%u\n", dax_region->align);
 }
 static DEVICE_ATTR_RO(align);
 
@@ -144,117 +75,6 @@ static const struct attribute_group *dax_region_attribute_groups[] = {
         NULL,
 };
 
-static struct inode *dax_alloc_inode(struct super_block *sb)
-{
-        return kmem_cache_alloc(dax_cache, GFP_KERNEL);
-}
-
-static void dax_i_callback(struct rcu_head *head)
-{
-        struct inode *inode = container_of(head, struct inode, i_rcu);
-
-        kmem_cache_free(dax_cache, inode);
-}
-
-static void dax_destroy_inode(struct inode *inode)
-{
-        call_rcu(&inode->i_rcu, dax_i_callback);
-}
-
-static const struct super_operations dax_sops = {
-        .statfs = simple_statfs,
-        .alloc_inode = dax_alloc_inode,
-        .destroy_inode = dax_destroy_inode,
-        .drop_inode = generic_delete_inode,
-};
-
-static struct dentry *dax_mount(struct file_system_type *fs_type,
-                int flags, const char *dev_name, void *data)
-{
-        return mount_pseudo(fs_type, "dax:", &dax_sops, NULL, DAXFS_MAGIC);
-}
-
-static struct file_system_type dax_type = {
-        .name = "dax",
-        .mount = dax_mount,
-        .kill_sb = kill_anon_super,
-};
-
-static int dax_test(struct inode *inode, void *data)
-{
-        return inode->i_cdev == data;
-}
-
-static int dax_set(struct inode *inode, void *data)
-{
-        inode->i_cdev = data;
-        return 0;
-}
-
-static struct inode *dax_inode_get(struct cdev *cdev, dev_t devt)
-{
-        struct inode *inode;
-
-        inode = iget5_locked(dax_superblock, hash_32(devt + DAXFS_MAGIC, 31),
-                        dax_test, dax_set, cdev);
-
-        if (!inode)
-                return NULL;
-
-        if (inode->i_state & I_NEW) {
-                inode->i_mode = S_IFCHR;
-                inode->i_flags = S_DAX;
-                inode->i_rdev = devt;
-                mapping_set_gfp_mask(&inode->i_data, GFP_USER);
-                unlock_new_inode(inode);
-        }
-        return inode;
-}
-
-static void init_once(void *inode)
-{
-        inode_init_once(inode);
-}
-
-static int dax_inode_init(void)
-{
-        int rc;
-
-        dax_cache = kmem_cache_create("dax_cache", sizeof(struct inode), 0,
-                        (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
-                         SLAB_MEM_SPREAD|SLAB_ACCOUNT),
-                        init_once);
-        if (!dax_cache)
-                return -ENOMEM;
-
-        rc = register_filesystem(&dax_type);
-        if (rc)
-                goto err_register_fs;
-
-        dax_mnt = kern_mount(&dax_type);
-        if (IS_ERR(dax_mnt)) {
-                rc = PTR_ERR(dax_mnt);
-                goto err_mount;
-        }
-        dax_superblock = dax_mnt->mnt_sb;
-
-        return 0;
-
- err_mount:
-        unregister_filesystem(&dax_type);
- err_register_fs:
-        kmem_cache_destroy(dax_cache);
-
-        return rc;
-}
-
-static void dax_inode_exit(void)
-{
-        kern_unmount(dax_mnt);
-        unregister_filesystem(&dax_type);
-        kmem_cache_destroy(dax_cache);
-}
-
 static void dax_region_free(struct kref *kref)
 {
         struct dax_region *dax_region;
@@ -323,47 +143,47 @@ struct dax_region *alloc_dax_region(struct device *parent, int region_id,
 }
 EXPORT_SYMBOL_GPL(alloc_dax_region);
 
-static struct dax_dev *to_dax_dev(struct device *dev)
+static struct dev_dax *to_dev_dax(struct device *dev)
 {
-        return container_of(dev, struct dax_dev, dev);
+        return container_of(dev, struct dev_dax, dev);
 }
 
 static ssize_t size_show(struct device *dev,
                 struct device_attribute *attr, char *buf)
 {
-        struct dax_dev *dax_dev = to_dax_dev(dev);
+        struct dev_dax *dev_dax = to_dev_dax(dev);
         unsigned long long size = 0;
         int i;
 
-        for (i = 0; i < dax_dev->num_resources; i++)
-                size += resource_size(&dax_dev->res[i]);
+        for (i = 0; i < dev_dax->num_resources; i++)
+                size += resource_size(&dev_dax->res[i]);
 
         return sprintf(buf, "%llu\n", size);
 }
 static DEVICE_ATTR_RO(size);
 
-static struct attribute *dax_device_attributes[] = {
+static struct attribute *dev_dax_attributes[] = {
         &dev_attr_size.attr,
         NULL,
 };
 
-static const struct attribute_group dax_device_attribute_group = {
-        .attrs = dax_device_attributes,
+static const struct attribute_group dev_dax_attribute_group = {
+        .attrs = dev_dax_attributes,
 };
 
 static const struct attribute_group *dax_attribute_groups[] = {
-        &dax_device_attribute_group,
+        &dev_dax_attribute_group,
         NULL,
 };
 
-static int check_vma(struct dax_dev *dax_dev, struct vm_area_struct *vma,
+static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma,
                 const char *func)
 {
-        struct dax_region *dax_region = dax_dev->region;
-        struct device *dev = &dax_dev->dev;
+        struct dax_region *dax_region = dev_dax->region;
+        struct device *dev = &dev_dax->dev;
         unsigned long mask;
 
-        if (!dax_dev->alive)
+        if (!dax_alive(dev_dax->dax_dev))
                 return -ENXIO;
 
         /* prevent private mappings from being established */
@@ -397,23 +217,24 @@ static int check_vma(struct dax_dev *dax_dev, struct vm_area_struct *vma,
         return 0;
 }
 
-static phys_addr_t pgoff_to_phys(struct dax_dev *dax_dev, pgoff_t pgoff,
+/* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */
+__weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
                 unsigned long size)
 {
         struct resource *res;
         phys_addr_t phys;
         int i;
 
-        for (i = 0; i < dax_dev->num_resources; i++) {
-                res = &dax_dev->res[i];
+        for (i = 0; i < dev_dax->num_resources; i++) {
+                res = &dev_dax->res[i];
                 phys = pgoff * PAGE_SIZE + res->start;
                 if (phys >= res->start && phys <= res->end)
                         break;
                 pgoff -= PHYS_PFN(resource_size(res));
         }
 
-        if (i < dax_dev->num_resources) {
-                res = &dax_dev->res[i];
+        if (i < dev_dax->num_resources) {
+                res = &dev_dax->res[i];
                 if (phys + size - 1 <= res->end)
                         return phys;
         }
@@ -421,28 +242,29 @@ static phys_addr_t pgoff_to_phys(struct dax_dev *dax_dev, pgoff_t pgoff,
         return -1;
 }
 
-static int __dax_dev_pte_fault(struct dax_dev *dax_dev, struct vm_fault *vmf)
+static int __dev_dax_pte_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
 {
-        struct device *dev = &dax_dev->dev;
+        struct device *dev = &dev_dax->dev;
         struct dax_region *dax_region;
         int rc = VM_FAULT_SIGBUS;
         phys_addr_t phys;
         pfn_t pfn;
         unsigned int fault_size = PAGE_SIZE;
 
-        if (check_vma(dax_dev, vmf->vma, __func__))
+        if (check_vma(dev_dax, vmf->vma, __func__))
                 return VM_FAULT_SIGBUS;
 
-        dax_region = dax_dev->region;
+        dax_region = dev_dax->region;
         if (dax_region->align > PAGE_SIZE) {
-                dev_dbg(dev, "%s: alignment > fault size\n", __func__);
+                dev_dbg(dev, "%s: alignment (%#x) > fault size (%#x)\n",
+                        __func__, dax_region->align, fault_size);
                 return VM_FAULT_SIGBUS;
         }
 
         if (fault_size != dax_region->align)
                 return VM_FAULT_SIGBUS;
 
-        phys = pgoff_to_phys(dax_dev, vmf->pgoff, PAGE_SIZE);
+        phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE);
         if (phys == -1) {
                 dev_dbg(dev, "%s: pgoff_to_phys(%#lx) failed\n", __func__,
                                 vmf->pgoff);
@@ -461,28 +283,29 @@ static int __dax_dev_pte_fault(struct dax_dev *dax_dev, struct vm_fault *vmf)
         return VM_FAULT_NOPAGE;
 }
 
-static int __dax_dev_pmd_fault(struct dax_dev *dax_dev, struct vm_fault *vmf)
+static int __dev_dax_pmd_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
 {
         unsigned long pmd_addr = vmf->address & PMD_MASK;
-        struct device *dev = &dax_dev->dev;
+        struct device *dev = &dev_dax->dev;
         struct dax_region *dax_region;
         phys_addr_t phys;
         pgoff_t pgoff;
         pfn_t pfn;
         unsigned int fault_size = PMD_SIZE;
 
-        if (check_vma(dax_dev, vmf->vma, __func__))
+        if (check_vma(dev_dax, vmf->vma, __func__))
                 return VM_FAULT_SIGBUS;
 
-        dax_region = dax_dev->region;
+        dax_region = dev_dax->region;
         if (dax_region->align > PMD_SIZE) {
-                dev_dbg(dev, "%s: alignment > fault size\n", __func__);
+                dev_dbg(dev, "%s: alignment (%#x) > fault size (%#x)\n",
+                        __func__, dax_region->align, fault_size);
                 return VM_FAULT_SIGBUS;
         }
 
         /* dax pmd mappings require pfn_t_devmap() */
         if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) {
-                dev_dbg(dev, "%s: alignment > fault size\n", __func__);
+                dev_dbg(dev, "%s: region lacks devmap flags\n", __func__);
                 return VM_FAULT_SIGBUS;
         }
 
@@ -497,7 +320,7 @@ static int __dax_dev_pmd_fault(struct dax_dev *dax_dev, struct vm_fault *vmf)
                 return VM_FAULT_SIGBUS;
 
         pgoff = linear_page_index(vmf->vma, pmd_addr);
-        phys = pgoff_to_phys(dax_dev, pgoff, PMD_SIZE);
+        phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE);
         if (phys == -1) {
                 dev_dbg(dev, "%s: pgoff_to_phys(%#lx) failed\n", __func__,
                                 pgoff);
@@ -511,10 +334,10 @@ static int __dax_dev_pmd_fault(struct dax_dev *dax_dev, struct vm_fault *vmf)
 }
 
 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
-static int __dax_dev_pud_fault(struct dax_dev *dax_dev, struct vm_fault *vmf)
+static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
 {
         unsigned long pud_addr = vmf->address & PUD_MASK;
-        struct device *dev = &dax_dev->dev;
+        struct device *dev = &dev_dax->dev;
         struct dax_region *dax_region;
         phys_addr_t phys;
         pgoff_t pgoff;
@@ -522,18 +345,19 @@ static int __dax_dev_pud_fault(struct dax_dev *dax_dev, struct vm_fault *vmf)
         unsigned int fault_size = PUD_SIZE;
 
 
-        if (check_vma(dax_dev, vmf->vma, __func__))
+        if (check_vma(dev_dax, vmf->vma, __func__))
                 return VM_FAULT_SIGBUS;
 
-        dax_region = dax_dev->region;
+        dax_region = dev_dax->region;
         if (dax_region->align > PUD_SIZE) {
-                dev_dbg(dev, "%s: alignment > fault size\n", __func__);
+                dev_dbg(dev, "%s: alignment (%#x) > fault size (%#x)\n",
+                        __func__, dax_region->align, fault_size);
                 return VM_FAULT_SIGBUS;
         }
 
         /* dax pud mappings require pfn_t_devmap() */
         if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) {
-                dev_dbg(dev, "%s: alignment > fault size\n", __func__);
+                dev_dbg(dev, "%s: region lacks devmap flags\n", __func__);
                 return VM_FAULT_SIGBUS;
         }
 
@@ -548,7 +372,7 @@ static int __dax_dev_pud_fault(struct dax_dev *dax_dev, struct vm_fault *vmf)
                 return VM_FAULT_SIGBUS;
 
         pgoff = linear_page_index(vmf->vma, pud_addr);
-        phys = pgoff_to_phys(dax_dev, pgoff, PUD_SIZE);
+        phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE);
         if (phys == -1) {
                 dev_dbg(dev, "%s: pgoff_to_phys(%#lx) failed\n", __func__,
                                 pgoff);
@@ -561,65 +385,71 @@ static int __dax_dev_pud_fault(struct dax_dev *dax_dev, struct vm_fault *vmf)
                         vmf->flags & FAULT_FLAG_WRITE);
 }
 #else
-static int __dax_dev_pud_fault(struct dax_dev *dax_dev, struct vm_fault *vmf)
+static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
 {
         return VM_FAULT_FALLBACK;
 }
 #endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
 
-static int dax_dev_huge_fault(struct vm_fault *vmf,
+static int dev_dax_huge_fault(struct vm_fault *vmf,
                 enum page_entry_size pe_size)
 {
         int rc, id;
         struct file *filp = vmf->vma->vm_file;
-        struct dax_dev *dax_dev = filp->private_data;
+        struct dev_dax *dev_dax = filp->private_data;
 
-        dev_dbg(&dax_dev->dev, "%s: %s: %s (%#lx - %#lx)\n", __func__,
+        dev_dbg(&dev_dax->dev, "%s: %s: %s (%#lx - %#lx) size = %d\n", __func__,
                         current->comm, (vmf->flags & FAULT_FLAG_WRITE)
                         ? "write" : "read",
-                        vmf->vma->vm_start, vmf->vma->vm_end);
+                        vmf->vma->vm_start, vmf->vma->vm_end, pe_size);
 
-        id = srcu_read_lock(&dax_srcu);
+        id = dax_read_lock();
         switch (pe_size) {
         case PE_SIZE_PTE:
-                rc = __dax_dev_pte_fault(dax_dev, vmf);
+                rc = __dev_dax_pte_fault(dev_dax, vmf);
                 break;
         case PE_SIZE_PMD:
-                rc = __dax_dev_pmd_fault(dax_dev, vmf);
+                rc = __dev_dax_pmd_fault(dev_dax, vmf);
                 break;
         case PE_SIZE_PUD:
-                rc = __dax_dev_pud_fault(dax_dev, vmf);
+                rc = __dev_dax_pud_fault(dev_dax, vmf);
                 break;
         default:
-                return VM_FAULT_FALLBACK;
+                rc = VM_FAULT_SIGBUS;
         }
-        srcu_read_unlock(&dax_srcu, id);
+        dax_read_unlock(id);
 
         return rc;
 }
 
-static int dax_dev_fault(struct vm_fault *vmf)
+static int dev_dax_fault(struct vm_fault *vmf)
 {
-        return dax_dev_huge_fault(vmf, PE_SIZE_PTE);
+        return dev_dax_huge_fault(vmf, PE_SIZE_PTE);
 }
 
-static const struct vm_operations_struct dax_dev_vm_ops = {
-        .fault = dax_dev_fault,
-        .huge_fault = dax_dev_huge_fault,
+static const struct vm_operations_struct dax_vm_ops = {
+        .fault = dev_dax_fault,
+        .huge_fault = dev_dax_huge_fault,
 };
 
 static int dax_mmap(struct file *filp, struct vm_area_struct *vma)
 {
-        struct dax_dev *dax_dev = filp->private_data;
-        int rc;
+        struct dev_dax *dev_dax = filp->private_data;
+        int rc, id;
 
-        dev_dbg(&dax_dev->dev, "%s\n", __func__);
+        dev_dbg(&dev_dax->dev, "%s\n", __func__);
 
-        rc = check_vma(dax_dev, vma, __func__);
+        /*
+         * We lock to check dax_dev liveness and will re-check at
+         * fault time.
+         */
+        id = dax_read_lock();
+        rc = check_vma(dev_dax, vma, __func__);
+        dax_read_unlock(id);
         if (rc)
                 return rc;
 
-        vma->vm_ops = &dax_dev_vm_ops;
+        vma->vm_ops = &dax_vm_ops;
         vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE;
         return 0;
 }
@@ -630,13 +460,13 @@ static unsigned long dax_get_unmapped_area(struct file *filp,
                 unsigned long flags)
 {
         unsigned long off, off_end, off_align, len_align, addr_align, align;
-        struct dax_dev *dax_dev = filp ? filp->private_data : NULL;
+        struct dev_dax *dev_dax = filp ? filp->private_data : NULL;
         struct dax_region *dax_region;
 
-        if (!dax_dev || addr)
+        if (!dev_dax || addr)
                 goto out;
 
-        dax_region = dax_dev->region;
+        dax_region = dev_dax->region;
         align = dax_region->align;
         off = pgoff << PAGE_SHIFT;
         off_end = off + len;
@@ -661,14 +491,15 @@ static unsigned long dax_get_unmapped_area(struct file *filp,
 
 static int dax_open(struct inode *inode, struct file *filp)
 {
-        struct dax_dev *dax_dev;
+        struct dax_device *dax_dev = inode_dax(inode);
+        struct inode *__dax_inode = dax_inode(dax_dev);
+        struct dev_dax *dev_dax = dax_get_private(dax_dev);
 
-        dax_dev = container_of(inode->i_cdev, struct dax_dev, cdev);
-        dev_dbg(&dax_dev->dev, "%s\n", __func__);
-        inode->i_mapping = dax_dev->inode->i_mapping;
-        inode->i_mapping->host = dax_dev->inode;
+        dev_dbg(&dev_dax->dev, "%s\n", __func__);
+        inode->i_mapping = __dax_inode->i_mapping;
+        inode->i_mapping->host = __dax_inode;
         filp->f_mapping = inode->i_mapping;
-        filp->private_data = dax_dev;
+        filp->private_data = dev_dax;
         inode->i_flags = S_DAX;
 
         return 0;
@@ -676,9 +507,9 @@ static int dax_open(struct inode *inode, struct file *filp)
 
 static int dax_release(struct inode *inode, struct file *filp)
 {
-        struct dax_dev *dax_dev = filp->private_data;
+        struct dev_dax *dev_dax = filp->private_data;
 
-        dev_dbg(&dax_dev->dev, "%s\n", __func__);
+        dev_dbg(&dev_dax->dev, "%s\n", __func__);
         return 0;
 }
 
@@ -691,55 +522,54 @@ static const struct file_operations dax_fops = {
         .mmap = dax_mmap,
 };
 
-static void dax_dev_release(struct device *dev)
+static void dev_dax_release(struct device *dev)
 {
-        struct dax_dev *dax_dev = to_dax_dev(dev);
-        struct dax_region *dax_region = dax_dev->region;
+        struct dev_dax *dev_dax = to_dev_dax(dev);
+        struct dax_region *dax_region = dev_dax->region;
+        struct dax_device *dax_dev = dev_dax->dax_dev;
 
-        ida_simple_remove(&dax_region->ida, dax_dev->id);
-        ida_simple_remove(&dax_minor_ida, MINOR(dev->devt));
+        ida_simple_remove(&dax_region->ida, dev_dax->id);
         dax_region_put(dax_region);
-        iput(dax_dev->inode);
-        kfree(dax_dev);
+        put_dax(dax_dev);
+        kfree(dev_dax);
 }
 
-static void kill_dax_dev(struct dax_dev *dax_dev)
+static void kill_dev_dax(struct dev_dax *dev_dax)
 {
-        /*
-         * Note, rcu is not protecting the liveness of dax_dev, rcu is
-         * ensuring that any fault handlers that might have seen
-         * dax_dev->alive == true, have completed.  Any fault handlers
-         * that start after synchronize_srcu() has started will abort
-         * upon seeing dax_dev->alive == false.
-         */
-        dax_dev->alive = false;
-        synchronize_srcu(&dax_srcu);
-        unmap_mapping_range(dax_dev->inode->i_mapping, 0, 0, 1);
+        struct dax_device *dax_dev = dev_dax->dax_dev;
+        struct inode *inode = dax_inode(dax_dev);
+
+        kill_dax(dax_dev);
+        unmap_mapping_range(inode->i_mapping, 0, 0, 1);
 }
 
-static void unregister_dax_dev(void *dev)
+static void unregister_dev_dax(void *dev)
 {
-        struct dax_dev *dax_dev = to_dax_dev(dev);
+        struct dev_dax *dev_dax = to_dev_dax(dev);
+        struct dax_device *dax_dev = dev_dax->dax_dev;
+        struct inode *inode = dax_inode(dax_dev);
+        struct cdev *cdev = inode->i_cdev;
 
         dev_dbg(dev, "%s\n", __func__);
 
-        kill_dax_dev(dax_dev);
-        cdev_device_del(&dax_dev->cdev, dev);
+        kill_dev_dax(dev_dax);
+        cdev_device_del(cdev, dev);
         put_device(dev);
 }
 
-struct dax_dev *devm_create_dax_dev(struct dax_region *dax_region,
+struct dev_dax *devm_create_dev_dax(struct dax_region *dax_region,
                 struct resource *res, int count)
 {
         struct device *parent = dax_region->dev;
-        struct dax_dev *dax_dev;
-        int rc = 0, minor, i;
+        struct dax_device *dax_dev;
+        struct dev_dax *dev_dax;
+        struct inode *inode;
         struct device *dev;
         struct cdev *cdev;
-        dev_t dev_t;
+        int rc = 0, i;
 
-        dax_dev = kzalloc(sizeof(*dax_dev) + sizeof(*res) * count, GFP_KERNEL);
-        if (!dax_dev)
+        dev_dax = kzalloc(sizeof(*dev_dax) + sizeof(*res) * count, GFP_KERNEL);
+        if (!dev_dax)
                 return ERR_PTR(-ENOMEM);
 
         for (i = 0; i < count; i++) {
@@ -749,110 +579,79 @@ struct dax_dev *devm_create_dax_dev(struct dax_region *dax_region,
                         rc = -EINVAL;
                         break;
                 }
-                dax_dev->res[i].start = res[i].start;
-                dax_dev->res[i].end = res[i].end;
+                dev_dax->res[i].start = res[i].start;
+                dev_dax->res[i].end = res[i].end;
         }
 
         if (i < count)
                 goto err_id;
 
-        dax_dev->id = ida_simple_get(&dax_region->ida, 0, 0, GFP_KERNEL);
-        if (dax_dev->id < 0) {
-                rc = dax_dev->id;
+        dev_dax->id = ida_simple_get(&dax_region->ida, 0, 0, GFP_KERNEL);
+        if (dev_dax->id < 0) {
+                rc = dev_dax->id;
                 goto err_id;
         }
 
-        minor = ida_simple_get(&dax_minor_ida, 0, 0, GFP_KERNEL);
-        if (minor < 0) {
-                rc = minor;
-                goto err_minor;
-        }
-
-        dev_t = MKDEV(MAJOR(dax_devt), minor);
-        dev = &dax_dev->dev;
-        dax_dev->inode = dax_inode_get(&dax_dev->cdev, dev_t);
-        if (!dax_dev->inode) {
-                rc = -ENOMEM;
-                goto err_inode;
-        }
+        /*
+         * No 'host' or dax_operations since there is no access to this
+         * device outside of mmap of the resulting character device.
+         */
+        dax_dev = alloc_dax(dev_dax, NULL, NULL);
+        if (!dax_dev)
+                goto err_dax;
 
         /* from here on we're committed to teardown via dax_dev_release() */
+        dev = &dev_dax->dev;
         device_initialize(dev);
 
-        cdev = &dax_dev->cdev;
+        inode = dax_inode(dax_dev);
+        cdev = inode->i_cdev;
         cdev_init(cdev, &dax_fops);
         cdev->owner = parent->driver->owner;
 
-        dax_dev->num_resources = count;
-        dax_dev->alive = true;
-        dax_dev->region = dax_region;
+        dev_dax->num_resources = count;
+        dev_dax->dax_dev = dax_dev;
+        dev_dax->region = dax_region;
         kref_get(&dax_region->kref);
 
-        dev->devt = dev_t;
+        dev->devt = inode->i_rdev;
         dev->class = dax_class;
         dev->parent = parent;
         dev->groups = dax_attribute_groups;
-        dev->release = dax_dev_release;
-        dev_set_name(dev, "dax%d.%d", dax_region->id, dax_dev->id);
+        dev->release = dev_dax_release;
+        dev_set_name(dev, "dax%d.%d", dax_region->id, dev_dax->id);
 
         rc = cdev_device_add(cdev, dev);
         if (rc) {
-                kill_dax_dev(dax_dev);
+                kill_dev_dax(dev_dax);
                 put_device(dev);
                 return ERR_PTR(rc);
         }
 
-        rc = devm_add_action_or_reset(dax_region->dev, unregister_dax_dev, dev);
+        rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev);
         if (rc)
                 return ERR_PTR(rc);
 
-        return dax_dev;
+        return dev_dax;
 
- err_inode:
-        ida_simple_remove(&dax_minor_ida, minor);
- err_minor:
-        ida_simple_remove(&dax_region->ida, dax_dev->id);
+ err_dax:
+        ida_simple_remove(&dax_region->ida, dev_dax->id);
  err_id:
-        kfree(dax_dev);
+        kfree(dev_dax);
 
         return ERR_PTR(rc);
 }
-EXPORT_SYMBOL_GPL(devm_create_dax_dev);
+EXPORT_SYMBOL_GPL(devm_create_dev_dax);
 
 static int __init dax_init(void)
 {
-        int rc;
-
-        rc = dax_inode_init();
-        if (rc)
-                return rc;
-
-        nr_dax = max(nr_dax, 256);
-        rc = alloc_chrdev_region(&dax_devt, 0, nr_dax, "dax");
-        if (rc)
-                goto err_chrdev;
-
         dax_class = class_create(THIS_MODULE, "dax");
-        if (IS_ERR(dax_class)) {
-                rc = PTR_ERR(dax_class);
-                goto err_class;
-        }
-
-        return 0;
-
- err_class:
-        unregister_chrdev_region(dax_devt, nr_dax);
- err_chrdev:
-        dax_inode_exit();
-        return rc;
+        return PTR_ERR_OR_ZERO(dax_class);
 }
 
 static void __exit dax_exit(void)
 {
         class_destroy(dax_class);
-        unregister_chrdev_region(dax_devt, nr_dax);
-        ida_destroy(&dax_minor_ida);
-        dax_inode_exit();
 }
 
 MODULE_AUTHOR("Intel Corporation");
diff --git a/drivers/dax/pmem.c b/drivers/dax/pmem.c
index cb0d742fa23f..9f2a0b4fd801 100644
--- a/drivers/dax/pmem.c
+++ b/drivers/dax/pmem.c
@@ -16,7 +16,7 @@
 #include <linux/pfn_t.h>
 #include "../nvdimm/pfn.h"
 #include "../nvdimm/nd.h"
-#include "dax.h"
+#include "device-dax.h"
 
 struct dax_pmem {
         struct device *dev;
@@ -61,8 +61,8 @@ static int dax_pmem_probe(struct device *dev)
         int rc;
         void *addr;
         struct resource res;
-        struct dax_dev *dax_dev;
         struct nd_pfn_sb *pfn_sb;
+        struct dev_dax *dev_dax;
         struct dax_pmem *dax_pmem;
         struct nd_region *nd_region;
         struct nd_namespace_io *nsio;
@@ -130,12 +130,12 @@ static int dax_pmem_probe(struct device *dev)
                 return -ENOMEM;
 
         /* TODO: support for subdividing a dax region... */
-        dax_dev = devm_create_dax_dev(dax_region, &res, 1);
+        dev_dax = devm_create_dev_dax(dax_region, &res, 1);
 
-        /* child dax_dev instances now own the lifetime of the dax_region */
+        /* child dev_dax instances now own the lifetime of the dax_region */
         dax_region_put(dax_region);
 
-        return PTR_ERR_OR_ZERO(dax_dev);
+        return PTR_ERR_OR_ZERO(dev_dax);
 }
 
 static struct nd_device_driver dax_pmem_driver = {
diff --git a/drivers/dax/super.c b/drivers/dax/super.c
new file mode 100644
index 000000000000..465dcd7317d5
--- /dev/null
+++ b/drivers/dax/super.c
@@ -0,0 +1,425 @@
+/*
+ * Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+#include <linux/pagemap.h>
+#include <linux/module.h>
+#include <linux/mount.h>
+#include <linux/magic.h>
+#include <linux/cdev.h>
+#include <linux/hash.h>
+#include <linux/slab.h>
+#include <linux/dax.h>
+#include <linux/fs.h>
+
+static int nr_dax = CONFIG_NR_DEV_DAX;
+module_param(nr_dax, int, S_IRUGO);
+MODULE_PARM_DESC(nr_dax, "max number of dax device instances");
+
+static dev_t dax_devt;
+DEFINE_STATIC_SRCU(dax_srcu);
+static struct vfsmount *dax_mnt;
+static DEFINE_IDA(dax_minor_ida);
+static struct kmem_cache *dax_cache __read_mostly;
+static struct super_block *dax_superblock __read_mostly;
+
+#define DAX_HASH_SIZE (PAGE_SIZE / sizeof(struct hlist_head))
+static struct hlist_head dax_host_list[DAX_HASH_SIZE];
+static DEFINE_SPINLOCK(dax_host_lock);
+
+int dax_read_lock(void)
+{
+        return srcu_read_lock(&dax_srcu);
+}
+EXPORT_SYMBOL_GPL(dax_read_lock);
+
+void dax_read_unlock(int id)
+{
+        srcu_read_unlock(&dax_srcu, id);
+}
+EXPORT_SYMBOL_GPL(dax_read_unlock);
+
+/**
+ * struct dax_device - anchor object for dax services
+ * @inode: core vfs
+ * @cdev: optional character interface for "device dax"
+ * @host: optional name for lookups where the device path is not available
+ * @private: dax driver private data
+ * @alive: !alive + rcu grace period == no new operations / mappings
+ */
+struct dax_device {
+        struct hlist_node list;
+        struct inode inode;
+        struct cdev cdev;
+        const char *host;
+        void *private;
+        bool alive;
+        const struct dax_operations *ops;
+};
+
+/**
+ * dax_direct_access() - translate a device pgoff to an absolute pfn
+ * @dax_dev: a dax_device instance representing the logical memory range
+ * @pgoff: offset in pages from the start of the device to translate
+ * @nr_pages: number of consecutive pages caller can handle relative to @pfn
+ * @kaddr: output parameter that returns a virtual address mapping of pfn
+ * @pfn: output parameter that returns an absolute pfn translation of @pgoff
+ *
+ * Return: negative errno if an error occurs, otherwise the number of
+ * pages accessible at the device relative @pgoff.
+ */
+long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
+                void **kaddr, pfn_t *pfn)
+{
+        long avail;
+
+        /*
+         * The device driver is allowed to sleep, in order to make the
+         * memory directly accessible.
+         */
+        might_sleep();
+
+        if (!dax_dev)
+                return -EOPNOTSUPP;
+
+        if (!dax_alive(dax_dev))
+                return -ENXIO;
+
+        if (nr_pages < 0)
+                return nr_pages;
+
+        avail = dax_dev->ops->direct_access(dax_dev, pgoff, nr_pages,
+                        kaddr, pfn);
+        if (!avail)
+                return -ERANGE;
+        return min(avail, nr_pages);
+}
+EXPORT_SYMBOL_GPL(dax_direct_access);
+
+bool dax_alive(struct dax_device *dax_dev)
+{
+        lockdep_assert_held(&dax_srcu);
+        return dax_dev->alive;
+}
+EXPORT_SYMBOL_GPL(dax_alive);
+
+static int dax_host_hash(const char *host)
+{
+        return hashlen_hash(hashlen_string("DAX", host)) % DAX_HASH_SIZE;
+}
+
+/*
+ * Note, rcu is not protecting the liveness of dax_dev, rcu is ensuring
+ * that any fault handlers or operations that might have seen
+ * dax_alive(), have completed.  Any operations that start after
+ * synchronize_srcu() has run will abort upon seeing !dax_alive().
+ */
+void kill_dax(struct dax_device *dax_dev)
+{
+        if (!dax_dev)
+                return;
+
+        dax_dev->alive = false;
+
+        synchronize_srcu(&dax_srcu);
+
+        spin_lock(&dax_host_lock);
+        hlist_del_init(&dax_dev->list);
+        spin_unlock(&dax_host_lock);
+
+        dax_dev->private = NULL;
+}
+EXPORT_SYMBOL_GPL(kill_dax);
+
+static struct inode *dax_alloc_inode(struct super_block *sb)
+{
+        struct dax_device *dax_dev;
+
+        dax_dev = kmem_cache_alloc(dax_cache, GFP_KERNEL);
+        return &dax_dev->inode;
+}
+
+static struct dax_device *to_dax_dev(struct inode *inode)
+{
+        return container_of(inode, struct dax_device, inode);
+}
+
+static void dax_i_callback(struct rcu_head *head)
+{
+        struct inode *inode = container_of(head, struct inode, i_rcu);
+        struct dax_device *dax_dev = to_dax_dev(inode);
+
+        kfree(dax_dev->host);
+        dax_dev->host = NULL;
+        ida_simple_remove(&dax_minor_ida, MINOR(inode->i_rdev));
+        kmem_cache_free(dax_cache, dax_dev);
+}
+
+static void dax_destroy_inode(struct inode *inode)
+{
+        struct dax_device *dax_dev = to_dax_dev(inode);
+
+        WARN_ONCE(dax_dev->alive,
+                        "kill_dax() must be called before final iput()\n");
+        call_rcu(&inode->i_rcu, dax_i_callback);
+}
+
+static const struct super_operations dax_sops = {
+        .statfs = simple_statfs,
+        .alloc_inode = dax_alloc_inode,
+        .destroy_inode = dax_destroy_inode,
+        .drop_inode = generic_delete_inode,
+};
+
+static struct dentry *dax_mount(struct file_system_type *fs_type,
+                int flags, const char *dev_name, void *data)
+{
+        return mount_pseudo(fs_type, "dax:", &dax_sops, NULL, DAXFS_MAGIC);
+}
+
+static struct file_system_type dax_fs_type = {
+        .name = "dax",
+        .mount = dax_mount,
+        .kill_sb = kill_anon_super,
+};
+
+static int dax_test(struct inode *inode, void *data)
+{
+        dev_t devt = *(dev_t *) data;
+
+        return inode->i_rdev == devt;
+}
+
+static int dax_set(struct inode *inode, void *data)
+{
+        dev_t devt = *(dev_t *) data;
+
+        inode->i_rdev = devt;
+        return 0;
+}
+
+static struct dax_device *dax_dev_get(dev_t devt)
+{
+        struct dax_device *dax_dev;
+        struct inode *inode;
+
+        inode = iget5_locked(dax_superblock, hash_32(devt + DAXFS_MAGIC, 31),
+                        dax_test, dax_set, &devt);
+
+        if (!inode)
+                return NULL;
+
+        dax_dev = to_dax_dev(inode);
+        if (inode->i_state & I_NEW) {
+                dax_dev->alive = true;
+                inode->i_cdev = &dax_dev->cdev;
+                inode->i_mode = S_IFCHR;
+                inode->i_flags = S_DAX;
+                mapping_set_gfp_mask(&inode->i_data, GFP_USER);
+                unlock_new_inode(inode);
+        }
+
+        return dax_dev;
+}
+
+static void dax_add_host(struct dax_device *dax_dev, const char *host)
+{
+        int hash;
+
+        /*
+         * Unconditionally init dax_dev since it's coming from a
+         * non-zeroed slab cache
+         */
+        INIT_HLIST_NODE(&dax_dev->list);
+        dax_dev->host = host;
+        if (!host)
+                return;
+
+        hash = dax_host_hash(host);
+        spin_lock(&dax_host_lock);
+        hlist_add_head(&dax_dev->list, &dax_host_list[hash]);
+        spin_unlock(&dax_host_lock);
+}
+
+struct dax_device *alloc_dax(void *private, const char *__host,
+                const struct dax_operations *ops)
+{
+        struct dax_device *dax_dev;
+        const char *host;
+        dev_t devt;
+        int minor;
+
+        host = kstrdup(__host, GFP_KERNEL);
+        if (__host && !host)
+                return NULL;
+
+        minor = ida_simple_get(&dax_minor_ida, 0, nr_dax, GFP_KERNEL);
+        if (minor < 0)
+                goto err_minor;
+
+        devt = MKDEV(MAJOR(dax_devt), minor);
+        dax_dev = dax_dev_get(devt);
+        if (!dax_dev)
+                goto err_dev;
+
+        dax_add_host(dax_dev, host);
+        dax_dev->ops = ops;
+        dax_dev->private = private;
+        return dax_dev;
+
+ err_dev:
+        ida_simple_remove(&dax_minor_ida, minor);
+ err_minor:
+        kfree(host);
+        return NULL;
+}
+EXPORT_SYMBOL_GPL(alloc_dax);
+
+void put_dax(struct dax_device *dax_dev)
+{
+        if (!dax_dev)
+                return;
+        iput(&dax_dev->inode);
+}
+EXPORT_SYMBOL_GPL(put_dax);
+
+/**
+ * dax_get_by_host() - temporary lookup mechanism for filesystem-dax
+ * @host: alternate name for the device registered by a dax driver
+ */
+struct dax_device *dax_get_by_host(const char *host)
+{
+        struct dax_device *dax_dev, *found = NULL;
+        int hash, id;
+
+        if (!host)
+                return NULL;
+
+        hash = dax_host_hash(host);
+
+        id = dax_read_lock();
+        spin_lock(&dax_host_lock);
+        hlist_for_each_entry(dax_dev, &dax_host_list[hash], list) {
+                if (!dax_alive(dax_dev)
+                                || strcmp(host, dax_dev->host) != 0)
+                        continue;
+
+                if (igrab(&dax_dev->inode))
+                        found = dax_dev;
+                break;
+        }
+        spin_unlock(&dax_host_lock);
+        dax_read_unlock(id);
+
+        return found;
+}
+EXPORT_SYMBOL_GPL(dax_get_by_host);
+
+/**
+ * inode_dax: convert a public inode into its dax_dev
+ * @inode: An inode with i_cdev pointing to a dax_dev
+ *
+ * Note this is not equivalent to to_dax_dev() which is for private
+ * internal use where we know the inode filesystem type == dax_fs_type.
+ */
+struct dax_device *inode_dax(struct inode *inode)
+{
+        struct cdev *cdev = inode->i_cdev;
+
+        return container_of(cdev, struct dax_device, cdev);
+}
+EXPORT_SYMBOL_GPL(inode_dax);
+
+struct inode *dax_inode(struct dax_device *dax_dev)
+{
+        return &dax_dev->inode;
+}
+EXPORT_SYMBOL_GPL(dax_inode);
+
+void *dax_get_private(struct dax_device *dax_dev)
+{
+        return dax_dev->private;
+}
+EXPORT_SYMBOL_GPL(dax_get_private);
+
+static void init_once(void *_dax_dev)
+{
+        struct dax_device *dax_dev = _dax_dev;
+        struct inode *inode = &dax_dev->inode;
+
+        inode_init_once(inode);
+}
+
+static int __dax_fs_init(void)
+{
+        int rc;
+
+        dax_cache = kmem_cache_create("dax_cache", sizeof(struct dax_device), 0,
+                        (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
+                         SLAB_MEM_SPREAD|SLAB_ACCOUNT),
+                        init_once);
+        if (!dax_cache)
+                return -ENOMEM;
+
+        rc = register_filesystem(&dax_fs_type);
+        if (rc)
+                goto err_register_fs;
+
+        dax_mnt = kern_mount(&dax_fs_type);
+        if (IS_ERR(dax_mnt)) {
+                rc = PTR_ERR(dax_mnt);
+                goto err_mount;
+        }
+        dax_superblock = dax_mnt->mnt_sb;
+
+        return 0;
+
+ err_mount:
+        unregister_filesystem(&dax_fs_type);
+ err_register_fs:
+        kmem_cache_destroy(dax_cache);
+
+        return rc;
+}
+
+static void __dax_fs_exit(void)
+{
+        kern_unmount(dax_mnt);
+        unregister_filesystem(&dax_fs_type);
+        kmem_cache_destroy(dax_cache);
+}
+
+static int __init dax_fs_init(void)
+{
+        int rc;
+
+        rc = __dax_fs_init();
+        if (rc)
+                return rc;
+
+        nr_dax = max(nr_dax, 256);
+        rc = alloc_chrdev_region(&dax_devt, 0, nr_dax, "dax");
+        if (rc)
+                __dax_fs_exit();
+        return rc;
+}
+
+static void __exit dax_fs_exit(void)
+{
+        unregister_chrdev_region(dax_devt, nr_dax);
+        ida_destroy(&dax_minor_ida);
+        __dax_fs_exit();
+}
+
+MODULE_AUTHOR("Intel Corporation");
+MODULE_LICENSE("GPL v2");
+subsys_initcall(dax_fs_init);
+module_exit(dax_fs_exit);
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index 7468a22f9d10..349ff8813401 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -200,6 +200,7 @@ config BLK_DEV_DM_BUILTIN
 config BLK_DEV_DM
         tristate "Device mapper support"
         select BLK_DEV_DM_BUILTIN
+        select DAX
         ---help---
           Device-mapper is a low level volume manager.  It works by allowing
           people to specify mappings for ranges of logical sectors.  Various
diff --git a/drivers/md/dm-core.h b/drivers/md/dm-core.h
index 97db4d11c05a..52ca8d059e82 100644
--- a/drivers/md/dm-core.h
+++ b/drivers/md/dm-core.h
@@ -58,6 +58,7 @@ struct mapped_device {
         struct target_type *immutable_target_type;
 
         struct gendisk *disk;
+        struct dax_device *dax_dev;
         char name[16];
 
         void *interface_ptr;
diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c
index a5120961632a..7d42a9d9f406 100644
--- a/drivers/md/dm-linear.c
+++ b/drivers/md/dm-linear.c
@@ -9,6 +9,7 @@
 #include <linux/init.h>
 #include <linux/blkdev.h>
 #include <linux/bio.h>
+#include <linux/dax.h>
 #include <linux/slab.h>
 #include <linux/device-mapper.h>
 
@@ -142,22 +143,20 @@ static int linear_iterate_devices(struct dm_target *ti,
         return fn(ti, lc->dev, lc->start, ti->len, data);
 }
 
-static long linear_direct_access(struct dm_target *ti, sector_t sector,
-                                 void **kaddr, pfn_t *pfn, long size)
+static long linear_dax_direct_access(struct dm_target *ti, pgoff_t pgoff,
+                long nr_pages, void **kaddr, pfn_t *pfn)
 {
+        long ret;
         struct linear_c *lc = ti->private;
         struct block_device *bdev = lc->dev->bdev;
-        struct blk_dax_ctl dax = {
-                .sector = linear_map_sector(ti, sector),
-                .size = size,
-        };
-        long ret;
-
-        ret = bdev_direct_access(bdev, &dax);
-        *kaddr = dax.addr;
-        *pfn = dax.pfn;
-
-        return ret;
+        struct dax_device *dax_dev = lc->dev->dax_dev;
+        sector_t dev_sector, sector = pgoff * PAGE_SECTORS;
+
+        dev_sector = linear_map_sector(ti, sector);
+        ret = bdev_dax_pgoff(bdev, dev_sector, nr_pages * PAGE_SIZE, &pgoff);
+        if (ret)
+                return ret;
+        return dax_direct_access(dax_dev, pgoff, nr_pages, kaddr, pfn);
 }
 
 static struct target_type linear_target = {
@@ -171,7 +170,7 @@ static struct target_type linear_target = {
         .status = linear_status,
         .prepare_ioctl = linear_prepare_ioctl,
         .iterate_devices = linear_iterate_devices,
-        .direct_access = linear_direct_access,
+        .direct_access = linear_dax_direct_access,
 };
 
 int __init dm_linear_init(void)
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
index c65feeada864..e152d9817c81 100644
--- a/drivers/md/dm-snap.c
+++ b/drivers/md/dm-snap.c
@@ -2302,8 +2302,8 @@ static int origin_map(struct dm_target *ti, struct bio *bio)
         return do_origin(o->dev, bio);
 }
 
-static long origin_direct_access(struct dm_target *ti, sector_t sector,
-                void **kaddr, pfn_t *pfn, long size)
+static long origin_dax_direct_access(struct dm_target *ti, pgoff_t pgoff,
+                long nr_pages, void **kaddr, pfn_t *pfn)
 {
         DMWARN("device does not support dax.");
         return -EIO;
@@ -2368,7 +2368,7 @@ static struct target_type origin_target = {
         .postsuspend = origin_postsuspend,
         .status  = origin_status,
         .iterate_devices = origin_iterate_devices,
-        .direct_access = origin_direct_access,
+        .direct_access = origin_dax_direct_access,
 };
 
 static struct target_type snapshot_target = {
diff --git a/drivers/md/dm-stripe.c b/drivers/md/dm-stripe.c
index 4b50ae115c6d..75152482f3ad 100644
--- a/drivers/md/dm-stripe.c
+++ b/drivers/md/dm-stripe.c
@@ -11,6 +11,7 @@
 #include <linux/init.h>
 #include <linux/blkdev.h>
 #include <linux/bio.h>
+#include <linux/dax.h>
 #include <linux/slab.h>
 #include <linux/log2.h>
 
@@ -310,27 +311,25 @@ static int stripe_map(struct dm_target *ti, struct bio *bio)
         return DM_MAPIO_REMAPPED;
 }
 
-static long stripe_direct_access(struct dm_target *ti, sector_t sector,
-                                 void **kaddr, pfn_t *pfn, long size)
+static long stripe_dax_direct_access(struct dm_target *ti, pgoff_t pgoff,
+                long nr_pages, void **kaddr, pfn_t *pfn)
 {
+        sector_t dev_sector, sector = pgoff * PAGE_SECTORS;
         struct stripe_c *sc = ti->private;
-        uint32_t stripe;
+        struct dax_device *dax_dev;
         struct block_device *bdev;
-        struct blk_dax_ctl dax = {
-                .size = size,
-        };
+        uint32_t stripe;
         long ret;
 
-        stripe_map_sector(sc, sector, &stripe, &dax.sector);
-
-        dax.sector += sc->stripe[stripe].physical_start;
+        stripe_map_sector(sc, sector, &stripe, &dev_sector);
+        dev_sector += sc->stripe[stripe].physical_start;
+        dax_dev = sc->stripe[stripe].dev->dax_dev;
         bdev = sc->stripe[stripe].dev->bdev;
 
-        ret = bdev_direct_access(bdev, &dax);
-        *kaddr = dax.addr;
-        *pfn = dax.pfn;
-
-        return ret;
+        ret = bdev_dax_pgoff(bdev, dev_sector, nr_pages * PAGE_SIZE, &pgoff);
+        if (ret)
+                return ret;
+        return dax_direct_access(dax_dev, pgoff, nr_pages, kaddr, pfn);
 }
 
 /*
@@ -451,7 +450,7 @@ static struct target_type stripe_target = {
         .status = stripe_status,
         .iterate_devices = stripe_iterate_devices,
         .io_hints = stripe_io_hints,
-        .direct_access = stripe_direct_access,
+        .direct_access = stripe_dax_direct_access,
 };
 
 int __init dm_stripe_init(void)
diff --git a/drivers/md/dm-target.c b/drivers/md/dm-target.c
index 6264ff00dcf0..b242b750542f 100644
--- a/drivers/md/dm-target.c
+++ b/drivers/md/dm-target.c
@@ -142,8 +142,8 @@ static void io_err_release_clone_rq(struct request *clone)
 {
 }
 
-static long io_err_direct_access(struct dm_target *ti, sector_t sector,
-                                 void **kaddr, pfn_t *pfn, long size)
+static long io_err_dax_direct_access(struct dm_target *ti, pgoff_t pgoff,
+                long nr_pages, void **kaddr, pfn_t *pfn)
 {
         return -EIO;
 }
@@ -157,7 +157,7 @@ static struct target_type error_target = {
         .map  = io_err_map,
         .clone_and_map_rq = io_err_clone_and_map_rq,
         .release_clone_rq = io_err_release_clone_rq,
-        .direct_access = io_err_direct_access,
+        .direct_access = io_err_dax_direct_access,
 };
 
 int __init dm_target_init(void)
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 268edf402bbb..6ef9500226c0 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -16,6 +16,7 @@
 #include <linux/blkpg.h>
 #include <linux/bio.h>
 #include <linux/mempool.h>
+#include <linux/dax.h>
 #include <linux/slab.h>
 #include <linux/idr.h>
 #include <linux/hdreg.h>
@@ -629,6 +630,7 @@ static int open_table_device(struct table_device *td, dev_t dev,
         }
 
         td->dm_dev.bdev = bdev;
+        td->dm_dev.dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
         return 0;
 }
 
@@ -642,7 +644,9 @@ static void close_table_device(struct table_device *td, struct mapped_device *md
 
         bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
         blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
+        put_dax(td->dm_dev.dax_dev);
         td->dm_dev.bdev = NULL;
+        td->dm_dev.dax_dev = NULL;
 }
 
 static struct table_device *find_table_device(struct list_head *l, dev_t dev,
@@ -920,31 +924,49 @@ int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
 }
 EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
 
-static long dm_blk_direct_access(struct block_device *bdev, sector_t sector,
-                                 void **kaddr, pfn_t *pfn, long size)
+static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,
+                sector_t sector, int *srcu_idx)
 {
-        struct mapped_device *md = bdev->bd_disk->private_data;
         struct dm_table *map;
         struct dm_target *ti;
-        int srcu_idx;
-        long len, ret = -EIO;
 
-        map = dm_get_live_table(md, &srcu_idx);
+        map = dm_get_live_table(md, srcu_idx);
         if (!map)
-                goto out;
+                return NULL;
 
         ti = dm_table_find_target(map, sector);
         if (!dm_target_is_valid(ti))
-                goto out;
+                return NULL;
+
+        return ti;
+}
+
+static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
+                long nr_pages, void **kaddr, pfn_t *pfn)
+{
+        struct mapped_device *md = dax_get_private(dax_dev);
+        sector_t sector = pgoff * PAGE_SECTORS;
+        struct dm_target *ti;
+        long len, ret = -EIO;
+        int srcu_idx;
 
-        len = max_io_len(sector, ti) << SECTOR_SHIFT;
-        size = min(len, size);
+        ti = dm_dax_get_live_target(md, sector, &srcu_idx);
 
+        if (!ti)
+                goto out;
+        if (!ti->type->direct_access)
+                goto out;
+        len = max_io_len(sector, ti) / PAGE_SECTORS;
+        if (len < 1)
+                goto out;
+        nr_pages = min(len, nr_pages);
         if (ti->type->direct_access)
-                ret = ti->type->direct_access(ti, sector, kaddr, pfn, size);
-out:
+                ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn);
+
+ out:
         dm_put_live_table(md, srcu_idx);
-        return min(ret, size);
+
+        return ret;
 }
 
 /*
@@ -1471,6 +1493,7 @@ static int next_free_minor(int *minor)
 }
 
 static const struct block_device_operations dm_blk_dops;
+static const struct dax_operations dm_dax_ops;
 
 static void dm_wq_work(struct work_struct *work);
 
@@ -1517,6 +1540,12 @@ static void cleanup_mapped_device(struct mapped_device *md)
         if (md->bs)
                 bioset_free(md->bs);
 
+        if (md->dax_dev) {
+                kill_dax(md->dax_dev);
+                put_dax(md->dax_dev);
+                md->dax_dev = NULL;
+        }
+
         if (md->disk) {
                 spin_lock(&_minor_lock);
                 md->disk->private_data = NULL;
@@ -1544,6 +1573,7 @@ static void cleanup_mapped_device(struct mapped_device *md)
 static struct mapped_device *alloc_dev(int minor)
 {
         int r, numa_node_id = dm_get_numa_node();
+        struct dax_device *dax_dev;
         struct mapped_device *md;
         void *old_md;
 
@@ -1608,6 +1638,12 @@ static struct mapped_device *alloc_dev(int minor)
         md->disk->queue = md->queue;
         md->disk->private_data = md;
         sprintf(md->disk->disk_name, "dm-%d", minor);
+
+        dax_dev = alloc_dax(md, md->disk->disk_name, &dm_dax_ops);
+        if (!dax_dev)
+                goto bad;
+        md->dax_dev = dax_dev;
+
         add_disk(md->disk);
         format_dev_t(md->name, MKDEV(_major, minor));
 
@@ -2816,12 +2852,15 @@ static const struct block_device_operations dm_blk_dops = {
         .open = dm_blk_open,
         .release = dm_blk_close,
         .ioctl = dm_blk_ioctl,
-        .direct_access = dm_blk_direct_access,
         .getgeo = dm_blk_getgeo,
         .pr_ops = &dm_pr_ops,
         .owner = THIS_MODULE
 };
 
+static const struct dax_operations dm_dax_ops = {
+        .direct_access = dm_dax_direct_access,
+};
+
 /*
  * module hooks
  */
diff --git a/drivers/nvdimm/Kconfig b/drivers/nvdimm/Kconfig
index 59e750183b7f..5bdd499b5f4f 100644
--- a/drivers/nvdimm/Kconfig
+++ b/drivers/nvdimm/Kconfig
@@ -20,6 +20,7 @@ if LIBNVDIMM
 config BLK_DEV_PMEM
         tristate "PMEM: Persistent memory block device support"
         default LIBNVDIMM
+        select DAX
         select ND_BTT if BTT
         select ND_PFN if NVDIMM_PFN
         help
diff --git a/drivers/nvdimm/btt_devs.c b/drivers/nvdimm/btt_devs.c
index 97dd2925ed6e..4b76af2b8715 100644
--- a/drivers/nvdimm/btt_devs.c
+++ b/drivers/nvdimm/btt_devs.c
@@ -314,7 +314,7 @@ int nd_btt_probe(struct device *dev, struct nd_namespace_common *ndns)
         if (rc < 0) {
                 struct nd_btt *nd_btt = to_nd_btt(btt_dev);
 
-                __nd_detach_ndns(btt_dev, &nd_btt->ndns);
+                nd_detach_ndns(btt_dev, &nd_btt->ndns);
                 put_device(btt_dev);
         }
 
diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c
index 351bac8f6503..e9361bffe5ee 100644
--- a/drivers/nvdimm/bus.c
+++ b/drivers/nvdimm/bus.c
@@ -27,6 +27,7 @@
 #include <linux/nd.h>
 #include "nd-core.h"
 #include "nd.h"
+#include "pfn.h"
 
 int nvdimm_major;
 static int nvdimm_bus_major;
@@ -171,6 +172,57 @@ void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event)
 }
 EXPORT_SYMBOL_GPL(nvdimm_region_notify);
 
+struct clear_badblocks_context {
+        resource_size_t phys, cleared;
+};
+
+static int nvdimm_clear_badblocks_region(struct device *dev, void *data)
+{
+        struct clear_badblocks_context *ctx = data;
+        struct nd_region *nd_region;
+        resource_size_t ndr_end;
+        sector_t sector;
+
+        /* make sure device is a region */
+        if (!is_nd_pmem(dev))
+                return 0;
+
+        nd_region = to_nd_region(dev);
+        ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1;
+
+        /* make sure we are in the region */
+        if (ctx->phys < nd_region->ndr_start
+                        || (ctx->phys + ctx->cleared) > ndr_end)
+                return 0;
+
+        sector = (ctx->phys - nd_region->ndr_start) / 512;
+        badblocks_clear(&nd_region->bb, sector, ctx->cleared / 512);
+
+        return 0;
+}
+
+static void nvdimm_clear_badblocks_regions(struct nvdimm_bus *nvdimm_bus,
+                phys_addr_t phys, u64 cleared)
+{
+        struct clear_badblocks_context ctx = {
+                .phys = phys,
+                .cleared = cleared,
+        };
+
+        device_for_each_child(&nvdimm_bus->dev, &ctx,
+                        nvdimm_clear_badblocks_region);
+}
+
+static void nvdimm_account_cleared_poison(struct nvdimm_bus *nvdimm_bus,
+                phys_addr_t phys, u64 cleared)
+{
+        if (cleared > 0)
+                nvdimm_forget_poison(nvdimm_bus, phys, cleared);
+
+        if (cleared > 0 && cleared / 512)
+                nvdimm_clear_badblocks_regions(nvdimm_bus, phys, cleared);
+}
+
 long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
                 unsigned int len)
 {
@@ -218,7 +270,8 @@ long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
         if (cmd_rc < 0)
                 return cmd_rc;
 
-        nvdimm_clear_from_poison_list(nvdimm_bus, phys, len);
+        nvdimm_account_cleared_poison(nvdimm_bus, phys, clear_err.cleared);
+
         return clear_err.cleared;
 }
 EXPORT_SYMBOL_GPL(nvdimm_clear_poison);
@@ -286,6 +339,7 @@ struct nvdimm_bus *nvdimm_bus_register(struct device *parent,
         init_waitqueue_head(&nvdimm_bus->probe_wait);
         nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL);
         mutex_init(&nvdimm_bus->reconfig_mutex);
+        spin_lock_init(&nvdimm_bus->poison_lock);
         if (nvdimm_bus->id < 0) {
                 kfree(nvdimm_bus);
                 return NULL;
@@ -354,9 +408,9 @@ static int nd_bus_remove(struct device *dev)
         nd_synchronize();
         device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister);
 
-        nvdimm_bus_lock(&nvdimm_bus->dev);
+        spin_lock(&nvdimm_bus->poison_lock);
         free_poison_list(&nvdimm_bus->poison_list);
-        nvdimm_bus_unlock(&nvdimm_bus->dev);
+        spin_unlock(&nvdimm_bus->poison_lock);
 
         nvdimm_bus_destroy_ndctl(nvdimm_bus);
 
@@ -769,16 +823,55 @@ void wait_nvdimm_bus_probe_idle(struct device *dev)
         } while (true);
 }
 
-static int pmem_active(struct device *dev, void *data)
+static int nd_pmem_forget_poison_check(struct device *dev, void *data)
 {
-        if (is_nd_pmem(dev) && dev->driver)
+        struct nd_cmd_clear_error *clear_err =
+                (struct nd_cmd_clear_error *)data;
+        struct nd_btt *nd_btt = is_nd_btt(dev) ? to_nd_btt(dev) : NULL;
+        struct nd_pfn *nd_pfn = is_nd_pfn(dev) ? to_nd_pfn(dev) : NULL;
+        struct nd_dax *nd_dax = is_nd_dax(dev) ? to_nd_dax(dev) : NULL;
+        struct nd_namespace_common *ndns = NULL;
+        struct nd_namespace_io *nsio;
+        resource_size_t offset = 0, end_trunc = 0, start, end, pstart, pend;
+
+        if (nd_dax || !dev->driver)
+                return 0;
+
+        start = clear_err->address;
+        end = clear_err->address + clear_err->cleared - 1;
+
+        if (nd_btt || nd_pfn || nd_dax) {
+                if (nd_btt)
+                        ndns = nd_btt->ndns;
+                else if (nd_pfn)
+                        ndns = nd_pfn->ndns;
+                else if (nd_dax)
+                        ndns = nd_dax->nd_pfn.ndns;
+
+                if (!ndns)
+                        return 0;
+        } else
+                ndns = to_ndns(dev);
+
+        nsio = to_nd_namespace_io(&ndns->dev);
+        pstart = nsio->res.start + offset;
+        pend = nsio->res.end - end_trunc;
+
+        if ((pstart >= start) && (pend <= end))
                 return -EBUSY;
+
         return 0;
+
+}
+
+static int nd_ns_forget_poison_check(struct device *dev, void *data)
+{
+        return device_for_each_child(dev, data, nd_pmem_forget_poison_check);
 }
 
 /* set_config requires an idle interleave set */
 static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus,
-                struct nvdimm *nvdimm, unsigned int cmd)
+                struct nvdimm *nvdimm, unsigned int cmd, void *data)
 {
         struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
 
@@ -792,8 +885,8 @@ static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus,
 
         /* require clear error to go through the pmem driver */
         if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR)
-                return device_for_each_child(&nvdimm_bus->dev, NULL,
-                                pmem_active);
+                return device_for_each_child(&nvdimm_bus->dev, data,
+                                nd_ns_forget_poison_check);
 
         if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA)
                 return 0;
@@ -820,7 +913,7 @@ static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
         const char *cmd_name, *dimm_name;
         unsigned long cmd_mask;
         void *buf;
-        int rc, i;
+        int rc, i, cmd_rc;
 
         if (nvdimm) {
                 desc = nd_cmd_dimm_desc(cmd);
@@ -927,13 +1020,20 @@ static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
         }
 
         nvdimm_bus_lock(&nvdimm_bus->dev);
-        rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, cmd);
+        rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, cmd, buf);
         if (rc)
                 goto out_unlock;
 
-        rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, NULL);
+        rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, &cmd_rc);
         if (rc < 0)
                 goto out_unlock;
+
+        if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR && cmd_rc >= 0) {
+                struct nd_cmd_clear_error *clear_err = buf;
+
+                nvdimm_account_cleared_poison(nvdimm_bus, clear_err->address,
+                                clear_err->cleared);
+        }
         nvdimm_bus_unlock(&nvdimm_bus->dev);
 
         if (copy_to_user(p, buf, buf_len))
diff --git a/drivers/nvdimm/claim.c b/drivers/nvdimm/claim.c
index ca6d572c48fc..93d128da1c92 100644
--- a/drivers/nvdimm/claim.c
+++ b/drivers/nvdimm/claim.c
@@ -21,8 +21,13 @@
 void __nd_detach_ndns(struct device *dev, struct nd_namespace_common **_ndns)
 {
         struct nd_namespace_common *ndns = *_ndns;
+        struct nvdimm_bus *nvdimm_bus;
 
-        lockdep_assert_held(&ndns->dev.mutex);
+        if (!ndns)
+                return;
+
+        nvdimm_bus = walk_to_nvdimm_bus(&ndns->dev);
+        lockdep_assert_held(&nvdimm_bus->reconfig_mutex);
         dev_WARN_ONCE(dev, ndns->claim != dev, "%s: invalid claim\n", __func__);
         ndns->claim = NULL;
         *_ndns = NULL;
@@ -37,18 +42,20 @@ void nd_detach_ndns(struct device *dev,
         if (!ndns)
                 return;
         get_device(&ndns->dev);
-        device_lock(&ndns->dev);
+        nvdimm_bus_lock(&ndns->dev);
         __nd_detach_ndns(dev, _ndns);
-        device_unlock(&ndns->dev);
+        nvdimm_bus_unlock(&ndns->dev);
         put_device(&ndns->dev);
 }
 
 bool __nd_attach_ndns(struct device *dev, struct nd_namespace_common *attach,
                 struct nd_namespace_common **_ndns)
 {
+        struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&attach->dev);
+
         if (attach->claim)
                 return false;
-        lockdep_assert_held(&attach->dev.mutex);
+        lockdep_assert_held(&nvdimm_bus->reconfig_mutex);
         dev_WARN_ONCE(dev, *_ndns, "%s: invalid claim\n", __func__);
         attach->claim = dev;
         *_ndns = attach;
@@ -61,9 +68,9 @@ bool nd_attach_ndns(struct device *dev, struct nd_namespace_common *attach,
 {
         bool claimed;
 
-        device_lock(&attach->dev);
+        nvdimm_bus_lock(&attach->dev);
         claimed = __nd_attach_ndns(dev, attach, _ndns);
-        device_unlock(&attach->dev);
+        nvdimm_bus_unlock(&attach->dev);
         return claimed;
 }
 
@@ -114,7 +121,7 @@ static void nd_detach_and_reset(struct device *dev,
                 struct nd_namespace_common **_ndns)
 {
         /* detach the namespace and destroy / reset the device */
-        nd_detach_ndns(dev, _ndns);
+        __nd_detach_ndns(dev, _ndns);
         if (is_idle(dev, *_ndns)) {
                 nd_device_unregister(dev, ND_ASYNC);
         } else if (is_nd_btt(dev)) {
@@ -184,7 +191,7 @@ ssize_t nd_namespace_store(struct device *dev,
         }
 
         WARN_ON_ONCE(!is_nvdimm_bus_locked(dev));
-        if (!nd_attach_ndns(dev, ndns, _ndns)) {
+        if (!__nd_attach_ndns(dev, ndns, _ndns)) {
                 dev_dbg(dev, "%s already claimed\n",
                                 dev_name(&ndns->dev));
                 len = -EBUSY;
@@ -239,22 +246,24 @@ static int nsio_rw_bytes(struct nd_namespace_common *ndns,
         if (rw == READ) {
                 if (unlikely(is_bad_pmem(&nsio->bb, sector, sz_align)))
                         return -EIO;
-                return memcpy_from_pmem(buf, nsio->addr + offset, size);
+                return memcpy_mcsafe(buf, nsio->addr + offset, size);
         }
 
         if (unlikely(is_bad_pmem(&nsio->bb, sector, sz_align))) {
                 /*
                  * FIXME: nsio_rw_bytes() may be called from atomic
-                 * context in the btt case and nvdimm_clear_poison()
-                 * takes a sleeping lock. Until the locking can be
-                 * reworked this capability requires that the namespace
-                 * is not claimed by btt.
+                 * context in the btt case and the ACPI DSM path for
+                 * clearing the error takes sleeping locks and allocates
+                 * memory. An explicit error clearing path, and support
+                 * for tracking badblocks in BTT metadata is needed to
+                 * work around this collision.
                  */
                 if (IS_ALIGNED(offset, 512) && IS_ALIGNED(size, 512)
                                 && (!ndns->claim || !is_nd_btt(ndns->claim))) {
                         long cleared;
 
-                        cleared = nvdimm_clear_poison(&ndns->dev, offset, size);
+                        cleared = nvdimm_clear_poison(&ndns->dev,
+                                        nsio->res.start + offset, size);
                         if (cleared < size)
                                 rc = -EIO;
                         if (cleared > 0 && cleared / 512) {
diff --git a/drivers/nvdimm/core.c b/drivers/nvdimm/core.c
index 9303cfeb8bee..2dee908e4bae 100644
--- a/drivers/nvdimm/core.c
+++ b/drivers/nvdimm/core.c
@@ -518,6 +518,15 @@ void nvdimm_badblocks_populate(struct nd_region *nd_region,
 }
 EXPORT_SYMBOL_GPL(nvdimm_badblocks_populate);
 
+static void append_poison_entry(struct nvdimm_bus *nvdimm_bus,
+                struct nd_poison *pl, u64 addr, u64 length)
+{
+        lockdep_assert_held(&nvdimm_bus->poison_lock);
+        pl->start = addr;
+        pl->length = length;
+        list_add_tail(&pl->list, &nvdimm_bus->poison_list);
+}
+
 static int add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length,
                         gfp_t flags)
 {
@@ -527,19 +536,24 @@ static int add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length,
         if (!pl)
                 return -ENOMEM;
 
-        pl->start = addr;
-        pl->length = length;
-        list_add_tail(&pl->list, &nvdimm_bus->poison_list);
-
+        append_poison_entry(nvdimm_bus, pl, addr, length);
         return 0;
 }
 
 static int bus_add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length)
 {
-        struct nd_poison *pl;
+        struct nd_poison *pl, *pl_new;
 
-        if (list_empty(&nvdimm_bus->poison_list))
-                return add_poison(nvdimm_bus, addr, length, GFP_KERNEL);
+        spin_unlock(&nvdimm_bus->poison_lock);
+        pl_new = kzalloc(sizeof(*pl_new), GFP_KERNEL);
+        spin_lock(&nvdimm_bus->poison_lock);
+
+        if (list_empty(&nvdimm_bus->poison_list)) {
+                if (!pl_new)
+                        return -ENOMEM;
+                append_poison_entry(nvdimm_bus, pl_new, addr, length);
+                return 0;
+        }
 
         /*
          * There is a chance this is a duplicate, check for those first.
@@ -551,6 +565,7 @@ static int bus_add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length)
                         /* If length has changed, update this list entry */
                         if (pl->length != length)
                                 pl->length = length;
+                        kfree(pl_new);
                         return 0;
                 }
 
@@ -559,29 +574,33 @@ static int bus_add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length)
          * as any overlapping ranges will get resolved when the list is consumed
          * and converted to badblocks
          */
-        return add_poison(nvdimm_bus, addr, length, GFP_KERNEL);
+        if (!pl_new)
+                return -ENOMEM;
+        append_poison_entry(nvdimm_bus, pl_new, addr, length);
+
+        return 0;
 }
 
 int nvdimm_bus_add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length)
 {
         int rc;
 
-        nvdimm_bus_lock(&nvdimm_bus->dev);
+        spin_lock(&nvdimm_bus->poison_lock);
         rc = bus_add_poison(nvdimm_bus, addr, length);
-        nvdimm_bus_unlock(&nvdimm_bus->dev);
+        spin_unlock(&nvdimm_bus->poison_lock);
 
         return rc;
 }
 EXPORT_SYMBOL_GPL(nvdimm_bus_add_poison);
 
-void nvdimm_clear_from_poison_list(struct nvdimm_bus *nvdimm_bus,
-                phys_addr_t start, unsigned int len)
+void nvdimm_forget_poison(struct nvdimm_bus *nvdimm_bus, phys_addr_t start,
+                unsigned int len)
 {
         struct list_head *poison_list = &nvdimm_bus->poison_list;
         u64 clr_end = start + len - 1;
         struct nd_poison *pl, *next;
 
-        nvdimm_bus_lock(&nvdimm_bus->dev);
+        spin_lock(&nvdimm_bus->poison_lock);
         WARN_ON_ONCE(list_empty(poison_list));
 
         /*
@@ -628,15 +647,15 @@ void nvdimm_clear_from_poison_list(struct nvdimm_bus *nvdimm_bus,
                         u64 new_len = pl_end - new_start + 1;
 
                         /* Add new entry covering the right half */
-                        add_poison(nvdimm_bus, new_start, new_len, GFP_NOIO);
+                        add_poison(nvdimm_bus, new_start, new_len, GFP_NOWAIT);
                         /* Adjust this entry to cover the left half */
                         pl->length = start - pl->start;
                         continue;
                 }
         }
-        nvdimm_bus_unlock(&nvdimm_bus->dev);
+        spin_unlock(&nvdimm_bus->poison_lock);
 }
-EXPORT_SYMBOL_GPL(nvdimm_clear_from_poison_list);
+EXPORT_SYMBOL_GPL(nvdimm_forget_poison);
 
 #ifdef CONFIG_BLK_DEV_INTEGRITY
 int nd_integrity_init(struct gendisk *disk, unsigned long meta_size)
diff --git a/drivers/nvdimm/dax_devs.c b/drivers/nvdimm/dax_devs.c
index 45fa82cae87c..c1b6556aea6e 100644
--- a/drivers/nvdimm/dax_devs.c
+++ b/drivers/nvdimm/dax_devs.c
@@ -124,7 +124,7 @@ int nd_dax_probe(struct device *dev, struct nd_namespace_common *ndns)
         dev_dbg(dev, "%s: dax: %s\n", __func__,
                         rc == 0 ? dev_name(dax_dev) : "<none>");
         if (rc < 0) {
-                __nd_detach_ndns(dax_dev, &nd_pfn->ndns);
+                nd_detach_ndns(dax_dev, &nd_pfn->ndns);
                 put_device(dax_dev);
         } else
                 __nd_device_register(dax_dev);
diff --git a/drivers/nvdimm/dimm.c b/drivers/nvdimm/dimm.c
index ee0b412827bf..e0f0e3ce1a32 100644
--- a/drivers/nvdimm/dimm.c
+++ b/drivers/nvdimm/dimm.c
@@ -49,6 +49,8 @@ static int nvdimm_probe(struct device *dev)
         kref_init(&ndd->kref);
 
         rc = nvdimm_init_nsarea(ndd);
+        if (rc == -EACCES)
+                nvdimm_set_locked(dev);
         if (rc)
                 goto err;
 
diff --git a/drivers/nvdimm/dimm_devs.c b/drivers/nvdimm/dimm_devs.c
index 8b721321be5b..fac1e9fbd11d 100644
--- a/drivers/nvdimm/dimm_devs.c
+++ b/drivers/nvdimm/dimm_devs.c
@@ -34,7 +34,7 @@ int nvdimm_check_config_data(struct device *dev)
 
         if (!nvdimm->cmd_mask ||
             !test_bit(ND_CMD_GET_CONFIG_DATA, &nvdimm->cmd_mask)) {
-                if (nvdimm->flags & NDD_ALIASING)
+                if (test_bit(NDD_ALIASING, &nvdimm->flags))
                         return -ENXIO;
                 else
                         return -ENOTTY;
@@ -67,6 +67,7 @@ int nvdimm_init_nsarea(struct nvdimm_drvdata *ndd)
         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(ndd->dev);
         struct nvdimm_bus_descriptor *nd_desc;
         int rc = validate_dimm(ndd);
+        int cmd_rc = 0;
 
         if (rc)
                 return rc;
@@ -76,8 +77,11 @@ int nvdimm_init_nsarea(struct nvdimm_drvdata *ndd)
 
         memset(cmd, 0, sizeof(*cmd));
         nd_desc = nvdimm_bus->nd_desc;
-        return nd_desc->ndctl(nd_desc, to_nvdimm(ndd->dev),
-                        ND_CMD_GET_CONFIG_SIZE, cmd, sizeof(*cmd), NULL);
+        rc = nd_desc->ndctl(nd_desc, to_nvdimm(ndd->dev),
+                        ND_CMD_GET_CONFIG_SIZE, cmd, sizeof(*cmd), &cmd_rc);
+        if (rc < 0)
+                return rc;
+        return cmd_rc;
 }
 
 int nvdimm_init_config_data(struct nvdimm_drvdata *ndd)
@@ -188,7 +192,14 @@ void nvdimm_set_aliasing(struct device *dev)
 {
         struct nvdimm *nvdimm = to_nvdimm(dev);
 
-        nvdimm->flags |= NDD_ALIASING;
+        set_bit(NDD_ALIASING, &nvdimm->flags);
+}
+
+void nvdimm_set_locked(struct device *dev)
+{
+        struct nvdimm *nvdimm = to_nvdimm(dev);
+
+        set_bit(NDD_LOCKED, &nvdimm->flags);
 }
 
 static void nvdimm_release(struct device *dev)
diff --git a/drivers/nvdimm/namespace_devs.c b/drivers/nvdimm/namespace_devs.c
index 1b481a5fb966..2f9dfbd2dbec 100644
--- a/drivers/nvdimm/namespace_devs.c
+++ b/drivers/nvdimm/namespace_devs.c
@@ -2236,14 +2236,21 @@ static int init_active_labels(struct nd_region *nd_region)
                 int count, j;
 
                 /*
-                 * If the dimm is disabled then prevent the region from
-                 * being activated if it aliases DPA.
+                 * If the dimm is disabled then we may need to prevent
+                 * the region from being activated.
                  */
                 if (!ndd) {
-                        if ((nvdimm->flags & NDD_ALIASING) == 0)
+                        if (test_bit(NDD_LOCKED, &nvdimm->flags))
+                                /* fail, label data may be unreadable */;
+                        else if (test_bit(NDD_ALIASING, &nvdimm->flags))
+                                /* fail, labels needed to disambiguate dpa */;
+                        else
                                 return 0;
-                        dev_dbg(&nd_region->dev, "%s: is disabled, failing probe\n",
-                                        dev_name(&nd_mapping->nvdimm->dev));
+
+                        dev_err(&nd_region->dev, "%s: is %s, failing probe\n",
+                                        dev_name(&nd_mapping->nvdimm->dev),
+                                        test_bit(NDD_LOCKED, &nvdimm->flags)
+                                        ? "locked" : "disabled");
                         return -ENXIO;
                 }
                 nd_mapping->ndd = ndd;
diff --git a/drivers/nvdimm/nd-core.h b/drivers/nvdimm/nd-core.h
index 8623e57c2ce3..4c4bd209e725 100644
--- a/drivers/nvdimm/nd-core.h
+++ b/drivers/nvdimm/nd-core.h
@@ -32,6 +32,7 @@ struct nvdimm_bus {
         struct list_head poison_list;
         struct list_head mapping_list;
         struct mutex reconfig_mutex;
+        spinlock_t poison_lock;
 };
 
 struct nvdimm {
diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h
index 2a99c83aa19f..77d032192bf7 100644
--- a/drivers/nvdimm/nd.h
+++ b/drivers/nvdimm/nd.h
@@ -154,6 +154,7 @@ struct nd_region {
         u64 ndr_start;
         int id, num_lanes, ro, numa_node;
         void *provider_data;
+        struct badblocks bb;
         struct nd_interleave_set *nd_set;
         struct nd_percpu_lane __percpu *lane;
         struct nd_mapping mapping[0];
@@ -239,6 +240,7 @@ int nvdimm_set_config_data(struct nvdimm_drvdata *ndd, size_t offset,
 long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
                 unsigned int len);
 void nvdimm_set_aliasing(struct device *dev);
+void nvdimm_set_locked(struct device *dev);
 struct nd_btt *to_nd_btt(struct device *dev);
 
 struct nd_gen_sb {
diff --git a/drivers/nvdimm/pfn_devs.c b/drivers/nvdimm/pfn_devs.c
index 6c033c9a2f06..335c8175410b 100644
--- a/drivers/nvdimm/pfn_devs.c
+++ b/drivers/nvdimm/pfn_devs.c
@@ -484,7 +484,7 @@ int nd_pfn_probe(struct device *dev, struct nd_namespace_common *ndns)
         dev_dbg(dev, "%s: pfn: %s\n", __func__,
                         rc == 0 ? dev_name(pfn_dev) : "<none>");
         if (rc < 0) {
-                __nd_detach_ndns(pfn_dev, &nd_pfn->ndns);
+                nd_detach_ndns(pfn_dev, &nd_pfn->ndns);
                 put_device(pfn_dev);
         } else
                 __nd_device_register(pfn_dev);
@@ -538,7 +538,8 @@ static struct vmem_altmap *__nvdimm_setup_pfn(struct nd_pfn *nd_pfn,
                 nd_pfn->npfns = le64_to_cpu(pfn_sb->npfns);
                 altmap = NULL;
         } else if (nd_pfn->mode == PFN_MODE_PMEM) {
-                nd_pfn->npfns = (resource_size(res) - offset) / PAGE_SIZE;
+                nd_pfn->npfns = PFN_SECTION_ALIGN_UP((resource_size(res)
+                                        - offset) / PAGE_SIZE);
                 if (le64_to_cpu(nd_pfn->pfn_sb->npfns) > nd_pfn->npfns)
                         dev_info(&nd_pfn->dev,
                                         "number of pfns truncated from %lld to %ld\n",
@@ -625,7 +626,8 @@ static int nd_pfn_init(struct nd_pfn *nd_pfn)
          */
         start += start_pad;
         size = resource_size(&nsio->res);
-        npfns = (size - start_pad - end_trunc - SZ_8K) / SZ_4K;
+        npfns = PFN_SECTION_ALIGN_UP((size - start_pad - end_trunc - SZ_8K)
+                        / PAGE_SIZE);
         if (nd_pfn->mode == PFN_MODE_PMEM) {
                 /*
                  * vmemmap_populate_hugepages() allocates the memmap array in
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index fbc640bf06b0..c544d466ea51 100644
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -29,6 +29,7 @@
 #include <linux/pfn_t.h>
 #include <linux/slab.h>
 #include <linux/pmem.h>
+#include <linux/dax.h>
 #include <linux/nd.h>
 #include "pmem.h"
 #include "pfn.h"
@@ -89,7 +90,7 @@ static int read_pmem(struct page *page, unsigned int off,
         int rc;
         void *mem = kmap_atomic(page);
 
-        rc = memcpy_from_pmem(mem + off, pmem_addr, len);
+        rc = memcpy_mcsafe(mem + off, pmem_addr, len);
         kunmap_atomic(mem);
         if (rc)
                 return -EIO;
@@ -200,13 +201,13 @@ static int pmem_rw_page(struct block_device *bdev, sector_t sector,
 }
 
 /* see "strong" declaration in tools/testing/nvdimm/pmem-dax.c */
-__weak long pmem_direct_access(struct block_device *bdev, sector_t sector,
-                      void **kaddr, pfn_t *pfn, long size)
+__weak long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff,
+                long nr_pages, void **kaddr, pfn_t *pfn)
 {
-        struct pmem_device *pmem = bdev->bd_queue->queuedata;
-        resource_size_t offset = sector * 512 + pmem->data_offset;
+        resource_size_t offset = PFN_PHYS(pgoff) + pmem->data_offset;
 
-        if (unlikely(is_bad_pmem(&pmem->bb, sector, size)))
+        if (unlikely(is_bad_pmem(&pmem->bb, PFN_PHYS(pgoff) / 512,
+                                        PFN_PHYS(nr_pages))))
                 return -EIO;
         *kaddr = pmem->virt_addr + offset;
         *pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags);
@@ -216,17 +217,28 @@ __weak long pmem_direct_access(struct block_device *bdev, sector_t sector,
          * requested range.
          */
         if (unlikely(pmem->bb.count))
-                return size;
-        return pmem->size - pmem->pfn_pad - offset;
+                return nr_pages;
+        return PHYS_PFN(pmem->size - pmem->pfn_pad - offset);
 }
 
 static const struct block_device_operations pmem_fops = {
         .owner =                THIS_MODULE,
         .rw_page =              pmem_rw_page,
-        .direct_access =        pmem_direct_access,
         .revalidate_disk =      nvdimm_revalidate_disk,
 };
 
+static long pmem_dax_direct_access(struct dax_device *dax_dev,
+                pgoff_t pgoff, long nr_pages, void **kaddr, pfn_t *pfn)
+{
+        struct pmem_device *pmem = dax_get_private(dax_dev);
+
+        return __pmem_direct_access(pmem, pgoff, nr_pages, kaddr, pfn);
+}
+
+static const struct dax_operations pmem_dax_ops = {
+        .direct_access = pmem_dax_direct_access,
+};
+
 static void pmem_release_queue(void *q)
 {
         blk_cleanup_queue(q);
@@ -237,10 +249,14 @@ static void pmem_freeze_queue(void *q)
         blk_freeze_queue_start(q);
 }
 
-static void pmem_release_disk(void *disk)
+static void pmem_release_disk(void *__pmem)
 {
-        del_gendisk(disk);
-        put_disk(disk);
+        struct pmem_device *pmem = __pmem;
+
+        kill_dax(pmem->dax_dev);
+        put_dax(pmem->dax_dev);
+        del_gendisk(pmem->disk);
+        put_disk(pmem->disk);
 }
 
 static int pmem_attach_disk(struct device *dev,
@@ -251,6 +267,7 @@ static int pmem_attach_disk(struct device *dev,
         struct vmem_altmap __altmap, *altmap = NULL;
         struct resource *res = &nsio->res;
         struct nd_pfn *nd_pfn = NULL;
+        struct dax_device *dax_dev;
         int nid = dev_to_node(dev);
         struct nd_pfn_sb *pfn_sb;
         struct pmem_device *pmem;
@@ -334,6 +351,7 @@ static int pmem_attach_disk(struct device *dev,
         disk = alloc_disk_node(0, nid);
         if (!disk)
                 return -ENOMEM;
+        pmem->disk = disk;
 
         disk->fops              = &pmem_fops;
         disk->queue             = q;
@@ -345,9 +363,16 @@ static int pmem_attach_disk(struct device *dev,
                 return -ENOMEM;
         nvdimm_badblocks_populate(nd_region, &pmem->bb, res);
         disk->bb = &pmem->bb;
-        device_add_disk(dev, disk);
 
-        if (devm_add_action_or_reset(dev, pmem_release_disk, disk))
+        dax_dev = alloc_dax(pmem, disk->disk_name, &pmem_dax_ops);
+        if (!dax_dev) {
+                put_disk(disk);
+                return -ENOMEM;
+        }
+        pmem->dax_dev = dax_dev;
+
+        device_add_disk(dev, disk);
+        if (devm_add_action_or_reset(dev, pmem_release_disk, pmem))
                 return -ENOMEM;
 
         revalidate_disk(disk);
@@ -397,12 +422,12 @@ static void nd_pmem_shutdown(struct device *dev)
 
 static void nd_pmem_notify(struct device *dev, enum nvdimm_event event)
 {
-        struct pmem_device *pmem = dev_get_drvdata(dev);
-        struct nd_region *nd_region = to_region(pmem);
+        struct nd_region *nd_region;
         resource_size_t offset = 0, end_trunc = 0;
         struct nd_namespace_common *ndns;
         struct nd_namespace_io *nsio;
         struct resource res;
+        struct badblocks *bb;
 
         if (event != NVDIMM_REVALIDATE_POISON)
                 return;
@@ -411,20 +436,33 @@ static void nd_pmem_notify(struct device *dev, enum nvdimm_event event)
                 struct nd_btt *nd_btt = to_nd_btt(dev);
 
                 ndns = nd_btt->ndns;
-        } else if (is_nd_pfn(dev)) {
-                struct nd_pfn *nd_pfn = to_nd_pfn(dev);
-                struct nd_pfn_sb *pfn_sb = nd_pfn->pfn_sb;
+                nd_region = to_nd_region(ndns->dev.parent);
+                nsio = to_nd_namespace_io(&ndns->dev);
+                bb = &nsio->bb;
+        } else {
+                struct pmem_device *pmem = dev_get_drvdata(dev);
 
-                ndns = nd_pfn->ndns;
-                offset = pmem->data_offset + __le32_to_cpu(pfn_sb->start_pad);
-                end_trunc = __le32_to_cpu(pfn_sb->end_trunc);
-        } else
-                ndns = to_ndns(dev);
+                nd_region = to_region(pmem);
+                bb = &pmem->bb;
+
+                if (is_nd_pfn(dev)) {
+                        struct nd_pfn *nd_pfn = to_nd_pfn(dev);
+                        struct nd_pfn_sb *pfn_sb = nd_pfn->pfn_sb;
+
+                        ndns = nd_pfn->ndns;
+                        offset = pmem->data_offset +
+                                        __le32_to_cpu(pfn_sb->start_pad);
+                        end_trunc = __le32_to_cpu(pfn_sb->end_trunc);
+                } else {
+                        ndns = to_ndns(dev);
+                }
+
+                nsio = to_nd_namespace_io(&ndns->dev);
+        }
 
-        nsio = to_nd_namespace_io(&ndns->dev);
         res.start = nsio->res.start + offset;
         res.end = nsio->res.end - end_trunc;
-        nvdimm_badblocks_populate(nd_region, &pmem->bb, &res);
+        nvdimm_badblocks_populate(nd_region, bb, &res);
 }
 
 MODULE_ALIAS("pmem");
diff --git a/drivers/nvdimm/pmem.h b/drivers/nvdimm/pmem.h
index b4ee4f71b4a1..7f4dbd72a90a 100644
--- a/drivers/nvdimm/pmem.h
+++ b/drivers/nvdimm/pmem.h
@@ -5,8 +5,6 @@
 #include <linux/pfn_t.h>
 #include <linux/fs.h>
 
-long pmem_direct_access(struct block_device *bdev, sector_t sector,
-                      void **kaddr, pfn_t *pfn, long size);
 /* this definition is in it's own header for tools/testing/nvdimm to consume */
 struct pmem_device {
         /* One contiguous memory region per device */
@@ -20,5 +18,10 @@ struct pmem_device {
         /* trim size when namespace capacity has been section aligned */
         u32                     pfn_pad;
         struct badblocks        bb;
+        struct dax_device       *dax_dev;
+        struct gendisk          *disk;
 };
+
+long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff,
+                long nr_pages, void **kaddr, pfn_t *pfn);
 #endif /* __NVDIMM_PMEM_H__ */
diff --git a/drivers/nvdimm/region.c b/drivers/nvdimm/region.c
index 8f241772ec0b..869a886c292e 100644
--- a/drivers/nvdimm/region.c
+++ b/drivers/nvdimm/region.c
@@ -14,6 +14,7 @@
 #include <linux/module.h>
 #include <linux/device.h>
 #include <linux/nd.h>
+#include "nd-core.h"
 #include "nd.h"
 
 static int nd_region_probe(struct device *dev)
@@ -52,6 +53,17 @@ static int nd_region_probe(struct device *dev)
         if (rc && err && rc == err)
                 return -ENODEV;
 
+        if (is_nd_pmem(&nd_region->dev)) {
+                struct resource ndr_res;
+
+                if (devm_init_badblocks(dev, &nd_region->bb))
+                        return -ENODEV;
+                ndr_res.start = nd_region->ndr_start;
+                ndr_res.end = nd_region->ndr_start + nd_region->ndr_size - 1;
+                nvdimm_badblocks_populate(nd_region,
+                                &nd_region->bb, &ndr_res);
+        }
+
         nd_region->btt_seed = nd_btt_create(nd_region);
         nd_region->pfn_seed = nd_pfn_create(nd_region);
         nd_region->dax_seed = nd_dax_create(nd_region);
@@ -104,6 +116,18 @@ static int child_notify(struct device *dev, void *data)
 
 static void nd_region_notify(struct device *dev, enum nvdimm_event event)
 {
+        if (event == NVDIMM_REVALIDATE_POISON) {
+                struct nd_region *nd_region = to_nd_region(dev);
+                struct resource res;
+
+                if (is_nd_pmem(&nd_region->dev)) {
+                        res.start = nd_region->ndr_start;
+                        res.end = nd_region->ndr_start +
+                                nd_region->ndr_size - 1;
+                        nvdimm_badblocks_populate(nd_region,
+                                        &nd_region->bb, &res);
+                }
+        }
         device_for_each_child(dev, &event, child_notify);
 }
 
diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c
index b7cb5066d961..b550edf2571f 100644
--- a/drivers/nvdimm/region_devs.c
+++ b/drivers/nvdimm/region_devs.c
@@ -222,7 +222,7 @@ int nd_region_to_nstype(struct nd_region *nd_region)
                         struct nd_mapping *nd_mapping = &nd_region->mapping[i];
                         struct nvdimm *nvdimm = nd_mapping->nvdimm;
 
-                        if (nvdimm->flags & NDD_ALIASING)
+                        if (test_bit(NDD_ALIASING, &nvdimm->flags))
                                 alias++;
                 }
                 if (alias)
@@ -255,6 +255,35 @@ static ssize_t size_show(struct device *dev,
 }
 static DEVICE_ATTR_RO(size);
 
+static ssize_t deep_flush_show(struct device *dev,
+                struct device_attribute *attr, char *buf)
+{
+        struct nd_region *nd_region = to_nd_region(dev);
+
+        /*
+         * NOTE: in the nvdimm_has_flush() error case this attribute is
+         * not visible.
+         */
+        return sprintf(buf, "%d\n", nvdimm_has_flush(nd_region));
+}
+
+static ssize_t deep_flush_store(struct device *dev, struct device_attribute *attr,
+                const char *buf, size_t len)
+{
+        bool flush;
+        int rc = strtobool(buf, &flush);
+        struct nd_region *nd_region = to_nd_region(dev);
+
+        if (rc)
+                return rc;
+        if (!flush)
+                return -EINVAL;
+        nvdimm_flush(nd_region);
+
+        return len;
+}
+static DEVICE_ATTR_RW(deep_flush);
+
 static ssize_t mappings_show(struct device *dev,
                 struct device_attribute *attr, char *buf)
 {
@@ -448,6 +477,25 @@ static ssize_t read_only_store(struct device *dev,
 }
 static DEVICE_ATTR_RW(read_only);
 
+static ssize_t region_badblocks_show(struct device *dev,
+                struct device_attribute *attr, char *buf)
+{
+        struct nd_region *nd_region = to_nd_region(dev);
+
+        return badblocks_show(&nd_region->bb, buf, 0);
+}
+
+static DEVICE_ATTR(badblocks, 0444, region_badblocks_show, NULL);
+
+static ssize_t resource_show(struct device *dev,
+                struct device_attribute *attr, char *buf)
+{
+        struct nd_region *nd_region = to_nd_region(dev);
+
+        return sprintf(buf, "%#llx\n", nd_region->ndr_start);
+}
+static DEVICE_ATTR_RO(resource);
+
 static struct attribute *nd_region_attributes[] = {
         &dev_attr_size.attr,
         &dev_attr_nstype.attr,
@@ -455,11 +503,14 @@ static struct attribute *nd_region_attributes[] = {
         &dev_attr_btt_seed.attr,
         &dev_attr_pfn_seed.attr,
         &dev_attr_dax_seed.attr,
+        &dev_attr_deep_flush.attr,
         &dev_attr_read_only.attr,
         &dev_attr_set_cookie.attr,
         &dev_attr_available_size.attr,
         &dev_attr_namespace_seed.attr,
         &dev_attr_init_namespaces.attr,
+        &dev_attr_badblocks.attr,
+        &dev_attr_resource.attr,
         NULL,
 };
 
@@ -476,6 +527,23 @@ static umode_t region_visible(struct kobject *kobj, struct attribute *a, int n)
         if (!is_nd_pmem(dev) && a == &dev_attr_dax_seed.attr)
                 return 0;
 
+        if (!is_nd_pmem(dev) && a == &dev_attr_badblocks.attr)
+                return 0;
+
+        if (!is_nd_pmem(dev) && a == &dev_attr_resource.attr)
+                return 0;
+
+        if (a == &dev_attr_deep_flush.attr) {
+                int has_flush = nvdimm_has_flush(nd_region);
+
+                if (has_flush == 1)
+                        return a->mode;
+                else if (has_flush == 0)
+                        return 0444;
+                else
+                        return 0;
+        }
+
         if (a != &dev_attr_set_cookie.attr
                         && a != &dev_attr_available_size.attr)
                 return a->mode;
@@ -813,7 +881,7 @@ static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus,
                         return NULL;
                 }
 
-                if (nvdimm->flags & NDD_UNARMED)
+                if (test_bit(NDD_UNARMED, &nvdimm->flags))
                         ro = 1;
         }
 
@@ -968,17 +1036,20 @@ EXPORT_SYMBOL_GPL(nvdimm_flush);
  */
 int nvdimm_has_flush(struct nd_region *nd_region)
 {
-        struct nd_region_data *ndrd = dev_get_drvdata(&nd_region->dev);
         int i;
 
         /* no nvdimm == flushing capability unknown */
         if (nd_region->ndr_mappings == 0)
                 return -ENXIO;
 
-        for (i = 0; i < nd_region->ndr_mappings; i++)
-                /* flush hints present, flushing required */
-                if (ndrd_get_flush_wpq(ndrd, i, 0))
+        for (i = 0; i < nd_region->ndr_mappings; i++) {
+                struct nd_mapping *nd_mapping = &nd_region->mapping[i];
+                struct nvdimm *nvdimm = nd_mapping->nvdimm;
+
+                /* flush hints present / available */
+                if (nvdimm->num_flush)
                         return 1;
+        }
 
         /*
          * The platform defines dimm devices without hints, assume
diff --git a/drivers/s390/block/Kconfig b/drivers/s390/block/Kconfig
index 4a3b62326183..0acb8c2f9475 100644
--- a/drivers/s390/block/Kconfig
+++ b/drivers/s390/block/Kconfig
@@ -14,6 +14,7 @@ config BLK_DEV_XPRAM
 
 config DCSSBLK
         def_tristate m
+        select DAX
         prompt "DCSSBLK support"
         depends on S390 && BLOCK
         help
diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c
index 415d10a67b7a..36e5280af3e4 100644
--- a/drivers/s390/block/dcssblk.c
+++ b/drivers/s390/block/dcssblk.c
@@ -18,6 +18,7 @@
 #include <linux/interrupt.h>
 #include <linux/platform_device.h>
 #include <linux/pfn_t.h>
+#include <linux/dax.h>
 #include <asm/extmem.h>
 #include <asm/io.h>
 
@@ -30,8 +31,8 @@ static int dcssblk_open(struct block_device *bdev, fmode_t mode);
 static void dcssblk_release(struct gendisk *disk, fmode_t mode);
 static blk_qc_t dcssblk_make_request(struct request_queue *q,
                                                 struct bio *bio);
-static long dcssblk_direct_access(struct block_device *bdev, sector_t secnum,
-                         void **kaddr, pfn_t *pfn, long size);
+static long dcssblk_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
+                long nr_pages, void **kaddr, pfn_t *pfn);
 
 static char dcssblk_segments[DCSSBLK_PARM_LEN] = "\0";
 
@@ -40,7 +41,10 @@ static const struct block_device_operations dcssblk_devops = {
         .owner          = THIS_MODULE,
         .open           = dcssblk_open,
         .release        = dcssblk_release,
-        .direct_access  = dcssblk_direct_access,
+};
+
+static const struct dax_operations dcssblk_dax_ops = {
+        .direct_access = dcssblk_dax_direct_access,
 };
 
 struct dcssblk_dev_info {
@@ -57,6 +61,7 @@ struct dcssblk_dev_info {
         struct request_queue *dcssblk_queue;
         int num_of_segments;
         struct list_head seg_list;
+        struct dax_device *dax_dev;
 };
 
 struct segment_info {
@@ -389,6 +394,8 @@ removeseg:
         }
         list_del(&dev_info->lh);
 
+        kill_dax(dev_info->dax_dev);
+        put_dax(dev_info->dax_dev);
         del_gendisk(dev_info->gd);
         blk_cleanup_queue(dev_info->dcssblk_queue);
         dev_info->gd->queue = NULL;
@@ -654,6 +661,13 @@ dcssblk_add_store(struct device *dev, struct device_attribute *attr, const char
         if (rc)
                 goto put_dev;
 
+        dev_info->dax_dev = alloc_dax(dev_info, dev_info->gd->disk_name,
+                        &dcssblk_dax_ops);
+        if (!dev_info->dax_dev) {
+                rc = -ENOMEM;
+                goto put_dev;
+        }
+
         get_device(&dev_info->dev);
         device_add_disk(&dev_info->dev, dev_info->gd);
 
@@ -752,6 +766,8 @@ dcssblk_remove_store(struct device *dev, struct device_attribute *attr, const ch
         }
 
         list_del(&dev_info->lh);
+        kill_dax(dev_info->dax_dev);
+        put_dax(dev_info->dax_dev);
         del_gendisk(dev_info->gd);
         blk_cleanup_queue(dev_info->dcssblk_queue);
         dev_info->gd->queue = NULL;
@@ -883,21 +899,26 @@ fail:
 }
 
 static long
-dcssblk_direct_access (struct block_device *bdev, sector_t secnum,
-                        void **kaddr, pfn_t *pfn, long size)
+__dcssblk_direct_access(struct dcssblk_dev_info *dev_info, pgoff_t pgoff,
+                long nr_pages, void **kaddr, pfn_t *pfn)
 {
-        struct dcssblk_dev_info *dev_info;
-        unsigned long offset, dev_sz;
+        resource_size_t offset = pgoff * PAGE_SIZE;
+        unsigned long dev_sz;
 
-        dev_info = bdev->bd_disk->private_data;
-        if (!dev_info)
-                return -ENODEV;
         dev_sz = dev_info->end - dev_info->start + 1;
-        offset = secnum * 512;
         *kaddr = (void *) dev_info->start + offset;
         *pfn = __pfn_to_pfn_t(PFN_DOWN(dev_info->start + offset), PFN_DEV);
 
-        return dev_sz - offset;
+        return (dev_sz - offset) / PAGE_SIZE;
+}
+
+static long
+dcssblk_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
+                long nr_pages, void **kaddr, pfn_t *pfn)
+{
+        struct dcssblk_dev_info *dev_info = dax_get_private(dax_dev);
+
+        return __dcssblk_direct_access(dev_info, pgoff, nr_pages, kaddr, pfn);
 }
 
 static void
diff --git a/fs/block_dev.c b/fs/block_dev.c
index 0d435c794d76..2a305c1a2d88 100644
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -18,6 +18,7 @@
 #include <linux/module.h>
 #include <linux/blkpg.h>
 #include <linux/magic.h>
+#include <linux/dax.h>
 #include <linux/buffer_head.h>
 #include <linux/swap.h>
 #include <linux/pagevec.h>
@@ -716,50 +717,18 @@ int bdev_write_page(struct block_device *bdev, sector_t sector,
 }
 EXPORT_SYMBOL_GPL(bdev_write_page);
 
-/**
- * bdev_direct_access() - Get the address for directly-accessibly memory
- * @bdev: The device containing the memory
- * @dax: control and output parameters for ->direct_access
- *
- * If a block device is made up of directly addressable memory, this function
- * will tell the caller the PFN and the address of the memory.  The address
- * may be directly dereferenced within the kernel without the need to call
- * ioremap(), kmap() or similar.  The PFN is suitable for inserting into
- * page tables.
- *
- * Return: negative errno if an error occurs, otherwise the number of bytes
- * accessible at this address.
- */
-long bdev_direct_access(struct block_device *bdev, struct blk_dax_ctl *dax)
+int bdev_dax_pgoff(struct block_device *bdev, sector_t sector, size_t size,
+                pgoff_t *pgoff)
 {
-        sector_t sector = dax->sector;
-        long avail, size = dax->size;
-        const struct block_device_operations *ops = bdev->bd_disk->fops;
+        phys_addr_t phys_off = (get_start_sect(bdev) + sector) * 512;
 
-        /*
-         * The device driver is allowed to sleep, in order to make the
-         * memory directly accessible.
-         */
-        might_sleep();
-
-        if (size < 0)
-                return size;
-        if (!blk_queue_dax(bdev_get_queue(bdev)) || !ops->direct_access)
-                return -EOPNOTSUPP;
-        if ((sector + DIV_ROUND_UP(size, 512)) >
-                                        part_nr_sects_read(bdev->bd_part))
-                return -ERANGE;
-        sector += get_start_sect(bdev);
-        if (sector % (PAGE_SIZE / 512))
+        if (pgoff)
+                *pgoff = PHYS_PFN(phys_off);
+        if (phys_off % PAGE_SIZE || size % PAGE_SIZE)
                 return -EINVAL;
-        avail = ops->direct_access(bdev, sector, &dax->addr, &dax->pfn, size);
-        if (!avail)
-                return -ERANGE;
-        if (avail > 0 && avail & ~PAGE_MASK)
-                return -ENXIO;
-        return min(avail, size);
+        return 0;
 }
-EXPORT_SYMBOL_GPL(bdev_direct_access);
+EXPORT_SYMBOL(bdev_dax_pgoff);
 
 /**
  * bdev_dax_supported() - Check if the device supports dax for filesystem
@@ -773,62 +742,46 @@ EXPORT_SYMBOL_GPL(bdev_direct_access);
  */
 int bdev_dax_supported(struct super_block *sb, int blocksize)
 {
-        struct blk_dax_ctl dax = {
-                .sector = 0,
-                .size = PAGE_SIZE,
-        };
-        int err;
+        struct block_device *bdev = sb->s_bdev;
+        struct dax_device *dax_dev;
+        pgoff_t pgoff;
+        int err, id;
+        void *kaddr;
+        pfn_t pfn;
+        long len;
 
         if (blocksize != PAGE_SIZE) {
                 vfs_msg(sb, KERN_ERR, "error: unsupported blocksize for dax");
                 return -EINVAL;
         }
 
-        err = bdev_direct_access(sb->s_bdev, &dax);
-        if (err < 0) {
-                switch (err) {
-                case -EOPNOTSUPP:
-                        vfs_msg(sb, KERN_ERR,
-                                "error: device does not support dax");
-                        break;
-                case -EINVAL:
-                        vfs_msg(sb, KERN_ERR,
-                                "error: unaligned partition for dax");
-                        break;
-                default:
-                        vfs_msg(sb, KERN_ERR,
-                                "error: dax access failed (%d)", err);
-                }
+        err = bdev_dax_pgoff(bdev, 0, PAGE_SIZE, &pgoff);
+        if (err) {
+                vfs_msg(sb, KERN_ERR, "error: unaligned partition for dax");
                 return err;
         }
 
-        return 0;
-}
-EXPORT_SYMBOL_GPL(bdev_dax_supported);
-
-/**
- * bdev_dax_capable() - Return if the raw device is capable for dax
- * @bdev: The device for raw block device access
- */
-bool bdev_dax_capable(struct block_device *bdev)
-{
-        struct blk_dax_ctl dax = {
-                .size = PAGE_SIZE,
-        };
+        dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
+        if (!dax_dev) {
+                vfs_msg(sb, KERN_ERR, "error: device does not support dax");
+                return -EOPNOTSUPP;
+        }
 
-        if (!IS_ENABLED(CONFIG_FS_DAX))
-                return false;
+        id = dax_read_lock();
+        len = dax_direct_access(dax_dev, pgoff, 1, &kaddr, &pfn);
+        dax_read_unlock(id);
 
-        dax.sector = 0;
-        if (bdev_direct_access(bdev, &dax) < 0)
-                return false;
+        put_dax(dax_dev);
 
-        dax.sector = bdev->bd_part->nr_sects - (PAGE_SIZE / 512);
-        if (bdev_direct_access(bdev, &dax) < 0)
-                return false;
+        if (len < 1) {
+                vfs_msg(sb, KERN_ERR,
+                                "error: dax access failed (%ld)", len);
+                return len < 0 ? len : -EIO;
+        }
 
-        return true;
+        return 0;
 }
+EXPORT_SYMBOL_GPL(bdev_dax_supported);
 
 /*
  * pseudo-fs
diff --git a/fs/dax.c b/fs/dax.c
index 6433650be833..43bbd6d1037d 100644
--- a/fs/dax.c
+++ b/fs/dax.c
@@ -55,32 +55,6 @@ static int __init init_dax_wait_table(void)
 }
 fs_initcall(init_dax_wait_table);
 
-static long dax_map_atomic(struct block_device *bdev, struct blk_dax_ctl *dax)
-{
-        struct request_queue *q = bdev->bd_queue;
-        long rc = -EIO;
-
-        dax->addr = ERR_PTR(-EIO);
-        if (blk_queue_enter(q, true) != 0)
-                return rc;
-
-        rc = bdev_direct_access(bdev, dax);
-        if (rc < 0) {
-                dax->addr = ERR_PTR(rc);
-                blk_queue_exit(q);
-                return rc;
-        }
-        return rc;
-}
-
-static void dax_unmap_atomic(struct block_device *bdev,
-                const struct blk_dax_ctl *dax)
-{
-        if (IS_ERR(dax->addr))
-                return;
-        blk_queue_exit(bdev->bd_queue);
-}
-
 static int dax_is_pmd_entry(void *entry)
 {
         return (unsigned long)entry & RADIX_DAX_PMD;
@@ -101,26 +75,6 @@ static int dax_is_empty_entry(void *entry)
         return (unsigned long)entry & RADIX_DAX_EMPTY;
 }
 
-struct page *read_dax_sector(struct block_device *bdev, sector_t n)
-{
-        struct page *page = alloc_pages(GFP_KERNEL, 0);
-        struct blk_dax_ctl dax = {
-                .size = PAGE_SIZE,
-                .sector = n & ~((((int) PAGE_SIZE) / 512) - 1),
-        };
-        long rc;
-
-        if (!page)
-                return ERR_PTR(-ENOMEM);
-
-        rc = dax_map_atomic(bdev, &dax);
-        if (rc < 0)
-                return ERR_PTR(rc);
-        memcpy_from_pmem(page_address(page), dax.addr, PAGE_SIZE);
-        dax_unmap_atomic(bdev, &dax);
-        return page;
-}
-
 /*
  * DAX radix tree locking
  */
@@ -582,21 +536,30 @@ static int dax_load_hole(struct address_space *mapping, void **entry,
         return ret;
 }
 
-static int copy_user_dax(struct block_device *bdev, sector_t sector, size_t size,
-                struct page *to, unsigned long vaddr)
+static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev,
+                sector_t sector, size_t size, struct page *to,
+                unsigned long vaddr)
 {
-        struct blk_dax_ctl dax = {
-                .sector = sector,
-                .size = size,
-        };
-        void *vto;
-
-        if (dax_map_atomic(bdev, &dax) < 0)
-                return PTR_ERR(dax.addr);
+        void *vto, *kaddr;
+        pgoff_t pgoff;
+        pfn_t pfn;
+        long rc;
+        int id;
+
+        rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);
+        if (rc)
+                return rc;
+
+        id = dax_read_lock();
+        rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, &pfn);
+        if (rc < 0) {
+                dax_read_unlock(id);
+                return rc;
+        }
         vto = kmap_atomic(to);
-        copy_user_page(vto, (void __force *)dax.addr, vaddr, to);
+        copy_user_page(vto, (void __force *)kaddr, vaddr, to);
         kunmap_atomic(vto);
-        dax_unmap_atomic(bdev, &dax);
+        dax_read_unlock(id);
         return 0;
 }
 
@@ -764,12 +727,16 @@ unlock_pte:
 }
 
 static int dax_writeback_one(struct block_device *bdev,
-                struct address_space *mapping, pgoff_t index, void *entry)
+                struct dax_device *dax_dev, struct address_space *mapping,
+                pgoff_t index, void *entry)
 {
         struct radix_tree_root *page_tree = &mapping->page_tree;
-        struct blk_dax_ctl dax;
-        void *entry2, **slot;
-        int ret = 0;
+        void *entry2, **slot, *kaddr;
+        long ret = 0, id;
+        sector_t sector;
+        pgoff_t pgoff;
+        size_t size;
+        pfn_t pfn;
 
         /*
          * A page got tagged dirty in DAX mapping? Something is seriously
@@ -818,26 +785,29 @@ static int dax_writeback_one(struct block_device *bdev,
          * 'entry'.  This allows us to flush for PMD_SIZE and not have to
          * worry about partial PMD writebacks.
          */
-        dax.sector = dax_radix_sector(entry);
-        dax.size = PAGE_SIZE << dax_radix_order(entry);
+        sector = dax_radix_sector(entry);
+        size = PAGE_SIZE << dax_radix_order(entry);
+
+        id = dax_read_lock();
+        ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
+        if (ret)
+                goto dax_unlock;
 
         /*
-         * We cannot hold tree_lock while calling dax_map_atomic() because it
-         * eventually calls cond_resched().
+         * dax_direct_access() may sleep, so cannot hold tree_lock over
+         * its invocation.
          */
-        ret = dax_map_atomic(bdev, &dax);
-        if (ret < 0) {
-                put_locked_mapping_entry(mapping, index, entry);
-                return ret;
-        }
+        ret = dax_direct_access(dax_dev, pgoff, size / PAGE_SIZE, &kaddr, &pfn);
+        if (ret < 0)
+                goto dax_unlock;
 
-        if (WARN_ON_ONCE(ret < dax.size)) {
+        if (WARN_ON_ONCE(ret < size / PAGE_SIZE)) {
                 ret = -EIO;
-                goto unmap;
+                goto dax_unlock;
         }
 
-        dax_mapping_entry_mkclean(mapping, index, pfn_t_to_pfn(dax.pfn));
-        wb_cache_pmem(dax.addr, dax.size);
+        dax_mapping_entry_mkclean(mapping, index, pfn_t_to_pfn(pfn));
+        wb_cache_pmem(kaddr, size);
         /*
          * After we have flushed the cache, we can clear the dirty tag. There
          * cannot be new dirty data in the pfn after the flush has completed as
@@ -847,8 +817,8 @@ static int dax_writeback_one(struct block_device *bdev,
         spin_lock_irq(&mapping->tree_lock);
         radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_DIRTY);
         spin_unlock_irq(&mapping->tree_lock);
- unmap:
-        dax_unmap_atomic(bdev, &dax);
+ dax_unlock:
+        dax_read_unlock(id);
         put_locked_mapping_entry(mapping, index, entry);
         return ret;
 
@@ -869,6 +839,7 @@ int dax_writeback_mapping_range(struct address_space *mapping,
         struct inode *inode = mapping->host;
         pgoff_t start_index, end_index;
         pgoff_t indices[PAGEVEC_SIZE];
+        struct dax_device *dax_dev;
         struct pagevec pvec;
         bool done = false;
         int i, ret = 0;
@@ -879,6 +850,10 @@ int dax_writeback_mapping_range(struct address_space *mapping,
         if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
                 return 0;
 
+        dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
+        if (!dax_dev)
+                return -EIO;
+
         start_index = wbc->range_start >> PAGE_SHIFT;
         end_index = wbc->range_end >> PAGE_SHIFT;
 
@@ -899,38 +874,49 @@ int dax_writeback_mapping_range(struct address_space *mapping,
                                 break;
                         }
 
-                        ret = dax_writeback_one(bdev, mapping, indices[i],
-                                        pvec.pages[i]);
-                        if (ret < 0)
+                        ret = dax_writeback_one(bdev, dax_dev, mapping,
+                                        indices[i], pvec.pages[i]);
+                        if (ret < 0) {
+                                put_dax(dax_dev);
                                 return ret;
+                        }
                 }
         }
+        put_dax(dax_dev);
         return 0;
 }
 EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);
 
 static int dax_insert_mapping(struct address_space *mapping,
-                struct block_device *bdev, sector_t sector, size_t size,
-                void **entryp, struct vm_area_struct *vma, struct vm_fault *vmf)
+                struct block_device *bdev, struct dax_device *dax_dev,
+                sector_t sector, size_t size, void **entryp,
+                struct vm_area_struct *vma, struct vm_fault *vmf)
 {
         unsigned long vaddr = vmf->address;
-        struct blk_dax_ctl dax = {
-                .sector = sector,
-                .size = size,
-        };
-        void *ret;
         void *entry = *entryp;
+        void *ret, *kaddr;
+        pgoff_t pgoff;
+        int id, rc;
+        pfn_t pfn;
 
-        if (dax_map_atomic(bdev, &dax) < 0)
-                return PTR_ERR(dax.addr);
-        dax_unmap_atomic(bdev, &dax);
+        rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);
+        if (rc)
+                return rc;
 
-        ret = dax_insert_mapping_entry(mapping, vmf, entry, dax.sector, 0);
+        id = dax_read_lock();
+        rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, &pfn);
+        if (rc < 0) {
+                dax_read_unlock(id);
+                return rc;
+        }
+        dax_read_unlock(id);
+
+        ret = dax_insert_mapping_entry(mapping, vmf, entry, sector, 0);
         if (IS_ERR(ret))
                 return PTR_ERR(ret);
         *entryp = ret;
 
-        return vm_insert_mixed(vma, vaddr, dax.pfn);
+        return vm_insert_mixed(vma, vaddr, pfn);
 }
 
 /**
@@ -979,24 +965,34 @@ static bool dax_range_is_aligned(struct block_device *bdev,
         return true;
 }
 
-int __dax_zero_page_range(struct block_device *bdev, sector_t sector,
-                unsigned int offset, unsigned int length)
+int __dax_zero_page_range(struct block_device *bdev,
+                struct dax_device *dax_dev, sector_t sector,
+                unsigned int offset, unsigned int size)
 {
-        struct blk_dax_ctl dax = {
-                .sector         = sector,
-                .size           = PAGE_SIZE,
-        };
-
-        if (dax_range_is_aligned(bdev, offset, length)) {
-                sector_t start_sector = dax.sector + (offset >> 9);
+        if (dax_range_is_aligned(bdev, offset, size)) {
+                sector_t start_sector = sector + (offset >> 9);
 
                 return blkdev_issue_zeroout(bdev, start_sector,
-                                length >> 9, GFP_NOFS, 0);
+                                size >> 9, GFP_NOFS, 0);
         } else {
-                if (dax_map_atomic(bdev, &dax) < 0)
-                        return PTR_ERR(dax.addr);
-                clear_pmem(dax.addr + offset, length);
-                dax_unmap_atomic(bdev, &dax);
+                pgoff_t pgoff;
+                long rc, id;
+                void *kaddr;
+                pfn_t pfn;
+
+                rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);
+                if (rc)
+                        return rc;
+
+                id = dax_read_lock();
+                rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr,
+                                &pfn);
+                if (rc < 0) {
+                        dax_read_unlock(id);
+                        return rc;
+                }
+                clear_pmem(kaddr + offset, size);
+                dax_read_unlock(id);
         }
         return 0;
 }
@@ -1011,9 +1007,12 @@ static loff_t
 dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
                 struct iomap *iomap)
 {
+        struct block_device *bdev = iomap->bdev;
+        struct dax_device *dax_dev = iomap->dax_dev;
         struct iov_iter *iter = data;
         loff_t end = pos + length, done = 0;
         ssize_t ret = 0;
+        int id;
 
         if (iov_iter_rw(iter) == READ) {
                 end = min(end, i_size_read(inode));
@@ -1038,34 +1037,42 @@ dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
                                               (end - 1) >> PAGE_SHIFT);
         }
 
+        id = dax_read_lock();
         while (pos < end) {
                 unsigned offset = pos & (PAGE_SIZE - 1);
-                struct blk_dax_ctl dax = { 0 };
+                const size_t size = ALIGN(length + offset, PAGE_SIZE);
+                const sector_t sector = dax_iomap_sector(iomap, pos);
                 ssize_t map_len;
+                pgoff_t pgoff;
+                void *kaddr;
+                pfn_t pfn;
 
                 if (fatal_signal_pending(current)) {
                         ret = -EINTR;
                         break;
                 }
 
-                dax.sector = dax_iomap_sector(iomap, pos);
-                dax.size = (length + offset + PAGE_SIZE - 1) & PAGE_MASK;
-                map_len = dax_map_atomic(iomap->bdev, &dax);
+                ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
+                if (ret)
+                        break;
+
+                map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size),
+                                &kaddr, &pfn);
                 if (map_len < 0) {
                         ret = map_len;
                         break;
                 }
 
-                dax.addr += offset;
+                map_len = PFN_PHYS(map_len);
+                kaddr += offset;
                 map_len -= offset;
                 if (map_len > end - pos)
                         map_len = end - pos;
 
                 if (iov_iter_rw(iter) == WRITE)
-                        map_len = copy_from_iter_pmem(dax.addr, map_len, iter);
+                        map_len = copy_from_iter_pmem(kaddr, map_len, iter);
                 else
-                        map_len = copy_to_iter(dax.addr, map_len, iter);
-                dax_unmap_atomic(iomap->bdev, &dax);
+                        map_len = copy_to_iter(kaddr, map_len, iter);
                 if (map_len <= 0) {
                         ret = map_len ? map_len : -EFAULT;
                         break;
@@ -1075,6 +1082,7 @@ dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
                 length -= map_len;
                 done += map_len;
         }
+        dax_read_unlock(id);
 
         return done ? done : ret;
 }
@@ -1181,8 +1189,8 @@ static int dax_iomap_pte_fault(struct vm_fault *vmf,
                         clear_user_highpage(vmf->cow_page, vaddr);
                         break;
                 case IOMAP_MAPPED:
-                        error = copy_user_dax(iomap.bdev, sector, PAGE_SIZE,
-                                        vmf->cow_page, vaddr);
+                        error = copy_user_dax(iomap.bdev, iomap.dax_dev,
+                                        sector, PAGE_SIZE, vmf->cow_page, vaddr);
                         break;
                 default:
                         WARN_ON_ONCE(1);
@@ -1207,8 +1215,8 @@ static int dax_iomap_pte_fault(struct vm_fault *vmf,
                         mem_cgroup_count_vm_event(vmf->vma->vm_mm, PGMAJFAULT);
                         major = VM_FAULT_MAJOR;
                 }
-                error = dax_insert_mapping(mapping, iomap.bdev, sector,
-                                PAGE_SIZE, &entry, vmf->vma, vmf);
+                error = dax_insert_mapping(mapping, iomap.bdev, iomap.dax_dev,
+                                sector, PAGE_SIZE, &entry, vmf->vma, vmf);
                 /* -EBUSY is fine, somebody else faulted on the same PTE */
                 if (error == -EBUSY)
                         error = 0;
@@ -1258,41 +1266,48 @@ static int dax_pmd_insert_mapping(struct vm_fault *vmf, struct iomap *iomap,
                 loff_t pos, void **entryp)
 {
         struct address_space *mapping = vmf->vma->vm_file->f_mapping;
+        const sector_t sector = dax_iomap_sector(iomap, pos);
+        struct dax_device *dax_dev = iomap->dax_dev;
         struct block_device *bdev = iomap->bdev;
         struct inode *inode = mapping->host;
-        struct blk_dax_ctl dax = {
-                .sector = dax_iomap_sector(iomap, pos),
-                .size = PMD_SIZE,
-        };
-        long length = dax_map_atomic(bdev, &dax);
-        void *ret = NULL;
-
-        if (length < 0) /* dax_map_atomic() failed */
+        const size_t size = PMD_SIZE;
+        void *ret = NULL, *kaddr;
+        long length = 0;
+        pgoff_t pgoff;
+        pfn_t pfn;
+        int id;
+
+        if (bdev_dax_pgoff(bdev, sector, size, &pgoff) != 0)
                 goto fallback;
-        if (length < PMD_SIZE)
-                goto unmap_fallback;
-        if (pfn_t_to_pfn(dax.pfn) & PG_PMD_COLOUR)
-                goto unmap_fallback;
-        if (!pfn_t_devmap(dax.pfn))
-                goto unmap_fallback;
-
-        dax_unmap_atomic(bdev, &dax);
 
-        ret = dax_insert_mapping_entry(mapping, vmf, *entryp, dax.sector,
+        id = dax_read_lock();
+        length = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, &pfn);
+        if (length < 0)
+                goto unlock_fallback;
+        length = PFN_PHYS(length);
+
+        if (length < size)
+                goto unlock_fallback;
+        if (pfn_t_to_pfn(pfn) & PG_PMD_COLOUR)
+                goto unlock_fallback;
+        if (!pfn_t_devmap(pfn))
+                goto unlock_fallback;
+        dax_read_unlock(id);
+
+        ret = dax_insert_mapping_entry(mapping, vmf, *entryp, sector,
                         RADIX_DAX_PMD);
         if (IS_ERR(ret))
                 goto fallback;
         *entryp = ret;
 
-        trace_dax_pmd_insert_mapping(inode, vmf, length, dax.pfn, ret);
+        trace_dax_pmd_insert_mapping(inode, vmf, length, pfn, ret);
         return vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd,
-                        dax.pfn, vmf->flags & FAULT_FLAG_WRITE);
+                        pfn, vmf->flags & FAULT_FLAG_WRITE);
 
- unmap_fallback:
-        dax_unmap_atomic(bdev, &dax);
+unlock_fallback:
+        dax_read_unlock(id);
 fallback:
-        trace_dax_pmd_insert_mapping_fallback(inode, vmf, length,
-                        dax.pfn, ret);
+        trace_dax_pmd_insert_mapping_fallback(inode, vmf, length, pfn, ret);
         return VM_FAULT_FALLBACK;
 }
 
diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c
index 3a38c1b84e3c..26d77f9f8c12 100644
--- a/fs/ext2/inode.c
+++ b/fs/ext2/inode.c
@@ -799,6 +799,7 @@ int ext2_get_block(struct inode *inode, sector_t iblock,
 static int ext2_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
                 unsigned flags, struct iomap *iomap)
 {
+        struct block_device *bdev;
         unsigned int blkbits = inode->i_blkbits;
         unsigned long first_block = offset >> blkbits;
         unsigned long max_blocks = (length + (1 << blkbits) - 1) >> blkbits;
@@ -812,8 +813,13 @@ static int ext2_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
                 return ret;
 
         iomap->flags = 0;
-        iomap->bdev = inode->i_sb->s_bdev;
+        bdev = inode->i_sb->s_bdev;
+        iomap->bdev = bdev;
         iomap->offset = (u64)first_block << blkbits;
+        if (blk_queue_dax(bdev->bd_queue))
+                iomap->dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
+        else
+                iomap->dax_dev = NULL;
 
         if (ret == 0) {
                 iomap->type = IOMAP_HOLE;
@@ -835,6 +841,7 @@ static int
 ext2_iomap_end(struct inode *inode, loff_t offset, loff_t length,
                 ssize_t written, unsigned flags, struct iomap *iomap)
 {
+        put_dax(iomap->dax_dev);
         if (iomap->type == IOMAP_MAPPED &&
             written < length &&
             (flags & IOMAP_WRITE))
diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
index 10b574ab354b..f0729b0705c7 100644
--- a/fs/ext4/inode.c
+++ b/fs/ext4/inode.c
@@ -3305,6 +3305,7 @@ static int ext4_releasepage(struct page *page, gfp_t wait)
 static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
                             unsigned flags, struct iomap *iomap)
 {
+        struct block_device *bdev;
         unsigned int blkbits = inode->i_blkbits;
         unsigned long first_block = offset >> blkbits;
         unsigned long last_block = (offset + length - 1) >> blkbits;
@@ -3373,7 +3374,12 @@ retry:
         }
 
         iomap->flags = 0;
-        iomap->bdev = inode->i_sb->s_bdev;
+        bdev = inode->i_sb->s_bdev;
+        iomap->bdev = bdev;
+        if (blk_queue_dax(bdev->bd_queue))
+                iomap->dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
+        else
+                iomap->dax_dev = NULL;
         iomap->offset = first_block << blkbits;
 
         if (ret == 0) {
@@ -3406,6 +3412,7 @@ static int ext4_iomap_end(struct inode *inode, loff_t offset, loff_t length,
         int blkbits = inode->i_blkbits;
         bool truncate = false;
 
+        put_dax(iomap->dax_dev);
         if (!(flags & IOMAP_WRITE) || (flags & IOMAP_FAULT))
                 return 0;
 
diff --git a/fs/iomap.c b/fs/iomap.c
index 1c25ae30500e..4add7d4ad006 100644
--- a/fs/iomap.c
+++ b/fs/iomap.c
@@ -360,7 +360,8 @@ static int iomap_dax_zero(loff_t pos, unsigned offset, unsigned bytes,
         sector_t sector = iomap->blkno +
                 (((pos & ~(PAGE_SIZE - 1)) - iomap->offset) >> 9);
 
-        return __dax_zero_page_range(iomap->bdev, sector, offset, bytes);
+        return __dax_zero_page_range(iomap->bdev, iomap->dax_dev, sector,
+                        offset, bytes);
 }
 
 static loff_t
diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c
index 288ee5b840d7..4b47403f8089 100644
--- a/fs/xfs/xfs_iomap.c
+++ b/fs/xfs/xfs_iomap.c
@@ -976,6 +976,7 @@ xfs_file_iomap_begin(
         int                     nimaps = 1, error = 0;
         bool                    shared = false, trimmed = false;
         unsigned                lockmode;
+        struct block_device     *bdev;
 
         if (XFS_FORCED_SHUTDOWN(mp))
                 return -EIO;
@@ -1063,6 +1064,14 @@ xfs_file_iomap_begin(
         }
 
         xfs_bmbt_to_iomap(ip, iomap, &imap);
+
+        /* optionally associate a dax device with the iomap bdev */
+        bdev = iomap->bdev;
+        if (blk_queue_dax(bdev->bd_queue))
+                iomap->dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
+        else
+                iomap->dax_dev = NULL;
+
         if (shared)
                 iomap->flags |= IOMAP_F_SHARED;
         return 0;
@@ -1140,6 +1149,7 @@ xfs_file_iomap_end(
         unsigned                flags,
         struct iomap            *iomap)
 {
+        put_dax(iomap->dax_dev);
         if ((flags & IOMAP_WRITE) && iomap->type == IOMAP_DELALLOC)
                 return xfs_file_iomap_end_delalloc(XFS_I(inode), offset,
                                 length, written, iomap);
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 83d28623645f..5493a66dc710 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -1923,28 +1923,12 @@ static inline bool integrity_req_gap_front_merge(struct request *req,
 
 #endif /* CONFIG_BLK_DEV_INTEGRITY */
 
-/**
- * struct blk_dax_ctl - control and output parameters for ->direct_access
- * @sector: (input) offset relative to a block_device
- * @addr: (output) kernel virtual address for @sector populated by driver
- * @pfn: (output) page frame number for @addr populated by driver
- * @size: (input) number of bytes requested
- */
-struct blk_dax_ctl {
-        sector_t sector;
-        void *addr;
-        long size;
-        pfn_t pfn;
-};
-
 struct block_device_operations {
         int (*open) (struct block_device *, fmode_t);
         void (*release) (struct gendisk *, fmode_t);
         int (*rw_page)(struct block_device *, sector_t, struct page *, bool);
         int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
         int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
-        long (*direct_access)(struct block_device *, sector_t, void **, pfn_t *,
-                        long);
         unsigned int (*check_events) (struct gendisk *disk,
                                       unsigned int clearing);
         /* ->media_changed() is DEPRECATED, use ->check_events() instead */
@@ -1963,9 +1947,8 @@ extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int,
 extern int bdev_read_page(struct block_device *, sector_t, struct page *);
 extern int bdev_write_page(struct block_device *, sector_t, struct page *,
                                                 struct writeback_control *);
-extern long bdev_direct_access(struct block_device *, struct blk_dax_ctl *);
 extern int bdev_dax_supported(struct super_block *, int);
-extern bool bdev_dax_capable(struct block_device *);
+int bdev_dax_pgoff(struct block_device *, sector_t, size_t, pgoff_t *pgoff);
 #else /* CONFIG_BLOCK */
 
 struct block_device;
diff --git a/include/linux/dax.h b/include/linux/dax.h
index d8a3dc042e1c..d3158e74a59e 100644
--- a/include/linux/dax.h
+++ b/include/linux/dax.h
@@ -7,6 +7,28 @@
 #include <asm/pgtable.h>
 
 struct iomap_ops;
+struct dax_device;
+struct dax_operations {
+        /*
+         * direct_access: translate a device-relative
+         * logical-page-offset into an absolute physical pfn. Return the
+         * number of pages available for DAX at that pfn.
+         */
+        long (*direct_access)(struct dax_device *, pgoff_t, long,
+                        void **, pfn_t *);
+};
+
+int dax_read_lock(void);
+void dax_read_unlock(int id);
+struct dax_device *dax_get_by_host(const char *host);
+struct dax_device *alloc_dax(void *private, const char *host,
+                const struct dax_operations *ops);
+void put_dax(struct dax_device *dax_dev);
+bool dax_alive(struct dax_device *dax_dev);
+void kill_dax(struct dax_device *dax_dev);
+void *dax_get_private(struct dax_device *dax_dev);
+long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
+                void **kaddr, pfn_t *pfn);
 
 /*
  * We use lowest available bit in exceptional entry for locking, one bit for
@@ -48,17 +70,13 @@ void dax_wake_mapping_entry_waiter(struct address_space *mapping,
                 pgoff_t index, void *entry, bool wake_all);
 
 #ifdef CONFIG_FS_DAX
-struct page *read_dax_sector(struct block_device *bdev, sector_t n);
-int __dax_zero_page_range(struct block_device *bdev, sector_t sector,
+int __dax_zero_page_range(struct block_device *bdev,
+                struct dax_device *dax_dev, sector_t sector,
                 unsigned int offset, unsigned int length);
 #else
-static inline struct page *read_dax_sector(struct block_device *bdev,
-                sector_t n)
-{
-        return ERR_PTR(-ENXIO);
-}
 static inline int __dax_zero_page_range(struct block_device *bdev,
-                sector_t sector, unsigned int offset, unsigned int length)
+                struct dax_device *dax_dev, sector_t sector,
+                unsigned int offset, unsigned int length)
 {
         return -ENXIO;
 }
diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h
index 78ad0624cdae..f4c639c0c362 100644
--- a/include/linux/device-mapper.h
+++ b/include/linux/device-mapper.h
@@ -130,13 +130,15 @@ typedef int (*dm_busy_fn) (struct dm_target *ti);
  *  < 0 : error
  * >= 0 : the number of bytes accessible at the address
  */
-typedef long (*dm_direct_access_fn) (struct dm_target *ti, sector_t sector,
-                                     void **kaddr, pfn_t *pfn, long size);
+typedef long (*dm_dax_direct_access_fn) (struct dm_target *ti, pgoff_t pgoff,
+                long nr_pages, void **kaddr, pfn_t *pfn);
+#define PAGE_SECTORS (PAGE_SIZE / 512)
 
 void dm_error(const char *message);
 
 struct dm_dev {
         struct block_device *bdev;
+        struct dax_device *dax_dev;
         fmode_t mode;
         char name[16];
 };
@@ -178,7 +180,7 @@ struct target_type {
         dm_busy_fn busy;
         dm_iterate_devices_fn iterate_devices;
         dm_io_hints_fn io_hints;
-        dm_direct_access_fn direct_access;
+        dm_dax_direct_access_fn direct_access;
 
         /* For internal device-mapper use. */
         struct list_head list;
diff --git a/include/linux/iomap.h b/include/linux/iomap.h
index 7291810067eb..f753e788da31 100644
--- a/include/linux/iomap.h
+++ b/include/linux/iomap.h
@@ -41,6 +41,7 @@ struct iomap {
         u16                     type;   /* type of mapping */
         u16                     flags;  /* flags for mapping */
         struct block_device     *bdev;  /* block device for I/O */
+        struct dax_device       *dax_dev; /* dax_dev for dax operations */
 };
 
 /*
diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h
index 77e7af32543f..6c807017128d 100644
--- a/include/linux/libnvdimm.h
+++ b/include/linux/libnvdimm.h
@@ -20,9 +20,11 @@
 
 enum {
         /* when a dimm supports both PMEM and BLK access a label is required */
-        NDD_ALIASING = 1 << 0,
+        NDD_ALIASING = 0,
         /* unarmed memory devices may not persist writes */
-        NDD_UNARMED = 1 << 1,
+        NDD_UNARMED = 1,
+        /* locked memory devices should not be accessed */
+        NDD_LOCKED = 2,
 
         /* need to set a limit somewhere, but yes, this is likely overkill */
         ND_IOCTL_MAX_BUFLEN = SZ_4M,
@@ -120,7 +122,7 @@ static inline struct nd_blk_region_desc *to_blk_region_desc(
 }
 
 int nvdimm_bus_add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length);
-void nvdimm_clear_from_poison_list(struct nvdimm_bus *nvdimm_bus,
+void nvdimm_forget_poison(struct nvdimm_bus *nvdimm_bus,
                 phys_addr_t start, unsigned int len);
 struct nvdimm_bus *nvdimm_bus_register(struct device *parent,
                 struct nvdimm_bus_descriptor *nfit_desc);
diff --git a/include/linux/pmem.h b/include/linux/pmem.h
index e856c2cb0fe8..71ecf3d46aac 100644
--- a/include/linux/pmem.h
+++ b/include/linux/pmem.h
@@ -31,12 +31,6 @@ static inline void arch_memcpy_to_pmem(void *dst, const void *src, size_t n)
         BUG();
 }
 
-static inline int arch_memcpy_from_pmem(void *dst, const void *src, size_t n)
-{
-        BUG();
-        return -EFAULT;
-}
-
 static inline size_t arch_copy_from_iter_pmem(void *addr, size_t bytes,
                 struct iov_iter *i)
 {
@@ -65,23 +59,6 @@ static inline bool arch_has_pmem_api(void)
         return IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API);
 }
 
-/*
- * memcpy_from_pmem - read from persistent memory with error handling
- * @dst: destination buffer
- * @src: source buffer
- * @size: transfer length
- *
- * Returns 0 on success negative error code on failure.
- */
-static inline int memcpy_from_pmem(void *dst, void const *src, size_t size)
-{
-        if (arch_has_pmem_api())
-                return arch_memcpy_from_pmem(dst, src, size);
-        else
-                memcpy(dst, src, size);
-        return 0;
-}
-
 /**
  * memcpy_to_pmem - copy data to persistent memory
  * @dst: destination buffer for the copy
diff --git a/include/linux/string.h b/include/linux/string.h
index c4011b28f3d8..537918f8a98e 100644
--- a/include/linux/string.h
+++ b/include/linux/string.h
@@ -114,6 +114,14 @@ extern int memcmp(const void *,const void *,__kernel_size_t);
 #ifndef __HAVE_ARCH_MEMCHR
 extern void * memchr(const void *,int,__kernel_size_t);
 #endif
+#ifndef __HAVE_ARCH_MEMCPY_MCSAFE
+static inline __must_check int memcpy_mcsafe(void *dst, const void *src,
+                size_t cnt)
+{
+        memcpy(dst, src, cnt);
+        return 0;
+}
+#endif
 void *memchr_inv(const void *s, int c, size_t n);
 char *strreplace(char *s, char old, char new);
 
diff --git a/include/uapi/linux/ndctl.h b/include/uapi/linux/ndctl.h
index ede5c6a62164..7ad3863cb88b 100644
--- a/include/uapi/linux/ndctl.h
+++ b/include/uapi/linux/ndctl.h
@@ -169,6 +169,7 @@ enum {
 enum {
         ND_ARS_VOLATILE = 1,
         ND_ARS_PERSISTENT = 2,
+        ND_CONFIG_LOCKED = 1,
 };
 
 static inline const char *nvdimm_bus_cmd_name(unsigned cmd)
diff --git a/tools/testing/nvdimm/Kbuild b/tools/testing/nvdimm/Kbuild
index 405212be044a..d870520da68b 100644
--- a/tools/testing/nvdimm/Kbuild
+++ b/tools/testing/nvdimm/Kbuild
@@ -28,7 +28,10 @@ obj-$(CONFIG_ND_BTT) += nd_btt.o
 obj-$(CONFIG_ND_BLK) += nd_blk.o
 obj-$(CONFIG_X86_PMEM_LEGACY) += nd_e820.o
 obj-$(CONFIG_ACPI_NFIT) += nfit.o
-obj-$(CONFIG_DEV_DAX) += dax.o
+ifeq ($(CONFIG_DAX),m)
+obj-$(CONFIG_DAX) += dax.o
+endif
+obj-$(CONFIG_DEV_DAX) += device_dax.o
 obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem.o
 
 nfit-y := $(ACPI_SRC)/core.o
@@ -48,9 +51,13 @@ nd_blk-y += config_check.o
 nd_e820-y := $(NVDIMM_SRC)/e820.o
 nd_e820-y += config_check.o
 
-dax-y := $(DAX_SRC)/dax.o
+dax-y := $(DAX_SRC)/super.o
 dax-y += config_check.o
 
+device_dax-y := $(DAX_SRC)/device.o
+device_dax-y += dax-dev.o
+device_dax-y += config_check.o
+
 dax_pmem-y := $(DAX_SRC)/pmem.o
 dax_pmem-y += config_check.o
 
diff --git a/tools/testing/nvdimm/dax-dev.c b/tools/testing/nvdimm/dax-dev.c
new file mode 100644
index 000000000000..36ee3d8797c3
--- /dev/null
+++ b/tools/testing/nvdimm/dax-dev.c
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+#include "test/nfit_test.h"
+#include <linux/mm.h>
+#include "../../../drivers/dax/dax-private.h"
+
+phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
+                unsigned long size)
+{
+        struct resource *res;
+        phys_addr_t addr;
+        int i;
+
+        for (i = 0; i < dev_dax->num_resources; i++) {
+                res = &dev_dax->res[i];
+                addr = pgoff * PAGE_SIZE + res->start;
+                if (addr >= res->start && addr <= res->end)
+                        break;
+                pgoff -= PHYS_PFN(resource_size(res));
+        }
+
+        if (i < dev_dax->num_resources) {
+                res = &dev_dax->res[i];
+                if (addr + size - 1 <= res->end) {
+                        if (get_nfit_res(addr)) {
+                                struct page *page;
+
+                                if (dev_dax->region->align > PAGE_SIZE)
+                                        return -1;
+
+                                page = vmalloc_to_page((void *)addr);
+                                return PFN_PHYS(page_to_pfn(page));
+                        } else
+                                return addr;
+                }
+        }
+
+        return -1;
+}
diff --git a/tools/testing/nvdimm/pmem-dax.c b/tools/testing/nvdimm/pmem-dax.c
index c9b8c48f85fc..b53596ad601b 100644
--- a/tools/testing/nvdimm/pmem-dax.c
+++ b/tools/testing/nvdimm/pmem-dax.c
@@ -15,13 +15,13 @@
 #include <pmem.h>
 #include <nd.h>
 
-long pmem_direct_access(struct block_device *bdev, sector_t sector,
-                void **kaddr, pfn_t *pfn, long size)
+long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff,
+                long nr_pages, void **kaddr, pfn_t *pfn)
 {
-        struct pmem_device *pmem = bdev->bd_queue->queuedata;
-        resource_size_t offset = sector * 512 + pmem->data_offset;
+        resource_size_t offset = PFN_PHYS(pgoff) + pmem->data_offset;
 
-        if (unlikely(is_bad_pmem(&pmem->bb, sector, size)))
+        if (unlikely(is_bad_pmem(&pmem->bb, PFN_PHYS(pgoff) / 512,
+                                        PFN_PHYS(nr_pages))))
                 return -EIO;
 
         /*
@@ -34,11 +34,10 @@ long pmem_direct_access(struct block_device *bdev, sector_t sector,
                 *kaddr = pmem->virt_addr + offset;
                 page = vmalloc_to_page(pmem->virt_addr + offset);
                 *pfn = page_to_pfn_t(page);
-                dev_dbg_ratelimited(disk_to_dev(bdev->bd_disk)->parent,
-                                "%s: sector: %#llx pfn: %#lx\n", __func__,
-                                (unsigned long long) sector, page_to_pfn(page));
+                pr_debug_ratelimited("%s: pmem: %p pgoff: %#lx pfn: %#lx\n",
+                                __func__, pmem, pgoff, page_to_pfn(page));
 
-                return PAGE_SIZE;
+                return 1;
         }
 
         *kaddr = pmem->virt_addr + offset;
@@ -49,6 +48,6 @@ long pmem_direct_access(struct block_device *bdev, sector_t sector,
          * requested range.
          */
         if (unlikely(pmem->bb.count))
-                return size;
-        return pmem->size - pmem->pfn_pad - offset;
+                return nr_pages;
+        return PHYS_PFN(pmem->size - pmem->pfn_pad - offset);
 }
diff --git a/tools/testing/nvdimm/test/nfit.c b/tools/testing/nvdimm/test/nfit.c
index 798f17655433..c2187178fb13 100644
--- a/tools/testing/nvdimm/test/nfit.c
+++ b/tools/testing/nvdimm/test/nfit.c
@@ -132,6 +132,7 @@ static u32 handle[] = {
         [3] = NFIT_DIMM_HANDLE(0, 0, 1, 0, 1),
         [4] = NFIT_DIMM_HANDLE(0, 1, 0, 0, 0),
         [5] = NFIT_DIMM_HANDLE(1, 0, 0, 0, 0),
+        [6] = NFIT_DIMM_HANDLE(1, 0, 0, 0, 1),
 };
 
 static unsigned long dimm_fail_cmd_flags[NUM_DCR];
@@ -728,8 +729,8 @@ static int nfit_test0_alloc(struct nfit_test *t)
 static int nfit_test1_alloc(struct nfit_test *t)
 {
         size_t nfit_size = sizeof(struct acpi_nfit_system_address) * 2
-                + sizeof(struct acpi_nfit_memory_map)
-                + offsetof(struct acpi_nfit_control_region, window_size);
+                + sizeof(struct acpi_nfit_memory_map) * 2
+                + offsetof(struct acpi_nfit_control_region, window_size) * 2;
         int i;
 
         t->nfit_buf = test_alloc(t, nfit_size, &t->nfit_dma);
@@ -906,6 +907,7 @@ static void nfit_test0_setup(struct nfit_test *t)
         memdev->address = 0;
         memdev->interleave_index = 0;
         memdev->interleave_ways = 2;
+        memdev->flags = ACPI_NFIT_MEM_HEALTH_ENABLED;
 
         /* mem-region2 (spa1, dimm0) */
         memdev = nfit_buf + offset + sizeof(struct acpi_nfit_memory_map) * 2;
@@ -921,6 +923,7 @@ static void nfit_test0_setup(struct nfit_test *t)
         memdev->address = SPA0_SIZE/2;
         memdev->interleave_index = 0;
         memdev->interleave_ways = 4;
+        memdev->flags = ACPI_NFIT_MEM_HEALTH_ENABLED;
 
         /* mem-region3 (spa1, dimm1) */
         memdev = nfit_buf + offset + sizeof(struct acpi_nfit_memory_map) * 3;
@@ -951,6 +954,7 @@ static void nfit_test0_setup(struct nfit_test *t)
         memdev->address = SPA0_SIZE/2;
         memdev->interleave_index = 0;
         memdev->interleave_ways = 4;
+        memdev->flags = ACPI_NFIT_MEM_HEALTH_ENABLED;
 
         /* mem-region5 (spa1, dimm3) */
         memdev = nfit_buf + offset + sizeof(struct acpi_nfit_memory_map) * 5;
@@ -1086,6 +1090,7 @@ static void nfit_test0_setup(struct nfit_test *t)
         memdev->address = 0;
         memdev->interleave_index = 0;
         memdev->interleave_ways = 1;
+        memdev->flags = ACPI_NFIT_MEM_HEALTH_ENABLED;
 
         offset = offset + sizeof(struct acpi_nfit_memory_map) * 14;
         /* dcr-descriptor0: blk */
@@ -1384,6 +1389,7 @@ static void nfit_test0_setup(struct nfit_test *t)
                 memdev->address = 0;
                 memdev->interleave_index = 0;
                 memdev->interleave_ways = 1;
+                memdev->flags = ACPI_NFIT_MEM_HEALTH_ENABLED;
 
                 /* mem-region16 (spa/bdw4, dimm4) */
                 memdev = nfit_buf + offset +
@@ -1486,6 +1492,34 @@ static void nfit_test1_setup(struct nfit_test *t)
         dcr->code = NFIT_FIC_BYTE;
         dcr->windows = 0;
 
+        offset += dcr->header.length;
+        memdev = nfit_buf + offset;
+        memdev->header.type = ACPI_NFIT_TYPE_MEMORY_MAP;
+        memdev->header.length = sizeof(*memdev);
+        memdev->device_handle = handle[6];
+        memdev->physical_id = 0;
+        memdev->region_id = 0;
+        memdev->range_index = 0;
+        memdev->region_index = 0+2;
+        memdev->region_size = SPA2_SIZE;
+        memdev->region_offset = 0;
+        memdev->address = 0;
+        memdev->interleave_index = 0;
+        memdev->interleave_ways = 1;
+        memdev->flags = ACPI_NFIT_MEM_MAP_FAILED;
+
+        /* dcr-descriptor1 */
+        offset += sizeof(*memdev);
+        dcr = nfit_buf + offset;
+        dcr->header.type = ACPI_NFIT_TYPE_CONTROL_REGION;
+        dcr->header.length = offsetof(struct acpi_nfit_control_region,
+                        window_size);
+        dcr->region_index = 0+2;
+        dcr_common_init(dcr);
+        dcr->serial_number = ~handle[6];
+        dcr->code = NFIT_FIC_BYTE;
+        dcr->windows = 0;
+
         post_ars_status(&t->ars_state, t->spa_set_dma[0], SPA2_SIZE);
 
         acpi_desc = &t->acpi_desc;
@@ -1817,6 +1851,10 @@ static int nfit_test_probe(struct platform_device *pdev)
         if (rc)
                 return rc;
 
+        rc = devm_add_action_or_reset(&pdev->dev, acpi_nfit_shutdown, acpi_desc);
+        if (rc)
+                return rc;
+
         if (nfit_test->setup != nfit_test0_setup)
                 return 0;
 
@@ -1907,7 +1945,7 @@ static __init int nfit_test_init(void)
                 case 1:
                         nfit_test->num_pm = 1;
                         nfit_test->dcr_idx = NUM_DCR;
-                        nfit_test->num_dcr = 1;
+                        nfit_test->num_dcr = 2;
                         nfit_test->alloc = nfit_test1_alloc;
                         nfit_test->setup = nfit_test1_setup;
                         break;
@@ -1924,6 +1962,7 @@ static __init int nfit_test_init(void)
                         put_device(&pdev->dev);
                         goto err_register;
                 }
+                get_device(&pdev->dev);
 
                 rc = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
                 if (rc)
@@ -1942,6 +1981,10 @@ static __init int nfit_test_init(void)
                 if (instances[i])
                         platform_device_unregister(&instances[i]->pdev);
         nfit_test_teardown();
+        for (i = 0; i < NUM_NFITS; i++)
+                if (instances[i])
+                        put_device(&instances[i]->pdev.dev);
+
         return rc;
 }
 
@@ -1949,10 +1992,13 @@ static __exit void nfit_test_exit(void)
 {
         int i;
 
-        platform_driver_unregister(&nfit_test_driver);
         for (i = 0; i < NUM_NFITS; i++)
                 platform_device_unregister(&instances[i]->pdev);
+        platform_driver_unregister(&nfit_test_driver);
         nfit_test_teardown();
+
+        for (i = 0; i < NUM_NFITS; i++)
+                put_device(&instances[i]->pdev.dev);
         class_destroy(nfit_test_dimm);
 }