mm/device-public-memory: device memory cache coherent with CPU

Platform with advance system bus (like CAPI or CCIX) allow device memory
to be accessible from CPU in a cache coherent fashion.  Add a new type of
ZONE_DEVICE to represent such memory.  The use case are the same as for
the un-addressable device memory but without all the corners cases.

Link: http://lkml.kernel.org/r/20170817000548.32038-19-jglisse@redhat.com
Signed-off-by: Jérôme Glisse <jglisse@redhat.com>
Cc: Aneesh Kumar <aneesh.kumar@linux.vnet.ibm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Balbir Singh <bsingharora@gmail.com>
Cc: David Nellans <dnellans@nvidia.com>
Cc: Evgeny Baskakov <ebaskakov@nvidia.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Mark Hairgrove <mhairgrove@nvidia.com>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Sherry Cheung <SCheung@nvidia.com>
Cc: Subhash Gutti <sgutti@nvidia.com>
Cc: Vladimir Davydov <vdavydov.dev@gmail.com>
Cc: Bob Liu <liubo95@huawei.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 38 insertions, 19 deletions

diff --git a/mm/migrate.c b/mm/migrate.c
index e581253ef330..618aeb5e9cde 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -36,6 +36,7 @@
 #include <linux/hugetlb.h>
 #include <linux/hugetlb_cgroup.h>
 #include <linux/gfp.h>
+#include <linux/pfn_t.h>
 #include <linux/memremap.h>
 #include <linux/userfaultfd_k.h>
 #include <linux/balloon_compaction.h>
@@ -239,10 +240,14 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
                 if (is_write_migration_entry(entry))
                         pte = maybe_mkwrite(pte, vma);
 
-                if (unlikely(is_zone_device_page(new)) &&
-                    is_device_private_page(new)) {
-                        entry = make_device_private_entry(new, pte_write(pte));
-                        pte = swp_entry_to_pte(entry);
+                if (unlikely(is_zone_device_page(new))) {
+                        if (is_device_private_page(new)) {
+                                entry = make_device_private_entry(new, pte_write(pte));
+                                pte = swp_entry_to_pte(entry);
+                        } else if (is_device_public_page(new)) {
+                                pte = pte_mkdevmap(pte);
+                                flush_dcache_page(new);
+                        }
                 } else
                         flush_dcache_page(new);
 
@@ -437,12 +442,11 @@ int migrate_page_move_mapping(struct address_space *mapping,
         void **pslot;
 
         /*
-         * ZONE_DEVICE pages have 1 refcount always held by their device
-         *
-         * Note that DAX memory will never reach that point as it does not have
-         * the MEMORY_DEVICE_ALLOW_MIGRATE flag set (see memory_hotplug.h).
+         * Device public or private pages have an extra refcount as they are
+         * ZONE_DEVICE pages.
          */
-        expected_count += is_zone_device_page(page);
+        expected_count += is_device_private_page(page);
+        expected_count += is_device_public_page(page);
 
         if (!mapping) {
                 /* Anonymous page without mapping */
@@ -2123,7 +2127,6 @@ out_unlock:
 
 #endif /* CONFIG_NUMA */
 
-
 struct migrate_vma {
         struct vm_area_struct   *vma;
         unsigned long           *dst;
@@ -2263,7 +2266,7 @@ again:
                                 pfn = 0;
                                 goto next;
                         }
-                        page = vm_normal_page(migrate->vma, addr, pte);
+                        page = _vm_normal_page(migrate->vma, addr, pte, true);
                         mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
                         mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
                 }
@@ -2406,10 +2409,19 @@ static bool migrate_vma_check_page(struct page *page)
                 if (is_device_private_page(page))
                         return true;
 
-                /* Other ZONE_DEVICE memory type are not supported */
-                return false;
+                /*
+                 * Only allow device public page to be migrated and account for
+                 * the extra reference count imply by ZONE_DEVICE pages.
+                 */
+                if (!is_device_public_page(page))
+                        return false;
+                extra++;
         }
 
+        /* For file back page */
+        if (page_mapping(page))
+                extra += 1 + page_has_private(page);
+
         if ((page_count(page) - extra) > page_mapcount(page))
                 return false;
 
@@ -2647,11 +2659,18 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate,
          */
         __SetPageUptodate(page);
 
-        if (is_zone_device_page(page) && is_device_private_page(page)) {
-                swp_entry_t swp_entry;
-
-                swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE);
-                entry = swp_entry_to_pte(swp_entry);
+        if (is_zone_device_page(page)) {
+                if (is_device_private_page(page)) {
+                        swp_entry_t swp_entry;
+
+                        swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE);
+                        entry = swp_entry_to_pte(swp_entry);
+                } else if (is_device_public_page(page)) {
+                        entry = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot)));
+                        if (vma->vm_flags & VM_WRITE)
+                                entry = pte_mkwrite(pte_mkdirty(entry));
+                        entry = pte_mkdevmap(entry);
+                }
         } else {
                 entry = mk_pte(page, vma->vm_page_prot);
                 if (vma->vm_flags & VM_WRITE)
@@ -2768,7 +2787,7 @@ static void migrate_vma_pages(struct migrate_vma *migrate)
                                         migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
                                         continue;
                                 }
-                        } else {
+                        } else if (!is_device_public_page(newpage)) {
                                 /*
                                  * Other types of ZONE_DEVICE page are not
                                  * supported.