mm: move the powerpc hugepd code to mm/gup.c

While only powerpc supports the hugepd case, the code is pretty generic
and I'd like to keep all GUP internals in one place.

Link: http://lkml.kernel.org/r/20190625143715.1689-15-hch@lst.de
Signed-off-by: Christoph Hellwig <hch@lst.de>
Cc: Andrey Konovalov <andreyknvl@google.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: David Miller <davem@davemloft.net>
Cc: James Hogan <jhogan@kernel.org>
Cc: Jason Gunthorpe <jgg@mellanox.com>
Cc: Khalid Aziz <khalid.aziz@oracle.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Paul Burton <paul.burton@mips.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Rich Felker <dalias@libc.org>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

2 files changed, 92 insertions, 0 deletions

diff --git a/mm/Kconfig b/mm/Kconfig
index 48840b28482b..0b4352557dd5 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -769,4 +769,14 @@ config GUP_GET_PTE_LOW_HIGH
 config ARCH_HAS_PTE_SPECIAL
         bool
 
+#
+# Some architectures require a special hugepage directory format that is
+# required to support multiple hugepage sizes. For example a4fe3ce76
+# "powerpc/mm: Allow more flexible layouts for hugepage pagetables"
+# introduced it on powerpc.  This allows for a more flexible hugepage
+# pagetable layouts.
+#
+config ARCH_HAS_HUGEPD
+        bool
+
 endmenu
diff --git a/mm/gup.c b/mm/gup.c
index 9d68cef2fa90..2f8bf7a71c74 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -1966,6 +1966,88 @@ static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
 }
 #endif
 
+#ifdef CONFIG_ARCH_HAS_HUGEPD
+static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
+                                      unsigned long sz)
+{
+        unsigned long __boundary = (addr + sz) & ~(sz-1);
+        return (__boundary - 1 < end - 1) ? __boundary : end;
+}
+
+static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
+                       unsigned long end, int write, struct page **pages, int *nr)
+{
+        unsigned long pte_end;
+        struct page *head, *page;
+        pte_t pte;
+        int refs;
+
+        pte_end = (addr + sz) & ~(sz-1);
+        if (pte_end < end)
+                end = pte_end;
+
+        pte = READ_ONCE(*ptep);
+
+        if (!pte_access_permitted(pte, write))
+                return 0;
+
+        /* hugepages are never "special" */
+        VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
+
+        refs = 0;
+        head = pte_page(pte);
+
+        page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
+        do {
+                VM_BUG_ON(compound_head(page) != head);
+                pages[*nr] = page;
+                (*nr)++;
+                page++;
+                refs++;
+        } while (addr += PAGE_SIZE, addr != end);
+
+        if (!page_cache_add_speculative(head, refs)) {
+                *nr -= refs;
+                return 0;
+        }
+
+        if (unlikely(pte_val(pte) != pte_val(*ptep))) {
+                /* Could be optimized better */
+                *nr -= refs;
+                while (refs--)
+                        put_page(head);
+                return 0;
+        }
+
+        return 1;
+}
+
+static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
+                unsigned int pdshift, unsigned long end, int write,
+                struct page **pages, int *nr)
+{
+        pte_t *ptep;
+        unsigned long sz = 1UL << hugepd_shift(hugepd);
+        unsigned long next;
+
+        ptep = hugepte_offset(hugepd, addr, pdshift);
+        do {
+                next = hugepte_addr_end(addr, end, sz);
+                if (!gup_hugepte(ptep, sz, addr, end, write, pages, nr))
+                        return 0;
+        } while (ptep++, addr = next, addr != end);
+
+        return 1;
+}
+#else
+static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
+                unsigned pdshift, unsigned long end, int write,
+                struct page **pages, int *nr)
+{
+        return 0;
+}
+#endif /* CONFIG_ARCH_HAS_HUGEPD */
+
 static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
                 unsigned long end, unsigned int flags, struct page **pages, int *nr)
 {