1 files changed, 69 insertions, 21 deletions

diff --git a/mm/percpu.c b/mm/percpu.c
index 5adfc268b408..6e09741ddc62 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -46,8 +46,6 @@
  *
  * To use this allocator, arch code should do the followings.
  *
- * - drop CONFIG_HAVE_LEGACY_PER_CPU_AREA
- *
  * - define __addr_to_pcpu_ptr() and __pcpu_ptr_to_addr() to translate
  *   regular address to percpu pointer and back if they need to be
  *   different from the default
@@ -74,6 +72,7 @@
 #include <asm/cacheflush.h>
 #include <asm/sections.h>
 #include <asm/tlbflush.h>
+#include <asm/io.h>
 
 #define PCPU_SLOT_BASE_SHIFT            5       /* 1-31 shares the same slot */
 #define PCPU_DFL_MAP_ALLOC              16      /* start a map with 16 ents */
@@ -81,13 +80,15 @@
 /* default addr <-> pcpu_ptr mapping, override in asm/percpu.h if necessary */
 #ifndef __addr_to_pcpu_ptr
 #define __addr_to_pcpu_ptr(addr)                                        \
-        (void *)((unsigned long)(addr) - (unsigned long)pcpu_base_addr  \
-                 + (unsigned long)__per_cpu_start)
+        (void __percpu *)((unsigned long)(addr) -                       \
+                          (unsigned long)pcpu_base_addr +               \
+                          (unsigned long)__per_cpu_start)
 #endif
 #ifndef __pcpu_ptr_to_addr
 #define __pcpu_ptr_to_addr(ptr)                                         \
-        (void *)((unsigned long)(ptr) + (unsigned long)pcpu_base_addr   \
-                 - (unsigned long)__per_cpu_start)
+        (void __force *)((unsigned long)(ptr) +                         \
+                         (unsigned long)pcpu_base_addr -                \
+                         (unsigned long)__per_cpu_start)
 #endif
 
 struct pcpu_chunk {
@@ -914,11 +915,10 @@ static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size)
         int rs, re;
 
         /* quick path, check whether it's empty already */
-        pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
-                if (rs == page_start && re == page_end)
-                        return;
-                break;
-        }
+        rs = page_start;
+        pcpu_next_unpop(chunk, &rs, &re, page_end);
+        if (rs == page_start && re == page_end)
+                return;
 
         /* immutable chunks can't be depopulated */
         WARN_ON(chunk->immutable);
@@ -969,11 +969,10 @@ static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size)
         int rs, re, rc;
 
         /* quick path, check whether all pages are already there */
-        pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end) {
-                if (rs == page_start && re == page_end)
-                        goto clear;
-                break;
-        }
+        rs = page_start;
+        pcpu_next_pop(chunk, &rs, &re, page_end);
+        if (rs == page_start && re == page_end)
+                goto clear;
 
         /* need to allocate and map pages, this chunk can't be immutable */
         WARN_ON(chunk->immutable);
@@ -1068,7 +1067,7 @@ static struct pcpu_chunk *alloc_pcpu_chunk(void)
  * RETURNS:
  * Percpu pointer to the allocated area on success, NULL on failure.
  */
-static void *pcpu_alloc(size_t size, size_t align, bool reserved)
+static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved)
 {
         static int warn_limit = 10;
         struct pcpu_chunk *chunk;
@@ -1197,7 +1196,7 @@ fail_unlock_mutex:
  * RETURNS:
  * Percpu pointer to the allocated area on success, NULL on failure.
  */
-void *__alloc_percpu(size_t size, size_t align)
+void __percpu *__alloc_percpu(size_t size, size_t align)
 {
         return pcpu_alloc(size, align, false);
 }
@@ -1218,7 +1217,7 @@ EXPORT_SYMBOL_GPL(__alloc_percpu);
  * RETURNS:
  * Percpu pointer to the allocated area on success, NULL on failure.
  */
-void *__alloc_reserved_percpu(size_t size, size_t align)
+void __percpu *__alloc_reserved_percpu(size_t size, size_t align)
 {
         return pcpu_alloc(size, align, true);
 }
@@ -1270,9 +1269,9 @@ static void pcpu_reclaim(struct work_struct *work)
  * CONTEXT:
  * Can be called from atomic context.
  */
-void free_percpu(void *ptr)
+void free_percpu(void __percpu *ptr)
 {
-        void *addr = __pcpu_ptr_to_addr(ptr);
+        void *addr;
         struct pcpu_chunk *chunk;
         unsigned long flags;
         int off;
@@ -1280,6 +1279,8 @@ void free_percpu(void *ptr)
         if (!ptr)
                 return;
 
+        addr = __pcpu_ptr_to_addr(ptr);
+
         spin_lock_irqsave(&pcpu_lock, flags);
 
         chunk = pcpu_chunk_addr_search(addr);
@@ -1302,6 +1303,53 @@ void free_percpu(void *ptr)
 }
 EXPORT_SYMBOL_GPL(free_percpu);
 
+/**
+ * is_kernel_percpu_address - test whether address is from static percpu area
+ * @addr: address to test
+ *
+ * Test whether @addr belongs to in-kernel static percpu area.  Module
+ * static percpu areas are not considered.  For those, use
+ * is_module_percpu_address().
+ *
+ * RETURNS:
+ * %true if @addr is from in-kernel static percpu area, %false otherwise.
+ */
+bool is_kernel_percpu_address(unsigned long addr)
+{
+        const size_t static_size = __per_cpu_end - __per_cpu_start;
+        void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr);
+        unsigned int cpu;
+
+        for_each_possible_cpu(cpu) {
+                void *start = per_cpu_ptr(base, cpu);
+
+                if ((void *)addr >= start && (void *)addr < start + static_size)
+                        return true;
+        }
+        return false;
+}
+
+/**
+ * per_cpu_ptr_to_phys - convert translated percpu address to physical address
+ * @addr: the address to be converted to physical address
+ *
+ * Given @addr which is dereferenceable address obtained via one of
+ * percpu access macros, this function translates it into its physical
+ * address.  The caller is responsible for ensuring @addr stays valid
+ * until this function finishes.
+ *
+ * RETURNS:
+ * The physical address for @addr.
+ */
+phys_addr_t per_cpu_ptr_to_phys(void *addr)
+{
+        if ((unsigned long)addr < VMALLOC_START ||
+                        (unsigned long)addr >= VMALLOC_END)
+                return __pa(addr);
+        else
+                return page_to_phys(vmalloc_to_page(addr));
+}
+
 static inline size_t pcpu_calc_fc_sizes(size_t static_size,
                                         size_t reserved_size,
                                         ssize_t *dyn_sizep)