1 files changed, 40 insertions, 22 deletions

diff --git a/mm/percpu.c b/mm/percpu.c
index bf80e55dbed7..3bb810a72006 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -116,9 +116,9 @@ static int pcpu_atom_size __read_mostly;
 static int pcpu_nr_slots __read_mostly;
 static size_t pcpu_chunk_struct_size __read_mostly;
 
-/* cpus with the lowest and highest unit numbers */
-static unsigned int pcpu_first_unit_cpu __read_mostly;
-static unsigned int pcpu_last_unit_cpu __read_mostly;
+/* cpus with the lowest and highest unit addresses */
+static unsigned int pcpu_low_unit_cpu __read_mostly;
+static unsigned int pcpu_high_unit_cpu __read_mostly;
 
 /* the address of the first chunk which starts with the kernel static area */
 void *pcpu_base_addr __read_mostly;
@@ -273,11 +273,11 @@ static void __maybe_unused pcpu_next_pop(struct pcpu_chunk *chunk,
              (rs) = (re) + 1, pcpu_next_pop((chunk), &(rs), &(re), (end)))
 
 /**
- * pcpu_mem_alloc - allocate memory
+ * pcpu_mem_zalloc - allocate memory
  * @size: bytes to allocate
  *
  * Allocate @size bytes.  If @size is smaller than PAGE_SIZE,
- * kzalloc() is used; otherwise, vmalloc() is used.  The returned
+ * kzalloc() is used; otherwise, vzalloc() is used.  The returned
  * memory is always zeroed.
  *
  * CONTEXT:
@@ -286,7 +286,7 @@ static void __maybe_unused pcpu_next_pop(struct pcpu_chunk *chunk,
  * RETURNS:
  * Pointer to the allocated area on success, NULL on failure.
  */
-static void *pcpu_mem_alloc(size_t size)
+static void *pcpu_mem_zalloc(size_t size)
 {
         if (WARN_ON_ONCE(!slab_is_available()))
                 return NULL;
@@ -302,7 +302,7 @@ static void *pcpu_mem_alloc(size_t size)
  * @ptr: memory to free
  * @size: size of the area
  *
- * Free @ptr.  @ptr should have been allocated using pcpu_mem_alloc().
+ * Free @ptr.  @ptr should have been allocated using pcpu_mem_zalloc().
  */
 static void pcpu_mem_free(void *ptr, size_t size)
 {
@@ -384,7 +384,7 @@ static int pcpu_extend_area_map(struct pcpu_chunk *chunk, int new_alloc)
         size_t old_size = 0, new_size = new_alloc * sizeof(new[0]);
         unsigned long flags;
 
-        new = pcpu_mem_alloc(new_size);
+        new = pcpu_mem_zalloc(new_size);
         if (!new)
                 return -ENOMEM;
 
@@ -604,11 +604,12 @@ static struct pcpu_chunk *pcpu_alloc_chunk(void)
 {
         struct pcpu_chunk *chunk;
 
-        chunk = pcpu_mem_alloc(pcpu_chunk_struct_size);
+        chunk = pcpu_mem_zalloc(pcpu_chunk_struct_size);
         if (!chunk)
                 return NULL;
 
-        chunk->map = pcpu_mem_alloc(PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0]));
+        chunk->map = pcpu_mem_zalloc(PCPU_DFL_MAP_ALLOC *
+                                                sizeof(chunk->map[0]));
         if (!chunk->map) {
                 kfree(chunk);
                 return NULL;
@@ -977,6 +978,17 @@ bool is_kernel_percpu_address(unsigned long addr)
  * address.  The caller is responsible for ensuring @addr stays valid
  * until this function finishes.
  *
+ * percpu allocator has special setup for the first chunk, which currently
+ * supports either embedding in linear address space or vmalloc mapping,
+ * and, from the second one, the backing allocator (currently either vm or
+ * km) provides translation.
+ *
+ * The addr can be tranlated simply without checking if it falls into the
+ * first chunk. But the current code reflects better how percpu allocator
+ * actually works, and the verification can discover both bugs in percpu
+ * allocator itself and per_cpu_ptr_to_phys() callers. So we keep current
+ * code.
+ *
  * RETURNS:
  * The physical address for @addr.
  */
@@ -984,19 +996,19 @@ phys_addr_t per_cpu_ptr_to_phys(void *addr)
 {
         void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr);
         bool in_first_chunk = false;
-        unsigned long first_start, first_end;
+        unsigned long first_low, first_high;
         unsigned int cpu;
 
         /*
-         * The following test on first_start/end isn't strictly
+         * The following test on unit_low/high isn't strictly
          * necessary but will speed up lookups of addresses which
          * aren't in the first chunk.
          */
-        first_start = pcpu_chunk_addr(pcpu_first_chunk, pcpu_first_unit_cpu, 0);
-        first_end = pcpu_chunk_addr(pcpu_first_chunk, pcpu_last_unit_cpu,
-                                    pcpu_unit_pages);
-        if ((unsigned long)addr >= first_start &&
-            (unsigned long)addr < first_end) {
+        first_low = pcpu_chunk_addr(pcpu_first_chunk, pcpu_low_unit_cpu, 0);
+        first_high = pcpu_chunk_addr(pcpu_first_chunk, pcpu_high_unit_cpu,
+                                     pcpu_unit_pages);
+        if ((unsigned long)addr >= first_low &&
+            (unsigned long)addr < first_high) {
                 for_each_possible_cpu(cpu) {
                         void *start = per_cpu_ptr(base, cpu);
 
@@ -1233,7 +1245,9 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
 
         for (cpu = 0; cpu < nr_cpu_ids; cpu++)
                 unit_map[cpu] = UINT_MAX;
-        pcpu_first_unit_cpu = NR_CPUS;
+
+        pcpu_low_unit_cpu = NR_CPUS;
+        pcpu_high_unit_cpu = NR_CPUS;
 
         for (group = 0, unit = 0; group < ai->nr_groups; group++, unit += i) {
                 const struct pcpu_group_info *gi = &ai->groups[group];
@@ -1253,9 +1267,13 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
                         unit_map[cpu] = unit + i;
                         unit_off[cpu] = gi->base_offset + i * ai->unit_size;
 
-                        if (pcpu_first_unit_cpu == NR_CPUS)
-                                pcpu_first_unit_cpu = cpu;
-                        pcpu_last_unit_cpu = cpu;
+                        /* determine low/high unit_cpu */
+                        if (pcpu_low_unit_cpu == NR_CPUS ||
+                            unit_off[cpu] < unit_off[pcpu_low_unit_cpu])
+                                pcpu_low_unit_cpu = cpu;
+                        if (pcpu_high_unit_cpu == NR_CPUS ||
+                            unit_off[cpu] > unit_off[pcpu_high_unit_cpu])
+                                pcpu_high_unit_cpu = cpu;
                 }
         }
         pcpu_nr_units = unit;
@@ -1889,7 +1907,7 @@ void __init percpu_init_late(void)
 
                 BUILD_BUG_ON(size > PAGE_SIZE);
 
-                map = pcpu_mem_alloc(size);
+                map = pcpu_mem_zalloc(size);
                 BUG_ON(!map);
 
                 spin_lock_irqsave(&pcpu_lock, flags);