mm, vmacache: hash addresses based on pmd

When perf profiling a wide variety of different workloads, it was found
that vmacache_find() had higher than expected cost: up to 0.08% of cpu
utilization in some cases.  This was found to rival other core VM
functions such as alloc_pages_vma() with thp enabled and default
mempolicy, and the conditionals in __get_vma_policy().

VMACACHE_HASH() determines which of the four per-task_struct slots a vma
is cached for a particular address.  This currently depends on the pfn,
so pfn 5212 occupies a different vmacache slot than its neighboring pfn
5213.

vmacache_find() iterates through all four of current's vmacache slots
when looking up an address.  Hashing based on pfn, an address has
~1/VMACACHE_SIZE chance of being cached in the first vmacache slot, or
about 25%, *if* the vma is cached.

This patch hashes an address by its pmd instead of pte to optimize for
workloads with good spatial locality.  This results in a higher
probability of vmas being cached in the first slot that is checked:
normally ~70% on the same workloads instead of 25%.

[rientjes@google.com: various updates]
  Link: http://lkml.kernel.org/r/alpine.DEB.2.21.1807231532290.109445@chino.kir.corp.google.com
Link: http://lkml.kernel.org/r/alpine.DEB.2.21.1807091749150.114630@chino.kir.corp.google.com
Signed-off-by: David Rientjes <rientjes@google.com>
Reviewed-by: Andrew Morton <akpm@linux-foundation.org>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 29 insertions, 9 deletions

diff --git a/mm/vmacache.c b/mm/vmacache.c
index db7596eb6132..ea517bef7dc5 100644
--- a/mm/vmacache.c
+++ b/mm/vmacache.c
@@ -6,6 +6,18 @@
 #include <linux/sched/task.h>
 #include <linux/mm.h>
 #include <linux/vmacache.h>
+#include <asm/pgtable.h>
+
+/*
+ * Hash based on the pmd of addr if configured with MMU, which provides a good
+ * hit rate for workloads with spatial locality.  Otherwise, use pages.
+ */
+#ifdef CONFIG_MMU
+#define VMACACHE_SHIFT  PMD_SHIFT
+#else
+#define VMACACHE_SHIFT  PAGE_SHIFT
+#endif
+#define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK)
 
 /*
  * Flush vma caches for threads that share a given mm.
@@ -87,6 +99,7 @@ static bool vmacache_valid(struct mm_struct *mm)
 
 struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
 {
+        int idx = VMACACHE_HASH(addr);
         int i;
 
         count_vm_vmacache_event(VMACACHE_FIND_CALLS);
@@ -95,16 +108,20 @@ struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
                 return NULL;
 
         for (i = 0; i < VMACACHE_SIZE; i++) {
-                struct vm_area_struct *vma = current->vmacache.vmas[i];
+                struct vm_area_struct *vma = current->vmacache.vmas[idx];
 
-                if (!vma)
-                        continue;
-                if (WARN_ON_ONCE(vma->vm_mm != mm))
-                        break;
-                if (vma->vm_start <= addr && vma->vm_end > addr) {
-                        count_vm_vmacache_event(VMACACHE_FIND_HITS);
-                        return vma;
+                if (vma) {
+#ifdef CONFIG_DEBUG_VM_VMACACHE
+                        if (WARN_ON_ONCE(vma->vm_mm != mm))
+                                break;
+#endif
+                        if (vma->vm_start <= addr && vma->vm_end > addr) {
+                                count_vm_vmacache_event(VMACACHE_FIND_HITS);
+                                return vma;
+                        }
                 }
+                if (++idx == VMACACHE_SIZE)
+                        idx = 0;
         }
 
         return NULL;
@@ -115,6 +132,7 @@ struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
                                            unsigned long start,
                                            unsigned long end)
 {
+        int idx = VMACACHE_HASH(start);
         int i;
 
         count_vm_vmacache_event(VMACACHE_FIND_CALLS);
@@ -123,12 +141,14 @@ struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
                 return NULL;
 
         for (i = 0; i < VMACACHE_SIZE; i++) {
-                struct vm_area_struct *vma = current->vmacache.vmas[i];
+                struct vm_area_struct *vma = current->vmacache.vmas[idx];
 
                 if (vma && vma->vm_start == start && vma->vm_end == end) {
                         count_vm_vmacache_event(VMACACHE_FIND_HITS);
                         return vma;
                 }
+                if (++idx == VMACACHE_SIZE)
+                        idx = 0;
         }
 
         return NULL;