x86, pat: Update the page flags for memtype atomically instead of using memtype_lock

While testing an application using the xpmem (out of kernel) driver, we
noticed a significant page fault rate reduction of x86_64 with respect
to ia64.  For one test running with 32 cpus, one thread per cpu, it
took 01:08 for each of the threads to vm_insert_pfn 2GB worth of pages.
For the same test running on 256 cpus, one thread per cpu, it took 14:48
to vm_insert_pfn 2 GB worth of pages.

The slowdown was tracked to lookup_memtype which acquires the
spinlock memtype_lock.  This heavily contended lock was slowing down
vm_insert_pfn().

With the cmpxchg on page->flags method, both the 32 cpu and 256 cpu
cases take approx 00:01.3 seconds to complete.

Signed-off-by: Robin Holt <holt@sgi.com>
LKML-Reference: <20100423153627.751194346@gulag1.americas.sgi.com>
Cc: Venkatesh Pallipadi <venkatesh.pallipadi@gmail.com>
Cc: Rafael Wysocki <rjw@novell.com>
Reviewed-by: Suresh Siddha <suresh.b.siddha@intel.com>
Signed-off-by: H. Peter Anvin <hpa@zytor.com>

1 files changed, 0 insertions, 8 deletions

diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index 951011166ef5..501fc60e5e4d 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -190,8 +190,6 @@ static int pat_pagerange_is_ram(unsigned long start, unsigned long end)
  * Here we do two pass:
  * - Find the memtype of all the pages in the range, look for any conflicts
  * - In case of no conflicts, set the new memtype for pages in the range
- *
- * Caller must hold memtype_lock for atomicity.
  */
 static int reserve_ram_pages_type(u64 start, u64 end, unsigned long req_type,
                                   unsigned long *new_type)
@@ -297,9 +295,7 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
         is_range_ram = pat_pagerange_is_ram(start, end);
         if (is_range_ram == 1) {
 
-                spin_lock(&memtype_lock);
                 err = reserve_ram_pages_type(start, end, req_type, new_type);
-                spin_unlock(&memtype_lock);
 
                 return err;
         } else if (is_range_ram < 0) {
@@ -351,9 +347,7 @@ int free_memtype(u64 start, u64 end)
         is_range_ram = pat_pagerange_is_ram(start, end);
         if (is_range_ram == 1) {
 
-                spin_lock(&memtype_lock);
                 err = free_ram_pages_type(start, end);
-                spin_unlock(&memtype_lock);
 
                 return err;
         } else if (is_range_ram < 0) {
@@ -394,10 +388,8 @@ static unsigned long lookup_memtype(u64 paddr)
 
         if (pat_pagerange_is_ram(paddr, paddr + PAGE_SIZE)) {
                 struct page *page;
-                spin_lock(&memtype_lock);
                 page = pfn_to_page(paddr >> PAGE_SHIFT);
                 rettype = get_page_memtype(page);
-                spin_unlock(&memtype_lock);
                 /*
                  * -1 from get_page_memtype() implies RAM page is in its
                  * default state and not reserved, and hence of type WB