page allocator: do not check NUMA node ID when the caller knows the node is valid

Callers of alloc_pages_node() can optionally specify -1 as a node to mean
"allocate from the current node".  However, a number of the callers in
fast paths know for a fact their node is valid.  To avoid a comparison and
branch, this patch adds alloc_pages_exact_node() that only checks the nid
with VM_BUG_ON().  Callers that know their node is valid are then
converted.

Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Reviewed-by: Christoph Lameter <cl@linux-foundation.org>
Reviewed-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Reviewed-by: Pekka Enberg <penberg@cs.helsinki.fi>
Acked-by: Paul Mundt <lethal@linux-sh.org>	[for the SLOB NUMA bits]
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Dave Hansen <dave@linux.vnet.ibm.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

6 files changed, 9 insertions, 8 deletions

diff --git a/arch/ia64/hp/common/sba_iommu.c b/arch/ia64/hp/common/sba_iommu.c
index 56ceb68eb99d..fe63b2dc9d07 100644
--- a/arch/ia64/hp/common/sba_iommu.c
+++ b/arch/ia64/hp/common/sba_iommu.c
@@ -1131,7 +1131,7 @@ sba_alloc_coherent (struct device *dev, size_t size, dma_addr_t *dma_handle, gfp
 #ifdef CONFIG_NUMA
         {
                 struct page *page;
-                page = alloc_pages_node(ioc->node == MAX_NUMNODES ?
+                page = alloc_pages_exact_node(ioc->node == MAX_NUMNODES ?
                                         numa_node_id() : ioc->node, flags,
                                         get_order(size));
 
diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c
index 8f33a8840422..5b17bd402275 100644
--- a/arch/ia64/kernel/mca.c
+++ b/arch/ia64/kernel/mca.c
@@ -1829,8 +1829,7 @@ ia64_mca_cpu_init(void *cpu_data)
                         data = mca_bootmem();
                         first_time = 0;
                 } else
-                        data = page_address(alloc_pages_node(numa_node_id(),
-                                        GFP_KERNEL, get_order(sz)));
+                        data = __get_free_pages(GFP_KERNEL, get_order(sz));
                 if (!data)
                         panic("Could not allocate MCA memory for cpu %d\n",
                                         cpu);
diff --git a/arch/ia64/kernel/uncached.c b/arch/ia64/kernel/uncached.c
index 8eff8c1d40a6..6ba72ab42fcc 100644
--- a/arch/ia64/kernel/uncached.c
+++ b/arch/ia64/kernel/uncached.c
@@ -98,7 +98,8 @@ static int uncached_add_chunk(struct uncached_pool *uc_pool, int nid)
 
         /* attempt to allocate a granule's worth of cached memory pages */
 
-        page = alloc_pages_node(nid, GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
+        page = alloc_pages_exact_node(nid,
+                                GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
                                 IA64_GRANULE_SHIFT-PAGE_SHIFT);
         if (!page) {
                 mutex_unlock(&uc_pool->add_chunk_mutex);
diff --git a/arch/ia64/sn/pci/pci_dma.c b/arch/ia64/sn/pci/pci_dma.c
index d876423e4e75..98b684928e12 100644
--- a/arch/ia64/sn/pci/pci_dma.c
+++ b/arch/ia64/sn/pci/pci_dma.c
@@ -90,7 +90,8 @@ static void *sn_dma_alloc_coherent(struct device *dev, size_t size,
          */
         node = pcibus_to_node(pdev->bus);
         if (likely(node >=0)) {
-                struct page *p = alloc_pages_node(node, flags, get_order(size));
+                struct page *p = alloc_pages_exact_node(node,
+                                                flags, get_order(size));
 
                 if (likely(p))
                         cpuaddr = page_address(p);
diff --git a/arch/powerpc/platforms/cell/ras.c b/arch/powerpc/platforms/cell/ras.c
index 296b5268754e..5e0a191764fc 100644
--- a/arch/powerpc/platforms/cell/ras.c
+++ b/arch/powerpc/platforms/cell/ras.c
@@ -122,8 +122,8 @@ static int __init cbe_ptcal_enable_on_node(int nid, int order)
 
         area->nid = nid;
         area->order = order;
-        area->pages = alloc_pages_node(area->nid, GFP_KERNEL | GFP_THISNODE,
-                                        area->order);
+        area->pages = alloc_pages_exact_node(area->nid, GFP_KERNEL|GFP_THISNODE,
+                                                area->order);
 
         if (!area->pages) {
                 printk(KERN_WARNING "%s: no page on node %d\n",
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 32d6ae8fb60e..e770bf349ec4 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -1277,7 +1277,7 @@ static struct vmcs *alloc_vmcs_cpu(int cpu)
         struct page *pages;
         struct vmcs *vmcs;
 
-        pages = alloc_pages_node(node, GFP_KERNEL, vmcs_config.order);
+        pages = alloc_pages_exact_node(node, GFP_KERNEL, vmcs_config.order);
         if (!pages)
                 return NULL;
         vmcs = page_address(pages);