11 files changed, 77 insertions, 30 deletions

diff --git a/drivers/block/as-iosched.c b/drivers/block/as-iosched.c
index 638db06de2be..3410b4d294b9 100644
--- a/drivers/block/as-iosched.c
+++ b/drivers/block/as-iosched.c
@@ -1871,20 +1871,22 @@ static int as_init_queue(request_queue_t *q, elevator_t *e)
         if (!arq_pool)
                 return -ENOMEM;
 
-        ad = kmalloc(sizeof(*ad), GFP_KERNEL);
+        ad = kmalloc_node(sizeof(*ad), GFP_KERNEL, q->node);
         if (!ad)
                 return -ENOMEM;
         memset(ad, 0, sizeof(*ad));
 
         ad->q = q; /* Identify what queue the data belongs to */
 
-        ad->hash = kmalloc(sizeof(struct list_head)*AS_HASH_ENTRIES,GFP_KERNEL);
+        ad->hash = kmalloc_node(sizeof(struct list_head)*AS_HASH_ENTRIES,
+                                GFP_KERNEL, q->node);
         if (!ad->hash) {
                 kfree(ad);
                 return -ENOMEM;
         }
 
-        ad->arq_pool = mempool_create(BLKDEV_MIN_RQ, mempool_alloc_slab, mempool_free_slab, arq_pool);
+        ad->arq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
+                                mempool_free_slab, arq_pool, q->node);
         if (!ad->arq_pool) {
                 kfree(ad->hash);
                 kfree(ad);
diff --git a/drivers/block/deadline-iosched.c b/drivers/block/deadline-iosched.c
index 7f79f3dd0165..4bc2fea73273 100644
--- a/drivers/block/deadline-iosched.c
+++ b/drivers/block/deadline-iosched.c
@@ -711,18 +711,20 @@ static int deadline_init_queue(request_queue_t *q, elevator_t *e)
         if (!drq_pool)
                 return -ENOMEM;
 
-        dd = kmalloc(sizeof(*dd), GFP_KERNEL);
+        dd = kmalloc_node(sizeof(*dd), GFP_KERNEL, q->node);
         if (!dd)
                 return -ENOMEM;
         memset(dd, 0, sizeof(*dd));
 
-        dd->hash = kmalloc(sizeof(struct list_head)*DL_HASH_ENTRIES,GFP_KERNEL);
+        dd->hash = kmalloc_node(sizeof(struct list_head)*DL_HASH_ENTRIES,
+                                GFP_KERNEL, q->node);
         if (!dd->hash) {
                 kfree(dd);
                 return -ENOMEM;
         }
 
-        dd->drq_pool = mempool_create(BLKDEV_MIN_RQ, mempool_alloc_slab, mempool_free_slab, drq_pool);
+        dd->drq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
+                                        mempool_free_slab, drq_pool, q->node);
         if (!dd->drq_pool) {
                 kfree(dd->hash);
                 kfree(dd);
diff --git a/drivers/block/genhd.c b/drivers/block/genhd.c
index 53f7d846b747..43805e4d31e9 100644
--- a/drivers/block/genhd.c
+++ b/drivers/block/genhd.c
@@ -582,10 +582,16 @@ struct seq_operations diskstats_op = {
         .show   = diskstats_show
 };
 
-
 struct gendisk *alloc_disk(int minors)
 {
-        struct gendisk *disk = kmalloc(sizeof(struct gendisk), GFP_KERNEL);
+        return alloc_disk_node(minors, -1);
+}
+
+struct gendisk *alloc_disk_node(int minors, int node_id)
+{
+        struct gendisk *disk;
+
+        disk = kmalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id);
         if (disk) {
                 memset(disk, 0, sizeof(struct gendisk));
                 if (!init_disk_stats(disk)) {
@@ -594,7 +600,7 @@ struct gendisk *alloc_disk(int minors)
                 }
                 if (minors > 1) {
                         int size = (minors - 1) * sizeof(struct hd_struct *);
-                        disk->part = kmalloc(size, GFP_KERNEL);
+                        disk->part = kmalloc_node(size, GFP_KERNEL, node_id);
                         if (!disk->part) {
                                 kfree(disk);
                                 return NULL;
@@ -610,6 +616,7 @@ struct gendisk *alloc_disk(int minors)
 }
 
 EXPORT_SYMBOL(alloc_disk);
+EXPORT_SYMBOL(alloc_disk_node);
 
 struct kobject *get_disk(struct gendisk *disk)
 {
diff --git a/drivers/block/ll_rw_blk.c b/drivers/block/ll_rw_blk.c
index 81fe3a0c1fe7..cd8cf302068c 100644
--- a/drivers/block/ll_rw_blk.c
+++ b/drivers/block/ll_rw_blk.c
@@ -28,6 +28,7 @@
 #include <linux/slab.h>
 #include <linux/swap.h>
 #include <linux/writeback.h>
+#include <linux/blkdev.h>
 
 /*
  * for max sense size
@@ -1645,7 +1646,8 @@ static int blk_init_free_list(request_queue_t *q)
         init_waitqueue_head(&rl->wait[WRITE]);
         init_waitqueue_head(&rl->drain);
 
-        rl->rq_pool = mempool_create(BLKDEV_MIN_RQ, mempool_alloc_slab, mempool_free_slab, request_cachep);
+        rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
+                                mempool_free_slab, request_cachep, q->node);
 
         if (!rl->rq_pool)
                 return -ENOMEM;
@@ -1657,8 +1659,15 @@ static int __make_request(request_queue_t *, struct bio *);
 
 request_queue_t *blk_alloc_queue(int gfp_mask)
 {
-        request_queue_t *q = kmem_cache_alloc(requestq_cachep, gfp_mask);
+        return blk_alloc_queue_node(gfp_mask, -1);
+}
+EXPORT_SYMBOL(blk_alloc_queue);
+
+request_queue_t *blk_alloc_queue_node(int gfp_mask, int node_id)
+{
+        request_queue_t *q;
 
+        q = kmem_cache_alloc_node(requestq_cachep, gfp_mask, node_id);
         if (!q)
                 return NULL;
 
@@ -1671,8 +1680,7 @@ request_queue_t *blk_alloc_queue(int gfp_mask)
 
         return q;
 }
-
-EXPORT_SYMBOL(blk_alloc_queue);
+EXPORT_SYMBOL(blk_alloc_queue_node);
 
 /**
  * blk_init_queue  - prepare a request queue for use with a block device
@@ -1705,13 +1713,22 @@ EXPORT_SYMBOL(blk_alloc_queue);
  *    blk_init_queue() must be paired with a blk_cleanup_queue() call
  *    when the block device is deactivated (such as at module unload).
  **/
+
 request_queue_t *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
 {
-        request_queue_t *q = blk_alloc_queue(GFP_KERNEL);
+        return blk_init_queue_node(rfn, lock, -1);
+}
+EXPORT_SYMBOL(blk_init_queue);
+
+request_queue_t *
+blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
+{
+        request_queue_t *q = blk_alloc_queue_node(GFP_KERNEL, node_id);
 
         if (!q)
                 return NULL;
 
+        q->node = node_id;
         if (blk_init_free_list(q))
                 goto out_init;
 
@@ -1754,8 +1771,7 @@ out_init:
         kmem_cache_free(requestq_cachep, q);
         return NULL;
 }
-
-EXPORT_SYMBOL(blk_init_queue);
+EXPORT_SYMBOL(blk_init_queue_node);
 
 int blk_get_queue(request_queue_t *q)
 {
diff --git a/drivers/ide/ide-disk.c b/drivers/ide/ide-disk.c
index 3302cd8eab4c..d6f934886b04 100644
--- a/drivers/ide/ide-disk.c
+++ b/drivers/ide/ide-disk.c
@@ -1215,7 +1215,8 @@ static int ide_disk_probe(struct device *dev)
         if (!idkp)
                 goto failed;
 
-        g = alloc_disk(1 << PARTN_BITS);
+        g = alloc_disk_node(1 << PARTN_BITS,
+                        pcibus_to_node(drive->hwif->pci_dev->bus));
         if (!g)
                 goto out_free_idkp;
 
diff --git a/drivers/ide/ide-probe.c b/drivers/ide/ide-probe.c
index 5d876f53c697..7df85af75371 100644
--- a/drivers/ide/ide-probe.c
+++ b/drivers/ide/ide-probe.c
@@ -977,8 +977,9 @@ static int ide_init_queue(ide_drive_t *drive)
          *      limits and LBA48 we could raise it but as yet
          *      do not.
          */
-         
-        q = blk_init_queue(do_ide_request, &ide_lock);
+
+        q = blk_init_queue_node(do_ide_request, &ide_lock,
+                                pcibus_to_node(drive->hwif->pci_dev->bus));
         if (!q)
                 return 1;
 
@@ -1095,7 +1096,8 @@ static int init_irq (ide_hwif_t *hwif)
                 hwgroup->hwif->next = hwif;
                 spin_unlock_irq(&ide_lock);
         } else {
-                hwgroup = kmalloc(sizeof(ide_hwgroup_t),GFP_KERNEL);
+                hwgroup = kmalloc_node(sizeof(ide_hwgroup_t), GFP_KERNEL,
+                        pcibus_to_node(hwif->drives[0].hwif->pci_dev->bus));
                 if (!hwgroup)
                         goto out_up;
 
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 4a99b76c5a33..235c3414d268 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -396,6 +396,7 @@ struct request_queue
          */
         unsigned int            sg_timeout;
         unsigned int            sg_reserved_size;
+        int                     node;
 
         struct list_head        drain_list;
 
@@ -615,6 +616,8 @@ static inline void blkdev_dequeue_request(struct request *req)
 /*
  * Access functions for manipulating queue properties
  */
+extern request_queue_t *blk_init_queue_node(request_fn_proc *rfn,
+                                        spinlock_t *lock, int node_id);
 extern request_queue_t *blk_init_queue(request_fn_proc *, spinlock_t *);
 extern void blk_cleanup_queue(request_queue_t *);
 extern void blk_queue_make_request(request_queue_t *, make_request_fn *);
@@ -646,7 +649,8 @@ extern void blk_wait_queue_drained(request_queue_t *, int);
 extern void blk_finish_queue_drain(request_queue_t *);
 
 int blk_get_queue(request_queue_t *);
-request_queue_t *blk_alloc_queue(int);
+request_queue_t *blk_alloc_queue(int gfp_mask);
+request_queue_t *blk_alloc_queue_node(int,int);
 #define blk_put_queue(q) blk_cleanup_queue((q))
 
 /*
diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index 47dedaf971d6..af26dc718ef6 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -403,6 +403,7 @@ extern int rescan_partitions(struct gendisk *disk, struct block_device *bdev);
 extern void add_partition(struct gendisk *, int, sector_t, sector_t);
 extern void delete_partition(struct gendisk *, int);
 
+extern struct gendisk *alloc_disk_node(int minors, int node_id);
 extern struct gendisk *alloc_disk(int minors);
 extern struct kobject *get_disk(struct gendisk *disk);
 extern void put_disk(struct gendisk *disk);
diff --git a/include/linux/ide.h b/include/linux/ide.h
index 336d6e509f59..92129078d4f3 100644
--- a/include/linux/ide.h
+++ b/include/linux/ide.h
@@ -917,7 +917,7 @@ typedef struct hwif_s {
         unsigned dma;
 
         void (*led_act)(void *data, int rw);
-} ide_hwif_t;
+} ____cacheline_maxaligned_in_smp ide_hwif_t;
 
 /*
  *  internal ide interrupt handler type
diff --git a/include/linux/mempool.h b/include/linux/mempool.h
index 4a36edf1c974..796220ce47cc 100644
--- a/include/linux/mempool.h
+++ b/include/linux/mempool.h
@@ -20,9 +20,14 @@ typedef struct mempool_s {
         mempool_free_t *free;
         wait_queue_head_t wait;
 } mempool_t;
-extern mempool_t * mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
-                                 mempool_free_t *free_fn, void *pool_data);
-extern int mempool_resize(mempool_t *pool, int new_min_nr, unsigned int __nocast gfp_mask);
+
+extern mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
+                        mempool_free_t *free_fn, void *pool_data);
+extern mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
+                        mempool_free_t *free_fn, void *pool_data, int nid);
+
+extern int mempool_resize(mempool_t *pool, int new_min_nr,
+                        unsigned int __nocast gfp_mask);
 extern void mempool_destroy(mempool_t *pool);
 extern void * mempool_alloc(mempool_t *pool, unsigned int __nocast gfp_mask);
 extern void mempool_free(void *element, mempool_t *pool);
diff --git a/mm/mempool.c b/mm/mempool.c
index c9f3d4620428..920c8c3ab1b8 100644
--- a/mm/mempool.c
+++ b/mm/mempool.c
@@ -51,16 +51,23 @@ static void free_pool(mempool_t *pool)
  * functions might sleep - as long as the mempool_alloc function is not called
  * from IRQ contexts.
  */
-mempool_t * mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
+mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
                                 mempool_free_t *free_fn, void *pool_data)
 {
-        mempool_t *pool;
+        return  mempool_create_node(min_nr,alloc_fn,free_fn, pool_data,-1);
+}
+EXPORT_SYMBOL(mempool_create);
 
-        pool = kmalloc(sizeof(*pool), GFP_KERNEL);
+mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
+                        mempool_free_t *free_fn, void *pool_data, int node_id)
+{
+        mempool_t *pool;
+        pool = kmalloc_node(sizeof(*pool), GFP_KERNEL, node_id);
         if (!pool)
                 return NULL;
         memset(pool, 0, sizeof(*pool));
-        pool->elements = kmalloc(min_nr * sizeof(void *), GFP_KERNEL);
+        pool->elements = kmalloc_node(min_nr * sizeof(void *),
+                                        GFP_KERNEL, node_id);
         if (!pool->elements) {
                 kfree(pool);
                 return NULL;
@@ -87,7 +94,7 @@ mempool_t * mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
         }
         return pool;
 }
-EXPORT_SYMBOL(mempool_create);
+EXPORT_SYMBOL(mempool_create_node);
 
 /**
  * mempool_resize - resize an existing memory pool