7 files changed, 199 insertions, 144 deletions

diff --git a/drivers/block/as-iosched.c b/drivers/block/as-iosched.c
index 638db06de2be..3410b4d294b9 100644
--- a/drivers/block/as-iosched.c
+++ b/drivers/block/as-iosched.c
@@ -1871,20 +1871,22 @@ static int as_init_queue(request_queue_t *q, elevator_t *e)
         if (!arq_pool)
                 return -ENOMEM;
 
-        ad = kmalloc(sizeof(*ad), GFP_KERNEL);
+        ad = kmalloc_node(sizeof(*ad), GFP_KERNEL, q->node);
         if (!ad)
                 return -ENOMEM;
         memset(ad, 0, sizeof(*ad));
 
         ad->q = q; /* Identify what queue the data belongs to */
 
-        ad->hash = kmalloc(sizeof(struct list_head)*AS_HASH_ENTRIES,GFP_KERNEL);
+        ad->hash = kmalloc_node(sizeof(struct list_head)*AS_HASH_ENTRIES,
+                                GFP_KERNEL, q->node);
         if (!ad->hash) {
                 kfree(ad);
                 return -ENOMEM;
         }
 
-        ad->arq_pool = mempool_create(BLKDEV_MIN_RQ, mempool_alloc_slab, mempool_free_slab, arq_pool);
+        ad->arq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
+                                mempool_free_slab, arq_pool, q->node);
         if (!ad->arq_pool) {
                 kfree(ad->hash);
                 kfree(ad);
diff --git a/drivers/block/deadline-iosched.c b/drivers/block/deadline-iosched.c
index 7f79f3dd0165..4bc2fea73273 100644
--- a/drivers/block/deadline-iosched.c
+++ b/drivers/block/deadline-iosched.c
@@ -711,18 +711,20 @@ static int deadline_init_queue(request_queue_t *q, elevator_t *e)
         if (!drq_pool)
                 return -ENOMEM;
 
-        dd = kmalloc(sizeof(*dd), GFP_KERNEL);
+        dd = kmalloc_node(sizeof(*dd), GFP_KERNEL, q->node);
         if (!dd)
                 return -ENOMEM;
         memset(dd, 0, sizeof(*dd));
 
-        dd->hash = kmalloc(sizeof(struct list_head)*DL_HASH_ENTRIES,GFP_KERNEL);
+        dd->hash = kmalloc_node(sizeof(struct list_head)*DL_HASH_ENTRIES,
+                                GFP_KERNEL, q->node);
         if (!dd->hash) {
                 kfree(dd);
                 return -ENOMEM;
         }
 
-        dd->drq_pool = mempool_create(BLKDEV_MIN_RQ, mempool_alloc_slab, mempool_free_slab, drq_pool);
+        dd->drq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
+                                        mempool_free_slab, drq_pool, q->node);
         if (!dd->drq_pool) {
                 kfree(dd->hash);
                 kfree(dd);
diff --git a/drivers/block/genhd.c b/drivers/block/genhd.c
index 53f7d846b747..47fd3659a061 100644
--- a/drivers/block/genhd.c
+++ b/drivers/block/genhd.c
@@ -40,7 +40,7 @@ static inline int major_to_index(int major)
 
 #ifdef CONFIG_PROC_FS
 /* get block device names in somewhat random order */
-int get_blkdev_list(char *p)
+int get_blkdev_list(char *p, int used)
 {
         struct blk_major_name *n;
         int i, len;
@@ -49,10 +49,18 @@ int get_blkdev_list(char *p)
 
         down(&block_subsys_sem);
         for (i = 0; i < ARRAY_SIZE(major_names); i++) {
-                for (n = major_names[i]; n; n = n->next)
+                for (n = major_names[i]; n; n = n->next) {
+                        /*
+                         * If the curent string plus the 5 extra characters
+                         * in the line would run us off the page, then we're done
+                         */
+                        if ((len + used + strlen(n->name) + 5) >= PAGE_SIZE)
+                                goto page_full;
                         len += sprintf(p+len, "%3d %s\n",
                                        n->major, n->name);
+                }
         }
+page_full:
         up(&block_subsys_sem);
 
         return len;
@@ -582,10 +590,16 @@ struct seq_operations diskstats_op = {
         .show   = diskstats_show
 };
 
-
 struct gendisk *alloc_disk(int minors)
 {
-        struct gendisk *disk = kmalloc(sizeof(struct gendisk), GFP_KERNEL);
+        return alloc_disk_node(minors, -1);
+}
+
+struct gendisk *alloc_disk_node(int minors, int node_id)
+{
+        struct gendisk *disk;
+
+        disk = kmalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id);
         if (disk) {
                 memset(disk, 0, sizeof(struct gendisk));
                 if (!init_disk_stats(disk)) {
@@ -594,7 +608,7 @@ struct gendisk *alloc_disk(int minors)
                 }
                 if (minors > 1) {
                         int size = (minors - 1) * sizeof(struct hd_struct *);
-                        disk->part = kmalloc(size, GFP_KERNEL);
+                        disk->part = kmalloc_node(size, GFP_KERNEL, node_id);
                         if (!disk->part) {
                                 kfree(disk);
                                 return NULL;
@@ -610,6 +624,7 @@ struct gendisk *alloc_disk(int minors)
 }
 
 EXPORT_SYMBOL(alloc_disk);
+EXPORT_SYMBOL(alloc_disk_node);
 
 struct kobject *get_disk(struct gendisk *disk)
 {
diff --git a/drivers/block/ioctl.c b/drivers/block/ioctl.c
index 6d7bcc9da9e7..6e278474f9a8 100644
--- a/drivers/block/ioctl.c
+++ b/drivers/block/ioctl.c
@@ -133,11 +133,9 @@ static int put_u64(unsigned long arg, u64 val)
         return put_user(val, (u64 __user *)arg);
 }
 
-int blkdev_ioctl(struct inode *inode, struct file *file, unsigned cmd,
-                        unsigned long arg)
+static int blkdev_locked_ioctl(struct file *file, struct block_device *bdev,
+                                unsigned cmd, unsigned long arg)
 {
-        struct block_device *bdev = inode->i_bdev;
-        struct gendisk *disk = bdev->bd_disk;
         struct backing_dev_info *bdi;
         int ret, n;
 
@@ -190,36 +188,72 @@ int blkdev_ioctl(struct inode *inode, struct file *file, unsigned cmd,
                 return put_ulong(arg, bdev->bd_inode->i_size >> 9);
         case BLKGETSIZE64:
                 return put_u64(arg, bdev->bd_inode->i_size);
+        }
+        return -ENOIOCTLCMD;
+}
+
+static int blkdev_driver_ioctl(struct inode *inode, struct file *file,
+                struct gendisk *disk, unsigned cmd, unsigned long arg)
+{
+        int ret;
+        if (disk->fops->unlocked_ioctl)
+                return disk->fops->unlocked_ioctl(file, cmd, arg);
+
+        if (disk->fops->ioctl) {
+                lock_kernel();
+                ret = disk->fops->ioctl(inode, file, cmd, arg);
+                unlock_kernel();
+                return ret;
+        }
+
+        return -ENOTTY;
+}
+
+int blkdev_ioctl(struct inode *inode, struct file *file, unsigned cmd,
+                        unsigned long arg)
+{
+        struct block_device *bdev = inode->i_bdev;
+        struct gendisk *disk = bdev->bd_disk;
+        int ret, n;
+
+        switch(cmd) {
         case BLKFLSBUF:
                 if (!capable(CAP_SYS_ADMIN))
                         return -EACCES;
-                if (disk->fops->ioctl) {
-                        ret = disk->fops->ioctl(inode, file, cmd, arg);
-                        /* -EINVAL to handle old uncorrected drivers */
-                        if (ret != -EINVAL && ret != -ENOTTY)
-                                return ret;
-                }
+
+                ret = blkdev_driver_ioctl(inode, file, disk, cmd, arg);
+                /* -EINVAL to handle old uncorrected drivers */
+                if (ret != -EINVAL && ret != -ENOTTY)
+                        return ret;
+
+                lock_kernel();
                 fsync_bdev(bdev);
                 invalidate_bdev(bdev, 0);
+                unlock_kernel();
                 return 0;
+
         case BLKROSET:
-                if (disk->fops->ioctl) {
-                        ret = disk->fops->ioctl(inode, file, cmd, arg);
-                        /* -EINVAL to handle old uncorrected drivers */
-                        if (ret != -EINVAL && ret != -ENOTTY)
-                                return ret;
-                }
+                ret = blkdev_driver_ioctl(inode, file, disk, cmd, arg);
+                /* -EINVAL to handle old uncorrected drivers */
+                if (ret != -EINVAL && ret != -ENOTTY)
+                        return ret;
                 if (!capable(CAP_SYS_ADMIN))
                         return -EACCES;
                 if (get_user(n, (int __user *)(arg)))
                         return -EFAULT;
+                lock_kernel();
                 set_device_ro(bdev, n);
+                unlock_kernel();
                 return 0;
-        default:
-                if (disk->fops->ioctl)
-                        return disk->fops->ioctl(inode, file, cmd, arg);
         }
-        return -ENOTTY;
+
+        lock_kernel();
+        ret = blkdev_locked_ioctl(file, bdev, cmd, arg);
+        unlock_kernel();
+        if (ret != -ENOIOCTLCMD)
+                return ret;
+
+        return blkdev_driver_ioctl(inode, file, disk, cmd, arg);
 }
 
 /* Most of the generic ioctls are handled in the normal fallback path.
diff --git a/drivers/block/ll_rw_blk.c b/drivers/block/ll_rw_blk.c
index 81fe3a0c1fe7..fd94ea27d594 100644
--- a/drivers/block/ll_rw_blk.c
+++ b/drivers/block/ll_rw_blk.c
@@ -28,6 +28,7 @@
 #include <linux/slab.h>
 #include <linux/swap.h>
 #include <linux/writeback.h>
+#include <linux/blkdev.h>
 
 /*
  * for max sense size
@@ -716,7 +717,7 @@ struct request *blk_queue_find_tag(request_queue_t *q, int tag)
 {
         struct blk_queue_tag *bqt = q->queue_tags;
 
-        if (unlikely(bqt == NULL || tag >= bqt->real_max_depth))
+        if (unlikely(bqt == NULL || tag >= bqt->max_depth))
                 return NULL;
 
         return bqt->tag_index[tag];
@@ -774,9 +775,9 @@ EXPORT_SYMBOL(blk_queue_free_tags);
 static int
 init_tag_map(request_queue_t *q, struct blk_queue_tag *tags, int depth)
 {
-        int bits, i;
         struct request **tag_index;
         unsigned long *tag_map;
+        int nr_ulongs;
 
         if (depth > q->nr_requests * 2) {
                 depth = q->nr_requests * 2;
@@ -788,24 +789,17 @@ init_tag_map(request_queue_t *q, struct blk_queue_tag *tags, int depth)
         if (!tag_index)
                 goto fail;
 
-        bits = (depth / BLK_TAGS_PER_LONG) + 1;
-        tag_map = kmalloc(bits * sizeof(unsigned long), GFP_ATOMIC);
+        nr_ulongs = ALIGN(depth, BITS_PER_LONG) / BITS_PER_LONG;
+        tag_map = kmalloc(nr_ulongs * sizeof(unsigned long), GFP_ATOMIC);
         if (!tag_map)
                 goto fail;
 
         memset(tag_index, 0, depth * sizeof(struct request *));
-        memset(tag_map, 0, bits * sizeof(unsigned long));
+        memset(tag_map, 0, nr_ulongs * sizeof(unsigned long));
         tags->max_depth = depth;
-        tags->real_max_depth = bits * BITS_PER_LONG;
         tags->tag_index = tag_index;
         tags->tag_map = tag_map;
 
-        /*
-         * set the upper bits if the depth isn't a multiple of the word size
-         */
-        for (i = depth; i < bits * BLK_TAGS_PER_LONG; i++)
-                __set_bit(i, tag_map);
-
         return 0;
 fail:
         kfree(tag_index);
@@ -870,32 +864,24 @@ int blk_queue_resize_tags(request_queue_t *q, int new_depth)
         struct blk_queue_tag *bqt = q->queue_tags;
         struct request **tag_index;
         unsigned long *tag_map;
-        int bits, max_depth;
+        int max_depth, nr_ulongs;
 
         if (!bqt)
                 return -ENXIO;
 
         /*
-         * don't bother sizing down
-         */
-        if (new_depth <= bqt->real_max_depth) {
-                bqt->max_depth = new_depth;
-                return 0;
-        }
-
-        /*
          * save the old state info, so we can copy it back
          */
         tag_index = bqt->tag_index;
         tag_map = bqt->tag_map;
-        max_depth = bqt->real_max_depth;
+        max_depth = bqt->max_depth;
 
         if (init_tag_map(q, bqt, new_depth))
                 return -ENOMEM;
 
         memcpy(bqt->tag_index, tag_index, max_depth * sizeof(struct request *));
-        bits = max_depth / BLK_TAGS_PER_LONG;
-        memcpy(bqt->tag_map, tag_map, bits * sizeof(unsigned long));
+        nr_ulongs = ALIGN(max_depth, BITS_PER_LONG) / BITS_PER_LONG;
+        memcpy(bqt->tag_map, tag_map, nr_ulongs * sizeof(unsigned long));
 
         kfree(tag_index);
         kfree(tag_map);
@@ -925,11 +911,16 @@ void blk_queue_end_tag(request_queue_t *q, struct request *rq)
 
         BUG_ON(tag == -1);
 
-        if (unlikely(tag >= bqt->real_max_depth))
+        if (unlikely(tag >= bqt->max_depth))
+                /*
+                 * This can happen after tag depth has been reduced.
+                 * FIXME: how about a warning or info message here?
+                 */
                 return;
 
         if (unlikely(!__test_and_clear_bit(tag, bqt->tag_map))) {
-                printk("attempt to clear non-busy tag (%d)\n", tag);
+                printk(KERN_ERR "%s: attempt to clear non-busy tag (%d)\n",
+                       __FUNCTION__, tag);
                 return;
         }
 
@@ -938,7 +929,8 @@ void blk_queue_end_tag(request_queue_t *q, struct request *rq)
         rq->tag = -1;
 
         if (unlikely(bqt->tag_index[tag] == NULL))
-                printk("tag %d is missing\n", tag);
+                printk(KERN_ERR "%s: tag %d is missing\n",
+                       __FUNCTION__, tag);
 
         bqt->tag_index[tag] = NULL;
         bqt->busy--;
@@ -967,24 +959,20 @@ EXPORT_SYMBOL(blk_queue_end_tag);
 int blk_queue_start_tag(request_queue_t *q, struct request *rq)
 {
         struct blk_queue_tag *bqt = q->queue_tags;
-        unsigned long *map = bqt->tag_map;
-        int tag = 0;
+        int tag;
 
         if (unlikely((rq->flags & REQ_QUEUED))) {
                 printk(KERN_ERR 
-                       "request %p for device [%s] already tagged %d",
-                       rq, rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->tag);
+                       "%s: request %p for device [%s] already tagged %d",
+                       __FUNCTION__, rq,
+                       rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->tag);
                 BUG();
         }
 
-        for (map = bqt->tag_map; *map == -1UL; map++) {
-                tag += BLK_TAGS_PER_LONG;
-
-                if (tag >= bqt->max_depth)
-                        return 1;
-        }
+        tag = find_first_zero_bit(bqt->tag_map, bqt->max_depth);
+        if (tag >= bqt->max_depth)
+                return 1;
 
-        tag += ffz(*map);
         __set_bit(tag, bqt->tag_map);
 
         rq->flags |= REQ_QUEUED;
@@ -1020,7 +1008,8 @@ void blk_queue_invalidate_tags(request_queue_t *q)
                 rq = list_entry_rq(tmp);
 
                 if (rq->tag == -1) {
-                        printk("bad tag found on list\n");
+                        printk(KERN_ERR
+                               "%s: bad tag found on list\n", __FUNCTION__);
                         list_del_init(&rq->queuelist);
                         rq->flags &= ~REQ_QUEUED;
                 } else
@@ -1450,7 +1439,7 @@ EXPORT_SYMBOL(blk_remove_plug);
  */
 void __generic_unplug_device(request_queue_t *q)
 {
-        if (test_bit(QUEUE_FLAG_STOPPED, &q->queue_flags))
+        if (unlikely(test_bit(QUEUE_FLAG_STOPPED, &q->queue_flags)))
                 return;
 
         if (!blk_remove_plug(q))
@@ -1645,7 +1634,8 @@ static int blk_init_free_list(request_queue_t *q)
         init_waitqueue_head(&rl->wait[WRITE]);
         init_waitqueue_head(&rl->drain);
 
-        rl->rq_pool = mempool_create(BLKDEV_MIN_RQ, mempool_alloc_slab, mempool_free_slab, request_cachep);
+        rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
+                                mempool_free_slab, request_cachep, q->node);
 
         if (!rl->rq_pool)
                 return -ENOMEM;
@@ -1657,8 +1647,15 @@ static int __make_request(request_queue_t *, struct bio *);
 
 request_queue_t *blk_alloc_queue(int gfp_mask)
 {
-        request_queue_t *q = kmem_cache_alloc(requestq_cachep, gfp_mask);
+        return blk_alloc_queue_node(gfp_mask, -1);
+}
+EXPORT_SYMBOL(blk_alloc_queue);
+
+request_queue_t *blk_alloc_queue_node(int gfp_mask, int node_id)
+{
+        request_queue_t *q;
 
+        q = kmem_cache_alloc_node(requestq_cachep, gfp_mask, node_id);
         if (!q)
                 return NULL;
 
@@ -1671,8 +1668,7 @@ request_queue_t *blk_alloc_queue(int gfp_mask)
 
         return q;
 }
-
-EXPORT_SYMBOL(blk_alloc_queue);
+EXPORT_SYMBOL(blk_alloc_queue_node);
 
 /**
  * blk_init_queue  - prepare a request queue for use with a block device
@@ -1705,13 +1701,22 @@ EXPORT_SYMBOL(blk_alloc_queue);
  *    blk_init_queue() must be paired with a blk_cleanup_queue() call
  *    when the block device is deactivated (such as at module unload).
  **/
+
 request_queue_t *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
 {
-        request_queue_t *q = blk_alloc_queue(GFP_KERNEL);
+        return blk_init_queue_node(rfn, lock, -1);
+}
+EXPORT_SYMBOL(blk_init_queue);
+
+request_queue_t *
+blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
+{
+        request_queue_t *q = blk_alloc_queue_node(GFP_KERNEL, node_id);
 
         if (!q)
                 return NULL;
 
+        q->node = node_id;
         if (blk_init_free_list(q))
                 goto out_init;
 
@@ -1754,12 +1759,11 @@ out_init:
         kmem_cache_free(requestq_cachep, q);
         return NULL;
 }
-
-EXPORT_SYMBOL(blk_init_queue);
+EXPORT_SYMBOL(blk_init_queue_node);
 
 int blk_get_queue(request_queue_t *q)
 {
-        if (!test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)) {
+        if (likely(!test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) {
                 atomic_inc(&q->refcnt);
                 return 0;
         }
@@ -1838,7 +1842,6 @@ static void __freed_request(request_queue_t *q, int rw)
                 clear_queue_congested(q, rw);
 
         if (rl->count[rw] + 1 <= q->nr_requests) {
-                smp_mb();
                 if (waitqueue_active(&rl->wait[rw]))
                         wake_up(&rl->wait[rw]);
 
@@ -1966,7 +1969,6 @@ static struct request *get_request_wait(request_queue_t *q, int rw)
         DEFINE_WAIT(wait);
         struct request *rq;
 
-        generic_unplug_device(q);
         do {
                 struct request_list *rl = &q->rq;
 
@@ -1978,6 +1980,7 @@ static struct request *get_request_wait(request_queue_t *q, int rw)
                 if (!rq) {
                         struct io_context *ioc;
 
+                        generic_unplug_device(q);
                         io_schedule();
 
                         /*
@@ -2581,7 +2584,7 @@ static int __make_request(request_queue_t *q, struct bio *bio)
         spin_lock_prefetch(q->queue_lock);
 
         barrier = bio_barrier(bio);
-        if (barrier && (q->ordered == QUEUE_ORDERED_NONE)) {
+        if (unlikely(barrier) && (q->ordered == QUEUE_ORDERED_NONE)) {
                 err = -EOPNOTSUPP;
                 goto end_io;
         }
@@ -2682,7 +2685,7 @@ get_rq:
         /*
          * REQ_BARRIER implies no merging, but lets make it explicit
          */
-        if (barrier)
+        if (unlikely(barrier))
                 req->flags |= (REQ_HARDBARRIER | REQ_NOMERGE);
 
         req->errors = 0;
@@ -2806,7 +2809,7 @@ static inline void block_wait_queue_running(request_queue_t *q)
 {
         DEFINE_WAIT(wait);
 
-        while (test_bit(QUEUE_FLAG_DRAIN, &q->queue_flags)) {
+        while (unlikely(test_bit(QUEUE_FLAG_DRAIN, &q->queue_flags))) {
                 struct request_list *rl = &q->rq;
 
                 prepare_to_wait_exclusive(&rl->drain, &wait,
@@ -2915,7 +2918,7 @@ end_io:
                         goto end_io;
                 }
 
-                if (test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))
+                if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)))
                         goto end_io;
 
                 block_wait_queue_running(q);
diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index 6f011d0d8e97..b35e08876dd4 100644
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -472,17 +472,11 @@ static int do_bio_filebacked(struct loop_device *lo, struct bio *bio)
  */
 static void loop_add_bio(struct loop_device *lo, struct bio *bio)
 {
-        unsigned long flags;
-
-        spin_lock_irqsave(&lo->lo_lock, flags);
         if (lo->lo_biotail) {
                 lo->lo_biotail->bi_next = bio;
                 lo->lo_biotail = bio;
         } else
                 lo->lo_bio = lo->lo_biotail = bio;
-        spin_unlock_irqrestore(&lo->lo_lock, flags);
-
-        up(&lo->lo_bh_mutex);
 }
 
 /*
@@ -492,14 +486,12 @@ static struct bio *loop_get_bio(struct loop_device *lo)
 {
         struct bio *bio;
 
-        spin_lock_irq(&lo->lo_lock);
         if ((bio = lo->lo_bio)) {
                 if (bio == lo->lo_biotail)
                         lo->lo_biotail = NULL;
                 lo->lo_bio = bio->bi_next;
                 bio->bi_next = NULL;
         }
-        spin_unlock_irq(&lo->lo_lock);
 
         return bio;
 }
@@ -509,35 +501,28 @@ static int loop_make_request(request_queue_t *q, struct bio *old_bio)
         struct loop_device *lo = q->queuedata;
         int rw = bio_rw(old_bio);
 
-        if (!lo)
-                goto out;
+        if (rw == READA)
+                rw = READ;
+
+        BUG_ON(!lo || (rw != READ && rw != WRITE));
 
         spin_lock_irq(&lo->lo_lock);
         if (lo->lo_state != Lo_bound)
-                goto inactive;
-        atomic_inc(&lo->lo_pending);
-        spin_unlock_irq(&lo->lo_lock);
-
-        if (rw == WRITE) {
-                if (lo->lo_flags & LO_FLAGS_READ_ONLY)
-                        goto err;
-        } else if (rw == READA) {
-                rw = READ;
-        } else if (rw != READ) {
-                printk(KERN_ERR "loop: unknown command (%x)\n", rw);
-                goto err;
-        }
+                goto out;
+        if (unlikely(rw == WRITE && (lo->lo_flags & LO_FLAGS_READ_ONLY)))
+                goto out;
+        lo->lo_pending++;
         loop_add_bio(lo, old_bio);
+        spin_unlock_irq(&lo->lo_lock);
+        up(&lo->lo_bh_mutex);
         return 0;
-err:
-        if (atomic_dec_and_test(&lo->lo_pending))
-                up(&lo->lo_bh_mutex);
+
 out:
+        if (lo->lo_pending == 0)
+                up(&lo->lo_bh_mutex);
+        spin_unlock_irq(&lo->lo_lock);
         bio_io_error(old_bio, old_bio->bi_size);
         return 0;
-inactive:
-        spin_unlock_irq(&lo->lo_lock);
-        goto out;
 }
 
 /*
@@ -560,13 +545,11 @@ static void do_loop_switch(struct loop_device *, struct switch_request *);
 
 static inline void loop_handle_bio(struct loop_device *lo, struct bio *bio)
 {
-        int ret;
-
         if (unlikely(!bio->bi_bdev)) {
                 do_loop_switch(lo, bio->bi_private);
                 bio_put(bio);
         } else {
-                ret = do_bio_filebacked(lo, bio);
+                int ret = do_bio_filebacked(lo, bio);
                 bio_endio(bio, bio->bi_size, ret);
         }
 }
@@ -594,7 +577,7 @@ static int loop_thread(void *data)
         set_user_nice(current, -20);
 
         lo->lo_state = Lo_bound;
-        atomic_inc(&lo->lo_pending);
+        lo->lo_pending = 1;
 
         /*
          * up sem, we are running
@@ -602,26 +585,37 @@ static int loop_thread(void *data)
         up(&lo->lo_sem);
 
         for (;;) {
-                down_interruptible(&lo->lo_bh_mutex);
+                int pending;
+
                 /*
-                 * could be upped because of tear-down, not because of
-                 * pending work
+                 * interruptible just to not contribute to load avg
                  */
-                if (!atomic_read(&lo->lo_pending))
+                if (down_interruptible(&lo->lo_bh_mutex))
+                        continue;
+
+                spin_lock_irq(&lo->lo_lock);
+
+                /*
+                 * could be upped because of tear-down, not pending work
+                 */
+                if (unlikely(!lo->lo_pending)) {
+                        spin_unlock_irq(&lo->lo_lock);
                         break;
+                }
 
                 bio = loop_get_bio(lo);
-                if (!bio) {
-                        printk("loop: missing bio\n");
-                        continue;
-                }
+                lo->lo_pending--;
+                pending = lo->lo_pending;
+                spin_unlock_irq(&lo->lo_lock);
+
+                BUG_ON(!bio);
                 loop_handle_bio(lo, bio);
 
                 /*
                  * upped both for pending work and tear-down, lo_pending
                  * will hit zero then
                  */
-                if (atomic_dec_and_test(&lo->lo_pending))
+                if (unlikely(!pending))
                         break;
         }
 
@@ -900,7 +894,8 @@ static int loop_clr_fd(struct loop_device *lo, struct block_device *bdev)
 
         spin_lock_irq(&lo->lo_lock);
         lo->lo_state = Lo_rundown;
-        if (atomic_dec_and_test(&lo->lo_pending))
+        lo->lo_pending--;
+        if (!lo->lo_pending)
                 up(&lo->lo_bh_mutex);
         spin_unlock_irq(&lo->lo_lock);
 
diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c
index bc56770bcc90..7f3d78de265c 100644
--- a/drivers/block/pktcdvd.c
+++ b/drivers/block/pktcdvd.c
@@ -467,14 +467,12 @@ static int pkt_set_speed(struct pktcdvd_device *pd, unsigned write_speed, unsign
  * Queue a bio for processing by the low-level CD device. Must be called
  * from process context.
  */
-static void pkt_queue_bio(struct pktcdvd_device *pd, struct bio *bio, int high_prio_read)
+static void pkt_queue_bio(struct pktcdvd_device *pd, struct bio *bio)
 {
         spin_lock(&pd->iosched.lock);
         if (bio_data_dir(bio) == READ) {
                 pkt_add_list_last(bio, &pd->iosched.read_queue,
                                   &pd->iosched.read_queue_tail);
-                if (high_prio_read)
-                        pd->iosched.high_prio_read = 1;
         } else {
                 pkt_add_list_last(bio, &pd->iosched.write_queue,
                                   &pd->iosched.write_queue_tail);
@@ -490,15 +488,16 @@ static void pkt_queue_bio(struct pktcdvd_device *pd, struct bio *bio, int high_p
  * requirements for CDRW drives:
  * - A cache flush command must be inserted before a read request if the
  *   previous request was a write.
- * - Switching between reading and writing is slow, so don't it more often
+ * - Switching between reading and writing is slow, so don't do it more often
  *   than necessary.
+ * - Optimize for throughput at the expense of latency. This means that streaming
+ *   writes will never be interrupted by a read, but if the drive has to seek
+ *   before the next write, switch to reading instead if there are any pending
+ *   read requests.
  * - Set the read speed according to current usage pattern. When only reading
  *   from the device, it's best to use the highest possible read speed, but
  *   when switching often between reading and writing, it's better to have the
  *   same read and write speeds.
- * - Reads originating from user space should have higher priority than reads
- *   originating from pkt_gather_data, because some process is usually waiting
- *   on reads of the first kind.
  */
 static void pkt_iosched_process_queue(struct pktcdvd_device *pd)
 {
@@ -512,21 +511,24 @@ static void pkt_iosched_process_queue(struct pktcdvd_device *pd)
 
         for (;;) {
                 struct bio *bio;
-                int reads_queued, writes_queued, high_prio_read;
+                int reads_queued, writes_queued;
 
                 spin_lock(&pd->iosched.lock);
                 reads_queued = (pd->iosched.read_queue != NULL);
                 writes_queued = (pd->iosched.write_queue != NULL);
-                if (!reads_queued)
-                        pd->iosched.high_prio_read = 0;
-                high_prio_read = pd->iosched.high_prio_read;
                 spin_unlock(&pd->iosched.lock);
 
                 if (!reads_queued && !writes_queued)
                         break;
 
                 if (pd->iosched.writing) {
-                        if (high_prio_read || (!writes_queued && reads_queued)) {
+                        int need_write_seek = 1;
+                        spin_lock(&pd->iosched.lock);
+                        bio = pd->iosched.write_queue;
+                        spin_unlock(&pd->iosched.lock);
+                        if (bio && (bio->bi_sector == pd->iosched.last_write))
+                                need_write_seek = 0;
+                        if (need_write_seek && reads_queued) {
                                 if (atomic_read(&pd->cdrw.pending_bios) > 0) {
                                         VPRINTK("pktcdvd: write, waiting\n");
                                         break;
@@ -559,8 +561,10 @@ static void pkt_iosched_process_queue(struct pktcdvd_device *pd)
 
                 if (bio_data_dir(bio) == READ)
                         pd->iosched.successive_reads += bio->bi_size >> 10;
-                else
+                else {
                         pd->iosched.successive_reads = 0;
+                        pd->iosched.last_write = bio->bi_sector + bio_sectors(bio);
+                }
                 if (pd->iosched.successive_reads >= HI_SPEED_SWITCH) {
                         if (pd->read_speed == pd->write_speed) {
                                 pd->read_speed = MAX_SPEED;
@@ -765,7 +769,7 @@ static void pkt_gather_data(struct pktcdvd_device *pd, struct packet_data *pkt)
 
                 atomic_inc(&pkt->io_wait);
                 bio->bi_rw = READ;
-                pkt_queue_bio(pd, bio, 0);
+                pkt_queue_bio(pd, bio);
                 frames_read++;
         }
 
@@ -1062,7 +1066,7 @@ static void pkt_start_write(struct pktcdvd_device *pd, struct packet_data *pkt)
 
         atomic_set(&pkt->io_wait, 1);
         pkt->w_bio->bi_rw = WRITE;
-        pkt_queue_bio(pd, pkt->w_bio, 0);
+        pkt_queue_bio(pd, pkt->w_bio);
 }
 
 static void pkt_finish_packet(struct packet_data *pkt, int uptodate)
@@ -2120,7 +2124,7 @@ static int pkt_make_request(request_queue_t *q, struct bio *bio)
                 cloned_bio->bi_private = psd;
                 cloned_bio->bi_end_io = pkt_end_io_read_cloned;
                 pd->stats.secs_r += bio->bi_size >> 9;
-                pkt_queue_bio(pd, cloned_bio, 1);
+                pkt_queue_bio(pd, cloned_bio);
                 return 0;
         }