1 files changed, 220 insertions, 114 deletions

diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
index 6a386ab4f7eb..c70604a20897 100644
--- a/drivers/md/dm-mpath.c
+++ b/drivers/md/dm-mpath.c
@@ -8,7 +8,6 @@
 #include <linux/device-mapper.h>
 
 #include "dm-path-selector.h"
-#include "dm-bio-record.h"
 #include "dm-uevent.h"
 
 #include <linux/ctype.h>
@@ -35,6 +34,7 @@ struct pgpath {
 
         struct dm_path path;
         struct work_struct deactivate_path;
+        struct work_struct activate_path;
 };
 
 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
@@ -64,8 +64,6 @@ struct multipath {
         spinlock_t lock;
 
         const char *hw_handler_name;
-        struct work_struct activate_path;
-        struct pgpath *pgpath_to_activate;
         unsigned nr_priority_groups;
         struct list_head priority_groups;
         unsigned pg_init_required;      /* pg_init needs calling? */
@@ -84,7 +82,7 @@ struct multipath {
         unsigned pg_init_count;         /* Number of times pg_init called */
 
         struct work_struct process_queued_ios;
-        struct bio_list queued_ios;
+        struct list_head queued_ios;
         unsigned queue_size;
 
         struct work_struct trigger_event;
@@ -101,7 +99,7 @@ struct multipath {
  */
 struct dm_mpath_io {
         struct pgpath *pgpath;
-        struct dm_bio_details details;
+        size_t nr_bytes;
 };
 
 typedef int (*action_fn) (struct pgpath *pgpath);
@@ -128,6 +126,7 @@ static struct pgpath *alloc_pgpath(void)
         if (pgpath) {
                 pgpath->is_active = 1;
                 INIT_WORK(&pgpath->deactivate_path, deactivate_path);
+                INIT_WORK(&pgpath->activate_path, activate_path);
         }
 
         return pgpath;
@@ -160,7 +159,6 @@ static struct priority_group *alloc_priority_group(void)
 
 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
 {
-        unsigned long flags;
         struct pgpath *pgpath, *tmp;
         struct multipath *m = ti->private;
 
@@ -169,10 +167,6 @@ static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
                 if (m->hw_handler_name)
                         scsi_dh_detach(bdev_get_queue(pgpath->path.dev->bdev));
                 dm_put_device(ti, pgpath->path.dev);
-                spin_lock_irqsave(&m->lock, flags);
-                if (m->pgpath_to_activate == pgpath)
-                        m->pgpath_to_activate = NULL;
-                spin_unlock_irqrestore(&m->lock, flags);
                 free_pgpath(pgpath);
         }
 }
@@ -198,11 +192,11 @@ static struct multipath *alloc_multipath(struct dm_target *ti)
         m = kzalloc(sizeof(*m), GFP_KERNEL);
         if (m) {
                 INIT_LIST_HEAD(&m->priority_groups);
+                INIT_LIST_HEAD(&m->queued_ios);
                 spin_lock_init(&m->lock);
                 m->queue_io = 1;
                 INIT_WORK(&m->process_queued_ios, process_queued_ios);
                 INIT_WORK(&m->trigger_event, trigger_event);
-                INIT_WORK(&m->activate_path, activate_path);
                 m->mpio_pool = mempool_create_slab_pool(MIN_IOS, _mpio_cache);
                 if (!m->mpio_pool) {
                         kfree(m);
@@ -250,11 +244,12 @@ static void __switch_pg(struct multipath *m, struct pgpath *pgpath)
         m->pg_init_count = 0;
 }
 
-static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg)
+static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg,
+                               size_t nr_bytes)
 {
         struct dm_path *path;
 
-        path = pg->ps.type->select_path(&pg->ps, &m->repeat_count);
+        path = pg->ps.type->select_path(&pg->ps, &m->repeat_count, nr_bytes);
         if (!path)
                 return -ENXIO;
 
@@ -266,7 +261,7 @@ static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg)
         return 0;
 }
 
-static void __choose_pgpath(struct multipath *m)
+static void __choose_pgpath(struct multipath *m, size_t nr_bytes)
 {
         struct priority_group *pg;
         unsigned bypassed = 1;
@@ -278,12 +273,12 @@ static void __choose_pgpath(struct multipath *m)
         if (m->next_pg) {
                 pg = m->next_pg;
                 m->next_pg = NULL;
-                if (!__choose_path_in_pg(m, pg))
+                if (!__choose_path_in_pg(m, pg, nr_bytes))
                         return;
         }
 
         /* Don't change PG until it has no remaining paths */
-        if (m->current_pg && !__choose_path_in_pg(m, m->current_pg))
+        if (m->current_pg && !__choose_path_in_pg(m, m->current_pg, nr_bytes))
                 return;
 
         /*
@@ -295,7 +290,7 @@ static void __choose_pgpath(struct multipath *m)
                 list_for_each_entry(pg, &m->priority_groups, list) {
                         if (pg->bypassed == bypassed)
                                 continue;
-                        if (!__choose_path_in_pg(m, pg))
+                        if (!__choose_path_in_pg(m, pg, nr_bytes))
                                 return;
                 }
         } while (bypassed--);
@@ -322,19 +317,21 @@ static int __must_push_back(struct multipath *m)
                 dm_noflush_suspending(m->ti));
 }
 
-static int map_io(struct multipath *m, struct bio *bio,
+static int map_io(struct multipath *m, struct request *clone,
                   struct dm_mpath_io *mpio, unsigned was_queued)
 {
         int r = DM_MAPIO_REMAPPED;
+        size_t nr_bytes = blk_rq_bytes(clone);
         unsigned long flags;
         struct pgpath *pgpath;
+        struct block_device *bdev;
 
         spin_lock_irqsave(&m->lock, flags);
 
         /* Do we need to select a new pgpath? */
         if (!m->current_pgpath ||
             (!m->queue_io && (m->repeat_count && --m->repeat_count == 0)))
-                __choose_pgpath(m);
+                __choose_pgpath(m, nr_bytes);
 
         pgpath = m->current_pgpath;
 
@@ -344,21 +341,28 @@ static int map_io(struct multipath *m, struct bio *bio,
         if ((pgpath && m->queue_io) ||
             (!pgpath && m->queue_if_no_path)) {
                 /* Queue for the daemon to resubmit */
-                bio_list_add(&m->queued_ios, bio);
+                list_add_tail(&clone->queuelist, &m->queued_ios);
                 m->queue_size++;
                 if ((m->pg_init_required && !m->pg_init_in_progress) ||
                     !m->queue_io)
                         queue_work(kmultipathd, &m->process_queued_ios);
                 pgpath = NULL;
                 r = DM_MAPIO_SUBMITTED;
-        } else if (pgpath)
-                bio->bi_bdev = pgpath->path.dev->bdev;
-        else if (__must_push_back(m))
+        } else if (pgpath) {
+                bdev = pgpath->path.dev->bdev;
+                clone->q = bdev_get_queue(bdev);
+                clone->rq_disk = bdev->bd_disk;
+        } else if (__must_push_back(m))
                 r = DM_MAPIO_REQUEUE;
         else
                 r = -EIO;       /* Failed */
 
         mpio->pgpath = pgpath;
+        mpio->nr_bytes = nr_bytes;
+
+        if (r == DM_MAPIO_REMAPPED && pgpath->pg->ps.type->start_io)
+                pgpath->pg->ps.type->start_io(&pgpath->pg->ps, &pgpath->path,
+                                              nr_bytes);
 
         spin_unlock_irqrestore(&m->lock, flags);
 
@@ -396,30 +400,31 @@ static void dispatch_queued_ios(struct multipath *m)
 {
         int r;
         unsigned long flags;
-        struct bio *bio = NULL, *next;
         struct dm_mpath_io *mpio;
         union map_info *info;
+        struct request *clone, *n;
+        LIST_HEAD(cl);
 
         spin_lock_irqsave(&m->lock, flags);
-        bio = bio_list_get(&m->queued_ios);
+        list_splice_init(&m->queued_ios, &cl);
         spin_unlock_irqrestore(&m->lock, flags);
 
-        while (bio) {
-                next = bio->bi_next;
-                bio->bi_next = NULL;
+        list_for_each_entry_safe(clone, n, &cl, queuelist) {
+                list_del_init(&clone->queuelist);
 
-                info = dm_get_mapinfo(bio);
+                info = dm_get_rq_mapinfo(clone);
                 mpio = info->ptr;
 
-                r = map_io(m, bio, mpio, 1);
-                if (r < 0)
-                        bio_endio(bio, r);
-                else if (r == DM_MAPIO_REMAPPED)
-                        generic_make_request(bio);
-                else if (r == DM_MAPIO_REQUEUE)
-                        bio_endio(bio, -EIO);
-
-                bio = next;
+                r = map_io(m, clone, mpio, 1);
+                if (r < 0) {
+                        mempool_free(mpio, m->mpio_pool);
+                        dm_kill_unmapped_request(clone, r);
+                } else if (r == DM_MAPIO_REMAPPED)
+                        dm_dispatch_request(clone);
+                else if (r == DM_MAPIO_REQUEUE) {
+                        mempool_free(mpio, m->mpio_pool);
+                        dm_requeue_unmapped_request(clone);
+                }
         }
 }
 
@@ -427,8 +432,8 @@ static void process_queued_ios(struct work_struct *work)
 {
         struct multipath *m =
                 container_of(work, struct multipath, process_queued_ios);
-        struct pgpath *pgpath = NULL;
-        unsigned init_required = 0, must_queue = 1;
+        struct pgpath *pgpath = NULL, *tmp;
+        unsigned must_queue = 1;
         unsigned long flags;
 
         spin_lock_irqsave(&m->lock, flags);
@@ -437,7 +442,7 @@ static void process_queued_ios(struct work_struct *work)
                 goto out;
 
         if (!m->current_pgpath)
-                __choose_pgpath(m);
+                __choose_pgpath(m, 0);
 
         pgpath = m->current_pgpath;
 
@@ -446,19 +451,15 @@ static void process_queued_ios(struct work_struct *work)
                 must_queue = 0;
 
         if (m->pg_init_required && !m->pg_init_in_progress && pgpath) {
-                m->pgpath_to_activate = pgpath;
                 m->pg_init_count++;
                 m->pg_init_required = 0;
-                m->pg_init_in_progress = 1;
-                init_required = 1;
+                list_for_each_entry(tmp, &pgpath->pg->pgpaths, list) {
+                        if (queue_work(kmpath_handlerd, &tmp->activate_path))
+                                m->pg_init_in_progress++;
+                }
         }
-
 out:
         spin_unlock_irqrestore(&m->lock, flags);
-
-        if (init_required)
-                queue_work(kmpath_handlerd, &m->activate_path);
-
         if (!must_queue)
                 dispatch_queued_ios(m);
 }
@@ -553,6 +554,12 @@ static int parse_path_selector(struct arg_set *as, struct priority_group *pg,
                 return -EINVAL;
         }
 
+        if (ps_argc > as->argc) {
+                dm_put_path_selector(pst);
+                ti->error = "not enough arguments for path selector";
+                return -EINVAL;
+        }
+
         r = pst->create(&pg->ps, ps_argc, as->argv);
         if (r) {
                 dm_put_path_selector(pst);
@@ -591,9 +598,20 @@ static struct pgpath *parse_path(struct arg_set *as, struct path_selector *ps,
         }
 
         if (m->hw_handler_name) {
-                r = scsi_dh_attach(bdev_get_queue(p->path.dev->bdev),
-                                   m->hw_handler_name);
+                struct request_queue *q = bdev_get_queue(p->path.dev->bdev);
+
+                r = scsi_dh_attach(q, m->hw_handler_name);
+                if (r == -EBUSY) {
+                        /*
+                         * Already attached to different hw_handler,
+                         * try to reattach with correct one.
+                         */
+                        scsi_dh_detach(q);
+                        r = scsi_dh_attach(q, m->hw_handler_name);
+                }
+
                 if (r < 0) {
+                        ti->error = "error attaching hardware handler";
                         dm_put_device(ti, p->path.dev);
                         goto bad;
                 }
@@ -699,6 +717,11 @@ static int parse_hw_handler(struct arg_set *as, struct multipath *m)
         if (!hw_argc)
                 return 0;
 
+        if (hw_argc > as->argc) {
+                ti->error = "not enough arguments for hardware handler";
+                return -EINVAL;
+        }
+
         m->hw_handler_name = kstrdup(shift(as), GFP_KERNEL);
         request_module("scsi_dh_%s", m->hw_handler_name);
         if (scsi_dh_handler_exist(m->hw_handler_name) == 0) {
@@ -823,6 +846,8 @@ static int multipath_ctr(struct dm_target *ti, unsigned int argc,
                 goto bad;
         }
 
+        ti->num_flush_requests = 1;
+
         return 0;
 
  bad:
@@ -836,25 +861,29 @@ static void multipath_dtr(struct dm_target *ti)
 
         flush_workqueue(kmpath_handlerd);
         flush_workqueue(kmultipathd);
+        flush_scheduled_work();
         free_multipath(m);
 }
 
 /*
- * Map bios, recording original fields for later in case we have to resubmit
+ * Map cloned requests
  */
-static int multipath_map(struct dm_target *ti, struct bio *bio,
+static int multipath_map(struct dm_target *ti, struct request *clone,
                          union map_info *map_context)
 {
         int r;
         struct dm_mpath_io *mpio;
         struct multipath *m = (struct multipath *) ti->private;
 
-        mpio = mempool_alloc(m->mpio_pool, GFP_NOIO);
-        dm_bio_record(&mpio->details, bio);
+        mpio = mempool_alloc(m->mpio_pool, GFP_ATOMIC);
+        if (!mpio)
+                /* ENOMEM, requeue */
+                return DM_MAPIO_REQUEUE;
+        memset(mpio, 0, sizeof(*mpio));
 
         map_context->ptr = mpio;
-        bio->bi_rw |= (1 << BIO_RW_FAILFAST_TRANSPORT);
-        r = map_io(m, bio, mpio, 0);
+        clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
+        r = map_io(m, clone, mpio, 0);
         if (r < 0 || r == DM_MAPIO_REQUEUE)
                 mempool_free(mpio, m->mpio_pool);
 
@@ -924,9 +953,13 @@ static int reinstate_path(struct pgpath *pgpath)
 
         pgpath->is_active = 1;
 
-        m->current_pgpath = NULL;
-        if (!m->nr_valid_paths++ && m->queue_size)
+        if (!m->nr_valid_paths++ && m->queue_size) {
+                m->current_pgpath = NULL;
                 queue_work(kmultipathd, &m->process_queued_ios);
+        } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
+                if (queue_work(kmpath_handlerd, &pgpath->activate_path))
+                        m->pg_init_in_progress++;
+        }
 
         dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
                       pgpath->path.dev->name, m->nr_valid_paths);
@@ -1102,87 +1135,70 @@ static void pg_init_done(struct dm_path *path, int errors)
 
         spin_lock_irqsave(&m->lock, flags);
         if (errors) {
-                DMERR("Could not failover device. Error %d.", errors);
-                m->current_pgpath = NULL;
-                m->current_pg = NULL;
+                if (pgpath == m->current_pgpath) {
+                        DMERR("Could not failover device. Error %d.", errors);
+                        m->current_pgpath = NULL;
+                        m->current_pg = NULL;
+                }
         } else if (!m->pg_init_required) {
                 m->queue_io = 0;
                 pg->bypassed = 0;
         }
 
-        m->pg_init_in_progress = 0;
-        queue_work(kmultipathd, &m->process_queued_ios);
+        m->pg_init_in_progress--;
+        if (!m->pg_init_in_progress)
+                queue_work(kmultipathd, &m->process_queued_ios);
         spin_unlock_irqrestore(&m->lock, flags);
 }
 
 static void activate_path(struct work_struct *work)
 {
         int ret;
-        struct multipath *m =
-                container_of(work, struct multipath, activate_path);
-        struct dm_path *path;
-        unsigned long flags;
+        struct pgpath *pgpath =
+                container_of(work, struct pgpath, activate_path);
 
-        spin_lock_irqsave(&m->lock, flags);
-        path = &m->pgpath_to_activate->path;
-        m->pgpath_to_activate = NULL;
-        spin_unlock_irqrestore(&m->lock, flags);
-        if (!path)
-                return;
-        ret = scsi_dh_activate(bdev_get_queue(path->dev->bdev));
-        pg_init_done(path, ret);
+        ret = scsi_dh_activate(bdev_get_queue(pgpath->path.dev->bdev));
+        pg_init_done(&pgpath->path, ret);
 }
 
 /*
  * end_io handling
  */
-static int do_end_io(struct multipath *m, struct bio *bio,
+static int do_end_io(struct multipath *m, struct request *clone,
                      int error, struct dm_mpath_io *mpio)
 {
+        /*
+         * We don't queue any clone request inside the multipath target
+         * during end I/O handling, since those clone requests don't have
+         * bio clones.  If we queue them inside the multipath target,
+         * we need to make bio clones, that requires memory allocation.
+         * (See drivers/md/dm.c:end_clone_bio() about why the clone requests
+         *  don't have bio clones.)
+         * Instead of queueing the clone request here, we queue the original
+         * request into dm core, which will remake a clone request and
+         * clone bios for it and resubmit it later.
+         */
+        int r = DM_ENDIO_REQUEUE;
         unsigned long flags;
 
-        if (!error)
+        if (!error && !clone->errors)
                 return 0;       /* I/O complete */
 
-        if ((error == -EWOULDBLOCK) && bio_rw_ahead(bio))
-                return error;
-
         if (error == -EOPNOTSUPP)
                 return error;
 
-        spin_lock_irqsave(&m->lock, flags);
-        if (!m->nr_valid_paths) {
-                if (__must_push_back(m)) {
-                        spin_unlock_irqrestore(&m->lock, flags);
-                        return DM_ENDIO_REQUEUE;
-                } else if (!m->queue_if_no_path) {
-                        spin_unlock_irqrestore(&m->lock, flags);
-                        return -EIO;
-                } else {
-                        spin_unlock_irqrestore(&m->lock, flags);
-                        goto requeue;
-                }
-        }
-        spin_unlock_irqrestore(&m->lock, flags);
-
         if (mpio->pgpath)
                 fail_path(mpio->pgpath);
 
-      requeue:
-        dm_bio_restore(&mpio->details, bio);
-
-        /* queue for the daemon to resubmit or fail */
         spin_lock_irqsave(&m->lock, flags);
-        bio_list_add(&m->queued_ios, bio);
-        m->queue_size++;
-        if (!m->queue_io)
-                queue_work(kmultipathd, &m->process_queued_ios);
+        if (!m->nr_valid_paths && !m->queue_if_no_path && !__must_push_back(m))
+                r = -EIO;
         spin_unlock_irqrestore(&m->lock, flags);
 
-        return DM_ENDIO_INCOMPLETE;     /* io not complete */
+        return r;
 }
 
-static int multipath_end_io(struct dm_target *ti, struct bio *bio,
+static int multipath_end_io(struct dm_target *ti, struct request *clone,
                             int error, union map_info *map_context)
 {
         struct multipath *m = ti->private;
@@ -1191,14 +1207,13 @@ static int multipath_end_io(struct dm_target *ti, struct bio *bio,
         struct path_selector *ps;
         int r;
 
-        r  = do_end_io(m, bio, error, mpio);
+        r  = do_end_io(m, clone, error, mpio);
         if (pgpath) {
                 ps = &pgpath->pg->ps;
                 if (ps->type->end_io)
-                        ps->type->end_io(ps, &pgpath->path);
+                        ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
         }
-        if (r != DM_ENDIO_INCOMPLETE)
-                mempool_free(mpio, m->mpio_pool);
+        mempool_free(mpio, m->mpio_pool);
 
         return r;
 }
@@ -1411,7 +1426,7 @@ static int multipath_ioctl(struct dm_target *ti, unsigned int cmd,
         spin_lock_irqsave(&m->lock, flags);
 
         if (!m->current_pgpath)
-                __choose_pgpath(m);
+                __choose_pgpath(m, 0);
 
         if (m->current_pgpath) {
                 bdev = m->current_pgpath->path.dev->bdev;
@@ -1428,22 +1443,113 @@ static int multipath_ioctl(struct dm_target *ti, unsigned int cmd,
         return r ? : __blkdev_driver_ioctl(bdev, mode, cmd, arg);
 }
 
+static int multipath_iterate_devices(struct dm_target *ti,
+                                     iterate_devices_callout_fn fn, void *data)
+{
+        struct multipath *m = ti->private;
+        struct priority_group *pg;
+        struct pgpath *p;
+        int ret = 0;
+
+        list_for_each_entry(pg, &m->priority_groups, list) {
+                list_for_each_entry(p, &pg->pgpaths, list) {
+                        ret = fn(ti, p->path.dev, ti->begin, data);
+                        if (ret)
+                                goto out;
+                }
+        }
+
+out:
+        return ret;
+}
+
+static int __pgpath_busy(struct pgpath *pgpath)
+{
+        struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
+
+        return dm_underlying_device_busy(q);
+}
+
+/*
+ * We return "busy", only when we can map I/Os but underlying devices
+ * are busy (so even if we map I/Os now, the I/Os will wait on
+ * the underlying queue).
+ * In other words, if we want to kill I/Os or queue them inside us
+ * due to map unavailability, we don't return "busy".  Otherwise,
+ * dm core won't give us the I/Os and we can't do what we want.
+ */
+static int multipath_busy(struct dm_target *ti)
+{
+        int busy = 0, has_active = 0;
+        struct multipath *m = ti->private;
+        struct priority_group *pg;
+        struct pgpath *pgpath;
+        unsigned long flags;
+
+        spin_lock_irqsave(&m->lock, flags);
+
+        /* Guess which priority_group will be used at next mapping time */
+        if (unlikely(!m->current_pgpath && m->next_pg))
+                pg = m->next_pg;
+        else if (likely(m->current_pg))
+                pg = m->current_pg;
+        else
+                /*
+                 * We don't know which pg will be used at next mapping time.
+                 * We don't call __choose_pgpath() here to avoid to trigger
+                 * pg_init just by busy checking.
+                 * So we don't know whether underlying devices we will be using
+                 * at next mapping time are busy or not. Just try mapping.
+                 */
+                goto out;
+
+        /*
+         * If there is one non-busy active path at least, the path selector
+         * will be able to select it. So we consider such a pg as not busy.
+         */
+        busy = 1;
+        list_for_each_entry(pgpath, &pg->pgpaths, list)
+                if (pgpath->is_active) {
+                        has_active = 1;
+
+                        if (!__pgpath_busy(pgpath)) {
+                                busy = 0;
+                                break;
+                        }
+                }
+
+        if (!has_active)
+                /*
+                 * No active path in this pg, so this pg won't be used and
+                 * the current_pg will be changed at next mapping time.
+                 * We need to try mapping to determine it.
+                 */
+                busy = 0;
+
+out:
+        spin_unlock_irqrestore(&m->lock, flags);
+
+        return busy;
+}
+
 /*-----------------------------------------------------------------
  * Module setup
  *---------------------------------------------------------------*/
 static struct target_type multipath_target = {
         .name = "multipath",
-        .version = {1, 0, 5},
+        .version = {1, 1, 0},
         .module = THIS_MODULE,
         .ctr = multipath_ctr,
         .dtr = multipath_dtr,
-        .map = multipath_map,
-        .end_io = multipath_end_io,
+        .map_rq = multipath_map,
+        .rq_end_io = multipath_end_io,
         .presuspend = multipath_presuspend,
         .resume = multipath_resume,
         .status = multipath_status,
         .message = multipath_message,
         .ioctl  = multipath_ioctl,
+        .iterate_devices = multipath_iterate_devices,
+        .busy = multipath_busy,
 };
 
 static int __init dm_multipath_init(void)