diff options
| -rw-r--r-- | drivers/md/dm.c | 206 |
1 files changed, 22 insertions, 184 deletions
diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 32e6622767ad..65114e4d9f65 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c | |||
| @@ -149,20 +149,9 @@ struct mapped_device { | |||
| 149 | int flush_error; | 149 | int flush_error; |
| 150 | 150 | ||
| 151 | /* | 151 | /* |
| 152 | * Protect barrier_error from concurrent endio processing | 152 | * Processing queue (flush) |
| 153 | * in request-based dm. | ||
| 154 | */ | ||
| 155 | spinlock_t barrier_error_lock; | ||
| 156 | int barrier_error; | ||
| 157 | |||
| 158 | /* | ||
| 159 | * Processing queue (flush/barriers) | ||
| 160 | */ | 153 | */ |
| 161 | struct workqueue_struct *wq; | 154 | struct workqueue_struct *wq; |
| 162 | struct work_struct barrier_work; | ||
| 163 | |||
| 164 | /* A pointer to the currently processing pre/post flush request */ | ||
| 165 | struct request *flush_request; | ||
| 166 | 155 | ||
| 167 | /* | 156 | /* |
| 168 | * The current mapping. | 157 | * The current mapping. |
| @@ -750,23 +739,6 @@ static void end_clone_bio(struct bio *clone, int error) | |||
| 750 | blk_update_request(tio->orig, 0, nr_bytes); | 739 | blk_update_request(tio->orig, 0, nr_bytes); |
| 751 | } | 740 | } |
| 752 | 741 | ||
| 753 | static void store_barrier_error(struct mapped_device *md, int error) | ||
| 754 | { | ||
| 755 | unsigned long flags; | ||
| 756 | |||
| 757 | spin_lock_irqsave(&md->barrier_error_lock, flags); | ||
| 758 | /* | ||
| 759 | * Basically, the first error is taken, but: | ||
| 760 | * -EOPNOTSUPP supersedes any I/O error. | ||
| 761 | * Requeue request supersedes any I/O error but -EOPNOTSUPP. | ||
| 762 | */ | ||
| 763 | if (!md->barrier_error || error == -EOPNOTSUPP || | ||
| 764 | (md->barrier_error != -EOPNOTSUPP && | ||
| 765 | error == DM_ENDIO_REQUEUE)) | ||
| 766 | md->barrier_error = error; | ||
| 767 | spin_unlock_irqrestore(&md->barrier_error_lock, flags); | ||
| 768 | } | ||
| 769 | |||
| 770 | /* | 742 | /* |
| 771 | * Don't touch any member of the md after calling this function because | 743 | * Don't touch any member of the md after calling this function because |
| 772 | * the md may be freed in dm_put() at the end of this function. | 744 | * the md may be freed in dm_put() at the end of this function. |
| @@ -804,13 +776,11 @@ static void free_rq_clone(struct request *clone) | |||
| 804 | static void dm_end_request(struct request *clone, int error) | 776 | static void dm_end_request(struct request *clone, int error) |
| 805 | { | 777 | { |
| 806 | int rw = rq_data_dir(clone); | 778 | int rw = rq_data_dir(clone); |
| 807 | int run_queue = 1; | ||
| 808 | bool is_barrier = clone->cmd_flags & REQ_HARDBARRIER; | ||
| 809 | struct dm_rq_target_io *tio = clone->end_io_data; | 779 | struct dm_rq_target_io *tio = clone->end_io_data; |
| 810 | struct mapped_device *md = tio->md; | 780 | struct mapped_device *md = tio->md; |
| 811 | struct request *rq = tio->orig; | 781 | struct request *rq = tio->orig; |
| 812 | 782 | ||
| 813 | if (rq->cmd_type == REQ_TYPE_BLOCK_PC && !is_barrier) { | 783 | if (rq->cmd_type == REQ_TYPE_BLOCK_PC) { |
| 814 | rq->errors = clone->errors; | 784 | rq->errors = clone->errors; |
| 815 | rq->resid_len = clone->resid_len; | 785 | rq->resid_len = clone->resid_len; |
| 816 | 786 | ||
| @@ -824,15 +794,8 @@ static void dm_end_request(struct request *clone, int error) | |||
| 824 | } | 794 | } |
| 825 | 795 | ||
| 826 | free_rq_clone(clone); | 796 | free_rq_clone(clone); |
| 827 | 797 | blk_end_request_all(rq, error); | |
| 828 | if (unlikely(is_barrier)) { | 798 | rq_completed(md, rw, true); |
| 829 | if (unlikely(error)) | ||
| 830 | store_barrier_error(md, error); | ||
| 831 | run_queue = 0; | ||
| 832 | } else | ||
| 833 | blk_end_request_all(rq, error); | ||
| 834 | |||
| 835 | rq_completed(md, rw, run_queue); | ||
| 836 | } | 799 | } |
| 837 | 800 | ||
| 838 | static void dm_unprep_request(struct request *rq) | 801 | static void dm_unprep_request(struct request *rq) |
| @@ -857,16 +820,6 @@ void dm_requeue_unmapped_request(struct request *clone) | |||
| 857 | struct request_queue *q = rq->q; | 820 | struct request_queue *q = rq->q; |
| 858 | unsigned long flags; | 821 | unsigned long flags; |
| 859 | 822 | ||
| 860 | if (unlikely(clone->cmd_flags & REQ_HARDBARRIER)) { | ||
| 861 | /* | ||
| 862 | * Barrier clones share an original request. | ||
| 863 | * Leave it to dm_end_request(), which handles this special | ||
| 864 | * case. | ||
| 865 | */ | ||
| 866 | dm_end_request(clone, DM_ENDIO_REQUEUE); | ||
| 867 | return; | ||
| 868 | } | ||
| 869 | |||
| 870 | dm_unprep_request(rq); | 823 | dm_unprep_request(rq); |
| 871 | 824 | ||
| 872 | spin_lock_irqsave(q->queue_lock, flags); | 825 | spin_lock_irqsave(q->queue_lock, flags); |
| @@ -956,19 +909,6 @@ static void dm_complete_request(struct request *clone, int error) | |||
| 956 | struct dm_rq_target_io *tio = clone->end_io_data; | 909 | struct dm_rq_target_io *tio = clone->end_io_data; |
| 957 | struct request *rq = tio->orig; | 910 | struct request *rq = tio->orig; |
| 958 | 911 | ||
| 959 | if (unlikely(clone->cmd_flags & REQ_HARDBARRIER)) { | ||
| 960 | /* | ||
| 961 | * Barrier clones share an original request. So can't use | ||
| 962 | * softirq_done with the original. | ||
| 963 | * Pass the clone to dm_done() directly in this special case. | ||
| 964 | * It is safe (even if clone->q->queue_lock is held here) | ||
| 965 | * because there is no I/O dispatching during the completion | ||
| 966 | * of barrier clone. | ||
| 967 | */ | ||
| 968 | dm_done(clone, error, true); | ||
| 969 | return; | ||
| 970 | } | ||
| 971 | |||
| 972 | tio->error = error; | 912 | tio->error = error; |
| 973 | rq->completion_data = clone; | 913 | rq->completion_data = clone; |
| 974 | blk_complete_request(rq); | 914 | blk_complete_request(rq); |
| @@ -985,17 +925,6 @@ void dm_kill_unmapped_request(struct request *clone, int error) | |||
| 985 | struct dm_rq_target_io *tio = clone->end_io_data; | 925 | struct dm_rq_target_io *tio = clone->end_io_data; |
| 986 | struct request *rq = tio->orig; | 926 | struct request *rq = tio->orig; |
| 987 | 927 | ||
| 988 | if (unlikely(clone->cmd_flags & REQ_HARDBARRIER)) { | ||
| 989 | /* | ||
| 990 | * Barrier clones share an original request. | ||
| 991 | * Leave it to dm_end_request(), which handles this special | ||
| 992 | * case. | ||
| 993 | */ | ||
| 994 | BUG_ON(error > 0); | ||
| 995 | dm_end_request(clone, error); | ||
| 996 | return; | ||
| 997 | } | ||
| 998 | |||
| 999 | rq->cmd_flags |= REQ_FAILED; | 928 | rq->cmd_flags |= REQ_FAILED; |
| 1000 | dm_complete_request(clone, error); | 929 | dm_complete_request(clone, error); |
| 1001 | } | 930 | } |
| @@ -1536,14 +1465,6 @@ static int dm_request(struct request_queue *q, struct bio *bio) | |||
| 1536 | return _dm_request(q, bio); | 1465 | return _dm_request(q, bio); |
| 1537 | } | 1466 | } |
| 1538 | 1467 | ||
| 1539 | static bool dm_rq_is_flush_request(struct request *rq) | ||
| 1540 | { | ||
| 1541 | if (rq->cmd_flags & REQ_FLUSH) | ||
| 1542 | return true; | ||
| 1543 | else | ||
| 1544 | return false; | ||
| 1545 | } | ||
| 1546 | |||
| 1547 | void dm_dispatch_request(struct request *rq) | 1468 | void dm_dispatch_request(struct request *rq) |
| 1548 | { | 1469 | { |
| 1549 | int r; | 1470 | int r; |
| @@ -1591,22 +1512,15 @@ static int setup_clone(struct request *clone, struct request *rq, | |||
| 1591 | { | 1512 | { |
| 1592 | int r; | 1513 | int r; |
| 1593 | 1514 | ||
| 1594 | if (dm_rq_is_flush_request(rq)) { | 1515 | r = blk_rq_prep_clone(clone, rq, tio->md->bs, GFP_ATOMIC, |
| 1595 | blk_rq_init(NULL, clone); | 1516 | dm_rq_bio_constructor, tio); |
| 1596 | clone->cmd_type = REQ_TYPE_FS; | 1517 | if (r) |
| 1597 | clone->cmd_flags |= (REQ_HARDBARRIER | WRITE); | 1518 | return r; |
| 1598 | } else { | ||
| 1599 | r = blk_rq_prep_clone(clone, rq, tio->md->bs, GFP_ATOMIC, | ||
| 1600 | dm_rq_bio_constructor, tio); | ||
| 1601 | if (r) | ||
| 1602 | return r; | ||
| 1603 | |||
| 1604 | clone->cmd = rq->cmd; | ||
| 1605 | clone->cmd_len = rq->cmd_len; | ||
| 1606 | clone->sense = rq->sense; | ||
| 1607 | clone->buffer = rq->buffer; | ||
| 1608 | } | ||
| 1609 | 1519 | ||
| 1520 | clone->cmd = rq->cmd; | ||
| 1521 | clone->cmd_len = rq->cmd_len; | ||
| 1522 | clone->sense = rq->sense; | ||
| 1523 | clone->buffer = rq->buffer; | ||
| 1610 | clone->end_io = end_clone_request; | 1524 | clone->end_io = end_clone_request; |
| 1611 | clone->end_io_data = tio; | 1525 | clone->end_io_data = tio; |
| 1612 | 1526 | ||
| @@ -1647,9 +1561,6 @@ static int dm_prep_fn(struct request_queue *q, struct request *rq) | |||
| 1647 | struct mapped_device *md = q->queuedata; | 1561 | struct mapped_device *md = q->queuedata; |
| 1648 | struct request *clone; | 1562 | struct request *clone; |
| 1649 | 1563 | ||
| 1650 | if (unlikely(dm_rq_is_flush_request(rq))) | ||
| 1651 | return BLKPREP_OK; | ||
| 1652 | |||
| 1653 | if (unlikely(rq->special)) { | 1564 | if (unlikely(rq->special)) { |
| 1654 | DMWARN("Already has something in rq->special."); | 1565 | DMWARN("Already has something in rq->special."); |
| 1655 | return BLKPREP_KILL; | 1566 | return BLKPREP_KILL; |
| @@ -1726,6 +1637,7 @@ static void dm_request_fn(struct request_queue *q) | |||
| 1726 | struct dm_table *map = dm_get_live_table(md); | 1637 | struct dm_table *map = dm_get_live_table(md); |
| 1727 | struct dm_target *ti; | 1638 | struct dm_target *ti; |
| 1728 | struct request *rq, *clone; | 1639 | struct request *rq, *clone; |
| 1640 | sector_t pos; | ||
| 1729 | 1641 | ||
| 1730 | /* | 1642 | /* |
| 1731 | * For suspend, check blk_queue_stopped() and increment | 1643 | * For suspend, check blk_queue_stopped() and increment |
| @@ -1738,15 +1650,14 @@ static void dm_request_fn(struct request_queue *q) | |||
| 1738 | if (!rq) | 1650 | if (!rq) |
| 1739 | goto plug_and_out; | 1651 | goto plug_and_out; |
| 1740 | 1652 | ||
| 1741 | if (unlikely(dm_rq_is_flush_request(rq))) { | 1653 | /* always use block 0 to find the target for flushes for now */ |
| 1742 | BUG_ON(md->flush_request); | 1654 | pos = 0; |
| 1743 | md->flush_request = rq; | 1655 | if (!(rq->cmd_flags & REQ_FLUSH)) |
| 1744 | blk_start_request(rq); | 1656 | pos = blk_rq_pos(rq); |
| 1745 | queue_work(md->wq, &md->barrier_work); | 1657 | |
| 1746 | goto out; | 1658 | ti = dm_table_find_target(map, pos); |
| 1747 | } | 1659 | BUG_ON(!dm_target_is_valid(ti)); |
| 1748 | 1660 | ||
| 1749 | ti = dm_table_find_target(map, blk_rq_pos(rq)); | ||
| 1750 | if (ti->type->busy && ti->type->busy(ti)) | 1661 | if (ti->type->busy && ti->type->busy(ti)) |
| 1751 | goto plug_and_out; | 1662 | goto plug_and_out; |
| 1752 | 1663 | ||
| @@ -1917,7 +1828,6 @@ out: | |||
| 1917 | static const struct block_device_operations dm_blk_dops; | 1828 | static const struct block_device_operations dm_blk_dops; |
| 1918 | 1829 | ||
| 1919 | static void dm_wq_work(struct work_struct *work); | 1830 | static void dm_wq_work(struct work_struct *work); |
| 1920 | static void dm_rq_barrier_work(struct work_struct *work); | ||
| 1921 | 1831 | ||
| 1922 | static void dm_init_md_queue(struct mapped_device *md) | 1832 | static void dm_init_md_queue(struct mapped_device *md) |
| 1923 | { | 1833 | { |
| @@ -1972,7 +1882,6 @@ static struct mapped_device *alloc_dev(int minor) | |||
| 1972 | mutex_init(&md->suspend_lock); | 1882 | mutex_init(&md->suspend_lock); |
| 1973 | mutex_init(&md->type_lock); | 1883 | mutex_init(&md->type_lock); |
| 1974 | spin_lock_init(&md->deferred_lock); | 1884 | spin_lock_init(&md->deferred_lock); |
| 1975 | spin_lock_init(&md->barrier_error_lock); | ||
| 1976 | rwlock_init(&md->map_lock); | 1885 | rwlock_init(&md->map_lock); |
| 1977 | atomic_set(&md->holders, 1); | 1886 | atomic_set(&md->holders, 1); |
| 1978 | atomic_set(&md->open_count, 0); | 1887 | atomic_set(&md->open_count, 0); |
| @@ -1995,7 +1904,6 @@ static struct mapped_device *alloc_dev(int minor) | |||
| 1995 | atomic_set(&md->pending[1], 0); | 1904 | atomic_set(&md->pending[1], 0); |
| 1996 | init_waitqueue_head(&md->wait); | 1905 | init_waitqueue_head(&md->wait); |
| 1997 | INIT_WORK(&md->work, dm_wq_work); | 1906 | INIT_WORK(&md->work, dm_wq_work); |
| 1998 | INIT_WORK(&md->barrier_work, dm_rq_barrier_work); | ||
| 1999 | init_waitqueue_head(&md->eventq); | 1907 | init_waitqueue_head(&md->eventq); |
| 2000 | 1908 | ||
| 2001 | md->disk->major = _major; | 1909 | md->disk->major = _major; |
| @@ -2245,8 +2153,6 @@ static int dm_init_request_based_queue(struct mapped_device *md) | |||
| 2245 | blk_queue_softirq_done(md->queue, dm_softirq_done); | 2153 | blk_queue_softirq_done(md->queue, dm_softirq_done); |
| 2246 | blk_queue_prep_rq(md->queue, dm_prep_fn); | 2154 | blk_queue_prep_rq(md->queue, dm_prep_fn); |
| 2247 | blk_queue_lld_busy(md->queue, dm_lld_busy); | 2155 | blk_queue_lld_busy(md->queue, dm_lld_busy); |
| 2248 | /* no flush support for request based dm yet */ | ||
| 2249 | blk_queue_flush(md->queue, 0); | ||
| 2250 | 2156 | ||
| 2251 | elv_register_queue(md->queue); | 2157 | elv_register_queue(md->queue); |
| 2252 | 2158 | ||
| @@ -2483,73 +2389,6 @@ static void dm_queue_flush(struct mapped_device *md) | |||
| 2483 | queue_work(md->wq, &md->work); | 2389 | queue_work(md->wq, &md->work); |
| 2484 | } | 2390 | } |
| 2485 | 2391 | ||
| 2486 | static void dm_rq_set_target_request_nr(struct request *clone, unsigned request_nr) | ||
| 2487 | { | ||
| 2488 | struct dm_rq_target_io *tio = clone->end_io_data; | ||
| 2489 | |||
| 2490 | tio->info.target_request_nr = request_nr; | ||
| 2491 | } | ||
| 2492 | |||
| 2493 | /* Issue barrier requests to targets and wait for their completion. */ | ||
| 2494 | static int dm_rq_barrier(struct mapped_device *md) | ||
| 2495 | { | ||
| 2496 | int i, j; | ||
| 2497 | struct dm_table *map = dm_get_live_table(md); | ||
| 2498 | unsigned num_targets = dm_table_get_num_targets(map); | ||
| 2499 | struct dm_target *ti; | ||
| 2500 | struct request *clone; | ||
| 2501 | |||
| 2502 | md->barrier_error = 0; | ||
| 2503 | |||
| 2504 | for (i = 0; i < num_targets; i++) { | ||
| 2505 | ti = dm_table_get_target(map, i); | ||
| 2506 | for (j = 0; j < ti->num_flush_requests; j++) { | ||
| 2507 | clone = clone_rq(md->flush_request, md, GFP_NOIO); | ||
| 2508 | dm_rq_set_target_request_nr(clone, j); | ||
| 2509 | atomic_inc(&md->pending[rq_data_dir(clone)]); | ||
| 2510 | map_request(ti, clone, md); | ||
| 2511 | } | ||
| 2512 | } | ||
| 2513 | |||
| 2514 | dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE); | ||
| 2515 | dm_table_put(map); | ||
| 2516 | |||
| 2517 | return md->barrier_error; | ||
| 2518 | } | ||
| 2519 | |||
| 2520 | static void dm_rq_barrier_work(struct work_struct *work) | ||
| 2521 | { | ||
| 2522 | int error; | ||
| 2523 | struct mapped_device *md = container_of(work, struct mapped_device, | ||
| 2524 | barrier_work); | ||
| 2525 | struct request_queue *q = md->queue; | ||
| 2526 | struct request *rq; | ||
| 2527 | unsigned long flags; | ||
| 2528 | |||
| 2529 | /* | ||
| 2530 | * Hold the md reference here and leave it at the last part so that | ||
| 2531 | * the md can't be deleted by device opener when the barrier request | ||
| 2532 | * completes. | ||
| 2533 | */ | ||
| 2534 | dm_get(md); | ||
| 2535 | |||
| 2536 | error = dm_rq_barrier(md); | ||
| 2537 | |||
| 2538 | rq = md->flush_request; | ||
| 2539 | md->flush_request = NULL; | ||
| 2540 | |||
| 2541 | if (error == DM_ENDIO_REQUEUE) { | ||
| 2542 | spin_lock_irqsave(q->queue_lock, flags); | ||
| 2543 | blk_requeue_request(q, rq); | ||
| 2544 | spin_unlock_irqrestore(q->queue_lock, flags); | ||
| 2545 | } else | ||
| 2546 | blk_end_request_all(rq, error); | ||
| 2547 | |||
| 2548 | blk_run_queue(q); | ||
| 2549 | |||
| 2550 | dm_put(md); | ||
| 2551 | } | ||
| 2552 | |||
| 2553 | /* | 2392 | /* |
| 2554 | * Swap in a new table, returning the old one for the caller to destroy. | 2393 | * Swap in a new table, returning the old one for the caller to destroy. |
| 2555 | */ | 2394 | */ |
| @@ -2686,9 +2525,8 @@ int dm_suspend(struct mapped_device *md, unsigned suspend_flags) | |||
| 2686 | up_write(&md->io_lock); | 2525 | up_write(&md->io_lock); |
| 2687 | 2526 | ||
| 2688 | /* | 2527 | /* |
| 2689 | * Request-based dm uses md->wq for barrier (dm_rq_barrier_work) which | 2528 | * Stop md->queue before flushing md->wq in case request-based |
| 2690 | * can be kicked until md->queue is stopped. So stop md->queue before | 2529 | * dm defers requests to md->wq from md->queue. |
| 2691 | * flushing md->wq. | ||
| 2692 | */ | 2530 | */ |
| 2693 | if (dm_request_based(md)) | 2531 | if (dm_request_based(md)) |
| 2694 | stop_queue(md->queue); | 2532 | stop_queue(md->queue); |
