aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--block/blk-core.c84
-rw-r--r--block/blk-merge.c12
-rw-r--r--block/genhd.c97
-rw-r--r--drivers/block/aoe/aoecmd.c12
-rw-r--r--drivers/md/dm.c27
-rw-r--r--drivers/md/linear.c9
-rw-r--r--drivers/md/md.c4
-rw-r--r--drivers/md/multipath.c9
-rw-r--r--drivers/md/raid0.c9
-rw-r--r--drivers/md/raid1.c9
-rw-r--r--drivers/md/raid10.c9
-rw-r--r--drivers/md/raid5.c9
-rw-r--r--fs/partitions/check.c12
-rw-r--r--include/linux/genhd.h159
14 files changed, 165 insertions, 296 deletions
diff --git a/block/blk-core.c b/block/blk-core.c
index 505ec61067df..98138f002524 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -61,21 +61,17 @@ static void drive_stat_acct(struct request *rq, int new_io)
61 if (!blk_fs_request(rq) || !rq->rq_disk) 61 if (!blk_fs_request(rq) || !rq->rq_disk)
62 return; 62 return;
63 63
64 cpu = disk_stat_lock(); 64 cpu = part_stat_lock();
65 part = disk_map_sector_rcu(rq->rq_disk, rq->sector); 65 part = disk_map_sector_rcu(rq->rq_disk, rq->sector);
66 66
67 if (!new_io) 67 if (!new_io)
68 all_stat_inc(cpu, rq->rq_disk, part, merges[rw], rq->sector); 68 part_stat_inc(cpu, part, merges[rw]);
69 else { 69 else {
70 disk_round_stats(cpu, rq->rq_disk); 70 part_round_stats(cpu, part);
71 rq->rq_disk->in_flight++; 71 part_inc_in_flight(part);
72 if (part) {
73 part_round_stats(cpu, part);
74 part->in_flight++;
75 }
76 } 72 }
77 73
78 disk_stat_unlock(); 74 part_stat_unlock();
79} 75}
80 76
81void blk_queue_congestion_threshold(struct request_queue *q) 77void blk_queue_congestion_threshold(struct request_queue *q)
@@ -983,8 +979,22 @@ static inline void add_request(struct request_queue *q, struct request *req)
983 __elv_add_request(q, req, ELEVATOR_INSERT_SORT, 0); 979 __elv_add_request(q, req, ELEVATOR_INSERT_SORT, 0);
984} 980}
985 981
986/* 982static void part_round_stats_single(int cpu, struct hd_struct *part,
987 * disk_round_stats() - Round off the performance stats on a struct 983 unsigned long now)
984{
985 if (now == part->stamp)
986 return;
987
988 if (part->in_flight) {
989 __part_stat_add(cpu, part, time_in_queue,
990 part->in_flight * (now - part->stamp));
991 __part_stat_add(cpu, part, io_ticks, (now - part->stamp));
992 }
993 part->stamp = now;
994}
995
996/**
997 * part_round_stats() - Round off the performance stats on a struct
988 * disk_stats. 998 * disk_stats.
989 * 999 *
990 * The average IO queue length and utilisation statistics are maintained 1000 * The average IO queue length and utilisation statistics are maintained
@@ -998,36 +1008,15 @@ static inline void add_request(struct request_queue *q, struct request *req)
998 * /proc/diskstats. This accounts immediately for all queue usage up to 1008 * /proc/diskstats. This accounts immediately for all queue usage up to
999 * the current jiffies and restarts the counters again. 1009 * the current jiffies and restarts the counters again.
1000 */ 1010 */
1001void disk_round_stats(int cpu, struct gendisk *disk)
1002{
1003 unsigned long now = jiffies;
1004
1005 if (now == disk->stamp)
1006 return;
1007
1008 if (disk->in_flight) {
1009 disk_stat_add(cpu, disk, time_in_queue,
1010 disk->in_flight * (now - disk->stamp));
1011 disk_stat_add(cpu, disk, io_ticks, (now - disk->stamp));
1012 }
1013 disk->stamp = now;
1014}
1015EXPORT_SYMBOL_GPL(disk_round_stats);
1016
1017void part_round_stats(int cpu, struct hd_struct *part) 1011void part_round_stats(int cpu, struct hd_struct *part)
1018{ 1012{
1019 unsigned long now = jiffies; 1013 unsigned long now = jiffies;
1020 1014
1021 if (now == part->stamp) 1015 if (part->partno)
1022 return; 1016 part_round_stats_single(cpu, &part_to_disk(part)->part0, now);
1023 1017 part_round_stats_single(cpu, part, now);
1024 if (part->in_flight) {
1025 part_stat_add(cpu, part, time_in_queue,
1026 part->in_flight * (now - part->stamp));
1027 part_stat_add(cpu, part, io_ticks, (now - part->stamp));
1028 }
1029 part->stamp = now;
1030} 1018}
1019EXPORT_SYMBOL_GPL(part_round_stats);
1031 1020
1032/* 1021/*
1033 * queue lock must be held 1022 * queue lock must be held
@@ -1567,11 +1556,10 @@ static int __end_that_request_first(struct request *req, int error,
1567 struct hd_struct *part; 1556 struct hd_struct *part;
1568 int cpu; 1557 int cpu;
1569 1558
1570 cpu = disk_stat_lock(); 1559 cpu = part_stat_lock();
1571 part = disk_map_sector_rcu(req->rq_disk, req->sector); 1560 part = disk_map_sector_rcu(req->rq_disk, req->sector);
1572 all_stat_add(cpu, req->rq_disk, part, sectors[rw], 1561 part_stat_add(cpu, part, sectors[rw], nr_bytes >> 9);
1573 nr_bytes >> 9, req->sector); 1562 part_stat_unlock();
1574 disk_stat_unlock();
1575 } 1563 }
1576 1564
1577 total_bytes = bio_nbytes = 0; 1565 total_bytes = bio_nbytes = 0;
@@ -1758,19 +1746,15 @@ static void end_that_request_last(struct request *req, int error)
1758 struct hd_struct *part; 1746 struct hd_struct *part;
1759 int cpu; 1747 int cpu;
1760 1748
1761 cpu = disk_stat_lock(); 1749 cpu = part_stat_lock();
1762 part = disk_map_sector_rcu(disk, req->sector); 1750 part = disk_map_sector_rcu(disk, req->sector);
1763 1751
1764 all_stat_inc(cpu, disk, part, ios[rw], req->sector); 1752 part_stat_inc(cpu, part, ios[rw]);
1765 all_stat_add(cpu, disk, part, ticks[rw], duration, req->sector); 1753 part_stat_add(cpu, part, ticks[rw], duration);
1766 disk_round_stats(cpu, disk); 1754 part_round_stats(cpu, part);
1767 disk->in_flight--; 1755 part_dec_in_flight(part);
1768 if (part) {
1769 part_round_stats(cpu, part);
1770 part->in_flight--;
1771 }
1772 1756
1773 disk_stat_unlock(); 1757 part_stat_unlock();
1774 } 1758 }
1775 1759
1776 if (req->end_io) 1760 if (req->end_io)
diff --git a/block/blk-merge.c b/block/blk-merge.c
index d926a24bf1fd..c77196d55899 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -390,17 +390,13 @@ static int attempt_merge(struct request_queue *q, struct request *req,
390 struct hd_struct *part; 390 struct hd_struct *part;
391 int cpu; 391 int cpu;
392 392
393 cpu = disk_stat_lock(); 393 cpu = part_stat_lock();
394 part = disk_map_sector_rcu(req->rq_disk, req->sector); 394 part = disk_map_sector_rcu(req->rq_disk, req->sector);
395 395
396 disk_round_stats(cpu, req->rq_disk); 396 part_round_stats(cpu, part);
397 req->rq_disk->in_flight--; 397 part_dec_in_flight(part);
398 if (part) {
399 part_round_stats(cpu, part);
400 part->in_flight--;
401 }
402 398
403 disk_stat_unlock(); 399 part_stat_unlock();
404 } 400 }
405 401
406 req->ioprio = ioprio_best(req->ioprio, next->ioprio); 402 req->ioprio = ioprio_best(req->ioprio, next->ioprio);
diff --git a/block/genhd.c b/block/genhd.c
index 06a252f2b967..e1cb96fb883e 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -176,7 +176,7 @@ EXPORT_SYMBOL_GPL(disk_part_iter_exit);
176 * while preemption is disabled. 176 * while preemption is disabled.
177 * 177 *
178 * RETURNS: 178 * RETURNS:
179 * Found partition on success, NULL if there's no matching partition. 179 * Found partition on success, part0 is returned if no partition matches
180 */ 180 */
181struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector) 181struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector)
182{ 182{
@@ -189,7 +189,7 @@ struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector)
189 sector < part->start_sect + part->nr_sects) 189 sector < part->start_sect + part->nr_sects)
190 return part; 190 return part;
191 } 191 }
192 return NULL; 192 return &disk->part0;
193} 193}
194EXPORT_SYMBOL_GPL(disk_map_sector_rcu); 194EXPORT_SYMBOL_GPL(disk_map_sector_rcu);
195 195
@@ -580,24 +580,24 @@ void __init printk_all_partitions(void)
580 * numbers in hex - the same format as the root= 580 * numbers in hex - the same format as the root=
581 * option takes. 581 * option takes.
582 */ 582 */
583 printk("%s %10llu %s", 583 disk_part_iter_init(&piter, disk, DISK_PITER_INCL_PART0);
584 bdevt_str(disk_devt(disk), devt_buf), 584 while ((part = disk_part_iter_next(&piter))) {
585 (unsigned long long)get_capacity(disk) >> 1, 585 bool is_part0 = part == &disk->part0;
586 disk_name(disk, 0, name_buf));
587 if (disk->driverfs_dev != NULL &&
588 disk->driverfs_dev->driver != NULL)
589 printk(" driver: %s\n",
590 disk->driverfs_dev->driver->name);
591 else
592 printk(" (driver?)\n");
593 586
594 /* now show the partitions */ 587 printk("%s%s %10llu %s", is_part0 ? "" : " ",
595 disk_part_iter_init(&piter, disk, 0);
596 while ((part = disk_part_iter_next(&piter)))
597 printk(" %s %10llu %s\n",
598 bdevt_str(part_devt(part), devt_buf), 588 bdevt_str(part_devt(part), devt_buf),
599 (unsigned long long)part->nr_sects >> 1, 589 (unsigned long long)part->nr_sects >> 1,
600 disk_name(disk, part->partno, name_buf)); 590 disk_name(disk, part->partno, name_buf));
591 if (is_part0) {
592 if (disk->driverfs_dev != NULL &&
593 disk->driverfs_dev->driver != NULL)
594 printk(" driver: %s\n",
595 disk->driverfs_dev->driver->name);
596 else
597 printk(" (driver?)\n");
598 } else
599 printk("\n");
600 }
601 disk_part_iter_exit(&piter); 601 disk_part_iter_exit(&piter);
602 } 602 }
603 class_dev_iter_exit(&iter); 603 class_dev_iter_exit(&iter);
@@ -674,12 +674,7 @@ static int show_partition(struct seq_file *seqf, void *v)
674 return 0; 674 return 0;
675 675
676 /* show the full disk and all non-0 size partitions of it */ 676 /* show the full disk and all non-0 size partitions of it */
677 seq_printf(seqf, "%4d %7d %10llu %s\n", 677 disk_part_iter_init(&piter, sgp, DISK_PITER_INCL_PART0);
678 MAJOR(disk_devt(sgp)), MINOR(disk_devt(sgp)),
679 (unsigned long long)get_capacity(sgp) >> 1,
680 disk_name(sgp, 0, buf));
681
682 disk_part_iter_init(&piter, sgp, 0);
683 while ((part = disk_part_iter_next(&piter))) 678 while ((part = disk_part_iter_next(&piter)))
684 seq_printf(seqf, "%4d %7d %10llu %s\n", 679 seq_printf(seqf, "%4d %7d %10llu %s\n",
685 MAJOR(part_devt(part)), MINOR(part_devt(part)), 680 MAJOR(part_devt(part)), MINOR(part_devt(part)),
@@ -768,40 +763,13 @@ static ssize_t disk_capability_show(struct device *dev,
768 return sprintf(buf, "%x\n", disk->flags); 763 return sprintf(buf, "%x\n", disk->flags);
769} 764}
770 765
771static ssize_t disk_stat_show(struct device *dev,
772 struct device_attribute *attr, char *buf)
773{
774 struct gendisk *disk = dev_to_disk(dev);
775 int cpu;
776
777 cpu = disk_stat_lock();
778 disk_round_stats(cpu, disk);
779 disk_stat_unlock();
780 return sprintf(buf,
781 "%8lu %8lu %8llu %8u "
782 "%8lu %8lu %8llu %8u "
783 "%8u %8u %8u"
784 "\n",
785 disk_stat_read(disk, ios[READ]),
786 disk_stat_read(disk, merges[READ]),
787 (unsigned long long)disk_stat_read(disk, sectors[READ]),
788 jiffies_to_msecs(disk_stat_read(disk, ticks[READ])),
789 disk_stat_read(disk, ios[WRITE]),
790 disk_stat_read(disk, merges[WRITE]),
791 (unsigned long long)disk_stat_read(disk, sectors[WRITE]),
792 jiffies_to_msecs(disk_stat_read(disk, ticks[WRITE])),
793 disk->in_flight,
794 jiffies_to_msecs(disk_stat_read(disk, io_ticks)),
795 jiffies_to_msecs(disk_stat_read(disk, time_in_queue)));
796}
797
798static DEVICE_ATTR(range, S_IRUGO, disk_range_show, NULL); 766static DEVICE_ATTR(range, S_IRUGO, disk_range_show, NULL);
799static DEVICE_ATTR(ext_range, S_IRUGO, disk_ext_range_show, NULL); 767static DEVICE_ATTR(ext_range, S_IRUGO, disk_ext_range_show, NULL);
800static DEVICE_ATTR(removable, S_IRUGO, disk_removable_show, NULL); 768static DEVICE_ATTR(removable, S_IRUGO, disk_removable_show, NULL);
801static DEVICE_ATTR(ro, S_IRUGO, disk_ro_show, NULL); 769static DEVICE_ATTR(ro, S_IRUGO, disk_ro_show, NULL);
802static DEVICE_ATTR(size, S_IRUGO, part_size_show, NULL); 770static DEVICE_ATTR(size, S_IRUGO, part_size_show, NULL);
803static DEVICE_ATTR(capability, S_IRUGO, disk_capability_show, NULL); 771static DEVICE_ATTR(capability, S_IRUGO, disk_capability_show, NULL);
804static DEVICE_ATTR(stat, S_IRUGO, disk_stat_show, NULL); 772static DEVICE_ATTR(stat, S_IRUGO, part_stat_show, NULL);
805#ifdef CONFIG_FAIL_MAKE_REQUEST 773#ifdef CONFIG_FAIL_MAKE_REQUEST
806static struct device_attribute dev_attr_fail = 774static struct device_attribute dev_attr_fail =
807 __ATTR(make-it-fail, S_IRUGO|S_IWUSR, part_fail_show, part_fail_store); 775 __ATTR(make-it-fail, S_IRUGO|S_IWUSR, part_fail_show, part_fail_store);
@@ -836,7 +804,7 @@ static void disk_release(struct device *dev)
836 804
837 kfree(disk->random); 805 kfree(disk->random);
838 kfree(disk->__part); 806 kfree(disk->__part);
839 free_disk_stats(disk); 807 free_part_stats(&disk->part0);
840 kfree(disk); 808 kfree(disk);
841} 809}
842struct class block_class = { 810struct class block_class = {
@@ -873,28 +841,11 @@ static int diskstats_show(struct seq_file *seqf, void *v)
873 "\n\n"); 841 "\n\n");
874 */ 842 */
875 843
876 cpu = disk_stat_lock(); 844 disk_part_iter_init(&piter, gp, DISK_PITER_INCL_PART0);
877 disk_round_stats(cpu, gp);
878 disk_stat_unlock();
879 seq_printf(seqf, "%4d %7d %s %lu %lu %llu %u %lu %lu %llu %u %u %u %u\n",
880 MAJOR(disk_devt(gp)), MINOR(disk_devt(gp)),
881 disk_name(gp, 0, buf),
882 disk_stat_read(gp, ios[0]), disk_stat_read(gp, merges[0]),
883 (unsigned long long)disk_stat_read(gp, sectors[0]),
884 jiffies_to_msecs(disk_stat_read(gp, ticks[0])),
885 disk_stat_read(gp, ios[1]), disk_stat_read(gp, merges[1]),
886 (unsigned long long)disk_stat_read(gp, sectors[1]),
887 jiffies_to_msecs(disk_stat_read(gp, ticks[1])),
888 gp->in_flight,
889 jiffies_to_msecs(disk_stat_read(gp, io_ticks)),
890 jiffies_to_msecs(disk_stat_read(gp, time_in_queue)));
891
892 /* now show all non-0 size partitions of it */
893 disk_part_iter_init(&piter, gp, 0);
894 while ((hd = disk_part_iter_next(&piter))) { 845 while ((hd = disk_part_iter_next(&piter))) {
895 cpu = disk_stat_lock(); 846 cpu = part_stat_lock();
896 part_round_stats(cpu, hd); 847 part_round_stats(cpu, hd);
897 disk_stat_unlock(); 848 part_stat_unlock();
898 seq_printf(seqf, "%4d %7d %s %lu %lu %llu " 849 seq_printf(seqf, "%4d %7d %s %lu %lu %llu "
899 "%u %lu %lu %llu %u %u %u %u\n", 850 "%u %lu %lu %llu %u %u %u %u\n",
900 MAJOR(part_devt(hd)), MINOR(part_devt(hd)), 851 MAJOR(part_devt(hd)), MINOR(part_devt(hd)),
@@ -1000,7 +951,7 @@ struct gendisk *alloc_disk_ext_node(int minors, int ext_minors, int node_id)
1000 int tot_minors = minors + ext_minors; 951 int tot_minors = minors + ext_minors;
1001 int size = tot_minors * sizeof(struct hd_struct *); 952 int size = tot_minors * sizeof(struct hd_struct *);
1002 953
1003 if (!init_disk_stats(disk)) { 954 if (!init_part_stats(&disk->part0)) {
1004 kfree(disk); 955 kfree(disk);
1005 return NULL; 956 return NULL;
1006 } 957 }
@@ -1008,7 +959,7 @@ struct gendisk *alloc_disk_ext_node(int minors, int ext_minors, int node_id)
1008 disk->__part = kmalloc_node(size, GFP_KERNEL | __GFP_ZERO, 959 disk->__part = kmalloc_node(size, GFP_KERNEL | __GFP_ZERO,
1009 node_id); 960 node_id);
1010 if (!disk->__part) { 961 if (!disk->__part) {
1011 free_disk_stats(disk); 962 free_part_stats(&disk->part0);
1012 kfree(disk); 963 kfree(disk);
1013 return NULL; 964 return NULL;
1014 } 965 }
diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c
index 934800f979c9..961d29a53cab 100644
--- a/drivers/block/aoe/aoecmd.c
+++ b/drivers/block/aoe/aoecmd.c
@@ -758,15 +758,15 @@ diskstats(struct gendisk *disk, struct bio *bio, ulong duration, sector_t sector
758 struct hd_struct *part; 758 struct hd_struct *part;
759 int cpu; 759 int cpu;
760 760
761 cpu = disk_stat_lock(); 761 cpu = part_stat_lock();
762 part = disk_map_sector_rcu(disk, sector); 762 part = disk_map_sector_rcu(disk, sector);
763 763
764 all_stat_inc(cpu, disk, part, ios[rw], sector); 764 part_stat_inc(cpu, part, ios[rw]);
765 all_stat_add(cpu, disk, part, ticks[rw], duration, sector); 765 part_stat_add(cpu, part, ticks[rw], duration);
766 all_stat_add(cpu, disk, part, sectors[rw], n_sect, sector); 766 part_stat_add(cpu, part, sectors[rw], n_sect);
767 all_stat_add(cpu, disk, part, io_ticks, duration, sector); 767 part_stat_add(cpu, part, io_ticks, duration);
768 768
769 disk_stat_unlock(); 769 part_stat_unlock();
770} 770}
771 771
772void 772void
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 637806695bb9..327de03a5bdf 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -381,10 +381,10 @@ static void start_io_acct(struct dm_io *io)
381 381
382 io->start_time = jiffies; 382 io->start_time = jiffies;
383 383
384 cpu = disk_stat_lock(); 384 cpu = part_stat_lock();
385 disk_round_stats(cpu, dm_disk(md)); 385 part_round_stats(cpu, &dm_disk(md)->part0);
386 disk_stat_unlock(); 386 part_stat_unlock();
387 dm_disk(md)->in_flight = atomic_inc_return(&md->pending); 387 dm_disk(md)->part0.in_flight = atomic_inc_return(&md->pending);
388} 388}
389 389
390static int end_io_acct(struct dm_io *io) 390static int end_io_acct(struct dm_io *io)
@@ -395,12 +395,13 @@ static int end_io_acct(struct dm_io *io)
395 int pending, cpu; 395 int pending, cpu;
396 int rw = bio_data_dir(bio); 396 int rw = bio_data_dir(bio);
397 397
398 cpu = disk_stat_lock(); 398 cpu = part_stat_lock();
399 disk_round_stats(cpu, dm_disk(md)); 399 part_round_stats(cpu, &dm_disk(md)->part0);
400 disk_stat_add(cpu, dm_disk(md), ticks[rw], duration); 400 part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
401 disk_stat_unlock(); 401 part_stat_unlock();
402 402
403 dm_disk(md)->in_flight = pending = atomic_dec_return(&md->pending); 403 dm_disk(md)->part0.in_flight = pending =
404 atomic_dec_return(&md->pending);
404 405
405 return !pending; 406 return !pending;
406} 407}
@@ -899,10 +900,10 @@ static int dm_request(struct request_queue *q, struct bio *bio)
899 900
900 down_read(&md->io_lock); 901 down_read(&md->io_lock);
901 902
902 cpu = disk_stat_lock(); 903 cpu = part_stat_lock();
903 disk_stat_inc(cpu, dm_disk(md), ios[rw]); 904 part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
904 disk_stat_add(cpu, dm_disk(md), sectors[rw], bio_sectors(bio)); 905 part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
905 disk_stat_unlock(); 906 part_stat_unlock();
906 907
907 /* 908 /*
908 * If we're suspended we have to queue 909 * If we're suspended we have to queue
diff --git a/drivers/md/linear.c b/drivers/md/linear.c
index 00cbc8e47294..c80ea90593d3 100644
--- a/drivers/md/linear.c
+++ b/drivers/md/linear.c
@@ -325,10 +325,11 @@ static int linear_make_request (struct request_queue *q, struct bio *bio)
325 return 0; 325 return 0;
326 } 326 }
327 327
328 cpu = disk_stat_lock(); 328 cpu = part_stat_lock();
329 disk_stat_inc(cpu, mddev->gendisk, ios[rw]); 329 part_stat_inc(cpu, &mddev->gendisk->part0, ios[rw]);
330 disk_stat_add(cpu, mddev->gendisk, sectors[rw], bio_sectors(bio)); 330 part_stat_add(cpu, &mddev->gendisk->part0, sectors[rw],
331 disk_stat_unlock(); 331 bio_sectors(bio));
332 part_stat_unlock();
332 333
333 tmp_dev = which_dev(mddev, bio->bi_sector); 334 tmp_dev = which_dev(mddev, bio->bi_sector);
334 block = bio->bi_sector >> 1; 335 block = bio->bi_sector >> 1;
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 2bd9cf416123..0a3a4bdcd4af 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -5546,8 +5546,8 @@ static int is_mddev_idle(mddev_t *mddev)
5546 rcu_read_lock(); 5546 rcu_read_lock();
5547 rdev_for_each_rcu(rdev, mddev) { 5547 rdev_for_each_rcu(rdev, mddev) {
5548 struct gendisk *disk = rdev->bdev->bd_contains->bd_disk; 5548 struct gendisk *disk = rdev->bdev->bd_contains->bd_disk;
5549 curr_events = disk_stat_read(disk, sectors[0]) + 5549 curr_events = part_stat_read(&disk->part0, sectors[0]) +
5550 disk_stat_read(disk, sectors[1]) - 5550 part_stat_read(&disk->part0, sectors[1]) -
5551 atomic_read(&disk->sync_io); 5551 atomic_read(&disk->sync_io);
5552 /* sync IO will cause sync_io to increase before the disk_stats 5552 /* sync IO will cause sync_io to increase before the disk_stats
5553 * as sync_io is counted when a request starts, and 5553 * as sync_io is counted when a request starts, and
diff --git a/drivers/md/multipath.c b/drivers/md/multipath.c
index 182f5a94cdc5..8bb8794129b3 100644
--- a/drivers/md/multipath.c
+++ b/drivers/md/multipath.c
@@ -159,10 +159,11 @@ static int multipath_make_request (struct request_queue *q, struct bio * bio)
159 mp_bh->master_bio = bio; 159 mp_bh->master_bio = bio;
160 mp_bh->mddev = mddev; 160 mp_bh->mddev = mddev;
161 161
162 cpu = disk_stat_lock(); 162 cpu = part_stat_lock();
163 disk_stat_inc(cpu, mddev->gendisk, ios[rw]); 163 part_stat_inc(cpu, &mddev->gendisk->part0, ios[rw]);
164 disk_stat_add(cpu, mddev->gendisk, sectors[rw], bio_sectors(bio)); 164 part_stat_add(cpu, &mddev->gendisk->part0, sectors[rw],
165 disk_stat_unlock(); 165 bio_sectors(bio));
166 part_stat_unlock();
166 167
167 mp_bh->path = multipath_map(conf); 168 mp_bh->path = multipath_map(conf);
168 if (mp_bh->path < 0) { 169 if (mp_bh->path < 0) {
diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c
index e26030fa59ab..f52f442a735f 100644
--- a/drivers/md/raid0.c
+++ b/drivers/md/raid0.c
@@ -406,10 +406,11 @@ static int raid0_make_request (struct request_queue *q, struct bio *bio)
406 return 0; 406 return 0;
407 } 407 }
408 408
409 cpu = disk_stat_lock(); 409 cpu = part_stat_lock();
410 disk_stat_inc(cpu, mddev->gendisk, ios[rw]); 410 part_stat_inc(cpu, &mddev->gendisk->part0, ios[rw]);
411 disk_stat_add(cpu, mddev->gendisk, sectors[rw], bio_sectors(bio)); 411 part_stat_add(cpu, &mddev->gendisk->part0, sectors[rw],
412 disk_stat_unlock(); 412 bio_sectors(bio));
413 part_stat_unlock();
413 414
414 chunk_size = mddev->chunk_size >> 10; 415 chunk_size = mddev->chunk_size >> 10;
415 chunk_sects = mddev->chunk_size >> 9; 416 chunk_sects = mddev->chunk_size >> 9;
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index babb13036f93..b9764429d856 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -804,10 +804,11 @@ static int make_request(struct request_queue *q, struct bio * bio)
804 804
805 bitmap = mddev->bitmap; 805 bitmap = mddev->bitmap;
806 806
807 cpu = disk_stat_lock(); 807 cpu = part_stat_lock();
808 disk_stat_inc(cpu, mddev->gendisk, ios[rw]); 808 part_stat_inc(cpu, &mddev->gendisk->part0, ios[rw]);
809 disk_stat_add(cpu, mddev->gendisk, sectors[rw], bio_sectors(bio)); 809 part_stat_add(cpu, &mddev->gendisk->part0, sectors[rw],
810 disk_stat_unlock(); 810 bio_sectors(bio));
811 part_stat_unlock();
811 812
812 /* 813 /*
813 * make_request() can abort the operation when READA is being 814 * make_request() can abort the operation when READA is being
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index 5ec80da0a9d7..5f990133f5ef 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -844,10 +844,11 @@ static int make_request(struct request_queue *q, struct bio * bio)
844 */ 844 */
845 wait_barrier(conf); 845 wait_barrier(conf);
846 846
847 cpu = disk_stat_lock(); 847 cpu = part_stat_lock();
848 disk_stat_inc(cpu, mddev->gendisk, ios[rw]); 848 part_stat_inc(cpu, &mddev->gendisk->part0, ios[rw]);
849 disk_stat_add(cpu, mddev->gendisk, sectors[rw], bio_sectors(bio)); 849 part_stat_add(cpu, &mddev->gendisk->part0, sectors[rw],
850 disk_stat_unlock(); 850 bio_sectors(bio));
851 part_stat_unlock();
851 852
852 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); 853 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
853 854
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 5899f211515f..ae16794bef20 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -3396,10 +3396,11 @@ static int make_request(struct request_queue *q, struct bio * bi)
3396 3396
3397 md_write_start(mddev, bi); 3397 md_write_start(mddev, bi);
3398 3398
3399 cpu = disk_stat_lock(); 3399 cpu = part_stat_lock();
3400 disk_stat_inc(cpu, mddev->gendisk, ios[rw]); 3400 part_stat_inc(cpu, &mddev->gendisk->part0, ios[rw]);
3401 disk_stat_add(cpu, mddev->gendisk, sectors[rw], bio_sectors(bi)); 3401 part_stat_add(cpu, &mddev->gendisk->part0, sectors[rw],
3402 disk_stat_unlock(); 3402 bio_sectors(bi));
3403 part_stat_unlock();
3403 3404
3404 if (rw == READ && 3405 if (rw == READ &&
3405 mddev->reshape_position == MaxSector && 3406 mddev->reshape_position == MaxSector &&
diff --git a/fs/partitions/check.c b/fs/partitions/check.c
index 60592d9f43b6..f517869e8d10 100644
--- a/fs/partitions/check.c
+++ b/fs/partitions/check.c
@@ -210,15 +210,15 @@ ssize_t part_size_show(struct device *dev,
210 return sprintf(buf, "%llu\n",(unsigned long long)p->nr_sects); 210 return sprintf(buf, "%llu\n",(unsigned long long)p->nr_sects);
211} 211}
212 212
213static ssize_t part_stat_show(struct device *dev, 213ssize_t part_stat_show(struct device *dev,
214 struct device_attribute *attr, char *buf) 214 struct device_attribute *attr, char *buf)
215{ 215{
216 struct hd_struct *p = dev_to_part(dev); 216 struct hd_struct *p = dev_to_part(dev);
217 int cpu; 217 int cpu;
218 218
219 cpu = disk_stat_lock(); 219 cpu = part_stat_lock();
220 part_round_stats(cpu, p); 220 part_round_stats(cpu, p);
221 disk_stat_unlock(); 221 part_stat_unlock();
222 return sprintf(buf, 222 return sprintf(buf,
223 "%8lu %8lu %8llu %8u " 223 "%8lu %8lu %8llu %8u "
224 "%8lu %8lu %8llu %8u " 224 "%8lu %8lu %8llu %8u "
@@ -575,8 +575,8 @@ void del_gendisk(struct gendisk *disk)
575 set_capacity(disk, 0); 575 set_capacity(disk, 0);
576 disk->flags &= ~GENHD_FL_UP; 576 disk->flags &= ~GENHD_FL_UP;
577 unlink_gendisk(disk); 577 unlink_gendisk(disk);
578 disk_stat_set_all(disk, 0); 578 part_stat_set_all(&disk->part0, 0);
579 disk->stamp = 0; 579 disk->part0.stamp = 0;
580 580
581 kobject_put(disk->part0.holder_dir); 581 kobject_put(disk->part0.holder_dir);
582 kobject_put(disk->slave_dir); 582 kobject_put(disk->slave_dir);
diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index 3d15b42dc352..c90e1b4fbe5a 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -145,13 +145,6 @@ struct gendisk {
145 struct timer_rand_state *random; 145 struct timer_rand_state *random;
146 146
147 atomic_t sync_io; /* RAID */ 147 atomic_t sync_io; /* RAID */
148 unsigned long stamp;
149 int in_flight;
150#ifdef CONFIG_SMP
151 struct disk_stats *dkstats;
152#else
153 struct disk_stats dkstats;
154#endif
155 struct work_struct async_notify; 148 struct work_struct async_notify;
156#ifdef CONFIG_BLK_DEV_INTEGRITY 149#ifdef CONFIG_BLK_DEV_INTEGRITY
157 struct blk_integrity *integrity; 150 struct blk_integrity *integrity;
@@ -232,46 +225,18 @@ extern struct hd_struct *disk_map_sector_rcu(struct gendisk *disk,
232 * internal use only. 225 * internal use only.
233 */ 226 */
234#ifdef CONFIG_SMP 227#ifdef CONFIG_SMP
235#define disk_stat_lock() ({ rcu_read_lock(); get_cpu(); }) 228#define part_stat_lock() ({ rcu_read_lock(); get_cpu(); })
236#define disk_stat_unlock() do { put_cpu(); rcu_read_unlock(); } while (0) 229#define part_stat_unlock() do { put_cpu(); rcu_read_unlock(); } while (0)
237
238#define disk_stat_add(cpu, gendiskp, field, addnd) \
239 (per_cpu_ptr(gendiskp->dkstats, cpu)->field += addnd)
240
241#define disk_stat_read(gendiskp, field) \
242({ \
243 typeof(gendiskp->dkstats->field) res = 0; \
244 int i; \
245 for_each_possible_cpu(i) \
246 res += per_cpu_ptr(gendiskp->dkstats, i)->field; \
247 res; \
248})
249
250static inline void disk_stat_set_all(struct gendisk *gendiskp, int value)
251{
252 int i;
253
254 for_each_possible_cpu(i)
255 memset(per_cpu_ptr(gendiskp->dkstats, i), value,
256 sizeof(struct disk_stats));
257}
258 230
259#define part_stat_add(cpu, part, field, addnd) \ 231#define __part_stat_add(cpu, part, field, addnd) \
260 (per_cpu_ptr(part->dkstats, cpu)->field += addnd) 232 (per_cpu_ptr((part)->dkstats, (cpu))->field += (addnd))
261
262#define all_stat_add(cpu, gendiskp, part, field, addnd, sector) \
263({ \
264 if (part) \
265 part_stat_add(cpu, part, field, addnd); \
266 disk_stat_add(cpu, gendiskp, field, addnd); \
267})
268 233
269#define part_stat_read(part, field) \ 234#define part_stat_read(part, field) \
270({ \ 235({ \
271 typeof(part->dkstats->field) res = 0; \ 236 typeof((part)->dkstats->field) res = 0; \
272 int i; \ 237 int i; \
273 for_each_possible_cpu(i) \ 238 for_each_possible_cpu(i) \
274 res += per_cpu_ptr(part->dkstats, i)->field; \ 239 res += per_cpu_ptr((part)->dkstats, i)->field; \
275 res; \ 240 res; \
276}) 241})
277 242
@@ -284,109 +249,73 @@ static inline void part_stat_set_all(struct hd_struct *part, int value)
284 sizeof(struct disk_stats)); 249 sizeof(struct disk_stats));
285} 250}
286 251
287#else /* !CONFIG_SMP */ 252static inline int init_part_stats(struct hd_struct *part)
288#define disk_stat_lock() ({ rcu_read_lock(); 0; })
289#define disk_stat_unlock() rcu_read_unlock()
290
291#define disk_stat_add(cpu, gendiskp, field, addnd) \
292 (gendiskp->dkstats.field += addnd)
293#define disk_stat_read(gendiskp, field) (gendiskp->dkstats.field)
294
295static inline void disk_stat_set_all(struct gendisk *gendiskp, int value)
296{ 253{
297 memset(&gendiskp->dkstats, value, sizeof (struct disk_stats)); 254 part->dkstats = alloc_percpu(struct disk_stats);
255 if (!part->dkstats)
256 return 0;
257 return 1;
298} 258}
299 259
300#define part_stat_add(cpu, part, field, addnd) \ 260static inline void free_part_stats(struct hd_struct *part)
301 (part->dkstats.field += addnd)
302
303#define all_stat_add(cpu, gendiskp, part, field, addnd, sector) \
304({ \
305 if (part) \
306 part_stat_add(cpu, part, field, addnd); \
307 disk_stat_add(cpu, gendiskp, field, addnd); \
308})
309
310#define part_stat_read(part, field) (part->dkstats.field)
311
312static inline void part_stat_set_all(struct hd_struct *part, int value)
313{ 261{
314 memset(&part->dkstats, value, sizeof(struct disk_stats)); 262 free_percpu(part->dkstats);
315} 263}
316 264
317#endif /* CONFIG_SMP */ 265#else /* !CONFIG_SMP */
318 266#define part_stat_lock() ({ rcu_read_lock(); 0; })
319#define disk_stat_dec(cpu, gendiskp, field) \ 267#define part_stat_unlock() rcu_read_unlock()
320 disk_stat_add(cpu, gendiskp, field, -1)
321#define disk_stat_inc(cpu, gendiskp, field) \
322 disk_stat_add(cpu, gendiskp, field, 1)
323#define disk_stat_sub(cpu, gendiskp, field, subnd) \
324 disk_stat_add(cpu, gendiskp, field, -subnd)
325
326#define part_stat_dec(cpu, gendiskp, field) \
327 part_stat_add(cpu, gendiskp, field, -1)
328#define part_stat_inc(cpu, gendiskp, field) \
329 part_stat_add(cpu, gendiskp, field, 1)
330#define part_stat_sub(cpu, gendiskp, field, subnd) \
331 part_stat_add(cpu, gendiskp, field, -subnd)
332 268
333#define all_stat_dec(cpu, gendiskp, field, sector) \ 269#define __part_stat_add(cpu, part, field, addnd) \
334 all_stat_add(cpu, gendiskp, field, -1, sector) 270 ((part)->dkstats.field += addnd)
335#define all_stat_inc(cpu, gendiskp, part, field, sector) \
336 all_stat_add(cpu, gendiskp, part, field, 1, sector)
337#define all_stat_sub(cpu, gendiskp, part, field, subnd, sector) \
338 all_stat_add(cpu, gendiskp, part, field, -subnd, sector)
339 271
340/* Inlines to alloc and free disk stats in struct gendisk */ 272#define part_stat_read(part, field) ((part)->dkstats.field)
341#ifdef CONFIG_SMP
342static inline int init_disk_stats(struct gendisk *disk)
343{
344 disk->dkstats = alloc_percpu(struct disk_stats);
345 if (!disk->dkstats)
346 return 0;
347 return 1;
348}
349 273
350static inline void free_disk_stats(struct gendisk *disk) 274static inline void part_stat_set_all(struct hd_struct *part, int value)
351{ 275{
352 free_percpu(disk->dkstats); 276 memset(&part->dkstats, value, sizeof(struct disk_stats));
353} 277}
354 278
355static inline int init_part_stats(struct hd_struct *part) 279static inline int init_part_stats(struct hd_struct *part)
356{ 280{
357 part->dkstats = alloc_percpu(struct disk_stats);
358 if (!part->dkstats)
359 return 0;
360 return 1; 281 return 1;
361} 282}
362 283
363static inline void free_part_stats(struct hd_struct *part) 284static inline void free_part_stats(struct hd_struct *part)
364{ 285{
365 free_percpu(part->dkstats);
366} 286}
367 287
368#else /* CONFIG_SMP */ 288#endif /* CONFIG_SMP */
369static inline int init_disk_stats(struct gendisk *disk)
370{
371 return 1;
372}
373 289
374static inline void free_disk_stats(struct gendisk *disk) 290#define part_stat_add(cpu, part, field, addnd) do { \
375{ 291 __part_stat_add((cpu), (part), field, addnd); \
376} 292 if ((part)->partno) \
293 __part_stat_add((cpu), &part_to_disk((part))->part0, \
294 field, addnd); \
295} while (0)
377 296
378static inline int init_part_stats(struct hd_struct *part) 297#define part_stat_dec(cpu, gendiskp, field) \
298 part_stat_add(cpu, gendiskp, field, -1)
299#define part_stat_inc(cpu, gendiskp, field) \
300 part_stat_add(cpu, gendiskp, field, 1)
301#define part_stat_sub(cpu, gendiskp, field, subnd) \
302 part_stat_add(cpu, gendiskp, field, -subnd)
303
304static inline void part_inc_in_flight(struct hd_struct *part)
379{ 305{
380 return 1; 306 part->in_flight++;
307 if (part->partno)
308 part_to_disk(part)->part0.in_flight++;
381} 309}
382 310
383static inline void free_part_stats(struct hd_struct *part) 311static inline void part_dec_in_flight(struct hd_struct *part)
384{ 312{
313 part->in_flight--;
314 if (part->partno)
315 part_to_disk(part)->part0.in_flight--;
385} 316}
386#endif /* CONFIG_SMP */
387 317
388/* drivers/block/ll_rw_blk.c */ 318/* drivers/block/ll_rw_blk.c */
389extern void disk_round_stats(int cpu, struct gendisk *disk);
390extern void part_round_stats(int cpu, struct hd_struct *part); 319extern void part_round_stats(int cpu, struct hd_struct *part);
391 320
392/* drivers/block/genhd.c */ 321/* drivers/block/genhd.c */
@@ -595,6 +524,8 @@ extern void blk_unregister_region(dev_t devt, unsigned long range);
595 524
596extern ssize_t part_size_show(struct device *dev, 525extern ssize_t part_size_show(struct device *dev,
597 struct device_attribute *attr, char *buf); 526 struct device_attribute *attr, char *buf);
527extern ssize_t part_stat_show(struct device *dev,
528 struct device_attribute *attr, char *buf);
598#ifdef CONFIG_FAIL_MAKE_REQUEST 529#ifdef CONFIG_FAIL_MAKE_REQUEST
599extern ssize_t part_fail_show(struct device *dev, 530extern ssize_t part_fail_show(struct device *dev,
600 struct device_attribute *attr, char *buf); 531 struct device_attribute *attr, char *buf);
generated by cgit v1.2.2 (git 2.25.0) at 2024-09-21 07:41:48 -0400