block: move stats from disk to part0

Move stats related fields - stamp, in_flight, dkstats - from disk to
part0 and unify stat handling such that...

* part_stat_*() now updates part0 together if the specified partition
  is not part0.  ie. part_stat_*() are now essentially all_stat_*().

* {disk|all}_stat_*() are gone.

* part_round_stats() is updated similary.  It handles part0 stats
  automatically and disk_round_stats() is killed.

* part_{inc|dec}_in_fligh() is implemented which automatically updates
  part0 stats for parts other than part0.

* disk_map_sector_rcu() is updated to return part0 if no part matches.
  Combined with the above changes, this makes NULL special case
  handling in callers unnecessary.

* Separate stats show code paths for disk are collapsed into part
  stats show code paths.

* Rename disk_stat_lock/unlock() to part_stat_lock/unlock()

While at it, reposition stat handling macros a bit and add missing
parentheses around macro parameters.

Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

3 files changed, 62 insertions, 131 deletions

diff --git a/block/blk-core.c b/block/blk-core.c
index 505ec61067df..98138f002524 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -61,21 +61,17 @@ static void drive_stat_acct(struct request *rq, int new_io)
         if (!blk_fs_request(rq) || !rq->rq_disk)
                 return;
 
-        cpu = disk_stat_lock();
+        cpu = part_stat_lock();
         part = disk_map_sector_rcu(rq->rq_disk, rq->sector);
 
         if (!new_io)
-                all_stat_inc(cpu, rq->rq_disk, part, merges[rw], rq->sector);
+                part_stat_inc(cpu, part, merges[rw]);
         else {
-                disk_round_stats(cpu, rq->rq_disk);
-                rq->rq_disk->in_flight++;
-                if (part) {
-                        part_round_stats(cpu, part);
-                        part->in_flight++;
-                }
+                part_round_stats(cpu, part);
+                part_inc_in_flight(part);
         }
 
-        disk_stat_unlock();
+        part_stat_unlock();
 }
 
 void blk_queue_congestion_threshold(struct request_queue *q)
@@ -983,8 +979,22 @@ static inline void add_request(struct request_queue *q, struct request *req)
         __elv_add_request(q, req, ELEVATOR_INSERT_SORT, 0);
 }
 
-/*
- * disk_round_stats()   - Round off the performance stats on a struct
+static void part_round_stats_single(int cpu, struct hd_struct *part,
+                                    unsigned long now)
+{
+        if (now == part->stamp)
+                return;
+
+        if (part->in_flight) {
+                __part_stat_add(cpu, part, time_in_queue,
+                                part->in_flight * (now - part->stamp));
+                __part_stat_add(cpu, part, io_ticks, (now - part->stamp));
+        }
+        part->stamp = now;
+}
+
+/**
+ * part_round_stats()   - Round off the performance stats on a struct
  * disk_stats.
  *
  * The average IO queue length and utilisation statistics are maintained
@@ -998,36 +1008,15 @@ static inline void add_request(struct request_queue *q, struct request *req)
  * /proc/diskstats.  This accounts immediately for all queue usage up to
  * the current jiffies and restarts the counters again.
  */
-void disk_round_stats(int cpu, struct gendisk *disk)
-{
-        unsigned long now = jiffies;
-
-        if (now == disk->stamp)
-                return;
-
-        if (disk->in_flight) {
-                disk_stat_add(cpu, disk, time_in_queue,
-                              disk->in_flight * (now - disk->stamp));
-                disk_stat_add(cpu, disk, io_ticks, (now - disk->stamp));
-        }
-        disk->stamp = now;
-}
-EXPORT_SYMBOL_GPL(disk_round_stats);
-
 void part_round_stats(int cpu, struct hd_struct *part)
 {
         unsigned long now = jiffies;
 
-        if (now == part->stamp)
-                return;
-
-        if (part->in_flight) {
-                part_stat_add(cpu, part, time_in_queue,
-                              part->in_flight * (now - part->stamp));
-                part_stat_add(cpu, part, io_ticks, (now - part->stamp));
-        }
-        part->stamp = now;
+        if (part->partno)
+                part_round_stats_single(cpu, &part_to_disk(part)->part0, now);
+        part_round_stats_single(cpu, part, now);
 }
+EXPORT_SYMBOL_GPL(part_round_stats);
 
 /*
  * queue lock must be held
@@ -1567,11 +1556,10 @@ static int __end_that_request_first(struct request *req, int error,
                 struct hd_struct *part;
                 int cpu;
 
-                cpu = disk_stat_lock();
+                cpu = part_stat_lock();
                 part = disk_map_sector_rcu(req->rq_disk, req->sector);
-                all_stat_add(cpu, req->rq_disk, part, sectors[rw],
-                             nr_bytes >> 9, req->sector);
-                disk_stat_unlock();
+                part_stat_add(cpu, part, sectors[rw], nr_bytes >> 9);
+                part_stat_unlock();
         }
 
         total_bytes = bio_nbytes = 0;
@@ -1758,19 +1746,15 @@ static void end_that_request_last(struct request *req, int error)
                 struct hd_struct *part;
                 int cpu;
 
-                cpu = disk_stat_lock();
+                cpu = part_stat_lock();
                 part = disk_map_sector_rcu(disk, req->sector);
 
-                all_stat_inc(cpu, disk, part, ios[rw], req->sector);
-                all_stat_add(cpu, disk, part, ticks[rw], duration, req->sector);
-                disk_round_stats(cpu, disk);
-                disk->in_flight--;
-                if (part) {
-                        part_round_stats(cpu, part);
-                        part->in_flight--;
-                }
+                part_stat_inc(cpu, part, ios[rw]);
+                part_stat_add(cpu, part, ticks[rw], duration);
+                part_round_stats(cpu, part);
+                part_dec_in_flight(part);
 
-                disk_stat_unlock();
+                part_stat_unlock();
         }
 
         if (req->end_io)
diff --git a/block/blk-merge.c b/block/blk-merge.c
index d926a24bf1fd..c77196d55899 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -390,17 +390,13 @@ static int attempt_merge(struct request_queue *q, struct request *req,
                 struct hd_struct *part;
                 int cpu;
 
-                cpu = disk_stat_lock();
+                cpu = part_stat_lock();
                 part = disk_map_sector_rcu(req->rq_disk, req->sector);
 
-                disk_round_stats(cpu, req->rq_disk);
-                req->rq_disk->in_flight--;
-                if (part) {
-                        part_round_stats(cpu, part);
-                        part->in_flight--;
-                }
+                part_round_stats(cpu, part);
+                part_dec_in_flight(part);
 
-                disk_stat_unlock();
+                part_stat_unlock();
         }
 
         req->ioprio = ioprio_best(req->ioprio, next->ioprio);
diff --git a/block/genhd.c b/block/genhd.c
index 06a252f2b967..e1cb96fb883e 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -176,7 +176,7 @@ EXPORT_SYMBOL_GPL(disk_part_iter_exit);
  * while preemption is disabled.
  *
  * RETURNS:
- * Found partition on success, NULL if there's no matching partition.
+ * Found partition on success, part0 is returned if no partition matches
  */
 struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector)
 {
@@ -189,7 +189,7 @@ struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector)
                     sector < part->start_sect + part->nr_sects)
                         return part;
         }
-        return NULL;
+        return &disk->part0;
 }
 EXPORT_SYMBOL_GPL(disk_map_sector_rcu);
 
@@ -580,24 +580,24 @@ void __init printk_all_partitions(void)
                  * numbers in hex - the same format as the root=
                  * option takes.
                  */
-                printk("%s %10llu %s",
-                       bdevt_str(disk_devt(disk), devt_buf),
-                       (unsigned long long)get_capacity(disk) >> 1,
-                       disk_name(disk, 0, name_buf));
-                if (disk->driverfs_dev != NULL &&
-                    disk->driverfs_dev->driver != NULL)
-                        printk(" driver: %s\n",
-                               disk->driverfs_dev->driver->name);
-                else
-                        printk(" (driver?)\n");
+                disk_part_iter_init(&piter, disk, DISK_PITER_INCL_PART0);
+                while ((part = disk_part_iter_next(&piter))) {
+                        bool is_part0 = part == &disk->part0;
 
-                /* now show the partitions */
-                disk_part_iter_init(&piter, disk, 0);
-                while ((part = disk_part_iter_next(&piter)))
-                        printk("  %s %10llu %s\n",
+                        printk("%s%s %10llu %s", is_part0 ? "" : "  ",
                                bdevt_str(part_devt(part), devt_buf),
                                (unsigned long long)part->nr_sects >> 1,
                                disk_name(disk, part->partno, name_buf));
+                        if (is_part0) {
+                                if (disk->driverfs_dev != NULL &&
+                                    disk->driverfs_dev->driver != NULL)
+                                        printk(" driver: %s\n",
+                                              disk->driverfs_dev->driver->name);
+                                else
+                                        printk(" (driver?)\n");
+                        } else
+                                printk("\n");
+                }
                 disk_part_iter_exit(&piter);
         }
         class_dev_iter_exit(&iter);
@@ -674,12 +674,7 @@ static int show_partition(struct seq_file *seqf, void *v)
                 return 0;
 
         /* show the full disk and all non-0 size partitions of it */
-        seq_printf(seqf, "%4d  %7d %10llu %s\n",
-                MAJOR(disk_devt(sgp)), MINOR(disk_devt(sgp)),
-                (unsigned long long)get_capacity(sgp) >> 1,
-                disk_name(sgp, 0, buf));
-
-        disk_part_iter_init(&piter, sgp, 0);
+        disk_part_iter_init(&piter, sgp, DISK_PITER_INCL_PART0);
         while ((part = disk_part_iter_next(&piter)))
                 seq_printf(seqf, "%4d  %7d %10llu %s\n",
                            MAJOR(part_devt(part)), MINOR(part_devt(part)),
@@ -768,40 +763,13 @@ static ssize_t disk_capability_show(struct device *dev,
         return sprintf(buf, "%x\n", disk->flags);
 }
 
-static ssize_t disk_stat_show(struct device *dev,
-                              struct device_attribute *attr, char *buf)
-{
-        struct gendisk *disk = dev_to_disk(dev);
-        int cpu;
-
-        cpu = disk_stat_lock();
-        disk_round_stats(cpu, disk);
-        disk_stat_unlock();
-        return sprintf(buf,
-                "%8lu %8lu %8llu %8u "
-                "%8lu %8lu %8llu %8u "
-                "%8u %8u %8u"
-                "\n",
-                disk_stat_read(disk, ios[READ]),
-                disk_stat_read(disk, merges[READ]),
-                (unsigned long long)disk_stat_read(disk, sectors[READ]),
-                jiffies_to_msecs(disk_stat_read(disk, ticks[READ])),
-                disk_stat_read(disk, ios[WRITE]),
-                disk_stat_read(disk, merges[WRITE]),
-                (unsigned long long)disk_stat_read(disk, sectors[WRITE]),
-                jiffies_to_msecs(disk_stat_read(disk, ticks[WRITE])),
-                disk->in_flight,
-                jiffies_to_msecs(disk_stat_read(disk, io_ticks)),
-                jiffies_to_msecs(disk_stat_read(disk, time_in_queue)));
-}
-
 static DEVICE_ATTR(range, S_IRUGO, disk_range_show, NULL);
 static DEVICE_ATTR(ext_range, S_IRUGO, disk_ext_range_show, NULL);
 static DEVICE_ATTR(removable, S_IRUGO, disk_removable_show, NULL);
 static DEVICE_ATTR(ro, S_IRUGO, disk_ro_show, NULL);
 static DEVICE_ATTR(size, S_IRUGO, part_size_show, NULL);
 static DEVICE_ATTR(capability, S_IRUGO, disk_capability_show, NULL);
-static DEVICE_ATTR(stat, S_IRUGO, disk_stat_show, NULL);
+static DEVICE_ATTR(stat, S_IRUGO, part_stat_show, NULL);
 #ifdef CONFIG_FAIL_MAKE_REQUEST
 static struct device_attribute dev_attr_fail =
         __ATTR(make-it-fail, S_IRUGO|S_IWUSR, part_fail_show, part_fail_store);
@@ -836,7 +804,7 @@ static void disk_release(struct device *dev)
 
         kfree(disk->random);
         kfree(disk->__part);
-        free_disk_stats(disk);
+        free_part_stats(&disk->part0);
         kfree(disk);
 }
 struct class block_class = {
@@ -873,28 +841,11 @@ static int diskstats_show(struct seq_file *seqf, void *v)
                                 "\n\n");
         */
  
-        cpu = disk_stat_lock();
-        disk_round_stats(cpu, gp);
-        disk_stat_unlock();
-        seq_printf(seqf, "%4d %7d %s %lu %lu %llu %u %lu %lu %llu %u %u %u %u\n",
-                MAJOR(disk_devt(gp)), MINOR(disk_devt(gp)),
-                disk_name(gp, 0, buf),
-                disk_stat_read(gp, ios[0]), disk_stat_read(gp, merges[0]),
-                (unsigned long long)disk_stat_read(gp, sectors[0]),
-                jiffies_to_msecs(disk_stat_read(gp, ticks[0])),
-                disk_stat_read(gp, ios[1]), disk_stat_read(gp, merges[1]),
-                (unsigned long long)disk_stat_read(gp, sectors[1]),
-                jiffies_to_msecs(disk_stat_read(gp, ticks[1])),
-                gp->in_flight,
-                jiffies_to_msecs(disk_stat_read(gp, io_ticks)),
-                jiffies_to_msecs(disk_stat_read(gp, time_in_queue)));
-
-        /* now show all non-0 size partitions of it */
-        disk_part_iter_init(&piter, gp, 0);
+        disk_part_iter_init(&piter, gp, DISK_PITER_INCL_PART0);
         while ((hd = disk_part_iter_next(&piter))) {
-                cpu = disk_stat_lock();
+                cpu = part_stat_lock();
                 part_round_stats(cpu, hd);
-                disk_stat_unlock();
+                part_stat_unlock();
                 seq_printf(seqf, "%4d %7d %s %lu %lu %llu "
                            "%u %lu %lu %llu %u %u %u %u\n",
                            MAJOR(part_devt(hd)), MINOR(part_devt(hd)),
@@ -1000,7 +951,7 @@ struct gendisk *alloc_disk_ext_node(int minors, int ext_minors, int node_id)
                 int tot_minors = minors + ext_minors;
                 int size = tot_minors * sizeof(struct hd_struct *);
 
-                if (!init_disk_stats(disk)) {
+                if (!init_part_stats(&disk->part0)) {
                         kfree(disk);
                         return NULL;
                 }
@@ -1008,7 +959,7 @@ struct gendisk *alloc_disk_ext_node(int minors, int ext_minors, int node_id)
                 disk->__part = kmalloc_node(size, GFP_KERNEL | __GFP_ZERO,
                                             node_id);
                 if (!disk->__part) {
-                        free_disk_stats(disk);
+                                free_part_stats(&disk->part0);
                         kfree(disk);
                         return NULL;
                 }