block: move stats from disk to part0

Move stats related fields - stamp, in_flight, dkstats - from disk to
part0 and unify stat handling such that...

* part_stat_*() now updates part0 together if the specified partition
  is not part0.  ie. part_stat_*() are now essentially all_stat_*().

* {disk|all}_stat_*() are gone.

* part_round_stats() is updated similary.  It handles part0 stats
  automatically and disk_round_stats() is killed.

* part_{inc|dec}_in_fligh() is implemented which automatically updates
  part0 stats for parts other than part0.

* disk_map_sector_rcu() is updated to return part0 if no part matches.
  Combined with the above changes, this makes NULL special case
  handling in callers unnecessary.

* Separate stats show code paths for disk are collapsed into part
  stats show code paths.

* Rename disk_stat_lock/unlock() to part_stat_lock/unlock()

While at it, reposition stat handling macros a bit and add missing
parentheses around macro parameters.

Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

1 files changed, 45 insertions, 114 deletions

diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index 3d15b42dc352..c90e1b4fbe5a 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -145,13 +145,6 @@ struct gendisk {
         struct timer_rand_state *random;
 
         atomic_t sync_io;               /* RAID */
-        unsigned long stamp;
-        int in_flight;
-#ifdef  CONFIG_SMP
-        struct disk_stats *dkstats;
-#else
-        struct disk_stats dkstats;
-#endif
         struct work_struct async_notify;
 #ifdef  CONFIG_BLK_DEV_INTEGRITY
         struct blk_integrity *integrity;
@@ -232,46 +225,18 @@ extern struct hd_struct *disk_map_sector_rcu(struct gendisk *disk,
  * internal use only.
  */
 #ifdef  CONFIG_SMP
-#define disk_stat_lock()        ({ rcu_read_lock(); get_cpu(); })
-#define disk_stat_unlock()      do { put_cpu(); rcu_read_unlock(); } while (0)
-
-#define disk_stat_add(cpu, gendiskp, field, addnd)                      \
-        (per_cpu_ptr(gendiskp->dkstats, cpu)->field += addnd)
-
-#define disk_stat_read(gendiskp, field)                                 \
-({                                                                      \
-        typeof(gendiskp->dkstats->field) res = 0;                       \
-        int i;                                                          \
-        for_each_possible_cpu(i)                                        \
-                res += per_cpu_ptr(gendiskp->dkstats, i)->field;        \
-        res;                                                            \
-})
-
-static inline void disk_stat_set_all(struct gendisk *gendiskp, int value)
-{
-        int i;
-
-        for_each_possible_cpu(i)
-                memset(per_cpu_ptr(gendiskp->dkstats, i), value,
-                                sizeof(struct disk_stats));
-}               
+#define part_stat_lock()        ({ rcu_read_lock(); get_cpu(); })
+#define part_stat_unlock()      do { put_cpu(); rcu_read_unlock(); } while (0)
 
-#define part_stat_add(cpu, part, field, addnd)                          \
-        (per_cpu_ptr(part->dkstats, cpu)->field += addnd)
-
-#define all_stat_add(cpu, gendiskp, part, field, addnd, sector)         \
-({                                                                      \
-        if (part)                                                       \
-                part_stat_add(cpu, part, field, addnd);                 \
-        disk_stat_add(cpu, gendiskp, field, addnd);                     \
-})
+#define __part_stat_add(cpu, part, field, addnd)                        \
+        (per_cpu_ptr((part)->dkstats, (cpu))->field += (addnd))
 
 #define part_stat_read(part, field)                                     \
 ({                                                                      \
-        typeof(part->dkstats->field) res = 0;                           \
+        typeof((part)->dkstats->field) res = 0;                         \
         int i;                                                          \
         for_each_possible_cpu(i)                                        \
-                res += per_cpu_ptr(part->dkstats, i)->field;            \
+                res += per_cpu_ptr((part)->dkstats, i)->field;          \
         res;                                                            \
 })
 
@@ -284,109 +249,73 @@ static inline void part_stat_set_all(struct hd_struct *part, int value)
                                 sizeof(struct disk_stats));
 }
 
-#else /* !CONFIG_SMP */
-#define disk_stat_lock()        ({ rcu_read_lock(); 0; })
-#define disk_stat_unlock()      rcu_read_unlock()
-
-#define disk_stat_add(cpu, gendiskp, field, addnd)                      \
-        (gendiskp->dkstats.field += addnd)
-#define disk_stat_read(gendiskp, field) (gendiskp->dkstats.field)
-
-static inline void disk_stat_set_all(struct gendisk *gendiskp, int value)
+static inline int init_part_stats(struct hd_struct *part)
 {
-        memset(&gendiskp->dkstats, value, sizeof (struct disk_stats));
+        part->dkstats = alloc_percpu(struct disk_stats);
+        if (!part->dkstats)
+                return 0;
+        return 1;
 }
 
-#define part_stat_add(cpu, part, field, addnd)                          \
-        (part->dkstats.field += addnd)
-
-#define all_stat_add(cpu, gendiskp, part, field, addnd, sector)         \
-({                                                                      \
-        if (part)                                                       \
-                part_stat_add(cpu, part, field, addnd);                 \
-        disk_stat_add(cpu, gendiskp, field, addnd);                     \
-})
-
-#define part_stat_read(part, field)     (part->dkstats.field)
-
-static inline void part_stat_set_all(struct hd_struct *part, int value)
+static inline void free_part_stats(struct hd_struct *part)
 {
-        memset(&part->dkstats, value, sizeof(struct disk_stats));
+        free_percpu(part->dkstats);
 }
 
-#endif /* CONFIG_SMP */
-
-#define disk_stat_dec(cpu, gendiskp, field)                             \
-        disk_stat_add(cpu, gendiskp, field, -1)
-#define disk_stat_inc(cpu, gendiskp, field)                             \
-        disk_stat_add(cpu, gendiskp, field, 1)
-#define disk_stat_sub(cpu, gendiskp, field, subnd)                      \
-        disk_stat_add(cpu, gendiskp, field, -subnd)
-
-#define part_stat_dec(cpu, gendiskp, field)                             \
-        part_stat_add(cpu, gendiskp, field, -1)
-#define part_stat_inc(cpu, gendiskp, field)                             \
-        part_stat_add(cpu, gendiskp, field, 1)
-#define part_stat_sub(cpu, gendiskp, field, subnd)                      \
-        part_stat_add(cpu, gendiskp, field, -subnd)
+#else /* !CONFIG_SMP */
+#define part_stat_lock()        ({ rcu_read_lock(); 0; })
+#define part_stat_unlock()      rcu_read_unlock()
 
-#define all_stat_dec(cpu, gendiskp, field, sector)                      \
-        all_stat_add(cpu, gendiskp, field, -1, sector)
-#define all_stat_inc(cpu, gendiskp, part, field, sector)                \
-        all_stat_add(cpu, gendiskp, part, field, 1, sector)
-#define all_stat_sub(cpu, gendiskp, part, field, subnd, sector)         \
-        all_stat_add(cpu, gendiskp, part, field, -subnd, sector)
+#define __part_stat_add(cpu, part, field, addnd)                                \
+        ((part)->dkstats.field += addnd)
 
-/* Inlines to alloc and free disk stats in struct gendisk */
-#ifdef  CONFIG_SMP
-static inline int init_disk_stats(struct gendisk *disk)
-{
-        disk->dkstats = alloc_percpu(struct disk_stats);
-        if (!disk->dkstats)
-                return 0;
-        return 1;
-}
+#define part_stat_read(part, field)     ((part)->dkstats.field)
 
-static inline void free_disk_stats(struct gendisk *disk)
+static inline void part_stat_set_all(struct hd_struct *part, int value)
 {
-        free_percpu(disk->dkstats);
+        memset(&part->dkstats, value, sizeof(struct disk_stats));
 }
 
 static inline int init_part_stats(struct hd_struct *part)
 {
-        part->dkstats = alloc_percpu(struct disk_stats);
-        if (!part->dkstats)
-                return 0;
         return 1;
 }
 
 static inline void free_part_stats(struct hd_struct *part)
 {
-        free_percpu(part->dkstats);
 }
 
-#else   /* CONFIG_SMP */
-static inline int init_disk_stats(struct gendisk *disk)
-{
-        return 1;
-}
+#endif /* CONFIG_SMP */
 
-static inline void free_disk_stats(struct gendisk *disk)
-{
-}
+#define part_stat_add(cpu, part, field, addnd)  do {                    \
+        __part_stat_add((cpu), (part), field, addnd);                   \
+        if ((part)->partno)                                             \
+                __part_stat_add((cpu), &part_to_disk((part))->part0,    \
+                                field, addnd);                          \
+} while (0)
 
-static inline int init_part_stats(struct hd_struct *part)
+#define part_stat_dec(cpu, gendiskp, field)                             \
+        part_stat_add(cpu, gendiskp, field, -1)
+#define part_stat_inc(cpu, gendiskp, field)                             \
+        part_stat_add(cpu, gendiskp, field, 1)
+#define part_stat_sub(cpu, gendiskp, field, subnd)                      \
+        part_stat_add(cpu, gendiskp, field, -subnd)
+
+static inline void part_inc_in_flight(struct hd_struct *part)
 {
-        return 1;
+        part->in_flight++;
+        if (part->partno)
+                part_to_disk(part)->part0.in_flight++;
 }
 
-static inline void free_part_stats(struct hd_struct *part)
+static inline void part_dec_in_flight(struct hd_struct *part)
 {
+        part->in_flight--;
+        if (part->partno)
+                part_to_disk(part)->part0.in_flight--;
 }
-#endif  /* CONFIG_SMP */
 
 /* drivers/block/ll_rw_blk.c */
-extern void disk_round_stats(int cpu, struct gendisk *disk);
 extern void part_round_stats(int cpu, struct hd_struct *part);
 
 /* drivers/block/genhd.c */
@@ -595,6 +524,8 @@ extern void blk_unregister_region(dev_t devt, unsigned long range);
 
 extern ssize_t part_size_show(struct device *dev,
                               struct device_attribute *attr, char *buf);
+extern ssize_t part_stat_show(struct device *dev,
+                              struct device_attribute *attr, char *buf);
 #ifdef CONFIG_FAIL_MAKE_REQUEST
 extern ssize_t part_fail_show(struct device *dev,
                               struct device_attribute *attr, char *buf);