Merge branch 'for-4.3/blkcg' of git://git.kernel.dk/linux-block

Pull blk-cg updates from Jens Axboe:
 "A bit later in the cycle, but this has been in the block tree for a a
  while.  This is basically four patchsets from Tejun, that improve our
  buffered cgroup writeback.  It was dependent on the other cgroup
  changes, but they went in earlier in this cycle.

  Series 1 is set of 5 patches that has cgroup writeback updates:

   - bdi_writeback iteration fix which could lead to some wb's being
     skipped or repeated during e.g. sync under memory pressure.

   - Simplification of wb work wait mechanism.

   - Writeback tracepoints updated to report cgroup.

  Series 2 is is a set of updates for the CFQ cgroup writeback handling:

     cfq has always charged all async IOs to the root cgroup.  It didn't
     have much choice as writeback didn't know about cgroups and there
     was no way to tell who to blame for a given writeback IO.
     writeback finally grew support for cgroups and now tags each
     writeback IO with the appropriate cgroup to charge it against.

     This patchset updates cfq so that it follows the blkcg each bio is
     tagged with.  Async cfq_queues are now shared across cfq_group,
     which is per-cgroup, instead of per-request_queue cfq_data.  This
     makes all IOs follow the weight based IO resource distribution
     implemented by cfq.

     - Switched from GFP_ATOMIC to GFP_NOWAIT as suggested by Jeff.

     - Other misc review points addressed, acks added and rebased.

  Series 3 is the blkcg policy cleanup patches:

     This patchset contains assorted cleanups for blkcg_policy methods
     and blk[c]g_policy_data handling.

     - alloc/free added for blkg_policy_data.  exit dropped.

     - alloc/free added for blkcg_policy_data.

     - blk-throttle's async percpu allocation is replaced with direct
       allocation.

     - all methods now take blk[c]g_policy_data instead of blkcg_gq or
       blkcg.

  And finally, series 4 is a set of patches cleaning up the blkcg stats
  handling:

    blkcg's stats have always been somwhat of a mess.  This patchset
    tries to improve the situation a bit.

     - The following patches added to consolidate blkcg entry point and
       blkg creation.  This is in itself is an improvement and helps
       colllecting common stats on bio issue.

     - per-blkg stats now accounted on bio issue rather than request
       completion so that bio based and request based drivers can behave
       the same way.  The issue was spotted by Vivek.

     - cfq-iosched implements custom recursive stats and blk-throttle
       implements custom per-cpu stats.  This patchset make blkcg core
       support both by default.

     - cfq-iosched and blk-throttle keep track of the same stats
       multiple times.  Unify them"

* 'for-4.3/blkcg' of git://git.kernel.dk/linux-block: (45 commits)
  blkcg: use CGROUP_WEIGHT_* scale for io.weight on the unified hierarchy
  blkcg: s/CFQ_WEIGHT_*/CFQ_WEIGHT_LEGACY_*/
  blkcg: implement interface for the unified hierarchy
  blkcg: misc preparations for unified hierarchy interface
  blkcg: separate out tg_conf_updated() from tg_set_conf()
  blkcg: move body parsing from blkg_conf_prep() to its callers
  blkcg: mark existing cftypes as legacy
  blkcg: rename subsystem name from blkio to io
  blkcg: refine error codes returned during blkcg configuration
  blkcg: remove unnecessary NULL checks from __cfqg_set_weight_device()
  blkcg: reduce stack usage of blkg_rwstat_recursive_sum()
  blkcg: remove cfqg_stats->sectors
  blkcg: move io_service_bytes and io_serviced stats into blkcg_gq
  blkcg: make blkg_[rw]stat_recursive_sum() to be able to index into blkcg_gq
  blkcg: make blkcg_[rw]stat per-cpu
  blkcg: add blkg_[rw]stat->aux_cnt and replace cfq_group->dead_stats with it
  blkcg: consolidate blkg creation in blkcg_bio_issue_check()
  blk-throttle: improve queue bypass handling
  blkcg: move root blkg lookup optimization from throtl_lookup_tg() to __blkg_lookup()
  blkcg: inline [__]blkg_lookup()
  ...

1 files changed, 43 insertions, 96 deletions

diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index ae0f438c2ee6..24489126f8ca 100644
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -53,8 +53,6 @@ struct wb_writeback_work {
         unsigned int for_background:1;
         unsigned int for_sync:1;        /* sync(2) WB_SYNC_ALL writeback */
         unsigned int auto_free:1;       /* free on completion */
-        unsigned int single_wait:1;
-        unsigned int single_done:1;
         enum wb_reason reason;          /* why was writeback initiated? */
 
         struct list_head list;          /* pending work list */
@@ -178,14 +176,11 @@ static void wb_wakeup(struct bdi_writeback *wb)
 static void wb_queue_work(struct bdi_writeback *wb,
                           struct wb_writeback_work *work)
 {
-        trace_writeback_queue(wb->bdi, work);
+        trace_writeback_queue(wb, work);
 
         spin_lock_bh(&wb->work_lock);
-        if (!test_bit(WB_registered, &wb->state)) {
-                if (work->single_wait)
-                        work->single_done = 1;
+        if (!test_bit(WB_registered, &wb->state))
                 goto out_unlock;
-        }
         if (work->done)
                 atomic_inc(&work->done->cnt);
         list_add_tail(&work->list, &wb->work_list);
@@ -706,7 +701,7 @@ EXPORT_SYMBOL_GPL(wbc_account_io);
 
 /**
  * inode_congested - test whether an inode is congested
- * @inode: inode to test for congestion
+ * @inode: inode to test for congestion (may be NULL)
  * @cong_bits: mask of WB_[a]sync_congested bits to test
  *
  * Tests whether @inode is congested.  @cong_bits is the mask of congestion
@@ -716,6 +711,9 @@ EXPORT_SYMBOL_GPL(wbc_account_io);
  * determined by whether the cgwb (cgroup bdi_writeback) for the blkcg
  * associated with @inode is congested; otherwise, the root wb's congestion
  * state is used.
+ *
+ * @inode is allowed to be NULL as this function is often called on
+ * mapping->host which is NULL for the swapper space.
  */
 int inode_congested(struct inode *inode, int cong_bits)
 {
@@ -738,32 +736,6 @@ int inode_congested(struct inode *inode, int cong_bits)
 EXPORT_SYMBOL_GPL(inode_congested);
 
 /**
- * wb_wait_for_single_work - wait for completion of a single bdi_writeback_work
- * @bdi: bdi the work item was issued to
- * @work: work item to wait for
- *
- * Wait for the completion of @work which was issued to one of @bdi's
- * bdi_writeback's.  The caller must have set @work->single_wait before
- * issuing it.  This wait operates independently fo
- * wb_wait_for_completion() and also disables automatic freeing of @work.
- */
-static void wb_wait_for_single_work(struct backing_dev_info *bdi,
-                                    struct wb_writeback_work *work)
-{
-        if (WARN_ON_ONCE(!work->single_wait))
-                return;
-
-        wait_event(bdi->wb_waitq, work->single_done);
-
-        /*
-         * Paired with smp_wmb() in wb_do_writeback() and ensures that all
-         * modifications to @work prior to assertion of ->single_done is
-         * visible to the caller once this function returns.
-         */
-        smp_rmb();
-}
-
-/**
  * wb_split_bdi_pages - split nr_pages to write according to bandwidth
  * @wb: target bdi_writeback to split @nr_pages to
  * @nr_pages: number of pages to write for the whole bdi
@@ -792,38 +764,6 @@ static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages)
 }
 
 /**
- * wb_clone_and_queue_work - clone a wb_writeback_work and issue it to a wb
- * @wb: target bdi_writeback
- * @base_work: source wb_writeback_work
- *
- * Try to make a clone of @base_work and issue it to @wb.  If cloning
- * succeeds, %true is returned; otherwise, @base_work is issued directly
- * and %false is returned.  In the latter case, the caller is required to
- * wait for @base_work's completion using wb_wait_for_single_work().
- *
- * A clone is auto-freed on completion.  @base_work never is.
- */
-static bool wb_clone_and_queue_work(struct bdi_writeback *wb,
-                                    struct wb_writeback_work *base_work)
-{
-        struct wb_writeback_work *work;
-
-        work = kmalloc(sizeof(*work), GFP_ATOMIC);
-        if (work) {
-                *work = *base_work;
-                work->auto_free = 1;
-                work->single_wait = 0;
-        } else {
-                work = base_work;
-                work->auto_free = 0;
-                work->single_wait = 1;
-        }
-        work->single_done = 0;
-        wb_queue_work(wb, work);
-        return work != base_work;
-}
-
-/**
  * bdi_split_work_to_wbs - split a wb_writeback_work to all wb's of a bdi
  * @bdi: target backing_dev_info
  * @base_work: wb_writeback_work to issue
@@ -838,15 +778,19 @@ static void bdi_split_work_to_wbs(struct backing_dev_info *bdi,
                                   struct wb_writeback_work *base_work,
                                   bool skip_if_busy)
 {
-        long nr_pages = base_work->nr_pages;
-        int next_blkcg_id = 0;
+        int next_memcg_id = 0;
         struct bdi_writeback *wb;
         struct wb_iter iter;
 
         might_sleep();
 restart:
         rcu_read_lock();
-        bdi_for_each_wb(wb, bdi, &iter, next_blkcg_id) {
+        bdi_for_each_wb(wb, bdi, &iter, next_memcg_id) {
+                DEFINE_WB_COMPLETION_ONSTACK(fallback_work_done);
+                struct wb_writeback_work fallback_work;
+                struct wb_writeback_work *work;
+                long nr_pages;
+
                 /* SYNC_ALL writes out I_DIRTY_TIME too */
                 if (!wb_has_dirty_io(wb) &&
                     (base_work->sync_mode == WB_SYNC_NONE ||
@@ -855,13 +799,30 @@ restart:
                 if (skip_if_busy && writeback_in_progress(wb))
                         continue;
 
-                base_work->nr_pages = wb_split_bdi_pages(wb, nr_pages);
-                if (!wb_clone_and_queue_work(wb, base_work)) {
-                        next_blkcg_id = wb->blkcg_css->id + 1;
-                        rcu_read_unlock();
-                        wb_wait_for_single_work(bdi, base_work);
-                        goto restart;
+                nr_pages = wb_split_bdi_pages(wb, base_work->nr_pages);
+
+                work = kmalloc(sizeof(*work), GFP_ATOMIC);
+                if (work) {
+                        *work = *base_work;
+                        work->nr_pages = nr_pages;
+                        work->auto_free = 1;
+                        wb_queue_work(wb, work);
+                        continue;
                 }
+
+                /* alloc failed, execute synchronously using on-stack fallback */
+                work = &fallback_work;
+                *work = *base_work;
+                work->nr_pages = nr_pages;
+                work->auto_free = 0;
+                work->done = &fallback_work_done;
+
+                wb_queue_work(wb, work);
+
+                next_memcg_id = wb->memcg_css->id + 1;
+                rcu_read_unlock();
+                wb_wait_for_completion(bdi, &fallback_work_done);
+                goto restart;
         }
         rcu_read_unlock();
 }
@@ -902,8 +863,6 @@ static void bdi_split_work_to_wbs(struct backing_dev_info *bdi,
 
         if (!skip_if_busy || !writeback_in_progress(&bdi->wb)) {
                 base_work->auto_free = 0;
-                base_work->single_wait = 0;
-                base_work->single_done = 0;
                 wb_queue_work(&bdi->wb, base_work);
         }
 }
@@ -924,7 +883,7 @@ void wb_start_writeback(struct bdi_writeback *wb, long nr_pages,
          */
         work = kzalloc(sizeof(*work), GFP_ATOMIC);
         if (!work) {
-                trace_writeback_nowork(wb->bdi);
+                trace_writeback_nowork(wb);
                 wb_wakeup(wb);
                 return;
         }
@@ -954,7 +913,7 @@ void wb_start_background_writeback(struct bdi_writeback *wb)
          * We just wake up the flusher thread. It will perform background
          * writeback as soon as there is no other work to do.
          */
-        trace_writeback_wake_background(wb->bdi);
+        trace_writeback_wake_background(wb);
         wb_wakeup(wb);
 }
 
@@ -1660,14 +1619,14 @@ static long wb_writeback(struct bdi_writeback *wb,
                 } else if (work->for_background)
                         oldest_jif = jiffies;
 
-                trace_writeback_start(wb->bdi, work);
+                trace_writeback_start(wb, work);
                 if (list_empty(&wb->b_io))
                         queue_io(wb, work);
                 if (work->sb)
                         progress = writeback_sb_inodes(work->sb, wb, work);
                 else
                         progress = __writeback_inodes_wb(wb, work);
-                trace_writeback_written(wb->bdi, work);
+                trace_writeback_written(wb, work);
 
                 wb_update_bandwidth(wb, wb_start);
 
@@ -1692,7 +1651,7 @@ static long wb_writeback(struct bdi_writeback *wb,
                  * we'll just busyloop.
                  */
                 if (!list_empty(&wb->b_more_io))  {
-                        trace_writeback_wait(wb->bdi, work);
+                        trace_writeback_wait(wb, work);
                         inode = wb_inode(wb->b_more_io.prev);
                         spin_lock(&inode->i_lock);
                         spin_unlock(&wb->list_lock);
@@ -1797,26 +1756,14 @@ static long wb_do_writeback(struct bdi_writeback *wb)
         set_bit(WB_writeback_running, &wb->state);
         while ((work = get_next_work_item(wb)) != NULL) {
                 struct wb_completion *done = work->done;
-                bool need_wake_up = false;
 
-                trace_writeback_exec(wb->bdi, work);
+                trace_writeback_exec(wb, work);
 
                 wrote += wb_writeback(wb, work);
 
-                if (work->single_wait) {
-                        WARN_ON_ONCE(work->auto_free);
-                        /* paired w/ rmb in wb_wait_for_single_work() */
-                        smp_wmb();
-                        work->single_done = 1;
-                        need_wake_up = true;
-                } else if (work->auto_free) {
+                if (work->auto_free)
                         kfree(work);
-                }
-
                 if (done && atomic_dec_and_test(&done->cnt))
-                        need_wake_up = true;
-
-                if (need_wake_up)
                         wake_up_all(&wb->bdi->wb_waitq);
         }