Merge branch 'for-3.9' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq

Pull workqueue changes from Tejun Heo:
 "A lot of reorganization is going on mostly to prepare for worker pools
  with custom attributes so that workqueue can replace custom pool
  implementations in places including writeback and btrfs and make CPU
  assignment in crypto more flexible.

  workqueue evolved from purely per-cpu design and implementation, so
  there are a lot of assumptions regarding being bound to CPUs and even
  unbound workqueues are implemented as an extension of the model -
  workqueues running on the special unbound CPU.  Bulk of changes this
  round are about promoting worker_pools as the top level abstraction
  replacing global_cwq (global cpu workqueue).  At this point, I'm
  fairly confident about getting custom worker pools working pretty soon
  and ready for the next merge window.

  Lai's patches are replacing the convoluted mb() dancing workqueue has
  been doing with much simpler mechanism which only depends on
  assignment atomicity of long.  For details, please read the commit
  message of 0b3dae68ac ("workqueue: simplify is-work-item-queued-here
  test").  While the change ends up adding one pointer to struct
  delayed_work, the inflation in percentage is less than five percent
  and it decouples delayed_work logic a lot more cleaner from usual work
  handling, removes the unusual memory barrier dancing, and allows for
  further simplification, so I think the trade-off is acceptable.

  There will be two more workqueue related pull requests and there are
  some shared commits among them.  I'll write further pull requests
  assuming this pull request is pulled first."

* 'for-3.9' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq: (37 commits)
  workqueue: un-GPL function delayed_work_timer_fn()
  workqueue: rename cpu_workqueue to pool_workqueue
  workqueue: reimplement is_chained_work() using current_wq_worker()
  workqueue: fix is_chained_work() regression
  workqueue: pick cwq instead of pool in __queue_work()
  workqueue: make get_work_pool_id() cheaper
  workqueue: move nr_running into worker_pool
  workqueue: cosmetic update in try_to_grab_pending()
  workqueue: simplify is-work-item-queued-here test
  workqueue: make work->data point to pool after try_to_grab_pending()
  workqueue: add delayed_work->wq to simplify reentrancy handling
  workqueue: make work_busy() test WORK_STRUCT_PENDING first
  workqueue: replace WORK_CPU_NONE/LAST with WORK_CPU_END
  workqueue: post global_cwq removal cleanups
  workqueue: rename nr_running variables
  workqueue: remove global_cwq
  workqueue: remove worker_pool->gcwq
  workqueue: replace for_each_worker_pool() with for_each_std_worker_pool()
  workqueue: make freezing/thawing per-pool
  workqueue: make hotplug processing per-pool
  ...

3 files changed, 28 insertions, 18 deletions

diff --git a/include/linux/async.h b/include/linux/async.h
index 7a24fe9b44b4..345169cfa304 100644
--- a/include/linux/async.h
+++ b/include/linux/async.h
@@ -52,4 +52,5 @@ extern void async_synchronize_full_domain(struct async_domain *domain);
 extern void async_synchronize_cookie(async_cookie_t cookie);
 extern void async_synchronize_cookie_domain(async_cookie_t cookie,
                                             struct async_domain *domain);
+extern bool current_is_async(void);
 #endif
diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index 2b58905d3504..8afab27cdbc2 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -27,7 +27,7 @@ void delayed_work_timer_fn(unsigned long __data);
 enum {
         WORK_STRUCT_PENDING_BIT = 0,    /* work item is pending execution */
         WORK_STRUCT_DELAYED_BIT = 1,    /* work item is delayed */
-        WORK_STRUCT_CWQ_BIT     = 2,    /* data points to cwq */
+        WORK_STRUCT_PWQ_BIT     = 2,    /* data points to pwq */
         WORK_STRUCT_LINKED_BIT  = 3,    /* next work is linked to this one */
 #ifdef CONFIG_DEBUG_OBJECTS_WORK
         WORK_STRUCT_STATIC_BIT  = 4,    /* static initializer (debugobjects) */
@@ -40,7 +40,7 @@ enum {
 
         WORK_STRUCT_PENDING     = 1 << WORK_STRUCT_PENDING_BIT,
         WORK_STRUCT_DELAYED     = 1 << WORK_STRUCT_DELAYED_BIT,
-        WORK_STRUCT_CWQ         = 1 << WORK_STRUCT_CWQ_BIT,
+        WORK_STRUCT_PWQ         = 1 << WORK_STRUCT_PWQ_BIT,
         WORK_STRUCT_LINKED      = 1 << WORK_STRUCT_LINKED_BIT,
 #ifdef CONFIG_DEBUG_OBJECTS_WORK
         WORK_STRUCT_STATIC      = 1 << WORK_STRUCT_STATIC_BIT,
@@ -57,29 +57,36 @@ enum {
 
         /* special cpu IDs */
         WORK_CPU_UNBOUND        = NR_CPUS,
-        WORK_CPU_NONE           = NR_CPUS + 1,
-        WORK_CPU_LAST           = WORK_CPU_NONE,
+        WORK_CPU_END            = NR_CPUS + 1,
 
         /*
-         * Reserve 7 bits off of cwq pointer w/ debugobjects turned
-         * off.  This makes cwqs aligned to 256 bytes and allows 15
-         * workqueue flush colors.
+         * Reserve 7 bits off of pwq pointer w/ debugobjects turned off.
+         * This makes pwqs aligned to 256 bytes and allows 15 workqueue
+         * flush colors.
          */
         WORK_STRUCT_FLAG_BITS   = WORK_STRUCT_COLOR_SHIFT +
                                   WORK_STRUCT_COLOR_BITS,
 
-        /* data contains off-queue information when !WORK_STRUCT_CWQ */
+        /* data contains off-queue information when !WORK_STRUCT_PWQ */
         WORK_OFFQ_FLAG_BASE     = WORK_STRUCT_FLAG_BITS,
 
         WORK_OFFQ_CANCELING     = (1 << WORK_OFFQ_FLAG_BASE),
 
+        /*
+         * When a work item is off queue, its high bits point to the last
+         * pool it was on.  Cap at 31 bits and use the highest number to
+         * indicate that no pool is associated.
+         */
         WORK_OFFQ_FLAG_BITS     = 1,
-        WORK_OFFQ_CPU_SHIFT     = WORK_OFFQ_FLAG_BASE + WORK_OFFQ_FLAG_BITS,
+        WORK_OFFQ_POOL_SHIFT    = WORK_OFFQ_FLAG_BASE + WORK_OFFQ_FLAG_BITS,
+        WORK_OFFQ_LEFT          = BITS_PER_LONG - WORK_OFFQ_POOL_SHIFT,
+        WORK_OFFQ_POOL_BITS     = WORK_OFFQ_LEFT <= 31 ? WORK_OFFQ_LEFT : 31,
+        WORK_OFFQ_POOL_NONE     = (1LU << WORK_OFFQ_POOL_BITS) - 1,
 
         /* convenience constants */
         WORK_STRUCT_FLAG_MASK   = (1UL << WORK_STRUCT_FLAG_BITS) - 1,
         WORK_STRUCT_WQ_DATA_MASK = ~WORK_STRUCT_FLAG_MASK,
-        WORK_STRUCT_NO_CPU      = (unsigned long)WORK_CPU_NONE << WORK_OFFQ_CPU_SHIFT,
+        WORK_STRUCT_NO_POOL     = (unsigned long)WORK_OFFQ_POOL_NONE << WORK_OFFQ_POOL_SHIFT,
 
         /* bit mask for work_busy() return values */
         WORK_BUSY_PENDING       = 1 << 0,
@@ -95,13 +102,16 @@ struct work_struct {
 #endif
 };
 
-#define WORK_DATA_INIT()        ATOMIC_LONG_INIT(WORK_STRUCT_NO_CPU)
+#define WORK_DATA_INIT()        ATOMIC_LONG_INIT(WORK_STRUCT_NO_POOL)
 #define WORK_DATA_STATIC_INIT() \
-        ATOMIC_LONG_INIT(WORK_STRUCT_NO_CPU | WORK_STRUCT_STATIC)
+        ATOMIC_LONG_INIT(WORK_STRUCT_NO_POOL | WORK_STRUCT_STATIC)
 
 struct delayed_work {
         struct work_struct work;
         struct timer_list timer;
+
+        /* target workqueue and CPU ->timer uses to queue ->work */
+        struct workqueue_struct *wq;
         int cpu;
 };
 
@@ -426,7 +436,6 @@ extern bool cancel_delayed_work_sync(struct delayed_work *dwork);
 extern void workqueue_set_max_active(struct workqueue_struct *wq,
                                      int max_active);
 extern bool workqueue_congested(unsigned int cpu, struct workqueue_struct *wq);
-extern unsigned int work_cpu(struct work_struct *work);
 extern unsigned int work_busy(struct work_struct *work);
 
 /*
diff --git a/include/trace/events/workqueue.h b/include/trace/events/workqueue.h
index f28d1b65f178..bf0e18ba6cfb 100644
--- a/include/trace/events/workqueue.h
+++ b/include/trace/events/workqueue.h
@@ -27,7 +27,7 @@ DECLARE_EVENT_CLASS(workqueue_work,
 /**
  * workqueue_queue_work - called when a work gets queued
  * @req_cpu:    the requested cpu
- * @cwq:        pointer to struct cpu_workqueue_struct
+ * @pwq:        pointer to struct pool_workqueue
  * @work:       pointer to struct work_struct
  *
  * This event occurs when a work is queued immediately or once a
@@ -36,10 +36,10 @@ DECLARE_EVENT_CLASS(workqueue_work,
  */
 TRACE_EVENT(workqueue_queue_work,
 
-        TP_PROTO(unsigned int req_cpu, struct cpu_workqueue_struct *cwq,
+        TP_PROTO(unsigned int req_cpu, struct pool_workqueue *pwq,
                  struct work_struct *work),
 
-        TP_ARGS(req_cpu, cwq, work),
+        TP_ARGS(req_cpu, pwq, work),
 
         TP_STRUCT__entry(
                 __field( void *,        work    )
@@ -52,9 +52,9 @@ TRACE_EVENT(workqueue_queue_work,
         TP_fast_assign(
                 __entry->work           = work;
                 __entry->function       = work->func;
-                __entry->workqueue      = cwq->wq;
+                __entry->workqueue      = pwq->wq;
                 __entry->req_cpu        = req_cpu;
-                __entry->cpu            = cwq->pool->gcwq->cpu;
+                __entry->cpu            = pwq->pool->cpu;
         ),
 
         TP_printk("work struct=%p function=%pf workqueue=%p req_cpu=%u cpu=%u",