Merge branch 'for-3.6' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq

Pull workqueue changes from Tejun Heo:
 "There are three major changes.

   - WQ_HIGHPRI has been reimplemented so that high priority work items
     are served by worker threads with -20 nice value from dedicated
     highpri worker pools.

   - CPU hotplug support has been reimplemented such that idle workers
     are kept across CPU hotplug events.  This makes CPU hotplug cheaper
     (for PM) and makes the code simpler.

   - flush_kthread_work() has been reimplemented so that a work item can
     be freed while executing.  This removes an annoying behavior
     difference between kthread_worker and workqueue."

* 'for-3.6' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq:
  workqueue: fix spurious CPU locality WARN from process_one_work()
  kthread_worker: reimplement flush_kthread_work() to allow freeing the work item being executed
  kthread_worker: reorganize to prepare for flush_kthread_work() reimplementation
  workqueue: simplify CPU hotplug code
  workqueue: remove CPU offline trustee
  workqueue: don't butcher idle workers on an offline CPU
  workqueue: reimplement CPU online rebinding to handle idle workers
  workqueue: drop @bind from create_worker()
  workqueue: use mutex for global_cwq manager exclusion
  workqueue: ROGUE workers are UNBOUND workers
  workqueue: drop CPU_DYING notifier operation
  workqueue: perform cpu down operations from low priority cpu_notifier()
  workqueue: reimplement WQ_HIGHPRI using a separate worker_pool
  workqueue: introduce NR_WORKER_POOLS and for_each_worker_pool()
  workqueue: separate out worker_pool flags
  workqueue: use @pool instead of @gcwq or @cpu where applicable
  workqueue: factor out worker_pool from global_cwq
  workqueue: don't use WQ_HIGHPRI for unbound workqueues

1 files changed, 52 insertions, 36 deletions

diff --git a/kernel/kthread.c b/kernel/kthread.c
index 3d3de633702e..b579af57ea10 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -360,16 +360,12 @@ repeat:
                                         struct kthread_work, node);
                 list_del_init(&work->node);
         }
+        worker->current_work = work;
         spin_unlock_irq(&worker->lock);
 
         if (work) {
                 __set_current_state(TASK_RUNNING);
                 work->func(work);
-                smp_wmb();      /* wmb worker-b0 paired with flush-b1 */
-                work->done_seq = work->queue_seq;
-                smp_mb();       /* mb worker-b1 paired with flush-b0 */
-                if (atomic_read(&work->flushing))
-                        wake_up_all(&work->done);
         } else if (!freezing(current))
                 schedule();
 
@@ -378,6 +374,19 @@ repeat:
 }
 EXPORT_SYMBOL_GPL(kthread_worker_fn);
 
+/* insert @work before @pos in @worker */
+static void insert_kthread_work(struct kthread_worker *worker,
+                               struct kthread_work *work,
+                               struct list_head *pos)
+{
+        lockdep_assert_held(&worker->lock);
+
+        list_add_tail(&work->node, pos);
+        work->worker = worker;
+        if (likely(worker->task))
+                wake_up_process(worker->task);
+}
+
 /**
  * queue_kthread_work - queue a kthread_work
  * @worker: target kthread_worker
@@ -395,10 +404,7 @@ bool queue_kthread_work(struct kthread_worker *worker,
 
         spin_lock_irqsave(&worker->lock, flags);
         if (list_empty(&work->node)) {
-                list_add_tail(&work->node, &worker->work_list);
-                work->queue_seq++;
-                if (likely(worker->task))
-                        wake_up_process(worker->task);
+                insert_kthread_work(worker, work, &worker->work_list);
                 ret = true;
         }
         spin_unlock_irqrestore(&worker->lock, flags);
@@ -406,6 +412,18 @@ bool queue_kthread_work(struct kthread_worker *worker,
 }
 EXPORT_SYMBOL_GPL(queue_kthread_work);
 
+struct kthread_flush_work {
+        struct kthread_work     work;
+        struct completion       done;
+};
+
+static void kthread_flush_work_fn(struct kthread_work *work)
+{
+        struct kthread_flush_work *fwork =
+                container_of(work, struct kthread_flush_work, work);
+        complete(&fwork->done);
+}
+
 /**
  * flush_kthread_work - flush a kthread_work
  * @work: work to flush
@@ -414,39 +432,37 @@ EXPORT_SYMBOL_GPL(queue_kthread_work);
  */
 void flush_kthread_work(struct kthread_work *work)
 {
-        int seq = work->queue_seq;
-
-        atomic_inc(&work->flushing);
+        struct kthread_flush_work fwork = {
+                KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
+                COMPLETION_INITIALIZER_ONSTACK(fwork.done),
+        };
+        struct kthread_worker *worker;
+        bool noop = false;
 
-        /*
-         * mb flush-b0 paired with worker-b1, to make sure either
-         * worker sees the above increment or we see done_seq update.
-         */
-        smp_mb__after_atomic_inc();
+retry:
+        worker = work->worker;
+        if (!worker)
+                return;
 
-        /* A - B <= 0 tests whether B is in front of A regardless of overflow */
-        wait_event(work->done, seq - work->done_seq <= 0);
-        atomic_dec(&work->flushing);
+        spin_lock_irq(&worker->lock);
+        if (work->worker != worker) {
+                spin_unlock_irq(&worker->lock);
+                goto retry;
+        }
 
-        /*
-         * rmb flush-b1 paired with worker-b0, to make sure our caller
-         * sees every change made by work->func().
-         */
-        smp_mb__after_atomic_dec();
-}
-EXPORT_SYMBOL_GPL(flush_kthread_work);
+        if (!list_empty(&work->node))
+                insert_kthread_work(worker, &fwork.work, work->node.next);
+        else if (worker->current_work == work)
+                insert_kthread_work(worker, &fwork.work, worker->work_list.next);
+        else
+                noop = true;
 
-struct kthread_flush_work {
-        struct kthread_work     work;
-        struct completion       done;
-};
+        spin_unlock_irq(&worker->lock);
 
-static void kthread_flush_work_fn(struct kthread_work *work)
-{
-        struct kthread_flush_work *fwork =
-                container_of(work, struct kthread_flush_work, work);
-        complete(&fwork->done);
+        if (!noop)
+                wait_for_completion(&fwork.done);
 }
+EXPORT_SYMBOL_GPL(flush_kthread_work);
 
 /**
  * flush_kthread_worker - flush all current works on a kthread_worker