workqueue: remove CPU offline trustee

With the previous changes, a disassociated global_cwq now can run as
an unbound one on its own - it can create workers as necessary to
drain remaining works after the CPU has been brought down and manage
the number of workers using the usual idle timer mechanism making
trustee completely redundant except for the actual unbinding
operation.

This patch removes the trustee and let a disassociated global_cwq
manage itself.  Unbinding is moved to a work item (for CPU affinity)
which is scheduled and flushed from CPU_DONW_PREPARE.

This patch moves nr_running clearing outside gcwq and manager locks to
simplify the code.  As nr_running is unused at the point, this is
safe.

Signed-off-by: Tejun Heo <tj@kernel.org>
Acked-by: "Rafael J. Wysocki" <rjw@sisk.pl>

1 files changed, 36 insertions, 252 deletions

diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index acfabb22e2c4..d1545daa74ad 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -79,13 +79,6 @@ enum {
         WORKER_NOT_RUNNING      = WORKER_PREP | WORKER_REBIND | WORKER_UNBOUND |
                                   WORKER_CPU_INTENSIVE,
 
-        /* gcwq->trustee_state */
-        TRUSTEE_START           = 0,            /* start */
-        TRUSTEE_IN_CHARGE       = 1,            /* trustee in charge of gcwq */
-        TRUSTEE_BUTCHER         = 2,            /* butcher workers */
-        TRUSTEE_RELEASE         = 3,            /* release workers */
-        TRUSTEE_DONE            = 4,            /* trustee is done */
-
         NR_WORKER_POOLS         = 2,            /* # worker pools per gcwq */
 
         BUSY_WORKER_HASH_ORDER  = 6,            /* 64 pointers */
@@ -100,7 +93,6 @@ enum {
                                                    (min two ticks) */
         MAYDAY_INTERVAL         = HZ / 10,      /* and then every 100ms */
         CREATE_COOLDOWN         = HZ,           /* time to breath after fail */
-        TRUSTEE_COOLDOWN        = HZ / 10,      /* for trustee draining */
 
         /*
          * Rescue workers are used only on emergencies and shared by
@@ -194,10 +186,6 @@ struct global_cwq {
         struct worker_pool      pools[2];       /* normal and highpri pools */
 
         wait_queue_head_t       rebind_hold;    /* rebind hold wait */
-
-        struct task_struct      *trustee;       /* L: for gcwq shutdown */
-        unsigned int            trustee_state;  /* L: trustee state */
-        wait_queue_head_t       trustee_wait;   /* trustee wait */
 } ____cacheline_aligned_in_smp;
 
 /*
@@ -753,11 +741,11 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task,
          * worklist not empty test sequence is in insert_work().
          * Please read comment there.
          *
-         * NOT_RUNNING is clear.  This means that trustee is not in
-         * charge and we're running on the local cpu w/ rq lock held
-         * and preemption disabled, which in turn means that none else
-         * could be manipulating idle_list, so dereferencing idle_list
-         * without gcwq lock is safe.
+         * NOT_RUNNING is clear.  This means that we're bound to and
+         * running on the local cpu w/ rq lock held and preemption
+         * disabled, which in turn means that none else could be
+         * manipulating idle_list, so dereferencing idle_list without gcwq
+         * lock is safe.
          */
         if (atomic_dec_and_test(nr_running) && !list_empty(&pool->worklist))
                 to_wakeup = first_worker(pool);
@@ -1217,19 +1205,16 @@ static void worker_enter_idle(struct worker *worker)
         /* idle_list is LIFO */
         list_add(&worker->entry, &pool->idle_list);
 
-        if (likely(gcwq->trustee_state != TRUSTEE_DONE)) {
-                if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
-                        mod_timer(&pool->idle_timer,
-                                  jiffies + IDLE_WORKER_TIMEOUT);
-        } else
-                wake_up_all(&gcwq->trustee_wait);
+        if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
+                mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
 
         /*
-         * Sanity check nr_running.  Because trustee releases gcwq->lock
-         * between setting %WORKER_UNBOUND and zapping nr_running, the
-         * warning may trigger spuriously.  Check iff trustee is idle.
+         * Sanity check nr_running.  Because gcwq_unbind_fn() releases
+         * gcwq->lock between setting %WORKER_UNBOUND and zapping
+         * nr_running, the warning may trigger spuriously.  Check iff
+         * unbind is not in progress.
          */
-        WARN_ON_ONCE(gcwq->trustee_state == TRUSTEE_DONE &&
+        WARN_ON_ONCE(!(gcwq->flags & GCWQ_DISASSOCIATED) &&
                      pool->nr_workers == pool->nr_idle &&
                      atomic_read(get_pool_nr_running(pool)));
 }
@@ -3367,46 +3352,9 @@ EXPORT_SYMBOL_GPL(work_busy);
  * gcwqs serve mix of short, long and very long running works making
  * blocked draining impractical.
  *
- * This is solved by allowing a gcwq to be detached from CPU, running it
- * with unbound workers and allowing it to be reattached later if the cpu
- * comes back online.  A separate thread is created to govern a gcwq in
- * such state and is called the trustee of the gcwq.
- *
- * Trustee states and their descriptions.
- *
- * START        Command state used on startup.  On CPU_DOWN_PREPARE, a
- *              new trustee is started with this state.
- *
- * IN_CHARGE    Once started, trustee will enter this state after
- *              assuming the manager role and making all existing
- *              workers rogue.  DOWN_PREPARE waits for trustee to
- *              enter this state.  After reaching IN_CHARGE, trustee
- *              tries to execute the pending worklist until it's empty
- *              and the state is set to BUTCHER, or the state is set
- *              to RELEASE.
- *
- * BUTCHER      Command state which is set by the cpu callback after
- *              the cpu has went down.  Once this state is set trustee
- *              knows that there will be no new works on the worklist
- *              and once the worklist is empty it can proceed to
- *              killing idle workers.
- *
- * RELEASE      Command state which is set by the cpu callback if the
- *              cpu down has been canceled or it has come online
- *              again.  After recognizing this state, trustee stops
- *              trying to drain or butcher and clears ROGUE, rebinds
- *              all remaining workers back to the cpu and releases
- *              manager role.
- *
- * DONE         Trustee will enter this state after BUTCHER or RELEASE
- *              is complete.
- *
- *          trustee                 CPU                draining
- *         took over                down               complete
- * START -----------> IN_CHARGE -----------> BUTCHER -----------> DONE
- *                        |                     |                  ^
- *                        | CPU is back online  v   return workers |
- *                         ----------------> RELEASE --------------
+ * This is solved by allowing a gcwq to be disassociated from the CPU
+ * running as an unbound one and allowing it to be reattached later if the
+ * cpu comes back online.
  */
 
 /* claim manager positions of all pools */
@@ -3427,61 +3375,11 @@ static void gcwq_release_management(struct global_cwq *gcwq)
                 mutex_unlock(&pool->manager_mutex);
 }
 
-/**
- * trustee_wait_event_timeout - timed event wait for trustee
- * @cond: condition to wait for
- * @timeout: timeout in jiffies
- *
- * wait_event_timeout() for trustee to use.  Handles locking and
- * checks for RELEASE request.
- *
- * CONTEXT:
- * spin_lock_irq(gcwq->lock) which may be released and regrabbed
- * multiple times.  To be used by trustee.
- *
- * RETURNS:
- * Positive indicating left time if @cond is satisfied, 0 if timed
- * out, -1 if canceled.
- */
-#define trustee_wait_event_timeout(cond, timeout) ({                    \
-        long __ret = (timeout);                                         \
-        while (!((cond) || (gcwq->trustee_state == TRUSTEE_RELEASE)) && \
-               __ret) {                                                 \
-                spin_unlock_irq(&gcwq->lock);                           \
-                __wait_event_timeout(gcwq->trustee_wait, (cond) ||      \
-                        (gcwq->trustee_state == TRUSTEE_RELEASE),       \
-                        __ret);                                         \
-                spin_lock_irq(&gcwq->lock);                             \
-        }                                                               \
-        gcwq->trustee_state == TRUSTEE_RELEASE ? -1 : (__ret);          \
-})
-
-/**
- * trustee_wait_event - event wait for trustee
- * @cond: condition to wait for
- *
- * wait_event() for trustee to use.  Automatically handles locking and
- * checks for CANCEL request.
- *
- * CONTEXT:
- * spin_lock_irq(gcwq->lock) which may be released and regrabbed
- * multiple times.  To be used by trustee.
- *
+static void gcwq_unbind_fn(struct work_struct *work)