1 files changed, 47 insertions, 35 deletions

diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 987293d03ebc..b010eac595d2 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -305,6 +305,9 @@ static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
 /* I: attributes used when instantiating standard unbound pools on demand */
 static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
 
+/* I: attributes used when instantiating ordered pools on demand */
+static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
+
 struct workqueue_struct *system_wq __read_mostly;
 EXPORT_SYMBOL(system_wq);
 struct workqueue_struct *system_highpri_wq __read_mostly;
@@ -518,14 +521,21 @@ static inline void debug_work_activate(struct work_struct *work) { }
 static inline void debug_work_deactivate(struct work_struct *work) { }
 #endif
 
-/* allocate ID and assign it to @pool */
+/**
+ * worker_pool_assign_id - allocate ID and assing it to @pool
+ * @pool: the pool pointer of interest
+ *
+ * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
+ * successfully, -errno on failure.
+ */
 static int worker_pool_assign_id(struct worker_pool *pool)
 {
         int ret;
 
         lockdep_assert_held(&wq_pool_mutex);
 
-        ret = idr_alloc(&worker_pool_idr, pool, 0, 0, GFP_KERNEL);
+        ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
+                        GFP_KERNEL);
         if (ret >= 0) {
                 pool->id = ret;
                 return 0;
@@ -1320,7 +1330,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq,
 
         debug_work_activate(work);
 
-        /* if dying, only works from the same workqueue are allowed */
+        /* if draining, only works from the same workqueue are allowed */
         if (unlikely(wq->flags & __WQ_DRAINING) &&
             WARN_ON_ONCE(!is_chained_work(wq)))
                 return;
@@ -1736,16 +1746,17 @@ static struct worker *create_worker(struct worker_pool *pool)
         if (IS_ERR(worker->task))
                 goto fail;
 
+        set_user_nice(worker->task, pool->attrs->nice);
+
+        /* prevent userland from meddling with cpumask of workqueue workers */
+        worker->task->flags |= PF_NO_SETAFFINITY;
+
         /*
          * set_cpus_allowed_ptr() will fail if the cpumask doesn't have any
          * online CPUs.  It'll be re-applied when any of the CPUs come up.
          */
-        set_user_nice(worker->task, pool->attrs->nice);
         set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
 
-        /* prevent userland from meddling with cpumask of workqueue workers */
-        worker->task->flags |= PF_NO_SETAFFINITY;
-
         /*
          * The caller is responsible for ensuring %POOL_DISASSOCIATED
          * remains stable across this function.  See the comments above the
@@ -2840,19 +2851,6 @@ already_gone:
         return false;
 }
 
-static bool __flush_work(struct work_struct *work)
-{
-        struct wq_barrier barr;
-
-        if (start_flush_work(work, &barr)) {
-                wait_for_completion(&barr.done);
-                destroy_work_on_stack(&barr.work);
-                return true;
-        } else {
-                return false;
-        }
-}
-
 /**
  * flush_work - wait for a work to finish executing the last queueing instance
  * @work: the work to flush
@@ -2866,10 +2864,18 @@ static bool __flush_work(struct work_struct *work)
  */
 bool flush_work(struct work_struct *work)
 {
+        struct wq_barrier barr;
+
         lock_map_acquire(&work->lockdep_map);
         lock_map_release(&work->lockdep_map);
 
-        return __flush_work(work);
+        if (start_flush_work(work, &barr)) {
+                wait_for_completion(&barr.done);
+                destroy_work_on_stack(&barr.work);
+                return true;
+        } else {
+                return false;
+        }
 }
 EXPORT_SYMBOL_GPL(flush_work);
 
@@ -4106,7 +4112,7 @@ out_unlock:
 static int alloc_and_link_pwqs(struct workqueue_struct *wq)
 {
         bool highpri = wq->flags & WQ_HIGHPRI;
-        int cpu;
+        int cpu, ret;
 
         if (!(wq->flags & WQ_UNBOUND)) {
                 wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
@@ -4126,6 +4132,13 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq)
                         mutex_unlock(&wq->mutex);
                 }
                 return 0;
+        } else if (wq->flags & __WQ_ORDERED) {
+                ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
+                /* there should only be single pwq for ordering guarantee */
+                WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||
+                              wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
+                     "ordering guarantee broken for workqueue %s\n", wq->name);
+                return ret;
         } else {
                 return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
         }
@@ -4814,14 +4827,7 @@ long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
 
         INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
         schedule_work_on(cpu, &wfc.work);
-
-        /*
-         * The work item is on-stack and can't lead to deadlock through
-         * flushing.  Use __flush_work() to avoid spurious lockdep warnings
-         * when work_on_cpu()s are nested.
-         */
-        __flush_work(&wfc.work);
-
+        flush_work(&wfc.work);
         return wfc.ret;
 }
 EXPORT_SYMBOL_GPL(work_on_cpu);
@@ -5009,10 +5015,6 @@ static int __init init_workqueues(void)
         int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
         int i, cpu;
 
-        /* make sure we have enough bits for OFFQ pool ID */
-        BUILD_BUG_ON((1LU << (BITS_PER_LONG - WORK_OFFQ_POOL_SHIFT)) <
-                     WORK_CPU_END * NR_STD_WORKER_POOLS);
-
         WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
 
         pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
@@ -5051,13 +5053,23 @@ static int __init init_workqueues(void)
                 }
         }
 
-        /* create default unbound wq attrs */
+        /* create default unbound and ordered wq attrs */
         for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
                 struct workqueue_attrs *attrs;
 
                 BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
                 attrs->nice = std_nice[i];
                 unbound_std_wq_attrs[i] = attrs;
+
+                /*
+                 * An ordered wq should have only one pwq as ordering is
+                 * guaranteed by max_active which is enforced by pwqs.
+                 * Turn off NUMA so that dfl_pwq is used for all nodes.
+                 */
+                BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
+                attrs->nice = std_nice[i];
+                attrs->no_numa = true;
+                ordered_wq_attrs[i] = attrs;
         }
 
         system_wq = alloc_workqueue("events", 0, 0);