3 files changed, 76 insertions, 17 deletions

diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index 26cc1df280d6..91d416f9c0a7 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -358,6 +358,8 @@ extern struct workqueue_struct *system_freezable_wq;
 extern struct workqueue_struct *system_power_efficient_wq;
 extern struct workqueue_struct *system_freezable_power_efficient_wq;
 
+extern bool wq_online;
+
 extern struct workqueue_struct *
 __alloc_workqueue_key(const char *fmt, unsigned int flags, int max_active,
         struct lock_class_key *key, const char *lock_name, ...) __printf(1, 6);
@@ -594,7 +596,7 @@ static inline bool schedule_delayed_work(struct delayed_work *dwork,
  */
 static inline bool keventd_up(void)
 {
-        return system_wq != NULL;
+        return wq_online;
 }
 
 #ifndef CONFIG_SMP
@@ -631,4 +633,7 @@ int workqueue_online_cpu(unsigned int cpu);
 int workqueue_offline_cpu(unsigned int cpu);
 #endif
 
+int __init workqueue_init_early(void);
+int __init workqueue_init(void);
+
 #endif
diff --git a/init/main.c b/init/main.c
index a8a58e2794a5..5c4fd68a8671 100644
--- a/init/main.c
+++ b/init/main.c
@@ -551,6 +551,14 @@ asmlinkage __visible void __init start_kernel(void)
                  "Interrupts were enabled *very* early, fixing it\n"))
                 local_irq_disable();
         idr_init_cache();
+
+        /*
+         * Allow workqueue creation and work item queueing/cancelling
+         * early.  Work item execution depends on kthreads and starts after
+         * workqueue_init().
+         */
+        workqueue_init_early();
+
         rcu_init();
 
         /* trace_printk() and trace points may be used after this */
@@ -1005,6 +1013,8 @@ static noinline void __init kernel_init_freeable(void)
 
         smp_prepare_cpus(setup_max_cpus);
 
+        workqueue_init();
+
         do_pre_smp_initcalls();
         lockup_detector_init();
 
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 4eaec8b86d65..15d0811c9e91 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -290,6 +290,8 @@ module_param_named(disable_numa, wq_disable_numa, bool, 0444);
 static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
 module_param_named(power_efficient, wq_power_efficient, bool, 0444);
 
+bool wq_online;                         /* can kworkers be created yet? */
+
 static bool wq_numa_enabled;            /* unbound NUMA affinity enabled */
 
 /* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
@@ -2583,6 +2585,9 @@ void flush_workqueue(struct workqueue_struct *wq)
         };
         int next_color;
 
+        if (WARN_ON(!wq_online))
+                return;
+
         lock_map_acquire(&wq->lockdep_map);
         lock_map_release(&wq->lockdep_map);
 
@@ -2843,6 +2848,9 @@ bool flush_work(struct work_struct *work)
 {
         struct wq_barrier barr;
 
+        if (WARN_ON(!wq_online))
+                return false;
+
         lock_map_acquire(&work->lockdep_map);
         lock_map_release(&work->lockdep_map);
 
@@ -2913,7 +2921,13 @@ static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
         mark_work_canceling(work);
         local_irq_restore(flags);
 
-        flush_work(work);
+        /*
+         * This allows canceling during early boot.  We know that @work
+         * isn't executing.
+         */
+        if (wq_online)
+                flush_work(work);
+
         clear_work_data(work);
 
         /*
@@ -3352,7 +3366,7 @@ static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
                 goto fail;
 
         /* create and start the initial worker */
-        if (!create_worker(pool))
+        if (wq_online && !create_worker(pool))
                 goto fail;
 
         /* install */
@@ -3417,6 +3431,7 @@ static void pwq_adjust_max_active(struct pool_workqueue *pwq)
 {
         struct workqueue_struct *wq = pwq->wq;
         bool freezable = wq->flags & WQ_FREEZABLE;
+        unsigned long flags;
 
         /* for @wq->saved_max_active */
         lockdep_assert_held(&wq->mutex);
@@ -3425,7 +3440,8 @@ static void pwq_adjust_max_active(struct pool_workqueue *pwq)
         if (!freezable && pwq->max_active == wq->saved_max_active)
                 return;
 
-        spin_lock_irq(&pwq->pool->lock);
+        /* this function can be called during early boot w/ irq disabled */
+        spin_lock_irqsave(&pwq->pool->lock, flags);
 
         /*
          * During [un]freezing, the caller is responsible for ensuring that
@@ -3448,7 +3464,7 @@ static void pwq_adjust_max_active(struct pool_workqueue *pwq)
                 pwq->max_active = 0;
         }
 
-        spin_unlock_irq(&pwq->pool->lock);
+        spin_unlock_irqrestore(&pwq->pool->lock, flags);
 }
 
 /* initialize newly alloced @pwq which is associated with @wq and @pool */
@@ -5457,7 +5473,17 @@ static void __init wq_numa_init(void)
         wq_numa_enabled = true;
 }
 
-static int __init init_workqueues(void)
+/**
+ * workqueue_init_early - early init for workqueue subsystem
+ *
+ * This is the first half of two-staged workqueue subsystem initialization
+ * and invoked as soon as the bare basics - memory allocation, cpumasks and
+ * idr are up.  It sets up all the data structures and system workqueues
+ * and allows early boot code to create workqueues and queue/cancel work
+ * items.  Actual work item execution starts only after kthreads can be
+ * created and scheduled right before early initcalls.
+ */
+int __init workqueue_init_early(void)
 {
         int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
         int i, cpu;
@@ -5490,16 +5516,6 @@ static int __init init_workqueues(void)
                 }
         }
 
-        /* create the initial worker */
-        for_each_online_cpu(cpu) {
-                struct worker_pool *pool;
-
-                for_each_cpu_worker_pool(pool, cpu) {
-                        pool->flags &= ~POOL_DISASSOCIATED;
-                        BUG_ON(!create_worker(pool));
-                }
-        }
-
         /* create default unbound and ordered wq attrs */
         for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
                 struct workqueue_attrs *attrs;
@@ -5536,8 +5552,36 @@ static int __init init_workqueues(void)
                !system_power_efficient_wq ||
                !system_freezable_power_efficient_wq);
 
+        return 0;
+}
+
+/**
+ * workqueue_init - bring workqueue subsystem fully online
+ *
+ * This is the latter half of two-staged workqueue subsystem initialization
+ * and invoked as soon as kthreads can be created and scheduled.
+ * Workqueues have been created and work items queued on them, but there
+ * are no kworkers executing the work items yet.  Populate the worker pools
+ * with the initial workers and enable future kworker creations.
+ */
+int __init workqueue_init(void)
+{
+        struct worker_pool *pool;
+        int cpu, bkt;
+
+        /* create the initial workers */
+        for_each_online_cpu(cpu) {
+                for_each_cpu_worker_pool(pool, cpu) {
+                        pool->flags &= ~POOL_DISASSOCIATED;
+                        BUG_ON(!create_worker(pool));
+                }
+        }
+
+        hash_for_each(unbound_pool_hash, bkt, pool, hash_node)
+                BUG_ON(!create_worker(pool));
+
+        wq_online = true;
         wq_watchdog_init();
 
         return 0;
 }
-early_initcall(init_workqueues);