aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/workqueue.h15
-rw-r--r--kernel/workqueue.c218
2 files changed, 173 insertions, 60 deletions
diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index 139069a6286c..67ce734747f6 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -51,7 +51,8 @@ enum {
51 WORK_NO_COLOR = WORK_NR_COLORS, 51 WORK_NO_COLOR = WORK_NR_COLORS,
52 52
53 /* special cpu IDs */ 53 /* special cpu IDs */
54 WORK_CPU_NONE = NR_CPUS, 54 WORK_CPU_UNBOUND = NR_CPUS,
55 WORK_CPU_NONE = NR_CPUS + 1,
55 WORK_CPU_LAST = WORK_CPU_NONE, 56 WORK_CPU_LAST = WORK_CPU_NONE,
56 57
57 /* 58 /*
@@ -237,11 +238,17 @@ enum {
237 WQ_RESCUER = 1 << 3, /* has an rescue worker */ 238 WQ_RESCUER = 1 << 3, /* has an rescue worker */
238 WQ_HIGHPRI = 1 << 4, /* high priority */ 239 WQ_HIGHPRI = 1 << 4, /* high priority */
239 WQ_CPU_INTENSIVE = 1 << 5, /* cpu instensive workqueue */ 240 WQ_CPU_INTENSIVE = 1 << 5, /* cpu instensive workqueue */
241 WQ_UNBOUND = 1 << 6, /* not bound to any cpu */
240 242
241 WQ_MAX_ACTIVE = 512, /* I like 512, better ideas? */ 243 WQ_MAX_ACTIVE = 512, /* I like 512, better ideas? */
244 WQ_MAX_UNBOUND_PER_CPU = 4, /* 4 * #cpus for unbound wq */
242 WQ_DFL_ACTIVE = WQ_MAX_ACTIVE / 2, 245 WQ_DFL_ACTIVE = WQ_MAX_ACTIVE / 2,
243}; 246};
244 247
248/* unbound wq's aren't per-cpu, scale max_active according to #cpus */
249#define WQ_UNBOUND_MAX_ACTIVE \
250 max_t(int, WQ_MAX_ACTIVE, num_possible_cpus() * WQ_MAX_UNBOUND_PER_CPU)
251
245/* 252/*
246 * System-wide workqueues which are always present. 253 * System-wide workqueues which are always present.
247 * 254 *
@@ -256,10 +263,16 @@ enum {
256 * system_nrt_wq is non-reentrant and guarantees that any given work 263 * system_nrt_wq is non-reentrant and guarantees that any given work
257 * item is never executed in parallel by multiple CPUs. Queue 264 * item is never executed in parallel by multiple CPUs. Queue
258 * flushing might take relatively long. 265 * flushing might take relatively long.
266 *
267 * system_unbound_wq is unbound workqueue. Workers are not bound to
268 * any specific CPU, not concurrency managed, and all queued works are
269 * executed immediately as long as max_active limit is not reached and
270 * resources are available.
259 */ 271 */
260extern struct workqueue_struct *system_wq; 272extern struct workqueue_struct *system_wq;
261extern struct workqueue_struct *system_long_wq; 273extern struct workqueue_struct *system_long_wq;
262extern struct workqueue_struct *system_nrt_wq; 274extern struct workqueue_struct *system_nrt_wq;
275extern struct workqueue_struct *system_unbound_wq;
263 276
264extern struct workqueue_struct * 277extern struct workqueue_struct *
265__alloc_workqueue_key(const char *name, unsigned int flags, int max_active, 278__alloc_workqueue_key(const char *name, unsigned int flags, int max_active,
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index a105ddf55f79..4608563cdd63 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -53,9 +53,10 @@ enum {
53 WORKER_ROGUE = 1 << 4, /* not bound to any cpu */ 53 WORKER_ROGUE = 1 << 4, /* not bound to any cpu */
54 WORKER_REBIND = 1 << 5, /* mom is home, come back */ 54 WORKER_REBIND = 1 << 5, /* mom is home, come back */
55 WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */ 55 WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
56 WORKER_UNBOUND = 1 << 7, /* worker is unbound */
56 57
57 WORKER_NOT_RUNNING = WORKER_PREP | WORKER_ROGUE | WORKER_REBIND | 58 WORKER_NOT_RUNNING = WORKER_PREP | WORKER_ROGUE | WORKER_REBIND |
58 WORKER_CPU_INTENSIVE, 59 WORKER_CPU_INTENSIVE | WORKER_UNBOUND,
59 60
60 /* gcwq->trustee_state */ 61 /* gcwq->trustee_state */
61 TRUSTEE_START = 0, /* start */ 62 TRUSTEE_START = 0, /* start */
@@ -96,7 +97,7 @@ enum {
96 * X: During normal operation, modification requires gcwq->lock and 97 * X: During normal operation, modification requires gcwq->lock and
97 * should be done only from local cpu. Either disabling preemption 98 * should be done only from local cpu. Either disabling preemption
98 * on local cpu or grabbing gcwq->lock is enough for read access. 99 * on local cpu or grabbing gcwq->lock is enough for read access.
99 * While trustee is in charge, it's identical to L. 100 * If GCWQ_DISASSOCIATED is set, it's identical to L.
100 * 101 *
101 * F: wq->flush_mutex protected. 102 * F: wq->flush_mutex protected.
102 * 103 *
@@ -220,14 +221,52 @@ struct workqueue_struct {
220struct workqueue_struct *system_wq __read_mostly; 221struct workqueue_struct *system_wq __read_mostly;
221struct workqueue_struct *system_long_wq __read_mostly; 222struct workqueue_struct *system_long_wq __read_mostly;
222struct workqueue_struct *system_nrt_wq __read_mostly; 223struct workqueue_struct *system_nrt_wq __read_mostly;
224struct workqueue_struct *system_unbound_wq __read_mostly;
223EXPORT_SYMBOL_GPL(system_wq); 225EXPORT_SYMBOL_GPL(system_wq);
224EXPORT_SYMBOL_GPL(system_long_wq); 226EXPORT_SYMBOL_GPL(system_long_wq);
225EXPORT_SYMBOL_GPL(system_nrt_wq); 227EXPORT_SYMBOL_GPL(system_nrt_wq);
228EXPORT_SYMBOL_GPL(system_unbound_wq);
226 229
227#define for_each_busy_worker(worker, i, pos, gcwq) \ 230#define for_each_busy_worker(worker, i, pos, gcwq) \
228 for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) \ 231 for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) \
229 hlist_for_each_entry(worker, pos, &gcwq->busy_hash[i], hentry) 232 hlist_for_each_entry(worker, pos, &gcwq->busy_hash[i], hentry)
230 233
234static inline int __next_gcwq_cpu(int cpu, const struct cpumask *mask,
235 unsigned int sw)
236{
237 if (cpu < nr_cpu_ids) {
238 if (sw & 1) {
239 cpu = cpumask_next(cpu, mask);
240 if (cpu < nr_cpu_ids)
241 return cpu;
242 }
243 if (sw & 2)
244 return WORK_CPU_UNBOUND;
245 }
246 return WORK_CPU_NONE;
247}
248
249static inline int __next_wq_cpu(int cpu, const struct cpumask *mask,
250 struct workqueue_struct *wq)
251{
252 return __next_gcwq_cpu(cpu, mask, !(wq->flags & WQ_UNBOUND) ? 1 : 2);
253}
254
255#define for_each_gcwq_cpu(cpu) \
256 for ((cpu) = __next_gcwq_cpu(-1, cpu_possible_mask, 3); \
257 (cpu) < WORK_CPU_NONE; \
258 (cpu) = __next_gcwq_cpu((cpu), cpu_possible_mask, 3))
259
260#define for_each_online_gcwq_cpu(cpu) \
261 for ((cpu) = __next_gcwq_cpu(-1, cpu_online_mask, 3); \
262 (cpu) < WORK_CPU_NONE; \
263 (cpu) = __next_gcwq_cpu((cpu), cpu_online_mask, 3))
264
265#define for_each_cwq_cpu(cpu, wq) \
266 for ((cpu) = __next_wq_cpu(-1, cpu_possible_mask, (wq)); \
267 (cpu) < WORK_CPU_NONE; \
268 (cpu) = __next_wq_cpu((cpu), cpu_possible_mask, (wq)))
269
231#ifdef CONFIG_DEBUG_OBJECTS_WORK 270#ifdef CONFIG_DEBUG_OBJECTS_WORK
232 271
233static struct debug_obj_descr work_debug_descr; 272static struct debug_obj_descr work_debug_descr;
@@ -351,26 +390,46 @@ static bool workqueue_freezing; /* W: have wqs started freezing? */
351static DEFINE_PER_CPU(struct global_cwq, global_cwq); 390static DEFINE_PER_CPU(struct global_cwq, global_cwq);
352static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, gcwq_nr_running); 391static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, gcwq_nr_running);
353 392
393/*
394 * Global cpu workqueue and nr_running counter for unbound gcwq. The
395 * gcwq is always online, has GCWQ_DISASSOCIATED set, and all its
396 * workers have WORKER_UNBOUND set.
397 */
398static struct global_cwq unbound_global_cwq;
399static atomic_t unbound_gcwq_nr_running = ATOMIC_INIT(0); /* always 0 */
400
354static int worker_thread(void *__worker); 401static int worker_thread(void *__worker);
355 402
356static struct global_cwq *get_gcwq(unsigned int cpu) 403static struct global_cwq *get_gcwq(unsigned int cpu)
357{ 404{
358 return &per_cpu(global_cwq, cpu); 405 if (cpu != WORK_CPU_UNBOUND)
406 return &per_cpu(global_cwq, cpu);
407 else
408 return &unbound_global_cwq;
359} 409}
360 410
361static atomic_t *get_gcwq_nr_running(unsigned int cpu) 411static atomic_t *get_gcwq_nr_running(unsigned int cpu)
362{ 412{
363 return &per_cpu(gcwq_nr_running, cpu); 413 if (cpu != WORK_CPU_UNBOUND)
414 return &per_cpu(gcwq_nr_running, cpu);
415 else
416 return &unbound_gcwq_nr_running;
364} 417}
365 418
366static struct cpu_workqueue_struct *get_cwq(unsigned int cpu, 419static struct cpu_workqueue_struct *get_cwq(unsigned int cpu,
367 struct workqueue_struct *wq) 420 struct workqueue_struct *wq)
368{ 421{
369#ifndef CONFIG_SMP 422 if (!(wq->flags & WQ_UNBOUND)) {
370 return wq->cpu_wq.single; 423 if (likely(cpu < nr_cpu_ids)) {
424#ifdef CONFIG_SMP
425 return per_cpu_ptr(wq->cpu_wq.pcpu, cpu);
371#else 426#else
372 return per_cpu_ptr(wq->cpu_wq.pcpu, cpu); 427 return wq->cpu_wq.single;
373#endif 428#endif
429 }
430 } else if (likely(cpu == WORK_CPU_UNBOUND))
431 return wq->cpu_wq.single;
432 return NULL;
374} 433}
375 434
376static unsigned int work_color_to_flags(int color) 435static unsigned int work_color_to_flags(int color)
@@ -453,7 +512,7 @@ static struct global_cwq *get_work_gcwq(struct work_struct *work)
453 if (cpu == WORK_CPU_NONE) 512 if (cpu == WORK_CPU_NONE)
454 return NULL; 513 return NULL;
455 514
456 BUG_ON(cpu >= nr_cpu_ids); 515 BUG_ON(cpu >= nr_cpu_ids && cpu != WORK_CPU_UNBOUND);
457 return get_gcwq(cpu); 516 return get_gcwq(cpu);
458} 517}
459 518
@@ -869,11 +928,14 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,
869 928
870 debug_work_activate(work); 929 debug_work_activate(work);
871 930
931 if (unlikely(cpu == WORK_CPU_UNBOUND))
932 cpu = raw_smp_processor_id();
933
872 /* 934 /*
873 * Determine gcwq to use. SINGLE_CPU is inherently 935 * Determine gcwq to use. SINGLE_CPU is inherently
874 * NON_REENTRANT, so test it first. 936 * NON_REENTRANT, so test it first.
875 */ 937 */
876 if (!(wq->flags & WQ_SINGLE_CPU)) { 938 if (!(wq->flags & (WQ_SINGLE_CPU | WQ_UNBOUND))) {
877 struct global_cwq *last_gcwq; 939 struct global_cwq *last_gcwq;
878 940
879 /* 941 /*
@@ -900,7 +962,7 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,
900 } 962 }
901 } else 963 } else
902 spin_lock_irqsave(&gcwq->lock, flags); 964 spin_lock_irqsave(&gcwq->lock, flags);
903 } else { 965 } else if (!(wq->flags & WQ_UNBOUND)) {
904 unsigned int req_cpu = cpu; 966 unsigned int req_cpu = cpu;
905 967
906 /* 968 /*
@@ -932,6 +994,9 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,
932 spin_unlock_irqrestore(&gcwq->lock, flags); 994 spin_unlock_irqrestore(&gcwq->lock, flags);
933 goto retry; 995 goto retry;
934 } 996 }
997 } else {
998 gcwq = get_gcwq(WORK_CPU_UNBOUND);
999 spin_lock_irqsave(&gcwq->lock, flags);
935 } 1000 }
936 1001
937 /* gcwq determined, get cwq and queue */ 1002 /* gcwq determined, get cwq and queue */
@@ -1166,7 +1231,8 @@ static bool worker_maybe_bind_and_lock(struct worker *worker)
1166 * it races with cpu hotunplug operation. Verify 1231 * it races with cpu hotunplug operation. Verify
1167 * against GCWQ_DISASSOCIATED. 1232 * against GCWQ_DISASSOCIATED.
1168 */ 1233 */
1169 set_cpus_allowed_ptr(task, get_cpu_mask(gcwq->cpu)); 1234 if (!(gcwq->flags & GCWQ_DISASSOCIATED))
1235 set_cpus_allowed_ptr(task, get_cpu_mask(gcwq->cpu));
1170 1236
1171 spin_lock_irq(&gcwq->lock); 1237 spin_lock_irq(&gcwq->lock);
1172 if (gcwq->flags & GCWQ_DISASSOCIATED) 1238 if (gcwq->flags & GCWQ_DISASSOCIATED)
@@ -1231,8 +1297,9 @@ static struct worker *alloc_worker(void)
1231 */ 1297 */
1232static struct worker *create_worker(struct global_cwq *gcwq, bool bind) 1298static struct worker *create_worker(struct global_cwq *gcwq, bool bind)
1233{ 1299{
1234 int id = -1; 1300 bool on_unbound_cpu = gcwq->cpu == WORK_CPU_UNBOUND;
1235 struct worker *worker = NULL; 1301 struct worker *worker = NULL;
1302 int id = -1;
1236 1303
1237 spin_lock_irq(&gcwq->lock); 1304 spin_lock_irq(&gcwq->lock);
1238 while (ida_get_new(&gcwq->worker_ida, &id)) { 1305 while (ida_get_new(&gcwq->worker_ida, &id)) {
@@ -1250,8 +1317,12 @@ static struct worker *create_worker(struct global_cwq *gcwq, bool bind)
1250 worker->gcwq = gcwq; 1317 worker->gcwq = gcwq;
1251 worker->id = id; 1318 worker->id = id;
1252 1319
1253 worker->task = kthread_create(worker_thread, worker, "kworker/%u:%d", 1320 if (!on_unbound_cpu)
1254 gcwq->cpu, id); 1321 worker->task = kthread_create(worker_thread, worker,
1322 "kworker/%u:%d", gcwq->cpu, id);
1323 else
1324 worker->task = kthread_create(worker_thread, worker,
1325 "kworker/u:%d", id);
1255 if (IS_ERR(worker->task)) 1326 if (IS_ERR(worker->task))
1256 goto fail; 1327 goto fail;
1257 1328
@@ -1260,10 +1331,13 @@ static struct worker *create_worker(struct global_cwq *gcwq, bool bind)
1260 * online later on. Make sure every worker has 1331 * online later on. Make sure every worker has
1261 * PF_THREAD_BOUND set. 1332 * PF_THREAD_BOUND set.
1262 */ 1333 */
1263 if (bind) 1334 if (bind && !on_unbound_cpu)
1264 kthread_bind(worker->task, gcwq->cpu); 1335 kthread_bind(worker->task, gcwq->cpu);
1265 else 1336 else {
1266 worker->task->flags |= PF_THREAD_BOUND; 1337 worker->task->flags |= PF_THREAD_BOUND;
1338 if (on_unbound_cpu)
1339 worker->flags |= WORKER_UNBOUND;
1340 }
1267 1341
1268 return worker; 1342 return worker;
1269fail: 1343fail:
@@ -1358,12 +1432,17 @@ static bool send_mayday(struct work_struct *work)
1358{ 1432{
1359 struct cpu_workqueue_struct *cwq = get_work_cwq(work); 1433 struct cpu_workqueue_struct *cwq = get_work_cwq(work);
1360 struct workqueue_struct *wq = cwq->wq; 1434 struct workqueue_struct *wq = cwq->wq;
1435 unsigned int cpu;
1361 1436
1362 if (!(wq->flags & WQ_RESCUER)) 1437 if (!(wq->flags & WQ_RESCUER))
1363 return false; 1438 return false;
1364 1439
1365 /* mayday mayday mayday */ 1440 /* mayday mayday mayday */
1366 if (!cpumask_test_and_set_cpu(cwq->gcwq->cpu, wq->mayday_mask)) 1441 cpu = cwq->gcwq->cpu;
1442 /* WORK_CPU_UNBOUND can't be set in cpumask, use cpu 0 instead */
1443 if (cpu == WORK_CPU_UNBOUND)
1444 cpu = 0;
1445 if (!cpumask_test_and_set_cpu(cpu, wq->mayday_mask))
1367 wake_up_process(wq->rescuer->task); 1446 wake_up_process(wq->rescuer->task);
1368 return true; 1447 return true;
1369} 1448}
@@ -1882,6 +1961,7 @@ static int rescuer_thread(void *__wq)
1882 struct workqueue_struct *wq = __wq; 1961 struct workqueue_struct *wq = __wq;
1883 struct worker *rescuer = wq->rescuer; 1962 struct worker *rescuer = wq->rescuer;
1884 struct list_head *scheduled = &rescuer->scheduled; 1963 struct list_head *scheduled = &rescuer->scheduled;
1964 bool is_unbound = wq->flags & WQ_UNBOUND;
1885 unsigned int cpu; 1965 unsigned int cpu;
1886 1966
1887 set_user_nice(current, RESCUER_NICE_LEVEL); 1967 set_user_nice(current, RESCUER_NICE_LEVEL);
@@ -1891,8 +1971,13 @@ repeat:
1891 if (kthread_should_stop()) 1971 if (kthread_should_stop())
1892 return 0; 1972 return 0;
1893 1973
1974 /*
1975 * See whether any cpu is asking for help. Unbounded
1976 * workqueues use cpu 0 in mayday_mask for CPU_UNBOUND.
1977 */
1894 for_each_cpu(cpu, wq->mayday_mask) { 1978 for_each_cpu(cpu, wq->mayday_mask) {
1895 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); 1979 unsigned int tcpu = is_unbound ? WORK_CPU_UNBOUND : cpu;
1980 struct cpu_workqueue_struct *cwq = get_cwq(tcpu, wq);
1896 struct global_cwq *gcwq = cwq->gcwq; 1981 struct global_cwq *gcwq = cwq->gcwq;
1897 struct work_struct *work, *n; 1982 struct work_struct *work, *n;
1898 1983
@@ -2034,7 +2119,7 @@ static bool flush_workqueue_prep_cwqs(struct workqueue_struct *wq,
2034 atomic_set(&wq->nr_cwqs_to_flush, 1); 2119 atomic_set(&wq->nr_cwqs_to_flush, 1);
2035 } 2120 }
2036 2121
2037 for_each_possible_cpu(cpu) { 2122 for_each_cwq_cpu(cpu, wq) {
2038 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); 2123 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
2039 struct global_cwq *gcwq = cwq->gcwq; 2124 struct global_cwq *gcwq = cwq->gcwq;
2040 2125
@@ -2344,7 +2429,7 @@ static void wait_on_work(struct work_struct *work)
2344 lock_map_acquire(&work->lockdep_map); 2429 lock_map_acquire(&work->lockdep_map);
2345 lock_map_release(&work->lockdep_map); 2430 lock_map_release(&work->lockdep_map);
2346 2431
2347 for_each_possible_cpu(cpu) 2432 for_each_gcwq_cpu(cpu)
2348 wait_on_cpu_work(get_gcwq(cpu), work); 2433 wait_on_cpu_work(get_gcwq(cpu), work);
2349} 2434}
2350 2435
@@ -2590,23 +2675,25 @@ static int alloc_cwqs(struct workqueue_struct *wq)
2590 const size_t size = sizeof(struct cpu_workqueue_struct); 2675 const size_t size = sizeof(struct cpu_workqueue_struct);
2591 const size_t align = max_t(size_t, 1 << WORK_STRUCT_FLAG_BITS, 2676 const size_t align = max_t(size_t, 1 << WORK_STRUCT_FLAG_BITS,
2592 __alignof__(unsigned long long)); 2677 __alignof__(unsigned long long));
2593#ifndef CONFIG_SMP
2594 void *ptr;
2595 2678
2596 /* 2679 if (CONFIG_SMP && !(wq->flags & WQ_UNBOUND)) {
2597 * Allocate enough room to align cwq and put an extra pointer 2680 /* on SMP, percpu allocator can align itself */
2598 * at the end pointing back to the originally allocated 2681 wq->cpu_wq.pcpu = __alloc_percpu(size, align);
2599 * pointer which will be used for free. 2682 } else {
2600 */ 2683 void *ptr;
2601 ptr = kzalloc(size + align + sizeof(void *), GFP_KERNEL); 2684
2602 if (ptr) { 2685 /*
2603 wq->cpu_wq.single = PTR_ALIGN(ptr, align); 2686 * Allocate enough room to align cwq and put an extra
2604 *(void **)(wq->cpu_wq.single + 1) = ptr; 2687 * pointer at the end pointing back to the originally
2688 * allocated pointer which will be used for free.
2689 */
2690 ptr = kzalloc(size + align + sizeof(void *), GFP_KERNEL);
2691 if (ptr) {
2692 wq->cpu_wq.single = PTR_ALIGN(ptr, align);
2693 *(void **)(wq->cpu_wq.single + 1) = ptr;
2694 }
2605 } 2695 }
2606#else 2696
2607 /* On SMP, percpu allocator can align itself */
2608 wq->cpu_wq.pcpu = __alloc_percpu(size, align);
2609#endif
2610 /* just in case, make sure it's actually aligned */ 2697 /* just in case, make sure it's actually aligned */
2611 BUG_ON(!IS_ALIGNED(wq->cpu_wq.v, align)); 2698 BUG_ON(!IS_ALIGNED(wq->cpu_wq.v, align));
2612 return wq->cpu_wq.v ? 0 : -ENOMEM; 2699 return wq->cpu_wq.v ? 0 : -ENOMEM;
@@ -2614,23 +2701,25 @@ static int alloc_cwqs(struct workqueue_struct *wq)
2614 2701
2615static void free_cwqs(struct workqueue_struct *wq) 2702static void free_cwqs(struct workqueue_struct *wq)
2616{ 2703{
2617#ifndef CONFIG_SMP 2704 if (CONFIG_SMP && !(wq->flags & WQ_UNBOUND))
2618 /* on UP, the pointer to free is stored right after the cwq */ 2705 free_percpu(wq->cpu_wq.pcpu);
2619 if (wq->cpu_wq.single) 2706 else if (wq->cpu_wq.single) {
2707 /* the pointer to free is stored right after the cwq */
2620 kfree(*(void **)(wq->cpu_wq.single + 1)); 2708 kfree(*(void **)(wq->cpu_wq.single + 1));
2621#else 2709 }
2622 free_percpu(wq->cpu_wq.pcpu);
2623#endif
2624} 2710}
2625 2711
2626static int wq_clamp_max_active(int max_active, const char *name) 2712static int wq_clamp_max_active(int max_active, unsigned int flags,
2713 const char *name)
2627{ 2714{
2628 if (max_active < 1 || max_active > WQ_MAX_ACTIVE) 2715 int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
2716
2717 if (max_active < 1 || max_active > lim)
2629 printk(KERN_WARNING "workqueue: max_active %d requested for %s " 2718 printk(KERN_WARNING "workqueue: max_active %d requested for %s "
2630 "is out of range, clamping between %d and %d\n", 2719 "is out of range, clamping between %d and %d\n",
2631 max_active, name, 1, WQ_MAX_ACTIVE); 2720 max_active, name, 1, lim);
2632 2721
2633 return clamp_val(max_active, 1, WQ_MAX_ACTIVE); 2722 return clamp_val(max_active, 1, lim);
2634} 2723}
2635 2724
2636struct workqueue_struct *__alloc_workqueue_key(const char *name, 2725struct workqueue_struct *__alloc_workqueue_key(const char *name,
@@ -2642,8 +2731,15 @@ struct workqueue_struct *__alloc_workqueue_key(const char *name,
2642 struct workqueue_struct *wq; 2731 struct workqueue_struct *wq;
2643 unsigned int cpu; 2732 unsigned int cpu;
2644 2733
2734 /*
2735 * Unbound workqueues aren't concurrency managed and should be
2736 * dispatched to workers immediately.
2737 */
2738 if (flags & WQ_UNBOUND)
2739 flags |= WQ_HIGHPRI;
2740
2645 max_active = max_active ?: WQ_DFL_ACTIVE; 2741 max_active = max_active ?: WQ_DFL_ACTIVE;
2646 max_active = wq_clamp_max_active(max_active, name); 2742 max_active = wq_clamp_max_active(max_active, flags, name);
2647 2743
2648 wq = kzalloc(sizeof(*wq), GFP_KERNEL); 2744 wq = kzalloc(sizeof(*wq), GFP_KERNEL);
2649 if (!wq) 2745 if (!wq)
@@ -2664,7 +2760,7 @@ struct workqueue_struct *__alloc_workqueue_key(const char *name,
2664 if (alloc_cwqs(wq) < 0) 2760 if (alloc_cwqs(wq) < 0)
2665 goto err; 2761 goto err;
2666 2762
2667 for_each_possible_cpu(cpu) { 2763 for_each_cwq_cpu(cpu, wq) {
2668 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); 2764 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
2669 struct global_cwq *gcwq = get_gcwq(cpu); 2765 struct global_cwq *gcwq = get_gcwq(cpu);
2670 2766
@@ -2703,7 +2799,7 @@ struct workqueue_struct *__alloc_workqueue_key(const char *name,
2703 spin_lock(&workqueue_lock); 2799 spin_lock(&workqueue_lock);
2704 2800
2705 if (workqueue_freezing && wq->flags & WQ_FREEZEABLE) 2801 if (workqueue_freezing && wq->flags & WQ_FREEZEABLE)
2706 for_each_possible_cpu(cpu) 2802 for_each_cwq_cpu(cpu, wq)
2707 get_cwq(cpu, wq)->max_active = 0; 2803 get_cwq(cpu, wq)->max_active = 0;
2708 2804
2709 list_add(&wq->list, &workqueues); 2805 list_add(&wq->list, &workqueues);
@@ -2743,7 +2839,7 @@ void destroy_workqueue(struct workqueue_struct *wq)
2743 spin_unlock(&workqueue_lock); 2839 spin_unlock(&workqueue_lock);
2744 2840
2745 /* sanity check */ 2841 /* sanity check */
2746 for_each_possible_cpu(cpu) { 2842 for_each_cwq_cpu(cpu, wq) {
2747 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); 2843 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
2748 int i; 2844 int i;
2749 2845
@@ -2777,13 +2873,13 @@ void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
2777{ 2873{
2778 unsigned int cpu; 2874 unsigned int cpu;
2779 2875
2780 max_active = wq_clamp_max_active(max_active, wq->name); 2876 max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
2781 2877
2782 spin_lock(&workqueue_lock); 2878 spin_lock(&workqueue_lock);
2783 2879
2784 wq->saved_max_active = max_active; 2880 wq->saved_max_active = max_active;
2785 2881
2786 for_each_possible_cpu(cpu) { 2882 for_each_cwq_cpu(cpu, wq) {
2787 struct global_cwq *gcwq = get_gcwq(cpu); 2883 struct global_cwq *gcwq = get_gcwq(cpu);
2788 2884
2789 spin_lock_irq(&gcwq->lock); 2885 spin_lock_irq(&gcwq->lock);
@@ -3310,7 +3406,7 @@ void freeze_workqueues_begin(void)
3310 BUG_ON(workqueue_freezing); 3406 BUG_ON(workqueue_freezing);
3311 workqueue_freezing = true; 3407 workqueue_freezing = true;
3312 3408
3313 for_each_possible_cpu(cpu) { 3409 for_each_gcwq_cpu(cpu) {
3314 struct global_cwq *gcwq = get_gcwq(cpu); 3410 struct global_cwq *gcwq = get_gcwq(cpu);
3315 struct workqueue_struct *wq; 3411 struct workqueue_struct *wq;
3316 3412
@@ -3322,7 +3418,7 @@ void freeze_workqueues_begin(void)
3322 list_for_each_entry(wq, &workqueues, list) { 3418 list_for_each_entry(wq, &workqueues, list) {
3323 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); 3419 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
3324 3420
3325 if (wq->flags & WQ_FREEZEABLE) 3421 if (cwq && wq->flags & WQ_FREEZEABLE)
3326 cwq->max_active = 0; 3422 cwq->max_active = 0;
3327 } 3423 }
3328 3424
@@ -3354,7 +3450,7 @@ bool freeze_workqueues_busy(void)
3354 3450
3355 BUG_ON(!workqueue_freezing); 3451 BUG_ON(!workqueue_freezing);
3356 3452
3357 for_each_possible_cpu(cpu) { 3453 for_each_gcwq_cpu(cpu) {
3358 struct workqueue_struct *wq; 3454 struct workqueue_struct *wq;
3359 /* 3455 /*
3360 * nr_active is monotonically decreasing. It's safe 3456 * nr_active is monotonically decreasing. It's safe
@@ -3363,7 +3459,7 @@ bool freeze_workqueues_busy(void)
3363 list_for_each_entry(wq, &workqueues, list) { 3459 list_for_each_entry(wq, &workqueues, list) {
3364 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); 3460 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
3365 3461
3366 if (!(wq->flags & WQ_FREEZEABLE)) 3462 if (!cwq || !(wq->flags & WQ_FREEZEABLE))
3367 continue; 3463 continue;
3368 3464
3369 BUG_ON(cwq->nr_active < 0); 3465 BUG_ON(cwq->nr_active < 0);
@@ -3396,7 +3492,7 @@ void thaw_workqueues(void)
3396 if (!workqueue_freezing) 3492 if (!workqueue_freezing)
3397 goto out_unlock; 3493 goto out_unlock;
3398 3494
3399 for_each_possible_cpu(cpu) { 3495 for_each_gcwq_cpu(cpu) {
3400 struct global_cwq *gcwq = get_gcwq(cpu); 3496 struct global_cwq *gcwq = get_gcwq(cpu);
3401 struct workqueue_struct *wq; 3497 struct workqueue_struct *wq;
3402 3498
@@ -3408,7 +3504,7 @@ void thaw_workqueues(void)
3408 list_for_each_entry(wq, &workqueues, list) { 3504 list_for_each_entry(wq, &workqueues, list) {
3409 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); 3505 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
3410 3506
3411 if (!(wq->flags & WQ_FREEZEABLE)) 3507 if (!cwq || !(wq->flags & WQ_FREEZEABLE))
3412 continue; 3508 continue;
3413 3509
3414 /* restore max_active and repopulate worklist */ 3510 /* restore max_active and repopulate worklist */
@@ -3451,12 +3547,14 @@ void __init init_workqueues(void)
3451 hotcpu_notifier(workqueue_cpu_callback, CPU_PRI_WORKQUEUE); 3547 hotcpu_notifier(workqueue_cpu_callback, CPU_PRI_WORKQUEUE);
3452 3548
3453 /* initialize gcwqs */ 3549 /* initialize gcwqs */
3454 for_each_possible_cpu(cpu) { 3550 for_each_gcwq_cpu(cpu) {
3455 struct global_cwq *gcwq = get_gcwq(cpu); 3551 struct global_cwq *gcwq = get_gcwq(cpu);
3456 3552
3457 spin_lock_init(&gcwq->lock); 3553 spin_lock_init(&gcwq->lock);
3458 INIT_LIST_HEAD(&gcwq->worklist); 3554 INIT_LIST_HEAD(&gcwq->worklist);
3459 gcwq->cpu = cpu; 3555 gcwq->cpu = cpu;
3556 if (cpu == WORK_CPU_UNBOUND)
3557 gcwq->flags |= GCWQ_DISASSOCIATED;
3460 3558
3461 INIT_LIST_HEAD(&gcwq->idle_list); 3559 INIT_LIST_HEAD(&gcwq->idle_list);
3462 for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) 3560 for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++)
@@ -3476,7 +3574,7 @@ void __init init_workqueues(void)
3476 } 3574 }
3477 3575
3478 /* create the initial worker */ 3576 /* create the initial worker */
3479 for_each_online_cpu(cpu) { 3577 for_each_online_gcwq_cpu(cpu) {
3480 struct global_cwq *gcwq = get_gcwq(cpu); 3578 struct global_cwq *gcwq = get_gcwq(cpu);
3481 struct worker *worker; 3579 struct worker *worker;
3482 3580
@@ -3490,5 +3588,7 @@ void __init init_workqueues(void)
3490 system_wq = alloc_workqueue("events", 0, 0); 3588 system_wq = alloc_workqueue("events", 0, 0);
3491 system_long_wq = alloc_workqueue("events_long", 0, 0); 3589 system_long_wq = alloc_workqueue("events_long", 0, 0);
3492 system_nrt_wq = alloc_workqueue("events_nrt", WQ_NON_REENTRANT, 0); 3590 system_nrt_wq = alloc_workqueue("events_nrt", WQ_NON_REENTRANT, 0);
3591 system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
3592 WQ_UNBOUND_MAX_ACTIVE);
3493 BUG_ON(!system_wq || !system_long_wq || !system_nrt_wq); 3593 BUG_ON(!system_wq || !system_long_wq || !system_nrt_wq);
3494} 3594}