diff options
-rw-r--r-- | include/linux/workqueue.h | 15 | ||||
-rw-r--r-- | kernel/workqueue.c | 218 |
2 files changed, 173 insertions, 60 deletions
diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index 139069a6286c..67ce734747f6 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h | |||
@@ -51,7 +51,8 @@ enum { | |||
51 | WORK_NO_COLOR = WORK_NR_COLORS, | 51 | WORK_NO_COLOR = WORK_NR_COLORS, |
52 | 52 | ||
53 | /* special cpu IDs */ | 53 | /* special cpu IDs */ |
54 | WORK_CPU_NONE = NR_CPUS, | 54 | WORK_CPU_UNBOUND = NR_CPUS, |
55 | WORK_CPU_NONE = NR_CPUS + 1, | ||
55 | WORK_CPU_LAST = WORK_CPU_NONE, | 56 | WORK_CPU_LAST = WORK_CPU_NONE, |
56 | 57 | ||
57 | /* | 58 | /* |
@@ -237,11 +238,17 @@ enum { | |||
237 | WQ_RESCUER = 1 << 3, /* has an rescue worker */ | 238 | WQ_RESCUER = 1 << 3, /* has an rescue worker */ |
238 | WQ_HIGHPRI = 1 << 4, /* high priority */ | 239 | WQ_HIGHPRI = 1 << 4, /* high priority */ |
239 | WQ_CPU_INTENSIVE = 1 << 5, /* cpu instensive workqueue */ | 240 | WQ_CPU_INTENSIVE = 1 << 5, /* cpu instensive workqueue */ |
241 | WQ_UNBOUND = 1 << 6, /* not bound to any cpu */ | ||
240 | 242 | ||
241 | WQ_MAX_ACTIVE = 512, /* I like 512, better ideas? */ | 243 | WQ_MAX_ACTIVE = 512, /* I like 512, better ideas? */ |
244 | WQ_MAX_UNBOUND_PER_CPU = 4, /* 4 * #cpus for unbound wq */ | ||
242 | WQ_DFL_ACTIVE = WQ_MAX_ACTIVE / 2, | 245 | WQ_DFL_ACTIVE = WQ_MAX_ACTIVE / 2, |
243 | }; | 246 | }; |
244 | 247 | ||
248 | /* unbound wq's aren't per-cpu, scale max_active according to #cpus */ | ||
249 | #define WQ_UNBOUND_MAX_ACTIVE \ | ||
250 | max_t(int, WQ_MAX_ACTIVE, num_possible_cpus() * WQ_MAX_UNBOUND_PER_CPU) | ||
251 | |||
245 | /* | 252 | /* |
246 | * System-wide workqueues which are always present. | 253 | * System-wide workqueues which are always present. |
247 | * | 254 | * |
@@ -256,10 +263,16 @@ enum { | |||
256 | * system_nrt_wq is non-reentrant and guarantees that any given work | 263 | * system_nrt_wq is non-reentrant and guarantees that any given work |
257 | * item is never executed in parallel by multiple CPUs. Queue | 264 | * item is never executed in parallel by multiple CPUs. Queue |
258 | * flushing might take relatively long. | 265 | * flushing might take relatively long. |
266 | * | ||
267 | * system_unbound_wq is unbound workqueue. Workers are not bound to | ||
268 | * any specific CPU, not concurrency managed, and all queued works are | ||
269 | * executed immediately as long as max_active limit is not reached and | ||
270 | * resources are available. | ||
259 | */ | 271 | */ |
260 | extern struct workqueue_struct *system_wq; | 272 | extern struct workqueue_struct *system_wq; |
261 | extern struct workqueue_struct *system_long_wq; | 273 | extern struct workqueue_struct *system_long_wq; |
262 | extern struct workqueue_struct *system_nrt_wq; | 274 | extern struct workqueue_struct *system_nrt_wq; |
275 | extern struct workqueue_struct *system_unbound_wq; | ||
263 | 276 | ||
264 | extern struct workqueue_struct * | 277 | extern struct workqueue_struct * |
265 | __alloc_workqueue_key(const char *name, unsigned int flags, int max_active, | 278 | __alloc_workqueue_key(const char *name, unsigned int flags, int max_active, |
diff --git a/kernel/workqueue.c b/kernel/workqueue.c index a105ddf55f79..4608563cdd63 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c | |||
@@ -53,9 +53,10 @@ enum { | |||
53 | WORKER_ROGUE = 1 << 4, /* not bound to any cpu */ | 53 | WORKER_ROGUE = 1 << 4, /* not bound to any cpu */ |
54 | WORKER_REBIND = 1 << 5, /* mom is home, come back */ | 54 | WORKER_REBIND = 1 << 5, /* mom is home, come back */ |
55 | WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */ | 55 | WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */ |
56 | WORKER_UNBOUND = 1 << 7, /* worker is unbound */ | ||
56 | 57 | ||
57 | WORKER_NOT_RUNNING = WORKER_PREP | WORKER_ROGUE | WORKER_REBIND | | 58 | WORKER_NOT_RUNNING = WORKER_PREP | WORKER_ROGUE | WORKER_REBIND | |
58 | WORKER_CPU_INTENSIVE, | 59 | WORKER_CPU_INTENSIVE | WORKER_UNBOUND, |
59 | 60 | ||
60 | /* gcwq->trustee_state */ | 61 | /* gcwq->trustee_state */ |
61 | TRUSTEE_START = 0, /* start */ | 62 | TRUSTEE_START = 0, /* start */ |
@@ -96,7 +97,7 @@ enum { | |||
96 | * X: During normal operation, modification requires gcwq->lock and | 97 | * X: During normal operation, modification requires gcwq->lock and |
97 | * should be done only from local cpu. Either disabling preemption | 98 | * should be done only from local cpu. Either disabling preemption |
98 | * on local cpu or grabbing gcwq->lock is enough for read access. | 99 | * on local cpu or grabbing gcwq->lock is enough for read access. |
99 | * While trustee is in charge, it's identical to L. | 100 | * If GCWQ_DISASSOCIATED is set, it's identical to L. |
100 | * | 101 | * |
101 | * F: wq->flush_mutex protected. | 102 | * F: wq->flush_mutex protected. |
102 | * | 103 | * |
@@ -220,14 +221,52 @@ struct workqueue_struct { | |||
220 | struct workqueue_struct *system_wq __read_mostly; | 221 | struct workqueue_struct *system_wq __read_mostly; |
221 | struct workqueue_struct *system_long_wq __read_mostly; | 222 | struct workqueue_struct *system_long_wq __read_mostly; |
222 | struct workqueue_struct *system_nrt_wq __read_mostly; | 223 | struct workqueue_struct *system_nrt_wq __read_mostly; |
224 | struct workqueue_struct *system_unbound_wq __read_mostly; | ||
223 | EXPORT_SYMBOL_GPL(system_wq); | 225 | EXPORT_SYMBOL_GPL(system_wq); |
224 | EXPORT_SYMBOL_GPL(system_long_wq); | 226 | EXPORT_SYMBOL_GPL(system_long_wq); |
225 | EXPORT_SYMBOL_GPL(system_nrt_wq); | 227 | EXPORT_SYMBOL_GPL(system_nrt_wq); |
228 | EXPORT_SYMBOL_GPL(system_unbound_wq); | ||
226 | 229 | ||
227 | #define for_each_busy_worker(worker, i, pos, gcwq) \ | 230 | #define for_each_busy_worker(worker, i, pos, gcwq) \ |
228 | for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) \ | 231 | for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) \ |
229 | hlist_for_each_entry(worker, pos, &gcwq->busy_hash[i], hentry) | 232 | hlist_for_each_entry(worker, pos, &gcwq->busy_hash[i], hentry) |
230 | 233 | ||
234 | static inline int __next_gcwq_cpu(int cpu, const struct cpumask *mask, | ||
235 | unsigned int sw) | ||
236 | { | ||
237 | if (cpu < nr_cpu_ids) { | ||
238 | if (sw & 1) { | ||
239 | cpu = cpumask_next(cpu, mask); | ||
240 | if (cpu < nr_cpu_ids) | ||
241 | return cpu; | ||
242 | } | ||
243 | if (sw & 2) | ||
244 | return WORK_CPU_UNBOUND; | ||
245 | } | ||
246 | return WORK_CPU_NONE; | ||
247 | } | ||
248 | |||
249 | static inline int __next_wq_cpu(int cpu, const struct cpumask *mask, | ||
250 | struct workqueue_struct *wq) | ||
251 | { | ||
252 | return __next_gcwq_cpu(cpu, mask, !(wq->flags & WQ_UNBOUND) ? 1 : 2); | ||
253 | } | ||
254 | |||
255 | #define for_each_gcwq_cpu(cpu) \ | ||
256 | for ((cpu) = __next_gcwq_cpu(-1, cpu_possible_mask, 3); \ | ||
257 | (cpu) < WORK_CPU_NONE; \ | ||
258 | (cpu) = __next_gcwq_cpu((cpu), cpu_possible_mask, 3)) | ||
259 | |||
260 | #define for_each_online_gcwq_cpu(cpu) \ | ||
261 | for ((cpu) = __next_gcwq_cpu(-1, cpu_online_mask, 3); \ | ||
262 | (cpu) < WORK_CPU_NONE; \ | ||
263 | (cpu) = __next_gcwq_cpu((cpu), cpu_online_mask, 3)) | ||
264 | |||
265 | #define for_each_cwq_cpu(cpu, wq) \ | ||
266 | for ((cpu) = __next_wq_cpu(-1, cpu_possible_mask, (wq)); \ | ||
267 | (cpu) < WORK_CPU_NONE; \ | ||
268 | (cpu) = __next_wq_cpu((cpu), cpu_possible_mask, (wq))) | ||
269 | |||
231 | #ifdef CONFIG_DEBUG_OBJECTS_WORK | 270 | #ifdef CONFIG_DEBUG_OBJECTS_WORK |
232 | 271 | ||
233 | static struct debug_obj_descr work_debug_descr; | 272 | static struct debug_obj_descr work_debug_descr; |
@@ -351,26 +390,46 @@ static bool workqueue_freezing; /* W: have wqs started freezing? */ | |||
351 | static DEFINE_PER_CPU(struct global_cwq, global_cwq); | 390 | static DEFINE_PER_CPU(struct global_cwq, global_cwq); |
352 | static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, gcwq_nr_running); | 391 | static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, gcwq_nr_running); |
353 | 392 | ||
393 | /* | ||
394 | * Global cpu workqueue and nr_running counter for unbound gcwq. The | ||
395 | * gcwq is always online, has GCWQ_DISASSOCIATED set, and all its | ||
396 | * workers have WORKER_UNBOUND set. | ||
397 | */ | ||
398 | static struct global_cwq unbound_global_cwq; | ||
399 | static atomic_t unbound_gcwq_nr_running = ATOMIC_INIT(0); /* always 0 */ | ||
400 | |||
354 | static int worker_thread(void *__worker); | 401 | static int worker_thread(void *__worker); |
355 | 402 | ||
356 | static struct global_cwq *get_gcwq(unsigned int cpu) | 403 | static struct global_cwq *get_gcwq(unsigned int cpu) |
357 | { | 404 | { |
358 | return &per_cpu(global_cwq, cpu); | 405 | if (cpu != WORK_CPU_UNBOUND) |
406 | return &per_cpu(global_cwq, cpu); | ||
407 | else | ||
408 | return &unbound_global_cwq; | ||
359 | } | 409 | } |
360 | 410 | ||
361 | static atomic_t *get_gcwq_nr_running(unsigned int cpu) | 411 | static atomic_t *get_gcwq_nr_running(unsigned int cpu) |
362 | { | 412 | { |
363 | return &per_cpu(gcwq_nr_running, cpu); | 413 | if (cpu != WORK_CPU_UNBOUND) |
414 | return &per_cpu(gcwq_nr_running, cpu); | ||
415 | else | ||
416 | return &unbound_gcwq_nr_running; | ||
364 | } | 417 | } |
365 | 418 | ||
366 | static struct cpu_workqueue_struct *get_cwq(unsigned int cpu, | 419 | static struct cpu_workqueue_struct *get_cwq(unsigned int cpu, |
367 | struct workqueue_struct *wq) | 420 | struct workqueue_struct *wq) |
368 | { | 421 | { |
369 | #ifndef CONFIG_SMP | 422 | if (!(wq->flags & WQ_UNBOUND)) { |
370 | return wq->cpu_wq.single; | 423 | if (likely(cpu < nr_cpu_ids)) { |
424 | #ifdef CONFIG_SMP | ||
425 | return per_cpu_ptr(wq->cpu_wq.pcpu, cpu); | ||
371 | #else | 426 | #else |
372 | return per_cpu_ptr(wq->cpu_wq.pcpu, cpu); | 427 | return wq->cpu_wq.single; |
373 | #endif | 428 | #endif |
429 | } | ||
430 | } else if (likely(cpu == WORK_CPU_UNBOUND)) | ||
431 | return wq->cpu_wq.single; | ||
432 | return NULL; | ||
374 | } | 433 | } |
375 | 434 | ||
376 | static unsigned int work_color_to_flags(int color) | 435 | static unsigned int work_color_to_flags(int color) |
@@ -453,7 +512,7 @@ static struct global_cwq *get_work_gcwq(struct work_struct *work) | |||
453 | if (cpu == WORK_CPU_NONE) | 512 | if (cpu == WORK_CPU_NONE) |
454 | return NULL; | 513 | return NULL; |
455 | 514 | ||
456 | BUG_ON(cpu >= nr_cpu_ids); | 515 | BUG_ON(cpu >= nr_cpu_ids && cpu != WORK_CPU_UNBOUND); |
457 | return get_gcwq(cpu); | 516 | return get_gcwq(cpu); |
458 | } | 517 | } |
459 | 518 | ||
@@ -869,11 +928,14 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, | |||
869 | 928 | ||
870 | debug_work_activate(work); | 929 | debug_work_activate(work); |
871 | 930 | ||
931 | if (unlikely(cpu == WORK_CPU_UNBOUND)) | ||
932 | cpu = raw_smp_processor_id(); | ||
933 | |||
872 | /* | 934 | /* |
873 | * Determine gcwq to use. SINGLE_CPU is inherently | 935 | * Determine gcwq to use. SINGLE_CPU is inherently |
874 | * NON_REENTRANT, so test it first. | 936 | * NON_REENTRANT, so test it first. |
875 | */ | 937 | */ |
876 | if (!(wq->flags & WQ_SINGLE_CPU)) { | 938 | if (!(wq->flags & (WQ_SINGLE_CPU | WQ_UNBOUND))) { |
877 | struct global_cwq *last_gcwq; | 939 | struct global_cwq *last_gcwq; |
878 | 940 | ||
879 | /* | 941 | /* |
@@ -900,7 +962,7 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, | |||
900 | } | 962 | } |
901 | } else | 963 | } else |
902 | spin_lock_irqsave(&gcwq->lock, flags); | 964 | spin_lock_irqsave(&gcwq->lock, flags); |
903 | } else { | 965 | } else if (!(wq->flags & WQ_UNBOUND)) { |
904 | unsigned int req_cpu = cpu; | 966 | unsigned int req_cpu = cpu; |
905 | 967 | ||
906 | /* | 968 | /* |
@@ -932,6 +994,9 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, | |||
932 | spin_unlock_irqrestore(&gcwq->lock, flags); | 994 | spin_unlock_irqrestore(&gcwq->lock, flags); |
933 | goto retry; | 995 | goto retry; |
934 | } | 996 | } |
997 | } else { | ||
998 | gcwq = get_gcwq(WORK_CPU_UNBOUND); | ||
999 | spin_lock_irqsave(&gcwq->lock, flags); | ||
935 | } | 1000 | } |
936 | 1001 | ||
937 | /* gcwq determined, get cwq and queue */ | 1002 | /* gcwq determined, get cwq and queue */ |
@@ -1166,7 +1231,8 @@ static bool worker_maybe_bind_and_lock(struct worker *worker) | |||
1166 | * it races with cpu hotunplug operation. Verify | 1231 | * it races with cpu hotunplug operation. Verify |
1167 | * against GCWQ_DISASSOCIATED. | 1232 | * against GCWQ_DISASSOCIATED. |
1168 | */ | 1233 | */ |
1169 | set_cpus_allowed_ptr(task, get_cpu_mask(gcwq->cpu)); | 1234 | if (!(gcwq->flags & GCWQ_DISASSOCIATED)) |
1235 | set_cpus_allowed_ptr(task, get_cpu_mask(gcwq->cpu)); | ||
1170 | 1236 | ||
1171 | spin_lock_irq(&gcwq->lock); | 1237 | spin_lock_irq(&gcwq->lock); |
1172 | if (gcwq->flags & GCWQ_DISASSOCIATED) | 1238 | if (gcwq->flags & GCWQ_DISASSOCIATED) |
@@ -1231,8 +1297,9 @@ static struct worker *alloc_worker(void) | |||
1231 | */ | 1297 | */ |
1232 | static struct worker *create_worker(struct global_cwq *gcwq, bool bind) | 1298 | static struct worker *create_worker(struct global_cwq *gcwq, bool bind) |
1233 | { | 1299 | { |
1234 | int id = -1; | 1300 | bool on_unbound_cpu = gcwq->cpu == WORK_CPU_UNBOUND; |
1235 | struct worker *worker = NULL; | 1301 | struct worker *worker = NULL; |
1302 | int id = -1; | ||
1236 | 1303 | ||
1237 | spin_lock_irq(&gcwq->lock); | 1304 | spin_lock_irq(&gcwq->lock); |
1238 | while (ida_get_new(&gcwq->worker_ida, &id)) { | 1305 | while (ida_get_new(&gcwq->worker_ida, &id)) { |
@@ -1250,8 +1317,12 @@ static struct worker *create_worker(struct global_cwq *gcwq, bool bind) | |||
1250 | worker->gcwq = gcwq; | 1317 | worker->gcwq = gcwq; |
1251 | worker->id = id; | 1318 | worker->id = id; |
1252 | 1319 | ||
1253 | worker->task = kthread_create(worker_thread, worker, "kworker/%u:%d", | 1320 | if (!on_unbound_cpu) |
1254 | gcwq->cpu, id); | 1321 | worker->task = kthread_create(worker_thread, worker, |
1322 | "kworker/%u:%d", gcwq->cpu, id); | ||
1323 | else | ||
1324 | worker->task = kthread_create(worker_thread, worker, | ||
1325 | "kworker/u:%d", id); | ||
1255 | if (IS_ERR(worker->task)) | 1326 | if (IS_ERR(worker->task)) |
1256 | goto fail; | 1327 | goto fail; |
1257 | 1328 | ||
@@ -1260,10 +1331,13 @@ static struct worker *create_worker(struct global_cwq *gcwq, bool bind) | |||
1260 | * online later on. Make sure every worker has | 1331 | * online later on. Make sure every worker has |
1261 | * PF_THREAD_BOUND set. | 1332 | * PF_THREAD_BOUND set. |
1262 | */ | 1333 | */ |
1263 | if (bind) | 1334 | if (bind && !on_unbound_cpu) |
1264 | kthread_bind(worker->task, gcwq->cpu); | 1335 | kthread_bind(worker->task, gcwq->cpu); |
1265 | else | 1336 | else { |
1266 | worker->task->flags |= PF_THREAD_BOUND; | 1337 | worker->task->flags |= PF_THREAD_BOUND; |
1338 | if (on_unbound_cpu) | ||
1339 | worker->flags |= WORKER_UNBOUND; | ||
1340 | } | ||
1267 | 1341 | ||
1268 | return worker; | 1342 | return worker; |
1269 | fail: | 1343 | fail: |
@@ -1358,12 +1432,17 @@ static bool send_mayday(struct work_struct *work) | |||
1358 | { | 1432 | { |
1359 | struct cpu_workqueue_struct *cwq = get_work_cwq(work); | 1433 | struct cpu_workqueue_struct *cwq = get_work_cwq(work); |
1360 | struct workqueue_struct *wq = cwq->wq; | 1434 | struct workqueue_struct *wq = cwq->wq; |
1435 | unsigned int cpu; | ||
1361 | 1436 | ||
1362 | if (!(wq->flags & WQ_RESCUER)) | 1437 | if (!(wq->flags & WQ_RESCUER)) |
1363 | return false; | 1438 | return false; |
1364 | 1439 | ||
1365 | /* mayday mayday mayday */ | 1440 | /* mayday mayday mayday */ |
1366 | if (!cpumask_test_and_set_cpu(cwq->gcwq->cpu, wq->mayday_mask)) | 1441 | cpu = cwq->gcwq->cpu; |
1442 | /* WORK_CPU_UNBOUND can't be set in cpumask, use cpu 0 instead */ | ||
1443 | if (cpu == WORK_CPU_UNBOUND) | ||
1444 | cpu = 0; | ||
1445 | if (!cpumask_test_and_set_cpu(cpu, wq->mayday_mask)) | ||
1367 | wake_up_process(wq->rescuer->task); | 1446 | wake_up_process(wq->rescuer->task); |
1368 | return true; | 1447 | return true; |
1369 | } | 1448 | } |
@@ -1882,6 +1961,7 @@ static int rescuer_thread(void *__wq) | |||
1882 | struct workqueue_struct *wq = __wq; | 1961 | struct workqueue_struct *wq = __wq; |
1883 | struct worker *rescuer = wq->rescuer; | 1962 | struct worker *rescuer = wq->rescuer; |
1884 | struct list_head *scheduled = &rescuer->scheduled; | 1963 | struct list_head *scheduled = &rescuer->scheduled; |
1964 | bool is_unbound = wq->flags & WQ_UNBOUND; | ||
1885 | unsigned int cpu; | 1965 | unsigned int cpu; |
1886 | 1966 | ||
1887 | set_user_nice(current, RESCUER_NICE_LEVEL); | 1967 | set_user_nice(current, RESCUER_NICE_LEVEL); |
@@ -1891,8 +1971,13 @@ repeat: | |||
1891 | if (kthread_should_stop()) | 1971 | if (kthread_should_stop()) |
1892 | return 0; | 1972 | return 0; |
1893 | 1973 | ||
1974 | /* | ||
1975 | * See whether any cpu is asking for help. Unbounded | ||
1976 | * workqueues use cpu 0 in mayday_mask for CPU_UNBOUND. | ||
1977 | */ | ||
1894 | for_each_cpu(cpu, wq->mayday_mask) { | 1978 | for_each_cpu(cpu, wq->mayday_mask) { |
1895 | struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); | 1979 | unsigned int tcpu = is_unbound ? WORK_CPU_UNBOUND : cpu; |
1980 | struct cpu_workqueue_struct *cwq = get_cwq(tcpu, wq); | ||
1896 | struct global_cwq *gcwq = cwq->gcwq; | 1981 | struct global_cwq *gcwq = cwq->gcwq; |
1897 | struct work_struct *work, *n; | 1982 | struct work_struct *work, *n; |
1898 | 1983 | ||
@@ -2034,7 +2119,7 @@ static bool flush_workqueue_prep_cwqs(struct workqueue_struct *wq, | |||
2034 | atomic_set(&wq->nr_cwqs_to_flush, 1); | 2119 | atomic_set(&wq->nr_cwqs_to_flush, 1); |
2035 | } | 2120 | } |
2036 | 2121 | ||
2037 | for_each_possible_cpu(cpu) { | 2122 | for_each_cwq_cpu(cpu, wq) { |
2038 | struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); | 2123 | struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); |
2039 | struct global_cwq *gcwq = cwq->gcwq; | 2124 | struct global_cwq *gcwq = cwq->gcwq; |
2040 | 2125 | ||
@@ -2344,7 +2429,7 @@ static void wait_on_work(struct work_struct *work) | |||
2344 | lock_map_acquire(&work->lockdep_map); | 2429 | lock_map_acquire(&work->lockdep_map); |
2345 | lock_map_release(&work->lockdep_map); | 2430 | lock_map_release(&work->lockdep_map); |
2346 | 2431 | ||
2347 | for_each_possible_cpu(cpu) | 2432 | for_each_gcwq_cpu(cpu) |
2348 | wait_on_cpu_work(get_gcwq(cpu), work); | 2433 | wait_on_cpu_work(get_gcwq(cpu), work); |
2349 | } | 2434 | } |
2350 | 2435 | ||
@@ -2590,23 +2675,25 @@ static int alloc_cwqs(struct workqueue_struct *wq) | |||
2590 | const size_t size = sizeof(struct cpu_workqueue_struct); | 2675 | const size_t size = sizeof(struct cpu_workqueue_struct); |
2591 | const size_t align = max_t(size_t, 1 << WORK_STRUCT_FLAG_BITS, | 2676 | const size_t align = max_t(size_t, 1 << WORK_STRUCT_FLAG_BITS, |
2592 | __alignof__(unsigned long long)); | 2677 | __alignof__(unsigned long long)); |
2593 | #ifndef CONFIG_SMP | ||
2594 | void *ptr; | ||
2595 | 2678 | ||
2596 | /* | 2679 | if (CONFIG_SMP && !(wq->flags & WQ_UNBOUND)) { |
2597 | * Allocate enough room to align cwq and put an extra pointer | 2680 | /* on SMP, percpu allocator can align itself */ |
2598 | * at the end pointing back to the originally allocated | 2681 | wq->cpu_wq.pcpu = __alloc_percpu(size, align); |
2599 | * pointer which will be used for free. | 2682 | } else { |
2600 | */ | 2683 | void *ptr; |
2601 | ptr = kzalloc(size + align + sizeof(void *), GFP_KERNEL); | 2684 | |
2602 | if (ptr) { | 2685 | /* |
2603 | wq->cpu_wq.single = PTR_ALIGN(ptr, align); | 2686 | * Allocate enough room to align cwq and put an extra |
2604 | *(void **)(wq->cpu_wq.single + 1) = ptr; | 2687 | * pointer at the end pointing back to the originally |
2688 | * allocated pointer which will be used for free. | ||
2689 | */ | ||
2690 | ptr = kzalloc(size + align + sizeof(void *), GFP_KERNEL); | ||
2691 | if (ptr) { | ||
2692 | wq->cpu_wq.single = PTR_ALIGN(ptr, align); | ||
2693 | *(void **)(wq->cpu_wq.single + 1) = ptr; | ||
2694 | } | ||
2605 | } | 2695 | } |
2606 | #else | 2696 | |
2607 | /* On SMP, percpu allocator can align itself */ | ||
2608 | wq->cpu_wq.pcpu = __alloc_percpu(size, align); | ||
2609 | #endif | ||
2610 | /* just in case, make sure it's actually aligned */ | 2697 | /* just in case, make sure it's actually aligned */ |
2611 | BUG_ON(!IS_ALIGNED(wq->cpu_wq.v, align)); | 2698 | BUG_ON(!IS_ALIGNED(wq->cpu_wq.v, align)); |
2612 | return wq->cpu_wq.v ? 0 : -ENOMEM; | 2699 | return wq->cpu_wq.v ? 0 : -ENOMEM; |
@@ -2614,23 +2701,25 @@ static int alloc_cwqs(struct workqueue_struct *wq) | |||
2614 | 2701 | ||
2615 | static void free_cwqs(struct workqueue_struct *wq) | 2702 | static void free_cwqs(struct workqueue_struct *wq) |
2616 | { | 2703 | { |
2617 | #ifndef CONFIG_SMP | 2704 | if (CONFIG_SMP && !(wq->flags & WQ_UNBOUND)) |
2618 | /* on UP, the pointer to free is stored right after the cwq */ | 2705 | free_percpu(wq->cpu_wq.pcpu); |
2619 | if (wq->cpu_wq.single) | 2706 | else if (wq->cpu_wq.single) { |
2707 | /* the pointer to free is stored right after the cwq */ | ||
2620 | kfree(*(void **)(wq->cpu_wq.single + 1)); | 2708 | kfree(*(void **)(wq->cpu_wq.single + 1)); |
2621 | #else | 2709 | } |
2622 | free_percpu(wq->cpu_wq.pcpu); | ||
2623 | #endif | ||
2624 | } | 2710 | } |
2625 | 2711 | ||
2626 | static int wq_clamp_max_active(int max_active, const char *name) | 2712 | static int wq_clamp_max_active(int max_active, unsigned int flags, |
2713 | const char *name) | ||
2627 | { | 2714 | { |
2628 | if (max_active < 1 || max_active > WQ_MAX_ACTIVE) | 2715 | int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE; |
2716 | |||
2717 | if (max_active < 1 || max_active > lim) | ||
2629 | printk(KERN_WARNING "workqueue: max_active %d requested for %s " | 2718 | printk(KERN_WARNING "workqueue: max_active %d requested for %s " |
2630 | "is out of range, clamping between %d and %d\n", | 2719 | "is out of range, clamping between %d and %d\n", |
2631 | max_active, name, 1, WQ_MAX_ACTIVE); | 2720 | max_active, name, 1, lim); |
2632 | 2721 | ||
2633 | return clamp_val(max_active, 1, WQ_MAX_ACTIVE); | 2722 | return clamp_val(max_active, 1, lim); |
2634 | } | 2723 | } |
2635 | 2724 | ||
2636 | struct workqueue_struct *__alloc_workqueue_key(const char *name, | 2725 | struct workqueue_struct *__alloc_workqueue_key(const char *name, |
@@ -2642,8 +2731,15 @@ struct workqueue_struct *__alloc_workqueue_key(const char *name, | |||
2642 | struct workqueue_struct *wq; | 2731 | struct workqueue_struct *wq; |
2643 | unsigned int cpu; | 2732 | unsigned int cpu; |
2644 | 2733 | ||
2734 | /* | ||
2735 | * Unbound workqueues aren't concurrency managed and should be | ||
2736 | * dispatched to workers immediately. | ||
2737 | */ | ||
2738 | if (flags & WQ_UNBOUND) | ||
2739 | flags |= WQ_HIGHPRI; | ||
2740 | |||
2645 | max_active = max_active ?: WQ_DFL_ACTIVE; | 2741 | max_active = max_active ?: WQ_DFL_ACTIVE; |
2646 | max_active = wq_clamp_max_active(max_active, name); | 2742 | max_active = wq_clamp_max_active(max_active, flags, name); |
2647 | 2743 | ||
2648 | wq = kzalloc(sizeof(*wq), GFP_KERNEL); | 2744 | wq = kzalloc(sizeof(*wq), GFP_KERNEL); |
2649 | if (!wq) | 2745 | if (!wq) |
@@ -2664,7 +2760,7 @@ struct workqueue_struct *__alloc_workqueue_key(const char *name, | |||
2664 | if (alloc_cwqs(wq) < 0) | 2760 | if (alloc_cwqs(wq) < 0) |
2665 | goto err; | 2761 | goto err; |
2666 | 2762 | ||
2667 | for_each_possible_cpu(cpu) { | 2763 | for_each_cwq_cpu(cpu, wq) { |
2668 | struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); | 2764 | struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); |
2669 | struct global_cwq *gcwq = get_gcwq(cpu); | 2765 | struct global_cwq *gcwq = get_gcwq(cpu); |
2670 | 2766 | ||
@@ -2703,7 +2799,7 @@ struct workqueue_struct *__alloc_workqueue_key(const char *name, | |||
2703 | spin_lock(&workqueue_lock); | 2799 | spin_lock(&workqueue_lock); |
2704 | 2800 | ||
2705 | if (workqueue_freezing && wq->flags & WQ_FREEZEABLE) | 2801 | if (workqueue_freezing && wq->flags & WQ_FREEZEABLE) |
2706 | for_each_possible_cpu(cpu) | 2802 | for_each_cwq_cpu(cpu, wq) |
2707 | get_cwq(cpu, wq)->max_active = 0; | 2803 | get_cwq(cpu, wq)->max_active = 0; |
2708 | 2804 | ||
2709 | list_add(&wq->list, &workqueues); | 2805 | list_add(&wq->list, &workqueues); |
@@ -2743,7 +2839,7 @@ void destroy_workqueue(struct workqueue_struct *wq) | |||
2743 | spin_unlock(&workqueue_lock); | 2839 | spin_unlock(&workqueue_lock); |
2744 | 2840 | ||
2745 | /* sanity check */ | 2841 | /* sanity check */ |
2746 | for_each_possible_cpu(cpu) { | 2842 | for_each_cwq_cpu(cpu, wq) { |
2747 | struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); | 2843 | struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); |
2748 | int i; | 2844 | int i; |
2749 | 2845 | ||
@@ -2777,13 +2873,13 @@ void workqueue_set_max_active(struct workqueue_struct *wq, int max_active) | |||
2777 | { | 2873 | { |
2778 | unsigned int cpu; | 2874 | unsigned int cpu; |
2779 | 2875 | ||
2780 | max_active = wq_clamp_max_active(max_active, wq->name); | 2876 | max_active = wq_clamp_max_active(max_active, wq->flags, wq->name); |
2781 | 2877 | ||
2782 | spin_lock(&workqueue_lock); | 2878 | spin_lock(&workqueue_lock); |
2783 | 2879 | ||
2784 | wq->saved_max_active = max_active; | 2880 | wq->saved_max_active = max_active; |
2785 | 2881 | ||
2786 | for_each_possible_cpu(cpu) { | 2882 | for_each_cwq_cpu(cpu, wq) { |
2787 | struct global_cwq *gcwq = get_gcwq(cpu); | 2883 | struct global_cwq *gcwq = get_gcwq(cpu); |
2788 | 2884 | ||
2789 | spin_lock_irq(&gcwq->lock); | 2885 | spin_lock_irq(&gcwq->lock); |
@@ -3310,7 +3406,7 @@ void freeze_workqueues_begin(void) | |||
3310 | BUG_ON(workqueue_freezing); | 3406 | BUG_ON(workqueue_freezing); |
3311 | workqueue_freezing = true; | 3407 | workqueue_freezing = true; |
3312 | 3408 | ||
3313 | for_each_possible_cpu(cpu) { | 3409 | for_each_gcwq_cpu(cpu) { |
3314 | struct global_cwq *gcwq = get_gcwq(cpu); | 3410 | struct global_cwq *gcwq = get_gcwq(cpu); |
3315 | struct workqueue_struct *wq; | 3411 | struct workqueue_struct *wq; |
3316 | 3412 | ||
@@ -3322,7 +3418,7 @@ void freeze_workqueues_begin(void) | |||
3322 | list_for_each_entry(wq, &workqueues, list) { | 3418 | list_for_each_entry(wq, &workqueues, list) { |
3323 | struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); | 3419 | struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); |
3324 | 3420 | ||
3325 | if (wq->flags & WQ_FREEZEABLE) | 3421 | if (cwq && wq->flags & WQ_FREEZEABLE) |
3326 | cwq->max_active = 0; | 3422 | cwq->max_active = 0; |
3327 | } | 3423 | } |
3328 | 3424 | ||
@@ -3354,7 +3450,7 @@ bool freeze_workqueues_busy(void) | |||
3354 | 3450 | ||
3355 | BUG_ON(!workqueue_freezing); | 3451 | BUG_ON(!workqueue_freezing); |
3356 | 3452 | ||
3357 | for_each_possible_cpu(cpu) { | 3453 | for_each_gcwq_cpu(cpu) { |
3358 | struct workqueue_struct *wq; | 3454 | struct workqueue_struct *wq; |
3359 | /* | 3455 | /* |
3360 | * nr_active is monotonically decreasing. It's safe | 3456 | * nr_active is monotonically decreasing. It's safe |
@@ -3363,7 +3459,7 @@ bool freeze_workqueues_busy(void) | |||
3363 | list_for_each_entry(wq, &workqueues, list) { | 3459 | list_for_each_entry(wq, &workqueues, list) { |
3364 | struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); | 3460 | struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); |
3365 | 3461 | ||
3366 | if (!(wq->flags & WQ_FREEZEABLE)) | 3462 | if (!cwq || !(wq->flags & WQ_FREEZEABLE)) |
3367 | continue; | 3463 | continue; |
3368 | 3464 | ||
3369 | BUG_ON(cwq->nr_active < 0); | 3465 | BUG_ON(cwq->nr_active < 0); |
@@ -3396,7 +3492,7 @@ void thaw_workqueues(void) | |||
3396 | if (!workqueue_freezing) | 3492 | if (!workqueue_freezing) |
3397 | goto out_unlock; | 3493 | goto out_unlock; |
3398 | 3494 | ||
3399 | for_each_possible_cpu(cpu) { | 3495 | for_each_gcwq_cpu(cpu) { |
3400 | struct global_cwq *gcwq = get_gcwq(cpu); | 3496 | struct global_cwq *gcwq = get_gcwq(cpu); |
3401 | struct workqueue_struct *wq; | 3497 | struct workqueue_struct *wq; |
3402 | 3498 | ||
@@ -3408,7 +3504,7 @@ void thaw_workqueues(void) | |||
3408 | list_for_each_entry(wq, &workqueues, list) { | 3504 | list_for_each_entry(wq, &workqueues, list) { |
3409 | struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); | 3505 | struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); |
3410 | 3506 | ||
3411 | if (!(wq->flags & WQ_FREEZEABLE)) | 3507 | if (!cwq || !(wq->flags & WQ_FREEZEABLE)) |
3412 | continue; | 3508 | continue; |
3413 | 3509 | ||
3414 | /* restore max_active and repopulate worklist */ | 3510 | /* restore max_active and repopulate worklist */ |
@@ -3451,12 +3547,14 @@ void __init init_workqueues(void) | |||
3451 | hotcpu_notifier(workqueue_cpu_callback, CPU_PRI_WORKQUEUE); | 3547 | hotcpu_notifier(workqueue_cpu_callback, CPU_PRI_WORKQUEUE); |
3452 | 3548 | ||
3453 | /* initialize gcwqs */ | 3549 | /* initialize gcwqs */ |
3454 | for_each_possible_cpu(cpu) { | 3550 | for_each_gcwq_cpu(cpu) { |
3455 | struct global_cwq *gcwq = get_gcwq(cpu); | 3551 | struct global_cwq *gcwq = get_gcwq(cpu); |
3456 | 3552 | ||
3457 | spin_lock_init(&gcwq->lock); | 3553 | spin_lock_init(&gcwq->lock); |
3458 | INIT_LIST_HEAD(&gcwq->worklist); | 3554 | INIT_LIST_HEAD(&gcwq->worklist); |
3459 | gcwq->cpu = cpu; | 3555 | gcwq->cpu = cpu; |
3556 | if (cpu == WORK_CPU_UNBOUND) | ||
3557 | gcwq->flags |= GCWQ_DISASSOCIATED; | ||
3460 | 3558 | ||
3461 | INIT_LIST_HEAD(&gcwq->idle_list); | 3559 | INIT_LIST_HEAD(&gcwq->idle_list); |
3462 | for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) | 3560 | for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) |
@@ -3476,7 +3574,7 @@ void __init init_workqueues(void) | |||
3476 | } | 3574 | } |
3477 | 3575 | ||
3478 | /* create the initial worker */ | 3576 | /* create the initial worker */ |
3479 | for_each_online_cpu(cpu) { | 3577 | for_each_online_gcwq_cpu(cpu) { |
3480 | struct global_cwq *gcwq = get_gcwq(cpu); | 3578 | struct global_cwq *gcwq = get_gcwq(cpu); |
3481 | struct worker *worker; | 3579 | struct worker *worker; |
3482 | 3580 | ||
@@ -3490,5 +3588,7 @@ void __init init_workqueues(void) | |||
3490 | system_wq = alloc_workqueue("events", 0, 0); | 3588 | system_wq = alloc_workqueue("events", 0, 0); |
3491 | system_long_wq = alloc_workqueue("events_long", 0, 0); | 3589 | system_long_wq = alloc_workqueue("events_long", 0, 0); |
3492 | system_nrt_wq = alloc_workqueue("events_nrt", WQ_NON_REENTRANT, 0); | 3590 | system_nrt_wq = alloc_workqueue("events_nrt", WQ_NON_REENTRANT, 0); |
3591 | system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND, | ||
3592 | WQ_UNBOUND_MAX_ACTIVE); | ||
3493 | BUG_ON(!system_wq || !system_long_wq || !system_nrt_wq); | 3593 | BUG_ON(!system_wq || !system_long_wq || !system_nrt_wq); |
3494 | } | 3594 | } |