aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/workqueue.h14
-rw-r--r--kernel/workqueue.c36
2 files changed, 21 insertions, 29 deletions
diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index d74a529ed13..5f76001c4e6 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -25,17 +25,19 @@ typedef void (*work_func_t)(struct work_struct *work);
25 25
26enum { 26enum {
27 WORK_STRUCT_PENDING_BIT = 0, /* work item is pending execution */ 27 WORK_STRUCT_PENDING_BIT = 0, /* work item is pending execution */
28 WORK_STRUCT_LINKED_BIT = 1, /* next work is linked to this one */ 28 WORK_STRUCT_CWQ_BIT = 1, /* data points to cwq */
29 WORK_STRUCT_LINKED_BIT = 2, /* next work is linked to this one */
29#ifdef CONFIG_DEBUG_OBJECTS_WORK 30#ifdef CONFIG_DEBUG_OBJECTS_WORK
30 WORK_STRUCT_STATIC_BIT = 2, /* static initializer (debugobjects) */ 31 WORK_STRUCT_STATIC_BIT = 3, /* static initializer (debugobjects) */
31 WORK_STRUCT_COLOR_SHIFT = 3, /* color for workqueue flushing */ 32 WORK_STRUCT_COLOR_SHIFT = 4, /* color for workqueue flushing */
32#else 33#else
33 WORK_STRUCT_COLOR_SHIFT = 2, /* color for workqueue flushing */ 34 WORK_STRUCT_COLOR_SHIFT = 3, /* color for workqueue flushing */
34#endif 35#endif
35 36
36 WORK_STRUCT_COLOR_BITS = 4, 37 WORK_STRUCT_COLOR_BITS = 4,
37 38
38 WORK_STRUCT_PENDING = 1 << WORK_STRUCT_PENDING_BIT, 39 WORK_STRUCT_PENDING = 1 << WORK_STRUCT_PENDING_BIT,
40 WORK_STRUCT_CWQ = 1 << WORK_STRUCT_CWQ_BIT,
39 WORK_STRUCT_LINKED = 1 << WORK_STRUCT_LINKED_BIT, 41 WORK_STRUCT_LINKED = 1 << WORK_STRUCT_LINKED_BIT,
40#ifdef CONFIG_DEBUG_OBJECTS_WORK 42#ifdef CONFIG_DEBUG_OBJECTS_WORK
41 WORK_STRUCT_STATIC = 1 << WORK_STRUCT_STATIC_BIT, 43 WORK_STRUCT_STATIC = 1 << WORK_STRUCT_STATIC_BIT,
@@ -56,8 +58,8 @@ enum {
56 WORK_CPU_LAST = WORK_CPU_NONE, 58 WORK_CPU_LAST = WORK_CPU_NONE,
57 59
58 /* 60 /*
59 * Reserve 6 bits off of cwq pointer w/ debugobjects turned 61 * Reserve 7 bits off of cwq pointer w/ debugobjects turned
60 * off. This makes cwqs aligned to 64 bytes which isn't too 62 * off. This makes cwqs aligned to 128 bytes which isn't too
61 * excessive while allowing 15 workqueue flush colors. 63 * excessive while allowing 15 workqueue flush colors.
62 */ 64 */
63 WORK_STRUCT_FLAG_BITS = WORK_STRUCT_COLOR_SHIFT + 65 WORK_STRUCT_FLAG_BITS = WORK_STRUCT_COLOR_SHIFT +
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index c11edc9c936..e5cb7faac58 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -468,10 +468,9 @@ static int work_next_color(int color)
468} 468}
469 469
470/* 470/*
471 * Work data points to the cwq while a work is on queue. Once 471 * A work's data points to the cwq with WORK_STRUCT_CWQ set while the
472 * execution starts, it points to the cpu the work was last on. This 472 * work is on queue. Once execution starts, WORK_STRUCT_CWQ is
473 * can be distinguished by comparing the data value against 473 * cleared and the work data contains the cpu number it was last on.
474 * PAGE_OFFSET.
475 * 474 *
476 * set_work_{cwq|cpu}() and clear_work_data() can be used to set the 475 * set_work_{cwq|cpu}() and clear_work_data() can be used to set the
477 * cwq, cpu or clear work->data. These functions should only be 476 * cwq, cpu or clear work->data. These functions should only be
@@ -494,7 +493,7 @@ static void set_work_cwq(struct work_struct *work,
494 unsigned long extra_flags) 493 unsigned long extra_flags)
495{ 494{
496 set_work_data(work, (unsigned long)cwq, 495 set_work_data(work, (unsigned long)cwq,
497 WORK_STRUCT_PENDING | extra_flags); 496 WORK_STRUCT_PENDING | WORK_STRUCT_CWQ | extra_flags);
498} 497}
499 498
500static void set_work_cpu(struct work_struct *work, unsigned int cpu) 499static void set_work_cpu(struct work_struct *work, unsigned int cpu)
@@ -507,25 +506,24 @@ static void clear_work_data(struct work_struct *work)
507 set_work_data(work, WORK_STRUCT_NO_CPU, 0); 506 set_work_data(work, WORK_STRUCT_NO_CPU, 0);
508} 507}
509 508
510static inline unsigned long get_work_data(struct work_struct *work)
511{
512 return atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK;
513}
514
515static struct cpu_workqueue_struct *get_work_cwq(struct work_struct *work) 509static struct cpu_workqueue_struct *get_work_cwq(struct work_struct *work)
516{ 510{
517 unsigned long data = get_work_data(work); 511 unsigned long data = atomic_long_read(&work->data);
518 512
519 return data >= PAGE_OFFSET ? (void *)data : NULL; 513 if (data & WORK_STRUCT_CWQ)
514 return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
515 else
516 return NULL;
520} 517}
521 518
522static struct global_cwq *get_work_gcwq(struct work_struct *work) 519static struct global_cwq *get_work_gcwq(struct work_struct *work)
523{ 520{
524 unsigned long data = get_work_data(work); 521 unsigned long data = atomic_long_read(&work->data);
525 unsigned int cpu; 522 unsigned int cpu;
526 523
527 if (data >= PAGE_OFFSET) 524 if (data & WORK_STRUCT_CWQ)
528 return ((struct cpu_workqueue_struct *)data)->gcwq; 525 return ((struct cpu_workqueue_struct *)
526 (data & WORK_STRUCT_WQ_DATA_MASK))->gcwq;
529 527
530 cpu = data >> WORK_STRUCT_FLAG_BITS; 528 cpu = data >> WORK_STRUCT_FLAG_BITS;
531 if (cpu == WORK_CPU_NONE) 529 if (cpu == WORK_CPU_NONE)
@@ -3501,14 +3499,6 @@ void __init init_workqueues(void)
3501 unsigned int cpu; 3499 unsigned int cpu;
3502 int i; 3500 int i;
3503 3501
3504 /*
3505 * The pointer part of work->data is either pointing to the
3506 * cwq or contains the cpu number the work ran last on. Make
3507 * sure cpu number won't overflow into kernel pointer area so
3508 * that they can be distinguished.
3509 */
3510 BUILD_BUG_ON(WORK_CPU_LAST << WORK_STRUCT_FLAG_BITS >= PAGE_OFFSET);
3511
3512 hotcpu_notifier(workqueue_cpu_callback, CPU_PRI_WORKQUEUE); 3502 hotcpu_notifier(workqueue_cpu_callback, CPU_PRI_WORKQUEUE);
3513 3503
3514 /* initialize gcwqs */ 3504 /* initialize gcwqs */