aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/workqueue.c100
1 files changed, 18 insertions, 82 deletions
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 4608563cdd63..20d6237d7498 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -206,8 +206,6 @@ struct workqueue_struct {
206 struct list_head flusher_queue; /* F: flush waiters */ 206 struct list_head flusher_queue; /* F: flush waiters */
207 struct list_head flusher_overflow; /* F: flush overflow list */ 207 struct list_head flusher_overflow; /* F: flush overflow list */
208 208
209 unsigned long single_cpu; /* cpu for single cpu wq */
210
211 cpumask_var_t mayday_mask; /* cpus requesting rescue */ 209 cpumask_var_t mayday_mask; /* cpus requesting rescue */
212 struct worker *rescuer; /* I: rescue worker */ 210 struct worker *rescuer; /* I: rescue worker */
213 211
@@ -889,34 +887,6 @@ static void insert_work(struct cpu_workqueue_struct *cwq,
889 wake_up_worker(gcwq); 887 wake_up_worker(gcwq);
890} 888}
891 889
892/**
893 * cwq_unbind_single_cpu - unbind cwq from single cpu workqueue processing
894 * @cwq: cwq to unbind
895 *
896 * Try to unbind @cwq from single cpu workqueue processing. If
897 * @cwq->wq is frozen, unbind is delayed till the workqueue is thawed.
898 *
899 * CONTEXT:
900 * spin_lock_irq(gcwq->lock).
901 */
902static void cwq_unbind_single_cpu(struct cpu_workqueue_struct *cwq)
903{
904 struct workqueue_struct *wq = cwq->wq;
905 struct global_cwq *gcwq = cwq->gcwq;
906
907 BUG_ON(wq->single_cpu != gcwq->cpu);
908 /*
909 * Unbind from workqueue if @cwq is not frozen. If frozen,
910 * thaw_workqueues() will either restart processing on this
911 * cpu or unbind if empty. This keeps works queued while
912 * frozen fully ordered and flushable.
913 */
914 if (likely(!(gcwq->flags & GCWQ_FREEZING))) {
915 smp_wmb(); /* paired with cmpxchg() in __queue_work() */
916 wq->single_cpu = WORK_CPU_NONE;
917 }
918}
919
920static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, 890static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,
921 struct work_struct *work) 891 struct work_struct *work)
922{ 892{
@@ -924,20 +894,16 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,
924 struct cpu_workqueue_struct *cwq; 894 struct cpu_workqueue_struct *cwq;
925 struct list_head *worklist; 895 struct list_head *worklist;
926 unsigned long flags; 896 unsigned long flags;
927 bool arbitrate;
928 897
929 debug_work_activate(work); 898 debug_work_activate(work);
930 899
931 if (unlikely(cpu == WORK_CPU_UNBOUND)) 900 /* determine gcwq to use */
932 cpu = raw_smp_processor_id(); 901 if (!(wq->flags & WQ_UNBOUND)) {
933
934 /*
935 * Determine gcwq to use. SINGLE_CPU is inherently
936 * NON_REENTRANT, so test it first.
937 */
938 if (!(wq->flags & (WQ_SINGLE_CPU | WQ_UNBOUND))) {
939 struct global_cwq *last_gcwq; 902 struct global_cwq *last_gcwq;
940 903
904 if (unlikely(cpu == WORK_CPU_UNBOUND))
905 cpu = raw_smp_processor_id();
906
941 /* 907 /*
942 * It's multi cpu. If @wq is non-reentrant and @work 908 * It's multi cpu. If @wq is non-reentrant and @work
943 * was previously on a different cpu, it might still 909 * was previously on a different cpu, it might still
@@ -962,38 +928,6 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,
962 } 928 }
963 } else 929 } else
964 spin_lock_irqsave(&gcwq->lock, flags); 930 spin_lock_irqsave(&gcwq->lock, flags);
965 } else if (!(wq->flags & WQ_UNBOUND)) {
966 unsigned int req_cpu = cpu;
967
968 /*
969 * It's a bit more complex for single cpu workqueues.
970 * We first need to determine which cpu is going to be
971 * used. If no cpu is currently serving this
972 * workqueue, arbitrate using atomic accesses to
973 * wq->single_cpu; otherwise, use the current one.
974 */
975 retry:
976 cpu = wq->single_cpu;
977 arbitrate = cpu == WORK_CPU_NONE;
978 if (arbitrate)
979 cpu = req_cpu;
980
981 gcwq = get_gcwq(cpu);
982 spin_lock_irqsave(&gcwq->lock, flags);
983
984 /*
985 * The following cmpxchg() is a full barrier paired
986 * with smp_wmb() in cwq_unbind_single_cpu() and
987 * guarantees that all changes to wq->st_* fields are
988 * visible on the new cpu after this point.
989 */
990 if (arbitrate)
991 cmpxchg(&wq->single_cpu, WORK_CPU_NONE, cpu);
992
993 if (unlikely(wq->single_cpu != cpu)) {
994 spin_unlock_irqrestore(&gcwq->lock, flags);
995 goto retry;
996 }
997 } else { 931 } else {
998 gcwq = get_gcwq(WORK_CPU_UNBOUND); 932 gcwq = get_gcwq(WORK_CPU_UNBOUND);
999 spin_lock_irqsave(&gcwq->lock, flags); 933 spin_lock_irqsave(&gcwq->lock, flags);
@@ -1105,19 +1039,30 @@ int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1105 struct work_struct *work = &dwork->work; 1039 struct work_struct *work = &dwork->work;
1106 1040
1107 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { 1041 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1108 struct global_cwq *gcwq = get_work_gcwq(work); 1042 unsigned int lcpu;
1109 unsigned int lcpu = gcwq ? gcwq->cpu : raw_smp_processor_id();
1110 1043
1111 BUG_ON(timer_pending(timer)); 1044 BUG_ON(timer_pending(timer));
1112 BUG_ON(!list_empty(&work->entry)); 1045 BUG_ON(!list_empty(&work->entry));
1113 1046
1114 timer_stats_timer_set_start_info(&dwork->timer); 1047 timer_stats_timer_set_start_info(&dwork->timer);
1048
1115 /* 1049 /*
1116 * This stores cwq for the moment, for the timer_fn. 1050 * This stores cwq for the moment, for the timer_fn.
1117 * Note that the work's gcwq is preserved to allow 1051 * Note that the work's gcwq is preserved to allow
1118 * reentrance detection for delayed works. 1052 * reentrance detection for delayed works.
1119 */ 1053 */
1054 if (!(wq->flags & WQ_UNBOUND)) {
1055 struct global_cwq *gcwq = get_work_gcwq(work);
1056
1057 if (gcwq && gcwq->cpu != WORK_CPU_UNBOUND)
1058 lcpu = gcwq->cpu;
1059 else
1060 lcpu = raw_smp_processor_id();
1061 } else
1062 lcpu = WORK_CPU_UNBOUND;
1063
1120 set_work_cwq(work, get_cwq(lcpu, wq), 0); 1064 set_work_cwq(work, get_cwq(lcpu, wq), 0);
1065
1121 timer->expires = jiffies + delay; 1066 timer->expires = jiffies + delay;
1122 timer->data = (unsigned long)dwork; 1067 timer->data = (unsigned long)dwork;
1123 timer->function = delayed_work_timer_fn; 1068 timer->function = delayed_work_timer_fn;
@@ -1696,9 +1641,6 @@ static void cwq_dec_nr_in_flight(struct cpu_workqueue_struct *cwq, int color)
1696 /* one down, submit a delayed one */ 1641 /* one down, submit a delayed one */
1697 if (cwq->nr_active < cwq->max_active) 1642 if (cwq->nr_active < cwq->max_active)
1698 cwq_activate_first_delayed(cwq); 1643 cwq_activate_first_delayed(cwq);
1699 } else if (!cwq->nr_active && cwq->wq->flags & WQ_SINGLE_CPU) {
1700 /* this was the last work, unbind from single cpu */
1701 cwq_unbind_single_cpu(cwq);
1702 } 1644 }
1703 1645
1704 /* is flush in progress and are we at the flushing tip? */ 1646 /* is flush in progress and are we at the flushing tip? */
@@ -2751,7 +2693,6 @@ struct workqueue_struct *__alloc_workqueue_key(const char *name,
2751 atomic_set(&wq->nr_cwqs_to_flush, 0); 2693 atomic_set(&wq->nr_cwqs_to_flush, 0);
2752 INIT_LIST_HEAD(&wq->flusher_queue); 2694 INIT_LIST_HEAD(&wq->flusher_queue);
2753 INIT_LIST_HEAD(&wq->flusher_overflow); 2695 INIT_LIST_HEAD(&wq->flusher_overflow);
2754 wq->single_cpu = WORK_CPU_NONE;
2755 2696
2756 wq->name = name; 2697 wq->name = name;
2757 lockdep_init_map(&wq->lockdep_map, lock_name, key, 0); 2698 lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
@@ -3513,11 +3454,6 @@ void thaw_workqueues(void)
3513 while (!list_empty(&cwq->delayed_works) && 3454 while (!list_empty(&cwq->delayed_works) &&
3514 cwq->nr_active < cwq->max_active) 3455 cwq->nr_active < cwq->max_active)
3515 cwq_activate_first_delayed(cwq); 3456 cwq_activate_first_delayed(cwq);
3516
3517 /* perform delayed unbind from single cpu if empty */
3518 if (wq->single_cpu == gcwq->cpu &&
3519 !cwq->nr_active && list_empty(&cwq->delayed_works))
3520 cwq_unbind_single_cpu(cwq);
3521 } 3457 }
3522 3458
3523 wake_up_worker(gcwq); 3459 wake_up_worker(gcwq);