diff options
-rw-r--r-- | include/linux/padata.h | 53 | ||||
-rw-r--r-- | kernel/padata.c | 68 |
2 files changed, 110 insertions, 11 deletions
diff --git a/include/linux/padata.h b/include/linux/padata.h index 64836a63bd17..8d8406246eef 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h | |||
@@ -26,6 +26,17 @@ | |||
26 | #include <linux/list.h> | 26 | #include <linux/list.h> |
27 | #include <linux/timer.h> | 27 | #include <linux/timer.h> |
28 | 28 | ||
29 | /** | ||
30 | * struct padata_priv - Embedded to the users data structure. | ||
31 | * | ||
32 | * @list: List entry, to attach to the padata lists. | ||
33 | * @pd: Pointer to the internal control structure. | ||
34 | * @cb_cpu: Callback cpu for serializatioon. | ||
35 | * @seq_nr: Sequence number of the parallelized data object. | ||
36 | * @info: Used to pass information from the parallel to the serial function. | ||
37 | * @parallel: Parallel execution function. | ||
38 | * @serial: Serial complete function. | ||
39 | */ | ||
29 | struct padata_priv { | 40 | struct padata_priv { |
30 | struct list_head list; | 41 | struct list_head list; |
31 | struct parallel_data *pd; | 42 | struct parallel_data *pd; |
@@ -36,11 +47,29 @@ struct padata_priv { | |||
36 | void (*serial)(struct padata_priv *padata); | 47 | void (*serial)(struct padata_priv *padata); |
37 | }; | 48 | }; |
38 | 49 | ||
50 | /** | ||
51 | * struct padata_list | ||
52 | * | ||
53 | * @list: List head. | ||
54 | * @lock: List lock. | ||
55 | */ | ||
39 | struct padata_list { | 56 | struct padata_list { |
40 | struct list_head list; | 57 | struct list_head list; |
41 | spinlock_t lock; | 58 | spinlock_t lock; |
42 | }; | 59 | }; |
43 | 60 | ||
61 | /** | ||
62 | * struct padata_queue - The percpu padata queues. | ||
63 | * | ||
64 | * @parallel: List to wait for parallelization. | ||
65 | * @reorder: List to wait for reordering after parallel processing. | ||
66 | * @serial: List to wait for serialization after reordering. | ||
67 | * @pwork: work struct for parallelization. | ||
68 | * @swork: work struct for serialization. | ||
69 | * @pd: Backpointer to the internal control structure. | ||
70 | * @num_obj: Number of objects that are processed by this cpu. | ||
71 | * @cpu_index: Index of the cpu. | ||
72 | */ | ||
44 | struct padata_queue { | 73 | struct padata_queue { |
45 | struct padata_list parallel; | 74 | struct padata_list parallel; |
46 | struct padata_list reorder; | 75 | struct padata_list reorder; |
@@ -52,6 +81,20 @@ struct padata_queue { | |||
52 | int cpu_index; | 81 | int cpu_index; |
53 | }; | 82 | }; |
54 | 83 | ||
84 | /** | ||
85 | * struct parallel_data - Internal control structure, covers everything | ||
86 | * that depends on the cpumask in use. | ||
87 | * | ||
88 | * @pinst: padata instance. | ||
89 | * @queue: percpu padata queues. | ||
90 | * @seq_nr: The sequence number that will be attached to the next object. | ||
91 | * @reorder_objects: Number of objects waiting in the reorder queues. | ||
92 | * @refcnt: Number of objects holding a reference on this parallel_data. | ||
93 | * @max_seq_nr: Maximal used sequence number. | ||
94 | * @cpumask: cpumask in use. | ||
95 | * @lock: Reorder lock. | ||
96 | * @timer: Reorder timer. | ||
97 | */ | ||
55 | struct parallel_data { | 98 | struct parallel_data { |
56 | struct padata_instance *pinst; | 99 | struct padata_instance *pinst; |
57 | struct padata_queue *queue; | 100 | struct padata_queue *queue; |
@@ -64,6 +107,16 @@ struct parallel_data { | |||
64 | struct timer_list timer; | 107 | struct timer_list timer; |
65 | }; | 108 | }; |
66 | 109 | ||
110 | /** | ||
111 | * struct padata_instance - The overall control structure. | ||
112 | * | ||
113 | * @cpu_notifier: cpu hotplug notifier. | ||
114 | * @wq: The workqueue in use. | ||
115 | * @pd: The internal control structure. | ||
116 | * @cpumask: User supplied cpumask. | ||
117 | * @lock: padata instance lock. | ||
118 | * @flags: padata flags. | ||
119 | */ | ||
67 | struct padata_instance { | 120 | struct padata_instance { |
68 | struct notifier_block cpu_notifier; | 121 | struct notifier_block cpu_notifier; |
69 | struct workqueue_struct *wq; | 122 | struct workqueue_struct *wq; |
diff --git a/kernel/padata.c b/kernel/padata.c index ec6b8b7cf951..ca89dfb69805 100644 --- a/kernel/padata.c +++ b/kernel/padata.c | |||
@@ -88,7 +88,7 @@ static void padata_parallel_worker(struct work_struct *work) | |||
88 | local_bh_enable(); | 88 | local_bh_enable(); |
89 | } | 89 | } |
90 | 90 | ||
91 | /* | 91 | /** |
92 | * padata_do_parallel - padata parallelization function | 92 | * padata_do_parallel - padata parallelization function |
93 | * | 93 | * |
94 | * @pinst: padata instance | 94 | * @pinst: padata instance |
@@ -152,6 +152,23 @@ out: | |||
152 | } | 152 | } |
153 | EXPORT_SYMBOL(padata_do_parallel); | 153 | EXPORT_SYMBOL(padata_do_parallel); |
154 | 154 | ||
155 | /* | ||
156 | * padata_get_next - Get the next object that needs serialization. | ||
157 | * | ||
158 | * Return values are: | ||
159 | * | ||
160 | * A pointer to the control struct of the next object that needs | ||
161 | * serialization, if present in one of the percpu reorder queues. | ||
162 | * | ||
163 | * NULL, if all percpu reorder queues are empty. | ||
164 | * | ||
165 | * -EINPROGRESS, if the next object that needs serialization will | ||
166 | * be parallel processed by another cpu and is not yet present in | ||
167 | * the cpu's reorder queue. | ||
168 | * | ||
169 | * -ENODATA, if this cpu has to do the parallel processing for | ||
170 | * the next object. | ||
171 | */ | ||
155 | static struct padata_priv *padata_get_next(struct parallel_data *pd) | 172 | static struct padata_priv *padata_get_next(struct parallel_data *pd) |
156 | { | 173 | { |
157 | int cpu, num_cpus, empty, calc_seq_nr; | 174 | int cpu, num_cpus, empty, calc_seq_nr; |
@@ -173,7 +190,7 @@ static struct padata_priv *padata_get_next(struct parallel_data *pd) | |||
173 | 190 | ||
174 | /* | 191 | /* |
175 | * Calculate the seq_nr of the object that should be | 192 | * Calculate the seq_nr of the object that should be |
176 | * next in this queue. | 193 | * next in this reorder queue. |
177 | */ | 194 | */ |
178 | overrun = 0; | 195 | overrun = 0; |
179 | calc_seq_nr = (atomic_read(&queue->num_obj) * num_cpus) | 196 | calc_seq_nr = (atomic_read(&queue->num_obj) * num_cpus) |
@@ -248,15 +265,36 @@ static void padata_reorder(struct parallel_data *pd) | |||
248 | struct padata_queue *queue; | 265 | struct padata_queue *queue; |
249 | struct padata_instance *pinst = pd->pinst; | 266 | struct padata_instance *pinst = pd->pinst; |
250 | 267 | ||
268 | /* | ||
269 | * We need to ensure that only one cpu can work on dequeueing of | ||
270 | * the reorder queue the time. Calculating in which percpu reorder | ||
271 | * queue the next object will arrive takes some time. A spinlock | ||
272 | * would be highly contended. Also it is not clear in which order | ||
273 | * the objects arrive to the reorder queues. So a cpu could wait to | ||
274 | * get the lock just to notice that there is nothing to do at the | ||
275 | * moment. Therefore we use a trylock and let the holder of the lock | ||
276 | * care for all the objects enqueued during the holdtime of the lock. | ||
277 | */ | ||
251 | if (!spin_trylock_bh(&pd->lock)) | 278 | if (!spin_trylock_bh(&pd->lock)) |
252 | return; | 279 | return; |
253 | 280 | ||
254 | while (1) { | 281 | while (1) { |
255 | padata = padata_get_next(pd); | 282 | padata = padata_get_next(pd); |
256 | 283 | ||
284 | /* | ||
285 | * All reorder queues are empty, or the next object that needs | ||
286 | * serialization is parallel processed by another cpu and is | ||
287 | * still on it's way to the cpu's reorder queue, nothing to | ||
288 | * do for now. | ||
289 | */ | ||
257 | if (!padata || PTR_ERR(padata) == -EINPROGRESS) | 290 | if (!padata || PTR_ERR(padata) == -EINPROGRESS) |
258 | break; | 291 | break; |
259 | 292 | ||
293 | /* | ||
294 | * This cpu has to do the parallel processing of the next | ||
295 | * object. It's waiting in the cpu's parallelization queue, | ||
296 | * so exit imediately. | ||
297 | */ | ||
260 | if (PTR_ERR(padata) == -ENODATA) { | 298 | if (PTR_ERR(padata) == -ENODATA) { |
261 | del_timer(&pd->timer); | 299 | del_timer(&pd->timer); |
262 | spin_unlock_bh(&pd->lock); | 300 | spin_unlock_bh(&pd->lock); |
@@ -274,6 +312,11 @@ static void padata_reorder(struct parallel_data *pd) | |||
274 | 312 | ||
275 | spin_unlock_bh(&pd->lock); | 313 | spin_unlock_bh(&pd->lock); |
276 | 314 | ||
315 | /* | ||
316 | * The next object that needs serialization might have arrived to | ||
317 | * the reorder queues in the meantime, we will be called again | ||
318 | * from the timer function if noone else cares for it. | ||
319 | */ | ||
277 | if (atomic_read(&pd->reorder_objects) | 320 | if (atomic_read(&pd->reorder_objects) |
278 | && !(pinst->flags & PADATA_RESET)) | 321 | && !(pinst->flags & PADATA_RESET)) |
279 | mod_timer(&pd->timer, jiffies + HZ); | 322 | mod_timer(&pd->timer, jiffies + HZ); |
@@ -318,7 +361,7 @@ static void padata_serial_worker(struct work_struct *work) | |||
318 | local_bh_enable(); | 361 | local_bh_enable(); |
319 | } | 362 | } |
320 | 363 | ||
321 | /* | 364 | /** |
322 | * padata_do_serial - padata serialization function | 365 | * padata_do_serial - padata serialization function |
323 | * | 366 | * |
324 | * @padata: object to be serialized. | 367 | * @padata: object to be serialized. |
@@ -348,6 +391,7 @@ void padata_do_serial(struct padata_priv *padata) | |||
348 | } | 391 | } |
349 | EXPORT_SYMBOL(padata_do_serial); | 392 | EXPORT_SYMBOL(padata_do_serial); |
350 | 393 | ||
394 | /* Allocate and initialize the internal cpumask dependend resources. */ | ||
351 | static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst, | 395 | static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst, |
352 | const struct cpumask *cpumask) | 396 | const struct cpumask *cpumask) |
353 | { | 397 | { |
@@ -417,6 +461,7 @@ static void padata_free_pd(struct parallel_data *pd) | |||
417 | kfree(pd); | 461 | kfree(pd); |
418 | } | 462 | } |
419 | 463 | ||
464 | /* Flush all objects out of the padata queues. */ | ||
420 | static void padata_flush_queues(struct parallel_data *pd) | 465 | static void padata_flush_queues(struct parallel_data *pd) |
421 | { | 466 | { |
422 | int cpu; | 467 | int cpu; |
@@ -440,6 +485,7 @@ static void padata_flush_queues(struct parallel_data *pd) | |||
440 | BUG_ON(atomic_read(&pd->refcnt) != 0); | 485 | BUG_ON(atomic_read(&pd->refcnt) != 0); |
441 | } | 486 | } |
442 | 487 | ||
488 | /* Replace the internal control stucture with a new one. */ | ||
443 | static void padata_replace(struct padata_instance *pinst, | 489 | static void padata_replace(struct padata_instance *pinst, |
444 | struct parallel_data *pd_new) | 490 | struct parallel_data *pd_new) |
445 | { | 491 | { |
@@ -457,7 +503,7 @@ static void padata_replace(struct padata_instance *pinst, | |||
457 | pinst->flags &= ~PADATA_RESET; | 503 | pinst->flags &= ~PADATA_RESET; |
458 | } | 504 | } |
459 | 505 | ||
460 | /* | 506 | /** |
461 | * padata_set_cpumask - set the cpumask that padata should use | 507 | * padata_set_cpumask - set the cpumask that padata should use |
462 | * | 508 | * |
463 | * @pinst: padata instance | 509 | * @pinst: padata instance |
@@ -507,7 +553,7 @@ static int __padata_add_cpu(struct padata_instance *pinst, int cpu) | |||
507 | return 0; | 553 | return 0; |
508 | } | 554 | } |
509 | 555 | ||
510 | /* | 556 | /** |
511 | * padata_add_cpu - add a cpu to the padata cpumask | 557 | * padata_add_cpu - add a cpu to the padata cpumask |
512 | * | 558 | * |
513 | * @pinst: padata instance | 559 | * @pinst: padata instance |
@@ -545,7 +591,7 @@ static int __padata_remove_cpu(struct padata_instance *pinst, int cpu) | |||
545 | return 0; | 591 | return 0; |
546 | } | 592 | } |
547 | 593 | ||
548 | /* | 594 | /** |
549 | * padata_remove_cpu - remove a cpu from the padata cpumask | 595 | * padata_remove_cpu - remove a cpu from the padata cpumask |
550 | * | 596 | * |
551 | * @pinst: padata instance | 597 | * @pinst: padata instance |
@@ -568,7 +614,7 @@ int padata_remove_cpu(struct padata_instance *pinst, int cpu) | |||
568 | } | 614 | } |
569 | EXPORT_SYMBOL(padata_remove_cpu); | 615 | EXPORT_SYMBOL(padata_remove_cpu); |
570 | 616 | ||
571 | /* | 617 | /** |
572 | * padata_start - start the parallel processing | 618 | * padata_start - start the parallel processing |
573 | * | 619 | * |
574 | * @pinst: padata instance to start | 620 | * @pinst: padata instance to start |
@@ -581,7 +627,7 @@ void padata_start(struct padata_instance *pinst) | |||
581 | } | 627 | } |
582 | EXPORT_SYMBOL(padata_start); | 628 | EXPORT_SYMBOL(padata_start); |
583 | 629 | ||
584 | /* | 630 | /** |
585 | * padata_stop - stop the parallel processing | 631 | * padata_stop - stop the parallel processing |
586 | * | 632 | * |
587 | * @pinst: padata instance to stop | 633 | * @pinst: padata instance to stop |
@@ -648,7 +694,7 @@ static int padata_cpu_callback(struct notifier_block *nfb, | |||
648 | } | 694 | } |
649 | #endif | 695 | #endif |
650 | 696 | ||
651 | /* | 697 | /** |
652 | * padata_alloc - allocate and initialize a padata instance | 698 | * padata_alloc - allocate and initialize a padata instance |
653 | * | 699 | * |
654 | * @cpumask: cpumask that padata uses for parallelization | 700 | * @cpumask: cpumask that padata uses for parallelization |
@@ -703,10 +749,10 @@ err: | |||
703 | } | 749 | } |
704 | EXPORT_SYMBOL(padata_alloc); | 750 | EXPORT_SYMBOL(padata_alloc); |
705 | 751 | ||
706 | /* | 752 | /** |
707 | * padata_free - free a padata instance | 753 | * padata_free - free a padata instance |
708 | * | 754 | * |
709 | * @ padata_inst: padata instance to free | 755 | * @padata_inst: padata instance to free |
710 | */ | 756 | */ |
711 | void padata_free(struct padata_instance *pinst) | 757 | void padata_free(struct padata_instance *pinst) |
712 | { | 758 | { |