padata: ensure padata_do_serial() runs on the correct CPU

If the algorithm we're parallelizing is asynchronous we might change
CPUs between padata_do_parallel() and padata_do_serial(). However, we
don't expect this to happen as we need to enqueue the padata object into
the per-cpu reorder queue we took it from, i.e. the same-cpu's parallel
queue.

Ensure we're not switching CPUs for a given padata object by tracking
the CPU within the padata object. If the serial callback gets called on
the wrong CPU, defer invoking padata_reorder() via a kernel worker on
the CPU we're expected to run on.

Signed-off-by: Mathias Krause <minipli@googlemail.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

1 files changed, 19 insertions, 1 deletions

diff --git a/kernel/padata.c b/kernel/padata.c
index b4066147bce4..f262c9a4e70a 100644
--- a/kernel/padata.c
+++ b/kernel/padata.c
@@ -131,6 +131,7 @@ int padata_do_parallel(struct padata_instance *pinst,
         padata->cb_cpu = cb_cpu;
 
         target_cpu = padata_cpu_hash(pd);
+        padata->cpu = target_cpu;
         queue = per_cpu_ptr(pd->pqueue, target_cpu);
 
         spin_lock(&queue->parallel.lock);
@@ -363,10 +364,21 @@ void padata_do_serial(struct padata_priv *padata)
         int cpu;
         struct padata_parallel_queue *pqueue;
         struct parallel_data *pd;
+        int reorder_via_wq = 0;
 
         pd = padata->pd;
 
         cpu = get_cpu();
+
+        /* We need to run on the same CPU padata_do_parallel(.., padata, ..)
+         * was called on -- or, at least, enqueue the padata object into the
+         * correct per-cpu queue.
+         */
+        if (cpu != padata->cpu) {
+                reorder_via_wq = 1;
+                cpu = padata->cpu;
+        }
+
         pqueue = per_cpu_ptr(pd->pqueue, cpu);
 
         spin_lock(&pqueue->reorder.lock);
@@ -376,7 +388,13 @@ void padata_do_serial(struct padata_priv *padata)
 
         put_cpu();
 
-        padata_reorder(pd);
+        /* If we're running on the wrong CPU, call padata_reorder() via a
+         * kernel worker.
+         */
+        if (reorder_via_wq)
+                queue_work_on(cpu, pd->pinst->wq, &pqueue->reorder_work);
+        else
+                padata_reorder(pd);
 }
 EXPORT_SYMBOL(padata_do_serial);