1 files changed, 23 insertions, 34 deletions

diff --git a/kernel/padata.c b/kernel/padata.c
index b45259931512..89fe3d1b9efb 100644
--- a/kernel/padata.c
+++ b/kernel/padata.c
@@ -1,6 +1,8 @@
 /*
  * padata.c - generic interface to process data streams in parallel
  *
+ * See Documentation/padata.txt for an api documentation.
+ *
  * Copyright (C) 2008, 2009 secunet Security Networks AG
  * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
  *
@@ -29,7 +31,6 @@
 #include <linux/sysfs.h>
 #include <linux/rcupdate.h>
 
-#define MAX_SEQ_NR (INT_MAX - NR_CPUS)
 #define MAX_OBJ_NUM 1000
 
 static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index)
@@ -43,18 +44,19 @@ static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index)
         return target_cpu;
 }
 
-static int padata_cpu_hash(struct padata_priv *padata)
+static int padata_cpu_hash(struct parallel_data *pd)
 {
         int cpu_index;
-        struct parallel_data *pd;
-
-        pd =  padata->pd;
 
         /*
          * Hash the sequence numbers to the cpus by taking
          * seq_nr mod. number of cpus in use.
          */
-        cpu_index =  padata->seq_nr % cpumask_weight(pd->cpumask.pcpu);
+
+        spin_lock(&pd->seq_lock);
+        cpu_index =  pd->seq_nr % cpumask_weight(pd->cpumask.pcpu);
+        pd->seq_nr++;
+        spin_unlock(&pd->seq_lock);
 
         return padata_index_to_cpu(pd, cpu_index);
 }
@@ -132,12 +134,7 @@ int padata_do_parallel(struct padata_instance *pinst,
         padata->pd = pd;
         padata->cb_cpu = cb_cpu;
 
-        if (unlikely(atomic_read(&pd->seq_nr) == pd->max_seq_nr))
-                atomic_set(&pd->seq_nr, -1);
-
-        padata->seq_nr = atomic_inc_return(&pd->seq_nr);
-
-        target_cpu = padata_cpu_hash(padata);
+        target_cpu = padata_cpu_hash(pd);
         queue = per_cpu_ptr(pd->pqueue, target_cpu);
 
         spin_lock(&queue->parallel.lock);
@@ -173,7 +170,7 @@ EXPORT_SYMBOL(padata_do_parallel);
 static struct padata_priv *padata_get_next(struct parallel_data *pd)
 {
         int cpu, num_cpus;
-        int next_nr, next_index;
+        unsigned int next_nr, next_index;
         struct padata_parallel_queue *queue, *next_queue;
         struct padata_priv *padata;
         struct padata_list *reorder;
@@ -189,14 +186,6 @@ static struct padata_priv *padata_get_next(struct parallel_data *pd)
         cpu = padata_index_to_cpu(pd, next_index);
         next_queue = per_cpu_ptr(pd->pqueue, cpu);
 
-        if (unlikely(next_nr > pd->max_seq_nr)) {
-                next_nr = next_nr - pd->max_seq_nr - 1;
-                next_index = next_nr % num_cpus;
-                cpu = padata_index_to_cpu(pd, next_index);
-                next_queue = per_cpu_ptr(pd->pqueue, cpu);
-                pd->processed = 0;
-        }
-
         padata = NULL;
 
         reorder = &next_queue->reorder;
@@ -205,8 +194,6 @@ static struct padata_priv *padata_get_next(struct parallel_data *pd)
                 padata = list_entry(reorder->list.next,
                                     struct padata_priv, list);
 
-                BUG_ON(next_nr != padata->seq_nr);
-
                 spin_lock(&reorder->lock);
                 list_del_init(&padata->list);
                 atomic_dec(&pd->reorder_objects);
@@ -230,6 +217,7 @@ out:
 
 static void padata_reorder(struct parallel_data *pd)
 {
+        int cb_cpu;
         struct padata_priv *padata;
         struct padata_serial_queue *squeue;
         struct padata_instance *pinst = pd->pinst;
@@ -270,13 +258,14 @@ static void padata_reorder(struct parallel_data *pd)
                         return;
                 }
 
-                squeue = per_cpu_ptr(pd->squeue, padata->cb_cpu);
+                cb_cpu = padata->cb_cpu;
+                squeue = per_cpu_ptr(pd->squeue, cb_cpu);
 
                 spin_lock(&squeue->serial.lock);
                 list_add_tail(&padata->list, &squeue->serial.list);
                 spin_unlock(&squeue->serial.lock);
 
-                queue_work_on(padata->cb_cpu, pinst->wq, &squeue->work);
+                queue_work_on(cb_cpu, pinst->wq, &squeue->work);
         }
 
         spin_unlock_bh(&pd->lock);
@@ -367,13 +356,13 @@ static int padata_setup_cpumasks(struct parallel_data *pd,
         if (!alloc_cpumask_var(&pd->cpumask.pcpu, GFP_KERNEL))
                 return -ENOMEM;
 
-        cpumask_and(pd->cpumask.pcpu, pcpumask, cpu_active_mask);
+        cpumask_and(pd->cpumask.pcpu, pcpumask, cpu_online_mask);
         if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL)) {
                 free_cpumask_var(pd->cpumask.cbcpu);
                 return -ENOMEM;
         }
 
-        cpumask_and(pd->cpumask.cbcpu, cbcpumask, cpu_active_mask);
+        cpumask_and(pd->cpumask.cbcpu, cbcpumask, cpu_online_mask);
         return 0;
 }
 
@@ -400,7 +389,7 @@ static void padata_init_squeues(struct parallel_data *pd)
 /* Initialize all percpu queues used by parallel workers */
 static void padata_init_pqueues(struct parallel_data *pd)
 {
-        int cpu_index, num_cpus, cpu;
+        int cpu_index, cpu;
         struct padata_parallel_queue *pqueue;
 
         cpu_index = 0;
@@ -415,9 +404,6 @@ static void padata_init_pqueues(struct parallel_data *pd)
                 INIT_WORK(&pqueue->work, padata_parallel_worker);
                 atomic_set(&pqueue->num_obj, 0);
         }
-
-        num_cpus = cpumask_weight(pd->cpumask.pcpu);
-        pd->max_seq_nr = num_cpus ? (MAX_SEQ_NR / num_cpus) * num_cpus - 1 : 0;
 }
 
 /* Allocate and initialize the internal cpumask dependend resources. */
@@ -444,7 +430,7 @@ static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst,
         padata_init_pqueues(pd);
         padata_init_squeues(pd);
         setup_timer(&pd->timer, padata_reorder_timer, (unsigned long)pd);
-        atomic_set(&pd->seq_nr, -1);
+        pd->seq_nr = 0;
         atomic_set(&pd->reorder_objects, 0);
         atomic_set(&pd->refcnt, 0);
         pd->pinst = pinst;
@@ -580,7 +566,7 @@ EXPORT_SYMBOL(padata_unregister_cpumask_notifier);
 static bool padata_validate_cpumask(struct padata_instance *pinst,
                                     const struct cpumask *cpumask)
 {
-        if (!cpumask_intersects(cpumask, cpu_active_mask)) {
+        if (!cpumask_intersects(cpumask, cpu_online_mask)) {
                 pinst->flags |= PADATA_INVALID;
                 return false;
         }
@@ -694,7 +680,7 @@ static int __padata_add_cpu(struct padata_instance *pinst, int cpu)
 {
         struct parallel_data *pd;
 
-        if (cpumask_test_cpu(cpu, cpu_active_mask)) {
+        if (cpumask_test_cpu(cpu, cpu_online_mask)) {
                 pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu,
                                      pinst->cpumask.cbcpu);
                 if (!pd)
@@ -762,6 +748,9 @@ static int __padata_remove_cpu(struct padata_instance *pinst, int cpu)
                         return -ENOMEM;
 
                 padata_replace(pinst, pd);
+
+                cpumask_clear_cpu(cpu, pd->cpumask.cbcpu);
+                cpumask_clear_cpu(cpu, pd->cpumask.pcpu);
         }
 
         return 0;