summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDaniel Jordan <daniel.m.jordan@oracle.com>2019-09-05 21:40:24 -0400
committerHerbert Xu <herbert@gondor.apana.org.au>2019-09-13 07:15:40 -0400
commite6ce0e0807e90d38a2cefa524ac253d7a85c3f2f (patch)
tree6ab1e49a566d3d8eb3ca968f03882b4f076f8c48
parent509b3204890ab31c3e652c26424a0706bb809933 (diff)
padata: make padata_do_parallel find alternate callback CPU
padata_do_parallel currently returns -EINVAL if the callback CPU isn't in the callback cpumask. pcrypt tries to prevent this situation by keeping its own callback cpumask in sync with padata's and checks that the callback CPU it passes to padata is valid. Make padata handle this instead. padata_do_parallel now takes a pointer to the callback CPU and updates it for the caller if an alternate CPU is used. Overall behavior in terms of which callback CPUs are chosen stays the same. Prepares for removal of the padata cpumask notifier in pcrypt, which will fix a lockdep complaint about nested acquisition of the CPU hotplug lock later in the series. Signed-off-by: Daniel Jordan <daniel.m.jordan@oracle.com> Acked-by: Steffen Klassert <steffen.klassert@secunet.com> Cc: Herbert Xu <herbert@gondor.apana.org.au> Cc: Lai Jiangshan <jiangshanlai@gmail.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
-rw-r--r--crypto/pcrypt.c33
-rw-r--r--include/linux/padata.h2
-rw-r--r--kernel/padata.c27
3 files changed, 23 insertions, 39 deletions
diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c
index d67293063c7f..efca962ab12a 100644
--- a/crypto/pcrypt.c
+++ b/crypto/pcrypt.c
@@ -57,35 +57,6 @@ struct pcrypt_aead_ctx {
57 unsigned int cb_cpu; 57 unsigned int cb_cpu;
58}; 58};
59 59
60static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu,
61 struct padata_pcrypt *pcrypt)
62{
63 unsigned int cpu_index, cpu, i;
64 struct pcrypt_cpumask *cpumask;
65
66 cpu = *cb_cpu;
67
68 rcu_read_lock_bh();
69 cpumask = rcu_dereference_bh(pcrypt->cb_cpumask);
70 if (cpumask_test_cpu(cpu, cpumask->mask))
71 goto out;
72
73 if (!cpumask_weight(cpumask->mask))
74 goto out;
75
76 cpu_index = cpu % cpumask_weight(cpumask->mask);
77
78 cpu = cpumask_first(cpumask->mask);
79 for (i = 0; i < cpu_index; i++)
80 cpu = cpumask_next(cpu, cpumask->mask);
81
82 *cb_cpu = cpu;
83
84out:
85 rcu_read_unlock_bh();
86 return padata_do_parallel(pcrypt->pinst, padata, cpu);
87}
88
89static int pcrypt_aead_setkey(struct crypto_aead *parent, 60static int pcrypt_aead_setkey(struct crypto_aead *parent,
90 const u8 *key, unsigned int keylen) 61 const u8 *key, unsigned int keylen)
91{ 62{
@@ -157,7 +128,7 @@ static int pcrypt_aead_encrypt(struct aead_request *req)
157 req->cryptlen, req->iv); 128 req->cryptlen, req->iv);
158 aead_request_set_ad(creq, req->assoclen); 129 aead_request_set_ad(creq, req->assoclen);
159 130
160 err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt); 131 err = padata_do_parallel(pencrypt.pinst, padata, &ctx->cb_cpu);
161 if (!err) 132 if (!err)
162 return -EINPROGRESS; 133 return -EINPROGRESS;
163 134
@@ -199,7 +170,7 @@ static int pcrypt_aead_decrypt(struct aead_request *req)
199 req->cryptlen, req->iv); 170 req->cryptlen, req->iv);
200 aead_request_set_ad(creq, req->assoclen); 171 aead_request_set_ad(creq, req->assoclen);
201 172
202 err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pdecrypt); 173 err = padata_do_parallel(pdecrypt.pinst, padata, &ctx->cb_cpu);
203 if (!err) 174 if (!err)
204 return -EINPROGRESS; 175 return -EINPROGRESS;
205 176
diff --git a/include/linux/padata.h b/include/linux/padata.h
index 839d9319920a..f7851f8e2190 100644
--- a/include/linux/padata.h
+++ b/include/linux/padata.h
@@ -154,7 +154,7 @@ struct padata_instance {
154extern struct padata_instance *padata_alloc_possible(const char *name); 154extern struct padata_instance *padata_alloc_possible(const char *name);
155extern void padata_free(struct padata_instance *pinst); 155extern void padata_free(struct padata_instance *pinst);
156extern int padata_do_parallel(struct padata_instance *pinst, 156extern int padata_do_parallel(struct padata_instance *pinst,
157 struct padata_priv *padata, int cb_cpu); 157 struct padata_priv *padata, int *cb_cpu);
158extern void padata_do_serial(struct padata_priv *padata); 158extern void padata_do_serial(struct padata_priv *padata);
159extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, 159extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type,
160 cpumask_var_t cpumask); 160 cpumask_var_t cpumask);
diff --git a/kernel/padata.c b/kernel/padata.c
index 58728cd7f40c..9a17922ec436 100644
--- a/kernel/padata.c
+++ b/kernel/padata.c
@@ -94,17 +94,19 @@ static void padata_parallel_worker(struct work_struct *parallel_work)
94 * 94 *
95 * @pinst: padata instance 95 * @pinst: padata instance
96 * @padata: object to be parallelized 96 * @padata: object to be parallelized
97 * @cb_cpu: cpu the serialization callback function will run on, 97 * @cb_cpu: pointer to the CPU that the serialization callback function should
98 * must be in the serial cpumask of padata(i.e. cpumask.cbcpu). 98 * run on. If it's not in the serial cpumask of @pinst
99 * (i.e. cpumask.cbcpu), this function selects a fallback CPU and if
100 * none found, returns -EINVAL.
99 * 101 *
100 * The parallelization callback function will run with BHs off. 102 * The parallelization callback function will run with BHs off.
101 * Note: Every object which is parallelized by padata_do_parallel 103 * Note: Every object which is parallelized by padata_do_parallel
102 * must be seen by padata_do_serial. 104 * must be seen by padata_do_serial.
103 */ 105 */
104int padata_do_parallel(struct padata_instance *pinst, 106int padata_do_parallel(struct padata_instance *pinst,
105 struct padata_priv *padata, int cb_cpu) 107 struct padata_priv *padata, int *cb_cpu)
106{ 108{
107 int target_cpu, err; 109 int i, cpu, cpu_index, target_cpu, err;
108 struct padata_parallel_queue *queue; 110 struct padata_parallel_queue *queue;
109 struct parallel_data *pd; 111 struct parallel_data *pd;
110 112
@@ -116,8 +118,19 @@ int padata_do_parallel(struct padata_instance *pinst,
116 if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID) 118 if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID)
117 goto out; 119 goto out;
118 120
119 if (!cpumask_test_cpu(cb_cpu, pd->cpumask.cbcpu)) 121 if (!cpumask_test_cpu(*cb_cpu, pd->cpumask.cbcpu)) {
120 goto out; 122 if (!cpumask_weight(pd->cpumask.cbcpu))
123 goto out;
124
125 /* Select an alternate fallback CPU and notify the caller. */
126 cpu_index = *cb_cpu % cpumask_weight(pd->cpumask.cbcpu);
127
128 cpu = cpumask_first(pd->cpumask.cbcpu);
129 for (i = 0; i < cpu_index; i++)
130 cpu = cpumask_next(cpu, pd->cpumask.cbcpu);
131
132 *cb_cpu = cpu;
133 }
121 134
122 err = -EBUSY; 135 err = -EBUSY;
123 if ((pinst->flags & PADATA_RESET)) 136 if ((pinst->flags & PADATA_RESET))
@@ -129,7 +142,7 @@ int padata_do_parallel(struct padata_instance *pinst,
129 err = 0; 142 err = 0;
130 atomic_inc(&pd->refcnt); 143 atomic_inc(&pd->refcnt);
131 padata->pd = pd; 144 padata->pd = pd;
132 padata->cb_cpu = cb_cpu; 145 padata->cb_cpu = *cb_cpu;
133 146
134 target_cpu = padata_cpu_hash(pd); 147 target_cpu = padata_cpu_hash(pd);
135 padata->cpu = target_cpu; 148 padata->cpu = target_cpu;