diff options
Diffstat (limited to 'drivers/infiniband/hw/ehca/ehca_cq.c')
-rw-r--r-- | drivers/infiniband/hw/ehca/ehca_cq.c | 427 |
1 files changed, 427 insertions, 0 deletions
diff --git a/drivers/infiniband/hw/ehca/ehca_cq.c b/drivers/infiniband/hw/ehca/ehca_cq.c new file mode 100644 index 00000000000..458fe19648a --- /dev/null +++ b/drivers/infiniband/hw/ehca/ehca_cq.c | |||
@@ -0,0 +1,427 @@ | |||
1 | /* | ||
2 | * IBM eServer eHCA Infiniband device driver for Linux on POWER | ||
3 | * | ||
4 | * Completion queue handling | ||
5 | * | ||
6 | * Authors: Waleri Fomin <fomin@de.ibm.com> | ||
7 | * Khadija Souissi <souissi@de.ibm.com> | ||
8 | * Reinhard Ernst <rernst@de.ibm.com> | ||
9 | * Heiko J Schick <schickhj@de.ibm.com> | ||
10 | * Hoang-Nam Nguyen <hnguyen@de.ibm.com> | ||
11 | * | ||
12 | * | ||
13 | * Copyright (c) 2005 IBM Corporation | ||
14 | * | ||
15 | * All rights reserved. | ||
16 | * | ||
17 | * This source code is distributed under a dual license of GPL v2.0 and OpenIB | ||
18 | * BSD. | ||
19 | * | ||
20 | * OpenIB BSD License | ||
21 | * | ||
22 | * Redistribution and use in source and binary forms, with or without | ||
23 | * modification, are permitted provided that the following conditions are met: | ||
24 | * | ||
25 | * Redistributions of source code must retain the above copyright notice, this | ||
26 | * list of conditions and the following disclaimer. | ||
27 | * | ||
28 | * Redistributions in binary form must reproduce the above copyright notice, | ||
29 | * this list of conditions and the following disclaimer in the documentation | ||
30 | * and/or other materials | ||
31 | * provided with the distribution. | ||
32 | * | ||
33 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
34 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
35 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
36 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | ||
37 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
38 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
39 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | ||
40 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER | ||
41 | * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
42 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
43 | * POSSIBILITY OF SUCH DAMAGE. | ||
44 | */ | ||
45 | |||
46 | #include <asm/current.h> | ||
47 | |||
48 | #include "ehca_iverbs.h" | ||
49 | #include "ehca_classes.h" | ||
50 | #include "ehca_irq.h" | ||
51 | #include "hcp_if.h" | ||
52 | |||
53 | static struct kmem_cache *cq_cache; | ||
54 | |||
55 | int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp) | ||
56 | { | ||
57 | unsigned int qp_num = qp->real_qp_num; | ||
58 | unsigned int key = qp_num & (QP_HASHTAB_LEN-1); | ||
59 | unsigned long spl_flags; | ||
60 | |||
61 | spin_lock_irqsave(&cq->spinlock, spl_flags); | ||
62 | hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]); | ||
63 | spin_unlock_irqrestore(&cq->spinlock, spl_flags); | ||
64 | |||
65 | ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x", | ||
66 | cq->cq_number, qp_num); | ||
67 | |||
68 | return 0; | ||
69 | } | ||
70 | |||
71 | int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num) | ||
72 | { | ||
73 | int ret = -EINVAL; | ||
74 | unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1); | ||
75 | struct hlist_node *iter; | ||
76 | struct ehca_qp *qp; | ||
77 | unsigned long spl_flags; | ||
78 | |||
79 | spin_lock_irqsave(&cq->spinlock, spl_flags); | ||
80 | hlist_for_each(iter, &cq->qp_hashtab[key]) { | ||
81 | qp = hlist_entry(iter, struct ehca_qp, list_entries); | ||
82 | if (qp->real_qp_num == real_qp_num) { | ||
83 | hlist_del(iter); | ||
84 | ehca_dbg(cq->ib_cq.device, | ||
85 | "removed qp from cq .cq_num=%x real_qp_num=%x", | ||
86 | cq->cq_number, real_qp_num); | ||
87 | ret = 0; | ||
88 | break; | ||
89 | } | ||
90 | } | ||
91 | spin_unlock_irqrestore(&cq->spinlock, spl_flags); | ||
92 | if (ret) | ||
93 | ehca_err(cq->ib_cq.device, | ||
94 | "qp not found cq_num=%x real_qp_num=%x", | ||
95 | cq->cq_number, real_qp_num); | ||
96 | |||
97 | return ret; | ||
98 | } | ||
99 | |||
100 | struct ehca_qp* ehca_cq_get_qp(struct ehca_cq *cq, int real_qp_num) | ||
101 | { | ||
102 | struct ehca_qp *ret = NULL; | ||
103 | unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1); | ||
104 | struct hlist_node *iter; | ||
105 | struct ehca_qp *qp; | ||
106 | hlist_for_each(iter, &cq->qp_hashtab[key]) { | ||
107 | qp = hlist_entry(iter, struct ehca_qp, list_entries); | ||
108 | if (qp->real_qp_num == real_qp_num) { | ||
109 | ret = qp; | ||
110 | break; | ||
111 | } | ||
112 | } | ||
113 | return ret; | ||
114 | } | ||
115 | |||
116 | struct ib_cq *ehca_create_cq(struct ib_device *device, int cqe, | ||
117 | struct ib_ucontext *context, | ||
118 | struct ib_udata *udata) | ||
119 | { | ||
120 | static const u32 additional_cqe = 20; | ||
121 | struct ib_cq *cq; | ||
122 | struct ehca_cq *my_cq; | ||
123 | struct ehca_shca *shca = | ||
124 | container_of(device, struct ehca_shca, ib_device); | ||
125 | struct ipz_adapter_handle adapter_handle; | ||
126 | struct ehca_alloc_cq_parms param; /* h_call's out parameters */ | ||
127 | struct h_galpa gal; | ||
128 | void *vpage; | ||
129 | u32 counter; | ||
130 | u64 rpage, cqx_fec, h_ret; | ||
131 | int ipz_rc, ret, i; | ||
132 | unsigned long flags; | ||
133 | |||
134 | if (cqe >= 0xFFFFFFFF - 64 - additional_cqe) | ||
135 | return ERR_PTR(-EINVAL); | ||
136 | |||
137 | my_cq = kmem_cache_alloc(cq_cache, SLAB_KERNEL); | ||
138 | if (!my_cq) { | ||
139 | ehca_err(device, "Out of memory for ehca_cq struct device=%p", | ||
140 | device); | ||
141 | return ERR_PTR(-ENOMEM); | ||
142 | } | ||
143 | |||
144 | memset(my_cq, 0, sizeof(struct ehca_cq)); | ||
145 | memset(¶m, 0, sizeof(struct ehca_alloc_cq_parms)); | ||
146 | |||
147 | spin_lock_init(&my_cq->spinlock); | ||
148 | spin_lock_init(&my_cq->cb_lock); | ||
149 | spin_lock_init(&my_cq->task_lock); | ||
150 | my_cq->ownpid = current->tgid; | ||
151 | |||
152 | cq = &my_cq->ib_cq; | ||
153 | |||
154 | adapter_handle = shca->ipz_hca_handle; | ||
155 | param.eq_handle = shca->eq.ipz_eq_handle; | ||
156 | |||
157 | do { | ||
158 | if (!idr_pre_get(&ehca_cq_idr, GFP_KERNEL)) { | ||
159 | cq = ERR_PTR(-ENOMEM); | ||
160 | ehca_err(device, "Can't reserve idr nr. device=%p", | ||
161 | device); | ||
162 | goto create_cq_exit1; | ||
163 | } | ||
164 | |||
165 | spin_lock_irqsave(&ehca_cq_idr_lock, flags); | ||
166 | ret = idr_get_new(&ehca_cq_idr, my_cq, &my_cq->token); | ||
167 | spin_unlock_irqrestore(&ehca_cq_idr_lock, flags); | ||
168 | |||
169 | } while (ret == -EAGAIN); | ||
170 | |||
171 | if (ret) { | ||
172 | cq = ERR_PTR(-ENOMEM); | ||
173 | ehca_err(device, "Can't allocate new idr entry. device=%p", | ||
174 | device); | ||
175 | goto create_cq_exit1; | ||
176 | } | ||
177 | |||
178 | /* | ||
179 | * CQs maximum depth is 4GB-64, but we need additional 20 as buffer | ||
180 | * for receiving errors CQEs. | ||
181 | */ | ||
182 | param.nr_cqe = cqe + additional_cqe; | ||
183 | h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, ¶m); | ||
184 | |||
185 | if (h_ret != H_SUCCESS) { | ||
186 | ehca_err(device, "hipz_h_alloc_resource_cq() failed " | ||
187 | "h_ret=%lx device=%p", h_ret, device); | ||
188 | cq = ERR_PTR(ehca2ib_return_code(h_ret)); | ||
189 | goto create_cq_exit2; | ||
190 | } | ||
191 | |||
192 | ipz_rc = ipz_queue_ctor(&my_cq->ipz_queue, param.act_pages, | ||
193 | EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0); | ||
194 | if (!ipz_rc) { | ||
195 | ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%x device=%p", | ||
196 | ipz_rc, device); | ||
197 | cq = ERR_PTR(-EINVAL); | ||
198 | goto create_cq_exit3; | ||
199 | } | ||
200 | |||
201 | for (counter = 0; counter < param.act_pages; counter++) { | ||
202 | vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue); | ||
203 | if (!vpage) { | ||
204 | ehca_err(device, "ipz_qpageit_get_inc() " | ||
205 | "returns NULL device=%p", device); | ||
206 | cq = ERR_PTR(-EAGAIN); | ||
207 | goto create_cq_exit4; | ||
208 | } | ||
209 | rpage = virt_to_abs(vpage); | ||
210 | |||
211 | h_ret = hipz_h_register_rpage_cq(adapter_handle, | ||
212 | my_cq->ipz_cq_handle, | ||
213 | &my_cq->pf, | ||
214 | 0, | ||
215 | 0, | ||
216 | rpage, | ||
217 | 1, | ||
218 | my_cq->galpas. | ||
219 | kernel); | ||
220 | |||
221 | if (h_ret < H_SUCCESS) { | ||
222 | ehca_err(device, "hipz_h_register_rpage_cq() failed " | ||
223 | "ehca_cq=%p cq_num=%x h_ret=%lx counter=%i " | ||
224 | "act_pages=%i", my_cq, my_cq->cq_number, | ||
225 | h_ret, counter, param.act_pages); | ||
226 | cq = ERR_PTR(-EINVAL); | ||
227 | goto create_cq_exit4; | ||
228 | } | ||
229 | |||
230 | if (counter == (param.act_pages - 1)) { | ||
231 | vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue); | ||
232 | if ((h_ret != H_SUCCESS) || vpage) { | ||
233 | ehca_err(device, "Registration of pages not " | ||
234 | "complete ehca_cq=%p cq_num=%x " | ||
235 | "h_ret=%lx", my_cq, my_cq->cq_number, | ||
236 | h_ret); | ||
237 | cq = ERR_PTR(-EAGAIN); | ||
238 | goto create_cq_exit4; | ||
239 | } | ||
240 | } else { | ||
241 | if (h_ret != H_PAGE_REGISTERED) { | ||
242 | ehca_err(device, "Registration of page failed " | ||
243 | "ehca_cq=%p cq_num=%x h_ret=%lx" | ||
244 | "counter=%i act_pages=%i", | ||
245 | my_cq, my_cq->cq_number, | ||
246 | h_ret, counter, param.act_pages); | ||
247 | cq = ERR_PTR(-ENOMEM); | ||
248 | goto create_cq_exit4; | ||
249 | } | ||
250 | } | ||
251 | } | ||
252 | |||
253 | ipz_qeit_reset(&my_cq->ipz_queue); | ||
254 | |||
255 | gal = my_cq->galpas.kernel; | ||
256 | cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec)); | ||
257 | ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%lx", | ||
258 | my_cq, my_cq->cq_number, cqx_fec); | ||
259 | |||
260 | my_cq->ib_cq.cqe = my_cq->nr_of_entries = | ||
261 | param.act_nr_of_entries - additional_cqe; | ||
262 | my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff; | ||
263 | |||
264 | for (i = 0; i < QP_HASHTAB_LEN; i++) | ||
265 | INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]); | ||
266 | |||
267 | if (context) { | ||
268 | struct ipz_queue *ipz_queue = &my_cq->ipz_queue; | ||
269 | struct ehca_create_cq_resp resp; | ||
270 | struct vm_area_struct *vma; | ||
271 | memset(&resp, 0, sizeof(resp)); | ||
272 | resp.cq_number = my_cq->cq_number; | ||
273 | resp.token = my_cq->token; | ||
274 | resp.ipz_queue.qe_size = ipz_queue->qe_size; | ||
275 | resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg; | ||
276 | resp.ipz_queue.queue_length = ipz_queue->queue_length; | ||
277 | resp.ipz_queue.pagesize = ipz_queue->pagesize; | ||
278 | resp.ipz_queue.toggle_state = ipz_queue->toggle_state; | ||
279 | ret = ehca_mmap_nopage(((u64)(my_cq->token) << 32) | 0x12000000, | ||
280 | ipz_queue->queue_length, | ||
281 | (void**)&resp.ipz_queue.queue, | ||
282 | &vma); | ||
283 | if (ret) { | ||
284 | ehca_err(device, "Could not mmap queue pages"); | ||
285 | cq = ERR_PTR(ret); | ||
286 | goto create_cq_exit4; | ||
287 | } | ||
288 | my_cq->uspace_queue = resp.ipz_queue.queue; | ||
289 | resp.galpas = my_cq->galpas; | ||
290 | ret = ehca_mmap_register(my_cq->galpas.user.fw_handle, | ||
291 | (void**)&resp.galpas.kernel.fw_handle, | ||
292 | &vma); | ||
293 | if (ret) { | ||
294 | ehca_err(device, "Could not mmap fw_handle"); | ||
295 | cq = ERR_PTR(ret); | ||
296 | goto create_cq_exit5; | ||
297 | } | ||
298 | my_cq->uspace_fwh = (u64)resp.galpas.kernel.fw_handle; | ||
299 | if (ib_copy_to_udata(udata, &resp, sizeof(resp))) { | ||
300 | ehca_err(device, "Copy to udata failed."); | ||
301 | goto create_cq_exit6; | ||
302 | } | ||
303 | } | ||
304 | |||
305 | return cq; | ||
306 | |||
307 | create_cq_exit6: | ||
308 | ehca_munmap(my_cq->uspace_fwh, EHCA_PAGESIZE); | ||
309 | |||
310 | create_cq_exit5: | ||
311 | ehca_munmap(my_cq->uspace_queue, my_cq->ipz_queue.queue_length); | ||
312 | |||
313 | create_cq_exit4: | ||
314 | ipz_queue_dtor(&my_cq->ipz_queue); | ||
315 | |||
316 | create_cq_exit3: | ||
317 | h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1); | ||
318 | if (h_ret != H_SUCCESS) | ||
319 | ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p " | ||
320 | "cq_num=%x h_ret=%lx", my_cq, my_cq->cq_number, h_ret); | ||
321 | |||
322 | create_cq_exit2: | ||
323 | spin_lock_irqsave(&ehca_cq_idr_lock, flags); | ||
324 | idr_remove(&ehca_cq_idr, my_cq->token); | ||
325 | spin_unlock_irqrestore(&ehca_cq_idr_lock, flags); | ||
326 | |||
327 | create_cq_exit1: | ||
328 | kmem_cache_free(cq_cache, my_cq); | ||
329 | |||
330 | return cq; | ||
331 | } | ||
332 | |||
333 | int ehca_destroy_cq(struct ib_cq *cq) | ||
334 | { | ||
335 | u64 h_ret; | ||
336 | int ret; | ||
337 | struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq); | ||
338 | int cq_num = my_cq->cq_number; | ||
339 | struct ib_device *device = cq->device; | ||
340 | struct ehca_shca *shca = container_of(device, struct ehca_shca, | ||
341 | ib_device); | ||
342 | struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle; | ||
343 | u32 cur_pid = current->tgid; | ||
344 | unsigned long flags; | ||
345 | |||
346 | spin_lock_irqsave(&ehca_cq_idr_lock, flags); | ||
347 | while (my_cq->nr_callbacks) | ||
348 | yield(); | ||
349 | |||
350 | idr_remove(&ehca_cq_idr, my_cq->token); | ||
351 | spin_unlock_irqrestore(&ehca_cq_idr_lock, flags); | ||
352 | |||
353 | if (my_cq->uspace_queue && my_cq->ownpid != cur_pid) { | ||
354 | ehca_err(device, "Invalid caller pid=%x ownpid=%x", | ||
355 | cur_pid, my_cq->ownpid); | ||
356 | return -EINVAL; | ||
357 | } | ||
358 | |||
359 | /* un-mmap if vma alloc */ | ||
360 | if (my_cq->uspace_queue ) { | ||
361 | ret = ehca_munmap(my_cq->uspace_queue, | ||
362 | my_cq->ipz_queue.queue_length); | ||
363 | if (ret) | ||
364 | ehca_err(device, "Could not munmap queue ehca_cq=%p " | ||
365 | "cq_num=%x", my_cq, cq_num); | ||
366 | ret = ehca_munmap(my_cq->uspace_fwh, EHCA_PAGESIZE); | ||
367 | if (ret) | ||
368 | ehca_err(device, "Could not munmap fwh ehca_cq=%p " | ||
369 | "cq_num=%x", my_cq, cq_num); | ||
370 | } | ||
371 | |||
372 | h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0); | ||
373 | if (h_ret == H_R_STATE) { | ||
374 | /* cq in err: read err data and destroy it forcibly */ | ||
375 | ehca_dbg(device, "ehca_cq=%p cq_num=%x ressource=%lx in err " | ||
376 | "state. Try to delete it forcibly.", | ||
377 | my_cq, cq_num, my_cq->ipz_cq_handle.handle); | ||
378 | ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle); | ||
379 | h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1); | ||
380 | if (h_ret == H_SUCCESS) | ||
381 | ehca_dbg(device, "cq_num=%x deleted successfully.", | ||
382 | cq_num); | ||
383 | } | ||
384 | if (h_ret != H_SUCCESS) { | ||
385 | ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%lx " | ||
386 | "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num); | ||
387 | return ehca2ib_return_code(h_ret); | ||
388 | } | ||
389 | ipz_queue_dtor(&my_cq->ipz_queue); | ||
390 | kmem_cache_free(cq_cache, my_cq); | ||
391 | |||
392 | return 0; | ||
393 | } | ||
394 | |||
395 | int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata) | ||
396 | { | ||
397 | struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq); | ||
398 | u32 cur_pid = current->tgid; | ||
399 | |||
400 | if (my_cq->uspace_queue && my_cq->ownpid != cur_pid) { | ||
401 | ehca_err(cq->device, "Invalid caller pid=%x ownpid=%x", | ||
402 | cur_pid, my_cq->ownpid); | ||
403 | return -EINVAL; | ||
404 | } | ||
405 | |||
406 | /* TODO: proper resize needs to be done */ | ||
407 | ehca_err(cq->device, "not implemented yet"); | ||
408 | |||
409 | return -EFAULT; | ||
410 | } | ||
411 | |||
412 | int ehca_init_cq_cache(void) | ||
413 | { | ||
414 | cq_cache = kmem_cache_create("ehca_cache_cq", | ||
415 | sizeof(struct ehca_cq), 0, | ||
416 | SLAB_HWCACHE_ALIGN, | ||
417 | NULL, NULL); | ||
418 | if (!cq_cache) | ||
419 | return -ENOMEM; | ||
420 | return 0; | ||
421 | } | ||
422 | |||
423 | void ehca_cleanup_cq_cache(void) | ||
424 | { | ||
425 | if (cq_cache) | ||
426 | kmem_cache_destroy(cq_cache); | ||
427 | } | ||