aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/infiniband/hw/ipath/ipath_ruc.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/infiniband/hw/ipath/ipath_ruc.c')
-rw-r--r--drivers/infiniband/hw/ipath/ipath_ruc.c552
1 files changed, 552 insertions, 0 deletions
diff --git a/drivers/infiniband/hw/ipath/ipath_ruc.c b/drivers/infiniband/hw/ipath/ipath_ruc.c
new file mode 100644
index 000000000000..f232e77b78ee
--- /dev/null
+++ b/drivers/infiniband/hw/ipath/ipath_ruc.c
@@ -0,0 +1,552 @@
1/*
2 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include "ipath_verbs.h"
34
35/*
36 * Convert the AETH RNR timeout code into the number of milliseconds.
37 */
38const u32 ib_ipath_rnr_table[32] = {
39 656, /* 0 */
40 1, /* 1 */
41 1, /* 2 */
42 1, /* 3 */
43 1, /* 4 */
44 1, /* 5 */
45 1, /* 6 */
46 1, /* 7 */
47 1, /* 8 */
48 1, /* 9 */
49 1, /* A */
50 1, /* B */
51 1, /* C */
52 1, /* D */
53 2, /* E */
54 2, /* F */
55 3, /* 10 */
56 4, /* 11 */
57 6, /* 12 */
58 8, /* 13 */
59 11, /* 14 */
60 16, /* 15 */
61 21, /* 16 */
62 31, /* 17 */
63 41, /* 18 */
64 62, /* 19 */
65 82, /* 1A */
66 123, /* 1B */
67 164, /* 1C */
68 246, /* 1D */
69 328, /* 1E */
70 492 /* 1F */
71};
72
73/**
74 * ipath_insert_rnr_queue - put QP on the RNR timeout list for the device
75 * @qp: the QP
76 *
77 * XXX Use a simple list for now. We might need a priority
78 * queue if we have lots of QPs waiting for RNR timeouts
79 * but that should be rare.
80 */
81void ipath_insert_rnr_queue(struct ipath_qp *qp)
82{
83 struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
84 unsigned long flags;
85
86 spin_lock_irqsave(&dev->pending_lock, flags);
87 if (list_empty(&dev->rnrwait))
88 list_add(&qp->timerwait, &dev->rnrwait);
89 else {
90 struct list_head *l = &dev->rnrwait;
91 struct ipath_qp *nqp = list_entry(l->next, struct ipath_qp,
92 timerwait);
93
94 while (qp->s_rnr_timeout >= nqp->s_rnr_timeout) {
95 qp->s_rnr_timeout -= nqp->s_rnr_timeout;
96 l = l->next;
97 if (l->next == &dev->rnrwait)
98 break;
99 nqp = list_entry(l->next, struct ipath_qp,
100 timerwait);
101 }
102 list_add(&qp->timerwait, l);
103 }
104 spin_unlock_irqrestore(&dev->pending_lock, flags);
105}
106
107/**
108 * ipath_get_rwqe - copy the next RWQE into the QP's RWQE
109 * @qp: the QP
110 * @wr_id_only: update wr_id only, not SGEs
111 *
112 * Return 0 if no RWQE is available, otherwise return 1.
113 *
114 * Called at interrupt level with the QP r_rq.lock held.
115 */
116int ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only)
117{
118 struct ipath_rq *rq;
119 struct ipath_srq *srq;
120 struct ipath_rwqe *wqe;
121 int ret;
122
123 if (!qp->ibqp.srq) {
124 rq = &qp->r_rq;
125 if (unlikely(rq->tail == rq->head)) {
126 ret = 0;
127 goto bail;
128 }
129 wqe = get_rwqe_ptr(rq, rq->tail);
130 qp->r_wr_id = wqe->wr_id;
131 if (!wr_id_only) {
132 qp->r_sge.sge = wqe->sg_list[0];
133 qp->r_sge.sg_list = wqe->sg_list + 1;
134 qp->r_sge.num_sge = wqe->num_sge;
135 qp->r_len = wqe->length;
136 }
137 if (++rq->tail >= rq->size)
138 rq->tail = 0;
139 ret = 1;
140 goto bail;
141 }
142
143 srq = to_isrq(qp->ibqp.srq);
144 rq = &srq->rq;
145 spin_lock(&rq->lock);
146 if (unlikely(rq->tail == rq->head)) {
147 spin_unlock(&rq->lock);
148 ret = 0;
149 goto bail;
150 }
151 wqe = get_rwqe_ptr(rq, rq->tail);
152 qp->r_wr_id = wqe->wr_id;
153 if (!wr_id_only) {
154 qp->r_sge.sge = wqe->sg_list[0];
155 qp->r_sge.sg_list = wqe->sg_list + 1;
156 qp->r_sge.num_sge = wqe->num_sge;
157 qp->r_len = wqe->length;
158 }
159 if (++rq->tail >= rq->size)
160 rq->tail = 0;
161 if (srq->ibsrq.event_handler) {
162 struct ib_event ev;
163 u32 n;
164
165 if (rq->head < rq->tail)
166 n = rq->size + rq->head - rq->tail;
167 else
168 n = rq->head - rq->tail;
169 if (n < srq->limit) {
170 srq->limit = 0;
171 spin_unlock(&rq->lock);
172 ev.device = qp->ibqp.device;
173 ev.element.srq = qp->ibqp.srq;
174 ev.event = IB_EVENT_SRQ_LIMIT_REACHED;
175 srq->ibsrq.event_handler(&ev,
176 srq->ibsrq.srq_context);
177 } else
178 spin_unlock(&rq->lock);
179 } else
180 spin_unlock(&rq->lock);
181 ret = 1;
182
183bail:
184 return ret;
185}
186
187/**
188 * ipath_ruc_loopback - handle UC and RC lookback requests
189 * @sqp: the loopback QP
190 * @wc: the work completion entry
191 *
192 * This is called from ipath_do_uc_send() or ipath_do_rc_send() to
193 * forward a WQE addressed to the same HCA.
194 * Note that although we are single threaded due to the tasklet, we still
195 * have to protect against post_send(). We don't have to worry about
196 * receive interrupts since this is a connected protocol and all packets
197 * will pass through here.
198 */
199void ipath_ruc_loopback(struct ipath_qp *sqp, struct ib_wc *wc)
200{
201 struct ipath_ibdev *dev = to_idev(sqp->ibqp.device);
202 struct ipath_qp *qp;
203 struct ipath_swqe *wqe;
204 struct ipath_sge *sge;
205 unsigned long flags;
206 u64 sdata;
207
208 qp = ipath_lookup_qpn(&dev->qp_table, sqp->remote_qpn);
209 if (!qp) {
210 dev->n_pkt_drops++;
211 return;
212 }
213
214again:
215 spin_lock_irqsave(&sqp->s_lock, flags);
216
217 if (!(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_SEND_OK)) {
218 spin_unlock_irqrestore(&sqp->s_lock, flags);
219 goto done;
220 }
221
222 /* Get the next send request. */
223 if (sqp->s_last == sqp->s_head) {
224 /* Send work queue is empty. */
225 spin_unlock_irqrestore(&sqp->s_lock, flags);
226 goto done;
227 }
228
229 /*
230 * We can rely on the entry not changing without the s_lock
231 * being held until we update s_last.
232 */
233 wqe = get_swqe_ptr(sqp, sqp->s_last);
234 spin_unlock_irqrestore(&sqp->s_lock, flags);
235
236 wc->wc_flags = 0;
237 wc->imm_data = 0;
238
239 sqp->s_sge.sge = wqe->sg_list[0];
240 sqp->s_sge.sg_list = wqe->sg_list + 1;
241 sqp->s_sge.num_sge = wqe->wr.num_sge;
242 sqp->s_len = wqe->length;
243 switch (wqe->wr.opcode) {
244 case IB_WR_SEND_WITH_IMM:
245 wc->wc_flags = IB_WC_WITH_IMM;
246 wc->imm_data = wqe->wr.imm_data;
247 /* FALLTHROUGH */
248 case IB_WR_SEND:
249 spin_lock_irqsave(&qp->r_rq.lock, flags);
250 if (!ipath_get_rwqe(qp, 0)) {
251 rnr_nak:
252 spin_unlock_irqrestore(&qp->r_rq.lock, flags);
253 /* Handle RNR NAK */
254 if (qp->ibqp.qp_type == IB_QPT_UC)
255 goto send_comp;
256 if (sqp->s_rnr_retry == 0) {
257 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
258 goto err;
259 }
260 if (sqp->s_rnr_retry_cnt < 7)
261 sqp->s_rnr_retry--;
262 dev->n_rnr_naks++;
263 sqp->s_rnr_timeout =
264 ib_ipath_rnr_table[sqp->s_min_rnr_timer];
265 ipath_insert_rnr_queue(sqp);
266 goto done;
267 }
268 spin_unlock_irqrestore(&qp->r_rq.lock, flags);
269 break;
270
271 case IB_WR_RDMA_WRITE_WITH_IMM:
272 wc->wc_flags = IB_WC_WITH_IMM;
273 wc->imm_data = wqe->wr.imm_data;
274 spin_lock_irqsave(&qp->r_rq.lock, flags);
275 if (!ipath_get_rwqe(qp, 1))
276 goto rnr_nak;
277 spin_unlock_irqrestore(&qp->r_rq.lock, flags);
278 /* FALLTHROUGH */
279 case IB_WR_RDMA_WRITE:
280 if (wqe->length == 0)
281 break;
282 if (unlikely(!ipath_rkey_ok(dev, &qp->r_sge, wqe->length,
283 wqe->wr.wr.rdma.remote_addr,
284 wqe->wr.wr.rdma.rkey,
285 IB_ACCESS_REMOTE_WRITE))) {
286 acc_err:
287 wc->status = IB_WC_REM_ACCESS_ERR;
288 err:
289 wc->wr_id = wqe->wr.wr_id;
290 wc->opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
291 wc->vendor_err = 0;
292 wc->byte_len = 0;
293 wc->qp_num = sqp->ibqp.qp_num;
294 wc->src_qp = sqp->remote_qpn;
295 wc->pkey_index = 0;
296 wc->slid = sqp->remote_ah_attr.dlid;
297 wc->sl = sqp->remote_ah_attr.sl;
298 wc->dlid_path_bits = 0;
299 wc->port_num = 0;
300 ipath_sqerror_qp(sqp, wc);
301 goto done;
302 }
303 break;
304
305 case IB_WR_RDMA_READ:
306 if (unlikely(!ipath_rkey_ok(dev, &sqp->s_sge, wqe->length,
307 wqe->wr.wr.rdma.remote_addr,
308 wqe->wr.wr.rdma.rkey,
309 IB_ACCESS_REMOTE_READ)))
310 goto acc_err;
311 if (unlikely(!(qp->qp_access_flags &
312 IB_ACCESS_REMOTE_READ)))
313 goto acc_err;
314 qp->r_sge.sge = wqe->sg_list[0];
315 qp->r_sge.sg_list = wqe->sg_list + 1;
316 qp->r_sge.num_sge = wqe->wr.num_sge;
317 break;
318
319 case IB_WR_ATOMIC_CMP_AND_SWP:
320 case IB_WR_ATOMIC_FETCH_AND_ADD:
321 if (unlikely(!ipath_rkey_ok(dev, &qp->r_sge, sizeof(u64),
322 wqe->wr.wr.rdma.remote_addr,
323 wqe->wr.wr.rdma.rkey,
324 IB_ACCESS_REMOTE_ATOMIC)))
325 goto acc_err;
326 /* Perform atomic OP and save result. */
327 sdata = wqe->wr.wr.atomic.swap;
328 spin_lock_irqsave(&dev->pending_lock, flags);
329 qp->r_atomic_data = *(u64 *) qp->r_sge.sge.vaddr;
330 if (wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
331 *(u64 *) qp->r_sge.sge.vaddr =
332 qp->r_atomic_data + sdata;
333 else if (qp->r_atomic_data == wqe->wr.wr.atomic.compare_add)
334 *(u64 *) qp->r_sge.sge.vaddr = sdata;
335 spin_unlock_irqrestore(&dev->pending_lock, flags);
336 *(u64 *) sqp->s_sge.sge.vaddr = qp->r_atomic_data;
337 goto send_comp;
338
339 default:
340 goto done;
341 }
342
343 sge = &sqp->s_sge.sge;
344 while (sqp->s_len) {
345 u32 len = sqp->s_len;
346
347 if (len > sge->length)
348 len = sge->length;
349 BUG_ON(len == 0);
350 ipath_copy_sge(&qp->r_sge, sge->vaddr, len);
351 sge->vaddr += len;
352 sge->length -= len;
353 sge->sge_length -= len;
354 if (sge->sge_length == 0) {
355 if (--sqp->s_sge.num_sge)
356 *sge = *sqp->s_sge.sg_list++;
357 } else if (sge->length == 0 && sge->mr != NULL) {
358 if (++sge->n >= IPATH_SEGSZ) {
359 if (++sge->m >= sge->mr->mapsz)
360 break;
361 sge->n = 0;
362 }
363 sge->vaddr =
364 sge->mr->map[sge->m]->segs[sge->n].vaddr;
365 sge->length =
366 sge->mr->map[sge->m]->segs[sge->n].length;
367 }
368 sqp->s_len -= len;
369 }
370
371 if (wqe->wr.opcode == IB_WR_RDMA_WRITE ||
372 wqe->wr.opcode == IB_WR_RDMA_READ)
373 goto send_comp;
374
375 if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM)
376 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
377 else
378 wc->opcode = IB_WC_RECV;
379 wc->wr_id = qp->r_wr_id;
380 wc->status = IB_WC_SUCCESS;
381 wc->vendor_err = 0;
382 wc->byte_len = wqe->length;
383 wc->qp_num = qp->ibqp.qp_num;
384 wc->src_qp = qp->remote_qpn;
385 /* XXX do we know which pkey matched? Only needed for GSI. */
386 wc->pkey_index = 0;
387 wc->slid = qp->remote_ah_attr.dlid;
388 wc->sl = qp->remote_ah_attr.sl;
389 wc->dlid_path_bits = 0;
390 /* Signal completion event if the solicited bit is set. */
391 ipath_cq_enter(to_icq(qp->ibqp.recv_cq), wc,
392 wqe->wr.send_flags & IB_SEND_SOLICITED);
393
394send_comp:
395 sqp->s_rnr_retry = sqp->s_rnr_retry_cnt;
396
397 if (!test_bit(IPATH_S_SIGNAL_REQ_WR, &sqp->s_flags) ||
398 (wqe->wr.send_flags & IB_SEND_SIGNALED)) {
399 wc->wr_id = wqe->wr.wr_id;
400 wc->status = IB_WC_SUCCESS;
401 wc->opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
402 wc->vendor_err = 0;
403 wc->byte_len = wqe->length;
404 wc->qp_num = sqp->ibqp.qp_num;
405 wc->src_qp = 0;
406 wc->pkey_index = 0;
407 wc->slid = 0;
408 wc->sl = 0;
409 wc->dlid_path_bits = 0;
410 wc->port_num = 0;
411 ipath_cq_enter(to_icq(sqp->ibqp.send_cq), wc, 0);
412 }
413
414 /* Update s_last now that we are finished with the SWQE */
415 spin_lock_irqsave(&sqp->s_lock, flags);
416 if (++sqp->s_last >= sqp->s_size)
417 sqp->s_last = 0;
418 spin_unlock_irqrestore(&sqp->s_lock, flags);
419 goto again;
420
421done:
422 if (atomic_dec_and_test(&qp->refcount))
423 wake_up(&qp->wait);
424}
425
426/**
427 * ipath_no_bufs_available - tell the layer driver we need buffers
428 * @qp: the QP that caused the problem
429 * @dev: the device we ran out of buffers on
430 *
431 * Called when we run out of PIO buffers.
432 */
433void ipath_no_bufs_available(struct ipath_qp *qp, struct ipath_ibdev *dev)
434{
435 unsigned long flags;
436
437 spin_lock_irqsave(&dev->pending_lock, flags);
438 if (qp->piowait.next == LIST_POISON1)
439 list_add_tail(&qp->piowait, &dev->piowait);
440 spin_unlock_irqrestore(&dev->pending_lock, flags);
441 /*
442 * Note that as soon as ipath_layer_want_buffer() is called and
443 * possibly before it returns, ipath_ib_piobufavail()
444 * could be called. If we are still in the tasklet function,
445 * tasklet_hi_schedule() will not call us until the next time
446 * tasklet_hi_schedule() is called.
447 * We clear the tasklet flag now since we are committing to return
448 * from the tasklet function.
449 */
450 clear_bit(IPATH_S_BUSY, &qp->s_flags);
451 tasklet_unlock(&qp->s_task);
452 ipath_layer_want_buffer(dev->dd);
453 dev->n_piowait++;
454}
455
456/**
457 * ipath_post_rc_send - post RC and UC sends
458 * @qp: the QP to post on
459 * @wr: the work request to send
460 */
461int ipath_post_rc_send(struct ipath_qp *qp, struct ib_send_wr *wr)
462{
463 struct ipath_swqe *wqe;
464 unsigned long flags;
465 u32 next;
466 int i, j;
467 int acc;
468 int ret;
469
470 /*
471 * Don't allow RDMA reads or atomic operations on UC or
472 * undefined operations.
473 * Make sure buffer is large enough to hold the result for atomics.
474 */
475 if (qp->ibqp.qp_type == IB_QPT_UC) {
476 if ((unsigned) wr->opcode >= IB_WR_RDMA_READ) {
477 ret = -EINVAL;
478 goto bail;
479 }
480 } else if ((unsigned) wr->opcode > IB_WR_ATOMIC_FETCH_AND_ADD) {
481 ret = -EINVAL;
482 goto bail;
483 } else if (wr->opcode >= IB_WR_ATOMIC_CMP_AND_SWP &&
484 (wr->num_sge == 0 ||
485 wr->sg_list[0].length < sizeof(u64) ||
486 wr->sg_list[0].addr & (sizeof(u64) - 1))) {
487 ret = -EINVAL;
488 goto bail;
489 }
490 /* IB spec says that num_sge == 0 is OK. */
491 if (wr->num_sge > qp->s_max_sge) {
492 ret = -ENOMEM;
493 goto bail;
494 }
495 spin_lock_irqsave(&qp->s_lock, flags);
496 next = qp->s_head + 1;
497 if (next >= qp->s_size)
498 next = 0;
499 if (next == qp->s_last) {
500 spin_unlock_irqrestore(&qp->s_lock, flags);
501 ret = -EINVAL;
502 goto bail;
503 }
504
505 wqe = get_swqe_ptr(qp, qp->s_head);
506 wqe->wr = *wr;
507 wqe->ssn = qp->s_ssn++;
508 wqe->sg_list[0].mr = NULL;
509 wqe->sg_list[0].vaddr = NULL;
510 wqe->sg_list[0].length = 0;
511 wqe->sg_list[0].sge_length = 0;
512 wqe->length = 0;
513 acc = wr->opcode >= IB_WR_RDMA_READ ? IB_ACCESS_LOCAL_WRITE : 0;
514 for (i = 0, j = 0; i < wr->num_sge; i++) {
515 if (to_ipd(qp->ibqp.pd)->user && wr->sg_list[i].lkey == 0) {
516 spin_unlock_irqrestore(&qp->s_lock, flags);
517 ret = -EINVAL;
518 goto bail;
519 }
520 if (wr->sg_list[i].length == 0)
521 continue;
522 if (!ipath_lkey_ok(&to_idev(qp->ibqp.device)->lk_table,
523 &wqe->sg_list[j], &wr->sg_list[i],
524 acc)) {
525 spin_unlock_irqrestore(&qp->s_lock, flags);
526 ret = -EINVAL;
527 goto bail;
528 }
529 wqe->length += wr->sg_list[i].length;
530 j++;
531 }
532 wqe->wr.num_sge = j;
533 qp->s_head = next;
534 /*
535 * Wake up the send tasklet if the QP is not waiting
536 * for an RNR timeout.
537 */
538 next = qp->s_rnr_timeout;
539 spin_unlock_irqrestore(&qp->s_lock, flags);
540
541 if (next == 0) {
542 if (qp->ibqp.qp_type == IB_QPT_UC)
543 ipath_do_uc_send((unsigned long) qp);
544 else
545 ipath_do_rc_send((unsigned long) qp);
546 }
547
548 ret = 0;
549
550bail:
551 return ret;
552}
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-10 21:12:15 -0500