diff options
author | Roland Dreier <rolandd@cisco.com> | 2006-01-30 17:31:33 -0500 |
---|---|---|
committer | Roland Dreier <rolandd@cisco.com> | 2006-03-20 13:08:08 -0500 |
commit | 4885bf64bc2eb242ebebe67077cfe3688283b586 (patch) | |
tree | ed40e3eb7c05e923919f1e8ef2a9186b32c916a4 /drivers/infiniband/hw/mthca/mthca_cq.c | |
parent | 33b9b3ee9709b19c4f02ab91571d53540d05c3d1 (diff) |
IB/mthca: Add device-specific support for resizing CQs
Add low-level driver support for resizing CQs (both kernel and
userspace) to mthca.
Signed-off-by: Roland Dreier <rolandd@cisco.com>
Diffstat (limited to 'drivers/infiniband/hw/mthca/mthca_cq.c')
-rw-r--r-- | drivers/infiniband/hw/mthca/mthca_cq.c | 132 |
1 files changed, 105 insertions, 27 deletions
diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c index 78d9cc119f33..76aabc5bf371 100644 --- a/drivers/infiniband/hw/mthca/mthca_cq.c +++ b/drivers/infiniband/hw/mthca/mthca_cq.c | |||
@@ -1,7 +1,7 @@ | |||
1 | /* | 1 | /* |
2 | * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. | 2 | * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. |
3 | * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. | 3 | * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. |
4 | * Copyright (c) 2005 Cisco Systems, Inc. All rights reserved. | 4 | * Copyright (c) 2005, 2006 Cisco Systems, Inc. All rights reserved. |
5 | * Copyright (c) 2005 Mellanox Technologies. All rights reserved. | 5 | * Copyright (c) 2005 Mellanox Technologies. All rights reserved. |
6 | * Copyright (c) 2004 Voltaire, Inc. All rights reserved. | 6 | * Copyright (c) 2004 Voltaire, Inc. All rights reserved. |
7 | * | 7 | * |
@@ -150,24 +150,29 @@ struct mthca_err_cqe { | |||
150 | #define MTHCA_ARBEL_CQ_DB_REQ_NOT (2 << 24) | 150 | #define MTHCA_ARBEL_CQ_DB_REQ_NOT (2 << 24) |
151 | #define MTHCA_ARBEL_CQ_DB_REQ_NOT_MULT (3 << 24) | 151 | #define MTHCA_ARBEL_CQ_DB_REQ_NOT_MULT (3 << 24) |
152 | 152 | ||
153 | static inline struct mthca_cqe *get_cqe(struct mthca_cq *cq, int entry) | 153 | static inline struct mthca_cqe *get_cqe_from_buf(struct mthca_cq_buf *buf, |
154 | int entry) | ||
154 | { | 155 | { |
155 | if (cq->is_direct) | 156 | if (buf->is_direct) |
156 | return cq->queue.direct.buf + (entry * MTHCA_CQ_ENTRY_SIZE); | 157 | return buf->queue.direct.buf + (entry * MTHCA_CQ_ENTRY_SIZE); |
157 | else | 158 | else |
158 | return cq->queue.page_list[entry * MTHCA_CQ_ENTRY_SIZE / PAGE_SIZE].buf | 159 | return buf->queue.page_list[entry * MTHCA_CQ_ENTRY_SIZE / PAGE_SIZE].buf |
159 | + (entry * MTHCA_CQ_ENTRY_SIZE) % PAGE_SIZE; | 160 | + (entry * MTHCA_CQ_ENTRY_SIZE) % PAGE_SIZE; |
160 | } | 161 | } |
161 | 162 | ||
162 | static inline struct mthca_cqe *cqe_sw(struct mthca_cq *cq, int i) | 163 | static inline struct mthca_cqe *get_cqe(struct mthca_cq *cq, int entry) |
164 | { | ||
165 | return get_cqe_from_buf(&cq->buf, entry); | ||
166 | } | ||
167 | |||
168 | static inline struct mthca_cqe *cqe_sw(struct mthca_cqe *cqe) | ||
163 | { | 169 | { |
164 | struct mthca_cqe *cqe = get_cqe(cq, i); | ||
165 | return MTHCA_CQ_ENTRY_OWNER_HW & cqe->owner ? NULL : cqe; | 170 | return MTHCA_CQ_ENTRY_OWNER_HW & cqe->owner ? NULL : cqe; |
166 | } | 171 | } |
167 | 172 | ||
168 | static inline struct mthca_cqe *next_cqe_sw(struct mthca_cq *cq) | 173 | static inline struct mthca_cqe *next_cqe_sw(struct mthca_cq *cq) |
169 | { | 174 | { |
170 | return cqe_sw(cq, cq->cons_index & cq->ibcq.cqe); | 175 | return cqe_sw(get_cqe(cq, cq->cons_index & cq->ibcq.cqe)); |
171 | } | 176 | } |
172 | 177 | ||
173 | static inline void set_cqe_hw(struct mthca_cqe *cqe) | 178 | static inline void set_cqe_hw(struct mthca_cqe *cqe) |
@@ -289,7 +294,7 @@ void mthca_cq_clean(struct mthca_dev *dev, u32 cqn, u32 qpn, | |||
289 | * from our QP and therefore don't need to be checked. | 294 | * from our QP and therefore don't need to be checked. |
290 | */ | 295 | */ |
291 | for (prod_index = cq->cons_index; | 296 | for (prod_index = cq->cons_index; |
292 | cqe_sw(cq, prod_index & cq->ibcq.cqe); | 297 | cqe_sw(get_cqe(cq, prod_index & cq->ibcq.cqe)); |
293 | ++prod_index) | 298 | ++prod_index) |
294 | if (prod_index == cq->cons_index + cq->ibcq.cqe) | 299 | if (prod_index == cq->cons_index + cq->ibcq.cqe) |
295 | break; | 300 | break; |
@@ -324,6 +329,53 @@ void mthca_cq_clean(struct mthca_dev *dev, u32 cqn, u32 qpn, | |||
324 | wake_up(&cq->wait); | 329 | wake_up(&cq->wait); |
325 | } | 330 | } |
326 | 331 | ||
332 | void mthca_cq_resize_copy_cqes(struct mthca_cq *cq) | ||
333 | { | ||
334 | int i; | ||
335 | |||
336 | /* | ||
337 | * In Tavor mode, the hardware keeps the consumer and producer | ||
338 | * indices mod the CQ size. Since we might be making the CQ | ||
339 | * bigger, we need to deal with the case where the producer | ||
340 | * index wrapped around before the CQ was resized. | ||
341 | */ | ||
342 | if (!mthca_is_memfree(to_mdev(cq->ibcq.device)) && | ||
343 | cq->ibcq.cqe < cq->resize_buf->cqe) { | ||
344 | cq->cons_index &= cq->ibcq.cqe; | ||
345 | if (cqe_sw(get_cqe(cq, cq->ibcq.cqe))) | ||
346 | cq->cons_index -= cq->ibcq.cqe + 1; | ||
347 | } | ||
348 | |||
349 | for (i = cq->cons_index; cqe_sw(get_cqe(cq, i & cq->ibcq.cqe)); ++i) | ||
350 | memcpy(get_cqe_from_buf(&cq->resize_buf->buf, | ||
351 | i & cq->resize_buf->cqe), | ||
352 | get_cqe(cq, i & cq->ibcq.cqe), MTHCA_CQ_ENTRY_SIZE); | ||
353 | } | ||
354 | |||
355 | int mthca_alloc_cq_buf(struct mthca_dev *dev, struct mthca_cq_buf *buf, int nent) | ||
356 | { | ||
357 | int ret; | ||
358 | int i; | ||
359 | |||
360 | ret = mthca_buf_alloc(dev, nent * MTHCA_CQ_ENTRY_SIZE, | ||
361 | MTHCA_MAX_DIRECT_CQ_SIZE, | ||
362 | &buf->queue, &buf->is_direct, | ||
363 | &dev->driver_pd, 1, &buf->mr); | ||
364 | if (ret) | ||
365 | return ret; | ||
366 | |||
367 | for (i = 0; i < nent; ++i) | ||
368 | set_cqe_hw(get_cqe_from_buf(buf, i)); | ||
369 | |||
370 | return 0; | ||
371 | } | ||
372 | |||
373 | void mthca_free_cq_buf(struct mthca_dev *dev, struct mthca_cq_buf *buf, int cqe) | ||
374 | { | ||
375 | mthca_buf_free(dev, (cqe + 1) * MTHCA_CQ_ENTRY_SIZE, &buf->queue, | ||
376 | buf->is_direct, &buf->mr); | ||
377 | } | ||
378 | |||
327 | static void handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq, | 379 | static void handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq, |
328 | struct mthca_qp *qp, int wqe_index, int is_send, | 380 | struct mthca_qp *qp, int wqe_index, int is_send, |
329 | struct mthca_err_cqe *cqe, | 381 | struct mthca_err_cqe *cqe, |
@@ -609,11 +661,14 @@ int mthca_poll_cq(struct ib_cq *ibcq, int num_entries, | |||
609 | 661 | ||
610 | spin_lock_irqsave(&cq->lock, flags); | 662 | spin_lock_irqsave(&cq->lock, flags); |
611 | 663 | ||
612 | for (npolled = 0; npolled < num_entries; ++npolled) { | 664 | npolled = 0; |
665 | repoll: | ||
666 | while (npolled < num_entries) { | ||
613 | err = mthca_poll_one(dev, cq, &qp, | 667 | err = mthca_poll_one(dev, cq, &qp, |
614 | &freed, entry + npolled); | 668 | &freed, entry + npolled); |
615 | if (err) | 669 | if (err) |
616 | break; | 670 | break; |
671 | ++npolled; | ||
617 | } | 672 | } |
618 | 673 | ||
619 | if (freed) { | 674 | if (freed) { |
@@ -621,6 +676,42 @@ int mthca_poll_cq(struct ib_cq *ibcq, int num_entries, | |||
621 | update_cons_index(dev, cq, freed); | 676 | update_cons_index(dev, cq, freed); |
622 | } | 677 | } |
623 | 678 | ||
679 | /* | ||
680 | * If a CQ resize is in progress and we discovered that the | ||
681 | * old buffer is empty, then peek in the new buffer, and if | ||
682 | * it's not empty, switch to the new buffer and continue | ||
683 | * polling there. | ||
684 | */ | ||
685 | if (unlikely(err == -EAGAIN && cq->resize_buf && | ||
686 | cq->resize_buf->state == CQ_RESIZE_READY)) { | ||
687 | /* | ||
688 | * In Tavor mode, the hardware keeps the producer | ||
689 | * index modulo the CQ size. Since we might be making | ||
690 | * the CQ bigger, we need to mask our consumer index | ||
691 | * using the size of the old CQ buffer before looking | ||
692 | * in the new CQ buffer. | ||
693 | */ | ||
694 | if (!mthca_is_memfree(dev)) | ||
695 | cq->cons_index &= cq->ibcq.cqe; | ||
696 | |||
697 | if (cqe_sw(get_cqe_from_buf(&cq->resize_buf->buf, | ||
698 | cq->cons_index & cq->resize_buf->cqe))) { | ||
699 | struct mthca_cq_buf tbuf; | ||
700 | int tcqe; | ||
701 | |||
702 | tbuf = cq->buf; | ||
703 | tcqe = cq->ibcq.cqe; | ||
704 | cq->buf = cq->resize_buf->buf; | ||
705 | cq->ibcq.cqe = cq->resize_buf->cqe; | ||
706 | |||
707 | cq->resize_buf->buf = tbuf; | ||
708 | cq->resize_buf->cqe = tcqe; | ||
709 | cq->resize_buf->state = CQ_RESIZE_SWAPPED; | ||
710 | |||
711 | goto repoll; | ||
712 | } | ||
713 | } | ||
714 | |||
624 | spin_unlock_irqrestore(&cq->lock, flags); | 715 | spin_unlock_irqrestore(&cq->lock, flags); |
625 | 716 | ||
626 | return err == 0 || err == -EAGAIN ? npolled : err; | 717 | return err == 0 || err == -EAGAIN ? npolled : err; |
@@ -679,22 +770,14 @@ int mthca_arbel_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify notify) | |||
679 | return 0; | 770 | return 0; |
680 | } | 771 | } |
681 | 772 | ||
682 | static void mthca_free_cq_buf(struct mthca_dev *dev, struct mthca_cq *cq) | ||
683 | { | ||
684 | mthca_buf_free(dev, (cq->ibcq.cqe + 1) * MTHCA_CQ_ENTRY_SIZE, | ||
685 | &cq->queue, cq->is_direct, &cq->mr); | ||
686 | } | ||
687 | |||
688 | int mthca_init_cq(struct mthca_dev *dev, int nent, | 773 | int mthca_init_cq(struct mthca_dev *dev, int nent, |
689 | struct mthca_ucontext *ctx, u32 pdn, | 774 | struct mthca_ucontext *ctx, u32 pdn, |
690 | struct mthca_cq *cq) | 775 | struct mthca_cq *cq) |
691 | { | 776 | { |
692 | int size = nent * MTHCA_CQ_ENTRY_SIZE; | ||
693 | struct mthca_mailbox *mailbox; | 777 | struct mthca_mailbox *mailbox; |
694 | struct mthca_cq_context *cq_context; | 778 | struct mthca_cq_context *cq_context; |
695 | int err = -ENOMEM; | 779 | int err = -ENOMEM; |
696 | u8 status; | 780 | u8 status; |
697 | int i; | ||
698 | 781 | ||
699 | cq->ibcq.cqe = nent - 1; | 782 | cq->ibcq.cqe = nent - 1; |
700 | cq->is_kernel = !ctx; | 783 | cq->is_kernel = !ctx; |
@@ -732,14 +815,9 @@ int mthca_init_cq(struct mthca_dev *dev, int nent, | |||
732 | cq_context = mailbox->buf; | 815 | cq_context = mailbox->buf; |
733 | 816 | ||
734 | if (cq->is_kernel) { | 817 | if (cq->is_kernel) { |
735 | err = mthca_buf_alloc(dev, size, MTHCA_MAX_DIRECT_CQ_SIZE, | 818 | err = mthca_alloc_cq_buf(dev, &cq->buf, nent); |
736 | &cq->queue, &cq->is_direct, | ||
737 | &dev->driver_pd, 1, &cq->mr); | ||
738 | if (err) | 819 | if (err) |
739 | goto err_out_mailbox; | 820 | goto err_out_mailbox; |
740 | |||
741 | for (i = 0; i < nent; ++i) | ||
742 | set_cqe_hw(get_cqe(cq, i)); | ||
743 | } | 821 | } |
744 | 822 | ||
745 | spin_lock_init(&cq->lock); | 823 | spin_lock_init(&cq->lock); |
@@ -758,7 +836,7 @@ int mthca_init_cq(struct mthca_dev *dev, int nent, | |||
758 | cq_context->error_eqn = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn); | 836 | cq_context->error_eqn = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn); |
759 | cq_context->comp_eqn = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_COMP].eqn); | 837 | cq_context->comp_eqn = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_COMP].eqn); |
760 | cq_context->pd = cpu_to_be32(pdn); | 838 | cq_context->pd = cpu_to_be32(pdn); |
761 | cq_context->lkey = cpu_to_be32(cq->mr.ibmr.lkey); | 839 | cq_context->lkey = cpu_to_be32(cq->buf.mr.ibmr.lkey); |
762 | cq_context->cqn = cpu_to_be32(cq->cqn); | 840 | cq_context->cqn = cpu_to_be32(cq->cqn); |
763 | 841 | ||
764 | if (mthca_is_memfree(dev)) { | 842 | if (mthca_is_memfree(dev)) { |
@@ -796,7 +874,7 @@ int mthca_init_cq(struct mthca_dev *dev, int nent, | |||
796 | 874 | ||
797 | err_out_free_mr: | 875 | err_out_free_mr: |
798 | if (cq->is_kernel) | 876 | if (cq->is_kernel) |
799 | mthca_free_cq_buf(dev, cq); | 877 | mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); |
800 | 878 | ||
801 | err_out_mailbox: | 879 | err_out_mailbox: |
802 | mthca_free_mailbox(dev, mailbox); | 880 | mthca_free_mailbox(dev, mailbox); |
@@ -862,7 +940,7 @@ void mthca_free_cq(struct mthca_dev *dev, | |||
862 | wait_event(cq->wait, !atomic_read(&cq->refcount)); | 940 | wait_event(cq->wait, !atomic_read(&cq->refcount)); |
863 | 941 | ||
864 | if (cq->is_kernel) { | 942 | if (cq->is_kernel) { |
865 | mthca_free_cq_buf(dev, cq); | 943 | mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); |
866 | if (mthca_is_memfree(dev)) { | 944 | if (mthca_is_memfree(dev)) { |
867 | mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index); | 945 | mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index); |
868 | mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index); | 946 | mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index); |