aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/infiniband/hw
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/infiniband/hw')
-rw-r--r--drivers/infiniband/hw/mthca/Kconfig16
-rw-r--r--drivers/infiniband/hw/mthca/Makefile12
-rw-r--r--drivers/infiniband/hw/mthca/mthca_allocator.c179
-rw-r--r--drivers/infiniband/hw/mthca/mthca_av.c241
-rw-r--r--drivers/infiniband/hw/mthca/mthca_cmd.c1767
-rw-r--r--drivers/infiniband/hw/mthca/mthca_cmd.h310
-rw-r--r--drivers/infiniband/hw/mthca/mthca_config_reg.h51
-rw-r--r--drivers/infiniband/hw/mthca/mthca_cq.c918
-rw-r--r--drivers/infiniband/hw/mthca/mthca_dev.h437
-rw-r--r--drivers/infiniband/hw/mthca/mthca_doorbell.h95
-rw-r--r--drivers/infiniband/hw/mthca/mthca_eq.c964
-rw-r--r--drivers/infiniband/hw/mthca/mthca_mad.c323
-rw-r--r--drivers/infiniband/hw/mthca/mthca_main.c1123
-rw-r--r--drivers/infiniband/hw/mthca/mthca_mcg.c376
-rw-r--r--drivers/infiniband/hw/mthca/mthca_memfree.c465
-rw-r--r--drivers/infiniband/hw/mthca/mthca_memfree.h161
-rw-r--r--drivers/infiniband/hw/mthca/mthca_mr.c416
-rw-r--r--drivers/infiniband/hw/mthca/mthca_pd.c80
-rw-r--r--drivers/infiniband/hw/mthca/mthca_profile.c266
-rw-r--r--drivers/infiniband/hw/mthca/mthca_profile.h58
-rw-r--r--drivers/infiniband/hw/mthca/mthca_provider.c660
-rw-r--r--drivers/infiniband/hw/mthca/mthca_provider.h251
-rw-r--r--drivers/infiniband/hw/mthca/mthca_qp.c2056
-rw-r--r--drivers/infiniband/hw/mthca/mthca_reset.c232
-rw-r--r--drivers/infiniband/hw/mthca/mthca_uar.c78
25 files changed, 11535 insertions, 0 deletions
diff --git a/drivers/infiniband/hw/mthca/Kconfig b/drivers/infiniband/hw/mthca/Kconfig
new file mode 100644
index 000000000000..e88be85b3d5c
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/Kconfig
@@ -0,0 +1,16 @@
1config INFINIBAND_MTHCA
2 tristate "Mellanox HCA support"
3 depends on PCI && INFINIBAND
4 ---help---
5 This is a low-level driver for Mellanox InfiniHost host
6 channel adapters (HCAs), including the MT23108 PCI-X HCA
7 ("Tavor") and the MT25208 PCI Express HCA ("Arbel").
8
9config INFINIBAND_MTHCA_DEBUG
10 bool "Verbose debugging output"
11 depends on INFINIBAND_MTHCA
12 default n
13 ---help---
14 This option causes the mthca driver produce a bunch of debug
15 messages. Select this is you are developing the driver or
16 trying to diagnose a problem.
diff --git a/drivers/infiniband/hw/mthca/Makefile b/drivers/infiniband/hw/mthca/Makefile
new file mode 100644
index 000000000000..5dcbd43073e2
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/Makefile
@@ -0,0 +1,12 @@
1EXTRA_CFLAGS += -Idrivers/infiniband/include
2
3ifdef CONFIG_INFINIBAND_MTHCA_DEBUG
4EXTRA_CFLAGS += -DDEBUG
5endif
6
7obj-$(CONFIG_INFINIBAND_MTHCA) += ib_mthca.o
8
9ib_mthca-y := mthca_main.o mthca_cmd.o mthca_profile.o mthca_reset.o \
10 mthca_allocator.o mthca_eq.o mthca_pd.o mthca_cq.o \
11 mthca_mr.o mthca_qp.o mthca_av.o mthca_mcg.o mthca_mad.o \
12 mthca_provider.o mthca_memfree.o mthca_uar.o
diff --git a/drivers/infiniband/hw/mthca/mthca_allocator.c b/drivers/infiniband/hw/mthca/mthca_allocator.c
new file mode 100644
index 000000000000..b1db48dd91d6
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_allocator.c
@@ -0,0 +1,179 @@
1/*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_allocator.c 1349 2004-12-16 21:09:43Z roland $
33 */
34
35#include <linux/errno.h>
36#include <linux/slab.h>
37#include <linux/bitmap.h>
38
39#include "mthca_dev.h"
40
41/* Trivial bitmap-based allocator */
42u32 mthca_alloc(struct mthca_alloc *alloc)
43{
44 u32 obj;
45
46 spin_lock(&alloc->lock);
47 obj = find_next_zero_bit(alloc->table, alloc->max, alloc->last);
48 if (obj >= alloc->max) {
49 alloc->top = (alloc->top + alloc->max) & alloc->mask;
50 obj = find_first_zero_bit(alloc->table, alloc->max);
51 }
52
53 if (obj < alloc->max) {
54 set_bit(obj, alloc->table);
55 obj |= alloc->top;
56 } else
57 obj = -1;
58
59 spin_unlock(&alloc->lock);
60
61 return obj;
62}
63
64void mthca_free(struct mthca_alloc *alloc, u32 obj)
65{
66 obj &= alloc->max - 1;
67 spin_lock(&alloc->lock);
68 clear_bit(obj, alloc->table);
69 alloc->last = min(alloc->last, obj);
70 alloc->top = (alloc->top + alloc->max) & alloc->mask;
71 spin_unlock(&alloc->lock);
72}
73
74int mthca_alloc_init(struct mthca_alloc *alloc, u32 num, u32 mask,
75 u32 reserved)
76{
77 int i;
78
79 /* num must be a power of 2 */
80 if (num != 1 << (ffs(num) - 1))
81 return -EINVAL;
82
83 alloc->last = 0;
84 alloc->top = 0;
85 alloc->max = num;
86 alloc->mask = mask;
87 spin_lock_init(&alloc->lock);
88 alloc->table = kmalloc(BITS_TO_LONGS(num) * sizeof (long),
89 GFP_KERNEL);
90 if (!alloc->table)
91 return -ENOMEM;
92
93 bitmap_zero(alloc->table, num);
94 for (i = 0; i < reserved; ++i)
95 set_bit(i, alloc->table);
96
97 return 0;
98}
99
100void mthca_alloc_cleanup(struct mthca_alloc *alloc)
101{
102 kfree(alloc->table);
103}
104
105/*
106 * Array of pointers with lazy allocation of leaf pages. Callers of
107 * _get, _set and _clear methods must use a lock or otherwise
108 * serialize access to the array.
109 */
110
111void *mthca_array_get(struct mthca_array *array, int index)
112{
113 int p = (index * sizeof (void *)) >> PAGE_SHIFT;
114
115 if (array->page_list[p].page) {
116 int i = index & (PAGE_SIZE / sizeof (void *) - 1);
117 return array->page_list[p].page[i];
118 } else
119 return NULL;
120}
121
122int mthca_array_set(struct mthca_array *array, int index, void *value)
123{
124 int p = (index * sizeof (void *)) >> PAGE_SHIFT;
125
126 /* Allocate with GFP_ATOMIC because we'll be called with locks held. */
127 if (!array->page_list[p].page)
128 array->page_list[p].page = (void **) get_zeroed_page(GFP_ATOMIC);
129
130 if (!array->page_list[p].page)
131 return -ENOMEM;
132
133 array->page_list[p].page[index & (PAGE_SIZE / sizeof (void *) - 1)] =
134 value;
135 ++array->page_list[p].used;
136
137 return 0;
138}
139
140void mthca_array_clear(struct mthca_array *array, int index)
141{
142 int p = (index * sizeof (void *)) >> PAGE_SHIFT;
143
144 if (--array->page_list[p].used == 0) {
145 free_page((unsigned long) array->page_list[p].page);
146 array->page_list[p].page = NULL;
147 }
148
149 if (array->page_list[p].used < 0)
150 pr_debug("Array %p index %d page %d with ref count %d < 0\n",
151 array, index, p, array->page_list[p].used);
152}
153
154int mthca_array_init(struct mthca_array *array, int nent)
155{
156 int npage = (nent * sizeof (void *) + PAGE_SIZE - 1) / PAGE_SIZE;
157 int i;
158
159 array->page_list = kmalloc(npage * sizeof *array->page_list, GFP_KERNEL);
160 if (!array->page_list)
161 return -ENOMEM;
162
163 for (i = 0; i < npage; ++i) {
164 array->page_list[i].page = NULL;
165 array->page_list[i].used = 0;
166 }
167
168 return 0;
169}
170
171void mthca_array_cleanup(struct mthca_array *array, int nent)
172{
173 int i;
174
175 for (i = 0; i < (nent * sizeof (void *) + PAGE_SIZE - 1) / PAGE_SIZE; ++i)
176 free_page((unsigned long) array->page_list[i].page);
177
178 kfree(array->page_list);
179}
diff --git a/drivers/infiniband/hw/mthca/mthca_av.c b/drivers/infiniband/hw/mthca/mthca_av.c
new file mode 100644
index 000000000000..426d32778e9c
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_av.c
@@ -0,0 +1,241 @@
1/*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_av.c 1349 2004-12-16 21:09:43Z roland $
33 */
34
35#include <linux/init.h>
36
37#include <ib_verbs.h>
38#include <ib_cache.h>
39
40#include "mthca_dev.h"
41
42struct mthca_av {
43 u32 port_pd;
44 u8 reserved1;
45 u8 g_slid;
46 u16 dlid;
47 u8 reserved2;
48 u8 gid_index;
49 u8 msg_sr;
50 u8 hop_limit;
51 u32 sl_tclass_flowlabel;
52 u32 dgid[4];
53};
54
55int mthca_create_ah(struct mthca_dev *dev,
56 struct mthca_pd *pd,
57 struct ib_ah_attr *ah_attr,
58 struct mthca_ah *ah)
59{
60 u32 index = -1;
61 struct mthca_av *av = NULL;
62
63 ah->type = MTHCA_AH_PCI_POOL;
64
65 if (dev->hca_type == ARBEL_NATIVE) {
66 ah->av = kmalloc(sizeof *ah->av, GFP_KERNEL);
67 if (!ah->av)
68 return -ENOMEM;
69
70 ah->type = MTHCA_AH_KMALLOC;
71 av = ah->av;
72 } else if (!atomic_read(&pd->sqp_count) &&
73 !(dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) {
74 index = mthca_alloc(&dev->av_table.alloc);
75
76 /* fall back to allocate in host memory */
77 if (index == -1)
78 goto on_hca_fail;
79
80 av = kmalloc(sizeof *av, GFP_KERNEL);
81 if (!av)
82 goto on_hca_fail;
83
84 ah->type = MTHCA_AH_ON_HCA;
85 ah->avdma = dev->av_table.ddr_av_base +
86 index * MTHCA_AV_SIZE;
87 }
88
89on_hca_fail:
90 if (ah->type == MTHCA_AH_PCI_POOL) {
91 ah->av = pci_pool_alloc(dev->av_table.pool,
92 SLAB_KERNEL, &ah->avdma);
93 if (!ah->av)
94 return -ENOMEM;
95
96 av = ah->av;
97 }
98
99 ah->key = pd->ntmr.ibmr.lkey;
100
101 memset(av, 0, MTHCA_AV_SIZE);
102
103 av->port_pd = cpu_to_be32(pd->pd_num | (ah_attr->port_num << 24));
104 av->g_slid = ah_attr->src_path_bits;
105 av->dlid = cpu_to_be16(ah_attr->dlid);
106 av->msg_sr = (3 << 4) | /* 2K message */
107 ah_attr->static_rate;
108 av->sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 28);
109 if (ah_attr->ah_flags & IB_AH_GRH) {
110 av->g_slid |= 0x80;
111 av->gid_index = (ah_attr->port_num - 1) * dev->limits.gid_table_len +
112 ah_attr->grh.sgid_index;
113 av->hop_limit = ah_attr->grh.hop_limit;
114 av->sl_tclass_flowlabel |=
115 cpu_to_be32((ah_attr->grh.traffic_class << 20) |
116 ah_attr->grh.flow_label);
117 memcpy(av->dgid, ah_attr->grh.dgid.raw, 16);
118 } else {
119 /* Arbel workaround -- low byte of GID must be 2 */
120 av->dgid[3] = cpu_to_be32(2);
121 }
122
123 if (0) {
124 int j;
125
126 mthca_dbg(dev, "Created UDAV at %p/%08lx:\n",
127 av, (unsigned long) ah->avdma);
128 for (j = 0; j < 8; ++j)
129 printk(KERN_DEBUG " [%2x] %08x\n",
130 j * 4, be32_to_cpu(((u32 *) av)[j]));
131 }
132
133 if (ah->type == MTHCA_AH_ON_HCA) {
134 memcpy_toio(dev->av_table.av_map + index * MTHCA_AV_SIZE,
135 av, MTHCA_AV_SIZE);
136 kfree(av);
137 }
138
139 return 0;
140}
141
142int mthca_destroy_ah(struct mthca_dev *dev, struct mthca_ah *ah)
143{
144 switch (ah->type) {
145 case MTHCA_AH_ON_HCA:
146 mthca_free(&dev->av_table.alloc,
147 (ah->avdma - dev->av_table.ddr_av_base) /
148 MTHCA_AV_SIZE);
149 break;
150
151 case MTHCA_AH_PCI_POOL:
152 pci_pool_free(dev->av_table.pool, ah->av, ah->avdma);
153 break;
154
155 case MTHCA_AH_KMALLOC:
156 kfree(ah->av);
157 break;
158 }
159
160 return 0;
161}
162
163int mthca_read_ah(struct mthca_dev *dev, struct mthca_ah *ah,
164 struct ib_ud_header *header)
165{
166 if (ah->type == MTHCA_AH_ON_HCA)
167 return -EINVAL;
168
169 header->lrh.service_level = be32_to_cpu(ah->av->sl_tclass_flowlabel) >> 28;
170 header->lrh.destination_lid = ah->av->dlid;
171 header->lrh.source_lid = ah->av->g_slid & 0x7f;
172 if (ah->av->g_slid & 0x80) {
173 header->grh_present = 1;
174 header->grh.traffic_class =
175 (be32_to_cpu(ah->av->sl_tclass_flowlabel) >> 20) & 0xff;
176 header->grh.flow_label =
177 ah->av->sl_tclass_flowlabel & cpu_to_be32(0xfffff);
178 ib_get_cached_gid(&dev->ib_dev,
179 be32_to_cpu(ah->av->port_pd) >> 24,
180 ah->av->gid_index,
181 &header->grh.source_gid);
182 memcpy(header->grh.destination_gid.raw,
183 ah->av->dgid, 16);
184 } else {
185 header->grh_present = 0;
186 }
187
188 return 0;
189}
190
191int __devinit mthca_init_av_table(struct mthca_dev *dev)
192{
193 int err;
194
195 if (dev->hca_type == ARBEL_NATIVE)
196 return 0;
197
198 err = mthca_alloc_init(&dev->av_table.alloc,
199 dev->av_table.num_ddr_avs,
200 dev->av_table.num_ddr_avs - 1,
201 0);
202 if (err)
203 return err;
204
205 dev->av_table.pool = pci_pool_create("mthca_av", dev->pdev,
206 MTHCA_AV_SIZE,
207 MTHCA_AV_SIZE, 0);
208 if (!dev->av_table.pool)
209 goto out_free_alloc;
210
211 if (!(dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) {
212 dev->av_table.av_map = ioremap(pci_resource_start(dev->pdev, 4) +
213 dev->av_table.ddr_av_base -
214 dev->ddr_start,
215 dev->av_table.num_ddr_avs *
216 MTHCA_AV_SIZE);
217 if (!dev->av_table.av_map)
218 goto out_free_pool;
219 } else
220 dev->av_table.av_map = NULL;
221
222 return 0;
223
224 out_free_pool:
225 pci_pool_destroy(dev->av_table.pool);
226
227 out_free_alloc:
228 mthca_alloc_cleanup(&dev->av_table.alloc);
229 return -ENOMEM;
230}
231
232void __devexit mthca_cleanup_av_table(struct mthca_dev *dev)
233{
234 if (dev->hca_type == ARBEL_NATIVE)
235 return;
236
237 if (dev->av_table.av_map)
238 iounmap(dev->av_table.av_map);
239 pci_pool_destroy(dev->av_table.pool);
240 mthca_alloc_cleanup(&dev->av_table.alloc);
241}
diff --git a/drivers/infiniband/hw/mthca/mthca_cmd.c b/drivers/infiniband/hw/mthca/mthca_cmd.c
new file mode 100644
index 000000000000..9def0981f630
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_cmd.c
@@ -0,0 +1,1767 @@
1/*
2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_cmd.c 1349 2004-12-16 21:09:43Z roland $
33 */
34
35#include <linux/sched.h>
36#include <linux/pci.h>
37#include <linux/errno.h>
38#include <asm/io.h>
39#include <ib_mad.h>
40
41#include "mthca_dev.h"
42#include "mthca_config_reg.h"
43#include "mthca_cmd.h"
44#include "mthca_memfree.h"
45
46#define CMD_POLL_TOKEN 0xffff
47
48enum {
49 HCR_IN_PARAM_OFFSET = 0x00,
50 HCR_IN_MODIFIER_OFFSET = 0x08,
51 HCR_OUT_PARAM_OFFSET = 0x0c,
52 HCR_TOKEN_OFFSET = 0x14,
53 HCR_STATUS_OFFSET = 0x18,
54
55 HCR_OPMOD_SHIFT = 12,
56 HCA_E_BIT = 22,
57 HCR_GO_BIT = 23
58};
59
60enum {
61 /* initialization and general commands */
62 CMD_SYS_EN = 0x1,
63 CMD_SYS_DIS = 0x2,
64 CMD_MAP_FA = 0xfff,
65 CMD_UNMAP_FA = 0xffe,
66 CMD_RUN_FW = 0xff6,
67 CMD_MOD_STAT_CFG = 0x34,
68 CMD_QUERY_DEV_LIM = 0x3,
69 CMD_QUERY_FW = 0x4,
70 CMD_ENABLE_LAM = 0xff8,
71 CMD_DISABLE_LAM = 0xff7,
72 CMD_QUERY_DDR = 0x5,
73 CMD_QUERY_ADAPTER = 0x6,
74 CMD_INIT_HCA = 0x7,
75 CMD_CLOSE_HCA = 0x8,
76 CMD_INIT_IB = 0x9,
77 CMD_CLOSE_IB = 0xa,
78 CMD_QUERY_HCA = 0xb,
79 CMD_SET_IB = 0xc,
80 CMD_ACCESS_DDR = 0x2e,
81 CMD_MAP_ICM = 0xffa,
82 CMD_UNMAP_ICM = 0xff9,
83 CMD_MAP_ICM_AUX = 0xffc,
84 CMD_UNMAP_ICM_AUX = 0xffb,
85 CMD_SET_ICM_SIZE = 0xffd,
86
87 /* TPT commands */
88 CMD_SW2HW_MPT = 0xd,
89 CMD_QUERY_MPT = 0xe,
90 CMD_HW2SW_MPT = 0xf,
91 CMD_READ_MTT = 0x10,
92 CMD_WRITE_MTT = 0x11,
93 CMD_SYNC_TPT = 0x2f,
94
95 /* EQ commands */
96 CMD_MAP_EQ = 0x12,
97 CMD_SW2HW_EQ = 0x13,
98 CMD_HW2SW_EQ = 0x14,
99 CMD_QUERY_EQ = 0x15,
100
101 /* CQ commands */
102 CMD_SW2HW_CQ = 0x16,
103 CMD_HW2SW_CQ = 0x17,
104 CMD_QUERY_CQ = 0x18,
105 CMD_RESIZE_CQ = 0x2c,
106
107 /* SRQ commands */
108 CMD_SW2HW_SRQ = 0x35,
109 CMD_HW2SW_SRQ = 0x36,
110 CMD_QUERY_SRQ = 0x37,
111
112 /* QP/EE commands */
113 CMD_RST2INIT_QPEE = 0x19,
114 CMD_INIT2RTR_QPEE = 0x1a,
115 CMD_RTR2RTS_QPEE = 0x1b,
116 CMD_RTS2RTS_QPEE = 0x1c,
117 CMD_SQERR2RTS_QPEE = 0x1d,
118 CMD_2ERR_QPEE = 0x1e,
119 CMD_RTS2SQD_QPEE = 0x1f,
120 CMD_SQD2SQD_QPEE = 0x38,
121 CMD_SQD2RTS_QPEE = 0x20,
122 CMD_ERR2RST_QPEE = 0x21,
123 CMD_QUERY_QPEE = 0x22,
124 CMD_INIT2INIT_QPEE = 0x2d,
125 CMD_SUSPEND_QPEE = 0x32,
126 CMD_UNSUSPEND_QPEE = 0x33,
127 /* special QPs and management commands */
128 CMD_CONF_SPECIAL_QP = 0x23,
129 CMD_MAD_IFC = 0x24,
130
131 /* multicast commands */
132 CMD_READ_MGM = 0x25,
133 CMD_WRITE_MGM = 0x26,
134 CMD_MGID_HASH = 0x27,
135
136 /* miscellaneous commands */
137 CMD_DIAG_RPRT = 0x30,
138 CMD_NOP = 0x31,
139
140 /* debug commands */
141 CMD_QUERY_DEBUG_MSG = 0x2a,
142 CMD_SET_DEBUG_MSG = 0x2b,
143};
144
145/*
146 * According to Mellanox code, FW may be starved and never complete
147 * commands. So we can't use strict timeouts described in PRM -- we
148 * just arbitrarily select 60 seconds for now.
149 */
150#if 0
151/*
152 * Round up and add 1 to make sure we get the full wait time (since we
153 * will be starting in the middle of a jiffy)
154 */
155enum {
156 CMD_TIME_CLASS_A = (HZ + 999) / 1000 + 1,
157 CMD_TIME_CLASS_B = (HZ + 99) / 100 + 1,
158 CMD_TIME_CLASS_C = (HZ + 9) / 10 + 1
159};
160#else
161enum {
162 CMD_TIME_CLASS_A = 60 * HZ,
163 CMD_TIME_CLASS_B = 60 * HZ,
164 CMD_TIME_CLASS_C = 60 * HZ
165};
166#endif
167
168enum {
169 GO_BIT_TIMEOUT = HZ * 10
170};
171
172struct mthca_cmd_context {
173 struct completion done;
174 struct timer_list timer;
175 int result;
176 int next;
177 u64 out_param;
178 u16 token;
179 u8 status;
180};
181
182static inline int go_bit(struct mthca_dev *dev)
183{
184 return readl(dev->hcr + HCR_STATUS_OFFSET) &
185 swab32(1 << HCR_GO_BIT);
186}
187
188static int mthca_cmd_post(struct mthca_dev *dev,
189 u64 in_param,
190 u64 out_param,
191 u32 in_modifier,
192 u8 op_modifier,
193 u16 op,
194 u16 token,
195 int event)
196{
197 int err = 0;
198
199 if (down_interruptible(&dev->cmd.hcr_sem))
200 return -EINTR;
201
202 if (event) {
203 unsigned long end = jiffies + GO_BIT_TIMEOUT;
204
205 while (go_bit(dev) && time_before(jiffies, end)) {
206 set_current_state(TASK_RUNNING);
207 schedule();
208 }
209 }
210
211 if (go_bit(dev)) {
212 err = -EAGAIN;
213 goto out;
214 }
215
216 /*
217 * We use writel (instead of something like memcpy_toio)
218 * because writes of less than 32 bits to the HCR don't work
219 * (and some architectures such as ia64 implement memcpy_toio
220 * in terms of writeb).
221 */
222 __raw_writel(cpu_to_be32(in_param >> 32), dev->hcr + 0 * 4);
223 __raw_writel(cpu_to_be32(in_param & 0xfffffffful), dev->hcr + 1 * 4);
224 __raw_writel(cpu_to_be32(in_modifier), dev->hcr + 2 * 4);
225 __raw_writel(cpu_to_be32(out_param >> 32), dev->hcr + 3 * 4);
226 __raw_writel(cpu_to_be32(out_param & 0xfffffffful), dev->hcr + 4 * 4);
227 __raw_writel(cpu_to_be32(token << 16), dev->hcr + 5 * 4);
228
229 /* __raw_writel may not order writes. */
230 wmb();
231
232 __raw_writel(cpu_to_be32((1 << HCR_GO_BIT) |
233 (event ? (1 << HCA_E_BIT) : 0) |
234 (op_modifier << HCR_OPMOD_SHIFT) |
235 op), dev->hcr + 6 * 4);
236
237out:
238 up(&dev->cmd.hcr_sem);
239 return err;
240}
241
242static int mthca_cmd_poll(struct mthca_dev *dev,
243 u64 in_param,
244 u64 *out_param,
245 int out_is_imm,
246 u32 in_modifier,
247 u8 op_modifier,
248 u16 op,
249 unsigned long timeout,
250 u8 *status)
251{
252 int err = 0;
253 unsigned long end;
254
255 if (down_interruptible(&dev->cmd.poll_sem))
256 return -EINTR;
257
258 err = mthca_cmd_post(dev, in_param,
259 out_param ? *out_param : 0,
260 in_modifier, op_modifier,
261 op, CMD_POLL_TOKEN, 0);
262 if (err)
263 goto out;
264
265 end = timeout + jiffies;
266 while (go_bit(dev) && time_before(jiffies, end)) {
267 set_current_state(TASK_RUNNING);
268 schedule();
269 }
270
271 if (go_bit(dev)) {
272 err = -EBUSY;
273 goto out;
274 }
275
276 if (out_is_imm) {
277 memcpy_fromio(out_param, dev->hcr + HCR_OUT_PARAM_OFFSET, sizeof (u64));
278 be64_to_cpus(out_param);
279 }
280
281 *status = be32_to_cpu(__raw_readl(dev->hcr + HCR_STATUS_OFFSET)) >> 24;
282
283out:
284 up(&dev->cmd.poll_sem);
285 return err;
286}
287
288void mthca_cmd_event(struct mthca_dev *dev,
289 u16 token,
290 u8 status,
291 u64 out_param)
292{
293 struct mthca_cmd_context *context =
294 &dev->cmd.context[token & dev->cmd.token_mask];
295
296 /* previously timed out command completing at long last */
297 if (token != context->token)
298 return;
299
300 context->result = 0;
301 context->status = status;
302 context->out_param = out_param;
303
304 context->token += dev->cmd.token_mask + 1;
305
306 complete(&context->done);
307}
308
309static void event_timeout(unsigned long context_ptr)
310{
311 struct mthca_cmd_context *context =
312 (struct mthca_cmd_context *) context_ptr;
313
314 context->result = -EBUSY;
315 complete(&context->done);
316}
317
318static int mthca_cmd_wait(struct mthca_dev *dev,
319 u64 in_param,
320 u64 *out_param,
321 int out_is_imm,
322 u32 in_modifier,
323 u8 op_modifier,
324 u16 op,
325 unsigned long timeout,
326 u8 *status)
327{
328 int err = 0;
329 struct mthca_cmd_context *context;
330
331 if (down_interruptible(&dev->cmd.event_sem))
332 return -EINTR;
333
334 spin_lock(&dev->cmd.context_lock);
335 BUG_ON(dev->cmd.free_head < 0);
336 context = &dev->cmd.context[dev->cmd.free_head];
337 dev->cmd.free_head = context->next;
338 spin_unlock(&dev->cmd.context_lock);
339
340 init_completion(&context->done);
341
342 err = mthca_cmd_post(dev, in_param,
343 out_param ? *out_param : 0,
344 in_modifier, op_modifier,
345 op, context->token, 1);
346 if (err)
347 goto out;
348
349 context->timer.expires = jiffies + timeout;
350 add_timer(&context->timer);
351
352 wait_for_completion(&context->done);
353 del_timer_sync(&context->timer);
354
355 err = context->result;
356 if (err)
357 goto out;
358
359 *status = context->status;
360 if (*status)
361 mthca_dbg(dev, "Command %02x completed with status %02x\n",
362 op, *status);
363
364 if (out_is_imm)
365 *out_param = context->out_param;
366
367out:
368 spin_lock(&dev->cmd.context_lock);
369 context->next = dev->cmd.free_head;
370 dev->cmd.free_head = context - dev->cmd.context;
371 spin_unlock(&dev->cmd.context_lock);
372
373 up(&dev->cmd.event_sem);
374 return err;
375}
376
377/* Invoke a command with an output mailbox */
378static int mthca_cmd_box(struct mthca_dev *dev,
379 u64 in_param,
380 u64 out_param,
381 u32 in_modifier,
382 u8 op_modifier,
383 u16 op,
384 unsigned long timeout,
385 u8 *status)
386{
387 if (dev->cmd.use_events)
388 return mthca_cmd_wait(dev, in_param, &out_param, 0,
389 in_modifier, op_modifier, op,
390 timeout, status);
391 else
392 return mthca_cmd_poll(dev, in_param, &out_param, 0,
393 in_modifier, op_modifier, op,
394 timeout, status);
395}
396
397/* Invoke a command with no output parameter */
398static int mthca_cmd(struct mthca_dev *dev,
399 u64 in_param,
400 u32 in_modifier,
401 u8 op_modifier,
402 u16 op,
403 unsigned long timeout,
404 u8 *status)
405{
406 return mthca_cmd_box(dev, in_param, 0, in_modifier,
407 op_modifier, op, timeout, status);
408}
409
410/*
411 * Invoke a command with an immediate output parameter (and copy the
412 * output into the caller's out_param pointer after the command
413 * executes).
414 */
415static int mthca_cmd_imm(struct mthca_dev *dev,
416 u64 in_param,
417 u64 *out_param,
418 u32 in_modifier,
419 u8 op_modifier,
420 u16 op,
421 unsigned long timeout,
422 u8 *status)
423{
424 if (dev->cmd.use_events)
425 return mthca_cmd_wait(dev, in_param, out_param, 1,
426 in_modifier, op_modifier, op,
427 timeout, status);
428 else
429 return mthca_cmd_poll(dev, in_param, out_param, 1,
430 in_modifier, op_modifier, op,
431 timeout, status);
432}
433
434/*
435 * Switch to using events to issue FW commands (should be called after
436 * event queue to command events has been initialized).
437 */
438int mthca_cmd_use_events(struct mthca_dev *dev)
439{
440 int i;
441
442 dev->cmd.context = kmalloc(dev->cmd.max_cmds *
443 sizeof (struct mthca_cmd_context),
444 GFP_KERNEL);
445 if (!dev->cmd.context)
446 return -ENOMEM;
447
448 for (i = 0; i < dev->cmd.max_cmds; ++i) {
449 dev->cmd.context[i].token = i;
450 dev->cmd.context[i].next = i + 1;
451 init_timer(&dev->cmd.context[i].timer);
452 dev->cmd.context[i].timer.data =
453 (unsigned long) &dev->cmd.context[i];
454 dev->cmd.context[i].timer.function = event_timeout;
455 }
456
457 dev->cmd.context[dev->cmd.max_cmds - 1].next = -1;
458 dev->cmd.free_head = 0;
459
460 sema_init(&dev->cmd.event_sem, dev->cmd.max_cmds);
461 spin_lock_init(&dev->cmd.context_lock);
462
463 for (dev->cmd.token_mask = 1;
464 dev->cmd.token_mask < dev->cmd.max_cmds;
465 dev->cmd.token_mask <<= 1)
466 ; /* nothing */
467 --dev->cmd.token_mask;
468
469 dev->cmd.use_events = 1;
470 down(&dev->cmd.poll_sem);
471
472 return 0;
473}
474
475/*
476 * Switch back to polling (used when shutting down the device)
477 */
478void mthca_cmd_use_polling(struct mthca_dev *dev)
479{
480 int i;
481
482 dev->cmd.use_events = 0;
483
484 for (i = 0; i < dev->cmd.max_cmds; ++i)
485 down(&dev->cmd.event_sem);
486
487 kfree(dev->cmd.context);
488
489 up(&dev->cmd.poll_sem);
490}
491
492int mthca_SYS_EN(struct mthca_dev *dev, u8 *status)
493{
494 u64 out;
495 int ret;
496
497 ret = mthca_cmd_imm(dev, 0, &out, 0, 0, CMD_SYS_EN, HZ, status);
498
499 if (*status == MTHCA_CMD_STAT_DDR_MEM_ERR)
500 mthca_warn(dev, "SYS_EN DDR error: syn=%x, sock=%d, "
501 "sladdr=%d, SPD source=%s\n",
502 (int) (out >> 6) & 0xf, (int) (out >> 4) & 3,
503 (int) (out >> 1) & 7, (int) out & 1 ? "NVMEM" : "DIMM");
504
505 return ret;
506}
507
508int mthca_SYS_DIS(struct mthca_dev *dev, u8 *status)
509{
510 return mthca_cmd(dev, 0, 0, 0, CMD_SYS_DIS, HZ, status);
511}
512
513static int mthca_map_cmd(struct mthca_dev *dev, u16 op, struct mthca_icm *icm,
514 u64 virt, u8 *status)
515{
516 u32 *inbox;
517 dma_addr_t indma;
518 struct mthca_icm_iter iter;
519 int lg;
520 int nent = 0;
521 int i;
522 int err = 0;
523 int ts = 0, tc = 0;
524
525 inbox = pci_alloc_consistent(dev->pdev, PAGE_SIZE, &indma);
526 if (!inbox)
527 return -ENOMEM;
528
529 memset(inbox, 0, PAGE_SIZE);
530
531 for (mthca_icm_first(icm, &iter);
532 !mthca_icm_last(&iter);
533 mthca_icm_next(&iter)) {
534 /*
535 * We have to pass pages that are aligned to their
536 * size, so find the least significant 1 in the
537 * address or size and use that as our log2 size.
538 */
539 lg = ffs(mthca_icm_addr(&iter) | mthca_icm_size(&iter)) - 1;
540 if (lg < 12) {
541 mthca_warn(dev, "Got FW area not aligned to 4K (%llx/%lx).\n",
542 (unsigned long long) mthca_icm_addr(&iter),
543 mthca_icm_size(&iter));
544 err = -EINVAL;
545 goto out;
546 }
547 for (i = 0; i < mthca_icm_size(&iter) / (1 << lg); ++i, ++nent) {
548 if (virt != -1) {
549 *((__be64 *) (inbox + nent * 4)) =
550 cpu_to_be64(virt);
551 virt += 1 << lg;
552 }
553
554 *((__be64 *) (inbox + nent * 4 + 2)) =
555 cpu_to_be64((mthca_icm_addr(&iter) +
556 (i << lg)) | (lg - 12));
557 ts += 1 << (lg - 10);
558 ++tc;
559
560 if (nent == PAGE_SIZE / 16) {
561 err = mthca_cmd(dev, indma, nent, 0, op,
562 CMD_TIME_CLASS_B, status);
563 if (err || *status)
564 goto out;
565 nent = 0;
566 }
567 }
568 }
569
570 if (nent)
571 err = mthca_cmd(dev, indma, nent, 0, op,
572 CMD_TIME_CLASS_B, status);
573
574 switch (op) {
575 case CMD_MAP_FA:
576 mthca_dbg(dev, "Mapped %d chunks/%d KB for FW.\n", tc, ts);
577 break;
578 case CMD_MAP_ICM_AUX:
579 mthca_dbg(dev, "Mapped %d chunks/%d KB for ICM aux.\n", tc, ts);
580 break;
581 case CMD_MAP_ICM:
582 mthca_dbg(dev, "Mapped %d chunks/%d KB at %llx for ICM.\n",
583 tc, ts, (unsigned long long) virt - (ts << 10));
584 break;
585 }
586
587out:
588 pci_free_consistent(dev->pdev, PAGE_SIZE, inbox, indma);
589 return err;
590}
591
592int mthca_MAP_FA(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status)
593{
594 return mthca_map_cmd(dev, CMD_MAP_FA, icm, -1, status);
595}
596
597int mthca_UNMAP_FA(struct mthca_dev *dev, u8 *status)
598{
599 return mthca_cmd(dev, 0, 0, 0, CMD_UNMAP_FA, CMD_TIME_CLASS_B, status);
600}
601
602int mthca_RUN_FW(struct mthca_dev *dev, u8 *status)
603{
604 return mthca_cmd(dev, 0, 0, 0, CMD_RUN_FW, CMD_TIME_CLASS_A, status);
605}
606
607int mthca_QUERY_FW(struct mthca_dev *dev, u8 *status)
608{
609 u32 *outbox;
610 dma_addr_t outdma;
611 int err = 0;
612 u8 lg;
613
614#define QUERY_FW_OUT_SIZE 0x100
615#define QUERY_FW_VER_OFFSET 0x00
616#define QUERY_FW_MAX_CMD_OFFSET 0x0f
617#define QUERY_FW_ERR_START_OFFSET 0x30
618#define QUERY_FW_ERR_SIZE_OFFSET 0x38
619
620#define QUERY_FW_START_OFFSET 0x20
621#define QUERY_FW_END_OFFSET 0x28
622
623#define QUERY_FW_SIZE_OFFSET 0x00
624#define QUERY_FW_CLR_INT_BASE_OFFSET 0x20
625#define QUERY_FW_EQ_ARM_BASE_OFFSET 0x40
626#define QUERY_FW_EQ_SET_CI_BASE_OFFSET 0x48
627
628 outbox = pci_alloc_consistent(dev->pdev, QUERY_FW_OUT_SIZE, &outdma);
629 if (!outbox) {
630 return -ENOMEM;
631 }
632
633 err = mthca_cmd_box(dev, 0, outdma, 0, 0, CMD_QUERY_FW,
634 CMD_TIME_CLASS_A, status);
635
636 if (err)
637 goto out;
638
639 MTHCA_GET(dev->fw_ver, outbox, QUERY_FW_VER_OFFSET);
640 /*
641 * FW subminor version is at more signifant bits than minor
642 * version, so swap here.
643 */
644 dev->fw_ver = (dev->fw_ver & 0xffff00000000ull) |
645 ((dev->fw_ver & 0xffff0000ull) >> 16) |
646 ((dev->fw_ver & 0x0000ffffull) << 16);
647
648 MTHCA_GET(lg, outbox, QUERY_FW_MAX_CMD_OFFSET);
649 dev->cmd.max_cmds = 1 << lg;
650
651 mthca_dbg(dev, "FW version %012llx, max commands %d\n",
652 (unsigned long long) dev->fw_ver, dev->cmd.max_cmds);
653
654 if (dev->hca_type == ARBEL_NATIVE) {
655 MTHCA_GET(dev->fw.arbel.fw_pages, outbox, QUERY_FW_SIZE_OFFSET);
656 MTHCA_GET(dev->fw.arbel.clr_int_base, outbox, QUERY_FW_CLR_INT_BASE_OFFSET);
657 MTHCA_GET(dev->fw.arbel.eq_arm_base, outbox, QUERY_FW_EQ_ARM_BASE_OFFSET);
658 MTHCA_GET(dev->fw.arbel.eq_set_ci_base, outbox, QUERY_FW_EQ_SET_CI_BASE_OFFSET);
659 mthca_dbg(dev, "FW size %d KB\n", dev->fw.arbel.fw_pages << 2);
660
661 /*
662 * Arbel page size is always 4 KB; round up number of
663 * system pages needed.
664 */
665 dev->fw.arbel.fw_pages =
666 (dev->fw.arbel.fw_pages + (1 << (PAGE_SHIFT - 12)) - 1) >>
667 (PAGE_SHIFT - 12);
668
669 mthca_dbg(dev, "Clear int @ %llx, EQ arm @ %llx, EQ set CI @ %llx\n",
670 (unsigned long long) dev->fw.arbel.clr_int_base,
671 (unsigned long long) dev->fw.arbel.eq_arm_base,
672 (unsigned long long) dev->fw.arbel.eq_set_ci_base);
673 } else {
674 MTHCA_GET(dev->fw.tavor.fw_start, outbox, QUERY_FW_START_OFFSET);
675 MTHCA_GET(dev->fw.tavor.fw_end, outbox, QUERY_FW_END_OFFSET);
676
677 mthca_dbg(dev, "FW size %d KB (start %llx, end %llx)\n",
678 (int) ((dev->fw.tavor.fw_end - dev->fw.tavor.fw_start) >> 10),
679 (unsigned long long) dev->fw.tavor.fw_start,
680 (unsigned long long) dev->fw.tavor.fw_end);
681 }
682
683out:
684 pci_free_consistent(dev->pdev, QUERY_FW_OUT_SIZE, outbox, outdma);
685 return err;
686}
687
688int mthca_ENABLE_LAM(struct mthca_dev *dev, u8 *status)
689{
690 u8 info;
691 u32 *outbox;
692 dma_addr_t outdma;
693 int err = 0;
694
695#define ENABLE_LAM_OUT_SIZE 0x100
696#define ENABLE_LAM_START_OFFSET 0x00
697#define ENABLE_LAM_END_OFFSET 0x08
698#define ENABLE_LAM_INFO_OFFSET 0x13
699
700#define ENABLE_LAM_INFO_HIDDEN_FLAG (1 << 4)
701#define ENABLE_LAM_INFO_ECC_MASK 0x3
702
703 outbox = pci_alloc_consistent(dev->pdev, ENABLE_LAM_OUT_SIZE, &outdma);
704 if (!outbox)
705 return -ENOMEM;
706
707 err = mthca_cmd_box(dev, 0, outdma, 0, 0, CMD_ENABLE_LAM,
708 CMD_TIME_CLASS_C, status);
709
710 if (err)
711 goto out;
712
713 if (*status == MTHCA_CMD_STAT_LAM_NOT_PRE)
714 goto out;
715
716 MTHCA_GET(dev->ddr_start, outbox, ENABLE_LAM_START_OFFSET);
717 MTHCA_GET(dev->ddr_end, outbox, ENABLE_LAM_END_OFFSET);
718 MTHCA_GET(info, outbox, ENABLE_LAM_INFO_OFFSET);
719
720 if (!!(info & ENABLE_LAM_INFO_HIDDEN_FLAG) !=
721 !!(dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) {
722 mthca_info(dev, "FW reports that HCA-attached memory "
723 "is %s hidden; does not match PCI config\n",
724 (info & ENABLE_LAM_INFO_HIDDEN_FLAG) ?
725 "" : "not");
726 }
727 if (info & ENABLE_LAM_INFO_HIDDEN_FLAG)
728 mthca_dbg(dev, "HCA-attached memory is hidden.\n");
729
730 mthca_dbg(dev, "HCA memory size %d KB (start %llx, end %llx)\n",
731 (int) ((dev->ddr_end - dev->ddr_start) >> 10),
732 (unsigned long long) dev->ddr_start,
733 (unsigned long long) dev->ddr_end);
734
735out:
736 pci_free_consistent(dev->pdev, ENABLE_LAM_OUT_SIZE, outbox, outdma);
737 return err;
738}
739
740int mthca_DISABLE_LAM(struct mthca_dev *dev, u8 *status)
741{
742 return mthca_cmd(dev, 0, 0, 0, CMD_SYS_DIS, CMD_TIME_CLASS_C, status);
743}
744
745int mthca_QUERY_DDR(struct mthca_dev *dev, u8 *status)
746{
747 u8 info;
748 u32 *outbox;
749 dma_addr_t outdma;
750 int err = 0;
751
752#define QUERY_DDR_OUT_SIZE 0x100
753#define QUERY_DDR_START_OFFSET 0x00
754#define QUERY_DDR_END_OFFSET 0x08
755#define QUERY_DDR_INFO_OFFSET 0x13
756
757#define QUERY_DDR_INFO_HIDDEN_FLAG (1 << 4)
758#define QUERY_DDR_INFO_ECC_MASK 0x3
759
760 outbox = pci_alloc_consistent(dev->pdev, QUERY_DDR_OUT_SIZE, &outdma);
761 if (!outbox)
762 return -ENOMEM;
763
764 err = mthca_cmd_box(dev, 0, outdma, 0, 0, CMD_QUERY_DDR,
765 CMD_TIME_CLASS_A, status);
766
767 if (err)
768 goto out;
769
770 MTHCA_GET(dev->ddr_start, outbox, QUERY_DDR_START_OFFSET);
771 MTHCA_GET(dev->ddr_end, outbox, QUERY_DDR_END_OFFSET);
772 MTHCA_GET(info, outbox, QUERY_DDR_INFO_OFFSET);
773
774 if (!!(info & QUERY_DDR_INFO_HIDDEN_FLAG) !=
775 !!(dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) {
776 mthca_info(dev, "FW reports that HCA-attached memory "
777 "is %s hidden; does not match PCI config\n",
778 (info & QUERY_DDR_INFO_HIDDEN_FLAG) ?
779 "" : "not");
780 }
781 if (info & QUERY_DDR_INFO_HIDDEN_FLAG)
782 mthca_dbg(dev, "HCA-attached memory is hidden.\n");
783
784 mthca_dbg(dev, "HCA memory size %d KB (start %llx, end %llx)\n",
785 (int) ((dev->ddr_end - dev->ddr_start) >> 10),
786 (unsigned long long) dev->ddr_start,
787 (unsigned long long) dev->ddr_end);
788
789out:
790 pci_free_consistent(dev->pdev, QUERY_DDR_OUT_SIZE, outbox, outdma);
791 return err;
792}
793
794int mthca_QUERY_DEV_LIM(struct mthca_dev *dev,
795 struct mthca_dev_lim *dev_lim, u8 *status)
796{
797 u32 *outbox;
798 dma_addr_t outdma;
799 u8 field;
800 u16 size;
801 int err;
802
803#define QUERY_DEV_LIM_OUT_SIZE 0x100
804#define QUERY_DEV_LIM_MAX_SRQ_SZ_OFFSET 0x10
805#define QUERY_DEV_LIM_MAX_QP_SZ_OFFSET 0x11
806#define QUERY_DEV_LIM_RSVD_QP_OFFSET 0x12
807#define QUERY_DEV_LIM_MAX_QP_OFFSET 0x13
808#define QUERY_DEV_LIM_RSVD_SRQ_OFFSET 0x14
809#define QUERY_DEV_LIM_MAX_SRQ_OFFSET 0x15
810#define QUERY_DEV_LIM_RSVD_EEC_OFFSET 0x16
811#define QUERY_DEV_LIM_MAX_EEC_OFFSET 0x17
812#define QUERY_DEV_LIM_MAX_CQ_SZ_OFFSET 0x19
813#define QUERY_DEV_LIM_RSVD_CQ_OFFSET 0x1a
814#define QUERY_DEV_LIM_MAX_CQ_OFFSET 0x1b
815#define QUERY_DEV_LIM_MAX_MPT_OFFSET 0x1d
816#define QUERY_DEV_LIM_RSVD_EQ_OFFSET 0x1e
817#define QUERY_DEV_LIM_MAX_EQ_OFFSET 0x1f
818#define QUERY_DEV_LIM_RSVD_MTT_OFFSET 0x20
819#define QUERY_DEV_LIM_MAX_MRW_SZ_OFFSET 0x21
820#define QUERY_DEV_LIM_RSVD_MRW_OFFSET 0x22
821#define QUERY_DEV_LIM_MAX_MTT_SEG_OFFSET 0x23
822#define QUERY_DEV_LIM_MAX_AV_OFFSET 0x27
823#define QUERY_DEV_LIM_MAX_REQ_QP_OFFSET 0x29
824#define QUERY_DEV_LIM_MAX_RES_QP_OFFSET 0x2b
825#define QUERY_DEV_LIM_MAX_RDMA_OFFSET 0x2f
826#define QUERY_DEV_LIM_RSZ_SRQ_OFFSET 0x33
827#define QUERY_DEV_LIM_ACK_DELAY_OFFSET 0x35
828#define QUERY_DEV_LIM_MTU_WIDTH_OFFSET 0x36
829#define QUERY_DEV_LIM_VL_PORT_OFFSET 0x37
830#define QUERY_DEV_LIM_MAX_GID_OFFSET 0x3b
831#define QUERY_DEV_LIM_MAX_PKEY_OFFSET 0x3f
832#define QUERY_DEV_LIM_FLAGS_OFFSET 0x44
833#define QUERY_DEV_LIM_RSVD_UAR_OFFSET 0x48
834#define QUERY_DEV_LIM_UAR_SZ_OFFSET 0x49
835#define QUERY_DEV_LIM_PAGE_SZ_OFFSET 0x4b
836#define QUERY_DEV_LIM_MAX_SG_OFFSET 0x51
837#define QUERY_DEV_LIM_MAX_DESC_SZ_OFFSET 0x52
838#define QUERY_DEV_LIM_MAX_SG_RQ_OFFSET 0x55
839#define QUERY_DEV_LIM_MAX_DESC_SZ_RQ_OFFSET 0x56
840#define QUERY_DEV_LIM_MAX_QP_MCG_OFFSET 0x61
841#define QUERY_DEV_LIM_RSVD_MCG_OFFSET 0x62
842#define QUERY_DEV_LIM_MAX_MCG_OFFSET 0x63
843#define QUERY_DEV_LIM_RSVD_PD_OFFSET 0x64
844#define QUERY_DEV_LIM_MAX_PD_OFFSET 0x65
845#define QUERY_DEV_LIM_RSVD_RDD_OFFSET 0x66
846#define QUERY_DEV_LIM_MAX_RDD_OFFSET 0x67
847#define QUERY_DEV_LIM_EEC_ENTRY_SZ_OFFSET 0x80
848#define QUERY_DEV_LIM_QPC_ENTRY_SZ_OFFSET 0x82
849#define QUERY_DEV_LIM_EEEC_ENTRY_SZ_OFFSET 0x84
850#define QUERY_DEV_LIM_EQPC_ENTRY_SZ_OFFSET 0x86
851#define QUERY_DEV_LIM_EQC_ENTRY_SZ_OFFSET 0x88
852#define QUERY_DEV_LIM_CQC_ENTRY_SZ_OFFSET 0x8a
853#define QUERY_DEV_LIM_SRQ_ENTRY_SZ_OFFSET 0x8c
854#define QUERY_DEV_LIM_UAR_ENTRY_SZ_OFFSET 0x8e
855#define QUERY_DEV_LIM_MTT_ENTRY_SZ_OFFSET 0x90
856#define QUERY_DEV_LIM_MPT_ENTRY_SZ_OFFSET 0x92
857#define QUERY_DEV_LIM_PBL_SZ_OFFSET 0x96
858#define QUERY_DEV_LIM_BMME_FLAGS_OFFSET 0x97
859#define QUERY_DEV_LIM_RSVD_LKEY_OFFSET 0x98
860#define QUERY_DEV_LIM_LAMR_OFFSET 0x9f
861#define QUERY_DEV_LIM_MAX_ICM_SZ_OFFSET 0xa0
862
863 outbox = pci_alloc_consistent(dev->pdev, QUERY_DEV_LIM_OUT_SIZE, &outdma);
864 if (!outbox)
865 return -ENOMEM;
866
867 err = mthca_cmd_box(dev, 0, outdma, 0, 0, CMD_QUERY_DEV_LIM,
868 CMD_TIME_CLASS_A, status);
869
870 if (err)
871 goto out;
872
873 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_SRQ_SZ_OFFSET);
874 dev_lim->max_srq_sz = 1 << field;
875 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_QP_SZ_OFFSET);
876 dev_lim->max_qp_sz = 1 << field;
877 MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_QP_OFFSET);
878 dev_lim->reserved_qps = 1 << (field & 0xf);
879 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_QP_OFFSET);
880 dev_lim->max_qps = 1 << (field & 0x1f);
881 MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_SRQ_OFFSET);
882 dev_lim->reserved_srqs = 1 << (field >> 4);
883 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_SRQ_OFFSET);
884 dev_lim->max_srqs = 1 << (field & 0x1f);
885 MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_EEC_OFFSET);
886 dev_lim->reserved_eecs = 1 << (field & 0xf);
887 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_EEC_OFFSET);
888 dev_lim->max_eecs = 1 << (field & 0x1f);
889 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_CQ_SZ_OFFSET);
890 dev_lim->max_cq_sz = 1 << field;
891 MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_CQ_OFFSET);
892 dev_lim->reserved_cqs = 1 << (field & 0xf);
893 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_CQ_OFFSET);
894 dev_lim->max_cqs = 1 << (field & 0x1f);
895 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_MPT_OFFSET);
896 dev_lim->max_mpts = 1 << (field & 0x3f);
897 MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_EQ_OFFSET);
898 dev_lim->reserved_eqs = 1 << (field & 0xf);
899 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_EQ_OFFSET);
900 dev_lim->max_eqs = 1 << (field & 0x7);
901 MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_MTT_OFFSET);
902 dev_lim->reserved_mtts = 1 << (field >> 4);
903 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_MRW_SZ_OFFSET);
904 dev_lim->max_mrw_sz = 1 << field;
905 MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_MRW_OFFSET);
906 dev_lim->reserved_mrws = 1 << (field & 0xf);
907 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_MTT_SEG_OFFSET);
908 dev_lim->max_mtt_seg = 1 << (field & 0x3f);
909 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_REQ_QP_OFFSET);
910 dev_lim->max_requester_per_qp = 1 << (field & 0x3f);
911 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_RES_QP_OFFSET);
912 dev_lim->max_responder_per_qp = 1 << (field & 0x3f);
913 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_RDMA_OFFSET);
914 dev_lim->max_rdma_global = 1 << (field & 0x3f);
915 MTHCA_GET(field, outbox, QUERY_DEV_LIM_ACK_DELAY_OFFSET);
916 dev_lim->local_ca_ack_delay = field & 0x1f;
917 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MTU_WIDTH_OFFSET);
918 dev_lim->max_mtu = field >> 4;
919 dev_lim->max_port_width = field & 0xf;
920 MTHCA_GET(field, outbox, QUERY_DEV_LIM_VL_PORT_OFFSET);
921 dev_lim->max_vl = field >> 4;
922 dev_lim->num_ports = field & 0xf;
923 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_GID_OFFSET);
924 dev_lim->max_gids = 1 << (field & 0xf);
925 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_PKEY_OFFSET);
926 dev_lim->max_pkeys = 1 << (field & 0xf);
927 MTHCA_GET(dev_lim->flags, outbox, QUERY_DEV_LIM_FLAGS_OFFSET);
928 MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_UAR_OFFSET);
929 dev_lim->reserved_uars = field >> 4;
930 MTHCA_GET(field, outbox, QUERY_DEV_LIM_UAR_SZ_OFFSET);
931 dev_lim->uar_size = 1 << ((field & 0x3f) + 20);
932 MTHCA_GET(field, outbox, QUERY_DEV_LIM_PAGE_SZ_OFFSET);
933 dev_lim->min_page_sz = 1 << field;
934 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_SG_OFFSET);
935 dev_lim->max_sg = field;
936
937 MTHCA_GET(size, outbox, QUERY_DEV_LIM_MAX_DESC_SZ_OFFSET);
938 dev_lim->max_desc_sz = size;
939
940 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_QP_MCG_OFFSET);
941 dev_lim->max_qp_per_mcg = 1 << field;
942 MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_MCG_OFFSET);
943 dev_lim->reserved_mgms = field & 0xf;
944 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_MCG_OFFSET);
945 dev_lim->max_mcgs = 1 << field;
946 MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_PD_OFFSET);
947 dev_lim->reserved_pds = field >> 4;
948 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_PD_OFFSET);
949 dev_lim->max_pds = 1 << (field & 0x3f);
950 MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_RDD_OFFSET);
951 dev_lim->reserved_rdds = field >> 4;
952 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_RDD_OFFSET);
953 dev_lim->max_rdds = 1 << (field & 0x3f);
954
955 MTHCA_GET(size, outbox, QUERY_DEV_LIM_EEC_ENTRY_SZ_OFFSET);
956 dev_lim->eec_entry_sz = size;
957 MTHCA_GET(size, outbox, QUERY_DEV_LIM_QPC_ENTRY_SZ_OFFSET);
958 dev_lim->qpc_entry_sz = size;
959 MTHCA_GET(size, outbox, QUERY_DEV_LIM_EEEC_ENTRY_SZ_OFFSET);
960 dev_lim->eeec_entry_sz = size;
961 MTHCA_GET(size, outbox, QUERY_DEV_LIM_EQPC_ENTRY_SZ_OFFSET);
962 dev_lim->eqpc_entry_sz = size;
963 MTHCA_GET(size, outbox, QUERY_DEV_LIM_EQC_ENTRY_SZ_OFFSET);
964 dev_lim->eqc_entry_sz = size;
965 MTHCA_GET(size, outbox, QUERY_DEV_LIM_CQC_ENTRY_SZ_OFFSET);
966 dev_lim->cqc_entry_sz = size;
967 MTHCA_GET(size, outbox, QUERY_DEV_LIM_SRQ_ENTRY_SZ_OFFSET);
968 dev_lim->srq_entry_sz = size;
969 MTHCA_GET(size, outbox, QUERY_DEV_LIM_UAR_ENTRY_SZ_OFFSET);
970 dev_lim->uar_scratch_entry_sz = size;
971
972 mthca_dbg(dev, "Max QPs: %d, reserved QPs: %d, entry size: %d\n",
973 dev_lim->max_qps, dev_lim->reserved_qps, dev_lim->qpc_entry_sz);
974 mthca_dbg(dev, "Max CQs: %d, reserved CQs: %d, entry size: %d\n",
975 dev_lim->max_cqs, dev_lim->reserved_cqs, dev_lim->cqc_entry_sz);
976 mthca_dbg(dev, "Max EQs: %d, reserved EQs: %d, entry size: %d\n",
977 dev_lim->max_eqs, dev_lim->reserved_eqs, dev_lim->eqc_entry_sz);
978 mthca_dbg(dev, "reserved MPTs: %d, reserved MTTs: %d\n",
979 dev_lim->reserved_mrws, dev_lim->reserved_mtts);
980 mthca_dbg(dev, "Max PDs: %d, reserved PDs: %d, reserved UARs: %d\n",
981 dev_lim->max_pds, dev_lim->reserved_pds, dev_lim->reserved_uars);
982 mthca_dbg(dev, "Max QP/MCG: %d, reserved MGMs: %d\n",
983 dev_lim->max_pds, dev_lim->reserved_mgms);
984
985 mthca_dbg(dev, "Flags: %08x\n", dev_lim->flags);
986
987 if (dev->hca_type == ARBEL_NATIVE) {
988 MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSZ_SRQ_OFFSET);
989 dev_lim->hca.arbel.resize_srq = field & 1;
990 MTHCA_GET(size, outbox, QUERY_DEV_LIM_MTT_ENTRY_SZ_OFFSET);
991 dev_lim->mtt_seg_sz = size;
992 MTHCA_GET(size, outbox, QUERY_DEV_LIM_MPT_ENTRY_SZ_OFFSET);
993 dev_lim->mpt_entry_sz = size;
994 MTHCA_GET(field, outbox, QUERY_DEV_LIM_PBL_SZ_OFFSET);
995 dev_lim->hca.arbel.max_pbl_sz = 1 << (field & 0x3f);
996 MTHCA_GET(dev_lim->hca.arbel.bmme_flags, outbox,
997 QUERY_DEV_LIM_BMME_FLAGS_OFFSET);
998 MTHCA_GET(dev_lim->hca.arbel.reserved_lkey, outbox,
999 QUERY_DEV_LIM_RSVD_LKEY_OFFSET);
1000 MTHCA_GET(field, outbox, QUERY_DEV_LIM_LAMR_OFFSET);
1001 dev_lim->hca.arbel.lam_required = field & 1;
1002 MTHCA_GET(dev_lim->hca.arbel.max_icm_sz, outbox,
1003 QUERY_DEV_LIM_MAX_ICM_SZ_OFFSET);
1004
1005 if (dev_lim->hca.arbel.bmme_flags & 1)
1006 mthca_dbg(dev, "Base MM extensions: yes "
1007 "(flags %d, max PBL %d, rsvd L_Key %08x)\n",
1008 dev_lim->hca.arbel.bmme_flags,
1009 dev_lim->hca.arbel.max_pbl_sz,
1010 dev_lim->hca.arbel.reserved_lkey);
1011 else
1012 mthca_dbg(dev, "Base MM extensions: no\n");
1013
1014 mthca_dbg(dev, "Max ICM size %lld MB\n",
1015 (unsigned long long) dev_lim->hca.arbel.max_icm_sz >> 20);
1016 } else {
1017 MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_AV_OFFSET);
1018 dev_lim->hca.tavor.max_avs = 1 << (field & 0x3f);
1019 dev_lim->mtt_seg_sz = MTHCA_MTT_SEG_SIZE;
1020 dev_lim->mpt_entry_sz = MTHCA_MPT_ENTRY_SIZE;
1021 }
1022
1023out:
1024 pci_free_consistent(dev->pdev, QUERY_DEV_LIM_OUT_SIZE, outbox, outdma);
1025 return err;
1026}
1027
1028int mthca_QUERY_ADAPTER(struct mthca_dev *dev,
1029 struct mthca_adapter *adapter, u8 *status)
1030{
1031 u32 *outbox;
1032 dma_addr_t outdma;
1033 int err;
1034
1035#define QUERY_ADAPTER_OUT_SIZE 0x100
1036#define QUERY_ADAPTER_VENDOR_ID_OFFSET 0x00
1037#define QUERY_ADAPTER_DEVICE_ID_OFFSET 0x04
1038#define QUERY_ADAPTER_REVISION_ID_OFFSET 0x08
1039#define QUERY_ADAPTER_INTA_PIN_OFFSET 0x10
1040
1041 outbox = pci_alloc_consistent(dev->pdev, QUERY_ADAPTER_OUT_SIZE, &outdma);
1042 if (!outbox)
1043 return -ENOMEM;
1044
1045 err = mthca_cmd_box(dev, 0, outdma, 0, 0, CMD_QUERY_ADAPTER,
1046 CMD_TIME_CLASS_A, status);
1047
1048 if (err)
1049 goto out;
1050
1051 MTHCA_GET(adapter->vendor_id, outbox, QUERY_ADAPTER_VENDOR_ID_OFFSET);
1052 MTHCA_GET(adapter->device_id, outbox, QUERY_ADAPTER_DEVICE_ID_OFFSET);
1053 MTHCA_GET(adapter->revision_id, outbox, QUERY_ADAPTER_REVISION_ID_OFFSET);
1054 MTHCA_GET(adapter->inta_pin, outbox, QUERY_ADAPTER_INTA_PIN_OFFSET);
1055
1056out:
1057 pci_free_consistent(dev->pdev, QUERY_DEV_LIM_OUT_SIZE, outbox, outdma);
1058 return err;
1059}
1060
1061int mthca_INIT_HCA(struct mthca_dev *dev,
1062 struct mthca_init_hca_param *param,
1063 u8 *status)
1064{
1065 u32 *inbox;
1066 dma_addr_t indma;
1067 int err;
1068
1069#define INIT_HCA_IN_SIZE 0x200
1070#define INIT_HCA_FLAGS_OFFSET 0x014
1071#define INIT_HCA_QPC_OFFSET 0x020
1072#define INIT_HCA_QPC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x10)
1073#define INIT_HCA_LOG_QP_OFFSET (INIT_HCA_QPC_OFFSET + 0x17)
1074#define INIT_HCA_EEC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x20)
1075#define INIT_HCA_LOG_EEC_OFFSET (INIT_HCA_QPC_OFFSET + 0x27)
1076#define INIT_HCA_SRQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x28)
1077#define INIT_HCA_LOG_SRQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x2f)
1078#define INIT_HCA_CQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x30)
1079#define INIT_HCA_LOG_CQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x37)
1080#define INIT_HCA_EQPC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x40)
1081#define INIT_HCA_EEEC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x50)
1082#define INIT_HCA_EQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x60)
1083#define INIT_HCA_LOG_EQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x67)
1084#define INIT_HCA_RDB_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x70)
1085#define INIT_HCA_UDAV_OFFSET 0x0b0
1086#define INIT_HCA_UDAV_LKEY_OFFSET (INIT_HCA_UDAV_OFFSET + 0x0)
1087#define INIT_HCA_UDAV_PD_OFFSET (INIT_HCA_UDAV_OFFSET + 0x4)
1088#define INIT_HCA_MCAST_OFFSET 0x0c0
1089#define INIT_HCA_MC_BASE_OFFSET (INIT_HCA_MCAST_OFFSET + 0x00)
1090#define INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x12)
1091#define INIT_HCA_MC_HASH_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x16)
1092#define INIT_HCA_LOG_MC_TABLE_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x1b)
1093#define INIT_HCA_TPT_OFFSET 0x0f0
1094#define INIT_HCA_MPT_BASE_OFFSET (INIT_HCA_TPT_OFFSET + 0x00)
1095#define INIT_HCA_MTT_SEG_SZ_OFFSET (INIT_HCA_TPT_OFFSET + 0x09)
1096#define INIT_HCA_LOG_MPT_SZ_OFFSET (INIT_HCA_TPT_OFFSET + 0x0b)
1097#define INIT_HCA_MTT_BASE_OFFSET (INIT_HCA_TPT_OFFSET + 0x10)
1098#define INIT_HCA_UAR_OFFSET 0x120
1099#define INIT_HCA_UAR_BASE_OFFSET (INIT_HCA_UAR_OFFSET + 0x00)
1100#define INIT_HCA_UARC_SZ_OFFSET (INIT_HCA_UAR_OFFSET + 0x09)
1101#define INIT_HCA_LOG_UAR_SZ_OFFSET (INIT_HCA_UAR_OFFSET + 0x0a)
1102#define INIT_HCA_UAR_PAGE_SZ_OFFSET (INIT_HCA_UAR_OFFSET + 0x0b)
1103#define INIT_HCA_UAR_SCATCH_BASE_OFFSET (INIT_HCA_UAR_OFFSET + 0x10)
1104#define INIT_HCA_UAR_CTX_BASE_OFFSET (INIT_HCA_UAR_OFFSET + 0x18)
1105
1106 inbox = pci_alloc_consistent(dev->pdev, INIT_HCA_IN_SIZE, &indma);
1107 if (!inbox)
1108 return -ENOMEM;
1109
1110 memset(inbox, 0, INIT_HCA_IN_SIZE);
1111
1112#if defined(__LITTLE_ENDIAN)
1113 *(inbox + INIT_HCA_FLAGS_OFFSET / 4) &= ~cpu_to_be32(1 << 1);
1114#elif defined(__BIG_ENDIAN)
1115 *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 1);
1116#else
1117#error Host endianness not defined
1118#endif
1119 /* Check port for UD address vector: */
1120 *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1);
1121
1122 /* We leave wqe_quota, responder_exu, etc as 0 (default) */
1123
1124 /* QPC/EEC/CQC/EQC/RDB attributes */
1125
1126 MTHCA_PUT(inbox, param->qpc_base, INIT_HCA_QPC_BASE_OFFSET);
1127 MTHCA_PUT(inbox, param->log_num_qps, INIT_HCA_LOG_QP_OFFSET);
1128 MTHCA_PUT(inbox, param->eec_base, INIT_HCA_EEC_BASE_OFFSET);
1129 MTHCA_PUT(inbox, param->log_num_eecs, INIT_HCA_LOG_EEC_OFFSET);
1130 MTHCA_PUT(inbox, param->srqc_base, INIT_HCA_SRQC_BASE_OFFSET);
1131 MTHCA_PUT(inbox, param->log_num_srqs, INIT_HCA_LOG_SRQ_OFFSET);
1132 MTHCA_PUT(inbox, param->cqc_base, INIT_HCA_CQC_BASE_OFFSET);
1133 MTHCA_PUT(inbox, param->log_num_cqs, INIT_HCA_LOG_CQ_OFFSET);
1134 MTHCA_PUT(inbox, param->eqpc_base, INIT_HCA_EQPC_BASE_OFFSET);
1135 MTHCA_PUT(inbox, param->eeec_base, INIT_HCA_EEEC_BASE_OFFSET);
1136 MTHCA_PUT(inbox, param->eqc_base, INIT_HCA_EQC_BASE_OFFSET);
1137 MTHCA_PUT(inbox, param->log_num_eqs, INIT_HCA_LOG_EQ_OFFSET);
1138 MTHCA_PUT(inbox, param->rdb_base, INIT_HCA_RDB_BASE_OFFSET);
1139
1140 /* UD AV attributes */
1141
1142 /* multicast attributes */
1143
1144 MTHCA_PUT(inbox, param->mc_base, INIT_HCA_MC_BASE_OFFSET);
1145 MTHCA_PUT(inbox, param->log_mc_entry_sz, INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET);
1146 MTHCA_PUT(inbox, param->mc_hash_sz, INIT_HCA_MC_HASH_SZ_OFFSET);
1147 MTHCA_PUT(inbox, param->log_mc_table_sz, INIT_HCA_LOG_MC_TABLE_SZ_OFFSET);
1148
1149 /* TPT attributes */
1150
1151 MTHCA_PUT(inbox, param->mpt_base, INIT_HCA_MPT_BASE_OFFSET);
1152 if (dev->hca_type != ARBEL_NATIVE)
1153 MTHCA_PUT(inbox, param->mtt_seg_sz, INIT_HCA_MTT_SEG_SZ_OFFSET);
1154 MTHCA_PUT(inbox, param->log_mpt_sz, INIT_HCA_LOG_MPT_SZ_OFFSET);
1155 MTHCA_PUT(inbox, param->mtt_base, INIT_HCA_MTT_BASE_OFFSET);
1156
1157 /* UAR attributes */
1158 {
1159 u8 uar_page_sz = PAGE_SHIFT - 12;
1160 MTHCA_PUT(inbox, uar_page_sz, INIT_HCA_UAR_PAGE_SZ_OFFSET);
1161 }
1162
1163 MTHCA_PUT(inbox, param->uar_scratch_base, INIT_HCA_UAR_SCATCH_BASE_OFFSET);
1164
1165 if (dev->hca_type == ARBEL_NATIVE) {
1166 MTHCA_PUT(inbox, param->log_uarc_sz, INIT_HCA_UARC_SZ_OFFSET);
1167 MTHCA_PUT(inbox, param->log_uar_sz, INIT_HCA_LOG_UAR_SZ_OFFSET);
1168 MTHCA_PUT(inbox, param->uarc_base, INIT_HCA_UAR_CTX_BASE_OFFSET);
1169 }
1170
1171 err = mthca_cmd(dev, indma, 0, 0, CMD_INIT_HCA,
1172 HZ, status);
1173
1174 pci_free_consistent(dev->pdev, INIT_HCA_IN_SIZE, inbox, indma);
1175 return err;
1176}
1177
1178int mthca_INIT_IB(struct mthca_dev *dev,
1179 struct mthca_init_ib_param *param,
1180 int port, u8 *status)
1181{
1182 u32 *inbox;
1183 dma_addr_t indma;
1184 int err;
1185 u32 flags;
1186
1187#define INIT_IB_IN_SIZE 56
1188#define INIT_IB_FLAGS_OFFSET 0x00
1189#define INIT_IB_FLAG_SIG (1 << 18)
1190#define INIT_IB_FLAG_NG (1 << 17)
1191#define INIT_IB_FLAG_G0 (1 << 16)
1192#define INIT_IB_FLAG_1X (1 << 8)
1193#define INIT_IB_FLAG_4X (1 << 9)
1194#define INIT_IB_FLAG_12X (1 << 11)
1195#define INIT_IB_VL_SHIFT 4
1196#define INIT_IB_MTU_SHIFT 12
1197#define INIT_IB_MAX_GID_OFFSET 0x06
1198#define INIT_IB_MAX_PKEY_OFFSET 0x0a
1199#define INIT_IB_GUID0_OFFSET 0x10
1200#define INIT_IB_NODE_GUID_OFFSET 0x18
1201#define INIT_IB_SI_GUID_OFFSET 0x20
1202
1203 inbox = pci_alloc_consistent(dev->pdev, INIT_IB_IN_SIZE, &indma);
1204 if (!inbox)
1205 return -ENOMEM;
1206
1207 memset(inbox, 0, INIT_IB_IN_SIZE);
1208
1209 flags = 0;
1210 flags |= param->enable_1x ? INIT_IB_FLAG_1X : 0;
1211 flags |= param->enable_4x ? INIT_IB_FLAG_4X : 0;
1212 flags |= param->set_guid0 ? INIT_IB_FLAG_G0 : 0;
1213 flags |= param->set_node_guid ? INIT_IB_FLAG_NG : 0;
1214 flags |= param->set_si_guid ? INIT_IB_FLAG_SIG : 0;
1215 flags |= param->vl_cap << INIT_IB_VL_SHIFT;
1216 flags |= param->mtu_cap << INIT_IB_MTU_SHIFT;
1217 MTHCA_PUT(inbox, flags, INIT_IB_FLAGS_OFFSET);
1218
1219 MTHCA_PUT(inbox, param->gid_cap, INIT_IB_MAX_GID_OFFSET);
1220 MTHCA_PUT(inbox, param->pkey_cap, INIT_IB_MAX_PKEY_OFFSET);
1221 MTHCA_PUT(inbox, param->guid0, INIT_IB_GUID0_OFFSET);
1222 MTHCA_PUT(inbox, param->node_guid, INIT_IB_NODE_GUID_OFFSET);
1223 MTHCA_PUT(inbox, param->si_guid, INIT_IB_SI_GUID_OFFSET);
1224
1225 err = mthca_cmd(dev, indma, port, 0, CMD_INIT_IB,
1226 CMD_TIME_CLASS_A, status);
1227
1228 pci_free_consistent(dev->pdev, INIT_HCA_IN_SIZE, inbox, indma);
1229 return err;
1230}
1231
1232int mthca_CLOSE_IB(struct mthca_dev *dev, int port, u8 *status)
1233{
1234 return mthca_cmd(dev, 0, port, 0, CMD_CLOSE_IB, HZ, status);
1235}
1236
1237int mthca_CLOSE_HCA(struct mthca_dev *dev, int panic, u8 *status)
1238{
1239 return mthca_cmd(dev, 0, 0, panic, CMD_CLOSE_HCA, HZ, status);
1240}
1241
1242int mthca_SET_IB(struct mthca_dev *dev, struct mthca_set_ib_param *param,
1243 int port, u8 *status)
1244{
1245 u32 *inbox;
1246 dma_addr_t indma;
1247 int err;
1248 u32 flags = 0;
1249
1250#define SET_IB_IN_SIZE 0x40
1251#define SET_IB_FLAGS_OFFSET 0x00
1252#define SET_IB_FLAG_SIG (1 << 18)
1253#define SET_IB_FLAG_RQK (1 << 0)
1254#define SET_IB_CAP_MASK_OFFSET 0x04
1255#define SET_IB_SI_GUID_OFFSET 0x08
1256
1257 inbox = pci_alloc_consistent(dev->pdev, SET_IB_IN_SIZE, &indma);
1258 if (!inbox)
1259 return -ENOMEM;
1260
1261 memset(inbox, 0, SET_IB_IN_SIZE);
1262
1263 flags |= param->set_si_guid ? SET_IB_FLAG_SIG : 0;
1264 flags |= param->reset_qkey_viol ? SET_IB_FLAG_RQK : 0;
1265 MTHCA_PUT(inbox, flags, SET_IB_FLAGS_OFFSET);
1266
1267 MTHCA_PUT(inbox, param->cap_mask, SET_IB_CAP_MASK_OFFSET);
1268 MTHCA_PUT(inbox, param->si_guid, SET_IB_SI_GUID_OFFSET);
1269
1270 err = mthca_cmd(dev, indma, port, 0, CMD_SET_IB,
1271 CMD_TIME_CLASS_B, status);
1272
1273 pci_free_consistent(dev->pdev, INIT_HCA_IN_SIZE, inbox, indma);
1274 return err;
1275}
1276
1277int mthca_MAP_ICM(struct mthca_dev *dev, struct mthca_icm *icm, u64 virt, u8 *status)
1278{
1279 return mthca_map_cmd(dev, CMD_MAP_ICM, icm, virt, status);
1280}
1281
1282int mthca_MAP_ICM_page(struct mthca_dev *dev, u64 dma_addr, u64 virt, u8 *status)
1283{
1284 u64 *inbox;
1285 dma_addr_t indma;
1286 int err;
1287
1288 inbox = pci_alloc_consistent(dev->pdev, 16, &indma);
1289 if (!inbox)
1290 return -ENOMEM;
1291
1292 inbox[0] = cpu_to_be64(virt);
1293 inbox[1] = cpu_to_be64(dma_addr);
1294
1295 err = mthca_cmd(dev, indma, 1, 0, CMD_MAP_ICM, CMD_TIME_CLASS_B, status);
1296
1297 pci_free_consistent(dev->pdev, 16, inbox, indma);
1298
1299 if (!err)
1300 mthca_dbg(dev, "Mapped page at %llx for ICM.\n",
1301 (unsigned long long) virt);
1302
1303 return err;
1304}
1305
1306int mthca_UNMAP_ICM(struct mthca_dev *dev, u64 virt, u32 page_count, u8 *status)
1307{
1308 mthca_dbg(dev, "Unmapping %d pages at %llx from ICM.\n",
1309 page_count, (unsigned long long) virt);
1310
1311 return mthca_cmd(dev, virt, page_count, 0, CMD_UNMAP_ICM, CMD_TIME_CLASS_B, status);
1312}
1313
1314int mthca_MAP_ICM_AUX(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status)
1315{
1316 return mthca_map_cmd(dev, CMD_MAP_ICM_AUX, icm, -1, status);
1317}
1318
1319int mthca_UNMAP_ICM_AUX(struct mthca_dev *dev, u8 *status)
1320{
1321 return mthca_cmd(dev, 0, 0, 0, CMD_UNMAP_ICM_AUX, CMD_TIME_CLASS_B, status);
1322}
1323
1324int mthca_SET_ICM_SIZE(struct mthca_dev *dev, u64 icm_size, u64 *aux_pages,
1325 u8 *status)
1326{
1327 int ret = mthca_cmd_imm(dev, icm_size, aux_pages, 0, 0, CMD_SET_ICM_SIZE,
1328 CMD_TIME_CLASS_A, status);
1329
1330 if (ret || status)
1331 return ret;
1332
1333 /*
1334 * Arbel page size is always 4 KB; round up number of system
1335 * pages needed.
1336 */
1337 *aux_pages = (*aux_pages + (1 << (PAGE_SHIFT - 12)) - 1) >> (PAGE_SHIFT - 12);
1338
1339 return 0;
1340}
1341
1342int mthca_SW2HW_MPT(struct mthca_dev *dev, void *mpt_entry,
1343 int mpt_index, u8 *status)
1344{
1345 dma_addr_t indma;
1346 int err;
1347
1348 indma = pci_map_single(dev->pdev, mpt_entry,
1349 MTHCA_MPT_ENTRY_SIZE,
1350 PCI_DMA_TODEVICE);
1351 if (pci_dma_mapping_error(indma))
1352 return -ENOMEM;
1353
1354 err = mthca_cmd(dev, indma, mpt_index, 0, CMD_SW2HW_MPT,
1355 CMD_TIME_CLASS_B, status);
1356
1357 pci_unmap_single(dev->pdev, indma,
1358 MTHCA_MPT_ENTRY_SIZE, PCI_DMA_TODEVICE);
1359 return err;
1360}
1361
1362int mthca_HW2SW_MPT(struct mthca_dev *dev, void *mpt_entry,
1363 int mpt_index, u8 *status)
1364{
1365 dma_addr_t outdma = 0;
1366 int err;
1367
1368 if (mpt_entry) {
1369 outdma = pci_map_single(dev->pdev, mpt_entry,
1370 MTHCA_MPT_ENTRY_SIZE,
1371 PCI_DMA_FROMDEVICE);
1372 if (pci_dma_mapping_error(outdma))
1373 return -ENOMEM;
1374 }
1375
1376 err = mthca_cmd_box(dev, 0, outdma, mpt_index, !mpt_entry,
1377 CMD_HW2SW_MPT,
1378 CMD_TIME_CLASS_B, status);
1379
1380 if (mpt_entry)
1381 pci_unmap_single(dev->pdev, outdma,
1382 MTHCA_MPT_ENTRY_SIZE,
1383 PCI_DMA_FROMDEVICE);
1384 return err;
1385}
1386
1387int mthca_WRITE_MTT(struct mthca_dev *dev, u64 *mtt_entry,
1388 int num_mtt, u8 *status)
1389{
1390 dma_addr_t indma;
1391 int err;
1392
1393 indma = pci_map_single(dev->pdev, mtt_entry,
1394 (num_mtt + 2) * 8,
1395 PCI_DMA_TODEVICE);
1396 if (pci_dma_mapping_error(indma))
1397 return -ENOMEM;
1398
1399 err = mthca_cmd(dev, indma, num_mtt, 0, CMD_WRITE_MTT,
1400 CMD_TIME_CLASS_B, status);
1401
1402 pci_unmap_single(dev->pdev, indma,
1403 (num_mtt + 2) * 8, PCI_DMA_TODEVICE);
1404 return err;
1405}
1406
1407int mthca_MAP_EQ(struct mthca_dev *dev, u64 event_mask, int unmap,
1408 int eq_num, u8 *status)
1409{
1410 mthca_dbg(dev, "%s mask %016llx for eqn %d\n",
1411 unmap ? "Clearing" : "Setting",
1412 (unsigned long long) event_mask, eq_num);
1413 return mthca_cmd(dev, event_mask, (unmap << 31) | eq_num,
1414 0, CMD_MAP_EQ, CMD_TIME_CLASS_B, status);
1415}
1416
1417int mthca_SW2HW_EQ(struct mthca_dev *dev, void *eq_context,
1418 int eq_num, u8 *status)
1419{
1420 dma_addr_t indma;
1421 int err;
1422
1423 indma = pci_map_single(dev->pdev, eq_context,
1424 MTHCA_EQ_CONTEXT_SIZE,
1425 PCI_DMA_TODEVICE);
1426 if (pci_dma_mapping_error(indma))
1427 return -ENOMEM;
1428
1429 err = mthca_cmd(dev, indma, eq_num, 0, CMD_SW2HW_EQ,
1430 CMD_TIME_CLASS_A, status);
1431
1432 pci_unmap_single(dev->pdev, indma,
1433 MTHCA_EQ_CONTEXT_SIZE, PCI_DMA_TODEVICE);
1434 return err;
1435}
1436
1437int mthca_HW2SW_EQ(struct mthca_dev *dev, void *eq_context,
1438 int eq_num, u8 *status)
1439{
1440 dma_addr_t outdma = 0;
1441 int err;
1442
1443 outdma = pci_map_single(dev->pdev, eq_context,
1444 MTHCA_EQ_CONTEXT_SIZE,
1445 PCI_DMA_FROMDEVICE);
1446 if (pci_dma_mapping_error(outdma))
1447 return -ENOMEM;
1448
1449 err = mthca_cmd_box(dev, 0, outdma, eq_num, 0,
1450 CMD_HW2SW_EQ,
1451 CMD_TIME_CLASS_A, status);
1452
1453 pci_unmap_single(dev->pdev, outdma,
1454 MTHCA_EQ_CONTEXT_SIZE,
1455 PCI_DMA_FROMDEVICE);
1456 return err;
1457}
1458
1459int mthca_SW2HW_CQ(struct mthca_dev *dev, void *cq_context,
1460 int cq_num, u8 *status)
1461{
1462 dma_addr_t indma;
1463 int err;
1464
1465 indma = pci_map_single(dev->pdev, cq_context,
1466 MTHCA_CQ_CONTEXT_SIZE,
1467 PCI_DMA_TODEVICE);
1468 if (pci_dma_mapping_error(indma))
1469 return -ENOMEM;
1470
1471 err = mthca_cmd(dev, indma, cq_num, 0, CMD_SW2HW_CQ,
1472 CMD_TIME_CLASS_A, status);
1473
1474 pci_unmap_single(dev->pdev, indma,
1475 MTHCA_CQ_CONTEXT_SIZE, PCI_DMA_TODEVICE);
1476 return err;
1477}
1478
1479int mthca_HW2SW_CQ(struct mthca_dev *dev, void *cq_context,
1480 int cq_num, u8 *status)
1481{
1482 dma_addr_t outdma = 0;
1483 int err;
1484
1485 outdma = pci_map_single(dev->pdev, cq_context,
1486 MTHCA_CQ_CONTEXT_SIZE,
1487 PCI_DMA_FROMDEVICE);
1488 if (pci_dma_mapping_error(outdma))
1489 return -ENOMEM;
1490
1491 err = mthca_cmd_box(dev, 0, outdma, cq_num, 0,
1492 CMD_HW2SW_CQ,
1493 CMD_TIME_CLASS_A, status);
1494
1495 pci_unmap_single(dev->pdev, outdma,
1496 MTHCA_CQ_CONTEXT_SIZE,
1497 PCI_DMA_FROMDEVICE);
1498 return err;
1499}
1500
1501int mthca_MODIFY_QP(struct mthca_dev *dev, int trans, u32 num,
1502 int is_ee, void *qp_context, u32 optmask,
1503 u8 *status)
1504{
1505 static const u16 op[] = {
1506 [MTHCA_TRANS_RST2INIT] = CMD_RST2INIT_QPEE,
1507 [MTHCA_TRANS_INIT2INIT] = CMD_INIT2INIT_QPEE,
1508 [MTHCA_TRANS_INIT2RTR] = CMD_INIT2RTR_QPEE,
1509 [MTHCA_TRANS_RTR2RTS] = CMD_RTR2RTS_QPEE,
1510 [MTHCA_TRANS_RTS2RTS] = CMD_RTS2RTS_QPEE,
1511 [MTHCA_TRANS_SQERR2RTS] = CMD_SQERR2RTS_QPEE,
1512 [MTHCA_TRANS_ANY2ERR] = CMD_2ERR_QPEE,
1513 [MTHCA_TRANS_RTS2SQD] = CMD_RTS2SQD_QPEE,
1514 [MTHCA_TRANS_SQD2SQD] = CMD_SQD2SQD_QPEE,
1515 [MTHCA_TRANS_SQD2RTS] = CMD_SQD2RTS_QPEE,
1516 [MTHCA_TRANS_ANY2RST] = CMD_ERR2RST_QPEE
1517 };
1518 u8 op_mod = 0;
1519
1520 dma_addr_t indma;
1521 int err;
1522
1523 if (trans < 0 || trans >= ARRAY_SIZE(op))
1524 return -EINVAL;
1525
1526 if (trans == MTHCA_TRANS_ANY2RST) {
1527 indma = 0;
1528 op_mod = 3; /* don't write outbox, any->reset */
1529
1530 /* For debugging */
1531 qp_context = pci_alloc_consistent(dev->pdev, MTHCA_QP_CONTEXT_SIZE,
1532 &indma);
1533 op_mod = 2; /* write outbox, any->reset */
1534 } else {
1535 indma = pci_map_single(dev->pdev, qp_context,
1536 MTHCA_QP_CONTEXT_SIZE,
1537 PCI_DMA_TODEVICE);
1538 if (pci_dma_mapping_error(indma))
1539 return -ENOMEM;
1540
1541 if (0) {
1542 int i;
1543 mthca_dbg(dev, "Dumping QP context:\n");
1544 printk(" opt param mask: %08x\n", be32_to_cpup(qp_context));
1545 for (i = 0; i < 0x100 / 4; ++i) {
1546 if (i % 8 == 0)
1547 printk(" [%02x] ", i * 4);
1548 printk(" %08x", be32_to_cpu(((u32 *) qp_context)[i + 2]));
1549 if ((i + 1) % 8 == 0)
1550 printk("\n");
1551 }
1552 }
1553 }
1554
1555 if (trans == MTHCA_TRANS_ANY2RST) {
1556 err = mthca_cmd_box(dev, 0, indma, (!!is_ee << 24) | num,
1557 op_mod, op[trans], CMD_TIME_CLASS_C, status);
1558
1559 if (0) {
1560 int i;
1561 mthca_dbg(dev, "Dumping QP context:\n");
1562 printk(" %08x\n", be32_to_cpup(qp_context));
1563 for (i = 0; i < 0x100 / 4; ++i) {
1564 if (i % 8 == 0)
1565 printk("[%02x] ", i * 4);
1566 printk(" %08x", be32_to_cpu(((u32 *) qp_context)[i + 2]));
1567 if ((i + 1) % 8 == 0)
1568 printk("\n");
1569 }
1570 }
1571
1572 } else
1573 err = mthca_cmd(dev, indma, (!!is_ee << 24) | num,
1574 op_mod, op[trans], CMD_TIME_CLASS_C, status);
1575
1576 if (trans != MTHCA_TRANS_ANY2RST)
1577 pci_unmap_single(dev->pdev, indma,
1578 MTHCA_QP_CONTEXT_SIZE, PCI_DMA_TODEVICE);
1579 else
1580 pci_free_consistent(dev->pdev, MTHCA_QP_CONTEXT_SIZE,
1581 qp_context, indma);
1582 return err;
1583}
1584
1585int mthca_QUERY_QP(struct mthca_dev *dev, u32 num, int is_ee,
1586 void *qp_context, u8 *status)
1587{
1588 dma_addr_t outdma = 0;
1589 int err;
1590
1591 outdma = pci_map_single(dev->pdev, qp_context,
1592 MTHCA_QP_CONTEXT_SIZE,
1593 PCI_DMA_FROMDEVICE);
1594 if (pci_dma_mapping_error(outdma))
1595 return -ENOMEM;
1596
1597 err = mthca_cmd_box(dev, 0, outdma, (!!is_ee << 24) | num, 0,
1598 CMD_QUERY_QPEE,
1599 CMD_TIME_CLASS_A, status);
1600
1601 pci_unmap_single(dev->pdev, outdma,
1602 MTHCA_QP_CONTEXT_SIZE,
1603 PCI_DMA_FROMDEVICE);
1604 return err;
1605}
1606
1607int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn,
1608 u8 *status)
1609{
1610 u8 op_mod;
1611
1612 switch (type) {
1613 case IB_QPT_SMI:
1614 op_mod = 0;
1615 break;
1616 case IB_QPT_GSI:
1617 op_mod = 1;
1618 break;
1619 case IB_QPT_RAW_IPV6:
1620 op_mod = 2;
1621 break;
1622 case IB_QPT_RAW_ETY:
1623 op_mod = 3;
1624 break;
1625 default:
1626 return -EINVAL;
1627 }
1628
1629 return mthca_cmd(dev, 0, qpn, op_mod, CMD_CONF_SPECIAL_QP,
1630 CMD_TIME_CLASS_B, status);
1631}
1632
1633int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey,
1634 int port, struct ib_wc* in_wc, struct ib_grh* in_grh,
1635 void *in_mad, void *response_mad, u8 *status)
1636{
1637 void *box;
1638 dma_addr_t dma;
1639 int err;
1640 u32 in_modifier = port;
1641 u8 op_modifier = 0;
1642
1643#define MAD_IFC_BOX_SIZE 0x400
1644#define MAD_IFC_MY_QPN_OFFSET 0x100
1645#define MAD_IFC_RQPN_OFFSET 0x104
1646#define MAD_IFC_SL_OFFSET 0x108
1647#define MAD_IFC_G_PATH_OFFSET 0x109
1648#define MAD_IFC_RLID_OFFSET 0x10a
1649#define MAD_IFC_PKEY_OFFSET 0x10e
1650#define MAD_IFC_GRH_OFFSET 0x140
1651
1652 box = pci_alloc_consistent(dev->pdev, MAD_IFC_BOX_SIZE, &dma);
1653 if (!box)
1654 return -ENOMEM;
1655
1656 memcpy(box, in_mad, 256);
1657
1658 /*
1659 * Key check traps can't be generated unless we have in_wc to
1660 * tell us where to send the trap.
1661 */
1662 if (ignore_mkey || !in_wc)
1663 op_modifier |= 0x1;
1664 if (ignore_bkey || !in_wc)
1665 op_modifier |= 0x2;
1666
1667 if (in_wc) {
1668 u8 val;
1669
1670 memset(box + 256, 0, 256);
1671
1672 MTHCA_PUT(box, in_wc->qp_num, MAD_IFC_MY_QPN_OFFSET);
1673 MTHCA_PUT(box, in_wc->src_qp, MAD_IFC_RQPN_OFFSET);
1674
1675 val = in_wc->sl << 4;
1676 MTHCA_PUT(box, val, MAD_IFC_SL_OFFSET);
1677
1678 val = in_wc->dlid_path_bits |
1679 (in_wc->wc_flags & IB_WC_GRH ? 0x80 : 0);
1680 MTHCA_PUT(box, val, MAD_IFC_GRH_OFFSET);
1681
1682 MTHCA_PUT(box, in_wc->slid, MAD_IFC_RLID_OFFSET);
1683 MTHCA_PUT(box, in_wc->pkey_index, MAD_IFC_PKEY_OFFSET);
1684
1685 if (in_grh)
1686 memcpy((u8 *) box + MAD_IFC_GRH_OFFSET, in_grh, 40);
1687
1688 op_modifier |= 0x10;
1689
1690 in_modifier |= in_wc->slid << 16;
1691 }
1692
1693 err = mthca_cmd_box(dev, dma, dma + 512, in_modifier, op_modifier,
1694 CMD_MAD_IFC, CMD_TIME_CLASS_C, status);
1695
1696 if (!err && !*status)
1697 memcpy(response_mad, box + 512, 256);
1698
1699 pci_free_consistent(dev->pdev, MAD_IFC_BOX_SIZE, box, dma);
1700 return err;
1701}
1702
1703int mthca_READ_MGM(struct mthca_dev *dev, int index, void *mgm,
1704 u8 *status)
1705{
1706 dma_addr_t outdma = 0;
1707 int err;
1708
1709 outdma = pci_map_single(dev->pdev, mgm,
1710 MTHCA_MGM_ENTRY_SIZE,
1711 PCI_DMA_FROMDEVICE);
1712 if (pci_dma_mapping_error(outdma))
1713 return -ENOMEM;
1714
1715 err = mthca_cmd_box(dev, 0, outdma, index, 0,
1716 CMD_READ_MGM,
1717 CMD_TIME_CLASS_A, status);
1718
1719 pci_unmap_single(dev->pdev, outdma,
1720 MTHCA_MGM_ENTRY_SIZE,
1721 PCI_DMA_FROMDEVICE);
1722 return err;
1723}
1724
1725int mthca_WRITE_MGM(struct mthca_dev *dev, int index, void *mgm,
1726 u8 *status)
1727{
1728 dma_addr_t indma;
1729 int err;
1730
1731 indma = pci_map_single(dev->pdev, mgm,
1732 MTHCA_MGM_ENTRY_SIZE,
1733 PCI_DMA_TODEVICE);
1734 if (pci_dma_mapping_error(indma))
1735 return -ENOMEM;
1736
1737 err = mthca_cmd(dev, indma, index, 0, CMD_WRITE_MGM,
1738 CMD_TIME_CLASS_A, status);
1739
1740 pci_unmap_single(dev->pdev, indma,
1741 MTHCA_MGM_ENTRY_SIZE, PCI_DMA_TODEVICE);
1742 return err;
1743}
1744
1745int mthca_MGID_HASH(struct mthca_dev *dev, void *gid, u16 *hash,
1746 u8 *status)
1747{
1748 dma_addr_t indma;
1749 u64 imm;
1750 int err;
1751
1752 indma = pci_map_single(dev->pdev, gid, 16, PCI_DMA_TODEVICE);
1753 if (pci_dma_mapping_error(indma))
1754 return -ENOMEM;
1755
1756 err = mthca_cmd_imm(dev, indma, &imm, 0, 0, CMD_MGID_HASH,
1757 CMD_TIME_CLASS_A, status);
1758 *hash = imm;
1759
1760 pci_unmap_single(dev->pdev, indma, 16, PCI_DMA_TODEVICE);
1761 return err;
1762}
1763
1764int mthca_NOP(struct mthca_dev *dev, u8 *status)
1765{
1766 return mthca_cmd(dev, 0, 0x1f, 0, CMD_NOP, msecs_to_jiffies(100), status);
1767}
diff --git a/drivers/infiniband/hw/mthca/mthca_cmd.h b/drivers/infiniband/hw/mthca/mthca_cmd.h
new file mode 100644
index 000000000000..a8bc6aa36ff1
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_cmd.h
@@ -0,0 +1,310 @@
1/*
2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_cmd.h 1349 2004-12-16 21:09:43Z roland $
33 */
34
35#ifndef MTHCA_CMD_H
36#define MTHCA_CMD_H
37
38#include <ib_verbs.h>
39
40#define MTHCA_CMD_MAILBOX_ALIGN 16UL
41#define MTHCA_CMD_MAILBOX_EXTRA (MTHCA_CMD_MAILBOX_ALIGN - 1)
42
43enum {
44 /* command completed successfully: */
45 MTHCA_CMD_STAT_OK = 0x00,
46 /* Internal error (such as a bus error) occurred while processing command: */
47 MTHCA_CMD_STAT_INTERNAL_ERR = 0x01,
48 /* Operation/command not supported or opcode modifier not supported: */
49 MTHCA_CMD_STAT_BAD_OP = 0x02,
50 /* Parameter not supported or parameter out of range: */
51 MTHCA_CMD_STAT_BAD_PARAM = 0x03,
52 /* System not enabled or bad system state: */
53 MTHCA_CMD_STAT_BAD_SYS_STATE = 0x04,
54 /* Attempt to access reserved or unallocaterd resource: */
55 MTHCA_CMD_STAT_BAD_RESOURCE = 0x05,
56 /* Requested resource is currently executing a command, or is otherwise busy: */
57 MTHCA_CMD_STAT_RESOURCE_BUSY = 0x06,
58 /* memory error: */
59 MTHCA_CMD_STAT_DDR_MEM_ERR = 0x07,
60 /* Required capability exceeds device limits: */
61 MTHCA_CMD_STAT_EXCEED_LIM = 0x08,
62 /* Resource is not in the appropriate state or ownership: */
63 MTHCA_CMD_STAT_BAD_RES_STATE = 0x09,
64 /* Index out of range: */
65 MTHCA_CMD_STAT_BAD_INDEX = 0x0a,
66 /* FW image corrupted: */
67 MTHCA_CMD_STAT_BAD_NVMEM = 0x0b,
68 /* Attempt to modify a QP/EE which is not in the presumed state: */
69 MTHCA_CMD_STAT_BAD_QPEE_STATE = 0x10,
70 /* Bad segment parameters (Address/Size): */
71 MTHCA_CMD_STAT_BAD_SEG_PARAM = 0x20,
72 /* Memory Region has Memory Windows bound to: */
73 MTHCA_CMD_STAT_REG_BOUND = 0x21,
74 /* HCA local attached memory not present: */
75 MTHCA_CMD_STAT_LAM_NOT_PRE = 0x22,
76 /* Bad management packet (silently discarded): */
77 MTHCA_CMD_STAT_BAD_PKT = 0x30,
78 /* More outstanding CQEs in CQ than new CQ size: */
79 MTHCA_CMD_STAT_BAD_SIZE = 0x40
80};
81
82enum {
83 MTHCA_TRANS_INVALID = 0,
84 MTHCA_TRANS_RST2INIT,
85 MTHCA_TRANS_INIT2INIT,
86 MTHCA_TRANS_INIT2RTR,
87 MTHCA_TRANS_RTR2RTS,
88 MTHCA_TRANS_RTS2RTS,
89 MTHCA_TRANS_SQERR2RTS,
90 MTHCA_TRANS_ANY2ERR,
91 MTHCA_TRANS_RTS2SQD,
92 MTHCA_TRANS_SQD2SQD,
93 MTHCA_TRANS_SQD2RTS,
94 MTHCA_TRANS_ANY2RST,
95};
96
97enum {
98 DEV_LIM_FLAG_RC = 1 << 0,
99 DEV_LIM_FLAG_UC = 1 << 1,
100 DEV_LIM_FLAG_UD = 1 << 2,
101 DEV_LIM_FLAG_RD = 1 << 3,
102 DEV_LIM_FLAG_RAW_IPV6 = 1 << 4,
103 DEV_LIM_FLAG_RAW_ETHER = 1 << 5,
104 DEV_LIM_FLAG_SRQ = 1 << 6,
105 DEV_LIM_FLAG_BAD_PKEY_CNTR = 1 << 8,
106 DEV_LIM_FLAG_BAD_QKEY_CNTR = 1 << 9,
107 DEV_LIM_FLAG_MW = 1 << 16,
108 DEV_LIM_FLAG_AUTO_PATH_MIG = 1 << 17,
109 DEV_LIM_FLAG_ATOMIC = 1 << 18,
110 DEV_LIM_FLAG_RAW_MULTI = 1 << 19,
111 DEV_LIM_FLAG_UD_AV_PORT_ENFORCE = 1 << 20,
112 DEV_LIM_FLAG_UD_MULTI = 1 << 21,
113};
114
115struct mthca_dev_lim {
116 int max_srq_sz;
117 int max_qp_sz;
118 int reserved_qps;
119 int max_qps;
120 int reserved_srqs;
121 int max_srqs;
122 int reserved_eecs;
123 int max_eecs;
124 int max_cq_sz;
125 int reserved_cqs;
126 int max_cqs;
127 int max_mpts;
128 int reserved_eqs;
129 int max_eqs;
130 int reserved_mtts;
131 int max_mrw_sz;
132 int reserved_mrws;
133 int max_mtt_seg;
134 int max_requester_per_qp;
135 int max_responder_per_qp;
136 int max_rdma_global;
137 int local_ca_ack_delay;
138 int max_mtu;
139 int max_port_width;
140 int max_vl;
141 int num_ports;
142 int max_gids;
143 int max_pkeys;
144 u32 flags;
145 int reserved_uars;
146 int uar_size;
147 int min_page_sz;
148 int max_sg;
149 int max_desc_sz;
150 int max_qp_per_mcg;
151 int reserved_mgms;
152 int max_mcgs;
153 int reserved_pds;
154 int max_pds;
155 int reserved_rdds;
156 int max_rdds;
157 int eec_entry_sz;
158 int qpc_entry_sz;
159 int eeec_entry_sz;
160 int eqpc_entry_sz;
161 int eqc_entry_sz;
162 int cqc_entry_sz;
163 int srq_entry_sz;
164 int uar_scratch_entry_sz;
165 int mtt_seg_sz;
166 int mpt_entry_sz;
167 union {
168 struct {
169 int max_avs;
170 } tavor;
171 struct {
172 int resize_srq;
173 int max_pbl_sz;
174 u8 bmme_flags;
175 u32 reserved_lkey;
176 int lam_required;
177 u64 max_icm_sz;
178 } arbel;
179 } hca;
180};
181
182struct mthca_adapter {
183 u32 vendor_id;
184 u32 device_id;
185 u32 revision_id;
186 u8 inta_pin;
187};
188
189struct mthca_init_hca_param {
190 u64 qpc_base;
191 u64 eec_base;
192 u64 srqc_base;
193 u64 cqc_base;
194 u64 eqpc_base;
195 u64 eeec_base;
196 u64 eqc_base;
197 u64 rdb_base;
198 u64 mc_base;
199 u64 mpt_base;
200 u64 mtt_base;
201 u64 uar_scratch_base;
202 u64 uarc_base;
203 u16 log_mc_entry_sz;
204 u16 mc_hash_sz;
205 u8 log_num_qps;
206 u8 log_num_eecs;
207 u8 log_num_srqs;
208 u8 log_num_cqs;
209 u8 log_num_eqs;
210 u8 log_mc_table_sz;
211 u8 mtt_seg_sz;
212 u8 log_mpt_sz;
213 u8 log_uar_sz;
214 u8 log_uarc_sz;
215};
216
217struct mthca_init_ib_param {
218 int enable_1x;
219 int enable_4x;
220 int vl_cap;
221 int mtu_cap;
222 u16 gid_cap;
223 u16 pkey_cap;
224 int set_guid0;
225 u64 guid0;
226 int set_node_guid;
227 u64 node_guid;
228 int set_si_guid;
229 u64 si_guid;
230};
231
232struct mthca_set_ib_param {
233 int set_si_guid;
234 int reset_qkey_viol;
235 u64 si_guid;
236 u32 cap_mask;
237};
238
239int mthca_cmd_use_events(struct mthca_dev *dev);
240void mthca_cmd_use_polling(struct mthca_dev *dev);
241void mthca_cmd_event(struct mthca_dev *dev, u16 token,
242 u8 status, u64 out_param);
243
244int mthca_SYS_EN(struct mthca_dev *dev, u8 *status);
245int mthca_SYS_DIS(struct mthca_dev *dev, u8 *status);
246int mthca_MAP_FA(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status);
247int mthca_UNMAP_FA(struct mthca_dev *dev, u8 *status);
248int mthca_RUN_FW(struct mthca_dev *dev, u8 *status);
249int mthca_QUERY_FW(struct mthca_dev *dev, u8 *status);
250int mthca_ENABLE_LAM(struct mthca_dev *dev, u8 *status);
251int mthca_DISABLE_LAM(struct mthca_dev *dev, u8 *status);
252int mthca_QUERY_DDR(struct mthca_dev *dev, u8 *status);
253int mthca_QUERY_DEV_LIM(struct mthca_dev *dev,
254 struct mthca_dev_lim *dev_lim, u8 *status);
255int mthca_QUERY_ADAPTER(struct mthca_dev *dev,
256 struct mthca_adapter *adapter, u8 *status);
257int mthca_INIT_HCA(struct mthca_dev *dev,
258 struct mthca_init_hca_param *param,
259 u8 *status);
260int mthca_INIT_IB(struct mthca_dev *dev,
261 struct mthca_init_ib_param *param,
262 int port, u8 *status);
263int mthca_CLOSE_IB(struct mthca_dev *dev, int port, u8 *status);
264int mthca_CLOSE_HCA(struct mthca_dev *dev, int panic, u8 *status);
265int mthca_SET_IB(struct mthca_dev *dev, struct mthca_set_ib_param *param,
266 int port, u8 *status);
267int mthca_MAP_ICM(struct mthca_dev *dev, struct mthca_icm *icm, u64 virt, u8 *status);
268int mthca_MAP_ICM_page(struct mthca_dev *dev, u64 dma_addr, u64 virt, u8 *status);
269int mthca_UNMAP_ICM(struct mthca_dev *dev, u64 virt, u32 page_count, u8 *status);
270int mthca_MAP_ICM_AUX(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status);
271int mthca_UNMAP_ICM_AUX(struct mthca_dev *dev, u8 *status);
272int mthca_SET_ICM_SIZE(struct mthca_dev *dev, u64 icm_size, u64 *aux_pages,
273 u8 *status);
274int mthca_SW2HW_MPT(struct mthca_dev *dev, void *mpt_entry,
275 int mpt_index, u8 *status);
276int mthca_HW2SW_MPT(struct mthca_dev *dev, void *mpt_entry,
277 int mpt_index, u8 *status);
278int mthca_WRITE_MTT(struct mthca_dev *dev, u64 *mtt_entry,
279 int num_mtt, u8 *status);
280int mthca_MAP_EQ(struct mthca_dev *dev, u64 event_mask, int unmap,
281 int eq_num, u8 *status);
282int mthca_SW2HW_EQ(struct mthca_dev *dev, void *eq_context,
283 int eq_num, u8 *status);
284int mthca_HW2SW_EQ(struct mthca_dev *dev, void *eq_context,
285 int eq_num, u8 *status);
286int mthca_SW2HW_CQ(struct mthca_dev *dev, void *cq_context,
287 int cq_num, u8 *status);
288int mthca_HW2SW_CQ(struct mthca_dev *dev, void *cq_context,
289 int cq_num, u8 *status);
290int mthca_MODIFY_QP(struct mthca_dev *dev, int trans, u32 num,
291 int is_ee, void *qp_context, u32 optmask,
292 u8 *status);
293int mthca_QUERY_QP(struct mthca_dev *dev, u32 num, int is_ee,
294 void *qp_context, u8 *status);
295int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn,
296 u8 *status);
297int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey,
298 int port, struct ib_wc* in_wc, struct ib_grh* in_grh,
299 void *in_mad, void *response_mad, u8 *status);
300int mthca_READ_MGM(struct mthca_dev *dev, int index, void *mgm,
301 u8 *status);
302int mthca_WRITE_MGM(struct mthca_dev *dev, int index, void *mgm,
303 u8 *status);
304int mthca_MGID_HASH(struct mthca_dev *dev, void *gid, u16 *hash,
305 u8 *status);
306int mthca_NOP(struct mthca_dev *dev, u8 *status);
307
308#define MAILBOX_ALIGN(x) ((void *) ALIGN((unsigned long) (x), MTHCA_CMD_MAILBOX_ALIGN))
309
310#endif /* MTHCA_CMD_H */
diff --git a/drivers/infiniband/hw/mthca/mthca_config_reg.h b/drivers/infiniband/hw/mthca/mthca_config_reg.h
new file mode 100644
index 000000000000..b4bfbbfe2c3d
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_config_reg.h
@@ -0,0 +1,51 @@
1/*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_config_reg.h 1349 2004-12-16 21:09:43Z roland $
33 */
34
35#ifndef MTHCA_CONFIG_REG_H
36#define MTHCA_CONFIG_REG_H
37
38#include <asm/page.h>
39
40#define MTHCA_HCR_BASE 0x80680
41#define MTHCA_HCR_SIZE 0x0001c
42#define MTHCA_ECR_BASE 0x80700
43#define MTHCA_ECR_SIZE 0x00008
44#define MTHCA_ECR_CLR_BASE 0x80708
45#define MTHCA_ECR_CLR_SIZE 0x00008
46#define MTHCA_MAP_ECR_SIZE (MTHCA_ECR_SIZE + MTHCA_ECR_CLR_SIZE)
47#define MTHCA_CLR_INT_BASE 0xf00d8
48#define MTHCA_CLR_INT_SIZE 0x00008
49#define MTHCA_EQ_SET_CI_SIZE (8 * 32)
50
51#endif /* MTHCA_CONFIG_REG_H */
diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c
new file mode 100644
index 000000000000..5dead2df7eb0
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_cq.c
@@ -0,0 +1,918 @@
1/*
2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_cq.c 1369 2004-12-20 16:17:07Z roland $
33 */
34
35#include <linux/init.h>
36#include <linux/hardirq.h>
37
38#include <ib_pack.h>
39
40#include "mthca_dev.h"
41#include "mthca_cmd.h"
42#include "mthca_memfree.h"
43
44enum {
45 MTHCA_MAX_DIRECT_CQ_SIZE = 4 * PAGE_SIZE
46};
47
48enum {
49 MTHCA_CQ_ENTRY_SIZE = 0x20
50};
51
52/*
53 * Must be packed because start is 64 bits but only aligned to 32 bits.
54 */
55struct mthca_cq_context {
56 u32 flags;
57 u64 start;
58 u32 logsize_usrpage;
59 u32 error_eqn; /* Tavor only */
60 u32 comp_eqn;
61 u32 pd;
62 u32 lkey;
63 u32 last_notified_index;
64 u32 solicit_producer_index;
65 u32 consumer_index;
66 u32 producer_index;
67 u32 cqn;
68 u32 ci_db; /* Arbel only */
69 u32 state_db; /* Arbel only */
70 u32 reserved;
71} __attribute__((packed));
72
73#define MTHCA_CQ_STATUS_OK ( 0 << 28)
74#define MTHCA_CQ_STATUS_OVERFLOW ( 9 << 28)
75#define MTHCA_CQ_STATUS_WRITE_FAIL (10 << 28)
76#define MTHCA_CQ_FLAG_TR ( 1 << 18)
77#define MTHCA_CQ_FLAG_OI ( 1 << 17)
78#define MTHCA_CQ_STATE_DISARMED ( 0 << 8)
79#define MTHCA_CQ_STATE_ARMED ( 1 << 8)
80#define MTHCA_CQ_STATE_ARMED_SOL ( 4 << 8)
81#define MTHCA_EQ_STATE_FIRED (10 << 8)
82
83enum {
84 MTHCA_ERROR_CQE_OPCODE_MASK = 0xfe
85};
86
87enum {
88 SYNDROME_LOCAL_LENGTH_ERR = 0x01,
89 SYNDROME_LOCAL_QP_OP_ERR = 0x02,
90 SYNDROME_LOCAL_EEC_OP_ERR = 0x03,
91 SYNDROME_LOCAL_PROT_ERR = 0x04,
92 SYNDROME_WR_FLUSH_ERR = 0x05,
93 SYNDROME_MW_BIND_ERR = 0x06,
94 SYNDROME_BAD_RESP_ERR = 0x10,
95 SYNDROME_LOCAL_ACCESS_ERR = 0x11,
96 SYNDROME_REMOTE_INVAL_REQ_ERR = 0x12,
97 SYNDROME_REMOTE_ACCESS_ERR = 0x13,
98 SYNDROME_REMOTE_OP_ERR = 0x14,
99 SYNDROME_RETRY_EXC_ERR = 0x15,
100 SYNDROME_RNR_RETRY_EXC_ERR = 0x16,
101 SYNDROME_LOCAL_RDD_VIOL_ERR = 0x20,
102 SYNDROME_REMOTE_INVAL_RD_REQ_ERR = 0x21,
103 SYNDROME_REMOTE_ABORTED_ERR = 0x22,
104 SYNDROME_INVAL_EECN_ERR = 0x23,
105 SYNDROME_INVAL_EEC_STATE_ERR = 0x24
106};
107
108struct mthca_cqe {
109 u32 my_qpn;
110 u32 my_ee;
111 u32 rqpn;
112 u16 sl_g_mlpath;
113 u16 rlid;
114 u32 imm_etype_pkey_eec;
115 u32 byte_cnt;
116 u32 wqe;
117 u8 opcode;
118 u8 is_send;
119 u8 reserved;
120 u8 owner;
121};
122
123struct mthca_err_cqe {
124 u32 my_qpn;
125 u32 reserved1[3];
126 u8 syndrome;
127 u8 reserved2;
128 u16 db_cnt;
129 u32 reserved3;
130 u32 wqe;
131 u8 opcode;
132 u8 reserved4[2];
133 u8 owner;
134};
135
136#define MTHCA_CQ_ENTRY_OWNER_SW (0 << 7)
137#define MTHCA_CQ_ENTRY_OWNER_HW (1 << 7)
138
139#define MTHCA_TAVOR_CQ_DB_INC_CI (1 << 24)
140#define MTHCA_TAVOR_CQ_DB_REQ_NOT (2 << 24)
141#define MTHCA_TAVOR_CQ_DB_REQ_NOT_SOL (3 << 24)
142#define MTHCA_TAVOR_CQ_DB_SET_CI (4 << 24)
143#define MTHCA_TAVOR_CQ_DB_REQ_NOT_MULT (5 << 24)
144
145#define MTHCA_ARBEL_CQ_DB_REQ_NOT_SOL (1 << 24)
146#define MTHCA_ARBEL_CQ_DB_REQ_NOT (2 << 24)
147#define MTHCA_ARBEL_CQ_DB_REQ_NOT_MULT (3 << 24)
148
149static inline struct mthca_cqe *get_cqe(struct mthca_cq *cq, int entry)
150{
151 if (cq->is_direct)
152 return cq->queue.direct.buf + (entry * MTHCA_CQ_ENTRY_SIZE);
153 else
154 return cq->queue.page_list[entry * MTHCA_CQ_ENTRY_SIZE / PAGE_SIZE].buf
155 + (entry * MTHCA_CQ_ENTRY_SIZE) % PAGE_SIZE;
156}
157
158static inline struct mthca_cqe *cqe_sw(struct mthca_cq *cq, int i)
159{
160 struct mthca_cqe *cqe = get_cqe(cq, i);
161 return MTHCA_CQ_ENTRY_OWNER_HW & cqe->owner ? NULL : cqe;
162}
163
164static inline struct mthca_cqe *next_cqe_sw(struct mthca_cq *cq)
165{
166 return cqe_sw(cq, cq->cons_index & cq->ibcq.cqe);
167}
168
169static inline void set_cqe_hw(struct mthca_cqe *cqe)
170{
171 cqe->owner = MTHCA_CQ_ENTRY_OWNER_HW;
172}
173
174/*
175 * incr is ignored in native Arbel (mem-free) mode, so cq->cons_index
176 * should be correct before calling update_cons_index().
177 */
178static inline void update_cons_index(struct mthca_dev *dev, struct mthca_cq *cq,
179 int incr)
180{
181 u32 doorbell[2];
182
183 if (dev->hca_type == ARBEL_NATIVE) {
184 *cq->set_ci_db = cpu_to_be32(cq->cons_index);
185 wmb();
186 } else {
187 doorbell[0] = cpu_to_be32(MTHCA_TAVOR_CQ_DB_INC_CI | cq->cqn);
188 doorbell[1] = cpu_to_be32(incr - 1);
189
190 mthca_write64(doorbell,
191 dev->kar + MTHCA_CQ_DOORBELL,
192 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
193 }
194}
195
196void mthca_cq_event(struct mthca_dev *dev, u32 cqn)
197{
198 struct mthca_cq *cq;
199
200 cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1));
201
202 if (!cq) {
203 mthca_warn(dev, "Completion event for bogus CQ %08x\n", cqn);
204 return;
205 }
206
207 ++cq->arm_sn;
208
209 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
210}
211
212void mthca_cq_clean(struct mthca_dev *dev, u32 cqn, u32 qpn)
213{
214 struct mthca_cq *cq;
215 struct mthca_cqe *cqe;
216 int prod_index;
217 int nfreed = 0;
218
219 spin_lock_irq(&dev->cq_table.lock);
220 cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1));
221 if (cq)
222 atomic_inc(&cq->refcount);
223 spin_unlock_irq(&dev->cq_table.lock);
224
225 if (!cq)
226 return;
227
228 spin_lock_irq(&cq->lock);
229
230 /*
231 * First we need to find the current producer index, so we
232 * know where to start cleaning from. It doesn't matter if HW
233 * adds new entries after this loop -- the QP we're worried
234 * about is already in RESET, so the new entries won't come
235 * from our QP and therefore don't need to be checked.
236 */
237 for (prod_index = cq->cons_index;
238 cqe_sw(cq, prod_index & cq->ibcq.cqe);
239 ++prod_index)
240 if (prod_index == cq->cons_index + cq->ibcq.cqe)
241 break;
242
243 if (0)
244 mthca_dbg(dev, "Cleaning QPN %06x from CQN %06x; ci %d, pi %d\n",
245 qpn, cqn, cq->cons_index, prod_index);
246
247 /*
248 * Now sweep backwards through the CQ, removing CQ entries
249 * that match our QP by copying older entries on top of them.
250 */
251 while (prod_index > cq->cons_index) {
252 cqe = get_cqe(cq, (prod_index - 1) & cq->ibcq.cqe);
253 if (cqe->my_qpn == cpu_to_be32(qpn))
254 ++nfreed;
255 else if (nfreed)
256 memcpy(get_cqe(cq, (prod_index - 1 + nfreed) &
257 cq->ibcq.cqe),
258 cqe,
259 MTHCA_CQ_ENTRY_SIZE);
260 --prod_index;
261 }
262
263 if (nfreed) {
264 wmb();
265 cq->cons_index += nfreed;
266 update_cons_index(dev, cq, nfreed);
267 }
268
269 spin_unlock_irq(&cq->lock);
270 if (atomic_dec_and_test(&cq->refcount))
271 wake_up(&cq->wait);
272}
273
274static int handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq,
275 struct mthca_qp *qp, int wqe_index, int is_send,
276 struct mthca_err_cqe *cqe,
277 struct ib_wc *entry, int *free_cqe)
278{
279 int err;
280 int dbd;
281 u32 new_wqe;
282
283 if (1 && cqe->syndrome != SYNDROME_WR_FLUSH_ERR) {
284 int j;
285
286 mthca_dbg(dev, "%x/%d: error CQE -> QPN %06x, WQE @ %08x\n",
287 cq->cqn, cq->cons_index, be32_to_cpu(cqe->my_qpn),
288 be32_to_cpu(cqe->wqe));
289
290 for (j = 0; j < 8; ++j)
291 printk(KERN_DEBUG " [%2x] %08x\n",
292 j * 4, be32_to_cpu(((u32 *) cqe)[j]));
293 }
294
295 /*
296 * For completions in error, only work request ID, status (and
297 * freed resource count for RD) have to be set.
298 */
299 switch (cqe->syndrome) {
300 case SYNDROME_LOCAL_LENGTH_ERR:
301 entry->status = IB_WC_LOC_LEN_ERR;
302 break;
303 case SYNDROME_LOCAL_QP_OP_ERR:
304 entry->status = IB_WC_LOC_QP_OP_ERR;
305 break;
306 case SYNDROME_LOCAL_EEC_OP_ERR:
307 entry->status = IB_WC_LOC_EEC_OP_ERR;
308 break;
309 case SYNDROME_LOCAL_PROT_ERR:
310 entry->status = IB_WC_LOC_PROT_ERR;
311 break;
312 case SYNDROME_WR_FLUSH_ERR:
313 entry->status = IB_WC_WR_FLUSH_ERR;
314 break;
315 case SYNDROME_MW_BIND_ERR:
316 entry->status = IB_WC_MW_BIND_ERR;
317 break;
318 case SYNDROME_BAD_RESP_ERR:
319 entry->status = IB_WC_BAD_RESP_ERR;
320 break;
321 case SYNDROME_LOCAL_ACCESS_ERR:
322 entry->status = IB_WC_LOC_ACCESS_ERR;
323 break;
324 case SYNDROME_REMOTE_INVAL_REQ_ERR:
325 entry->status = IB_WC_REM_INV_REQ_ERR;
326 break;
327 case SYNDROME_REMOTE_ACCESS_ERR:
328 entry->status = IB_WC_REM_ACCESS_ERR;
329 break;
330 case SYNDROME_REMOTE_OP_ERR:
331 entry->status = IB_WC_REM_OP_ERR;
332 break;
333 case SYNDROME_RETRY_EXC_ERR:
334 entry->status = IB_WC_RETRY_EXC_ERR;
335 break;
336 case SYNDROME_RNR_RETRY_EXC_ERR:
337 entry->status = IB_WC_RNR_RETRY_EXC_ERR;
338 break;
339 case SYNDROME_LOCAL_RDD_VIOL_ERR:
340 entry->status = IB_WC_LOC_RDD_VIOL_ERR;
341 break;
342 case SYNDROME_REMOTE_INVAL_RD_REQ_ERR:
343 entry->status = IB_WC_REM_INV_RD_REQ_ERR;
344 break;
345 case SYNDROME_REMOTE_ABORTED_ERR:
346 entry->status = IB_WC_REM_ABORT_ERR;
347 break;
348 case SYNDROME_INVAL_EECN_ERR:
349 entry->status = IB_WC_INV_EECN_ERR;
350 break;
351 case SYNDROME_INVAL_EEC_STATE_ERR:
352 entry->status = IB_WC_INV_EEC_STATE_ERR;
353 break;
354 default:
355 entry->status = IB_WC_GENERAL_ERR;
356 break;
357 }
358
359 err = mthca_free_err_wqe(dev, qp, is_send, wqe_index, &dbd, &new_wqe);
360 if (err)
361 return err;
362
363 /*
364 * If we're at the end of the WQE chain, or we've used up our
365 * doorbell count, free the CQE. Otherwise just update it for
366 * the next poll operation.
367 */
368 if (!(new_wqe & cpu_to_be32(0x3f)) || (!cqe->db_cnt && dbd))
369 return 0;
370
371 cqe->db_cnt = cpu_to_be16(be16_to_cpu(cqe->db_cnt) - dbd);
372 cqe->wqe = new_wqe;
373 cqe->syndrome = SYNDROME_WR_FLUSH_ERR;
374
375 *free_cqe = 0;
376
377 return 0;
378}
379
380static void dump_cqe(struct mthca_cqe *cqe)
381{
382 int j;
383
384 for (j = 0; j < 8; ++j)
385 printk(KERN_DEBUG " [%2x] %08x\n",
386 j * 4, be32_to_cpu(((u32 *) cqe)[j]));
387}
388
389static inline int mthca_poll_one(struct mthca_dev *dev,
390 struct mthca_cq *cq,
391 struct mthca_qp **cur_qp,
392 int *freed,
393 struct ib_wc *entry)
394{
395 struct mthca_wq *wq;
396 struct mthca_cqe *cqe;
397 int wqe_index;
398 int is_error;
399 int is_send;
400 int free_cqe = 1;
401 int err = 0;
402
403 cqe = next_cqe_sw(cq);
404 if (!cqe)
405 return -EAGAIN;
406
407 /*
408 * Make sure we read CQ entry contents after we've checked the
409 * ownership bit.
410 */
411 rmb();
412
413 if (0) {
414 mthca_dbg(dev, "%x/%d: CQE -> QPN %06x, WQE @ %08x\n",
415 cq->cqn, cq->cons_index, be32_to_cpu(cqe->my_qpn),
416 be32_to_cpu(cqe->wqe));
417
418 dump_cqe(cqe);
419 }
420
421 is_error = (cqe->opcode & MTHCA_ERROR_CQE_OPCODE_MASK) ==
422 MTHCA_ERROR_CQE_OPCODE_MASK;
423 is_send = is_error ? cqe->opcode & 0x01 : cqe->is_send & 0x80;
424
425 if (!*cur_qp || be32_to_cpu(cqe->my_qpn) != (*cur_qp)->qpn) {
426 /*
427 * We do not have to take the QP table lock here,
428 * because CQs will be locked while QPs are removed
429 * from the table.
430 */
431 *cur_qp = mthca_array_get(&dev->qp_table.qp,
432 be32_to_cpu(cqe->my_qpn) &
433 (dev->limits.num_qps - 1));
434 if (!*cur_qp) {
435 mthca_warn(dev, "CQ entry for unknown QP %06x\n",
436 be32_to_cpu(cqe->my_qpn) & 0xffffff);
437 err = -EINVAL;
438 goto out;
439 }
440 }
441
442 entry->qp_num = (*cur_qp)->qpn;
443
444 if (is_send) {
445 wq = &(*cur_qp)->sq;
446 wqe_index = ((be32_to_cpu(cqe->wqe) - (*cur_qp)->send_wqe_offset)
447 >> wq->wqe_shift);
448 entry->wr_id = (*cur_qp)->wrid[wqe_index +
449 (*cur_qp)->rq.max];
450 } else {
451 wq = &(*cur_qp)->rq;
452 wqe_index = be32_to_cpu(cqe->wqe) >> wq->wqe_shift;
453 entry->wr_id = (*cur_qp)->wrid[wqe_index];
454 }
455
456 if (wq->last_comp < wqe_index)
457 wq->tail += wqe_index - wq->last_comp;
458 else
459 wq->tail += wqe_index + wq->max - wq->last_comp;
460
461 wq->last_comp = wqe_index;
462
463 if (0)
464 mthca_dbg(dev, "%s completion for QP %06x, index %d (nr %d)\n",
465 is_send ? "Send" : "Receive",
466 (*cur_qp)->qpn, wqe_index, wq->max);
467
468 if (is_error) {
469 err = handle_error_cqe(dev, cq, *cur_qp, wqe_index, is_send,
470 (struct mthca_err_cqe *) cqe,
471 entry, &free_cqe);
472 goto out;
473 }
474
475 if (is_send) {
476 entry->opcode = IB_WC_SEND; /* XXX */
477 } else {
478 entry->byte_len = be32_to_cpu(cqe->byte_cnt);
479 switch (cqe->opcode & 0x1f) {
480 case IB_OPCODE_SEND_LAST_WITH_IMMEDIATE:
481 case IB_OPCODE_SEND_ONLY_WITH_IMMEDIATE:
482 entry->wc_flags = IB_WC_WITH_IMM;
483 entry->imm_data = cqe->imm_etype_pkey_eec;
484 entry->opcode = IB_WC_RECV;
485 break;
486 case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE:
487 case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE:
488 entry->wc_flags = IB_WC_WITH_IMM;
489 entry->imm_data = cqe->imm_etype_pkey_eec;
490 entry->opcode = IB_WC_RECV_RDMA_WITH_IMM;
491 break;
492 default:
493 entry->wc_flags = 0;
494 entry->opcode = IB_WC_RECV;
495 break;
496 }
497 entry->slid = be16_to_cpu(cqe->rlid);
498 entry->sl = be16_to_cpu(cqe->sl_g_mlpath) >> 12;
499 entry->src_qp = be32_to_cpu(cqe->rqpn) & 0xffffff;
500 entry->dlid_path_bits = be16_to_cpu(cqe->sl_g_mlpath) & 0x7f;
501 entry->pkey_index = be32_to_cpu(cqe->imm_etype_pkey_eec) >> 16;
502 entry->wc_flags |= be16_to_cpu(cqe->sl_g_mlpath) & 0x80 ?
503 IB_WC_GRH : 0;
504 }
505
506 entry->status = IB_WC_SUCCESS;
507
508 out:
509 if (likely(free_cqe)) {
510 set_cqe_hw(cqe);
511 ++(*freed);
512 ++cq->cons_index;
513 }
514
515 return err;
516}
517
518int mthca_poll_cq(struct ib_cq *ibcq, int num_entries,
519 struct ib_wc *entry)
520{
521 struct mthca_dev *dev = to_mdev(ibcq->device);
522 struct mthca_cq *cq = to_mcq(ibcq);
523 struct mthca_qp *qp = NULL;
524 unsigned long flags;
525 int err = 0;
526 int freed = 0;
527 int npolled;
528
529 spin_lock_irqsave(&cq->lock, flags);
530
531 for (npolled = 0; npolled < num_entries; ++npolled) {
532 err = mthca_poll_one(dev, cq, &qp,
533 &freed, entry + npolled);
534 if (err)
535 break;
536 }
537
538 if (freed) {
539 wmb();
540 update_cons_index(dev, cq, freed);
541 }
542
543 spin_unlock_irqrestore(&cq->lock, flags);
544
545 return err == 0 || err == -EAGAIN ? npolled : err;
546}
547
548int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify)
549{
550 u32 doorbell[2];
551
552 doorbell[0] = cpu_to_be32((notify == IB_CQ_SOLICITED ?
553 MTHCA_TAVOR_CQ_DB_REQ_NOT_SOL :
554 MTHCA_TAVOR_CQ_DB_REQ_NOT) |
555 to_mcq(cq)->cqn);
556 doorbell[1] = 0xffffffff;
557
558 mthca_write64(doorbell,
559 to_mdev(cq->device)->kar + MTHCA_CQ_DOORBELL,
560 MTHCA_GET_DOORBELL_LOCK(&to_mdev(cq->device)->doorbell_lock));
561
562 return 0;
563}
564
565int mthca_arbel_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify notify)
566{
567 struct mthca_cq *cq = to_mcq(ibcq);
568 u32 doorbell[2];
569 u32 sn;
570 u32 ci;
571
572 sn = cq->arm_sn & 3;
573 ci = cpu_to_be32(cq->cons_index);
574
575 doorbell[0] = ci;
576 doorbell[1] = cpu_to_be32((cq->cqn << 8) | (2 << 5) | (sn << 3) |
577 (notify == IB_CQ_SOLICITED ? 1 : 2));
578
579 mthca_write_db_rec(doorbell, cq->arm_db);
580
581 /*
582 * Make sure that the doorbell record in host memory is
583 * written before ringing the doorbell via PCI MMIO.
584 */
585 wmb();
586
587 doorbell[0] = cpu_to_be32((sn << 28) |
588 (notify == IB_CQ_SOLICITED ?
589 MTHCA_ARBEL_CQ_DB_REQ_NOT_SOL :
590 MTHCA_ARBEL_CQ_DB_REQ_NOT) |
591 cq->cqn);
592 doorbell[1] = ci;
593
594 mthca_write64(doorbell,
595 to_mdev(ibcq->device)->kar + MTHCA_CQ_DOORBELL,
596 MTHCA_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->doorbell_lock));
597
598 return 0;
599}
600
601static void mthca_free_cq_buf(struct mthca_dev *dev, struct mthca_cq *cq)
602{
603 int i;
604 int size;
605
606 if (cq->is_direct)
607 pci_free_consistent(dev->pdev,
608 (cq->ibcq.cqe + 1) * MTHCA_CQ_ENTRY_SIZE,
609 cq->queue.direct.buf,
610 pci_unmap_addr(&cq->queue.direct,
611 mapping));
612 else {
613 size = (cq->ibcq.cqe + 1) * MTHCA_CQ_ENTRY_SIZE;
614 for (i = 0; i < (size + PAGE_SIZE - 1) / PAGE_SIZE; ++i)
615 if (cq->queue.page_list[i].buf)
616 pci_free_consistent(dev->pdev, PAGE_SIZE,
617 cq->queue.page_list[i].buf,
618 pci_unmap_addr(&cq->queue.page_list[i],
619 mapping));
620
621 kfree(cq->queue.page_list);
622 }
623}
624
625static int mthca_alloc_cq_buf(struct mthca_dev *dev, int size,
626 struct mthca_cq *cq)
627{
628 int err = -ENOMEM;
629 int npages, shift;
630 u64 *dma_list = NULL;
631 dma_addr_t t;
632 int i;
633
634 if (size <= MTHCA_MAX_DIRECT_CQ_SIZE) {
635 cq->is_direct = 1;
636 npages = 1;
637 shift = get_order(size) + PAGE_SHIFT;
638
639 cq->queue.direct.buf = pci_alloc_consistent(dev->pdev,
640 size, &t);
641 if (!cq->queue.direct.buf)
642 return -ENOMEM;
643
644 pci_unmap_addr_set(&cq->queue.direct, mapping, t);
645
646 memset(cq->queue.direct.buf, 0, size);
647
648 while (t & ((1 << shift) - 1)) {
649 --shift;
650 npages *= 2;
651 }
652
653 dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
654 if (!dma_list)
655 goto err_free;
656
657 for (i = 0; i < npages; ++i)
658 dma_list[i] = t + i * (1 << shift);
659 } else {
660 cq->is_direct = 0;
661 npages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
662 shift = PAGE_SHIFT;
663
664 dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
665 if (!dma_list)
666 return -ENOMEM;
667
668 cq->queue.page_list = kmalloc(npages * sizeof *cq->queue.page_list,
669 GFP_KERNEL);
670 if (!cq->queue.page_list)
671 goto err_out;
672
673 for (i = 0; i < npages; ++i)
674 cq->queue.page_list[i].buf = NULL;
675
676 for (i = 0; i < npages; ++i) {
677 cq->queue.page_list[i].buf =
678 pci_alloc_consistent(dev->pdev, PAGE_SIZE, &t);
679 if (!cq->queue.page_list[i].buf)
680 goto err_free;
681
682 dma_list[i] = t;
683 pci_unmap_addr_set(&cq->queue.page_list[i], mapping, t);
684
685 memset(cq->queue.page_list[i].buf, 0, PAGE_SIZE);
686 }
687 }
688
689 err = mthca_mr_alloc_phys(dev, dev->driver_pd.pd_num,
690 dma_list, shift, npages,
691 0, size,
692 MTHCA_MPT_FLAG_LOCAL_WRITE |
693 MTHCA_MPT_FLAG_LOCAL_READ,
694 &cq->mr);
695 if (err)
696 goto err_free;
697
698 kfree(dma_list);
699
700 return 0;
701
702err_free:
703 mthca_free_cq_buf(dev, cq);
704
705err_out:
706 kfree(dma_list);
707
708 return err;
709}
710
711int mthca_init_cq(struct mthca_dev *dev, int nent,
712 struct mthca_cq *cq)
713{
714 int size = nent * MTHCA_CQ_ENTRY_SIZE;
715 void *mailbox = NULL;
716 struct mthca_cq_context *cq_context;
717 int err = -ENOMEM;
718 u8 status;
719 int i;
720
721 might_sleep();
722
723 cq->ibcq.cqe = nent - 1;
724
725 cq->cqn = mthca_alloc(&dev->cq_table.alloc);
726 if (cq->cqn == -1)
727 return -ENOMEM;
728
729 if (dev->hca_type == ARBEL_NATIVE) {
730 cq->arm_sn = 1;
731
732 err = mthca_table_get(dev, dev->cq_table.table, cq->cqn);
733 if (err)
734 goto err_out;
735
736 err = -ENOMEM;
737
738 cq->set_ci_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_SET_CI,
739 cq->cqn, &cq->set_ci_db);
740 if (cq->set_ci_db_index < 0)
741 goto err_out_icm;
742
743 cq->arm_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_ARM,
744 cq->cqn, &cq->arm_db);
745 if (cq->arm_db_index < 0)
746 goto err_out_ci;
747 }
748
749 mailbox = kmalloc(sizeof (struct mthca_cq_context) + MTHCA_CMD_MAILBOX_EXTRA,
750 GFP_KERNEL);
751 if (!mailbox)
752 goto err_out_mailbox;
753
754 cq_context = MAILBOX_ALIGN(mailbox);
755
756 err = mthca_alloc_cq_buf(dev, size, cq);
757 if (err)
758 goto err_out_mailbox;
759
760 for (i = 0; i < nent; ++i)
761 set_cqe_hw(get_cqe(cq, i));
762
763 spin_lock_init(&cq->lock);
764 atomic_set(&cq->refcount, 1);
765 init_waitqueue_head(&cq->wait);
766
767 memset(cq_context, 0, sizeof *cq_context);
768 cq_context->flags = cpu_to_be32(MTHCA_CQ_STATUS_OK |
769 MTHCA_CQ_STATE_DISARMED |
770 MTHCA_CQ_FLAG_TR);
771 cq_context->start = cpu_to_be64(0);
772 cq_context->logsize_usrpage = cpu_to_be32((ffs(nent) - 1) << 24 |
773 dev->driver_uar.index);
774 cq_context->error_eqn = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn);
775 cq_context->comp_eqn = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_COMP].eqn);
776 cq_context->pd = cpu_to_be32(dev->driver_pd.pd_num);
777 cq_context->lkey = cpu_to_be32(cq->mr.ibmr.lkey);
778 cq_context->cqn = cpu_to_be32(cq->cqn);
779
780 if (dev->hca_type == ARBEL_NATIVE) {
781 cq_context->ci_db = cpu_to_be32(cq->set_ci_db_index);
782 cq_context->state_db = cpu_to_be32(cq->arm_db_index);
783 }
784
785 err = mthca_SW2HW_CQ(dev, cq_context, cq->cqn, &status);
786 if (err) {
787 mthca_warn(dev, "SW2HW_CQ failed (%d)\n", err);
788 goto err_out_free_mr;
789 }
790
791 if (status) {
792 mthca_warn(dev, "SW2HW_CQ returned status 0x%02x\n",
793 status);
794 err = -EINVAL;
795 goto err_out_free_mr;
796 }
797
798 spin_lock_irq(&dev->cq_table.lock);
799 if (mthca_array_set(&dev->cq_table.cq,
800 cq->cqn & (dev->limits.num_cqs - 1),
801 cq)) {
802 spin_unlock_irq(&dev->cq_table.lock);
803 goto err_out_free_mr;
804 }
805 spin_unlock_irq(&dev->cq_table.lock);
806
807 cq->cons_index = 0;
808
809 kfree(mailbox);
810
811 return 0;
812
813err_out_free_mr:
814 mthca_free_mr(dev, &cq->mr);
815 mthca_free_cq_buf(dev, cq);
816
817err_out_mailbox:
818 kfree(mailbox);
819
820 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index);
821
822err_out_ci:
823 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index);
824
825err_out_icm:
826 mthca_table_put(dev, dev->cq_table.table, cq->cqn);
827
828err_out:
829 mthca_free(&dev->cq_table.alloc, cq->cqn);
830
831 return err;
832}
833
834void mthca_free_cq(struct mthca_dev *dev,
835 struct mthca_cq *cq)
836{
837 void *mailbox;
838 int err;
839 u8 status;
840
841 might_sleep();
842
843 mailbox = kmalloc(sizeof (struct mthca_cq_context) + MTHCA_CMD_MAILBOX_EXTRA,
844 GFP_KERNEL);
845 if (!mailbox) {
846 mthca_warn(dev, "No memory for mailbox to free CQ.\n");
847 return;
848 }
849
850 err = mthca_HW2SW_CQ(dev, MAILBOX_ALIGN(mailbox), cq->cqn, &status);
851 if (err)
852 mthca_warn(dev, "HW2SW_CQ failed (%d)\n", err);
853 else if (status)
854 mthca_warn(dev, "HW2SW_CQ returned status 0x%02x\n",
855 status);
856
857 if (0) {
858 u32 *ctx = MAILBOX_ALIGN(mailbox);
859 int j;
860
861 printk(KERN_ERR "context for CQN %x (cons index %x, next sw %d)\n",
862 cq->cqn, cq->cons_index, !!next_cqe_sw(cq));
863 for (j = 0; j < 16; ++j)
864 printk(KERN_ERR "[%2x] %08x\n", j * 4, be32_to_cpu(ctx[j]));
865 }
866
867 spin_lock_irq(&dev->cq_table.lock);
868 mthca_array_clear(&dev->cq_table.cq,
869 cq->cqn & (dev->limits.num_cqs - 1));
870 spin_unlock_irq(&dev->cq_table.lock);
871
872 if (dev->mthca_flags & MTHCA_FLAG_MSI_X)
873 synchronize_irq(dev->eq_table.eq[MTHCA_EQ_COMP].msi_x_vector);
874 else
875 synchronize_irq(dev->pdev->irq);
876
877 atomic_dec(&cq->refcount);
878 wait_event(cq->wait, !atomic_read(&cq->refcount));
879
880 mthca_free_mr(dev, &cq->mr);
881 mthca_free_cq_buf(dev, cq);
882
883 if (dev->hca_type == ARBEL_NATIVE) {
884 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index);
885 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index);
886 mthca_table_put(dev, dev->cq_table.table, cq->cqn);
887 }
888
889 mthca_free(&dev->cq_table.alloc, cq->cqn);
890 kfree(mailbox);
891}
892
893int __devinit mthca_init_cq_table(struct mthca_dev *dev)
894{
895 int err;
896
897 spin_lock_init(&dev->cq_table.lock);
898
899 err = mthca_alloc_init(&dev->cq_table.alloc,
900 dev->limits.num_cqs,
901 (1 << 24) - 1,
902 dev->limits.reserved_cqs);
903 if (err)
904 return err;
905
906 err = mthca_array_init(&dev->cq_table.cq,
907 dev->limits.num_cqs);
908 if (err)
909 mthca_alloc_cleanup(&dev->cq_table.alloc);
910
911 return err;
912}
913
914void __devexit mthca_cleanup_cq_table(struct mthca_dev *dev)
915{
916 mthca_array_cleanup(&dev->cq_table.cq, dev->limits.num_cqs);
917 mthca_alloc_cleanup(&dev->cq_table.alloc);
918}
diff --git a/drivers/infiniband/hw/mthca/mthca_dev.h b/drivers/infiniband/hw/mthca/mthca_dev.h
new file mode 100644
index 000000000000..56b2bfb5adb1
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_dev.h
@@ -0,0 +1,437 @@
1/*
2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_dev.h 1349 2004-12-16 21:09:43Z roland $
33 */
34
35#ifndef MTHCA_DEV_H
36#define MTHCA_DEV_H
37
38#include <linux/spinlock.h>
39#include <linux/kernel.h>
40#include <linux/pci.h>
41#include <linux/dma-mapping.h>
42#include <asm/semaphore.h>
43
44#include "mthca_provider.h"
45#include "mthca_doorbell.h"
46
47#define DRV_NAME "ib_mthca"
48#define PFX DRV_NAME ": "
49#define DRV_VERSION "0.06-pre"
50#define DRV_RELDATE "November 8, 2004"
51
52/* Types of supported HCA */
53enum {
54 TAVOR, /* MT23108 */
55 ARBEL_COMPAT, /* MT25208 in Tavor compat mode */
56 ARBEL_NATIVE /* MT25208 with extended features */
57};
58
59enum {
60 MTHCA_FLAG_DDR_HIDDEN = 1 << 1,
61 MTHCA_FLAG_SRQ = 1 << 2,
62 MTHCA_FLAG_MSI = 1 << 3,
63 MTHCA_FLAG_MSI_X = 1 << 4,
64 MTHCA_FLAG_NO_LAM = 1 << 5
65};
66
67enum {
68 MTHCA_MAX_PORTS = 2
69};
70
71enum {
72 MTHCA_EQ_CONTEXT_SIZE = 0x40,
73 MTHCA_CQ_CONTEXT_SIZE = 0x40,
74 MTHCA_QP_CONTEXT_SIZE = 0x200,
75 MTHCA_RDB_ENTRY_SIZE = 0x20,
76 MTHCA_AV_SIZE = 0x20,
77 MTHCA_MGM_ENTRY_SIZE = 0x40,
78
79 /* Arbel FW gives us these, but we need them for Tavor */
80 MTHCA_MPT_ENTRY_SIZE = 0x40,
81 MTHCA_MTT_SEG_SIZE = 0x40,
82};
83
84enum {
85 MTHCA_EQ_CMD,
86 MTHCA_EQ_ASYNC,
87 MTHCA_EQ_COMP,
88 MTHCA_NUM_EQ
89};
90
91struct mthca_cmd {
92 int use_events;
93 struct semaphore hcr_sem;
94 struct semaphore poll_sem;
95 struct semaphore event_sem;
96 int max_cmds;
97 spinlock_t context_lock;
98 int free_head;
99 struct mthca_cmd_context *context;
100 u16 token_mask;
101};
102
103struct mthca_limits {
104 int num_ports;
105 int vl_cap;
106 int mtu_cap;
107 int gid_table_len;
108 int pkey_table_len;
109 int local_ca_ack_delay;
110 int num_uars;
111 int max_sg;
112 int num_qps;
113 int reserved_qps;
114 int num_srqs;
115 int reserved_srqs;
116 int num_eecs;
117 int reserved_eecs;
118 int num_cqs;
119 int reserved_cqs;
120 int num_eqs;
121 int reserved_eqs;
122 int num_mpts;
123 int num_mtt_segs;
124 int mtt_seg_size;
125 int reserved_mtts;
126 int reserved_mrws;
127 int reserved_uars;
128 int num_mgms;
129 int num_amgms;
130 int reserved_mcgs;
131 int num_pds;
132 int reserved_pds;
133};
134
135struct mthca_alloc {
136 u32 last;
137 u32 top;
138 u32 max;
139 u32 mask;
140 spinlock_t lock;
141 unsigned long *table;
142};
143
144struct mthca_array {
145 struct {
146 void **page;
147 int used;
148 } *page_list;
149};
150
151struct mthca_uar_table {
152 struct mthca_alloc alloc;
153 u64 uarc_base;
154 int uarc_size;
155};
156
157struct mthca_pd_table {
158 struct mthca_alloc alloc;
159};
160
161struct mthca_mr_table {
162 struct mthca_alloc mpt_alloc;
163 int max_mtt_order;
164 unsigned long **mtt_buddy;
165 u64 mtt_base;
166 struct mthca_icm_table *mtt_table;
167 struct mthca_icm_table *mpt_table;
168};
169
170struct mthca_eq_table {
171 struct mthca_alloc alloc;
172 void __iomem *clr_int;
173 u32 clr_mask;
174 u32 arm_mask;
175 struct mthca_eq eq[MTHCA_NUM_EQ];
176 u64 icm_virt;
177 struct page *icm_page;
178 dma_addr_t icm_dma;
179 int have_irq;
180 u8 inta_pin;
181};
182
183struct mthca_cq_table {
184 struct mthca_alloc alloc;
185 spinlock_t lock;
186 struct mthca_array cq;
187 struct mthca_icm_table *table;
188};
189
190struct mthca_qp_table {
191 struct mthca_alloc alloc;
192 u32 rdb_base;
193 int rdb_shift;
194 int sqp_start;
195 spinlock_t lock;
196 struct mthca_array qp;
197 struct mthca_icm_table *qp_table;
198 struct mthca_icm_table *eqp_table;
199};
200
201struct mthca_av_table {
202 struct pci_pool *pool;
203 int num_ddr_avs;
204 u64 ddr_av_base;
205 void __iomem *av_map;
206 struct mthca_alloc alloc;
207};
208
209struct mthca_mcg_table {
210 struct semaphore sem;
211 struct mthca_alloc alloc;
212 struct mthca_icm_table *table;
213};
214
215struct mthca_dev {
216 struct ib_device ib_dev;
217 struct pci_dev *pdev;
218
219 int hca_type;
220 unsigned long mthca_flags;
221 unsigned long device_cap_flags;
222
223 u32 rev_id;
224
225 /* firmware info */
226 u64 fw_ver;
227 union {
228 struct {
229 u64 fw_start;
230 u64 fw_end;
231 } tavor;
232 struct {
233 u64 clr_int_base;
234 u64 eq_arm_base;
235 u64 eq_set_ci_base;
236 struct mthca_icm *fw_icm;
237 struct mthca_icm *aux_icm;
238 u16 fw_pages;
239 } arbel;
240 } fw;
241
242 u64 ddr_start;
243 u64 ddr_end;
244
245 MTHCA_DECLARE_DOORBELL_LOCK(doorbell_lock)
246 struct semaphore cap_mask_mutex;
247
248 void __iomem *hcr;
249 void __iomem *kar;
250 void __iomem *clr_base;
251 union {
252 struct {
253 void __iomem *ecr_base;
254 } tavor;
255 struct {
256 void __iomem *eq_arm;
257 void __iomem *eq_set_ci_base;
258 } arbel;
259 } eq_regs;
260
261 struct mthca_cmd cmd;
262 struct mthca_limits limits;
263
264 struct mthca_uar_table uar_table;
265 struct mthca_pd_table pd_table;
266 struct mthca_mr_table mr_table;
267 struct mthca_eq_table eq_table;
268 struct mthca_cq_table cq_table;
269 struct mthca_qp_table qp_table;
270 struct mthca_av_table av_table;
271 struct mthca_mcg_table mcg_table;
272
273 struct mthca_uar driver_uar;
274 struct mthca_db_table *db_tab;
275 struct mthca_pd driver_pd;
276 struct mthca_mr driver_mr;
277
278 struct ib_mad_agent *send_agent[MTHCA_MAX_PORTS][2];
279 struct ib_ah *sm_ah[MTHCA_MAX_PORTS];
280 spinlock_t sm_lock;
281};
282
283#define mthca_dbg(mdev, format, arg...) \
284 dev_dbg(&mdev->pdev->dev, format, ## arg)
285#define mthca_err(mdev, format, arg...) \
286 dev_err(&mdev->pdev->dev, format, ## arg)
287#define mthca_info(mdev, format, arg...) \
288 dev_info(&mdev->pdev->dev, format, ## arg)
289#define mthca_warn(mdev, format, arg...) \
290 dev_warn(&mdev->pdev->dev, format, ## arg)
291
292extern void __buggy_use_of_MTHCA_GET(void);
293extern void __buggy_use_of_MTHCA_PUT(void);
294
295#define MTHCA_GET(dest, source, offset) \
296 do { \
297 void *__p = (char *) (source) + (offset); \
298 switch (sizeof (dest)) { \
299 case 1: (dest) = *(u8 *) __p; break; \
300 case 2: (dest) = be16_to_cpup(__p); break; \
301 case 4: (dest) = be32_to_cpup(__p); break; \
302 case 8: (dest) = be64_to_cpup(__p); break; \
303 default: __buggy_use_of_MTHCA_GET(); \
304 } \
305 } while (0)
306
307#define MTHCA_PUT(dest, source, offset) \
308 do { \
309 __typeof__(source) *__p = \
310 (__typeof__(source) *) ((char *) (dest) + (offset)); \
311 switch (sizeof(source)) { \
312 case 1: *__p = (source); break; \
313 case 2: *__p = cpu_to_be16(source); break; \
314 case 4: *__p = cpu_to_be32(source); break; \
315 case 8: *__p = cpu_to_be64(source); break; \
316 default: __buggy_use_of_MTHCA_PUT(); \
317 } \
318 } while (0)
319
320int mthca_reset(struct mthca_dev *mdev);
321
322u32 mthca_alloc(struct mthca_alloc *alloc);
323void mthca_free(struct mthca_alloc *alloc, u32 obj);
324int mthca_alloc_init(struct mthca_alloc *alloc, u32 num, u32 mask,
325 u32 reserved);
326void mthca_alloc_cleanup(struct mthca_alloc *alloc);
327void *mthca_array_get(struct mthca_array *array, int index);
328int mthca_array_set(struct mthca_array *array, int index, void *value);
329void mthca_array_clear(struct mthca_array *array, int index);
330int mthca_array_init(struct mthca_array *array, int nent);
331void mthca_array_cleanup(struct mthca_array *array, int nent);
332
333int mthca_init_uar_table(struct mthca_dev *dev);
334int mthca_init_pd_table(struct mthca_dev *dev);
335int mthca_init_mr_table(struct mthca_dev *dev);
336int mthca_init_eq_table(struct mthca_dev *dev);
337int mthca_init_cq_table(struct mthca_dev *dev);
338int mthca_init_qp_table(struct mthca_dev *dev);
339int mthca_init_av_table(struct mthca_dev *dev);
340int mthca_init_mcg_table(struct mthca_dev *dev);
341
342void mthca_cleanup_uar_table(struct mthca_dev *dev);
343void mthca_cleanup_pd_table(struct mthca_dev *dev);
344void mthca_cleanup_mr_table(struct mthca_dev *dev);
345void mthca_cleanup_eq_table(struct mthca_dev *dev);
346void mthca_cleanup_cq_table(struct mthca_dev *dev);
347void mthca_cleanup_qp_table(struct mthca_dev *dev);
348void mthca_cleanup_av_table(struct mthca_dev *dev);
349void mthca_cleanup_mcg_table(struct mthca_dev *dev);
350
351int mthca_register_device(struct mthca_dev *dev);
352void mthca_unregister_device(struct mthca_dev *dev);
353
354int mthca_uar_alloc(struct mthca_dev *dev, struct mthca_uar *uar);
355void mthca_uar_free(struct mthca_dev *dev, struct mthca_uar *uar);
356
357int mthca_pd_alloc(struct mthca_dev *dev, struct mthca_pd *pd);
358void mthca_pd_free(struct mthca_dev *dev, struct mthca_pd *pd);
359
360int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd,
361 u32 access, struct mthca_mr *mr);
362int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd,
363 u64 *buffer_list, int buffer_size_shift,
364 int list_len, u64 iova, u64 total_size,
365 u32 access, struct mthca_mr *mr);
366void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr);
367
368int mthca_map_eq_icm(struct mthca_dev *dev, u64 icm_virt);
369void mthca_unmap_eq_icm(struct mthca_dev *dev);
370
371int mthca_poll_cq(struct ib_cq *ibcq, int num_entries,
372 struct ib_wc *entry);
373int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify);
374int mthca_arbel_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify);
375int mthca_init_cq(struct mthca_dev *dev, int nent,
376 struct mthca_cq *cq);
377void mthca_free_cq(struct mthca_dev *dev,
378 struct mthca_cq *cq);
379void mthca_cq_event(struct mthca_dev *dev, u32 cqn);
380void mthca_cq_clean(struct mthca_dev *dev, u32 cqn, u32 qpn);
381
382void mthca_qp_event(struct mthca_dev *dev, u32 qpn,
383 enum ib_event_type event_type);
384int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask);
385int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
386 struct ib_send_wr **bad_wr);
387int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
388 struct ib_recv_wr **bad_wr);
389int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
390 struct ib_send_wr **bad_wr);
391int mthca_arbel_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
392 struct ib_recv_wr **bad_wr);
393int mthca_free_err_wqe(struct mthca_dev *dev, struct mthca_qp *qp, int is_send,
394 int index, int *dbd, u32 *new_wqe);
395int mthca_alloc_qp(struct mthca_dev *dev,
396 struct mthca_pd *pd,
397 struct mthca_cq *send_cq,
398 struct mthca_cq *recv_cq,
399 enum ib_qp_type type,
400 enum ib_sig_type send_policy,
401 struct mthca_qp *qp);
402int mthca_alloc_sqp(struct mthca_dev *dev,
403 struct mthca_pd *pd,
404 struct mthca_cq *send_cq,
405 struct mthca_cq *recv_cq,
406 enum ib_sig_type send_policy,
407 int qpn,
408 int port,
409 struct mthca_sqp *sqp);
410void mthca_free_qp(struct mthca_dev *dev, struct mthca_qp *qp);
411int mthca_create_ah(struct mthca_dev *dev,
412 struct mthca_pd *pd,
413 struct ib_ah_attr *ah_attr,
414 struct mthca_ah *ah);
415int mthca_destroy_ah(struct mthca_dev *dev, struct mthca_ah *ah);
416int mthca_read_ah(struct mthca_dev *dev, struct mthca_ah *ah,
417 struct ib_ud_header *header);
418
419int mthca_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid);
420int mthca_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid);
421
422int mthca_process_mad(struct ib_device *ibdev,
423 int mad_flags,
424 u8 port_num,
425 struct ib_wc *in_wc,
426 struct ib_grh *in_grh,
427 struct ib_mad *in_mad,
428 struct ib_mad *out_mad);
429int mthca_create_agents(struct mthca_dev *dev);
430void mthca_free_agents(struct mthca_dev *dev);
431
432static inline struct mthca_dev *to_mdev(struct ib_device *ibdev)
433{
434 return container_of(ibdev, struct mthca_dev, ib_dev);
435}
436
437#endif /* MTHCA_DEV_H */
diff --git a/drivers/infiniband/hw/mthca/mthca_doorbell.h b/drivers/infiniband/hw/mthca/mthca_doorbell.h
new file mode 100644
index 000000000000..78b183cab54c
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_doorbell.h
@@ -0,0 +1,95 @@
1/*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_doorbell.h 1349 2004-12-16 21:09:43Z roland $
33 */
34
35#include <linux/types.h>
36
37#define MTHCA_RD_DOORBELL 0x00
38#define MTHCA_SEND_DOORBELL 0x10
39#define MTHCA_RECEIVE_DOORBELL 0x18
40#define MTHCA_CQ_DOORBELL 0x20
41#define MTHCA_EQ_DOORBELL 0x28
42
43#if BITS_PER_LONG == 64
44/*
45 * Assume that we can just write a 64-bit doorbell atomically. s390
46 * actually doesn't have writeq() but S/390 systems don't even have
47 * PCI so we won't worry about it.
48 */
49
50#define MTHCA_DECLARE_DOORBELL_LOCK(name)
51#define MTHCA_INIT_DOORBELL_LOCK(ptr) do { } while (0)
52#define MTHCA_GET_DOORBELL_LOCK(ptr) (NULL)
53
54static inline void mthca_write64(u32 val[2], void __iomem *dest,
55 spinlock_t *doorbell_lock)
56{
57 __raw_writeq(*(u64 *) val, dest);
58}
59
60static inline void mthca_write_db_rec(u32 val[2], u32 *db)
61{
62 *(u64 *) db = *(u64 *) val;
63}
64
65#else
66
67/*
68 * Just fall back to a spinlock to protect the doorbell if
69 * BITS_PER_LONG is 32 -- there's no portable way to do atomic 64-bit
70 * MMIO writes.
71 */
72
73#define MTHCA_DECLARE_DOORBELL_LOCK(name) spinlock_t name;
74#define MTHCA_INIT_DOORBELL_LOCK(ptr) spin_lock_init(ptr)
75#define MTHCA_GET_DOORBELL_LOCK(ptr) (ptr)
76
77static inline void mthca_write64(u32 val[2], void __iomem *dest,
78 spinlock_t *doorbell_lock)
79{
80 unsigned long flags;
81
82 spin_lock_irqsave(doorbell_lock, flags);
83 __raw_writel(val[0], dest);
84 __raw_writel(val[1], dest + 4);
85 spin_unlock_irqrestore(doorbell_lock, flags);
86}
87
88static inline void mthca_write_db_rec(u32 val[2], u32 *db)
89{
90 db[0] = val[0];
91 wmb();
92 db[1] = val[1];
93}
94
95#endif
diff --git a/drivers/infiniband/hw/mthca/mthca_eq.c b/drivers/infiniband/hw/mthca/mthca_eq.c
new file mode 100644
index 000000000000..623daab5c92b
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_eq.c
@@ -0,0 +1,964 @@
1/*
2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_eq.c 1382 2004-12-24 02:21:02Z roland $
33 */
34
35#include <linux/init.h>
36#include <linux/errno.h>
37#include <linux/interrupt.h>
38#include <linux/pci.h>
39
40#include "mthca_dev.h"
41#include "mthca_cmd.h"
42#include "mthca_config_reg.h"
43
44enum {
45 MTHCA_NUM_ASYNC_EQE = 0x80,
46 MTHCA_NUM_CMD_EQE = 0x80,
47 MTHCA_EQ_ENTRY_SIZE = 0x20
48};
49
50/*
51 * Must be packed because start is 64 bits but only aligned to 32 bits.
52 */
53struct mthca_eq_context {
54 u32 flags;
55 u64 start;
56 u32 logsize_usrpage;
57 u32 tavor_pd; /* reserved for Arbel */
58 u8 reserved1[3];
59 u8 intr;
60 u32 arbel_pd; /* lost_count for Tavor */
61 u32 lkey;
62 u32 reserved2[2];
63 u32 consumer_index;
64 u32 producer_index;
65 u32 reserved3[4];
66} __attribute__((packed));
67
68#define MTHCA_EQ_STATUS_OK ( 0 << 28)
69#define MTHCA_EQ_STATUS_OVERFLOW ( 9 << 28)
70#define MTHCA_EQ_STATUS_WRITE_FAIL (10 << 28)
71#define MTHCA_EQ_OWNER_SW ( 0 << 24)
72#define MTHCA_EQ_OWNER_HW ( 1 << 24)
73#define MTHCA_EQ_FLAG_TR ( 1 << 18)
74#define MTHCA_EQ_FLAG_OI ( 1 << 17)
75#define MTHCA_EQ_STATE_ARMED ( 1 << 8)
76#define MTHCA_EQ_STATE_FIRED ( 2 << 8)
77#define MTHCA_EQ_STATE_ALWAYS_ARMED ( 3 << 8)
78#define MTHCA_EQ_STATE_ARBEL ( 8 << 8)
79
80enum {
81 MTHCA_EVENT_TYPE_COMP = 0x00,
82 MTHCA_EVENT_TYPE_PATH_MIG = 0x01,
83 MTHCA_EVENT_TYPE_COMM_EST = 0x02,
84 MTHCA_EVENT_TYPE_SQ_DRAINED = 0x03,
85 MTHCA_EVENT_TYPE_SRQ_LAST_WQE = 0x13,
86 MTHCA_EVENT_TYPE_CQ_ERROR = 0x04,
87 MTHCA_EVENT_TYPE_WQ_CATAS_ERROR = 0x05,
88 MTHCA_EVENT_TYPE_EEC_CATAS_ERROR = 0x06,
89 MTHCA_EVENT_TYPE_PATH_MIG_FAILED = 0x07,
90 MTHCA_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10,
91 MTHCA_EVENT_TYPE_WQ_ACCESS_ERROR = 0x11,
92 MTHCA_EVENT_TYPE_SRQ_CATAS_ERROR = 0x12,
93 MTHCA_EVENT_TYPE_LOCAL_CATAS_ERROR = 0x08,
94 MTHCA_EVENT_TYPE_PORT_CHANGE = 0x09,
95 MTHCA_EVENT_TYPE_EQ_OVERFLOW = 0x0f,
96 MTHCA_EVENT_TYPE_ECC_DETECT = 0x0e,
97 MTHCA_EVENT_TYPE_CMD = 0x0a
98};
99
100#define MTHCA_ASYNC_EVENT_MASK ((1ULL << MTHCA_EVENT_TYPE_PATH_MIG) | \
101 (1ULL << MTHCA_EVENT_TYPE_COMM_EST) | \
102 (1ULL << MTHCA_EVENT_TYPE_SQ_DRAINED) | \
103 (1ULL << MTHCA_EVENT_TYPE_CQ_ERROR) | \
104 (1ULL << MTHCA_EVENT_TYPE_WQ_CATAS_ERROR) | \
105 (1ULL << MTHCA_EVENT_TYPE_EEC_CATAS_ERROR) | \
106 (1ULL << MTHCA_EVENT_TYPE_PATH_MIG_FAILED) | \
107 (1ULL << MTHCA_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \
108 (1ULL << MTHCA_EVENT_TYPE_WQ_ACCESS_ERROR) | \
109 (1ULL << MTHCA_EVENT_TYPE_LOCAL_CATAS_ERROR) | \
110 (1ULL << MTHCA_EVENT_TYPE_PORT_CHANGE) | \
111 (1ULL << MTHCA_EVENT_TYPE_ECC_DETECT))
112#define MTHCA_SRQ_EVENT_MASK (1ULL << MTHCA_EVENT_TYPE_SRQ_CATAS_ERROR) | \
113 (1ULL << MTHCA_EVENT_TYPE_SRQ_LAST_WQE)
114#define MTHCA_CMD_EVENT_MASK (1ULL << MTHCA_EVENT_TYPE_CMD)
115
116#define MTHCA_EQ_DB_INC_CI (1 << 24)
117#define MTHCA_EQ_DB_REQ_NOT (2 << 24)
118#define MTHCA_EQ_DB_DISARM_CQ (3 << 24)
119#define MTHCA_EQ_DB_SET_CI (4 << 24)
120#define MTHCA_EQ_DB_ALWAYS_ARM (5 << 24)
121
122struct mthca_eqe {
123 u8 reserved1;
124 u8 type;
125 u8 reserved2;
126 u8 subtype;
127 union {
128 u32 raw[6];
129 struct {
130 u32 cqn;
131 } __attribute__((packed)) comp;
132 struct {
133 u16 reserved1;
134 u16 token;
135 u32 reserved2;
136 u8 reserved3[3];
137 u8 status;
138 u64 out_param;
139 } __attribute__((packed)) cmd;
140 struct {
141 u32 qpn;
142 } __attribute__((packed)) qp;
143 struct {
144 u32 cqn;
145 u32 reserved1;
146 u8 reserved2[3];
147 u8 syndrome;
148 } __attribute__((packed)) cq_err;
149 struct {
150 u32 reserved1[2];
151 u32 port;
152 } __attribute__((packed)) port_change;
153 } event;
154 u8 reserved3[3];
155 u8 owner;
156} __attribute__((packed));
157
158#define MTHCA_EQ_ENTRY_OWNER_SW (0 << 7)
159#define MTHCA_EQ_ENTRY_OWNER_HW (1 << 7)
160
161static inline u64 async_mask(struct mthca_dev *dev)
162{
163 return dev->mthca_flags & MTHCA_FLAG_SRQ ?
164 MTHCA_ASYNC_EVENT_MASK | MTHCA_SRQ_EVENT_MASK :
165 MTHCA_ASYNC_EVENT_MASK;
166}
167
168static inline void tavor_set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci)
169{
170 u32 doorbell[2];
171
172 doorbell[0] = cpu_to_be32(MTHCA_EQ_DB_SET_CI | eq->eqn);
173 doorbell[1] = cpu_to_be32(ci & (eq->nent - 1));
174
175 /*
176 * This barrier makes sure that all updates to ownership bits
177 * done by set_eqe_hw() hit memory before the consumer index
178 * is updated. set_eq_ci() allows the HCA to possibly write
179 * more EQ entries, and we want to avoid the exceedingly
180 * unlikely possibility of the HCA writing an entry and then
181 * having set_eqe_hw() overwrite the owner field.
182 */
183 wmb();
184 mthca_write64(doorbell,
185 dev->kar + MTHCA_EQ_DOORBELL,
186 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
187}
188
189static inline void arbel_set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci)
190{
191 /* See comment in tavor_set_eq_ci() above. */
192 wmb();
193 __raw_writel(cpu_to_be32(ci), dev->eq_regs.arbel.eq_set_ci_base +
194 eq->eqn * 8);
195 /* We still want ordering, just not swabbing, so add a barrier */
196 mb();
197}
198
199static inline void set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci)
200{
201 if (dev->hca_type == ARBEL_NATIVE)
202 arbel_set_eq_ci(dev, eq, ci);
203 else
204 tavor_set_eq_ci(dev, eq, ci);
205}
206
207static inline void tavor_eq_req_not(struct mthca_dev *dev, int eqn)
208{
209 u32 doorbell[2];
210
211 doorbell[0] = cpu_to_be32(MTHCA_EQ_DB_REQ_NOT | eqn);
212 doorbell[1] = 0;
213
214 mthca_write64(doorbell,
215 dev->kar + MTHCA_EQ_DOORBELL,
216 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
217}
218
219static inline void arbel_eq_req_not(struct mthca_dev *dev, u32 eqn_mask)
220{
221 writel(eqn_mask, dev->eq_regs.arbel.eq_arm);
222}
223
224static inline void disarm_cq(struct mthca_dev *dev, int eqn, int cqn)
225{
226 if (dev->hca_type != ARBEL_NATIVE) {
227 u32 doorbell[2];
228
229 doorbell[0] = cpu_to_be32(MTHCA_EQ_DB_DISARM_CQ | eqn);
230 doorbell[1] = cpu_to_be32(cqn);
231
232 mthca_write64(doorbell,
233 dev->kar + MTHCA_EQ_DOORBELL,
234 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
235 }
236}
237
238static inline struct mthca_eqe *get_eqe(struct mthca_eq *eq, u32 entry)
239{
240 unsigned long off = (entry & (eq->nent - 1)) * MTHCA_EQ_ENTRY_SIZE;
241 return eq->page_list[off / PAGE_SIZE].buf + off % PAGE_SIZE;
242}
243
244static inline struct mthca_eqe* next_eqe_sw(struct mthca_eq *eq)
245{
246 struct mthca_eqe* eqe;
247 eqe = get_eqe(eq, eq->cons_index);
248 return (MTHCA_EQ_ENTRY_OWNER_HW & eqe->owner) ? NULL : eqe;
249}
250
251static inline void set_eqe_hw(struct mthca_eqe *eqe)
252{
253 eqe->owner = MTHCA_EQ_ENTRY_OWNER_HW;
254}
255
256static void port_change(struct mthca_dev *dev, int port, int active)
257{
258 struct ib_event record;
259
260 mthca_dbg(dev, "Port change to %s for port %d\n",
261 active ? "active" : "down", port);
262
263 record.device = &dev->ib_dev;
264 record.event = active ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
265 record.element.port_num = port;
266
267 ib_dispatch_event(&record);
268}
269
270static int mthca_eq_int(struct mthca_dev *dev, struct mthca_eq *eq)
271{
272 struct mthca_eqe *eqe;
273 int disarm_cqn;
274 int eqes_found = 0;
275
276 while ((eqe = next_eqe_sw(eq))) {
277 int set_ci = 0;
278
279 /*
280 * Make sure we read EQ entry contents after we've
281 * checked the ownership bit.
282 */
283 rmb();
284
285 switch (eqe->type) {
286 case MTHCA_EVENT_TYPE_COMP:
287 disarm_cqn = be32_to_cpu(eqe->event.comp.cqn) & 0xffffff;
288 disarm_cq(dev, eq->eqn, disarm_cqn);
289 mthca_cq_event(dev, disarm_cqn);
290 break;
291
292 case MTHCA_EVENT_TYPE_PATH_MIG:
293 mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
294 IB_EVENT_PATH_MIG);
295 break;
296
297 case MTHCA_EVENT_TYPE_COMM_EST:
298 mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
299 IB_EVENT_COMM_EST);
300 break;
301
302 case MTHCA_EVENT_TYPE_SQ_DRAINED:
303 mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
304 IB_EVENT_SQ_DRAINED);
305 break;
306
307 case MTHCA_EVENT_TYPE_WQ_CATAS_ERROR:
308 mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
309 IB_EVENT_QP_FATAL);
310 break;
311
312 case MTHCA_EVENT_TYPE_PATH_MIG_FAILED:
313 mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
314 IB_EVENT_PATH_MIG_ERR);
315 break;
316
317 case MTHCA_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
318 mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
319 IB_EVENT_QP_REQ_ERR);
320 break;
321
322 case MTHCA_EVENT_TYPE_WQ_ACCESS_ERROR:
323 mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
324 IB_EVENT_QP_ACCESS_ERR);
325 break;
326
327 case MTHCA_EVENT_TYPE_CMD:
328 mthca_cmd_event(dev,
329 be16_to_cpu(eqe->event.cmd.token),
330 eqe->event.cmd.status,
331 be64_to_cpu(eqe->event.cmd.out_param));
332 /*
333 * cmd_event() may add more commands.
334 * The card will think the queue has overflowed if
335 * we don't tell it we've been processing events.
336 */
337 set_ci = 1;
338 break;
339
340 case MTHCA_EVENT_TYPE_PORT_CHANGE:
341 port_change(dev,
342 (be32_to_cpu(eqe->event.port_change.port) >> 28) & 3,
343 eqe->subtype == 0x4);
344 break;
345
346 case MTHCA_EVENT_TYPE_CQ_ERROR:
347 mthca_warn(dev, "CQ %s on CQN %08x\n",
348 eqe->event.cq_err.syndrome == 1 ?
349 "overrun" : "access violation",
350 be32_to_cpu(eqe->event.cq_err.cqn));
351 break;
352
353 case MTHCA_EVENT_TYPE_EQ_OVERFLOW:
354 mthca_warn(dev, "EQ overrun on EQN %d\n", eq->eqn);
355 break;
356
357 case MTHCA_EVENT_TYPE_EEC_CATAS_ERROR:
358 case MTHCA_EVENT_TYPE_SRQ_CATAS_ERROR:
359 case MTHCA_EVENT_TYPE_LOCAL_CATAS_ERROR:
360 case MTHCA_EVENT_TYPE_ECC_DETECT:
361 default:
362 mthca_warn(dev, "Unhandled event %02x(%02x) on EQ %d\n",
363 eqe->type, eqe->subtype, eq->eqn);
364 break;
365 };
366
367 set_eqe_hw(eqe);
368 ++eq->cons_index;
369 eqes_found = 1;
370
371 if (unlikely(set_ci)) {
372 /*
373 * Conditional on hca_type is OK here because
374 * this is a rare case, not the fast path.
375 */
376 set_eq_ci(dev, eq, eq->cons_index);
377 set_ci = 0;
378 }
379 }
380
381 /*
382 * Rely on caller to set consumer index so that we don't have
383 * to test hca_type in our interrupt handling fast path.
384 */
385 return eqes_found;
386}
387
388static irqreturn_t mthca_tavor_interrupt(int irq, void *dev_ptr, struct pt_regs *regs)
389{
390 struct mthca_dev *dev = dev_ptr;
391 u32 ecr;
392 int i;
393
394 if (dev->eq_table.clr_mask)
395 writel(dev->eq_table.clr_mask, dev->eq_table.clr_int);
396
397 ecr = readl(dev->eq_regs.tavor.ecr_base + 4);
398 if (ecr) {
399 writel(ecr, dev->eq_regs.tavor.ecr_base +
400 MTHCA_ECR_CLR_BASE - MTHCA_ECR_BASE + 4);
401
402 for (i = 0; i < MTHCA_NUM_EQ; ++i)
403 if (ecr & dev->eq_table.eq[i].eqn_mask &&
404 mthca_eq_int(dev, &dev->eq_table.eq[i])) {
405 tavor_set_eq_ci(dev, &dev->eq_table.eq[i],
406 dev->eq_table.eq[i].cons_index);
407 tavor_eq_req_not(dev, dev->eq_table.eq[i].eqn);
408 }
409 }
410
411 return IRQ_RETVAL(ecr);
412}
413
414static irqreturn_t mthca_tavor_msi_x_interrupt(int irq, void *eq_ptr,
415 struct pt_regs *regs)
416{
417 struct mthca_eq *eq = eq_ptr;
418 struct mthca_dev *dev = eq->dev;
419
420 mthca_eq_int(dev, eq);
421 tavor_set_eq_ci(dev, eq, eq->cons_index);
422 tavor_eq_req_not(dev, eq->eqn);
423
424 /* MSI-X vectors always belong to us */
425 return IRQ_HANDLED;
426}
427
428static irqreturn_t mthca_arbel_interrupt(int irq, void *dev_ptr, struct pt_regs *regs)
429{
430 struct mthca_dev *dev = dev_ptr;
431 int work = 0;
432 int i;
433
434 if (dev->eq_table.clr_mask)
435 writel(dev->eq_table.clr_mask, dev->eq_table.clr_int);
436
437 for (i = 0; i < MTHCA_NUM_EQ; ++i)
438 if (mthca_eq_int(dev, &dev->eq_table.eq[i])) {
439 work = 1;
440 arbel_set_eq_ci(dev, &dev->eq_table.eq[i],
441 dev->eq_table.eq[i].cons_index);
442 }
443
444 arbel_eq_req_not(dev, dev->eq_table.arm_mask);
445
446 return IRQ_RETVAL(work);
447}
448
449static irqreturn_t mthca_arbel_msi_x_interrupt(int irq, void *eq_ptr,
450 struct pt_regs *regs)
451{
452 struct mthca_eq *eq = eq_ptr;
453 struct mthca_dev *dev = eq->dev;
454
455 mthca_eq_int(dev, eq);
456 arbel_set_eq_ci(dev, eq, eq->cons_index);
457 arbel_eq_req_not(dev, eq->eqn_mask);
458
459 /* MSI-X vectors always belong to us */
460 return IRQ_HANDLED;
461}
462
463static int __devinit mthca_create_eq(struct mthca_dev *dev,
464 int nent,
465 u8 intr,
466 struct mthca_eq *eq)
467{
468 int npages = (nent * MTHCA_EQ_ENTRY_SIZE + PAGE_SIZE - 1) /
469 PAGE_SIZE;
470 u64 *dma_list = NULL;
471 dma_addr_t t;
472 void *mailbox = NULL;
473 struct mthca_eq_context *eq_context;
474 int err = -ENOMEM;
475 int i;
476 u8 status;
477
478 /* Make sure EQ size is aligned to a power of 2 size. */
479 for (i = 1; i < nent; i <<= 1)
480 ; /* nothing */
481 nent = i;
482
483 eq->dev = dev;
484
485 eq->page_list = kmalloc(npages * sizeof *eq->page_list,
486 GFP_KERNEL);
487 if (!eq->page_list)
488 goto err_out;
489
490 for (i = 0; i < npages; ++i)
491 eq->page_list[i].buf = NULL;
492
493 dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
494 if (!dma_list)
495 goto err_out_free;
496
497 mailbox = kmalloc(sizeof *eq_context + MTHCA_CMD_MAILBOX_EXTRA,
498 GFP_KERNEL);
499 if (!mailbox)
500 goto err_out_free;
501 eq_context = MAILBOX_ALIGN(mailbox);
502
503 for (i = 0; i < npages; ++i) {
504 eq->page_list[i].buf = pci_alloc_consistent(dev->pdev,
505 PAGE_SIZE, &t);
506 if (!eq->page_list[i].buf)
507 goto err_out_free;
508
509 dma_list[i] = t;
510 pci_unmap_addr_set(&eq->page_list[i], mapping, t);
511
512 memset(eq->page_list[i].buf, 0, PAGE_SIZE);
513 }
514
515 for (i = 0; i < nent; ++i)
516 set_eqe_hw(get_eqe(eq, i));
517
518 eq->eqn = mthca_alloc(&dev->eq_table.alloc);
519 if (eq->eqn == -1)
520 goto err_out_free;
521
522 err = mthca_mr_alloc_phys(dev, dev->driver_pd.pd_num,
523 dma_list, PAGE_SHIFT, npages,
524 0, npages * PAGE_SIZE,
525 MTHCA_MPT_FLAG_LOCAL_WRITE |
526 MTHCA_MPT_FLAG_LOCAL_READ,
527 &eq->mr);
528 if (err)
529 goto err_out_free_eq;
530
531 eq->nent = nent;
532
533 memset(eq_context, 0, sizeof *eq_context);
534 eq_context->flags = cpu_to_be32(MTHCA_EQ_STATUS_OK |
535 MTHCA_EQ_OWNER_HW |
536 MTHCA_EQ_STATE_ARMED |
537 MTHCA_EQ_FLAG_TR);
538 if (dev->hca_type == ARBEL_NATIVE)
539 eq_context->flags |= cpu_to_be32(MTHCA_EQ_STATE_ARBEL);
540
541 eq_context->logsize_usrpage = cpu_to_be32((ffs(nent) - 1) << 24);
542 if (dev->hca_type == ARBEL_NATIVE) {
543 eq_context->arbel_pd = cpu_to_be32(dev->driver_pd.pd_num);
544 } else {
545 eq_context->logsize_usrpage |= cpu_to_be32(dev->driver_uar.index);
546 eq_context->tavor_pd = cpu_to_be32(dev->driver_pd.pd_num);
547 }
548 eq_context->intr = intr;
549 eq_context->lkey = cpu_to_be32(eq->mr.ibmr.lkey);
550
551 err = mthca_SW2HW_EQ(dev, eq_context, eq->eqn, &status);
552 if (err) {
553 mthca_warn(dev, "SW2HW_EQ failed (%d)\n", err);
554 goto err_out_free_mr;
555 }
556 if (status) {
557 mthca_warn(dev, "SW2HW_EQ returned status 0x%02x\n",
558 status);
559 err = -EINVAL;
560 goto err_out_free_mr;
561 }
562
563 kfree(dma_list);
564 kfree(mailbox);
565
566 eq->eqn_mask = swab32(1 << eq->eqn);
567 eq->cons_index = 0;
568
569 dev->eq_table.arm_mask |= eq->eqn_mask;
570
571 mthca_dbg(dev, "Allocated EQ %d with %d entries\n",
572 eq->eqn, nent);
573
574 return err;
575
576 err_out_free_mr:
577 mthca_free_mr(dev, &eq->mr);
578
579 err_out_free_eq:
580 mthca_free(&dev->eq_table.alloc, eq->eqn);
581
582 err_out_free:
583 for (i = 0; i < npages; ++i)
584 if (eq->page_list[i].buf)
585 pci_free_consistent(dev->pdev, PAGE_SIZE,
586 eq->page_list[i].buf,
587 pci_unmap_addr(&eq->page_list[i],
588 mapping));
589
590 kfree(eq->page_list);
591 kfree(dma_list);
592 kfree(mailbox);
593
594 err_out:
595 return err;
596}
597
598static void mthca_free_eq(struct mthca_dev *dev,
599 struct mthca_eq *eq)
600{
601 void *mailbox = NULL;
602 int err;
603 u8 status;
604 int npages = (eq->nent * MTHCA_EQ_ENTRY_SIZE + PAGE_SIZE - 1) /
605 PAGE_SIZE;
606 int i;
607
608 mailbox = kmalloc(sizeof (struct mthca_eq_context) + MTHCA_CMD_MAILBOX_EXTRA,
609 GFP_KERNEL);
610 if (!mailbox)
611 return;
612
613 err = mthca_HW2SW_EQ(dev, MAILBOX_ALIGN(mailbox),
614 eq->eqn, &status);
615 if (err)
616 mthca_warn(dev, "HW2SW_EQ failed (%d)\n", err);
617 if (status)
618 mthca_warn(dev, "HW2SW_EQ returned status 0x%02x\n",
619 status);
620
621 dev->eq_table.arm_mask &= ~eq->eqn_mask;
622
623 if (0) {
624 mthca_dbg(dev, "Dumping EQ context %02x:\n", eq->eqn);
625 for (i = 0; i < sizeof (struct mthca_eq_context) / 4; ++i) {
626 if (i % 4 == 0)
627 printk("[%02x] ", i * 4);
628 printk(" %08x", be32_to_cpup(MAILBOX_ALIGN(mailbox) + i * 4));
629 if ((i + 1) % 4 == 0)
630 printk("\n");
631 }
632 }
633
634 mthca_free_mr(dev, &eq->mr);
635 for (i = 0; i < npages; ++i)
636 pci_free_consistent(dev->pdev, PAGE_SIZE,
637 eq->page_list[i].buf,
638 pci_unmap_addr(&eq->page_list[i], mapping));
639
640 kfree(eq->page_list);
641 kfree(mailbox);
642}
643
644static void mthca_free_irqs(struct mthca_dev *dev)
645{
646 int i;
647
648 if (dev->eq_table.have_irq)
649 free_irq(dev->pdev->irq, dev);
650 for (i = 0; i < MTHCA_NUM_EQ; ++i)
651 if (dev->eq_table.eq[i].have_irq)
652 free_irq(dev->eq_table.eq[i].msi_x_vector,
653 dev->eq_table.eq + i);
654}
655
656static int __devinit mthca_map_reg(struct mthca_dev *dev,
657 unsigned long offset, unsigned long size,
658 void __iomem **map)
659{
660 unsigned long base = pci_resource_start(dev->pdev, 0);
661
662 if (!request_mem_region(base + offset, size, DRV_NAME))
663 return -EBUSY;
664
665 *map = ioremap(base + offset, size);
666 if (!*map) {
667 release_mem_region(base + offset, size);
668 return -ENOMEM;
669 }
670
671 return 0;
672}
673
674static void mthca_unmap_reg(struct mthca_dev *dev, unsigned long offset,
675 unsigned long size, void __iomem *map)
676{
677 unsigned long base = pci_resource_start(dev->pdev, 0);
678
679 release_mem_region(base + offset, size);
680 iounmap(map);
681}
682
683static int __devinit mthca_map_eq_regs(struct mthca_dev *dev)
684{
685 unsigned long mthca_base;
686
687 mthca_base = pci_resource_start(dev->pdev, 0);
688
689 if (dev->hca_type == ARBEL_NATIVE) {
690 /*
691 * We assume that the EQ arm and EQ set CI registers
692 * fall within the first BAR. We can't trust the
693 * values firmware gives us, since those addresses are
694 * valid on the HCA's side of the PCI bus but not
695 * necessarily the host side.
696 */
697 if (mthca_map_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
698 dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE,
699 &dev->clr_base)) {
700 mthca_err(dev, "Couldn't map interrupt clear register, "
701 "aborting.\n");
702 return -ENOMEM;
703 }
704
705 /*
706 * Add 4 because we limit ourselves to EQs 0 ... 31,
707 * so we only need the low word of the register.
708 */
709 if (mthca_map_reg(dev, ((pci_resource_len(dev->pdev, 0) - 1) &
710 dev->fw.arbel.eq_arm_base) + 4, 4,
711 &dev->eq_regs.arbel.eq_arm)) {
712 mthca_err(dev, "Couldn't map interrupt clear register, "
713 "aborting.\n");
714 mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
715 dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE,
716 dev->clr_base);
717 return -ENOMEM;
718 }
719
720 if (mthca_map_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
721 dev->fw.arbel.eq_set_ci_base,
722 MTHCA_EQ_SET_CI_SIZE,
723 &dev->eq_regs.arbel.eq_set_ci_base)) {
724 mthca_err(dev, "Couldn't map interrupt clear register, "
725 "aborting.\n");
726 mthca_unmap_reg(dev, ((pci_resource_len(dev->pdev, 0) - 1) &
727 dev->fw.arbel.eq_arm_base) + 4, 4,
728 dev->eq_regs.arbel.eq_arm);
729 mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
730 dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE,
731 dev->clr_base);
732 return -ENOMEM;
733 }
734 } else {
735 if (mthca_map_reg(dev, MTHCA_CLR_INT_BASE, MTHCA_CLR_INT_SIZE,
736 &dev->clr_base)) {
737 mthca_err(dev, "Couldn't map interrupt clear register, "
738 "aborting.\n");
739 return -ENOMEM;
740 }
741
742 if (mthca_map_reg(dev, MTHCA_ECR_BASE,
743 MTHCA_ECR_SIZE + MTHCA_ECR_CLR_SIZE,
744 &dev->eq_regs.tavor.ecr_base)) {
745 mthca_err(dev, "Couldn't map ecr register, "
746 "aborting.\n");
747 mthca_unmap_reg(dev, MTHCA_CLR_INT_BASE, MTHCA_CLR_INT_SIZE,
748 dev->clr_base);
749 return -ENOMEM;
750 }
751 }
752
753 return 0;
754
755}
756
757static void __devexit mthca_unmap_eq_regs(struct mthca_dev *dev)
758{
759 if (dev->hca_type == ARBEL_NATIVE) {
760 mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
761 dev->fw.arbel.eq_set_ci_base,
762 MTHCA_EQ_SET_CI_SIZE,
763 dev->eq_regs.arbel.eq_set_ci_base);
764 mthca_unmap_reg(dev, ((pci_resource_len(dev->pdev, 0) - 1) &
765 dev->fw.arbel.eq_arm_base) + 4, 4,
766 dev->eq_regs.arbel.eq_arm);
767 mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
768 dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE,
769 dev->clr_base);
770 } else {
771 mthca_unmap_reg(dev, MTHCA_ECR_BASE,
772 MTHCA_ECR_SIZE + MTHCA_ECR_CLR_SIZE,
773 dev->eq_regs.tavor.ecr_base);
774 mthca_unmap_reg(dev, MTHCA_CLR_INT_BASE, MTHCA_CLR_INT_SIZE,
775 dev->clr_base);
776 }
777}
778
779int __devinit mthca_map_eq_icm(struct mthca_dev *dev, u64 icm_virt)
780{
781 int ret;
782 u8 status;
783
784 /*
785 * We assume that mapping one page is enough for the whole EQ
786 * context table. This is fine with all current HCAs, because
787 * we only use 32 EQs and each EQ uses 32 bytes of context
788 * memory, or 1 KB total.
789 */
790 dev->eq_table.icm_virt = icm_virt;
791 dev->eq_table.icm_page = alloc_page(GFP_HIGHUSER);
792 if (!dev->eq_table.icm_page)
793 return -ENOMEM;
794 dev->eq_table.icm_dma = pci_map_page(dev->pdev, dev->eq_table.icm_page, 0,
795 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
796 if (pci_dma_mapping_error(dev->eq_table.icm_dma)) {
797 __free_page(dev->eq_table.icm_page);
798 return -ENOMEM;
799 }
800
801 ret = mthca_MAP_ICM_page(dev, dev->eq_table.icm_dma, icm_virt, &status);
802 if (!ret && status)
803 ret = -EINVAL;
804 if (ret) {
805 pci_unmap_page(dev->pdev, dev->eq_table.icm_dma, PAGE_SIZE,
806 PCI_DMA_BIDIRECTIONAL);
807 __free_page(dev->eq_table.icm_page);
808 }
809
810 return ret;
811}
812
813void __devexit mthca_unmap_eq_icm(struct mthca_dev *dev)
814{
815 u8 status;
816
817 mthca_UNMAP_ICM(dev, dev->eq_table.icm_virt, PAGE_SIZE / 4096, &status);
818 pci_unmap_page(dev->pdev, dev->eq_table.icm_dma, PAGE_SIZE,
819 PCI_DMA_BIDIRECTIONAL);
820 __free_page(dev->eq_table.icm_page);
821}
822
823int __devinit mthca_init_eq_table(struct mthca_dev *dev)
824{
825 int err;
826 u8 status;
827 u8 intr;
828 int i;
829
830 err = mthca_alloc_init(&dev->eq_table.alloc,
831 dev->limits.num_eqs,
832 dev->limits.num_eqs - 1,
833 dev->limits.reserved_eqs);
834 if (err)
835 return err;
836
837 err = mthca_map_eq_regs(dev);
838 if (err)
839 goto err_out_free;
840
841 if (dev->mthca_flags & MTHCA_FLAG_MSI ||
842 dev->mthca_flags & MTHCA_FLAG_MSI_X) {
843 dev->eq_table.clr_mask = 0;
844 } else {
845 dev->eq_table.clr_mask =
846 swab32(1 << (dev->eq_table.inta_pin & 31));
847 dev->eq_table.clr_int = dev->clr_base +
848 (dev->eq_table.inta_pin < 31 ? 4 : 0);
849 }
850
851 dev->eq_table.arm_mask = 0;
852
853 intr = (dev->mthca_flags & MTHCA_FLAG_MSI) ?
854 128 : dev->eq_table.inta_pin;
855
856 err = mthca_create_eq(dev, dev->limits.num_cqs,
857 (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 128 : intr,
858 &dev->eq_table.eq[MTHCA_EQ_COMP]);
859 if (err)
860 goto err_out_unmap;
861
862 err = mthca_create_eq(dev, MTHCA_NUM_ASYNC_EQE,
863 (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 129 : intr,
864 &dev->eq_table.eq[MTHCA_EQ_ASYNC]);
865 if (err)
866 goto err_out_comp;
867
868 err = mthca_create_eq(dev, MTHCA_NUM_CMD_EQE,
869 (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 130 : intr,
870 &dev->eq_table.eq[MTHCA_EQ_CMD]);
871 if (err)
872 goto err_out_async;
873
874 if (dev->mthca_flags & MTHCA_FLAG_MSI_X) {
875 static const char *eq_name[] = {
876 [MTHCA_EQ_COMP] = DRV_NAME " (comp)",
877 [MTHCA_EQ_ASYNC] = DRV_NAME " (async)",
878 [MTHCA_EQ_CMD] = DRV_NAME " (cmd)"
879 };
880
881 for (i = 0; i < MTHCA_NUM_EQ; ++i) {
882 err = request_irq(dev->eq_table.eq[i].msi_x_vector,
883 dev->hca_type == ARBEL_NATIVE ?
884 mthca_arbel_msi_x_interrupt :
885 mthca_tavor_msi_x_interrupt,
886 0, eq_name[i], dev->eq_table.eq + i);
887 if (err)
888 goto err_out_cmd;
889 dev->eq_table.eq[i].have_irq = 1;
890 }
891 } else {
892 err = request_irq(dev->pdev->irq,
893 dev->hca_type == ARBEL_NATIVE ?
894 mthca_arbel_interrupt :
895 mthca_tavor_interrupt,
896 SA_SHIRQ, DRV_NAME, dev);
897 if (err)
898 goto err_out_cmd;
899 dev->eq_table.have_irq = 1;
900 }
901
902 err = mthca_MAP_EQ(dev, async_mask(dev),
903 0, dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, &status);
904 if (err)
905 mthca_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n",
906 dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, err);
907 if (status)
908 mthca_warn(dev, "MAP_EQ for async EQ %d returned status 0x%02x\n",
909 dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, status);
910
911 err = mthca_MAP_EQ(dev, MTHCA_CMD_EVENT_MASK,
912 0, dev->eq_table.eq[MTHCA_EQ_CMD].eqn, &status);
913 if (err)
914 mthca_warn(dev, "MAP_EQ for cmd EQ %d failed (%d)\n",
915 dev->eq_table.eq[MTHCA_EQ_CMD].eqn, err);
916 if (status)
917 mthca_warn(dev, "MAP_EQ for cmd EQ %d returned status 0x%02x\n",
918 dev->eq_table.eq[MTHCA_EQ_CMD].eqn, status);
919
920 for (i = 0; i < MTHCA_EQ_CMD; ++i)
921 if (dev->hca_type == ARBEL_NATIVE)
922 arbel_eq_req_not(dev, dev->eq_table.eq[i].eqn_mask);
923 else
924 tavor_eq_req_not(dev, dev->eq_table.eq[i].eqn);
925
926 return 0;
927
928err_out_cmd:
929 mthca_free_irqs(dev);
930 mthca_free_eq(dev, &dev->eq_table.eq[MTHCA_EQ_CMD]);
931
932err_out_async:
933 mthca_free_eq(dev, &dev->eq_table.eq[MTHCA_EQ_ASYNC]);
934
935err_out_comp:
936 mthca_free_eq(dev, &dev->eq_table.eq[MTHCA_EQ_COMP]);
937
938err_out_unmap:
939 mthca_unmap_eq_regs(dev);
940
941err_out_free:
942 mthca_alloc_cleanup(&dev->eq_table.alloc);
943 return err;
944}
945
946void __devexit mthca_cleanup_eq_table(struct mthca_dev *dev)
947{
948 u8 status;
949 int i;
950
951 mthca_free_irqs(dev);
952
953 mthca_MAP_EQ(dev, async_mask(dev),
954 1, dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, &status);
955 mthca_MAP_EQ(dev, MTHCA_CMD_EVENT_MASK,
956 1, dev->eq_table.eq[MTHCA_EQ_CMD].eqn, &status);
957
958 for (i = 0; i < MTHCA_NUM_EQ; ++i)
959 mthca_free_eq(dev, &dev->eq_table.eq[i]);
960
961 mthca_unmap_eq_regs(dev);
962
963 mthca_alloc_cleanup(&dev->eq_table.alloc);
964}
diff --git a/drivers/infiniband/hw/mthca/mthca_mad.c b/drivers/infiniband/hw/mthca/mthca_mad.c
new file mode 100644
index 000000000000..7df223642015
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_mad.c
@@ -0,0 +1,323 @@
1/*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_mad.c 1349 2004-12-16 21:09:43Z roland $
33 */
34
35#include <ib_verbs.h>
36#include <ib_mad.h>
37#include <ib_smi.h>
38
39#include "mthca_dev.h"
40#include "mthca_cmd.h"
41
42enum {
43 MTHCA_VENDOR_CLASS1 = 0x9,
44 MTHCA_VENDOR_CLASS2 = 0xa
45};
46
47struct mthca_trap_mad {
48 struct ib_mad *mad;
49 DECLARE_PCI_UNMAP_ADDR(mapping)
50};
51
52static void update_sm_ah(struct mthca_dev *dev,
53 u8 port_num, u16 lid, u8 sl)
54{
55 struct ib_ah *new_ah;
56 struct ib_ah_attr ah_attr;
57 unsigned long flags;
58
59 if (!dev->send_agent[port_num - 1][0])
60 return;
61
62 memset(&ah_attr, 0, sizeof ah_attr);
63 ah_attr.dlid = lid;
64 ah_attr.sl = sl;
65 ah_attr.port_num = port_num;
66
67 new_ah = ib_create_ah(dev->send_agent[port_num - 1][0]->qp->pd,
68 &ah_attr);
69 if (IS_ERR(new_ah))
70 return;
71
72 spin_lock_irqsave(&dev->sm_lock, flags);
73 if (dev->sm_ah[port_num - 1])
74 ib_destroy_ah(dev->sm_ah[port_num - 1]);
75 dev->sm_ah[port_num - 1] = new_ah;
76 spin_unlock_irqrestore(&dev->sm_lock, flags);
77}
78
79/*
80 * Snoop SM MADs for port info and P_Key table sets, so we can
81 * synthesize LID change and P_Key change events.
82 */
83static void smp_snoop(struct ib_device *ibdev,
84 u8 port_num,
85 struct ib_mad *mad)
86{
87 struct ib_event event;
88
89 if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED ||
90 mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) &&
91 mad->mad_hdr.method == IB_MGMT_METHOD_SET) {
92 if (mad->mad_hdr.attr_id == IB_SMP_ATTR_PORT_INFO) {
93 update_sm_ah(to_mdev(ibdev), port_num,
94 be16_to_cpup((__be16 *) (mad->data + 58)),
95 (*(u8 *) (mad->data + 76)) & 0xf);
96
97 event.device = ibdev;
98 event.event = IB_EVENT_LID_CHANGE;
99 event.element.port_num = port_num;
100 ib_dispatch_event(&event);
101 }
102
103 if (mad->mad_hdr.attr_id == IB_SMP_ATTR_PKEY_TABLE) {
104 event.device = ibdev;
105 event.event = IB_EVENT_PKEY_CHANGE;
106 event.element.port_num = port_num;
107 ib_dispatch_event(&event);
108 }
109 }
110}
111
112static void forward_trap(struct mthca_dev *dev,
113 u8 port_num,
114 struct ib_mad *mad)
115{
116 int qpn = mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED;
117 struct mthca_trap_mad *tmad;
118 struct ib_sge gather_list;
119 struct ib_send_wr *bad_wr, wr = {
120 .opcode = IB_WR_SEND,
121 .sg_list = &gather_list,
122 .num_sge = 1,
123 .send_flags = IB_SEND_SIGNALED,
124 .wr = {
125 .ud = {
126 .remote_qpn = qpn,
127 .remote_qkey = qpn ? IB_QP1_QKEY : 0,
128 .timeout_ms = 0
129 }
130 }
131 };
132 struct ib_mad_agent *agent = dev->send_agent[port_num - 1][qpn];
133 int ret;
134 unsigned long flags;
135
136 if (agent) {
137 tmad = kmalloc(sizeof *tmad, GFP_KERNEL);
138 if (!tmad)
139 return;
140
141 tmad->mad = kmalloc(sizeof *tmad->mad, GFP_KERNEL);
142 if (!tmad->mad) {
143 kfree(tmad);
144 return;
145 }
146
147 memcpy(tmad->mad, mad, sizeof *mad);
148
149 wr.wr.ud.mad_hdr = &tmad->mad->mad_hdr;
150 wr.wr_id = (unsigned long) tmad;
151
152 gather_list.addr = dma_map_single(agent->device->dma_device,
153 tmad->mad,
154 sizeof *tmad->mad,
155 DMA_TO_DEVICE);
156 gather_list.length = sizeof *tmad->mad;
157 gather_list.lkey = to_mpd(agent->qp->pd)->ntmr.ibmr.lkey;
158 pci_unmap_addr_set(tmad, mapping, gather_list.addr);
159
160 /*
161 * We rely here on the fact that MLX QPs don't use the
162 * address handle after the send is posted (this is
163 * wrong following the IB spec strictly, but we know
164 * it's OK for our devices).
165 */
166 spin_lock_irqsave(&dev->sm_lock, flags);
167 wr.wr.ud.ah = dev->sm_ah[port_num - 1];
168 if (wr.wr.ud.ah)
169 ret = ib_post_send_mad(agent, &wr, &bad_wr);
170 else
171 ret = -EINVAL;
172 spin_unlock_irqrestore(&dev->sm_lock, flags);
173
174 if (ret) {
175 dma_unmap_single(agent->device->dma_device,
176 pci_unmap_addr(tmad, mapping),
177 sizeof *tmad->mad,
178 DMA_TO_DEVICE);
179 kfree(tmad->mad);
180 kfree(tmad);
181 }
182 }
183}
184
185int mthca_process_mad(struct ib_device *ibdev,
186 int mad_flags,
187 u8 port_num,
188 struct ib_wc *in_wc,
189 struct ib_grh *in_grh,
190 struct ib_mad *in_mad,
191 struct ib_mad *out_mad)
192{
193 int err;
194 u8 status;
195 u16 slid = in_wc ? in_wc->slid : IB_LID_PERMISSIVE;
196
197 /* Forward locally generated traps to the SM */
198 if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP &&
199 slid == 0) {
200 forward_trap(to_mdev(ibdev), port_num, in_mad);
201 return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;
202 }
203
204 /*
205 * Only handle SM gets, sets and trap represses for SM class
206 *
207 * Only handle PMA and Mellanox vendor-specific class gets and
208 * sets for other classes.
209 */
210 if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED ||
211 in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
212 if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET &&
213 in_mad->mad_hdr.method != IB_MGMT_METHOD_SET &&
214 in_mad->mad_hdr.method != IB_MGMT_METHOD_TRAP_REPRESS)
215 return IB_MAD_RESULT_SUCCESS;
216
217 /*
218 * Don't process SMInfo queries or vendor-specific
219 * MADs -- the SMA can't handle them.
220 */
221 if (in_mad->mad_hdr.attr_id == IB_SMP_ATTR_SM_INFO ||
222 ((in_mad->mad_hdr.attr_id & IB_SMP_ATTR_VENDOR_MASK) ==
223 IB_SMP_ATTR_VENDOR_MASK))
224 return IB_MAD_RESULT_SUCCESS;
225 } else if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT ||
226 in_mad->mad_hdr.mgmt_class == MTHCA_VENDOR_CLASS1 ||
227 in_mad->mad_hdr.mgmt_class == MTHCA_VENDOR_CLASS2) {
228 if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET &&
229 in_mad->mad_hdr.method != IB_MGMT_METHOD_SET)
230 return IB_MAD_RESULT_SUCCESS;
231 } else
232 return IB_MAD_RESULT_SUCCESS;
233
234 err = mthca_MAD_IFC(to_mdev(ibdev),
235 mad_flags & IB_MAD_IGNORE_MKEY,
236 mad_flags & IB_MAD_IGNORE_BKEY,
237 port_num, in_wc, in_grh, in_mad, out_mad,
238 &status);
239 if (err) {
240 mthca_err(to_mdev(ibdev), "MAD_IFC failed\n");
241 return IB_MAD_RESULT_FAILURE;
242 }
243 if (status == MTHCA_CMD_STAT_BAD_PKT)
244 return IB_MAD_RESULT_SUCCESS;
245 if (status) {
246 mthca_err(to_mdev(ibdev), "MAD_IFC returned status %02x\n",
247 status);
248 return IB_MAD_RESULT_FAILURE;
249 }
250
251 if (!out_mad->mad_hdr.status)
252 smp_snoop(ibdev, port_num, in_mad);
253
254 /* set return bit in status of directed route responses */
255 if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
256 out_mad->mad_hdr.status |= cpu_to_be16(1 << 15);
257
258 if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP_REPRESS)
259 /* no response for trap repress */
260 return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;
261
262 return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
263}
264
265static void send_handler(struct ib_mad_agent *agent,
266 struct ib_mad_send_wc *mad_send_wc)
267{
268 struct mthca_trap_mad *tmad =
269 (void *) (unsigned long) mad_send_wc->wr_id;
270
271 dma_unmap_single(agent->device->dma_device,
272 pci_unmap_addr(tmad, mapping),
273 sizeof *tmad->mad,
274 DMA_TO_DEVICE);
275 kfree(tmad->mad);
276 kfree(tmad);
277}
278
279int mthca_create_agents(struct mthca_dev *dev)
280{
281 struct ib_mad_agent *agent;
282 int p, q;
283
284 spin_lock_init(&dev->sm_lock);
285
286 for (p = 0; p < dev->limits.num_ports; ++p)
287 for (q = 0; q <= 1; ++q) {
288 agent = ib_register_mad_agent(&dev->ib_dev, p + 1,
289 q ? IB_QPT_GSI : IB_QPT_SMI,
290 NULL, 0, send_handler,
291 NULL, NULL);
292 if (IS_ERR(agent))
293 goto err;
294 dev->send_agent[p][q] = agent;
295 }
296
297 return 0;
298
299err:
300 for (p = 0; p < dev->limits.num_ports; ++p)
301 for (q = 0; q <= 1; ++q)
302 if (dev->send_agent[p][q])
303 ib_unregister_mad_agent(dev->send_agent[p][q]);
304
305 return PTR_ERR(agent);
306}
307
308void mthca_free_agents(struct mthca_dev *dev)
309{
310 struct ib_mad_agent *agent;
311 int p, q;
312
313 for (p = 0; p < dev->limits.num_ports; ++p) {
314 for (q = 0; q <= 1; ++q) {
315 agent = dev->send_agent[p][q];
316 dev->send_agent[p][q] = NULL;
317 ib_unregister_mad_agent(agent);
318 }
319
320 if (dev->sm_ah[p])
321 ib_destroy_ah(dev->sm_ah[p]);
322 }
323}
diff --git a/drivers/infiniband/hw/mthca/mthca_main.c b/drivers/infiniband/hw/mthca/mthca_main.c
new file mode 100644
index 000000000000..9e782bc1c38d
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_main.c
@@ -0,0 +1,1123 @@
1/*
2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_main.c 1396 2004-12-28 04:10:27Z roland $
33 */
34
35#include <linux/config.h>
36#include <linux/version.h>
37#include <linux/module.h>
38#include <linux/init.h>
39#include <linux/errno.h>
40#include <linux/pci.h>
41#include <linux/interrupt.h>
42
43#include "mthca_dev.h"
44#include "mthca_config_reg.h"
45#include "mthca_cmd.h"
46#include "mthca_profile.h"
47#include "mthca_memfree.h"
48
49MODULE_AUTHOR("Roland Dreier");
50MODULE_DESCRIPTION("Mellanox InfiniBand HCA low-level driver");
51MODULE_LICENSE("Dual BSD/GPL");
52MODULE_VERSION(DRV_VERSION);
53
54#ifdef CONFIG_PCI_MSI
55
56static int msi_x = 0;
57module_param(msi_x, int, 0444);
58MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero");
59
60static int msi = 0;
61module_param(msi, int, 0444);
62MODULE_PARM_DESC(msi, "attempt to use MSI if nonzero");
63
64#else /* CONFIG_PCI_MSI */
65
66#define msi_x (0)
67#define msi (0)
68
69#endif /* CONFIG_PCI_MSI */
70
71static const char mthca_version[] __devinitdata =
72 "ib_mthca: Mellanox InfiniBand HCA driver v"
73 DRV_VERSION " (" DRV_RELDATE ")\n";
74
75static struct mthca_profile default_profile = {
76 .num_qp = 1 << 16,
77 .rdb_per_qp = 4,
78 .num_cq = 1 << 16,
79 .num_mcg = 1 << 13,
80 .num_mpt = 1 << 17,
81 .num_mtt = 1 << 20,
82 .num_udav = 1 << 15, /* Tavor only */
83 .uarc_size = 1 << 18, /* Arbel only */
84};
85
86static int __devinit mthca_tune_pci(struct mthca_dev *mdev)
87{
88 int cap;
89 u16 val;
90
91 /* First try to max out Read Byte Count */
92 cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_PCIX);
93 if (cap) {
94 if (pci_read_config_word(mdev->pdev, cap + PCI_X_CMD, &val)) {
95 mthca_err(mdev, "Couldn't read PCI-X command register, "
96 "aborting.\n");
97 return -ENODEV;
98 }
99 val = (val & ~PCI_X_CMD_MAX_READ) | (3 << 2);
100 if (pci_write_config_word(mdev->pdev, cap + PCI_X_CMD, val)) {
101 mthca_err(mdev, "Couldn't write PCI-X command register, "
102 "aborting.\n");
103 return -ENODEV;
104 }
105 } else if (mdev->hca_type == TAVOR)
106 mthca_info(mdev, "No PCI-X capability, not setting RBC.\n");
107
108 cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_EXP);
109 if (cap) {
110 if (pci_read_config_word(mdev->pdev, cap + PCI_EXP_DEVCTL, &val)) {
111 mthca_err(mdev, "Couldn't read PCI Express device control "
112 "register, aborting.\n");
113 return -ENODEV;
114 }
115 val = (val & ~PCI_EXP_DEVCTL_READRQ) | (5 << 12);
116 if (pci_write_config_word(mdev->pdev, cap + PCI_EXP_DEVCTL, val)) {
117 mthca_err(mdev, "Couldn't write PCI Express device control "
118 "register, aborting.\n");
119 return -ENODEV;
120 }
121 } else if (mdev->hca_type == ARBEL_NATIVE ||
122 mdev->hca_type == ARBEL_COMPAT)
123 mthca_info(mdev, "No PCI Express capability, "
124 "not setting Max Read Request Size.\n");
125
126 return 0;
127}
128
129static int __devinit mthca_dev_lim(struct mthca_dev *mdev, struct mthca_dev_lim *dev_lim)
130{
131 int err;
132 u8 status;
133
134 err = mthca_QUERY_DEV_LIM(mdev, dev_lim, &status);
135 if (err) {
136 mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n");
137 return err;
138 }
139 if (status) {
140 mthca_err(mdev, "QUERY_DEV_LIM returned status 0x%02x, "
141 "aborting.\n", status);
142 return -EINVAL;
143 }
144 if (dev_lim->min_page_sz > PAGE_SIZE) {
145 mthca_err(mdev, "HCA minimum page size of %d bigger than "
146 "kernel PAGE_SIZE of %ld, aborting.\n",
147 dev_lim->min_page_sz, PAGE_SIZE);
148 return -ENODEV;
149 }
150 if (dev_lim->num_ports > MTHCA_MAX_PORTS) {
151 mthca_err(mdev, "HCA has %d ports, but we only support %d, "
152 "aborting.\n",
153 dev_lim->num_ports, MTHCA_MAX_PORTS);
154 return -ENODEV;
155 }
156
157 mdev->limits.num_ports = dev_lim->num_ports;
158 mdev->limits.vl_cap = dev_lim->max_vl;
159 mdev->limits.mtu_cap = dev_lim->max_mtu;
160 mdev->limits.gid_table_len = dev_lim->max_gids;
161 mdev->limits.pkey_table_len = dev_lim->max_pkeys;
162 mdev->limits.local_ca_ack_delay = dev_lim->local_ca_ack_delay;
163 mdev->limits.max_sg = dev_lim->max_sg;
164 mdev->limits.reserved_qps = dev_lim->reserved_qps;
165 mdev->limits.reserved_srqs = dev_lim->reserved_srqs;
166 mdev->limits.reserved_eecs = dev_lim->reserved_eecs;
167 mdev->limits.reserved_cqs = dev_lim->reserved_cqs;
168 mdev->limits.reserved_eqs = dev_lim->reserved_eqs;
169 mdev->limits.reserved_mtts = dev_lim->reserved_mtts;
170 mdev->limits.reserved_mrws = dev_lim->reserved_mrws;
171 mdev->limits.reserved_uars = dev_lim->reserved_uars;
172 mdev->limits.reserved_pds = dev_lim->reserved_pds;
173
174 /* IB_DEVICE_RESIZE_MAX_WR not supported by driver.
175 May be doable since hardware supports it for SRQ.
176
177 IB_DEVICE_N_NOTIFY_CQ is supported by hardware but not by driver.
178
179 IB_DEVICE_SRQ_RESIZE is supported by hardware but SRQ is not
180 supported by driver. */
181 mdev->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT |
182 IB_DEVICE_PORT_ACTIVE_EVENT |
183 IB_DEVICE_SYS_IMAGE_GUID |
184 IB_DEVICE_RC_RNR_NAK_GEN;
185
186 if (dev_lim->flags & DEV_LIM_FLAG_BAD_PKEY_CNTR)
187 mdev->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
188
189 if (dev_lim->flags & DEV_LIM_FLAG_BAD_QKEY_CNTR)
190 mdev->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
191
192 if (dev_lim->flags & DEV_LIM_FLAG_RAW_MULTI)
193 mdev->device_cap_flags |= IB_DEVICE_RAW_MULTI;
194
195 if (dev_lim->flags & DEV_LIM_FLAG_AUTO_PATH_MIG)
196 mdev->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
197
198 if (dev_lim->flags & DEV_LIM_FLAG_UD_AV_PORT_ENFORCE)
199 mdev->device_cap_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE;
200
201 if (dev_lim->flags & DEV_LIM_FLAG_SRQ)
202 mdev->mthca_flags |= MTHCA_FLAG_SRQ;
203
204 return 0;
205}
206
207static int __devinit mthca_init_tavor(struct mthca_dev *mdev)
208{
209 u8 status;
210 int err;
211 struct mthca_dev_lim dev_lim;
212 struct mthca_profile profile;
213 struct mthca_init_hca_param init_hca;
214 struct mthca_adapter adapter;
215
216 err = mthca_SYS_EN(mdev, &status);
217 if (err) {
218 mthca_err(mdev, "SYS_EN command failed, aborting.\n");
219 return err;
220 }
221 if (status) {
222 mthca_err(mdev, "SYS_EN returned status 0x%02x, "
223 "aborting.\n", status);
224 return -EINVAL;
225 }
226
227 err = mthca_QUERY_FW(mdev, &status);
228 if (err) {
229 mthca_err(mdev, "QUERY_FW command failed, aborting.\n");
230 goto err_disable;
231 }
232 if (status) {
233 mthca_err(mdev, "QUERY_FW returned status 0x%02x, "
234 "aborting.\n", status);
235 err = -EINVAL;
236 goto err_disable;
237 }
238 err = mthca_QUERY_DDR(mdev, &status);
239 if (err) {
240 mthca_err(mdev, "QUERY_DDR command failed, aborting.\n");
241 goto err_disable;
242 }
243 if (status) {
244 mthca_err(mdev, "QUERY_DDR returned status 0x%02x, "
245 "aborting.\n", status);
246 err = -EINVAL;
247 goto err_disable;
248 }
249
250 err = mthca_dev_lim(mdev, &dev_lim);
251
252 profile = default_profile;
253 profile.num_uar = dev_lim.uar_size / PAGE_SIZE;
254 profile.uarc_size = 0;
255
256 err = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca);
257 if (err < 0)
258 goto err_disable;
259
260 err = mthca_INIT_HCA(mdev, &init_hca, &status);
261 if (err) {
262 mthca_err(mdev, "INIT_HCA command failed, aborting.\n");
263 goto err_disable;
264 }
265 if (status) {
266 mthca_err(mdev, "INIT_HCA returned status 0x%02x, "
267 "aborting.\n", status);
268 err = -EINVAL;
269 goto err_disable;
270 }
271
272 err = mthca_QUERY_ADAPTER(mdev, &adapter, &status);
273 if (err) {
274 mthca_err(mdev, "QUERY_ADAPTER command failed, aborting.\n");
275 goto err_close;
276 }
277 if (status) {
278 mthca_err(mdev, "QUERY_ADAPTER returned status 0x%02x, "
279 "aborting.\n", status);
280 err = -EINVAL;
281 goto err_close;
282 }
283
284 mdev->eq_table.inta_pin = adapter.inta_pin;
285 mdev->rev_id = adapter.revision_id;
286
287 return 0;
288
289err_close:
290 mthca_CLOSE_HCA(mdev, 0, &status);
291
292err_disable:
293 mthca_SYS_DIS(mdev, &status);
294
295 return err;
296}
297
298static int __devinit mthca_load_fw(struct mthca_dev *mdev)
299{
300 u8 status;
301 int err;
302
303 /* FIXME: use HCA-attached memory for FW if present */
304
305 mdev->fw.arbel.fw_icm =
306 mthca_alloc_icm(mdev, mdev->fw.arbel.fw_pages,
307 GFP_HIGHUSER | __GFP_NOWARN);
308 if (!mdev->fw.arbel.fw_icm) {
309 mthca_err(mdev, "Couldn't allocate FW area, aborting.\n");
310 return -ENOMEM;
311 }
312
313 err = mthca_MAP_FA(mdev, mdev->fw.arbel.fw_icm, &status);
314 if (err) {
315 mthca_err(mdev, "MAP_FA command failed, aborting.\n");
316 goto err_free;
317 }
318 if (status) {
319 mthca_err(mdev, "MAP_FA returned status 0x%02x, aborting.\n", status);
320 err = -EINVAL;
321 goto err_free;
322 }
323 err = mthca_RUN_FW(mdev, &status);
324 if (err) {
325 mthca_err(mdev, "RUN_FW command failed, aborting.\n");
326 goto err_unmap_fa;
327 }
328 if (status) {
329 mthca_err(mdev, "RUN_FW returned status 0x%02x, aborting.\n", status);
330 err = -EINVAL;
331 goto err_unmap_fa;
332 }
333
334 return 0;
335
336err_unmap_fa:
337 mthca_UNMAP_FA(mdev, &status);
338
339err_free:
340 mthca_free_icm(mdev, mdev->fw.arbel.fw_icm);
341 return err;
342}
343
344static int __devinit mthca_init_icm(struct mthca_dev *mdev,
345 struct mthca_dev_lim *dev_lim,
346 struct mthca_init_hca_param *init_hca,
347 u64 icm_size)
348{
349 u64 aux_pages;
350 u8 status;
351 int err;
352
353 err = mthca_SET_ICM_SIZE(mdev, icm_size, &aux_pages, &status);
354 if (err) {
355 mthca_err(mdev, "SET_ICM_SIZE command failed, aborting.\n");
356 return err;
357 }
358 if (status) {
359 mthca_err(mdev, "SET_ICM_SIZE returned status 0x%02x, "
360 "aborting.\n", status);
361 return -EINVAL;
362 }
363
364 mthca_dbg(mdev, "%lld KB of HCA context requires %lld KB aux memory.\n",
365 (unsigned long long) icm_size >> 10,
366 (unsigned long long) aux_pages << 2);
367
368 mdev->fw.arbel.aux_icm = mthca_alloc_icm(mdev, aux_pages,
369 GFP_HIGHUSER | __GFP_NOWARN);
370 if (!mdev->fw.arbel.aux_icm) {
371 mthca_err(mdev, "Couldn't allocate aux memory, aborting.\n");
372 return -ENOMEM;
373 }
374
375 err = mthca_MAP_ICM_AUX(mdev, mdev->fw.arbel.aux_icm, &status);
376 if (err) {
377 mthca_err(mdev, "MAP_ICM_AUX command failed, aborting.\n");
378 goto err_free_aux;
379 }
380 if (status) {
381 mthca_err(mdev, "MAP_ICM_AUX returned status 0x%02x, aborting.\n", status);
382 err = -EINVAL;
383 goto err_free_aux;
384 }
385
386 err = mthca_map_eq_icm(mdev, init_hca->eqc_base);
387 if (err) {
388 mthca_err(mdev, "Failed to map EQ context memory, aborting.\n");
389 goto err_unmap_aux;
390 }
391
392 mdev->mr_table.mtt_table = mthca_alloc_icm_table(mdev, init_hca->mtt_base,
393 init_hca->mtt_seg_sz,
394 mdev->limits.num_mtt_segs,
395 mdev->limits.reserved_mtts, 1);
396 if (!mdev->mr_table.mtt_table) {
397 mthca_err(mdev, "Failed to map MTT context memory, aborting.\n");
398 err = -ENOMEM;
399 goto err_unmap_eq;
400 }
401
402 mdev->mr_table.mpt_table = mthca_alloc_icm_table(mdev, init_hca->mpt_base,
403 dev_lim->mpt_entry_sz,
404 mdev->limits.num_mpts,
405 mdev->limits.reserved_mrws, 1);
406 if (!mdev->mr_table.mpt_table) {
407 mthca_err(mdev, "Failed to map MPT context memory, aborting.\n");
408 err = -ENOMEM;
409 goto err_unmap_mtt;
410 }
411
412 mdev->qp_table.qp_table = mthca_alloc_icm_table(mdev, init_hca->qpc_base,
413 dev_lim->qpc_entry_sz,
414 mdev->limits.num_qps,
415 mdev->limits.reserved_qps, 0);
416 if (!mdev->qp_table.qp_table) {
417 mthca_err(mdev, "Failed to map QP context memory, aborting.\n");
418 err = -ENOMEM;
419 goto err_unmap_mpt;
420 }
421
422 mdev->qp_table.eqp_table = mthca_alloc_icm_table(mdev, init_hca->eqpc_base,
423 dev_lim->eqpc_entry_sz,
424 mdev->limits.num_qps,
425 mdev->limits.reserved_qps, 0);
426 if (!mdev->qp_table.eqp_table) {
427 mthca_err(mdev, "Failed to map EQP context memory, aborting.\n");
428 err = -ENOMEM;
429 goto err_unmap_qp;
430 }
431
432 mdev->cq_table.table = mthca_alloc_icm_table(mdev, init_hca->cqc_base,
433 dev_lim->cqc_entry_sz,
434 mdev->limits.num_cqs,
435 mdev->limits.reserved_cqs, 0);
436 if (!mdev->cq_table.table) {
437 mthca_err(mdev, "Failed to map CQ context memory, aborting.\n");
438 err = -ENOMEM;
439 goto err_unmap_eqp;
440 }
441
442 /*
443 * It's not strictly required, but for simplicity just map the
444 * whole multicast group table now. The table isn't very big
445 * and it's a lot easier than trying to track ref counts.
446 */
447 mdev->mcg_table.table = mthca_alloc_icm_table(mdev, init_hca->mc_base,
448 MTHCA_MGM_ENTRY_SIZE,
449 mdev->limits.num_mgms +
450 mdev->limits.num_amgms,
451 mdev->limits.num_mgms +
452 mdev->limits.num_amgms,
453 0);
454 if (!mdev->mcg_table.table) {
455 mthca_err(mdev, "Failed to map MCG context memory, aborting.\n");
456 err = -ENOMEM;
457 goto err_unmap_cq;
458 }
459
460 return 0;
461
462err_unmap_cq:
463 mthca_free_icm_table(mdev, mdev->cq_table.table);
464
465err_unmap_eqp:
466 mthca_free_icm_table(mdev, mdev->qp_table.eqp_table);
467
468err_unmap_qp:
469 mthca_free_icm_table(mdev, mdev->qp_table.qp_table);
470
471err_unmap_mpt:
472 mthca_free_icm_table(mdev, mdev->mr_table.mpt_table);
473
474err_unmap_mtt:
475 mthca_free_icm_table(mdev, mdev->mr_table.mtt_table);
476
477err_unmap_eq:
478 mthca_unmap_eq_icm(mdev);
479
480err_unmap_aux:
481 mthca_UNMAP_ICM_AUX(mdev, &status);
482
483err_free_aux:
484 mthca_free_icm(mdev, mdev->fw.arbel.aux_icm);
485
486 return err;
487}
488
489static int __devinit mthca_init_arbel(struct mthca_dev *mdev)
490{
491 struct mthca_dev_lim dev_lim;
492 struct mthca_profile profile;
493 struct mthca_init_hca_param init_hca;
494 struct mthca_adapter adapter;
495 u64 icm_size;
496 u8 status;
497 int err;
498
499 err = mthca_QUERY_FW(mdev, &status);
500 if (err) {
501 mthca_err(mdev, "QUERY_FW command failed, aborting.\n");
502 return err;
503 }
504 if (status) {
505 mthca_err(mdev, "QUERY_FW returned status 0x%02x, "
506 "aborting.\n", status);
507 return -EINVAL;
508 }
509
510 err = mthca_ENABLE_LAM(mdev, &status);
511 if (err) {
512 mthca_err(mdev, "ENABLE_LAM command failed, aborting.\n");
513 return err;
514 }
515 if (status == MTHCA_CMD_STAT_LAM_NOT_PRE) {
516 mthca_dbg(mdev, "No HCA-attached memory (running in MemFree mode)\n");
517 mdev->mthca_flags |= MTHCA_FLAG_NO_LAM;
518 } else if (status) {
519 mthca_err(mdev, "ENABLE_LAM returned status 0x%02x, "
520 "aborting.\n", status);
521 return -EINVAL;
522 }
523
524 err = mthca_load_fw(mdev);
525 if (err) {
526 mthca_err(mdev, "Failed to start FW, aborting.\n");
527 goto err_disable;
528 }
529
530 err = mthca_dev_lim(mdev, &dev_lim);
531 if (err) {
532 mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n");
533 goto err_stop_fw;
534 }
535
536 profile = default_profile;
537 profile.num_uar = dev_lim.uar_size / PAGE_SIZE;
538 profile.num_udav = 0;
539
540 icm_size = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca);
541 if ((int) icm_size < 0) {
542 err = icm_size;
543 goto err_stop_fw;
544 }
545
546 err = mthca_init_icm(mdev, &dev_lim, &init_hca, icm_size);
547 if (err)
548 goto err_stop_fw;
549
550 err = mthca_INIT_HCA(mdev, &init_hca, &status);
551 if (err) {
552 mthca_err(mdev, "INIT_HCA command failed, aborting.\n");
553 goto err_free_icm;
554 }
555 if (status) {
556 mthca_err(mdev, "INIT_HCA returned status 0x%02x, "
557 "aborting.\n", status);
558 err = -EINVAL;
559 goto err_free_icm;
560 }
561
562 err = mthca_QUERY_ADAPTER(mdev, &adapter, &status);
563 if (err) {
564 mthca_err(mdev, "QUERY_ADAPTER command failed, aborting.\n");
565 goto err_free_icm;
566 }
567 if (status) {
568 mthca_err(mdev, "QUERY_ADAPTER returned status 0x%02x, "
569 "aborting.\n", status);
570 err = -EINVAL;
571 goto err_free_icm;
572 }
573
574 mdev->eq_table.inta_pin = adapter.inta_pin;
575 mdev->rev_id = adapter.revision_id;
576
577 return 0;
578
579err_free_icm:
580 mthca_free_icm_table(mdev, mdev->cq_table.table);
581 mthca_free_icm_table(mdev, mdev->qp_table.eqp_table);
582 mthca_free_icm_table(mdev, mdev->qp_table.qp_table);
583 mthca_free_icm_table(mdev, mdev->mr_table.mpt_table);
584 mthca_free_icm_table(mdev, mdev->mr_table.mtt_table);
585 mthca_unmap_eq_icm(mdev);
586
587 mthca_UNMAP_ICM_AUX(mdev, &status);
588 mthca_free_icm(mdev, mdev->fw.arbel.aux_icm);
589
590err_stop_fw:
591 mthca_UNMAP_FA(mdev, &status);
592 mthca_free_icm(mdev, mdev->fw.arbel.fw_icm);
593
594err_disable:
595 if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM))
596 mthca_DISABLE_LAM(mdev, &status);
597
598 return err;
599}
600
601static int __devinit mthca_init_hca(struct mthca_dev *mdev)
602{
603 if (mdev->hca_type == ARBEL_NATIVE)
604 return mthca_init_arbel(mdev);
605 else
606 return mthca_init_tavor(mdev);
607}
608
609static int __devinit mthca_setup_hca(struct mthca_dev *dev)
610{
611 int err;
612 u8 status;
613
614 MTHCA_INIT_DOORBELL_LOCK(&dev->doorbell_lock);
615
616 err = mthca_init_uar_table(dev);
617 if (err) {
618 mthca_err(dev, "Failed to initialize "
619 "user access region table, aborting.\n");
620 return err;
621 }
622
623 err = mthca_uar_alloc(dev, &dev->driver_uar);
624 if (err) {
625 mthca_err(dev, "Failed to allocate driver access region, "
626 "aborting.\n");
627 goto err_uar_table_free;
628 }
629
630 dev->kar = ioremap(dev->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE);
631 if (!dev->kar) {
632 mthca_err(dev, "Couldn't map kernel access region, "
633 "aborting.\n");
634 err = -ENOMEM;
635 goto err_uar_free;
636 }
637
638 err = mthca_init_pd_table(dev);
639 if (err) {
640 mthca_err(dev, "Failed to initialize "
641 "protection domain table, aborting.\n");
642 goto err_kar_unmap;
643 }
644
645 err = mthca_init_mr_table(dev);
646 if (err) {
647 mthca_err(dev, "Failed to initialize "
648 "memory region table, aborting.\n");
649 goto err_pd_table_free;
650 }
651
652 err = mthca_pd_alloc(dev, &dev->driver_pd);
653 if (err) {
654 mthca_err(dev, "Failed to create driver PD, "
655 "aborting.\n");
656 goto err_mr_table_free;
657 }
658
659 err = mthca_init_eq_table(dev);
660 if (err) {
661 mthca_err(dev, "Failed to initialize "
662 "event queue table, aborting.\n");
663 goto err_pd_free;
664 }
665
666 err = mthca_cmd_use_events(dev);
667 if (err) {
668 mthca_err(dev, "Failed to switch to event-driven "
669 "firmware commands, aborting.\n");
670 goto err_eq_table_free;
671 }
672
673 err = mthca_NOP(dev, &status);
674 if (err || status) {
675 mthca_err(dev, "NOP command failed to generate interrupt, aborting.\n");
676 if (dev->mthca_flags & (MTHCA_FLAG_MSI | MTHCA_FLAG_MSI_X))
677 mthca_err(dev, "Try again with MSI/MSI-X disabled.\n");
678 else
679 mthca_err(dev, "BIOS or ACPI interrupt routing problem?\n");
680
681 goto err_cmd_poll;
682 }
683
684 mthca_dbg(dev, "NOP command IRQ test passed\n");
685
686 err = mthca_init_cq_table(dev);
687 if (err) {
688 mthca_err(dev, "Failed to initialize "
689 "completion queue table, aborting.\n");
690 goto err_cmd_poll;
691 }
692
693 err = mthca_init_qp_table(dev);
694 if (err) {
695 mthca_err(dev, "Failed to initialize "
696 "queue pair table, aborting.\n");
697 goto err_cq_table_free;
698 }
699
700 err = mthca_init_av_table(dev);
701 if (err) {
702 mthca_err(dev, "Failed to initialize "
703 "address vector table, aborting.\n");
704 goto err_qp_table_free;
705 }
706
707 err = mthca_init_mcg_table(dev);
708 if (err) {
709 mthca_err(dev, "Failed to initialize "
710 "multicast group table, aborting.\n");
711 goto err_av_table_free;
712 }
713
714 return 0;
715
716err_av_table_free:
717 mthca_cleanup_av_table(dev);
718
719err_qp_table_free:
720 mthca_cleanup_qp_table(dev);
721
722err_cq_table_free:
723 mthca_cleanup_cq_table(dev);
724
725err_cmd_poll:
726 mthca_cmd_use_polling(dev);
727
728err_eq_table_free:
729 mthca_cleanup_eq_table(dev);
730
731err_pd_free:
732 mthca_pd_free(dev, &dev->driver_pd);
733
734err_mr_table_free:
735 mthca_cleanup_mr_table(dev);
736
737err_pd_table_free:
738 mthca_cleanup_pd_table(dev);
739
740err_kar_unmap:
741 iounmap(dev->kar);
742
743err_uar_free:
744 mthca_uar_free(dev, &dev->driver_uar);
745
746err_uar_table_free:
747 mthca_cleanup_uar_table(dev);
748 return err;
749}
750
751static int __devinit mthca_request_regions(struct pci_dev *pdev,
752 int ddr_hidden)
753{
754 int err;
755
756 /*
757 * We can't just use pci_request_regions() because the MSI-X
758 * table is right in the middle of the first BAR. If we did
759 * pci_request_region and grab all of the first BAR, then
760 * setting up MSI-X would fail, since the PCI core wants to do
761 * request_mem_region on the MSI-X vector table.
762 *
763 * So just request what we need right now, and request any
764 * other regions we need when setting up EQs.
765 */
766 if (!request_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE,
767 MTHCA_HCR_SIZE, DRV_NAME))
768 return -EBUSY;
769
770 err = pci_request_region(pdev, 2, DRV_NAME);
771 if (err)
772 goto err_bar2_failed;
773
774 if (!ddr_hidden) {
775 err = pci_request_region(pdev, 4, DRV_NAME);
776 if (err)
777 goto err_bar4_failed;
778 }
779
780 return 0;
781
782err_bar4_failed:
783 pci_release_region(pdev, 2);
784
785err_bar2_failed:
786 release_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE,
787 MTHCA_HCR_SIZE);
788
789 return err;
790}
791
792static void mthca_release_regions(struct pci_dev *pdev,
793 int ddr_hidden)
794{
795 if (!ddr_hidden)
796 pci_release_region(pdev, 4);
797
798 pci_release_region(pdev, 2);
799
800 release_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE,
801 MTHCA_HCR_SIZE);
802}
803
804static int __devinit mthca_enable_msi_x(struct mthca_dev *mdev)
805{
806 struct msix_entry entries[3];
807 int err;
808
809 entries[0].entry = 0;
810 entries[1].entry = 1;
811 entries[2].entry = 2;
812
813 err = pci_enable_msix(mdev->pdev, entries, ARRAY_SIZE(entries));
814 if (err) {
815 if (err > 0)
816 mthca_info(mdev, "Only %d MSI-X vectors available, "
817 "not using MSI-X\n", err);
818 return err;
819 }
820
821 mdev->eq_table.eq[MTHCA_EQ_COMP ].msi_x_vector = entries[0].vector;
822 mdev->eq_table.eq[MTHCA_EQ_ASYNC].msi_x_vector = entries[1].vector;
823 mdev->eq_table.eq[MTHCA_EQ_CMD ].msi_x_vector = entries[2].vector;
824
825 return 0;
826}
827
828static void mthca_close_hca(struct mthca_dev *mdev)
829{
830 u8 status;
831
832 mthca_CLOSE_HCA(mdev, 0, &status);
833
834 if (mdev->hca_type == ARBEL_NATIVE) {
835 mthca_free_icm_table(mdev, mdev->cq_table.table);
836 mthca_free_icm_table(mdev, mdev->qp_table.eqp_table);
837 mthca_free_icm_table(mdev, mdev->qp_table.qp_table);
838 mthca_free_icm_table(mdev, mdev->mr_table.mpt_table);
839 mthca_free_icm_table(mdev, mdev->mr_table.mtt_table);
840 mthca_unmap_eq_icm(mdev);
841
842 mthca_UNMAP_ICM_AUX(mdev, &status);
843 mthca_free_icm(mdev, mdev->fw.arbel.aux_icm);
844
845 mthca_UNMAP_FA(mdev, &status);
846 mthca_free_icm(mdev, mdev->fw.arbel.fw_icm);
847
848 if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM))
849 mthca_DISABLE_LAM(mdev, &status);
850 } else
851 mthca_SYS_DIS(mdev, &status);
852}
853
854static int __devinit mthca_init_one(struct pci_dev *pdev,
855 const struct pci_device_id *id)
856{
857 static int mthca_version_printed = 0;
858 static int mthca_memfree_warned = 0;
859 int ddr_hidden = 0;
860 int err;
861 struct mthca_dev *mdev;
862
863 if (!mthca_version_printed) {
864 printk(KERN_INFO "%s", mthca_version);
865 ++mthca_version_printed;
866 }
867
868 printk(KERN_INFO PFX "Initializing %s (%s)\n",
869 pci_pretty_name(pdev), pci_name(pdev));
870
871 err = pci_enable_device(pdev);
872 if (err) {
873 dev_err(&pdev->dev, "Cannot enable PCI device, "
874 "aborting.\n");
875 return err;
876 }
877
878 /*
879 * Check for BARs. We expect 0: 1MB, 2: 8MB, 4: DDR (may not
880 * be present)
881 */
882 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) ||
883 pci_resource_len(pdev, 0) != 1 << 20) {
884 dev_err(&pdev->dev, "Missing DCS, aborting.");
885 err = -ENODEV;
886 goto err_disable_pdev;
887 }
888 if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM) ||
889 pci_resource_len(pdev, 2) != 1 << 23) {
890 dev_err(&pdev->dev, "Missing UAR, aborting.");
891 err = -ENODEV;
892 goto err_disable_pdev;
893 }
894 if (!(pci_resource_flags(pdev, 4) & IORESOURCE_MEM))
895 ddr_hidden = 1;
896
897 err = mthca_request_regions(pdev, ddr_hidden);
898 if (err) {
899 dev_err(&pdev->dev, "Cannot obtain PCI resources, "
900 "aborting.\n");
901 goto err_disable_pdev;
902 }
903
904 pci_set_master(pdev);
905
906 err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
907 if (err) {
908 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n");
909 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
910 if (err) {
911 dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n");
912 goto err_free_res;
913 }
914 }
915 err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
916 if (err) {
917 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit "
918 "consistent PCI DMA mask.\n");
919 err = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
920 if (err) {
921 dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, "
922 "aborting.\n");
923 goto err_free_res;
924 }
925 }
926
927 mdev = (struct mthca_dev *) ib_alloc_device(sizeof *mdev);
928 if (!mdev) {
929 dev_err(&pdev->dev, "Device struct alloc failed, "
930 "aborting.\n");
931 err = -ENOMEM;
932 goto err_free_res;
933 }
934
935 mdev->pdev = pdev;
936 mdev->hca_type = id->driver_data;
937
938 if (mdev->hca_type == ARBEL_NATIVE && !mthca_memfree_warned++)
939 mthca_warn(mdev, "Warning: native MT25208 mode support is incomplete. "
940 "Your HCA may not work properly.\n");
941
942 if (ddr_hidden)
943 mdev->mthca_flags |= MTHCA_FLAG_DDR_HIDDEN;
944
945 /*
946 * Now reset the HCA before we touch the PCI capabilities or
947 * attempt a firmware command, since a boot ROM may have left
948 * the HCA in an undefined state.
949 */
950 err = mthca_reset(mdev);
951 if (err) {
952 mthca_err(mdev, "Failed to reset HCA, aborting.\n");
953 goto err_free_dev;
954 }
955
956 if (msi_x && !mthca_enable_msi_x(mdev))
957 mdev->mthca_flags |= MTHCA_FLAG_MSI_X;
958 if (msi && !(mdev->mthca_flags & MTHCA_FLAG_MSI_X) &&
959 !pci_enable_msi(pdev))
960 mdev->mthca_flags |= MTHCA_FLAG_MSI;
961
962 sema_init(&mdev->cmd.hcr_sem, 1);
963 sema_init(&mdev->cmd.poll_sem, 1);
964 mdev->cmd.use_events = 0;
965
966 mdev->hcr = ioremap(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE, MTHCA_HCR_SIZE);
967 if (!mdev->hcr) {
968 mthca_err(mdev, "Couldn't map command register, "
969 "aborting.\n");
970 err = -ENOMEM;
971 goto err_free_dev;
972 }
973
974 err = mthca_tune_pci(mdev);
975 if (err)
976 goto err_iounmap;
977
978 err = mthca_init_hca(mdev);
979 if (err)
980 goto err_iounmap;
981
982 err = mthca_setup_hca(mdev);
983 if (err)
984 goto err_close;
985
986 err = mthca_register_device(mdev);
987 if (err)
988 goto err_cleanup;
989
990 err = mthca_create_agents(mdev);
991 if (err)
992 goto err_unregister;
993
994 pci_set_drvdata(pdev, mdev);
995
996 return 0;
997
998err_unregister:
999 mthca_unregister_device(mdev);
1000
1001err_cleanup:
1002 mthca_cleanup_mcg_table(mdev);
1003 mthca_cleanup_av_table(mdev);
1004 mthca_cleanup_qp_table(mdev);
1005 mthca_cleanup_cq_table(mdev);
1006 mthca_cmd_use_polling(mdev);
1007 mthca_cleanup_eq_table(mdev);
1008
1009 mthca_pd_free(mdev, &mdev->driver_pd);
1010
1011 mthca_cleanup_mr_table(mdev);
1012 mthca_cleanup_pd_table(mdev);
1013 mthca_cleanup_uar_table(mdev);
1014
1015err_close:
1016 mthca_close_hca(mdev);
1017
1018err_iounmap:
1019 iounmap(mdev->hcr);
1020
1021err_free_dev:
1022 if (mdev->mthca_flags & MTHCA_FLAG_MSI_X)
1023 pci_disable_msix(pdev);
1024 if (mdev->mthca_flags & MTHCA_FLAG_MSI)
1025 pci_disable_msi(pdev);
1026
1027 ib_dealloc_device(&mdev->ib_dev);
1028
1029err_free_res:
1030 mthca_release_regions(pdev, ddr_hidden);
1031
1032err_disable_pdev:
1033 pci_disable_device(pdev);
1034 pci_set_drvdata(pdev, NULL);
1035 return err;
1036}
1037
1038static void __devexit mthca_remove_one(struct pci_dev *pdev)
1039{
1040 struct mthca_dev *mdev = pci_get_drvdata(pdev);
1041 u8 status;
1042 int p;
1043
1044 if (mdev) {
1045 mthca_free_agents(mdev);
1046 mthca_unregister_device(mdev);
1047
1048 for (p = 1; p <= mdev->limits.num_ports; ++p)
1049 mthca_CLOSE_IB(mdev, p, &status);
1050
1051 mthca_cleanup_mcg_table(mdev);
1052 mthca_cleanup_av_table(mdev);
1053 mthca_cleanup_qp_table(mdev);
1054 mthca_cleanup_cq_table(mdev);
1055 mthca_cmd_use_polling(mdev);
1056 mthca_cleanup_eq_table(mdev);
1057
1058 mthca_pd_free(mdev, &mdev->driver_pd);
1059
1060 mthca_cleanup_mr_table(mdev);
1061 mthca_cleanup_pd_table(mdev);
1062
1063 iounmap(mdev->kar);
1064 mthca_uar_free(mdev, &mdev->driver_uar);
1065 mthca_cleanup_uar_table(mdev);
1066
1067 mthca_close_hca(mdev);
1068
1069 iounmap(mdev->hcr);
1070
1071 if (mdev->mthca_flags & MTHCA_FLAG_MSI_X)
1072 pci_disable_msix(pdev);
1073 if (mdev->mthca_flags & MTHCA_FLAG_MSI)
1074 pci_disable_msi(pdev);
1075
1076 ib_dealloc_device(&mdev->ib_dev);
1077 mthca_release_regions(pdev, mdev->mthca_flags &
1078 MTHCA_FLAG_DDR_HIDDEN);
1079 pci_disable_device(pdev);
1080 pci_set_drvdata(pdev, NULL);
1081 }
1082}
1083
1084static struct pci_device_id mthca_pci_table[] = {
1085 { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_TAVOR),
1086 .driver_data = TAVOR },
1087 { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_TAVOR),
1088 .driver_data = TAVOR },
1089 { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT),
1090 .driver_data = ARBEL_COMPAT },
1091 { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT),
1092 .driver_data = ARBEL_COMPAT },
1093 { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL),
1094 .driver_data = ARBEL_NATIVE },
1095 { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL),
1096 .driver_data = ARBEL_NATIVE },
1097 { 0, }
1098};
1099
1100MODULE_DEVICE_TABLE(pci, mthca_pci_table);
1101
1102static struct pci_driver mthca_driver = {
1103 .name = "ib_mthca",
1104 .id_table = mthca_pci_table,
1105 .probe = mthca_init_one,
1106 .remove = __devexit_p(mthca_remove_one)
1107};
1108
1109static int __init mthca_init(void)
1110{
1111 int ret;
1112
1113 ret = pci_register_driver(&mthca_driver);
1114 return ret < 0 ? ret : 0;
1115}
1116
1117static void __exit mthca_cleanup(void)
1118{
1119 pci_unregister_driver(&mthca_driver);
1120}
1121
1122module_init(mthca_init);
1123module_exit(mthca_cleanup);
diff --git a/drivers/infiniband/hw/mthca/mthca_mcg.c b/drivers/infiniband/hw/mthca/mthca_mcg.c
new file mode 100644
index 000000000000..70a6553a588e
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_mcg.c
@@ -0,0 +1,376 @@
1/*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_mcg.c 1349 2004-12-16 21:09:43Z roland $
33 */
34
35#include <linux/init.h>
36
37#include "mthca_dev.h"
38#include "mthca_cmd.h"
39
40enum {
41 MTHCA_QP_PER_MGM = 4 * (MTHCA_MGM_ENTRY_SIZE / 16 - 2)
42};
43
44struct mthca_mgm {
45 u32 next_gid_index;
46 u32 reserved[3];
47 u8 gid[16];
48 u32 qp[MTHCA_QP_PER_MGM];
49};
50
51static const u8 zero_gid[16]; /* automatically initialized to 0 */
52
53/*
54 * Caller must hold MCG table semaphore. gid and mgm parameters must
55 * be properly aligned for command interface.
56 *
57 * Returns 0 unless a firmware command error occurs.
58 *
59 * If GID is found in MGM or MGM is empty, *index = *hash, *prev = -1
60 * and *mgm holds MGM entry.
61 *
62 * if GID is found in AMGM, *index = index in AMGM, *prev = index of
63 * previous entry in hash chain and *mgm holds AMGM entry.
64 *
65 * If no AMGM exists for given gid, *index = -1, *prev = index of last
66 * entry in hash chain and *mgm holds end of hash chain.
67 */
68static int find_mgm(struct mthca_dev *dev,
69 u8 *gid, struct mthca_mgm *mgm,
70 u16 *hash, int *prev, int *index)
71{
72 void *mailbox;
73 u8 *mgid;
74 int err;
75 u8 status;
76
77 mailbox = kmalloc(16 + MTHCA_CMD_MAILBOX_EXTRA, GFP_KERNEL);
78 if (!mailbox)
79 return -ENOMEM;
80 mgid = MAILBOX_ALIGN(mailbox);
81
82 memcpy(mgid, gid, 16);
83
84 err = mthca_MGID_HASH(dev, mgid, hash, &status);
85 if (err)
86 goto out;
87 if (status) {
88 mthca_err(dev, "MGID_HASH returned status %02x\n", status);
89 err = -EINVAL;
90 goto out;
91 }
92
93 if (0)
94 mthca_dbg(dev, "Hash for %04x:%04x:%04x:%04x:"
95 "%04x:%04x:%04x:%04x is %04x\n",
96 be16_to_cpu(((u16 *) gid)[0]), be16_to_cpu(((u16 *) gid)[1]),
97 be16_to_cpu(((u16 *) gid)[2]), be16_to_cpu(((u16 *) gid)[3]),
98 be16_to_cpu(((u16 *) gid)[4]), be16_to_cpu(((u16 *) gid)[5]),
99 be16_to_cpu(((u16 *) gid)[6]), be16_to_cpu(((u16 *) gid)[7]),
100 *hash);
101
102 *index = *hash;
103 *prev = -1;
104
105 do {
106 err = mthca_READ_MGM(dev, *index, mgm, &status);
107 if (err)
108 goto out;
109 if (status) {
110 mthca_err(dev, "READ_MGM returned status %02x\n", status);
111 return -EINVAL;
112 }
113
114 if (!memcmp(mgm->gid, zero_gid, 16)) {
115 if (*index != *hash) {
116 mthca_err(dev, "Found zero MGID in AMGM.\n");
117 err = -EINVAL;
118 }
119 goto out;
120 }
121
122 if (!memcmp(mgm->gid, gid, 16))
123 goto out;
124
125 *prev = *index;
126 *index = be32_to_cpu(mgm->next_gid_index) >> 5;
127 } while (*index);
128
129 *index = -1;
130
131 out:
132 kfree(mailbox);
133 return err;
134}
135
136int mthca_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
137{
138 struct mthca_dev *dev = to_mdev(ibqp->device);
139 void *mailbox;
140 struct mthca_mgm *mgm;
141 u16 hash;
142 int index, prev;
143 int link = 0;
144 int i;
145 int err;
146 u8 status;
147
148 mailbox = kmalloc(sizeof *mgm + MTHCA_CMD_MAILBOX_EXTRA, GFP_KERNEL);
149 if (!mailbox)
150 return -ENOMEM;
151 mgm = MAILBOX_ALIGN(mailbox);
152
153 if (down_interruptible(&dev->mcg_table.sem))
154 return -EINTR;
155
156 err = find_mgm(dev, gid->raw, mgm, &hash, &prev, &index);
157 if (err)
158 goto out;
159
160 if (index != -1) {
161 if (!memcmp(mgm->gid, zero_gid, 16))
162 memcpy(mgm->gid, gid->raw, 16);
163 } else {
164 link = 1;
165
166 index = mthca_alloc(&dev->mcg_table.alloc);
167 if (index == -1) {
168 mthca_err(dev, "No AMGM entries left\n");
169 err = -ENOMEM;
170 goto out;
171 }
172
173 err = mthca_READ_MGM(dev, index, mgm, &status);
174 if (err)
175 goto out;
176 if (status) {
177 mthca_err(dev, "READ_MGM returned status %02x\n", status);
178 err = -EINVAL;
179 goto out;
180 }
181
182 memcpy(mgm->gid, gid->raw, 16);
183 mgm->next_gid_index = 0;
184 }
185
186 for (i = 0; i < MTHCA_QP_PER_MGM; ++i)
187 if (!(mgm->qp[i] & cpu_to_be32(1 << 31))) {
188 mgm->qp[i] = cpu_to_be32(ibqp->qp_num | (1 << 31));
189 break;
190 }
191
192 if (i == MTHCA_QP_PER_MGM) {
193 mthca_err(dev, "MGM at index %x is full.\n", index);
194 err = -ENOMEM;
195 goto out;
196 }
197
198 err = mthca_WRITE_MGM(dev, index, mgm, &status);
199 if (err)
200 goto out;
201 if (status) {
202 mthca_err(dev, "WRITE_MGM returned status %02x\n", status);
203 err = -EINVAL;
204 }
205
206 if (!link)
207 goto out;
208
209 err = mthca_READ_MGM(dev, prev, mgm, &status);
210 if (err)
211 goto out;
212 if (status) {
213 mthca_err(dev, "READ_MGM returned status %02x\n", status);
214 err = -EINVAL;
215 goto out;
216 }
217
218 mgm->next_gid_index = cpu_to_be32(index << 5);
219
220 err = mthca_WRITE_MGM(dev, prev, mgm, &status);
221 if (err)
222 goto out;
223 if (status) {
224 mthca_err(dev, "WRITE_MGM returned status %02x\n", status);
225 err = -EINVAL;
226 }
227
228 out:
229 up(&dev->mcg_table.sem);
230 kfree(mailbox);
231 return err;
232}
233
234int mthca_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
235{
236 struct mthca_dev *dev = to_mdev(ibqp->device);
237 void *mailbox;
238 struct mthca_mgm *mgm;
239 u16 hash;
240 int prev, index;
241 int i, loc;
242 int err;
243 u8 status;
244
245 mailbox = kmalloc(sizeof *mgm + MTHCA_CMD_MAILBOX_EXTRA, GFP_KERNEL);
246 if (!mailbox)
247 return -ENOMEM;
248 mgm = MAILBOX_ALIGN(mailbox);
249
250 if (down_interruptible(&dev->mcg_table.sem))
251 return -EINTR;
252
253 err = find_mgm(dev, gid->raw, mgm, &hash, &prev, &index);
254 if (err)
255 goto out;
256
257 if (index == -1) {
258 mthca_err(dev, "MGID %04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x "
259 "not found\n",
260 be16_to_cpu(((u16 *) gid->raw)[0]),
261 be16_to_cpu(((u16 *) gid->raw)[1]),
262 be16_to_cpu(((u16 *) gid->raw)[2]),
263 be16_to_cpu(((u16 *) gid->raw)[3]),
264 be16_to_cpu(((u16 *) gid->raw)[4]),
265 be16_to_cpu(((u16 *) gid->raw)[5]),
266 be16_to_cpu(((u16 *) gid->raw)[6]),
267 be16_to_cpu(((u16 *) gid->raw)[7]));
268 err = -EINVAL;
269 goto out;
270 }
271
272 for (loc = -1, i = 0; i < MTHCA_QP_PER_MGM; ++i) {
273 if (mgm->qp[i] == cpu_to_be32(ibqp->qp_num | (1 << 31)))
274 loc = i;
275 if (!(mgm->qp[i] & cpu_to_be32(1 << 31)))
276 break;
277 }
278
279 if (loc == -1) {
280 mthca_err(dev, "QP %06x not found in MGM\n", ibqp->qp_num);
281 err = -EINVAL;
282 goto out;
283 }
284
285 mgm->qp[loc] = mgm->qp[i - 1];
286 mgm->qp[i - 1] = 0;
287
288 err = mthca_WRITE_MGM(dev, index, mgm, &status);
289 if (err)
290 goto out;
291 if (status) {
292 mthca_err(dev, "WRITE_MGM returned status %02x\n", status);
293 err = -EINVAL;
294 goto out;
295 }
296
297 if (i != 1)
298 goto out;
299
300 goto out;
301
302 if (prev == -1) {
303 /* Remove entry from MGM */
304 if (be32_to_cpu(mgm->next_gid_index) >> 5) {
305 err = mthca_READ_MGM(dev,
306 be32_to_cpu(mgm->next_gid_index) >> 5,
307 mgm, &status);
308 if (err)
309 goto out;
310 if (status) {
311 mthca_err(dev, "READ_MGM returned status %02x\n",
312 status);
313 err = -EINVAL;
314 goto out;
315 }
316 } else
317 memset(mgm->gid, 0, 16);
318
319 err = mthca_WRITE_MGM(dev, index, mgm, &status);
320 if (err)
321 goto out;
322 if (status) {
323 mthca_err(dev, "WRITE_MGM returned status %02x\n", status);
324 err = -EINVAL;
325 goto out;
326 }
327 } else {
328 /* Remove entry from AMGM */
329 index = be32_to_cpu(mgm->next_gid_index) >> 5;
330 err = mthca_READ_MGM(dev, prev, mgm, &status);
331 if (err)
332 goto out;
333 if (status) {
334 mthca_err(dev, "READ_MGM returned status %02x\n", status);
335 err = -EINVAL;
336 goto out;
337 }
338
339 mgm->next_gid_index = cpu_to_be32(index << 5);
340
341 err = mthca_WRITE_MGM(dev, prev, mgm, &status);
342 if (err)
343 goto out;
344 if (status) {
345 mthca_err(dev, "WRITE_MGM returned status %02x\n", status);
346 err = -EINVAL;
347 goto out;
348 }
349 }
350
351 out:
352 up(&dev->mcg_table.sem);
353 kfree(mailbox);
354 return err;
355}
356
357int __devinit mthca_init_mcg_table(struct mthca_dev *dev)
358{
359 int err;
360
361 err = mthca_alloc_init(&dev->mcg_table.alloc,
362 dev->limits.num_amgms,
363 dev->limits.num_amgms - 1,
364 0);
365 if (err)
366 return err;
367
368 init_MUTEX(&dev->mcg_table.sem);
369
370 return 0;
371}
372
373void __devexit mthca_cleanup_mcg_table(struct mthca_dev *dev)
374{
375 mthca_alloc_cleanup(&dev->mcg_table.alloc);
376}
diff --git a/drivers/infiniband/hw/mthca/mthca_memfree.c b/drivers/infiniband/hw/mthca/mthca_memfree.c
new file mode 100644
index 000000000000..7730b5960616
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_memfree.c
@@ -0,0 +1,465 @@
1/*
2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id$
33 */
34
35#include "mthca_memfree.h"
36#include "mthca_dev.h"
37#include "mthca_cmd.h"
38
39/*
40 * We allocate in as big chunks as we can, up to a maximum of 256 KB
41 * per chunk.
42 */
43enum {
44 MTHCA_ICM_ALLOC_SIZE = 1 << 18,
45 MTHCA_TABLE_CHUNK_SIZE = 1 << 18
46};
47
48void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm)
49{
50 struct mthca_icm_chunk *chunk, *tmp;
51 int i;
52
53 if (!icm)
54 return;
55
56 list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) {
57 if (chunk->nsg > 0)
58 pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages,
59 PCI_DMA_BIDIRECTIONAL);
60
61 for (i = 0; i < chunk->npages; ++i)
62 __free_pages(chunk->mem[i].page,
63 get_order(chunk->mem[i].length));
64
65 kfree(chunk);
66 }
67
68 kfree(icm);
69}
70
71struct mthca_icm *mthca_alloc_icm(struct mthca_dev *dev, int npages,
72 unsigned int gfp_mask)
73{
74 struct mthca_icm *icm;
75 struct mthca_icm_chunk *chunk = NULL;
76 int cur_order;
77
78 icm = kmalloc(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
79 if (!icm)
80 return icm;
81
82 icm->refcount = 0;
83 INIT_LIST_HEAD(&icm->chunk_list);
84
85 cur_order = get_order(MTHCA_ICM_ALLOC_SIZE);
86
87 while (npages > 0) {
88 if (!chunk) {
89 chunk = kmalloc(sizeof *chunk,
90 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
91 if (!chunk)
92 goto fail;
93
94 chunk->npages = 0;
95 chunk->nsg = 0;
96 list_add_tail(&chunk->list, &icm->chunk_list);
97 }
98
99 while (1 << cur_order > npages)
100 --cur_order;
101
102 chunk->mem[chunk->npages].page = alloc_pages(gfp_mask, cur_order);
103 if (chunk->mem[chunk->npages].page) {
104 chunk->mem[chunk->npages].length = PAGE_SIZE << cur_order;
105 chunk->mem[chunk->npages].offset = 0;
106
107 if (++chunk->npages == MTHCA_ICM_CHUNK_LEN) {
108 chunk->nsg = pci_map_sg(dev->pdev, chunk->mem,
109 chunk->npages,
110 PCI_DMA_BIDIRECTIONAL);
111
112 if (chunk->nsg <= 0)
113 goto fail;
114
115 chunk = NULL;
116 }
117
118 npages -= 1 << cur_order;
119 } else {
120 --cur_order;
121 if (cur_order < 0)
122 goto fail;
123 }
124 }
125
126 if (chunk) {
127 chunk->nsg = pci_map_sg(dev->pdev, chunk->mem,
128 chunk->npages,
129 PCI_DMA_BIDIRECTIONAL);
130
131 if (chunk->nsg <= 0)
132 goto fail;
133 }
134
135 return icm;
136
137fail:
138 mthca_free_icm(dev, icm);
139 return NULL;
140}
141
142int mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int obj)
143{
144 int i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE;
145 int ret = 0;
146 u8 status;
147
148 down(&table->mutex);
149
150 if (table->icm[i]) {
151 ++table->icm[i]->refcount;
152 goto out;
153 }
154
155 table->icm[i] = mthca_alloc_icm(dev, MTHCA_TABLE_CHUNK_SIZE >> PAGE_SHIFT,
156 (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) |
157 __GFP_NOWARN);
158 if (!table->icm[i]) {
159 ret = -ENOMEM;
160 goto out;
161 }
162
163 if (mthca_MAP_ICM(dev, table->icm[i], table->virt + i * MTHCA_TABLE_CHUNK_SIZE,
164 &status) || status) {
165 mthca_free_icm(dev, table->icm[i]);
166 table->icm[i] = NULL;
167 ret = -ENOMEM;
168 goto out;
169 }
170
171 ++table->icm[i]->refcount;
172
173out:
174 up(&table->mutex);
175 return ret;
176}
177
178void mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int obj)
179{
180 int i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE;
181 u8 status;
182
183 down(&table->mutex);
184
185 if (--table->icm[i]->refcount == 0) {
186 mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE,
187 MTHCA_TABLE_CHUNK_SIZE >> 12, &status);
188 mthca_free_icm(dev, table->icm[i]);
189 table->icm[i] = NULL;
190 }
191
192 up(&table->mutex);
193}
194
195struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev,
196 u64 virt, int obj_size,
197 int nobj, int reserved,
198 int use_lowmem)
199{
200 struct mthca_icm_table *table;
201 int num_icm;
202 int i;
203 u8 status;
204
205 num_icm = obj_size * nobj / MTHCA_TABLE_CHUNK_SIZE;
206
207 table = kmalloc(sizeof *table + num_icm * sizeof *table->icm, GFP_KERNEL);
208 if (!table)
209 return NULL;
210
211 table->virt = virt;
212 table->num_icm = num_icm;
213 table->num_obj = nobj;
214 table->obj_size = obj_size;
215 table->lowmem = use_lowmem;
216 init_MUTEX(&table->mutex);
217
218 for (i = 0; i < num_icm; ++i)
219 table->icm[i] = NULL;
220
221 for (i = 0; i * MTHCA_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) {
222 table->icm[i] = mthca_alloc_icm(dev, MTHCA_TABLE_CHUNK_SIZE >> PAGE_SHIFT,
223 (use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) |
224 __GFP_NOWARN);
225 if (!table->icm[i])
226 goto err;
227 if (mthca_MAP_ICM(dev, table->icm[i], virt + i * MTHCA_TABLE_CHUNK_SIZE,
228 &status) || status) {
229 mthca_free_icm(dev, table->icm[i]);
230 table->icm[i] = NULL;
231 goto err;
232 }
233
234 /*
235 * Add a reference to this ICM chunk so that it never
236 * gets freed (since it contains reserved firmware objects).
237 */
238 ++table->icm[i]->refcount;
239 }
240
241 return table;
242
243err:
244 for (i = 0; i < num_icm; ++i)
245 if (table->icm[i]) {
246 mthca_UNMAP_ICM(dev, virt + i * MTHCA_TABLE_CHUNK_SIZE,
247 MTHCA_TABLE_CHUNK_SIZE >> 12, &status);
248 mthca_free_icm(dev, table->icm[i]);
249 }
250
251 kfree(table);
252
253 return NULL;
254}
255
256void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table)
257{
258 int i;
259 u8 status;
260
261 for (i = 0; i < table->num_icm; ++i)
262 if (table->icm[i]) {
263 mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE,
264 MTHCA_TABLE_CHUNK_SIZE >> 12, &status);
265 mthca_free_icm(dev, table->icm[i]);
266 }
267
268 kfree(table);
269}
270
271static u64 mthca_uarc_virt(struct mthca_dev *dev, int page)
272{
273 return dev->uar_table.uarc_base +
274 dev->driver_uar.index * dev->uar_table.uarc_size +
275 page * 4096;
276}
277
278int mthca_alloc_db(struct mthca_dev *dev, int type, u32 qn, u32 **db)
279{
280 int group;
281 int start, end, dir;
282 int i, j;
283 struct mthca_db_page *page;
284 int ret = 0;
285 u8 status;
286
287 down(&dev->db_tab->mutex);
288
289 switch (type) {
290 case MTHCA_DB_TYPE_CQ_ARM:
291 case MTHCA_DB_TYPE_SQ:
292 group = 0;
293 start = 0;
294 end = dev->db_tab->max_group1;
295 dir = 1;
296 break;
297
298 case MTHCA_DB_TYPE_CQ_SET_CI:
299 case MTHCA_DB_TYPE_RQ:
300 case MTHCA_DB_TYPE_SRQ:
301 group = 1;
302 start = dev->db_tab->npages - 1;
303 end = dev->db_tab->min_group2;
304 dir = -1;
305 break;
306
307 default:
308 return -1;
309 }
310
311 for (i = start; i != end; i += dir)
312 if (dev->db_tab->page[i].db_rec &&
313 !bitmap_full(dev->db_tab->page[i].used,
314 MTHCA_DB_REC_PER_PAGE)) {
315 page = dev->db_tab->page + i;
316 goto found;
317 }
318
319 if (dev->db_tab->max_group1 >= dev->db_tab->min_group2 - 1) {
320 ret = -ENOMEM;
321 goto out;
322 }
323
324 page = dev->db_tab->page + end;
325 page->db_rec = dma_alloc_coherent(&dev->pdev->dev, 4096,
326 &page->mapping, GFP_KERNEL);
327 if (!page->db_rec) {
328 ret = -ENOMEM;
329 goto out;
330 }
331 memset(page->db_rec, 0, 4096);
332
333 ret = mthca_MAP_ICM_page(dev, page->mapping, mthca_uarc_virt(dev, i), &status);
334 if (!ret && status)
335 ret = -EINVAL;
336 if (ret) {
337 dma_free_coherent(&dev->pdev->dev, 4096,
338 page->db_rec, page->mapping);
339 goto out;
340 }
341
342 bitmap_zero(page->used, MTHCA_DB_REC_PER_PAGE);
343 if (group == 0)
344 ++dev->db_tab->max_group1;
345 else
346 --dev->db_tab->min_group2;
347
348found:
349 j = find_first_zero_bit(page->used, MTHCA_DB_REC_PER_PAGE);
350 set_bit(j, page->used);
351
352 if (group == 1)
353 j = MTHCA_DB_REC_PER_PAGE - 1 - j;
354
355 ret = i * MTHCA_DB_REC_PER_PAGE + j;
356
357 page->db_rec[j] = cpu_to_be64((qn << 8) | (type << 5));
358
359 *db = (u32 *) &page->db_rec[j];
360
361out:
362 up(&dev->db_tab->mutex);
363
364 return ret;
365}
366
367void mthca_free_db(struct mthca_dev *dev, int type, int db_index)
368{
369 int i, j;
370 struct mthca_db_page *page;
371 u8 status;
372
373 i = db_index / MTHCA_DB_REC_PER_PAGE;
374 j = db_index % MTHCA_DB_REC_PER_PAGE;
375
376 page = dev->db_tab->page + i;
377
378 down(&dev->db_tab->mutex);
379
380 page->db_rec[j] = 0;
381 if (i >= dev->db_tab->min_group2)
382 j = MTHCA_DB_REC_PER_PAGE - 1 - j;
383 clear_bit(j, page->used);
384
385 if (bitmap_empty(page->used, MTHCA_DB_REC_PER_PAGE) &&
386 i >= dev->db_tab->max_group1 - 1) {
387 mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, i), 1, &status);
388
389 dma_free_coherent(&dev->pdev->dev, 4096,
390 page->db_rec, page->mapping);
391 page->db_rec = NULL;
392
393 if (i == dev->db_tab->max_group1) {
394 --dev->db_tab->max_group1;
395 /* XXX may be able to unmap more pages now */
396 }
397 if (i == dev->db_tab->min_group2)
398 ++dev->db_tab->min_group2;
399 }
400
401 up(&dev->db_tab->mutex);
402}
403
404int mthca_init_db_tab(struct mthca_dev *dev)
405{
406 int i;
407
408 if (dev->hca_type != ARBEL_NATIVE)
409 return 0;
410
411 dev->db_tab = kmalloc(sizeof *dev->db_tab, GFP_KERNEL);
412 if (!dev->db_tab)
413 return -ENOMEM;
414
415 init_MUTEX(&dev->db_tab->mutex);
416
417 dev->db_tab->npages = dev->uar_table.uarc_size / PAGE_SIZE;
418 dev->db_tab->max_group1 = 0;
419 dev->db_tab->min_group2 = dev->db_tab->npages - 1;
420
421 dev->db_tab->page = kmalloc(dev->db_tab->npages *
422 sizeof *dev->db_tab->page,
423 GFP_KERNEL);
424 if (!dev->db_tab->page) {
425 kfree(dev->db_tab);
426 return -ENOMEM;
427 }
428
429 for (i = 0; i < dev->db_tab->npages; ++i)
430 dev->db_tab->page[i].db_rec = NULL;
431
432 return 0;
433}
434
435void mthca_cleanup_db_tab(struct mthca_dev *dev)
436{
437 int i;
438 u8 status;
439
440 if (dev->hca_type != ARBEL_NATIVE)
441 return;
442
443 /*
444 * Because we don't always free our UARC pages when they
445 * become empty to make mthca_free_db() simpler we need to
446 * make a sweep through the doorbell pages and free any
447 * leftover pages now.
448 */
449 for (i = 0; i < dev->db_tab->npages; ++i) {
450 if (!dev->db_tab->page[i].db_rec)
451 continue;
452
453 if (!bitmap_empty(dev->db_tab->page[i].used, MTHCA_DB_REC_PER_PAGE))
454 mthca_warn(dev, "Kernel UARC page %d not empty\n", i);
455
456 mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, i), 1, &status);
457
458 dma_free_coherent(&dev->pdev->dev, 4096,
459 dev->db_tab->page[i].db_rec,
460 dev->db_tab->page[i].mapping);
461 }
462
463 kfree(dev->db_tab->page);
464 kfree(dev->db_tab);
465}
diff --git a/drivers/infiniband/hw/mthca/mthca_memfree.h b/drivers/infiniband/hw/mthca/mthca_memfree.h
new file mode 100644
index 000000000000..a8fa97e140f5
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_memfree.h
@@ -0,0 +1,161 @@
1/*
2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id$
33 */
34
35#ifndef MTHCA_MEMFREE_H
36#define MTHCA_MEMFREE_H
37
38#include <linux/list.h>
39#include <linux/pci.h>
40
41#include <asm/semaphore.h>
42
43#define MTHCA_ICM_CHUNK_LEN \
44 ((256 - sizeof (struct list_head) - 2 * sizeof (int)) / \
45 (sizeof (struct scatterlist)))
46
47struct mthca_icm_chunk {
48 struct list_head list;
49 int npages;
50 int nsg;
51 struct scatterlist mem[MTHCA_ICM_CHUNK_LEN];
52};
53
54struct mthca_icm {
55 struct list_head chunk_list;
56 int refcount;
57};
58
59struct mthca_icm_table {
60 u64 virt;
61 int num_icm;
62 int num_obj;
63 int obj_size;
64 int lowmem;
65 struct semaphore mutex;
66 struct mthca_icm *icm[0];
67};
68
69struct mthca_icm_iter {
70 struct mthca_icm *icm;
71 struct mthca_icm_chunk *chunk;
72 int page_idx;
73};
74
75struct mthca_dev;
76
77struct mthca_icm *mthca_alloc_icm(struct mthca_dev *dev, int npages,
78 unsigned int gfp_mask);
79void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm);
80
81struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev,
82 u64 virt, int obj_size,
83 int nobj, int reserved,
84 int use_lowmem);
85void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table);
86int mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int obj);
87void mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int obj);
88
89static inline void mthca_icm_first(struct mthca_icm *icm,
90 struct mthca_icm_iter *iter)
91{
92 iter->icm = icm;
93 iter->chunk = list_empty(&icm->chunk_list) ?
94 NULL : list_entry(icm->chunk_list.next,
95 struct mthca_icm_chunk, list);
96 iter->page_idx = 0;
97}
98
99static inline int mthca_icm_last(struct mthca_icm_iter *iter)
100{
101 return !iter->chunk;
102}
103
104static inline void mthca_icm_next(struct mthca_icm_iter *iter)
105{
106 if (++iter->page_idx >= iter->chunk->nsg) {
107 if (iter->chunk->list.next == &iter->icm->chunk_list) {
108 iter->chunk = NULL;
109 return;
110 }
111
112 iter->chunk = list_entry(iter->chunk->list.next,
113 struct mthca_icm_chunk, list);
114 iter->page_idx = 0;
115 }
116}
117
118static inline dma_addr_t mthca_icm_addr(struct mthca_icm_iter *iter)
119{
120 return sg_dma_address(&iter->chunk->mem[iter->page_idx]);
121}
122
123static inline unsigned long mthca_icm_size(struct mthca_icm_iter *iter)
124{
125 return sg_dma_len(&iter->chunk->mem[iter->page_idx]);
126}
127
128enum {
129 MTHCA_DB_REC_PER_PAGE = 4096 / 8
130};
131
132struct mthca_db_page {
133 DECLARE_BITMAP(used, MTHCA_DB_REC_PER_PAGE);
134 u64 *db_rec;
135 dma_addr_t mapping;
136};
137
138struct mthca_db_table {
139 int npages;
140 int max_group1;
141 int min_group2;
142 struct mthca_db_page *page;
143 struct semaphore mutex;
144};
145
146enum {
147 MTHCA_DB_TYPE_INVALID = 0x0,
148 MTHCA_DB_TYPE_CQ_SET_CI = 0x1,
149 MTHCA_DB_TYPE_CQ_ARM = 0x2,
150 MTHCA_DB_TYPE_SQ = 0x3,
151 MTHCA_DB_TYPE_RQ = 0x4,
152 MTHCA_DB_TYPE_SRQ = 0x5,
153 MTHCA_DB_TYPE_GROUP_SEP = 0x7
154};
155
156int mthca_init_db_tab(struct mthca_dev *dev);
157void mthca_cleanup_db_tab(struct mthca_dev *dev);
158int mthca_alloc_db(struct mthca_dev *dev, int type, u32 qn, u32 **db);
159void mthca_free_db(struct mthca_dev *dev, int type, int db_index);
160
161#endif /* MTHCA_MEMFREE_H */
diff --git a/drivers/infiniband/hw/mthca/mthca_mr.c b/drivers/infiniband/hw/mthca/mthca_mr.c
new file mode 100644
index 000000000000..80a0cd97881b
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_mr.c
@@ -0,0 +1,416 @@
1/*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_mr.c 1349 2004-12-16 21:09:43Z roland $
33 */
34
35#include <linux/slab.h>
36#include <linux/init.h>
37#include <linux/errno.h>
38
39#include "mthca_dev.h"
40#include "mthca_cmd.h"
41
42/*
43 * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits.
44 */
45struct mthca_mpt_entry {
46 u32 flags;
47 u32 page_size;
48 u32 key;
49 u32 pd;
50 u64 start;
51 u64 length;
52 u32 lkey;
53 u32 window_count;
54 u32 window_count_limit;
55 u64 mtt_seg;
56 u32 mtt_sz; /* Arbel only */
57 u32 reserved[2];
58} __attribute__((packed));
59
60#define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28)
61#define MTHCA_MPT_FLAG_MIO (1 << 17)
62#define MTHCA_MPT_FLAG_BIND_ENABLE (1 << 15)
63#define MTHCA_MPT_FLAG_PHYSICAL (1 << 9)
64#define MTHCA_MPT_FLAG_REGION (1 << 8)
65
66#define MTHCA_MTT_FLAG_PRESENT 1
67
68/*
69 * Buddy allocator for MTT segments (currently not very efficient
70 * since it doesn't keep a free list and just searches linearly
71 * through the bitmaps)
72 */
73
74static u32 mthca_alloc_mtt(struct mthca_dev *dev, int order)
75{
76 int o;
77 int m;
78 u32 seg;
79
80 spin_lock(&dev->mr_table.mpt_alloc.lock);
81
82 for (o = order; o <= dev->mr_table.max_mtt_order; ++o) {
83 m = 1 << (dev->mr_table.max_mtt_order - o);
84 seg = find_first_bit(dev->mr_table.mtt_buddy[o], m);
85 if (seg < m)
86 goto found;
87 }
88
89 spin_unlock(&dev->mr_table.mpt_alloc.lock);
90 return -1;
91
92 found:
93 clear_bit(seg, dev->mr_table.mtt_buddy[o]);
94
95 while (o > order) {
96 --o;
97 seg <<= 1;
98 set_bit(seg ^ 1, dev->mr_table.mtt_buddy[o]);
99 }
100
101 spin_unlock(&dev->mr_table.mpt_alloc.lock);
102
103 seg <<= order;
104
105 return seg;
106}
107
108static void mthca_free_mtt(struct mthca_dev *dev, u32 seg, int order)
109{
110 seg >>= order;
111
112 spin_lock(&dev->mr_table.mpt_alloc.lock);
113
114 while (test_bit(seg ^ 1, dev->mr_table.mtt_buddy[order])) {
115 clear_bit(seg ^ 1, dev->mr_table.mtt_buddy[order]);
116 seg >>= 1;
117 ++order;
118 }
119
120 set_bit(seg, dev->mr_table.mtt_buddy[order]);
121
122 spin_unlock(&dev->mr_table.mpt_alloc.lock);
123}
124
125static inline u32 hw_index_to_key(struct mthca_dev *dev, u32 ind)
126{
127 if (dev->hca_type == ARBEL_NATIVE)
128 return (ind >> 24) | (ind << 8);
129 else
130 return ind;
131}
132
133static inline u32 key_to_hw_index(struct mthca_dev *dev, u32 key)
134{
135 if (dev->hca_type == ARBEL_NATIVE)
136 return (key << 24) | (key >> 8);
137 else
138 return key;
139}
140
141int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd,
142 u32 access, struct mthca_mr *mr)
143{
144 void *mailbox;
145 struct mthca_mpt_entry *mpt_entry;
146 u32 key;
147 int err;
148 u8 status;
149
150 might_sleep();
151
152 mr->order = -1;
153 key = mthca_alloc(&dev->mr_table.mpt_alloc);
154 if (key == -1)
155 return -ENOMEM;
156 mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
157
158 mailbox = kmalloc(sizeof *mpt_entry + MTHCA_CMD_MAILBOX_EXTRA,
159 GFP_KERNEL);
160 if (!mailbox) {
161 mthca_free(&dev->mr_table.mpt_alloc, mr->ibmr.lkey);
162 return -ENOMEM;
163 }
164 mpt_entry = MAILBOX_ALIGN(mailbox);
165
166 mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS |
167 MTHCA_MPT_FLAG_MIO |
168 MTHCA_MPT_FLAG_PHYSICAL |
169 MTHCA_MPT_FLAG_REGION |
170 access);
171 mpt_entry->page_size = 0;
172 mpt_entry->key = cpu_to_be32(key);
173 mpt_entry->pd = cpu_to_be32(pd);
174 mpt_entry->start = 0;
175 mpt_entry->length = ~0ULL;
176
177 memset(&mpt_entry->lkey, 0,
178 sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey));
179
180 err = mthca_SW2HW_MPT(dev, mpt_entry,
181 key & (dev->limits.num_mpts - 1),
182 &status);
183 if (err)
184 mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);
185 else if (status) {
186 mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n",
187 status);
188 err = -EINVAL;
189 }
190
191 kfree(mailbox);
192 return err;
193}
194
195int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd,
196 u64 *buffer_list, int buffer_size_shift,
197 int list_len, u64 iova, u64 total_size,
198 u32 access, struct mthca_mr *mr)
199{
200 void *mailbox;
201 u64 *mtt_entry;
202 struct mthca_mpt_entry *mpt_entry;
203 u32 key;
204 int err = -ENOMEM;
205 u8 status;
206 int i;
207
208 might_sleep();
209 WARN_ON(buffer_size_shift >= 32);
210
211 key = mthca_alloc(&dev->mr_table.mpt_alloc);
212 if (key == -1)
213 return -ENOMEM;
214 mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
215
216 for (i = dev->limits.mtt_seg_size / 8, mr->order = 0;
217 i < list_len;
218 i <<= 1, ++mr->order)
219 ; /* nothing */
220
221 mr->first_seg = mthca_alloc_mtt(dev, mr->order);
222 if (mr->first_seg == -1)
223 goto err_out_mpt_free;
224
225 /*
226 * If list_len is odd, we add one more dummy entry for
227 * firmware efficiency.
228 */
229 mailbox = kmalloc(max(sizeof *mpt_entry,
230 (size_t) 8 * (list_len + (list_len & 1) + 2)) +
231 MTHCA_CMD_MAILBOX_EXTRA,
232 GFP_KERNEL);
233 if (!mailbox)
234 goto err_out_free_mtt;
235
236 mtt_entry = MAILBOX_ALIGN(mailbox);
237
238 mtt_entry[0] = cpu_to_be64(dev->mr_table.mtt_base +
239 mr->first_seg * dev->limits.mtt_seg_size);
240 mtt_entry[1] = 0;
241 for (i = 0; i < list_len; ++i)
242 mtt_entry[i + 2] = cpu_to_be64(buffer_list[i] |
243 MTHCA_MTT_FLAG_PRESENT);
244 if (list_len & 1) {
245 mtt_entry[i + 2] = 0;
246 ++list_len;
247 }
248
249 if (0) {
250 mthca_dbg(dev, "Dumping MPT entry\n");
251 for (i = 0; i < list_len + 2; ++i)
252 printk(KERN_ERR "[%2d] %016llx\n",
253 i, (unsigned long long) be64_to_cpu(mtt_entry[i]));
254 }
255
256 err = mthca_WRITE_MTT(dev, mtt_entry, list_len, &status);
257 if (err) {
258 mthca_warn(dev, "WRITE_MTT failed (%d)\n", err);
259 goto err_out_mailbox_free;
260 }
261 if (status) {
262 mthca_warn(dev, "WRITE_MTT returned status 0x%02x\n",
263 status);
264 err = -EINVAL;
265 goto err_out_mailbox_free;
266 }
267
268 mpt_entry = MAILBOX_ALIGN(mailbox);
269
270 mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS |
271 MTHCA_MPT_FLAG_MIO |
272 MTHCA_MPT_FLAG_REGION |
273 access);
274
275 mpt_entry->page_size = cpu_to_be32(buffer_size_shift - 12);
276 mpt_entry->key = cpu_to_be32(key);
277 mpt_entry->pd = cpu_to_be32(pd);
278 mpt_entry->start = cpu_to_be64(iova);
279 mpt_entry->length = cpu_to_be64(total_size);
280 memset(&mpt_entry->lkey, 0,
281 sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey));
282 mpt_entry->mtt_seg = cpu_to_be64(dev->mr_table.mtt_base +
283 mr->first_seg * dev->limits.mtt_seg_size);
284
285 if (0) {
286 mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey);
287 for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) {
288 if (i % 4 == 0)
289 printk("[%02x] ", i * 4);
290 printk(" %08x", be32_to_cpu(((u32 *) mpt_entry)[i]));
291 if ((i + 1) % 4 == 0)
292 printk("\n");
293 }
294 }
295
296 err = mthca_SW2HW_MPT(dev, mpt_entry,
297 key & (dev->limits.num_mpts - 1),
298 &status);
299 if (err)
300 mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);
301 else if (status) {
302 mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n",
303 status);
304 err = -EINVAL;
305 }
306
307 kfree(mailbox);
308 return err;
309
310 err_out_mailbox_free:
311 kfree(mailbox);
312
313 err_out_free_mtt:
314 mthca_free_mtt(dev, mr->first_seg, mr->order);
315
316 err_out_mpt_free:
317 mthca_free(&dev->mr_table.mpt_alloc, mr->ibmr.lkey);
318 return err;
319}
320
321void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr)
322{
323 int err;
324 u8 status;
325
326 might_sleep();
327
328 err = mthca_HW2SW_MPT(dev, NULL,
329 key_to_hw_index(dev, mr->ibmr.lkey) &
330 (dev->limits.num_mpts - 1),
331 &status);
332 if (err)
333 mthca_warn(dev, "HW2SW_MPT failed (%d)\n", err);
334 else if (status)
335 mthca_warn(dev, "HW2SW_MPT returned status 0x%02x\n",
336 status);
337
338 if (mr->order >= 0)
339 mthca_free_mtt(dev, mr->first_seg, mr->order);
340
341 mthca_free(&dev->mr_table.mpt_alloc, key_to_hw_index(dev, mr->ibmr.lkey));
342}
343
344int __devinit mthca_init_mr_table(struct mthca_dev *dev)
345{
346 int err;
347 int i, s;
348
349 err = mthca_alloc_init(&dev->mr_table.mpt_alloc,
350 dev->limits.num_mpts,
351 ~0, dev->limits.reserved_mrws);
352 if (err)
353 return err;
354
355 err = -ENOMEM;
356
357 for (i = 1, dev->mr_table.max_mtt_order = 0;
358 i < dev->limits.num_mtt_segs;
359 i <<= 1, ++dev->mr_table.max_mtt_order)
360 ; /* nothing */
361
362 dev->mr_table.mtt_buddy = kmalloc((dev->mr_table.max_mtt_order + 1) *
363 sizeof (long *),
364 GFP_KERNEL);
365 if (!dev->mr_table.mtt_buddy)
366 goto err_out;
367
368 for (i = 0; i <= dev->mr_table.max_mtt_order; ++i)
369 dev->mr_table.mtt_buddy[i] = NULL;
370
371 for (i = 0; i <= dev->mr_table.max_mtt_order; ++i) {
372 s = BITS_TO_LONGS(1 << (dev->mr_table.max_mtt_order - i));
373 dev->mr_table.mtt_buddy[i] = kmalloc(s * sizeof (long),
374 GFP_KERNEL);
375 if (!dev->mr_table.mtt_buddy[i])
376 goto err_out_free;
377 bitmap_zero(dev->mr_table.mtt_buddy[i],
378 1 << (dev->mr_table.max_mtt_order - i));
379 }
380
381 set_bit(0, dev->mr_table.mtt_buddy[dev->mr_table.max_mtt_order]);
382
383 for (i = 0; i < dev->mr_table.max_mtt_order; ++i)
384 if (1 << i >= dev->limits.reserved_mtts)
385 break;
386
387 if (i == dev->mr_table.max_mtt_order) {
388 mthca_err(dev, "MTT table of order %d is "
389 "too small.\n", i);
390 goto err_out_free;
391 }
392
393 (void) mthca_alloc_mtt(dev, i);
394
395 return 0;
396
397 err_out_free:
398 for (i = 0; i <= dev->mr_table.max_mtt_order; ++i)
399 kfree(dev->mr_table.mtt_buddy[i]);
400
401 err_out:
402 mthca_alloc_cleanup(&dev->mr_table.mpt_alloc);
403
404 return err;
405}
406
407void __devexit mthca_cleanup_mr_table(struct mthca_dev *dev)
408{
409 int i;
410
411 /* XXX check if any MRs are still allocated? */
412 for (i = 0; i <= dev->mr_table.max_mtt_order; ++i)
413 kfree(dev->mr_table.mtt_buddy[i]);
414 kfree(dev->mr_table.mtt_buddy);
415 mthca_alloc_cleanup(&dev->mr_table.mpt_alloc);
416}
diff --git a/drivers/infiniband/hw/mthca/mthca_pd.c b/drivers/infiniband/hw/mthca/mthca_pd.c
new file mode 100644
index 000000000000..ea66847e4ea3
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_pd.c
@@ -0,0 +1,80 @@
1/*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_pd.c 1349 2004-12-16 21:09:43Z roland $
33 */
34
35#include <linux/init.h>
36#include <linux/errno.h>
37
38#include "mthca_dev.h"
39
40int mthca_pd_alloc(struct mthca_dev *dev, struct mthca_pd *pd)
41{
42 int err;
43
44 might_sleep();
45
46 atomic_set(&pd->sqp_count, 0);
47 pd->pd_num = mthca_alloc(&dev->pd_table.alloc);
48 if (pd->pd_num == -1)
49 return -ENOMEM;
50
51 err = mthca_mr_alloc_notrans(dev, pd->pd_num,
52 MTHCA_MPT_FLAG_LOCAL_READ |
53 MTHCA_MPT_FLAG_LOCAL_WRITE,
54 &pd->ntmr);
55 if (err)
56 mthca_free(&dev->pd_table.alloc, pd->pd_num);
57
58 return err;
59}
60
61void mthca_pd_free(struct mthca_dev *dev, struct mthca_pd *pd)
62{
63 might_sleep();
64 mthca_free_mr(dev, &pd->ntmr);
65 mthca_free(&dev->pd_table.alloc, pd->pd_num);
66}
67
68int __devinit mthca_init_pd_table(struct mthca_dev *dev)
69{
70 return mthca_alloc_init(&dev->pd_table.alloc,
71 dev->limits.num_pds,
72 (1 << 24) - 1,
73 dev->limits.reserved_pds);
74}
75
76void __devexit mthca_cleanup_pd_table(struct mthca_dev *dev)
77{
78 /* XXX check if any PDs are still allocated? */
79 mthca_alloc_cleanup(&dev->pd_table.alloc);
80}
diff --git a/drivers/infiniband/hw/mthca/mthca_profile.c b/drivers/infiniband/hw/mthca/mthca_profile.c
new file mode 100644
index 000000000000..7881a8a919ca
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_profile.c
@@ -0,0 +1,266 @@
1/*
2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_profile.c 1349 2004-12-16 21:09:43Z roland $
33 */
34
35#include <linux/module.h>
36#include <linux/moduleparam.h>
37
38#include "mthca_profile.h"
39
40enum {
41 MTHCA_RES_QP,
42 MTHCA_RES_EEC,
43 MTHCA_RES_SRQ,
44 MTHCA_RES_CQ,
45 MTHCA_RES_EQP,
46 MTHCA_RES_EEEC,
47 MTHCA_RES_EQ,
48 MTHCA_RES_RDB,
49 MTHCA_RES_MCG,
50 MTHCA_RES_MPT,
51 MTHCA_RES_MTT,
52 MTHCA_RES_UAR,
53 MTHCA_RES_UDAV,
54 MTHCA_RES_UARC,
55 MTHCA_RES_NUM
56};
57
58enum {
59 MTHCA_NUM_EQS = 32,
60 MTHCA_NUM_PDS = 1 << 15
61};
62
63u64 mthca_make_profile(struct mthca_dev *dev,
64 struct mthca_profile *request,
65 struct mthca_dev_lim *dev_lim,
66 struct mthca_init_hca_param *init_hca)
67{
68 struct mthca_resource {
69 u64 size;
70 u64 start;
71 int type;
72 int num;
73 int log_num;
74 };
75
76 u64 mem_base, mem_avail;
77 u64 total_size = 0;
78 struct mthca_resource *profile;
79 struct mthca_resource tmp;
80 int i, j;
81
82 profile = kmalloc(MTHCA_RES_NUM * sizeof *profile, GFP_KERNEL);
83 if (!profile)
84 return -ENOMEM;
85
86 memset(profile, 0, MTHCA_RES_NUM * sizeof *profile);
87
88 profile[MTHCA_RES_QP].size = dev_lim->qpc_entry_sz;
89 profile[MTHCA_RES_EEC].size = dev_lim->eec_entry_sz;
90 profile[MTHCA_RES_SRQ].size = dev_lim->srq_entry_sz;
91 profile[MTHCA_RES_CQ].size = dev_lim->cqc_entry_sz;
92 profile[MTHCA_RES_EQP].size = dev_lim->eqpc_entry_sz;
93 profile[MTHCA_RES_EEEC].size = dev_lim->eeec_entry_sz;
94 profile[MTHCA_RES_EQ].size = dev_lim->eqc_entry_sz;
95 profile[MTHCA_RES_RDB].size = MTHCA_RDB_ENTRY_SIZE;
96 profile[MTHCA_RES_MCG].size = MTHCA_MGM_ENTRY_SIZE;
97 profile[MTHCA_RES_MPT].size = dev_lim->mpt_entry_sz;
98 profile[MTHCA_RES_MTT].size = dev_lim->mtt_seg_sz;
99 profile[MTHCA_RES_UAR].size = dev_lim->uar_scratch_entry_sz;
100 profile[MTHCA_RES_UDAV].size = MTHCA_AV_SIZE;
101 profile[MTHCA_RES_UARC].size = request->uarc_size;
102
103 profile[MTHCA_RES_QP].num = request->num_qp;
104 profile[MTHCA_RES_EQP].num = request->num_qp;
105 profile[MTHCA_RES_RDB].num = request->num_qp * request->rdb_per_qp;
106 profile[MTHCA_RES_CQ].num = request->num_cq;
107 profile[MTHCA_RES_EQ].num = MTHCA_NUM_EQS;
108 profile[MTHCA_RES_MCG].num = request->num_mcg;
109 profile[MTHCA_RES_MPT].num = request->num_mpt;
110 profile[MTHCA_RES_MTT].num = request->num_mtt;
111 profile[MTHCA_RES_UAR].num = request->num_uar;
112 profile[MTHCA_RES_UARC].num = request->num_uar;
113 profile[MTHCA_RES_UDAV].num = request->num_udav;
114
115 for (i = 0; i < MTHCA_RES_NUM; ++i) {
116 profile[i].type = i;
117 profile[i].log_num = max(ffs(profile[i].num) - 1, 0);
118 profile[i].size *= profile[i].num;
119 if (dev->hca_type == ARBEL_NATIVE)
120 profile[i].size = max(profile[i].size, (u64) PAGE_SIZE);
121 }
122
123 if (dev->hca_type == ARBEL_NATIVE) {
124 mem_base = 0;
125 mem_avail = dev_lim->hca.arbel.max_icm_sz;
126 } else {
127 mem_base = dev->ddr_start;
128 mem_avail = dev->fw.tavor.fw_start - dev->ddr_start;
129 }
130
131 /*
132 * Sort the resources in decreasing order of size. Since they
133 * all have sizes that are powers of 2, we'll be able to keep
134 * resources aligned to their size and pack them without gaps
135 * using the sorted order.
136 */
137 for (i = MTHCA_RES_NUM; i > 0; --i)
138 for (j = 1; j < i; ++j) {
139 if (profile[j].size > profile[j - 1].size) {
140 tmp = profile[j];
141 profile[j] = profile[j - 1];
142 profile[j - 1] = tmp;
143 }
144 }
145
146 for (i = 0; i < MTHCA_RES_NUM; ++i) {
147 if (profile[i].size) {
148 profile[i].start = mem_base + total_size;
149 total_size += profile[i].size;
150 }
151 if (total_size > mem_avail) {
152 mthca_err(dev, "Profile requires 0x%llx bytes; "
153 "won't in 0x%llx bytes of context memory.\n",
154 (unsigned long long) total_size,
155 (unsigned long long) mem_avail);
156 kfree(profile);
157 return -ENOMEM;
158 }
159
160 if (profile[i].size)
161 mthca_dbg(dev, "profile[%2d]--%2d/%2d @ 0x%16llx "
162 "(size 0x%8llx)\n",
163 i, profile[i].type, profile[i].log_num,
164 (unsigned long long) profile[i].start,
165 (unsigned long long) profile[i].size);
166 }
167
168 if (dev->hca_type == ARBEL_NATIVE)
169 mthca_dbg(dev, "HCA context memory: reserving %d KB\n",
170 (int) (total_size >> 10));
171 else
172 mthca_dbg(dev, "HCA memory: allocated %d KB/%d KB (%d KB free)\n",
173 (int) (total_size >> 10), (int) (mem_avail >> 10),
174 (int) ((mem_avail - total_size) >> 10));
175
176 for (i = 0; i < MTHCA_RES_NUM; ++i) {
177 switch (profile[i].type) {
178 case MTHCA_RES_QP:
179 dev->limits.num_qps = profile[i].num;
180 init_hca->qpc_base = profile[i].start;
181 init_hca->log_num_qps = profile[i].log_num;
182 break;
183 case MTHCA_RES_EEC:
184 dev->limits.num_eecs = profile[i].num;
185 init_hca->eec_base = profile[i].start;
186 init_hca->log_num_eecs = profile[i].log_num;
187 break;
188 case MTHCA_RES_SRQ:
189 dev->limits.num_srqs = profile[i].num;
190 init_hca->srqc_base = profile[i].start;
191 init_hca->log_num_srqs = profile[i].log_num;
192 break;
193 case MTHCA_RES_CQ:
194 dev->limits.num_cqs = profile[i].num;
195 init_hca->cqc_base = profile[i].start;
196 init_hca->log_num_cqs = profile[i].log_num;
197 break;
198 case MTHCA_RES_EQP:
199 init_hca->eqpc_base = profile[i].start;
200 break;
201 case MTHCA_RES_EEEC:
202 init_hca->eeec_base = profile[i].start;
203 break;
204 case MTHCA_RES_EQ:
205 dev->limits.num_eqs = profile[i].num;
206 init_hca->eqc_base = profile[i].start;
207 init_hca->log_num_eqs = profile[i].log_num;
208 break;
209 case MTHCA_RES_RDB:
210 for (dev->qp_table.rdb_shift = 0;
211 profile[MTHCA_RES_QP].num << dev->qp_table.rdb_shift <
212 profile[i].num;
213 ++dev->qp_table.rdb_shift)
214 ; /* nothing */
215 dev->qp_table.rdb_base = (u32) profile[i].start;
216 init_hca->rdb_base = profile[i].start;
217 break;
218 case MTHCA_RES_MCG:
219 dev->limits.num_mgms = profile[i].num >> 1;
220 dev->limits.num_amgms = profile[i].num >> 1;
221 init_hca->mc_base = profile[i].start;
222 init_hca->log_mc_entry_sz = ffs(MTHCA_MGM_ENTRY_SIZE) - 1;
223 init_hca->log_mc_table_sz = profile[i].log_num;
224 init_hca->mc_hash_sz = 1 << (profile[i].log_num - 1);
225 break;
226 case MTHCA_RES_MPT:
227 dev->limits.num_mpts = profile[i].num;
228 init_hca->mpt_base = profile[i].start;
229 init_hca->log_mpt_sz = profile[i].log_num;
230 break;
231 case MTHCA_RES_MTT:
232 dev->limits.num_mtt_segs = profile[i].num;
233 dev->limits.mtt_seg_size = dev_lim->mtt_seg_sz;
234 dev->mr_table.mtt_base = profile[i].start;
235 init_hca->mtt_base = profile[i].start;
236 init_hca->mtt_seg_sz = ffs(dev_lim->mtt_seg_sz) - 7;
237 break;
238 case MTHCA_RES_UAR:
239 dev->limits.num_uars = profile[i].num;
240 init_hca->uar_scratch_base = profile[i].start;
241 break;
242 case MTHCA_RES_UDAV:
243 dev->av_table.ddr_av_base = profile[i].start;
244 dev->av_table.num_ddr_avs = profile[i].num;
245 break;
246 case MTHCA_RES_UARC:
247 dev->uar_table.uarc_size = request->uarc_size;
248 dev->uar_table.uarc_base = profile[i].start;
249 init_hca->uarc_base = profile[i].start;
250 init_hca->log_uarc_sz = ffs(request->uarc_size) - 13;
251 init_hca->log_uar_sz = ffs(request->num_uar) - 1;
252 break;
253 default:
254 break;
255 }
256 }
257
258 /*
259 * PDs don't take any HCA memory, but we assign them as part
260 * of the HCA profile anyway.
261 */
262 dev->limits.num_pds = MTHCA_NUM_PDS;
263
264 kfree(profile);
265 return total_size;
266}
diff --git a/drivers/infiniband/hw/mthca/mthca_profile.h b/drivers/infiniband/hw/mthca/mthca_profile.h
new file mode 100644
index 000000000000..daaf7999486c
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_profile.h
@@ -0,0 +1,58 @@
1/*
2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_profile.h 1349 2004-12-16 21:09:43Z roland $
33 */
34
35#ifndef MTHCA_PROFILE_H
36#define MTHCA_PROFILE_H
37
38#include "mthca_dev.h"
39#include "mthca_cmd.h"
40
41struct mthca_profile {
42 int num_qp;
43 int rdb_per_qp;
44 int num_cq;
45 int num_mcg;
46 int num_mpt;
47 int num_mtt;
48 int num_udav;
49 int num_uar;
50 int uarc_size;
51};
52
53u64 mthca_make_profile(struct mthca_dev *mdev,
54 struct mthca_profile *request,
55 struct mthca_dev_lim *dev_lim,
56 struct mthca_init_hca_param *init_hca);
57
58#endif /* MTHCA_PROFILE_H */
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
new file mode 100644
index 000000000000..bbf74cf43343
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -0,0 +1,660 @@
1/*
2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_provider.c 1397 2004-12-28 05:09:00Z roland $
33 */
34
35#include <ib_smi.h>
36
37#include "mthca_dev.h"
38#include "mthca_cmd.h"
39
40static int mthca_query_device(struct ib_device *ibdev,
41 struct ib_device_attr *props)
42{
43 struct ib_smp *in_mad = NULL;
44 struct ib_smp *out_mad = NULL;
45 int err = -ENOMEM;
46 struct mthca_dev* mdev = to_mdev(ibdev);
47
48 u8 status;
49
50 in_mad = kmalloc(sizeof *in_mad, GFP_KERNEL);
51 out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
52 if (!in_mad || !out_mad)
53 goto out;
54
55 props->fw_ver = mdev->fw_ver;
56
57 memset(in_mad, 0, sizeof *in_mad);
58 in_mad->base_version = 1;
59 in_mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
60 in_mad->class_version = 1;
61 in_mad->method = IB_MGMT_METHOD_GET;
62 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
63
64 err = mthca_MAD_IFC(mdev, 1, 1,
65 1, NULL, NULL, in_mad, out_mad,
66 &status);
67 if (err)
68 goto out;
69 if (status) {
70 err = -EINVAL;
71 goto out;
72 }
73
74 props->device_cap_flags = mdev->device_cap_flags;
75 props->vendor_id = be32_to_cpup((u32 *) (out_mad->data + 36)) &
76 0xffffff;
77 props->vendor_part_id = be16_to_cpup((u16 *) (out_mad->data + 30));
78 props->hw_ver = be16_to_cpup((u16 *) (out_mad->data + 32));
79 memcpy(&props->sys_image_guid, out_mad->data + 4, 8);
80 memcpy(&props->node_guid, out_mad->data + 12, 8);
81
82 err = 0;
83 out:
84 kfree(in_mad);
85 kfree(out_mad);
86 return err;
87}
88
89static int mthca_query_port(struct ib_device *ibdev,
90 u8 port, struct ib_port_attr *props)
91{
92 struct ib_smp *in_mad = NULL;
93 struct ib_smp *out_mad = NULL;
94 int err = -ENOMEM;
95 u8 status;
96
97 in_mad = kmalloc(sizeof *in_mad, GFP_KERNEL);
98 out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
99 if (!in_mad || !out_mad)
100 goto out;
101
102 memset(in_mad, 0, sizeof *in_mad);
103 in_mad->base_version = 1;
104 in_mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
105 in_mad->class_version = 1;
106 in_mad->method = IB_MGMT_METHOD_GET;
107 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;
108 in_mad->attr_mod = cpu_to_be32(port);
109
110 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
111 port, NULL, NULL, in_mad, out_mad,
112 &status);
113 if (err)
114 goto out;
115 if (status) {
116 err = -EINVAL;
117 goto out;
118 }
119
120 props->lid = be16_to_cpup((u16 *) (out_mad->data + 16));
121 props->lmc = out_mad->data[34] & 0x7;
122 props->sm_lid = be16_to_cpup((u16 *) (out_mad->data + 18));
123 props->sm_sl = out_mad->data[36] & 0xf;
124 props->state = out_mad->data[32] & 0xf;
125 props->phys_state = out_mad->data[33] >> 4;
126 props->port_cap_flags = be32_to_cpup((u32 *) (out_mad->data + 20));
127 props->gid_tbl_len = to_mdev(ibdev)->limits.gid_table_len;
128 props->pkey_tbl_len = to_mdev(ibdev)->limits.pkey_table_len;
129 props->qkey_viol_cntr = be16_to_cpup((u16 *) (out_mad->data + 48));
130 props->active_width = out_mad->data[31] & 0xf;
131 props->active_speed = out_mad->data[35] >> 4;
132
133 out:
134 kfree(in_mad);
135 kfree(out_mad);
136 return err;
137}
138
139static int mthca_modify_port(struct ib_device *ibdev,
140 u8 port, int port_modify_mask,
141 struct ib_port_modify *props)
142{
143 struct mthca_set_ib_param set_ib;
144 struct ib_port_attr attr;
145 int err;
146 u8 status;
147
148 if (down_interruptible(&to_mdev(ibdev)->cap_mask_mutex))
149 return -ERESTARTSYS;
150
151 err = mthca_query_port(ibdev, port, &attr);
152 if (err)
153 goto out;
154
155 set_ib.set_si_guid = 0;
156 set_ib.reset_qkey_viol = !!(port_modify_mask & IB_PORT_RESET_QKEY_CNTR);
157
158 set_ib.cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) &
159 ~props->clr_port_cap_mask;
160
161 err = mthca_SET_IB(to_mdev(ibdev), &set_ib, port, &status);
162 if (err)
163 goto out;
164 if (status) {
165 err = -EINVAL;
166 goto out;
167 }
168
169out:
170 up(&to_mdev(ibdev)->cap_mask_mutex);
171 return err;
172}
173
174static int mthca_query_pkey(struct ib_device *ibdev,
175 u8 port, u16 index, u16 *pkey)
176{
177 struct ib_smp *in_mad = NULL;
178 struct ib_smp *out_mad = NULL;
179 int err = -ENOMEM;
180 u8 status;
181
182 in_mad = kmalloc(sizeof *in_mad, GFP_KERNEL);
183 out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
184 if (!in_mad || !out_mad)
185 goto out;
186
187 memset(in_mad, 0, sizeof *in_mad);
188 in_mad->base_version = 1;
189 in_mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
190 in_mad->class_version = 1;
191 in_mad->method = IB_MGMT_METHOD_GET;
192 in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE;
193 in_mad->attr_mod = cpu_to_be32(index / 32);
194
195 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
196 port, NULL, NULL, in_mad, out_mad,
197 &status);
198 if (err)
199 goto out;
200 if (status) {
201 err = -EINVAL;
202 goto out;
203 }
204
205 *pkey = be16_to_cpu(((u16 *) out_mad->data)[index % 32]);
206
207 out:
208 kfree(in_mad);
209 kfree(out_mad);
210 return err;
211}
212
213static int mthca_query_gid(struct ib_device *ibdev, u8 port,
214 int index, union ib_gid *gid)
215{
216 struct ib_smp *in_mad = NULL;
217 struct ib_smp *out_mad = NULL;
218 int err = -ENOMEM;
219 u8 status;
220
221 in_mad = kmalloc(sizeof *in_mad, GFP_KERNEL);
222 out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
223 if (!in_mad || !out_mad)
224 goto out;
225
226 memset(in_mad, 0, sizeof *in_mad);
227 in_mad->base_version = 1;
228 in_mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
229 in_mad->class_version = 1;
230 in_mad->method = IB_MGMT_METHOD_GET;
231 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;
232 in_mad->attr_mod = cpu_to_be32(port);
233
234 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
235 port, NULL, NULL, in_mad, out_mad,
236 &status);
237 if (err)
238 goto out;
239 if (status) {
240 err = -EINVAL;
241 goto out;
242 }
243
244 memcpy(gid->raw, out_mad->data + 8, 8);
245
246 memset(in_mad, 0, sizeof *in_mad);
247 in_mad->base_version = 1;
248 in_mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
249 in_mad->class_version = 1;
250 in_mad->method = IB_MGMT_METHOD_GET;
251 in_mad->attr_id = IB_SMP_ATTR_GUID_INFO;
252 in_mad->attr_mod = cpu_to_be32(index / 8);
253
254 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
255 port, NULL, NULL, in_mad, out_mad,
256 &status);
257 if (err)
258 goto out;
259 if (status) {
260 err = -EINVAL;
261 goto out;
262 }
263
264 memcpy(gid->raw + 8, out_mad->data + (index % 8) * 16, 8);
265
266 out:
267 kfree(in_mad);
268 kfree(out_mad);
269 return err;
270}
271
272static struct ib_pd *mthca_alloc_pd(struct ib_device *ibdev)
273{
274 struct mthca_pd *pd;
275 int err;
276
277 pd = kmalloc(sizeof *pd, GFP_KERNEL);
278 if (!pd)
279 return ERR_PTR(-ENOMEM);
280
281 err = mthca_pd_alloc(to_mdev(ibdev), pd);
282 if (err) {
283 kfree(pd);
284 return ERR_PTR(err);
285 }
286
287 return &pd->ibpd;
288}
289
290static int mthca_dealloc_pd(struct ib_pd *pd)
291{
292 mthca_pd_free(to_mdev(pd->device), to_mpd(pd));
293 kfree(pd);
294
295 return 0;
296}
297
298static struct ib_ah *mthca_ah_create(struct ib_pd *pd,
299 struct ib_ah_attr *ah_attr)
300{
301 int err;
302 struct mthca_ah *ah;
303
304 ah = kmalloc(sizeof *ah, GFP_KERNEL);
305 if (!ah)
306 return ERR_PTR(-ENOMEM);
307
308 err = mthca_create_ah(to_mdev(pd->device), to_mpd(pd), ah_attr, ah);
309 if (err) {
310 kfree(ah);
311 return ERR_PTR(err);
312 }
313
314 return &ah->ibah;
315}
316
317static int mthca_ah_destroy(struct ib_ah *ah)
318{
319 mthca_destroy_ah(to_mdev(ah->device), to_mah(ah));
320 kfree(ah);
321
322 return 0;
323}
324
325static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
326 struct ib_qp_init_attr *init_attr)
327{
328 struct mthca_qp *qp;
329 int err;
330
331 switch (init_attr->qp_type) {
332 case IB_QPT_RC:
333 case IB_QPT_UC:
334 case IB_QPT_UD:
335 {
336 qp = kmalloc(sizeof *qp, GFP_KERNEL);
337 if (!qp)
338 return ERR_PTR(-ENOMEM);
339
340 qp->sq.max = init_attr->cap.max_send_wr;
341 qp->rq.max = init_attr->cap.max_recv_wr;
342 qp->sq.max_gs = init_attr->cap.max_send_sge;
343 qp->rq.max_gs = init_attr->cap.max_recv_sge;
344
345 err = mthca_alloc_qp(to_mdev(pd->device), to_mpd(pd),
346 to_mcq(init_attr->send_cq),
347 to_mcq(init_attr->recv_cq),
348 init_attr->qp_type, init_attr->sq_sig_type,
349 qp);
350 qp->ibqp.qp_num = qp->qpn;
351 break;
352 }
353 case IB_QPT_SMI:
354 case IB_QPT_GSI:
355 {
356 qp = kmalloc(sizeof (struct mthca_sqp), GFP_KERNEL);
357 if (!qp)
358 return ERR_PTR(-ENOMEM);
359
360 qp->sq.max = init_attr->cap.max_send_wr;
361 qp->rq.max = init_attr->cap.max_recv_wr;
362 qp->sq.max_gs = init_attr->cap.max_send_sge;
363 qp->rq.max_gs = init_attr->cap.max_recv_sge;
364
365 qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1;
366
367 err = mthca_alloc_sqp(to_mdev(pd->device), to_mpd(pd),
368 to_mcq(init_attr->send_cq),
369 to_mcq(init_attr->recv_cq),
370 init_attr->sq_sig_type,
371 qp->ibqp.qp_num, init_attr->port_num,
372 to_msqp(qp));
373 break;
374 }
375 default:
376 /* Don't support raw QPs */
377 return ERR_PTR(-ENOSYS);
378 }
379
380 if (err) {
381 kfree(qp);
382 return ERR_PTR(err);
383 }
384
385 init_attr->cap.max_inline_data = 0;
386
387 return &qp->ibqp;
388}
389
390static int mthca_destroy_qp(struct ib_qp *qp)
391{
392 mthca_free_qp(to_mdev(qp->device), to_mqp(qp));
393 kfree(qp);
394 return 0;
395}
396
397static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, int entries)
398{
399 struct mthca_cq *cq;
400 int nent;
401 int err;
402
403 cq = kmalloc(sizeof *cq, GFP_KERNEL);
404 if (!cq)
405 return ERR_PTR(-ENOMEM);
406
407 for (nent = 1; nent <= entries; nent <<= 1)
408 ; /* nothing */
409
410 err = mthca_init_cq(to_mdev(ibdev), nent, cq);
411 if (err) {
412 kfree(cq);
413 cq = ERR_PTR(err);
414 }
415
416 return &cq->ibcq;
417}
418
419static int mthca_destroy_cq(struct ib_cq *cq)
420{
421 mthca_free_cq(to_mdev(cq->device), to_mcq(cq));
422 kfree(cq);
423
424 return 0;
425}
426
427static inline u32 convert_access(int acc)
428{
429 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MTHCA_MPT_FLAG_ATOMIC : 0) |
430 (acc & IB_ACCESS_REMOTE_WRITE ? MTHCA_MPT_FLAG_REMOTE_WRITE : 0) |
431 (acc & IB_ACCESS_REMOTE_READ ? MTHCA_MPT_FLAG_REMOTE_READ : 0) |
432 (acc & IB_ACCESS_LOCAL_WRITE ? MTHCA_MPT_FLAG_LOCAL_WRITE : 0) |
433 MTHCA_MPT_FLAG_LOCAL_READ;
434}
435
436static struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, int acc)
437{
438 struct mthca_mr *mr;
439 int err;
440
441 mr = kmalloc(sizeof *mr, GFP_KERNEL);
442 if (!mr)
443 return ERR_PTR(-ENOMEM);
444
445 err = mthca_mr_alloc_notrans(to_mdev(pd->device),
446 to_mpd(pd)->pd_num,
447 convert_access(acc), mr);
448
449 if (err) {
450 kfree(mr);
451 return ERR_PTR(err);
452 }
453
454 return &mr->ibmr;
455}
456
457static struct ib_mr *mthca_reg_phys_mr(struct ib_pd *pd,
458 struct ib_phys_buf *buffer_list,
459 int num_phys_buf,
460 int acc,
461 u64 *iova_start)
462{
463 struct mthca_mr *mr;
464 u64 *page_list;
465 u64 total_size;
466 u64 mask;
467 int shift;
468 int npages;
469 int err;
470 int i, j, n;
471
472 /* First check that we have enough alignment */
473 if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK))
474 return ERR_PTR(-EINVAL);
475
476 if (num_phys_buf > 1 &&
477 ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK))
478 return ERR_PTR(-EINVAL);
479
480 mask = 0;
481 total_size = 0;
482 for (i = 0; i < num_phys_buf; ++i) {
483 if (buffer_list[i].addr & ~PAGE_MASK)
484 return ERR_PTR(-EINVAL);
485 if (i != 0 && i != num_phys_buf - 1 &&
486 (buffer_list[i].size & ~PAGE_MASK))
487 return ERR_PTR(-EINVAL);
488
489 total_size += buffer_list[i].size;
490 if (i > 0)
491 mask |= buffer_list[i].addr;
492 }
493
494 /* Find largest page shift we can use to cover buffers */
495 for (shift = PAGE_SHIFT; shift < 31; ++shift)
496 if (num_phys_buf > 1) {
497 if ((1ULL << shift) & mask)
498 break;
499 } else {
500 if (1ULL << shift >=
501 buffer_list[0].size +
502 (buffer_list[0].addr & ((1ULL << shift) - 1)))
503 break;
504 }
505
506 buffer_list[0].size += buffer_list[0].addr & ((1ULL << shift) - 1);
507 buffer_list[0].addr &= ~0ull << shift;
508
509 mr = kmalloc(sizeof *mr, GFP_KERNEL);
510 if (!mr)
511 return ERR_PTR(-ENOMEM);
512
513 npages = 0;
514 for (i = 0; i < num_phys_buf; ++i)
515 npages += (buffer_list[i].size + (1ULL << shift) - 1) >> shift;
516
517 if (!npages)
518 return &mr->ibmr;
519
520 page_list = kmalloc(npages * sizeof *page_list, GFP_KERNEL);
521 if (!page_list) {
522 kfree(mr);
523 return ERR_PTR(-ENOMEM);
524 }
525
526 n = 0;
527 for (i = 0; i < num_phys_buf; ++i)
528 for (j = 0;
529 j < (buffer_list[i].size + (1ULL << shift) - 1) >> shift;
530 ++j)
531 page_list[n++] = buffer_list[i].addr + ((u64) j << shift);
532
533 mthca_dbg(to_mdev(pd->device), "Registering memory at %llx (iova %llx) "
534 "in PD %x; shift %d, npages %d.\n",
535 (unsigned long long) buffer_list[0].addr,
536 (unsigned long long) *iova_start,
537 to_mpd(pd)->pd_num,
538 shift, npages);
539
540 err = mthca_mr_alloc_phys(to_mdev(pd->device),
541 to_mpd(pd)->pd_num,
542 page_list, shift, npages,
543 *iova_start, total_size,
544 convert_access(acc), mr);
545
546 if (err) {
547 kfree(mr);
548 return ERR_PTR(err);
549 }
550
551 kfree(page_list);
552 return &mr->ibmr;
553}
554
555static int mthca_dereg_mr(struct ib_mr *mr)
556{
557 mthca_free_mr(to_mdev(mr->device), to_mmr(mr));
558 kfree(mr);
559 return 0;
560}
561
562static ssize_t show_rev(struct class_device *cdev, char *buf)
563{
564 struct mthca_dev *dev = container_of(cdev, struct mthca_dev, ib_dev.class_dev);
565 return sprintf(buf, "%x\n", dev->rev_id);
566}
567
568static ssize_t show_fw_ver(struct class_device *cdev, char *buf)
569{
570 struct mthca_dev *dev = container_of(cdev, struct mthca_dev, ib_dev.class_dev);
571 return sprintf(buf, "%x.%x.%x\n", (int) (dev->fw_ver >> 32),
572 (int) (dev->fw_ver >> 16) & 0xffff,
573 (int) dev->fw_ver & 0xffff);
574}
575
576static ssize_t show_hca(struct class_device *cdev, char *buf)
577{
578 struct mthca_dev *dev = container_of(cdev, struct mthca_dev, ib_dev.class_dev);
579 switch (dev->hca_type) {
580 case TAVOR: return sprintf(buf, "MT23108\n");
581 case ARBEL_COMPAT: return sprintf(buf, "MT25208 (MT23108 compat mode)\n");
582 case ARBEL_NATIVE: return sprintf(buf, "MT25208\n");
583 default: return sprintf(buf, "unknown\n");
584 }
585}
586
587static CLASS_DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
588static CLASS_DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL);
589static CLASS_DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
590
591static struct class_device_attribute *mthca_class_attributes[] = {
592 &class_device_attr_hw_rev,
593 &class_device_attr_fw_ver,
594 &class_device_attr_hca_type
595};
596
597int mthca_register_device(struct mthca_dev *dev)
598{
599 int ret;
600 int i;
601
602 strlcpy(dev->ib_dev.name, "mthca%d", IB_DEVICE_NAME_MAX);
603 dev->ib_dev.node_type = IB_NODE_CA;
604 dev->ib_dev.phys_port_cnt = dev->limits.num_ports;
605 dev->ib_dev.dma_device = &dev->pdev->dev;
606 dev->ib_dev.class_dev.dev = &dev->pdev->dev;
607 dev->ib_dev.query_device = mthca_query_device;
608 dev->ib_dev.query_port = mthca_query_port;
609 dev->ib_dev.modify_port = mthca_modify_port;
610 dev->ib_dev.query_pkey = mthca_query_pkey;
611 dev->ib_dev.query_gid = mthca_query_gid;
612 dev->ib_dev.alloc_pd = mthca_alloc_pd;
613 dev->ib_dev.dealloc_pd = mthca_dealloc_pd;
614 dev->ib_dev.create_ah = mthca_ah_create;
615 dev->ib_dev.destroy_ah = mthca_ah_destroy;
616 dev->ib_dev.create_qp = mthca_create_qp;
617 dev->ib_dev.modify_qp = mthca_modify_qp;
618 dev->ib_dev.destroy_qp = mthca_destroy_qp;
619 dev->ib_dev.create_cq = mthca_create_cq;
620 dev->ib_dev.destroy_cq = mthca_destroy_cq;
621 dev->ib_dev.poll_cq = mthca_poll_cq;
622 dev->ib_dev.get_dma_mr = mthca_get_dma_mr;
623 dev->ib_dev.reg_phys_mr = mthca_reg_phys_mr;
624 dev->ib_dev.dereg_mr = mthca_dereg_mr;
625 dev->ib_dev.attach_mcast = mthca_multicast_attach;
626 dev->ib_dev.detach_mcast = mthca_multicast_detach;
627 dev->ib_dev.process_mad = mthca_process_mad;
628
629 if (dev->hca_type == ARBEL_NATIVE) {
630 dev->ib_dev.req_notify_cq = mthca_arbel_arm_cq;
631 dev->ib_dev.post_send = mthca_arbel_post_send;
632 dev->ib_dev.post_recv = mthca_arbel_post_receive;
633 } else {
634 dev->ib_dev.req_notify_cq = mthca_tavor_arm_cq;
635 dev->ib_dev.post_send = mthca_tavor_post_send;
636 dev->ib_dev.post_recv = mthca_tavor_post_receive;
637 }
638
639 init_MUTEX(&dev->cap_mask_mutex);
640
641 ret = ib_register_device(&dev->ib_dev);
642 if (ret)
643 return ret;
644
645 for (i = 0; i < ARRAY_SIZE(mthca_class_attributes); ++i) {
646 ret = class_device_create_file(&dev->ib_dev.class_dev,
647 mthca_class_attributes[i]);
648 if (ret) {
649 ib_unregister_device(&dev->ib_dev);
650 return ret;
651 }
652 }
653
654 return 0;
655}
656
657void mthca_unregister_device(struct mthca_dev *dev)
658{
659 ib_unregister_device(&dev->ib_dev);
660}
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.h b/drivers/infiniband/hw/mthca/mthca_provider.h
new file mode 100644
index 000000000000..0598f3905d9a
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_provider.h
@@ -0,0 +1,251 @@
1/*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_provider.h 1349 2004-12-16 21:09:43Z roland $
33 */
34
35#ifndef MTHCA_PROVIDER_H
36#define MTHCA_PROVIDER_H
37
38#include <ib_verbs.h>
39#include <ib_pack.h>
40
41#define MTHCA_MPT_FLAG_ATOMIC (1 << 14)
42#define MTHCA_MPT_FLAG_REMOTE_WRITE (1 << 13)
43#define MTHCA_MPT_FLAG_REMOTE_READ (1 << 12)
44#define MTHCA_MPT_FLAG_LOCAL_WRITE (1 << 11)
45#define MTHCA_MPT_FLAG_LOCAL_READ (1 << 10)
46
47struct mthca_buf_list {
48 void *buf;
49 DECLARE_PCI_UNMAP_ADDR(mapping)
50};
51
52struct mthca_uar {
53 unsigned long pfn;
54 int index;
55};
56
57struct mthca_mr {
58 struct ib_mr ibmr;
59 int order;
60 u32 first_seg;
61};
62
63struct mthca_pd {
64 struct ib_pd ibpd;
65 u32 pd_num;
66 atomic_t sqp_count;
67 struct mthca_mr ntmr;
68};
69
70struct mthca_eq {
71 struct mthca_dev *dev;
72 int eqn;
73 u32 eqn_mask;
74 u32 cons_index;
75 u16 msi_x_vector;
76 u16 msi_x_entry;
77 int have_irq;
78 int nent;
79 struct mthca_buf_list *page_list;
80 struct mthca_mr mr;
81};
82
83struct mthca_av;
84
85enum mthca_ah_type {
86 MTHCA_AH_ON_HCA,
87 MTHCA_AH_PCI_POOL,
88 MTHCA_AH_KMALLOC
89};
90
91struct mthca_ah {
92 struct ib_ah ibah;
93 enum mthca_ah_type type;
94 u32 key;
95 struct mthca_av *av;
96 dma_addr_t avdma;
97};
98
99/*
100 * Quick description of our CQ/QP locking scheme:
101 *
102 * We have one global lock that protects dev->cq/qp_table. Each
103 * struct mthca_cq/qp also has its own lock. An individual qp lock
104 * may be taken inside of an individual cq lock. Both cqs attached to
105 * a qp may be locked, with the send cq locked first. No other
106 * nesting should be done.
107 *
108 * Each struct mthca_cq/qp also has an atomic_t ref count. The
109 * pointer from the cq/qp_table to the struct counts as one reference.
110 * This reference also is good for access through the consumer API, so
111 * modifying the CQ/QP etc doesn't need to take another reference.
112 * Access because of a completion being polled does need a reference.
113 *
114 * Finally, each struct mthca_cq/qp has a wait_queue_head_t for the
115 * destroy function to sleep on.
116 *
117 * This means that access from the consumer API requires nothing but
118 * taking the struct's lock.
119 *
120 * Access because of a completion event should go as follows:
121 * - lock cq/qp_table and look up struct
122 * - increment ref count in struct
123 * - drop cq/qp_table lock
124 * - lock struct, do your thing, and unlock struct
125 * - decrement ref count; if zero, wake up waiters
126 *
127 * To destroy a CQ/QP, we can do the following:
128 * - lock cq/qp_table, remove pointer, unlock cq/qp_table lock
129 * - decrement ref count
130 * - wait_event until ref count is zero
131 *
132 * It is the consumer's responsibilty to make sure that no QP
133 * operations (WQE posting or state modification) are pending when the
134 * QP is destroyed. Also, the consumer must make sure that calls to
135 * qp_modify are serialized.
136 *
137 * Possible optimizations (wait for profile data to see if/where we
138 * have locks bouncing between CPUs):
139 * - split cq/qp table lock into n separate (cache-aligned) locks,
140 * indexed (say) by the page in the table
141 * - split QP struct lock into three (one for common info, one for the
142 * send queue and one for the receive queue)
143 */
144
145struct mthca_cq {
146 struct ib_cq ibcq;
147 spinlock_t lock;
148 atomic_t refcount;
149 int cqn;
150 u32 cons_index;
151 int is_direct;
152
153 /* Next fields are Arbel only */
154 int set_ci_db_index;
155 u32 *set_ci_db;
156 int arm_db_index;
157 u32 *arm_db;
158 int arm_sn;
159
160 union {
161 struct mthca_buf_list direct;
162 struct mthca_buf_list *page_list;
163 } queue;
164 struct mthca_mr mr;
165 wait_queue_head_t wait;
166};
167
168struct mthca_wq {
169 spinlock_t lock;
170 int max;
171 unsigned next_ind;
172 unsigned last_comp;
173 unsigned head;
174 unsigned tail;
175 void *last;
176 int max_gs;
177 int wqe_shift;
178
179 int db_index; /* Arbel only */
180 u32 *db;
181};
182
183struct mthca_qp {
184 struct ib_qp ibqp;
185 atomic_t refcount;
186 u32 qpn;
187 int is_direct;
188 u8 transport;
189 u8 state;
190 u8 atomic_rd_en;
191 u8 resp_depth;
192
193 struct mthca_mr mr;
194
195 struct mthca_wq rq;
196 struct mthca_wq sq;
197 enum ib_sig_type sq_policy;
198 int send_wqe_offset;
199
200 u64 *wrid;
201 union {
202 struct mthca_buf_list direct;
203 struct mthca_buf_list *page_list;
204 } queue;
205
206 wait_queue_head_t wait;
207};
208
209struct mthca_sqp {
210 struct mthca_qp qp;
211 int port;
212 int pkey_index;
213 u32 qkey;
214 u32 send_psn;
215 struct ib_ud_header ud_header;
216 int header_buf_size;
217 void *header_buf;
218 dma_addr_t header_dma;
219};
220
221static inline struct mthca_mr *to_mmr(struct ib_mr *ibmr)
222{
223 return container_of(ibmr, struct mthca_mr, ibmr);
224}
225
226static inline struct mthca_pd *to_mpd(struct ib_pd *ibpd)
227{
228 return container_of(ibpd, struct mthca_pd, ibpd);
229}
230
231static inline struct mthca_ah *to_mah(struct ib_ah *ibah)
232{
233 return container_of(ibah, struct mthca_ah, ibah);
234}
235
236static inline struct mthca_cq *to_mcq(struct ib_cq *ibcq)
237{
238 return container_of(ibcq, struct mthca_cq, ibcq);
239}
240
241static inline struct mthca_qp *to_mqp(struct ib_qp *ibqp)
242{
243 return container_of(ibqp, struct mthca_qp, ibqp);
244}
245
246static inline struct mthca_sqp *to_msqp(struct mthca_qp *qp)
247{
248 return container_of(qp, struct mthca_sqp, qp);
249}
250
251#endif /* MTHCA_PROVIDER_H */
diff --git a/drivers/infiniband/hw/mthca/mthca_qp.c b/drivers/infiniband/hw/mthca/mthca_qp.c
new file mode 100644
index 000000000000..7e4bbbd31f07
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_qp.c
@@ -0,0 +1,2056 @@
1/*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_qp.c 1355 2004-12-17 15:23:43Z roland $
33 */
34
35#include <linux/init.h>
36
37#include <ib_verbs.h>
38#include <ib_cache.h>
39#include <ib_pack.h>
40
41#include "mthca_dev.h"
42#include "mthca_cmd.h"
43#include "mthca_memfree.h"
44
45enum {
46 MTHCA_MAX_DIRECT_QP_SIZE = 4 * PAGE_SIZE,
47 MTHCA_ACK_REQ_FREQ = 10,
48 MTHCA_FLIGHT_LIMIT = 9,
49 MTHCA_UD_HEADER_SIZE = 72 /* largest UD header possible */
50};
51
52enum {
53 MTHCA_QP_STATE_RST = 0,
54 MTHCA_QP_STATE_INIT = 1,
55 MTHCA_QP_STATE_RTR = 2,
56 MTHCA_QP_STATE_RTS = 3,
57 MTHCA_QP_STATE_SQE = 4,
58 MTHCA_QP_STATE_SQD = 5,
59 MTHCA_QP_STATE_ERR = 6,
60 MTHCA_QP_STATE_DRAINING = 7
61};
62
63enum {
64 MTHCA_QP_ST_RC = 0x0,
65 MTHCA_QP_ST_UC = 0x1,
66 MTHCA_QP_ST_RD = 0x2,
67 MTHCA_QP_ST_UD = 0x3,
68 MTHCA_QP_ST_MLX = 0x7
69};
70
71enum {
72 MTHCA_QP_PM_MIGRATED = 0x3,
73 MTHCA_QP_PM_ARMED = 0x0,
74 MTHCA_QP_PM_REARM = 0x1
75};
76
77enum {
78 /* qp_context flags */
79 MTHCA_QP_BIT_DE = 1 << 8,
80 /* params1 */
81 MTHCA_QP_BIT_SRE = 1 << 15,
82 MTHCA_QP_BIT_SWE = 1 << 14,
83 MTHCA_QP_BIT_SAE = 1 << 13,
84 MTHCA_QP_BIT_SIC = 1 << 4,
85 MTHCA_QP_BIT_SSC = 1 << 3,
86 /* params2 */
87 MTHCA_QP_BIT_RRE = 1 << 15,
88 MTHCA_QP_BIT_RWE = 1 << 14,
89 MTHCA_QP_BIT_RAE = 1 << 13,
90 MTHCA_QP_BIT_RIC = 1 << 4,
91 MTHCA_QP_BIT_RSC = 1 << 3
92};
93
94struct mthca_qp_path {
95 u32 port_pkey;
96 u8 rnr_retry;
97 u8 g_mylmc;
98 u16 rlid;
99 u8 ackto;
100 u8 mgid_index;
101 u8 static_rate;
102 u8 hop_limit;
103 u32 sl_tclass_flowlabel;
104 u8 rgid[16];
105} __attribute__((packed));
106
107struct mthca_qp_context {
108 u32 flags;
109 u32 tavor_sched_queue; /* Reserved on Arbel */
110 u8 mtu_msgmax;
111 u8 rq_size_stride; /* Reserved on Tavor */
112 u8 sq_size_stride; /* Reserved on Tavor */
113 u8 rlkey_arbel_sched_queue; /* Reserved on Tavor */
114 u32 usr_page;
115 u32 local_qpn;
116 u32 remote_qpn;
117 u32 reserved1[2];
118 struct mthca_qp_path pri_path;
119 struct mthca_qp_path alt_path;
120 u32 rdd;
121 u32 pd;
122 u32 wqe_base;
123 u32 wqe_lkey;
124 u32 params1;
125 u32 reserved2;
126 u32 next_send_psn;
127 u32 cqn_snd;
128 u32 snd_wqe_base_l; /* Next send WQE on Tavor */
129 u32 snd_db_index; /* (debugging only entries) */
130 u32 last_acked_psn;
131 u32 ssn;
132 u32 params2;
133 u32 rnr_nextrecvpsn;
134 u32 ra_buff_indx;
135 u32 cqn_rcv;
136 u32 rcv_wqe_base_l; /* Next recv WQE on Tavor */
137 u32 rcv_db_index; /* (debugging only entries) */
138 u32 qkey;
139 u32 srqn;
140 u32 rmsn;
141 u16 rq_wqe_counter; /* reserved on Tavor */
142 u16 sq_wqe_counter; /* reserved on Tavor */
143 u32 reserved3[18];
144} __attribute__((packed));
145
146struct mthca_qp_param {
147 u32 opt_param_mask;
148 u32 reserved1;
149 struct mthca_qp_context context;
150 u32 reserved2[62];
151} __attribute__((packed));
152
153enum {
154 MTHCA_QP_OPTPAR_ALT_ADDR_PATH = 1 << 0,
155 MTHCA_QP_OPTPAR_RRE = 1 << 1,
156 MTHCA_QP_OPTPAR_RAE = 1 << 2,
157 MTHCA_QP_OPTPAR_RWE = 1 << 3,
158 MTHCA_QP_OPTPAR_PKEY_INDEX = 1 << 4,
159 MTHCA_QP_OPTPAR_Q_KEY = 1 << 5,
160 MTHCA_QP_OPTPAR_RNR_TIMEOUT = 1 << 6,
161 MTHCA_QP_OPTPAR_PRIMARY_ADDR_PATH = 1 << 7,
162 MTHCA_QP_OPTPAR_SRA_MAX = 1 << 8,
163 MTHCA_QP_OPTPAR_RRA_MAX = 1 << 9,
164 MTHCA_QP_OPTPAR_PM_STATE = 1 << 10,
165 MTHCA_QP_OPTPAR_PORT_NUM = 1 << 11,
166 MTHCA_QP_OPTPAR_RETRY_COUNT = 1 << 12,
167 MTHCA_QP_OPTPAR_ALT_RNR_RETRY = 1 << 13,
168 MTHCA_QP_OPTPAR_ACK_TIMEOUT = 1 << 14,
169 MTHCA_QP_OPTPAR_RNR_RETRY = 1 << 15,
170 MTHCA_QP_OPTPAR_SCHED_QUEUE = 1 << 16
171};
172
173enum {
174 MTHCA_OPCODE_NOP = 0x00,
175 MTHCA_OPCODE_RDMA_WRITE = 0x08,
176 MTHCA_OPCODE_RDMA_WRITE_IMM = 0x09,
177 MTHCA_OPCODE_SEND = 0x0a,
178 MTHCA_OPCODE_SEND_IMM = 0x0b,
179 MTHCA_OPCODE_RDMA_READ = 0x10,
180 MTHCA_OPCODE_ATOMIC_CS = 0x11,
181 MTHCA_OPCODE_ATOMIC_FA = 0x12,
182 MTHCA_OPCODE_BIND_MW = 0x18,
183 MTHCA_OPCODE_INVALID = 0xff
184};
185
186enum {
187 MTHCA_NEXT_DBD = 1 << 7,
188 MTHCA_NEXT_FENCE = 1 << 6,
189 MTHCA_NEXT_CQ_UPDATE = 1 << 3,
190 MTHCA_NEXT_EVENT_GEN = 1 << 2,
191 MTHCA_NEXT_SOLICIT = 1 << 1,
192
193 MTHCA_MLX_VL15 = 1 << 17,
194 MTHCA_MLX_SLR = 1 << 16
195};
196
197struct mthca_next_seg {
198 u32 nda_op; /* [31:6] next WQE [4:0] next opcode */
199 u32 ee_nds; /* [31:8] next EE [7] DBD [6] F [5:0] next WQE size */
200 u32 flags; /* [3] CQ [2] Event [1] Solicit */
201 u32 imm; /* immediate data */
202};
203
204struct mthca_tavor_ud_seg {
205 u32 reserved1;
206 u32 lkey;
207 u64 av_addr;
208 u32 reserved2[4];
209 u32 dqpn;
210 u32 qkey;
211 u32 reserved3[2];
212};
213
214struct mthca_arbel_ud_seg {
215 u32 av[8];
216 u32 dqpn;
217 u32 qkey;
218 u32 reserved[2];
219};
220
221struct mthca_bind_seg {
222 u32 flags; /* [31] Atomic [30] rem write [29] rem read */
223 u32 reserved;
224 u32 new_rkey;
225 u32 lkey;
226 u64 addr;
227 u64 length;
228};
229
230struct mthca_raddr_seg {
231 u64 raddr;
232 u32 rkey;
233 u32 reserved;
234};
235
236struct mthca_atomic_seg {
237 u64 swap_add;
238 u64 compare;
239};
240
241struct mthca_data_seg {
242 u32 byte_count;
243 u32 lkey;
244 u64 addr;
245};
246
247struct mthca_mlx_seg {
248 u32 nda_op;
249 u32 nds;
250 u32 flags; /* [17] VL15 [16] SLR [14:12] static rate
251 [11:8] SL [3] C [2] E */
252 u16 rlid;
253 u16 vcrc;
254};
255
256static const u8 mthca_opcode[] = {
257 [IB_WR_SEND] = MTHCA_OPCODE_SEND,
258 [IB_WR_SEND_WITH_IMM] = MTHCA_OPCODE_SEND_IMM,
259 [IB_WR_RDMA_WRITE] = MTHCA_OPCODE_RDMA_WRITE,
260 [IB_WR_RDMA_WRITE_WITH_IMM] = MTHCA_OPCODE_RDMA_WRITE_IMM,
261 [IB_WR_RDMA_READ] = MTHCA_OPCODE_RDMA_READ,
262 [IB_WR_ATOMIC_CMP_AND_SWP] = MTHCA_OPCODE_ATOMIC_CS,
263 [IB_WR_ATOMIC_FETCH_AND_ADD] = MTHCA_OPCODE_ATOMIC_FA,
264};
265
266static int is_sqp(struct mthca_dev *dev, struct mthca_qp *qp)
267{
268 return qp->qpn >= dev->qp_table.sqp_start &&
269 qp->qpn <= dev->qp_table.sqp_start + 3;
270}
271
272static int is_qp0(struct mthca_dev *dev, struct mthca_qp *qp)
273{
274 return qp->qpn >= dev->qp_table.sqp_start &&
275 qp->qpn <= dev->qp_table.sqp_start + 1;
276}
277
278static void *get_recv_wqe(struct mthca_qp *qp, int n)
279{
280 if (qp->is_direct)
281 return qp->queue.direct.buf + (n << qp->rq.wqe_shift);
282 else
283 return qp->queue.page_list[(n << qp->rq.wqe_shift) >> PAGE_SHIFT].buf +
284 ((n << qp->rq.wqe_shift) & (PAGE_SIZE - 1));
285}
286
287static void *get_send_wqe(struct mthca_qp *qp, int n)
288{
289 if (qp->is_direct)
290 return qp->queue.direct.buf + qp->send_wqe_offset +
291 (n << qp->sq.wqe_shift);
292 else
293 return qp->queue.page_list[(qp->send_wqe_offset +
294 (n << qp->sq.wqe_shift)) >>
295 PAGE_SHIFT].buf +
296 ((qp->send_wqe_offset + (n << qp->sq.wqe_shift)) &
297 (PAGE_SIZE - 1));
298}
299
300void mthca_qp_event(struct mthca_dev *dev, u32 qpn,
301 enum ib_event_type event_type)
302{
303 struct mthca_qp *qp;
304 struct ib_event event;
305
306 spin_lock(&dev->qp_table.lock);
307 qp = mthca_array_get(&dev->qp_table.qp, qpn & (dev->limits.num_qps - 1));
308 if (qp)
309 atomic_inc(&qp->refcount);
310 spin_unlock(&dev->qp_table.lock);
311
312 if (!qp) {
313 mthca_warn(dev, "Async event for bogus QP %08x\n", qpn);
314 return;
315 }
316
317 event.device = &dev->ib_dev;
318 event.event = event_type;
319 event.element.qp = &qp->ibqp;
320 if (qp->ibqp.event_handler)
321 qp->ibqp.event_handler(&event, qp->ibqp.qp_context);
322
323 if (atomic_dec_and_test(&qp->refcount))
324 wake_up(&qp->wait);
325}
326
327static int to_mthca_state(enum ib_qp_state ib_state)
328{
329 switch (ib_state) {
330 case IB_QPS_RESET: return MTHCA_QP_STATE_RST;
331 case IB_QPS_INIT: return MTHCA_QP_STATE_INIT;
332 case IB_QPS_RTR: return MTHCA_QP_STATE_RTR;
333 case IB_QPS_RTS: return MTHCA_QP_STATE_RTS;
334 case IB_QPS_SQD: return MTHCA_QP_STATE_SQD;
335 case IB_QPS_SQE: return MTHCA_QP_STATE_SQE;
336 case IB_QPS_ERR: return MTHCA_QP_STATE_ERR;
337 default: return -1;
338 }
339}
340
341enum { RC, UC, UD, RD, RDEE, MLX, NUM_TRANS };
342
343static int to_mthca_st(int transport)
344{
345 switch (transport) {
346 case RC: return MTHCA_QP_ST_RC;
347 case UC: return MTHCA_QP_ST_UC;
348 case UD: return MTHCA_QP_ST_UD;
349 case RD: return MTHCA_QP_ST_RD;
350 case MLX: return MTHCA_QP_ST_MLX;
351 default: return -1;
352 }
353}
354
355static const struct {
356 int trans;
357 u32 req_param[NUM_TRANS];
358 u32 opt_param[NUM_TRANS];
359} state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
360 [IB_QPS_RESET] = {
361 [IB_QPS_RESET] = { .trans = MTHCA_TRANS_ANY2RST },
362 [IB_QPS_ERR] = { .trans = MTHCA_TRANS_ANY2ERR },
363 [IB_QPS_INIT] = {
364 .trans = MTHCA_TRANS_RST2INIT,
365 .req_param = {
366 [UD] = (IB_QP_PKEY_INDEX |
367 IB_QP_PORT |
368 IB_QP_QKEY),
369 [RC] = (IB_QP_PKEY_INDEX |
370 IB_QP_PORT |
371 IB_QP_ACCESS_FLAGS),
372 [MLX] = (IB_QP_PKEY_INDEX |
373 IB_QP_QKEY),
374 },
375 /* bug-for-bug compatibility with VAPI: */
376 .opt_param = {
377 [MLX] = IB_QP_PORT
378 }
379 },
380 },
381 [IB_QPS_INIT] = {
382 [IB_QPS_RESET] = { .trans = MTHCA_TRANS_ANY2RST },
383 [IB_QPS_ERR] = { .trans = MTHCA_TRANS_ANY2ERR },
384 [IB_QPS_INIT] = {
385 .trans = MTHCA_TRANS_INIT2INIT,
386 .opt_param = {
387 [UD] = (IB_QP_PKEY_INDEX |
388 IB_QP_PORT |
389 IB_QP_QKEY),
390 [RC] = (IB_QP_PKEY_INDEX |
391 IB_QP_PORT |
392 IB_QP_ACCESS_FLAGS),
393 [MLX] = (IB_QP_PKEY_INDEX |
394 IB_QP_QKEY),
395 }
396 },
397 [IB_QPS_RTR] = {
398 .trans = MTHCA_TRANS_INIT2RTR,
399 .req_param = {
400 [RC] = (IB_QP_AV |
401 IB_QP_PATH_MTU |
402 IB_QP_DEST_QPN |
403 IB_QP_RQ_PSN |
404 IB_QP_MAX_DEST_RD_ATOMIC |
405 IB_QP_MIN_RNR_TIMER),
406 },
407 .opt_param = {
408 [UD] = (IB_QP_PKEY_INDEX |
409 IB_QP_QKEY),
410 [RC] = (IB_QP_ALT_PATH |
411 IB_QP_ACCESS_FLAGS |
412 IB_QP_PKEY_INDEX),
413 [MLX] = (IB_QP_PKEY_INDEX |
414 IB_QP_QKEY),
415 }
416 }
417 },
418 [IB_QPS_RTR] = {
419 [IB_QPS_RESET] = { .trans = MTHCA_TRANS_ANY2RST },
420 [IB_QPS_ERR] = { .trans = MTHCA_TRANS_ANY2ERR },
421 [IB_QPS_RTS] = {
422 .trans = MTHCA_TRANS_RTR2RTS,
423 .req_param = {
424 [UD] = IB_QP_SQ_PSN,
425 [RC] = (IB_QP_TIMEOUT |
426 IB_QP_RETRY_CNT |
427 IB_QP_RNR_RETRY |
428 IB_QP_SQ_PSN |
429 IB_QP_MAX_QP_RD_ATOMIC),
430 [MLX] = IB_QP_SQ_PSN,
431 },
432 .opt_param = {
433 [UD] = (IB_QP_CUR_STATE |
434 IB_QP_QKEY),
435 [RC] = (IB_QP_CUR_STATE |
436 IB_QP_ALT_PATH |
437 IB_QP_ACCESS_FLAGS |
438 IB_QP_PKEY_INDEX |
439 IB_QP_MIN_RNR_TIMER |
440 IB_QP_PATH_MIG_STATE),
441 [MLX] = (IB_QP_CUR_STATE |
442 IB_QP_QKEY),
443 }
444 }
445 },
446 [IB_QPS_RTS] = {
447 [IB_QPS_RESET] = { .trans = MTHCA_TRANS_ANY2RST },
448 [IB_QPS_ERR] = { .trans = MTHCA_TRANS_ANY2ERR },
449 [IB_QPS_RTS] = {
450 .trans = MTHCA_TRANS_RTS2RTS,
451 .opt_param = {
452 [UD] = (IB_QP_CUR_STATE |
453 IB_QP_QKEY),
454 [RC] = (IB_QP_ACCESS_FLAGS |
455 IB_QP_ALT_PATH |
456 IB_QP_PATH_MIG_STATE |
457 IB_QP_MIN_RNR_TIMER),
458 [MLX] = (IB_QP_CUR_STATE |
459 IB_QP_QKEY),
460 }
461 },
462 [IB_QPS_SQD] = {
463 .trans = MTHCA_TRANS_RTS2SQD,
464 },
465 },
466 [IB_QPS_SQD] = {
467 [IB_QPS_RESET] = { .trans = MTHCA_TRANS_ANY2RST },
468 [IB_QPS_ERR] = { .trans = MTHCA_TRANS_ANY2ERR },
469 [IB_QPS_RTS] = {
470 .trans = MTHCA_TRANS_SQD2RTS,
471 .opt_param = {
472 [UD] = (IB_QP_CUR_STATE |
473 IB_QP_QKEY),
474 [RC] = (IB_QP_CUR_STATE |
475 IB_QP_ALT_PATH |
476 IB_QP_ACCESS_FLAGS |
477 IB_QP_MIN_RNR_TIMER |
478 IB_QP_PATH_MIG_STATE),
479 [MLX] = (IB_QP_CUR_STATE |
480 IB_QP_QKEY),
481 }
482 },
483 [IB_QPS_SQD] = {
484 .trans = MTHCA_TRANS_SQD2SQD,
485 .opt_param = {
486 [UD] = (IB_QP_PKEY_INDEX |
487 IB_QP_QKEY),
488 [RC] = (IB_QP_AV |
489 IB_QP_TIMEOUT |
490 IB_QP_RETRY_CNT |
491 IB_QP_RNR_RETRY |
492 IB_QP_MAX_QP_RD_ATOMIC |
493 IB_QP_MAX_DEST_RD_ATOMIC |
494 IB_QP_CUR_STATE |
495 IB_QP_ALT_PATH |
496 IB_QP_ACCESS_FLAGS |
497 IB_QP_PKEY_INDEX |
498 IB_QP_MIN_RNR_TIMER |
499 IB_QP_PATH_MIG_STATE),
500 [MLX] = (IB_QP_PKEY_INDEX |
501 IB_QP_QKEY),
502 }
503 }
504 },
505 [IB_QPS_SQE] = {
506 [IB_QPS_RESET] = { .trans = MTHCA_TRANS_ANY2RST },
507 [IB_QPS_ERR] = { .trans = MTHCA_TRANS_ANY2ERR },
508 [IB_QPS_RTS] = {
509 .trans = MTHCA_TRANS_SQERR2RTS,
510 .opt_param = {
511 [UD] = (IB_QP_CUR_STATE |
512 IB_QP_QKEY),
513 [RC] = (IB_QP_CUR_STATE |
514 IB_QP_MIN_RNR_TIMER),
515 [MLX] = (IB_QP_CUR_STATE |
516 IB_QP_QKEY),
517 }
518 }
519 },
520 [IB_QPS_ERR] = {
521 [IB_QPS_RESET] = { .trans = MTHCA_TRANS_ANY2RST },
522 [IB_QPS_ERR] = { .trans = MTHCA_TRANS_ANY2ERR }
523 }
524};
525
526static void store_attrs(struct mthca_sqp *sqp, struct ib_qp_attr *attr,
527 int attr_mask)
528{
529 if (attr_mask & IB_QP_PKEY_INDEX)
530 sqp->pkey_index = attr->pkey_index;
531 if (attr_mask & IB_QP_QKEY)
532 sqp->qkey = attr->qkey;
533 if (attr_mask & IB_QP_SQ_PSN)
534 sqp->send_psn = attr->sq_psn;
535}
536
537static void init_port(struct mthca_dev *dev, int port)
538{
539 int err;
540 u8 status;
541 struct mthca_init_ib_param param;
542
543 memset(&param, 0, sizeof param);
544
545 param.enable_1x = 1;
546 param.enable_4x = 1;
547 param.vl_cap = dev->limits.vl_cap;
548 param.mtu_cap = dev->limits.mtu_cap;
549 param.gid_cap = dev->limits.gid_table_len;
550 param.pkey_cap = dev->limits.pkey_table_len;
551
552 err = mthca_INIT_IB(dev, &param, port, &status);
553 if (err)
554 mthca_warn(dev, "INIT_IB failed, return code %d.\n", err);
555 if (status)
556 mthca_warn(dev, "INIT_IB returned status %02x.\n", status);
557}
558
559int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask)
560{
561 struct mthca_dev *dev = to_mdev(ibqp->device);
562 struct mthca_qp *qp = to_mqp(ibqp);
563 enum ib_qp_state cur_state, new_state;
564 void *mailbox = NULL;
565 struct mthca_qp_param *qp_param;
566 struct mthca_qp_context *qp_context;
567 u32 req_param, opt_param;
568 u8 status;
569 int err;
570
571 if (attr_mask & IB_QP_CUR_STATE) {
572 if (attr->cur_qp_state != IB_QPS_RTR &&
573 attr->cur_qp_state != IB_QPS_RTS &&
574 attr->cur_qp_state != IB_QPS_SQD &&
575 attr->cur_qp_state != IB_QPS_SQE)
576 return -EINVAL;
577 else
578 cur_state = attr->cur_qp_state;
579 } else {
580 spin_lock_irq(&qp->sq.lock);
581 spin_lock(&qp->rq.lock);
582 cur_state = qp->state;
583 spin_unlock(&qp->rq.lock);
584 spin_unlock_irq(&qp->sq.lock);
585 }
586
587 if (attr_mask & IB_QP_STATE) {
588 if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR)
589 return -EINVAL;
590 new_state = attr->qp_state;
591 } else
592 new_state = cur_state;
593
594 if (state_table[cur_state][new_state].trans == MTHCA_TRANS_INVALID) {
595 mthca_dbg(dev, "Illegal QP transition "
596 "%d->%d\n", cur_state, new_state);
597 return -EINVAL;
598 }
599
600 req_param = state_table[cur_state][new_state].req_param[qp->transport];
601 opt_param = state_table[cur_state][new_state].opt_param[qp->transport];
602
603 if ((req_param & attr_mask) != req_param) {
604 mthca_dbg(dev, "QP transition "
605 "%d->%d missing req attr 0x%08x\n",
606 cur_state, new_state,
607 req_param & ~attr_mask);
608 return -EINVAL;
609 }
610
611 if (attr_mask & ~(req_param | opt_param | IB_QP_STATE)) {
612 mthca_dbg(dev, "QP transition (transport %d) "
613 "%d->%d has extra attr 0x%08x\n",
614 qp->transport,
615 cur_state, new_state,
616 attr_mask & ~(req_param | opt_param |
617 IB_QP_STATE));
618 return -EINVAL;
619 }
620
621 mailbox = kmalloc(sizeof (*qp_param) + MTHCA_CMD_MAILBOX_EXTRA, GFP_KERNEL);
622 if (!mailbox)
623 return -ENOMEM;
624 qp_param = MAILBOX_ALIGN(mailbox);
625 qp_context = &qp_param->context;
626 memset(qp_param, 0, sizeof *qp_param);
627
628 qp_context->flags = cpu_to_be32((to_mthca_state(new_state) << 28) |
629 (to_mthca_st(qp->transport) << 16));
630 qp_context->flags |= cpu_to_be32(MTHCA_QP_BIT_DE);
631 if (!(attr_mask & IB_QP_PATH_MIG_STATE))
632 qp_context->flags |= cpu_to_be32(MTHCA_QP_PM_MIGRATED << 11);
633 else {
634 qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_PM_STATE);
635 switch (attr->path_mig_state) {
636 case IB_MIG_MIGRATED:
637 qp_context->flags |= cpu_to_be32(MTHCA_QP_PM_MIGRATED << 11);
638 break;
639 case IB_MIG_REARM:
640 qp_context->flags |= cpu_to_be32(MTHCA_QP_PM_REARM << 11);
641 break;
642 case IB_MIG_ARMED:
643 qp_context->flags |= cpu_to_be32(MTHCA_QP_PM_ARMED << 11);
644 break;
645 }
646 }
647
648 /* leave tavor_sched_queue as 0 */
649
650 if (qp->transport == MLX || qp->transport == UD)
651 qp_context->mtu_msgmax = (IB_MTU_2048 << 5) | 11;
652 else if (attr_mask & IB_QP_PATH_MTU)
653 qp_context->mtu_msgmax = (attr->path_mtu << 5) | 31;
654
655 if (dev->hca_type == ARBEL_NATIVE) {
656 qp_context->rq_size_stride =
657 ((ffs(qp->rq.max) - 1) << 3) | (qp->rq.wqe_shift - 4);
658 qp_context->sq_size_stride =
659 ((ffs(qp->sq.max) - 1) << 3) | (qp->sq.wqe_shift - 4);
660 }
661
662 /* leave arbel_sched_queue as 0 */
663
664 qp_context->usr_page = cpu_to_be32(dev->driver_uar.index);
665 qp_context->local_qpn = cpu_to_be32(qp->qpn);
666 if (attr_mask & IB_QP_DEST_QPN) {
667 qp_context->remote_qpn = cpu_to_be32(attr->dest_qp_num);
668 }
669
670 if (qp->transport == MLX)
671 qp_context->pri_path.port_pkey |=
672 cpu_to_be32(to_msqp(qp)->port << 24);
673 else {
674 if (attr_mask & IB_QP_PORT) {
675 qp_context->pri_path.port_pkey |=
676 cpu_to_be32(attr->port_num << 24);
677 qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_PORT_NUM);
678 }
679 }
680
681 if (attr_mask & IB_QP_PKEY_INDEX) {
682 qp_context->pri_path.port_pkey |=
683 cpu_to_be32(attr->pkey_index);
684 qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_PKEY_INDEX);
685 }
686
687 if (attr_mask & IB_QP_RNR_RETRY) {
688 qp_context->pri_path.rnr_retry = attr->rnr_retry << 5;
689 qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RNR_RETRY);
690 }
691
692 if (attr_mask & IB_QP_AV) {
693 qp_context->pri_path.g_mylmc = attr->ah_attr.src_path_bits & 0x7f;
694 qp_context->pri_path.rlid = cpu_to_be16(attr->ah_attr.dlid);
695 qp_context->pri_path.static_rate = (!!attr->ah_attr.static_rate) << 3;
696 if (attr->ah_attr.ah_flags & IB_AH_GRH) {
697 qp_context->pri_path.g_mylmc |= 1 << 7;
698 qp_context->pri_path.mgid_index = attr->ah_attr.grh.sgid_index;
699 qp_context->pri_path.hop_limit = attr->ah_attr.grh.hop_limit;
700 qp_context->pri_path.sl_tclass_flowlabel =
701 cpu_to_be32((attr->ah_attr.sl << 28) |
702 (attr->ah_attr.grh.traffic_class << 20) |
703 (attr->ah_attr.grh.flow_label));
704 memcpy(qp_context->pri_path.rgid,
705 attr->ah_attr.grh.dgid.raw, 16);
706 } else {
707 qp_context->pri_path.sl_tclass_flowlabel =
708 cpu_to_be32(attr->ah_attr.sl << 28);
709 }
710 qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_PRIMARY_ADDR_PATH);
711 }
712
713 if (attr_mask & IB_QP_TIMEOUT) {
714 qp_context->pri_path.ackto = attr->timeout;
715 qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_ACK_TIMEOUT);
716 }
717
718 /* XXX alt_path */
719
720 /* leave rdd as 0 */
721 qp_context->pd = cpu_to_be32(to_mpd(ibqp->pd)->pd_num);
722 /* leave wqe_base as 0 (we always create an MR based at 0 for WQs) */
723 qp_context->wqe_lkey = cpu_to_be32(qp->mr.ibmr.lkey);
724 qp_context->params1 = cpu_to_be32((MTHCA_ACK_REQ_FREQ << 28) |
725 (MTHCA_FLIGHT_LIMIT << 24) |
726 MTHCA_QP_BIT_SRE |
727 MTHCA_QP_BIT_SWE |
728 MTHCA_QP_BIT_SAE);
729 if (qp->sq_policy == IB_SIGNAL_ALL_WR)
730 qp_context->params1 |= cpu_to_be32(MTHCA_QP_BIT_SSC);
731 if (attr_mask & IB_QP_RETRY_CNT) {
732 qp_context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
733 qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RETRY_COUNT);
734 }
735
736 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
737 qp_context->params1 |= cpu_to_be32(min(attr->max_dest_rd_atomic ?
738 ffs(attr->max_dest_rd_atomic) - 1 : 0,
739 7) << 21);
740 qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_SRA_MAX);
741 }
742
743 if (attr_mask & IB_QP_SQ_PSN)
744 qp_context->next_send_psn = cpu_to_be32(attr->sq_psn);
745 qp_context->cqn_snd = cpu_to_be32(to_mcq(ibqp->send_cq)->cqn);
746
747 if (dev->hca_type == ARBEL_NATIVE) {
748 qp_context->snd_wqe_base_l = cpu_to_be32(qp->send_wqe_offset);
749 qp_context->snd_db_index = cpu_to_be32(qp->sq.db_index);
750 }
751
752 if (attr_mask & IB_QP_ACCESS_FLAGS) {
753 /*
754 * Only enable RDMA/atomics if we have responder
755 * resources set to a non-zero value.
756 */
757 if (qp->resp_depth) {
758 qp_context->params2 |=
759 cpu_to_be32(attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE ?
760 MTHCA_QP_BIT_RWE : 0);
761 qp_context->params2 |=
762 cpu_to_be32(attr->qp_access_flags & IB_ACCESS_REMOTE_READ ?
763 MTHCA_QP_BIT_RRE : 0);
764 qp_context->params2 |=
765 cpu_to_be32(attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC ?
766 MTHCA_QP_BIT_RAE : 0);
767 }
768
769 qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RWE |
770 MTHCA_QP_OPTPAR_RRE |
771 MTHCA_QP_OPTPAR_RAE);
772
773 qp->atomic_rd_en = attr->qp_access_flags;
774 }
775
776 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
777 u8 rra_max;
778
779 if (qp->resp_depth && !attr->max_rd_atomic) {
780 /*
781 * Lowering our responder resources to zero.
782 * Turn off RDMA/atomics as responder.
783 * (RWE/RRE/RAE in params2 already zero)
784 */
785 qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RWE |
786 MTHCA_QP_OPTPAR_RRE |
787 MTHCA_QP_OPTPAR_RAE);
788 }
789
790 if (!qp->resp_depth && attr->max_rd_atomic) {
791 /*
792 * Increasing our responder resources from
793 * zero. Turn on RDMA/atomics as appropriate.
794 */
795 qp_context->params2 |=
796 cpu_to_be32(qp->atomic_rd_en & IB_ACCESS_REMOTE_WRITE ?
797 MTHCA_QP_BIT_RWE : 0);
798 qp_context->params2 |=
799 cpu_to_be32(qp->atomic_rd_en & IB_ACCESS_REMOTE_READ ?
800 MTHCA_QP_BIT_RRE : 0);
801 qp_context->params2 |=
802 cpu_to_be32(qp->atomic_rd_en & IB_ACCESS_REMOTE_ATOMIC ?
803 MTHCA_QP_BIT_RAE : 0);
804
805 qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RWE |
806 MTHCA_QP_OPTPAR_RRE |
807 MTHCA_QP_OPTPAR_RAE);
808 }
809
810 for (rra_max = 0;
811 1 << rra_max < attr->max_rd_atomic &&
812 rra_max < dev->qp_table.rdb_shift;
813 ++rra_max)
814 ; /* nothing */
815
816 qp_context->params2 |= cpu_to_be32(rra_max << 21);
817 qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RRA_MAX);
818
819 qp->resp_depth = attr->max_rd_atomic;
820 }
821
822 qp_context->params2 |= cpu_to_be32(MTHCA_QP_BIT_RSC);
823
824 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
825 qp_context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
826 qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RNR_TIMEOUT);
827 }
828 if (attr_mask & IB_QP_RQ_PSN)
829 qp_context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
830
831 qp_context->ra_buff_indx =
832 cpu_to_be32(dev->qp_table.rdb_base +
833 ((qp->qpn & (dev->limits.num_qps - 1)) * MTHCA_RDB_ENTRY_SIZE <<
834 dev->qp_table.rdb_shift));
835
836 qp_context->cqn_rcv = cpu_to_be32(to_mcq(ibqp->recv_cq)->cqn);
837
838 if (dev->hca_type == ARBEL_NATIVE)
839 qp_context->rcv_db_index = cpu_to_be32(qp->rq.db_index);
840
841 if (attr_mask & IB_QP_QKEY) {
842 qp_context->qkey = cpu_to_be32(attr->qkey);
843 qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_Q_KEY);
844 }
845
846 err = mthca_MODIFY_QP(dev, state_table[cur_state][new_state].trans,
847 qp->qpn, 0, qp_param, 0, &status);
848 if (status) {
849 mthca_warn(dev, "modify QP %d returned status %02x.\n",
850 state_table[cur_state][new_state].trans, status);
851 err = -EINVAL;
852 }
853
854 if (!err)
855 qp->state = new_state;
856
857 kfree(mailbox);
858
859 if (is_sqp(dev, qp))
860 store_attrs(to_msqp(qp), attr, attr_mask);
861
862 /*
863 * If we are moving QP0 to RTR, bring the IB link up; if we
864 * are moving QP0 to RESET or ERROR, bring the link back down.
865 */
866 if (is_qp0(dev, qp)) {
867 if (cur_state != IB_QPS_RTR &&
868 new_state == IB_QPS_RTR)
869 init_port(dev, to_msqp(qp)->port);
870
871 if (cur_state != IB_QPS_RESET &&
872 cur_state != IB_QPS_ERR &&
873 (new_state == IB_QPS_RESET ||
874 new_state == IB_QPS_ERR))
875 mthca_CLOSE_IB(dev, to_msqp(qp)->port, &status);
876 }
877
878 return err;
879}
880
881/*
882 * Allocate and register buffer for WQEs. qp->rq.max, sq.max,
883 * rq.max_gs and sq.max_gs must all be assigned.
884 * mthca_alloc_wqe_buf will calculate rq.wqe_shift and
885 * sq.wqe_shift (as well as send_wqe_offset, is_direct, and
886 * queue)
887 */
888static int mthca_alloc_wqe_buf(struct mthca_dev *dev,
889 struct mthca_pd *pd,
890 struct mthca_qp *qp)
891{
892 int size;
893 int i;
894 int npages, shift;
895 dma_addr_t t;
896 u64 *dma_list = NULL;
897 int err = -ENOMEM;
898
899 size = sizeof (struct mthca_next_seg) +
900 qp->rq.max_gs * sizeof (struct mthca_data_seg);
901
902 for (qp->rq.wqe_shift = 6; 1 << qp->rq.wqe_shift < size;
903 qp->rq.wqe_shift++)
904 ; /* nothing */
905
906 size = sizeof (struct mthca_next_seg) +
907 qp->sq.max_gs * sizeof (struct mthca_data_seg);
908 switch (qp->transport) {
909 case MLX:
910 size += 2 * sizeof (struct mthca_data_seg);
911 break;
912 case UD:
913 if (dev->hca_type == ARBEL_NATIVE)
914 size += sizeof (struct mthca_arbel_ud_seg);
915 else
916 size += sizeof (struct mthca_tavor_ud_seg);
917 break;
918 default:
919 /* bind seg is as big as atomic + raddr segs */
920 size += sizeof (struct mthca_bind_seg);
921 }
922
923 for (qp->sq.wqe_shift = 6; 1 << qp->sq.wqe_shift < size;
924 qp->sq.wqe_shift++)
925 ; /* nothing */
926
927 qp->send_wqe_offset = ALIGN(qp->rq.max << qp->rq.wqe_shift,
928 1 << qp->sq.wqe_shift);
929 size = PAGE_ALIGN(qp->send_wqe_offset +
930 (qp->sq.max << qp->sq.wqe_shift));
931
932 qp->wrid = kmalloc((qp->rq.max + qp->sq.max) * sizeof (u64),
933 GFP_KERNEL);
934 if (!qp->wrid)
935 goto err_out;
936
937 if (size <= MTHCA_MAX_DIRECT_QP_SIZE) {
938 qp->is_direct = 1;
939 npages = 1;
940 shift = get_order(size) + PAGE_SHIFT;
941
942 if (0)
943 mthca_dbg(dev, "Creating direct QP of size %d (shift %d)\n",
944 size, shift);
945
946 qp->queue.direct.buf = pci_alloc_consistent(dev->pdev, size, &t);
947 if (!qp->queue.direct.buf)
948 goto err_out;
949
950 pci_unmap_addr_set(&qp->queue.direct, mapping, t);
951
952 memset(qp->queue.direct.buf, 0, size);
953
954 while (t & ((1 << shift) - 1)) {
955 --shift;
956 npages *= 2;
957 }
958
959 dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
960 if (!dma_list)
961 goto err_out_free;
962
963 for (i = 0; i < npages; ++i)
964 dma_list[i] = t + i * (1 << shift);
965 } else {
966 qp->is_direct = 0;
967 npages = size / PAGE_SIZE;
968 shift = PAGE_SHIFT;
969
970 if (0)
971 mthca_dbg(dev, "Creating indirect QP with %d pages\n", npages);
972
973 dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
974 if (!dma_list)
975 goto err_out;
976
977 qp->queue.page_list = kmalloc(npages *
978 sizeof *qp->queue.page_list,
979 GFP_KERNEL);
980 if (!qp->queue.page_list)
981 goto err_out;
982
983 for (i = 0; i < npages; ++i) {
984 qp->queue.page_list[i].buf =
985 pci_alloc_consistent(dev->pdev, PAGE_SIZE, &t);
986 if (!qp->queue.page_list[i].buf)
987 goto err_out_free;
988
989 memset(qp->queue.page_list[i].buf, 0, PAGE_SIZE);
990
991 pci_unmap_addr_set(&qp->queue.page_list[i], mapping, t);
992 dma_list[i] = t;
993 }
994 }
995
996 err = mthca_mr_alloc_phys(dev, pd->pd_num, dma_list, shift,
997 npages, 0, size,
998 MTHCA_MPT_FLAG_LOCAL_READ,
999 &qp->mr);
1000 if (err)
1001 goto err_out_free;
1002
1003 kfree(dma_list);
1004 return 0;
1005
1006 err_out_free:
1007 if (qp->is_direct) {
1008 pci_free_consistent(dev->pdev, size,
1009 qp->queue.direct.buf,
1010 pci_unmap_addr(&qp->queue.direct, mapping));
1011 } else
1012 for (i = 0; i < npages; ++i) {
1013 if (qp->queue.page_list[i].buf)
1014 pci_free_consistent(dev->pdev, PAGE_SIZE,
1015 qp->queue.page_list[i].buf,
1016 pci_unmap_addr(&qp->queue.page_list[i],
1017 mapping));
1018
1019 }
1020
1021 err_out:
1022 kfree(qp->wrid);
1023 kfree(dma_list);
1024 return err;
1025}
1026
1027static int mthca_alloc_memfree(struct mthca_dev *dev,
1028 struct mthca_qp *qp)
1029{
1030 int ret = 0;
1031
1032 if (dev->hca_type == ARBEL_NATIVE) {
1033 ret = mthca_table_get(dev, dev->qp_table.qp_table, qp->qpn);
1034 if (ret)
1035 return ret;
1036
1037 ret = mthca_table_get(dev, dev->qp_table.eqp_table, qp->qpn);
1038 if (ret)
1039 goto err_qpc;
1040
1041 qp->rq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_RQ,
1042 qp->qpn, &qp->rq.db);
1043 if (qp->rq.db_index < 0) {
1044 ret = -ENOMEM;
1045 goto err_eqpc;
1046 }
1047
1048 qp->sq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_SQ,
1049 qp->qpn, &qp->sq.db);
1050 if (qp->sq.db_index < 0) {
1051 ret = -ENOMEM;
1052 goto err_rq_db;
1053 }
1054 }
1055
1056 return 0;
1057
1058err_rq_db:
1059 mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index);
1060
1061err_eqpc:
1062 mthca_table_put(dev, dev->qp_table.eqp_table, qp->qpn);
1063
1064err_qpc:
1065 mthca_table_put(dev, dev->qp_table.qp_table, qp->qpn);
1066
1067 return ret;
1068}
1069
1070static void mthca_free_memfree(struct mthca_dev *dev,
1071 struct mthca_qp *qp)
1072{
1073 if (dev->hca_type == ARBEL_NATIVE) {
1074 mthca_free_db(dev, MTHCA_DB_TYPE_SQ, qp->sq.db_index);
1075 mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index);
1076 mthca_table_put(dev, dev->qp_table.eqp_table, qp->qpn);
1077 mthca_table_put(dev, dev->qp_table.qp_table, qp->qpn);
1078 }
1079}
1080
1081static void mthca_wq_init(struct mthca_wq* wq)
1082{
1083 spin_lock_init(&wq->lock);
1084 wq->next_ind = 0;
1085 wq->last_comp = wq->max - 1;
1086 wq->head = 0;
1087 wq->tail = 0;
1088 wq->last = NULL;
1089}
1090
1091static int mthca_alloc_qp_common(struct mthca_dev *dev,
1092 struct mthca_pd *pd,
1093 struct mthca_cq *send_cq,
1094 struct mthca_cq *recv_cq,
1095 enum ib_sig_type send_policy,
1096 struct mthca_qp *qp)
1097{
1098 struct mthca_next_seg *wqe;
1099 int ret;
1100 int i;
1101
1102 atomic_set(&qp->refcount, 1);
1103 qp->state = IB_QPS_RESET;
1104 qp->atomic_rd_en = 0;
1105 qp->resp_depth = 0;
1106 qp->sq_policy = send_policy;
1107 mthca_wq_init(&qp->sq);
1108 mthca_wq_init(&qp->rq);
1109
1110 ret = mthca_alloc_memfree(dev, qp);
1111 if (ret)
1112 return ret;
1113
1114 ret = mthca_alloc_wqe_buf(dev, pd, qp);
1115 if (ret) {
1116 mthca_free_memfree(dev, qp);
1117 return ret;
1118 }
1119
1120 if (dev->hca_type == ARBEL_NATIVE) {
1121 for (i = 0; i < qp->rq.max; ++i) {
1122 wqe = get_recv_wqe(qp, i);
1123 wqe->nda_op = cpu_to_be32(((i + 1) & (qp->rq.max - 1)) <<
1124 qp->rq.wqe_shift);
1125 wqe->ee_nds = cpu_to_be32(1 << (qp->rq.wqe_shift - 4));
1126 }
1127
1128 for (i = 0; i < qp->sq.max; ++i) {
1129 wqe = get_send_wqe(qp, i);
1130 wqe->nda_op = cpu_to_be32((((i + 1) & (qp->sq.max - 1)) <<
1131 qp->sq.wqe_shift) +
1132 qp->send_wqe_offset);
1133 }
1134 }
1135
1136 return 0;
1137}
1138
1139static void mthca_align_qp_size(struct mthca_dev *dev, struct mthca_qp *qp)
1140{
1141 int i;
1142
1143 if (dev->hca_type != ARBEL_NATIVE)
1144 return;
1145
1146 for (i = 0; 1 << i < qp->rq.max; ++i)
1147 ; /* nothing */
1148
1149 qp->rq.max = 1 << i;
1150
1151 for (i = 0; 1 << i < qp->sq.max; ++i)
1152 ; /* nothing */
1153
1154 qp->sq.max = 1 << i;
1155}
1156
1157int mthca_alloc_qp(struct mthca_dev *dev,
1158 struct mthca_pd *pd,
1159 struct mthca_cq *send_cq,
1160 struct mthca_cq *recv_cq,
1161 enum ib_qp_type type,
1162 enum ib_sig_type send_policy,
1163 struct mthca_qp *qp)
1164{
1165 int err;
1166
1167 mthca_align_qp_size(dev, qp);
1168
1169 switch (type) {
1170 case IB_QPT_RC: qp->transport = RC; break;
1171 case IB_QPT_UC: qp->transport = UC; break;
1172 case IB_QPT_UD: qp->transport = UD; break;
1173 default: return -EINVAL;
1174 }
1175
1176 qp->qpn = mthca_alloc(&dev->qp_table.alloc);
1177 if (qp->qpn == -1)
1178 return -ENOMEM;
1179
1180 err = mthca_alloc_qp_common(dev, pd, send_cq, recv_cq,
1181 send_policy, qp);
1182 if (err) {
1183 mthca_free(&dev->qp_table.alloc, qp->qpn);
1184 return err;
1185 }
1186
1187 spin_lock_irq(&dev->qp_table.lock);
1188 mthca_array_set(&dev->qp_table.qp,
1189 qp->qpn & (dev->limits.num_qps - 1), qp);
1190 spin_unlock_irq(&dev->qp_table.lock);
1191
1192 return 0;
1193}
1194
1195int mthca_alloc_sqp(struct mthca_dev *dev,
1196 struct mthca_pd *pd,
1197 struct mthca_cq *send_cq,
1198 struct mthca_cq *recv_cq,
1199 enum ib_sig_type send_policy,
1200 int qpn,
1201 int port,
1202 struct mthca_sqp *sqp)
1203{
1204 int err = 0;
1205 u32 mqpn = qpn * 2 + dev->qp_table.sqp_start + port - 1;
1206
1207 mthca_align_qp_size(dev, &sqp->qp);
1208
1209 sqp->header_buf_size = sqp->qp.sq.max * MTHCA_UD_HEADER_SIZE;
1210 sqp->header_buf = dma_alloc_coherent(&dev->pdev->dev, sqp->header_buf_size,
1211 &sqp->header_dma, GFP_KERNEL);
1212 if (!sqp->header_buf)
1213 return -ENOMEM;
1214
1215 spin_lock_irq(&dev->qp_table.lock);
1216 if (mthca_array_get(&dev->qp_table.qp, mqpn))
1217 err = -EBUSY;
1218 else
1219 mthca_array_set(&dev->qp_table.qp, mqpn, sqp);
1220 spin_unlock_irq(&dev->qp_table.lock);
1221
1222 if (err)
1223 goto err_out;
1224
1225 sqp->port = port;
1226 sqp->qp.qpn = mqpn;
1227 sqp->qp.transport = MLX;
1228
1229 err = mthca_alloc_qp_common(dev, pd, send_cq, recv_cq,
1230 send_policy, &sqp->qp);
1231 if (err)
1232 goto err_out_free;
1233
1234 atomic_inc(&pd->sqp_count);
1235
1236 return 0;
1237
1238 err_out_free:
1239 /*
1240 * Lock CQs here, so that CQ polling code can do QP lookup
1241 * without taking a lock.
1242 */
1243 spin_lock_irq(&send_cq->lock);
1244 if (send_cq != recv_cq)
1245 spin_lock(&recv_cq->lock);
1246
1247 spin_lock(&dev->qp_table.lock);
1248 mthca_array_clear(&dev->qp_table.qp, mqpn);
1249 spin_unlock(&dev->qp_table.lock);
1250
1251 if (send_cq != recv_cq)
1252 spin_unlock(&recv_cq->lock);
1253 spin_unlock_irq(&send_cq->lock);
1254
1255 err_out:
1256 dma_free_coherent(&dev->pdev->dev, sqp->header_buf_size,
1257 sqp->header_buf, sqp->header_dma);
1258
1259 return err;
1260}
1261
1262void mthca_free_qp(struct mthca_dev *dev,
1263 struct mthca_qp *qp)
1264{
1265 u8 status;
1266 int size;
1267 int i;
1268 struct mthca_cq *send_cq;
1269 struct mthca_cq *recv_cq;
1270
1271 send_cq = to_mcq(qp->ibqp.send_cq);
1272 recv_cq = to_mcq(qp->ibqp.recv_cq);
1273
1274 /*
1275 * Lock CQs here, so that CQ polling code can do QP lookup
1276 * without taking a lock.
1277 */
1278 spin_lock_irq(&send_cq->lock);
1279 if (send_cq != recv_cq)
1280 spin_lock(&recv_cq->lock);
1281
1282 spin_lock(&dev->qp_table.lock);
1283 mthca_array_clear(&dev->qp_table.qp,
1284 qp->qpn & (dev->limits.num_qps - 1));
1285 spin_unlock(&dev->qp_table.lock);
1286
1287 if (send_cq != recv_cq)
1288 spin_unlock(&recv_cq->lock);
1289 spin_unlock_irq(&send_cq->lock);
1290
1291 atomic_dec(&qp->refcount);
1292 wait_event(qp->wait, !atomic_read(&qp->refcount));
1293
1294 if (qp->state != IB_QPS_RESET)
1295 mthca_MODIFY_QP(dev, MTHCA_TRANS_ANY2RST, qp->qpn, 0, NULL, 0, &status);
1296
1297 mthca_cq_clean(dev, to_mcq(qp->ibqp.send_cq)->cqn, qp->qpn);
1298 if (qp->ibqp.send_cq != qp->ibqp.recv_cq)
1299 mthca_cq_clean(dev, to_mcq(qp->ibqp.recv_cq)->cqn, qp->qpn);
1300
1301 mthca_free_mr(dev, &qp->mr);
1302
1303 size = PAGE_ALIGN(qp->send_wqe_offset +
1304 (qp->sq.max << qp->sq.wqe_shift));
1305
1306 if (qp->is_direct) {
1307 pci_free_consistent(dev->pdev, size,
1308 qp->queue.direct.buf,
1309 pci_unmap_addr(&qp->queue.direct, mapping));
1310 } else {
1311 for (i = 0; i < size / PAGE_SIZE; ++i) {
1312 pci_free_consistent(dev->pdev, PAGE_SIZE,
1313 qp->queue.page_list[i].buf,
1314 pci_unmap_addr(&qp->queue.page_list[i],
1315 mapping));
1316 }
1317 }
1318
1319 kfree(qp->wrid);
1320
1321 mthca_free_memfree(dev, qp);
1322
1323 if (is_sqp(dev, qp)) {
1324 atomic_dec(&(to_mpd(qp->ibqp.pd)->sqp_count));
1325 dma_free_coherent(&dev->pdev->dev,
1326 to_msqp(qp)->header_buf_size,
1327 to_msqp(qp)->header_buf,
1328 to_msqp(qp)->header_dma);
1329 } else
1330 mthca_free(&dev->qp_table.alloc, qp->qpn);
1331}
1332
1333/* Create UD header for an MLX send and build a data segment for it */
1334static int build_mlx_header(struct mthca_dev *dev, struct mthca_sqp *sqp,
1335 int ind, struct ib_send_wr *wr,
1336 struct mthca_mlx_seg *mlx,
1337 struct mthca_data_seg *data)
1338{
1339 int header_size;
1340 int err;
1341
1342 ib_ud_header_init(256, /* assume a MAD */
1343 sqp->ud_header.grh_present,
1344 &sqp->ud_header);
1345
1346 err = mthca_read_ah(dev, to_mah(wr->wr.ud.ah), &sqp->ud_header);
1347 if (err)
1348 return err;
1349 mlx->flags &= ~cpu_to_be32(MTHCA_NEXT_SOLICIT | 1);
1350 mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MTHCA_MLX_VL15 : 0) |
1351 (sqp->ud_header.lrh.destination_lid == 0xffff ?
1352 MTHCA_MLX_SLR : 0) |
1353 (sqp->ud_header.lrh.service_level << 8));
1354 mlx->rlid = sqp->ud_header.lrh.destination_lid;
1355 mlx->vcrc = 0;
1356
1357 switch (wr->opcode) {
1358 case IB_WR_SEND:
1359 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
1360 sqp->ud_header.immediate_present = 0;
1361 break;
1362 case IB_WR_SEND_WITH_IMM:
1363 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
1364 sqp->ud_header.immediate_present = 1;
1365 sqp->ud_header.immediate_data = wr->imm_data;
1366 break;
1367 default:
1368 return -EINVAL;
1369 }
1370
1371 sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0;
1372 if (sqp->ud_header.lrh.destination_lid == 0xffff)
1373 sqp->ud_header.lrh.source_lid = 0xffff;
1374 sqp->ud_header.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED);
1375 if (!sqp->qp.ibqp.qp_num)
1376 ib_get_cached_pkey(&dev->ib_dev, sqp->port,
1377 sqp->pkey_index,
1378 &sqp->ud_header.bth.pkey);
1379 else
1380 ib_get_cached_pkey(&dev->ib_dev, sqp->port,
1381 wr->wr.ud.pkey_index,
1382 &sqp->ud_header.bth.pkey);
1383 cpu_to_be16s(&sqp->ud_header.bth.pkey);
1384 sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn);
1385 sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
1386 sqp->ud_header.deth.qkey = cpu_to_be32(wr->wr.ud.remote_qkey & 0x80000000 ?
1387 sqp->qkey : wr->wr.ud.remote_qkey);
1388 sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num);
1389
1390 header_size = ib_ud_header_pack(&sqp->ud_header,
1391 sqp->header_buf +
1392 ind * MTHCA_UD_HEADER_SIZE);
1393
1394 data->byte_count = cpu_to_be32(header_size);
1395 data->lkey = cpu_to_be32(to_mpd(sqp->qp.ibqp.pd)->ntmr.ibmr.lkey);
1396 data->addr = cpu_to_be64(sqp->header_dma +
1397 ind * MTHCA_UD_HEADER_SIZE);
1398
1399 return 0;
1400}
1401
1402static inline int mthca_wq_overflow(struct mthca_wq *wq, int nreq,
1403 struct ib_cq *ib_cq)
1404{
1405 unsigned cur;
1406 struct mthca_cq *cq;
1407
1408 cur = wq->head - wq->tail;
1409 if (likely(cur + nreq < wq->max))
1410 return 0;
1411
1412 cq = to_mcq(ib_cq);
1413 spin_lock(&cq->lock);
1414 cur = wq->head - wq->tail;
1415 spin_unlock(&cq->lock);
1416
1417 return cur + nreq >= wq->max;
1418}
1419
1420int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
1421 struct ib_send_wr **bad_wr)
1422{
1423 struct mthca_dev *dev = to_mdev(ibqp->device);
1424 struct mthca_qp *qp = to_mqp(ibqp);
1425 void *wqe;
1426 void *prev_wqe;
1427 unsigned long flags;
1428 int err = 0;
1429 int nreq;
1430 int i;
1431 int size;
1432 int size0 = 0;
1433 u32 f0 = 0;
1434 int ind;
1435 u8 op0 = 0;
1436
1437 spin_lock_irqsave(&qp->sq.lock, flags);
1438
1439 /* XXX check that state is OK to post send */
1440
1441 ind = qp->sq.next_ind;
1442
1443 for (nreq = 0; wr; ++nreq, wr = wr->next) {
1444 if (mthca_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
1445 mthca_err(dev, "SQ %06x full (%u head, %u tail,"
1446 " %d max, %d nreq)\n", qp->qpn,
1447 qp->sq.head, qp->sq.tail,
1448 qp->sq.max, nreq);
1449 err = -ENOMEM;
1450 *bad_wr = wr;
1451 goto out;
1452 }
1453
1454 wqe = get_send_wqe(qp, ind);
1455 prev_wqe = qp->sq.last;
1456 qp->sq.last = wqe;
1457
1458 ((struct mthca_next_seg *) wqe)->nda_op = 0;
1459 ((struct mthca_next_seg *) wqe)->ee_nds = 0;
1460 ((struct mthca_next_seg *) wqe)->flags =
1461 ((wr->send_flags & IB_SEND_SIGNALED) ?
1462 cpu_to_be32(MTHCA_NEXT_CQ_UPDATE) : 0) |
1463 ((wr->send_flags & IB_SEND_SOLICITED) ?
1464 cpu_to_be32(MTHCA_NEXT_SOLICIT) : 0) |
1465 cpu_to_be32(1);
1466 if (wr->opcode == IB_WR_SEND_WITH_IMM ||
1467 wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM)
1468 ((struct mthca_next_seg *) wqe)->flags = wr->imm_data;
1469
1470 wqe += sizeof (struct mthca_next_seg);
1471 size = sizeof (struct mthca_next_seg) / 16;
1472
1473 switch (qp->transport) {
1474 case RC:
1475 switch (wr->opcode) {
1476 case IB_WR_ATOMIC_CMP_AND_SWP:
1477 case IB_WR_ATOMIC_FETCH_AND_ADD:
1478 ((struct mthca_raddr_seg *) wqe)->raddr =
1479 cpu_to_be64(wr->wr.atomic.remote_addr);
1480 ((struct mthca_raddr_seg *) wqe)->rkey =
1481 cpu_to_be32(wr->wr.atomic.rkey);
1482 ((struct mthca_raddr_seg *) wqe)->reserved = 0;
1483
1484 wqe += sizeof (struct mthca_raddr_seg);
1485
1486 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
1487 ((struct mthca_atomic_seg *) wqe)->swap_add =
1488 cpu_to_be64(wr->wr.atomic.swap);
1489 ((struct mthca_atomic_seg *) wqe)->compare =
1490 cpu_to_be64(wr->wr.atomic.compare_add);
1491 } else {
1492 ((struct mthca_atomic_seg *) wqe)->swap_add =
1493 cpu_to_be64(wr->wr.atomic.compare_add);
1494 ((struct mthca_atomic_seg *) wqe)->compare = 0;
1495 }
1496
1497 wqe += sizeof (struct mthca_atomic_seg);
1498 size += sizeof (struct mthca_raddr_seg) / 16 +
1499 sizeof (struct mthca_atomic_seg);
1500 break;
1501
1502 case IB_WR_RDMA_WRITE:
1503 case IB_WR_RDMA_WRITE_WITH_IMM:
1504 case IB_WR_RDMA_READ:
1505 ((struct mthca_raddr_seg *) wqe)->raddr =
1506 cpu_to_be64(wr->wr.rdma.remote_addr);
1507 ((struct mthca_raddr_seg *) wqe)->rkey =
1508 cpu_to_be32(wr->wr.rdma.rkey);
1509 ((struct mthca_raddr_seg *) wqe)->reserved = 0;
1510 wqe += sizeof (struct mthca_raddr_seg);
1511 size += sizeof (struct mthca_raddr_seg) / 16;
1512 break;
1513
1514 default:
1515 /* No extra segments required for sends */
1516 break;
1517 }
1518
1519 break;
1520
1521 case UD:
1522 ((struct mthca_tavor_ud_seg *) wqe)->lkey =
1523 cpu_to_be32(to_mah(wr->wr.ud.ah)->key);
1524 ((struct mthca_tavor_ud_seg *) wqe)->av_addr =
1525 cpu_to_be64(to_mah(wr->wr.ud.ah)->avdma);
1526 ((struct mthca_tavor_ud_seg *) wqe)->dqpn =
1527 cpu_to_be32(wr->wr.ud.remote_qpn);
1528 ((struct mthca_tavor_ud_seg *) wqe)->qkey =
1529 cpu_to_be32(wr->wr.ud.remote_qkey);
1530
1531 wqe += sizeof (struct mthca_tavor_ud_seg);
1532 size += sizeof (struct mthca_tavor_ud_seg) / 16;
1533 break;
1534
1535 case MLX:
1536 err = build_mlx_header(dev, to_msqp(qp), ind, wr,
1537 wqe - sizeof (struct mthca_next_seg),
1538 wqe);
1539 if (err) {
1540 *bad_wr = wr;
1541 goto out;
1542 }
1543 wqe += sizeof (struct mthca_data_seg);
1544 size += sizeof (struct mthca_data_seg) / 16;
1545 break;
1546 }
1547
1548 if (wr->num_sge > qp->sq.max_gs) {
1549 mthca_err(dev, "too many gathers\n");
1550 err = -EINVAL;
1551 *bad_wr = wr;
1552 goto out;
1553 }
1554
1555 for (i = 0; i < wr->num_sge; ++i) {
1556 ((struct mthca_data_seg *) wqe)->byte_count =
1557 cpu_to_be32(wr->sg_list[i].length);
1558 ((struct mthca_data_seg *) wqe)->lkey =
1559 cpu_to_be32(wr->sg_list[i].lkey);
1560 ((struct mthca_data_seg *) wqe)->addr =
1561 cpu_to_be64(wr->sg_list[i].addr);
1562 wqe += sizeof (struct mthca_data_seg);
1563 size += sizeof (struct mthca_data_seg) / 16;
1564 }
1565
1566 /* Add one more inline data segment for ICRC */
1567 if (qp->transport == MLX) {
1568 ((struct mthca_data_seg *) wqe)->byte_count =
1569 cpu_to_be32((1 << 31) | 4);
1570 ((u32 *) wqe)[1] = 0;
1571 wqe += sizeof (struct mthca_data_seg);
1572 size += sizeof (struct mthca_data_seg) / 16;
1573 }
1574
1575 qp->wrid[ind + qp->rq.max] = wr->wr_id;
1576
1577 if (wr->opcode >= ARRAY_SIZE(mthca_opcode)) {
1578 mthca_err(dev, "opcode invalid\n");
1579 err = -EINVAL;
1580 *bad_wr = wr;
1581 goto out;
1582 }
1583
1584 if (prev_wqe) {
1585 ((struct mthca_next_seg *) prev_wqe)->nda_op =
1586 cpu_to_be32(((ind << qp->sq.wqe_shift) +
1587 qp->send_wqe_offset) |
1588 mthca_opcode[wr->opcode]);
1589 wmb();
1590 ((struct mthca_next_seg *) prev_wqe)->ee_nds =
1591 cpu_to_be32((size0 ? 0 : MTHCA_NEXT_DBD) | size);
1592 }
1593
1594 if (!size0) {
1595 size0 = size;
1596 op0 = mthca_opcode[wr->opcode];
1597 }
1598
1599 ++ind;
1600 if (unlikely(ind >= qp->sq.max))
1601 ind -= qp->sq.max;
1602 }
1603
1604out:
1605 if (likely(nreq)) {
1606 u32 doorbell[2];
1607
1608 doorbell[0] = cpu_to_be32(((qp->sq.next_ind << qp->sq.wqe_shift) +
1609 qp->send_wqe_offset) | f0 | op0);
1610 doorbell[1] = cpu_to_be32((qp->qpn << 8) | size0);
1611
1612 wmb();
1613
1614 mthca_write64(doorbell,
1615 dev->kar + MTHCA_SEND_DOORBELL,
1616 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
1617 }
1618
1619 qp->sq.next_ind = ind;
1620 qp->sq.head += nreq;
1621
1622 spin_unlock_irqrestore(&qp->sq.lock, flags);
1623 return err;
1624}
1625
1626int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
1627 struct ib_recv_wr **bad_wr)
1628{
1629 struct mthca_dev *dev = to_mdev(ibqp->device);
1630 struct mthca_qp *qp = to_mqp(ibqp);
1631 unsigned long flags;
1632 int err = 0;
1633 int nreq;
1634 int i;
1635 int size;
1636 int size0 = 0;
1637 int ind;
1638 void *wqe;
1639 void *prev_wqe;
1640
1641 spin_lock_irqsave(&qp->rq.lock, flags);
1642
1643 /* XXX check that state is OK to post receive */
1644
1645 ind = qp->rq.next_ind;
1646
1647 for (nreq = 0; wr; ++nreq, wr = wr->next) {
1648 if (mthca_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
1649 mthca_err(dev, "RQ %06x full (%u head, %u tail,"
1650 " %d max, %d nreq)\n", qp->qpn,
1651 qp->rq.head, qp->rq.tail,
1652 qp->rq.max, nreq);
1653 err = -ENOMEM;
1654 *bad_wr = wr;
1655 goto out;
1656 }
1657
1658 wqe = get_recv_wqe(qp, ind);
1659 prev_wqe = qp->rq.last;
1660 qp->rq.last = wqe;
1661
1662 ((struct mthca_next_seg *) wqe)->nda_op = 0;
1663 ((struct mthca_next_seg *) wqe)->ee_nds =
1664 cpu_to_be32(MTHCA_NEXT_DBD);
1665 ((struct mthca_next_seg *) wqe)->flags = 0;
1666
1667 wqe += sizeof (struct mthca_next_seg);
1668 size = sizeof (struct mthca_next_seg) / 16;
1669
1670 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
1671 err = -EINVAL;
1672 *bad_wr = wr;
1673 goto out;
1674 }
1675
1676 for (i = 0; i < wr->num_sge; ++i) {
1677 ((struct mthca_data_seg *) wqe)->byte_count =
1678 cpu_to_be32(wr->sg_list[i].length);
1679 ((struct mthca_data_seg *) wqe)->lkey =
1680 cpu_to_be32(wr->sg_list[i].lkey);
1681 ((struct mthca_data_seg *) wqe)->addr =
1682 cpu_to_be64(wr->sg_list[i].addr);
1683 wqe += sizeof (struct mthca_data_seg);
1684 size += sizeof (struct mthca_data_seg) / 16;
1685 }
1686
1687 qp->wrid[ind] = wr->wr_id;
1688
1689 if (likely(prev_wqe)) {
1690 ((struct mthca_next_seg *) prev_wqe)->nda_op =
1691 cpu_to_be32((ind << qp->rq.wqe_shift) | 1);
1692 wmb();
1693 ((struct mthca_next_seg *) prev_wqe)->ee_nds =
1694 cpu_to_be32(MTHCA_NEXT_DBD | size);
1695 }
1696
1697 if (!size0)
1698 size0 = size;
1699
1700 ++ind;
1701 if (unlikely(ind >= qp->rq.max))
1702 ind -= qp->rq.max;
1703 }
1704
1705out:
1706 if (likely(nreq)) {
1707 u32 doorbell[2];
1708
1709 doorbell[0] = cpu_to_be32((qp->rq.next_ind << qp->rq.wqe_shift) | size0);
1710 doorbell[1] = cpu_to_be32((qp->qpn << 8) | nreq);
1711
1712 wmb();
1713
1714 mthca_write64(doorbell,
1715 dev->kar + MTHCA_RECEIVE_DOORBELL,
1716 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
1717 }
1718
1719 qp->rq.next_ind = ind;
1720 qp->rq.head += nreq;
1721
1722 spin_unlock_irqrestore(&qp->rq.lock, flags);
1723 return err;
1724}
1725
1726int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
1727 struct ib_send_wr **bad_wr)
1728{
1729 struct mthca_dev *dev = to_mdev(ibqp->device);
1730 struct mthca_qp *qp = to_mqp(ibqp);
1731 void *wqe;
1732 void *prev_wqe;
1733 unsigned long flags;
1734 int err = 0;
1735 int nreq;
1736 int i;
1737 int size;
1738 int size0 = 0;
1739 u32 f0 = 0;
1740 int ind;
1741 u8 op0 = 0;
1742
1743 spin_lock_irqsave(&qp->sq.lock, flags);
1744
1745 /* XXX check that state is OK to post send */
1746
1747 ind = qp->sq.head & (qp->sq.max - 1);
1748
1749 for (nreq = 0; wr; ++nreq, wr = wr->next) {
1750 if (mthca_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
1751 mthca_err(dev, "SQ %06x full (%u head, %u tail,"
1752 " %d max, %d nreq)\n", qp->qpn,
1753 qp->sq.head, qp->sq.tail,
1754 qp->sq.max, nreq);
1755 err = -ENOMEM;
1756 *bad_wr = wr;
1757 goto out;
1758 }
1759
1760 wqe = get_send_wqe(qp, ind);
1761 prev_wqe = qp->sq.last;
1762 qp->sq.last = wqe;
1763
1764 ((struct mthca_next_seg *) wqe)->flags =
1765 ((wr->send_flags & IB_SEND_SIGNALED) ?
1766 cpu_to_be32(MTHCA_NEXT_CQ_UPDATE) : 0) |
1767 ((wr->send_flags & IB_SEND_SOLICITED) ?
1768 cpu_to_be32(MTHCA_NEXT_SOLICIT) : 0) |
1769 cpu_to_be32(1);
1770 if (wr->opcode == IB_WR_SEND_WITH_IMM ||
1771 wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM)
1772 ((struct mthca_next_seg *) wqe)->flags = wr->imm_data;
1773
1774 wqe += sizeof (struct mthca_next_seg);
1775 size = sizeof (struct mthca_next_seg) / 16;
1776
1777 switch (qp->transport) {
1778 case UD:
1779 memcpy(((struct mthca_arbel_ud_seg *) wqe)->av,
1780 to_mah(wr->wr.ud.ah)->av, MTHCA_AV_SIZE);
1781 ((struct mthca_arbel_ud_seg *) wqe)->dqpn =
1782 cpu_to_be32(wr->wr.ud.remote_qpn);
1783 ((struct mthca_arbel_ud_seg *) wqe)->qkey =
1784 cpu_to_be32(wr->wr.ud.remote_qkey);
1785
1786 wqe += sizeof (struct mthca_arbel_ud_seg);
1787 size += sizeof (struct mthca_arbel_ud_seg) / 16;
1788 break;
1789
1790 case MLX:
1791 err = build_mlx_header(dev, to_msqp(qp), ind, wr,
1792 wqe - sizeof (struct mthca_next_seg),
1793 wqe);
1794 if (err) {
1795 *bad_wr = wr;
1796 goto out;
1797 }
1798 wqe += sizeof (struct mthca_data_seg);
1799 size += sizeof (struct mthca_data_seg) / 16;
1800 break;
1801 }
1802
1803 if (wr->num_sge > qp->sq.max_gs) {
1804 mthca_err(dev, "too many gathers\n");
1805 err = -EINVAL;
1806 *bad_wr = wr;
1807 goto out;
1808 }
1809
1810 for (i = 0; i < wr->num_sge; ++i) {
1811 ((struct mthca_data_seg *) wqe)->byte_count =
1812 cpu_to_be32(wr->sg_list[i].length);
1813 ((struct mthca_data_seg *) wqe)->lkey =
1814 cpu_to_be32(wr->sg_list[i].lkey);
1815 ((struct mthca_data_seg *) wqe)->addr =
1816 cpu_to_be64(wr->sg_list[i].addr);
1817 wqe += sizeof (struct mthca_data_seg);
1818 size += sizeof (struct mthca_data_seg) / 16;
1819 }
1820
1821 /* Add one more inline data segment for ICRC */
1822 if (qp->transport == MLX) {
1823 ((struct mthca_data_seg *) wqe)->byte_count =
1824 cpu_to_be32((1 << 31) | 4);
1825 ((u32 *) wqe)[1] = 0;
1826 wqe += sizeof (struct mthca_data_seg);
1827 size += sizeof (struct mthca_data_seg) / 16;
1828 }
1829
1830 qp->wrid[ind + qp->rq.max] = wr->wr_id;
1831
1832 if (wr->opcode >= ARRAY_SIZE(mthca_opcode)) {
1833 mthca_err(dev, "opcode invalid\n");
1834 err = -EINVAL;
1835 *bad_wr = wr;
1836 goto out;
1837 }
1838
1839 if (likely(prev_wqe)) {
1840 ((struct mthca_next_seg *) prev_wqe)->nda_op =
1841 cpu_to_be32(((ind << qp->sq.wqe_shift) +
1842 qp->send_wqe_offset) |
1843 mthca_opcode[wr->opcode]);
1844 wmb();
1845 ((struct mthca_next_seg *) prev_wqe)->ee_nds =
1846 cpu_to_be32(MTHCA_NEXT_DBD | size);
1847 }
1848
1849 if (!size0) {
1850 size0 = size;
1851 op0 = mthca_opcode[wr->opcode];
1852 }
1853
1854 ++ind;
1855 if (unlikely(ind >= qp->sq.max))
1856 ind -= qp->sq.max;
1857 }
1858
1859out:
1860 if (likely(nreq)) {
1861 u32 doorbell[2];
1862
1863 doorbell[0] = cpu_to_be32((nreq << 24) |
1864 ((qp->sq.head & 0xffff) << 8) |
1865 f0 | op0);
1866 doorbell[1] = cpu_to_be32((qp->qpn << 8) | size0);
1867
1868 qp->sq.head += nreq;
1869
1870 /*
1871 * Make sure that descriptors are written before
1872 * doorbell record.
1873 */
1874 wmb();
1875 *qp->sq.db = cpu_to_be32(qp->sq.head & 0xffff);
1876
1877 /*
1878 * Make sure doorbell record is written before we
1879 * write MMIO send doorbell.
1880 */
1881 wmb();
1882 mthca_write64(doorbell,
1883 dev->kar + MTHCA_SEND_DOORBELL,
1884 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
1885 }
1886
1887 spin_unlock_irqrestore(&qp->sq.lock, flags);
1888 return err;
1889}
1890
1891int mthca_arbel_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
1892 struct ib_recv_wr **bad_wr)
1893{
1894 struct mthca_dev *dev = to_mdev(ibqp->device);
1895 struct mthca_qp *qp = to_mqp(ibqp);
1896 unsigned long flags;
1897 int err = 0;
1898 int nreq;
1899 int ind;
1900 int i;
1901 void *wqe;
1902
1903 spin_lock_irqsave(&qp->rq.lock, flags);
1904
1905 /* XXX check that state is OK to post receive */
1906
1907 ind = qp->rq.head & (qp->rq.max - 1);
1908
1909 for (nreq = 0; wr; ++nreq, wr = wr->next) {
1910 if (mthca_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
1911 mthca_err(dev, "RQ %06x full (%u head, %u tail,"
1912 " %d max, %d nreq)\n", qp->qpn,
1913 qp->rq.head, qp->rq.tail,
1914 qp->rq.max, nreq);
1915 err = -ENOMEM;
1916 *bad_wr = wr;
1917 goto out;
1918 }
1919
1920 wqe = get_recv_wqe(qp, ind);
1921
1922 ((struct mthca_next_seg *) wqe)->flags = 0;
1923
1924 wqe += sizeof (struct mthca_next_seg);
1925
1926 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
1927 err = -EINVAL;
1928 *bad_wr = wr;
1929 goto out;
1930 }
1931
1932 for (i = 0; i < wr->num_sge; ++i) {
1933 ((struct mthca_data_seg *) wqe)->byte_count =
1934 cpu_to_be32(wr->sg_list[i].length);
1935 ((struct mthca_data_seg *) wqe)->lkey =
1936 cpu_to_be32(wr->sg_list[i].lkey);
1937 ((struct mthca_data_seg *) wqe)->addr =
1938 cpu_to_be64(wr->sg_list[i].addr);
1939 wqe += sizeof (struct mthca_data_seg);
1940 }
1941
1942 if (i < qp->rq.max_gs) {
1943 ((struct mthca_data_seg *) wqe)->byte_count = 0;
1944 ((struct mthca_data_seg *) wqe)->lkey = cpu_to_be32(0x100);
1945 ((struct mthca_data_seg *) wqe)->addr = 0;
1946 }
1947
1948 qp->wrid[ind] = wr->wr_id;
1949
1950 ++ind;
1951 if (unlikely(ind >= qp->rq.max))
1952 ind -= qp->rq.max;
1953 }
1954out:
1955 if (likely(nreq)) {
1956 qp->rq.head += nreq;
1957
1958 /*
1959 * Make sure that descriptors are written before
1960 * doorbell record.
1961 */
1962 wmb();
1963 *qp->rq.db = cpu_to_be32(qp->rq.head & 0xffff);
1964 }
1965
1966 spin_unlock_irqrestore(&qp->rq.lock, flags);
1967 return err;
1968}
1969
1970int mthca_free_err_wqe(struct mthca_dev *dev, struct mthca_qp *qp, int is_send,
1971 int index, int *dbd, u32 *new_wqe)
1972{
1973 struct mthca_next_seg *next;
1974
1975 if (is_send)
1976 next = get_send_wqe(qp, index);
1977 else
1978 next = get_recv_wqe(qp, index);
1979
1980 if (dev->hca_type == ARBEL_NATIVE)
1981 *dbd = 1;
1982 else
1983 *dbd = !!(next->ee_nds & cpu_to_be32(MTHCA_NEXT_DBD));
1984 if (next->ee_nds & cpu_to_be32(0x3f))
1985 *new_wqe = (next->nda_op & cpu_to_be32(~0x3f)) |
1986 (next->ee_nds & cpu_to_be32(0x3f));
1987 else
1988 *new_wqe = 0;
1989
1990 return 0;
1991}
1992
1993int __devinit mthca_init_qp_table(struct mthca_dev *dev)
1994{
1995 int err;
1996 u8 status;
1997 int i;
1998
1999 spin_lock_init(&dev->qp_table.lock);
2000
2001 /*
2002 * We reserve 2 extra QPs per port for the special QPs. The
2003 * special QP for port 1 has to be even, so round up.
2004 */
2005 dev->qp_table.sqp_start = (dev->limits.reserved_qps + 1) & ~1UL;
2006 err = mthca_alloc_init(&dev->qp_table.alloc,
2007 dev->limits.num_qps,
2008 (1 << 24) - 1,
2009 dev->qp_table.sqp_start +
2010 MTHCA_MAX_PORTS * 2);
2011 if (err)
2012 return err;
2013
2014 err = mthca_array_init(&dev->qp_table.qp,
2015 dev->limits.num_qps);
2016 if (err) {
2017 mthca_alloc_cleanup(&dev->qp_table.alloc);
2018 return err;
2019 }
2020
2021 for (i = 0; i < 2; ++i) {
2022 err = mthca_CONF_SPECIAL_QP(dev, i ? IB_QPT_GSI : IB_QPT_SMI,
2023 dev->qp_table.sqp_start + i * 2,
2024 &status);
2025 if (err)
2026 goto err_out;
2027 if (status) {
2028 mthca_warn(dev, "CONF_SPECIAL_QP returned "
2029 "status %02x, aborting.\n",
2030 status);
2031 err = -EINVAL;
2032 goto err_out;
2033 }
2034 }
2035 return 0;
2036
2037 err_out:
2038 for (i = 0; i < 2; ++i)
2039 mthca_CONF_SPECIAL_QP(dev, i, 0, &status);
2040
2041 mthca_array_cleanup(&dev->qp_table.qp, dev->limits.num_qps);
2042 mthca_alloc_cleanup(&dev->qp_table.alloc);
2043
2044 return err;
2045}
2046
2047void __devexit mthca_cleanup_qp_table(struct mthca_dev *dev)
2048{
2049 int i;
2050 u8 status;
2051
2052 for (i = 0; i < 2; ++i)
2053 mthca_CONF_SPECIAL_QP(dev, i, 0, &status);
2054
2055 mthca_alloc_cleanup(&dev->qp_table.alloc);
2056}
diff --git a/drivers/infiniband/hw/mthca/mthca_reset.c b/drivers/infiniband/hw/mthca/mthca_reset.c
new file mode 100644
index 000000000000..ce3fff7d02b7
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_reset.c
@@ -0,0 +1,232 @@
1/*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_reset.c 1349 2004-12-16 21:09:43Z roland $
33 */
34
35#include <linux/config.h>
36#include <linux/init.h>
37#include <linux/errno.h>
38#include <linux/pci.h>
39#include <linux/delay.h>
40
41#include "mthca_dev.h"
42#include "mthca_cmd.h"
43
44int mthca_reset(struct mthca_dev *mdev)
45{
46 int i;
47 int err = 0;
48 u32 *hca_header = NULL;
49 u32 *bridge_header = NULL;
50 struct pci_dev *bridge = NULL;
51
52#define MTHCA_RESET_OFFSET 0xf0010
53#define MTHCA_RESET_VALUE swab32(1)
54
55 /*
56 * Reset the chip. This is somewhat ugly because we have to
57 * save off the PCI header before reset and then restore it
58 * after the chip reboots. We skip config space offsets 22
59 * and 23 since those have a special meaning.
60 *
61 * To make matters worse, for Tavor (PCI-X HCA) we have to
62 * find the associated bridge device and save off its PCI
63 * header as well.
64 */
65
66 if (mdev->hca_type == TAVOR) {
67 /* Look for the bridge -- its device ID will be 2 more
68 than HCA's device ID. */
69 while ((bridge = pci_get_device(mdev->pdev->vendor,
70 mdev->pdev->device + 2,
71 bridge)) != NULL) {
72 if (bridge->hdr_type == PCI_HEADER_TYPE_BRIDGE &&
73 bridge->subordinate == mdev->pdev->bus) {
74 mthca_dbg(mdev, "Found bridge: %s (%s)\n",
75 pci_pretty_name(bridge), pci_name(bridge));
76 break;
77 }
78 }
79
80 if (!bridge) {
81 /*
82 * Didn't find a bridge for a Tavor device --
83 * assume we're in no-bridge mode and hope for
84 * the best.
85 */
86 mthca_warn(mdev, "No bridge found for %s (%s)\n",
87 pci_pretty_name(mdev->pdev), pci_name(mdev->pdev));
88 }
89
90 }
91
92 /* For Arbel do we need to save off the full 4K PCI Express header?? */
93 hca_header = kmalloc(256, GFP_KERNEL);
94 if (!hca_header) {
95 err = -ENOMEM;
96 mthca_err(mdev, "Couldn't allocate memory to save HCA "
97 "PCI header, aborting.\n");
98 goto out;
99 }
100
101 for (i = 0; i < 64; ++i) {
102 if (i == 22 || i == 23)
103 continue;
104 if (pci_read_config_dword(mdev->pdev, i * 4, hca_header + i)) {
105 err = -ENODEV;
106 mthca_err(mdev, "Couldn't save HCA "
107 "PCI header, aborting.\n");
108 goto out;
109 }
110 }
111
112 if (bridge) {
113 bridge_header = kmalloc(256, GFP_KERNEL);
114 if (!bridge_header) {
115 err = -ENOMEM;
116 mthca_err(mdev, "Couldn't allocate memory to save HCA "
117 "bridge PCI header, aborting.\n");
118 goto out;
119 }
120
121 for (i = 0; i < 64; ++i) {
122 if (i == 22 || i == 23)
123 continue;
124 if (pci_read_config_dword(bridge, i * 4, bridge_header + i)) {
125 err = -ENODEV;
126 mthca_err(mdev, "Couldn't save HCA bridge "
127 "PCI header, aborting.\n");
128 goto out;
129 }
130 }
131 }
132
133 /* actually hit reset */
134 {
135 void __iomem *reset = ioremap(pci_resource_start(mdev->pdev, 0) +
136 MTHCA_RESET_OFFSET, 4);
137
138 if (!reset) {
139 err = -ENOMEM;
140 mthca_err(mdev, "Couldn't map HCA reset register, "
141 "aborting.\n");
142 goto out;
143 }
144
145 writel(MTHCA_RESET_VALUE, reset);
146 iounmap(reset);
147 }
148
149 /* Docs say to wait one second before accessing device */
150 msleep(1000);
151
152 /* Now wait for PCI device to start responding again */
153 {
154 u32 v;
155 int c = 0;
156
157 for (c = 0; c < 100; ++c) {
158 if (pci_read_config_dword(bridge ? bridge : mdev->pdev, 0, &v)) {
159 err = -ENODEV;
160 mthca_err(mdev, "Couldn't access HCA after reset, "
161 "aborting.\n");
162 goto out;
163 }
164
165 if (v != 0xffffffff)
166 goto good;
167
168 msleep(100);
169 }
170
171 err = -ENODEV;
172 mthca_err(mdev, "PCI device did not come back after reset, "
173 "aborting.\n");
174 goto out;
175 }
176
177good:
178 /* Now restore the PCI headers */
179 if (bridge) {
180 /*
181 * Bridge control register is at 0x3e, so we'll
182 * naturally restore it last in this loop.
183 */
184 for (i = 0; i < 16; ++i) {
185 if (i * 4 == PCI_COMMAND)
186 continue;
187
188 if (pci_write_config_dword(bridge, i * 4, bridge_header[i])) {
189 err = -ENODEV;
190 mthca_err(mdev, "Couldn't restore HCA bridge reg %x, "
191 "aborting.\n", i);
192 goto out;
193 }
194 }
195
196 if (pci_write_config_dword(bridge, PCI_COMMAND,
197 bridge_header[PCI_COMMAND / 4])) {
198 err = -ENODEV;
199 mthca_err(mdev, "Couldn't restore HCA bridge COMMAND, "
200 "aborting.\n");
201 goto out;
202 }
203 }
204
205 for (i = 0; i < 16; ++i) {
206 if (i * 4 == PCI_COMMAND)
207 continue;
208
209 if (pci_write_config_dword(mdev->pdev, i * 4, hca_header[i])) {
210 err = -ENODEV;
211 mthca_err(mdev, "Couldn't restore HCA reg %x, "
212 "aborting.\n", i);
213 goto out;
214 }
215 }
216
217 if (pci_write_config_dword(mdev->pdev, PCI_COMMAND,
218 hca_header[PCI_COMMAND / 4])) {
219 err = -ENODEV;
220 mthca_err(mdev, "Couldn't restore HCA COMMAND, "
221 "aborting.\n");
222 goto out;
223 }
224
225out:
226 if (bridge)
227 pci_dev_put(bridge);
228 kfree(bridge_header);
229 kfree(hca_header);
230
231 return err;
232}
diff --git a/drivers/infiniband/hw/mthca/mthca_uar.c b/drivers/infiniband/hw/mthca/mthca_uar.c
new file mode 100644
index 000000000000..1c8791ded6ff
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_uar.c
@@ -0,0 +1,78 @@
1/*
2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id$
33 */
34
35#include "mthca_dev.h"
36#include "mthca_memfree.h"
37
38int mthca_uar_alloc(struct mthca_dev *dev, struct mthca_uar *uar)
39{
40 uar->index = mthca_alloc(&dev->uar_table.alloc);
41 if (uar->index == -1)
42 return -ENOMEM;
43
44 uar->pfn = (pci_resource_start(dev->pdev, 2) >> PAGE_SHIFT) + uar->index;
45
46 return 0;
47}
48
49void mthca_uar_free(struct mthca_dev *dev, struct mthca_uar *uar)
50{
51 mthca_free(&dev->uar_table.alloc, uar->index);
52}
53
54int mthca_init_uar_table(struct mthca_dev *dev)
55{
56 int ret;
57
58 ret = mthca_alloc_init(&dev->uar_table.alloc,
59 dev->limits.num_uars,
60 dev->limits.num_uars - 1,
61 dev->limits.reserved_uars);
62 if (ret)
63 return ret;
64
65 ret = mthca_init_db_tab(dev);
66 if (ret)
67 mthca_alloc_cleanup(&dev->uar_table.alloc);
68
69 return ret;
70}
71
72void mthca_cleanup_uar_table(struct mthca_dev *dev)
73{
74 mthca_cleanup_db_tab(dev);
75
76 /* XXX check if any UARs are still allocated? */
77 mthca_alloc_cleanup(&dev->uar_table.alloc);
78}
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-19 06:57:17 -0500