aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/infiniband/hw/ehca/ehca_classes.h50
-rw-r--r--drivers/infiniband/hw/ehca/ehca_cq.c8
-rw-r--r--drivers/infiniband/hw/ehca/ehca_eq.c8
-rw-r--r--drivers/infiniband/hw/ehca/ehca_irq.c42
-rw-r--r--drivers/infiniband/hw/ehca/ehca_main.c49
-rw-r--r--drivers/infiniband/hw/ehca/ehca_mrmw.c371
-rw-r--r--drivers/infiniband/hw/ehca/ehca_mrmw.h2
-rw-r--r--drivers/infiniband/hw/ehca/ehca_pd.c25
-rw-r--r--drivers/infiniband/hw/ehca/ehca_qp.c178
-rw-r--r--drivers/infiniband/hw/ehca/ehca_tools.h19
-rw-r--r--drivers/infiniband/hw/ehca/ehca_uverbs.c2
-rw-r--r--drivers/infiniband/hw/ehca/hcp_if.c50
-rw-r--r--drivers/infiniband/hw/ehca/ipz_pt_fn.c222
-rw-r--r--drivers/infiniband/hw/ehca/ipz_pt_fn.h26
-rw-r--r--drivers/infiniband/hw/ipath/Makefile1
-rw-r--r--drivers/infiniband/hw/ipath/ipath_layer.c365
-rw-r--r--drivers/infiniband/hw/ipath/ipath_layer.h71
-rw-r--r--drivers/infiniband/hw/ipath/ipath_verbs.h2
-rw-r--r--drivers/infiniband/hw/mlx4/qp.c20
-rw-r--r--drivers/infiniband/hw/mthca/mthca_cmd.c3
-rw-r--r--drivers/net/mlx4/cmd.c3
21 files changed, 802 insertions, 715 deletions
diff --git a/drivers/infiniband/hw/ehca/ehca_classes.h b/drivers/infiniband/hw/ehca/ehca_classes.h
index 043e4fb23fb0..3725aa8664d9 100644
--- a/drivers/infiniband/hw/ehca/ehca_classes.h
+++ b/drivers/infiniband/hw/ehca/ehca_classes.h
@@ -43,7 +43,6 @@
43#ifndef __EHCA_CLASSES_H__ 43#ifndef __EHCA_CLASSES_H__
44#define __EHCA_CLASSES_H__ 44#define __EHCA_CLASSES_H__
45 45
46
47struct ehca_module; 46struct ehca_module;
48struct ehca_qp; 47struct ehca_qp;
49struct ehca_cq; 48struct ehca_cq;
@@ -100,6 +99,11 @@ struct ehca_sport {
100 struct ehca_sma_attr saved_attr; 99 struct ehca_sma_attr saved_attr;
101}; 100};
102 101
102#define HCA_CAP_MR_PGSIZE_4K 1
103#define HCA_CAP_MR_PGSIZE_64K 2
104#define HCA_CAP_MR_PGSIZE_1M 4
105#define HCA_CAP_MR_PGSIZE_16M 8
106
103struct ehca_shca { 107struct ehca_shca {
104 struct ib_device ib_device; 108 struct ib_device ib_device;
105 struct ibmebus_dev *ibmebus_dev; 109 struct ibmebus_dev *ibmebus_dev;
@@ -115,6 +119,8 @@ struct ehca_shca {
115 struct h_galpas galpas; 119 struct h_galpas galpas;
116 struct mutex modify_mutex; 120 struct mutex modify_mutex;
117 u64 hca_cap; 121 u64 hca_cap;
122 /* MR pgsize: bit 0-3 means 4K, 64K, 1M, 16M respectively */
123 u32 hca_cap_mr_pgsize;
118 int max_mtu; 124 int max_mtu;
119}; 125};
120 126
@@ -122,6 +128,10 @@ struct ehca_pd {
122 struct ib_pd ib_pd; 128 struct ib_pd ib_pd;
123 struct ipz_pd fw_pd; 129 struct ipz_pd fw_pd;
124 u32 ownpid; 130 u32 ownpid;
131 /* small queue mgmt */
132 struct mutex lock;
133 struct list_head free[2];
134 struct list_head full[2];
125}; 135};
126 136
127enum ehca_ext_qp_type { 137enum ehca_ext_qp_type {
@@ -206,6 +216,7 @@ struct ehca_mr {
206 enum ehca_mr_flag flags; 216 enum ehca_mr_flag flags;
207 u32 num_kpages; /* number of kernel pages */ 217 u32 num_kpages; /* number of kernel pages */
208 u32 num_hwpages; /* number of hw pages to form MR */ 218 u32 num_hwpages; /* number of hw pages to form MR */
219 u64 hwpage_size; /* hw page size used for this MR */
209 int acl; /* ACL (stored here for usage in reregister) */ 220 int acl; /* ACL (stored here for usage in reregister) */
210 u64 *start; /* virtual start address (stored here for */ 221 u64 *start; /* virtual start address (stored here for */
211 /* usage in reregister) */ 222 /* usage in reregister) */
@@ -240,6 +251,7 @@ struct ehca_mr_pginfo {
240 enum ehca_mr_pgi_type type; 251 enum ehca_mr_pgi_type type;
241 u64 num_kpages; 252 u64 num_kpages;
242 u64 kpage_cnt; 253 u64 kpage_cnt;
254 u64 hwpage_size; /* hw page size used for this MR */
243 u64 num_hwpages; /* number of hw pages */ 255 u64 num_hwpages; /* number of hw pages */
244 u64 hwpage_cnt; /* counter for hw pages */ 256 u64 hwpage_cnt; /* counter for hw pages */
245 u64 next_hwpage; /* next hw page in buffer/chunk/listelem */ 257 u64 next_hwpage; /* next hw page in buffer/chunk/listelem */
@@ -298,6 +310,8 @@ int ehca_init_av_cache(void);
298void ehca_cleanup_av_cache(void); 310void ehca_cleanup_av_cache(void);
299int ehca_init_mrmw_cache(void); 311int ehca_init_mrmw_cache(void);
300void ehca_cleanup_mrmw_cache(void); 312void ehca_cleanup_mrmw_cache(void);
313int ehca_init_small_qp_cache(void);
314void ehca_cleanup_small_qp_cache(void);
301 315
302extern rwlock_t ehca_qp_idr_lock; 316extern rwlock_t ehca_qp_idr_lock;
303extern rwlock_t ehca_cq_idr_lock; 317extern rwlock_t ehca_cq_idr_lock;
@@ -315,7 +329,7 @@ struct ipzu_queue_resp {
315 u32 queue_length; /* queue length allocated in bytes */ 329 u32 queue_length; /* queue length allocated in bytes */
316 u32 pagesize; 330 u32 pagesize;
317 u32 toggle_state; 331 u32 toggle_state;
318 u32 dummy; /* padding for 8 byte alignment */ 332 u32 offset; /* save offset within a page for small_qp */
319}; 333};
320 334
321struct ehca_create_cq_resp { 335struct ehca_create_cq_resp {
@@ -357,15 +371,29 @@ enum ehca_ll_comp_flags {
357 LLQP_COMP_MASK = 0x60, 371 LLQP_COMP_MASK = 0x60,
358}; 372};
359 373
374struct ehca_alloc_queue_parms {
375 /* input parameters */
376 int max_wr;
377 int max_sge;
378 int page_size;
379 int is_small;
380
381 /* output parameters */
382 u16 act_nr_wqes;
383 u8 act_nr_sges;
384 u32 queue_size; /* bytes for small queues, pages otherwise */
385};
386
360struct ehca_alloc_qp_parms { 387struct ehca_alloc_qp_parms {
361/* input parameters */ 388 struct ehca_alloc_queue_parms squeue;
389 struct ehca_alloc_queue_parms rqueue;
390
391 /* input parameters */
362 enum ehca_service_type servicetype; 392 enum ehca_service_type servicetype;
393 int qp_storage;
363 int sigtype; 394 int sigtype;
364 enum ehca_ext_qp_type ext_type; 395 enum ehca_ext_qp_type ext_type;
365 enum ehca_ll_comp_flags ll_comp_flags; 396 enum ehca_ll_comp_flags ll_comp_flags;
366
367 int max_send_wr, max_recv_wr;
368 int max_send_sge, max_recv_sge;
369 int ud_av_l_key_ctl; 397 int ud_av_l_key_ctl;
370 398
371 u32 token; 399 u32 token;
@@ -375,18 +403,10 @@ struct ehca_alloc_qp_parms {
375 403
376 u32 srq_qpn, srq_token, srq_limit; 404 u32 srq_qpn, srq_token, srq_limit;
377 405
378/* output parameters */ 406 /* output parameters */
379 u32 real_qp_num; 407 u32 real_qp_num;
380 struct ipz_qp_handle qp_handle; 408 struct ipz_qp_handle qp_handle;
381 struct h_galpas galpas; 409 struct h_galpas galpas;
382
383 u16 act_nr_send_wqes;
384 u16 act_nr_recv_wqes;
385 u8 act_nr_recv_sges;
386 u8 act_nr_send_sges;
387
388 u32 nr_rq_pages;
389 u32 nr_sq_pages;
390}; 410};
391 411
392int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp); 412int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp);
diff --git a/drivers/infiniband/hw/ehca/ehca_cq.c b/drivers/infiniband/hw/ehca/ehca_cq.c
index 1e8ca3fca4aa..81aff36101ba 100644
--- a/drivers/infiniband/hw/ehca/ehca_cq.c
+++ b/drivers/infiniband/hw/ehca/ehca_cq.c
@@ -190,8 +190,8 @@ struct ib_cq *ehca_create_cq(struct ib_device *device, int cqe, int comp_vector,
190 goto create_cq_exit2; 190 goto create_cq_exit2;
191 } 191 }
192 192
193 ipz_rc = ipz_queue_ctor(&my_cq->ipz_queue, param.act_pages, 193 ipz_rc = ipz_queue_ctor(NULL, &my_cq->ipz_queue, param.act_pages,
194 EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0); 194 EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0, 0);
195 if (!ipz_rc) { 195 if (!ipz_rc) {
196 ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%x device=%p", 196 ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%x device=%p",
197 ipz_rc, device); 197 ipz_rc, device);
@@ -285,7 +285,7 @@ struct ib_cq *ehca_create_cq(struct ib_device *device, int cqe, int comp_vector,
285 return cq; 285 return cq;
286 286
287create_cq_exit4: 287create_cq_exit4:
288 ipz_queue_dtor(&my_cq->ipz_queue); 288 ipz_queue_dtor(NULL, &my_cq->ipz_queue);
289 289
290create_cq_exit3: 290create_cq_exit3:
291 h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1); 291 h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
@@ -359,7 +359,7 @@ int ehca_destroy_cq(struct ib_cq *cq)
359 "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num); 359 "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num);
360 return ehca2ib_return_code(h_ret); 360 return ehca2ib_return_code(h_ret);
361 } 361 }
362 ipz_queue_dtor(&my_cq->ipz_queue); 362 ipz_queue_dtor(NULL, &my_cq->ipz_queue);
363 kmem_cache_free(cq_cache, my_cq); 363 kmem_cache_free(cq_cache, my_cq);
364 364
365 return 0; 365 return 0;
diff --git a/drivers/infiniband/hw/ehca/ehca_eq.c b/drivers/infiniband/hw/ehca/ehca_eq.c
index 4825975f88cf..1d41faa7a337 100644
--- a/drivers/infiniband/hw/ehca/ehca_eq.c
+++ b/drivers/infiniband/hw/ehca/ehca_eq.c
@@ -86,8 +86,8 @@ int ehca_create_eq(struct ehca_shca *shca,
86 return -EINVAL; 86 return -EINVAL;
87 } 87 }
88 88
89 ret = ipz_queue_ctor(&eq->ipz_queue, nr_pages, 89 ret = ipz_queue_ctor(NULL, &eq->ipz_queue, nr_pages,
90 EHCA_PAGESIZE, sizeof(struct ehca_eqe), 0); 90 EHCA_PAGESIZE, sizeof(struct ehca_eqe), 0, 0);
91 if (!ret) { 91 if (!ret) {
92 ehca_err(ib_dev, "Can't allocate EQ pages eq=%p", eq); 92 ehca_err(ib_dev, "Can't allocate EQ pages eq=%p", eq);
93 goto create_eq_exit1; 93 goto create_eq_exit1;
@@ -145,7 +145,7 @@ int ehca_create_eq(struct ehca_shca *shca,
145 return 0; 145 return 0;
146 146
147create_eq_exit2: 147create_eq_exit2:
148 ipz_queue_dtor(&eq->ipz_queue); 148 ipz_queue_dtor(NULL, &eq->ipz_queue);
149 149
150create_eq_exit1: 150create_eq_exit1:
151 hipz_h_destroy_eq(shca->ipz_hca_handle, eq); 151 hipz_h_destroy_eq(shca->ipz_hca_handle, eq);
@@ -181,7 +181,7 @@ int ehca_destroy_eq(struct ehca_shca *shca, struct ehca_eq *eq)
181 ehca_err(&shca->ib_device, "Can't free EQ resources."); 181 ehca_err(&shca->ib_device, "Can't free EQ resources.");
182 return -EINVAL; 182 return -EINVAL;
183 } 183 }
184 ipz_queue_dtor(&eq->ipz_queue); 184 ipz_queue_dtor(NULL, &eq->ipz_queue);
185 185
186 return 0; 186 return 0;
187} 187}
diff --git a/drivers/infiniband/hw/ehca/ehca_irq.c b/drivers/infiniband/hw/ehca/ehca_irq.c
index 4fb01fcb63ae..71c0799b3500 100644
--- a/drivers/infiniband/hw/ehca/ehca_irq.c
+++ b/drivers/infiniband/hw/ehca/ehca_irq.c
@@ -175,9 +175,8 @@ error_data1:
175 175
176} 176}
177 177
178static void qp_event_callback(struct ehca_shca *shca, 178static void qp_event_callback(struct ehca_shca *shca, u64 eqe,
179 u64 eqe, 179 enum ib_event_type event_type, int fatal)
180 enum ib_event_type event_type)
181{ 180{
182 struct ib_event event; 181 struct ib_event event;
183 struct ehca_qp *qp; 182 struct ehca_qp *qp;
@@ -191,16 +190,26 @@ static void qp_event_callback(struct ehca_shca *shca,
191 if (!qp) 190 if (!qp)
192 return; 191 return;
193 192
194 ehca_error_data(shca, qp, qp->ipz_qp_handle.handle); 193 if (fatal)
194 ehca_error_data(shca, qp, qp->ipz_qp_handle.handle);
195 195
196 if (!qp->ib_qp.event_handler) 196 event.device = &shca->ib_device;
197 return;
198 197
199 event.device = &shca->ib_device; 198 if (qp->ext_type == EQPT_SRQ) {
200 event.event = event_type; 199 if (!qp->ib_srq.event_handler)
201 event.element.qp = &qp->ib_qp; 200 return;
202 201
203 qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context); 202 event.event = fatal ? IB_EVENT_SRQ_ERR : event_type;
203 event.element.srq = &qp->ib_srq;
204 qp->ib_srq.event_handler(&event, qp->ib_srq.srq_context);
205 } else {
206 if (!qp->ib_qp.event_handler)
207 return;
208
209 event.event = event_type;
210 event.element.qp = &qp->ib_qp;
211 qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context);
212 }
204 213
205 return; 214 return;
206} 215}
@@ -234,17 +243,17 @@ static void parse_identifier(struct ehca_shca *shca, u64 eqe)
234 243
235 switch (identifier) { 244 switch (identifier) {
236 case 0x02: /* path migrated */ 245 case 0x02: /* path migrated */
237 qp_event_callback(shca, eqe, IB_EVENT_PATH_MIG); 246 qp_event_callback(shca, eqe, IB_EVENT_PATH_MIG, 0);
238 break; 247 break;
239 case 0x03: /* communication established */ 248 case 0x03: /* communication established */
240 qp_event_callback(shca, eqe, IB_EVENT_COMM_EST); 249 qp_event_callback(shca, eqe, IB_EVENT_COMM_EST, 0);
241 break; 250 break;
242 case 0x04: /* send queue drained */ 251 case 0x04: /* send queue drained */
243 qp_event_callback(shca, eqe, IB_EVENT_SQ_DRAINED); 252 qp_event_callback(shca, eqe, IB_EVENT_SQ_DRAINED, 0);
244 break; 253 break;
245 case 0x05: /* QP error */ 254 case 0x05: /* QP error */
246 case 0x06: /* QP error */ 255 case 0x06: /* QP error */
247 qp_event_callback(shca, eqe, IB_EVENT_QP_FATAL); 256 qp_event_callback(shca, eqe, IB_EVENT_QP_FATAL, 1);
248 break; 257 break;
249 case 0x07: /* CQ error */ 258 case 0x07: /* CQ error */
250 case 0x08: /* CQ error */ 259 case 0x08: /* CQ error */
@@ -278,6 +287,11 @@ static void parse_identifier(struct ehca_shca *shca, u64 eqe)
278 ehca_err(&shca->ib_device, "Interface trace stopped."); 287 ehca_err(&shca->ib_device, "Interface trace stopped.");
279 break; 288 break;
280 case 0x14: /* first error capture info available */ 289 case 0x14: /* first error capture info available */
290 ehca_info(&shca->ib_device, "First error capture available");
291 break;
292 case 0x15: /* SRQ limit reached */
293 qp_event_callback(shca, eqe, IB_EVENT_SRQ_LIMIT_REACHED, 0);
294 break;
281 default: 295 default:
282 ehca_err(&shca->ib_device, "Unknown identifier: %x on %s.", 296 ehca_err(&shca->ib_device, "Unknown identifier: %x on %s.",
283 identifier, shca->ib_device.name); 297 identifier, shca->ib_device.name);
diff --git a/drivers/infiniband/hw/ehca/ehca_main.c b/drivers/infiniband/hw/ehca/ehca_main.c
index 04c324330b7c..99036b65bb84 100644
--- a/drivers/infiniband/hw/ehca/ehca_main.c
+++ b/drivers/infiniband/hw/ehca/ehca_main.c
@@ -63,6 +63,7 @@ int ehca_port_act_time = 30;
63int ehca_poll_all_eqs = 1; 63int ehca_poll_all_eqs = 1;
64int ehca_static_rate = -1; 64int ehca_static_rate = -1;
65int ehca_scaling_code = 0; 65int ehca_scaling_code = 0;
66int ehca_mr_largepage = 0;
66 67
67module_param_named(open_aqp1, ehca_open_aqp1, int, 0); 68module_param_named(open_aqp1, ehca_open_aqp1, int, 0);
68module_param_named(debug_level, ehca_debug_level, int, 0); 69module_param_named(debug_level, ehca_debug_level, int, 0);
@@ -72,7 +73,8 @@ module_param_named(use_hp_mr, ehca_use_hp_mr, int, 0);
72module_param_named(port_act_time, ehca_port_act_time, int, 0); 73module_param_named(port_act_time, ehca_port_act_time, int, 0);
73module_param_named(poll_all_eqs, ehca_poll_all_eqs, int, 0); 74module_param_named(poll_all_eqs, ehca_poll_all_eqs, int, 0);
74module_param_named(static_rate, ehca_static_rate, int, 0); 75module_param_named(static_rate, ehca_static_rate, int, 0);
75module_param_named(scaling_code, ehca_scaling_code, int, 0); 76module_param_named(scaling_code, ehca_scaling_code, int, 0);
77module_param_named(mr_largepage, ehca_mr_largepage, int, 0);
76 78
77MODULE_PARM_DESC(open_aqp1, 79MODULE_PARM_DESC(open_aqp1,
78 "AQP1 on startup (0: no (default), 1: yes)"); 80 "AQP1 on startup (0: no (default), 1: yes)");
@@ -95,6 +97,9 @@ MODULE_PARM_DESC(static_rate,
95 "set permanent static rate (default: disabled)"); 97 "set permanent static rate (default: disabled)");
96MODULE_PARM_DESC(scaling_code, 98MODULE_PARM_DESC(scaling_code,
97 "set scaling code (0: disabled/default, 1: enabled)"); 99 "set scaling code (0: disabled/default, 1: enabled)");
100MODULE_PARM_DESC(mr_largepage,
101 "use large page for MR (0: use PAGE_SIZE (default), "
102 "1: use large page depending on MR size");
98 103
99DEFINE_RWLOCK(ehca_qp_idr_lock); 104DEFINE_RWLOCK(ehca_qp_idr_lock);
100DEFINE_RWLOCK(ehca_cq_idr_lock); 105DEFINE_RWLOCK(ehca_cq_idr_lock);
@@ -125,6 +130,23 @@ void ehca_free_fw_ctrlblock(void *ptr)
125} 130}
126#endif 131#endif
127 132
133int ehca2ib_return_code(u64 ehca_rc)
134{
135 switch (ehca_rc) {
136 case H_SUCCESS:
137 return 0;
138 case H_RESOURCE: /* Resource in use */
139 case H_BUSY:
140 return -EBUSY;
141 case H_NOT_ENOUGH_RESOURCES: /* insufficient resources */
142 case H_CONSTRAINED: /* resource constraint */
143 case H_NO_MEM:
144 return -ENOMEM;
145 default:
146 return -EINVAL;
147 }
148}
149
128static int ehca_create_slab_caches(void) 150static int ehca_create_slab_caches(void)
129{ 151{
130 int ret; 152 int ret;
@@ -159,6 +181,12 @@ static int ehca_create_slab_caches(void)
159 goto create_slab_caches5; 181 goto create_slab_caches5;
160 } 182 }
161 183
184 ret = ehca_init_small_qp_cache();
185 if (ret) {
186 ehca_gen_err("Cannot create small queue SLAB cache.");
187 goto create_slab_caches6;
188 }
189
162#ifdef CONFIG_PPC_64K_PAGES 190#ifdef CONFIG_PPC_64K_PAGES
163 ctblk_cache = kmem_cache_create("ehca_cache_ctblk", 191 ctblk_cache = kmem_cache_create("ehca_cache_ctblk",
164 EHCA_PAGESIZE, H_CB_ALIGNMENT, 192 EHCA_PAGESIZE, H_CB_ALIGNMENT,
@@ -166,12 +194,15 @@ static int ehca_create_slab_caches(void)
166 NULL); 194 NULL);
167 if (!ctblk_cache) { 195 if (!ctblk_cache) {
168 ehca_gen_err("Cannot create ctblk SLAB cache."); 196 ehca_gen_err("Cannot create ctblk SLAB cache.");
169 ehca_cleanup_mrmw_cache(); 197 ehca_cleanup_small_qp_cache();
170 goto create_slab_caches5; 198 goto create_slab_caches6;
171 } 199 }
172#endif 200#endif
173 return 0; 201 return 0;
174 202
203create_slab_caches6:
204 ehca_cleanup_mrmw_cache();
205
175create_slab_caches5: 206create_slab_caches5:
176 ehca_cleanup_av_cache(); 207 ehca_cleanup_av_cache();
177 208
@@ -189,6 +220,7 @@ create_slab_caches2:
189 220
190static void ehca_destroy_slab_caches(void) 221static void ehca_destroy_slab_caches(void)
191{ 222{
223 ehca_cleanup_small_qp_cache();
192 ehca_cleanup_mrmw_cache(); 224 ehca_cleanup_mrmw_cache();
193 ehca_cleanup_av_cache(); 225 ehca_cleanup_av_cache();
194 ehca_cleanup_qp_cache(); 226 ehca_cleanup_qp_cache();
@@ -295,6 +327,8 @@ int ehca_sense_attributes(struct ehca_shca *shca)
295 if (EHCA_BMASK_GET(hca_cap_descr[i].mask, shca->hca_cap)) 327 if (EHCA_BMASK_GET(hca_cap_descr[i].mask, shca->hca_cap))
296 ehca_gen_dbg(" %s", hca_cap_descr[i].descr); 328 ehca_gen_dbg(" %s", hca_cap_descr[i].descr);
297 329
330 shca->hca_cap_mr_pgsize = rblock->memory_page_size_supported;
331
298 port = (struct hipz_query_port *)rblock; 332 port = (struct hipz_query_port *)rblock;
299 h_ret = hipz_h_query_port(shca->ipz_hca_handle, 1, port); 333 h_ret = hipz_h_query_port(shca->ipz_hca_handle, 1, port);
300 if (h_ret != H_SUCCESS) { 334 if (h_ret != H_SUCCESS) {
@@ -590,6 +624,14 @@ static ssize_t ehca_show_adapter_handle(struct device *dev,
590} 624}
591static DEVICE_ATTR(adapter_handle, S_IRUGO, ehca_show_adapter_handle, NULL); 625static DEVICE_ATTR(adapter_handle, S_IRUGO, ehca_show_adapter_handle, NULL);
592 626
627static ssize_t ehca_show_mr_largepage(struct device *dev,
628 struct device_attribute *attr,
629 char *buf)
630{
631 return sprintf(buf, "%d\n", ehca_mr_largepage);
632}
633static DEVICE_ATTR(mr_largepage, S_IRUGO, ehca_show_mr_largepage, NULL);
634
593static struct attribute *ehca_dev_attrs[] = { 635static struct attribute *ehca_dev_attrs[] = {
594 &dev_attr_adapter_handle.attr, 636 &dev_attr_adapter_handle.attr,
595 &dev_attr_num_ports.attr, 637 &dev_attr_num_ports.attr,
@@ -606,6 +648,7 @@ static struct attribute *ehca_dev_attrs[] = {
606 &dev_attr_cur_mw.attr, 648 &dev_attr_cur_mw.attr,
607 &dev_attr_max_pd.attr, 649 &dev_attr_max_pd.attr,
608 &dev_attr_max_ah.attr, 650 &dev_attr_max_ah.attr,
651 &dev_attr_mr_largepage.attr,
609 NULL 652 NULL
610}; 653};
611 654
diff --git a/drivers/infiniband/hw/ehca/ehca_mrmw.c b/drivers/infiniband/hw/ehca/ehca_mrmw.c
index 9f4c9d46e8ef..c1b868b79d67 100644
--- a/drivers/infiniband/hw/ehca/ehca_mrmw.c
+++ b/drivers/infiniband/hw/ehca/ehca_mrmw.c
@@ -5,6 +5,7 @@
5 * 5 *
6 * Authors: Dietmar Decker <ddecker@de.ibm.com> 6 * Authors: Dietmar Decker <ddecker@de.ibm.com>
7 * Christoph Raisch <raisch@de.ibm.com> 7 * Christoph Raisch <raisch@de.ibm.com>
8 * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
8 * 9 *
9 * Copyright (c) 2005 IBM Corporation 10 * Copyright (c) 2005 IBM Corporation
10 * 11 *
@@ -56,6 +57,37 @@
56static struct kmem_cache *mr_cache; 57static struct kmem_cache *mr_cache;
57static struct kmem_cache *mw_cache; 58static struct kmem_cache *mw_cache;
58 59
60enum ehca_mr_pgsize {
61 EHCA_MR_PGSIZE4K = 0x1000L,
62 EHCA_MR_PGSIZE64K = 0x10000L,
63 EHCA_MR_PGSIZE1M = 0x100000L,
64 EHCA_MR_PGSIZE16M = 0x1000000L
65};
66
67extern int ehca_mr_largepage;
68
69static u32 ehca_encode_hwpage_size(u32 pgsize)
70{
71 u32 idx = 0;
72 pgsize >>= 12;
73 /*
74 * map mr page size into hw code:
75 * 0, 1, 2, 3 for 4K, 64K, 1M, 64M
76 */
77 while (!(pgsize & 1)) {
78 idx++;
79 pgsize >>= 4;
80 }
81 return idx;
82}
83
84static u64 ehca_get_max_hwpage_size(struct ehca_shca *shca)
85{
86 if (shca->hca_cap_mr_pgsize & HCA_CAP_MR_PGSIZE_16M)
87 return EHCA_MR_PGSIZE16M;
88 return EHCA_MR_PGSIZE4K;
89}
90
59static struct ehca_mr *ehca_mr_new(void) 91static struct ehca_mr *ehca_mr_new(void)
60{ 92{
61 struct ehca_mr *me; 93 struct ehca_mr *me;
@@ -207,19 +239,23 @@ struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
207 struct ehca_mr_pginfo pginfo; 239 struct ehca_mr_pginfo pginfo;
208 u32 num_kpages; 240 u32 num_kpages;
209 u32 num_hwpages; 241 u32 num_hwpages;
242 u64 hw_pgsize;
210 243
211 num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size, 244 num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size,
212 PAGE_SIZE); 245 PAGE_SIZE);
213 num_hwpages = NUM_CHUNKS(((u64)iova_start % EHCA_PAGESIZE) + 246 /* for kernel space we try most possible pgsize */
214 size, EHCA_PAGESIZE); 247 hw_pgsize = ehca_get_max_hwpage_size(shca);
248 num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size,
249 hw_pgsize);
215 memset(&pginfo, 0, sizeof(pginfo)); 250 memset(&pginfo, 0, sizeof(pginfo));
216 pginfo.type = EHCA_MR_PGI_PHYS; 251 pginfo.type = EHCA_MR_PGI_PHYS;
217 pginfo.num_kpages = num_kpages; 252 pginfo.num_kpages = num_kpages;
253 pginfo.hwpage_size = hw_pgsize;
218 pginfo.num_hwpages = num_hwpages; 254 pginfo.num_hwpages = num_hwpages;
219 pginfo.u.phy.num_phys_buf = num_phys_buf; 255 pginfo.u.phy.num_phys_buf = num_phys_buf;
220 pginfo.u.phy.phys_buf_array = phys_buf_array; 256 pginfo.u.phy.phys_buf_array = phys_buf_array;
221 pginfo.next_hwpage = (((u64)iova_start & ~PAGE_MASK) / 257 pginfo.next_hwpage =
222 EHCA_PAGESIZE); 258 ((u64)iova_start & ~(hw_pgsize - 1)) / hw_pgsize;
223 259
224 ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags, 260 ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags,
225 e_pd, &pginfo, &e_mr->ib.ib_mr.lkey, 261 e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
@@ -259,6 +295,7 @@ struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
259 int ret; 295 int ret;
260 u32 num_kpages; 296 u32 num_kpages;
261 u32 num_hwpages; 297 u32 num_hwpages;
298 u64 hwpage_size;
262 299
263 if (!pd) { 300 if (!pd) {
264 ehca_gen_err("bad pd=%p", pd); 301 ehca_gen_err("bad pd=%p", pd);
@@ -309,16 +346,32 @@ struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
309 346
310 /* determine number of MR pages */ 347 /* determine number of MR pages */
311 num_kpages = NUM_CHUNKS((virt % PAGE_SIZE) + length, PAGE_SIZE); 348 num_kpages = NUM_CHUNKS((virt % PAGE_SIZE) + length, PAGE_SIZE);
312 num_hwpages = NUM_CHUNKS((virt % EHCA_PAGESIZE) + length, 349 /* select proper hw_pgsize */
313 EHCA_PAGESIZE); 350 if (ehca_mr_largepage &&
351 (shca->hca_cap_mr_pgsize & HCA_CAP_MR_PGSIZE_16M)) {
352 if (length <= EHCA_MR_PGSIZE4K
353 && PAGE_SIZE == EHCA_MR_PGSIZE4K)
354 hwpage_size = EHCA_MR_PGSIZE4K;
355 else if (length <= EHCA_MR_PGSIZE64K)
356 hwpage_size = EHCA_MR_PGSIZE64K;
357 else if (length <= EHCA_MR_PGSIZE1M)
358 hwpage_size = EHCA_MR_PGSIZE1M;
359 else
360 hwpage_size = EHCA_MR_PGSIZE16M;
361 } else
362 hwpage_size = EHCA_MR_PGSIZE4K;
363 ehca_dbg(pd->device, "hwpage_size=%lx", hwpage_size);
314 364
365reg_user_mr_fallback:
366 num_hwpages = NUM_CHUNKS((virt % hwpage_size) + length, hwpage_size);
315 /* register MR on HCA */ 367 /* register MR on HCA */
316 memset(&pginfo, 0, sizeof(pginfo)); 368 memset(&pginfo, 0, sizeof(pginfo));
317 pginfo.type = EHCA_MR_PGI_USER; 369 pginfo.type = EHCA_MR_PGI_USER;
370 pginfo.hwpage_size = hwpage_size;
318 pginfo.num_kpages = num_kpages; 371 pginfo.num_kpages = num_kpages;
319 pginfo.num_hwpages = num_hwpages; 372 pginfo.num_hwpages = num_hwpages;
320 pginfo.u.usr.region = e_mr->umem; 373 pginfo.u.usr.region = e_mr->umem;
321 pginfo.next_hwpage = e_mr->umem->offset / EHCA_PAGESIZE; 374 pginfo.next_hwpage = e_mr->umem->offset / hwpage_size;
322 pginfo.u.usr.next_chunk = list_prepare_entry(pginfo.u.usr.next_chunk, 375 pginfo.u.usr.next_chunk = list_prepare_entry(pginfo.u.usr.next_chunk,
323 (&e_mr->umem->chunk_list), 376 (&e_mr->umem->chunk_list),
324 list); 377 list);
@@ -326,6 +379,18 @@ struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
326 ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags, 379 ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags,
327 e_pd, &pginfo, &e_mr->ib.ib_mr.lkey, 380 e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
328 &e_mr->ib.ib_mr.rkey); 381 &e_mr->ib.ib_mr.rkey);
382 if (ret == -EINVAL && pginfo.hwpage_size > PAGE_SIZE) {
383 ehca_warn(pd->device, "failed to register mr "
384 "with hwpage_size=%lx", hwpage_size);
385 ehca_info(pd->device, "try to register mr with "
386 "kpage_size=%lx", PAGE_SIZE);
387 /*
388 * this means kpages are not contiguous for a hw page
389 * try kernel page size as fallback solution
390 */
391 hwpage_size = PAGE_SIZE;
392 goto reg_user_mr_fallback;
393 }
329 if (ret) { 394 if (ret) {
330 ib_mr = ERR_PTR(ret); 395 ib_mr = ERR_PTR(ret);
331 goto reg_user_mr_exit2; 396 goto reg_user_mr_exit2;
@@ -452,6 +517,8 @@ int ehca_rereg_phys_mr(struct ib_mr *mr,
452 new_pd = container_of(mr->pd, struct ehca_pd, ib_pd); 517 new_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
453 518
454 if (mr_rereg_mask & IB_MR_REREG_TRANS) { 519 if (mr_rereg_mask & IB_MR_REREG_TRANS) {
520 u64 hw_pgsize = ehca_get_max_hwpage_size(shca);
521
455 new_start = iova_start; /* change address */ 522 new_start = iova_start; /* change address */
456 /* check physical buffer list and calculate size */ 523 /* check physical buffer list and calculate size */
457 ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, 524 ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array,
@@ -468,16 +535,17 @@ int ehca_rereg_phys_mr(struct ib_mr *mr,
468 } 535 }
469 num_kpages = NUM_CHUNKS(((u64)new_start % PAGE_SIZE) + 536 num_kpages = NUM_CHUNKS(((u64)new_start % PAGE_SIZE) +
470 new_size, PAGE_SIZE); 537 new_size, PAGE_SIZE);
471 num_hwpages = NUM_CHUNKS(((u64)new_start % EHCA_PAGESIZE) + 538 num_hwpages = NUM_CHUNKS(((u64)new_start % hw_pgsize) +
472 new_size, EHCA_PAGESIZE); 539 new_size, hw_pgsize);
473 memset(&pginfo, 0, sizeof(pginfo)); 540 memset(&pginfo, 0, sizeof(pginfo));
474 pginfo.type = EHCA_MR_PGI_PHYS; 541 pginfo.type = EHCA_MR_PGI_PHYS;
475 pginfo.num_kpages = num_kpages; 542 pginfo.num_kpages = num_kpages;
543 pginfo.hwpage_size = hw_pgsize;
476 pginfo.num_hwpages = num_hwpages; 544 pginfo.num_hwpages = num_hwpages;
477 pginfo.u.phy.num_phys_buf = num_phys_buf; 545 pginfo.u.phy.num_phys_buf = num_phys_buf;
478 pginfo.u.phy.phys_buf_array = phys_buf_array; 546 pginfo.u.phy.phys_buf_array = phys_buf_array;
479 pginfo.next_hwpage = (((u64)iova_start & ~PAGE_MASK) / 547 pginfo.next_hwpage =
480 EHCA_PAGESIZE); 548 ((u64)iova_start & ~(hw_pgsize - 1)) / hw_pgsize;
481 } 549 }
482 if (mr_rereg_mask & IB_MR_REREG_ACCESS) 550 if (mr_rereg_mask & IB_MR_REREG_ACCESS)
483 new_acl = mr_access_flags; 551 new_acl = mr_access_flags;
@@ -709,6 +777,7 @@ struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
709 int ret; 777 int ret;
710 u32 tmp_lkey, tmp_rkey; 778 u32 tmp_lkey, tmp_rkey;
711 struct ehca_mr_pginfo pginfo; 779 struct ehca_mr_pginfo pginfo;
780 u64 hw_pgsize;
712 781
713 /* check other parameters */ 782 /* check other parameters */
714 if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) && 783 if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
@@ -738,8 +807,8 @@ struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
738 ib_fmr = ERR_PTR(-EINVAL); 807 ib_fmr = ERR_PTR(-EINVAL);
739 goto alloc_fmr_exit0; 808 goto alloc_fmr_exit0;
740 } 809 }
741 if (((1 << fmr_attr->page_shift) != EHCA_PAGESIZE) && 810 hw_pgsize = ehca_get_max_hwpage_size(shca);
742 ((1 << fmr_attr->page_shift) != PAGE_SIZE)) { 811 if ((1 << fmr_attr->page_shift) != hw_pgsize) {
743 ehca_err(pd->device, "unsupported fmr_attr->page_shift=%x", 812 ehca_err(pd->device, "unsupported fmr_attr->page_shift=%x",
744 fmr_attr->page_shift); 813 fmr_attr->page_shift);
745 ib_fmr = ERR_PTR(-EINVAL); 814 ib_fmr = ERR_PTR(-EINVAL);
@@ -755,6 +824,10 @@ struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
755 824
756 /* register MR on HCA */ 825 /* register MR on HCA */
757 memset(&pginfo, 0, sizeof(pginfo)); 826 memset(&pginfo, 0, sizeof(pginfo));
827 /*
828 * pginfo.num_hwpages==0, ie register_rpages() will not be called
829 * but deferred to map_phys_fmr()
830 */
758 ret = ehca_reg_mr(shca, e_fmr, NULL, 831 ret = ehca_reg_mr(shca, e_fmr, NULL,
759 fmr_attr->max_pages * (1 << fmr_attr->page_shift), 832 fmr_attr->max_pages * (1 << fmr_attr->page_shift),
760 mr_access_flags, e_pd, &pginfo, 833 mr_access_flags, e_pd, &pginfo,
@@ -765,6 +838,7 @@ struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
765 } 838 }
766 839
767 /* successful */ 840 /* successful */
841 e_fmr->hwpage_size = hw_pgsize;
768 e_fmr->fmr_page_size = 1 << fmr_attr->page_shift; 842 e_fmr->fmr_page_size = 1 << fmr_attr->page_shift;
769 e_fmr->fmr_max_pages = fmr_attr->max_pages; 843 e_fmr->fmr_max_pages = fmr_attr->max_pages;
770 e_fmr->fmr_max_maps = fmr_attr->max_maps; 844 e_fmr->fmr_max_maps = fmr_attr->max_maps;
@@ -822,10 +896,12 @@ int ehca_map_phys_fmr(struct ib_fmr *fmr,
822 memset(&pginfo, 0, sizeof(pginfo)); 896 memset(&pginfo, 0, sizeof(pginfo));
823 pginfo.type = EHCA_MR_PGI_FMR; 897 pginfo.type = EHCA_MR_PGI_FMR;
824 pginfo.num_kpages = list_len; 898 pginfo.num_kpages = list_len;
825 pginfo.num_hwpages = list_len * (e_fmr->fmr_page_size / EHCA_PAGESIZE); 899 pginfo.hwpage_size = e_fmr->hwpage_size;
900 pginfo.num_hwpages =
901 list_len * e_fmr->fmr_page_size / pginfo.hwpage_size;
826 pginfo.u.fmr.page_list = page_list; 902 pginfo.u.fmr.page_list = page_list;
827 pginfo.next_hwpage = ((iova & (e_fmr->fmr_page_size-1)) / 903 pginfo.next_hwpage =
828 EHCA_PAGESIZE); 904 (iova & (e_fmr->fmr_page_size-1)) / pginfo.hwpage_size;
829 pginfo.u.fmr.fmr_pgsize = e_fmr->fmr_page_size; 905 pginfo.u.fmr.fmr_pgsize = e_fmr->fmr_page_size;
830 906
831 ret = ehca_rereg_mr(shca, e_fmr, (u64 *)iova, 907 ret = ehca_rereg_mr(shca, e_fmr, (u64 *)iova,
@@ -964,7 +1040,7 @@ int ehca_reg_mr(struct ehca_shca *shca,
964 struct ehca_mr_hipzout_parms hipzout; 1040 struct ehca_mr_hipzout_parms hipzout;
965 1041
966 ehca_mrmw_map_acl(acl, &hipz_acl); 1042 ehca_mrmw_map_acl(acl, &hipz_acl);
967 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl); 1043 ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl);
968 if (ehca_use_hp_mr == 1) 1044 if (ehca_use_hp_mr == 1)
969 hipz_acl |= 0x00000001; 1045 hipz_acl |= 0x00000001;
970 1046
@@ -987,6 +1063,7 @@ int ehca_reg_mr(struct ehca_shca *shca,
987 /* successful registration */ 1063 /* successful registration */
988 e_mr->num_kpages = pginfo->num_kpages; 1064 e_mr->num_kpages = pginfo->num_kpages;
989 e_mr->num_hwpages = pginfo->num_hwpages; 1065 e_mr->num_hwpages = pginfo->num_hwpages;
1066 e_mr->hwpage_size = pginfo->hwpage_size;
990 e_mr->start = iova_start; 1067 e_mr->start = iova_start;
991 e_mr->size = size; 1068 e_mr->size = size;
992 e_mr->acl = acl; 1069 e_mr->acl = acl;
@@ -1029,6 +1106,9 @@ int ehca_reg_mr_rpages(struct ehca_shca *shca,
1029 u32 i; 1106 u32 i;
1030 u64 *kpage; 1107 u64 *kpage;
1031 1108
1109 if (!pginfo->num_hwpages) /* in case of fmr */
1110 return 0;
1111
1032 kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL); 1112 kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
1033 if (!kpage) { 1113 if (!kpage) {
1034 ehca_err(&shca->ib_device, "kpage alloc failed"); 1114 ehca_err(&shca->ib_device, "kpage alloc failed");
@@ -1036,7 +1116,7 @@ int ehca_reg_mr_rpages(struct ehca_shca *shca,
1036 goto ehca_reg_mr_rpages_exit0; 1116 goto ehca_reg_mr_rpages_exit0;
1037 } 1117 }
1038 1118
1039 /* max 512 pages per shot */ 1119 /* max MAX_RPAGES ehca mr pages per register call */
1040 for (i = 0; i < NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES); i++) { 1120 for (i = 0; i < NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES); i++) {
1041 1121
1042 if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) { 1122 if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
@@ -1049,8 +1129,8 @@ int ehca_reg_mr_rpages(struct ehca_shca *shca,
1049 ret = ehca_set_pagebuf(pginfo, rnum, kpage); 1129 ret = ehca_set_pagebuf(pginfo, rnum, kpage);
1050 if (ret) { 1130 if (ret) {
1051 ehca_err(&shca->ib_device, "ehca_set_pagebuf " 1131 ehca_err(&shca->ib_device, "ehca_set_pagebuf "
1052 "bad rc, ret=%x rnum=%x kpage=%p", 1132 "bad rc, ret=%x rnum=%x kpage=%p",
1053 ret, rnum, kpage); 1133 ret, rnum, kpage);
1054 goto ehca_reg_mr_rpages_exit1; 1134 goto ehca_reg_mr_rpages_exit1;
1055 } 1135 }
1056 1136
@@ -1065,9 +1145,10 @@ int ehca_reg_mr_rpages(struct ehca_shca *shca,
1065 } else 1145 } else
1066 rpage = *kpage; 1146 rpage = *kpage;
1067 1147
1068 h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, e_mr, 1148 h_ret = hipz_h_register_rpage_mr(
1069 0, /* pagesize 4k */ 1149 shca->ipz_hca_handle, e_mr,
1070 0, rpage, rnum); 1150 ehca_encode_hwpage_size(pginfo->hwpage_size),
1151 0, rpage, rnum);
1071 1152
1072 if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) { 1153 if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
1073 /* 1154 /*
@@ -1131,7 +1212,7 @@ inline int ehca_rereg_mr_rereg1(struct ehca_shca *shca,
1131 struct ehca_mr_hipzout_parms hipzout; 1212 struct ehca_mr_hipzout_parms hipzout;
1132 1213
1133 ehca_mrmw_map_acl(acl, &hipz_acl); 1214 ehca_mrmw_map_acl(acl, &hipz_acl);
1134 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl); 1215 ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl);
1135 1216
1136 kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL); 1217 kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
1137 if (!kpage) { 1218 if (!kpage) {
@@ -1182,6 +1263,7 @@ inline int ehca_rereg_mr_rereg1(struct ehca_shca *shca,
1182 */ 1263 */
1183 e_mr->num_kpages = pginfo->num_kpages; 1264 e_mr->num_kpages = pginfo->num_kpages;
1184 e_mr->num_hwpages = pginfo->num_hwpages; 1265 e_mr->num_hwpages = pginfo->num_hwpages;
1266 e_mr->hwpage_size = pginfo->hwpage_size;
1185 e_mr->start = iova_start; 1267 e_mr->start = iova_start;
1186 e_mr->size = size; 1268 e_mr->size = size;
1187 e_mr->acl = acl; 1269 e_mr->acl = acl;
@@ -1268,13 +1350,14 @@ int ehca_rereg_mr(struct ehca_shca *shca,
1268 1350
1269 /* set some MR values */ 1351 /* set some MR values */
1270 e_mr->flags = save_mr.flags; 1352 e_mr->flags = save_mr.flags;
1353 e_mr->hwpage_size = save_mr.hwpage_size;
1271 e_mr->fmr_page_size = save_mr.fmr_page_size; 1354 e_mr->fmr_page_size = save_mr.fmr_page_size;
1272 e_mr->fmr_max_pages = save_mr.fmr_max_pages; 1355 e_mr->fmr_max_pages = save_mr.fmr_max_pages;
1273 e_mr->fmr_max_maps = save_mr.fmr_max_maps; 1356 e_mr->fmr_max_maps = save_mr.fmr_max_maps;
1274 e_mr->fmr_map_cnt = save_mr.fmr_map_cnt; 1357 e_mr->fmr_map_cnt = save_mr.fmr_map_cnt;
1275 1358
1276 ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl, 1359 ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl,
1277 e_pd, pginfo, lkey, rkey); 1360 e_pd, pginfo, lkey, rkey);
1278 if (ret) { 1361 if (ret) {
1279 u32 offset = (u64)(&e_mr->flags) - (u64)e_mr; 1362 u32 offset = (u64)(&e_mr->flags) - (u64)e_mr;
1280 memcpy(&e_mr->flags, &(save_mr.flags), 1363 memcpy(&e_mr->flags, &(save_mr.flags),
@@ -1355,6 +1438,7 @@ int ehca_unmap_one_fmr(struct ehca_shca *shca,
1355 1438
1356 /* set some MR values */ 1439 /* set some MR values */
1357 e_fmr->flags = save_fmr.flags; 1440 e_fmr->flags = save_fmr.flags;
1441 e_fmr->hwpage_size = save_fmr.hwpage_size;
1358 e_fmr->fmr_page_size = save_fmr.fmr_page_size; 1442 e_fmr->fmr_page_size = save_fmr.fmr_page_size;
1359 e_fmr->fmr_max_pages = save_fmr.fmr_max_pages; 1443 e_fmr->fmr_max_pages = save_fmr.fmr_max_pages;
1360 e_fmr->fmr_max_maps = save_fmr.fmr_max_maps; 1444 e_fmr->fmr_max_maps = save_fmr.fmr_max_maps;
@@ -1363,8 +1447,6 @@ int ehca_unmap_one_fmr(struct ehca_shca *shca,
1363 1447
1364 memset(&pginfo, 0, sizeof(pginfo)); 1448 memset(&pginfo, 0, sizeof(pginfo));
1365 pginfo.type = EHCA_MR_PGI_FMR; 1449 pginfo.type = EHCA_MR_PGI_FMR;
1366 pginfo.num_kpages = 0;
1367 pginfo.num_hwpages = 0;
1368 ret = ehca_reg_mr(shca, e_fmr, NULL, 1450 ret = ehca_reg_mr(shca, e_fmr, NULL,
1369 (e_fmr->fmr_max_pages * e_fmr->fmr_page_size), 1451 (e_fmr->fmr_max_pages * e_fmr->fmr_page_size),
1370 e_fmr->acl, e_pd, &pginfo, &tmp_lkey, 1452 e_fmr->acl, e_pd, &pginfo, &tmp_lkey,
@@ -1373,7 +1455,6 @@ int ehca_unmap_one_fmr(struct ehca_shca *shca,
1373 u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr; 1455 u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr;
1374 memcpy(&e_fmr->flags, &(save_mr.flags), 1456 memcpy(&e_fmr->flags, &(save_mr.flags),
1375 sizeof(struct ehca_mr) - offset); 1457 sizeof(struct ehca_mr) - offset);
1376 goto ehca_unmap_one_fmr_exit0;
1377 } 1458 }
1378 1459
1379ehca_unmap_one_fmr_exit0: 1460ehca_unmap_one_fmr_exit0:
@@ -1401,7 +1482,7 @@ int ehca_reg_smr(struct ehca_shca *shca,
1401 struct ehca_mr_hipzout_parms hipzout; 1482 struct ehca_mr_hipzout_parms hipzout;
1402 1483
1403 ehca_mrmw_map_acl(acl, &hipz_acl); 1484 ehca_mrmw_map_acl(acl, &hipz_acl);
1404 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl); 1485 ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl);
1405 1486
1406 h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr, 1487 h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
1407 (u64)iova_start, hipz_acl, e_pd->fw_pd, 1488 (u64)iova_start, hipz_acl, e_pd->fw_pd,
@@ -1420,6 +1501,7 @@ int ehca_reg_smr(struct ehca_shca *shca,
1420 /* successful registration */ 1501 /* successful registration */
1421 e_newmr->num_kpages = e_origmr->num_kpages; 1502 e_newmr->num_kpages = e_origmr->num_kpages;
1422 e_newmr->num_hwpages = e_origmr->num_hwpages; 1503 e_newmr->num_hwpages = e_origmr->num_hwpages;
1504 e_newmr->hwpage_size = e_origmr->hwpage_size;
1423 e_newmr->start = iova_start; 1505 e_newmr->start = iova_start;
1424 e_newmr->size = e_origmr->size; 1506 e_newmr->size = e_origmr->size;
1425 e_newmr->acl = acl; 1507 e_newmr->acl = acl;
@@ -1452,6 +1534,7 @@ int ehca_reg_internal_maxmr(
1452 struct ib_phys_buf ib_pbuf; 1534 struct ib_phys_buf ib_pbuf;
1453 u32 num_kpages; 1535 u32 num_kpages;
1454 u32 num_hwpages; 1536 u32 num_hwpages;
1537 u64 hw_pgsize;
1455 1538
1456 e_mr = ehca_mr_new(); 1539 e_mr = ehca_mr_new();
1457 if (!e_mr) { 1540 if (!e_mr) {
@@ -1468,13 +1551,15 @@ int ehca_reg_internal_maxmr(
1468 ib_pbuf.size = size_maxmr; 1551 ib_pbuf.size = size_maxmr;
1469 num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size_maxmr, 1552 num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size_maxmr,
1470 PAGE_SIZE); 1553 PAGE_SIZE);
1471 num_hwpages = NUM_CHUNKS(((u64)iova_start % EHCA_PAGESIZE) + size_maxmr, 1554 hw_pgsize = ehca_get_max_hwpage_size(shca);
1472 EHCA_PAGESIZE); 1555 num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size_maxmr,
1556 hw_pgsize);
1473 1557
1474 memset(&pginfo, 0, sizeof(pginfo)); 1558 memset(&pginfo, 0, sizeof(pginfo));
1475 pginfo.type = EHCA_MR_PGI_PHYS; 1559 pginfo.type = EHCA_MR_PGI_PHYS;
1476 pginfo.num_kpages = num_kpages; 1560 pginfo.num_kpages = num_kpages;
1477 pginfo.num_hwpages = num_hwpages; 1561 pginfo.num_hwpages = num_hwpages;
1562 pginfo.hwpage_size = hw_pgsize;
1478 pginfo.u.phy.num_phys_buf = 1; 1563 pginfo.u.phy.num_phys_buf = 1;
1479 pginfo.u.phy.phys_buf_array = &ib_pbuf; 1564 pginfo.u.phy.phys_buf_array = &ib_pbuf;
1480 1565
@@ -1523,7 +1608,7 @@ int ehca_reg_maxmr(struct ehca_shca *shca,
1523 struct ehca_mr_hipzout_parms hipzout; 1608 struct ehca_mr_hipzout_parms hipzout;
1524 1609
1525 ehca_mrmw_map_acl(acl, &hipz_acl); 1610 ehca_mrmw_map_acl(acl, &hipz_acl);
1526 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl); 1611 ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl);
1527 1612
1528 h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr, 1613 h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
1529 (u64)iova_start, hipz_acl, e_pd->fw_pd, 1614 (u64)iova_start, hipz_acl, e_pd->fw_pd,
@@ -1539,6 +1624,7 @@ int ehca_reg_maxmr(struct ehca_shca *shca,
1539 /* successful registration */ 1624 /* successful registration */
1540 e_newmr->num_kpages = e_origmr->num_kpages; 1625 e_newmr->num_kpages = e_origmr->num_kpages;
1541 e_newmr->num_hwpages = e_origmr->num_hwpages; 1626 e_newmr->num_hwpages = e_origmr->num_hwpages;
1627 e_newmr->hwpage_size = e_origmr->hwpage_size;
1542 e_newmr->start = iova_start; 1628 e_newmr->start = iova_start;
1543 e_newmr->size = e_origmr->size; 1629 e_newmr->size = e_origmr->size;
1544 e_newmr->acl = acl; 1630 e_newmr->acl = acl;
@@ -1684,6 +1770,7 @@ static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,
1684 u64 pgaddr; 1770 u64 pgaddr;
1685 u32 i = 0; 1771 u32 i = 0;
1686 u32 j = 0; 1772 u32 j = 0;
1773 int hwpages_per_kpage = PAGE_SIZE / pginfo->hwpage_size;
1687 1774
1688 /* loop over desired chunk entries */ 1775 /* loop over desired chunk entries */
1689 chunk = pginfo->u.usr.next_chunk; 1776 chunk = pginfo->u.usr.next_chunk;
@@ -1695,7 +1782,7 @@ static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,
1695 << PAGE_SHIFT ; 1782 << PAGE_SHIFT ;
1696 *kpage = phys_to_abs(pgaddr + 1783 *kpage = phys_to_abs(pgaddr +
1697 (pginfo->next_hwpage * 1784 (pginfo->next_hwpage *
1698 EHCA_PAGESIZE)); 1785 pginfo->hwpage_size));
1699 if ( !(*kpage) ) { 1786 if ( !(*kpage) ) {
1700 ehca_gen_err("pgaddr=%lx " 1787 ehca_gen_err("pgaddr=%lx "
1701 "chunk->page_list[i]=%lx " 1788 "chunk->page_list[i]=%lx "
@@ -1708,8 +1795,7 @@ static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,
1708 (pginfo->hwpage_cnt)++; 1795 (pginfo->hwpage_cnt)++;
1709 (pginfo->next_hwpage)++; 1796 (pginfo->next_hwpage)++;
1710 kpage++; 1797 kpage++;
1711 if (pginfo->next_hwpage % 1798 if (pginfo->next_hwpage % hwpages_per_kpage == 0) {
1712 (PAGE_SIZE / EHCA_PAGESIZE) == 0) {
1713 (pginfo->kpage_cnt)++; 1799 (pginfo->kpage_cnt)++;
1714 (pginfo->u.usr.next_nmap)++; 1800 (pginfo->u.usr.next_nmap)++;
1715 pginfo->next_hwpage = 0; 1801 pginfo->next_hwpage = 0;
@@ -1738,6 +1824,143 @@ static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,
1738 return ret; 1824 return ret;
1739} 1825}
1740 1826
1827/*
1828 * check given pages for contiguous layout
1829 * last page addr is returned in prev_pgaddr for further check
1830 */
1831static int ehca_check_kpages_per_ate(struct scatterlist *page_list,
1832 int start_idx, int end_idx,
1833 u64 *prev_pgaddr)
1834{
1835 int t;
1836 for (t = start_idx; t <= end_idx; t++) {
1837 u64 pgaddr = page_to_pfn(page_list[t].page) << PAGE_SHIFT;
1838 ehca_gen_dbg("chunk_page=%lx value=%016lx", pgaddr,
1839 *(u64 *)abs_to_virt(phys_to_abs(pgaddr)));
1840 if (pgaddr - PAGE_SIZE != *prev_pgaddr) {
1841 ehca_gen_err("uncontiguous page found pgaddr=%lx "
1842 "prev_pgaddr=%lx page_list_i=%x",
1843 pgaddr, *prev_pgaddr, t);
1844 return -EINVAL;
1845 }
1846 *prev_pgaddr = pgaddr;
1847 }
1848 return 0;
1849}
1850
1851/* PAGE_SIZE < pginfo->hwpage_size */
1852static int ehca_set_pagebuf_user2(struct ehca_mr_pginfo *pginfo,
1853 u32 number,
1854 u64 *kpage)
1855{
1856 int ret = 0;
1857 struct ib_umem_chunk *prev_chunk;
1858 struct ib_umem_chunk *chunk;
1859 u64 pgaddr, prev_pgaddr;
1860 u32 i = 0;
1861 u32 j = 0;
1862 int kpages_per_hwpage = pginfo->hwpage_size / PAGE_SIZE;
1863 int nr_kpages = kpages_per_hwpage;
1864
1865 /* loop over desired chunk entries */
1866 chunk = pginfo->u.usr.next_chunk;
1867 prev_chunk = pginfo->u.usr.next_chunk;
1868 list_for_each_entry_continue(
1869 chunk, (&(pginfo->u.usr.region->chunk_list)), list) {
1870 for (i = pginfo->u.usr.next_nmap; i < chunk->nmap; ) {
1871 if (nr_kpages == kpages_per_hwpage) {
1872 pgaddr = ( page_to_pfn(chunk->page_list[i].page)
1873 << PAGE_SHIFT );
1874 *kpage = phys_to_abs(pgaddr);
1875 if ( !(*kpage) ) {
1876 ehca_gen_err("pgaddr=%lx i=%x",
1877 pgaddr, i);
1878 ret = -EFAULT;
1879 return ret;
1880 }
1881 /*
1882 * The first page in a hwpage must be aligned;
1883 * the first MR page is exempt from this rule.
1884 */
1885 if (pgaddr & (pginfo->hwpage_size - 1)) {
1886 if (pginfo->hwpage_cnt) {
1887 ehca_gen_err(
1888 "invalid alignment "
1889 "pgaddr=%lx i=%x "
1890 "mr_pgsize=%lx",
1891 pgaddr, i,
1892 pginfo->hwpage_size);
1893 ret = -EFAULT;
1894 return ret;
1895 }
1896 /* first MR page */
1897 pginfo->kpage_cnt =
1898 (pgaddr &
1899 (pginfo->hwpage_size - 1)) >>
1900 PAGE_SHIFT;
1901 nr_kpages -= pginfo->kpage_cnt;
1902 *kpage = phys_to_abs(
1903 pgaddr &
1904 ~(pginfo->hwpage_size - 1));
1905 }
1906 ehca_gen_dbg("kpage=%lx chunk_page=%lx "
1907 "value=%016lx", *kpage, pgaddr,
1908 *(u64 *)abs_to_virt(
1909 phys_to_abs(pgaddr)));
1910 prev_pgaddr = pgaddr;
1911 i++;
1912 pginfo->kpage_cnt++;
1913 pginfo->u.usr.next_nmap++;
1914 nr_kpages--;
1915 if (!nr_kpages)
1916 goto next_kpage;
1917 continue;
1918 }
1919 if (i + nr_kpages > chunk->nmap) {
1920 ret = ehca_check_kpages_per_ate(
1921 chunk->page_list, i,
1922 chunk->nmap - 1, &prev_pgaddr);
1923 if (ret) return ret;
1924 pginfo->kpage_cnt += chunk->nmap - i;
1925 pginfo->u.usr.next_nmap += chunk->nmap - i;
1926 nr_kpages -= chunk->nmap - i;
1927 break;
1928 }
1929
1930 ret = ehca_check_kpages_per_ate(chunk->page_list, i,
1931 i + nr_kpages - 1,
1932 &prev_pgaddr);
1933 if (ret) return ret;
1934 i += nr_kpages;
1935 pginfo->kpage_cnt += nr_kpages;
1936 pginfo->u.usr.next_nmap += nr_kpages;
1937next_kpage:
1938 nr_kpages = kpages_per_hwpage;
1939 (pginfo->hwpage_cnt)++;
1940 kpage++;
1941 j++;
1942 if (j >= number) break;
1943 }
1944 if ((pginfo->u.usr.next_nmap >= chunk->nmap) &&
1945 (j >= number)) {
1946 pginfo->u.usr.next_nmap = 0;
1947 prev_chunk = chunk;
1948 break;
1949 } else if (pginfo->u.usr.next_nmap >= chunk->nmap) {
1950 pginfo->u.usr.next_nmap = 0;
1951 prev_chunk = chunk;
1952 } else if (j >= number)
1953 break;
1954 else
1955 prev_chunk = chunk;
1956 }
1957 pginfo->u.usr.next_chunk =
1958 list_prepare_entry(prev_chunk,
1959 (&(pginfo->u.usr.region->chunk_list)),
1960 list);
1961 return ret;
1962}
1963
1741int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo, 1964int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,
1742 u32 number, 1965 u32 number,
1743 u64 *kpage) 1966 u64 *kpage)
@@ -1750,9 +1973,10 @@ int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,
1750 /* loop over desired phys_buf_array entries */ 1973 /* loop over desired phys_buf_array entries */
1751 while (i < number) { 1974 while (i < number) {
1752 pbuf = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf; 1975 pbuf = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf;
1753 num_hw = NUM_CHUNKS((pbuf->addr % EHCA_PAGESIZE) + 1976 num_hw = NUM_CHUNKS((pbuf->addr % pginfo->hwpage_size) +
1754 pbuf->size, EHCA_PAGESIZE); 1977 pbuf->size, pginfo->hwpage_size);
1755 offs_hw = (pbuf->addr & ~PAGE_MASK) / EHCA_PAGESIZE; 1978 offs_hw = (pbuf->addr & ~(pginfo->hwpage_size - 1)) /
1979 pginfo->hwpage_size;
1756 while (pginfo->next_hwpage < offs_hw + num_hw) { 1980 while (pginfo->next_hwpage < offs_hw + num_hw) {
1757 /* sanity check */ 1981 /* sanity check */
1758 if ((pginfo->kpage_cnt >= pginfo->num_kpages) || 1982 if ((pginfo->kpage_cnt >= pginfo->num_kpages) ||
@@ -1768,21 +1992,23 @@ int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,
1768 return -EFAULT; 1992 return -EFAULT;
1769 } 1993 }
1770 *kpage = phys_to_abs( 1994 *kpage = phys_to_abs(
1771 (pbuf->addr & EHCA_PAGEMASK) 1995 (pbuf->addr & ~(pginfo->hwpage_size - 1)) +
1772 + (pginfo->next_hwpage * EHCA_PAGESIZE)); 1996 (pginfo->next_hwpage * pginfo->hwpage_size));
1773 if ( !(*kpage) && pbuf->addr ) { 1997 if ( !(*kpage) && pbuf->addr ) {
1774 ehca_gen_err("pbuf->addr=%lx " 1998 ehca_gen_err("pbuf->addr=%lx pbuf->size=%lx "
1775 "pbuf->size=%lx "
1776 "next_hwpage=%lx", pbuf->addr, 1999 "next_hwpage=%lx", pbuf->addr,
1777 pbuf->size, 2000 pbuf->size, pginfo->next_hwpage);
1778 pginfo->next_hwpage);
1779 return -EFAULT; 2001 return -EFAULT;
1780 } 2002 }
1781 (pginfo->hwpage_cnt)++; 2003 (pginfo->hwpage_cnt)++;
1782 (pginfo->next_hwpage)++; 2004 (pginfo->next_hwpage)++;
1783 if (pginfo->next_hwpage % 2005 if (PAGE_SIZE >= pginfo->hwpage_size) {
1784 (PAGE_SIZE / EHCA_PAGESIZE) == 0) 2006 if (pginfo->next_hwpage %
1785 (pginfo->kpage_cnt)++; 2007 (PAGE_SIZE / pginfo->hwpage_size) == 0)
2008 (pginfo->kpage_cnt)++;
2009 } else
2010 pginfo->kpage_cnt += pginfo->hwpage_size /
2011 PAGE_SIZE;
1786 kpage++; 2012 kpage++;
1787 i++; 2013 i++;
1788 if (i >= number) break; 2014 if (i >= number) break;
@@ -1806,8 +2032,8 @@ int ehca_set_pagebuf_fmr(struct ehca_mr_pginfo *pginfo,
1806 /* loop over desired page_list entries */ 2032 /* loop over desired page_list entries */
1807 fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem; 2033 fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem;
1808 for (i = 0; i < number; i++) { 2034 for (i = 0; i < number; i++) {
1809 *kpage = phys_to_abs((*fmrlist & EHCA_PAGEMASK) + 2035 *kpage = phys_to_abs((*fmrlist & ~(pginfo->hwpage_size - 1)) +
1810 pginfo->next_hwpage * EHCA_PAGESIZE); 2036 pginfo->next_hwpage * pginfo->hwpage_size);
1811 if ( !(*kpage) ) { 2037 if ( !(*kpage) ) {
1812 ehca_gen_err("*fmrlist=%lx fmrlist=%p " 2038 ehca_gen_err("*fmrlist=%lx fmrlist=%p "
1813 "next_listelem=%lx next_hwpage=%lx", 2039 "next_listelem=%lx next_hwpage=%lx",
@@ -1817,15 +2043,38 @@ int ehca_set_pagebuf_fmr(struct ehca_mr_pginfo *pginfo,
1817 return -EFAULT; 2043 return -EFAULT;
1818 } 2044 }
1819 (pginfo->hwpage_cnt)++; 2045 (pginfo->hwpage_cnt)++;
1820 (pginfo->next_hwpage)++; 2046 if (pginfo->u.fmr.fmr_pgsize >= pginfo->hwpage_size) {
1821 kpage++; 2047 if (pginfo->next_hwpage %
1822 if (pginfo->next_hwpage % 2048 (pginfo->u.fmr.fmr_pgsize /
1823 (pginfo->u.fmr.fmr_pgsize / EHCA_PAGESIZE) == 0) { 2049 pginfo->hwpage_size) == 0) {
1824 (pginfo->kpage_cnt)++; 2050 (pginfo->kpage_cnt)++;
1825 (pginfo->u.fmr.next_listelem)++; 2051 (pginfo->u.fmr.next_listelem)++;
1826 fmrlist++; 2052 fmrlist++;
1827 pginfo->next_hwpage = 0; 2053 pginfo->next_hwpage = 0;
2054 } else
2055 (pginfo->next_hwpage)++;
2056 } else {
2057 unsigned int cnt_per_hwpage = pginfo->hwpage_size /
2058 pginfo->u.fmr.fmr_pgsize;
2059 unsigned int j;
2060 u64 prev = *kpage;
2061 /* check if adrs are contiguous */
2062 for (j = 1; j < cnt_per_hwpage; j++) {
2063 u64 p = phys_to_abs(fmrlist[j] &
2064 ~(pginfo->hwpage_size - 1));
2065 if (prev + pginfo->u.fmr.fmr_pgsize != p) {
2066 ehca_gen_err("uncontiguous fmr pages "
2067 "found prev=%lx p=%lx "
2068 "idx=%x", prev, p, i + j);
2069 return -EINVAL;
2070 }
2071 prev = p;
2072 }
2073 pginfo->kpage_cnt += cnt_per_hwpage;
2074 pginfo->u.fmr.next_listelem += cnt_per_hwpage;
2075 fmrlist += cnt_per_hwpage;
1828 } 2076 }
2077 kpage++;
1829 } 2078 }
1830 return ret; 2079 return ret;
1831} 2080}
@@ -1842,7 +2091,9 @@ int ehca_set_pagebuf(struct ehca_mr_pginfo *pginfo,
1842 ret = ehca_set_pagebuf_phys(pginfo, number, kpage); 2091 ret = ehca_set_pagebuf_phys(pginfo, number, kpage);
1843 break; 2092 break;
1844 case EHCA_MR_PGI_USER: 2093 case EHCA_MR_PGI_USER:
1845 ret = ehca_set_pagebuf_user1(pginfo, number, kpage); 2094 ret = PAGE_SIZE >= pginfo->hwpage_size ?
2095 ehca_set_pagebuf_user1(pginfo, number, kpage) :
2096 ehca_set_pagebuf_user2(pginfo, number, kpage);
1846 break; 2097 break;
1847 case EHCA_MR_PGI_FMR: 2098 case EHCA_MR_PGI_FMR:
1848 ret = ehca_set_pagebuf_fmr(pginfo, number, kpage); 2099 ret = ehca_set_pagebuf_fmr(pginfo, number, kpage);
@@ -1895,9 +2146,9 @@ void ehca_mrmw_map_acl(int ib_acl,
1895/*----------------------------------------------------------------------*/ 2146/*----------------------------------------------------------------------*/
1896 2147
1897/* sets page size in hipz access control for MR/MW. */ 2148/* sets page size in hipz access control for MR/MW. */
1898void ehca_mrmw_set_pgsize_hipz_acl(u32 *hipz_acl) /*INOUT*/ 2149void ehca_mrmw_set_pgsize_hipz_acl(u32 pgsize, u32 *hipz_acl) /*INOUT*/
1899{ 2150{
1900 return; /* HCA supports only 4k */ 2151 *hipz_acl |= (ehca_encode_hwpage_size(pgsize) << 24);
1901} /* end ehca_mrmw_set_pgsize_hipz_acl() */ 2152} /* end ehca_mrmw_set_pgsize_hipz_acl() */
1902 2153
1903/*----------------------------------------------------------------------*/ 2154/*----------------------------------------------------------------------*/
diff --git a/drivers/infiniband/hw/ehca/ehca_mrmw.h b/drivers/infiniband/hw/ehca/ehca_mrmw.h
index 24f13fe3708b..bc8f4e31c123 100644
--- a/drivers/infiniband/hw/ehca/ehca_mrmw.h
+++ b/drivers/infiniband/hw/ehca/ehca_mrmw.h
@@ -111,7 +111,7 @@ int ehca_mr_is_maxmr(u64 size,
111void ehca_mrmw_map_acl(int ib_acl, 111void ehca_mrmw_map_acl(int ib_acl,
112 u32 *hipz_acl); 112 u32 *hipz_acl);
113 113
114void ehca_mrmw_set_pgsize_hipz_acl(u32 *hipz_acl); 114void ehca_mrmw_set_pgsize_hipz_acl(u32 pgsize, u32 *hipz_acl);
115 115
116void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl, 116void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
117 int *ib_acl); 117 int *ib_acl);
diff --git a/drivers/infiniband/hw/ehca/ehca_pd.c b/drivers/infiniband/hw/ehca/ehca_pd.c
index c85312ad292b..3dafd7ff36cd 100644
--- a/drivers/infiniband/hw/ehca/ehca_pd.c
+++ b/drivers/infiniband/hw/ehca/ehca_pd.c
@@ -49,6 +49,7 @@ struct ib_pd *ehca_alloc_pd(struct ib_device *device,
49 struct ib_ucontext *context, struct ib_udata *udata) 49 struct ib_ucontext *context, struct ib_udata *udata)
50{ 50{
51 struct ehca_pd *pd; 51 struct ehca_pd *pd;
52 int i;
52 53
53 pd = kmem_cache_zalloc(pd_cache, GFP_KERNEL); 54 pd = kmem_cache_zalloc(pd_cache, GFP_KERNEL);
54 if (!pd) { 55 if (!pd) {
@@ -58,6 +59,11 @@ struct ib_pd *ehca_alloc_pd(struct ib_device *device,
58 } 59 }
59 60
60 pd->ownpid = current->tgid; 61 pd->ownpid = current->tgid;
62 for (i = 0; i < 2; i++) {
63 INIT_LIST_HEAD(&pd->free[i]);
64 INIT_LIST_HEAD(&pd->full[i]);
65 }
66 mutex_init(&pd->lock);
61 67
62 /* 68 /*
63 * Kernel PD: when device = -1, 0 69 * Kernel PD: when device = -1, 0
@@ -81,6 +87,9 @@ int ehca_dealloc_pd(struct ib_pd *pd)
81{ 87{
82 u32 cur_pid = current->tgid; 88 u32 cur_pid = current->tgid;
83 struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd); 89 struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd);
90 int i, leftovers = 0;
91 extern struct kmem_cache *small_qp_cache;
92 struct ipz_small_queue_page *page, *tmp;
84 93
85 if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context && 94 if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
86 my_pd->ownpid != cur_pid) { 95 my_pd->ownpid != cur_pid) {
@@ -89,8 +98,20 @@ int ehca_dealloc_pd(struct ib_pd *pd)
89 return -EINVAL; 98 return -EINVAL;
90 } 99 }
91 100
92 kmem_cache_free(pd_cache, 101 for (i = 0; i < 2; i++) {
93 container_of(pd, struct ehca_pd, ib_pd)); 102 list_splice(&my_pd->full[i], &my_pd->free[i]);
103 list_for_each_entry_safe(page, tmp, &my_pd->free[i], list) {
104 leftovers = 1;
105 free_page(page->page);
106 kmem_cache_free(small_qp_cache, page);
107 }
108 }
109
110 if (leftovers)
111 ehca_warn(pd->device,
112 "Some small queue pages were not freed");
113
114 kmem_cache_free(pd_cache, my_pd);
94 115
95 return 0; 116 return 0;
96} 117}
diff --git a/drivers/infiniband/hw/ehca/ehca_qp.c b/drivers/infiniband/hw/ehca/ehca_qp.c
index a3146e696c5d..b178cba96345 100644
--- a/drivers/infiniband/hw/ehca/ehca_qp.c
+++ b/drivers/infiniband/hw/ehca/ehca_qp.c
@@ -275,34 +275,39 @@ static inline void queue2resp(struct ipzu_queue_resp *resp,
275 resp->toggle_state = queue->toggle_state; 275 resp->toggle_state = queue->toggle_state;
276} 276}
277 277
278static inline int ll_qp_msg_size(int nr_sge)
279{
280 return 128 << nr_sge;
281}
282
283/* 278/*
284 * init_qp_queue initializes/constructs r/squeue and registers queue pages. 279 * init_qp_queue initializes/constructs r/squeue and registers queue pages.
285 */ 280 */
286static inline int init_qp_queue(struct ehca_shca *shca, 281static inline int init_qp_queue(struct ehca_shca *shca,
282 struct ehca_pd *pd,
287 struct ehca_qp *my_qp, 283 struct ehca_qp *my_qp,
288 struct ipz_queue *queue, 284 struct ipz_queue *queue,
289 int q_type, 285 int q_type,
290 u64 expected_hret, 286 u64 expected_hret,
291 int nr_q_pages, 287 struct ehca_alloc_queue_parms *parms,
292 int wqe_size, 288 int wqe_size)
293 int nr_sges)
294{ 289{
295 int ret, cnt, ipz_rc; 290 int ret, cnt, ipz_rc, nr_q_pages;
296 void *vpage; 291 void *vpage;
297 u64 rpage, h_ret; 292 u64 rpage, h_ret;
298 struct ib_device *ib_dev = &shca->ib_device; 293 struct ib_device *ib_dev = &shca->ib_device;
299 struct ipz_adapter_handle ipz_hca_handle = shca->ipz_hca_handle; 294 struct ipz_adapter_handle ipz_hca_handle = shca->ipz_hca_handle;
300 295
301 if (!nr_q_pages) 296 if (!parms->queue_size)
302 return 0; 297 return 0;
303 298
304 ipz_rc = ipz_queue_ctor(queue, nr_q_pages, EHCA_PAGESIZE, 299 if (parms->is_small) {
305 wqe_size, nr_sges); 300 nr_q_pages = 1;
301 ipz_rc = ipz_queue_ctor(pd, queue, nr_q_pages,
302 128 << parms->page_size,
303 wqe_size, parms->act_nr_sges, 1);
304 } else {
305 nr_q_pages = parms->queue_size;
306 ipz_rc = ipz_queue_ctor(pd, queue, nr_q_pages,
307 EHCA_PAGESIZE, wqe_size,
308 parms->act_nr_sges, 0);
309 }
310
306 if (!ipz_rc) { 311 if (!ipz_rc) {
307 ehca_err(ib_dev, "Cannot allocate page for queue. ipz_rc=%x", 312 ehca_err(ib_dev, "Cannot allocate page for queue. ipz_rc=%x",
308 ipz_rc); 313 ipz_rc);
@@ -323,7 +328,7 @@ static inline int init_qp_queue(struct ehca_shca *shca,
323 h_ret = hipz_h_register_rpage_qp(ipz_hca_handle, 328 h_ret = hipz_h_register_rpage_qp(ipz_hca_handle,
324 my_qp->ipz_qp_handle, 329 my_qp->ipz_qp_handle,
325 NULL, 0, q_type, 330 NULL, 0, q_type,
326 rpage, 1, 331 rpage, parms->is_small ? 0 : 1,
327 my_qp->galpas.kernel); 332 my_qp->galpas.kernel);
328 if (cnt == (nr_q_pages - 1)) { /* last page! */ 333 if (cnt == (nr_q_pages - 1)) { /* last page! */
329 if (h_ret != expected_hret) { 334 if (h_ret != expected_hret) {
@@ -354,19 +359,55 @@ static inline int init_qp_queue(struct ehca_shca *shca,
354 return 0; 359 return 0;
355 360
356init_qp_queue1: 361init_qp_queue1:
357 ipz_queue_dtor(queue); 362 ipz_queue_dtor(pd, queue);
358 return ret; 363 return ret;
359} 364}
360 365
366static inline int ehca_calc_wqe_size(int act_nr_sge, int is_llqp)
367{
368 if (is_llqp)
369 return 128 << act_nr_sge;
370 else
371 return offsetof(struct ehca_wqe,
372 u.nud.sg_list[act_nr_sge]);
373}
374
375static void ehca_determine_small_queue(struct ehca_alloc_queue_parms *queue,
376 int req_nr_sge, int is_llqp)
377{
378 u32 wqe_size, q_size;
379 int act_nr_sge = req_nr_sge;
380
381 if (!is_llqp)
382 /* round up #SGEs so WQE size is a power of 2 */
383 for (act_nr_sge = 4; act_nr_sge <= 252;
384 act_nr_sge = 4 + 2 * act_nr_sge)
385 if (act_nr_sge >= req_nr_sge)
386 break;
387
388 wqe_size = ehca_calc_wqe_size(act_nr_sge, is_llqp);
389 q_size = wqe_size * (queue->max_wr + 1);
390
391 if (q_size <= 512)
392 queue->page_size = 2;
393 else if (q_size <= 1024)
394 queue->page_size = 3;
395 else
396 queue->page_size = 0;
397
398 queue->is_small = (queue->page_size != 0);
399}
400
361/* 401/*
362 * Create an ib_qp struct that is either a QP or an SRQ, depending on 402 * Create an ib_qp struct that is either a QP or an SRQ, depending on
363 * the value of the is_srq parameter. If init_attr and srq_init_attr share 403 * the value of the is_srq parameter. If init_attr and srq_init_attr share
364 * fields, the field out of init_attr is used. 404 * fields, the field out of init_attr is used.
365 */ 405 */
366struct ehca_qp *internal_create_qp(struct ib_pd *pd, 406static struct ehca_qp *internal_create_qp(
367 struct ib_qp_init_attr *init_attr, 407 struct ib_pd *pd,
368 struct ib_srq_init_attr *srq_init_attr, 408 struct ib_qp_init_attr *init_attr,
369 struct ib_udata *udata, int is_srq) 409 struct ib_srq_init_attr *srq_init_attr,
410 struct ib_udata *udata, int is_srq)
370{ 411{
371 struct ehca_qp *my_qp; 412 struct ehca_qp *my_qp;
372 struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd); 413 struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd);
@@ -552,10 +593,20 @@ struct ehca_qp *internal_create_qp(struct ib_pd *pd,
552 if (my_qp->recv_cq) 593 if (my_qp->recv_cq)
553 parms.recv_cq_handle = my_qp->recv_cq->ipz_cq_handle; 594 parms.recv_cq_handle = my_qp->recv_cq->ipz_cq_handle;
554 595
555 parms.max_send_wr = init_attr->cap.max_send_wr; 596 parms.squeue.max_wr = init_attr->cap.max_send_wr;
556 parms.max_recv_wr = init_attr->cap.max_recv_wr; 597 parms.rqueue.max_wr = init_attr->cap.max_recv_wr;
557 parms.max_send_sge = max_send_sge; 598 parms.squeue.max_sge = max_send_sge;
558 parms.max_recv_sge = max_recv_sge; 599 parms.rqueue.max_sge = max_recv_sge;
600
601 if (EHCA_BMASK_GET(HCA_CAP_MINI_QP, shca->hca_cap)
602 && !(context && udata)) { /* no small QP support in userspace ATM */
603 ehca_determine_small_queue(
604 &parms.squeue, max_send_sge, is_llqp);
605 ehca_determine_small_queue(
606 &parms.rqueue, max_recv_sge, is_llqp);
607 parms.qp_storage =
608 (parms.squeue.is_small || parms.rqueue.is_small);
609 }
559 610
560 h_ret = hipz_h_alloc_resource_qp(shca->ipz_hca_handle, &parms); 611 h_ret = hipz_h_alloc_resource_qp(shca->ipz_hca_handle, &parms);
561 if (h_ret != H_SUCCESS) { 612 if (h_ret != H_SUCCESS) {
@@ -569,50 +620,33 @@ struct ehca_qp *internal_create_qp(struct ib_pd *pd,
569 my_qp->ipz_qp_handle = parms.qp_handle; 620 my_qp->ipz_qp_handle = parms.qp_handle;
570 my_qp->galpas = parms.galpas; 621 my_qp->galpas = parms.galpas;
571 622
623 swqe_size = ehca_calc_wqe_size(parms.squeue.act_nr_sges, is_llqp);
624 rwqe_size = ehca_calc_wqe_size(parms.rqueue.act_nr_sges, is_llqp);
625
572 switch (qp_type) { 626 switch (qp_type) {
573 case IB_QPT_RC: 627 case IB_QPT_RC:
574 if (!is_llqp) { 628 if (is_llqp) {
575 swqe_size = offsetof(struct ehca_wqe, u.nud.sg_list[ 629 parms.squeue.act_nr_sges = 1;
576 (parms.act_nr_send_sges)]); 630 parms.rqueue.act_nr_sges = 1;
577 rwqe_size = offsetof(struct ehca_wqe, u.nud.sg_list[
578 (parms.act_nr_recv_sges)]);
579 } else { /* for LLQP we need to use msg size, not wqe size */
580 swqe_size = ll_qp_msg_size(max_send_sge);
581 rwqe_size = ll_qp_msg_size(max_recv_sge);
582 parms.act_nr_send_sges = 1;
583 parms.act_nr_recv_sges = 1;
584 } 631 }
585 break; 632 break;
586 case IB_QPT_UC:
587 swqe_size = offsetof(struct ehca_wqe,
588 u.nud.sg_list[parms.act_nr_send_sges]);
589 rwqe_size = offsetof(struct ehca_wqe,
590 u.nud.sg_list[parms.act_nr_recv_sges]);
591 break;
592
593 case IB_QPT_UD: 633 case IB_QPT_UD:
594 case IB_QPT_GSI: 634 case IB_QPT_GSI:
595 case IB_QPT_SMI: 635 case IB_QPT_SMI:
636 /* UD circumvention */
596 if (is_llqp) { 637 if (is_llqp) {
597 swqe_size = ll_qp_msg_size(parms.act_nr_send_sges); 638 parms.squeue.act_nr_sges = 1;
598 rwqe_size = ll_qp_msg_size(parms.act_nr_recv_sges); 639 parms.rqueue.act_nr_sges = 1;
599 parms.act_nr_send_sges = 1;
600 parms.act_nr_recv_sges = 1;
601 } else { 640 } else {
602 /* UD circumvention */ 641 parms.squeue.act_nr_sges -= 2;
603 parms.act_nr_send_sges -= 2; 642 parms.rqueue.act_nr_sges -= 2;
604 parms.act_nr_recv_sges -= 2;
605 swqe_size = offsetof(struct ehca_wqe, u.ud_av.sg_list[
606 parms.act_nr_send_sges]);
607 rwqe_size = offsetof(struct ehca_wqe, u.ud_av.sg_list[
608 parms.act_nr_recv_sges]);
609 } 643 }
610 644
611 if (IB_QPT_GSI == qp_type || IB_QPT_SMI == qp_type) { 645 if (IB_QPT_GSI == qp_type || IB_QPT_SMI == qp_type) {
612 parms.act_nr_send_wqes = init_attr->cap.max_send_wr; 646 parms.squeue.act_nr_wqes = init_attr->cap.max_send_wr;
613 parms.act_nr_recv_wqes = init_attr->cap.max_recv_wr; 647 parms.rqueue.act_nr_wqes = init_attr->cap.max_recv_wr;
614 parms.act_nr_send_sges = init_attr->cap.max_send_sge; 648 parms.squeue.act_nr_sges = init_attr->cap.max_send_sge;
615 parms.act_nr_recv_sges = init_attr->cap.max_recv_sge; 649 parms.rqueue.act_nr_sges = init_attr->cap.max_recv_sge;
616 ib_qp_num = (qp_type == IB_QPT_SMI) ? 0 : 1; 650 ib_qp_num = (qp_type == IB_QPT_SMI) ? 0 : 1;
617 } 651 }
618 652
@@ -625,10 +659,9 @@ struct ehca_qp *internal_create_qp(struct ib_pd *pd,
625 /* initialize r/squeue and register queue pages */ 659 /* initialize r/squeue and register queue pages */
626 if (HAS_SQ(my_qp)) { 660 if (HAS_SQ(my_qp)) {
627 ret = init_qp_queue( 661 ret = init_qp_queue(
628 shca, my_qp, &my_qp->ipz_squeue, 0, 662 shca, my_pd, my_qp, &my_qp->ipz_squeue, 0,
629 HAS_RQ(my_qp) ? H_PAGE_REGISTERED : H_SUCCESS, 663 HAS_RQ(my_qp) ? H_PAGE_REGISTERED : H_SUCCESS,
630 parms.nr_sq_pages, swqe_size, 664 &parms.squeue, swqe_size);
631 parms.act_nr_send_sges);
632 if (ret) { 665 if (ret) {
633 ehca_err(pd->device, "Couldn't initialize squeue " 666 ehca_err(pd->device, "Couldn't initialize squeue "
634 "and pages ret=%x", ret); 667 "and pages ret=%x", ret);
@@ -638,9 +671,8 @@ struct ehca_qp *internal_create_qp(struct ib_pd *pd,
638 671
639 if (HAS_RQ(my_qp)) { 672 if (HAS_RQ(my_qp)) {
640 ret = init_qp_queue( 673 ret = init_qp_queue(
641 shca, my_qp, &my_qp->ipz_rqueue, 1, 674 shca, my_pd, my_qp, &my_qp->ipz_rqueue, 1,
642 H_SUCCESS, parms.nr_rq_pages, rwqe_size, 675 H_SUCCESS, &parms.rqueue, rwqe_size);
643 parms.act_nr_recv_sges);
644 if (ret) { 676 if (ret) {
645 ehca_err(pd->device, "Couldn't initialize rqueue " 677 ehca_err(pd->device, "Couldn't initialize rqueue "
646 "and pages ret=%x", ret); 678 "and pages ret=%x", ret);
@@ -670,10 +702,10 @@ struct ehca_qp *internal_create_qp(struct ib_pd *pd,
670 } 702 }
671 703
672 init_attr->cap.max_inline_data = 0; /* not supported yet */ 704 init_attr->cap.max_inline_data = 0; /* not supported yet */
673 init_attr->cap.max_recv_sge = parms.act_nr_recv_sges; 705 init_attr->cap.max_recv_sge = parms.rqueue.act_nr_sges;
674 init_attr->cap.max_recv_wr = parms.act_nr_recv_wqes; 706 init_attr->cap.max_recv_wr = parms.rqueue.act_nr_wqes;
675 init_attr->cap.max_send_sge = parms.act_nr_send_sges; 707 init_attr->cap.max_send_sge = parms.squeue.act_nr_sges;
676 init_attr->cap.max_send_wr = parms.act_nr_send_wqes; 708 init_attr->cap.max_send_wr = parms.squeue.act_nr_wqes;
677 my_qp->init_attr = *init_attr; 709 my_qp->init_attr = *init_attr;
678 710
679 /* NOTE: define_apq0() not supported yet */ 711 /* NOTE: define_apq0() not supported yet */
@@ -707,6 +739,8 @@ struct ehca_qp *internal_create_qp(struct ib_pd *pd,
707 resp.ext_type = my_qp->ext_type; 739 resp.ext_type = my_qp->ext_type;
708 resp.qkey = my_qp->qkey; 740 resp.qkey = my_qp->qkey;
709 resp.real_qp_num = my_qp->real_qp_num; 741 resp.real_qp_num = my_qp->real_qp_num;
742 resp.ipz_rqueue.offset = my_qp->ipz_rqueue.offset;
743 resp.ipz_squeue.offset = my_qp->ipz_squeue.offset;
710 if (HAS_SQ(my_qp)) 744 if (HAS_SQ(my_qp))
711 queue2resp(&resp.ipz_squeue, &my_qp->ipz_squeue); 745 queue2resp(&resp.ipz_squeue, &my_qp->ipz_squeue);
712 if (HAS_RQ(my_qp)) 746 if (HAS_RQ(my_qp))
@@ -723,11 +757,11 @@ struct ehca_qp *internal_create_qp(struct ib_pd *pd,
723 757
724create_qp_exit4: 758create_qp_exit4:
725 if (HAS_RQ(my_qp)) 759 if (HAS_RQ(my_qp))
726 ipz_queue_dtor(&my_qp->ipz_rqueue); 760 ipz_queue_dtor(my_pd, &my_qp->ipz_rqueue);
727 761
728create_qp_exit3: 762create_qp_exit3:
729 if (HAS_SQ(my_qp)) 763 if (HAS_SQ(my_qp))
730 ipz_queue_dtor(&my_qp->ipz_squeue); 764 ipz_queue_dtor(my_pd, &my_qp->ipz_squeue);
731 765
732create_qp_exit2: 766create_qp_exit2:
733 hipz_h_destroy_qp(shca->ipz_hca_handle, my_qp); 767 hipz_h_destroy_qp(shca->ipz_hca_handle, my_qp);
@@ -752,8 +786,8 @@ struct ib_qp *ehca_create_qp(struct ib_pd *pd,
752 return IS_ERR(ret) ? (struct ib_qp *)ret : &ret->ib_qp; 786 return IS_ERR(ret) ? (struct ib_qp *)ret : &ret->ib_qp;
753} 787}
754 788
755int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp, 789static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
756 struct ib_uobject *uobject); 790 struct ib_uobject *uobject);
757 791
758struct ib_srq *ehca_create_srq(struct ib_pd *pd, 792struct ib_srq *ehca_create_srq(struct ib_pd *pd,
759 struct ib_srq_init_attr *srq_init_attr, 793 struct ib_srq_init_attr *srq_init_attr,
@@ -1669,8 +1703,8 @@ query_srq_exit1:
1669 return ret; 1703 return ret;
1670} 1704}
1671 1705
1672int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp, 1706static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
1673 struct ib_uobject *uobject) 1707 struct ib_uobject *uobject)
1674{ 1708{
1675 struct ehca_shca *shca = container_of(dev, struct ehca_shca, ib_device); 1709 struct ehca_shca *shca = container_of(dev, struct ehca_shca, ib_device);
1676 struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd, 1710 struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd,
@@ -1734,9 +1768,9 @@ int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
1734 } 1768 }
1735 1769
1736 if (HAS_RQ(my_qp)) 1770 if (HAS_RQ(my_qp))
1737 ipz_queue_dtor(&my_qp->ipz_rqueue); 1771 ipz_queue_dtor(my_pd, &my_qp->ipz_rqueue);
1738 if (HAS_SQ(my_qp)) 1772 if (HAS_SQ(my_qp))
1739 ipz_queue_dtor(&my_qp->ipz_squeue); 1773 ipz_queue_dtor(my_pd, &my_qp->ipz_squeue);
1740 kmem_cache_free(qp_cache, my_qp); 1774 kmem_cache_free(qp_cache, my_qp);
1741 return 0; 1775 return 0;
1742} 1776}
diff --git a/drivers/infiniband/hw/ehca/ehca_tools.h b/drivers/infiniband/hw/ehca/ehca_tools.h
index 678b81391861..57c77a715f46 100644
--- a/drivers/infiniband/hw/ehca/ehca_tools.h
+++ b/drivers/infiniband/hw/ehca/ehca_tools.h
@@ -154,24 +154,7 @@ extern int ehca_debug_level;
154#define EHCA_BMASK_GET(mask, value) \ 154#define EHCA_BMASK_GET(mask, value) \
155 (EHCA_BMASK_MASK(mask) & (((u64)(value)) >> EHCA_BMASK_SHIFTPOS(mask))) 155 (EHCA_BMASK_MASK(mask) & (((u64)(value)) >> EHCA_BMASK_SHIFTPOS(mask)))
156 156
157
158/* Converts ehca to ib return code */ 157/* Converts ehca to ib return code */
159static inline int ehca2ib_return_code(u64 ehca_rc) 158int ehca2ib_return_code(u64 ehca_rc);
160{
161 switch (ehca_rc) {
162 case H_SUCCESS:
163 return 0;
164 case H_RESOURCE: /* Resource in use */
165 case H_BUSY:
166 return -EBUSY;
167 case H_NOT_ENOUGH_RESOURCES: /* insufficient resources */
168 case H_CONSTRAINED: /* resource constraint */
169 case H_NO_MEM:
170 return -ENOMEM;
171 default:
172 return -EINVAL;
173 }
174}
175
176 159
177#endif /* EHCA_TOOLS_H */ 160#endif /* EHCA_TOOLS_H */
diff --git a/drivers/infiniband/hw/ehca/ehca_uverbs.c b/drivers/infiniband/hw/ehca/ehca_uverbs.c
index 05c415744e3b..4bc687fdf531 100644
--- a/drivers/infiniband/hw/ehca/ehca_uverbs.c
+++ b/drivers/infiniband/hw/ehca/ehca_uverbs.c
@@ -149,7 +149,7 @@ static int ehca_mmap_queue(struct vm_area_struct *vma, struct ipz_queue *queue,
149 ehca_gen_err("vm_insert_page() failed rc=%x", ret); 149 ehca_gen_err("vm_insert_page() failed rc=%x", ret);
150 return ret; 150 return ret;
151 } 151 }
152 start += PAGE_SIZE; 152 start += PAGE_SIZE;
153 } 153 }
154 vma->vm_private_data = mm_count; 154 vma->vm_private_data = mm_count;
155 (*mm_count)++; 155 (*mm_count)++;
diff --git a/drivers/infiniband/hw/ehca/hcp_if.c b/drivers/infiniband/hw/ehca/hcp_if.c
index 3394e05f4b4f..fdbfebea7d11 100644
--- a/drivers/infiniband/hw/ehca/hcp_if.c
+++ b/drivers/infiniband/hw/ehca/hcp_if.c
@@ -52,10 +52,13 @@
52#define H_ALL_RES_QP_ENHANCED_OPS EHCA_BMASK_IBM(9, 11) 52#define H_ALL_RES_QP_ENHANCED_OPS EHCA_BMASK_IBM(9, 11)
53#define H_ALL_RES_QP_PTE_PIN EHCA_BMASK_IBM(12, 12) 53#define H_ALL_RES_QP_PTE_PIN EHCA_BMASK_IBM(12, 12)
54#define H_ALL_RES_QP_SERVICE_TYPE EHCA_BMASK_IBM(13, 15) 54#define H_ALL_RES_QP_SERVICE_TYPE EHCA_BMASK_IBM(13, 15)
55#define H_ALL_RES_QP_STORAGE EHCA_BMASK_IBM(16, 17)
55#define H_ALL_RES_QP_LL_RQ_CQE_POSTING EHCA_BMASK_IBM(18, 18) 56#define H_ALL_RES_QP_LL_RQ_CQE_POSTING EHCA_BMASK_IBM(18, 18)
56#define H_ALL_RES_QP_LL_SQ_CQE_POSTING EHCA_BMASK_IBM(19, 21) 57#define H_ALL_RES_QP_LL_SQ_CQE_POSTING EHCA_BMASK_IBM(19, 21)
57#define H_ALL_RES_QP_SIGNALING_TYPE EHCA_BMASK_IBM(22, 23) 58#define H_ALL_RES_QP_SIGNALING_TYPE EHCA_BMASK_IBM(22, 23)
58#define H_ALL_RES_QP_UD_AV_LKEY_CTRL EHCA_BMASK_IBM(31, 31) 59#define H_ALL_RES_QP_UD_AV_LKEY_CTRL EHCA_BMASK_IBM(31, 31)
60#define H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE EHCA_BMASK_IBM(32, 35)
61#define H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE EHCA_BMASK_IBM(36, 39)
59#define H_ALL_RES_QP_RESOURCE_TYPE EHCA_BMASK_IBM(56, 63) 62#define H_ALL_RES_QP_RESOURCE_TYPE EHCA_BMASK_IBM(56, 63)
60 63
61#define H_ALL_RES_QP_MAX_OUTST_SEND_WR EHCA_BMASK_IBM(0, 15) 64#define H_ALL_RES_QP_MAX_OUTST_SEND_WR EHCA_BMASK_IBM(0, 15)
@@ -299,6 +302,11 @@ u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
299 | EHCA_BMASK_SET(H_ALL_RES_QP_PTE_PIN, 0) 302 | EHCA_BMASK_SET(H_ALL_RES_QP_PTE_PIN, 0)
300 | EHCA_BMASK_SET(H_ALL_RES_QP_SERVICE_TYPE, parms->servicetype) 303 | EHCA_BMASK_SET(H_ALL_RES_QP_SERVICE_TYPE, parms->servicetype)
301 | EHCA_BMASK_SET(H_ALL_RES_QP_SIGNALING_TYPE, parms->sigtype) 304 | EHCA_BMASK_SET(H_ALL_RES_QP_SIGNALING_TYPE, parms->sigtype)
305 | EHCA_BMASK_SET(H_ALL_RES_QP_STORAGE, parms->qp_storage)
306 | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE,
307 parms->squeue.page_size)
308 | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE,
309 parms->rqueue.page_size)
302 | EHCA_BMASK_SET(H_ALL_RES_QP_LL_RQ_CQE_POSTING, 310 | EHCA_BMASK_SET(H_ALL_RES_QP_LL_RQ_CQE_POSTING,
303 !!(parms->ll_comp_flags & LLQP_RECV_COMP)) 311 !!(parms->ll_comp_flags & LLQP_RECV_COMP))
304 | EHCA_BMASK_SET(H_ALL_RES_QP_LL_SQ_CQE_POSTING, 312 | EHCA_BMASK_SET(H_ALL_RES_QP_LL_SQ_CQE_POSTING,
@@ -309,13 +317,13 @@ u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
309 317
310 max_r10_reg = 318 max_r10_reg =
311 EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_SEND_WR, 319 EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_SEND_WR,
312 parms->max_send_wr + 1) 320 parms->squeue.max_wr + 1)
313 | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_RECV_WR, 321 | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_RECV_WR,
314 parms->max_recv_wr + 1) 322 parms->rqueue.max_wr + 1)
315 | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_SEND_SGE, 323 | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_SEND_SGE,
316 parms->max_send_sge) 324 parms->squeue.max_sge)
317 | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_RECV_SGE, 325 | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_RECV_SGE,
318 parms->max_recv_sge); 326 parms->rqueue.max_sge);
319 327
320 r11 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_QP_TOKEN, parms->srq_token); 328 r11 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_QP_TOKEN, parms->srq_token);
321 329
@@ -335,17 +343,17 @@ u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
335 343
336 parms->qp_handle.handle = outs[0]; 344 parms->qp_handle.handle = outs[0];
337 parms->real_qp_num = (u32)outs[1]; 345 parms->real_qp_num = (u32)outs[1];
338 parms->act_nr_send_wqes = 346 parms->squeue.act_nr_wqes =
339 (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_SEND_WR, outs[2]); 347 (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_SEND_WR, outs[2]);
340 parms->act_nr_recv_wqes = 348 parms->rqueue.act_nr_wqes =
341 (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_RECV_WR, outs[2]); 349 (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_RECV_WR, outs[2]);
342 parms->act_nr_send_sges = 350 parms->squeue.act_nr_sges =
343 (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_SEND_SGE, outs[3]); 351 (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_SEND_SGE, outs[3]);
344 parms->act_nr_recv_sges = 352 parms->rqueue.act_nr_sges =
345 (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_RECV_SGE, outs[3]); 353 (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_RECV_SGE, outs[3]);
346 parms->nr_sq_pages = 354 parms->squeue.queue_size =
347 (u32)EHCA_BMASK_GET(H_ALL_RES_QP_SQUEUE_SIZE_PAGES, outs[4]); 355 (u32)EHCA_BMASK_GET(H_ALL_RES_QP_SQUEUE_SIZE_PAGES, outs[4]);
348 parms->nr_rq_pages = 356 parms->rqueue.queue_size =
349 (u32)EHCA_BMASK_GET(H_ALL_RES_QP_RQUEUE_SIZE_PAGES, outs[4]); 357 (u32)EHCA_BMASK_GET(H_ALL_RES_QP_RQUEUE_SIZE_PAGES, outs[4]);
350 358
351 if (ret == H_SUCCESS) 359 if (ret == H_SUCCESS)
@@ -427,7 +435,8 @@ u64 hipz_h_register_rpage(const struct ipz_adapter_handle adapter_handle,
427{ 435{
428 return ehca_plpar_hcall_norets(H_REGISTER_RPAGES, 436 return ehca_plpar_hcall_norets(H_REGISTER_RPAGES,
429 adapter_handle.handle, /* r4 */ 437 adapter_handle.handle, /* r4 */
430 queue_type | pagesize << 8, /* r5 */ 438 (u64)queue_type | ((u64)pagesize) << 8,
439 /* r5 */
431 resource_handle, /* r6 */ 440 resource_handle, /* r6 */
432 logical_address_of_page, /* r7 */ 441 logical_address_of_page, /* r7 */
433 count, /* r8 */ 442 count, /* r8 */
@@ -496,7 +505,7 @@ u64 hipz_h_register_rpage_qp(const struct ipz_adapter_handle adapter_handle,
496 const u64 count, 505 const u64 count,
497 const struct h_galpa galpa) 506 const struct h_galpa galpa)
498{ 507{
499 if (count != 1) { 508 if (count > 1) {
500 ehca_gen_err("Page counter=%lx", count); 509 ehca_gen_err("Page counter=%lx", count);
501 return H_PARAMETER; 510 return H_PARAMETER;
502 } 511 }
@@ -724,6 +733,9 @@ u64 hipz_h_alloc_resource_mr(const struct ipz_adapter_handle adapter_handle,
724 u64 ret; 733 u64 ret;
725 u64 outs[PLPAR_HCALL9_BUFSIZE]; 734 u64 outs[PLPAR_HCALL9_BUFSIZE];
726 735
736 ehca_gen_dbg("kernel PAGE_SIZE=%x access_ctrl=%016x "
737 "vaddr=%lx length=%lx",
738 (u32)PAGE_SIZE, access_ctrl, vaddr, length);
727 ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs, 739 ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
728 adapter_handle.handle, /* r4 */ 740 adapter_handle.handle, /* r4 */
729 5, /* r5 */ 741 5, /* r5 */
@@ -746,8 +758,22 @@ u64 hipz_h_register_rpage_mr(const struct ipz_adapter_handle adapter_handle,
746 const u64 logical_address_of_page, 758 const u64 logical_address_of_page,
747 const u64 count) 759 const u64 count)
748{ 760{
761 extern int ehca_debug_level;
749 u64 ret; 762 u64 ret;
750 763
764 if (unlikely(ehca_debug_level >= 2)) {
765 if (count > 1) {
766 u64 *kpage;
767 int i;
768 kpage = (u64 *)abs_to_virt(logical_address_of_page);
769 for (i = 0; i < count; i++)
770 ehca_gen_dbg("kpage[%d]=%p",
771 i, (void *)kpage[i]);
772 } else
773 ehca_gen_dbg("kpage=%p",
774 (void *)logical_address_of_page);
775 }
776
751 if ((count > 1) && (logical_address_of_page & (EHCA_PAGESIZE-1))) { 777 if ((count > 1) && (logical_address_of_page & (EHCA_PAGESIZE-1))) {
752 ehca_gen_err("logical_address_of_page not on a 4k boundary " 778 ehca_gen_err("logical_address_of_page not on a 4k boundary "
753 "adapter_handle=%lx mr=%p mr_handle=%lx " 779 "adapter_handle=%lx mr=%p mr_handle=%lx "
diff --git a/drivers/infiniband/hw/ehca/ipz_pt_fn.c b/drivers/infiniband/hw/ehca/ipz_pt_fn.c
index 9606f13ed092..a090c679c397 100644
--- a/drivers/infiniband/hw/ehca/ipz_pt_fn.c
+++ b/drivers/infiniband/hw/ehca/ipz_pt_fn.c
@@ -40,6 +40,11 @@
40 40
41#include "ehca_tools.h" 41#include "ehca_tools.h"
42#include "ipz_pt_fn.h" 42#include "ipz_pt_fn.h"
43#include "ehca_classes.h"
44
45#define PAGES_PER_KPAGE (PAGE_SIZE >> EHCA_PAGESHIFT)
46
47struct kmem_cache *small_qp_cache;
43 48
44void *ipz_qpageit_get_inc(struct ipz_queue *queue) 49void *ipz_qpageit_get_inc(struct ipz_queue *queue)
45{ 50{
@@ -49,7 +54,7 @@ void *ipz_qpageit_get_inc(struct ipz_queue *queue)
49 queue->current_q_offset -= queue->pagesize; 54 queue->current_q_offset -= queue->pagesize;
50 ret = NULL; 55 ret = NULL;
51 } 56 }
52 if (((u64)ret) % EHCA_PAGESIZE) { 57 if (((u64)ret) % queue->pagesize) {
53 ehca_gen_err("ERROR!! not at PAGE-Boundary"); 58 ehca_gen_err("ERROR!! not at PAGE-Boundary");
54 return NULL; 59 return NULL;
55 } 60 }
@@ -83,80 +88,195 @@ int ipz_queue_abs_to_offset(struct ipz_queue *queue, u64 addr, u64 *q_offset)
83 return -EINVAL; 88 return -EINVAL;
84} 89}
85 90
86int ipz_queue_ctor(struct ipz_queue *queue, 91#if PAGE_SHIFT < EHCA_PAGESHIFT
87 const u32 nr_of_pages, 92#error Kernel pages must be at least as large than eHCA pages (4K) !
88 const u32 pagesize, const u32 qe_size, const u32 nr_of_sg) 93#endif
94
95/*
96 * allocate pages for queue:
97 * outer loop allocates whole kernel pages (page aligned) and
98 * inner loop divides a kernel page into smaller hca queue pages
99 */
100static int alloc_queue_pages(struct ipz_queue *queue, const u32 nr_of_pages)
89{ 101{
90 int pages_per_kpage = PAGE_SIZE >> EHCA_PAGESHIFT; 102 int k, f = 0;
91 int f; 103 u8 *kpage;
92 104
93 if (pagesize > PAGE_SIZE) {
94 ehca_gen_err("FATAL ERROR: pagesize=%x is greater "
95 "than kernel page size", pagesize);
96 return 0;
97 }
98 if (!pages_per_kpage) {
99 ehca_gen_err("FATAL ERROR: invalid kernel page size. "
100 "pages_per_kpage=%x", pages_per_kpage);
101 return 0;
102 }
103 queue->queue_length = nr_of_pages * pagesize;
104 queue->queue_pages = vmalloc(nr_of_pages * sizeof(void *));
105 if (!queue->queue_pages) {
106 ehca_gen_err("ERROR!! didn't get the memory");
107 return 0;
108 }
109 memset(queue->queue_pages, 0, nr_of_pages * sizeof(void *));
110 /*
111 * allocate pages for queue:
112 * outer loop allocates whole kernel pages (page aligned) and
113 * inner loop divides a kernel page into smaller hca queue pages
114 */
115 f = 0;
116 while (f < nr_of_pages) { 105 while (f < nr_of_pages) {
117 u8 *kpage = (u8 *)get_zeroed_page(GFP_KERNEL); 106 kpage = (u8 *)get_zeroed_page(GFP_KERNEL);
118 int k;
119 if (!kpage) 107 if (!kpage)
120 goto ipz_queue_ctor_exit0; /*NOMEM*/ 108 goto out;
121 for (k = 0; k < pages_per_kpage && f < nr_of_pages; k++) { 109
122 (queue->queue_pages)[f] = (struct ipz_page *)kpage; 110 for (k = 0; k < PAGES_PER_KPAGE && f < nr_of_pages; k++) {
111 queue->queue_pages[f] = (struct ipz_page *)kpage;
123 kpage += EHCA_PAGESIZE; 112 kpage += EHCA_PAGESIZE;
124 f++; 113 f++;
125 } 114 }
126 } 115 }
116 return 1;
127 117
128 queue->current_q_offset = 0; 118out:
119 for (f = 0; f < nr_of_pages && queue->queue_pages[f];
120 f += PAGES_PER_KPAGE)
121 free_page((unsigned long)(queue->queue_pages)[f]);
122 return 0;
123}
124
125static int alloc_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd)
126{
127 int order = ilog2(queue->pagesize) - 9;
128 struct ipz_small_queue_page *page;
129 unsigned long bit;
130
131 mutex_lock(&pd->lock);
132
133 if (!list_empty(&pd->free[order]))
134 page = list_entry(pd->free[order].next,
135 struct ipz_small_queue_page, list);
136 else {
137 page = kmem_cache_zalloc(small_qp_cache, GFP_KERNEL);
138 if (!page)
139 goto out;
140
141 page->page = get_zeroed_page(GFP_KERNEL);
142 if (!page->page) {
143 kmem_cache_free(small_qp_cache, page);
144 goto out;
145 }
146
147 list_add(&page->list, &pd->free[order]);
148 }
149
150 bit = find_first_zero_bit(page->bitmap, IPZ_SPAGE_PER_KPAGE >> order);
151 __set_bit(bit, page->bitmap);
152 page->fill++;
153
154 if (page->fill == IPZ_SPAGE_PER_KPAGE >> order)
155 list_move(&page->list, &pd->full[order]);
156
157 mutex_unlock(&pd->lock);
158
159 queue->queue_pages[0] = (void *)(page->page | (bit << (order + 9)));
160 queue->small_page = page;
161 return 1;
162
163out:
164 ehca_err(pd->ib_pd.device, "failed to allocate small queue page");
165 return 0;
166}
167
168static void free_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd)
169{
170 int order = ilog2(queue->pagesize) - 9;
171 struct ipz_small_queue_page *page = queue->small_page;
172 unsigned long bit;
173 int free_page = 0;
174
175 bit = ((unsigned long)queue->queue_pages[0] & PAGE_MASK)
176 >> (order + 9);
177
178 mutex_lock(&pd->lock);
179
180 __clear_bit(bit, page->bitmap);
181 page->fill--;
182
183 if (page->fill == 0) {
184 list_del(&page->list);
185 free_page = 1;
186 }
187
188 if (page->fill == (IPZ_SPAGE_PER_KPAGE >> order) - 1)
189 /* the page was full until we freed the chunk */
190 list_move_tail(&page->list, &pd->free[order]);
191
192 mutex_unlock(&pd->lock);
193
194 if (free_page) {
195 free_page(page->page);
196 kmem_cache_free(small_qp_cache, page);
197 }
198}
199
200int ipz_queue_ctor(struct ehca_pd *pd, struct ipz_queue *queue,
201 const u32 nr_of_pages, const u32 pagesize,
202 const u32 qe_size, const u32 nr_of_sg,
203 int is_small)
204{
205 if (pagesize > PAGE_SIZE) {
206 ehca_gen_err("FATAL ERROR: pagesize=%x "
207 "is greater than kernel page size", pagesize);
208 return 0;
209 }
210
211 /* init queue fields */
212 queue->queue_length = nr_of_pages * pagesize;
213 queue->pagesize = pagesize;
129 queue->qe_size = qe_size; 214 queue->qe_size = qe_size;
130 queue->act_nr_of_sg = nr_of_sg; 215 queue->act_nr_of_sg = nr_of_sg;
131 queue->pagesize = pagesize; 216 queue->current_q_offset = 0;
132 queue->toggle_state = 1; 217 queue->toggle_state = 1;
133 return 1; 218 queue->small_page = NULL;
134 219
135 ipz_queue_ctor_exit0: 220 /* allocate queue page pointers */
136 ehca_gen_err("Couldn't get alloc pages queue=%p f=%x nr_of_pages=%x", 221 queue->queue_pages = vmalloc(nr_of_pages * sizeof(void *));
137 queue, f, nr_of_pages); 222 if (!queue->queue_pages) {
138 for (f = 0; f < nr_of_pages; f += pages_per_kpage) { 223 ehca_gen_err("Couldn't allocate queue page list");
139 if (!(queue->queue_pages)[f]) 224 return 0;
140 break;
141 free_page((unsigned long)(queue->queue_pages)[f]);
142 } 225 }
226 memset(queue->queue_pages, 0, nr_of_pages * sizeof(void *));
227
228 /* allocate actual queue pages */
229 if (is_small) {
230 if (!alloc_small_queue_page(queue, pd))
231 goto ipz_queue_ctor_exit0;
232 } else
233 if (!alloc_queue_pages(queue, nr_of_pages))
234 goto ipz_queue_ctor_exit0;
235
236 return 1;
237
238ipz_queue_ctor_exit0:
239 ehca_gen_err("Couldn't alloc pages queue=%p "
240 "nr_of_pages=%x", queue, nr_of_pages);
241 vfree(queue->queue_pages);
242
143 return 0; 243 return 0;
144} 244}
145 245
146int ipz_queue_dtor(struct ipz_queue *queue) 246int ipz_queue_dtor(struct ehca_pd *pd, struct ipz_queue *queue)
147{ 247{
148 int pages_per_kpage = PAGE_SIZE >> EHCA_PAGESHIFT; 248 int i, nr_pages;
149 int g;
150 int nr_pages;
151 249
152 if (!queue || !queue->queue_pages) { 250 if (!queue || !queue->queue_pages) {
153 ehca_gen_dbg("queue or queue_pages is NULL"); 251 ehca_gen_dbg("queue or queue_pages is NULL");
154 return 0; 252 return 0;
155 } 253 }
156 nr_pages = queue->queue_length / queue->pagesize; 254
157 for (g = 0; g < nr_pages; g += pages_per_kpage) 255 if (queue->small_page)
158 free_page((unsigned long)(queue->queue_pages)[g]); 256 free_small_queue_page(queue, pd);
257 else {
258 nr_pages = queue->queue_length / queue->pagesize;
259 for (i = 0; i < nr_pages; i += PAGES_PER_KPAGE)
260 free_page((unsigned long)queue->queue_pages[i]);
261 }
262
159 vfree(queue->queue_pages); 263 vfree(queue->queue_pages);
160 264
161 return 1; 265 return 1;
162} 266}
267
268int ehca_init_small_qp_cache(void)
269{
270 small_qp_cache = kmem_cache_create("ehca_cache_small_qp",
271 sizeof(struct ipz_small_queue_page),
272 0, SLAB_HWCACHE_ALIGN, NULL);
273 if (!small_qp_cache)
274 return -ENOMEM;
275
276 return 0;
277}
278
279void ehca_cleanup_small_qp_cache(void)
280{
281 kmem_cache_destroy(small_qp_cache);
282}
diff --git a/drivers/infiniband/hw/ehca/ipz_pt_fn.h b/drivers/infiniband/hw/ehca/ipz_pt_fn.h
index 39a4f64aff41..c6937a044e8a 100644
--- a/drivers/infiniband/hw/ehca/ipz_pt_fn.h
+++ b/drivers/infiniband/hw/ehca/ipz_pt_fn.h
@@ -51,11 +51,25 @@
51#include "ehca_tools.h" 51#include "ehca_tools.h"
52#include "ehca_qes.h" 52#include "ehca_qes.h"
53 53
54struct ehca_pd;
55struct ipz_small_queue_page;
56
54/* struct generic ehca page */ 57/* struct generic ehca page */
55struct ipz_page { 58struct ipz_page {
56 u8 entries[EHCA_PAGESIZE]; 59 u8 entries[EHCA_PAGESIZE];
57}; 60};
58 61
62#define IPZ_SPAGE_PER_KPAGE (PAGE_SIZE / 512)
63
64struct ipz_small_queue_page {
65 unsigned long page;
66 unsigned long bitmap[IPZ_SPAGE_PER_KPAGE / BITS_PER_LONG];
67 int fill;
68 void *mapped_addr;
69 u32 mmap_count;
70 struct list_head list;
71};
72
59/* struct generic queue in linux kernel virtual memory (kv) */ 73/* struct generic queue in linux kernel virtual memory (kv) */
60struct ipz_queue { 74struct ipz_queue {
61 u64 current_q_offset; /* current queue entry */ 75 u64 current_q_offset; /* current queue entry */
@@ -66,7 +80,8 @@ struct ipz_queue {
66 u32 queue_length; /* queue length allocated in bytes */ 80 u32 queue_length; /* queue length allocated in bytes */
67 u32 pagesize; 81 u32 pagesize;
68 u32 toggle_state; /* toggle flag - per page */ 82 u32 toggle_state; /* toggle flag - per page */
69 u32 dummy3; /* 64 bit alignment */ 83 u32 offset; /* save offset within page for small_qp */
84 struct ipz_small_queue_page *small_page;
70}; 85};
71 86
72/* 87/*
@@ -188,9 +203,10 @@ struct ipz_qpt {
188 * see ipz_qpt_ctor() 203 * see ipz_qpt_ctor()
189 * returns true if ok, false if out of memory 204 * returns true if ok, false if out of memory
190 */ 205 */
191int ipz_queue_ctor(struct ipz_queue *queue, const u32 nr_of_pages, 206int ipz_queue_ctor(struct ehca_pd *pd, struct ipz_queue *queue,
192 const u32 pagesize, const u32 qe_size, 207 const u32 nr_of_pages, const u32 pagesize,
193 const u32 nr_of_sg); 208 const u32 qe_size, const u32 nr_of_sg,
209 int is_small);
194 210
195/* 211/*
196 * destructor for a ipz_queue_t 212 * destructor for a ipz_queue_t
@@ -198,7 +214,7 @@ int ipz_queue_ctor(struct ipz_queue *queue, const u32 nr_of_pages,
198 * see ipz_queue_ctor() 214 * see ipz_queue_ctor()
199 * returns true if ok, false if queue was NULL-ptr of free failed 215 * returns true if ok, false if queue was NULL-ptr of free failed
200 */ 216 */
201int ipz_queue_dtor(struct ipz_queue *queue); 217int ipz_queue_dtor(struct ehca_pd *pd, struct ipz_queue *queue);
202 218
203/* 219/*
204 * constructor for a ipz_qpt_t, 220 * constructor for a ipz_qpt_t,
diff --git a/drivers/infiniband/hw/ipath/Makefile b/drivers/infiniband/hw/ipath/Makefile
index ec2e603ea241..fe6738826865 100644
--- a/drivers/infiniband/hw/ipath/Makefile
+++ b/drivers/infiniband/hw/ipath/Makefile
@@ -14,7 +14,6 @@ ib_ipath-y := \
14 ipath_init_chip.o \ 14 ipath_init_chip.o \
15 ipath_intr.o \ 15 ipath_intr.o \
16 ipath_keys.o \ 16 ipath_keys.o \
17 ipath_layer.o \
18 ipath_mad.o \ 17 ipath_mad.o \
19 ipath_mmap.o \ 18 ipath_mmap.o \
20 ipath_mr.o \ 19 ipath_mr.o \
diff --git a/drivers/infiniband/hw/ipath/ipath_layer.c b/drivers/infiniband/hw/ipath/ipath_layer.c
deleted file mode 100644
index 82616b779e24..000000000000
--- a/drivers/infiniband/hw/ipath/ipath_layer.c
+++ /dev/null
@@ -1,365 +0,0 @@
1/*
2 * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
3 * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34/*
35 * These are the routines used by layered drivers, currently just the
36 * layered ethernet driver and verbs layer.
37 */
38
39#include <linux/io.h>
40#include <asm/byteorder.h>
41
42#include "ipath_kernel.h"
43#include "ipath_layer.h"
44#include "ipath_verbs.h"
45#include "ipath_common.h"
46
47/* Acquire before ipath_devs_lock. */
48static DEFINE_MUTEX(ipath_layer_mutex);
49
50u16 ipath_layer_rcv_opcode;
51
52static int (*layer_intr)(void *, u32);
53static int (*layer_rcv)(void *, void *, struct sk_buff *);
54static int (*layer_rcv_lid)(void *, void *);
55
56static void *(*layer_add_one)(int, struct ipath_devdata *);
57static void (*layer_remove_one)(void *);
58
59int __ipath_layer_intr(struct ipath_devdata *dd, u32 arg)
60{
61 int ret = -ENODEV;
62
63 if (dd->ipath_layer.l_arg && layer_intr)
64 ret = layer_intr(dd->ipath_layer.l_arg, arg);
65
66 return ret;
67}
68
69int ipath_layer_intr(struct ipath_devdata *dd, u32 arg)
70{
71 int ret;
72
73 mutex_lock(&ipath_layer_mutex);
74
75 ret = __ipath_layer_intr(dd, arg);
76
77 mutex_unlock(&ipath_layer_mutex);
78
79 return ret;
80}
81
82int __ipath_layer_rcv(struct ipath_devdata *dd, void *hdr,
83 struct sk_buff *skb)
84{
85 int ret = -ENODEV;
86
87 if (dd->ipath_layer.l_arg && layer_rcv)
88 ret = layer_rcv(dd->ipath_layer.l_arg, hdr, skb);
89
90 return ret;
91}
92
93int __ipath_layer_rcv_lid(struct ipath_devdata *dd, void *hdr)
94{
95 int ret = -ENODEV;
96
97 if (dd->ipath_layer.l_arg && layer_rcv_lid)
98 ret = layer_rcv_lid(dd->ipath_layer.l_arg, hdr);
99
100 return ret;
101}
102
103void ipath_layer_lid_changed(struct ipath_devdata *dd)
104{
105 mutex_lock(&ipath_layer_mutex);
106
107 if (dd->ipath_layer.l_arg && layer_intr)
108 layer_intr(dd->ipath_layer.l_arg, IPATH_LAYER_INT_LID);
109
110 mutex_unlock(&ipath_layer_mutex);
111}
112
113void ipath_layer_add(struct ipath_devdata *dd)
114{
115 mutex_lock(&ipath_layer_mutex);
116
117 if (layer_add_one)
118 dd->ipath_layer.l_arg =
119 layer_add_one(dd->ipath_unit, dd);
120
121 mutex_unlock(&ipath_layer_mutex);
122}
123
124void ipath_layer_remove(struct ipath_devdata *dd)
125{
126 mutex_lock(&ipath_layer_mutex);
127
128 if (dd->ipath_layer.l_arg && layer_remove_one) {
129 layer_remove_one(dd->ipath_layer.l_arg);
130 dd->ipath_layer.l_arg = NULL;
131 }
132
133 mutex_unlock(&ipath_layer_mutex);
134}
135
136int ipath_layer_register(void *(*l_add)(int, struct ipath_devdata *),
137 void (*l_remove)(void *),
138 int (*l_intr)(void *, u32),
139 int (*l_rcv)(void *, void *, struct sk_buff *),
140 u16 l_rcv_opcode,
141 int (*l_rcv_lid)(void *, void *))
142{
143 struct ipath_devdata *dd, *tmp;
144 unsigned long flags;
145
146 mutex_lock(&ipath_layer_mutex);
147
148 layer_add_one = l_add;
149 layer_remove_one = l_remove;
150 layer_intr = l_intr;
151 layer_rcv = l_rcv;
152 layer_rcv_lid = l_rcv_lid;
153 ipath_layer_rcv_opcode = l_rcv_opcode;
154
155 spin_lock_irqsave(&ipath_devs_lock, flags);
156
157 list_for_each_entry_safe(dd, tmp, &ipath_dev_list, ipath_list) {
158 if (!(dd->ipath_flags & IPATH_INITTED))
159 continue;
160
161 if (dd->ipath_layer.l_arg)
162 continue;
163
164 spin_unlock_irqrestore(&ipath_devs_lock, flags);
165 dd->ipath_layer.l_arg = l_add(dd->ipath_unit, dd);
166 spin_lock_irqsave(&ipath_devs_lock, flags);
167 }
168
169 spin_unlock_irqrestore(&ipath_devs_lock, flags);
170 mutex_unlock(&ipath_layer_mutex);
171
172 return 0;
173}
174
175EXPORT_SYMBOL_GPL(ipath_layer_register);
176
177void ipath_layer_unregister(void)
178{
179 struct ipath_devdata *dd, *tmp;
180 unsigned long flags;
181
182 mutex_lock(&ipath_layer_mutex);
183 spin_lock_irqsave(&ipath_devs_lock, flags);
184
185 list_for_each_entry_safe(dd, tmp, &ipath_dev_list, ipath_list) {
186 if (dd->ipath_layer.l_arg && layer_remove_one) {
187 spin_unlock_irqrestore(&ipath_devs_lock, flags);
188 layer_remove_one(dd->ipath_layer.l_arg);
189 spin_lock_irqsave(&ipath_devs_lock, flags);
190 dd->ipath_layer.l_arg = NULL;
191 }
192 }
193
194 spin_unlock_irqrestore(&ipath_devs_lock, flags);
195
196 layer_add_one = NULL;
197 layer_remove_one = NULL;
198 layer_intr = NULL;
199 layer_rcv = NULL;
200 layer_rcv_lid = NULL;
201
202 mutex_unlock(&ipath_layer_mutex);
203}
204
205EXPORT_SYMBOL_GPL(ipath_layer_unregister);
206
207int ipath_layer_open(struct ipath_devdata *dd, u32 * pktmax)
208{
209 int ret;
210 u32 intval = 0;
211
212 mutex_lock(&ipath_layer_mutex);
213
214 if (!dd->ipath_layer.l_arg) {
215 ret = -EINVAL;
216 goto bail;
217 }
218
219 ret = ipath_setrcvhdrsize(dd, IPATH_HEADER_QUEUE_WORDS);
220
221 if (ret < 0)
222 goto bail;
223
224 *pktmax = dd->ipath_ibmaxlen;
225
226 if (*dd->ipath_statusp & IPATH_STATUS_IB_READY)
227 intval |= IPATH_LAYER_INT_IF_UP;
228 if (dd->ipath_lid)
229 intval |= IPATH_LAYER_INT_LID;
230 if (dd->ipath_mlid)
231 intval |= IPATH_LAYER_INT_BCAST;
232 /*
233 * do this on open, in case low level is already up and
234 * just layered driver was reloaded, etc.
235 */
236 if (intval)
237 layer_intr(dd->ipath_layer.l_arg, intval);
238
239 ret = 0;
240bail:
241 mutex_unlock(&ipath_layer_mutex);
242
243 return ret;
244}
245
246EXPORT_SYMBOL_GPL(ipath_layer_open);
247
248u16 ipath_layer_get_lid(struct ipath_devdata *dd)
249{
250 return dd->ipath_lid;
251}
252
253EXPORT_SYMBOL_GPL(ipath_layer_get_lid);
254
255/**
256 * ipath_layer_get_mac - get the MAC address
257 * @dd: the infinipath device
258 * @mac: the MAC is put here
259 *
260 * This is the EUID-64 OUI octets (top 3), then
261 * skip the next 2 (which should both be zero or 0xff).
262 * The returned MAC is in network order
263 * mac points to at least 6 bytes of buffer
264 * We assume that by the time the LID is set, that the GUID is as valid
265 * as it's ever going to be, rather than adding yet another status bit.
266 */
267
268int ipath_layer_get_mac(struct ipath_devdata *dd, u8 * mac)
269{
270 u8 *guid;
271
272 guid = (u8 *) &dd->ipath_guid;
273
274 mac[0] = guid[0];
275 mac[1] = guid[1];
276 mac[2] = guid[2];
277 mac[3] = guid[5];
278 mac[4] = guid[6];
279 mac[5] = guid[7];
280 if ((guid[3] || guid[4]) && !(guid[3] == 0xff && guid[4] == 0xff))
281 ipath_dbg("Warning, guid bytes 3 and 4 not 0 or 0xffff: "
282 "%x %x\n", guid[3], guid[4]);
283 return 0;
284}
285
286EXPORT_SYMBOL_GPL(ipath_layer_get_mac);
287
288u16 ipath_layer_get_bcast(struct ipath_devdata *dd)
289{
290 return dd->ipath_mlid;
291}
292
293EXPORT_SYMBOL_GPL(ipath_layer_get_bcast);
294
295int ipath_layer_send_hdr(struct ipath_devdata *dd, struct ether_header *hdr)
296{
297 int ret = 0;
298 u32 __iomem *piobuf;
299 u32 plen, *uhdr;
300 size_t count;
301 __be16 vlsllnh;
302
303 if (!(dd->ipath_flags & IPATH_RCVHDRSZ_SET)) {
304 ipath_dbg("send while not open\n");
305 ret = -EINVAL;
306 } else
307 if ((dd->ipath_flags & (IPATH_LINKUNK | IPATH_LINKDOWN)) ||
308 dd->ipath_lid == 0) {
309 /*
310 * lid check is for when sma hasn't yet configured
311 */
312 ret = -ENETDOWN;
313 ipath_cdbg(VERBOSE, "send while not ready, "
314 "mylid=%u, flags=0x%x\n",
315 dd->ipath_lid, dd->ipath_flags);
316 }
317
318 vlsllnh = *((__be16 *) hdr);
319 if (vlsllnh != htons(IPATH_LRH_BTH)) {
320 ipath_dbg("Warning: lrh[0] wrong (%x, not %x); "
321 "not sending\n", be16_to_cpu(vlsllnh),
322 IPATH_LRH_BTH);
323 ret = -EINVAL;
324 }
325 if (ret)
326 goto done;
327
328 /* Get a PIO buffer to use. */
329 piobuf = ipath_getpiobuf(dd, NULL);
330 if (piobuf == NULL) {
331 ret = -EBUSY;
332 goto done;
333 }
334
335 plen = (sizeof(*hdr) >> 2); /* actual length */
336 ipath_cdbg(EPKT, "0x%x+1w pio %p\n", plen, piobuf);
337
338 writeq(plen+1, piobuf); /* len (+1 for pad) to pbc, no flags */
339 ipath_flush_wc();
340 piobuf += 2;
341 uhdr = (u32 *)hdr;
342 count = plen-1; /* amount we can copy before trigger word */
343 __iowrite32_copy(piobuf, uhdr, count);
344 ipath_flush_wc();
345 __raw_writel(uhdr[count], piobuf + count);
346 ipath_flush_wc(); /* ensure it's sent, now */
347
348 ipath_stats.sps_ether_spkts++; /* ether packet sent */
349
350done:
351 return ret;
352}
353
354EXPORT_SYMBOL_GPL(ipath_layer_send_hdr);
355
356int ipath_layer_set_piointbufavail_int(struct ipath_devdata *dd)
357{
358 set_bit(IPATH_S_PIOINTBUFAVAIL, &dd->ipath_sendctrl);
359
360 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
361 dd->ipath_sendctrl);
362 return 0;
363}
364
365EXPORT_SYMBOL_GPL(ipath_layer_set_piointbufavail_int);
diff --git a/drivers/infiniband/hw/ipath/ipath_layer.h b/drivers/infiniband/hw/ipath/ipath_layer.h
deleted file mode 100644
index 415709c4d85b..000000000000
--- a/drivers/infiniband/hw/ipath/ipath_layer.h
+++ /dev/null
@@ -1,71 +0,0 @@
1/*
2 * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
3 * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#ifndef _IPATH_LAYER_H
35#define _IPATH_LAYER_H
36
37/*
38 * This header file is for symbols shared between the infinipath driver
39 * and drivers layered upon it (such as ipath).
40 */
41
42struct sk_buff;
43struct ipath_devdata;
44struct ether_header;
45
46int ipath_layer_register(void *(*l_add)(int, struct ipath_devdata *),
47 void (*l_remove)(void *),
48 int (*l_intr)(void *, u32),
49 int (*l_rcv)(void *, void *,
50 struct sk_buff *),
51 u16 rcv_opcode,
52 int (*l_rcv_lid)(void *, void *));
53void ipath_layer_unregister(void);
54int ipath_layer_open(struct ipath_devdata *, u32 * pktmax);
55u16 ipath_layer_get_lid(struct ipath_devdata *dd);
56int ipath_layer_get_mac(struct ipath_devdata *dd, u8 *);
57u16 ipath_layer_get_bcast(struct ipath_devdata *dd);
58int ipath_layer_send_hdr(struct ipath_devdata *dd,
59 struct ether_header *hdr);
60int ipath_layer_set_piointbufavail_int(struct ipath_devdata *dd);
61
62/* ipath_ether interrupt values */
63#define IPATH_LAYER_INT_IF_UP 0x2
64#define IPATH_LAYER_INT_IF_DOWN 0x4
65#define IPATH_LAYER_INT_LID 0x8
66#define IPATH_LAYER_INT_SEND_CONTINUE 0x10
67#define IPATH_LAYER_INT_BCAST 0x40
68
69extern unsigned ipath_debug; /* debugging bit mask */
70
71#endif /* _IPATH_LAYER_H */
diff --git a/drivers/infiniband/hw/ipath/ipath_verbs.h b/drivers/infiniband/hw/ipath/ipath_verbs.h
index 9bbe81967f14..1a24c6a4a814 100644
--- a/drivers/infiniband/hw/ipath/ipath_verbs.h
+++ b/drivers/infiniband/hw/ipath/ipath_verbs.h
@@ -42,8 +42,6 @@
42#include <rdma/ib_pack.h> 42#include <rdma/ib_pack.h>
43#include <rdma/ib_user_verbs.h> 43#include <rdma/ib_user_verbs.h>
44 44
45#include "ipath_layer.h"
46
47#define IPATH_MAX_RDMA_ATOMIC 4 45#define IPATH_MAX_RDMA_ATOMIC 4
48 46
49#define QPN_MAX (1 << 24) 47#define QPN_MAX (1 << 24)
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index b5a24fbef70d..f6315dfb213e 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -415,9 +415,11 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
415 return 0; 415 return 0;
416 416
417err_wrid: 417err_wrid:
418 if (pd->uobject && !init_attr->srq) 418 if (pd->uobject) {
419 mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &qp->db); 419 if (!init_attr->srq)
420 else { 420 mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context),
421 &qp->db);
422 } else {
421 kfree(qp->sq.wrid); 423 kfree(qp->sq.wrid);
422 kfree(qp->rq.wrid); 424 kfree(qp->rq.wrid);
423 } 425 }
@@ -742,7 +744,7 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
742 if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) { 744 if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) {
743 printk(KERN_ERR "path MTU (%u) is invalid\n", 745 printk(KERN_ERR "path MTU (%u) is invalid\n",
744 attr->path_mtu); 746 attr->path_mtu);
745 return -EINVAL; 747 goto out;
746 } 748 }
747 context->mtu_msgmax = (attr->path_mtu << 5) | 31; 749 context->mtu_msgmax = (attr->path_mtu << 5) | 31;
748 } 750 }
@@ -781,10 +783,8 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
781 783
782 if (attr_mask & IB_QP_AV) { 784 if (attr_mask & IB_QP_AV) {
783 if (mlx4_set_path(dev, &attr->ah_attr, &context->pri_path, 785 if (mlx4_set_path(dev, &attr->ah_attr, &context->pri_path,
784 attr_mask & IB_QP_PORT ? attr->port_num : qp->port)) { 786 attr_mask & IB_QP_PORT ? attr->port_num : qp->port))
785 err = -EINVAL;
786 goto out; 787 goto out;
787 }
788 788
789 optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH | 789 optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |
790 MLX4_QP_OPTPAR_SCHED_QUEUE); 790 MLX4_QP_OPTPAR_SCHED_QUEUE);
@@ -798,15 +798,15 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
798 if (attr_mask & IB_QP_ALT_PATH) { 798 if (attr_mask & IB_QP_ALT_PATH) {
799 if (attr->alt_port_num == 0 || 799 if (attr->alt_port_num == 0 ||
800 attr->alt_port_num > dev->dev->caps.num_ports) 800 attr->alt_port_num > dev->dev->caps.num_ports)
801 return -EINVAL; 801 goto out;
802 802
803 if (attr->alt_pkey_index >= 803 if (attr->alt_pkey_index >=
804 dev->dev->caps.pkey_table_len[attr->alt_port_num]) 804 dev->dev->caps.pkey_table_len[attr->alt_port_num])
805 return -EINVAL; 805 goto out;
806 806
807 if (mlx4_set_path(dev, &attr->alt_ah_attr, &context->alt_path, 807 if (mlx4_set_path(dev, &attr->alt_ah_attr, &context->alt_path,
808 attr->alt_port_num)) 808 attr->alt_port_num))
809 return -EINVAL; 809 goto out;
810 810
811 context->alt_path.pkey_index = attr->alt_pkey_index; 811 context->alt_path.pkey_index = attr->alt_pkey_index;
812 context->alt_path.ackto = attr->alt_timeout << 3; 812 context->alt_path.ackto = attr->alt_timeout << 3;
diff --git a/drivers/infiniband/hw/mthca/mthca_cmd.c b/drivers/infiniband/hw/mthca/mthca_cmd.c
index f40558d76475..acc95892713a 100644
--- a/drivers/infiniband/hw/mthca/mthca_cmd.c
+++ b/drivers/infiniband/hw/mthca/mthca_cmd.c
@@ -357,8 +357,6 @@ void mthca_cmd_event(struct mthca_dev *dev,
357 context->status = status; 357 context->status = status;
358 context->out_param = out_param; 358 context->out_param = out_param;
359 359
360 context->token += dev->cmd.token_mask + 1;
361
362 complete(&context->done); 360 complete(&context->done);
363} 361}
364 362
@@ -380,6 +378,7 @@ static int mthca_cmd_wait(struct mthca_dev *dev,
380 spin_lock(&dev->cmd.context_lock); 378 spin_lock(&dev->cmd.context_lock);
381 BUG_ON(dev->cmd.free_head < 0); 379 BUG_ON(dev->cmd.free_head < 0);
382 context = &dev->cmd.context[dev->cmd.free_head]; 380 context = &dev->cmd.context[dev->cmd.free_head];
381 context->token += dev->cmd.token_mask + 1;
383 dev->cmd.free_head = context->next; 382 dev->cmd.free_head = context->next;
384 spin_unlock(&dev->cmd.context_lock); 383 spin_unlock(&dev->cmd.context_lock);
385 384
diff --git a/drivers/net/mlx4/cmd.c b/drivers/net/mlx4/cmd.c
index c1f81a993f5d..a9f31753661a 100644
--- a/drivers/net/mlx4/cmd.c
+++ b/drivers/net/mlx4/cmd.c
@@ -246,8 +246,6 @@ void mlx4_cmd_event(struct mlx4_dev *dev, u16 token, u8 status, u64 out_param)
246 context->result = mlx4_status_to_errno(status); 246 context->result = mlx4_status_to_errno(status);
247 context->out_param = out_param; 247 context->out_param = out_param;
248 248
249 context->token += priv->cmd.token_mask + 1;
250
251 complete(&context->done); 249 complete(&context->done);
252} 250}
253 251
@@ -264,6 +262,7 @@ static int mlx4_cmd_wait(struct mlx4_dev *dev, u64 in_param, u64 *out_param,
264 spin_lock(&cmd->context_lock); 262 spin_lock(&cmd->context_lock);
265 BUG_ON(cmd->free_head < 0); 263 BUG_ON(cmd->free_head < 0);
266 context = &cmd->context[cmd->free_head]; 264 context = &cmd->context[cmd->free_head];
265 context->token += cmd->token_mask + 1;
267 cmd->free_head = context->next; 266 cmd->free_head = context->next;
268 spin_unlock(&cmd->context_lock); 267 spin_unlock(&cmd->context_lock);
269 268
generated by cgit v1.2.2 (git 2.25.0) at 2025-10-02 10:36:32 -0400