aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/infiniband/hw/nes/nes_verbs.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/infiniband/hw/nes/nes_verbs.c')
-rw-r--r--drivers/infiniband/hw/nes/nes_verbs.c817
1 files changed, 416 insertions, 401 deletions
diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index a680c42d6e8..64d3136e374 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (c) 2006 - 2009 Intel-NE, Inc. All rights reserved. 2 * Copyright (c) 2006 - 2009 Intel Corporation. All rights reserved.
3 * 3 *
4 * This software is available to you under a choice of one of two 4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU 5 * licenses. You may choose to be licensed under the terms of the GNU
@@ -275,342 +275,236 @@ static int nes_bind_mw(struct ib_qp *ibqp, struct ib_mw *ibmw,
275} 275}
276 276
277 277
278/** 278/*
279 * nes_alloc_fmr 279 * nes_alloc_fast_mr
280 */ 280 */
281static struct ib_fmr *nes_alloc_fmr(struct ib_pd *ibpd, 281static int alloc_fast_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
282 int ibmr_access_flags, 282 u32 stag, u32 page_count)
283 struct ib_fmr_attr *ibfmr_attr)
284{ 283{
285 unsigned long flags;
286 struct nes_pd *nespd = to_nespd(ibpd);
287 struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
288 struct nes_device *nesdev = nesvnic->nesdev;
289 struct nes_adapter *nesadapter = nesdev->nesadapter;
290 struct nes_fmr *nesfmr;
291 struct nes_cqp_request *cqp_request;
292 struct nes_hw_cqp_wqe *cqp_wqe; 284 struct nes_hw_cqp_wqe *cqp_wqe;
285 struct nes_cqp_request *cqp_request;
286 unsigned long flags;
293 int ret; 287 int ret;
294 u32 stag; 288 struct nes_adapter *nesadapter = nesdev->nesadapter;
295 u32 stag_index = 0;
296 u32 next_stag_index = 0;
297 u32 driver_key = 0;
298 u32 opcode = 0; 289 u32 opcode = 0;
299 u8 stag_key = 0; 290 u16 major_code;
300 int i=0; 291 u64 region_length = page_count * PAGE_SIZE;
301 struct nes_vpbl vpbl;
302
303 get_random_bytes(&next_stag_index, sizeof(next_stag_index));
304 stag_key = (u8)next_stag_index;
305
306 driver_key = 0;
307
308 next_stag_index >>= 8;
309 next_stag_index %= nesadapter->max_mr;
310
311 ret = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs,
312 nesadapter->max_mr, &stag_index, &next_stag_index);
313 if (ret) {
314 goto failed_resource_alloc;
315 }
316
317 nesfmr = kzalloc(sizeof(*nesfmr), GFP_KERNEL);
318 if (!nesfmr) {
319 ret = -ENOMEM;
320 goto failed_fmr_alloc;
321 }
322
323 nesfmr->nesmr.mode = IWNES_MEMREG_TYPE_FMR;
324 if (ibfmr_attr->max_pages == 1) {
325 /* use zero length PBL */
326 nesfmr->nesmr.pbl_4k = 0;
327 nesfmr->nesmr.pbls_used = 0;
328 } else if (ibfmr_attr->max_pages <= 32) {
329 /* use PBL 256 */
330 nesfmr->nesmr.pbl_4k = 0;
331 nesfmr->nesmr.pbls_used = 1;
332 } else if (ibfmr_attr->max_pages <= 512) {
333 /* use 4K PBLs */
334 nesfmr->nesmr.pbl_4k = 1;
335 nesfmr->nesmr.pbls_used = 1;
336 } else {
337 /* use two level 4K PBLs */
338 /* add support for two level 256B PBLs */
339 nesfmr->nesmr.pbl_4k = 1;
340 nesfmr->nesmr.pbls_used = 1 + (ibfmr_attr->max_pages >> 9) +
341 ((ibfmr_attr->max_pages & 511) ? 1 : 0);
342 }
343 /* Register the region with the adapter */
344 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
345
346 /* track PBL resources */
347 if (nesfmr->nesmr.pbls_used != 0) {
348 if (nesfmr->nesmr.pbl_4k) {
349 if (nesfmr->nesmr.pbls_used > nesadapter->free_4kpbl) {
350 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
351 ret = -ENOMEM;
352 goto failed_vpbl_avail;
353 } else {
354 nesadapter->free_4kpbl -= nesfmr->nesmr.pbls_used;
355 }
356 } else {
357 if (nesfmr->nesmr.pbls_used > nesadapter->free_256pbl) {
358 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
359 ret = -ENOMEM;
360 goto failed_vpbl_avail;
361 } else {
362 nesadapter->free_256pbl -= nesfmr->nesmr.pbls_used;
363 }
364 }
365 }
366
367 /* one level pbl */
368 if (nesfmr->nesmr.pbls_used == 0) {
369 nesfmr->root_vpbl.pbl_vbase = NULL;
370 nes_debug(NES_DBG_MR, "zero level pbl \n");
371 } else if (nesfmr->nesmr.pbls_used == 1) {
372 /* can change it to kmalloc & dma_map_single */
373 nesfmr->root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
374 &nesfmr->root_vpbl.pbl_pbase);
375 if (!nesfmr->root_vpbl.pbl_vbase) {
376 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
377 ret = -ENOMEM;
378 goto failed_vpbl_alloc;
379 }
380 nesfmr->leaf_pbl_cnt = 0;
381 nes_debug(NES_DBG_MR, "one level pbl, root_vpbl.pbl_vbase=%p \n",
382 nesfmr->root_vpbl.pbl_vbase);
383 }
384 /* two level pbl */
385 else {
386 nesfmr->root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 8192,
387 &nesfmr->root_vpbl.pbl_pbase);
388 if (!nesfmr->root_vpbl.pbl_vbase) {
389 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
390 ret = -ENOMEM;
391 goto failed_vpbl_alloc;
392 }
393
394 nesfmr->leaf_pbl_cnt = nesfmr->nesmr.pbls_used-1;
395 nesfmr->root_vpbl.leaf_vpbl = kzalloc(sizeof(*nesfmr->root_vpbl.leaf_vpbl)*1024, GFP_ATOMIC);
396 if (!nesfmr->root_vpbl.leaf_vpbl) {
397 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
398 ret = -ENOMEM;
399 goto failed_leaf_vpbl_alloc;
400 }
401
402 nes_debug(NES_DBG_MR, "two level pbl, root_vpbl.pbl_vbase=%p"
403 " leaf_pbl_cnt=%d root_vpbl.leaf_vpbl=%p\n",
404 nesfmr->root_vpbl.pbl_vbase, nesfmr->leaf_pbl_cnt, nesfmr->root_vpbl.leaf_vpbl);
405
406 for (i=0; i<nesfmr->leaf_pbl_cnt; i++)
407 nesfmr->root_vpbl.leaf_vpbl[i].pbl_vbase = NULL;
408
409 for (i=0; i<nesfmr->leaf_pbl_cnt; i++) {
410 vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
411 &vpbl.pbl_pbase);
412
413 if (!vpbl.pbl_vbase) {
414 ret = -ENOMEM;
415 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
416 goto failed_leaf_vpbl_pages_alloc;
417 }
418
419 nesfmr->root_vpbl.pbl_vbase[i].pa_low = cpu_to_le32((u32)vpbl.pbl_pbase);
420 nesfmr->root_vpbl.pbl_vbase[i].pa_high = cpu_to_le32((u32)((((u64)vpbl.pbl_pbase)>>32)));
421 nesfmr->root_vpbl.leaf_vpbl[i] = vpbl;
422
423 nes_debug(NES_DBG_MR, "pbase_low=0x%x, pbase_high=0x%x, vpbl=%p\n",
424 nesfmr->root_vpbl.pbl_vbase[i].pa_low,
425 nesfmr->root_vpbl.pbl_vbase[i].pa_high,
426 &nesfmr->root_vpbl.leaf_vpbl[i]);
427 }
428 }
429 nesfmr->ib_qp = NULL;
430 nesfmr->access_rights =0;
431 292
432 stag = stag_index << 8;
433 stag |= driver_key;
434 stag += (u32)stag_key;
435 293
436 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
437 cqp_request = nes_get_cqp_request(nesdev); 294 cqp_request = nes_get_cqp_request(nesdev);
438 if (cqp_request == NULL) { 295 if (cqp_request == NULL) {
439 nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); 296 nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n");
440 ret = -ENOMEM; 297 return -ENOMEM;
441 goto failed_leaf_vpbl_pages_alloc;
442 } 298 }
299 nes_debug(NES_DBG_MR, "alloc_fast_reg_mr: page_count = %d, "
300 "region_length = %llu\n",
301 page_count, region_length);
443 cqp_request->waiting = 1; 302 cqp_request->waiting = 1;
444 cqp_wqe = &cqp_request->cqp_wqe; 303 cqp_wqe = &cqp_request->cqp_wqe;
445 304
446 nes_debug(NES_DBG_MR, "Registering STag 0x%08X, index = 0x%08X\n", 305 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
447 stag, stag_index); 306 if (nesadapter->free_4kpbl > 0) {
448 307 nesadapter->free_4kpbl--;
449 opcode = NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_VA_TO | NES_CQP_STAG_MR; 308 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
450 309 } else {
451 if (nesfmr->nesmr.pbl_4k == 1) 310 /* No 4kpbl's available: */
452 opcode |= NES_CQP_STAG_PBL_BLK_SIZE; 311 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
453 312 nes_debug(NES_DBG_MR, "Out of Pbls\n");
454 if (ibmr_access_flags & IB_ACCESS_REMOTE_WRITE) { 313 nes_free_cqp_request(nesdev, cqp_request);
455 opcode |= NES_CQP_STAG_RIGHTS_REMOTE_WRITE | 314 return -ENOMEM;
456 NES_CQP_STAG_RIGHTS_LOCAL_WRITE | NES_CQP_STAG_REM_ACC_EN;
457 nesfmr->access_rights |=
458 NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_RIGHTS_LOCAL_WRITE |
459 NES_CQP_STAG_REM_ACC_EN;
460 } 315 }
461 316
462 if (ibmr_access_flags & IB_ACCESS_REMOTE_READ) { 317 opcode = NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_MR |
463 opcode |= NES_CQP_STAG_RIGHTS_REMOTE_READ | 318 NES_CQP_STAG_PBL_BLK_SIZE | NES_CQP_STAG_VA_TO |
464 NES_CQP_STAG_RIGHTS_LOCAL_READ | NES_CQP_STAG_REM_ACC_EN; 319 NES_CQP_STAG_REM_ACC_EN;
465 nesfmr->access_rights |= 320 /*
466 NES_CQP_STAG_RIGHTS_REMOTE_READ | NES_CQP_STAG_RIGHTS_LOCAL_READ | 321 * The current OFED API does not support the zero based TO option.
467 NES_CQP_STAG_REM_ACC_EN; 322 * If added then need to changed the NES_CQP_STAG_VA* option. Also,
468 } 323 * the API does not support that ability to have the MR set for local
324 * access only when created and not allow the SQ op to override. Given
325 * this the remote enable must be set here.
326 */
469 327
470 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 328 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
471 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); 329 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode);
472 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX, (nespd->pd_id & 0x00007fff)); 330 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, 1);
473 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag);
474 331
475 cqp_wqe->wqe_words[NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX] = 332 cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] =
476 cpu_to_le32((nesfmr->nesmr.pbls_used>1) ? 333 cpu_to_le32((u32)(region_length >> 8) & 0xff000000);
477 (nesfmr->nesmr.pbls_used-1) : nesfmr->nesmr.pbls_used); 334 cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] |=
335 cpu_to_le32(nespd->pd_id & 0x00007fff);
336
337 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag);
338 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_VA_LOW_IDX, 0);
339 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_LOW_IDX, 0);
340 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, 0);
341 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (page_count * 8));
342 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE);
343 barrier();
478 344
479 atomic_set(&cqp_request->refcount, 2); 345 atomic_set(&cqp_request->refcount, 2);
480 nes_post_cqp_request(nesdev, cqp_request); 346 nes_post_cqp_request(nesdev, cqp_request);
481 347
482 /* Wait for CQP */ 348 /* Wait for CQP */
483 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), 349 ret = wait_event_timeout(cqp_request->waitq,
484 NES_EVENT_TIMEOUT); 350 (0 != cqp_request->request_done),
485 nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u," 351 NES_EVENT_TIMEOUT);
486 " CQP Major:Minor codes = 0x%04X:0x%04X.\n", 352
487 stag, ret, cqp_request->major_code, cqp_request->minor_code); 353 nes_debug(NES_DBG_MR, "Allocate STag 0x%08X completed, "
488 354 "wait_event_timeout ret = %u, CQP Major:Minor codes = "
489 if ((!ret) || (cqp_request->major_code)) { 355 "0x%04X:0x%04X.\n", stag, ret, cqp_request->major_code,
490 nes_put_cqp_request(nesdev, cqp_request); 356 cqp_request->minor_code);
491 ret = (!ret) ? -ETIME : -EIO; 357 major_code = cqp_request->major_code;
492 goto failed_leaf_vpbl_pages_alloc;
493 }
494 nes_put_cqp_request(nesdev, cqp_request); 358 nes_put_cqp_request(nesdev, cqp_request);
495 nesfmr->nesmr.ibfmr.lkey = stag;
496 nesfmr->nesmr.ibfmr.rkey = stag;
497 nesfmr->attr = *ibfmr_attr;
498
499 return &nesfmr->nesmr.ibfmr;
500
501 failed_leaf_vpbl_pages_alloc:
502 /* unroll all allocated pages */
503 for (i=0; i<nesfmr->leaf_pbl_cnt; i++) {
504 if (nesfmr->root_vpbl.leaf_vpbl[i].pbl_vbase) {
505 pci_free_consistent(nesdev->pcidev, 4096, nesfmr->root_vpbl.leaf_vpbl[i].pbl_vbase,
506 nesfmr->root_vpbl.leaf_vpbl[i].pbl_pbase);
507 }
508 }
509 if (nesfmr->root_vpbl.leaf_vpbl)
510 kfree(nesfmr->root_vpbl.leaf_vpbl);
511 359
512 failed_leaf_vpbl_alloc: 360 if (!ret || major_code) {
513 if (nesfmr->leaf_pbl_cnt == 0) {
514 if (nesfmr->root_vpbl.pbl_vbase)
515 pci_free_consistent(nesdev->pcidev, 4096, nesfmr->root_vpbl.pbl_vbase,
516 nesfmr->root_vpbl.pbl_pbase);
517 } else
518 pci_free_consistent(nesdev->pcidev, 8192, nesfmr->root_vpbl.pbl_vbase,
519 nesfmr->root_vpbl.pbl_pbase);
520
521 failed_vpbl_alloc:
522 if (nesfmr->nesmr.pbls_used != 0) {
523 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 361 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
524 if (nesfmr->nesmr.pbl_4k) 362 nesadapter->free_4kpbl++;
525 nesadapter->free_4kpbl += nesfmr->nesmr.pbls_used;
526 else
527 nesadapter->free_256pbl += nesfmr->nesmr.pbls_used;
528 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 363 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
529 } 364 }
530 365
531failed_vpbl_avail: 366 if (!ret)
532 kfree(nesfmr); 367 return -ETIME;
533 368 else if (major_code)
534 failed_fmr_alloc: 369 return -EIO;
535 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 370 return 0;
536
537 failed_resource_alloc:
538 return ERR_PTR(ret);
539} 371}
540 372
541 373/*
542/** 374 * nes_alloc_fast_reg_mr
543 * nes_dealloc_fmr
544 */ 375 */
545static int nes_dealloc_fmr(struct ib_fmr *ibfmr) 376struct ib_mr *nes_alloc_fast_reg_mr(struct ib_pd *ibpd, int max_page_list_len)
546{ 377{
547 unsigned long flags; 378 struct nes_pd *nespd = to_nespd(ibpd);
548 struct nes_mr *nesmr = to_nesmr_from_ibfmr(ibfmr); 379 struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
549 struct nes_fmr *nesfmr = to_nesfmr(nesmr);
550 struct nes_vnic *nesvnic = to_nesvnic(ibfmr->device);
551 struct nes_device *nesdev = nesvnic->nesdev; 380 struct nes_device *nesdev = nesvnic->nesdev;
552 struct nes_adapter *nesadapter = nesdev->nesadapter; 381 struct nes_adapter *nesadapter = nesdev->nesadapter;
553 int i = 0;
554 int rc;
555 382
556 /* free the resources */ 383 u32 next_stag_index;
557 if (nesfmr->leaf_pbl_cnt == 0) { 384 u8 stag_key = 0;
558 /* single PBL case */ 385 u32 driver_key = 0;
559 if (nesfmr->root_vpbl.pbl_vbase) 386 int err = 0;
560 pci_free_consistent(nesdev->pcidev, 4096, nesfmr->root_vpbl.pbl_vbase, 387 u32 stag_index = 0;
561 nesfmr->root_vpbl.pbl_pbase); 388 struct nes_mr *nesmr;
562 } else { 389 u32 stag;
563 for (i = 0; i < nesfmr->leaf_pbl_cnt; i++) { 390 int ret;
564 pci_free_consistent(nesdev->pcidev, 4096, nesfmr->root_vpbl.leaf_vpbl[i].pbl_vbase, 391 struct ib_mr *ibmr;
565 nesfmr->root_vpbl.leaf_vpbl[i].pbl_pbase); 392/*
566 } 393 * Note: Set to always use a fixed length single page entry PBL. This is to allow
567 kfree(nesfmr->root_vpbl.leaf_vpbl); 394 * for the fast_reg_mr operation to always know the size of the PBL.
568 pci_free_consistent(nesdev->pcidev, 8192, nesfmr->root_vpbl.pbl_vbase, 395 */
569 nesfmr->root_vpbl.pbl_pbase); 396 if (max_page_list_len > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64)))
570 } 397 return ERR_PTR(-E2BIG);
571 nesmr->ibmw.device = ibfmr->device;
572 nesmr->ibmw.pd = ibfmr->pd;
573 nesmr->ibmw.rkey = ibfmr->rkey;
574 nesmr->ibmw.uobject = NULL;
575 398
576 rc = nes_dealloc_mw(&nesmr->ibmw); 399 get_random_bytes(&next_stag_index, sizeof(next_stag_index));
400 stag_key = (u8)next_stag_index;
401 next_stag_index >>= 8;
402 next_stag_index %= nesadapter->max_mr;
577 403
578 if ((rc == 0) && (nesfmr->nesmr.pbls_used != 0)) { 404 err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs,
579 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 405 nesadapter->max_mr, &stag_index,
580 if (nesfmr->nesmr.pbl_4k) { 406 &next_stag_index);
581 nesadapter->free_4kpbl += nesfmr->nesmr.pbls_used; 407 if (err)
582 WARN_ON(nesadapter->free_4kpbl > nesadapter->max_4kpbl); 408 return ERR_PTR(err);
583 } else { 409
584 nesadapter->free_256pbl += nesfmr->nesmr.pbls_used; 410 nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL);
585 WARN_ON(nesadapter->free_256pbl > nesadapter->max_256pbl); 411 if (!nesmr) {
586 } 412 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
587 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 413 return ERR_PTR(-ENOMEM);
588 } 414 }
589 415
590 return rc; 416 stag = stag_index << 8;
591} 417 stag |= driver_key;
418 stag += (u32)stag_key;
592 419
420 nes_debug(NES_DBG_MR, "Allocating STag 0x%08X index = 0x%08X\n",
421 stag, stag_index);
593 422
594/** 423 ret = alloc_fast_reg_mr(nesdev, nespd, stag, max_page_list_len);
595 * nes_map_phys_fmr 424
425 if (ret == 0) {
426 nesmr->ibmr.rkey = stag;
427 nesmr->ibmr.lkey = stag;
428 nesmr->mode = IWNES_MEMREG_TYPE_FMEM;
429 ibmr = &nesmr->ibmr;
430 } else {
431 kfree(nesmr);
432 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
433 ibmr = ERR_PTR(-ENOMEM);
434 }
435 return ibmr;
436}
437
438/*
439 * nes_alloc_fast_reg_page_list
596 */ 440 */
597static int nes_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, 441static struct ib_fast_reg_page_list *nes_alloc_fast_reg_page_list(
598 int list_len, u64 iova) 442 struct ib_device *ibdev,
443 int page_list_len)
599{ 444{
600 return 0; 445 struct nes_vnic *nesvnic = to_nesvnic(ibdev);
601} 446 struct nes_device *nesdev = nesvnic->nesdev;
447 struct ib_fast_reg_page_list *pifrpl;
448 struct nes_ib_fast_reg_page_list *pnesfrpl;
602 449
450 if (page_list_len > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64)))
451 return ERR_PTR(-E2BIG);
452 /*
453 * Allocate the ib_fast_reg_page_list structure, the
454 * nes_fast_bpl structure, and the PLB table.
455 */
456 pnesfrpl = kmalloc(sizeof(struct nes_ib_fast_reg_page_list) +
457 page_list_len * sizeof(u64), GFP_KERNEL);
458
459 if (!pnesfrpl)
460 return ERR_PTR(-ENOMEM);
603 461
604/** 462 pifrpl = &pnesfrpl->ibfrpl;
605 * nes_unmap_frm 463 pifrpl->page_list = &pnesfrpl->pbl;
464 pifrpl->max_page_list_len = page_list_len;
465 /*
466 * Allocate the WQE PBL
467 */
468 pnesfrpl->nes_wqe_pbl.kva = pci_alloc_consistent(nesdev->pcidev,
469 page_list_len * sizeof(u64),
470 &pnesfrpl->nes_wqe_pbl.paddr);
471
472 if (!pnesfrpl->nes_wqe_pbl.kva) {
473 kfree(pnesfrpl);
474 return ERR_PTR(-ENOMEM);
475 }
476 nes_debug(NES_DBG_MR, "nes_alloc_fast_reg_pbl: nes_frpl = %p, "
477 "ibfrpl = %p, ibfrpl.page_list = %p, pbl.kva = %p, "
478 "pbl.paddr= %p\n", pnesfrpl, &pnesfrpl->ibfrpl,
479 pnesfrpl->ibfrpl.page_list, pnesfrpl->nes_wqe_pbl.kva,
480 (void *)pnesfrpl->nes_wqe_pbl.paddr);
481
482 return pifrpl;
483}
484
485/*
486 * nes_free_fast_reg_page_list
606 */ 487 */
607static int nes_unmap_fmr(struct list_head *ibfmr_list) 488static void nes_free_fast_reg_page_list(struct ib_fast_reg_page_list *pifrpl)
608{ 489{
609 return 0; 490 struct nes_vnic *nesvnic = to_nesvnic(pifrpl->device);
491 struct nes_device *nesdev = nesvnic->nesdev;
492 struct nes_ib_fast_reg_page_list *pnesfrpl;
493
494 pnesfrpl = container_of(pifrpl, struct nes_ib_fast_reg_page_list, ibfrpl);
495 /*
496 * Free the WQE PBL.
497 */
498 pci_free_consistent(nesdev->pcidev,
499 pifrpl->max_page_list_len * sizeof(u64),
500 pnesfrpl->nes_wqe_pbl.kva,
501 pnesfrpl->nes_wqe_pbl.paddr);
502 /*
503 * Free the PBL structure
504 */
505 kfree(pnesfrpl);
610} 506}
611 507
612
613
614/** 508/**
615 * nes_query_device 509 * nes_query_device
616 */ 510 */
@@ -633,23 +527,23 @@ static int nes_query_device(struct ib_device *ibdev, struct ib_device_attr *prop
633 props->max_qp_wr = nesdev->nesadapter->max_qp_wr - 2; 527 props->max_qp_wr = nesdev->nesadapter->max_qp_wr - 2;
634 props->max_sge = nesdev->nesadapter->max_sge; 528 props->max_sge = nesdev->nesadapter->max_sge;
635 props->max_cq = nesibdev->max_cq; 529 props->max_cq = nesibdev->max_cq;
636 props->max_cqe = nesdev->nesadapter->max_cqe - 1; 530 props->max_cqe = nesdev->nesadapter->max_cqe;
637 props->max_mr = nesibdev->max_mr; 531 props->max_mr = nesibdev->max_mr;
638 props->max_mw = nesibdev->max_mr; 532 props->max_mw = nesibdev->max_mr;
639 props->max_pd = nesibdev->max_pd; 533 props->max_pd = nesibdev->max_pd;
640 props->max_sge_rd = 1; 534 props->max_sge_rd = 1;
641 switch (nesdev->nesadapter->max_irrq_wr) { 535 switch (nesdev->nesadapter->max_irrq_wr) {
642 case 0: 536 case 0:
643 props->max_qp_rd_atom = 1; 537 props->max_qp_rd_atom = 2;
644 break; 538 break;
645 case 1: 539 case 1:
646 props->max_qp_rd_atom = 4; 540 props->max_qp_rd_atom = 8;
647 break; 541 break;
648 case 2: 542 case 2:
649 props->max_qp_rd_atom = 16; 543 props->max_qp_rd_atom = 32;
650 break; 544 break;
651 case 3: 545 case 3:
652 props->max_qp_rd_atom = 32; 546 props->max_qp_rd_atom = 64;
653 break; 547 break;
654 default: 548 default:
655 props->max_qp_rd_atom = 0; 549 props->max_qp_rd_atom = 0;
@@ -1121,6 +1015,7 @@ static int nes_setup_virt_qp(struct nes_qp *nesqp, struct nes_pbl *nespbl,
1121 kunmap(nesqp->page); 1015 kunmap(nesqp->page);
1122 return -ENOMEM; 1016 return -ENOMEM;
1123 } 1017 }
1018 nesqp->sq_kmapped = 1;
1124 nesqp->hwqp.q2_vbase = mem; 1019 nesqp->hwqp.q2_vbase = mem;
1125 mem += 256; 1020 mem += 256;
1126 memset(nesqp->hwqp.q2_vbase, 0, 256); 1021 memset(nesqp->hwqp.q2_vbase, 0, 256);
@@ -1198,7 +1093,10 @@ static inline void nes_free_qp_mem(struct nes_device *nesdev,
1198 pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase); 1093 pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase);
1199 pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase ); 1094 pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase );
1200 nesqp->pbl_vbase = NULL; 1095 nesqp->pbl_vbase = NULL;
1201 kunmap(nesqp->page); 1096 if (nesqp->sq_kmapped) {
1097 nesqp->sq_kmapped = 0;
1098 kunmap(nesqp->page);
1099 }
1202 } 1100 }
1203} 1101}
1204 1102
@@ -1504,8 +1402,6 @@ static struct ib_qp *nes_create_qp(struct ib_pd *ibpd,
1504 nes_debug(NES_DBG_QP, "QP%u structure located @%p.Size = %u.\n", 1402 nes_debug(NES_DBG_QP, "QP%u structure located @%p.Size = %u.\n",
1505 nesqp->hwqp.qp_id, nesqp, (u32)sizeof(*nesqp)); 1403 nesqp->hwqp.qp_id, nesqp, (u32)sizeof(*nesqp));
1506 spin_lock_init(&nesqp->lock); 1404 spin_lock_init(&nesqp->lock);
1507 init_waitqueue_head(&nesqp->state_waitq);
1508 init_waitqueue_head(&nesqp->kick_waitq);
1509 nes_add_ref(&nesqp->ibqp); 1405 nes_add_ref(&nesqp->ibqp);
1510 break; 1406 break;
1511 default: 1407 default:
@@ -1513,6 +1409,8 @@ static struct ib_qp *nes_create_qp(struct ib_pd *ibpd,
1513 return ERR_PTR(-EINVAL); 1409 return ERR_PTR(-EINVAL);
1514 } 1410 }
1515 1411
1412 nesqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR);
1413
1516 /* update the QP table */ 1414 /* update the QP table */
1517 nesdev->nesadapter->qp_table[nesqp->hwqp.qp_id-NES_FIRST_QPN] = nesqp; 1415 nesdev->nesadapter->qp_table[nesqp->hwqp.qp_id-NES_FIRST_QPN] = nesqp;
1518 nes_debug(NES_DBG_QP, "netdev refcnt=%u\n", 1416 nes_debug(NES_DBG_QP, "netdev refcnt=%u\n",
@@ -1607,8 +1505,10 @@ static int nes_destroy_qp(struct ib_qp *ibqp)
1607 nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index; 1505 nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index;
1608 } 1506 }
1609 } 1507 }
1610 if (nesqp->pbl_pbase) 1508 if (nesqp->pbl_pbase && nesqp->sq_kmapped) {
1509 nesqp->sq_kmapped = 0;
1611 kunmap(nesqp->page); 1510 kunmap(nesqp->page);
1511 }
1612 } else { 1512 } else {
1613 /* Clean any pending completions from the cq(s) */ 1513 /* Clean any pending completions from the cq(s) */
1614 if (nesqp->nesscq) 1514 if (nesqp->nesscq)
@@ -1649,6 +1549,9 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev, int entries,
1649 unsigned long flags; 1549 unsigned long flags;
1650 int ret; 1550 int ret;
1651 1551
1552 if (entries > nesadapter->max_cqe)
1553 return ERR_PTR(-EINVAL);
1554
1652 err = nes_alloc_resource(nesadapter, nesadapter->allocated_cqs, 1555 err = nes_alloc_resource(nesadapter, nesadapter->allocated_cqs,
1653 nesadapter->max_cq, &cq_num, &nesadapter->next_cq); 1556 nesadapter->max_cq, &cq_num, &nesadapter->next_cq);
1654 if (err) { 1557 if (err) {
@@ -2606,9 +2509,6 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2606 stag = stag_index << 8; 2509 stag = stag_index << 8;
2607 stag |= driver_key; 2510 stag |= driver_key;
2608 stag += (u32)stag_key; 2511 stag += (u32)stag_key;
2609 if (stag == 0) {
2610 stag = 1;
2611 }
2612 2512
2613 iova_start = virt; 2513 iova_start = virt;
2614 /* Make the leaf PBL the root if only one PBL */ 2514 /* Make the leaf PBL the root if only one PBL */
@@ -3109,7 +3009,6 @@ int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
3109 " already done based on hw state.\n", 3009 " already done based on hw state.\n",
3110 nesqp->hwqp.qp_id); 3010 nesqp->hwqp.qp_id);
3111 issue_modify_qp = 0; 3011 issue_modify_qp = 0;
3112 nesqp->in_disconnect = 0;
3113 } 3012 }
3114 switch (nesqp->hw_iwarp_state) { 3013 switch (nesqp->hw_iwarp_state) {
3115 case NES_AEQE_IWARP_STATE_CLOSING: 3014 case NES_AEQE_IWARP_STATE_CLOSING:
@@ -3122,7 +3021,6 @@ int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
3122 break; 3021 break;
3123 default: 3022 default:
3124 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING; 3023 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3125 nesqp->in_disconnect = 1;
3126 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING; 3024 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3127 break; 3025 break;
3128 } 3026 }
@@ -3139,7 +3037,6 @@ int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
3139 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE; 3037 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
3140 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE; 3038 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
3141 issue_modify_qp = 1; 3039 issue_modify_qp = 1;
3142 nesqp->in_disconnect = 1;
3143 break; 3040 break;
3144 case IB_QPS_ERR: 3041 case IB_QPS_ERR:
3145 case IB_QPS_RESET: 3042 case IB_QPS_RESET:
@@ -3162,7 +3059,6 @@ int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
3162 if ((nesqp->hw_tcp_state > NES_AEQE_TCP_STATE_CLOSED) && 3059 if ((nesqp->hw_tcp_state > NES_AEQE_TCP_STATE_CLOSED) &&
3163 (nesqp->hw_tcp_state != NES_AEQE_TCP_STATE_TIME_WAIT)) { 3060 (nesqp->hw_tcp_state != NES_AEQE_TCP_STATE_TIME_WAIT)) {
3164 next_iwarp_state |= NES_CQP_QP_RESET; 3061 next_iwarp_state |= NES_CQP_QP_RESET;
3165 nesqp->in_disconnect = 1;
3166 } else { 3062 } else {
3167 nes_debug(NES_DBG_MOD_QP, "QP%u NOT setting NES_CQP_QP_RESET since TCP state = %u\n", 3063 nes_debug(NES_DBG_MOD_QP, "QP%u NOT setting NES_CQP_QP_RESET since TCP state = %u\n",
3168 nesqp->hwqp.qp_id, nesqp->hw_tcp_state); 3064 nesqp->hwqp.qp_id, nesqp->hw_tcp_state);
@@ -3373,21 +3269,17 @@ static int nes_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
3373 struct nes_device *nesdev = nesvnic->nesdev; 3269 struct nes_device *nesdev = nesvnic->nesdev;
3374 struct nes_qp *nesqp = to_nesqp(ibqp); 3270 struct nes_qp *nesqp = to_nesqp(ibqp);
3375 struct nes_hw_qp_wqe *wqe; 3271 struct nes_hw_qp_wqe *wqe;
3376 int err; 3272 int err = 0;
3377 u32 qsize = nesqp->hwqp.sq_size; 3273 u32 qsize = nesqp->hwqp.sq_size;
3378 u32 head; 3274 u32 head;
3379 u32 wqe_misc; 3275 u32 wqe_misc = 0;
3380 u32 wqe_count; 3276 u32 wqe_count = 0;
3381 u32 counter; 3277 u32 counter;
3382 u32 total_payload_length;
3383
3384 err = 0;
3385 wqe_misc = 0;
3386 wqe_count = 0;
3387 total_payload_length = 0;
3388 3278
3389 if (nesqp->ibqp_state > IB_QPS_RTS) 3279 if (nesqp->ibqp_state > IB_QPS_RTS) {
3390 return -EINVAL; 3280 err = -EINVAL;
3281 goto out;
3282 }
3391 3283
3392 spin_lock_irqsave(&nesqp->lock, flags); 3284 spin_lock_irqsave(&nesqp->lock, flags);
3393 3285
@@ -3413,94 +3305,208 @@ static int nes_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
3413 u64temp = (u64)(ib_wr->wr_id); 3305 u64temp = (u64)(ib_wr->wr_id);
3414 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX, 3306 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX,
3415 u64temp); 3307 u64temp);
3416 switch (ib_wr->opcode) { 3308 switch (ib_wr->opcode) {
3417 case IB_WR_SEND: 3309 case IB_WR_SEND:
3418 if (ib_wr->send_flags & IB_SEND_SOLICITED) { 3310 case IB_WR_SEND_WITH_INV:
3419 wqe_misc = NES_IWARP_SQ_OP_SENDSE; 3311 if (IB_WR_SEND == ib_wr->opcode) {
3420 } else { 3312 if (ib_wr->send_flags & IB_SEND_SOLICITED)
3421 wqe_misc = NES_IWARP_SQ_OP_SEND; 3313 wqe_misc = NES_IWARP_SQ_OP_SENDSE;
3422 } 3314 else
3423 if (ib_wr->num_sge > nesdev->nesadapter->max_sge) { 3315 wqe_misc = NES_IWARP_SQ_OP_SEND;
3424 err = -EINVAL; 3316 } else {
3425 break; 3317 if (ib_wr->send_flags & IB_SEND_SOLICITED)
3426 } 3318 wqe_misc = NES_IWARP_SQ_OP_SENDSEINV;
3427 if (ib_wr->send_flags & IB_SEND_FENCE) { 3319 else
3428 wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE; 3320 wqe_misc = NES_IWARP_SQ_OP_SENDINV;
3429 }
3430 if ((ib_wr->send_flags & IB_SEND_INLINE) &&
3431 ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
3432 (ib_wr->sg_list[0].length <= 64)) {
3433 memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
3434 (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
3435 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX,
3436 ib_wr->sg_list[0].length);
3437 wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA;
3438 } else {
3439 fill_wqe_sg_send(wqe, ib_wr, 1);
3440 }
3441 3321
3442 break; 3322 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX,
3443 case IB_WR_RDMA_WRITE: 3323 ib_wr->ex.invalidate_rkey);
3444 wqe_misc = NES_IWARP_SQ_OP_RDMAW; 3324 }
3445 if (ib_wr->num_sge > nesdev->nesadapter->max_sge) {
3446 nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=%u\n",
3447 ib_wr->num_sge,
3448 nesdev->nesadapter->max_sge);
3449 err = -EINVAL;
3450 break;
3451 }
3452 if (ib_wr->send_flags & IB_SEND_FENCE) {
3453 wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE;
3454 }
3455 3325
3456 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX, 3326 if (ib_wr->num_sge > nesdev->nesadapter->max_sge) {
3457 ib_wr->wr.rdma.rkey); 3327 err = -EINVAL;
3458 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX, 3328 break;
3459 ib_wr->wr.rdma.remote_addr);
3460
3461 if ((ib_wr->send_flags & IB_SEND_INLINE) &&
3462 ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
3463 (ib_wr->sg_list[0].length <= 64)) {
3464 memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
3465 (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
3466 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX,
3467 ib_wr->sg_list[0].length);
3468 wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA;
3469 } else {
3470 fill_wqe_sg_send(wqe, ib_wr, 1);
3471 }
3472 wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] =
3473 wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX];
3474 break;
3475 case IB_WR_RDMA_READ:
3476 /* iWARP only supports 1 sge for RDMA reads */
3477 if (ib_wr->num_sge > 1) {
3478 nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=1\n",
3479 ib_wr->num_sge);
3480 err = -EINVAL;
3481 break;
3482 }
3483 wqe_misc = NES_IWARP_SQ_OP_RDMAR;
3484 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX,
3485 ib_wr->wr.rdma.remote_addr);
3486 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX,
3487 ib_wr->wr.rdma.rkey);
3488 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX,
3489 ib_wr->sg_list->length);
3490 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX,
3491 ib_wr->sg_list->addr);
3492 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX,
3493 ib_wr->sg_list->lkey);
3494 break;
3495 default:
3496 /* error */
3497 err = -EINVAL;
3498 break;
3499 } 3329 }
3500 3330
3501 if (ib_wr->send_flags & IB_SEND_SIGNALED) { 3331 if (ib_wr->send_flags & IB_SEND_FENCE)
3502 wqe_misc |= NES_IWARP_SQ_WQE_SIGNALED_COMPL; 3332 wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE;
3333
3334 if ((ib_wr->send_flags & IB_SEND_INLINE) &&
3335 ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
3336 (ib_wr->sg_list[0].length <= 64)) {
3337 memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
3338 (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
3339 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX,
3340 ib_wr->sg_list[0].length);
3341 wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA;
3342 } else {
3343 fill_wqe_sg_send(wqe, ib_wr, 1);
3344 }
3345
3346 break;
3347 case IB_WR_RDMA_WRITE:
3348 wqe_misc = NES_IWARP_SQ_OP_RDMAW;
3349 if (ib_wr->num_sge > nesdev->nesadapter->max_sge) {
3350 nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=%u\n",
3351 ib_wr->num_sge, nesdev->nesadapter->max_sge);
3352 err = -EINVAL;
3353 break;
3354 }
3355
3356 if (ib_wr->send_flags & IB_SEND_FENCE)
3357 wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE;
3358
3359 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX,
3360 ib_wr->wr.rdma.rkey);
3361 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX,
3362 ib_wr->wr.rdma.remote_addr);
3363
3364 if ((ib_wr->send_flags & IB_SEND_INLINE) &&
3365 ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
3366 (ib_wr->sg_list[0].length <= 64)) {
3367 memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
3368 (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
3369 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX,
3370 ib_wr->sg_list[0].length);
3371 wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA;
3372 } else {
3373 fill_wqe_sg_send(wqe, ib_wr, 1);
3374 }
3375
3376 wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] =
3377 wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX];
3378 break;
3379 case IB_WR_RDMA_READ:
3380 case IB_WR_RDMA_READ_WITH_INV:
3381 /* iWARP only supports 1 sge for RDMA reads */
3382 if (ib_wr->num_sge > 1) {
3383 nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=1\n",
3384 ib_wr->num_sge);
3385 err = -EINVAL;
3386 break;
3387 }
3388 if (ib_wr->opcode == IB_WR_RDMA_READ) {
3389 wqe_misc = NES_IWARP_SQ_OP_RDMAR;
3390 } else {
3391 wqe_misc = NES_IWARP_SQ_OP_RDMAR_LOCINV;
3392 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX,
3393 ib_wr->ex.invalidate_rkey);
3394 }
3395
3396 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX,
3397 ib_wr->wr.rdma.remote_addr);
3398 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX,
3399 ib_wr->wr.rdma.rkey);
3400 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX,
3401 ib_wr->sg_list->length);
3402 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX,
3403 ib_wr->sg_list->addr);
3404 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX,
3405 ib_wr->sg_list->lkey);
3406 break;
3407 case IB_WR_LOCAL_INV:
3408 wqe_misc = NES_IWARP_SQ_OP_LOCINV;
3409 set_wqe_32bit_value(wqe->wqe_words,
3410 NES_IWARP_SQ_LOCINV_WQE_INV_STAG_IDX,
3411 ib_wr->ex.invalidate_rkey);
3412 break;
3413 case IB_WR_FAST_REG_MR:
3414 {
3415 int i;
3416 int flags = ib_wr->wr.fast_reg.access_flags;
3417 struct nes_ib_fast_reg_page_list *pnesfrpl =
3418 container_of(ib_wr->wr.fast_reg.page_list,
3419 struct nes_ib_fast_reg_page_list,
3420 ibfrpl);
3421 u64 *src_page_list = pnesfrpl->ibfrpl.page_list;
3422 u64 *dst_page_list = pnesfrpl->nes_wqe_pbl.kva;
3423
3424 if (ib_wr->wr.fast_reg.page_list_len >
3425 (NES_4K_PBL_CHUNK_SIZE / sizeof(u64))) {
3426 nes_debug(NES_DBG_IW_TX, "SQ_FMR: bad page_list_len\n");
3427 err = -EINVAL;
3428 break;
3429 }
3430 wqe_misc = NES_IWARP_SQ_OP_FAST_REG;
3431 set_wqe_64bit_value(wqe->wqe_words,
3432 NES_IWARP_SQ_FMR_WQE_VA_FBO_LOW_IDX,
3433 ib_wr->wr.fast_reg.iova_start);
3434 set_wqe_32bit_value(wqe->wqe_words,
3435 NES_IWARP_SQ_FMR_WQE_LENGTH_LOW_IDX,
3436 ib_wr->wr.fast_reg.length);
3437 set_wqe_32bit_value(wqe->wqe_words,
3438 NES_IWARP_SQ_FMR_WQE_MR_STAG_IDX,
3439 ib_wr->wr.fast_reg.rkey);
3440 /* Set page size: */
3441 if (ib_wr->wr.fast_reg.page_shift == 12) {
3442 wqe_misc |= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_4K;
3443 } else if (ib_wr->wr.fast_reg.page_shift == 21) {
3444 wqe_misc |= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_2M;
3445 } else {
3446 nes_debug(NES_DBG_IW_TX, "Invalid page shift,"
3447 " ib_wr=%u, max=1\n", ib_wr->num_sge);
3448 err = -EINVAL;
3449 break;
3450 }
3451 /* Set access_flags */
3452 wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_READ;
3453 if (flags & IB_ACCESS_LOCAL_WRITE)
3454 wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_WRITE;
3455
3456 if (flags & IB_ACCESS_REMOTE_WRITE)
3457 wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_WRITE;
3458
3459 if (flags & IB_ACCESS_REMOTE_READ)
3460 wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_READ;
3461
3462 if (flags & IB_ACCESS_MW_BIND)
3463 wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_WINDOW_BIND;
3464
3465 /* Fill in PBL info: */
3466 if (ib_wr->wr.fast_reg.page_list_len >
3467 pnesfrpl->ibfrpl.max_page_list_len) {
3468 nes_debug(NES_DBG_IW_TX, "Invalid page list length,"
3469 " ib_wr=%p, value=%u, max=%u\n",
3470 ib_wr, ib_wr->wr.fast_reg.page_list_len,
3471 pnesfrpl->ibfrpl.max_page_list_len);
3472 err = -EINVAL;
3473 break;
3474 }
3475
3476 set_wqe_64bit_value(wqe->wqe_words,
3477 NES_IWARP_SQ_FMR_WQE_PBL_ADDR_LOW_IDX,
3478 pnesfrpl->nes_wqe_pbl.paddr);
3479
3480 set_wqe_32bit_value(wqe->wqe_words,
3481 NES_IWARP_SQ_FMR_WQE_PBL_LENGTH_IDX,
3482 ib_wr->wr.fast_reg.page_list_len * 8);
3483
3484 for (i = 0; i < ib_wr->wr.fast_reg.page_list_len; i++)
3485 dst_page_list[i] = cpu_to_le64(src_page_list[i]);
3486
3487 nes_debug(NES_DBG_IW_TX, "SQ_FMR: iova_start: %p, "
3488 "length: %d, rkey: %0x, pgl_paddr: %p, "
3489 "page_list_len: %u, wqe_misc: %x\n",
3490 (void *)ib_wr->wr.fast_reg.iova_start,
3491 ib_wr->wr.fast_reg.length,
3492 ib_wr->wr.fast_reg.rkey,
3493 (void *)pnesfrpl->nes_wqe_pbl.paddr,
3494 ib_wr->wr.fast_reg.page_list_len,
3495 wqe_misc);
3496 break;
3497 }
3498 default:
3499 /* error */
3500 err = -EINVAL;
3501 break;
3503 } 3502 }
3503
3504 if (err)
3505 break;
3506
3507 if ((ib_wr->send_flags & IB_SEND_SIGNALED) || nesqp->sig_all)
3508 wqe_misc |= NES_IWARP_SQ_WQE_SIGNALED_COMPL;
3509
3504 wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = cpu_to_le32(wqe_misc); 3510 wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = cpu_to_le32(wqe_misc);
3505 3511
3506 ib_wr = ib_wr->next; 3512 ib_wr = ib_wr->next;
@@ -3522,6 +3528,7 @@ static int nes_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
3522 3528
3523 spin_unlock_irqrestore(&nesqp->lock, flags); 3529 spin_unlock_irqrestore(&nesqp->lock, flags);
3524 3530
3531out:
3525 if (err) 3532 if (err)
3526 *bad_wr = ib_wr; 3533 *bad_wr = ib_wr;
3527 return err; 3534 return err;
@@ -3548,8 +3555,10 @@ static int nes_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
3548 u32 counter; 3555 u32 counter;
3549 u32 total_payload_length; 3556 u32 total_payload_length;
3550 3557
3551 if (nesqp->ibqp_state > IB_QPS_RTS) 3558 if (nesqp->ibqp_state > IB_QPS_RTS) {
3552 return -EINVAL; 3559 err = -EINVAL;
3560 goto out;
3561 }
3553 3562
3554 spin_lock_irqsave(&nesqp->lock, flags); 3563 spin_lock_irqsave(&nesqp->lock, flags);
3555 3564
@@ -3612,6 +3621,7 @@ static int nes_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
3612 3621
3613 spin_unlock_irqrestore(&nesqp->lock, flags); 3622 spin_unlock_irqrestore(&nesqp->lock, flags);
3614 3623
3624out:
3615 if (err) 3625 if (err)
3616 *bad_wr = ib_wr; 3626 *bad_wr = ib_wr;
3617 return err; 3627 return err;
@@ -3720,6 +3730,12 @@ static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)
3720 nes_debug(NES_DBG_CQ, "Operation = Send.\n"); 3730 nes_debug(NES_DBG_CQ, "Operation = Send.\n");
3721 entry->opcode = IB_WC_SEND; 3731 entry->opcode = IB_WC_SEND;
3722 break; 3732 break;
3733 case NES_IWARP_SQ_OP_LOCINV:
3734 entry->opcode = IB_WR_LOCAL_INV;
3735 break;
3736 case NES_IWARP_SQ_OP_FAST_REG:
3737 entry->opcode = IB_WC_FAST_REG_MR;
3738 break;
3723 } 3739 }
3724 3740
3725 nesqp->hwqp.sq_tail = (wqe_index+1)&(nesqp->hwqp.sq_size - 1); 3741 nesqp->hwqp.sq_tail = (wqe_index+1)&(nesqp->hwqp.sq_size - 1);
@@ -3890,10 +3906,9 @@ struct nes_ib_device *nes_init_ofa_device(struct net_device *netdev)
3890 nesibdev->ibdev.dealloc_mw = nes_dealloc_mw; 3906 nesibdev->ibdev.dealloc_mw = nes_dealloc_mw;
3891 nesibdev->ibdev.bind_mw = nes_bind_mw; 3907 nesibdev->ibdev.bind_mw = nes_bind_mw;
3892 3908
3893 nesibdev->ibdev.alloc_fmr = nes_alloc_fmr; 3909 nesibdev->ibdev.alloc_fast_reg_mr = nes_alloc_fast_reg_mr;
3894 nesibdev->ibdev.unmap_fmr = nes_unmap_fmr; 3910 nesibdev->ibdev.alloc_fast_reg_page_list = nes_alloc_fast_reg_page_list;
3895 nesibdev->ibdev.dealloc_fmr = nes_dealloc_fmr; 3911 nesibdev->ibdev.free_fast_reg_page_list = nes_free_fast_reg_page_list;
3896 nesibdev->ibdev.map_phys_fmr = nes_map_phys_fmr;
3897 3912
3898 nesibdev->ibdev.attach_mcast = nes_multicast_attach; 3913 nesibdev->ibdev.attach_mcast = nes_multicast_attach;
3899 nesibdev->ibdev.detach_mcast = nes_multicast_detach; 3914 nesibdev->ibdev.detach_mcast = nes_multicast_detach;
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-31 14:34:12 -0500