aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/scsi/be2iscsi/be_main.c
diff options
context:
space:
mode:
authorJayamohan Kallickal <jayamohank@serverengines.com>2009-10-23 02:22:33 -0400
committerJames Bottomley <James.Bottomley@suse.de>2009-12-04 13:00:38 -0500
commitbfead3b2cb4607c71831423c3ee97d22cd0c9dcb (patch)
tree13822ffd73826b315f8be0077c0dc9b65ab3ff86 /drivers/scsi/be2iscsi/be_main.c
parentb4a9c7ede96e90f7b1ec009ce7256059295e76df (diff)
[SCSI] be2iscsi: Adding msix and mcc_rings V3
This patch enables msix for be2iscsi. It also enables use of mcc_rings for fw commands. Since the mcc eq creation is dependent on msix I am sending as one patch Signed-off-by: Jayamohan Kallickal <jayamohank@serverengines.com> Signed-off-by: James Bottomley <James.Bottomley@suse.de>
Diffstat (limited to 'drivers/scsi/be2iscsi/be_main.c')
-rw-r--r--drivers/scsi/be2iscsi/be_main.c888
1 files changed, 656 insertions, 232 deletions
diff --git a/drivers/scsi/be2iscsi/be_main.c b/drivers/scsi/be2iscsi/be_main.c
index 4f1aca346e38..2c3e99eeff82 100644
--- a/drivers/scsi/be2iscsi/be_main.c
+++ b/drivers/scsi/be2iscsi/be_main.c
@@ -39,7 +39,7 @@
39 39
40static unsigned int be_iopoll_budget = 10; 40static unsigned int be_iopoll_budget = 10;
41static unsigned int be_max_phys_size = 64; 41static unsigned int be_max_phys_size = 64;
42static unsigned int enable_msix; 42static unsigned int enable_msix = 1;
43 43
44MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table); 44MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
45MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR); 45MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
@@ -58,6 +58,17 @@ static int beiscsi_slave_configure(struct scsi_device *sdev)
58 return 0; 58 return 0;
59} 59}
60 60
61/*------------------- PCI Driver operations and data ----------------- */
62static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
63 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
64 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
65 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
66 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
67 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID4) },
68 { 0 }
69};
70MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
71
61static struct scsi_host_template beiscsi_sht = { 72static struct scsi_host_template beiscsi_sht = {
62 .module = THIS_MODULE, 73 .module = THIS_MODULE,
63 .name = "ServerEngines 10Gbe open-iscsi Initiator Driver", 74 .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
@@ -76,16 +87,8 @@ static struct scsi_host_template beiscsi_sht = {
76 .cmd_per_lun = BEISCSI_CMD_PER_LUN, 87 .cmd_per_lun = BEISCSI_CMD_PER_LUN,
77 .use_clustering = ENABLE_CLUSTERING, 88 .use_clustering = ENABLE_CLUSTERING,
78}; 89};
79static struct scsi_transport_template *beiscsi_scsi_transport;
80 90
81/*------------------- PCI Driver operations and data ----------------- */ 91static struct scsi_transport_template *beiscsi_scsi_transport;
82static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
83 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
84 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
85 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
86 { 0 }
87};
88MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
89 92
90static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev) 93static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
91{ 94{
@@ -104,7 +107,6 @@ static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
104 shost->max_cmd_len = BEISCSI_MAX_CMD_LEN; 107 shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
105 shost->max_lun = BEISCSI_NUM_MAX_LUN; 108 shost->max_lun = BEISCSI_NUM_MAX_LUN;
106 shost->transportt = beiscsi_scsi_transport; 109 shost->transportt = beiscsi_scsi_transport;
107
108 phba = iscsi_host_priv(shost); 110 phba = iscsi_host_priv(shost);
109 memset(phba, 0, sizeof(*phba)); 111 memset(phba, 0, sizeof(*phba));
110 phba->shost = shost; 112 phba->shost = shost;
@@ -181,6 +183,7 @@ static int beiscsi_enable_pci(struct pci_dev *pcidev)
181 return ret; 183 return ret;
182 } 184 }
183 185
186 pci_set_master(pcidev);
184 if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) { 187 if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
185 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32)); 188 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
186 if (ret) { 189 if (ret) {
@@ -203,7 +206,6 @@ static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
203 status = beiscsi_map_pci_bars(phba, pdev); 206 status = beiscsi_map_pci_bars(phba, pdev);
204 if (status) 207 if (status)
205 return status; 208 return status;
206
207 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16; 209 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
208 mbox_mem_alloc->va = pci_alloc_consistent(pdev, 210 mbox_mem_alloc->va = pci_alloc_consistent(pdev,
209 mbox_mem_alloc->size, 211 mbox_mem_alloc->size,
@@ -219,6 +221,9 @@ static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
219 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16); 221 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
220 memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox)); 222 memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
221 spin_lock_init(&ctrl->mbox_lock); 223 spin_lock_init(&ctrl->mbox_lock);
224 spin_lock_init(&phba->ctrl.mcc_lock);
225 spin_lock_init(&phba->ctrl.mcc_cq_lock);
226
222 return status; 227 return status;
223} 228}
224 229
@@ -268,6 +273,113 @@ static void hwi_ring_eq_db(struct beiscsi_hba *phba,
268} 273}
269 274
270/** 275/**
276 * be_isr_mcc - The isr routine of the driver.
277 * @irq: Not used
278 * @dev_id: Pointer to host adapter structure
279 */
280static irqreturn_t be_isr_mcc(int irq, void *dev_id)
281{
282 struct beiscsi_hba *phba;
283 struct be_eq_entry *eqe = NULL;
284 struct be_queue_info *eq;
285 struct be_queue_info *mcc;
286 unsigned int num_eq_processed;
287 struct be_eq_obj *pbe_eq;
288 unsigned long flags;
289
290 pbe_eq = dev_id;
291 eq = &pbe_eq->q;
292 phba = pbe_eq->phba;
293 mcc = &phba->ctrl.mcc_obj.cq;
294 eqe = queue_tail_node(eq);
295 if (!eqe)
296 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
297
298 num_eq_processed = 0;
299
300 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
301 & EQE_VALID_MASK) {
302 if (((eqe->dw[offsetof(struct amap_eq_entry,
303 resource_id) / 32] &
304 EQE_RESID_MASK) >> 16) == mcc->id) {
305 spin_lock_irqsave(&phba->isr_lock, flags);
306 phba->todo_mcc_cq = 1;
307 spin_unlock_irqrestore(&phba->isr_lock, flags);
308 }
309 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
310 queue_tail_inc(eq);
311 eqe = queue_tail_node(eq);
312 num_eq_processed++;
313 }
314 if (phba->todo_mcc_cq)
315 queue_work(phba->wq, &phba->work_cqs);
316 if (num_eq_processed)
317 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
318
319 return IRQ_HANDLED;
320}
321
322/**
323 * be_isr_msix - The isr routine of the driver.
324 * @irq: Not used
325 * @dev_id: Pointer to host adapter structure
326 */
327static irqreturn_t be_isr_msix(int irq, void *dev_id)
328{
329 struct beiscsi_hba *phba;
330 struct be_eq_entry *eqe = NULL;
331 struct be_queue_info *eq;
332 struct be_queue_info *cq;
333 unsigned int num_eq_processed;
334 struct be_eq_obj *pbe_eq;
335 unsigned long flags;
336
337 pbe_eq = dev_id;
338 eq = &pbe_eq->q;
339 cq = pbe_eq->cq;
340 eqe = queue_tail_node(eq);
341 if (!eqe)
342 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
343
344 phba = pbe_eq->phba;
345 num_eq_processed = 0;
346 if (blk_iopoll_enabled) {
347 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
348 & EQE_VALID_MASK) {
349 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
350 blk_iopoll_sched(&pbe_eq->iopoll);
351
352 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
353 queue_tail_inc(eq);
354 eqe = queue_tail_node(eq);
355 num_eq_processed++;
356 }
357 if (num_eq_processed)
358 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
359
360 return IRQ_HANDLED;
361 } else {
362 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
363 & EQE_VALID_MASK) {
364 spin_lock_irqsave(&phba->isr_lock, flags);
365 phba->todo_cq = 1;
366 spin_unlock_irqrestore(&phba->isr_lock, flags);
367 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
368 queue_tail_inc(eq);
369 eqe = queue_tail_node(eq);
370 num_eq_processed++;
371 }
372 if (phba->todo_cq)
373 queue_work(phba->wq, &phba->work_cqs);
374
375 if (num_eq_processed)
376 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
377
378 return IRQ_HANDLED;
379 }
380}
381
382/**
271 * be_isr - The isr routine of the driver. 383 * be_isr - The isr routine of the driver.
272 * @irq: Not used 384 * @irq: Not used
273 * @dev_id: Pointer to host adapter structure 385 * @dev_id: Pointer to host adapter structure
@@ -280,48 +392,70 @@ static irqreturn_t be_isr(int irq, void *dev_id)
280 struct be_eq_entry *eqe = NULL; 392 struct be_eq_entry *eqe = NULL;
281 struct be_queue_info *eq; 393 struct be_queue_info *eq;
282 struct be_queue_info *cq; 394 struct be_queue_info *cq;
395 struct be_queue_info *mcc;
283 unsigned long flags, index; 396 unsigned long flags, index;
284 unsigned int num_eq_processed; 397 unsigned int num_mcceq_processed, num_ioeq_processed;
285 struct be_ctrl_info *ctrl; 398 struct be_ctrl_info *ctrl;
399 struct be_eq_obj *pbe_eq;
286 int isr; 400 int isr;
287 401
288 phba = dev_id; 402 phba = dev_id;
289 if (!enable_msix) { 403 ctrl = &phba->ctrl;;
290 ctrl = &phba->ctrl;; 404 isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
291 isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET + 405 (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
292 (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE)); 406 if (!isr)
293 if (!isr) 407 return IRQ_NONE;
294 return IRQ_NONE;
295 }
296 408
297 phwi_ctrlr = phba->phwi_ctrlr; 409 phwi_ctrlr = phba->phwi_ctrlr;
298 phwi_context = phwi_ctrlr->phwi_ctxt; 410 phwi_context = phwi_ctrlr->phwi_ctxt;
299 eq = &phwi_context->be_eq.q; 411 pbe_eq = &phwi_context->be_eq[0];
300 cq = &phwi_context->be_cq; 412
413 eq = &phwi_context->be_eq[0].q;
414 mcc = &phba->ctrl.mcc_obj.cq;
301 index = 0; 415 index = 0;
302 eqe = queue_tail_node(eq); 416 eqe = queue_tail_node(eq);
303 if (!eqe) 417 if (!eqe)
304 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n"); 418 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
305 419
306 num_eq_processed = 0; 420 num_ioeq_processed = 0;
421 num_mcceq_processed = 0;
307 if (blk_iopoll_enabled) { 422 if (blk_iopoll_enabled) {
308 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 423 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
309 & EQE_VALID_MASK) { 424 & EQE_VALID_MASK) {
310 if (!blk_iopoll_sched_prep(&phba->iopoll)) 425 if (((eqe->dw[offsetof(struct amap_eq_entry,
311 blk_iopoll_sched(&phba->iopoll); 426 resource_id) / 32] &
312 427 EQE_RESID_MASK) >> 16) == mcc->id) {
428 spin_lock_irqsave(&phba->isr_lock, flags);
429 phba->todo_mcc_cq = 1;
430 spin_unlock_irqrestore(&phba->isr_lock, flags);
431 num_mcceq_processed++;
432 } else {
433 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
434 blk_iopoll_sched(&pbe_eq->iopoll);
435 num_ioeq_processed++;
436 }
313 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 437 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
314 queue_tail_inc(eq); 438 queue_tail_inc(eq);
315 eqe = queue_tail_node(eq); 439 eqe = queue_tail_node(eq);
316 num_eq_processed++;
317 SE_DEBUG(DBG_LVL_8, "Valid EQE\n");
318 } 440 }
319 if (num_eq_processed) { 441 if (num_ioeq_processed || num_mcceq_processed) {
320 hwi_ring_eq_db(phba, eq->id, 0, num_eq_processed, 0, 1); 442 if (phba->todo_mcc_cq)
443 queue_work(phba->wq, &phba->work_cqs);
444
445 if ((num_mcceq_processed) && (!num_ioeq_processed))
446 hwi_ring_eq_db(phba, eq->id, 0,
447 (num_ioeq_processed +
448 num_mcceq_processed) , 1, 1);
449 else
450 hwi_ring_eq_db(phba, eq->id, 0,
451 (num_ioeq_processed +
452 num_mcceq_processed), 0, 1);
453
321 return IRQ_HANDLED; 454 return IRQ_HANDLED;
322 } else 455 } else
323 return IRQ_NONE; 456 return IRQ_NONE;
324 } else { 457 } else {
458 cq = &phwi_context->be_cq[0];
325 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 459 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
326 & EQE_VALID_MASK) { 460 & EQE_VALID_MASK) {
327 461
@@ -339,13 +473,14 @@ static irqreturn_t be_isr(int irq, void *dev_id)
339 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 473 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
340 queue_tail_inc(eq); 474 queue_tail_inc(eq);
341 eqe = queue_tail_node(eq); 475 eqe = queue_tail_node(eq);
342 num_eq_processed++; 476 num_ioeq_processed++;
343 } 477 }
344 if (phba->todo_cq || phba->todo_mcc_cq) 478 if (phba->todo_cq || phba->todo_mcc_cq)
345 queue_work(phba->wq, &phba->work_cqs); 479 queue_work(phba->wq, &phba->work_cqs);
346 480
347 if (num_eq_processed) { 481 if (num_ioeq_processed) {
348 hwi_ring_eq_db(phba, eq->id, 0, num_eq_processed, 1, 1); 482 hwi_ring_eq_db(phba, eq->id, 0,
483 num_ioeq_processed, 1, 1);
349 return IRQ_HANDLED; 484 return IRQ_HANDLED;
350 } else 485 } else
351 return IRQ_NONE; 486 return IRQ_NONE;
@@ -355,13 +490,32 @@ static irqreturn_t be_isr(int irq, void *dev_id)
355static int beiscsi_init_irqs(struct beiscsi_hba *phba) 490static int beiscsi_init_irqs(struct beiscsi_hba *phba)
356{ 491{
357 struct pci_dev *pcidev = phba->pcidev; 492 struct pci_dev *pcidev = phba->pcidev;
358 int ret; 493 struct hwi_controller *phwi_ctrlr;
494 struct hwi_context_memory *phwi_context;
495 int ret, msix_vec, i = 0;
496 char desc[32];
359 497
360 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED, "beiscsi", phba); 498 phwi_ctrlr = phba->phwi_ctrlr;
361 if (ret) { 499 phwi_context = phwi_ctrlr->phwi_ctxt;
362 shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-" 500
363 "Failed to register irq\\n"); 501 if (phba->msix_enabled) {
364 return ret; 502 for (i = 0; i < phba->num_cpus; i++) {
503 sprintf(desc, "beiscsi_msix_%04x", i);
504 msix_vec = phba->msix_entries[i].vector;
505 ret = request_irq(msix_vec, be_isr_msix, 0, desc,
506 &phwi_context->be_eq[i]);
507 }
508 msix_vec = phba->msix_entries[i].vector;
509 ret = request_irq(msix_vec, be_isr_mcc, 0, "beiscsi_msix_mcc",
510 &phwi_context->be_eq[i]);
511 } else {
512 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
513 "beiscsi", phba);
514 if (ret) {
515 shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
516 "Failed to register irq\\n");
517 return ret;
518 }
365 } 519 }
366 return 0; 520 return 0;
367} 521}
@@ -378,15 +532,6 @@ static void hwi_ring_cq_db(struct beiscsi_hba *phba,
378 iowrite32(val, phba->db_va + DB_CQ_OFFSET); 532 iowrite32(val, phba->db_va + DB_CQ_OFFSET);
379} 533}
380 534
381/*
382 * async pdus include
383 * a. unsolicited NOP-In (target initiated NOP-In)
384 * b. Async Messages
385 * c. Reject PDU
386 * d. Login response
387 * These headers arrive unprocessed by the EP firmware and iSCSI layer
388 * process them
389 */
390static unsigned int 535static unsigned int
391beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn, 536beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
392 struct beiscsi_hba *phba, 537 struct beiscsi_hba *phba,
@@ -397,6 +542,9 @@ beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
397{ 542{
398 struct iscsi_conn *conn = beiscsi_conn->conn; 543 struct iscsi_conn *conn = beiscsi_conn->conn;
399 struct iscsi_session *session = conn->session; 544 struct iscsi_session *session = conn->session;
545 struct iscsi_task *task;
546 struct beiscsi_io_task *io_task;
547 struct iscsi_hdr *login_hdr;
400 548
401 switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] & 549 switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
402 PDUBASE_OPCODE_MASK) { 550 PDUBASE_OPCODE_MASK) {
@@ -412,6 +560,10 @@ beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
412 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n"); 560 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
413 break; 561 break;
414 case ISCSI_OP_LOGIN_RSP: 562 case ISCSI_OP_LOGIN_RSP:
563 task = conn->login_task;
564 io_task = task->dd_data;
565 login_hdr = (struct iscsi_hdr *)ppdu;
566 login_hdr->itt = io_task->libiscsi_itt;
415 break; 567 break;
416 default: 568 default:
417 shost_printk(KERN_WARNING, phba->shost, 569 shost_printk(KERN_WARNING, phba->shost,
@@ -440,7 +592,8 @@ static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
440 io_sgl_alloc_index]; 592 io_sgl_alloc_index];
441 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL; 593 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
442 phba->io_sgl_hndl_avbl--; 594 phba->io_sgl_hndl_avbl--;
443 if (phba->io_sgl_alloc_index == (phba->params.ios_per_ctrl - 1)) 595 if (phba->io_sgl_alloc_index == (phba->params.
596 ios_per_ctrl - 1))
444 phba->io_sgl_alloc_index = 0; 597 phba->io_sgl_alloc_index = 0;
445 else 598 else
446 phba->io_sgl_alloc_index++; 599 phba->io_sgl_alloc_index++;
@@ -490,9 +643,18 @@ struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid,
490 643
491 phwi_ctrlr = phba->phwi_ctrlr; 644 phwi_ctrlr = phba->phwi_ctrlr;
492 pwrb_context = &phwi_ctrlr->wrb_context[cid]; 645 pwrb_context = &phwi_ctrlr->wrb_context[cid];
493 pwrb_handle = pwrb_context->pwrb_handle_base[index]; 646 if (pwrb_context->wrb_handles_available) {
494 pwrb_handle->wrb_index = index; 647 pwrb_handle = pwrb_context->pwrb_handle_base[
495 pwrb_handle->nxt_wrb_index = index; 648 pwrb_context->alloc_index];
649 pwrb_context->wrb_handles_available--;
650 pwrb_handle->nxt_wrb_index = pwrb_handle->wrb_index;
651 if (pwrb_context->alloc_index ==
652 (phba->params.wrbs_per_cxn - 1))
653 pwrb_context->alloc_index = 0;
654 else
655 pwrb_context->alloc_index++;
656 } else
657 pwrb_handle = NULL;
496 return pwrb_handle; 658 return pwrb_handle;
497} 659}
498 660
@@ -508,11 +670,19 @@ static void
508free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context, 670free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
509 struct wrb_handle *pwrb_handle) 671 struct wrb_handle *pwrb_handle)
510{ 672{
673
674 pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
675 pwrb_context->wrb_handles_available++;
676 if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
677 pwrb_context->free_index = 0;
678 else
679 pwrb_context->free_index++;
680
511 SE_DEBUG(DBG_LVL_8, 681 SE_DEBUG(DBG_LVL_8,
512 "FREE WRB: pwrb_handle=%p free_index=%d=0x%x" 682 "FREE WRB: pwrb_handle=%p free_index=0x%x"
513 "wrb_handles_available=%d \n", 683 "wrb_handles_available=%d \n",
514 pwrb_handle, pwrb_context->free_index, 684 pwrb_handle, pwrb_context->free_index,
515 pwrb_context->free_index, pwrb_context->wrb_handles_available); 685 pwrb_context->wrb_handles_available);
516} 686}
517 687
518static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba) 688static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
@@ -540,6 +710,8 @@ void
540free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) 710free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
541{ 711{
542 712
713 SE_DEBUG(DBG_LVL_8, "In free_mgmt_sgl_handle,eh_sgl_free_index=%d \n",
714 phba->eh_sgl_free_index);
543 if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) { 715 if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
544 /* 716 /*
545 * this can happen if clean_task is called on a task that 717 * this can happen if clean_task is called on a task that
@@ -572,10 +744,10 @@ be_complete_io(struct beiscsi_conn *beiscsi_conn,
572 u32 resid = 0, exp_cmdsn, max_cmdsn; 744 u32 resid = 0, exp_cmdsn, max_cmdsn;
573 u8 rsp, status, flags; 745 u8 rsp, status, flags;
574 746
575 exp_cmdsn = be32_to_cpu(psol-> 747 exp_cmdsn = (psol->
576 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32] 748 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
577 & SOL_EXP_CMD_SN_MASK); 749 & SOL_EXP_CMD_SN_MASK);
578 max_cmdsn = be32_to_cpu((psol-> 750 max_cmdsn = ((psol->
579 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32] 751 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
580 & SOL_EXP_CMD_SN_MASK) + 752 & SOL_EXP_CMD_SN_MASK) +
581 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) 753 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
@@ -610,9 +782,9 @@ be_complete_io(struct beiscsi_conn *beiscsi_conn,
610 } 782 }
611 783
612 if (status == SAM_STAT_CHECK_CONDITION) { 784 if (status == SAM_STAT_CHECK_CONDITION) {
785 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
613 sense = sts_bhs->sense_info + sizeof(unsigned short); 786 sense = sts_bhs->sense_info + sizeof(unsigned short);
614 sense_len = 787 sense_len = cpu_to_be16(*slen);
615 cpu_to_be16((unsigned short)(sts_bhs->sense_info[0]));
616 memcpy(task->sc->sense_buffer, sense, 788 memcpy(task->sc->sense_buffer, sense,
617 min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE)); 789 min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
618 } 790 }
@@ -620,8 +792,8 @@ be_complete_io(struct beiscsi_conn *beiscsi_conn,
620 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32] 792 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
621 & SOL_RES_CNT_MASK) 793 & SOL_RES_CNT_MASK)
622 conn->rxdata_octets += (psol-> 794 conn->rxdata_octets += (psol->
623 dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32] 795 dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
624 & SOL_RES_CNT_MASK); 796 & SOL_RES_CNT_MASK);
625 } 797 }
626unmap: 798unmap:
627 scsi_dma_unmap(io_task->scsi_cmnd); 799 scsi_dma_unmap(io_task->scsi_cmnd);
@@ -633,6 +805,7 @@ be_complete_logout(struct beiscsi_conn *beiscsi_conn,
633 struct iscsi_task *task, struct sol_cqe *psol) 805 struct iscsi_task *task, struct sol_cqe *psol)
634{ 806{
635 struct iscsi_logout_rsp *hdr; 807 struct iscsi_logout_rsp *hdr;
808 struct beiscsi_io_task *io_task = task->dd_data;
636 struct iscsi_conn *conn = beiscsi_conn->conn; 809 struct iscsi_conn *conn = beiscsi_conn->conn;
637 810
638 hdr = (struct iscsi_logout_rsp *)task->hdr; 811 hdr = (struct iscsi_logout_rsp *)task->hdr;
@@ -651,7 +824,7 @@ be_complete_logout(struct beiscsi_conn *beiscsi_conn,
651 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) 824 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
652 / 32] & SOL_CMD_WND_MASK) >> 24) - 1); 825 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
653 hdr->hlength = 0; 826 hdr->hlength = 0;
654 827 hdr->itt = io_task->libiscsi_itt;
655 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); 828 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
656} 829}
657 830
@@ -661,6 +834,7 @@ be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
661{ 834{
662 struct iscsi_tm_rsp *hdr; 835 struct iscsi_tm_rsp *hdr;
663 struct iscsi_conn *conn = beiscsi_conn->conn; 836 struct iscsi_conn *conn = beiscsi_conn->conn;
837 struct beiscsi_io_task *io_task = task->dd_data;
664 838
665 hdr = (struct iscsi_tm_rsp *)task->hdr; 839 hdr = (struct iscsi_tm_rsp *)task->hdr;
666 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32] 840 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
@@ -668,11 +842,12 @@ be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
668 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 842 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
669 32] & SOL_RESP_MASK); 843 32] & SOL_RESP_MASK);
670 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe, 844 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
671 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK); 845 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
672 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe, 846 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
673 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) + 847 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
674 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) 848 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
675 / 32] & SOL_CMD_WND_MASK) >> 24) - 1); 849 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
850 hdr->itt = io_task->libiscsi_itt;
676 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); 851 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
677} 852}
678 853
@@ -681,18 +856,25 @@ hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
681 struct beiscsi_hba *phba, struct sol_cqe *psol) 856 struct beiscsi_hba *phba, struct sol_cqe *psol)
682{ 857{
683 struct hwi_wrb_context *pwrb_context; 858 struct hwi_wrb_context *pwrb_context;
684 struct wrb_handle *pwrb_handle; 859 struct wrb_handle *pwrb_handle = NULL;
685 struct hwi_controller *phwi_ctrlr; 860 struct hwi_controller *phwi_ctrlr;
861 struct iscsi_task *task;
862 struct beiscsi_io_task *io_task;
686 struct iscsi_conn *conn = beiscsi_conn->conn; 863 struct iscsi_conn *conn = beiscsi_conn->conn;
687 struct iscsi_session *session = conn->session; 864 struct iscsi_session *session = conn->session;
688 865
689 phwi_ctrlr = phba->phwi_ctrlr; 866 phwi_ctrlr = phba->phwi_ctrlr;
690 pwrb_context = &phwi_ctrlr->wrb_context[((psol-> 867 pwrb_context = &phwi_ctrlr->wrb_context[((psol->
691 dw[offsetof(struct amap_sol_cqe, cid) / 32] & 868 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
692 SOL_CID_MASK) >> 6)]; 869 SOL_CID_MASK) >> 6)];
693 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol-> 870 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
694 dw[offsetof(struct amap_sol_cqe, wrb_index) / 871 dw[offsetof(struct amap_sol_cqe, wrb_index) /
695 32] & SOL_WRB_INDEX_MASK) >> 16)]; 872 32] & SOL_WRB_INDEX_MASK) >> 16)];
873 task = pwrb_handle->pio_handle;
874 io_task = task->dd_data;
875 spin_lock(&phba->mgmt_sgl_lock);
876 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
877 spin_unlock(&phba->mgmt_sgl_lock);
696 spin_lock_bh(&session->lock); 878 spin_lock_bh(&session->lock);
697 free_wrb_handle(phba, pwrb_context, pwrb_handle); 879 free_wrb_handle(phba, pwrb_context, pwrb_handle);
698 spin_unlock_bh(&session->lock); 880 spin_unlock_bh(&session->lock);
@@ -704,6 +886,7 @@ be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
704{ 886{
705 struct iscsi_nopin *hdr; 887 struct iscsi_nopin *hdr;
706 struct iscsi_conn *conn = beiscsi_conn->conn; 888 struct iscsi_conn *conn = beiscsi_conn->conn;
889 struct beiscsi_io_task *io_task = task->dd_data;
707 890
708 hdr = (struct iscsi_nopin *)task->hdr; 891 hdr = (struct iscsi_nopin *)task->hdr;
709 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32] 892 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
@@ -715,6 +898,7 @@ be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
715 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) 898 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
716 / 32] & SOL_CMD_WND_MASK) >> 24) - 1); 899 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
717 hdr->opcode = ISCSI_OP_NOOP_IN; 900 hdr->opcode = ISCSI_OP_NOOP_IN;
901 hdr->itt = io_task->libiscsi_itt;
718 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); 902 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
719} 903}
720 904
@@ -726,25 +910,25 @@ static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
726 struct iscsi_wrb *pwrb = NULL; 910 struct iscsi_wrb *pwrb = NULL;
727 struct hwi_controller *phwi_ctrlr; 911 struct hwi_controller *phwi_ctrlr;
728 struct iscsi_task *task; 912 struct iscsi_task *task;
729 struct beiscsi_io_task *io_task; 913 unsigned int type;
730 struct iscsi_conn *conn = beiscsi_conn->conn; 914 struct iscsi_conn *conn = beiscsi_conn->conn;
731 struct iscsi_session *session = conn->session; 915 struct iscsi_session *session = conn->session;
732 916
733 phwi_ctrlr = phba->phwi_ctrlr; 917 phwi_ctrlr = phba->phwi_ctrlr;
734
735 pwrb_context = &phwi_ctrlr-> 918 pwrb_context = &phwi_ctrlr->
736 wrb_context[((psol->dw[offsetof(struct amap_sol_cqe, cid) / 32] 919 wrb_context[((psol->dw[offsetof
737 & SOL_CID_MASK) >> 6)]; 920 (struct amap_sol_cqe, cid) / 32]
921 & SOL_CID_MASK) >> 6)];
738 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol-> 922 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
739 dw[offsetof(struct amap_sol_cqe, wrb_index) / 923 dw[offsetof(struct amap_sol_cqe, wrb_index) /
740 32] & SOL_WRB_INDEX_MASK) >> 16)]; 924 32] & SOL_WRB_INDEX_MASK) >> 16)];
741
742 task = pwrb_handle->pio_handle; 925 task = pwrb_handle->pio_handle;
743 io_task = task->dd_data;
744 spin_lock_bh(&session->lock);
745 pwrb = pwrb_handle->pwrb; 926 pwrb = pwrb_handle->pwrb;
746 switch ((pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] & 927 type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
747 WRB_TYPE_MASK) >> 28) { 928 WRB_TYPE_MASK) >> 28;
929
930 spin_lock_bh(&session->lock);
931 switch (type) {
748 case HWH_TYPE_IO: 932 case HWH_TYPE_IO:
749 case HWH_TYPE_IO_RD: 933 case HWH_TYPE_IO_RD:
750 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == 934 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
@@ -774,14 +958,14 @@ static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
774 958
775 default: 959 default:
776 shost_printk(KERN_WARNING, phba->shost, 960 shost_printk(KERN_WARNING, phba->shost,
777 "wrb_index 0x%x CID 0x%x\n", 961 "In hwi_complete_cmd, unknown type = %d"
778 ((psol->dw[offsetof(struct amap_iscsi_wrb, type) / 962 "wrb_index 0x%x CID 0x%x\n", type,
779 32] & SOL_WRB_INDEX_MASK) >> 16), 963 ((psol->dw[offsetof(struct amap_iscsi_wrb,
780 ((psol->dw[offsetof(struct amap_sol_cqe, cid) / 32] 964 type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
781 & SOL_CID_MASK) >> 6)); 965 ((psol->dw[offsetof(struct amap_sol_cqe,
966 cid) / 32] & SOL_CID_MASK) >> 6));
782 break; 967 break;
783 } 968 }
784
785 spin_unlock_bh(&session->lock); 969 spin_unlock_bh(&session->lock);
786} 970}
787 971
@@ -1208,21 +1392,20 @@ static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1208 hwi_post_async_buffers(phba, pasync_handle->is_header); 1392 hwi_post_async_buffers(phba, pasync_handle->is_header);
1209} 1393}
1210 1394
1211static unsigned int beiscsi_process_cq(struct beiscsi_hba *phba) 1395
1396static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
1212{ 1397{
1213 struct hwi_controller *phwi_ctrlr;
1214 struct hwi_context_memory *phwi_context;
1215 struct be_queue_info *cq; 1398 struct be_queue_info *cq;
1216 struct sol_cqe *sol; 1399 struct sol_cqe *sol;
1217 struct dmsg_cqe *dmsg; 1400 struct dmsg_cqe *dmsg;
1218 unsigned int num_processed = 0; 1401 unsigned int num_processed = 0;
1219 unsigned int tot_nump = 0; 1402 unsigned int tot_nump = 0;
1220 struct beiscsi_conn *beiscsi_conn; 1403 struct beiscsi_conn *beiscsi_conn;
1404 struct beiscsi_hba *phba;
1221 1405
1222 phwi_ctrlr = phba->phwi_ctrlr; 1406 cq = pbe_eq->cq;
1223 phwi_context = phwi_ctrlr->phwi_ctxt;
1224 cq = &phwi_context->be_cq;
1225 sol = queue_tail_node(cq); 1407 sol = queue_tail_node(cq);
1408 phba = pbe_eq->phba;
1226 1409
1227 while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] & 1410 while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1228 CQE_VALID_MASK) { 1411 CQE_VALID_MASK) {
@@ -1237,11 +1420,11 @@ static unsigned int beiscsi_process_cq(struct beiscsi_hba *phba)
1237 "Connection table empty for cid = %d\n", 1420 "Connection table empty for cid = %d\n",
1238 (u32)(sol->dw[offsetof(struct amap_sol_cqe, 1421 (u32)(sol->dw[offsetof(struct amap_sol_cqe,
1239 cid) / 32] & SOL_CID_MASK) >> 6); 1422 cid) / 32] & SOL_CID_MASK) >> 6);
1240 return 0; 1423 return 0;
1241 } 1424 }
1242 1425
1243 if (num_processed >= 32) { 1426 if (num_processed >= 32) {
1244 hwi_ring_cq_db(phba, phwi_context->be_cq.id, 1427 hwi_ring_cq_db(phba, cq->id,
1245 num_processed, 0, 0); 1428 num_processed, 0, 0);
1246 tot_nump += num_processed; 1429 tot_nump += num_processed;
1247 num_processed = 0; 1430 num_processed = 0;
@@ -1258,8 +1441,12 @@ static unsigned int beiscsi_process_cq(struct beiscsi_hba *phba)
1258 hwi_complete_drvr_msgs(beiscsi_conn, phba, sol); 1441 hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1259 break; 1442 break;
1260 case UNSOL_HDR_NOTIFY: 1443 case UNSOL_HDR_NOTIFY:
1444 SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n");
1445 hwi_process_default_pdu_ring(beiscsi_conn, phba,
1446 (struct i_t_dpdu_cqe *)sol);
1447 break;
1261 case UNSOL_DATA_NOTIFY: 1448 case UNSOL_DATA_NOTIFY:
1262 SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR/DATA_NOTIFY\n"); 1449 SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n");
1263 hwi_process_default_pdu_ring(beiscsi_conn, phba, 1450 hwi_process_default_pdu_ring(beiscsi_conn, phba,
1264 (struct i_t_dpdu_cqe *)sol); 1451 (struct i_t_dpdu_cqe *)sol);
1265 break; 1452 break;
@@ -1306,7 +1493,7 @@ static unsigned int beiscsi_process_cq(struct beiscsi_hba *phba)
1306 case CXN_KILLED_OVER_RUN_RESIDUAL: 1493 case CXN_KILLED_OVER_RUN_RESIDUAL:
1307 case CXN_KILLED_UNDER_RUN_RESIDUAL: 1494 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1308 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN: 1495 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
1309 SE_DEBUG(DBG_LVL_1, "CQ Error %d, resetting CID " 1496 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
1310 "0x%x...\n", 1497 "0x%x...\n",
1311 sol->dw[offsetof(struct amap_sol_cqe, code) / 1498 sol->dw[offsetof(struct amap_sol_cqe, code) /
1312 32] & CQE_CODE_MASK, 1499 32] & CQE_CODE_MASK,
@@ -1317,8 +1504,8 @@ static unsigned int beiscsi_process_cq(struct beiscsi_hba *phba)
1317 break; 1504 break;
1318 case CXN_KILLED_RST_SENT: 1505 case CXN_KILLED_RST_SENT:
1319 case CXN_KILLED_RST_RCVD: 1506 case CXN_KILLED_RST_RCVD:
1320 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset received/sent " 1507 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
1321 "on CID 0x%x...\n", 1508 "received/sent on CID 0x%x...\n",
1322 sol->dw[offsetof(struct amap_sol_cqe, code) / 1509 sol->dw[offsetof(struct amap_sol_cqe, code) /
1323 32] & CQE_CODE_MASK, 1510 32] & CQE_CODE_MASK,
1324 sol->dw[offsetof(struct amap_sol_cqe, cid) / 1511 sol->dw[offsetof(struct amap_sol_cqe, cid) /
@@ -1344,8 +1531,7 @@ static unsigned int beiscsi_process_cq(struct beiscsi_hba *phba)
1344 1531
1345 if (num_processed > 0) { 1532 if (num_processed > 0) {
1346 tot_nump += num_processed; 1533 tot_nump += num_processed;
1347 hwi_ring_cq_db(phba, phwi_context->be_cq.id, num_processed, 1534 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
1348 1, 0);
1349 } 1535 }
1350 return tot_nump; 1536 return tot_nump;
1351} 1537}
@@ -1353,21 +1539,30 @@ static unsigned int beiscsi_process_cq(struct beiscsi_hba *phba)
1353static void beiscsi_process_all_cqs(struct work_struct *work) 1539static void beiscsi_process_all_cqs(struct work_struct *work)
1354{ 1540{
1355 unsigned long flags; 1541 unsigned long flags;
1542 struct hwi_controller *phwi_ctrlr;
1543 struct hwi_context_memory *phwi_context;
1544 struct be_eq_obj *pbe_eq;
1356 struct beiscsi_hba *phba = 1545 struct beiscsi_hba *phba =
1357 container_of(work, struct beiscsi_hba, work_cqs); 1546 container_of(work, struct beiscsi_hba, work_cqs);
1358 1547
1548 phwi_ctrlr = phba->phwi_ctrlr;
1549 phwi_context = phwi_ctrlr->phwi_ctxt;
1550 if (phba->msix_enabled)
1551 pbe_eq = &phwi_context->be_eq[phba->num_cpus];
1552 else
1553 pbe_eq = &phwi_context->be_eq[0];
1554
1359 if (phba->todo_mcc_cq) { 1555 if (phba->todo_mcc_cq) {
1360 spin_lock_irqsave(&phba->isr_lock, flags); 1556 spin_lock_irqsave(&phba->isr_lock, flags);
1361 phba->todo_mcc_cq = 0; 1557 phba->todo_mcc_cq = 0;
1362 spin_unlock_irqrestore(&phba->isr_lock, flags); 1558 spin_unlock_irqrestore(&phba->isr_lock, flags);
1363 SE_DEBUG(DBG_LVL_1, "MCC Interrupt Not expected \n");
1364 } 1559 }
1365 1560
1366 if (phba->todo_cq) { 1561 if (phba->todo_cq) {
1367 spin_lock_irqsave(&phba->isr_lock, flags); 1562 spin_lock_irqsave(&phba->isr_lock, flags);
1368 phba->todo_cq = 0; 1563 phba->todo_cq = 0;
1369 spin_unlock_irqrestore(&phba->isr_lock, flags); 1564 spin_unlock_irqrestore(&phba->isr_lock, flags);
1370 beiscsi_process_cq(phba); 1565 beiscsi_process_cq(pbe_eq);
1371 } 1566 }
1372} 1567}
1373 1568
@@ -1375,19 +1570,15 @@ static int be_iopoll(struct blk_iopoll *iop, int budget)
1375{ 1570{
1376 static unsigned int ret; 1571 static unsigned int ret;
1377 struct beiscsi_hba *phba; 1572 struct beiscsi_hba *phba;
1573 struct be_eq_obj *pbe_eq;
1378 1574
1379 phba = container_of(iop, struct beiscsi_hba, iopoll); 1575 pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
1380 1576 ret = beiscsi_process_cq(pbe_eq);
1381 ret = beiscsi_process_cq(phba);
1382 if (ret < budget) { 1577 if (ret < budget) {
1383 struct hwi_controller *phwi_ctrlr; 1578 phba = pbe_eq->phba;
1384 struct hwi_context_memory *phwi_context;
1385
1386 phwi_ctrlr = phba->phwi_ctrlr;
1387 phwi_context = phwi_ctrlr->phwi_ctxt;
1388 blk_iopoll_complete(iop); 1579 blk_iopoll_complete(iop);
1389 hwi_ring_eq_db(phba, phwi_context->be_eq.q.id, 0, 1580 SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id);
1390 0, 1, 1); 1581 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
1391 } 1582 }
1392 return ret; 1583 return ret;
1393} 1584}
@@ -1537,14 +1728,12 @@ static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
1537 1728
1538static void beiscsi_find_mem_req(struct beiscsi_hba *phba) 1729static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
1539{ 1730{
1540 unsigned int num_cq_pages, num_eq_pages, num_async_pdu_buf_pages; 1731 unsigned int num_cq_pages, num_async_pdu_buf_pages;
1541 unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn; 1732 unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
1542 unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages; 1733 unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
1543 1734
1544 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \ 1735 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
1545 sizeof(struct sol_cqe)); 1736 sizeof(struct sol_cqe));
1546 num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
1547 sizeof(struct be_eq_entry));
1548 num_async_pdu_buf_pages = 1737 num_async_pdu_buf_pages =
1549 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \ 1738 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1550 phba->params.defpdu_hdr_sz); 1739 phba->params.defpdu_hdr_sz);
@@ -1565,8 +1754,6 @@ static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
1565 phba->mem_req[HWI_MEM_ADDN_CONTEXT] = 1754 phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
1566 sizeof(struct hwi_context_memory); 1755 sizeof(struct hwi_context_memory);
1567 1756
1568 phba->mem_req[HWI_MEM_CQ] = num_cq_pages * PAGE_SIZE;
1569 phba->mem_req[HWI_MEM_EQ] = num_eq_pages * PAGE_SIZE;
1570 1757
1571 phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb) 1758 phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
1572 * (phba->params.wrbs_per_cxn) 1759 * (phba->params.wrbs_per_cxn)
@@ -1751,8 +1938,6 @@ static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
1751 1938
1752 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) { 1939 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
1753 pwrb_context = &phwi_ctrlr->wrb_context[index]; 1940 pwrb_context = &phwi_ctrlr->wrb_context[index];
1754 SE_DEBUG(DBG_LVL_8, "cid=%d pwrb_context=%p \n", index,
1755 pwrb_context);
1756 pwrb_context->pwrb_handle_base = 1941 pwrb_context->pwrb_handle_base =
1757 kzalloc(sizeof(struct wrb_handle *) * 1942 kzalloc(sizeof(struct wrb_handle *) *
1758 phba->params.wrbs_per_cxn, GFP_KERNEL); 1943 phba->params.wrbs_per_cxn, GFP_KERNEL);
@@ -1767,6 +1952,7 @@ static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
1767 pwrb_context->pwrb_handle_basestd[j] = 1952 pwrb_context->pwrb_handle_basestd[j] =
1768 pwrb_handle; 1953 pwrb_handle;
1769 pwrb_context->wrb_handles_available++; 1954 pwrb_context->wrb_handles_available++;
1955 pwrb_handle->wrb_index = j;
1770 pwrb_handle++; 1956 pwrb_handle++;
1771 } 1957 }
1772 pwrb_context->free_index = 0; 1958 pwrb_context->free_index = 0;
@@ -1785,6 +1971,7 @@ static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
1785 pwrb_context->pwrb_handle_basestd[j] = 1971 pwrb_context->pwrb_handle_basestd[j] =
1786 pwrb_handle; 1972 pwrb_handle;
1787 pwrb_context->wrb_handles_available++; 1973 pwrb_context->wrb_handles_available++;
1974 pwrb_handle->wrb_index = j;
1788 pwrb_handle++; 1975 pwrb_handle++;
1789 } 1976 }
1790 pwrb_context->free_index = 0; 1977 pwrb_context->free_index = 0;
@@ -2042,79 +2229,126 @@ static int be_fill_queue(struct be_queue_info *q,
2042 return 0; 2229 return 0;
2043} 2230}
2044 2231
2045static int beiscsi_create_eq(struct beiscsi_hba *phba, 2232static int beiscsi_create_eqs(struct beiscsi_hba *phba,
2046 struct hwi_context_memory *phwi_context) 2233 struct hwi_context_memory *phwi_context)
2047{ 2234{
2048 unsigned int idx; 2235 unsigned int i, num_eq_pages;
2049 int ret; 2236 int ret, eq_for_mcc;
2050 struct be_queue_info *eq; 2237 struct be_queue_info *eq;
2051 struct be_dma_mem *mem; 2238 struct be_dma_mem *mem;
2052 struct be_mem_descriptor *mem_descr;
2053 void *eq_vaddress; 2239 void *eq_vaddress;
2240 dma_addr_t paddr;
2054 2241
2055 idx = 0; 2242 num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
2056 eq = &phwi_context->be_eq.q; 2243 sizeof(struct be_eq_entry));
2057 mem = &eq->dma_mem;
2058 mem_descr = phba->init_mem;
2059 mem_descr += HWI_MEM_EQ;
2060 eq_vaddress = mem_descr->mem_array[idx].virtual_address;
2061
2062 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2063 sizeof(struct be_eq_entry), eq_vaddress);
2064 if (ret) {
2065 shost_printk(KERN_ERR, phba->shost,
2066 "be_fill_queue Failed for EQ \n");
2067 return ret;
2068 }
2069 2244
2070 mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address; 2245 if (phba->msix_enabled)
2246 eq_for_mcc = 1;
2247 else
2248 eq_for_mcc = 0;
2249 for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2250 eq = &phwi_context->be_eq[i].q;
2251 mem = &eq->dma_mem;
2252 phwi_context->be_eq[i].phba = phba;
2253 eq_vaddress = pci_alloc_consistent(phba->pcidev,
2254 num_eq_pages * PAGE_SIZE,
2255 &paddr);
2256 if (!eq_vaddress)
2257 goto create_eq_error;
2258
2259 mem->va = eq_vaddress;
2260 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2261 sizeof(struct be_eq_entry), eq_vaddress);
2262 if (ret) {
2263 shost_printk(KERN_ERR, phba->shost,
2264 "be_fill_queue Failed for EQ \n");
2265 goto create_eq_error;
2266 }
2071 2267
2072 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq, 2268 mem->dma = paddr;
2073 phwi_context->be_eq.cur_eqd); 2269 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2074 if (ret) { 2270 phwi_context->cur_eqd);
2075 shost_printk(KERN_ERR, phba->shost, "beiscsi_cmd_eq_create" 2271 if (ret) {
2076 "Failedfor EQ \n"); 2272 shost_printk(KERN_ERR, phba->shost,
2077 return ret; 2273 "beiscsi_cmd_eq_create"
2274 "Failedfor EQ \n");
2275 goto create_eq_error;
2276 }
2277 SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id);
2078 } 2278 }
2079 SE_DEBUG(DBG_LVL_8, "eq id is %d\n", phwi_context->be_eq.q.id);
2080 return 0; 2279 return 0;
2280create_eq_error:
2281 for (i = 0; i < (phba->num_cpus + 1); i++) {
2282 eq = &phwi_context->be_eq[i].q;
2283 mem = &eq->dma_mem;
2284 if (mem->va)
2285 pci_free_consistent(phba->pcidev, num_eq_pages
2286 * PAGE_SIZE,
2287 mem->va, mem->dma);
2288 }
2289 return ret;
2081} 2290}
2082 2291
2083static int beiscsi_create_cq(struct beiscsi_hba *phba, 2292static int beiscsi_create_cqs(struct beiscsi_hba *phba,
2084 struct hwi_context_memory *phwi_context) 2293 struct hwi_context_memory *phwi_context)
2085{ 2294{
2086 unsigned int idx; 2295 unsigned int i, num_cq_pages;
2087 int ret; 2296 int ret;
2088 struct be_queue_info *cq, *eq; 2297 struct be_queue_info *cq, *eq;
2089 struct be_dma_mem *mem; 2298 struct be_dma_mem *mem;
2090 struct be_mem_descriptor *mem_descr; 2299 struct be_eq_obj *pbe_eq;
2091 void *cq_vaddress; 2300 void *cq_vaddress;
2301 dma_addr_t paddr;
2092 2302
2093 idx = 0; 2303 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2094 cq = &phwi_context->be_cq; 2304 sizeof(struct sol_cqe));
2095 eq = &phwi_context->be_eq.q;
2096 mem = &cq->dma_mem;
2097 mem_descr = phba->init_mem;
2098 mem_descr += HWI_MEM_CQ;
2099 cq_vaddress = mem_descr->mem_array[idx].virtual_address;
2100 ret = be_fill_queue(cq, phba->params.icds_per_ctrl / 2,
2101 sizeof(struct sol_cqe), cq_vaddress);
2102 if (ret) {
2103 shost_printk(KERN_ERR, phba->shost,
2104 "be_fill_queue Failed for ISCSI CQ \n");
2105 return ret;
2106 }
2107 2305
2108 mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address; 2306 for (i = 0; i < phba->num_cpus; i++) {
2109 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false, false, 0); 2307 cq = &phwi_context->be_cq[i];
2110 if (ret) { 2308 eq = &phwi_context->be_eq[i].q;
2111 shost_printk(KERN_ERR, phba->shost, 2309 pbe_eq = &phwi_context->be_eq[i];
2112 "beiscsi_cmd_eq_create Failed for ISCSI CQ \n"); 2310 pbe_eq->cq = cq;
2113 return ret; 2311 pbe_eq->phba = phba;
2312 mem = &cq->dma_mem;
2313 cq_vaddress = pci_alloc_consistent(phba->pcidev,
2314 num_cq_pages * PAGE_SIZE,
2315 &paddr);
2316 if (!cq_vaddress)
2317 goto create_cq_error;
2318 ret = be_fill_queue(cq, phba->params.icds_per_ctrl / 2,
2319 sizeof(struct sol_cqe), cq_vaddress);
2320 if (ret) {
2321 shost_printk(KERN_ERR, phba->shost,
2322 "be_fill_queue Failed for ISCSI CQ \n");
2323 goto create_cq_error;
2324 }
2325
2326 mem->dma = paddr;
2327 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
2328 false, 0);
2329 if (ret) {
2330 shost_printk(KERN_ERR, phba->shost,
2331 "beiscsi_cmd_eq_create"
2332 "Failed for ISCSI CQ \n");
2333 goto create_cq_error;
2334 }
2335 SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n",
2336 cq->id, eq->id);
2337 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
2114 } 2338 }
2115 SE_DEBUG(DBG_LVL_8, "iscsi cq id is %d\n", phwi_context->be_cq.id);
2116 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
2117 return 0; 2339 return 0;
2340
2341create_cq_error:
2342 for (i = 0; i < phba->num_cpus; i++) {
2343 cq = &phwi_context->be_cq[i];
2344 mem = &cq->dma_mem;
2345 if (mem->va)
2346 pci_free_consistent(phba->pcidev, num_cq_pages
2347 * PAGE_SIZE,
2348 mem->va, mem->dma);
2349 }
2350 return ret;
2351
2118} 2352}
2119 2353
2120static int 2354static int
@@ -2132,7 +2366,7 @@ beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2132 2366
2133 idx = 0; 2367 idx = 0;
2134 dq = &phwi_context->be_def_hdrq; 2368 dq = &phwi_context->be_def_hdrq;
2135 cq = &phwi_context->be_cq; 2369 cq = &phwi_context->be_cq[0];
2136 mem = &dq->dma_mem; 2370 mem = &dq->dma_mem;
2137 mem_descr = phba->init_mem; 2371 mem_descr = phba->init_mem;
2138 mem_descr += HWI_MEM_ASYNC_HEADER_RING; 2372 mem_descr += HWI_MEM_ASYNC_HEADER_RING;
@@ -2176,7 +2410,7 @@ beiscsi_create_def_data(struct beiscsi_hba *phba,
2176 2410
2177 idx = 0; 2411 idx = 0;
2178 dataq = &phwi_context->be_def_dataq; 2412 dataq = &phwi_context->be_def_dataq;
2179 cq = &phwi_context->be_cq; 2413 cq = &phwi_context->be_cq[0];
2180 mem = &dataq->dma_mem; 2414 mem = &dataq->dma_mem;
2181 mem_descr = phba->init_mem; 2415 mem_descr = phba->init_mem;
2182 mem_descr += HWI_MEM_ASYNC_DATA_RING; 2416 mem_descr += HWI_MEM_ASYNC_DATA_RING;
@@ -2239,6 +2473,30 @@ beiscsi_post_pages(struct beiscsi_hba *phba)
2239 return 0; 2473 return 0;
2240} 2474}
2241 2475
2476static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
2477{
2478 struct be_dma_mem *mem = &q->dma_mem;
2479 if (mem->va)
2480 pci_free_consistent(phba->pcidev, mem->size,
2481 mem->va, mem->dma);
2482}
2483
2484static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
2485 u16 len, u16 entry_size)
2486{
2487 struct be_dma_mem *mem = &q->dma_mem;
2488
2489 memset(q, 0, sizeof(*q));
2490 q->len = len;
2491 q->entry_size = entry_size;
2492 mem->size = len * entry_size;
2493 mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
2494 if (!mem->va)
2495 return -1;
2496 memset(mem->va, 0, mem->size);
2497 return 0;
2498}
2499
2242static int 2500static int
2243beiscsi_create_wrb_rings(struct beiscsi_hba *phba, 2501beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2244 struct hwi_context_memory *phwi_context, 2502 struct hwi_context_memory *phwi_context,
@@ -2328,13 +2586,29 @@ static void free_wrb_handles(struct beiscsi_hba *phba)
2328 } 2586 }
2329} 2587}
2330 2588
2589static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
2590{
2591 struct be_queue_info *q;
2592 struct be_ctrl_info *ctrl = &phba->ctrl;
2593
2594 q = &phba->ctrl.mcc_obj.q;
2595 if (q->created)
2596 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
2597 be_queue_free(phba, q);
2598
2599 q = &phba->ctrl.mcc_obj.cq;
2600 if (q->created)
2601 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2602 be_queue_free(phba, q);
2603}
2604
2331static void hwi_cleanup(struct beiscsi_hba *phba) 2605static void hwi_cleanup(struct beiscsi_hba *phba)
2332{ 2606{
2333 struct be_queue_info *q; 2607 struct be_queue_info *q;
2334 struct be_ctrl_info *ctrl = &phba->ctrl; 2608 struct be_ctrl_info *ctrl = &phba->ctrl;
2335 struct hwi_controller *phwi_ctrlr; 2609 struct hwi_controller *phwi_ctrlr;
2336 struct hwi_context_memory *phwi_context; 2610 struct hwi_context_memory *phwi_context;
2337 int i; 2611 int i, eq_num;
2338 2612
2339 phwi_ctrlr = phba->phwi_ctrlr; 2613 phwi_ctrlr = phba->phwi_ctrlr;
2340 phwi_context = phwi_ctrlr->phwi_ctxt; 2614 phwi_context = phwi_ctrlr->phwi_ctxt;
@@ -2343,7 +2617,6 @@ static void hwi_cleanup(struct beiscsi_hba *phba)
2343 if (q->created) 2617 if (q->created)
2344 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ); 2618 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
2345 } 2619 }
2346
2347 free_wrb_handles(phba); 2620 free_wrb_handles(phba);
2348 2621
2349 q = &phwi_context->be_def_hdrq; 2622 q = &phwi_context->be_def_hdrq;
@@ -2356,13 +2629,76 @@ static void hwi_cleanup(struct beiscsi_hba *phba)
2356 2629
2357 beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL); 2630 beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
2358 2631
2359 q = &phwi_context->be_cq; 2632 for (i = 0; i < (phba->num_cpus); i++) {
2360 if (q->created) 2633 q = &phwi_context->be_cq[i];
2361 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ); 2634 if (q->created)
2635 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2636 }
2637 if (phba->msix_enabled)
2638 eq_num = 1;
2639 else
2640 eq_num = 0;
2641 for (i = 0; i < (phba->num_cpus + eq_num); i++) {
2642 q = &phwi_context->be_eq[i].q;
2643 if (q->created)
2644 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
2645 }
2646 be_mcc_queues_destroy(phba);
2647}
2362 2648
2363 q = &phwi_context->be_eq.q; 2649static int be_mcc_queues_create(struct beiscsi_hba *phba,
2364 if (q->created) 2650 struct hwi_context_memory *phwi_context)
2365 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ); 2651{
2652 struct be_queue_info *q, *cq;
2653 struct be_ctrl_info *ctrl = &phba->ctrl;
2654
2655 /* Alloc MCC compl queue */
2656 cq = &phba->ctrl.mcc_obj.cq;
2657 if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
2658 sizeof(struct be_mcc_compl)))
2659 goto err;
2660 /* Ask BE to create MCC compl queue; */
2661 if (phba->msix_enabled) {
2662 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
2663 [phba->num_cpus].q, false, true, 0))
2664 goto mcc_cq_free;
2665 } else {
2666 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
2667 false, true, 0))
2668 goto mcc_cq_free;
2669 }
2670
2671 /* Alloc MCC queue */
2672 q = &phba->ctrl.mcc_obj.q;
2673 if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2674 goto mcc_cq_destroy;
2675
2676 /* Ask BE to create MCC queue */
2677 if (be_cmd_mccq_create(phba, q, cq))
2678 goto mcc_q_free;
2679
2680 return 0;
2681
2682mcc_q_free:
2683 be_queue_free(phba, q);
2684mcc_cq_destroy:
2685 beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
2686mcc_cq_free:
2687 be_queue_free(phba, cq);
2688err:
2689 return -1;
2690}
2691
2692static int find_num_cpus(void)
2693{
2694 int num_cpus = 0;
2695
2696 num_cpus = num_online_cpus();
2697 if (num_cpus >= MAX_CPUS)
2698 num_cpus = MAX_CPUS - 1;
2699
2700 SE_DEBUG(DBG_LVL_8, "num_cpus = %d \n", num_cpus);
2701 return num_cpus;
2366} 2702}
2367 2703
2368static int hwi_init_port(struct beiscsi_hba *phba) 2704static int hwi_init_port(struct beiscsi_hba *phba)
@@ -2376,20 +2712,23 @@ static int hwi_init_port(struct beiscsi_hba *phba)
2376 def_pdu_ring_sz = 2712 def_pdu_ring_sz =
2377 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr); 2713 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
2378 phwi_ctrlr = phba->phwi_ctrlr; 2714 phwi_ctrlr = phba->phwi_ctrlr;
2379
2380 phwi_context = phwi_ctrlr->phwi_ctxt; 2715 phwi_context = phwi_ctrlr->phwi_ctxt;
2381 phwi_context->be_eq.max_eqd = 0; 2716 phwi_context->max_eqd = 0;
2382 phwi_context->be_eq.min_eqd = 0; 2717 phwi_context->min_eqd = 0;
2383 phwi_context->be_eq.cur_eqd = 64; 2718 phwi_context->cur_eqd = 64;
2384 phwi_context->be_eq.enable_aic = false;
2385 be_cmd_fw_initialize(&phba->ctrl); 2719 be_cmd_fw_initialize(&phba->ctrl);
2386 status = beiscsi_create_eq(phba, phwi_context); 2720
2721 status = beiscsi_create_eqs(phba, phwi_context);
2387 if (status != 0) { 2722 if (status != 0) {
2388 shost_printk(KERN_ERR, phba->shost, "EQ not created \n"); 2723 shost_printk(KERN_ERR, phba->shost, "EQ not created \n");
2389 goto error; 2724 goto error;
2390 } 2725 }
2391 2726
2392 status = mgmt_check_supported_fw(ctrl); 2727 status = be_mcc_queues_create(phba, phwi_context);
2728 if (status != 0)
2729 goto error;
2730
2731 status = mgmt_check_supported_fw(ctrl, phba);
2393 if (status != 0) { 2732 if (status != 0) {
2394 shost_printk(KERN_ERR, phba->shost, 2733 shost_printk(KERN_ERR, phba->shost,
2395 "Unsupported fw version \n"); 2734 "Unsupported fw version \n");
@@ -2403,7 +2742,7 @@ static int hwi_init_port(struct beiscsi_hba *phba)
2403 goto error; 2742 goto error;
2404 } 2743 }
2405 2744
2406 status = beiscsi_create_cq(phba, phwi_context); 2745 status = beiscsi_create_cqs(phba, phwi_context);
2407 if (status != 0) { 2746 if (status != 0) {
2408 shost_printk(KERN_ERR, phba->shost, "CQ not created\n"); 2747 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
2409 goto error; 2748 goto error;
@@ -2447,7 +2786,6 @@ error:
2447 return -ENOMEM; 2786 return -ENOMEM;
2448} 2787}
2449 2788
2450
2451static int hwi_init_controller(struct beiscsi_hba *phba) 2789static int hwi_init_controller(struct beiscsi_hba *phba)
2452{ 2790{
2453 struct hwi_controller *phwi_ctrlr; 2791 struct hwi_controller *phwi_ctrlr;
@@ -2530,6 +2868,7 @@ static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
2530 2868
2531 phba->io_sgl_hndl_avbl = 0; 2869 phba->io_sgl_hndl_avbl = 0;
2532 phba->eh_sgl_hndl_avbl = 0; 2870 phba->eh_sgl_hndl_avbl = 0;
2871
2533 mem_descr_sglh = phba->init_mem; 2872 mem_descr_sglh = phba->init_mem;
2534 mem_descr_sglh += HWI_MEM_SGLH; 2873 mem_descr_sglh += HWI_MEM_SGLH;
2535 if (1 == mem_descr_sglh->num_elements) { 2874 if (1 == mem_descr_sglh->num_elements) {
@@ -2656,13 +2995,12 @@ static unsigned char hwi_enable_intr(struct beiscsi_hba *phba)
2656 struct hwi_context_memory *phwi_context; 2995 struct hwi_context_memory *phwi_context;
2657 struct be_queue_info *eq; 2996 struct be_queue_info *eq;
2658 u8 __iomem *addr; 2997 u8 __iomem *addr;
2659 u32 reg; 2998 u32 reg, i;
2660 u32 enabled; 2999 u32 enabled;
2661 3000
2662 phwi_ctrlr = phba->phwi_ctrlr; 3001 phwi_ctrlr = phba->phwi_ctrlr;
2663 phwi_context = phwi_ctrlr->phwi_ctxt; 3002 phwi_context = phwi_ctrlr->phwi_ctxt;
2664 3003
2665 eq = &phwi_context->be_eq.q;
2666 addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg + 3004 addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
2667 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET); 3005 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
2668 reg = ioread32(addr); 3006 reg = ioread32(addr);
@@ -2673,9 +3011,11 @@ static unsigned char hwi_enable_intr(struct beiscsi_hba *phba)
2673 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 3011 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
2674 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p \n", reg, addr); 3012 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p \n", reg, addr);
2675 iowrite32(reg, addr); 3013 iowrite32(reg, addr);
2676 SE_DEBUG(DBG_LVL_8, "eq->id=%d \n", eq->id); 3014 for (i = 0; i <= phba->num_cpus; i++) {
2677 3015 eq = &phwi_context->be_eq[i].q;
2678 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1); 3016 SE_DEBUG(DBG_LVL_8, "eq->id=%d \n", eq->id);
3017 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3018 }
2679 } else 3019 } else
2680 shost_printk(KERN_WARNING, phba->shost, 3020 shost_printk(KERN_WARNING, phba->shost,
2681 "In hwi_enable_intr, Not Enabled \n"); 3021 "In hwi_enable_intr, Not Enabled \n");
@@ -2738,17 +3078,25 @@ static void hwi_purge_eq(struct beiscsi_hba *phba)
2738 struct hwi_context_memory *phwi_context; 3078 struct hwi_context_memory *phwi_context;
2739 struct be_queue_info *eq; 3079 struct be_queue_info *eq;
2740 struct be_eq_entry *eqe = NULL; 3080 struct be_eq_entry *eqe = NULL;
3081 int i, eq_msix;
2741 3082
2742 phwi_ctrlr = phba->phwi_ctrlr; 3083 phwi_ctrlr = phba->phwi_ctrlr;
2743 phwi_context = phwi_ctrlr->phwi_ctxt; 3084 phwi_context = phwi_ctrlr->phwi_ctxt;
2744 eq = &phwi_context->be_eq.q; 3085 if (phba->msix_enabled)
2745 eqe = queue_tail_node(eq); 3086 eq_msix = 1;
3087 else
3088 eq_msix = 0;
2746 3089
2747 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 3090 for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
2748 & EQE_VALID_MASK) { 3091 eq = &phwi_context->be_eq[i].q;
2749 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
2750 queue_tail_inc(eq);
2751 eqe = queue_tail_node(eq); 3092 eqe = queue_tail_node(eq);
3093
3094 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3095 & EQE_VALID_MASK) {
3096 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3097 queue_tail_inc(eq);
3098 eqe = queue_tail_node(eq);
3099 }
2752 } 3100 }
2753} 3101}
2754 3102
@@ -2846,8 +3194,8 @@ beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
2846 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb)); 3194 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
2847 3195
2848 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK; 3196 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
2849 doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK) << 3197 doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
2850 DB_DEF_PDU_WRB_INDEX_SHIFT; 3198 << DB_DEF_PDU_WRB_INDEX_SHIFT;
2851 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 3199 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
2852 3200
2853 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET); 3201 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
@@ -2856,7 +3204,7 @@ beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
2856static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt, 3204static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
2857 int *index, int *age) 3205 int *index, int *age)
2858{ 3206{
2859 *index = be32_to_cpu(itt) >> 16; 3207 *index = (int)itt;
2860 if (age) 3208 if (age)
2861 *age = conn->session->age; 3209 *age = conn->session->age;
2862} 3210}
@@ -2885,15 +3233,13 @@ static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
2885 3233
2886 io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool, 3234 io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
2887 GFP_KERNEL, &paddr); 3235 GFP_KERNEL, &paddr);
2888
2889 if (!io_task->cmd_bhs) 3236 if (!io_task->cmd_bhs)
2890 return -ENOMEM; 3237 return -ENOMEM;
2891
2892 io_task->bhs_pa.u.a64.address = paddr; 3238 io_task->bhs_pa.u.a64.address = paddr;
3239 io_task->libiscsi_itt = (itt_t)task->itt;
2893 io_task->pwrb_handle = alloc_wrb_handle(phba, 3240 io_task->pwrb_handle = alloc_wrb_handle(phba,
2894 beiscsi_conn->beiscsi_conn_cid, 3241 beiscsi_conn->beiscsi_conn_cid,
2895 task->itt); 3242 task->itt);
2896 io_task->pwrb_handle->pio_handle = task;
2897 io_task->conn = beiscsi_conn; 3243 io_task->conn = beiscsi_conn;
2898 3244
2899 task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr; 3245 task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
@@ -2905,7 +3251,6 @@ static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
2905 spin_unlock(&phba->io_sgl_lock); 3251 spin_unlock(&phba->io_sgl_lock);
2906 if (!io_task->psgl_handle) 3252 if (!io_task->psgl_handle)
2907 goto free_hndls; 3253 goto free_hndls;
2908
2909 } else { 3254 } else {
2910 io_task->scsi_cmnd = NULL; 3255 io_task->scsi_cmnd = NULL;
2911 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) { 3256 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
@@ -2932,8 +3277,11 @@ static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
2932 goto free_hndls; 3277 goto free_hndls;
2933 } 3278 }
2934 } 3279 }
2935 itt = (itt_t) cpu_to_be32(((unsigned int)task->itt << 16) | 3280 itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
2936 (unsigned int)(io_task->psgl_handle->sgl_index)); 3281 wrb_index << 16) | (unsigned int)
3282 (io_task->psgl_handle->sgl_index));
3283 io_task->pwrb_handle->pio_handle = task;
3284
2937 io_task->cmd_bhs->iscsi_hdr.itt = itt; 3285 io_task->cmd_bhs->iscsi_hdr.itt = itt;
2938 return 0; 3286 return 0;
2939 3287
@@ -3006,7 +3354,6 @@ static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
3006 io_task->bhs_len = sizeof(struct be_cmd_bhs); 3354 io_task->bhs_len = sizeof(struct be_cmd_bhs);
3007 3355
3008 if (writedir) { 3356 if (writedir) {
3009 SE_DEBUG(DBG_LVL_4, " WRITE Command \t");
3010 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48); 3357 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
3011 AMAP_SET_BITS(struct amap_pdu_data_out, itt, 3358 AMAP_SET_BITS(struct amap_pdu_data_out, itt,
3012 &io_task->cmd_bhs->iscsi_data_pdu, 3359 &io_task->cmd_bhs->iscsi_data_pdu,
@@ -3016,11 +3363,12 @@ static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
3016 ISCSI_OPCODE_SCSI_DATA_OUT); 3363 ISCSI_OPCODE_SCSI_DATA_OUT);
3017 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit, 3364 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
3018 &io_task->cmd_bhs->iscsi_data_pdu, 1); 3365 &io_task->cmd_bhs->iscsi_data_pdu, 1);
3019 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_WR_CMD); 3366 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3367 INI_WR_CMD);
3020 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1); 3368 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3021 } else { 3369 } else {
3022 SE_DEBUG(DBG_LVL_4, "READ Command \t"); 3370 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3023 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_RD_CMD); 3371 INI_RD_CMD);
3024 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0); 3372 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
3025 } 3373 }
3026 memcpy(&io_task->cmd_bhs->iscsi_data_pdu. 3374 memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
@@ -3059,10 +3407,16 @@ static int beiscsi_mtask(struct iscsi_task *task)
3059 struct iscsi_conn *conn = task->conn; 3407 struct iscsi_conn *conn = task->conn;
3060 struct beiscsi_conn *beiscsi_conn = conn->dd_data; 3408 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3061 struct beiscsi_hba *phba = beiscsi_conn->phba; 3409 struct beiscsi_hba *phba = beiscsi_conn->phba;
3410 struct iscsi_session *session;
3062 struct iscsi_wrb *pwrb = NULL; 3411 struct iscsi_wrb *pwrb = NULL;
3412 struct hwi_controller *phwi_ctrlr;
3413 struct hwi_wrb_context *pwrb_context;
3414 struct wrb_handle *pwrb_handle;
3063 unsigned int doorbell = 0; 3415 unsigned int doorbell = 0;
3416 unsigned int i, cid;
3064 struct iscsi_task *aborted_task; 3417 struct iscsi_task *aborted_task;
3065 3418
3419 cid = beiscsi_conn->beiscsi_conn_cid;
3066 pwrb = io_task->pwrb_handle->pwrb; 3420 pwrb = io_task->pwrb_handle->pwrb;
3067 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 3421 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3068 be32_to_cpu(task->cmdsn)); 3422 be32_to_cpu(task->cmdsn));
@@ -3073,33 +3427,43 @@ static int beiscsi_mtask(struct iscsi_task *task)
3073 3427
3074 switch (task->hdr->opcode & ISCSI_OPCODE_MASK) { 3428 switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
3075 case ISCSI_OP_LOGIN: 3429 case ISCSI_OP_LOGIN:
3076 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, TGT_DM_CMD); 3430 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3431 TGT_DM_CMD);
3077 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); 3432 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3078 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1); 3433 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
3079 hwi_write_buffer(pwrb, task); 3434 hwi_write_buffer(pwrb, task);
3080 break; 3435 break;
3081 case ISCSI_OP_NOOP_OUT: 3436 case ISCSI_OP_NOOP_OUT:
3082 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_RD_CMD); 3437 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3438 INI_RD_CMD);
3083 hwi_write_buffer(pwrb, task); 3439 hwi_write_buffer(pwrb, task);
3084 break; 3440 break;
3085 case ISCSI_OP_TEXT: 3441 case ISCSI_OP_TEXT:
3086 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_WR_CMD); 3442 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3443 INI_WR_CMD);
3087 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1); 3444 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3088 hwi_write_buffer(pwrb, task); 3445 hwi_write_buffer(pwrb, task);
3089 break; 3446 break;
3090 case ISCSI_OP_SCSI_TMFUNC: 3447 case ISCSI_OP_SCSI_TMFUNC:
3091 aborted_task = iscsi_itt_to_task(conn, 3448 session = conn->session;
3092 ((struct iscsi_tm *)task->hdr)->rtt); 3449 i = ((struct iscsi_tm *)task->hdr)->rtt;
3450 phwi_ctrlr = phba->phwi_ctrlr;
3451 pwrb_context = &phwi_ctrlr->wrb_context[cid];
3452 pwrb_handle = pwrb_context->pwrb_handle_basestd[be32_to_cpu(i)
3453 >> 16];
3454 aborted_task = pwrb_handle->pio_handle;
3093 if (!aborted_task) 3455 if (!aborted_task)
3094 return 0; 3456 return 0;
3457
3095 aborted_io_task = aborted_task->dd_data; 3458 aborted_io_task = aborted_task->dd_data;
3096 if (!aborted_io_task->scsi_cmnd) 3459 if (!aborted_io_task->scsi_cmnd)
3097 return 0; 3460 return 0;
3098 3461
3099 mgmt_invalidate_icds(phba, 3462 mgmt_invalidate_icds(phba,
3100 aborted_io_task->psgl_handle->sgl_index, 3463 aborted_io_task->psgl_handle->sgl_index,
3101 beiscsi_conn->beiscsi_conn_cid); 3464 cid);
3102 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_TMF_CMD); 3465 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3466 INI_TMF_CMD);
3103 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); 3467 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3104 hwi_write_buffer(pwrb, task); 3468 hwi_write_buffer(pwrb, task);
3105 break; 3469 break;
@@ -3122,7 +3486,7 @@ static int beiscsi_mtask(struct iscsi_task *task)
3122 io_task->pwrb_handle->nxt_wrb_index); 3486 io_task->pwrb_handle->nxt_wrb_index);
3123 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb)); 3487 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3124 3488
3125 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK; 3489 doorbell |= cid & DB_WRB_POST_CID_MASK;
3126 doorbell |= (io_task->pwrb_handle->wrb_index & 3490 doorbell |= (io_task->pwrb_handle->wrb_index &
3127 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT; 3491 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3128 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 3492 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
@@ -3165,9 +3529,14 @@ static int beiscsi_task_xmit(struct iscsi_task *task)
3165 return beiscsi_iotask(task, sg, num_sg, xferlen, writedir); 3529 return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
3166} 3530}
3167 3531
3532
3168static void beiscsi_remove(struct pci_dev *pcidev) 3533static void beiscsi_remove(struct pci_dev *pcidev)
3169{ 3534{
3170 struct beiscsi_hba *phba = NULL; 3535 struct beiscsi_hba *phba = NULL;
3536 struct hwi_controller *phwi_ctrlr;
3537 struct hwi_context_memory *phwi_context;
3538 struct be_eq_obj *pbe_eq;
3539 unsigned int i, msix_vec;
3171 3540
3172 phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev); 3541 phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
3173 if (!phba) { 3542 if (!phba) {
@@ -3175,12 +3544,24 @@ static void beiscsi_remove(struct pci_dev *pcidev)
3175 return; 3544 return;
3176 } 3545 }
3177 3546
3547 phwi_ctrlr = phba->phwi_ctrlr;
3548 phwi_context = phwi_ctrlr->phwi_ctxt;
3178 hwi_disable_intr(phba); 3549 hwi_disable_intr(phba);
3179 if (phba->pcidev->irq) 3550 if (phba->msix_enabled) {
3180 free_irq(phba->pcidev->irq, phba); 3551 for (i = 0; i <= phba->num_cpus; i++) {
3552 msix_vec = phba->msix_entries[i].vector;
3553 free_irq(msix_vec, &phwi_context->be_eq[i]);
3554 }
3555 } else
3556 if (phba->pcidev->irq)
3557 free_irq(phba->pcidev->irq, phba);
3558 pci_disable_msix(phba->pcidev);
3181 destroy_workqueue(phba->wq); 3559 destroy_workqueue(phba->wq);
3182 if (blk_iopoll_enabled) 3560 if (blk_iopoll_enabled)
3183 blk_iopoll_disable(&phba->iopoll); 3561 for (i = 0; i < phba->num_cpus; i++) {
3562 pbe_eq = &phwi_context->be_eq[i];
3563 blk_iopoll_disable(&pbe_eq->iopoll);
3564 }
3184 3565
3185 beiscsi_clean_port(phba); 3566 beiscsi_clean_port(phba);
3186 beiscsi_free_mem(phba); 3567 beiscsi_free_mem(phba);
@@ -3194,11 +3575,29 @@ static void beiscsi_remove(struct pci_dev *pcidev)
3194 iscsi_host_free(phba->shost); 3575 iscsi_host_free(phba->shost);
3195} 3576}
3196 3577
3578static void beiscsi_msix_enable(struct beiscsi_hba *phba)
3579{
3580 int i, status;
3581
3582 for (i = 0; i <= phba->num_cpus; i++)
3583 phba->msix_entries[i].entry = i;
3584
3585 status = pci_enable_msix(phba->pcidev, phba->msix_entries,
3586 (phba->num_cpus + 1));
3587 if (!status)
3588 phba->msix_enabled = true;
3589
3590 return;
3591}
3592
3197static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev, 3593static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
3198 const struct pci_device_id *id) 3594 const struct pci_device_id *id)
3199{ 3595{
3200 struct beiscsi_hba *phba = NULL; 3596 struct beiscsi_hba *phba = NULL;
3201 int ret; 3597 struct hwi_controller *phwi_ctrlr;
3598 struct hwi_context_memory *phwi_context;
3599 struct be_eq_obj *pbe_eq;
3600 int ret, msix_vec, num_cpus, i;
3202 3601
3203 ret = beiscsi_enable_pci(pcidev); 3602 ret = beiscsi_enable_pci(pcidev);
3204 if (ret < 0) { 3603 if (ret < 0) {
@@ -3213,8 +3612,18 @@ static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
3213 " Failed in beiscsi_hba_alloc \n"); 3612 " Failed in beiscsi_hba_alloc \n");
3214 goto disable_pci; 3613 goto disable_pci;
3215 } 3614 }
3615 SE_DEBUG(DBG_LVL_8, " phba = %p \n", phba);
3216 3616
3217 pci_set_drvdata(pcidev, phba); 3617 pci_set_drvdata(pcidev, phba);
3618 if (enable_msix)
3619 num_cpus = find_num_cpus();
3620 else
3621 num_cpus = 1;
3622 phba->num_cpus = num_cpus;
3623 SE_DEBUG(DBG_LVL_8, "num_cpus = %d \n", phba->num_cpus);
3624
3625 if (enable_msix)
3626 beiscsi_msix_enable(phba);
3218 ret = be_ctrl_init(phba, pcidev); 3627 ret = be_ctrl_init(phba, pcidev);
3219 if (ret) { 3628 if (ret) {
3220 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-" 3629 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
@@ -3235,7 +3644,7 @@ static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
3235 3644
3236 snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u", 3645 snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
3237 phba->shost->host_no); 3646 phba->shost->host_no);
3238 phba->wq = create_singlethread_workqueue(phba->wq_name); 3647 phba->wq = create_workqueue(phba->wq_name);
3239 if (!phba->wq) { 3648 if (!phba->wq) {
3240 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-" 3649 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3241 "Failed to allocate work queue\n"); 3650 "Failed to allocate work queue\n");
@@ -3244,11 +3653,16 @@ static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
3244 3653
3245 INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs); 3654 INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
3246 3655
3656 phwi_ctrlr = phba->phwi_ctrlr;
3657 phwi_context = phwi_ctrlr->phwi_ctxt;
3247 if (blk_iopoll_enabled) { 3658 if (blk_iopoll_enabled) {
3248 blk_iopoll_init(&phba->iopoll, be_iopoll_budget, be_iopoll); 3659 for (i = 0; i < phba->num_cpus; i++) {
3249 blk_iopoll_enable(&phba->iopoll); 3660 pbe_eq = &phwi_context->be_eq[i];
3661 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
3662 be_iopoll);
3663 blk_iopoll_enable(&pbe_eq->iopoll);
3664 }
3250 } 3665 }
3251
3252 ret = beiscsi_init_irqs(phba); 3666 ret = beiscsi_init_irqs(phba);
3253 if (ret < 0) { 3667 if (ret < 0) {
3254 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-" 3668 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
@@ -3261,17 +3675,26 @@ static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
3261 "Failed to hwi_enable_intr\n"); 3675 "Failed to hwi_enable_intr\n");
3262 goto free_ctrlr; 3676 goto free_ctrlr;
3263 } 3677 }
3264
3265 SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED \n\n\n"); 3678 SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED \n\n\n");
3266 return 0; 3679 return 0;
3267 3680
3268free_ctrlr: 3681free_ctrlr:
3269 if (phba->pcidev->irq) 3682 if (phba->msix_enabled) {
3270 free_irq(phba->pcidev->irq, phba); 3683 for (i = 0; i <= phba->num_cpus; i++) {
3684 msix_vec = phba->msix_entries[i].vector;
3685 free_irq(msix_vec, &phwi_context->be_eq[i]);
3686 }
3687 } else
3688 if (phba->pcidev->irq)
3689 free_irq(phba->pcidev->irq, phba);
3690 pci_disable_msix(phba->pcidev);
3271free_blkenbld: 3691free_blkenbld:
3272 destroy_workqueue(phba->wq); 3692 destroy_workqueue(phba->wq);
3273 if (blk_iopoll_enabled) 3693 if (blk_iopoll_enabled)
3274 blk_iopoll_disable(&phba->iopoll); 3694 for (i = 0; i < phba->num_cpus; i++) {
3695 pbe_eq = &phwi_context->be_eq[i];
3696 blk_iopoll_disable(&pbe_eq->iopoll);
3697 }
3275free_twq: 3698free_twq:
3276 beiscsi_clean_port(phba); 3699 beiscsi_clean_port(phba);
3277 beiscsi_free_mem(phba); 3700 beiscsi_free_mem(phba);
@@ -3351,6 +3774,7 @@ static struct pci_driver beiscsi_pci_driver = {
3351 .id_table = beiscsi_pci_id_table 3774 .id_table = beiscsi_pci_id_table
3352}; 3775};
3353 3776
3777
3354static int __init beiscsi_module_init(void) 3778static int __init beiscsi_module_init(void)
3355{ 3779{
3356 int ret; 3780 int ret;