aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/scsi/csiostor/csio_wr.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/scsi/csiostor/csio_wr.c')
-rw-r--r--drivers/scsi/csiostor/csio_wr.c1632
1 files changed, 1632 insertions, 0 deletions
diff --git a/drivers/scsi/csiostor/csio_wr.c b/drivers/scsi/csiostor/csio_wr.c
new file mode 100644
index 000000000000..c32df1bdaa97
--- /dev/null
+++ b/drivers/scsi/csiostor/csio_wr.c
@@ -0,0 +1,1632 @@
1/*
2 * This file is part of the Chelsio FCoE driver for Linux.
3 *
4 * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#include <linux/kernel.h>
36#include <linux/string.h>
37#include <linux/compiler.h>
38#include <linux/slab.h>
39#include <asm/page.h>
40#include <linux/cache.h>
41
42#include "csio_hw.h"
43#include "csio_wr.h"
44#include "csio_mb.h"
45#include "csio_defs.h"
46
47int csio_intr_coalesce_cnt; /* value:SGE_INGRESS_RX_THRESHOLD[0] */
48static int csio_sge_thresh_reg; /* SGE_INGRESS_RX_THRESHOLD[0] */
49
50int csio_intr_coalesce_time = 10; /* value:SGE_TIMER_VALUE_1 */
51static int csio_sge_timer_reg = 1;
52
53#define CSIO_SET_FLBUF_SIZE(_hw, _reg, _val) \
54 csio_wr_reg32((_hw), (_val), SGE_FL_BUFFER_SIZE##_reg)
55
56static void
57csio_get_flbuf_size(struct csio_hw *hw, struct csio_sge *sge, uint32_t reg)
58{
59 sge->sge_fl_buf_size[reg] = csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE0 +
60 reg * sizeof(uint32_t));
61}
62
63/* Free list buffer size */
64static inline uint32_t
65csio_wr_fl_bufsz(struct csio_sge *sge, struct csio_dma_buf *buf)
66{
67 return sge->sge_fl_buf_size[buf->paddr & 0xF];
68}
69
70/* Size of the egress queue status page */
71static inline uint32_t
72csio_wr_qstat_pgsz(struct csio_hw *hw)
73{
74 return (hw->wrm.sge.sge_control & EGRSTATUSPAGESIZE(1)) ? 128 : 64;
75}
76
77/* Ring freelist doorbell */
78static inline void
79csio_wr_ring_fldb(struct csio_hw *hw, struct csio_q *flq)
80{
81 /*
82 * Ring the doorbell only when we have atleast CSIO_QCREDIT_SZ
83 * number of bytes in the freelist queue. This translates to atleast
84 * 8 freelist buffer pointers (since each pointer is 8 bytes).
85 */
86 if (flq->inc_idx >= 8) {
87 csio_wr_reg32(hw, DBPRIO(1) | QID(flq->un.fl.flid) |
88 PIDX(flq->inc_idx / 8),
89 MYPF_REG(SGE_PF_KDOORBELL));
90 flq->inc_idx &= 7;
91 }
92}
93
94/* Write a 0 cidx increment value to enable SGE interrupts for this queue */
95static void
96csio_wr_sge_intr_enable(struct csio_hw *hw, uint16_t iqid)
97{
98 csio_wr_reg32(hw, CIDXINC(0) |
99 INGRESSQID(iqid) |
100 TIMERREG(X_TIMERREG_RESTART_COUNTER),
101 MYPF_REG(SGE_PF_GTS));
102}
103
104/*
105 * csio_wr_fill_fl - Populate the FL buffers of a FL queue.
106 * @hw: HW module.
107 * @flq: Freelist queue.
108 *
109 * Fill up freelist buffer entries with buffers of size specified
110 * in the size register.
111 *
112 */
113static int
114csio_wr_fill_fl(struct csio_hw *hw, struct csio_q *flq)
115{
116 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
117 struct csio_sge *sge = &wrm->sge;
118 __be64 *d = (__be64 *)(flq->vstart);
119 struct csio_dma_buf *buf = &flq->un.fl.bufs[0];
120 uint64_t paddr;
121 int sreg = flq->un.fl.sreg;
122 int n = flq->credits;
123
124 while (n--) {
125 buf->len = sge->sge_fl_buf_size[sreg];
126 buf->vaddr = pci_alloc_consistent(hw->pdev, buf->len,
127 &buf->paddr);
128 if (!buf->vaddr) {
129 csio_err(hw, "Could only fill %d buffers!\n", n + 1);
130 return -ENOMEM;
131 }
132
133 paddr = buf->paddr | (sreg & 0xF);
134
135 *d++ = cpu_to_be64(paddr);
136 buf++;
137 }
138
139 return 0;
140}
141
142/*
143 * csio_wr_update_fl -
144 * @hw: HW module.
145 * @flq: Freelist queue.
146 *
147 *
148 */
149static inline void
150csio_wr_update_fl(struct csio_hw *hw, struct csio_q *flq, uint16_t n)
151{
152
153 flq->inc_idx += n;
154 flq->pidx += n;
155 if (unlikely(flq->pidx >= flq->credits))
156 flq->pidx -= (uint16_t)flq->credits;
157
158 CSIO_INC_STATS(flq, n_flq_refill);
159}
160
161/*
162 * csio_wr_alloc_q - Allocate a WR queue and initialize it.
163 * @hw: HW module
164 * @qsize: Size of the queue in bytes
165 * @wrsize: Since of WR in this queue, if fixed.
166 * @type: Type of queue (Ingress/Egress/Freelist)
167 * @owner: Module that owns this queue.
168 * @nflb: Number of freelist buffers for FL.
169 * @sreg: What is the FL buffer size register?
170 * @iq_int_handler: Ingress queue handler in INTx mode.
171 *
172 * This function allocates and sets up a queue for the caller
173 * of size qsize, aligned at the required boundary. This is subject to
174 * be free entries being available in the queue array. If one is found,
175 * it is initialized with the allocated queue, marked as being used (owner),
176 * and a handle returned to the caller in form of the queue's index
177 * into the q_arr array.
178 * If user has indicated a freelist (by specifying nflb > 0), create
179 * another queue (with its own index into q_arr) for the freelist. Allocate
180 * memory for DMA buffer metadata (vaddr, len etc). Save off the freelist
181 * idx in the ingress queue's flq.idx. This is how a Freelist is associated
182 * with its owning ingress queue.
183 */
184int
185csio_wr_alloc_q(struct csio_hw *hw, uint32_t qsize, uint32_t wrsize,
186 uint16_t type, void *owner, uint32_t nflb, int sreg,
187 iq_handler_t iq_intx_handler)
188{
189 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
190 struct csio_q *q, *flq;
191 int free_idx = wrm->free_qidx;
192 int ret_idx = free_idx;
193 uint32_t qsz;
194 int flq_idx;
195
196 if (free_idx >= wrm->num_q) {
197 csio_err(hw, "No more free queues.\n");
198 return -1;
199 }
200
201 switch (type) {
202 case CSIO_EGRESS:
203 qsz = ALIGN(qsize, CSIO_QCREDIT_SZ) + csio_wr_qstat_pgsz(hw);
204 break;
205 case CSIO_INGRESS:
206 switch (wrsize) {
207 case 16:
208 case 32:
209 case 64:
210 case 128:
211 break;
212 default:
213 csio_err(hw, "Invalid Ingress queue WR size:%d\n",
214 wrsize);
215 return -1;
216 }
217
218 /*
219 * Number of elements must be a multiple of 16
220 * So this includes status page size
221 */
222 qsz = ALIGN(qsize/wrsize, 16) * wrsize;
223
224 break;
225 case CSIO_FREELIST:
226 qsz = ALIGN(qsize/wrsize, 8) * wrsize + csio_wr_qstat_pgsz(hw);
227 break;
228 default:
229 csio_err(hw, "Invalid queue type: 0x%x\n", type);
230 return -1;
231 }
232
233 q = wrm->q_arr[free_idx];
234
235 q->vstart = pci_alloc_consistent(hw->pdev, qsz, &q->pstart);
236 if (!q->vstart) {
237 csio_err(hw,
238 "Failed to allocate DMA memory for "
239 "queue at id: %d size: %d\n", free_idx, qsize);
240 return -1;
241 }
242
243 /*
244 * We need to zero out the contents, importantly for ingress,
245 * since we start with a generatiom bit of 1 for ingress.
246 */
247 memset(q->vstart, 0, qsz);
248
249 q->type = type;
250 q->owner = owner;
251 q->pidx = q->cidx = q->inc_idx = 0;
252 q->size = qsz;
253 q->wr_sz = wrsize; /* If using fixed size WRs */
254
255 wrm->free_qidx++;
256
257 if (type == CSIO_INGRESS) {
258 /* Since queue area is set to zero */
259 q->un.iq.genbit = 1;
260
261 /*
262 * Ingress queue status page size is always the size of
263 * the ingress queue entry.
264 */
265 q->credits = (qsz - q->wr_sz) / q->wr_sz;
266 q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz
267 - q->wr_sz);
268
269 /* Allocate memory for FL if requested */
270 if (nflb > 0) {
271 flq_idx = csio_wr_alloc_q(hw, nflb * sizeof(__be64),
272 sizeof(__be64), CSIO_FREELIST,
273 owner, 0, sreg, NULL);
274 if (flq_idx == -1) {
275 csio_err(hw,
276 "Failed to allocate FL queue"
277 " for IQ idx:%d\n", free_idx);
278 return -1;
279 }
280
281 /* Associate the new FL with the Ingress quue */
282 q->un.iq.flq_idx = flq_idx;
283
284 flq = wrm->q_arr[q->un.iq.flq_idx];
285 flq->un.fl.bufs = kzalloc(flq->credits *
286 sizeof(struct csio_dma_buf),
287 GFP_KERNEL);
288 if (!flq->un.fl.bufs) {
289 csio_err(hw,
290 "Failed to allocate FL queue bufs"
291 " for IQ idx:%d\n", free_idx);
292 return -1;
293 }
294
295 flq->un.fl.packen = 0;
296 flq->un.fl.offset = 0;
297 flq->un.fl.sreg = sreg;
298
299 /* Fill up the free list buffers */
300 if (csio_wr_fill_fl(hw, flq))
301 return -1;
302
303 /*
304 * Make sure in a FLQ, atleast 1 credit (8 FL buffers)
305 * remains unpopulated,otherwise HW thinks
306 * FLQ is empty.
307 */
308 flq->pidx = flq->inc_idx = flq->credits - 8;
309 } else {
310 q->un.iq.flq_idx = -1;
311 }
312
313 /* Associate the IQ INTx handler. */
314 q->un.iq.iq_intx_handler = iq_intx_handler;
315
316 csio_q_iqid(hw, ret_idx) = CSIO_MAX_QID;
317
318 } else if (type == CSIO_EGRESS) {
319 q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / CSIO_QCREDIT_SZ;
320 q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz
321 - csio_wr_qstat_pgsz(hw));
322 csio_q_eqid(hw, ret_idx) = CSIO_MAX_QID;
323 } else { /* Freelist */
324 q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / sizeof(__be64);
325 q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz
326 - csio_wr_qstat_pgsz(hw));
327 csio_q_flid(hw, ret_idx) = CSIO_MAX_QID;
328 }
329
330 return ret_idx;
331}
332
333/*
334 * csio_wr_iq_create_rsp - Response handler for IQ creation.
335 * @hw: The HW module.
336 * @mbp: Mailbox.
337 * @iq_idx: Ingress queue that got created.
338 *
339 * Handle FW_IQ_CMD mailbox completion. Save off the assigned IQ/FL ids.
340 */
341static int
342csio_wr_iq_create_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)
343{
344 struct csio_iq_params iqp;
345 enum fw_retval retval;
346 uint32_t iq_id;
347 int flq_idx;
348
349 memset(&iqp, 0, sizeof(struct csio_iq_params));
350
351 csio_mb_iq_alloc_write_rsp(hw, mbp, &retval, &iqp);
352
353 if (retval != FW_SUCCESS) {
354 csio_err(hw, "IQ cmd returned 0x%x!\n", retval);
355 mempool_free(mbp, hw->mb_mempool);
356 return -EINVAL;
357 }
358
359 csio_q_iqid(hw, iq_idx) = iqp.iqid;
360 csio_q_physiqid(hw, iq_idx) = iqp.physiqid;
361 csio_q_pidx(hw, iq_idx) = csio_q_cidx(hw, iq_idx) = 0;
362 csio_q_inc_idx(hw, iq_idx) = 0;
363
364 /* Actual iq-id. */
365 iq_id = iqp.iqid - hw->wrm.fw_iq_start;
366
367 /* Set the iq-id to iq map table. */
368 if (iq_id >= CSIO_MAX_IQ) {
369 csio_err(hw,
370 "Exceeding MAX_IQ(%d) supported!"
371 " iqid:%d rel_iqid:%d FW iq_start:%d\n",
372 CSIO_MAX_IQ, iq_id, iqp.iqid, hw->wrm.fw_iq_start);
373 mempool_free(mbp, hw->mb_mempool);
374 return -EINVAL;
375 }
376 csio_q_set_intr_map(hw, iq_idx, iq_id);
377
378 /*
379 * During FW_IQ_CMD, FW sets interrupt_sent bit to 1 in the SGE
380 * ingress context of this queue. This will block interrupts to
381 * this queue until the next GTS write. Therefore, we do a
382 * 0-cidx increment GTS write for this queue just to clear the
383 * interrupt_sent bit. This will re-enable interrupts to this
384 * queue.
385 */
386 csio_wr_sge_intr_enable(hw, iqp.physiqid);
387
388 flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
389 if (flq_idx != -1) {
390 struct csio_q *flq = hw->wrm.q_arr[flq_idx];
391
392 csio_q_flid(hw, flq_idx) = iqp.fl0id;
393 csio_q_cidx(hw, flq_idx) = 0;
394 csio_q_pidx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8;
395 csio_q_inc_idx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8;
396
397 /* Now update SGE about the buffers allocated during init */
398 csio_wr_ring_fldb(hw, flq);
399 }
400
401 mempool_free(mbp, hw->mb_mempool);
402
403 return 0;
404}
405
406/*
407 * csio_wr_iq_create - Configure an Ingress queue with FW.
408 * @hw: The HW module.
409 * @priv: Private data object.
410 * @iq_idx: Ingress queue index in the WR module.
411 * @vec: MSIX vector.
412 * @portid: PCIE Channel to be associated with this queue.
413 * @async: Is this a FW asynchronous message handling queue?
414 * @cbfn: Completion callback.
415 *
416 * This API configures an ingress queue with FW by issuing a FW_IQ_CMD mailbox
417 * with alloc/write bits set.
418 */
419int
420csio_wr_iq_create(struct csio_hw *hw, void *priv, int iq_idx,
421 uint32_t vec, uint8_t portid, bool async,
422 void (*cbfn) (struct csio_hw *, struct csio_mb *))
423{
424 struct csio_mb *mbp;
425 struct csio_iq_params iqp;
426 int flq_idx;
427
428 memset(&iqp, 0, sizeof(struct csio_iq_params));
429 csio_q_portid(hw, iq_idx) = portid;
430
431 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
432 if (!mbp) {
433 csio_err(hw, "IQ command out of memory!\n");
434 return -ENOMEM;
435 }
436
437 switch (hw->intr_mode) {
438 case CSIO_IM_INTX:
439 case CSIO_IM_MSI:
440 /* For interrupt forwarding queue only */
441 if (hw->intr_iq_idx == iq_idx)
442 iqp.iqandst = X_INTERRUPTDESTINATION_PCIE;
443 else
444 iqp.iqandst = X_INTERRUPTDESTINATION_IQ;
445 iqp.iqandstindex =
446 csio_q_physiqid(hw, hw->intr_iq_idx);
447 break;
448 case CSIO_IM_MSIX:
449 iqp.iqandst = X_INTERRUPTDESTINATION_PCIE;
450 iqp.iqandstindex = (uint16_t)vec;
451 break;
452 case CSIO_IM_NONE:
453 mempool_free(mbp, hw->mb_mempool);
454 return -EINVAL;
455 }
456
457 /* Pass in the ingress queue cmd parameters */
458 iqp.pfn = hw->pfn;
459 iqp.vfn = 0;
460 iqp.iq_start = 1;
461 iqp.viid = 0;
462 iqp.type = FW_IQ_TYPE_FL_INT_CAP;
463 iqp.iqasynch = async;
464 if (csio_intr_coalesce_cnt)
465 iqp.iqanus = X_UPDATESCHEDULING_COUNTER_OPTTIMER;
466 else
467 iqp.iqanus = X_UPDATESCHEDULING_TIMER;
468 iqp.iqanud = X_UPDATEDELIVERY_INTERRUPT;
469 iqp.iqpciech = portid;
470 iqp.iqintcntthresh = (uint8_t)csio_sge_thresh_reg;
471
472 switch (csio_q_wr_sz(hw, iq_idx)) {
473 case 16:
474 iqp.iqesize = 0; break;
475 case 32:
476 iqp.iqesize = 1; break;
477 case 64:
478 iqp.iqesize = 2; break;
479 case 128:
480 iqp.iqesize = 3; break;
481 }
482
483 iqp.iqsize = csio_q_size(hw, iq_idx) /
484 csio_q_wr_sz(hw, iq_idx);
485 iqp.iqaddr = csio_q_pstart(hw, iq_idx);
486
487 flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
488 if (flq_idx != -1) {
489 struct csio_q *flq = hw->wrm.q_arr[flq_idx];
490
491 iqp.fl0paden = 1;
492 iqp.fl0packen = flq->un.fl.packen ? 1 : 0;
493 iqp.fl0fbmin = X_FETCHBURSTMIN_64B;
494 iqp.fl0fbmax = X_FETCHBURSTMAX_512B;
495 iqp.fl0size = csio_q_size(hw, flq_idx) / CSIO_QCREDIT_SZ;
496 iqp.fl0addr = csio_q_pstart(hw, flq_idx);
497 }
498
499 csio_mb_iq_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);
500
501 if (csio_mb_issue(hw, mbp)) {
502 csio_err(hw, "Issue of IQ cmd failed!\n");
503 mempool_free(mbp, hw->mb_mempool);
504 return -EINVAL;
505 }
506
507 if (cbfn != NULL)
508 return 0;
509
510 return csio_wr_iq_create_rsp(hw, mbp, iq_idx);
511}
512
513/*
514 * csio_wr_eq_create_rsp - Response handler for EQ creation.
515 * @hw: The HW module.
516 * @mbp: Mailbox.
517 * @eq_idx: Egress queue that got created.
518 *
519 * Handle FW_EQ_OFLD_CMD mailbox completion. Save off the assigned EQ ids.
520 */
521static int
522csio_wr_eq_cfg_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)
523{
524 struct csio_eq_params eqp;
525 enum fw_retval retval;
526
527 memset(&eqp, 0, sizeof(struct csio_eq_params));
528
529 csio_mb_eq_ofld_alloc_write_rsp(hw, mbp, &retval, &eqp);
530
531 if (retval != FW_SUCCESS) {
532 csio_err(hw, "EQ OFLD cmd returned 0x%x!\n", retval);
533 mempool_free(mbp, hw->mb_mempool);
534 return -EINVAL;
535 }
536
537 csio_q_eqid(hw, eq_idx) = (uint16_t)eqp.eqid;
538 csio_q_physeqid(hw, eq_idx) = (uint16_t)eqp.physeqid;
539 csio_q_pidx(hw, eq_idx) = csio_q_cidx(hw, eq_idx) = 0;
540 csio_q_inc_idx(hw, eq_idx) = 0;
541
542 mempool_free(mbp, hw->mb_mempool);
543
544 return 0;
545}
546
547/*
548 * csio_wr_eq_create - Configure an Egress queue with FW.
549 * @hw: HW module.
550 * @priv: Private data.
551 * @eq_idx: Egress queue index in the WR module.
552 * @iq_idx: Associated ingress queue index.
553 * @cbfn: Completion callback.
554 *
555 * This API configures a offload egress queue with FW by issuing a
556 * FW_EQ_OFLD_CMD (with alloc + write ) mailbox.
557 */
558int
559csio_wr_eq_create(struct csio_hw *hw, void *priv, int eq_idx,
560 int iq_idx, uint8_t portid,
561 void (*cbfn) (struct csio_hw *, struct csio_mb *))
562{
563 struct csio_mb *mbp;
564 struct csio_eq_params eqp;
565
566 memset(&eqp, 0, sizeof(struct csio_eq_params));
567
568 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
569 if (!mbp) {
570 csio_err(hw, "EQ command out of memory!\n");
571 return -ENOMEM;
572 }
573
574 eqp.pfn = hw->pfn;
575 eqp.vfn = 0;
576 eqp.eqstart = 1;
577 eqp.hostfcmode = X_HOSTFCMODE_STATUS_PAGE;
578 eqp.iqid = csio_q_iqid(hw, iq_idx);
579 eqp.fbmin = X_FETCHBURSTMIN_64B;
580 eqp.fbmax = X_FETCHBURSTMAX_512B;
581 eqp.cidxfthresh = 0;
582 eqp.pciechn = portid;
583 eqp.eqsize = csio_q_size(hw, eq_idx) / CSIO_QCREDIT_SZ;
584 eqp.eqaddr = csio_q_pstart(hw, eq_idx);
585
586 csio_mb_eq_ofld_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO,
587 &eqp, cbfn);
588
589 if (csio_mb_issue(hw, mbp)) {
590 csio_err(hw, "Issue of EQ OFLD cmd failed!\n");
591 mempool_free(mbp, hw->mb_mempool);
592 return -EINVAL;
593 }
594
595 if (cbfn != NULL)
596 return 0;
597
598 return csio_wr_eq_cfg_rsp(hw, mbp, eq_idx);
599}
600
601/*
602 * csio_wr_iq_destroy_rsp - Response handler for IQ removal.
603 * @hw: The HW module.
604 * @mbp: Mailbox.
605 * @iq_idx: Ingress queue that was freed.
606 *
607 * Handle FW_IQ_CMD (free) mailbox completion.
608 */
609static int
610csio_wr_iq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)
611{
612 enum fw_retval retval = csio_mb_fw_retval(mbp);
613 int rv = 0;
614
615 if (retval != FW_SUCCESS)
616 rv = -EINVAL;
617
618 mempool_free(mbp, hw->mb_mempool);
619
620 return rv;
621}
622
623/*
624 * csio_wr_iq_destroy - Free an ingress queue.
625 * @hw: The HW module.
626 * @priv: Private data object.
627 * @iq_idx: Ingress queue index to destroy
628 * @cbfn: Completion callback.
629 *
630 * This API frees an ingress queue by issuing the FW_IQ_CMD
631 * with the free bit set.
632 */
633static int
634csio_wr_iq_destroy(struct csio_hw *hw, void *priv, int iq_idx,
635 void (*cbfn)(struct csio_hw *, struct csio_mb *))
636{
637 int rv = 0;
638 struct csio_mb *mbp;
639 struct csio_iq_params iqp;
640 int flq_idx;
641
642 memset(&iqp, 0, sizeof(struct csio_iq_params));
643
644 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
645 if (!mbp)
646 return -ENOMEM;
647
648 iqp.pfn = hw->pfn;
649 iqp.vfn = 0;
650 iqp.iqid = csio_q_iqid(hw, iq_idx);
651 iqp.type = FW_IQ_TYPE_FL_INT_CAP;
652
653 flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
654 if (flq_idx != -1)
655 iqp.fl0id = csio_q_flid(hw, flq_idx);
656 else
657 iqp.fl0id = 0xFFFF;
658
659 iqp.fl1id = 0xFFFF;
660
661 csio_mb_iq_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);
662
663 rv = csio_mb_issue(hw, mbp);
664 if (rv != 0) {
665 mempool_free(mbp, hw->mb_mempool);
666 return rv;
667 }
668
669 if (cbfn != NULL)
670 return 0;
671
672 return csio_wr_iq_destroy_rsp(hw, mbp, iq_idx);
673}
674
675/*
676 * csio_wr_eq_destroy_rsp - Response handler for OFLD EQ creation.
677 * @hw: The HW module.
678 * @mbp: Mailbox.
679 * @eq_idx: Egress queue that was freed.
680 *
681 * Handle FW_OFLD_EQ_CMD (free) mailbox completion.
682 */
683static int
684csio_wr_eq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)
685{
686 enum fw_retval retval = csio_mb_fw_retval(mbp);
687 int rv = 0;
688
689 if (retval != FW_SUCCESS)
690 rv = -EINVAL;
691
692 mempool_free(mbp, hw->mb_mempool);
693
694 return rv;
695}
696
697/*
698 * csio_wr_eq_destroy - Free an Egress queue.
699 * @hw: The HW module.
700 * @priv: Private data object.
701 * @eq_idx: Egress queue index to destroy
702 * @cbfn: Completion callback.
703 *
704 * This API frees an Egress queue by issuing the FW_EQ_OFLD_CMD
705 * with the free bit set.
706 */
707static int
708csio_wr_eq_destroy(struct csio_hw *hw, void *priv, int eq_idx,
709 void (*cbfn) (struct csio_hw *, struct csio_mb *))
710{
711 int rv = 0;
712 struct csio_mb *mbp;
713 struct csio_eq_params eqp;
714
715 memset(&eqp, 0, sizeof(struct csio_eq_params));
716
717 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
718 if (!mbp)
719 return -ENOMEM;
720
721 eqp.pfn = hw->pfn;
722 eqp.vfn = 0;
723 eqp.eqid = csio_q_eqid(hw, eq_idx);
724
725 csio_mb_eq_ofld_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &eqp, cbfn);
726
727 rv = csio_mb_issue(hw, mbp);
728 if (rv != 0) {
729 mempool_free(mbp, hw->mb_mempool);
730 return rv;
731 }
732
733 if (cbfn != NULL)
734 return 0;
735
736 return csio_wr_eq_destroy_rsp(hw, mbp, eq_idx);
737}
738
739/*
740 * csio_wr_cleanup_eq_stpg - Cleanup Egress queue status page
741 * @hw: HW module
742 * @qidx: Egress queue index
743 *
744 * Cleanup the Egress queue status page.
745 */
746static void
747csio_wr_cleanup_eq_stpg(struct csio_hw *hw, int qidx)
748{
749 struct csio_q *q = csio_hw_to_wrm(hw)->q_arr[qidx];
750 struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;
751
752 memset(stp, 0, sizeof(*stp));
753}
754
755/*
756 * csio_wr_cleanup_iq_ftr - Cleanup Footer entries in IQ
757 * @hw: HW module
758 * @qidx: Ingress queue index
759 *
760 * Cleanup the footer entries in the given ingress queue,
761 * set to 1 the internal copy of genbit.
762 */
763static void
764csio_wr_cleanup_iq_ftr(struct csio_hw *hw, int qidx)
765{
766 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
767 struct csio_q *q = wrm->q_arr[qidx];
768 void *wr;
769 struct csio_iqwr_footer *ftr;
770 uint32_t i = 0;
771
772 /* set to 1 since we are just about zero out genbit */
773 q->un.iq.genbit = 1;
774
775 for (i = 0; i < q->credits; i++) {
776 /* Get the WR */
777 wr = (void *)((uintptr_t)q->vstart +
778 (i * q->wr_sz));
779 /* Get the footer */
780 ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
781 (q->wr_sz - sizeof(*ftr)));
782 /* Zero out footer */
783 memset(ftr, 0, sizeof(*ftr));
784 }
785}
786
787int
788csio_wr_destroy_queues(struct csio_hw *hw, bool cmd)
789{
790 int i, flq_idx;
791 struct csio_q *q;
792 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
793 int rv;
794
795 for (i = 0; i < wrm->free_qidx; i++) {
796 q = wrm->q_arr[i];
797
798 switch (q->type) {
799 case CSIO_EGRESS:
800 if (csio_q_eqid(hw, i) != CSIO_MAX_QID) {
801 csio_wr_cleanup_eq_stpg(hw, i);
802 if (!cmd) {
803 csio_q_eqid(hw, i) = CSIO_MAX_QID;
804 continue;
805 }
806
807 rv = csio_wr_eq_destroy(hw, NULL, i, NULL);
808 if ((rv == -EBUSY) || (rv == -ETIMEDOUT))
809 cmd = false;
810
811 csio_q_eqid(hw, i) = CSIO_MAX_QID;
812 }
813 case CSIO_INGRESS:
814 if (csio_q_iqid(hw, i) != CSIO_MAX_QID) {
815 csio_wr_cleanup_iq_ftr(hw, i);
816 if (!cmd) {
817 csio_q_iqid(hw, i) = CSIO_MAX_QID;
818 flq_idx = csio_q_iq_flq_idx(hw, i);
819 if (flq_idx != -1)
820 csio_q_flid(hw, flq_idx) =
821 CSIO_MAX_QID;
822 continue;
823 }
824
825 rv = csio_wr_iq_destroy(hw, NULL, i, NULL);
826 if ((rv == -EBUSY) || (rv == -ETIMEDOUT))
827 cmd = false;
828
829 csio_q_iqid(hw, i) = CSIO_MAX_QID;
830 flq_idx = csio_q_iq_flq_idx(hw, i);
831 if (flq_idx != -1)
832 csio_q_flid(hw, flq_idx) = CSIO_MAX_QID;
833 }
834 default:
835 break;
836 }
837 }
838
839 hw->flags &= ~CSIO_HWF_Q_FW_ALLOCED;
840
841 return 0;
842}
843
844/*
845 * csio_wr_get - Get requested size of WR entry/entries from queue.
846 * @hw: HW module.
847 * @qidx: Index of queue.
848 * @size: Cumulative size of Work request(s).
849 * @wrp: Work request pair.
850 *
851 * If requested credits are available, return the start address of the
852 * work request in the work request pair. Set pidx accordingly and
853 * return.
854 *
855 * NOTE about WR pair:
856 * ==================
857 * A WR can start towards the end of a queue, and then continue at the
858 * beginning, since the queue is considered to be circular. This will
859 * require a pair of address/size to be passed back to the caller -
860 * hence Work request pair format.
861 */
862int
863csio_wr_get(struct csio_hw *hw, int qidx, uint32_t size,
864 struct csio_wr_pair *wrp)
865{
866 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
867 struct csio_q *q = wrm->q_arr[qidx];
868 void *cwr = (void *)((uintptr_t)(q->vstart) +
869 (q->pidx * CSIO_QCREDIT_SZ));
870 struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;
871 uint16_t cidx = q->cidx = ntohs(stp->cidx);
872 uint16_t pidx = q->pidx;
873 uint32_t req_sz = ALIGN(size, CSIO_QCREDIT_SZ);
874 int req_credits = req_sz / CSIO_QCREDIT_SZ;
875 int credits;
876
877 CSIO_DB_ASSERT(q->owner != NULL);
878 CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));
879 CSIO_DB_ASSERT(cidx <= q->credits);
880
881 /* Calculate credits */
882 if (pidx > cidx) {
883 credits = q->credits - (pidx - cidx) - 1;
884 } else if (cidx > pidx) {
885 credits = cidx - pidx - 1;
886 } else {
887 /* cidx == pidx, empty queue */
888 credits = q->credits;
889 CSIO_INC_STATS(q, n_qempty);
890 }
891
892 /*
893 * Check if we have enough credits.
894 * credits = 1 implies queue is full.
895 */
896 if (!credits || (req_credits > credits)) {
897 CSIO_INC_STATS(q, n_qfull);
898 return -EBUSY;
899 }
900
901 /*
902 * If we are here, we have enough credits to satisfy the
903 * request. Check if we are near the end of q, and if WR spills over.
904 * If it does, use the first addr/size to cover the queue until
905 * the end. Fit the remainder portion of the request at the top
906 * of queue and return it in the second addr/len. Set pidx
907 * accordingly.
908 */
909 if (unlikely(((uintptr_t)cwr + req_sz) > (uintptr_t)(q->vwrap))) {
910 wrp->addr1 = cwr;
911 wrp->size1 = (uint32_t)((uintptr_t)q->vwrap - (uintptr_t)cwr);
912 wrp->addr2 = q->vstart;
913 wrp->size2 = req_sz - wrp->size1;
914 q->pidx = (uint16_t)(ALIGN(wrp->size2, CSIO_QCREDIT_SZ) /
915 CSIO_QCREDIT_SZ);
916 CSIO_INC_STATS(q, n_qwrap);
917 CSIO_INC_STATS(q, n_eq_wr_split);
918 } else {
919 wrp->addr1 = cwr;
920 wrp->size1 = req_sz;
921 wrp->addr2 = NULL;
922 wrp->size2 = 0;
923 q->pidx += (uint16_t)req_credits;
924
925 /* We are the end of queue, roll back pidx to top of queue */
926 if (unlikely(q->pidx == q->credits)) {
927 q->pidx = 0;
928 CSIO_INC_STATS(q, n_qwrap);
929 }
930 }
931
932 q->inc_idx = (uint16_t)req_credits;
933
934 CSIO_INC_STATS(q, n_tot_reqs);
935
936 return 0;
937}
938
939/*
940 * csio_wr_copy_to_wrp - Copies given data into WR.
941 * @data_buf - Data buffer
942 * @wrp - Work request pair.
943 * @wr_off - Work request offset.
944 * @data_len - Data length.
945 *
946 * Copies the given data in Work Request. Work request pair(wrp) specifies
947 * address information of Work request.
948 * Returns: none
949 */
950void
951csio_wr_copy_to_wrp(void *data_buf, struct csio_wr_pair *wrp,
952 uint32_t wr_off, uint32_t data_len)
953{
954 uint32_t nbytes;
955
956 /* Number of space available in buffer addr1 of WRP */
957 nbytes = ((wrp->size1 - wr_off) >= data_len) ?
958 data_len : (wrp->size1 - wr_off);
959
960 memcpy((uint8_t *) wrp->addr1 + wr_off, data_buf, nbytes);
961 data_len -= nbytes;
962
963 /* Write the remaining data from the begining of circular buffer */
964 if (data_len) {
965 CSIO_DB_ASSERT(data_len <= wrp->size2);
966 CSIO_DB_ASSERT(wrp->addr2 != NULL);
967 memcpy(wrp->addr2, (uint8_t *) data_buf + nbytes, data_len);
968 }
969}
970
971/*
972 * csio_wr_issue - Notify chip of Work request.
973 * @hw: HW module.
974 * @qidx: Index of queue.
975 * @prio: 0: Low priority, 1: High priority
976 *
977 * Rings the SGE Doorbell by writing the current producer index of the passed
978 * in queue into the register.
979 *
980 */
981int
982csio_wr_issue(struct csio_hw *hw, int qidx, bool prio)
983{
984 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
985 struct csio_q *q = wrm->q_arr[qidx];
986
987 CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));
988
989 wmb();
990 /* Ring SGE Doorbell writing q->pidx into it */
991 csio_wr_reg32(hw, DBPRIO(prio) | QID(q->un.eq.physeqid) |
992 PIDX(q->inc_idx), MYPF_REG(SGE_PF_KDOORBELL));
993 q->inc_idx = 0;
994
995 return 0;
996}
997
998static inline uint32_t
999csio_wr_avail_qcredits(struct csio_q *q)
1000{
1001 if (q->pidx > q->cidx)
1002 return q->pidx - q->cidx;
1003 else if (q->cidx > q->pidx)
1004 return q->credits - (q->cidx - q->pidx);
1005 else
1006 return 0; /* cidx == pidx, empty queue */
1007}
1008
1009/*
1010 * csio_wr_inval_flq_buf - Invalidate a free list buffer entry.
1011 * @hw: HW module.
1012 * @flq: The freelist queue.
1013 *
1014 * Invalidate the driver's version of a freelist buffer entry,
1015 * without freeing the associated the DMA memory. The entry
1016 * to be invalidated is picked up from the current Free list
1017 * queue cidx.
1018 *
1019 */
1020static inline void
1021csio_wr_inval_flq_buf(struct csio_hw *hw, struct csio_q *flq)
1022{
1023 flq->cidx++;
1024 if (flq->cidx == flq->credits) {
1025 flq->cidx = 0;
1026 CSIO_INC_STATS(flq, n_qwrap);
1027 }
1028}
1029
1030/*
1031 * csio_wr_process_fl - Process a freelist completion.
1032 * @hw: HW module.
1033 * @q: The ingress queue attached to the Freelist.
1034 * @wr: The freelist completion WR in the ingress queue.
1035 * @len_to_qid: The lower 32-bits of the first flit of the RSP footer
1036 * @iq_handler: Caller's handler for this completion.
1037 * @priv: Private pointer of caller
1038 *
1039 */
1040static inline void
1041csio_wr_process_fl(struct csio_hw *hw, struct csio_q *q,
1042 void *wr, uint32_t len_to_qid,
1043 void (*iq_handler)(struct csio_hw *, void *,
1044 uint32_t, struct csio_fl_dma_buf *,
1045 void *),
1046 void *priv)
1047{
1048 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1049 struct csio_sge *sge = &wrm->sge;
1050 struct csio_fl_dma_buf flb;
1051 struct csio_dma_buf *buf, *fbuf;
1052 uint32_t bufsz, len, lastlen = 0;
1053 int n;
1054 struct csio_q *flq = hw->wrm.q_arr[q->un.iq.flq_idx];
1055
1056 CSIO_DB_ASSERT(flq != NULL);
1057
1058 len = len_to_qid;
1059
1060 if (len & IQWRF_NEWBUF) {
1061 if (flq->un.fl.offset > 0) {
1062 csio_wr_inval_flq_buf(hw, flq);
1063 flq->un.fl.offset = 0;
1064 }
1065 len = IQWRF_LEN_GET(len);
1066 }
1067
1068 CSIO_DB_ASSERT(len != 0);
1069
1070 flb.totlen = len;
1071
1072 /* Consume all freelist buffers used for len bytes */
1073 for (n = 0, fbuf = flb.flbufs; ; n++, fbuf++) {
1074 buf = &flq->un.fl.bufs[flq->cidx];
1075 bufsz = csio_wr_fl_bufsz(sge, buf);
1076
1077 fbuf->paddr = buf->paddr;
1078 fbuf->vaddr = buf->vaddr;
1079
1080 flb.offset = flq->un.fl.offset;
1081 lastlen = min(bufsz, len);
1082 fbuf->len = lastlen;
1083
1084 len -= lastlen;
1085 if (!len)
1086 break;
1087 csio_wr_inval_flq_buf(hw, flq);
1088 }
1089
1090 flb.defer_free = flq->un.fl.packen ? 0 : 1;
1091
1092 iq_handler(hw, wr, q->wr_sz - sizeof(struct csio_iqwr_footer),
1093 &flb, priv);
1094
1095 if (flq->un.fl.packen)
1096 flq->un.fl.offset += ALIGN(lastlen, sge->csio_fl_align);
1097 else
1098 csio_wr_inval_flq_buf(hw, flq);
1099
1100}
1101
1102/*
1103 * csio_is_new_iqwr - Is this a new Ingress queue entry ?
1104 * @q: Ingress quueue.
1105 * @ftr: Ingress queue WR SGE footer.
1106 *
1107 * The entry is new if our generation bit matches the corresponding
1108 * bit in the footer of the current WR.
1109 */
1110static inline bool
1111csio_is_new_iqwr(struct csio_q *q, struct csio_iqwr_footer *ftr)
1112{
1113 return (q->un.iq.genbit == (ftr->u.type_gen >> IQWRF_GEN_SHIFT));
1114}
1115
1116/*
1117 * csio_wr_process_iq - Process elements in Ingress queue.
1118 * @hw: HW pointer
1119 * @qidx: Index of queue
1120 * @iq_handler: Handler for this queue
1121 * @priv: Caller's private pointer
1122 *
1123 * This routine walks through every entry of the ingress queue, calling
1124 * the provided iq_handler with the entry, until the generation bit
1125 * flips.
1126 */
1127int
1128csio_wr_process_iq(struct csio_hw *hw, struct csio_q *q,
1129 void (*iq_handler)(struct csio_hw *, void *,
1130 uint32_t, struct csio_fl_dma_buf *,
1131 void *),
1132 void *priv)
1133{
1134 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1135 void *wr = (void *)((uintptr_t)q->vstart + (q->cidx * q->wr_sz));
1136 struct csio_iqwr_footer *ftr;
1137 uint32_t wr_type, fw_qid, qid;
1138 struct csio_q *q_completed;
1139 struct csio_q *flq = csio_iq_has_fl(q) ?
1140 wrm->q_arr[q->un.iq.flq_idx] : NULL;
1141 int rv = 0;
1142
1143 /* Get the footer */
1144 ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
1145 (q->wr_sz - sizeof(*ftr)));
1146
1147 /*
1148 * When q wrapped around last time, driver should have inverted
1149 * ic.genbit as well.
1150 */
1151 while (csio_is_new_iqwr(q, ftr)) {
1152
1153 CSIO_DB_ASSERT(((uintptr_t)wr + q->wr_sz) <=
1154 (uintptr_t)q->vwrap);
1155 rmb();
1156 wr_type = IQWRF_TYPE_GET(ftr->u.type_gen);
1157
1158 switch (wr_type) {
1159 case X_RSPD_TYPE_CPL:
1160 /* Subtract footer from WR len */
1161 iq_handler(hw, wr, q->wr_sz - sizeof(*ftr), NULL, priv);
1162 break;
1163 case X_RSPD_TYPE_FLBUF:
1164 csio_wr_process_fl(hw, q, wr,
1165 ntohl(ftr->pldbuflen_qid),
1166 iq_handler, priv);
1167 break;
1168 case X_RSPD_TYPE_INTR:
1169 fw_qid = ntohl(ftr->pldbuflen_qid);
1170 qid = fw_qid - wrm->fw_iq_start;
1171 q_completed = hw->wrm.intr_map[qid];
1172
1173 if (unlikely(qid ==
1174 csio_q_physiqid(hw, hw->intr_iq_idx))) {
1175 /*
1176 * We are already in the Forward Interrupt
1177 * Interrupt Queue Service! Do-not service
1178 * again!
1179 *
1180 */
1181 } else {
1182 CSIO_DB_ASSERT(q_completed);
1183 CSIO_DB_ASSERT(
1184 q_completed->un.iq.iq_intx_handler);
1185
1186 /* Call the queue handler. */
1187 q_completed->un.iq.iq_intx_handler(hw, NULL,
1188 0, NULL, (void *)q_completed);
1189 }
1190 break;
1191 default:
1192 csio_warn(hw, "Unknown resp type 0x%x received\n",
1193 wr_type);
1194 CSIO_INC_STATS(q, n_rsp_unknown);
1195 break;
1196 }
1197
1198 /*
1199 * Ingress *always* has fixed size WR entries. Therefore,
1200 * there should always be complete WRs towards the end of
1201 * queue.
1202 */
1203 if (((uintptr_t)wr + q->wr_sz) == (uintptr_t)q->vwrap) {
1204
1205 /* Roll over to start of queue */
1206 q->cidx = 0;
1207 wr = q->vstart;
1208
1209 /* Toggle genbit */
1210 q->un.iq.genbit ^= 0x1;
1211
1212 CSIO_INC_STATS(q, n_qwrap);
1213 } else {
1214 q->cidx++;
1215 wr = (void *)((uintptr_t)(q->vstart) +
1216 (q->cidx * q->wr_sz));
1217 }
1218
1219 ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
1220 (q->wr_sz - sizeof(*ftr)));
1221 q->inc_idx++;
1222
1223 } /* while (q->un.iq.genbit == hdr->genbit) */
1224
1225 /*
1226 * We need to re-arm SGE interrupts in case we got a stray interrupt,
1227 * especially in msix mode. With INTx, this may be a common occurence.
1228 */
1229 if (unlikely(!q->inc_idx)) {
1230 CSIO_INC_STATS(q, n_stray_comp);
1231 rv = -EINVAL;
1232 goto restart;
1233 }
1234
1235 /* Replenish free list buffers if pending falls below low water mark */
1236 if (flq) {
1237 uint32_t avail = csio_wr_avail_qcredits(flq);
1238 if (avail <= 16) {
1239 /* Make sure in FLQ, atleast 1 credit (8 FL buffers)
1240 * remains unpopulated otherwise HW thinks
1241 * FLQ is empty.
1242 */
1243 csio_wr_update_fl(hw, flq, (flq->credits - 8) - avail);
1244 csio_wr_ring_fldb(hw, flq);
1245 }
1246 }
1247
1248restart:
1249 /* Now inform SGE about our incremental index value */
1250 csio_wr_reg32(hw, CIDXINC(q->inc_idx) |
1251 INGRESSQID(q->un.iq.physiqid) |
1252 TIMERREG(csio_sge_timer_reg),
1253 MYPF_REG(SGE_PF_GTS));
1254 q->stats.n_tot_rsps += q->inc_idx;
1255
1256 q->inc_idx = 0;
1257
1258 return rv;
1259}
1260
1261int
1262csio_wr_process_iq_idx(struct csio_hw *hw, int qidx,
1263 void (*iq_handler)(struct csio_hw *, void *,
1264 uint32_t, struct csio_fl_dma_buf *,
1265 void *),
1266 void *priv)
1267{
1268 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1269 struct csio_q *iq = wrm->q_arr[qidx];
1270
1271 return csio_wr_process_iq(hw, iq, iq_handler, priv);
1272}
1273
1274static int
1275csio_closest_timer(struct csio_sge *s, int time)
1276{
1277 int i, delta, match = 0, min_delta = INT_MAX;
1278
1279 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1280 delta = time - s->timer_val[i];
1281 if (delta < 0)
1282 delta = -delta;
1283 if (delta < min_delta) {
1284 min_delta = delta;
1285 match = i;
1286 }
1287 }
1288 return match;
1289}
1290
1291static int
1292csio_closest_thresh(struct csio_sge *s, int cnt)
1293{
1294 int i, delta, match = 0, min_delta = INT_MAX;
1295
1296 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1297 delta = cnt - s->counter_val[i];
1298 if (delta < 0)
1299 delta = -delta;
1300 if (delta < min_delta) {
1301 min_delta = delta;
1302 match = i;
1303 }
1304 }
1305 return match;
1306}
1307
1308static void
1309csio_wr_fixup_host_params(struct csio_hw *hw)
1310{
1311 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1312 struct csio_sge *sge = &wrm->sge;
1313 uint32_t clsz = L1_CACHE_BYTES;
1314 uint32_t s_hps = PAGE_SHIFT - 10;
1315 uint32_t ingpad = 0;
1316 uint32_t stat_len = clsz > 64 ? 128 : 64;
1317
1318 csio_wr_reg32(hw, HOSTPAGESIZEPF0(s_hps) | HOSTPAGESIZEPF1(s_hps) |
1319 HOSTPAGESIZEPF2(s_hps) | HOSTPAGESIZEPF3(s_hps) |
1320 HOSTPAGESIZEPF4(s_hps) | HOSTPAGESIZEPF5(s_hps) |
1321 HOSTPAGESIZEPF6(s_hps) | HOSTPAGESIZEPF7(s_hps),
1322 SGE_HOST_PAGE_SIZE);
1323
1324 sge->csio_fl_align = clsz < 32 ? 32 : clsz;
1325 ingpad = ilog2(sge->csio_fl_align) - 5;
1326
1327 csio_set_reg_field(hw, SGE_CONTROL, INGPADBOUNDARY_MASK |
1328 EGRSTATUSPAGESIZE(1),
1329 INGPADBOUNDARY(ingpad) |
1330 EGRSTATUSPAGESIZE(stat_len != 64));
1331
1332 /* FL BUFFER SIZE#0 is Page size i,e already aligned to cache line */
1333 csio_wr_reg32(hw, PAGE_SIZE, SGE_FL_BUFFER_SIZE0);
1334 csio_wr_reg32(hw,
1335 (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE2) +
1336 sge->csio_fl_align - 1) & ~(sge->csio_fl_align - 1),
1337 SGE_FL_BUFFER_SIZE2);
1338 csio_wr_reg32(hw,
1339 (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE3) +
1340 sge->csio_fl_align - 1) & ~(sge->csio_fl_align - 1),
1341 SGE_FL_BUFFER_SIZE3);
1342
1343 csio_wr_reg32(hw, HPZ0(PAGE_SHIFT - 12), ULP_RX_TDDP_PSZ);
1344
1345 /* default value of rx_dma_offset of the NIC driver */
1346 csio_set_reg_field(hw, SGE_CONTROL, PKTSHIFT_MASK,
1347 PKTSHIFT(CSIO_SGE_RX_DMA_OFFSET));
1348}
1349
1350static void
1351csio_init_intr_coalesce_parms(struct csio_hw *hw)
1352{
1353 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1354 struct csio_sge *sge = &wrm->sge;
1355
1356 csio_sge_thresh_reg = csio_closest_thresh(sge, csio_intr_coalesce_cnt);
1357 if (csio_intr_coalesce_cnt) {
1358 csio_sge_thresh_reg = 0;
1359 csio_sge_timer_reg = X_TIMERREG_RESTART_COUNTER;
1360 return;
1361 }
1362
1363 csio_sge_timer_reg = csio_closest_timer(sge, csio_intr_coalesce_time);
1364}
1365
1366/*
1367 * csio_wr_get_sge - Get SGE register values.
1368 * @hw: HW module.
1369 *
1370 * Used by non-master functions and by master-functions relying on config file.
1371 */
1372static void
1373csio_wr_get_sge(struct csio_hw *hw)
1374{
1375 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1376 struct csio_sge *sge = &wrm->sge;
1377 uint32_t ingpad;
1378 int i;
1379 u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5;
1380 u32 ingress_rx_threshold;
1381
1382 sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL);
1383
1384 ingpad = INGPADBOUNDARY_GET(sge->sge_control);
1385
1386 switch (ingpad) {
1387 case X_INGPCIEBOUNDARY_32B:
1388 sge->csio_fl_align = 32; break;
1389 case X_INGPCIEBOUNDARY_64B:
1390 sge->csio_fl_align = 64; break;
1391 case X_INGPCIEBOUNDARY_128B:
1392 sge->csio_fl_align = 128; break;
1393 case X_INGPCIEBOUNDARY_256B:
1394 sge->csio_fl_align = 256; break;
1395 case X_INGPCIEBOUNDARY_512B:
1396 sge->csio_fl_align = 512; break;
1397 case X_INGPCIEBOUNDARY_1024B:
1398 sge->csio_fl_align = 1024; break;
1399 case X_INGPCIEBOUNDARY_2048B:
1400 sge->csio_fl_align = 2048; break;
1401 case X_INGPCIEBOUNDARY_4096B:
1402 sge->csio_fl_align = 4096; break;
1403 }
1404
1405 for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)
1406 csio_get_flbuf_size(hw, sge, i);
1407
1408 timer_value_0_and_1 = csio_rd_reg32(hw, SGE_TIMER_VALUE_0_AND_1);
1409 timer_value_2_and_3 = csio_rd_reg32(hw, SGE_TIMER_VALUE_2_AND_3);
1410 timer_value_4_and_5 = csio_rd_reg32(hw, SGE_TIMER_VALUE_4_AND_5);
1411
1412 sge->timer_val[0] = (uint16_t)csio_core_ticks_to_us(hw,
1413 TIMERVALUE0_GET(timer_value_0_and_1));
1414 sge->timer_val[1] = (uint16_t)csio_core_ticks_to_us(hw,
1415 TIMERVALUE1_GET(timer_value_0_and_1));
1416 sge->timer_val[2] = (uint16_t)csio_core_ticks_to_us(hw,
1417 TIMERVALUE2_GET(timer_value_2_and_3));
1418 sge->timer_val[3] = (uint16_t)csio_core_ticks_to_us(hw,
1419 TIMERVALUE3_GET(timer_value_2_and_3));
1420 sge->timer_val[4] = (uint16_t)csio_core_ticks_to_us(hw,
1421 TIMERVALUE4_GET(timer_value_4_and_5));
1422 sge->timer_val[5] = (uint16_t)csio_core_ticks_to_us(hw,
1423 TIMERVALUE5_GET(timer_value_4_and_5));
1424
1425 ingress_rx_threshold = csio_rd_reg32(hw, SGE_INGRESS_RX_THRESHOLD);
1426 sge->counter_val[0] = THRESHOLD_0_GET(ingress_rx_threshold);
1427 sge->counter_val[1] = THRESHOLD_1_GET(ingress_rx_threshold);
1428 sge->counter_val[2] = THRESHOLD_2_GET(ingress_rx_threshold);
1429 sge->counter_val[3] = THRESHOLD_3_GET(ingress_rx_threshold);
1430
1431 csio_init_intr_coalesce_parms(hw);
1432}
1433
1434/*
1435 * csio_wr_set_sge - Initialize SGE registers
1436 * @hw: HW module.
1437 *
1438 * Used by Master function to initialize SGE registers in the absence
1439 * of a config file.
1440 */
1441static void
1442csio_wr_set_sge(struct csio_hw *hw)
1443{
1444 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1445 struct csio_sge *sge = &wrm->sge;
1446 int i;
1447
1448 /*
1449 * Set up our basic SGE mode to deliver CPL messages to our Ingress
1450 * Queue and Packet Date to the Free List.
1451 */
1452 csio_set_reg_field(hw, SGE_CONTROL, RXPKTCPLMODE(1), RXPKTCPLMODE(1));
1453
1454 sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL);
1455
1456 /* sge->csio_fl_align is set up by csio_wr_fixup_host_params(). */
1457
1458 /*
1459 * Set up to drop DOORBELL writes when the DOORBELL FIFO overflows
1460 * and generate an interrupt when this occurs so we can recover.
1461 */
1462 csio_set_reg_field(hw, SGE_DBFIFO_STATUS,
1463 HP_INT_THRESH(HP_INT_THRESH_MASK) |
1464 LP_INT_THRESH(LP_INT_THRESH_MASK),
1465 HP_INT_THRESH(CSIO_SGE_DBFIFO_INT_THRESH) |
1466 LP_INT_THRESH(CSIO_SGE_DBFIFO_INT_THRESH));
1467 csio_set_reg_field(hw, SGE_DOORBELL_CONTROL, ENABLE_DROP,
1468 ENABLE_DROP);
1469
1470 /* SGE_FL_BUFFER_SIZE0 is set up by csio_wr_fixup_host_params(). */
1471
1472 CSIO_SET_FLBUF_SIZE(hw, 1, CSIO_SGE_FLBUF_SIZE1);
1473 CSIO_SET_FLBUF_SIZE(hw, 2, CSIO_SGE_FLBUF_SIZE2);
1474 CSIO_SET_FLBUF_SIZE(hw, 3, CSIO_SGE_FLBUF_SIZE3);
1475 CSIO_SET_FLBUF_SIZE(hw, 4, CSIO_SGE_FLBUF_SIZE4);
1476 CSIO_SET_FLBUF_SIZE(hw, 5, CSIO_SGE_FLBUF_SIZE5);
1477 CSIO_SET_FLBUF_SIZE(hw, 6, CSIO_SGE_FLBUF_SIZE6);
1478 CSIO_SET_FLBUF_SIZE(hw, 7, CSIO_SGE_FLBUF_SIZE7);
1479 CSIO_SET_FLBUF_SIZE(hw, 8, CSIO_SGE_FLBUF_SIZE8);
1480
1481 for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)
1482 csio_get_flbuf_size(hw, sge, i);
1483
1484 /* Initialize interrupt coalescing attributes */
1485 sge->timer_val[0] = CSIO_SGE_TIMER_VAL_0;
1486 sge->timer_val[1] = CSIO_SGE_TIMER_VAL_1;
1487 sge->timer_val[2] = CSIO_SGE_TIMER_VAL_2;
1488 sge->timer_val[3] = CSIO_SGE_TIMER_VAL_3;
1489 sge->timer_val[4] = CSIO_SGE_TIMER_VAL_4;
1490 sge->timer_val[5] = CSIO_SGE_TIMER_VAL_5;
1491
1492 sge->counter_val[0] = CSIO_SGE_INT_CNT_VAL_0;
1493 sge->counter_val[1] = CSIO_SGE_INT_CNT_VAL_1;
1494 sge->counter_val[2] = CSIO_SGE_INT_CNT_VAL_2;
1495 sge->counter_val[3] = CSIO_SGE_INT_CNT_VAL_3;
1496
1497 csio_wr_reg32(hw, THRESHOLD_0(sge->counter_val[0]) |
1498 THRESHOLD_1(sge->counter_val[1]) |
1499 THRESHOLD_2(sge->counter_val[2]) |
1500 THRESHOLD_3(sge->counter_val[3]),
1501 SGE_INGRESS_RX_THRESHOLD);
1502
1503 csio_wr_reg32(hw,
1504 TIMERVALUE0(csio_us_to_core_ticks(hw, sge->timer_val[0])) |
1505 TIMERVALUE1(csio_us_to_core_ticks(hw, sge->timer_val[1])),
1506 SGE_TIMER_VALUE_0_AND_1);
1507
1508 csio_wr_reg32(hw,
1509 TIMERVALUE2(csio_us_to_core_ticks(hw, sge->timer_val[2])) |
1510 TIMERVALUE3(csio_us_to_core_ticks(hw, sge->timer_val[3])),
1511 SGE_TIMER_VALUE_2_AND_3);
1512
1513 csio_wr_reg32(hw,
1514 TIMERVALUE4(csio_us_to_core_ticks(hw, sge->timer_val[4])) |
1515 TIMERVALUE5(csio_us_to_core_ticks(hw, sge->timer_val[5])),
1516 SGE_TIMER_VALUE_4_AND_5);
1517
1518 csio_init_intr_coalesce_parms(hw);
1519}
1520
1521void
1522csio_wr_sge_init(struct csio_hw *hw)
1523{
1524 /*
1525 * If we are master:
1526 * - If we plan to use the config file, we need to fixup some
1527 * host specific registers, and read the rest of the SGE
1528 * configuration.
1529 * - If we dont plan to use the config file, we need to initialize
1530 * SGE entirely, including fixing the host specific registers.
1531 * If we arent the master, we are only allowed to read and work off of
1532 * the already initialized SGE values.
1533 *
1534 * Therefore, before calling this function, we assume that the master-
1535 * ship of the card, and whether to use config file or not, have
1536 * already been decided. In other words, CSIO_HWF_USING_SOFT_PARAMS and
1537 * CSIO_HWF_MASTER should be set/unset.
1538 */
1539 if (csio_is_hw_master(hw)) {
1540 csio_wr_fixup_host_params(hw);
1541
1542 if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS)
1543 csio_wr_get_sge(hw);
1544 else
1545 csio_wr_set_sge(hw);
1546 } else
1547 csio_wr_get_sge(hw);
1548}
1549
1550/*
1551 * csio_wrm_init - Initialize Work request module.
1552 * @wrm: WR module
1553 * @hw: HW pointer
1554 *
1555 * Allocates memory for an array of queue pointers starting at q_arr.
1556 */
1557int
1558csio_wrm_init(struct csio_wrm *wrm, struct csio_hw *hw)
1559{
1560 int i;
1561
1562 if (!wrm->num_q) {
1563 csio_err(hw, "Num queues is not set\n");
1564 return -EINVAL;
1565 }
1566
1567 wrm->q_arr = kzalloc(sizeof(struct csio_q *) * wrm->num_q, GFP_KERNEL);
1568 if (!wrm->q_arr)
1569 goto err;
1570
1571 for (i = 0; i < wrm->num_q; i++) {
1572 wrm->q_arr[i] = kzalloc(sizeof(struct csio_q), GFP_KERNEL);
1573 if (!wrm->q_arr[i]) {
1574 while (--i >= 0)
1575 kfree(wrm->q_arr[i]);
1576 goto err_free_arr;
1577 }
1578 }
1579 wrm->free_qidx = 0;
1580
1581 return 0;
1582
1583err_free_arr:
1584 kfree(wrm->q_arr);
1585err:
1586 return -ENOMEM;
1587}
1588
1589/*
1590 * csio_wrm_exit - Initialize Work request module.
1591 * @wrm: WR module
1592 * @hw: HW module
1593 *
1594 * Uninitialize WR module. Free q_arr and pointers in it.
1595 * We have the additional job of freeing the DMA memory associated
1596 * with the queues.
1597 */
1598void
1599csio_wrm_exit(struct csio_wrm *wrm, struct csio_hw *hw)
1600{
1601 int i;
1602 uint32_t j;
1603 struct csio_q *q;
1604 struct csio_dma_buf *buf;
1605
1606 for (i = 0; i < wrm->num_q; i++) {
1607 q = wrm->q_arr[i];
1608
1609 if (wrm->free_qidx && (i < wrm->free_qidx)) {
1610 if (q->type == CSIO_FREELIST) {
1611 if (!q->un.fl.bufs)
1612 continue;
1613 for (j = 0; j < q->credits; j++) {
1614 buf = &q->un.fl.bufs[j];
1615 if (!buf->vaddr)
1616 continue;
1617 pci_free_consistent(hw->pdev, buf->len,
1618 buf->vaddr,
1619 buf->paddr);
1620 }
1621 kfree(q->un.fl.bufs);
1622 }
1623 pci_free_consistent(hw->pdev, q->size,
1624 q->vstart, q->pstart);
1625 }
1626 kfree(q);
1627 }
1628
1629 hw->flags &= ~CSIO_HWF_Q_MEM_ALLOCED;
1630
1631 kfree(wrm->q_arr);
1632}
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-11 13:16:25 -0400