diff options
author | Ben Hutchings <bhutchings@solarflare.com> | 2009-11-29 10:14:45 -0500 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2009-11-29 20:23:55 -0500 |
commit | 8e730c15e1560415f33d7301b617be26050ffb86 (patch) | |
tree | 452b02c5cb91f48f392c635a0191e7dfecc9590d /drivers | |
parent | 744093c98363f8a65853aed39708c9effc80f8ff (diff) |
sfc: Move shared NIC code from falcon.c to new source file nic.c
Signed-off-by: Ben Hutchings <bhutchings@solarflare.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'drivers')
-rw-r--r-- | drivers/net/sfc/Makefile | 2 | ||||
-rw-r--r-- | drivers/net/sfc/falcon.c | 1536 | ||||
-rw-r--r-- | drivers/net/sfc/nic.c | 1548 |
3 files changed, 1549 insertions, 1537 deletions
diff --git a/drivers/net/sfc/Makefile b/drivers/net/sfc/Makefile index 7b52fe10d38f..223106b93447 100644 --- a/drivers/net/sfc/Makefile +++ b/drivers/net/sfc/Makefile | |||
@@ -1,4 +1,4 @@ | |||
1 | sfc-y += efx.o falcon.o tx.o rx.o falcon_gmac.o \ | 1 | sfc-y += efx.o nic.o falcon.o tx.o rx.o falcon_gmac.o \ |
2 | falcon_xmac.o selftest.o ethtool.o qt202x_phy.o \ | 2 | falcon_xmac.o selftest.o ethtool.o qt202x_phy.o \ |
3 | mdio_10g.o tenxpress.o falcon_boards.o | 3 | mdio_10g.o tenxpress.o falcon_boards.o |
4 | sfc-$(CONFIG_SFC_MTD) += mtd.o | 4 | sfc-$(CONFIG_SFC_MTD) += mtd.o |
diff --git a/drivers/net/sfc/falcon.c b/drivers/net/sfc/falcon.c index 64b47da12326..48d28d828d46 100644 --- a/drivers/net/sfc/falcon.c +++ b/drivers/net/sfc/falcon.c | |||
@@ -29,26 +29,6 @@ | |||
29 | 29 | ||
30 | /* Hardware control for SFC4000 (aka Falcon). */ | 30 | /* Hardware control for SFC4000 (aka Falcon). */ |
31 | 31 | ||
32 | /************************************************************************** | ||
33 | * | ||
34 | * Configurable values | ||
35 | * | ||
36 | ************************************************************************** | ||
37 | */ | ||
38 | |||
39 | /* This is set to 16 for a good reason. In summary, if larger than | ||
40 | * 16, the descriptor cache holds more than a default socket | ||
41 | * buffer's worth of packets (for UDP we can only have at most one | ||
42 | * socket buffer's worth outstanding). This combined with the fact | ||
43 | * that we only get 1 TX event per descriptor cache means the NIC | ||
44 | * goes idle. | ||
45 | */ | ||
46 | #define TX_DC_ENTRIES 16 | ||
47 | #define TX_DC_ENTRIES_ORDER 1 | ||
48 | |||
49 | #define RX_DC_ENTRIES 64 | ||
50 | #define RX_DC_ENTRIES_ORDER 3 | ||
51 | |||
52 | static const unsigned int | 32 | static const unsigned int |
53 | /* "Large" EEPROM device: Atmel AT25640 or similar | 33 | /* "Large" EEPROM device: Atmel AT25640 or similar |
54 | * 8 KB, 16-bit address, 32 B write block */ | 34 | * 8 KB, 16-bit address, 32 B write block */ |
@@ -63,87 +43,6 @@ default_flash_type = ((17 << SPI_DEV_TYPE_SIZE_LBN) | |||
63 | | (15 << SPI_DEV_TYPE_ERASE_SIZE_LBN) | 43 | | (15 << SPI_DEV_TYPE_ERASE_SIZE_LBN) |
64 | | (8 << SPI_DEV_TYPE_BLOCK_SIZE_LBN)); | 44 | | (8 << SPI_DEV_TYPE_BLOCK_SIZE_LBN)); |
65 | 45 | ||
66 | /* RX FIFO XOFF watermark | ||
67 | * | ||
68 | * When the amount of the RX FIFO increases used increases past this | ||
69 | * watermark send XOFF. Only used if RX flow control is enabled (ethtool -A) | ||
70 | * This also has an effect on RX/TX arbitration | ||
71 | */ | ||
72 | int efx_nic_rx_xoff_thresh = -1; | ||
73 | module_param_named(rx_xoff_thresh_bytes, efx_nic_rx_xoff_thresh, int, 0644); | ||
74 | MODULE_PARM_DESC(rx_xoff_thresh_bytes, "RX fifo XOFF threshold"); | ||
75 | |||
76 | /* RX FIFO XON watermark | ||
77 | * | ||
78 | * When the amount of the RX FIFO used decreases below this | ||
79 | * watermark send XON. Only used if TX flow control is enabled (ethtool -A) | ||
80 | * This also has an effect on RX/TX arbitration | ||
81 | */ | ||
82 | int efx_nic_rx_xon_thresh = -1; | ||
83 | module_param_named(rx_xon_thresh_bytes, efx_nic_rx_xon_thresh, int, 0644); | ||
84 | MODULE_PARM_DESC(rx_xon_thresh_bytes, "RX fifo XON threshold"); | ||
85 | |||
86 | /* If EFX_MAX_INT_ERRORS internal errors occur within | ||
87 | * EFX_INT_ERROR_EXPIRE seconds, we consider the NIC broken and | ||
88 | * disable it. | ||
89 | */ | ||
90 | #define EFX_INT_ERROR_EXPIRE 3600 | ||
91 | #define EFX_MAX_INT_ERRORS 5 | ||
92 | |||
93 | /* We poll for events every FLUSH_INTERVAL ms, and check FLUSH_POLL_COUNT times | ||
94 | */ | ||
95 | #define EFX_FLUSH_INTERVAL 10 | ||
96 | #define EFX_FLUSH_POLL_COUNT 100 | ||
97 | |||
98 | /************************************************************************** | ||
99 | * | ||
100 | * Falcon constants | ||
101 | * | ||
102 | ************************************************************************** | ||
103 | */ | ||
104 | |||
105 | /* Size and alignment of special buffers (4KB) */ | ||
106 | #define EFX_BUF_SIZE 4096 | ||
107 | |||
108 | /* Depth of RX flush request fifo */ | ||
109 | #define EFX_RX_FLUSH_COUNT 4 | ||
110 | |||
111 | /************************************************************************** | ||
112 | * | ||
113 | * Solarstorm hardware access | ||
114 | * | ||
115 | **************************************************************************/ | ||
116 | |||
117 | static inline void efx_write_buf_tbl(struct efx_nic *efx, efx_qword_t *value, | ||
118 | unsigned int index) | ||
119 | { | ||
120 | efx_sram_writeq(efx, efx->membase + efx->type->buf_tbl_base, | ||
121 | value, index); | ||
122 | } | ||
123 | |||
124 | /* Read the current event from the event queue */ | ||
125 | static inline efx_qword_t *efx_event(struct efx_channel *channel, | ||
126 | unsigned int index) | ||
127 | { | ||
128 | return (((efx_qword_t *) (channel->eventq.addr)) + index); | ||
129 | } | ||
130 | |||
131 | /* See if an event is present | ||
132 | * | ||
133 | * We check both the high and low dword of the event for all ones. We | ||
134 | * wrote all ones when we cleared the event, and no valid event can | ||
135 | * have all ones in either its high or low dwords. This approach is | ||
136 | * robust against reordering. | ||
137 | * | ||
138 | * Note that using a single 64-bit comparison is incorrect; even | ||
139 | * though the CPU read will be atomic, the DMA write may not be. | ||
140 | */ | ||
141 | static inline int efx_event_present(efx_qword_t *event) | ||
142 | { | ||
143 | return (!(EFX_DWORD_IS_ALL_ONES(event->dword[0]) | | ||
144 | EFX_DWORD_IS_ALL_ONES(event->dword[1]))); | ||
145 | } | ||
146 | |||
147 | /************************************************************************** | 46 | /************************************************************************** |
148 | * | 47 | * |
149 | * I2C bus - this is a bit-bashing interface using GPIO pins | 48 | * I2C bus - this is a bit-bashing interface using GPIO pins |
@@ -200,841 +99,6 @@ static struct i2c_algo_bit_data falcon_i2c_bit_operations = { | |||
200 | .timeout = DIV_ROUND_UP(HZ, 20), | 99 | .timeout = DIV_ROUND_UP(HZ, 20), |
201 | }; | 100 | }; |
202 | 101 | ||
203 | /************************************************************************** | ||
204 | * | ||
205 | * Special buffer handling | ||
206 | * Special buffers are used for event queues and the TX and RX | ||
207 | * descriptor rings. | ||
208 | * | ||
209 | *************************************************************************/ | ||
210 | |||
211 | /* | ||
212 | * Initialise a special buffer | ||
213 | * | ||
214 | * This will define a buffer (previously allocated via | ||
215 | * efx_alloc_special_buffer()) in the buffer table, allowing | ||
216 | * it to be used for event queues, descriptor rings etc. | ||
217 | */ | ||
218 | static void | ||
219 | efx_init_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer) | ||
220 | { | ||
221 | efx_qword_t buf_desc; | ||
222 | int index; | ||
223 | dma_addr_t dma_addr; | ||
224 | int i; | ||
225 | |||
226 | EFX_BUG_ON_PARANOID(!buffer->addr); | ||
227 | |||
228 | /* Write buffer descriptors to NIC */ | ||
229 | for (i = 0; i < buffer->entries; i++) { | ||
230 | index = buffer->index + i; | ||
231 | dma_addr = buffer->dma_addr + (i * 4096); | ||
232 | EFX_LOG(efx, "mapping special buffer %d at %llx\n", | ||
233 | index, (unsigned long long)dma_addr); | ||
234 | EFX_POPULATE_QWORD_3(buf_desc, | ||
235 | FRF_AZ_BUF_ADR_REGION, 0, | ||
236 | FRF_AZ_BUF_ADR_FBUF, dma_addr >> 12, | ||
237 | FRF_AZ_BUF_OWNER_ID_FBUF, 0); | ||
238 | efx_write_buf_tbl(efx, &buf_desc, index); | ||
239 | } | ||
240 | } | ||
241 | |||
242 | /* Unmaps a buffer and clears the buffer table entries */ | ||
243 | static void | ||
244 | efx_fini_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer) | ||
245 | { | ||
246 | efx_oword_t buf_tbl_upd; | ||
247 | unsigned int start = buffer->index; | ||
248 | unsigned int end = (buffer->index + buffer->entries - 1); | ||
249 | |||
250 | if (!buffer->entries) | ||
251 | return; | ||
252 | |||
253 | EFX_LOG(efx, "unmapping special buffers %d-%d\n", | ||
254 | buffer->index, buffer->index + buffer->entries - 1); | ||
255 | |||
256 | EFX_POPULATE_OWORD_4(buf_tbl_upd, | ||
257 | FRF_AZ_BUF_UPD_CMD, 0, | ||
258 | FRF_AZ_BUF_CLR_CMD, 1, | ||
259 | FRF_AZ_BUF_CLR_END_ID, end, | ||
260 | FRF_AZ_BUF_CLR_START_ID, start); | ||
261 | efx_writeo(efx, &buf_tbl_upd, FR_AZ_BUF_TBL_UPD); | ||
262 | } | ||
263 | |||
264 | /* | ||
265 | * Allocate a new special buffer | ||
266 | * | ||
267 | * This allocates memory for a new buffer, clears it and allocates a | ||
268 | * new buffer ID range. It does not write into the buffer table. | ||
269 | * | ||
270 | * This call will allocate 4KB buffers, since 8KB buffers can't be | ||
271 | * used for event queues and descriptor rings. | ||
272 | */ | ||
273 | static int efx_alloc_special_buffer(struct efx_nic *efx, | ||
274 | struct efx_special_buffer *buffer, | ||
275 | unsigned int len) | ||
276 | { | ||
277 | len = ALIGN(len, EFX_BUF_SIZE); | ||
278 | |||
279 | buffer->addr = pci_alloc_consistent(efx->pci_dev, len, | ||
280 | &buffer->dma_addr); | ||
281 | if (!buffer->addr) | ||
282 | return -ENOMEM; | ||
283 | buffer->len = len; | ||
284 | buffer->entries = len / EFX_BUF_SIZE; | ||
285 | BUG_ON(buffer->dma_addr & (EFX_BUF_SIZE - 1)); | ||
286 | |||
287 | /* All zeros is a potentially valid event so memset to 0xff */ | ||
288 | memset(buffer->addr, 0xff, len); | ||
289 | |||
290 | /* Select new buffer ID */ | ||
291 | buffer->index = efx->next_buffer_table; | ||
292 | efx->next_buffer_table += buffer->entries; | ||
293 | |||
294 | EFX_LOG(efx, "allocating special buffers %d-%d at %llx+%x " | ||
295 | "(virt %p phys %llx)\n", buffer->index, | ||
296 | buffer->index + buffer->entries - 1, | ||
297 | (u64)buffer->dma_addr, len, | ||
298 | buffer->addr, (u64)virt_to_phys(buffer->addr)); | ||
299 | |||
300 | return 0; | ||
301 | } | ||
302 | |||
303 | static void | ||
304 | efx_free_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer) | ||
305 | { | ||
306 | if (!buffer->addr) | ||
307 | return; | ||
308 | |||
309 | EFX_LOG(efx, "deallocating special buffers %d-%d at %llx+%x " | ||
310 | "(virt %p phys %llx)\n", buffer->index, | ||
311 | buffer->index + buffer->entries - 1, | ||
312 | (u64)buffer->dma_addr, buffer->len, | ||
313 | buffer->addr, (u64)virt_to_phys(buffer->addr)); | ||
314 | |||
315 | pci_free_consistent(efx->pci_dev, buffer->len, buffer->addr, | ||
316 | buffer->dma_addr); | ||
317 | buffer->addr = NULL; | ||
318 | buffer->entries = 0; | ||
319 | } | ||
320 | |||
321 | /************************************************************************** | ||
322 | * | ||
323 | * Generic buffer handling | ||
324 | * These buffers are used for interrupt status and MAC stats | ||
325 | * | ||
326 | **************************************************************************/ | ||
327 | |||
328 | int efx_nic_alloc_buffer(struct efx_nic *efx, struct efx_buffer *buffer, | ||
329 | unsigned int len) | ||
330 | { | ||
331 | buffer->addr = pci_alloc_consistent(efx->pci_dev, len, | ||
332 | &buffer->dma_addr); | ||
333 | if (!buffer->addr) | ||
334 | return -ENOMEM; | ||
335 | buffer->len = len; | ||
336 | memset(buffer->addr, 0, len); | ||
337 | return 0; | ||
338 | } | ||
339 | |||
340 | void efx_nic_free_buffer(struct efx_nic *efx, struct efx_buffer *buffer) | ||
341 | { | ||
342 | if (buffer->addr) { | ||
343 | pci_free_consistent(efx->pci_dev, buffer->len, | ||
344 | buffer->addr, buffer->dma_addr); | ||
345 | buffer->addr = NULL; | ||
346 | } | ||
347 | } | ||
348 | |||
349 | /************************************************************************** | ||
350 | * | ||
351 | * TX path | ||
352 | * | ||
353 | **************************************************************************/ | ||
354 | |||
355 | /* Returns a pointer to the specified transmit descriptor in the TX | ||
356 | * descriptor queue belonging to the specified channel. | ||
357 | */ | ||
358 | static inline efx_qword_t * | ||
359 | efx_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index) | ||
360 | { | ||
361 | return (((efx_qword_t *) (tx_queue->txd.addr)) + index); | ||
362 | } | ||
363 | |||
364 | /* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */ | ||
365 | static inline void efx_notify_tx_desc(struct efx_tx_queue *tx_queue) | ||
366 | { | ||
367 | unsigned write_ptr; | ||
368 | efx_dword_t reg; | ||
369 | |||
370 | write_ptr = tx_queue->write_count & EFX_TXQ_MASK; | ||
371 | EFX_POPULATE_DWORD_1(reg, FRF_AZ_TX_DESC_WPTR_DWORD, write_ptr); | ||
372 | efx_writed_page(tx_queue->efx, ®, | ||
373 | FR_AZ_TX_DESC_UPD_DWORD_P0, tx_queue->queue); | ||
374 | } | ||
375 | |||
376 | |||
377 | /* For each entry inserted into the software descriptor ring, create a | ||
378 | * descriptor in the hardware TX descriptor ring (in host memory), and | ||
379 | * write a doorbell. | ||
380 | */ | ||
381 | void efx_nic_push_buffers(struct efx_tx_queue *tx_queue) | ||
382 | { | ||
383 | |||
384 | struct efx_tx_buffer *buffer; | ||
385 | efx_qword_t *txd; | ||
386 | unsigned write_ptr; | ||
387 | |||
388 | BUG_ON(tx_queue->write_count == tx_queue->insert_count); | ||
389 | |||
390 | do { | ||
391 | write_ptr = tx_queue->write_count & EFX_TXQ_MASK; | ||
392 | buffer = &tx_queue->buffer[write_ptr]; | ||
393 | txd = efx_tx_desc(tx_queue, write_ptr); | ||
394 | ++tx_queue->write_count; | ||
395 | |||
396 | /* Create TX descriptor ring entry */ | ||
397 | EFX_POPULATE_QWORD_4(*txd, | ||
398 | FSF_AZ_TX_KER_CONT, buffer->continuation, | ||
399 | FSF_AZ_TX_KER_BYTE_COUNT, buffer->len, | ||
400 | FSF_AZ_TX_KER_BUF_REGION, 0, | ||
401 | FSF_AZ_TX_KER_BUF_ADDR, buffer->dma_addr); | ||
402 | } while (tx_queue->write_count != tx_queue->insert_count); | ||
403 | |||
404 | wmb(); /* Ensure descriptors are written before they are fetched */ | ||
405 | efx_notify_tx_desc(tx_queue); | ||
406 | } | ||
407 | |||
408 | /* Allocate hardware resources for a TX queue */ | ||
409 | int efx_nic_probe_tx(struct efx_tx_queue *tx_queue) | ||
410 | { | ||
411 | struct efx_nic *efx = tx_queue->efx; | ||
412 | BUILD_BUG_ON(EFX_TXQ_SIZE < 512 || EFX_TXQ_SIZE > 4096 || | ||
413 | EFX_TXQ_SIZE & EFX_TXQ_MASK); | ||
414 | return efx_alloc_special_buffer(efx, &tx_queue->txd, | ||
415 | EFX_TXQ_SIZE * sizeof(efx_qword_t)); | ||
416 | } | ||
417 | |||
418 | void efx_nic_init_tx(struct efx_tx_queue *tx_queue) | ||
419 | { | ||
420 | efx_oword_t tx_desc_ptr; | ||
421 | struct efx_nic *efx = tx_queue->efx; | ||
422 | |||
423 | tx_queue->flushed = FLUSH_NONE; | ||
424 | |||
425 | /* Pin TX descriptor ring */ | ||
426 | efx_init_special_buffer(efx, &tx_queue->txd); | ||
427 | |||
428 | /* Push TX descriptor ring to card */ | ||
429 | EFX_POPULATE_OWORD_10(tx_desc_ptr, | ||
430 | FRF_AZ_TX_DESCQ_EN, 1, | ||
431 | FRF_AZ_TX_ISCSI_DDIG_EN, 0, | ||
432 | FRF_AZ_TX_ISCSI_HDIG_EN, 0, | ||
433 | FRF_AZ_TX_DESCQ_BUF_BASE_ID, tx_queue->txd.index, | ||
434 | FRF_AZ_TX_DESCQ_EVQ_ID, | ||
435 | tx_queue->channel->channel, | ||
436 | FRF_AZ_TX_DESCQ_OWNER_ID, 0, | ||
437 | FRF_AZ_TX_DESCQ_LABEL, tx_queue->queue, | ||
438 | FRF_AZ_TX_DESCQ_SIZE, | ||
439 | __ffs(tx_queue->txd.entries), | ||
440 | FRF_AZ_TX_DESCQ_TYPE, 0, | ||
441 | FRF_BZ_TX_NON_IP_DROP_DIS, 1); | ||
442 | |||
443 | if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { | ||
444 | int csum = tx_queue->queue == EFX_TX_QUEUE_OFFLOAD_CSUM; | ||
445 | EFX_SET_OWORD_FIELD(tx_desc_ptr, FRF_BZ_TX_IP_CHKSM_DIS, !csum); | ||
446 | EFX_SET_OWORD_FIELD(tx_desc_ptr, FRF_BZ_TX_TCP_CHKSM_DIS, | ||
447 | !csum); | ||
448 | } | ||
449 | |||
450 | efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base, | ||
451 | tx_queue->queue); | ||
452 | |||
453 | if (efx_nic_rev(efx) < EFX_REV_FALCON_B0) { | ||
454 | efx_oword_t reg; | ||
455 | |||
456 | /* Only 128 bits in this register */ | ||
457 | BUILD_BUG_ON(EFX_TX_QUEUE_COUNT >= 128); | ||
458 | |||
459 | efx_reado(efx, ®, FR_AA_TX_CHKSM_CFG); | ||
460 | if (tx_queue->queue == EFX_TX_QUEUE_OFFLOAD_CSUM) | ||
461 | clear_bit_le(tx_queue->queue, (void *)®); | ||
462 | else | ||
463 | set_bit_le(tx_queue->queue, (void *)®); | ||
464 | efx_writeo(efx, ®, FR_AA_TX_CHKSM_CFG); | ||
465 | } | ||
466 | } | ||
467 | |||
468 | static void efx_flush_tx_queue(struct efx_tx_queue *tx_queue) | ||
469 | { | ||
470 | struct efx_nic *efx = tx_queue->efx; | ||
471 | efx_oword_t tx_flush_descq; | ||
472 | |||
473 | tx_queue->flushed = FLUSH_PENDING; | ||
474 | |||
475 | /* Post a flush command */ | ||
476 | EFX_POPULATE_OWORD_2(tx_flush_descq, | ||
477 | FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, | ||
478 | FRF_AZ_TX_FLUSH_DESCQ, tx_queue->queue); | ||
479 | efx_writeo(efx, &tx_flush_descq, FR_AZ_TX_FLUSH_DESCQ); | ||
480 | } | ||
481 | |||
482 | void efx_nic_fini_tx(struct efx_tx_queue *tx_queue) | ||
483 | { | ||
484 | struct efx_nic *efx = tx_queue->efx; | ||
485 | efx_oword_t tx_desc_ptr; | ||
486 | |||
487 | /* The queue should have been flushed */ | ||
488 | WARN_ON(tx_queue->flushed != FLUSH_DONE); | ||
489 | |||
490 | /* Remove TX descriptor ring from card */ | ||
491 | EFX_ZERO_OWORD(tx_desc_ptr); | ||
492 | efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base, | ||
493 | tx_queue->queue); | ||
494 | |||
495 | /* Unpin TX descriptor ring */ | ||
496 | efx_fini_special_buffer(efx, &tx_queue->txd); | ||
497 | } | ||
498 | |||
499 | /* Free buffers backing TX queue */ | ||
500 | void efx_nic_remove_tx(struct efx_tx_queue *tx_queue) | ||
501 | { | ||
502 | efx_free_special_buffer(tx_queue->efx, &tx_queue->txd); | ||
503 | } | ||
504 | |||
505 | /************************************************************************** | ||
506 | * | ||
507 | * RX path | ||
508 | * | ||
509 | **************************************************************************/ | ||
510 | |||
511 | /* Returns a pointer to the specified descriptor in the RX descriptor queue */ | ||
512 | static inline efx_qword_t * | ||
513 | efx_rx_desc(struct efx_rx_queue *rx_queue, unsigned int index) | ||
514 | { | ||
515 | return (((efx_qword_t *) (rx_queue->rxd.addr)) + index); | ||
516 | } | ||
517 | |||
518 | /* This creates an entry in the RX descriptor queue */ | ||
519 | static inline void | ||
520 | efx_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned index) | ||
521 | { | ||
522 | struct efx_rx_buffer *rx_buf; | ||
523 | efx_qword_t *rxd; | ||
524 | |||
525 | rxd = efx_rx_desc(rx_queue, index); | ||
526 | rx_buf = efx_rx_buffer(rx_queue, index); | ||
527 | EFX_POPULATE_QWORD_3(*rxd, | ||
528 | FSF_AZ_RX_KER_BUF_SIZE, | ||
529 | rx_buf->len - | ||
530 | rx_queue->efx->type->rx_buffer_padding, | ||
531 | FSF_AZ_RX_KER_BUF_REGION, 0, | ||
532 | FSF_AZ_RX_KER_BUF_ADDR, rx_buf->dma_addr); | ||
533 | } | ||
534 | |||
535 | /* This writes to the RX_DESC_WPTR register for the specified receive | ||
536 | * descriptor ring. | ||
537 | */ | ||
538 | void efx_nic_notify_rx_desc(struct efx_rx_queue *rx_queue) | ||
539 | { | ||
540 | efx_dword_t reg; | ||
541 | unsigned write_ptr; | ||
542 | |||
543 | while (rx_queue->notified_count != rx_queue->added_count) { | ||
544 | efx_build_rx_desc(rx_queue, | ||
545 | rx_queue->notified_count & | ||
546 | EFX_RXQ_MASK); | ||
547 | ++rx_queue->notified_count; | ||
548 | } | ||
549 | |||
550 | wmb(); | ||
551 | write_ptr = rx_queue->added_count & EFX_RXQ_MASK; | ||
552 | EFX_POPULATE_DWORD_1(reg, FRF_AZ_RX_DESC_WPTR_DWORD, write_ptr); | ||
553 | efx_writed_page(rx_queue->efx, ®, | ||
554 | FR_AZ_RX_DESC_UPD_DWORD_P0, rx_queue->queue); | ||
555 | } | ||
556 | |||
557 | int efx_nic_probe_rx(struct efx_rx_queue *rx_queue) | ||
558 | { | ||
559 | struct efx_nic *efx = rx_queue->efx; | ||
560 | BUILD_BUG_ON(EFX_RXQ_SIZE < 512 || EFX_RXQ_SIZE > 4096 || | ||
561 | EFX_RXQ_SIZE & EFX_RXQ_MASK); | ||
562 | return efx_alloc_special_buffer(efx, &rx_queue->rxd, | ||
563 | EFX_RXQ_SIZE * sizeof(efx_qword_t)); | ||
564 | } | ||
565 | |||
566 | void efx_nic_init_rx(struct efx_rx_queue *rx_queue) | ||
567 | { | ||
568 | efx_oword_t rx_desc_ptr; | ||
569 | struct efx_nic *efx = rx_queue->efx; | ||
570 | bool is_b0 = efx_nic_rev(efx) >= EFX_REV_FALCON_B0; | ||
571 | bool iscsi_digest_en = is_b0; | ||
572 | |||
573 | EFX_LOG(efx, "RX queue %d ring in special buffers %d-%d\n", | ||
574 | rx_queue->queue, rx_queue->rxd.index, | ||
575 | rx_queue->rxd.index + rx_queue->rxd.entries - 1); | ||
576 | |||
577 | rx_queue->flushed = FLUSH_NONE; | ||
578 | |||
579 | /* Pin RX descriptor ring */ | ||
580 | efx_init_special_buffer(efx, &rx_queue->rxd); | ||
581 | |||
582 | /* Push RX descriptor ring to card */ | ||
583 | EFX_POPULATE_OWORD_10(rx_desc_ptr, | ||
584 | FRF_AZ_RX_ISCSI_DDIG_EN, iscsi_digest_en, | ||
585 | FRF_AZ_RX_ISCSI_HDIG_EN, iscsi_digest_en, | ||
586 | FRF_AZ_RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index, | ||
587 | FRF_AZ_RX_DESCQ_EVQ_ID, | ||
588 | rx_queue->channel->channel, | ||
589 | FRF_AZ_RX_DESCQ_OWNER_ID, 0, | ||
590 | FRF_AZ_RX_DESCQ_LABEL, rx_queue->queue, | ||
591 | FRF_AZ_RX_DESCQ_SIZE, | ||
592 | __ffs(rx_queue->rxd.entries), | ||
593 | FRF_AZ_RX_DESCQ_TYPE, 0 /* kernel queue */ , | ||
594 | /* For >=B0 this is scatter so disable */ | ||
595 | FRF_AZ_RX_DESCQ_JUMBO, !is_b0, | ||
596 | FRF_AZ_RX_DESCQ_EN, 1); | ||
597 | efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base, | ||
598 | rx_queue->queue); | ||
599 | } | ||
600 | |||
601 | static void efx_flush_rx_queue(struct efx_rx_queue *rx_queue) | ||
602 | { | ||
603 | struct efx_nic *efx = rx_queue->efx; | ||
604 | efx_oword_t rx_flush_descq; | ||
605 | |||
606 | rx_queue->flushed = FLUSH_PENDING; | ||
607 | |||
608 | /* Post a flush command */ | ||
609 | EFX_POPULATE_OWORD_2(rx_flush_descq, | ||
610 | FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, | ||
611 | FRF_AZ_RX_FLUSH_DESCQ, rx_queue->queue); | ||
612 | efx_writeo(efx, &rx_flush_descq, FR_AZ_RX_FLUSH_DESCQ); | ||
613 | } | ||
614 | |||
615 | void efx_nic_fini_rx(struct efx_rx_queue *rx_queue) | ||
616 | { | ||
617 | efx_oword_t rx_desc_ptr; | ||
618 | struct efx_nic *efx = rx_queue->efx; | ||
619 | |||
620 | /* The queue should already have been flushed */ | ||
621 | WARN_ON(rx_queue->flushed != FLUSH_DONE); | ||
622 | |||
623 | /* Remove RX descriptor ring from card */ | ||
624 | EFX_ZERO_OWORD(rx_desc_ptr); | ||
625 | efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base, | ||
626 | rx_queue->queue); | ||
627 | |||
628 | /* Unpin RX descriptor ring */ | ||
629 | efx_fini_special_buffer(efx, &rx_queue->rxd); | ||
630 | } | ||
631 | |||
632 | /* Free buffers backing RX queue */ | ||
633 | void efx_nic_remove_rx(struct efx_rx_queue *rx_queue) | ||
634 | { | ||
635 | efx_free_special_buffer(rx_queue->efx, &rx_queue->rxd); | ||
636 | } | ||
637 | |||
638 | /************************************************************************** | ||
639 | * | ||
640 | * Event queue processing | ||
641 | * Event queues are processed by per-channel tasklets. | ||
642 | * | ||
643 | **************************************************************************/ | ||
644 | |||
645 | /* Update a channel's event queue's read pointer (RPTR) register | ||
646 | * | ||
647 | * This writes the EVQ_RPTR_REG register for the specified channel's | ||
648 | * event queue. | ||
649 | * | ||
650 | * Note that EVQ_RPTR_REG contains the index of the "last read" event, | ||
651 | * whereas channel->eventq_read_ptr contains the index of the "next to | ||
652 | * read" event. | ||
653 | */ | ||
654 | void efx_nic_eventq_read_ack(struct efx_channel *channel) | ||
655 | { | ||
656 | efx_dword_t reg; | ||
657 | struct efx_nic *efx = channel->efx; | ||
658 | |||
659 | EFX_POPULATE_DWORD_1(reg, FRF_AZ_EVQ_RPTR, channel->eventq_read_ptr); | ||
660 | efx_writed_table(efx, ®, efx->type->evq_rptr_tbl_base, | ||
661 | channel->channel); | ||
662 | } | ||
663 | |||
664 | /* Use HW to insert a SW defined event */ | ||
665 | void efx_generate_event(struct efx_channel *channel, efx_qword_t *event) | ||
666 | { | ||
667 | efx_oword_t drv_ev_reg; | ||
668 | |||
669 | BUILD_BUG_ON(FRF_AZ_DRV_EV_DATA_LBN != 0 || | ||
670 | FRF_AZ_DRV_EV_DATA_WIDTH != 64); | ||
671 | drv_ev_reg.u32[0] = event->u32[0]; | ||
672 | drv_ev_reg.u32[1] = event->u32[1]; | ||
673 | drv_ev_reg.u32[2] = 0; | ||
674 | drv_ev_reg.u32[3] = 0; | ||
675 | EFX_SET_OWORD_FIELD(drv_ev_reg, FRF_AZ_DRV_EV_QID, channel->channel); | ||
676 | efx_writeo(channel->efx, &drv_ev_reg, FR_AZ_DRV_EV); | ||
677 | } | ||
678 | |||
679 | /* Handle a transmit completion event | ||
680 | * | ||
681 | * The NIC batches TX completion events; the message we receive is of | ||
682 | * the form "complete all TX events up to this index". | ||
683 | */ | ||
684 | static void | ||
685 | efx_handle_tx_event(struct efx_channel *channel, efx_qword_t *event) | ||
686 | { | ||
687 | unsigned int tx_ev_desc_ptr; | ||
688 | unsigned int tx_ev_q_label; | ||
689 | struct efx_tx_queue *tx_queue; | ||
690 | struct efx_nic *efx = channel->efx; | ||
691 | |||
692 | if (likely(EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_COMP))) { | ||
693 | /* Transmit completion */ | ||
694 | tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_DESC_PTR); | ||
695 | tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL); | ||
696 | tx_queue = &efx->tx_queue[tx_ev_q_label]; | ||
697 | channel->irq_mod_score += | ||
698 | (tx_ev_desc_ptr - tx_queue->read_count) & | ||
699 | EFX_TXQ_MASK; | ||
700 | efx_xmit_done(tx_queue, tx_ev_desc_ptr); | ||
701 | } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) { | ||
702 | /* Rewrite the FIFO write pointer */ | ||
703 | tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL); | ||
704 | tx_queue = &efx->tx_queue[tx_ev_q_label]; | ||
705 | |||
706 | if (efx_dev_registered(efx)) | ||
707 | netif_tx_lock(efx->net_dev); | ||
708 | efx_notify_tx_desc(tx_queue); | ||
709 | if (efx_dev_registered(efx)) | ||
710 | netif_tx_unlock(efx->net_dev); | ||
711 | } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_PKT_ERR) && | ||
712 | EFX_WORKAROUND_10727(efx)) { | ||
713 | efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH); | ||
714 | } else { | ||
715 | EFX_ERR(efx, "channel %d unexpected TX event " | ||
716 | EFX_QWORD_FMT"\n", channel->channel, | ||
717 | EFX_QWORD_VAL(*event)); | ||
718 | } | ||
719 | } | ||
720 | |||
721 | /* Detect errors included in the rx_evt_pkt_ok bit. */ | ||
722 | static void efx_handle_rx_not_ok(struct efx_rx_queue *rx_queue, | ||
723 | const efx_qword_t *event, | ||
724 | bool *rx_ev_pkt_ok, | ||
725 | bool *discard) | ||
726 | { | ||
727 | struct efx_nic *efx = rx_queue->efx; | ||
728 | bool rx_ev_buf_owner_id_err, rx_ev_ip_hdr_chksum_err; | ||
729 | bool rx_ev_tcp_udp_chksum_err, rx_ev_eth_crc_err; | ||
730 | bool rx_ev_frm_trunc, rx_ev_drib_nib, rx_ev_tobe_disc; | ||
731 | bool rx_ev_other_err, rx_ev_pause_frm; | ||
732 | bool rx_ev_hdr_type, rx_ev_mcast_pkt; | ||
733 | unsigned rx_ev_pkt_type; | ||
734 | |||
735 | rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE); | ||
736 | rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT); | ||
737 | rx_ev_tobe_disc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_TOBE_DISC); | ||
738 | rx_ev_pkt_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_TYPE); | ||
739 | rx_ev_buf_owner_id_err = EFX_QWORD_FIELD(*event, | ||
740 | FSF_AZ_RX_EV_BUF_OWNER_ID_ERR); | ||
741 | rx_ev_ip_hdr_chksum_err = EFX_QWORD_FIELD(*event, | ||
742 | FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR); | ||
743 | rx_ev_tcp_udp_chksum_err = EFX_QWORD_FIELD(*event, | ||
744 | FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR); | ||
745 | rx_ev_eth_crc_err = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_ETH_CRC_ERR); | ||
746 | rx_ev_frm_trunc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_FRM_TRUNC); | ||
747 | rx_ev_drib_nib = ((efx_nic_rev(efx) >= EFX_REV_FALCON_B0) ? | ||
748 | 0 : EFX_QWORD_FIELD(*event, FSF_AA_RX_EV_DRIB_NIB)); | ||
749 | rx_ev_pause_frm = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PAUSE_FRM_ERR); | ||
750 | |||
751 | /* Every error apart from tobe_disc and pause_frm */ | ||
752 | rx_ev_other_err = (rx_ev_drib_nib | rx_ev_tcp_udp_chksum_err | | ||
753 | rx_ev_buf_owner_id_err | rx_ev_eth_crc_err | | ||
754 | rx_ev_frm_trunc | rx_ev_ip_hdr_chksum_err); | ||
755 | |||
756 | /* Count errors that are not in MAC stats. Ignore expected | ||
757 | * checksum errors during self-test. */ | ||
758 | if (rx_ev_frm_trunc) | ||
759 | ++rx_queue->channel->n_rx_frm_trunc; | ||
760 | else if (rx_ev_tobe_disc) | ||
761 | ++rx_queue->channel->n_rx_tobe_disc; | ||
762 | else if (!efx->loopback_selftest) { | ||
763 | if (rx_ev_ip_hdr_chksum_err) | ||
764 | ++rx_queue->channel->n_rx_ip_hdr_chksum_err; | ||
765 | else if (rx_ev_tcp_udp_chksum_err) | ||
766 | ++rx_queue->channel->n_rx_tcp_udp_chksum_err; | ||
767 | } | ||
768 | |||
769 | /* The frame must be discarded if any of these are true. */ | ||
770 | *discard = (rx_ev_eth_crc_err | rx_ev_frm_trunc | rx_ev_drib_nib | | ||
771 | rx_ev_tobe_disc | rx_ev_pause_frm); | ||
772 | |||
773 | /* TOBE_DISC is expected on unicast mismatches; don't print out an | ||
774 | * error message. FRM_TRUNC indicates RXDP dropped the packet due | ||
775 | * to a FIFO overflow. | ||
776 | */ | ||
777 | #ifdef EFX_ENABLE_DEBUG | ||
778 | if (rx_ev_other_err) { | ||
779 | EFX_INFO_RL(efx, " RX queue %d unexpected RX event " | ||
780 | EFX_QWORD_FMT "%s%s%s%s%s%s%s%s\n", | ||
781 | rx_queue->queue, EFX_QWORD_VAL(*event), | ||
782 | rx_ev_buf_owner_id_err ? " [OWNER_ID_ERR]" : "", | ||
783 | rx_ev_ip_hdr_chksum_err ? | ||
784 | " [IP_HDR_CHKSUM_ERR]" : "", | ||
785 | rx_ev_tcp_udp_chksum_err ? | ||
786 | " [TCP_UDP_CHKSUM_ERR]" : "", | ||
787 | rx_ev_eth_crc_err ? " [ETH_CRC_ERR]" : "", | ||
788 | rx_ev_frm_trunc ? " [FRM_TRUNC]" : "", | ||
789 | rx_ev_drib_nib ? " [DRIB_NIB]" : "", | ||
790 | rx_ev_tobe_disc ? " [TOBE_DISC]" : "", | ||
791 | rx_ev_pause_frm ? " [PAUSE]" : ""); | ||
792 | } | ||
793 | #endif | ||
794 | } | ||
795 | |||
796 | /* Handle receive events that are not in-order. */ | ||
797 | static void | ||
798 | efx_handle_rx_bad_index(struct efx_rx_queue *rx_queue, unsigned index) | ||
799 | { | ||
800 | struct efx_nic *efx = rx_queue->efx; | ||
801 | unsigned expected, dropped; | ||
802 | |||
803 | expected = rx_queue->removed_count & EFX_RXQ_MASK; | ||
804 | dropped = (index - expected) & EFX_RXQ_MASK; | ||
805 | EFX_INFO(efx, "dropped %d events (index=%d expected=%d)\n", | ||
806 | dropped, index, expected); | ||
807 | |||
808 | efx_schedule_reset(efx, EFX_WORKAROUND_5676(efx) ? | ||
809 | RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE); | ||
810 | } | ||
811 | |||
812 | /* Handle a packet received event | ||
813 | * | ||
814 | * The NIC gives a "discard" flag if it's a unicast packet with the | ||
815 | * wrong destination address | ||
816 | * Also "is multicast" and "matches multicast filter" flags can be used to | ||
817 | * discard non-matching multicast packets. | ||
818 | */ | ||
819 | static void | ||
820 | efx_handle_rx_event(struct efx_channel *channel, const efx_qword_t *event) | ||
821 | { | ||
822 | unsigned int rx_ev_desc_ptr, rx_ev_byte_cnt; | ||
823 | unsigned int rx_ev_hdr_type, rx_ev_mcast_pkt; | ||
824 | unsigned expected_ptr; | ||
825 | bool rx_ev_pkt_ok, discard = false, checksummed; | ||
826 | struct efx_rx_queue *rx_queue; | ||
827 | struct efx_nic *efx = channel->efx; | ||
828 | |||
829 | /* Basic packet information */ | ||
830 | rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_BYTE_CNT); | ||
831 | rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_OK); | ||
832 | rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE); | ||
833 | WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_JUMBO_CONT)); | ||
834 | WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_SOP) != 1); | ||
835 | WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_Q_LABEL) != | ||
836 | channel->channel); | ||
837 | |||
838 | rx_queue = &efx->rx_queue[channel->channel]; | ||
839 | |||
840 | rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_DESC_PTR); | ||
841 | expected_ptr = rx_queue->removed_count & EFX_RXQ_MASK; | ||
842 | if (unlikely(rx_ev_desc_ptr != expected_ptr)) | ||
843 | efx_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr); | ||
844 | |||
845 | if (likely(rx_ev_pkt_ok)) { | ||
846 | /* If packet is marked as OK and packet type is TCP/IP or | ||
847 | * UDP/IP, then we can rely on the hardware checksum. | ||
848 | */ | ||
849 | checksummed = | ||
850 | likely(efx->rx_checksum_enabled) && | ||
851 | (rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_TCP || | ||
852 | rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_UDP); | ||
853 | } else { | ||
854 | efx_handle_rx_not_ok(rx_queue, event, &rx_ev_pkt_ok, &discard); | ||
855 | checksummed = false; | ||
856 | } | ||
857 | |||
858 | /* Detect multicast packets that didn't match the filter */ | ||
859 | rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT); | ||
860 | if (rx_ev_mcast_pkt) { | ||
861 | unsigned int rx_ev_mcast_hash_match = | ||
862 | EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_HASH_MATCH); | ||
863 | |||
864 | if (unlikely(!rx_ev_mcast_hash_match)) { | ||
865 | ++channel->n_rx_mcast_mismatch; | ||
866 | discard = true; | ||
867 | } | ||
868 | } | ||
869 | |||
870 | channel->irq_mod_score += 2; | ||
871 | |||
872 | /* Handle received packet */ | ||
873 | efx_rx_packet(rx_queue, rx_ev_desc_ptr, rx_ev_byte_cnt, | ||
874 | checksummed, discard); | ||
875 | } | ||
876 | |||
877 | /* Global events are basically PHY events */ | ||
878 | static void | ||
879 | efx_handle_global_event(struct efx_channel *channel, efx_qword_t *event) | ||
880 | { | ||
881 | struct efx_nic *efx = channel->efx; | ||
882 | bool handled = false; | ||
883 | |||
884 | if (EFX_QWORD_FIELD(*event, FSF_AB_GLB_EV_G_PHY0_INTR) || | ||
885 | EFX_QWORD_FIELD(*event, FSF_AB_GLB_EV_XG_PHY0_INTR) || | ||
886 | EFX_QWORD_FIELD(*event, FSF_AB_GLB_EV_XFP_PHY0_INTR)) { | ||
887 | /* Ignored */ | ||
888 | handled = true; | ||
889 | } | ||
890 | |||
891 | if ((efx_nic_rev(efx) >= EFX_REV_FALCON_B0) && | ||
892 | EFX_QWORD_FIELD(*event, FSF_BB_GLB_EV_XG_MGT_INTR)) { | ||
893 | efx->xmac_poll_required = true; | ||
894 | handled = true; | ||
895 | } | ||
896 | |||
897 | if (efx_nic_rev(efx) <= EFX_REV_FALCON_A1 ? | ||
898 | EFX_QWORD_FIELD(*event, FSF_AA_GLB_EV_RX_RECOVERY) : | ||
899 | EFX_QWORD_FIELD(*event, FSF_BB_GLB_EV_RX_RECOVERY)) { | ||
900 | EFX_ERR(efx, "channel %d seen global RX_RESET " | ||
901 | "event. Resetting.\n", channel->channel); | ||
902 | |||
903 | atomic_inc(&efx->rx_reset); | ||
904 | efx_schedule_reset(efx, EFX_WORKAROUND_6555(efx) ? | ||
905 | RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE); | ||
906 | handled = true; | ||
907 | } | ||
908 | |||
909 | if (!handled) | ||
910 | EFX_ERR(efx, "channel %d unknown global event " | ||
911 | EFX_QWORD_FMT "\n", channel->channel, | ||
912 | EFX_QWORD_VAL(*event)); | ||
913 | } | ||
914 | |||
915 | static void | ||
916 | efx_handle_driver_event(struct efx_channel *channel, efx_qword_t *event) | ||
917 | { | ||
918 | struct efx_nic *efx = channel->efx; | ||
919 | unsigned int ev_sub_code; | ||
920 | unsigned int ev_sub_data; | ||
921 | |||
922 | ev_sub_code = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBCODE); | ||
923 | ev_sub_data = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA); | ||
924 | |||
925 | switch (ev_sub_code) { | ||
926 | case FSE_AZ_TX_DESCQ_FLS_DONE_EV: | ||
927 | EFX_TRACE(efx, "channel %d TXQ %d flushed\n", | ||
928 | channel->channel, ev_sub_data); | ||
929 | break; | ||
930 | case FSE_AZ_RX_DESCQ_FLS_DONE_EV: | ||
931 | EFX_TRACE(efx, "channel %d RXQ %d flushed\n", | ||
932 | channel->channel, ev_sub_data); | ||
933 | break; | ||
934 | case FSE_AZ_EVQ_INIT_DONE_EV: | ||
935 | EFX_LOG(efx, "channel %d EVQ %d initialised\n", | ||
936 | channel->channel, ev_sub_data); | ||
937 | break; | ||
938 | case FSE_AZ_SRM_UPD_DONE_EV: | ||
939 | EFX_TRACE(efx, "channel %d SRAM update done\n", | ||
940 | channel->channel); | ||
941 | break; | ||
942 | case FSE_AZ_WAKE_UP_EV: | ||
943 | EFX_TRACE(efx, "channel %d RXQ %d wakeup event\n", | ||
944 | channel->channel, ev_sub_data); | ||
945 | break; | ||
946 | case FSE_AZ_TIMER_EV: | ||
947 | EFX_TRACE(efx, "channel %d RX queue %d timer expired\n", | ||
948 | channel->channel, ev_sub_data); | ||
949 | break; | ||
950 | case FSE_AA_RX_RECOVER_EV: | ||
951 | EFX_ERR(efx, "channel %d seen DRIVER RX_RESET event. " | ||
952 | "Resetting.\n", channel->channel); | ||
953 | atomic_inc(&efx->rx_reset); | ||
954 | efx_schedule_reset(efx, | ||
955 | EFX_WORKAROUND_6555(efx) ? | ||
956 | RESET_TYPE_RX_RECOVERY : | ||
957 | RESET_TYPE_DISABLE); | ||
958 | break; | ||
959 | case FSE_BZ_RX_DSC_ERROR_EV: | ||
960 | EFX_ERR(efx, "RX DMA Q %d reports descriptor fetch error." | ||
961 | " RX Q %d is disabled.\n", ev_sub_data, ev_sub_data); | ||
962 | efx_schedule_reset(efx, RESET_TYPE_RX_DESC_FETCH); | ||
963 | break; | ||
964 | case FSE_BZ_TX_DSC_ERROR_EV: | ||
965 | EFX_ERR(efx, "TX DMA Q %d reports descriptor fetch error." | ||
966 | " TX Q %d is disabled.\n", ev_sub_data, ev_sub_data); | ||
967 | efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH); | ||
968 | break; | ||
969 | default: | ||
970 | EFX_TRACE(efx, "channel %d unknown driver event code %d " | ||
971 | "data %04x\n", channel->channel, ev_sub_code, | ||
972 | ev_sub_data); | ||
973 | break; | ||
974 | } | ||
975 | } | ||
976 | |||
977 | int efx_nic_process_eventq(struct efx_channel *channel, int rx_quota) | ||
978 | { | ||
979 | unsigned int read_ptr; | ||
980 | efx_qword_t event, *p_event; | ||
981 | int ev_code; | ||
982 | int rx_packets = 0; | ||
983 | |||
984 | read_ptr = channel->eventq_read_ptr; | ||
985 | |||
986 | do { | ||
987 | p_event = efx_event(channel, read_ptr); | ||
988 | event = *p_event; | ||
989 | |||
990 | if (!efx_event_present(&event)) | ||
991 | /* End of events */ | ||
992 | break; | ||
993 | |||
994 | EFX_TRACE(channel->efx, "channel %d event is "EFX_QWORD_FMT"\n", | ||
995 | channel->channel, EFX_QWORD_VAL(event)); | ||
996 | |||
997 | /* Clear this event by marking it all ones */ | ||
998 | EFX_SET_QWORD(*p_event); | ||
999 | |||
1000 | ev_code = EFX_QWORD_FIELD(event, FSF_AZ_EV_CODE); | ||
1001 | |||
1002 | switch (ev_code) { | ||
1003 | case FSE_AZ_EV_CODE_RX_EV: | ||
1004 | efx_handle_rx_event(channel, &event); | ||
1005 | ++rx_packets; | ||
1006 | break; | ||
1007 | case FSE_AZ_EV_CODE_TX_EV: | ||
1008 | efx_handle_tx_event(channel, &event); | ||
1009 | break; | ||
1010 | case FSE_AZ_EV_CODE_DRV_GEN_EV: | ||
1011 | channel->eventq_magic = EFX_QWORD_FIELD( | ||
1012 | event, FSF_AZ_DRV_GEN_EV_MAGIC); | ||
1013 | EFX_LOG(channel->efx, "channel %d received generated " | ||
1014 | "event "EFX_QWORD_FMT"\n", channel->channel, | ||
1015 | EFX_QWORD_VAL(event)); | ||
1016 | break; | ||
1017 | case FSE_AZ_EV_CODE_GLOBAL_EV: | ||
1018 | efx_handle_global_event(channel, &event); | ||
1019 | break; | ||
1020 | case FSE_AZ_EV_CODE_DRIVER_EV: | ||
1021 | efx_handle_driver_event(channel, &event); | ||
1022 | break; | ||
1023 | default: | ||
1024 | EFX_ERR(channel->efx, "channel %d unknown event type %d" | ||
1025 | " (data " EFX_QWORD_FMT ")\n", channel->channel, | ||
1026 | ev_code, EFX_QWORD_VAL(event)); | ||
1027 | } | ||
1028 | |||
1029 | /* Increment read pointer */ | ||
1030 | read_ptr = (read_ptr + 1) & EFX_EVQ_MASK; | ||
1031 | |||
1032 | } while (rx_packets < rx_quota); | ||
1033 | |||
1034 | channel->eventq_read_ptr = read_ptr; | ||
1035 | return rx_packets; | ||
1036 | } | ||
1037 | |||
1038 | static void falcon_push_irq_moderation(struct efx_channel *channel) | 102 | static void falcon_push_irq_moderation(struct efx_channel *channel) |
1039 | { | 103 | { |
1040 | efx_dword_t timer_cmd; | 104 | efx_dword_t timer_cmd; |
@@ -1056,135 +120,6 @@ static void falcon_push_irq_moderation(struct efx_channel *channel) | |||
1056 | BUILD_BUG_ON(FR_AA_TIMER_COMMAND_KER != FR_BZ_TIMER_COMMAND_P0); | 120 | BUILD_BUG_ON(FR_AA_TIMER_COMMAND_KER != FR_BZ_TIMER_COMMAND_P0); |
1057 | efx_writed_page_locked(efx, &timer_cmd, FR_BZ_TIMER_COMMAND_P0, | 121 | efx_writed_page_locked(efx, &timer_cmd, FR_BZ_TIMER_COMMAND_P0, |
1058 | channel->channel); | 122 | channel->channel); |
1059 | |||
1060 | } | ||
1061 | |||
1062 | /* Allocate buffer table entries for event queue */ | ||
1063 | int efx_nic_probe_eventq(struct efx_channel *channel) | ||
1064 | { | ||
1065 | struct efx_nic *efx = channel->efx; | ||
1066 | BUILD_BUG_ON(EFX_EVQ_SIZE < 512 || EFX_EVQ_SIZE > 32768 || | ||
1067 | EFX_EVQ_SIZE & EFX_EVQ_MASK); | ||
1068 | return efx_alloc_special_buffer(efx, &channel->eventq, | ||
1069 | EFX_EVQ_SIZE * sizeof(efx_qword_t)); | ||
1070 | } | ||
1071 | |||
1072 | void efx_nic_init_eventq(struct efx_channel *channel) | ||
1073 | { | ||
1074 | efx_oword_t evq_ptr; | ||
1075 | struct efx_nic *efx = channel->efx; | ||
1076 | |||
1077 | EFX_LOG(efx, "channel %d event queue in special buffers %d-%d\n", | ||
1078 | channel->channel, channel->eventq.index, | ||
1079 | channel->eventq.index + channel->eventq.entries - 1); | ||
1080 | |||
1081 | /* Pin event queue buffer */ | ||
1082 | efx_init_special_buffer(efx, &channel->eventq); | ||
1083 | |||
1084 | /* Fill event queue with all ones (i.e. empty events) */ | ||
1085 | memset(channel->eventq.addr, 0xff, channel->eventq.len); | ||
1086 | |||
1087 | /* Push event queue to card */ | ||
1088 | EFX_POPULATE_OWORD_3(evq_ptr, | ||
1089 | FRF_AZ_EVQ_EN, 1, | ||
1090 | FRF_AZ_EVQ_SIZE, __ffs(channel->eventq.entries), | ||
1091 | FRF_AZ_EVQ_BUF_BASE_ID, channel->eventq.index); | ||
1092 | efx_writeo_table(efx, &evq_ptr, efx->type->evq_ptr_tbl_base, | ||
1093 | channel->channel); | ||
1094 | |||
1095 | efx->type->push_irq_moderation(channel); | ||
1096 | } | ||
1097 | |||
1098 | void efx_nic_fini_eventq(struct efx_channel *channel) | ||
1099 | { | ||
1100 | efx_oword_t eventq_ptr; | ||
1101 | struct efx_nic *efx = channel->efx; | ||
1102 | |||
1103 | /* Remove event queue from card */ | ||
1104 | EFX_ZERO_OWORD(eventq_ptr); | ||
1105 | efx_writeo_table(efx, &eventq_ptr, efx->type->evq_ptr_tbl_base, | ||
1106 | channel->channel); | ||
1107 | |||
1108 | /* Unpin event queue */ | ||
1109 | efx_fini_special_buffer(efx, &channel->eventq); | ||
1110 | } | ||
1111 | |||
1112 | /* Free buffers backing event queue */ | ||
1113 | void efx_nic_remove_eventq(struct efx_channel *channel) | ||
1114 | { | ||
1115 | efx_free_special_buffer(channel->efx, &channel->eventq); | ||
1116 | } | ||
1117 | |||
1118 | |||
1119 | /* Generates a test event on the event queue. A subsequent call to | ||
1120 | * process_eventq() should pick up the event and place the value of | ||
1121 | * "magic" into channel->eventq_magic; | ||
1122 | */ | ||
1123 | void efx_nic_generate_test_event(struct efx_channel *channel, unsigned int magic) | ||
1124 | { | ||
1125 | efx_qword_t test_event; | ||
1126 | |||
1127 | EFX_POPULATE_QWORD_2(test_event, FSF_AZ_EV_CODE, | ||
1128 | FSE_AZ_EV_CODE_DRV_GEN_EV, | ||
1129 | FSF_AZ_DRV_GEN_EV_MAGIC, magic); | ||
1130 | efx_generate_event(channel, &test_event); | ||
1131 | } | ||
1132 | |||
1133 | /************************************************************************** | ||
1134 | * | ||
1135 | * Flush handling | ||
1136 | * | ||
1137 | **************************************************************************/ | ||
1138 | |||
1139 | |||
1140 | static void efx_poll_flush_events(struct efx_nic *efx) | ||
1141 | { | ||
1142 | struct efx_channel *channel = &efx->channel[0]; | ||
1143 | struct efx_tx_queue *tx_queue; | ||
1144 | struct efx_rx_queue *rx_queue; | ||
1145 | unsigned int read_ptr = channel->eventq_read_ptr; | ||
1146 | unsigned int end_ptr = (read_ptr - 1) & EFX_EVQ_MASK; | ||
1147 | |||
1148 | do { | ||
1149 | efx_qword_t *event = efx_event(channel, read_ptr); | ||
1150 | int ev_code, ev_sub_code, ev_queue; | ||
1151 | bool ev_failed; | ||
1152 | |||
1153 | if (!efx_event_present(event)) | ||
1154 | break; | ||
1155 | |||
1156 | ev_code = EFX_QWORD_FIELD(*event, FSF_AZ_EV_CODE); | ||
1157 | ev_sub_code = EFX_QWORD_FIELD(*event, | ||
1158 | FSF_AZ_DRIVER_EV_SUBCODE); | ||
1159 | if (ev_code == FSE_AZ_EV_CODE_DRIVER_EV && | ||
1160 | ev_sub_code == FSE_AZ_TX_DESCQ_FLS_DONE_EV) { | ||
1161 | ev_queue = EFX_QWORD_FIELD(*event, | ||
1162 | FSF_AZ_DRIVER_EV_SUBDATA); | ||
1163 | if (ev_queue < EFX_TX_QUEUE_COUNT) { | ||
1164 | tx_queue = efx->tx_queue + ev_queue; | ||
1165 | tx_queue->flushed = FLUSH_DONE; | ||
1166 | } | ||
1167 | } else if (ev_code == FSE_AZ_EV_CODE_DRIVER_EV && | ||
1168 | ev_sub_code == FSE_AZ_RX_DESCQ_FLS_DONE_EV) { | ||
1169 | ev_queue = EFX_QWORD_FIELD( | ||
1170 | *event, FSF_AZ_DRIVER_EV_RX_DESCQ_ID); | ||
1171 | ev_failed = EFX_QWORD_FIELD( | ||
1172 | *event, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL); | ||
1173 | if (ev_queue < efx->n_rx_queues) { | ||
1174 | rx_queue = efx->rx_queue + ev_queue; | ||
1175 | rx_queue->flushed = | ||
1176 | ev_failed ? FLUSH_FAILED : FLUSH_DONE; | ||
1177 | } | ||
1178 | } | ||
1179 | |||
1180 | /* We're about to destroy the queue anyway, so | ||
1181 | * it's ok to throw away every non-flush event */ | ||
1182 | EFX_SET_QWORD(*event); | ||
1183 | |||
1184 | read_ptr = (read_ptr + 1) & EFX_EVQ_MASK; | ||
1185 | } while (read_ptr != end_ptr); | ||
1186 | |||
1187 | channel->eventq_read_ptr = read_ptr; | ||
1188 | } | 123 | } |
1189 | 124 | ||
1190 | static void falcon_deconfigure_mac_wrapper(struct efx_nic *efx); | 125 | static void falcon_deconfigure_mac_wrapper(struct efx_nic *efx); |
@@ -1199,123 +134,6 @@ static void falcon_prepare_flush(struct efx_nic *efx) | |||
1199 | msleep(10); | 134 | msleep(10); |
1200 | } | 135 | } |
1201 | 136 | ||
1202 | /* Handle tx and rx flushes at the same time, since they run in | ||
1203 | * parallel in the hardware and there's no reason for us to | ||
1204 | * serialise them */ | ||
1205 | int efx_nic_flush_queues(struct efx_nic *efx) | ||
1206 | { | ||
1207 | struct efx_rx_queue *rx_queue; | ||
1208 | struct efx_tx_queue *tx_queue; | ||
1209 | int i, tx_pending, rx_pending; | ||
1210 | |||
1211 | /* If necessary prepare the hardware for flushing */ | ||
1212 | efx->type->prepare_flush(efx); | ||
1213 | |||
1214 | /* Flush all tx queues in parallel */ | ||
1215 | efx_for_each_tx_queue(tx_queue, efx) | ||
1216 | efx_flush_tx_queue(tx_queue); | ||
1217 | |||
1218 | /* The hardware supports four concurrent rx flushes, each of which may | ||
1219 | * need to be retried if there is an outstanding descriptor fetch */ | ||
1220 | for (i = 0; i < EFX_FLUSH_POLL_COUNT; ++i) { | ||
1221 | rx_pending = tx_pending = 0; | ||
1222 | efx_for_each_rx_queue(rx_queue, efx) { | ||
1223 | if (rx_queue->flushed == FLUSH_PENDING) | ||
1224 | ++rx_pending; | ||
1225 | } | ||
1226 | efx_for_each_rx_queue(rx_queue, efx) { | ||
1227 | if (rx_pending == EFX_RX_FLUSH_COUNT) | ||
1228 | break; | ||
1229 | if (rx_queue->flushed == FLUSH_FAILED || | ||
1230 | rx_queue->flushed == FLUSH_NONE) { | ||
1231 | efx_flush_rx_queue(rx_queue); | ||
1232 | ++rx_pending; | ||
1233 | } | ||
1234 | } | ||
1235 | efx_for_each_tx_queue(tx_queue, efx) { | ||
1236 | if (tx_queue->flushed != FLUSH_DONE) | ||
1237 | ++tx_pending; | ||
1238 | } | ||
1239 | |||
1240 | if (rx_pending == 0 && tx_pending == 0) | ||
1241 | return 0; | ||
1242 | |||
1243 | msleep(EFX_FLUSH_INTERVAL); | ||
1244 | efx_poll_flush_events(efx); | ||
1245 | } | ||
1246 | |||
1247 | /* Mark the queues as all flushed. We're going to return failure | ||
1248 | * leading to a reset, or fake up success anyway */ | ||
1249 | efx_for_each_tx_queue(tx_queue, efx) { | ||
1250 | if (tx_queue->flushed != FLUSH_DONE) | ||
1251 | EFX_ERR(efx, "tx queue %d flush command timed out\n", | ||
1252 | tx_queue->queue); | ||
1253 | tx_queue->flushed = FLUSH_DONE; | ||
1254 | } | ||
1255 | efx_for_each_rx_queue(rx_queue, efx) { | ||
1256 | if (rx_queue->flushed != FLUSH_DONE) | ||
1257 | EFX_ERR(efx, "rx queue %d flush command timed out\n", | ||
1258 | rx_queue->queue); | ||
1259 | rx_queue->flushed = FLUSH_DONE; | ||
1260 | } | ||
1261 | |||
1262 | if (EFX_WORKAROUND_7803(efx)) | ||
1263 | return 0; | ||
1264 | |||
1265 | return -ETIMEDOUT; | ||
1266 | } | ||
1267 | |||
1268 | /************************************************************************** | ||
1269 | * | ||
1270 | * Hardware interrupts | ||
1271 | * The hardware interrupt handler does very little work; all the event | ||
1272 | * queue processing is carried out by per-channel tasklets. | ||
1273 | * | ||
1274 | **************************************************************************/ | ||
1275 | |||
1276 | /* Enable/disable/generate interrupts */ | ||
1277 | static inline void efx_nic_interrupts(struct efx_nic *efx, | ||
1278 | bool enabled, bool force) | ||
1279 | { | ||
1280 | efx_oword_t int_en_reg_ker; | ||
1281 | |||
1282 | EFX_POPULATE_OWORD_2(int_en_reg_ker, | ||
1283 | FRF_AZ_KER_INT_KER, force, | ||
1284 | FRF_AZ_DRV_INT_EN_KER, enabled); | ||
1285 | efx_writeo(efx, &int_en_reg_ker, FR_AZ_INT_EN_KER); | ||
1286 | } | ||
1287 | |||
1288 | void efx_nic_enable_interrupts(struct efx_nic *efx) | ||
1289 | { | ||
1290 | struct efx_channel *channel; | ||
1291 | |||
1292 | EFX_ZERO_OWORD(*((efx_oword_t *) efx->irq_status.addr)); | ||
1293 | wmb(); /* Ensure interrupt vector is clear before interrupts enabled */ | ||
1294 | |||
1295 | /* Enable interrupts */ | ||
1296 | efx_nic_interrupts(efx, true, false); | ||
1297 | |||
1298 | /* Force processing of all the channels to get the EVQ RPTRs up to | ||
1299 | date */ | ||
1300 | efx_for_each_channel(channel, efx) | ||
1301 | efx_schedule_channel(channel); | ||
1302 | } | ||
1303 | |||
1304 | void efx_nic_disable_interrupts(struct efx_nic *efx) | ||
1305 | { | ||
1306 | /* Disable interrupts */ | ||
1307 | efx_nic_interrupts(efx, false, false); | ||
1308 | } | ||
1309 | |||
1310 | /* Generate a test interrupt | ||
1311 | * Interrupt must already have been enabled, otherwise nasty things | ||
1312 | * may happen. | ||
1313 | */ | ||
1314 | void efx_nic_generate_interrupt(struct efx_nic *efx) | ||
1315 | { | ||
1316 | efx_nic_interrupts(efx, true, true); | ||
1317 | } | ||
1318 | |||
1319 | /* Acknowledge a legacy interrupt from Falcon | 137 | /* Acknowledge a legacy interrupt from Falcon |
1320 | * | 138 | * |
1321 | * This acknowledges a legacy (not MSI) interrupt via INT_ACK_KER_REG. | 139 | * This acknowledges a legacy (not MSI) interrupt via INT_ACK_KER_REG. |
@@ -1335,102 +153,6 @@ inline void falcon_irq_ack_a1(struct efx_nic *efx) | |||
1335 | efx_readd(efx, ®, FR_AA_WORK_AROUND_BROKEN_PCI_READS); | 153 | efx_readd(efx, ®, FR_AA_WORK_AROUND_BROKEN_PCI_READS); |
1336 | } | 154 | } |
1337 | 155 | ||
1338 | /* Process a fatal interrupt | ||
1339 | * Disable bus mastering ASAP and schedule a reset | ||
1340 | */ | ||
1341 | irqreturn_t efx_nic_fatal_interrupt(struct efx_nic *efx) | ||
1342 | { | ||
1343 | struct falcon_nic_data *nic_data = efx->nic_data; | ||
1344 | efx_oword_t *int_ker = efx->irq_status.addr; | ||
1345 | efx_oword_t fatal_intr; | ||
1346 | int error, mem_perr; | ||
1347 | |||
1348 | efx_reado(efx, &fatal_intr, FR_AZ_FATAL_INTR_KER); | ||
1349 | error = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_FATAL_INTR); | ||
1350 | |||
1351 | EFX_ERR(efx, "SYSTEM ERROR " EFX_OWORD_FMT " status " | ||
1352 | EFX_OWORD_FMT ": %s\n", EFX_OWORD_VAL(*int_ker), | ||
1353 | EFX_OWORD_VAL(fatal_intr), | ||
1354 | error ? "disabling bus mastering" : "no recognised error"); | ||
1355 | if (error == 0) | ||
1356 | goto out; | ||
1357 | |||
1358 | /* If this is a memory parity error dump which blocks are offending */ | ||
1359 | mem_perr = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_MEM_PERR_INT_KER); | ||
1360 | if (mem_perr) { | ||
1361 | efx_oword_t reg; | ||
1362 | efx_reado(efx, ®, FR_AZ_MEM_STAT); | ||
1363 | EFX_ERR(efx, "SYSTEM ERROR: memory parity error " | ||
1364 | EFX_OWORD_FMT "\n", EFX_OWORD_VAL(reg)); | ||
1365 | } | ||
1366 | |||
1367 | /* Disable both devices */ | ||
1368 | pci_clear_master(efx->pci_dev); | ||
1369 | if (efx_nic_is_dual_func(efx)) | ||
1370 | pci_clear_master(nic_data->pci_dev2); | ||
1371 | efx_nic_disable_interrupts(efx); | ||
1372 | |||
1373 | /* Count errors and reset or disable the NIC accordingly */ | ||
1374 | if (efx->int_error_count == 0 || | ||
1375 | time_after(jiffies, efx->int_error_expire)) { | ||
1376 | efx->int_error_count = 0; | ||
1377 | efx->int_error_expire = | ||
1378 | jiffies + EFX_INT_ERROR_EXPIRE * HZ; | ||
1379 | } | ||
1380 | if (++efx->int_error_count < EFX_MAX_INT_ERRORS) { | ||
1381 | EFX_ERR(efx, "SYSTEM ERROR - reset scheduled\n"); | ||
1382 | efx_schedule_reset(efx, RESET_TYPE_INT_ERROR); | ||
1383 | } else { | ||
1384 | EFX_ERR(efx, "SYSTEM ERROR - max number of errors seen." | ||
1385 | "NIC will be disabled\n"); | ||
1386 | efx_schedule_reset(efx, RESET_TYPE_DISABLE); | ||
1387 | } | ||
1388 | out: | ||
1389 | return IRQ_HANDLED; | ||
1390 | } | ||
1391 | |||
1392 | /* Handle a legacy interrupt | ||
1393 | * Acknowledges the interrupt and schedule event queue processing. | ||
1394 | */ | ||
1395 | static irqreturn_t efx_legacy_interrupt(int irq, void *dev_id) | ||
1396 | { | ||
1397 | struct efx_nic *efx = dev_id; | ||
1398 | efx_oword_t *int_ker = efx->irq_status.addr; | ||
1399 | irqreturn_t result = IRQ_NONE; | ||
1400 | struct efx_channel *channel; | ||
1401 | efx_dword_t reg; | ||
1402 | u32 queues; | ||
1403 | int syserr; | ||
1404 | |||
1405 | /* Read the ISR which also ACKs the interrupts */ | ||
1406 | efx_readd(efx, ®, FR_BZ_INT_ISR0); | ||
1407 | queues = EFX_EXTRACT_DWORD(reg, 0, 31); | ||
1408 | |||
1409 | /* Check to see if we have a serious error condition */ | ||
1410 | syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT); | ||
1411 | if (unlikely(syserr)) | ||
1412 | return efx_nic_fatal_interrupt(efx); | ||
1413 | |||
1414 | /* Schedule processing of any interrupting queues */ | ||
1415 | efx_for_each_channel(channel, efx) { | ||
1416 | if ((queues & 1) || | ||
1417 | efx_event_present( | ||
1418 | efx_event(channel, channel->eventq_read_ptr))) { | ||
1419 | efx_schedule_channel(channel); | ||
1420 | result = IRQ_HANDLED; | ||
1421 | } | ||
1422 | queues >>= 1; | ||
1423 | } | ||
1424 | |||
1425 | if (result == IRQ_HANDLED) { | ||
1426 | efx->last_irq_cpu = raw_smp_processor_id(); | ||
1427 | EFX_TRACE(efx, "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n", | ||
1428 | irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg)); | ||
1429 | } | ||
1430 | |||
1431 | return result; | ||
1432 | } | ||
1433 | |||
1434 | 156 | ||
1435 | irqreturn_t falcon_legacy_interrupt_a1(int irq, void *dev_id) | 157 | irqreturn_t falcon_legacy_interrupt_a1(int irq, void *dev_id) |
1436 | { | 158 | { |
@@ -1477,126 +199,6 @@ irqreturn_t falcon_legacy_interrupt_a1(int irq, void *dev_id) | |||
1477 | 199 | ||
1478 | return IRQ_HANDLED; | 200 | return IRQ_HANDLED; |
1479 | } | 201 | } |
1480 | |||
1481 | /* Handle an MSI interrupt | ||
1482 | * | ||
1483 | * Handle an MSI hardware interrupt. This routine schedules event | ||
1484 | * queue processing. No interrupt acknowledgement cycle is necessary. | ||
1485 | * Also, we never need to check that the interrupt is for us, since | ||
1486 | * MSI interrupts cannot be shared. | ||
1487 | */ | ||
1488 | static irqreturn_t efx_msi_interrupt(int irq, void *dev_id) | ||
1489 | { | ||
1490 | struct efx_channel *channel = dev_id; | ||
1491 | struct efx_nic *efx = channel->efx; | ||
1492 | efx_oword_t *int_ker = efx->irq_status.addr; | ||
1493 | int syserr; | ||
1494 | |||
1495 | efx->last_irq_cpu = raw_smp_processor_id(); | ||
1496 | EFX_TRACE(efx, "IRQ %d on CPU %d status " EFX_OWORD_FMT "\n", | ||
1497 | irq, raw_smp_processor_id(), EFX_OWORD_VAL(*int_ker)); | ||
1498 | |||
1499 | /* Check to see if we have a serious error condition */ | ||
1500 | syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT); | ||
1501 | if (unlikely(syserr)) | ||
1502 | return efx_nic_fatal_interrupt(efx); | ||
1503 | |||
1504 | /* Schedule processing of the channel */ | ||
1505 | efx_schedule_channel(channel); | ||
1506 | |||
1507 | return IRQ_HANDLED; | ||
1508 | } | ||
1509 | |||
1510 | |||
1511 | /* Setup RSS indirection table. | ||
1512 | * This maps from the hash value of the packet to RXQ | ||
1513 | */ | ||
1514 | static void efx_setup_rss_indir_table(struct efx_nic *efx) | ||
1515 | { | ||
1516 | int i = 0; | ||
1517 | unsigned long offset; | ||
1518 | efx_dword_t dword; | ||
1519 | |||
1520 | if (efx_nic_rev(efx) < EFX_REV_FALCON_B0) | ||
1521 | return; | ||
1522 | |||
1523 | for (offset = FR_BZ_RX_INDIRECTION_TBL; | ||
1524 | offset < FR_BZ_RX_INDIRECTION_TBL + 0x800; | ||
1525 | offset += 0x10) { | ||
1526 | EFX_POPULATE_DWORD_1(dword, FRF_BZ_IT_QUEUE, | ||
1527 | i % efx->n_rx_queues); | ||
1528 | efx_writed(efx, &dword, offset); | ||
1529 | i++; | ||
1530 | } | ||
1531 | } | ||
1532 | |||
1533 | /* Hook interrupt handler(s) | ||
1534 | * Try MSI and then legacy interrupts. | ||
1535 | */ | ||
1536 | int efx_nic_init_interrupt(struct efx_nic *efx) | ||
1537 | { | ||
1538 | struct efx_channel *channel; | ||
1539 | int rc; | ||
1540 | |||
1541 | if (!EFX_INT_MODE_USE_MSI(efx)) { | ||
1542 | irq_handler_t handler; | ||
1543 | if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) | ||
1544 | handler = efx_legacy_interrupt; | ||
1545 | else | ||
1546 | handler = falcon_legacy_interrupt_a1; | ||
1547 | |||
1548 | rc = request_irq(efx->legacy_irq, handler, IRQF_SHARED, | ||
1549 | efx->name, efx); | ||
1550 | if (rc) { | ||
1551 | EFX_ERR(efx, "failed to hook legacy IRQ %d\n", | ||
1552 | efx->pci_dev->irq); | ||
1553 | goto fail1; | ||
1554 | } | ||
1555 | return 0; | ||
1556 | } | ||
1557 | |||
1558 | /* Hook MSI or MSI-X interrupt */ | ||
1559 | efx_for_each_channel(channel, efx) { | ||
1560 | rc = request_irq(channel->irq, efx_msi_interrupt, | ||
1561 | IRQF_PROBE_SHARED, /* Not shared */ | ||
1562 | channel->name, channel); | ||
1563 | if (rc) { | ||
1564 | EFX_ERR(efx, "failed to hook IRQ %d\n", channel->irq); | ||
1565 | goto fail2; | ||
1566 | } | ||
1567 | } | ||
1568 | |||
1569 | return 0; | ||
1570 | |||
1571 | fail2: | ||
1572 | efx_for_each_channel(channel, efx) | ||
1573 | free_irq(channel->irq, channel); | ||
1574 | fail1: | ||
1575 | return rc; | ||
1576 | } | ||
1577 | |||
1578 | void efx_nic_fini_interrupt(struct efx_nic *efx) | ||
1579 | { | ||
1580 | struct efx_channel *channel; | ||
1581 | efx_oword_t reg; | ||
1582 | |||
1583 | /* Disable MSI/MSI-X interrupts */ | ||
1584 | efx_for_each_channel(channel, efx) { | ||
1585 | if (channel->irq) | ||
1586 | free_irq(channel->irq, channel); | ||
1587 | } | ||
1588 | |||
1589 | /* ACK legacy interrupt */ | ||
1590 | if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) | ||
1591 | efx_reado(efx, ®, FR_BZ_INT_ISR0); | ||
1592 | else | ||
1593 | falcon_irq_ack_a1(efx); | ||
1594 | |||
1595 | /* Disable legacy interrupt */ | ||
1596 | if (efx->legacy_irq) | ||
1597 | free_irq(efx->legacy_irq, efx); | ||
1598 | } | ||
1599 | |||
1600 | /************************************************************************** | 202 | /************************************************************************** |
1601 | * | 203 | * |
1602 | * EEPROM/flash | 204 | * EEPROM/flash |
@@ -2440,68 +1042,6 @@ static const struct efx_nic_register_test falcon_b0_register_tests[] = { | |||
2440 | EFX_OWORD32(0x0003FF0F, 0x00000000, 0x00000000, 0x00000000) }, | 1042 | EFX_OWORD32(0x0003FF0F, 0x00000000, 0x00000000, 0x00000000) }, |
2441 | }; | 1043 | }; |
2442 | 1044 | ||
2443 | static bool efx_masked_compare_oword(const efx_oword_t *a, const efx_oword_t *b, | ||
2444 | const efx_oword_t *mask) | ||
2445 | { | ||
2446 | return ((a->u64[0] ^ b->u64[0]) & mask->u64[0]) || | ||
2447 | ((a->u64[1] ^ b->u64[1]) & mask->u64[1]); | ||
2448 | } | ||
2449 | |||
2450 | int efx_nic_test_registers(struct efx_nic *efx, | ||
2451 | const struct efx_nic_register_test *regs, | ||
2452 | size_t n_regs) | ||
2453 | { | ||
2454 | unsigned address = 0, i, j; | ||
2455 | efx_oword_t mask, imask, original, reg, buf; | ||
2456 | |||
2457 | /* Falcon should be in loopback to isolate the XMAC from the PHY */ | ||
2458 | WARN_ON(!LOOPBACK_INTERNAL(efx)); | ||
2459 | |||
2460 | for (i = 0; i < n_regs; ++i) { | ||
2461 | address = regs[i].address; | ||
2462 | mask = imask = regs[i].mask; | ||
2463 | EFX_INVERT_OWORD(imask); | ||
2464 | |||
2465 | efx_reado(efx, &original, address); | ||
2466 | |||
2467 | /* bit sweep on and off */ | ||
2468 | for (j = 0; j < 128; j++) { | ||
2469 | if (!EFX_EXTRACT_OWORD32(mask, j, j)) | ||
2470 | continue; | ||
2471 | |||
2472 | /* Test this testable bit can be set in isolation */ | ||
2473 | EFX_AND_OWORD(reg, original, mask); | ||
2474 | EFX_SET_OWORD32(reg, j, j, 1); | ||
2475 | |||
2476 | efx_writeo(efx, ®, address); | ||
2477 | efx_reado(efx, &buf, address); | ||
2478 | |||
2479 | if (efx_masked_compare_oword(®, &buf, &mask)) | ||
2480 | goto fail; | ||
2481 | |||
2482 | /* Test this testable bit can be cleared in isolation */ | ||
2483 | EFX_OR_OWORD(reg, original, mask); | ||
2484 | EFX_SET_OWORD32(reg, j, j, 0); | ||
2485 | |||
2486 | efx_writeo(efx, ®, address); | ||
2487 | efx_reado(efx, &buf, address); | ||
2488 | |||
2489 | if (efx_masked_compare_oword(®, &buf, &mask)) | ||
2490 | goto fail; | ||
2491 | } | ||
2492 | |||
2493 | efx_writeo(efx, &original, address); | ||
2494 | } | ||
2495 | |||
2496 | return 0; | ||
2497 | |||
2498 | fail: | ||
2499 | EFX_ERR(efx, "wrote "EFX_OWORD_FMT" read "EFX_OWORD_FMT | ||
2500 | " at address 0x%x mask "EFX_OWORD_FMT"\n", EFX_OWORD_VAL(reg), | ||
2501 | EFX_OWORD_VAL(buf), address, EFX_OWORD_VAL(mask)); | ||
2502 | return -EIO; | ||
2503 | } | ||
2504 | |||
2505 | static int falcon_b0_test_registers(struct efx_nic *efx) | 1045 | static int falcon_b0_test_registers(struct efx_nic *efx) |
2506 | { | 1046 | { |
2507 | return efx_nic_test_registers(efx, falcon_b0_register_tests, | 1047 | return efx_nic_test_registers(efx, falcon_b0_register_tests, |
@@ -2719,7 +1259,6 @@ static int falcon_spi_device_init(struct efx_nic *efx, | |||
2719 | return 0; | 1259 | return 0; |
2720 | } | 1260 | } |
2721 | 1261 | ||
2722 | |||
2723 | static void falcon_remove_spi_devices(struct efx_nic *efx) | 1262 | static void falcon_remove_spi_devices(struct efx_nic *efx) |
2724 | { | 1263 | { |
2725 | kfree(efx->spi_eeprom); | 1264 | kfree(efx->spi_eeprom); |
@@ -2789,14 +1328,6 @@ static int falcon_probe_nvconfig(struct efx_nic *efx) | |||
2789 | return rc; | 1328 | return rc; |
2790 | } | 1329 | } |
2791 | 1330 | ||
2792 | u32 efx_nic_fpga_ver(struct efx_nic *efx) | ||
2793 | { | ||
2794 | efx_oword_t altera_build; | ||
2795 | |||
2796 | efx_reado(efx, &altera_build, FR_AZ_ALTERA_BUILD); | ||
2797 | return EFX_OWORD_FIELD(altera_build, FRF_AZ_ALTERA_BUILD_VER); | ||
2798 | } | ||
2799 | |||
2800 | /* Probe all SPI devices on the NIC */ | 1331 | /* Probe all SPI devices on the NIC */ |
2801 | static void falcon_probe_spi_devices(struct efx_nic *efx) | 1332 | static void falcon_probe_spi_devices(struct efx_nic *efx) |
2802 | { | 1333 | { |
@@ -3006,73 +1537,6 @@ static void falcon_init_rx_cfg(struct efx_nic *efx) | |||
3006 | efx_writeo(efx, ®, FR_AZ_RX_CFG); | 1537 | efx_writeo(efx, ®, FR_AZ_RX_CFG); |
3007 | } | 1538 | } |
3008 | 1539 | ||
3009 | void efx_nic_init_common(struct efx_nic *efx) | ||
3010 | { | ||
3011 | efx_oword_t temp; | ||
3012 | |||
3013 | /* Set positions of descriptor caches in SRAM. */ | ||
3014 | EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_TX_DC_BASE_ADR, | ||
3015 | efx->type->tx_dc_base / 8); | ||
3016 | efx_writeo(efx, &temp, FR_AZ_SRM_TX_DC_CFG); | ||
3017 | EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_RX_DC_BASE_ADR, | ||
3018 | efx->type->rx_dc_base / 8); | ||
3019 | efx_writeo(efx, &temp, FR_AZ_SRM_RX_DC_CFG); | ||
3020 | |||
3021 | /* Set TX descriptor cache size. */ | ||
3022 | BUILD_BUG_ON(TX_DC_ENTRIES != (8 << TX_DC_ENTRIES_ORDER)); | ||
3023 | EFX_POPULATE_OWORD_1(temp, FRF_AZ_TX_DC_SIZE, TX_DC_ENTRIES_ORDER); | ||
3024 | efx_writeo(efx, &temp, FR_AZ_TX_DC_CFG); | ||
3025 | |||
3026 | /* Set RX descriptor cache size. Set low watermark to size-8, as | ||
3027 | * this allows most efficient prefetching. | ||
3028 | */ | ||
3029 | BUILD_BUG_ON(RX_DC_ENTRIES != (8 << RX_DC_ENTRIES_ORDER)); | ||
3030 | EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_SIZE, RX_DC_ENTRIES_ORDER); | ||
3031 | efx_writeo(efx, &temp, FR_AZ_RX_DC_CFG); | ||
3032 | EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_PF_LWM, RX_DC_ENTRIES - 8); | ||
3033 | efx_writeo(efx, &temp, FR_AZ_RX_DC_PF_WM); | ||
3034 | |||
3035 | /* Program INT_KER address */ | ||
3036 | EFX_POPULATE_OWORD_2(temp, | ||
3037 | FRF_AZ_NORM_INT_VEC_DIS_KER, | ||
3038 | EFX_INT_MODE_USE_MSI(efx), | ||
3039 | FRF_AZ_INT_ADR_KER, efx->irq_status.dma_addr); | ||
3040 | efx_writeo(efx, &temp, FR_AZ_INT_ADR_KER); | ||
3041 | |||
3042 | /* Enable all the genuinely fatal interrupts. (They are still | ||
3043 | * masked by the overall interrupt mask, controlled by | ||
3044 | * falcon_interrupts()). | ||
3045 | * | ||
3046 | * Note: All other fatal interrupts are enabled | ||
3047 | */ | ||
3048 | EFX_POPULATE_OWORD_3(temp, | ||
3049 | FRF_AZ_ILL_ADR_INT_KER_EN, 1, | ||
3050 | FRF_AZ_RBUF_OWN_INT_KER_EN, 1, | ||
3051 | FRF_AZ_TBUF_OWN_INT_KER_EN, 1); | ||
3052 | EFX_INVERT_OWORD(temp); | ||
3053 | efx_writeo(efx, &temp, FR_AZ_FATAL_INTR_KER); | ||
3054 | |||
3055 | efx_setup_rss_indir_table(efx); | ||
3056 | |||
3057 | /* Disable the ugly timer-based TX DMA backoff and allow TX DMA to be | ||
3058 | * controlled by the RX FIFO fill level. Set arbitration to one pkt/Q. | ||
3059 | */ | ||
3060 | efx_reado(efx, &temp, FR_AZ_TX_RESERVED); | ||
3061 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER, 0xfe); | ||
3062 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER_EN, 1); | ||
3063 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_ONE_PKT_PER_Q, 1); | ||
3064 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PUSH_EN, 0); | ||
3065 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_DIS_NON_IP_EV, 1); | ||
3066 | /* Enable SW_EV to inherit in char driver - assume harmless here */ | ||
3067 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_SOFT_EVT_EN, 1); | ||
3068 | /* Prefetch threshold 2 => fetch when descriptor cache half empty */ | ||
3069 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_THRESHOLD, 2); | ||
3070 | /* Squash TX of packets of 16 bytes or less */ | ||
3071 | if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) | ||
3072 | EFX_SET_OWORD_FIELD(temp, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); | ||
3073 | efx_writeo(efx, &temp, FR_AZ_TX_RESERVED); | ||
3074 | } | ||
3075 | |||
3076 | /* This call performs hardware-specific global initialisation, such as | 1540 | /* This call performs hardware-specific global initialisation, such as |
3077 | * defining the descriptor cache sizes and number of RSS channels. | 1541 | * defining the descriptor cache sizes and number of RSS channels. |
3078 | * It does not set up any buffers, descriptor rings or event queues. | 1542 | * It does not set up any buffers, descriptor rings or event queues. |
diff --git a/drivers/net/sfc/nic.c b/drivers/net/sfc/nic.c new file mode 100644 index 000000000000..55dbd7994b64 --- /dev/null +++ b/drivers/net/sfc/nic.c | |||
@@ -0,0 +1,1548 @@ | |||
1 | /**************************************************************************** | ||
2 | * Driver for Solarflare Solarstorm network controllers and boards | ||
3 | * Copyright 2005-2006 Fen Systems Ltd. | ||
4 | * Copyright 2006-2008 Solarflare Communications Inc. | ||
5 | * | ||
6 | * This program is free software; you can redistribute it and/or modify it | ||
7 | * under the terms of the GNU General Public License version 2 as published | ||
8 | * by the Free Software Foundation, incorporated herein by reference. | ||
9 | */ | ||
10 | |||
11 | #include <linux/bitops.h> | ||
12 | #include <linux/delay.h> | ||
13 | #include <linux/pci.h> | ||
14 | #include <linux/module.h> | ||
15 | #include <linux/seq_file.h> | ||
16 | #include "net_driver.h" | ||
17 | #include "bitfield.h" | ||
18 | #include "efx.h" | ||
19 | #include "nic.h" | ||
20 | #include "regs.h" | ||
21 | #include "io.h" | ||
22 | #include "workarounds.h" | ||
23 | |||
24 | /************************************************************************** | ||
25 | * | ||
26 | * Configurable values | ||
27 | * | ||
28 | ************************************************************************** | ||
29 | */ | ||
30 | |||
31 | /* This is set to 16 for a good reason. In summary, if larger than | ||
32 | * 16, the descriptor cache holds more than a default socket | ||
33 | * buffer's worth of packets (for UDP we can only have at most one | ||
34 | * socket buffer's worth outstanding). This combined with the fact | ||
35 | * that we only get 1 TX event per descriptor cache means the NIC | ||
36 | * goes idle. | ||
37 | */ | ||
38 | #define TX_DC_ENTRIES 16 | ||
39 | #define TX_DC_ENTRIES_ORDER 1 | ||
40 | |||
41 | #define RX_DC_ENTRIES 64 | ||
42 | #define RX_DC_ENTRIES_ORDER 3 | ||
43 | |||
44 | /* RX FIFO XOFF watermark | ||
45 | * | ||
46 | * When the amount of the RX FIFO increases used increases past this | ||
47 | * watermark send XOFF. Only used if RX flow control is enabled (ethtool -A) | ||
48 | * This also has an effect on RX/TX arbitration | ||
49 | */ | ||
50 | int efx_nic_rx_xoff_thresh = -1; | ||
51 | module_param_named(rx_xoff_thresh_bytes, efx_nic_rx_xoff_thresh, int, 0644); | ||
52 | MODULE_PARM_DESC(rx_xoff_thresh_bytes, "RX fifo XOFF threshold"); | ||
53 | |||
54 | /* RX FIFO XON watermark | ||
55 | * | ||
56 | * When the amount of the RX FIFO used decreases below this | ||
57 | * watermark send XON. Only used if TX flow control is enabled (ethtool -A) | ||
58 | * This also has an effect on RX/TX arbitration | ||
59 | */ | ||
60 | int efx_nic_rx_xon_thresh = -1; | ||
61 | module_param_named(rx_xon_thresh_bytes, efx_nic_rx_xon_thresh, int, 0644); | ||
62 | MODULE_PARM_DESC(rx_xon_thresh_bytes, "RX fifo XON threshold"); | ||
63 | |||
64 | /* If EFX_MAX_INT_ERRORS internal errors occur within | ||
65 | * EFX_INT_ERROR_EXPIRE seconds, we consider the NIC broken and | ||
66 | * disable it. | ||
67 | */ | ||
68 | #define EFX_INT_ERROR_EXPIRE 3600 | ||
69 | #define EFX_MAX_INT_ERRORS 5 | ||
70 | |||
71 | /* We poll for events every FLUSH_INTERVAL ms, and check FLUSH_POLL_COUNT times | ||
72 | */ | ||
73 | #define EFX_FLUSH_INTERVAL 10 | ||
74 | #define EFX_FLUSH_POLL_COUNT 100 | ||
75 | |||
76 | /* Size and alignment of special buffers (4KB) */ | ||
77 | #define EFX_BUF_SIZE 4096 | ||
78 | |||
79 | /* Depth of RX flush request fifo */ | ||
80 | #define EFX_RX_FLUSH_COUNT 4 | ||
81 | |||
82 | /************************************************************************** | ||
83 | * | ||
84 | * Solarstorm hardware access | ||
85 | * | ||
86 | **************************************************************************/ | ||
87 | |||
88 | static inline void efx_write_buf_tbl(struct efx_nic *efx, efx_qword_t *value, | ||
89 | unsigned int index) | ||
90 | { | ||
91 | efx_sram_writeq(efx, efx->membase + efx->type->buf_tbl_base, | ||
92 | value, index); | ||
93 | } | ||
94 | |||
95 | /* Read the current event from the event queue */ | ||
96 | static inline efx_qword_t *efx_event(struct efx_channel *channel, | ||
97 | unsigned int index) | ||
98 | { | ||
99 | return (((efx_qword_t *) (channel->eventq.addr)) + index); | ||
100 | } | ||
101 | |||
102 | /* See if an event is present | ||
103 | * | ||
104 | * We check both the high and low dword of the event for all ones. We | ||
105 | * wrote all ones when we cleared the event, and no valid event can | ||
106 | * have all ones in either its high or low dwords. This approach is | ||
107 | * robust against reordering. | ||
108 | * | ||
109 | * Note that using a single 64-bit comparison is incorrect; even | ||
110 | * though the CPU read will be atomic, the DMA write may not be. | ||
111 | */ | ||
112 | static inline int efx_event_present(efx_qword_t *event) | ||
113 | { | ||
114 | return (!(EFX_DWORD_IS_ALL_ONES(event->dword[0]) | | ||
115 | EFX_DWORD_IS_ALL_ONES(event->dword[1]))); | ||
116 | } | ||
117 | |||
118 | static bool efx_masked_compare_oword(const efx_oword_t *a, const efx_oword_t *b, | ||
119 | const efx_oword_t *mask) | ||
120 | { | ||
121 | return ((a->u64[0] ^ b->u64[0]) & mask->u64[0]) || | ||
122 | ((a->u64[1] ^ b->u64[1]) & mask->u64[1]); | ||
123 | } | ||
124 | |||
125 | int efx_nic_test_registers(struct efx_nic *efx, | ||
126 | const struct efx_nic_register_test *regs, | ||
127 | size_t n_regs) | ||
128 | { | ||
129 | unsigned address = 0, i, j; | ||
130 | efx_oword_t mask, imask, original, reg, buf; | ||
131 | |||
132 | /* Falcon should be in loopback to isolate the XMAC from the PHY */ | ||
133 | WARN_ON(!LOOPBACK_INTERNAL(efx)); | ||
134 | |||
135 | for (i = 0; i < n_regs; ++i) { | ||
136 | address = regs[i].address; | ||
137 | mask = imask = regs[i].mask; | ||
138 | EFX_INVERT_OWORD(imask); | ||
139 | |||
140 | efx_reado(efx, &original, address); | ||
141 | |||
142 | /* bit sweep on and off */ | ||
143 | for (j = 0; j < 128; j++) { | ||
144 | if (!EFX_EXTRACT_OWORD32(mask, j, j)) | ||
145 | continue; | ||
146 | |||
147 | /* Test this testable bit can be set in isolation */ | ||
148 | EFX_AND_OWORD(reg, original, mask); | ||
149 | EFX_SET_OWORD32(reg, j, j, 1); | ||
150 | |||
151 | efx_writeo(efx, ®, address); | ||
152 | efx_reado(efx, &buf, address); | ||
153 | |||
154 | if (efx_masked_compare_oword(®, &buf, &mask)) | ||
155 | goto fail; | ||
156 | |||
157 | /* Test this testable bit can be cleared in isolation */ | ||
158 | EFX_OR_OWORD(reg, original, mask); | ||
159 | EFX_SET_OWORD32(reg, j, j, 0); | ||
160 | |||
161 | efx_writeo(efx, ®, address); | ||
162 | efx_reado(efx, &buf, address); | ||
163 | |||
164 | if (efx_masked_compare_oword(®, &buf, &mask)) | ||
165 | goto fail; | ||
166 | } | ||
167 | |||
168 | efx_writeo(efx, &original, address); | ||
169 | } | ||
170 | |||
171 | return 0; | ||
172 | |||
173 | fail: | ||
174 | EFX_ERR(efx, "wrote "EFX_OWORD_FMT" read "EFX_OWORD_FMT | ||
175 | " at address 0x%x mask "EFX_OWORD_FMT"\n", EFX_OWORD_VAL(reg), | ||
176 | EFX_OWORD_VAL(buf), address, EFX_OWORD_VAL(mask)); | ||
177 | return -EIO; | ||
178 | } | ||
179 | |||
180 | /************************************************************************** | ||
181 | * | ||
182 | * Special buffer handling | ||
183 | * Special buffers are used for event queues and the TX and RX | ||
184 | * descriptor rings. | ||
185 | * | ||
186 | *************************************************************************/ | ||
187 | |||
188 | /* | ||
189 | * Initialise a special buffer | ||
190 | * | ||
191 | * This will define a buffer (previously allocated via | ||
192 | * efx_alloc_special_buffer()) in the buffer table, allowing | ||
193 | * it to be used for event queues, descriptor rings etc. | ||
194 | */ | ||
195 | static void | ||
196 | efx_init_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer) | ||
197 | { | ||
198 | efx_qword_t buf_desc; | ||
199 | int index; | ||
200 | dma_addr_t dma_addr; | ||
201 | int i; | ||
202 | |||
203 | EFX_BUG_ON_PARANOID(!buffer->addr); | ||
204 | |||
205 | /* Write buffer descriptors to NIC */ | ||
206 | for (i = 0; i < buffer->entries; i++) { | ||
207 | index = buffer->index + i; | ||
208 | dma_addr = buffer->dma_addr + (i * 4096); | ||
209 | EFX_LOG(efx, "mapping special buffer %d at %llx\n", | ||
210 | index, (unsigned long long)dma_addr); | ||
211 | EFX_POPULATE_QWORD_3(buf_desc, | ||
212 | FRF_AZ_BUF_ADR_REGION, 0, | ||
213 | FRF_AZ_BUF_ADR_FBUF, dma_addr >> 12, | ||
214 | FRF_AZ_BUF_OWNER_ID_FBUF, 0); | ||
215 | efx_write_buf_tbl(efx, &buf_desc, index); | ||
216 | } | ||
217 | } | ||
218 | |||
219 | /* Unmaps a buffer and clears the buffer table entries */ | ||
220 | static void | ||
221 | efx_fini_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer) | ||
222 | { | ||
223 | efx_oword_t buf_tbl_upd; | ||
224 | unsigned int start = buffer->index; | ||
225 | unsigned int end = (buffer->index + buffer->entries - 1); | ||
226 | |||
227 | if (!buffer->entries) | ||
228 | return; | ||
229 | |||
230 | EFX_LOG(efx, "unmapping special buffers %d-%d\n", | ||
231 | buffer->index, buffer->index + buffer->entries - 1); | ||
232 | |||
233 | EFX_POPULATE_OWORD_4(buf_tbl_upd, | ||
234 | FRF_AZ_BUF_UPD_CMD, 0, | ||
235 | FRF_AZ_BUF_CLR_CMD, 1, | ||
236 | FRF_AZ_BUF_CLR_END_ID, end, | ||
237 | FRF_AZ_BUF_CLR_START_ID, start); | ||
238 | efx_writeo(efx, &buf_tbl_upd, FR_AZ_BUF_TBL_UPD); | ||
239 | } | ||
240 | |||
241 | /* | ||
242 | * Allocate a new special buffer | ||
243 | * | ||
244 | * This allocates memory for a new buffer, clears it and allocates a | ||
245 | * new buffer ID range. It does not write into the buffer table. | ||
246 | * | ||
247 | * This call will allocate 4KB buffers, since 8KB buffers can't be | ||
248 | * used for event queues and descriptor rings. | ||
249 | */ | ||
250 | static int efx_alloc_special_buffer(struct efx_nic *efx, | ||
251 | struct efx_special_buffer *buffer, | ||
252 | unsigned int len) | ||
253 | { | ||
254 | len = ALIGN(len, EFX_BUF_SIZE); | ||
255 | |||
256 | buffer->addr = pci_alloc_consistent(efx->pci_dev, len, | ||
257 | &buffer->dma_addr); | ||
258 | if (!buffer->addr) | ||
259 | return -ENOMEM; | ||
260 | buffer->len = len; | ||
261 | buffer->entries = len / EFX_BUF_SIZE; | ||
262 | BUG_ON(buffer->dma_addr & (EFX_BUF_SIZE - 1)); | ||
263 | |||
264 | /* All zeros is a potentially valid event so memset to 0xff */ | ||
265 | memset(buffer->addr, 0xff, len); | ||
266 | |||
267 | /* Select new buffer ID */ | ||
268 | buffer->index = efx->next_buffer_table; | ||
269 | efx->next_buffer_table += buffer->entries; | ||
270 | |||
271 | EFX_LOG(efx, "allocating special buffers %d-%d at %llx+%x " | ||
272 | "(virt %p phys %llx)\n", buffer->index, | ||
273 | buffer->index + buffer->entries - 1, | ||
274 | (u64)buffer->dma_addr, len, | ||
275 | buffer->addr, (u64)virt_to_phys(buffer->addr)); | ||
276 | |||
277 | return 0; | ||
278 | } | ||
279 | |||
280 | static void | ||
281 | efx_free_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer) | ||
282 | { | ||
283 | if (!buffer->addr) | ||
284 | return; | ||
285 | |||
286 | EFX_LOG(efx, "deallocating special buffers %d-%d at %llx+%x " | ||
287 | "(virt %p phys %llx)\n", buffer->index, | ||
288 | buffer->index + buffer->entries - 1, | ||
289 | (u64)buffer->dma_addr, buffer->len, | ||
290 | buffer->addr, (u64)virt_to_phys(buffer->addr)); | ||
291 | |||
292 | pci_free_consistent(efx->pci_dev, buffer->len, buffer->addr, | ||
293 | buffer->dma_addr); | ||
294 | buffer->addr = NULL; | ||
295 | buffer->entries = 0; | ||
296 | } | ||
297 | |||
298 | /************************************************************************** | ||
299 | * | ||
300 | * Generic buffer handling | ||
301 | * These buffers are used for interrupt status and MAC stats | ||
302 | * | ||
303 | **************************************************************************/ | ||
304 | |||
305 | int efx_nic_alloc_buffer(struct efx_nic *efx, struct efx_buffer *buffer, | ||
306 | unsigned int len) | ||
307 | { | ||
308 | buffer->addr = pci_alloc_consistent(efx->pci_dev, len, | ||
309 | &buffer->dma_addr); | ||
310 | if (!buffer->addr) | ||
311 | return -ENOMEM; | ||
312 | buffer->len = len; | ||
313 | memset(buffer->addr, 0, len); | ||
314 | return 0; | ||
315 | } | ||
316 | |||
317 | void efx_nic_free_buffer(struct efx_nic *efx, struct efx_buffer *buffer) | ||
318 | { | ||
319 | if (buffer->addr) { | ||
320 | pci_free_consistent(efx->pci_dev, buffer->len, | ||
321 | buffer->addr, buffer->dma_addr); | ||
322 | buffer->addr = NULL; | ||
323 | } | ||
324 | } | ||
325 | |||
326 | /************************************************************************** | ||
327 | * | ||
328 | * TX path | ||
329 | * | ||
330 | **************************************************************************/ | ||
331 | |||
332 | /* Returns a pointer to the specified transmit descriptor in the TX | ||
333 | * descriptor queue belonging to the specified channel. | ||
334 | */ | ||
335 | static inline efx_qword_t * | ||
336 | efx_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index) | ||
337 | { | ||
338 | return (((efx_qword_t *) (tx_queue->txd.addr)) + index); | ||
339 | } | ||
340 | |||
341 | /* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */ | ||
342 | static inline void efx_notify_tx_desc(struct efx_tx_queue *tx_queue) | ||
343 | { | ||
344 | unsigned write_ptr; | ||
345 | efx_dword_t reg; | ||
346 | |||
347 | write_ptr = tx_queue->write_count & EFX_TXQ_MASK; | ||
348 | EFX_POPULATE_DWORD_1(reg, FRF_AZ_TX_DESC_WPTR_DWORD, write_ptr); | ||
349 | efx_writed_page(tx_queue->efx, ®, | ||
350 | FR_AZ_TX_DESC_UPD_DWORD_P0, tx_queue->queue); | ||
351 | } | ||
352 | |||
353 | |||
354 | /* For each entry inserted into the software descriptor ring, create a | ||
355 | * descriptor in the hardware TX descriptor ring (in host memory), and | ||
356 | * write a doorbell. | ||
357 | */ | ||
358 | void efx_nic_push_buffers(struct efx_tx_queue *tx_queue) | ||
359 | { | ||
360 | |||
361 | struct efx_tx_buffer *buffer; | ||
362 | efx_qword_t *txd; | ||
363 | unsigned write_ptr; | ||
364 | |||
365 | BUG_ON(tx_queue->write_count == tx_queue->insert_count); | ||
366 | |||
367 | do { | ||
368 | write_ptr = tx_queue->write_count & EFX_TXQ_MASK; | ||
369 | buffer = &tx_queue->buffer[write_ptr]; | ||
370 | txd = efx_tx_desc(tx_queue, write_ptr); | ||
371 | ++tx_queue->write_count; | ||
372 | |||
373 | /* Create TX descriptor ring entry */ | ||
374 | EFX_POPULATE_QWORD_4(*txd, | ||
375 | FSF_AZ_TX_KER_CONT, buffer->continuation, | ||
376 | FSF_AZ_TX_KER_BYTE_COUNT, buffer->len, | ||
377 | FSF_AZ_TX_KER_BUF_REGION, 0, | ||
378 | FSF_AZ_TX_KER_BUF_ADDR, buffer->dma_addr); | ||
379 | } while (tx_queue->write_count != tx_queue->insert_count); | ||
380 | |||
381 | wmb(); /* Ensure descriptors are written before they are fetched */ | ||
382 | efx_notify_tx_desc(tx_queue); | ||
383 | } | ||
384 | |||
385 | /* Allocate hardware resources for a TX queue */ | ||
386 | int efx_nic_probe_tx(struct efx_tx_queue *tx_queue) | ||
387 | { | ||
388 | struct efx_nic *efx = tx_queue->efx; | ||
389 | BUILD_BUG_ON(EFX_TXQ_SIZE < 512 || EFX_TXQ_SIZE > 4096 || | ||
390 | EFX_TXQ_SIZE & EFX_TXQ_MASK); | ||
391 | return efx_alloc_special_buffer(efx, &tx_queue->txd, | ||
392 | EFX_TXQ_SIZE * sizeof(efx_qword_t)); | ||
393 | } | ||
394 | |||
395 | void efx_nic_init_tx(struct efx_tx_queue *tx_queue) | ||
396 | { | ||
397 | efx_oword_t tx_desc_ptr; | ||
398 | struct efx_nic *efx = tx_queue->efx; | ||
399 | |||
400 | tx_queue->flushed = FLUSH_NONE; | ||
401 | |||
402 | /* Pin TX descriptor ring */ | ||
403 | efx_init_special_buffer(efx, &tx_queue->txd); | ||
404 | |||
405 | /* Push TX descriptor ring to card */ | ||
406 | EFX_POPULATE_OWORD_10(tx_desc_ptr, | ||
407 | FRF_AZ_TX_DESCQ_EN, 1, | ||
408 | FRF_AZ_TX_ISCSI_DDIG_EN, 0, | ||
409 | FRF_AZ_TX_ISCSI_HDIG_EN, 0, | ||
410 | FRF_AZ_TX_DESCQ_BUF_BASE_ID, tx_queue->txd.index, | ||
411 | FRF_AZ_TX_DESCQ_EVQ_ID, | ||
412 | tx_queue->channel->channel, | ||
413 | FRF_AZ_TX_DESCQ_OWNER_ID, 0, | ||
414 | FRF_AZ_TX_DESCQ_LABEL, tx_queue->queue, | ||
415 | FRF_AZ_TX_DESCQ_SIZE, | ||
416 | __ffs(tx_queue->txd.entries), | ||
417 | FRF_AZ_TX_DESCQ_TYPE, 0, | ||
418 | FRF_BZ_TX_NON_IP_DROP_DIS, 1); | ||
419 | |||
420 | if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { | ||
421 | int csum = tx_queue->queue == EFX_TX_QUEUE_OFFLOAD_CSUM; | ||
422 | EFX_SET_OWORD_FIELD(tx_desc_ptr, FRF_BZ_TX_IP_CHKSM_DIS, !csum); | ||
423 | EFX_SET_OWORD_FIELD(tx_desc_ptr, FRF_BZ_TX_TCP_CHKSM_DIS, | ||
424 | !csum); | ||
425 | } | ||
426 | |||
427 | efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base, | ||
428 | tx_queue->queue); | ||
429 | |||
430 | if (efx_nic_rev(efx) < EFX_REV_FALCON_B0) { | ||
431 | efx_oword_t reg; | ||
432 | |||
433 | /* Only 128 bits in this register */ | ||
434 | BUILD_BUG_ON(EFX_TX_QUEUE_COUNT >= 128); | ||
435 | |||
436 | efx_reado(efx, ®, FR_AA_TX_CHKSM_CFG); | ||
437 | if (tx_queue->queue == EFX_TX_QUEUE_OFFLOAD_CSUM) | ||
438 | clear_bit_le(tx_queue->queue, (void *)®); | ||
439 | else | ||
440 | set_bit_le(tx_queue->queue, (void *)®); | ||
441 | efx_writeo(efx, ®, FR_AA_TX_CHKSM_CFG); | ||
442 | } | ||
443 | } | ||
444 | |||
445 | static void efx_flush_tx_queue(struct efx_tx_queue *tx_queue) | ||
446 | { | ||
447 | struct efx_nic *efx = tx_queue->efx; | ||
448 | efx_oword_t tx_flush_descq; | ||
449 | |||
450 | tx_queue->flushed = FLUSH_PENDING; | ||
451 | |||
452 | /* Post a flush command */ | ||
453 | EFX_POPULATE_OWORD_2(tx_flush_descq, | ||
454 | FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, | ||
455 | FRF_AZ_TX_FLUSH_DESCQ, tx_queue->queue); | ||
456 | efx_writeo(efx, &tx_flush_descq, FR_AZ_TX_FLUSH_DESCQ); | ||
457 | } | ||
458 | |||
459 | void efx_nic_fini_tx(struct efx_tx_queue *tx_queue) | ||
460 | { | ||
461 | struct efx_nic *efx = tx_queue->efx; | ||
462 | efx_oword_t tx_desc_ptr; | ||
463 | |||
464 | /* The queue should have been flushed */ | ||
465 | WARN_ON(tx_queue->flushed != FLUSH_DONE); | ||
466 | |||
467 | /* Remove TX descriptor ring from card */ | ||
468 | EFX_ZERO_OWORD(tx_desc_ptr); | ||
469 | efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base, | ||
470 | tx_queue->queue); | ||
471 | |||
472 | /* Unpin TX descriptor ring */ | ||
473 | efx_fini_special_buffer(efx, &tx_queue->txd); | ||
474 | } | ||
475 | |||
476 | /* Free buffers backing TX queue */ | ||
477 | void efx_nic_remove_tx(struct efx_tx_queue *tx_queue) | ||
478 | { | ||
479 | efx_free_special_buffer(tx_queue->efx, &tx_queue->txd); | ||
480 | } | ||
481 | |||
482 | /************************************************************************** | ||
483 | * | ||
484 | * RX path | ||
485 | * | ||
486 | **************************************************************************/ | ||
487 | |||
488 | /* Returns a pointer to the specified descriptor in the RX descriptor queue */ | ||
489 | static inline efx_qword_t * | ||
490 | efx_rx_desc(struct efx_rx_queue *rx_queue, unsigned int index) | ||
491 | { | ||
492 | return (((efx_qword_t *) (rx_queue->rxd.addr)) + index); | ||
493 | } | ||
494 | |||
495 | /* This creates an entry in the RX descriptor queue */ | ||
496 | static inline void | ||
497 | efx_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned index) | ||
498 | { | ||
499 | struct efx_rx_buffer *rx_buf; | ||
500 | efx_qword_t *rxd; | ||
501 | |||
502 | rxd = efx_rx_desc(rx_queue, index); | ||
503 | rx_buf = efx_rx_buffer(rx_queue, index); | ||
504 | EFX_POPULATE_QWORD_3(*rxd, | ||
505 | FSF_AZ_RX_KER_BUF_SIZE, | ||
506 | rx_buf->len - | ||
507 | rx_queue->efx->type->rx_buffer_padding, | ||
508 | FSF_AZ_RX_KER_BUF_REGION, 0, | ||
509 | FSF_AZ_RX_KER_BUF_ADDR, rx_buf->dma_addr); | ||
510 | } | ||
511 | |||
512 | /* This writes to the RX_DESC_WPTR register for the specified receive | ||
513 | * descriptor ring. | ||
514 | */ | ||
515 | void efx_nic_notify_rx_desc(struct efx_rx_queue *rx_queue) | ||
516 | { | ||
517 | efx_dword_t reg; | ||
518 | unsigned write_ptr; | ||
519 | |||
520 | while (rx_queue->notified_count != rx_queue->added_count) { | ||
521 | efx_build_rx_desc(rx_queue, | ||
522 | rx_queue->notified_count & | ||
523 | EFX_RXQ_MASK); | ||
524 | ++rx_queue->notified_count; | ||
525 | } | ||
526 | |||
527 | wmb(); | ||
528 | write_ptr = rx_queue->added_count & EFX_RXQ_MASK; | ||
529 | EFX_POPULATE_DWORD_1(reg, FRF_AZ_RX_DESC_WPTR_DWORD, write_ptr); | ||
530 | efx_writed_page(rx_queue->efx, ®, | ||
531 | FR_AZ_RX_DESC_UPD_DWORD_P0, rx_queue->queue); | ||
532 | } | ||
533 | |||
534 | int efx_nic_probe_rx(struct efx_rx_queue *rx_queue) | ||
535 | { | ||
536 | struct efx_nic *efx = rx_queue->efx; | ||
537 | BUILD_BUG_ON(EFX_RXQ_SIZE < 512 || EFX_RXQ_SIZE > 4096 || | ||
538 | EFX_RXQ_SIZE & EFX_RXQ_MASK); | ||
539 | return efx_alloc_special_buffer(efx, &rx_queue->rxd, | ||
540 | EFX_RXQ_SIZE * sizeof(efx_qword_t)); | ||
541 | } | ||
542 | |||
543 | void efx_nic_init_rx(struct efx_rx_queue *rx_queue) | ||
544 | { | ||
545 | efx_oword_t rx_desc_ptr; | ||
546 | struct efx_nic *efx = rx_queue->efx; | ||
547 | bool is_b0 = efx_nic_rev(efx) >= EFX_REV_FALCON_B0; | ||
548 | bool iscsi_digest_en = is_b0; | ||
549 | |||
550 | EFX_LOG(efx, "RX queue %d ring in special buffers %d-%d\n", | ||
551 | rx_queue->queue, rx_queue->rxd.index, | ||
552 | rx_queue->rxd.index + rx_queue->rxd.entries - 1); | ||
553 | |||
554 | rx_queue->flushed = FLUSH_NONE; | ||
555 | |||
556 | /* Pin RX descriptor ring */ | ||
557 | efx_init_special_buffer(efx, &rx_queue->rxd); | ||
558 | |||
559 | /* Push RX descriptor ring to card */ | ||
560 | EFX_POPULATE_OWORD_10(rx_desc_ptr, | ||
561 | FRF_AZ_RX_ISCSI_DDIG_EN, iscsi_digest_en, | ||
562 | FRF_AZ_RX_ISCSI_HDIG_EN, iscsi_digest_en, | ||
563 | FRF_AZ_RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index, | ||
564 | FRF_AZ_RX_DESCQ_EVQ_ID, | ||
565 | rx_queue->channel->channel, | ||
566 | FRF_AZ_RX_DESCQ_OWNER_ID, 0, | ||
567 | FRF_AZ_RX_DESCQ_LABEL, rx_queue->queue, | ||
568 | FRF_AZ_RX_DESCQ_SIZE, | ||
569 | __ffs(rx_queue->rxd.entries), | ||
570 | FRF_AZ_RX_DESCQ_TYPE, 0 /* kernel queue */ , | ||
571 | /* For >=B0 this is scatter so disable */ | ||
572 | FRF_AZ_RX_DESCQ_JUMBO, !is_b0, | ||
573 | FRF_AZ_RX_DESCQ_EN, 1); | ||
574 | efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base, | ||
575 | rx_queue->queue); | ||
576 | } | ||
577 | |||
578 | static void efx_flush_rx_queue(struct efx_rx_queue *rx_queue) | ||
579 | { | ||
580 | struct efx_nic *efx = rx_queue->efx; | ||
581 | efx_oword_t rx_flush_descq; | ||
582 | |||
583 | rx_queue->flushed = FLUSH_PENDING; | ||
584 | |||
585 | /* Post a flush command */ | ||
586 | EFX_POPULATE_OWORD_2(rx_flush_descq, | ||
587 | FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, | ||
588 | FRF_AZ_RX_FLUSH_DESCQ, rx_queue->queue); | ||
589 | efx_writeo(efx, &rx_flush_descq, FR_AZ_RX_FLUSH_DESCQ); | ||
590 | } | ||
591 | |||
592 | void efx_nic_fini_rx(struct efx_rx_queue *rx_queue) | ||
593 | { | ||
594 | efx_oword_t rx_desc_ptr; | ||
595 | struct efx_nic *efx = rx_queue->efx; | ||
596 | |||
597 | /* The queue should already have been flushed */ | ||
598 | WARN_ON(rx_queue->flushed != FLUSH_DONE); | ||
599 | |||
600 | /* Remove RX descriptor ring from card */ | ||
601 | EFX_ZERO_OWORD(rx_desc_ptr); | ||
602 | efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base, | ||
603 | rx_queue->queue); | ||
604 | |||
605 | /* Unpin RX descriptor ring */ | ||
606 | efx_fini_special_buffer(efx, &rx_queue->rxd); | ||
607 | } | ||
608 | |||
609 | /* Free buffers backing RX queue */ | ||
610 | void efx_nic_remove_rx(struct efx_rx_queue *rx_queue) | ||
611 | { | ||
612 | efx_free_special_buffer(rx_queue->efx, &rx_queue->rxd); | ||
613 | } | ||
614 | |||
615 | /************************************************************************** | ||
616 | * | ||
617 | * Event queue processing | ||
618 | * Event queues are processed by per-channel tasklets. | ||
619 | * | ||
620 | **************************************************************************/ | ||
621 | |||
622 | /* Update a channel's event queue's read pointer (RPTR) register | ||
623 | * | ||
624 | * This writes the EVQ_RPTR_REG register for the specified channel's | ||
625 | * event queue. | ||
626 | * | ||
627 | * Note that EVQ_RPTR_REG contains the index of the "last read" event, | ||
628 | * whereas channel->eventq_read_ptr contains the index of the "next to | ||
629 | * read" event. | ||
630 | */ | ||
631 | void efx_nic_eventq_read_ack(struct efx_channel *channel) | ||
632 | { | ||
633 | efx_dword_t reg; | ||
634 | struct efx_nic *efx = channel->efx; | ||
635 | |||
636 | EFX_POPULATE_DWORD_1(reg, FRF_AZ_EVQ_RPTR, channel->eventq_read_ptr); | ||
637 | efx_writed_table(efx, ®, efx->type->evq_rptr_tbl_base, | ||
638 | channel->channel); | ||
639 | } | ||
640 | |||
641 | /* Use HW to insert a SW defined event */ | ||
642 | void efx_generate_event(struct efx_channel *channel, efx_qword_t *event) | ||
643 | { | ||
644 | efx_oword_t drv_ev_reg; | ||
645 | |||
646 | BUILD_BUG_ON(FRF_AZ_DRV_EV_DATA_LBN != 0 || | ||
647 | FRF_AZ_DRV_EV_DATA_WIDTH != 64); | ||
648 | drv_ev_reg.u32[0] = event->u32[0]; | ||
649 | drv_ev_reg.u32[1] = event->u32[1]; | ||
650 | drv_ev_reg.u32[2] = 0; | ||
651 | drv_ev_reg.u32[3] = 0; | ||
652 | EFX_SET_OWORD_FIELD(drv_ev_reg, FRF_AZ_DRV_EV_QID, channel->channel); | ||
653 | efx_writeo(channel->efx, &drv_ev_reg, FR_AZ_DRV_EV); | ||
654 | } | ||
655 | |||
656 | /* Handle a transmit completion event | ||
657 | * | ||
658 | * The NIC batches TX completion events; the message we receive is of | ||
659 | * the form "complete all TX events up to this index". | ||
660 | */ | ||
661 | static void | ||
662 | efx_handle_tx_event(struct efx_channel *channel, efx_qword_t *event) | ||
663 | { | ||
664 | unsigned int tx_ev_desc_ptr; | ||
665 | unsigned int tx_ev_q_label; | ||
666 | struct efx_tx_queue *tx_queue; | ||
667 | struct efx_nic *efx = channel->efx; | ||
668 | |||
669 | if (likely(EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_COMP))) { | ||
670 | /* Transmit completion */ | ||
671 | tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_DESC_PTR); | ||
672 | tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL); | ||
673 | tx_queue = &efx->tx_queue[tx_ev_q_label]; | ||
674 | channel->irq_mod_score += | ||
675 | (tx_ev_desc_ptr - tx_queue->read_count) & | ||
676 | EFX_TXQ_MASK; | ||
677 | efx_xmit_done(tx_queue, tx_ev_desc_ptr); | ||
678 | } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) { | ||
679 | /* Rewrite the FIFO write pointer */ | ||
680 | tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL); | ||
681 | tx_queue = &efx->tx_queue[tx_ev_q_label]; | ||
682 | |||
683 | if (efx_dev_registered(efx)) | ||
684 | netif_tx_lock(efx->net_dev); | ||
685 | efx_notify_tx_desc(tx_queue); | ||
686 | if (efx_dev_registered(efx)) | ||
687 | netif_tx_unlock(efx->net_dev); | ||
688 | } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_PKT_ERR) && | ||
689 | EFX_WORKAROUND_10727(efx)) { | ||
690 | efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH); | ||
691 | } else { | ||
692 | EFX_ERR(efx, "channel %d unexpected TX event " | ||
693 | EFX_QWORD_FMT"\n", channel->channel, | ||
694 | EFX_QWORD_VAL(*event)); | ||
695 | } | ||
696 | } | ||
697 | |||
698 | /* Detect errors included in the rx_evt_pkt_ok bit. */ | ||
699 | static void efx_handle_rx_not_ok(struct efx_rx_queue *rx_queue, | ||
700 | const efx_qword_t *event, | ||
701 | bool *rx_ev_pkt_ok, | ||
702 | bool *discard) | ||
703 | { | ||
704 | struct efx_nic *efx = rx_queue->efx; | ||
705 | bool rx_ev_buf_owner_id_err, rx_ev_ip_hdr_chksum_err; | ||
706 | bool rx_ev_tcp_udp_chksum_err, rx_ev_eth_crc_err; | ||
707 | bool rx_ev_frm_trunc, rx_ev_drib_nib, rx_ev_tobe_disc; | ||
708 | bool rx_ev_other_err, rx_ev_pause_frm; | ||
709 | bool rx_ev_hdr_type, rx_ev_mcast_pkt; | ||
710 | unsigned rx_ev_pkt_type; | ||
711 | |||
712 | rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE); | ||
713 | rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT); | ||
714 | rx_ev_tobe_disc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_TOBE_DISC); | ||
715 | rx_ev_pkt_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_TYPE); | ||
716 | rx_ev_buf_owner_id_err = EFX_QWORD_FIELD(*event, | ||
717 | FSF_AZ_RX_EV_BUF_OWNER_ID_ERR); | ||
718 | rx_ev_ip_hdr_chksum_err = EFX_QWORD_FIELD(*event, | ||
719 | FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR); | ||
720 | rx_ev_tcp_udp_chksum_err = EFX_QWORD_FIELD(*event, | ||
721 | FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR); | ||
722 | rx_ev_eth_crc_err = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_ETH_CRC_ERR); | ||
723 | rx_ev_frm_trunc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_FRM_TRUNC); | ||
724 | rx_ev_drib_nib = ((efx_nic_rev(efx) >= EFX_REV_FALCON_B0) ? | ||
725 | 0 : EFX_QWORD_FIELD(*event, FSF_AA_RX_EV_DRIB_NIB)); | ||
726 | rx_ev_pause_frm = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PAUSE_FRM_ERR); | ||
727 | |||
728 | /* Every error apart from tobe_disc and pause_frm */ | ||
729 | rx_ev_other_err = (rx_ev_drib_nib | rx_ev_tcp_udp_chksum_err | | ||
730 | rx_ev_buf_owner_id_err | rx_ev_eth_crc_err | | ||
731 | rx_ev_frm_trunc | rx_ev_ip_hdr_chksum_err); | ||
732 | |||
733 | /* Count errors that are not in MAC stats. Ignore expected | ||
734 | * checksum errors during self-test. */ | ||
735 | if (rx_ev_frm_trunc) | ||
736 | ++rx_queue->channel->n_rx_frm_trunc; | ||
737 | else if (rx_ev_tobe_disc) | ||
738 | ++rx_queue->channel->n_rx_tobe_disc; | ||
739 | else if (!efx->loopback_selftest) { | ||
740 | if (rx_ev_ip_hdr_chksum_err) | ||
741 | ++rx_queue->channel->n_rx_ip_hdr_chksum_err; | ||
742 | else if (rx_ev_tcp_udp_chksum_err) | ||
743 | ++rx_queue->channel->n_rx_tcp_udp_chksum_err; | ||
744 | } | ||
745 | |||
746 | /* The frame must be discarded if any of these are true. */ | ||
747 | *discard = (rx_ev_eth_crc_err | rx_ev_frm_trunc | rx_ev_drib_nib | | ||
748 | rx_ev_tobe_disc | rx_ev_pause_frm); | ||
749 | |||
750 | /* TOBE_DISC is expected on unicast mismatches; don't print out an | ||
751 | * error message. FRM_TRUNC indicates RXDP dropped the packet due | ||
752 | * to a FIFO overflow. | ||
753 | */ | ||
754 | #ifdef EFX_ENABLE_DEBUG | ||
755 | if (rx_ev_other_err) { | ||
756 | EFX_INFO_RL(efx, " RX queue %d unexpected RX event " | ||
757 | EFX_QWORD_FMT "%s%s%s%s%s%s%s%s\n", | ||
758 | rx_queue->queue, EFX_QWORD_VAL(*event), | ||
759 | rx_ev_buf_owner_id_err ? " [OWNER_ID_ERR]" : "", | ||
760 | rx_ev_ip_hdr_chksum_err ? | ||
761 | " [IP_HDR_CHKSUM_ERR]" : "", | ||
762 | rx_ev_tcp_udp_chksum_err ? | ||
763 | " [TCP_UDP_CHKSUM_ERR]" : "", | ||
764 | rx_ev_eth_crc_err ? " [ETH_CRC_ERR]" : "", | ||
765 | rx_ev_frm_trunc ? " [FRM_TRUNC]" : "", | ||
766 | rx_ev_drib_nib ? " [DRIB_NIB]" : "", | ||
767 | rx_ev_tobe_disc ? " [TOBE_DISC]" : "", | ||
768 | rx_ev_pause_frm ? " [PAUSE]" : ""); | ||
769 | } | ||
770 | #endif | ||
771 | } | ||
772 | |||
773 | /* Handle receive events that are not in-order. */ | ||
774 | static void | ||
775 | efx_handle_rx_bad_index(struct efx_rx_queue *rx_queue, unsigned index) | ||
776 | { | ||
777 | struct efx_nic *efx = rx_queue->efx; | ||
778 | unsigned expected, dropped; | ||
779 | |||
780 | expected = rx_queue->removed_count & EFX_RXQ_MASK; | ||
781 | dropped = (index - expected) & EFX_RXQ_MASK; | ||
782 | EFX_INFO(efx, "dropped %d events (index=%d expected=%d)\n", | ||
783 | dropped, index, expected); | ||
784 | |||
785 | efx_schedule_reset(efx, EFX_WORKAROUND_5676(efx) ? | ||
786 | RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE); | ||
787 | } | ||
788 | |||
789 | /* Handle a packet received event | ||
790 | * | ||
791 | * The NIC gives a "discard" flag if it's a unicast packet with the | ||
792 | * wrong destination address | ||
793 | * Also "is multicast" and "matches multicast filter" flags can be used to | ||
794 | * discard non-matching multicast packets. | ||
795 | */ | ||
796 | static void | ||
797 | efx_handle_rx_event(struct efx_channel *channel, const efx_qword_t *event) | ||
798 | { | ||
799 | unsigned int rx_ev_desc_ptr, rx_ev_byte_cnt; | ||
800 | unsigned int rx_ev_hdr_type, rx_ev_mcast_pkt; | ||
801 | unsigned expected_ptr; | ||
802 | bool rx_ev_pkt_ok, discard = false, checksummed; | ||
803 | struct efx_rx_queue *rx_queue; | ||
804 | struct efx_nic *efx = channel->efx; | ||
805 | |||
806 | /* Basic packet information */ | ||
807 | rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_BYTE_CNT); | ||
808 | rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_OK); | ||
809 | rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE); | ||
810 | WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_JUMBO_CONT)); | ||
811 | WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_SOP) != 1); | ||
812 | WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_Q_LABEL) != | ||
813 | channel->channel); | ||
814 | |||
815 | rx_queue = &efx->rx_queue[channel->channel]; | ||
816 | |||
817 | rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_DESC_PTR); | ||
818 | expected_ptr = rx_queue->removed_count & EFX_RXQ_MASK; | ||
819 | if (unlikely(rx_ev_desc_ptr != expected_ptr)) | ||
820 | efx_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr); | ||
821 | |||
822 | if (likely(rx_ev_pkt_ok)) { | ||
823 | /* If packet is marked as OK and packet type is TCP/IP or | ||
824 | * UDP/IP, then we can rely on the hardware checksum. | ||
825 | */ | ||
826 | checksummed = | ||
827 | likely(efx->rx_checksum_enabled) && | ||
828 | (rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_TCP || | ||
829 | rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_UDP); | ||
830 | } else { | ||
831 | efx_handle_rx_not_ok(rx_queue, event, &rx_ev_pkt_ok, &discard); | ||
832 | checksummed = false; | ||
833 | } | ||
834 | |||
835 | /* Detect multicast packets that didn't match the filter */ | ||
836 | rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT); | ||
837 | if (rx_ev_mcast_pkt) { | ||
838 | unsigned int rx_ev_mcast_hash_match = | ||
839 | EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_HASH_MATCH); | ||
840 | |||
841 | if (unlikely(!rx_ev_mcast_hash_match)) { | ||
842 | ++channel->n_rx_mcast_mismatch; | ||
843 | discard = true; | ||
844 | } | ||
845 | } | ||
846 | |||
847 | channel->irq_mod_score += 2; | ||
848 | |||
849 | /* Handle received packet */ | ||
850 | efx_rx_packet(rx_queue, rx_ev_desc_ptr, rx_ev_byte_cnt, | ||
851 | checksummed, discard); | ||
852 | } | ||
853 | |||
854 | /* Global events are basically PHY events */ | ||
855 | static void | ||
856 | efx_handle_global_event(struct efx_channel *channel, efx_qword_t *event) | ||
857 | { | ||
858 | struct efx_nic *efx = channel->efx; | ||
859 | bool handled = false; | ||
860 | |||
861 | if (EFX_QWORD_FIELD(*event, FSF_AB_GLB_EV_G_PHY0_INTR) || | ||
862 | EFX_QWORD_FIELD(*event, FSF_AB_GLB_EV_XG_PHY0_INTR) || | ||
863 | EFX_QWORD_FIELD(*event, FSF_AB_GLB_EV_XFP_PHY0_INTR)) { | ||
864 | /* Ignored */ | ||
865 | handled = true; | ||
866 | } | ||
867 | |||
868 | if ((efx_nic_rev(efx) >= EFX_REV_FALCON_B0) && | ||
869 | EFX_QWORD_FIELD(*event, FSF_BB_GLB_EV_XG_MGT_INTR)) { | ||
870 | efx->xmac_poll_required = true; | ||
871 | handled = true; | ||
872 | } | ||
873 | |||
874 | if (efx_nic_rev(efx) <= EFX_REV_FALCON_A1 ? | ||
875 | EFX_QWORD_FIELD(*event, FSF_AA_GLB_EV_RX_RECOVERY) : | ||
876 | EFX_QWORD_FIELD(*event, FSF_BB_GLB_EV_RX_RECOVERY)) { | ||
877 | EFX_ERR(efx, "channel %d seen global RX_RESET " | ||
878 | "event. Resetting.\n", channel->channel); | ||
879 | |||
880 | atomic_inc(&efx->rx_reset); | ||
881 | efx_schedule_reset(efx, EFX_WORKAROUND_6555(efx) ? | ||
882 | RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE); | ||
883 | handled = true; | ||
884 | } | ||
885 | |||
886 | if (!handled) | ||
887 | EFX_ERR(efx, "channel %d unknown global event " | ||
888 | EFX_QWORD_FMT "\n", channel->channel, | ||
889 | EFX_QWORD_VAL(*event)); | ||
890 | } | ||
891 | |||
892 | static void | ||
893 | efx_handle_driver_event(struct efx_channel *channel, efx_qword_t *event) | ||
894 | { | ||
895 | struct efx_nic *efx = channel->efx; | ||
896 | unsigned int ev_sub_code; | ||
897 | unsigned int ev_sub_data; | ||
898 | |||
899 | ev_sub_code = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBCODE); | ||
900 | ev_sub_data = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA); | ||
901 | |||
902 | switch (ev_sub_code) { | ||
903 | case FSE_AZ_TX_DESCQ_FLS_DONE_EV: | ||
904 | EFX_TRACE(efx, "channel %d TXQ %d flushed\n", | ||
905 | channel->channel, ev_sub_data); | ||
906 | break; | ||
907 | case FSE_AZ_RX_DESCQ_FLS_DONE_EV: | ||
908 | EFX_TRACE(efx, "channel %d RXQ %d flushed\n", | ||
909 | channel->channel, ev_sub_data); | ||
910 | break; | ||
911 | case FSE_AZ_EVQ_INIT_DONE_EV: | ||
912 | EFX_LOG(efx, "channel %d EVQ %d initialised\n", | ||
913 | channel->channel, ev_sub_data); | ||
914 | break; | ||
915 | case FSE_AZ_SRM_UPD_DONE_EV: | ||
916 | EFX_TRACE(efx, "channel %d SRAM update done\n", | ||
917 | channel->channel); | ||
918 | break; | ||
919 | case FSE_AZ_WAKE_UP_EV: | ||
920 | EFX_TRACE(efx, "channel %d RXQ %d wakeup event\n", | ||
921 | channel->channel, ev_sub_data); | ||
922 | break; | ||
923 | case FSE_AZ_TIMER_EV: | ||
924 | EFX_TRACE(efx, "channel %d RX queue %d timer expired\n", | ||
925 | channel->channel, ev_sub_data); | ||
926 | break; | ||
927 | case FSE_AA_RX_RECOVER_EV: | ||
928 | EFX_ERR(efx, "channel %d seen DRIVER RX_RESET event. " | ||
929 | "Resetting.\n", channel->channel); | ||
930 | atomic_inc(&efx->rx_reset); | ||
931 | efx_schedule_reset(efx, | ||
932 | EFX_WORKAROUND_6555(efx) ? | ||
933 | RESET_TYPE_RX_RECOVERY : | ||
934 | RESET_TYPE_DISABLE); | ||
935 | break; | ||
936 | case FSE_BZ_RX_DSC_ERROR_EV: | ||
937 | EFX_ERR(efx, "RX DMA Q %d reports descriptor fetch error." | ||
938 | " RX Q %d is disabled.\n", ev_sub_data, ev_sub_data); | ||
939 | efx_schedule_reset(efx, RESET_TYPE_RX_DESC_FETCH); | ||
940 | break; | ||
941 | case FSE_BZ_TX_DSC_ERROR_EV: | ||
942 | EFX_ERR(efx, "TX DMA Q %d reports descriptor fetch error." | ||
943 | " TX Q %d is disabled.\n", ev_sub_data, ev_sub_data); | ||
944 | efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH); | ||
945 | break; | ||
946 | default: | ||
947 | EFX_TRACE(efx, "channel %d unknown driver event code %d " | ||
948 | "data %04x\n", channel->channel, ev_sub_code, | ||
949 | ev_sub_data); | ||
950 | break; | ||
951 | } | ||
952 | } | ||
953 | |||
954 | int efx_nic_process_eventq(struct efx_channel *channel, int rx_quota) | ||
955 | { | ||
956 | unsigned int read_ptr; | ||
957 | efx_qword_t event, *p_event; | ||
958 | int ev_code; | ||
959 | int rx_packets = 0; | ||
960 | |||
961 | read_ptr = channel->eventq_read_ptr; | ||
962 | |||
963 | do { | ||
964 | p_event = efx_event(channel, read_ptr); | ||
965 | event = *p_event; | ||
966 | |||
967 | if (!efx_event_present(&event)) | ||
968 | /* End of events */ | ||
969 | break; | ||
970 | |||
971 | EFX_TRACE(channel->efx, "channel %d event is "EFX_QWORD_FMT"\n", | ||
972 | channel->channel, EFX_QWORD_VAL(event)); | ||
973 | |||
974 | /* Clear this event by marking it all ones */ | ||
975 | EFX_SET_QWORD(*p_event); | ||
976 | |||
977 | ev_code = EFX_QWORD_FIELD(event, FSF_AZ_EV_CODE); | ||
978 | |||
979 | switch (ev_code) { | ||
980 | case FSE_AZ_EV_CODE_RX_EV: | ||
981 | efx_handle_rx_event(channel, &event); | ||
982 | ++rx_packets; | ||
983 | break; | ||
984 | case FSE_AZ_EV_CODE_TX_EV: | ||
985 | efx_handle_tx_event(channel, &event); | ||
986 | break; | ||
987 | case FSE_AZ_EV_CODE_DRV_GEN_EV: | ||
988 | channel->eventq_magic = EFX_QWORD_FIELD( | ||
989 | event, FSF_AZ_DRV_GEN_EV_MAGIC); | ||
990 | EFX_LOG(channel->efx, "channel %d received generated " | ||
991 | "event "EFX_QWORD_FMT"\n", channel->channel, | ||
992 | EFX_QWORD_VAL(event)); | ||
993 | break; | ||
994 | case FSE_AZ_EV_CODE_GLOBAL_EV: | ||
995 | efx_handle_global_event(channel, &event); | ||
996 | break; | ||
997 | case FSE_AZ_EV_CODE_DRIVER_EV: | ||
998 | efx_handle_driver_event(channel, &event); | ||
999 | break; | ||
1000 | default: | ||
1001 | EFX_ERR(channel->efx, "channel %d unknown event type %d" | ||
1002 | " (data " EFX_QWORD_FMT ")\n", channel->channel, | ||
1003 | ev_code, EFX_QWORD_VAL(event)); | ||
1004 | } | ||
1005 | |||
1006 | /* Increment read pointer */ | ||
1007 | read_ptr = (read_ptr + 1) & EFX_EVQ_MASK; | ||
1008 | |||
1009 | } while (rx_packets < rx_quota); | ||
1010 | |||
1011 | channel->eventq_read_ptr = read_ptr; | ||
1012 | return rx_packets; | ||
1013 | } | ||
1014 | |||
1015 | |||
1016 | /* Allocate buffer table entries for event queue */ | ||
1017 | int efx_nic_probe_eventq(struct efx_channel *channel) | ||
1018 | { | ||
1019 | struct efx_nic *efx = channel->efx; | ||
1020 | BUILD_BUG_ON(EFX_EVQ_SIZE < 512 || EFX_EVQ_SIZE > 32768 || | ||
1021 | EFX_EVQ_SIZE & EFX_EVQ_MASK); | ||
1022 | return efx_alloc_special_buffer(efx, &channel->eventq, | ||
1023 | EFX_EVQ_SIZE * sizeof(efx_qword_t)); | ||
1024 | } | ||
1025 | |||
1026 | void efx_nic_init_eventq(struct efx_channel *channel) | ||
1027 | { | ||
1028 | efx_oword_t evq_ptr; | ||
1029 | struct efx_nic *efx = channel->efx; | ||
1030 | |||
1031 | EFX_LOG(efx, "channel %d event queue in special buffers %d-%d\n", | ||
1032 | channel->channel, channel->eventq.index, | ||
1033 | channel->eventq.index + channel->eventq.entries - 1); | ||
1034 | |||
1035 | /* Pin event queue buffer */ | ||
1036 | efx_init_special_buffer(efx, &channel->eventq); | ||
1037 | |||
1038 | /* Fill event queue with all ones (i.e. empty events) */ | ||
1039 | memset(channel->eventq.addr, 0xff, channel->eventq.len); | ||
1040 | |||
1041 | /* Push event queue to card */ | ||
1042 | EFX_POPULATE_OWORD_3(evq_ptr, | ||
1043 | FRF_AZ_EVQ_EN, 1, | ||
1044 | FRF_AZ_EVQ_SIZE, __ffs(channel->eventq.entries), | ||
1045 | FRF_AZ_EVQ_BUF_BASE_ID, channel->eventq.index); | ||
1046 | efx_writeo_table(efx, &evq_ptr, efx->type->evq_ptr_tbl_base, | ||
1047 | channel->channel); | ||
1048 | |||
1049 | efx->type->push_irq_moderation(channel); | ||
1050 | } | ||
1051 | |||
1052 | void efx_nic_fini_eventq(struct efx_channel *channel) | ||
1053 | { | ||
1054 | efx_oword_t eventq_ptr; | ||
1055 | struct efx_nic *efx = channel->efx; | ||
1056 | |||
1057 | /* Remove event queue from card */ | ||
1058 | EFX_ZERO_OWORD(eventq_ptr); | ||
1059 | efx_writeo_table(efx, &eventq_ptr, efx->type->evq_ptr_tbl_base, | ||
1060 | channel->channel); | ||
1061 | |||
1062 | /* Unpin event queue */ | ||
1063 | efx_fini_special_buffer(efx, &channel->eventq); | ||
1064 | } | ||
1065 | |||
1066 | /* Free buffers backing event queue */ | ||
1067 | void efx_nic_remove_eventq(struct efx_channel *channel) | ||
1068 | { | ||
1069 | efx_free_special_buffer(channel->efx, &channel->eventq); | ||
1070 | } | ||
1071 | |||
1072 | |||
1073 | /* Generates a test event on the event queue. A subsequent call to | ||
1074 | * process_eventq() should pick up the event and place the value of | ||
1075 | * "magic" into channel->eventq_magic; | ||
1076 | */ | ||
1077 | void efx_nic_generate_test_event(struct efx_channel *channel, unsigned int magic) | ||
1078 | { | ||
1079 | efx_qword_t test_event; | ||
1080 | |||
1081 | EFX_POPULATE_QWORD_2(test_event, FSF_AZ_EV_CODE, | ||
1082 | FSE_AZ_EV_CODE_DRV_GEN_EV, | ||
1083 | FSF_AZ_DRV_GEN_EV_MAGIC, magic); | ||
1084 | efx_generate_event(channel, &test_event); | ||
1085 | } | ||
1086 | |||
1087 | /************************************************************************** | ||
1088 | * | ||
1089 | * Flush handling | ||
1090 | * | ||
1091 | **************************************************************************/ | ||
1092 | |||
1093 | |||
1094 | static void efx_poll_flush_events(struct efx_nic *efx) | ||
1095 | { | ||
1096 | struct efx_channel *channel = &efx->channel[0]; | ||
1097 | struct efx_tx_queue *tx_queue; | ||
1098 | struct efx_rx_queue *rx_queue; | ||
1099 | unsigned int read_ptr = channel->eventq_read_ptr; | ||
1100 | unsigned int end_ptr = (read_ptr - 1) & EFX_EVQ_MASK; | ||
1101 | |||
1102 | do { | ||
1103 | efx_qword_t *event = efx_event(channel, read_ptr); | ||
1104 | int ev_code, ev_sub_code, ev_queue; | ||
1105 | bool ev_failed; | ||
1106 | |||
1107 | if (!efx_event_present(event)) | ||
1108 | break; | ||
1109 | |||
1110 | ev_code = EFX_QWORD_FIELD(*event, FSF_AZ_EV_CODE); | ||
1111 | ev_sub_code = EFX_QWORD_FIELD(*event, | ||
1112 | FSF_AZ_DRIVER_EV_SUBCODE); | ||
1113 | if (ev_code == FSE_AZ_EV_CODE_DRIVER_EV && | ||
1114 | ev_sub_code == FSE_AZ_TX_DESCQ_FLS_DONE_EV) { | ||
1115 | ev_queue = EFX_QWORD_FIELD(*event, | ||
1116 | FSF_AZ_DRIVER_EV_SUBDATA); | ||
1117 | if (ev_queue < EFX_TX_QUEUE_COUNT) { | ||
1118 | tx_queue = efx->tx_queue + ev_queue; | ||
1119 | tx_queue->flushed = FLUSH_DONE; | ||
1120 | } | ||
1121 | } else if (ev_code == FSE_AZ_EV_CODE_DRIVER_EV && | ||
1122 | ev_sub_code == FSE_AZ_RX_DESCQ_FLS_DONE_EV) { | ||
1123 | ev_queue = EFX_QWORD_FIELD( | ||
1124 | *event, FSF_AZ_DRIVER_EV_RX_DESCQ_ID); | ||
1125 | ev_failed = EFX_QWORD_FIELD( | ||
1126 | *event, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL); | ||
1127 | if (ev_queue < efx->n_rx_queues) { | ||
1128 | rx_queue = efx->rx_queue + ev_queue; | ||
1129 | rx_queue->flushed = | ||
1130 | ev_failed ? FLUSH_FAILED : FLUSH_DONE; | ||
1131 | } | ||
1132 | } | ||
1133 | |||
1134 | /* We're about to destroy the queue anyway, so | ||
1135 | * it's ok to throw away every non-flush event */ | ||
1136 | EFX_SET_QWORD(*event); | ||
1137 | |||
1138 | read_ptr = (read_ptr + 1) & EFX_EVQ_MASK; | ||
1139 | } while (read_ptr != end_ptr); | ||
1140 | |||
1141 | channel->eventq_read_ptr = read_ptr; | ||
1142 | } | ||
1143 | |||
1144 | /* Handle tx and rx flushes at the same time, since they run in | ||
1145 | * parallel in the hardware and there's no reason for us to | ||
1146 | * serialise them */ | ||
1147 | int efx_nic_flush_queues(struct efx_nic *efx) | ||
1148 | { | ||
1149 | struct efx_rx_queue *rx_queue; | ||
1150 | struct efx_tx_queue *tx_queue; | ||
1151 | int i, tx_pending, rx_pending; | ||
1152 | |||
1153 | /* If necessary prepare the hardware for flushing */ | ||
1154 | efx->type->prepare_flush(efx); | ||
1155 | |||
1156 | /* Flush all tx queues in parallel */ | ||
1157 | efx_for_each_tx_queue(tx_queue, efx) | ||
1158 | efx_flush_tx_queue(tx_queue); | ||
1159 | |||
1160 | /* The hardware supports four concurrent rx flushes, each of which may | ||
1161 | * need to be retried if there is an outstanding descriptor fetch */ | ||
1162 | for (i = 0; i < EFX_FLUSH_POLL_COUNT; ++i) { | ||
1163 | rx_pending = tx_pending = 0; | ||
1164 | efx_for_each_rx_queue(rx_queue, efx) { | ||
1165 | if (rx_queue->flushed == FLUSH_PENDING) | ||
1166 | ++rx_pending; | ||
1167 | } | ||
1168 | efx_for_each_rx_queue(rx_queue, efx) { | ||
1169 | if (rx_pending == EFX_RX_FLUSH_COUNT) | ||
1170 | break; | ||
1171 | if (rx_queue->flushed == FLUSH_FAILED || | ||
1172 | rx_queue->flushed == FLUSH_NONE) { | ||
1173 | efx_flush_rx_queue(rx_queue); | ||
1174 | ++rx_pending; | ||
1175 | } | ||
1176 | } | ||
1177 | efx_for_each_tx_queue(tx_queue, efx) { | ||
1178 | if (tx_queue->flushed != FLUSH_DONE) | ||
1179 | ++tx_pending; | ||
1180 | } | ||
1181 | |||
1182 | if (rx_pending == 0 && tx_pending == 0) | ||
1183 | return 0; | ||
1184 | |||
1185 | msleep(EFX_FLUSH_INTERVAL); | ||
1186 | efx_poll_flush_events(efx); | ||
1187 | } | ||
1188 | |||
1189 | /* Mark the queues as all flushed. We're going to return failure | ||
1190 | * leading to a reset, or fake up success anyway */ | ||
1191 | efx_for_each_tx_queue(tx_queue, efx) { | ||
1192 | if (tx_queue->flushed != FLUSH_DONE) | ||
1193 | EFX_ERR(efx, "tx queue %d flush command timed out\n", | ||
1194 | tx_queue->queue); | ||
1195 | tx_queue->flushed = FLUSH_DONE; | ||
1196 | } | ||
1197 | efx_for_each_rx_queue(rx_queue, efx) { | ||
1198 | if (rx_queue->flushed != FLUSH_DONE) | ||
1199 | EFX_ERR(efx, "rx queue %d flush command timed out\n", | ||
1200 | rx_queue->queue); | ||
1201 | rx_queue->flushed = FLUSH_DONE; | ||
1202 | } | ||
1203 | |||
1204 | if (EFX_WORKAROUND_7803(efx)) | ||
1205 | return 0; | ||
1206 | |||
1207 | return -ETIMEDOUT; | ||
1208 | } | ||
1209 | |||
1210 | /************************************************************************** | ||
1211 | * | ||
1212 | * Hardware interrupts | ||
1213 | * The hardware interrupt handler does very little work; all the event | ||
1214 | * queue processing is carried out by per-channel tasklets. | ||
1215 | * | ||
1216 | **************************************************************************/ | ||
1217 | |||
1218 | /* Enable/disable/generate interrupts */ | ||
1219 | static inline void efx_nic_interrupts(struct efx_nic *efx, | ||
1220 | bool enabled, bool force) | ||
1221 | { | ||
1222 | efx_oword_t int_en_reg_ker; | ||
1223 | |||
1224 | EFX_POPULATE_OWORD_2(int_en_reg_ker, | ||
1225 | FRF_AZ_KER_INT_KER, force, | ||
1226 | FRF_AZ_DRV_INT_EN_KER, enabled); | ||
1227 | efx_writeo(efx, &int_en_reg_ker, FR_AZ_INT_EN_KER); | ||
1228 | } | ||
1229 | |||
1230 | void efx_nic_enable_interrupts(struct efx_nic *efx) | ||
1231 | { | ||
1232 | struct efx_channel *channel; | ||
1233 | |||
1234 | EFX_ZERO_OWORD(*((efx_oword_t *) efx->irq_status.addr)); | ||
1235 | wmb(); /* Ensure interrupt vector is clear before interrupts enabled */ | ||
1236 | |||
1237 | /* Enable interrupts */ | ||
1238 | efx_nic_interrupts(efx, true, false); | ||
1239 | |||
1240 | /* Force processing of all the channels to get the EVQ RPTRs up to | ||
1241 | date */ | ||
1242 | efx_for_each_channel(channel, efx) | ||
1243 | efx_schedule_channel(channel); | ||
1244 | } | ||
1245 | |||
1246 | void efx_nic_disable_interrupts(struct efx_nic *efx) | ||
1247 | { | ||
1248 | /* Disable interrupts */ | ||
1249 | efx_nic_interrupts(efx, false, false); | ||
1250 | } | ||
1251 | |||
1252 | /* Generate a test interrupt | ||
1253 | * Interrupt must already have been enabled, otherwise nasty things | ||
1254 | * may happen. | ||
1255 | */ | ||
1256 | void efx_nic_generate_interrupt(struct efx_nic *efx) | ||
1257 | { | ||
1258 | efx_nic_interrupts(efx, true, true); | ||
1259 | } | ||
1260 | |||
1261 | /* Process a fatal interrupt | ||
1262 | * Disable bus mastering ASAP and schedule a reset | ||
1263 | */ | ||
1264 | irqreturn_t efx_nic_fatal_interrupt(struct efx_nic *efx) | ||
1265 | { | ||
1266 | struct falcon_nic_data *nic_data = efx->nic_data; | ||
1267 | efx_oword_t *int_ker = efx->irq_status.addr; | ||
1268 | efx_oword_t fatal_intr; | ||
1269 | int error, mem_perr; | ||
1270 | |||
1271 | efx_reado(efx, &fatal_intr, FR_AZ_FATAL_INTR_KER); | ||
1272 | error = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_FATAL_INTR); | ||
1273 | |||
1274 | EFX_ERR(efx, "SYSTEM ERROR " EFX_OWORD_FMT " status " | ||
1275 | EFX_OWORD_FMT ": %s\n", EFX_OWORD_VAL(*int_ker), | ||
1276 | EFX_OWORD_VAL(fatal_intr), | ||
1277 | error ? "disabling bus mastering" : "no recognised error"); | ||
1278 | if (error == 0) | ||
1279 | goto out; | ||
1280 | |||
1281 | /* If this is a memory parity error dump which blocks are offending */ | ||
1282 | mem_perr = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_MEM_PERR_INT_KER); | ||
1283 | if (mem_perr) { | ||
1284 | efx_oword_t reg; | ||
1285 | efx_reado(efx, ®, FR_AZ_MEM_STAT); | ||
1286 | EFX_ERR(efx, "SYSTEM ERROR: memory parity error " | ||
1287 | EFX_OWORD_FMT "\n", EFX_OWORD_VAL(reg)); | ||
1288 | } | ||
1289 | |||
1290 | /* Disable both devices */ | ||
1291 | pci_clear_master(efx->pci_dev); | ||
1292 | if (efx_nic_is_dual_func(efx)) | ||
1293 | pci_clear_master(nic_data->pci_dev2); | ||
1294 | efx_nic_disable_interrupts(efx); | ||
1295 | |||
1296 | /* Count errors and reset or disable the NIC accordingly */ | ||
1297 | if (efx->int_error_count == 0 || | ||
1298 | time_after(jiffies, efx->int_error_expire)) { | ||
1299 | efx->int_error_count = 0; | ||
1300 | efx->int_error_expire = | ||
1301 | jiffies + EFX_INT_ERROR_EXPIRE * HZ; | ||
1302 | } | ||
1303 | if (++efx->int_error_count < EFX_MAX_INT_ERRORS) { | ||
1304 | EFX_ERR(efx, "SYSTEM ERROR - reset scheduled\n"); | ||
1305 | efx_schedule_reset(efx, RESET_TYPE_INT_ERROR); | ||
1306 | } else { | ||
1307 | EFX_ERR(efx, "SYSTEM ERROR - max number of errors seen." | ||
1308 | "NIC will be disabled\n"); | ||
1309 | efx_schedule_reset(efx, RESET_TYPE_DISABLE); | ||
1310 | } | ||
1311 | out: | ||
1312 | return IRQ_HANDLED; | ||
1313 | } | ||
1314 | |||
1315 | /* Handle a legacy interrupt | ||
1316 | * Acknowledges the interrupt and schedule event queue processing. | ||
1317 | */ | ||
1318 | static irqreturn_t efx_legacy_interrupt(int irq, void *dev_id) | ||
1319 | { | ||
1320 | struct efx_nic *efx = dev_id; | ||
1321 | efx_oword_t *int_ker = efx->irq_status.addr; | ||
1322 | irqreturn_t result = IRQ_NONE; | ||
1323 | struct efx_channel *channel; | ||
1324 | efx_dword_t reg; | ||
1325 | u32 queues; | ||
1326 | int syserr; | ||
1327 | |||
1328 | /* Read the ISR which also ACKs the interrupts */ | ||
1329 | efx_readd(efx, ®, FR_BZ_INT_ISR0); | ||
1330 | queues = EFX_EXTRACT_DWORD(reg, 0, 31); | ||
1331 | |||
1332 | /* Check to see if we have a serious error condition */ | ||
1333 | syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT); | ||
1334 | if (unlikely(syserr)) | ||
1335 | return efx_nic_fatal_interrupt(efx); | ||
1336 | |||
1337 | /* Schedule processing of any interrupting queues */ | ||
1338 | efx_for_each_channel(channel, efx) { | ||
1339 | if ((queues & 1) || | ||
1340 | efx_event_present( | ||
1341 | efx_event(channel, channel->eventq_read_ptr))) { | ||
1342 | efx_schedule_channel(channel); | ||
1343 | result = IRQ_HANDLED; | ||
1344 | } | ||
1345 | queues >>= 1; | ||
1346 | } | ||
1347 | |||
1348 | if (result == IRQ_HANDLED) { | ||
1349 | efx->last_irq_cpu = raw_smp_processor_id(); | ||
1350 | EFX_TRACE(efx, "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n", | ||
1351 | irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg)); | ||
1352 | } | ||
1353 | |||
1354 | return result; | ||
1355 | } | ||
1356 | |||
1357 | /* Handle an MSI interrupt | ||
1358 | * | ||
1359 | * Handle an MSI hardware interrupt. This routine schedules event | ||
1360 | * queue processing. No interrupt acknowledgement cycle is necessary. | ||
1361 | * Also, we never need to check that the interrupt is for us, since | ||
1362 | * MSI interrupts cannot be shared. | ||
1363 | */ | ||
1364 | static irqreturn_t efx_msi_interrupt(int irq, void *dev_id) | ||
1365 | { | ||
1366 | struct efx_channel *channel = dev_id; | ||
1367 | struct efx_nic *efx = channel->efx; | ||
1368 | efx_oword_t *int_ker = efx->irq_status.addr; | ||
1369 | int syserr; | ||
1370 | |||
1371 | efx->last_irq_cpu = raw_smp_processor_id(); | ||
1372 | EFX_TRACE(efx, "IRQ %d on CPU %d status " EFX_OWORD_FMT "\n", | ||
1373 | irq, raw_smp_processor_id(), EFX_OWORD_VAL(*int_ker)); | ||
1374 | |||
1375 | /* Check to see if we have a serious error condition */ | ||
1376 | syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT); | ||
1377 | if (unlikely(syserr)) | ||
1378 | return efx_nic_fatal_interrupt(efx); | ||
1379 | |||
1380 | /* Schedule processing of the channel */ | ||
1381 | efx_schedule_channel(channel); | ||
1382 | |||
1383 | return IRQ_HANDLED; | ||
1384 | } | ||
1385 | |||
1386 | |||
1387 | /* Setup RSS indirection table. | ||
1388 | * This maps from the hash value of the packet to RXQ | ||
1389 | */ | ||
1390 | static void efx_setup_rss_indir_table(struct efx_nic *efx) | ||
1391 | { | ||
1392 | int i = 0; | ||
1393 | unsigned long offset; | ||
1394 | efx_dword_t dword; | ||
1395 | |||
1396 | if (efx_nic_rev(efx) < EFX_REV_FALCON_B0) | ||
1397 | return; | ||
1398 | |||
1399 | for (offset = FR_BZ_RX_INDIRECTION_TBL; | ||
1400 | offset < FR_BZ_RX_INDIRECTION_TBL + 0x800; | ||
1401 | offset += 0x10) { | ||
1402 | EFX_POPULATE_DWORD_1(dword, FRF_BZ_IT_QUEUE, | ||
1403 | i % efx->n_rx_queues); | ||
1404 | efx_writed(efx, &dword, offset); | ||
1405 | i++; | ||
1406 | } | ||
1407 | } | ||
1408 | |||
1409 | /* Hook interrupt handler(s) | ||
1410 | * Try MSI and then legacy interrupts. | ||
1411 | */ | ||
1412 | int efx_nic_init_interrupt(struct efx_nic *efx) | ||
1413 | { | ||
1414 | struct efx_channel *channel; | ||
1415 | int rc; | ||
1416 | |||
1417 | if (!EFX_INT_MODE_USE_MSI(efx)) { | ||
1418 | irq_handler_t handler; | ||
1419 | if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) | ||
1420 | handler = efx_legacy_interrupt; | ||
1421 | else | ||
1422 | handler = falcon_legacy_interrupt_a1; | ||
1423 | |||
1424 | rc = request_irq(efx->legacy_irq, handler, IRQF_SHARED, | ||
1425 | efx->name, efx); | ||
1426 | if (rc) { | ||
1427 | EFX_ERR(efx, "failed to hook legacy IRQ %d\n", | ||
1428 | efx->pci_dev->irq); | ||
1429 | goto fail1; | ||
1430 | } | ||
1431 | return 0; | ||
1432 | } | ||
1433 | |||
1434 | /* Hook MSI or MSI-X interrupt */ | ||
1435 | efx_for_each_channel(channel, efx) { | ||
1436 | rc = request_irq(channel->irq, efx_msi_interrupt, | ||
1437 | IRQF_PROBE_SHARED, /* Not shared */ | ||
1438 | channel->name, channel); | ||
1439 | if (rc) { | ||
1440 | EFX_ERR(efx, "failed to hook IRQ %d\n", channel->irq); | ||
1441 | goto fail2; | ||
1442 | } | ||
1443 | } | ||
1444 | |||
1445 | return 0; | ||
1446 | |||
1447 | fail2: | ||
1448 | efx_for_each_channel(channel, efx) | ||
1449 | free_irq(channel->irq, channel); | ||
1450 | fail1: | ||
1451 | return rc; | ||
1452 | } | ||
1453 | |||
1454 | void efx_nic_fini_interrupt(struct efx_nic *efx) | ||
1455 | { | ||
1456 | struct efx_channel *channel; | ||
1457 | efx_oword_t reg; | ||
1458 | |||
1459 | /* Disable MSI/MSI-X interrupts */ | ||
1460 | efx_for_each_channel(channel, efx) { | ||
1461 | if (channel->irq) | ||
1462 | free_irq(channel->irq, channel); | ||
1463 | } | ||
1464 | |||
1465 | /* ACK legacy interrupt */ | ||
1466 | if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) | ||
1467 | efx_reado(efx, ®, FR_BZ_INT_ISR0); | ||
1468 | else | ||
1469 | falcon_irq_ack_a1(efx); | ||
1470 | |||
1471 | /* Disable legacy interrupt */ | ||
1472 | if (efx->legacy_irq) | ||
1473 | free_irq(efx->legacy_irq, efx); | ||
1474 | } | ||
1475 | |||
1476 | u32 efx_nic_fpga_ver(struct efx_nic *efx) | ||
1477 | { | ||
1478 | efx_oword_t altera_build; | ||
1479 | efx_reado(efx, &altera_build, FR_AZ_ALTERA_BUILD); | ||
1480 | return EFX_OWORD_FIELD(altera_build, FRF_AZ_ALTERA_BUILD_VER); | ||
1481 | } | ||
1482 | |||
1483 | void efx_nic_init_common(struct efx_nic *efx) | ||
1484 | { | ||
1485 | efx_oword_t temp; | ||
1486 | |||
1487 | /* Set positions of descriptor caches in SRAM. */ | ||
1488 | EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_TX_DC_BASE_ADR, | ||
1489 | efx->type->tx_dc_base / 8); | ||
1490 | efx_writeo(efx, &temp, FR_AZ_SRM_TX_DC_CFG); | ||
1491 | EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_RX_DC_BASE_ADR, | ||
1492 | efx->type->rx_dc_base / 8); | ||
1493 | efx_writeo(efx, &temp, FR_AZ_SRM_RX_DC_CFG); | ||
1494 | |||
1495 | /* Set TX descriptor cache size. */ | ||
1496 | BUILD_BUG_ON(TX_DC_ENTRIES != (8 << TX_DC_ENTRIES_ORDER)); | ||
1497 | EFX_POPULATE_OWORD_1(temp, FRF_AZ_TX_DC_SIZE, TX_DC_ENTRIES_ORDER); | ||
1498 | efx_writeo(efx, &temp, FR_AZ_TX_DC_CFG); | ||
1499 | |||
1500 | /* Set RX descriptor cache size. Set low watermark to size-8, as | ||
1501 | * this allows most efficient prefetching. | ||
1502 | */ | ||
1503 | BUILD_BUG_ON(RX_DC_ENTRIES != (8 << RX_DC_ENTRIES_ORDER)); | ||
1504 | EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_SIZE, RX_DC_ENTRIES_ORDER); | ||
1505 | efx_writeo(efx, &temp, FR_AZ_RX_DC_CFG); | ||
1506 | EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_PF_LWM, RX_DC_ENTRIES - 8); | ||
1507 | efx_writeo(efx, &temp, FR_AZ_RX_DC_PF_WM); | ||
1508 | |||
1509 | /* Program INT_KER address */ | ||
1510 | EFX_POPULATE_OWORD_2(temp, | ||
1511 | FRF_AZ_NORM_INT_VEC_DIS_KER, | ||
1512 | EFX_INT_MODE_USE_MSI(efx), | ||
1513 | FRF_AZ_INT_ADR_KER, efx->irq_status.dma_addr); | ||
1514 | efx_writeo(efx, &temp, FR_AZ_INT_ADR_KER); | ||
1515 | |||
1516 | /* Enable all the genuinely fatal interrupts. (They are still | ||
1517 | * masked by the overall interrupt mask, controlled by | ||
1518 | * falcon_interrupts()). | ||
1519 | * | ||
1520 | * Note: All other fatal interrupts are enabled | ||
1521 | */ | ||
1522 | EFX_POPULATE_OWORD_3(temp, | ||
1523 | FRF_AZ_ILL_ADR_INT_KER_EN, 1, | ||
1524 | FRF_AZ_RBUF_OWN_INT_KER_EN, 1, | ||
1525 | FRF_AZ_TBUF_OWN_INT_KER_EN, 1); | ||
1526 | EFX_INVERT_OWORD(temp); | ||
1527 | efx_writeo(efx, &temp, FR_AZ_FATAL_INTR_KER); | ||
1528 | |||
1529 | efx_setup_rss_indir_table(efx); | ||
1530 | |||
1531 | /* Disable the ugly timer-based TX DMA backoff and allow TX DMA to be | ||
1532 | * controlled by the RX FIFO fill level. Set arbitration to one pkt/Q. | ||
1533 | */ | ||
1534 | efx_reado(efx, &temp, FR_AZ_TX_RESERVED); | ||
1535 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER, 0xfe); | ||
1536 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER_EN, 1); | ||
1537 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_ONE_PKT_PER_Q, 1); | ||
1538 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PUSH_EN, 0); | ||
1539 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_DIS_NON_IP_EV, 1); | ||
1540 | /* Enable SW_EV to inherit in char driver - assume harmless here */ | ||
1541 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_SOFT_EVT_EN, 1); | ||
1542 | /* Prefetch threshold 2 => fetch when descriptor cache half empty */ | ||
1543 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_THRESHOLD, 2); | ||
1544 | /* Squash TX of packets of 16 bytes or less */ | ||
1545 | if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) | ||
1546 | EFX_SET_OWORD_FIELD(temp, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); | ||
1547 | efx_writeo(efx, &temp, FR_AZ_TX_RESERVED); | ||
1548 | } | ||