aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/net/ethernet/freescale/Kconfig2
-rw-r--r--drivers/net/ethernet/freescale/Makefile1
-rw-r--r--drivers/net/ethernet/freescale/dpaa/Kconfig10
-rw-r--r--drivers/net/ethernet/freescale/dpaa/Makefile11
-rw-r--r--drivers/net/ethernet/freescale/dpaa/dpaa_eth.c2682
-rw-r--r--drivers/net/ethernet/freescale/dpaa/dpaa_eth.h144
6 files changed, 2850 insertions, 0 deletions
diff --git a/drivers/net/ethernet/freescale/Kconfig b/drivers/net/ethernet/freescale/Kconfig
index d1ca45fbb164..aa3f615886b4 100644
--- a/drivers/net/ethernet/freescale/Kconfig
+++ b/drivers/net/ethernet/freescale/Kconfig
@@ -93,4 +93,6 @@ config GIANFAR
93 and MPC86xx family of chips, the eTSEC on LS1021A and the FEC 93 and MPC86xx family of chips, the eTSEC on LS1021A and the FEC
94 on the 8540. 94 on the 8540.
95 95
96source "drivers/net/ethernet/freescale/dpaa/Kconfig"
97
96endif # NET_VENDOR_FREESCALE 98endif # NET_VENDOR_FREESCALE
diff --git a/drivers/net/ethernet/freescale/Makefile b/drivers/net/ethernet/freescale/Makefile
index cbe21dc7e37e..4a13115155c9 100644
--- a/drivers/net/ethernet/freescale/Makefile
+++ b/drivers/net/ethernet/freescale/Makefile
@@ -22,3 +22,4 @@ obj-$(CONFIG_UCC_GETH) += ucc_geth_driver.o
22ucc_geth_driver-objs := ucc_geth.o ucc_geth_ethtool.o 22ucc_geth_driver-objs := ucc_geth.o ucc_geth_ethtool.o
23 23
24obj-$(CONFIG_FSL_FMAN) += fman/ 24obj-$(CONFIG_FSL_FMAN) += fman/
25obj-$(CONFIG_FSL_DPAA_ETH) += dpaa/
diff --git a/drivers/net/ethernet/freescale/dpaa/Kconfig b/drivers/net/ethernet/freescale/dpaa/Kconfig
new file mode 100644
index 000000000000..f3a3454805f9
--- /dev/null
+++ b/drivers/net/ethernet/freescale/dpaa/Kconfig
@@ -0,0 +1,10 @@
1menuconfig FSL_DPAA_ETH
2 tristate "DPAA Ethernet"
3 depends on FSL_SOC && FSL_DPAA && FSL_FMAN
4 select PHYLIB
5 select FSL_FMAN_MAC
6 ---help---
7 Data Path Acceleration Architecture Ethernet driver,
8 supporting the Freescale QorIQ chips.
9 Depends on Freescale Buffer Manager and Queue Manager
10 driver and Frame Manager Driver.
diff --git a/drivers/net/ethernet/freescale/dpaa/Makefile b/drivers/net/ethernet/freescale/dpaa/Makefile
new file mode 100644
index 000000000000..fc76029281c7
--- /dev/null
+++ b/drivers/net/ethernet/freescale/dpaa/Makefile
@@ -0,0 +1,11 @@
1#
2# Makefile for the Freescale DPAA Ethernet controllers
3#
4
5# Include FMan headers
6FMAN = $(srctree)/drivers/net/ethernet/freescale/fman
7ccflags-y += -I$(FMAN)
8
9obj-$(CONFIG_FSL_DPAA_ETH) += fsl_dpa.o
10
11fsl_dpa-objs += dpaa_eth.o
diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
new file mode 100644
index 000000000000..8b9b0720f8b1
--- /dev/null
+++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
@@ -0,0 +1,2682 @@
1/* Copyright 2008 - 2016 Freescale Semiconductor Inc.
2 *
3 * Redistribution and use in source and binary forms, with or without
4 * modification, are permitted provided that the following conditions are met:
5 * * Redistributions of source code must retain the above copyright
6 * notice, this list of conditions and the following disclaimer.
7 * * Redistributions in binary form must reproduce the above copyright
8 * notice, this list of conditions and the following disclaimer in the
9 * documentation and/or other materials provided with the distribution.
10 * * Neither the name of Freescale Semiconductor nor the
11 * names of its contributors may be used to endorse or promote products
12 * derived from this software without specific prior written permission.
13 *
14 * ALTERNATIVELY, this software may be distributed under the terms of the
15 * GNU General Public License ("GPL") as published by the Free Software
16 * Foundation, either version 2 of that License or (at your option) any
17 * later version.
18 *
19 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
20 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
26 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
32
33#include <linux/init.h>
34#include <linux/module.h>
35#include <linux/of_platform.h>
36#include <linux/of_mdio.h>
37#include <linux/of_net.h>
38#include <linux/io.h>
39#include <linux/if_arp.h>
40#include <linux/if_vlan.h>
41#include <linux/icmp.h>
42#include <linux/ip.h>
43#include <linux/ipv6.h>
44#include <linux/udp.h>
45#include <linux/tcp.h>
46#include <linux/net.h>
47#include <linux/skbuff.h>
48#include <linux/etherdevice.h>
49#include <linux/if_ether.h>
50#include <linux/highmem.h>
51#include <linux/percpu.h>
52#include <linux/dma-mapping.h>
53#include <linux/sort.h>
54#include <soc/fsl/bman.h>
55#include <soc/fsl/qman.h>
56
57#include "fman.h"
58#include "fman_port.h"
59#include "mac.h"
60#include "dpaa_eth.h"
61
62static int debug = -1;
63module_param(debug, int, 0444);
64MODULE_PARM_DESC(debug, "Module/Driver verbosity level (0=none,...,16=all)");
65
66static u16 tx_timeout = 1000;
67module_param(tx_timeout, ushort, 0444);
68MODULE_PARM_DESC(tx_timeout, "The Tx timeout in ms");
69
70#define FM_FD_STAT_RX_ERRORS \
71 (FM_FD_ERR_DMA | FM_FD_ERR_PHYSICAL | \
72 FM_FD_ERR_SIZE | FM_FD_ERR_CLS_DISCARD | \
73 FM_FD_ERR_EXTRACTION | FM_FD_ERR_NO_SCHEME | \
74 FM_FD_ERR_PRS_TIMEOUT | FM_FD_ERR_PRS_ILL_INSTRUCT | \
75 FM_FD_ERR_PRS_HDR_ERR)
76
77#define FM_FD_STAT_TX_ERRORS \
78 (FM_FD_ERR_UNSUPPORTED_FORMAT | \
79 FM_FD_ERR_LENGTH | FM_FD_ERR_DMA)
80
81#define DPAA_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | \
82 NETIF_MSG_LINK | NETIF_MSG_IFUP | \
83 NETIF_MSG_IFDOWN)
84
85#define DPAA_INGRESS_CS_THRESHOLD 0x10000000
86/* Ingress congestion threshold on FMan ports
87 * The size in bytes of the ingress tail-drop threshold on FMan ports.
88 * Traffic piling up above this value will be rejected by QMan and discarded
89 * by FMan.
90 */
91
92/* Size in bytes of the FQ taildrop threshold */
93#define DPAA_FQ_TD 0x200000
94
95#define DPAA_CS_THRESHOLD_1G 0x06000000
96/* Egress congestion threshold on 1G ports, range 0x1000 .. 0x10000000
97 * The size in bytes of the egress Congestion State notification threshold on
98 * 1G ports. The 1G dTSECs can quite easily be flooded by cores doing Tx in a
99 * tight loop (e.g. by sending UDP datagrams at "while(1) speed"),
100 * and the larger the frame size, the more acute the problem.
101 * So we have to find a balance between these factors:
102 * - avoiding the device staying congested for a prolonged time (risking
103 * the netdev watchdog to fire - see also the tx_timeout module param);
104 * - affecting performance of protocols such as TCP, which otherwise
105 * behave well under the congestion notification mechanism;
106 * - preventing the Tx cores from tightly-looping (as if the congestion
107 * threshold was too low to be effective);
108 * - running out of memory if the CS threshold is set too high.
109 */
110
111#define DPAA_CS_THRESHOLD_10G 0x10000000
112/* The size in bytes of the egress Congestion State notification threshold on
113 * 10G ports, range 0x1000 .. 0x10000000
114 */
115
116/* Largest value that the FQD's OAL field can hold */
117#define FSL_QMAN_MAX_OAL 127
118
119/* Default alignment for start of data in an Rx FD */
120#define DPAA_FD_DATA_ALIGNMENT 16
121
122/* Values for the L3R field of the FM Parse Results
123 */
124/* L3 Type field: First IP Present IPv4 */
125#define FM_L3_PARSE_RESULT_IPV4 0x8000
126/* L3 Type field: First IP Present IPv6 */
127#define FM_L3_PARSE_RESULT_IPV6 0x4000
128/* Values for the L4R field of the FM Parse Results */
129/* L4 Type field: UDP */
130#define FM_L4_PARSE_RESULT_UDP 0x40
131/* L4 Type field: TCP */
132#define FM_L4_PARSE_RESULT_TCP 0x20
133
134#define DPAA_SGT_MAX_ENTRIES 16 /* maximum number of entries in SG Table */
135#define DPAA_BUFF_RELEASE_MAX 8 /* maximum number of buffers released at once */
136
137#define FSL_DPAA_BPID_INV 0xff
138#define FSL_DPAA_ETH_MAX_BUF_COUNT 128
139#define FSL_DPAA_ETH_REFILL_THRESHOLD 80
140
141#define DPAA_TX_PRIV_DATA_SIZE 16
142#define DPAA_PARSE_RESULTS_SIZE sizeof(struct fman_prs_result)
143#define DPAA_TIME_STAMP_SIZE 8
144#define DPAA_HASH_RESULTS_SIZE 8
145#define DPAA_RX_PRIV_DATA_SIZE (u16)(DPAA_TX_PRIV_DATA_SIZE + \
146 dpaa_rx_extra_headroom)
147
148#define DPAA_ETH_RX_QUEUES 128
149
150#define DPAA_ENQUEUE_RETRIES 100000
151
152enum port_type {RX, TX};
153
154struct fm_port_fqs {
155 struct dpaa_fq *tx_defq;
156 struct dpaa_fq *tx_errq;
157 struct dpaa_fq *rx_defq;
158 struct dpaa_fq *rx_errq;
159};
160
161/* All the dpa bps in use at any moment */
162static struct dpaa_bp *dpaa_bp_array[BM_MAX_NUM_OF_POOLS];
163
164/* The raw buffer size must be cacheline aligned */
165#define DPAA_BP_RAW_SIZE 4096
166/* When using more than one buffer pool, the raw sizes are as follows:
167 * 1 bp: 4KB
168 * 2 bp: 2KB, 4KB
169 * 3 bp: 1KB, 2KB, 4KB
170 * 4 bp: 1KB, 2KB, 4KB, 8KB
171 */
172static inline size_t bpool_buffer_raw_size(u8 index, u8 cnt)
173{
174 size_t res = DPAA_BP_RAW_SIZE / 4;
175 u8 i;
176
177 for (i = (cnt < 3) ? cnt : 3; i < 3 + index; i++)
178 res *= 2;
179 return res;
180}
181
182/* FMan-DMA requires 16-byte alignment for Rx buffers, but SKB_DATA_ALIGN is
183 * even stronger (SMP_CACHE_BYTES-aligned), so we just get away with that,
184 * via SKB_WITH_OVERHEAD(). We can't rely on netdev_alloc_frag() giving us
185 * half-page-aligned buffers, so we reserve some more space for start-of-buffer
186 * alignment.
187 */
188#define dpaa_bp_size(raw_size) SKB_WITH_OVERHEAD((raw_size) - SMP_CACHE_BYTES)
189
190static int dpaa_max_frm;
191
192static int dpaa_rx_extra_headroom;
193
194#define dpaa_get_max_mtu() \
195 (dpaa_max_frm - (VLAN_ETH_HLEN + ETH_FCS_LEN))
196
197static int dpaa_netdev_init(struct net_device *net_dev,
198 const struct net_device_ops *dpaa_ops,
199 u16 tx_timeout)
200{
201 struct dpaa_priv *priv = netdev_priv(net_dev);
202 struct device *dev = net_dev->dev.parent;
203 struct dpaa_percpu_priv *percpu_priv;
204 const u8 *mac_addr;
205 int i, err;
206
207 /* Although we access another CPU's private data here
208 * we do it at initialization so it is safe
209 */
210 for_each_possible_cpu(i) {
211 percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
212 percpu_priv->net_dev = net_dev;
213 }
214
215 net_dev->netdev_ops = dpaa_ops;
216 mac_addr = priv->mac_dev->addr;
217
218 net_dev->mem_start = priv->mac_dev->res->start;
219 net_dev->mem_end = priv->mac_dev->res->end;
220
221 net_dev->min_mtu = ETH_MIN_MTU;
222 net_dev->max_mtu = dpaa_get_max_mtu();
223
224 net_dev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
225 NETIF_F_LLTX);
226
227 net_dev->hw_features |= NETIF_F_SG | NETIF_F_HIGHDMA;
228 /* The kernels enables GSO automatically, if we declare NETIF_F_SG.
229 * For conformity, we'll still declare GSO explicitly.
230 */
231 net_dev->features |= NETIF_F_GSO;
232
233 net_dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
234 /* we do not want shared skbs on TX */
235 net_dev->priv_flags &= ~IFF_TX_SKB_SHARING;
236
237 net_dev->features |= net_dev->hw_features;
238 net_dev->vlan_features = net_dev->features;
239
240 memcpy(net_dev->perm_addr, mac_addr, net_dev->addr_len);
241 memcpy(net_dev->dev_addr, mac_addr, net_dev->addr_len);
242
243 net_dev->needed_headroom = priv->tx_headroom;
244 net_dev->watchdog_timeo = msecs_to_jiffies(tx_timeout);
245
246 /* start without the RUNNING flag, phylib controls it later */
247 netif_carrier_off(net_dev);
248
249 err = register_netdev(net_dev);
250 if (err < 0) {
251 dev_err(dev, "register_netdev() = %d\n", err);
252 return err;
253 }
254
255 return 0;
256}
257
258static int dpaa_stop(struct net_device *net_dev)
259{
260 struct mac_device *mac_dev;
261 struct dpaa_priv *priv;
262 int i, err, error;
263
264 priv = netdev_priv(net_dev);
265 mac_dev = priv->mac_dev;
266
267 netif_tx_stop_all_queues(net_dev);
268 /* Allow the Fman (Tx) port to process in-flight frames before we
269 * try switching it off.
270 */
271 usleep_range(5000, 10000);
272
273 err = mac_dev->stop(mac_dev);
274 if (err < 0)
275 netif_err(priv, ifdown, net_dev, "mac_dev->stop() = %d\n",
276 err);
277
278 for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++) {
279 error = fman_port_disable(mac_dev->port[i]);
280 if (error)
281 err = error;
282 }
283
284 if (net_dev->phydev)
285 phy_disconnect(net_dev->phydev);
286 net_dev->phydev = NULL;
287
288 return err;
289}
290
291static void dpaa_tx_timeout(struct net_device *net_dev)
292{
293 struct dpaa_percpu_priv *percpu_priv;
294 const struct dpaa_priv *priv;
295
296 priv = netdev_priv(net_dev);
297 percpu_priv = this_cpu_ptr(priv->percpu_priv);
298
299 netif_crit(priv, timer, net_dev, "Transmit timeout latency: %u ms\n",
300 jiffies_to_msecs(jiffies - dev_trans_start(net_dev)));
301
302 percpu_priv->stats.tx_errors++;
303}
304
305/* Calculates the statistics for the given device by adding the statistics
306 * collected by each CPU.
307 */
308static struct rtnl_link_stats64 *dpaa_get_stats64(struct net_device *net_dev,
309 struct rtnl_link_stats64 *s)
310{
311 int numstats = sizeof(struct rtnl_link_stats64) / sizeof(u64);
312 struct dpaa_priv *priv = netdev_priv(net_dev);
313 struct dpaa_percpu_priv *percpu_priv;
314 u64 *netstats = (u64 *)s;
315 u64 *cpustats;
316 int i, j;
317
318 for_each_possible_cpu(i) {
319 percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
320
321 cpustats = (u64 *)&percpu_priv->stats;
322
323 /* add stats from all CPUs */
324 for (j = 0; j < numstats; j++)
325 netstats[j] += cpustats[j];
326 }
327
328 return s;
329}
330
331static struct mac_device *dpaa_mac_dev_get(struct platform_device *pdev)
332{
333 struct platform_device *of_dev;
334 struct dpaa_eth_data *eth_data;
335 struct device *dpaa_dev, *dev;
336 struct device_node *mac_node;
337 struct mac_device *mac_dev;
338
339 dpaa_dev = &pdev->dev;
340 eth_data = dpaa_dev->platform_data;
341 if (!eth_data)
342 return ERR_PTR(-ENODEV);
343
344 mac_node = eth_data->mac_node;
345
346 of_dev = of_find_device_by_node(mac_node);
347 if (!of_dev) {
348 dev_err(dpaa_dev, "of_find_device_by_node(%s) failed\n",
349 mac_node->full_name);
350 of_node_put(mac_node);
351 return ERR_PTR(-EINVAL);
352 }
353 of_node_put(mac_node);
354
355 dev = &of_dev->dev;
356
357 mac_dev = dev_get_drvdata(dev);
358 if (!mac_dev) {
359 dev_err(dpaa_dev, "dev_get_drvdata(%s) failed\n",
360 dev_name(dev));
361 return ERR_PTR(-EINVAL);
362 }
363
364 return mac_dev;
365}
366
367static int dpaa_set_mac_address(struct net_device *net_dev, void *addr)
368{
369 const struct dpaa_priv *priv;
370 struct mac_device *mac_dev;
371 struct sockaddr old_addr;
372 int err;
373
374 priv = netdev_priv(net_dev);
375
376 memcpy(old_addr.sa_data, net_dev->dev_addr, ETH_ALEN);
377
378 err = eth_mac_addr(net_dev, addr);
379 if (err < 0) {
380 netif_err(priv, drv, net_dev, "eth_mac_addr() = %d\n", err);
381 return err;
382 }
383
384 mac_dev = priv->mac_dev;
385
386 err = mac_dev->change_addr(mac_dev->fman_mac,
387 (enet_addr_t *)net_dev->dev_addr);
388 if (err < 0) {
389 netif_err(priv, drv, net_dev, "mac_dev->change_addr() = %d\n",
390 err);
391 /* reverting to previous address */
392 eth_mac_addr(net_dev, &old_addr);
393
394 return err;
395 }
396
397 return 0;
398}
399
400static void dpaa_set_rx_mode(struct net_device *net_dev)
401{
402 const struct dpaa_priv *priv;
403 int err;
404
405 priv = netdev_priv(net_dev);
406
407 if (!!(net_dev->flags & IFF_PROMISC) != priv->mac_dev->promisc) {
408 priv->mac_dev->promisc = !priv->mac_dev->promisc;
409 err = priv->mac_dev->set_promisc(priv->mac_dev->fman_mac,
410 priv->mac_dev->promisc);
411 if (err < 0)
412 netif_err(priv, drv, net_dev,
413 "mac_dev->set_promisc() = %d\n",
414 err);
415 }
416
417 err = priv->mac_dev->set_multi(net_dev, priv->mac_dev);
418 if (err < 0)
419 netif_err(priv, drv, net_dev, "mac_dev->set_multi() = %d\n",
420 err);
421}
422
423static struct dpaa_bp *dpaa_bpid2pool(int bpid)
424{
425 if (WARN_ON(bpid < 0 || bpid >= BM_MAX_NUM_OF_POOLS))
426 return NULL;
427
428 return dpaa_bp_array[bpid];
429}
430
431/* checks if this bpool is already allocated */
432static bool dpaa_bpid2pool_use(int bpid)
433{
434 if (dpaa_bpid2pool(bpid)) {
435 atomic_inc(&dpaa_bp_array[bpid]->refs);
436 return true;
437 }
438
439 return false;
440}
441
442/* called only once per bpid by dpaa_bp_alloc_pool() */
443static void dpaa_bpid2pool_map(int bpid, struct dpaa_bp *dpaa_bp)
444{
445 dpaa_bp_array[bpid] = dpaa_bp;
446 atomic_set(&dpaa_bp->refs, 1);
447}
448
449static int dpaa_bp_alloc_pool(struct dpaa_bp *dpaa_bp)
450{
451 int err;
452
453 if (dpaa_bp->size == 0 || dpaa_bp->config_count == 0) {
454 pr_err("%s: Buffer pool is not properly initialized! Missing size or initial number of buffers\n",
455 __func__);
456 return -EINVAL;
457 }
458
459 /* If the pool is already specified, we only create one per bpid */
460 if (dpaa_bp->bpid != FSL_DPAA_BPID_INV &&
461 dpaa_bpid2pool_use(dpaa_bp->bpid))
462 return 0;
463
464 if (dpaa_bp->bpid == FSL_DPAA_BPID_INV) {
465 dpaa_bp->pool = bman_new_pool();
466 if (!dpaa_bp->pool) {
467 pr_err("%s: bman_new_pool() failed\n",
468 __func__);
469 return -ENODEV;
470 }
471
472 dpaa_bp->bpid = (u8)bman_get_bpid(dpaa_bp->pool);
473 }
474
475 if (dpaa_bp->seed_cb) {
476 err = dpaa_bp->seed_cb(dpaa_bp);
477 if (err)
478 goto pool_seed_failed;
479 }
480
481 dpaa_bpid2pool_map(dpaa_bp->bpid, dpaa_bp);
482
483 return 0;
484
485pool_seed_failed:
486 pr_err("%s: pool seeding failed\n", __func__);
487 bman_free_pool(dpaa_bp->pool);
488
489 return err;
490}
491
492/* remove and free all the buffers from the given buffer pool */
493static void dpaa_bp_drain(struct dpaa_bp *bp)
494{
495 u8 num = 8;
496 int ret;
497
498 do {
499 struct bm_buffer bmb[8];
500 int i;
501
502 ret = bman_acquire(bp->pool, bmb, num);
503 if (ret < 0) {
504 if (num == 8) {
505 /* we have less than 8 buffers left;
506 * drain them one by one
507 */
508 num = 1;
509 ret = 1;
510 continue;
511 } else {
512 /* Pool is fully drained */
513 break;
514 }
515 }
516
517 if (bp->free_buf_cb)
518 for (i = 0; i < num; i++)
519 bp->free_buf_cb(bp, &bmb[i]);
520 } while (ret > 0);
521}
522
523static void dpaa_bp_free(struct dpaa_bp *dpaa_bp)
524{
525 struct dpaa_bp *bp = dpaa_bpid2pool(dpaa_bp->bpid);
526
527 /* the mapping between bpid and dpaa_bp is done very late in the
528 * allocation procedure; if something failed before the mapping, the bp
529 * was not configured, therefore we don't need the below instructions
530 */
531 if (!bp)
532 return;
533
534 if (!atomic_dec_and_test(&bp->refs))
535 return;
536
537 if (bp->free_buf_cb)
538 dpaa_bp_drain(bp);
539
540 dpaa_bp_array[bp->bpid] = NULL;
541 bman_free_pool(bp->pool);
542}
543
544static void dpaa_bps_free(struct dpaa_priv *priv)
545{
546 int i;
547
548 for (i = 0; i < DPAA_BPS_NUM; i++)
549 dpaa_bp_free(priv->dpaa_bps[i]);
550}
551
552/* Use multiple WQs for FQ assignment:
553 * - Tx Confirmation queues go to WQ1.
554 * - Rx Error and Tx Error queues go to WQ2 (giving them a better chance
555 * to be scheduled, in case there are many more FQs in WQ3).
556 * - Rx Default and Tx queues go to WQ3 (no differentiation between
557 * Rx and Tx traffic).
558 * This ensures that Tx-confirmed buffers are timely released. In particular,
559 * it avoids congestion on the Tx Confirm FQs, which can pile up PFDRs if they
560 * are greatly outnumbered by other FQs in the system, while
561 * dequeue scheduling is round-robin.
562 */
563static inline void dpaa_assign_wq(struct dpaa_fq *fq)
564{
565 switch (fq->fq_type) {
566 case FQ_TYPE_TX_CONFIRM:
567 case FQ_TYPE_TX_CONF_MQ:
568 fq->wq = 1;
569 break;
570 case FQ_TYPE_RX_ERROR:
571 case FQ_TYPE_TX_ERROR:
572 fq->wq = 2;
573 break;
574 case FQ_TYPE_RX_DEFAULT:
575 case FQ_TYPE_TX:
576 fq->wq = 3;
577 break;
578 default:
579 WARN(1, "Invalid FQ type %d for FQID %d!\n",
580 fq->fq_type, fq->fqid);
581 }
582}
583
584static struct dpaa_fq *dpaa_fq_alloc(struct device *dev,
585 u32 start, u32 count,
586 struct list_head *list,
587 enum dpaa_fq_type fq_type)
588{
589 struct dpaa_fq *dpaa_fq;
590 int i;
591
592 dpaa_fq = devm_kzalloc(dev, sizeof(*dpaa_fq) * count,
593 GFP_KERNEL);
594 if (!dpaa_fq)
595 return NULL;
596
597 for (i = 0; i < count; i++) {
598 dpaa_fq[i].fq_type = fq_type;
599 dpaa_fq[i].fqid = start ? start + i : 0;
600 list_add_tail(&dpaa_fq[i].list, list);
601 }
602
603 for (i = 0; i < count; i++)
604 dpaa_assign_wq(dpaa_fq + i);
605
606 return dpaa_fq;
607}
608
609static int dpaa_alloc_all_fqs(struct device *dev, struct list_head *list,
610 struct fm_port_fqs *port_fqs)
611{
612 struct dpaa_fq *dpaa_fq;
613
614 dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_RX_ERROR);
615 if (!dpaa_fq)
616 goto fq_alloc_failed;
617
618 port_fqs->rx_errq = &dpaa_fq[0];
619
620 dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_RX_DEFAULT);
621 if (!dpaa_fq)
622 goto fq_alloc_failed;
623
624 port_fqs->rx_defq = &dpaa_fq[0];
625
626 if (!dpaa_fq_alloc(dev, 0, DPAA_ETH_TXQ_NUM, list, FQ_TYPE_TX_CONF_MQ))
627 goto fq_alloc_failed;
628
629 dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_TX_ERROR);
630 if (!dpaa_fq)
631 goto fq_alloc_failed;
632
633 port_fqs->tx_errq = &dpaa_fq[0];
634
635 dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_TX_CONFIRM);
636 if (!dpaa_fq)
637 goto fq_alloc_failed;
638
639 port_fqs->tx_defq = &dpaa_fq[0];
640
641 if (!dpaa_fq_alloc(dev, 0, DPAA_ETH_TXQ_NUM, list, FQ_TYPE_TX))
642 goto fq_alloc_failed;
643
644 return 0;
645
646fq_alloc_failed:
647 dev_err(dev, "dpaa_fq_alloc() failed\n");
648 return -ENOMEM;
649}
650
651static u32 rx_pool_channel;
652static DEFINE_SPINLOCK(rx_pool_channel_init);
653
654static int dpaa_get_channel(void)
655{
656 spin_lock(&rx_pool_channel_init);
657 if (!rx_pool_channel) {
658 u32 pool;
659 int ret;
660
661 ret = qman_alloc_pool(&pool);
662
663 if (!ret)
664 rx_pool_channel = pool;
665 }
666 spin_unlock(&rx_pool_channel_init);
667 if (!rx_pool_channel)
668 return -ENOMEM;
669 return rx_pool_channel;
670}
671
672static void dpaa_release_channel(void)
673{
674 qman_release_pool(rx_pool_channel);
675}
676
677static void dpaa_eth_add_channel(u16 channel)
678{
679 u32 pool = QM_SDQCR_CHANNELS_POOL_CONV(channel);
680 const cpumask_t *cpus = qman_affine_cpus();
681 struct qman_portal *portal;
682 int cpu;
683
684 for_each_cpu(cpu, cpus) {
685 portal = qman_get_affine_portal(cpu);
686 qman_p_static_dequeue_add(portal, pool);
687 }
688}
689
690/* Congestion group state change notification callback.
691 * Stops the device's egress queues while they are congested and
692 * wakes them upon exiting congested state.
693 * Also updates some CGR-related stats.
694 */
695static void dpaa_eth_cgscn(struct qman_portal *qm, struct qman_cgr *cgr,
696 int congested)
697{
698 struct dpaa_priv *priv = (struct dpaa_priv *)container_of(cgr,
699 struct dpaa_priv, cgr_data.cgr);
700
701 if (congested)
702 netif_tx_stop_all_queues(priv->net_dev);
703 else
704 netif_tx_wake_all_queues(priv->net_dev);
705}
706
707static int dpaa_eth_cgr_init(struct dpaa_priv *priv)
708{
709 struct qm_mcc_initcgr initcgr;
710 u32 cs_th;
711 int err;
712
713 err = qman_alloc_cgrid(&priv->cgr_data.cgr.cgrid);
714 if (err < 0) {
715 if (netif_msg_drv(priv))
716 pr_err("%s: Error %d allocating CGR ID\n",
717 __func__, err);
718 goto out_error;
719 }
720 priv->cgr_data.cgr.cb = dpaa_eth_cgscn;
721
722 /* Enable Congestion State Change Notifications and CS taildrop */
723 initcgr.we_mask = QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES;
724 initcgr.cgr.cscn_en = QM_CGR_EN;
725
726 /* Set different thresholds based on the MAC speed.
727 * This may turn suboptimal if the MAC is reconfigured at a speed
728 * lower than its max, e.g. if a dTSEC later negotiates a 100Mbps link.
729 * In such cases, we ought to reconfigure the threshold, too.
730 */
731 if (priv->mac_dev->if_support & SUPPORTED_10000baseT_Full)
732 cs_th = DPAA_CS_THRESHOLD_10G;
733 else
734 cs_th = DPAA_CS_THRESHOLD_1G;
735 qm_cgr_cs_thres_set64(&initcgr.cgr.cs_thres, cs_th, 1);
736
737 initcgr.we_mask |= QM_CGR_WE_CSTD_EN;
738 initcgr.cgr.cstd_en = QM_CGR_EN;
739
740 err = qman_create_cgr(&priv->cgr_data.cgr, QMAN_CGR_FLAG_USE_INIT,
741 &initcgr);
742 if (err < 0) {
743 if (netif_msg_drv(priv))
744 pr_err("%s: Error %d creating CGR with ID %d\n",
745 __func__, err, priv->cgr_data.cgr.cgrid);
746 qman_release_cgrid(priv->cgr_data.cgr.cgrid);
747 goto out_error;
748 }
749 if (netif_msg_drv(priv))
750 pr_debug("Created CGR %d for netdev with hwaddr %pM on QMan channel %d\n",
751 priv->cgr_data.cgr.cgrid, priv->mac_dev->addr,
752 priv->cgr_data.cgr.chan);
753
754out_error:
755 return err;
756}
757
758static inline void dpaa_setup_ingress(const struct dpaa_priv *priv,
759 struct dpaa_fq *fq,
760 const struct qman_fq *template)
761{
762 fq->fq_base = *template;
763 fq->net_dev = priv->net_dev;
764
765 fq->flags = QMAN_FQ_FLAG_NO_ENQUEUE;
766 fq->channel = priv->channel;
767}
768
769static inline void dpaa_setup_egress(const struct dpaa_priv *priv,
770 struct dpaa_fq *fq,
771 struct fman_port *port,
772 const struct qman_fq *template)
773{
774 fq->fq_base = *template;
775 fq->net_dev = priv->net_dev;
776
777 if (port) {
778 fq->flags = QMAN_FQ_FLAG_TO_DCPORTAL;
779 fq->channel = (u16)fman_port_get_qman_channel_id(port);
780 } else {
781 fq->flags = QMAN_FQ_FLAG_NO_MODIFY;
782 }
783}
784
785static void dpaa_fq_setup(struct dpaa_priv *priv,
786 const struct dpaa_fq_cbs *fq_cbs,
787 struct fman_port *tx_port)
788{
789 int egress_cnt = 0, conf_cnt = 0, num_portals = 0, cpu;
790 const cpumask_t *affine_cpus = qman_affine_cpus();
791 u16 portals[NR_CPUS];
792 struct dpaa_fq *fq;
793
794 for_each_cpu(cpu, affine_cpus)
795 portals[num_portals++] = qman_affine_channel(cpu);
796 if (num_portals == 0)
797 dev_err(priv->net_dev->dev.parent,
798 "No Qman software (affine) channels found");
799
800 /* Initialize each FQ in the list */
801 list_for_each_entry(fq, &priv->dpaa_fq_list, list) {
802 switch (fq->fq_type) {
803 case FQ_TYPE_RX_DEFAULT:
804 dpaa_setup_ingress(priv, fq, &fq_cbs->rx_defq);
805 break;
806 case FQ_TYPE_RX_ERROR:
807 dpaa_setup_ingress(priv, fq, &fq_cbs->rx_errq);
808 break;
809 case FQ_TYPE_TX:
810 dpaa_setup_egress(priv, fq, tx_port,
811 &fq_cbs->egress_ern);
812 /* If we have more Tx queues than the number of cores,
813 * just ignore the extra ones.
814 */
815 if (egress_cnt < DPAA_ETH_TXQ_NUM)
816 priv->egress_fqs[egress_cnt++] = &fq->fq_base;
817 break;
818 case FQ_TYPE_TX_CONF_MQ:
819 priv->conf_fqs[conf_cnt++] = &fq->fq_base;
820 /* fall through */
821 case FQ_TYPE_TX_CONFIRM:
822 dpaa_setup_ingress(priv, fq, &fq_cbs->tx_defq);
823 break;
824 case FQ_TYPE_TX_ERROR:
825 dpaa_setup_ingress(priv, fq, &fq_cbs->tx_errq);
826 break;
827 default:
828 dev_warn(priv->net_dev->dev.parent,
829 "Unknown FQ type detected!\n");
830 break;
831 }
832 }
833
834 /* Make sure all CPUs receive a corresponding Tx queue. */
835 while (egress_cnt < DPAA_ETH_TXQ_NUM) {
836 list_for_each_entry(fq, &priv->dpaa_fq_list, list) {
837 if (fq->fq_type != FQ_TYPE_TX)
838 continue;
839 priv->egress_fqs[egress_cnt++] = &fq->fq_base;
840 if (egress_cnt == DPAA_ETH_TXQ_NUM)
841 break;
842 }
843 }
844}
845
846static inline int dpaa_tx_fq_to_id(const struct dpaa_priv *priv,
847 struct qman_fq *tx_fq)
848{
849 int i;
850
851 for (i = 0; i < DPAA_ETH_TXQ_NUM; i++)
852 if (priv->egress_fqs[i] == tx_fq)
853 return i;
854
855 return -EINVAL;
856}
857
858static int dpaa_fq_init(struct dpaa_fq *dpaa_fq, bool td_enable)
859{
860 const struct dpaa_priv *priv;
861 struct qman_fq *confq = NULL;
862 struct qm_mcc_initfq initfq;
863 struct device *dev;
864 struct qman_fq *fq;
865 int queue_id;
866 int err;
867
868 priv = netdev_priv(dpaa_fq->net_dev);
869 dev = dpaa_fq->net_dev->dev.parent;
870
871 if (dpaa_fq->fqid == 0)
872 dpaa_fq->flags |= QMAN_FQ_FLAG_DYNAMIC_FQID;
873
874 dpaa_fq->init = !(dpaa_fq->flags & QMAN_FQ_FLAG_NO_MODIFY);
875
876 err = qman_create_fq(dpaa_fq->fqid, dpaa_fq->flags, &dpaa_fq->fq_base);
877 if (err) {
878 dev_err(dev, "qman_create_fq() failed\n");
879 return err;
880 }
881 fq = &dpaa_fq->fq_base;
882
883 if (dpaa_fq->init) {
884 memset(&initfq, 0, sizeof(initfq));
885
886 initfq.we_mask = QM_INITFQ_WE_FQCTRL;
887 /* Note: we may get to keep an empty FQ in cache */
888 initfq.fqd.fq_ctrl = QM_FQCTRL_PREFERINCACHE;
889
890 /* Try to reduce the number of portal interrupts for
891 * Tx Confirmation FQs.
892 */
893 if (dpaa_fq->fq_type == FQ_TYPE_TX_CONFIRM)
894 initfq.fqd.fq_ctrl |= QM_FQCTRL_HOLDACTIVE;
895
896 /* FQ placement */
897 initfq.we_mask |= QM_INITFQ_WE_DESTWQ;
898
899 qm_fqd_set_destwq(&initfq.fqd, dpaa_fq->channel, dpaa_fq->wq);
900
901 /* Put all egress queues in a congestion group of their own.
902 * Sensu stricto, the Tx confirmation queues are Rx FQs,
903 * rather than Tx - but they nonetheless account for the
904 * memory footprint on behalf of egress traffic. We therefore
905 * place them in the netdev's CGR, along with the Tx FQs.
906 */
907 if (dpaa_fq->fq_type == FQ_TYPE_TX ||
908 dpaa_fq->fq_type == FQ_TYPE_TX_CONFIRM ||
909 dpaa_fq->fq_type == FQ_TYPE_TX_CONF_MQ) {
910 initfq.we_mask |= QM_INITFQ_WE_CGID;
911 initfq.fqd.fq_ctrl |= QM_FQCTRL_CGE;
912 initfq.fqd.cgid = (u8)priv->cgr_data.cgr.cgrid;
913 /* Set a fixed overhead accounting, in an attempt to
914 * reduce the impact of fixed-size skb shells and the
915 * driver's needed headroom on system memory. This is
916 * especially the case when the egress traffic is
917 * composed of small datagrams.
918 * Unfortunately, QMan's OAL value is capped to an
919 * insufficient value, but even that is better than
920 * no overhead accounting at all.
921 */
922 initfq.we_mask |= QM_INITFQ_WE_OAC;
923 qm_fqd_set_oac(&initfq.fqd, QM_OAC_CG);
924 qm_fqd_set_oal(&initfq.fqd,
925 min(sizeof(struct sk_buff) +
926 priv->tx_headroom,
927 (size_t)FSL_QMAN_MAX_OAL));
928 }
929
930 if (td_enable) {
931 initfq.we_mask |= QM_INITFQ_WE_TDTHRESH;
932 qm_fqd_set_taildrop(&initfq.fqd, DPAA_FQ_TD, 1);
933 initfq.fqd.fq_ctrl = QM_FQCTRL_TDE;
934 }
935
936 if (dpaa_fq->fq_type == FQ_TYPE_TX) {
937 queue_id = dpaa_tx_fq_to_id(priv, &dpaa_fq->fq_base);
938 if (queue_id >= 0)
939 confq = priv->conf_fqs[queue_id];
940 if (confq) {
941 initfq.we_mask |= QM_INITFQ_WE_CONTEXTA;
942 /* ContextA: OVOM=1(use contextA2 bits instead of ICAD)
943 * A2V=1 (contextA A2 field is valid)
944 * A0V=1 (contextA A0 field is valid)
945 * B0V=1 (contextB field is valid)
946 * ContextA A2: EBD=1 (deallocate buffers inside FMan)
947 * ContextB B0(ASPID): 0 (absolute Virtual Storage ID)
948 */
949 initfq.fqd.context_a.hi = 0x1e000000;
950 initfq.fqd.context_a.lo = 0x80000000;
951 }
952 }
953
954 /* Put all the ingress queues in our "ingress CGR". */
955 if (priv->use_ingress_cgr &&
956 (dpaa_fq->fq_type == FQ_TYPE_RX_DEFAULT ||
957 dpaa_fq->fq_type == FQ_TYPE_RX_ERROR)) {
958 initfq.we_mask |= QM_INITFQ_WE_CGID;
959 initfq.fqd.fq_ctrl |= QM_FQCTRL_CGE;
960 initfq.fqd.cgid = (u8)priv->ingress_cgr.cgrid;
961 /* Set a fixed overhead accounting, just like for the
962 * egress CGR.
963 */
964 initfq.we_mask |= QM_INITFQ_WE_OAC;
965 qm_fqd_set_oac(&initfq.fqd, QM_OAC_CG);
966 qm_fqd_set_oal(&initfq.fqd,
967 min(sizeof(struct sk_buff) +
968 priv->tx_headroom,
969 (size_t)FSL_QMAN_MAX_OAL));
970 }
971
972 /* Initialization common to all ingress queues */
973 if (dpaa_fq->flags & QMAN_FQ_FLAG_NO_ENQUEUE) {
974 initfq.we_mask |= QM_INITFQ_WE_CONTEXTA;
975 initfq.fqd.fq_ctrl |=
976 QM_FQCTRL_HOLDACTIVE;
977 initfq.fqd.context_a.stashing.exclusive =
978 QM_STASHING_EXCL_DATA | QM_STASHING_EXCL_CTX |
979 QM_STASHING_EXCL_ANNOTATION;
980 qm_fqd_set_stashing(&initfq.fqd, 1, 2,
981 DIV_ROUND_UP(sizeof(struct qman_fq),
982 64));
983 }
984
985 err = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &initfq);
986 if (err < 0) {
987 dev_err(dev, "qman_init_fq(%u) = %d\n",
988 qman_fq_fqid(fq), err);
989 qman_destroy_fq(fq);
990 return err;
991 }
992 }
993
994 dpaa_fq->fqid = qman_fq_fqid(fq);
995
996 return 0;
997}
998
999static int dpaa_fq_free_entry(struct device *dev, struct qman_fq *fq)
1000{
1001 const struct dpaa_priv *priv;
1002 struct dpaa_fq *dpaa_fq;
1003 int err, error;
1004
1005 err = 0;
1006
1007 dpaa_fq = container_of(fq, struct dpaa_fq, fq_base);
1008 priv = netdev_priv(dpaa_fq->net_dev);
1009
1010 if (dpaa_fq->init) {
1011 err = qman_retire_fq(fq, NULL);
1012 if (err < 0 && netif_msg_drv(priv))
1013 dev_err(dev, "qman_retire_fq(%u) = %d\n",
1014 qman_fq_fqid(fq), err);
1015
1016 error = qman_oos_fq(fq);
1017 if (error < 0 && netif_msg_drv(priv)) {
1018 dev_err(dev, "qman_oos_fq(%u) = %d\n",
1019 qman_fq_fqid(fq), error);
1020 if (err >= 0)
1021 err = error;
1022 }
1023 }
1024
1025 qman_destroy_fq(fq);
1026 list_del(&dpaa_fq->list);
1027
1028 return err;
1029}
1030
1031static int dpaa_fq_free(struct device *dev, struct list_head *list)
1032{
1033 struct dpaa_fq *dpaa_fq, *tmp;
1034 int err, error;
1035
1036 err = 0;
1037 list_for_each_entry_safe(dpaa_fq, tmp, list, list) {
1038 error = dpaa_fq_free_entry(dev, (struct qman_fq *)dpaa_fq);
1039 if (error < 0 && err >= 0)
1040 err = error;
1041 }
1042
1043 return err;
1044}
1045
1046static void dpaa_eth_init_tx_port(struct fman_port *port, struct dpaa_fq *errq,
1047 struct dpaa_fq *defq,
1048 struct dpaa_buffer_layout *buf_layout)
1049{
1050 struct fman_buffer_prefix_content buf_prefix_content;
1051 struct fman_port_params params;
1052 int err;
1053
1054 memset(&params, 0, sizeof(params));
1055 memset(&buf_prefix_content, 0, sizeof(buf_prefix_content));
1056
1057 buf_prefix_content.priv_data_size = buf_layout->priv_data_size;
1058 buf_prefix_content.pass_prs_result = true;
1059 buf_prefix_content.pass_hash_result = true;
1060 buf_prefix_content.pass_time_stamp = false;
1061 buf_prefix_content.data_align = DPAA_FD_DATA_ALIGNMENT;
1062
1063 params.specific_params.non_rx_params.err_fqid = errq->fqid;
1064 params.specific_params.non_rx_params.dflt_fqid = defq->fqid;
1065
1066 err = fman_port_config(port, &params);
1067 if (err)
1068 pr_err("%s: fman_port_config failed\n", __func__);
1069
1070 err = fman_port_cfg_buf_prefix_content(port, &buf_prefix_content);
1071 if (err)
1072 pr_err("%s: fman_port_cfg_buf_prefix_content failed\n",
1073 __func__);
1074
1075 err = fman_port_init(port);
1076 if (err)
1077 pr_err("%s: fm_port_init failed\n", __func__);
1078}
1079
1080static void dpaa_eth_init_rx_port(struct fman_port *port, struct dpaa_bp **bps,
1081 size_t count, struct dpaa_fq *errq,
1082 struct dpaa_fq *defq,
1083 struct dpaa_buffer_layout *buf_layout)
1084{
1085 struct fman_buffer_prefix_content buf_prefix_content;
1086 struct fman_port_rx_params *rx_p;
1087 struct fman_port_params params;
1088 int i, err;
1089
1090 memset(&params, 0, sizeof(params));
1091 memset(&buf_prefix_content, 0, sizeof(buf_prefix_content));
1092
1093 buf_prefix_content.priv_data_size = buf_layout->priv_data_size;
1094 buf_prefix_content.pass_prs_result = true;
1095 buf_prefix_content.pass_hash_result = true;
1096 buf_prefix_content.pass_time_stamp = false;
1097 buf_prefix_content.data_align = DPAA_FD_DATA_ALIGNMENT;
1098
1099 rx_p = &params.specific_params.rx_params;
1100 rx_p->err_fqid = errq->fqid;
1101 rx_p->dflt_fqid = defq->fqid;
1102
1103 count = min(ARRAY_SIZE(rx_p->ext_buf_pools.ext_buf_pool), count);
1104 rx_p->ext_buf_pools.num_of_pools_used = (u8)count;
1105 for (i = 0; i < count; i++) {
1106 rx_p->ext_buf_pools.ext_buf_pool[i].id = bps[i]->bpid;
1107 rx_p->ext_buf_pools.ext_buf_pool[i].size = (u16)bps[i]->size;
1108 }
1109
1110 err = fman_port_config(port, &params);
1111 if (err)
1112 pr_err("%s: fman_port_config failed\n", __func__);
1113
1114 err = fman_port_cfg_buf_prefix_content(port, &buf_prefix_content);
1115 if (err)
1116 pr_err("%s: fman_port_cfg_buf_prefix_content failed\n",
1117 __func__);
1118
1119 err = fman_port_init(port);
1120 if (err)
1121 pr_err("%s: fm_port_init failed\n", __func__);
1122}
1123
1124static void dpaa_eth_init_ports(struct mac_device *mac_dev,
1125 struct dpaa_bp **bps, size_t count,
1126 struct fm_port_fqs *port_fqs,
1127 struct dpaa_buffer_layout *buf_layout,
1128 struct device *dev)
1129{
1130 struct fman_port *rxport = mac_dev->port[RX];
1131 struct fman_port *txport = mac_dev->port[TX];
1132
1133 dpaa_eth_init_tx_port(txport, port_fqs->tx_errq,
1134 port_fqs->tx_defq, &buf_layout[TX]);
1135 dpaa_eth_init_rx_port(rxport, bps, count, port_fqs->rx_errq,
1136 port_fqs->rx_defq, &buf_layout[RX]);
1137}
1138
1139static int dpaa_bman_release(const struct dpaa_bp *dpaa_bp,
1140 struct bm_buffer *bmb, int cnt)
1141{
1142 int err;
1143
1144 err = bman_release(dpaa_bp->pool, bmb, cnt);
1145 /* Should never occur, address anyway to avoid leaking the buffers */
1146 if (unlikely(WARN_ON(err)) && dpaa_bp->free_buf_cb)
1147 while (cnt-- > 0)
1148 dpaa_bp->free_buf_cb(dpaa_bp, &bmb[cnt]);
1149
1150 return cnt;
1151}
1152
1153static void dpaa_release_sgt_members(struct qm_sg_entry *sgt)
1154{
1155 struct bm_buffer bmb[DPAA_BUFF_RELEASE_MAX];
1156 struct dpaa_bp *dpaa_bp;
1157 int i = 0, j;
1158
1159 memset(bmb, 0, sizeof(bmb));
1160
1161 do {
1162 dpaa_bp = dpaa_bpid2pool(sgt[i].bpid);
1163 if (!dpaa_bp)
1164 return;
1165
1166 j = 0;
1167 do {
1168 WARN_ON(qm_sg_entry_is_ext(&sgt[i]));
1169
1170 bm_buffer_set64(&bmb[j], qm_sg_entry_get64(&sgt[i]));
1171
1172 j++; i++;
1173 } while (j < ARRAY_SIZE(bmb) &&
1174 !qm_sg_entry_is_final(&sgt[i - 1]) &&
1175 sgt[i - 1].bpid == sgt[i].bpid);
1176
1177 dpaa_bman_release(dpaa_bp, bmb, j);
1178 } while (!qm_sg_entry_is_final(&sgt[i - 1]));
1179}
1180
1181static void dpaa_fd_release(const struct net_device *net_dev,
1182 const struct qm_fd *fd)
1183{
1184 struct qm_sg_entry *sgt;
1185 struct dpaa_bp *dpaa_bp;
1186 struct bm_buffer bmb;
1187 dma_addr_t addr;
1188 void *vaddr;
1189
1190 bmb.data = 0;
1191 bm_buffer_set64(&bmb, qm_fd_addr(fd));
1192
1193 dpaa_bp = dpaa_bpid2pool(fd->bpid);
1194 if (!dpaa_bp)
1195 return;
1196
1197 if (qm_fd_get_format(fd) == qm_fd_sg) {
1198 vaddr = phys_to_virt(qm_fd_addr(fd));
1199 sgt = vaddr + qm_fd_get_offset(fd);
1200
1201 dma_unmap_single(dpaa_bp->dev, qm_fd_addr(fd), dpaa_bp->size,
1202 DMA_FROM_DEVICE);
1203
1204 dpaa_release_sgt_members(sgt);
1205
1206 addr = dma_map_single(dpaa_bp->dev, vaddr, dpaa_bp->size,
1207 DMA_FROM_DEVICE);
1208 if (dma_mapping_error(dpaa_bp->dev, addr)) {
1209 dev_err(dpaa_bp->dev, "DMA mapping failed");
1210 return;
1211 }
1212 bm_buffer_set64(&bmb, addr);
1213 }
1214
1215 dpaa_bman_release(dpaa_bp, &bmb, 1);
1216}
1217
1218/* Turn on HW checksum computation for this outgoing frame.
1219 * If the current protocol is not something we support in this regard
1220 * (or if the stack has already computed the SW checksum), we do nothing.
1221 *
1222 * Returns 0 if all goes well (or HW csum doesn't apply), and a negative value
1223 * otherwise.
1224 *
1225 * Note that this function may modify the fd->cmd field and the skb data buffer
1226 * (the Parse Results area).
1227 */
1228static int dpaa_enable_tx_csum(struct dpaa_priv *priv,
1229 struct sk_buff *skb,
1230 struct qm_fd *fd,
1231 char *parse_results)
1232{
1233 struct fman_prs_result *parse_result;
1234 u16 ethertype = ntohs(skb->protocol);
1235 struct ipv6hdr *ipv6h = NULL;
1236 struct iphdr *iph;
1237 int retval = 0;
1238 u8 l4_proto;
1239
1240 if (skb->ip_summed != CHECKSUM_PARTIAL)
1241 return 0;
1242
1243 /* Note: L3 csum seems to be already computed in sw, but we can't choose
1244 * L4 alone from the FM configuration anyway.
1245 */
1246
1247 /* Fill in some fields of the Parse Results array, so the FMan
1248 * can find them as if they came from the FMan Parser.
1249 */
1250 parse_result = (struct fman_prs_result *)parse_results;
1251
1252 /* If we're dealing with VLAN, get the real Ethernet type */
1253 if (ethertype == ETH_P_8021Q) {
1254 /* We can't always assume the MAC header is set correctly
1255 * by the stack, so reset to beginning of skb->data
1256 */
1257 skb_reset_mac_header(skb);
1258 ethertype = ntohs(vlan_eth_hdr(skb)->h_vlan_encapsulated_proto);
1259 }
1260
1261 /* Fill in the relevant L3 parse result fields
1262 * and read the L4 protocol type
1263 */
1264 switch (ethertype) {
1265 case ETH_P_IP:
1266 parse_result->l3r = cpu_to_be16(FM_L3_PARSE_RESULT_IPV4);
1267 iph = ip_hdr(skb);
1268 WARN_ON(!iph);
1269 l4_proto = iph->protocol;
1270 break;
1271 case ETH_P_IPV6:
1272 parse_result->l3r = cpu_to_be16(FM_L3_PARSE_RESULT_IPV6);
1273 ipv6h = ipv6_hdr(skb);
1274 WARN_ON(!ipv6h);
1275 l4_proto = ipv6h->nexthdr;
1276 break;
1277 default:
1278 /* We shouldn't even be here */
1279 if (net_ratelimit())
1280 netif_alert(priv, tx_err, priv->net_dev,
1281 "Can't compute HW csum for L3 proto 0x%x\n",
1282 ntohs(skb->protocol));
1283 retval = -EIO;
1284 goto return_error;
1285 }
1286
1287 /* Fill in the relevant L4 parse result fields */
1288 switch (l4_proto) {
1289 case IPPROTO_UDP:
1290 parse_result->l4r = FM_L4_PARSE_RESULT_UDP;
1291 break;
1292 case IPPROTO_TCP:
1293 parse_result->l4r = FM_L4_PARSE_RESULT_TCP;
1294 break;
1295 default:
1296 if (net_ratelimit())
1297 netif_alert(priv, tx_err, priv->net_dev,
1298 "Can't compute HW csum for L4 proto 0x%x\n",
1299 l4_proto);
1300 retval = -EIO;
1301 goto return_error;
1302 }
1303
1304 /* At index 0 is IPOffset_1 as defined in the Parse Results */
1305 parse_result->ip_off[0] = (u8)skb_network_offset(skb);
1306 parse_result->l4_off = (u8)skb_transport_offset(skb);
1307
1308 /* Enable L3 (and L4, if TCP or UDP) HW checksum. */
1309 fd->cmd |= FM_FD_CMD_RPD | FM_FD_CMD_DTC;
1310
1311 /* On P1023 and similar platforms fd->cmd interpretation could
1312 * be disabled by setting CONTEXT_A bit ICMD; currently this bit
1313 * is not set so we do not need to check; in the future, if/when
1314 * using context_a we need to check this bit
1315 */
1316
1317return_error:
1318 return retval;
1319}
1320
1321static int dpaa_bp_add_8_bufs(const struct dpaa_bp *dpaa_bp)
1322{
1323 struct device *dev = dpaa_bp->dev;
1324 struct bm_buffer bmb[8];
1325 dma_addr_t addr;
1326 void *new_buf;
1327 u8 i;
1328
1329 for (i = 0; i < 8; i++) {
1330 new_buf = netdev_alloc_frag(dpaa_bp->raw_size);
1331 if (unlikely(!new_buf)) {
1332 dev_err(dev, "netdev_alloc_frag() failed, size %zu\n",
1333 dpaa_bp->raw_size);
1334 goto release_previous_buffs;
1335 }
1336 new_buf = PTR_ALIGN(new_buf, SMP_CACHE_BYTES);
1337
1338 addr = dma_map_single(dev, new_buf,
1339 dpaa_bp->size, DMA_FROM_DEVICE);
1340 if (unlikely(dma_mapping_error(dev, addr))) {
1341 dev_err(dpaa_bp->dev, "DMA map failed");
1342 goto release_previous_buffs;
1343 }
1344
1345 bmb[i].data = 0;
1346 bm_buffer_set64(&bmb[i], addr);
1347 }
1348
1349release_bufs:
1350 return dpaa_bman_release(dpaa_bp, bmb, i);
1351
1352release_previous_buffs:
1353 WARN_ONCE(1, "dpaa_eth: failed to add buffers on Rx\n");
1354
1355 bm_buffer_set64(&bmb[i], 0);
1356 /* Avoid releasing a completely null buffer; bman_release() requires
1357 * at least one buffer.
1358 */
1359 if (likely(i))
1360 goto release_bufs;
1361
1362 return 0;
1363}
1364
1365static int dpaa_bp_seed(struct dpaa_bp *dpaa_bp)
1366{
1367 int i;
1368
1369 /* Give each CPU an allotment of "config_count" buffers */
1370 for_each_possible_cpu(i) {
1371 int *count_ptr = per_cpu_ptr(dpaa_bp->percpu_count, i);
1372 int j;
1373
1374 /* Although we access another CPU's counters here
1375 * we do it at boot time so it is safe
1376 */
1377 for (j = 0; j < dpaa_bp->config_count; j += 8)
1378 *count_ptr += dpaa_bp_add_8_bufs(dpaa_bp);
1379 }
1380 return 0;
1381}
1382
1383/* Add buffers/(pages) for Rx processing whenever bpool count falls below
1384 * REFILL_THRESHOLD.
1385 */
1386static int dpaa_eth_refill_bpool(struct dpaa_bp *dpaa_bp, int *countptr)
1387{
1388 int count = *countptr;
1389 int new_bufs;
1390
1391 if (unlikely(count < FSL_DPAA_ETH_REFILL_THRESHOLD)) {
1392 do {
1393 new_bufs = dpaa_bp_add_8_bufs(dpaa_bp);
1394 if (unlikely(!new_bufs)) {
1395 /* Avoid looping forever if we've temporarily
1396 * run out of memory. We'll try again at the
1397 * next NAPI cycle.
1398 */
1399 break;
1400 }
1401 count += new_bufs;
1402 } while (count < FSL_DPAA_ETH_MAX_BUF_COUNT);
1403
1404 *countptr = count;
1405 if (unlikely(count < FSL_DPAA_ETH_MAX_BUF_COUNT))
1406 return -ENOMEM;
1407 }
1408
1409 return 0;
1410}
1411
1412static int dpaa_eth_refill_bpools(struct dpaa_priv *priv)
1413{
1414 struct dpaa_bp *dpaa_bp;
1415 int *countptr;
1416 int res, i;
1417
1418 for (i = 0; i < DPAA_BPS_NUM; i++) {
1419 dpaa_bp = priv->dpaa_bps[i];
1420 if (!dpaa_bp)
1421 return -EINVAL;
1422 countptr = this_cpu_ptr(dpaa_bp->percpu_count);
1423 res = dpaa_eth_refill_bpool(dpaa_bp, countptr);
1424 if (res)
1425 return res;
1426 }
1427 return 0;
1428}
1429
1430/* Cleanup function for outgoing frame descriptors that were built on Tx path,
1431 * either contiguous frames or scatter/gather ones.
1432 * Skb freeing is not handled here.
1433 *
1434 * This function may be called on error paths in the Tx function, so guard
1435 * against cases when not all fd relevant fields were filled in.
1436 *
1437 * Return the skb backpointer, since for S/G frames the buffer containing it
1438 * gets freed here.
1439 */
1440static struct sk_buff *dpaa_cleanup_tx_fd(const struct dpaa_priv *priv,
1441 const struct qm_fd *fd)
1442{
1443 const enum dma_data_direction dma_dir = DMA_TO_DEVICE;
1444 struct device *dev = priv->net_dev->dev.parent;
1445 dma_addr_t addr = qm_fd_addr(fd);
1446 const struct qm_sg_entry *sgt;
1447 struct sk_buff **skbh, *skb;
1448 int nr_frags, i;
1449
1450 skbh = (struct sk_buff **)phys_to_virt(addr);
1451 skb = *skbh;
1452
1453 if (unlikely(qm_fd_get_format(fd) == qm_fd_sg)) {
1454 nr_frags = skb_shinfo(skb)->nr_frags;
1455 dma_unmap_single(dev, addr, qm_fd_get_offset(fd) +
1456 sizeof(struct qm_sg_entry) * (1 + nr_frags),
1457 dma_dir);
1458
1459 /* The sgt buffer has been allocated with netdev_alloc_frag(),
1460 * it's from lowmem.
1461 */
1462 sgt = phys_to_virt(addr + qm_fd_get_offset(fd));
1463
1464 /* sgt[0] is from lowmem, was dma_map_single()-ed */
1465 dma_unmap_single(dev, qm_sg_addr(&sgt[0]),
1466 qm_sg_entry_get_len(&sgt[0]), dma_dir);
1467
1468 /* remaining pages were mapped with skb_frag_dma_map() */
1469 for (i = 1; i < nr_frags; i++) {
1470 WARN_ON(qm_sg_entry_is_ext(&sgt[i]));
1471
1472 dma_unmap_page(dev, qm_sg_addr(&sgt[i]),
1473 qm_sg_entry_get_len(&sgt[i]), dma_dir);
1474 }
1475
1476 /* Free the page frag that we allocated on Tx */
1477 skb_free_frag(phys_to_virt(addr));
1478 } else {
1479 dma_unmap_single(dev, addr,
1480 skb_tail_pointer(skb) - (u8 *)skbh, dma_dir);
1481 }
1482
1483 return skb;
1484}
1485
1486/* Build a linear skb around the received buffer.
1487 * We are guaranteed there is enough room at the end of the data buffer to
1488 * accommodate the shared info area of the skb.
1489 */
1490static struct sk_buff *contig_fd_to_skb(const struct dpaa_priv *priv,
1491 const struct qm_fd *fd)
1492{
1493 ssize_t fd_off = qm_fd_get_offset(fd);
1494 dma_addr_t addr = qm_fd_addr(fd);
1495 struct dpaa_bp *dpaa_bp;
1496 struct sk_buff *skb;
1497 void *vaddr;
1498
1499 vaddr = phys_to_virt(addr);
1500 WARN_ON(!IS_ALIGNED((unsigned long)vaddr, SMP_CACHE_BYTES));
1501
1502 dpaa_bp = dpaa_bpid2pool(fd->bpid);
1503 if (!dpaa_bp)
1504 goto free_buffer;
1505
1506 skb = build_skb(vaddr, dpaa_bp->size +
1507 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
1508 if (unlikely(!skb)) {
1509 WARN_ONCE(1, "Build skb failure on Rx\n");
1510 goto free_buffer;
1511 }
1512 WARN_ON(fd_off != priv->rx_headroom);
1513 skb_reserve(skb, fd_off);
1514 skb_put(skb, qm_fd_get_length(fd));
1515
1516 skb->ip_summed = CHECKSUM_NONE;
1517
1518 return skb;
1519
1520free_buffer:
1521 skb_free_frag(vaddr);
1522 return NULL;
1523}
1524
1525/* Build an skb with the data of the first S/G entry in the linear portion and
1526 * the rest of the frame as skb fragments.
1527 *
1528 * The page fragment holding the S/G Table is recycled here.
1529 */
1530static struct sk_buff *sg_fd_to_skb(const struct dpaa_priv *priv,
1531 const struct qm_fd *fd)
1532{
1533 ssize_t fd_off = qm_fd_get_offset(fd);
1534 dma_addr_t addr = qm_fd_addr(fd);
1535 const struct qm_sg_entry *sgt;
1536 struct page *page, *head_page;
1537 struct dpaa_bp *dpaa_bp;
1538 void *vaddr, *sg_vaddr;
1539 int frag_off, frag_len;
1540 struct sk_buff *skb;
1541 dma_addr_t sg_addr;
1542 int page_offset;
1543 unsigned int sz;
1544 int *count_ptr;
1545 int i;
1546
1547 vaddr = phys_to_virt(addr);
1548 WARN_ON(!IS_ALIGNED((unsigned long)vaddr, SMP_CACHE_BYTES));
1549
1550 /* Iterate through the SGT entries and add data buffers to the skb */
1551 sgt = vaddr + fd_off;
1552 for (i = 0; i < DPAA_SGT_MAX_ENTRIES; i++) {
1553 /* Extension bit is not supported */
1554 WARN_ON(qm_sg_entry_is_ext(&sgt[i]));
1555
1556 sg_addr = qm_sg_addr(&sgt[i]);
1557 sg_vaddr = phys_to_virt(sg_addr);
1558 WARN_ON(!IS_ALIGNED((unsigned long)sg_vaddr,
1559 SMP_CACHE_BYTES));
1560
1561 /* We may use multiple Rx pools */
1562 dpaa_bp = dpaa_bpid2pool(sgt[i].bpid);
1563 if (!dpaa_bp)
1564 goto free_buffers;
1565
1566 count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);
1567 dma_unmap_single(dpaa_bp->dev, sg_addr, dpaa_bp->size,
1568 DMA_FROM_DEVICE);
1569 if (i == 0) {
1570 sz = dpaa_bp->size +
1571 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1572 skb = build_skb(sg_vaddr, sz);
1573 if (WARN_ON(unlikely(!skb)))
1574 goto free_buffers;
1575
1576 skb->ip_summed = CHECKSUM_NONE;
1577
1578 /* Make sure forwarded skbs will have enough space
1579 * on Tx, if extra headers are added.
1580 */
1581 WARN_ON(fd_off != priv->rx_headroom);
1582 skb_reserve(skb, fd_off);
1583 skb_put(skb, qm_sg_entry_get_len(&sgt[i]));
1584 } else {
1585 /* Not the first S/G entry; all data from buffer will
1586 * be added in an skb fragment; fragment index is offset
1587 * by one since first S/G entry was incorporated in the
1588 * linear part of the skb.
1589 *
1590 * Caution: 'page' may be a tail page.
1591 */
1592 page = virt_to_page(sg_vaddr);
1593 head_page = virt_to_head_page(sg_vaddr);
1594
1595 /* Compute offset in (possibly tail) page */
1596 page_offset = ((unsigned long)sg_vaddr &
1597 (PAGE_SIZE - 1)) +
1598 (page_address(page) - page_address(head_page));
1599 /* page_offset only refers to the beginning of sgt[i];
1600 * but the buffer itself may have an internal offset.
1601 */
1602 frag_off = qm_sg_entry_get_off(&sgt[i]) + page_offset;
1603 frag_len = qm_sg_entry_get_len(&sgt[i]);
1604 /* skb_add_rx_frag() does no checking on the page; if
1605 * we pass it a tail page, we'll end up with
1606 * bad page accounting and eventually with segafults.
1607 */
1608 skb_add_rx_frag(skb, i - 1, head_page, frag_off,
1609 frag_len, dpaa_bp->size);
1610 }
1611 /* Update the pool count for the current {cpu x bpool} */
1612 (*count_ptr)--;
1613
1614 if (qm_sg_entry_is_final(&sgt[i]))
1615 break;
1616 }
1617 WARN_ONCE(i == DPAA_SGT_MAX_ENTRIES, "No final bit on SGT\n");
1618
1619 /* free the SG table buffer */
1620 skb_free_frag(vaddr);
1621
1622 return skb;
1623
1624free_buffers:
1625 /* compensate sw bpool counter changes */
1626 for (i--; i > 0; i--) {
1627 dpaa_bp = dpaa_bpid2pool(sgt[i].bpid);
1628 if (dpaa_bp) {
1629 count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);
1630 (*count_ptr)++;
1631 }
1632 }
1633 /* free all the SG entries */
1634 for (i = 0; i < DPAA_SGT_MAX_ENTRIES ; i++) {
1635 sg_addr = qm_sg_addr(&sgt[i]);
1636 sg_vaddr = phys_to_virt(sg_addr);
1637 skb_free_frag(sg_vaddr);
1638 dpaa_bp = dpaa_bpid2pool(sgt[i].bpid);
1639 if (dpaa_bp) {
1640 count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);
1641 (*count_ptr)--;
1642 }
1643
1644 if (qm_sg_entry_is_final(&sgt[i]))
1645 break;
1646 }
1647 /* free the SGT fragment */
1648 skb_free_frag(vaddr);
1649
1650 return NULL;
1651}
1652
1653static int skb_to_contig_fd(struct dpaa_priv *priv,
1654 struct sk_buff *skb, struct qm_fd *fd,
1655 int *offset)
1656{
1657 struct net_device *net_dev = priv->net_dev;
1658 struct device *dev = net_dev->dev.parent;
1659 enum dma_data_direction dma_dir;
1660 unsigned char *buffer_start;
1661 struct sk_buff **skbh;
1662 dma_addr_t addr;
1663 int err;
1664
1665 /* We are guaranteed to have at least tx_headroom bytes
1666 * available, so just use that for offset.
1667 */
1668 fd->bpid = FSL_DPAA_BPID_INV;
1669 buffer_start = skb->data - priv->tx_headroom;
1670 dma_dir = DMA_TO_DEVICE;
1671
1672 skbh = (struct sk_buff **)buffer_start;
1673 *skbh = skb;
1674
1675 /* Enable L3/L4 hardware checksum computation.
1676 *
1677 * We must do this before dma_map_single(DMA_TO_DEVICE), because we may
1678 * need to write into the skb.
1679 */
1680 err = dpaa_enable_tx_csum(priv, skb, fd,
1681 ((char *)skbh) + DPAA_TX_PRIV_DATA_SIZE);
1682 if (unlikely(err < 0)) {
1683 if (net_ratelimit())
1684 netif_err(priv, tx_err, net_dev, "HW csum error: %d\n",
1685 err);
1686 return err;
1687 }
1688
1689 /* Fill in the rest of the FD fields */
1690 qm_fd_set_contig(fd, priv->tx_headroom, skb->len);
1691 fd->cmd |= FM_FD_CMD_FCO;
1692
1693 /* Map the entire buffer size that may be seen by FMan, but no more */
1694 addr = dma_map_single(dev, skbh,
1695 skb_tail_pointer(skb) - buffer_start, dma_dir);
1696 if (unlikely(dma_mapping_error(dev, addr))) {
1697 if (net_ratelimit())
1698 netif_err(priv, tx_err, net_dev, "dma_map_single() failed\n");
1699 return -EINVAL;
1700 }
1701 qm_fd_addr_set64(fd, addr);
1702
1703 return 0;
1704}
1705
1706static int skb_to_sg_fd(struct dpaa_priv *priv,
1707 struct sk_buff *skb, struct qm_fd *fd)
1708{
1709 const enum dma_data_direction dma_dir = DMA_TO_DEVICE;
1710 const int nr_frags = skb_shinfo(skb)->nr_frags;
1711 struct net_device *net_dev = priv->net_dev;
1712 struct device *dev = net_dev->dev.parent;
1713 struct qm_sg_entry *sgt;
1714 struct sk_buff **skbh;
1715 int i, j, err, sz;
1716 void *buffer_start;
1717 skb_frag_t *frag;
1718 dma_addr_t addr;
1719 size_t frag_len;
1720 void *sgt_buf;
1721
1722 /* get a page frag to store the SGTable */
1723 sz = SKB_DATA_ALIGN(priv->tx_headroom +
1724 sizeof(struct qm_sg_entry) * (1 + nr_frags));
1725 sgt_buf = netdev_alloc_frag(sz);
1726 if (unlikely(!sgt_buf)) {
1727 netdev_err(net_dev, "netdev_alloc_frag() failed for size %d\n",
1728 sz);
1729 return -ENOMEM;
1730 }
1731
1732 /* Enable L3/L4 hardware checksum computation.
1733 *
1734 * We must do this before dma_map_single(DMA_TO_DEVICE), because we may
1735 * need to write into the skb.
1736 */
1737 err = dpaa_enable_tx_csum(priv, skb, fd,
1738 sgt_buf + DPAA_TX_PRIV_DATA_SIZE);
1739 if (unlikely(err < 0)) {
1740 if (net_ratelimit())
1741 netif_err(priv, tx_err, net_dev, "HW csum error: %d\n",
1742 err);
1743 goto csum_failed;
1744 }
1745
1746 sgt = (struct qm_sg_entry *)(sgt_buf + priv->tx_headroom);
1747 qm_sg_entry_set_len(&sgt[0], skb_headlen(skb));
1748 sgt[0].bpid = FSL_DPAA_BPID_INV;
1749 sgt[0].offset = 0;
1750 addr = dma_map_single(dev, skb->data,
1751 skb_headlen(skb), dma_dir);
1752 if (unlikely(dma_mapping_error(dev, addr))) {
1753 dev_err(dev, "DMA mapping failed");
1754 err = -EINVAL;
1755 goto sg0_map_failed;
1756 }
1757 qm_sg_entry_set64(&sgt[0], addr);
1758
1759 /* populate the rest of SGT entries */
1760 frag = &skb_shinfo(skb)->frags[0];
1761 frag_len = frag->size;
1762 for (i = 1; i <= nr_frags; i++, frag++) {
1763 WARN_ON(!skb_frag_page(frag));
1764 addr = skb_frag_dma_map(dev, frag, 0,
1765 frag_len, dma_dir);
1766 if (unlikely(dma_mapping_error(dev, addr))) {
1767 dev_err(dev, "DMA mapping failed");
1768 err = -EINVAL;
1769 goto sg_map_failed;
1770 }
1771
1772 qm_sg_entry_set_len(&sgt[i], frag_len);
1773 sgt[i].bpid = FSL_DPAA_BPID_INV;
1774 sgt[i].offset = 0;
1775
1776 /* keep the offset in the address */
1777 qm_sg_entry_set64(&sgt[i], addr);
1778 frag_len = frag->size;
1779 }
1780 qm_sg_entry_set_f(&sgt[i - 1], frag_len);
1781
1782 qm_fd_set_sg(fd, priv->tx_headroom, skb->len);
1783
1784 /* DMA map the SGT page */
1785 buffer_start = (void *)sgt - priv->tx_headroom;
1786 skbh = (struct sk_buff **)buffer_start;
1787 *skbh = skb;
1788
1789 addr = dma_map_single(dev, buffer_start, priv->tx_headroom +
1790 sizeof(struct qm_sg_entry) * (1 + nr_frags),
1791 dma_dir);
1792 if (unlikely(dma_mapping_error(dev, addr))) {
1793 dev_err(dev, "DMA mapping failed");
1794 err = -EINVAL;
1795 goto sgt_map_failed;
1796 }
1797
1798 fd->bpid = FSL_DPAA_BPID_INV;
1799 fd->cmd |= FM_FD_CMD_FCO;
1800 qm_fd_addr_set64(fd, addr);
1801
1802 return 0;
1803
1804sgt_map_failed:
1805sg_map_failed:
1806 for (j = 0; j < i; j++)
1807 dma_unmap_page(dev, qm_sg_addr(&sgt[j]),
1808 qm_sg_entry_get_len(&sgt[j]), dma_dir);
1809sg0_map_failed:
1810csum_failed:
1811 skb_free_frag(sgt_buf);
1812
1813 return err;
1814}
1815
1816static inline int dpaa_xmit(struct dpaa_priv *priv,
1817 struct rtnl_link_stats64 *percpu_stats,
1818 int queue,
1819 struct qm_fd *fd)
1820{
1821 struct qman_fq *egress_fq;
1822 int err, i;
1823
1824 egress_fq = priv->egress_fqs[queue];
1825 if (fd->bpid == FSL_DPAA_BPID_INV)
1826 fd->cmd |= qman_fq_fqid(priv->conf_fqs[queue]);
1827
1828 for (i = 0; i < DPAA_ENQUEUE_RETRIES; i++) {
1829 err = qman_enqueue(egress_fq, fd);
1830 if (err != -EBUSY)
1831 break;
1832 }
1833
1834 if (unlikely(err < 0)) {
1835 percpu_stats->tx_errors++;
1836 percpu_stats->tx_fifo_errors++;
1837 return err;
1838 }
1839
1840 percpu_stats->tx_packets++;
1841 percpu_stats->tx_bytes += qm_fd_get_length(fd);
1842
1843 return 0;
1844}
1845
1846static int dpaa_start_xmit(struct sk_buff *skb, struct net_device *net_dev)
1847{
1848 const int queue_mapping = skb_get_queue_mapping(skb);
1849 bool nonlinear = skb_is_nonlinear(skb);
1850 struct rtnl_link_stats64 *percpu_stats;
1851 struct dpaa_percpu_priv *percpu_priv;
1852 struct dpaa_priv *priv;
1853 struct qm_fd fd;
1854 int offset = 0;
1855 int err = 0;
1856
1857 priv = netdev_priv(net_dev);
1858 percpu_priv = this_cpu_ptr(priv->percpu_priv);
1859 percpu_stats = &percpu_priv->stats;
1860
1861 qm_fd_clear_fd(&fd);
1862
1863 if (!nonlinear) {
1864 /* We're going to store the skb backpointer at the beginning
1865 * of the data buffer, so we need a privately owned skb
1866 *
1867 * We've made sure skb is not shared in dev->priv_flags,
1868 * we need to verify the skb head is not cloned
1869 */
1870 if (skb_cow_head(skb, priv->tx_headroom))
1871 goto enomem;
1872
1873 WARN_ON(skb_is_nonlinear(skb));
1874 }
1875
1876 /* MAX_SKB_FRAGS is equal or larger than our dpaa_SGT_MAX_ENTRIES;
1877 * make sure we don't feed FMan with more fragments than it supports.
1878 */
1879 if (nonlinear &&
1880 likely(skb_shinfo(skb)->nr_frags < DPAA_SGT_MAX_ENTRIES)) {
1881 /* Just create a S/G fd based on the skb */
1882 err = skb_to_sg_fd(priv, skb, &fd);
1883 } else {
1884 /* If the egress skb contains more fragments than we support
1885 * we have no choice but to linearize it ourselves.
1886 */
1887 if (unlikely(nonlinear) && __skb_linearize(skb))
1888 goto enomem;
1889
1890 /* Finally, create a contig FD from this skb */
1891 err = skb_to_contig_fd(priv, skb, &fd, &offset);
1892 }
1893 if (unlikely(err < 0))
1894 goto skb_to_fd_failed;
1895
1896 if (likely(dpaa_xmit(priv, percpu_stats, queue_mapping, &fd) == 0))
1897 return NETDEV_TX_OK;
1898
1899 dpaa_cleanup_tx_fd(priv, &fd);
1900skb_to_fd_failed:
1901enomem:
1902 percpu_stats->tx_errors++;
1903 dev_kfree_skb(skb);
1904 return NETDEV_TX_OK;
1905}
1906
1907static void dpaa_rx_error(struct net_device *net_dev,
1908 const struct dpaa_priv *priv,
1909 struct dpaa_percpu_priv *percpu_priv,
1910 const struct qm_fd *fd,
1911 u32 fqid)
1912{
1913 if (net_ratelimit())
1914 netif_err(priv, hw, net_dev, "Err FD status = 0x%08x\n",
1915 fd->status & FM_FD_STAT_RX_ERRORS);
1916
1917 percpu_priv->stats.rx_errors++;
1918
1919 dpaa_fd_release(net_dev, fd);
1920}
1921
1922static void dpaa_tx_error(struct net_device *net_dev,
1923 const struct dpaa_priv *priv,
1924 struct dpaa_percpu_priv *percpu_priv,
1925 const struct qm_fd *fd,
1926 u32 fqid)
1927{
1928 struct sk_buff *skb;
1929
1930 if (net_ratelimit())
1931 netif_warn(priv, hw, net_dev, "FD status = 0x%08x\n",
1932 fd->status & FM_FD_STAT_TX_ERRORS);
1933
1934 percpu_priv->stats.tx_errors++;
1935
1936 skb = dpaa_cleanup_tx_fd(priv, fd);
1937 dev_kfree_skb(skb);
1938}
1939
1940static int dpaa_eth_poll(struct napi_struct *napi, int budget)
1941{
1942 struct dpaa_napi_portal *np =
1943 container_of(napi, struct dpaa_napi_portal, napi);
1944
1945 int cleaned = qman_p_poll_dqrr(np->p, budget);
1946
1947 if (cleaned < budget) {
1948 napi_complete(napi);
1949 qman_p_irqsource_add(np->p, QM_PIRQ_DQRI);
1950
1951 } else if (np->down) {
1952 qman_p_irqsource_add(np->p, QM_PIRQ_DQRI);
1953 }
1954
1955 return cleaned;
1956}
1957
1958static void dpaa_tx_conf(struct net_device *net_dev,
1959 const struct dpaa_priv *priv,
1960 struct dpaa_percpu_priv *percpu_priv,
1961 const struct qm_fd *fd,
1962 u32 fqid)
1963{
1964 struct sk_buff *skb;
1965
1966 if (unlikely(fd->status & FM_FD_STAT_TX_ERRORS) != 0) {
1967 if (net_ratelimit())
1968 netif_warn(priv, hw, net_dev, "FD status = 0x%08x\n",
1969 fd->status & FM_FD_STAT_TX_ERRORS);
1970
1971 percpu_priv->stats.tx_errors++;
1972 }
1973
1974 skb = dpaa_cleanup_tx_fd(priv, fd);
1975
1976 consume_skb(skb);
1977}
1978
1979static inline int dpaa_eth_napi_schedule(struct dpaa_percpu_priv *percpu_priv,
1980 struct qman_portal *portal)
1981{
1982 if (unlikely(in_irq() || !in_serving_softirq())) {
1983 /* Disable QMan IRQ and invoke NAPI */
1984 qman_p_irqsource_remove(portal, QM_PIRQ_DQRI);
1985
1986 percpu_priv->np.p = portal;
1987 napi_schedule(&percpu_priv->np.napi);
1988 return 1;
1989 }
1990 return 0;
1991}
1992
1993static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal,
1994 struct qman_fq *fq,
1995 const struct qm_dqrr_entry *dq)
1996{
1997 struct dpaa_fq *dpaa_fq = container_of(fq, struct dpaa_fq, fq_base);
1998 struct dpaa_percpu_priv *percpu_priv;
1999 struct net_device *net_dev;
2000 struct dpaa_bp *dpaa_bp;
2001 struct dpaa_priv *priv;
2002
2003 net_dev = dpaa_fq->net_dev;
2004 priv = netdev_priv(net_dev);
2005 dpaa_bp = dpaa_bpid2pool(dq->fd.bpid);
2006 if (!dpaa_bp)
2007 return qman_cb_dqrr_consume;
2008
2009 percpu_priv = this_cpu_ptr(priv->percpu_priv);
2010
2011 if (dpaa_eth_napi_schedule(percpu_priv, portal))
2012 return qman_cb_dqrr_stop;
2013
2014 if (dpaa_eth_refill_bpools(priv))
2015 /* Unable to refill the buffer pool due to insufficient
2016 * system memory. Just release the frame back into the pool,
2017 * otherwise we'll soon end up with an empty buffer pool.
2018 */
2019 dpaa_fd_release(net_dev, &dq->fd);
2020 else
2021 dpaa_rx_error(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);
2022
2023 return qman_cb_dqrr_consume;
2024}
2025
2026static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
2027 struct qman_fq *fq,
2028 const struct qm_dqrr_entry *dq)
2029{
2030 struct rtnl_link_stats64 *percpu_stats;
2031 struct dpaa_percpu_priv *percpu_priv;
2032 const struct qm_fd *fd = &dq->fd;
2033 dma_addr_t addr = qm_fd_addr(fd);
2034 enum qm_fd_format fd_format;
2035 struct net_device *net_dev;
2036 u32 fd_status = fd->status;
2037 struct dpaa_bp *dpaa_bp;
2038 struct dpaa_priv *priv;
2039 unsigned int skb_len;
2040 struct sk_buff *skb;
2041 int *count_ptr;
2042
2043 net_dev = ((struct dpaa_fq *)fq)->net_dev;
2044 priv = netdev_priv(net_dev);
2045 dpaa_bp = dpaa_bpid2pool(dq->fd.bpid);
2046 if (!dpaa_bp)
2047 return qman_cb_dqrr_consume;
2048
2049 percpu_priv = this_cpu_ptr(priv->percpu_priv);
2050 percpu_stats = &percpu_priv->stats;
2051
2052 if (unlikely(dpaa_eth_napi_schedule(percpu_priv, portal)))
2053 return qman_cb_dqrr_stop;
2054
2055 /* Make sure we didn't run out of buffers */
2056 if (unlikely(dpaa_eth_refill_bpools(priv))) {
2057 /* Unable to refill the buffer pool due to insufficient
2058 * system memory. Just release the frame back into the pool,
2059 * otherwise we'll soon end up with an empty buffer pool.
2060 */
2061 dpaa_fd_release(net_dev, &dq->fd);
2062 return qman_cb_dqrr_consume;
2063 }
2064
2065 if (unlikely(fd_status & FM_FD_STAT_RX_ERRORS) != 0) {
2066 if (net_ratelimit())
2067 netif_warn(priv, hw, net_dev, "FD status = 0x%08x\n",
2068 fd_status & FM_FD_STAT_RX_ERRORS);
2069
2070 percpu_stats->rx_errors++;
2071 dpaa_fd_release(net_dev, fd);
2072 return qman_cb_dqrr_consume;
2073 }
2074
2075 dpaa_bp = dpaa_bpid2pool(fd->bpid);
2076 if (!dpaa_bp)
2077 return qman_cb_dqrr_consume;
2078
2079 dma_unmap_single(dpaa_bp->dev, addr, dpaa_bp->size, DMA_FROM_DEVICE);
2080
2081 /* prefetch the first 64 bytes of the frame or the SGT start */
2082 prefetch(phys_to_virt(addr) + qm_fd_get_offset(fd));
2083
2084 fd_format = qm_fd_get_format(fd);
2085 /* The only FD types that we may receive are contig and S/G */
2086 WARN_ON((fd_format != qm_fd_contig) && (fd_format != qm_fd_sg));
2087
2088 /* Account for either the contig buffer or the SGT buffer (depending on
2089 * which case we were in) having been removed from the pool.
2090 */
2091 count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);
2092 (*count_ptr)--;
2093
2094 if (likely(fd_format == qm_fd_contig))
2095 skb = contig_fd_to_skb(priv, fd);
2096 else
2097 skb = sg_fd_to_skb(priv, fd);
2098 if (!skb)
2099 return qman_cb_dqrr_consume;
2100
2101 skb->protocol = eth_type_trans(skb, net_dev);
2102
2103 skb_len = skb->len;
2104
2105 if (unlikely(netif_receive_skb(skb) == NET_RX_DROP))
2106 return qman_cb_dqrr_consume;
2107
2108 percpu_stats->rx_packets++;
2109 percpu_stats->rx_bytes += skb_len;
2110
2111 return qman_cb_dqrr_consume;
2112}
2113
2114static enum qman_cb_dqrr_result conf_error_dqrr(struct qman_portal *portal,
2115 struct qman_fq *fq,
2116 const struct qm_dqrr_entry *dq)
2117{
2118 struct dpaa_percpu_priv *percpu_priv;
2119 struct net_device *net_dev;
2120 struct dpaa_priv *priv;
2121
2122 net_dev = ((struct dpaa_fq *)fq)->net_dev;
2123 priv = netdev_priv(net_dev);
2124
2125 percpu_priv = this_cpu_ptr(priv->percpu_priv);
2126
2127 if (dpaa_eth_napi_schedule(percpu_priv, portal))
2128 return qman_cb_dqrr_stop;
2129
2130 dpaa_tx_error(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);
2131
2132 return qman_cb_dqrr_consume;
2133}
2134
2135static enum qman_cb_dqrr_result conf_dflt_dqrr(struct qman_portal *portal,
2136 struct qman_fq *fq,
2137 const struct qm_dqrr_entry *dq)
2138{
2139 struct dpaa_percpu_priv *percpu_priv;
2140 struct net_device *net_dev;
2141 struct dpaa_priv *priv;
2142
2143 net_dev = ((struct dpaa_fq *)fq)->net_dev;
2144 priv = netdev_priv(net_dev);
2145
2146 percpu_priv = this_cpu_ptr(priv->percpu_priv);
2147
2148 if (dpaa_eth_napi_schedule(percpu_priv, portal))
2149 return qman_cb_dqrr_stop;
2150
2151 dpaa_tx_conf(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);
2152
2153 return qman_cb_dqrr_consume;
2154}
2155
2156static void egress_ern(struct qman_portal *portal,
2157 struct qman_fq *fq,
2158 const union qm_mr_entry *msg)
2159{
2160 const struct qm_fd *fd = &msg->ern.fd;
2161 struct dpaa_percpu_priv *percpu_priv;
2162 const struct dpaa_priv *priv;
2163 struct net_device *net_dev;
2164 struct sk_buff *skb;
2165
2166 net_dev = ((struct dpaa_fq *)fq)->net_dev;
2167 priv = netdev_priv(net_dev);
2168 percpu_priv = this_cpu_ptr(priv->percpu_priv);
2169
2170 percpu_priv->stats.tx_dropped++;
2171 percpu_priv->stats.tx_fifo_errors++;
2172
2173 skb = dpaa_cleanup_tx_fd(priv, fd);
2174 dev_kfree_skb_any(skb);
2175}
2176
2177static const struct dpaa_fq_cbs dpaa_fq_cbs = {
2178 .rx_defq = { .cb = { .dqrr = rx_default_dqrr } },
2179 .tx_defq = { .cb = { .dqrr = conf_dflt_dqrr } },
2180 .rx_errq = { .cb = { .dqrr = rx_error_dqrr } },
2181 .tx_errq = { .cb = { .dqrr = conf_error_dqrr } },
2182 .egress_ern = { .cb = { .ern = egress_ern } }
2183};
2184
2185static void dpaa_eth_napi_enable(struct dpaa_priv *priv)
2186{
2187 struct dpaa_percpu_priv *percpu_priv;
2188 int i;
2189
2190 for_each_possible_cpu(i) {
2191 percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
2192
2193 percpu_priv->np.down = 0;
2194 napi_enable(&percpu_priv->np.napi);
2195 }
2196}
2197
2198static void dpaa_eth_napi_disable(struct dpaa_priv *priv)
2199{
2200 struct dpaa_percpu_priv *percpu_priv;
2201 int i;
2202
2203 for_each_possible_cpu(i) {
2204 percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
2205
2206 percpu_priv->np.down = 1;
2207 napi_disable(&percpu_priv->np.napi);
2208 }
2209}
2210
2211static int dpaa_open(struct net_device *net_dev)
2212{
2213 struct mac_device *mac_dev;
2214 struct dpaa_priv *priv;
2215 int err, i;
2216
2217 priv = netdev_priv(net_dev);
2218 mac_dev = priv->mac_dev;
2219 dpaa_eth_napi_enable(priv);
2220
2221 net_dev->phydev = mac_dev->init_phy(net_dev, priv->mac_dev);
2222 if (!net_dev->phydev) {
2223 netif_err(priv, ifup, net_dev, "init_phy() failed\n");
2224 return -ENODEV;
2225 }
2226
2227 for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++) {
2228 err = fman_port_enable(mac_dev->port[i]);
2229 if (err)
2230 goto mac_start_failed;
2231 }
2232
2233 err = priv->mac_dev->start(mac_dev);
2234 if (err < 0) {
2235 netif_err(priv, ifup, net_dev, "mac_dev->start() = %d\n", err);
2236 goto mac_start_failed;
2237 }
2238
2239 netif_tx_start_all_queues(net_dev);
2240
2241 return 0;
2242
2243mac_start_failed:
2244 for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++)
2245 fman_port_disable(mac_dev->port[i]);
2246
2247 dpaa_eth_napi_disable(priv);
2248
2249 return err;
2250}
2251
2252static int dpaa_eth_stop(struct net_device *net_dev)
2253{
2254 struct dpaa_priv *priv;
2255 int err;
2256
2257 err = dpaa_stop(net_dev);
2258
2259 priv = netdev_priv(net_dev);
2260 dpaa_eth_napi_disable(priv);
2261
2262 return err;
2263}
2264
2265static const struct net_device_ops dpaa_ops = {
2266 .ndo_open = dpaa_open,
2267 .ndo_start_xmit = dpaa_start_xmit,
2268 .ndo_stop = dpaa_eth_stop,
2269 .ndo_tx_timeout = dpaa_tx_timeout,
2270 .ndo_get_stats64 = dpaa_get_stats64,
2271 .ndo_set_mac_address = dpaa_set_mac_address,
2272 .ndo_validate_addr = eth_validate_addr,
2273 .ndo_set_rx_mode = dpaa_set_rx_mode,
2274};
2275
2276static int dpaa_napi_add(struct net_device *net_dev)
2277{
2278 struct dpaa_priv *priv = netdev_priv(net_dev);
2279 struct dpaa_percpu_priv *percpu_priv;
2280 int cpu;
2281
2282 for_each_possible_cpu(cpu) {
2283 percpu_priv = per_cpu_ptr(priv->percpu_priv, cpu);
2284
2285 netif_napi_add(net_dev, &percpu_priv->np.napi,
2286 dpaa_eth_poll, NAPI_POLL_WEIGHT);
2287 }
2288
2289 return 0;
2290}
2291
2292static void dpaa_napi_del(struct net_device *net_dev)
2293{
2294 struct dpaa_priv *priv = netdev_priv(net_dev);
2295 struct dpaa_percpu_priv *percpu_priv;
2296 int cpu;
2297
2298 for_each_possible_cpu(cpu) {
2299 percpu_priv = per_cpu_ptr(priv->percpu_priv, cpu);
2300
2301 netif_napi_del(&percpu_priv->np.napi);
2302 }
2303}
2304
2305static inline void dpaa_bp_free_pf(const struct dpaa_bp *bp,
2306 struct bm_buffer *bmb)
2307{
2308 dma_addr_t addr = bm_buf_addr(bmb);
2309
2310 dma_unmap_single(bp->dev, addr, bp->size, DMA_FROM_DEVICE);
2311
2312 skb_free_frag(phys_to_virt(addr));
2313}
2314
2315/* Alloc the dpaa_bp struct and configure default values */
2316static struct dpaa_bp *dpaa_bp_alloc(struct device *dev)
2317{
2318 struct dpaa_bp *dpaa_bp;
2319
2320 dpaa_bp = devm_kzalloc(dev, sizeof(*dpaa_bp), GFP_KERNEL);
2321 if (!dpaa_bp)
2322 return ERR_PTR(-ENOMEM);
2323
2324 dpaa_bp->bpid = FSL_DPAA_BPID_INV;
2325 dpaa_bp->percpu_count = devm_alloc_percpu(dev, *dpaa_bp->percpu_count);
2326 dpaa_bp->config_count = FSL_DPAA_ETH_MAX_BUF_COUNT;
2327
2328 dpaa_bp->seed_cb = dpaa_bp_seed;
2329 dpaa_bp->free_buf_cb = dpaa_bp_free_pf;
2330
2331 return dpaa_bp;
2332}
2333
2334/* Place all ingress FQs (Rx Default, Rx Error) in a dedicated CGR.
2335 * We won't be sending congestion notifications to FMan; for now, we just use
2336 * this CGR to generate enqueue rejections to FMan in order to drop the frames
2337 * before they reach our ingress queues and eat up memory.
2338 */
2339static int dpaa_ingress_cgr_init(struct dpaa_priv *priv)
2340{
2341 struct qm_mcc_initcgr initcgr;
2342 u32 cs_th;
2343 int err;
2344
2345 err = qman_alloc_cgrid(&priv->ingress_cgr.cgrid);
2346 if (err < 0) {
2347 if (netif_msg_drv(priv))
2348 pr_err("Error %d allocating CGR ID\n", err);
2349 goto out_error;
2350 }
2351
2352 /* Enable CS TD, but disable Congestion State Change Notifications. */
2353 initcgr.we_mask = QM_CGR_WE_CS_THRES;
2354 initcgr.cgr.cscn_en = QM_CGR_EN;
2355 cs_th = DPAA_INGRESS_CS_THRESHOLD;
2356 qm_cgr_cs_thres_set64(&initcgr.cgr.cs_thres, cs_th, 1);
2357
2358 initcgr.we_mask |= QM_CGR_WE_CSTD_EN;
2359 initcgr.cgr.cstd_en = QM_CGR_EN;
2360
2361 /* This CGR will be associated with the SWP affined to the current CPU.
2362 * However, we'll place all our ingress FQs in it.
2363 */
2364 err = qman_create_cgr(&priv->ingress_cgr, QMAN_CGR_FLAG_USE_INIT,
2365 &initcgr);
2366 if (err < 0) {
2367 if (netif_msg_drv(priv))
2368 pr_err("Error %d creating ingress CGR with ID %d\n",
2369 err, priv->ingress_cgr.cgrid);
2370 qman_release_cgrid(priv->ingress_cgr.cgrid);
2371 goto out_error;
2372 }
2373 if (netif_msg_drv(priv))
2374 pr_debug("Created ingress CGR %d for netdev with hwaddr %pM\n",
2375 priv->ingress_cgr.cgrid, priv->mac_dev->addr);
2376
2377 priv->use_ingress_cgr = true;
2378
2379out_error:
2380 return err;
2381}
2382
2383static const struct of_device_id dpaa_match[];
2384
2385static inline u16 dpaa_get_headroom(struct dpaa_buffer_layout *bl)
2386{
2387 u16 headroom;
2388
2389 /* The frame headroom must accommodate:
2390 * - the driver private data area
2391 * - parse results, hash results, timestamp if selected
2392 * If either hash results or time stamp are selected, both will
2393 * be copied to/from the frame headroom, as TS is located between PR and
2394 * HR in the IC and IC copy size has a granularity of 16bytes
2395 * (see description of FMBM_RICP and FMBM_TICP registers in DPAARM)
2396 *
2397 * Also make sure the headroom is a multiple of data_align bytes
2398 */
2399 headroom = (u16)(bl->priv_data_size + DPAA_PARSE_RESULTS_SIZE +
2400 DPAA_TIME_STAMP_SIZE + DPAA_HASH_RESULTS_SIZE);
2401
2402 return DPAA_FD_DATA_ALIGNMENT ? ALIGN(headroom,
2403 DPAA_FD_DATA_ALIGNMENT) :
2404 headroom;
2405}
2406
2407static int dpaa_eth_probe(struct platform_device *pdev)
2408{
2409 struct dpaa_bp *dpaa_bps[DPAA_BPS_NUM] = {NULL};
2410 struct dpaa_percpu_priv *percpu_priv;
2411 struct net_device *net_dev = NULL;
2412 struct dpaa_fq *dpaa_fq, *tmp;
2413 struct dpaa_priv *priv = NULL;
2414 struct fm_port_fqs port_fqs;
2415 struct mac_device *mac_dev;
2416 int err = 0, i, channel;
2417 struct device *dev;
2418
2419 dev = &pdev->dev;
2420
2421 /* Allocate this early, so we can store relevant information in
2422 * the private area
2423 */
2424 net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA_ETH_TXQ_NUM);
2425 if (!net_dev) {
2426 dev_err(dev, "alloc_etherdev_mq() failed\n");
2427 goto alloc_etherdev_mq_failed;
2428 }
2429
2430 /* Do this here, so we can be verbose early */
2431 SET_NETDEV_DEV(net_dev, dev);
2432 dev_set_drvdata(dev, net_dev);
2433
2434 priv = netdev_priv(net_dev);
2435 priv->net_dev = net_dev;
2436
2437 priv->msg_enable = netif_msg_init(debug, DPAA_MSG_DEFAULT);
2438
2439 mac_dev = dpaa_mac_dev_get(pdev);
2440 if (IS_ERR(mac_dev)) {
2441 dev_err(dev, "dpaa_mac_dev_get() failed\n");
2442 err = PTR_ERR(mac_dev);
2443 goto mac_probe_failed;
2444 }
2445
2446 /* If fsl_fm_max_frm is set to a higher value than the all-common 1500,
2447 * we choose conservatively and let the user explicitly set a higher
2448 * MTU via ifconfig. Otherwise, the user may end up with different MTUs
2449 * in the same LAN.
2450 * If on the other hand fsl_fm_max_frm has been chosen below 1500,
2451 * start with the maximum allowed.
2452 */
2453 net_dev->mtu = min(dpaa_get_max_mtu(), ETH_DATA_LEN);
2454
2455 netdev_dbg(net_dev, "Setting initial MTU on net device: %d\n",
2456 net_dev->mtu);
2457
2458 priv->buf_layout[RX].priv_data_size = DPAA_RX_PRIV_DATA_SIZE; /* Rx */
2459 priv->buf_layout[TX].priv_data_size = DPAA_TX_PRIV_DATA_SIZE; /* Tx */
2460
2461 /* device used for DMA mapping */
2462 arch_setup_dma_ops(dev, 0, 0, NULL, false);
2463 err = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(40));
2464 if (err) {
2465 dev_err(dev, "dma_coerce_mask_and_coherent() failed\n");
2466 goto dev_mask_failed;
2467 }
2468
2469 /* bp init */
2470 for (i = 0; i < DPAA_BPS_NUM; i++) {
2471 int err;
2472
2473 dpaa_bps[i] = dpaa_bp_alloc(dev);
2474 if (IS_ERR(dpaa_bps[i]))
2475 return PTR_ERR(dpaa_bps[i]);
2476 /* the raw size of the buffers used for reception */
2477 dpaa_bps[i]->raw_size = bpool_buffer_raw_size(i, DPAA_BPS_NUM);
2478 /* avoid runtime computations by keeping the usable size here */
2479 dpaa_bps[i]->size = dpaa_bp_size(dpaa_bps[i]->raw_size);
2480 dpaa_bps[i]->dev = dev;
2481
2482 err = dpaa_bp_alloc_pool(dpaa_bps[i]);
2483 if (err < 0) {
2484 dpaa_bps_free(priv);
2485 priv->dpaa_bps[i] = NULL;
2486 goto bp_create_failed;
2487 }
2488 priv->dpaa_bps[i] = dpaa_bps[i];
2489 }
2490
2491 INIT_LIST_HEAD(&priv->dpaa_fq_list);
2492
2493 memset(&port_fqs, 0, sizeof(port_fqs));
2494
2495 err = dpaa_alloc_all_fqs(dev, &priv->dpaa_fq_list, &port_fqs);
2496 if (err < 0) {
2497 dev_err(dev, "dpaa_alloc_all_fqs() failed\n");
2498 goto fq_probe_failed;
2499 }
2500
2501 priv->mac_dev = mac_dev;
2502
2503 channel = dpaa_get_channel();
2504 if (channel < 0) {
2505 dev_err(dev, "dpaa_get_channel() failed\n");
2506 err = channel;
2507 goto get_channel_failed;
2508 }
2509
2510 priv->channel = (u16)channel;
2511
2512 /* Start a thread that will walk the CPUs with affine portals
2513 * and add this pool channel to each's dequeue mask.
2514 */
2515 dpaa_eth_add_channel(priv->channel);
2516
2517 dpaa_fq_setup(priv, &dpaa_fq_cbs, priv->mac_dev->port[TX]);
2518
2519 /* Create a congestion group for this netdev, with
2520 * dynamically-allocated CGR ID.
2521 * Must be executed after probing the MAC, but before
2522 * assigning the egress FQs to the CGRs.
2523 */
2524 err = dpaa_eth_cgr_init(priv);
2525 if (err < 0) {
2526 dev_err(dev, "Error initializing CGR\n");
2527 goto tx_cgr_init_failed;
2528 }
2529
2530 err = dpaa_ingress_cgr_init(priv);
2531 if (err < 0) {
2532 dev_err(dev, "Error initializing ingress CGR\n");
2533 goto rx_cgr_init_failed;
2534 }
2535
2536 /* Add the FQs to the interface, and make them active */
2537 list_for_each_entry_safe(dpaa_fq, tmp, &priv->dpaa_fq_list, list) {
2538 err = dpaa_fq_init(dpaa_fq, false);
2539 if (err < 0)
2540 goto fq_alloc_failed;
2541 }
2542
2543 priv->tx_headroom = dpaa_get_headroom(&priv->buf_layout[TX]);
2544 priv->rx_headroom = dpaa_get_headroom(&priv->buf_layout[RX]);
2545
2546 /* All real interfaces need their ports initialized */
2547 dpaa_eth_init_ports(mac_dev, dpaa_bps, DPAA_BPS_NUM, &port_fqs,
2548 &priv->buf_layout[0], dev);
2549
2550 priv->percpu_priv = devm_alloc_percpu(dev, *priv->percpu_priv);
2551 if (!priv->percpu_priv) {
2552 dev_err(dev, "devm_alloc_percpu() failed\n");
2553 err = -ENOMEM;
2554 goto alloc_percpu_failed;
2555 }
2556 for_each_possible_cpu(i) {
2557 percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
2558 memset(percpu_priv, 0, sizeof(*percpu_priv));
2559 }
2560
2561 /* Initialize NAPI */
2562 err = dpaa_napi_add(net_dev);
2563 if (err < 0)
2564 goto napi_add_failed;
2565
2566 err = dpaa_netdev_init(net_dev, &dpaa_ops, tx_timeout);
2567 if (err < 0)
2568 goto netdev_init_failed;
2569
2570 netif_info(priv, probe, net_dev, "Probed interface %s\n",
2571 net_dev->name);
2572
2573 return 0;
2574
2575netdev_init_failed:
2576napi_add_failed:
2577 dpaa_napi_del(net_dev);
2578alloc_percpu_failed:
2579 dpaa_fq_free(dev, &priv->dpaa_fq_list);
2580fq_alloc_failed:
2581 qman_delete_cgr_safe(&priv->ingress_cgr);
2582 qman_release_cgrid(priv->ingress_cgr.cgrid);
2583rx_cgr_init_failed:
2584 qman_delete_cgr_safe(&priv->cgr_data.cgr);
2585 qman_release_cgrid(priv->cgr_data.cgr.cgrid);
2586tx_cgr_init_failed:
2587get_channel_failed:
2588 dpaa_bps_free(priv);
2589bp_create_failed:
2590fq_probe_failed:
2591dev_mask_failed:
2592mac_probe_failed:
2593 dev_set_drvdata(dev, NULL);
2594 free_netdev(net_dev);
2595alloc_etherdev_mq_failed:
2596 for (i = 0; i < DPAA_BPS_NUM && dpaa_bps[i]; i++) {
2597 if (atomic_read(&dpaa_bps[i]->refs) == 0)
2598 devm_kfree(dev, dpaa_bps[i]);
2599 }
2600 return err;
2601}
2602
2603static int dpaa_remove(struct platform_device *pdev)
2604{
2605 struct net_device *net_dev;
2606 struct dpaa_priv *priv;
2607 struct device *dev;
2608 int err;
2609
2610 dev = &pdev->dev;
2611 net_dev = dev_get_drvdata(dev);
2612
2613 priv = netdev_priv(net_dev);
2614
2615 dev_set_drvdata(dev, NULL);
2616 unregister_netdev(net_dev);
2617
2618 err = dpaa_fq_free(dev, &priv->dpaa_fq_list);
2619
2620 qman_delete_cgr_safe(&priv->ingress_cgr);
2621 qman_release_cgrid(priv->ingress_cgr.cgrid);
2622 qman_delete_cgr_safe(&priv->cgr_data.cgr);
2623 qman_release_cgrid(priv->cgr_data.cgr.cgrid);
2624
2625 dpaa_napi_del(net_dev);
2626
2627 dpaa_bps_free(priv);
2628
2629 free_netdev(net_dev);
2630
2631 return err;
2632}
2633
2634static struct platform_device_id dpaa_devtype[] = {
2635 {
2636 .name = "dpaa-ethernet",
2637 .driver_data = 0,
2638 }, {
2639 }
2640};
2641MODULE_DEVICE_TABLE(platform, dpaa_devtype);
2642
2643static struct platform_driver dpaa_driver = {
2644 .driver = {
2645 .name = KBUILD_MODNAME,
2646 },
2647 .id_table = dpaa_devtype,
2648 .probe = dpaa_eth_probe,
2649 .remove = dpaa_remove
2650};
2651
2652static int __init dpaa_load(void)
2653{
2654 int err;
2655
2656 pr_debug("FSL DPAA Ethernet driver\n");
2657
2658 /* initialize dpaa_eth mirror values */
2659 dpaa_rx_extra_headroom = fman_get_rx_extra_headroom();
2660 dpaa_max_frm = fman_get_max_frm();
2661
2662 err = platform_driver_register(&dpaa_driver);
2663 if (err < 0)
2664 pr_err("Error, platform_driver_register() = %d\n", err);
2665
2666 return err;
2667}
2668module_init(dpaa_load);
2669
2670static void __exit dpaa_unload(void)
2671{
2672 platform_driver_unregister(&dpaa_driver);
2673
2674 /* Only one channel is used and needs to be released after all
2675 * interfaces are removed
2676 */
2677 dpaa_release_channel();
2678}
2679module_exit(dpaa_unload);
2680
2681MODULE_LICENSE("Dual BSD/GPL");
2682MODULE_DESCRIPTION("FSL DPAA Ethernet driver");
diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.h b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.h
new file mode 100644
index 000000000000..fe98e089cf4d
--- /dev/null
+++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.h
@@ -0,0 +1,144 @@
1/* Copyright 2008 - 2016 Freescale Semiconductor Inc.
2 *
3 * Redistribution and use in source and binary forms, with or without
4 * modification, are permitted provided that the following conditions are met:
5 * * Redistributions of source code must retain the above copyright
6 * notice, this list of conditions and the following disclaimer.
7 * * Redistributions in binary form must reproduce the above copyright
8 * notice, this list of conditions and the following disclaimer in the
9 * documentation and/or other materials provided with the distribution.
10 * * Neither the name of Freescale Semiconductor nor the
11 * names of its contributors may be used to endorse or promote products
12 * derived from this software without specific prior written permission.
13 *
14 * ALTERNATIVELY, this software may be distributed under the terms of the
15 * GNU General Public License ("GPL") as published by the Free Software
16 * Foundation, either version 2 of that License or (at your option) any
17 * later version.
18 *
19 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
20 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
26 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31#ifndef __DPAA_H
32#define __DPAA_H
33
34#include <linux/netdevice.h>
35#include <soc/fsl/qman.h>
36#include <soc/fsl/bman.h>
37
38#include "fman.h"
39#include "mac.h"
40
41#define DPAA_ETH_TXQ_NUM NR_CPUS
42
43#define DPAA_BPS_NUM 3 /* number of bpools per interface */
44
45/* More detailed FQ types - used for fine-grained WQ assignments */
46enum dpaa_fq_type {
47 FQ_TYPE_RX_DEFAULT = 1, /* Rx Default FQs */
48 FQ_TYPE_RX_ERROR, /* Rx Error FQs */
49 FQ_TYPE_TX, /* "Real" Tx FQs */
50 FQ_TYPE_TX_CONFIRM, /* Tx default Conf FQ (actually an Rx FQ) */
51 FQ_TYPE_TX_CONF_MQ, /* Tx conf FQs (one for each Tx FQ) */
52 FQ_TYPE_TX_ERROR, /* Tx Error FQs (these are actually Rx FQs) */
53};
54
55struct dpaa_fq {
56 struct qman_fq fq_base;
57 struct list_head list;
58 struct net_device *net_dev;
59 bool init;
60 u32 fqid;
61 u32 flags;
62 u16 channel;
63 u8 wq;
64 enum dpaa_fq_type fq_type;
65};
66
67struct dpaa_fq_cbs {
68 struct qman_fq rx_defq;
69 struct qman_fq tx_defq;
70 struct qman_fq rx_errq;
71 struct qman_fq tx_errq;
72 struct qman_fq egress_ern;
73};
74
75struct dpaa_bp {
76 /* device used in the DMA mapping operations */
77 struct device *dev;
78 /* current number of buffers in the buffer pool alloted to each CPU */
79 int __percpu *percpu_count;
80 /* all buffers allocated for this pool have this raw size */
81 size_t raw_size;
82 /* all buffers in this pool have this same usable size */
83 size_t size;
84 /* the buffer pools are initialized with config_count buffers for each
85 * CPU; at runtime the number of buffers per CPU is constantly brought
86 * back to this level
87 */
88 u16 config_count;
89 u8 bpid;
90 struct bman_pool *pool;
91 /* bpool can be seeded before use by this cb */
92 int (*seed_cb)(struct dpaa_bp *);
93 /* bpool can be emptied before freeing by this cb */
94 void (*free_buf_cb)(const struct dpaa_bp *, struct bm_buffer *);
95 atomic_t refs;
96};
97
98struct dpaa_napi_portal {
99 struct napi_struct napi;
100 struct qman_portal *p;
101 bool down;
102};
103
104struct dpaa_percpu_priv {
105 struct net_device *net_dev;
106 struct dpaa_napi_portal np;
107 struct rtnl_link_stats64 stats;
108};
109
110struct dpaa_buffer_layout {
111 u16 priv_data_size;
112};
113
114struct dpaa_priv {
115 struct dpaa_percpu_priv __percpu *percpu_priv;
116 struct dpaa_bp *dpaa_bps[DPAA_BPS_NUM];
117 /* Store here the needed Tx headroom for convenience and speed
118 * (even though it can be computed based on the fields of buf_layout)
119 */
120 u16 tx_headroom;
121 struct net_device *net_dev;
122 struct mac_device *mac_dev;
123 struct qman_fq *egress_fqs[DPAA_ETH_TXQ_NUM];
124 struct qman_fq *conf_fqs[DPAA_ETH_TXQ_NUM];
125
126 u16 channel;
127 struct list_head dpaa_fq_list;
128
129 u32 msg_enable; /* net_device message level */
130
131 struct {
132 /* All egress queues to a given net device belong to one
133 * (and the same) congestion group.
134 */
135 struct qman_cgr cgr;
136 } cgr_data;
137 /* Use a per-port CGR for ingress traffic. */
138 bool use_ingress_cgr;
139 struct qman_cgr ingress_cgr;
140
141 struct dpaa_buffer_layout buf_layout[2];
142 u16 rx_headroom;
143};
144#endif /* __DPAA_H */
generated by cgit v1.2.2 (git 2.25.0) at 2025-10-03 02:51:39 -0400