1 files changed, 70 insertions, 168 deletions

diff --git a/net/core/dev.c b/net/core/dev.c
index 382df6c09eec..b715a55cccc4 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -170,25 +170,6 @@ static DEFINE_SPINLOCK(ptype_lock);
 static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
 static struct list_head ptype_all __read_mostly;        /* Taps */
 
-#ifdef CONFIG_NET_DMA
-struct net_dma {
-        struct dma_client client;
-        spinlock_t lock;
-        cpumask_t channel_mask;
-        struct dma_chan **channels;
-};
-
-static enum dma_state_client
-netdev_dma_event(struct dma_client *client, struct dma_chan *chan,
-        enum dma_state state);
-
-static struct net_dma net_dma = {
-        .client = {
-                .event_callback = netdev_dma_event,
-        },
-};
-#endif
-
 /*
  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
  * semaphore.
@@ -1107,6 +1088,11 @@ int dev_open(struct net_device *dev)
                 dev->flags |= IFF_UP;
 
                 /*
+                 *      Enable NET_DMA
+                 */
+                dmaengine_get();
+
+                /*
                  *      Initialize multicasting status
                  */
                 dev_set_rx_mode(dev);
@@ -1183,6 +1169,11 @@ int dev_close(struct net_device *dev)
          */
         call_netdevice_notifiers(NETDEV_DOWN, dev);
 
+        /*
+         *      Shutdown NET_DMA
+         */
+        dmaengine_put();
+
         return 0;
 }
 
@@ -2387,7 +2378,7 @@ void napi_gro_flush(struct napi_struct *napi)
 }
 EXPORT_SYMBOL(napi_gro_flush);
 
-static int __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
+int dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
 {
         struct sk_buff **pp = NULL;
         struct packet_type *ptype;
@@ -2417,11 +2408,14 @@ static int __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
 
                 for (p = napi->gro_list; p; p = p->next) {
                         count++;
-                        NAPI_GRO_CB(p)->same_flow =
-                                p->mac_len == mac_len &&
-                                !memcmp(skb_mac_header(p), skb_mac_header(skb),
-                                        mac_len);
-                        NAPI_GRO_CB(p)->flush = 0;
+
+                        if (!NAPI_GRO_CB(p)->same_flow)
+                                continue;
+
+                        if (p->mac_len != mac_len ||
+                            memcmp(skb_mac_header(p), skb_mac_header(skb),
+                                   mac_len))
+                                NAPI_GRO_CB(p)->same_flow = 0;
                 }
 
                 pp = ptype->gro_receive(&napi->gro_list, skb);
@@ -2463,6 +2457,19 @@ ok:
 normal:
         return -1;
 }
+EXPORT_SYMBOL(dev_gro_receive);
+
+static int __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
+{
+        struct sk_buff *p;
+
+        for (p = napi->gro_list; p; p = p->next) {
+                NAPI_GRO_CB(p)->same_flow = 1;
+                NAPI_GRO_CB(p)->flush = 0;
+        }
+
+        return dev_gro_receive(napi, skb);
+}
 
 int napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
 {
@@ -2479,11 +2486,26 @@ int napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
 }
 EXPORT_SYMBOL(napi_gro_receive);
 
-int napi_gro_frags(struct napi_struct *napi, struct napi_gro_fraginfo *info)
+void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
+{
+        skb_shinfo(skb)->nr_frags = 0;
+
+        skb->len -= skb->data_len;
+        skb->truesize -= skb->data_len;
+        skb->data_len = 0;
+
+        __skb_pull(skb, skb_headlen(skb));
+        skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
+
+        napi->skb = skb;
+}
+EXPORT_SYMBOL(napi_reuse_skb);
+
+struct sk_buff *napi_fraginfo_skb(struct napi_struct *napi,
+                                  struct napi_gro_fraginfo *info)
 {
         struct net_device *dev = napi->dev;
         struct sk_buff *skb = napi->skb;
-        int err = NET_RX_DROP;
 
         napi->skb = NULL;
 
@@ -2503,16 +2525,31 @@ int napi_gro_frags(struct napi_struct *napi, struct napi_gro_fraginfo *info)
         skb->len += info->len;
         skb->truesize += info->len;
 
-        if (!pskb_may_pull(skb, ETH_HLEN))
-                goto reuse;
-
-        err = NET_RX_SUCCESS;
+        if (!pskb_may_pull(skb, ETH_HLEN)) {
+                napi_reuse_skb(napi, skb);
+                goto out;
+        }
 
         skb->protocol = eth_type_trans(skb, dev);
 
         skb->ip_summed = info->ip_summed;
         skb->csum = info->csum;
 
+out:
+        return skb;
+}
+EXPORT_SYMBOL(napi_fraginfo_skb);
+
+int napi_gro_frags(struct napi_struct *napi, struct napi_gro_fraginfo *info)
+{
+        struct sk_buff *skb = napi_fraginfo_skb(napi, info);
+        int err = NET_RX_DROP;
+
+        if (!skb)
+                goto out;
+
+        err = NET_RX_SUCCESS;
+
         switch (__napi_gro_receive(napi, skb)) {
         case -1:
                 return netif_receive_skb(skb);
@@ -2521,17 +2558,7 @@ int napi_gro_frags(struct napi_struct *napi, struct napi_gro_fraginfo *info)
                 goto out;
         }
 
-reuse:
-        skb_shinfo(skb)->nr_frags = 0;
-
-        skb->len -= skb->data_len;
-        skb->truesize -= skb->data_len;
-        skb->data_len = 0;
-
-        __skb_pull(skb, skb_headlen(skb));
-        skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
-
-        napi->skb = skb;
+        napi_reuse_skb(napi, skb);
 
 out:
         return err;
@@ -2718,14 +2745,7 @@ out:
          * There may not be any more sk_buffs coming right now, so push
          * any pending DMA copies to hardware
          */
-        if (!cpus_empty(net_dma.channel_mask)) {
-                int chan_idx;
-                for_each_cpu_mask_nr(chan_idx, net_dma.channel_mask) {
-                        struct dma_chan *chan = net_dma.channels[chan_idx];
-                        if (chan)
-                                dma_async_memcpy_issue_pending(chan);
-                }
-        }
+        dma_issue_pending_all();
 #endif
 
         return;
@@ -4916,122 +4936,6 @@ static int dev_cpu_callback(struct notifier_block *nfb,
         return NOTIFY_OK;
 }
 
-#ifdef CONFIG_NET_DMA
-/**
- * net_dma_rebalance - try to maintain one DMA channel per CPU
- * @net_dma: DMA client and associated data (lock, channels, channel_mask)
- *
- * This is called when the number of channels allocated to the net_dma client
- * changes.  The net_dma client tries to have one DMA channel per CPU.
- */
-
-static void net_dma_rebalance(struct net_dma *net_dma)
-{
-        unsigned int cpu, i, n, chan_idx;
-        struct dma_chan *chan;
-
-        if (cpus_empty(net_dma->channel_mask)) {
-                for_each_online_cpu(cpu)
-                        rcu_assign_pointer(per_cpu(softnet_data, cpu).net_dma, NULL);
-                return;
-        }
-
-        i = 0;
-        cpu = first_cpu(cpu_online_map);
-
-        for_each_cpu_mask_nr(chan_idx, net_dma->channel_mask) {
-                chan = net_dma->channels[chan_idx];
-
-                n = ((num_online_cpus() / cpus_weight(net_dma->channel_mask))
-                   + (i < (num_online_cpus() %
-                        cpus_weight(net_dma->channel_mask)) ? 1 : 0));
-
-                while(n) {
-                        per_cpu(softnet_data, cpu).net_dma = chan;
-                        cpu = next_cpu(cpu, cpu_online_map);
-                        n--;
-                }
-                i++;
-        }
-}
-
-/**
- * netdev_dma_event - event callback for the net_dma_client
- * @client: should always be net_dma_client
- * @chan: DMA channel for the event
- * @state: DMA state to be handled
- */
-static enum dma_state_client
-netdev_dma_event(struct dma_client *client, struct dma_chan *chan,
-        enum dma_state state)
-{
-        int i, found = 0, pos = -1;
-        struct net_dma *net_dma =
-                container_of(client, struct net_dma, client);
-        enum dma_state_client ack = DMA_DUP; /* default: take no action */
-
-        spin_lock(&net_dma->lock);
-        switch (state) {
-        case DMA_RESOURCE_AVAILABLE:
-                for (i = 0; i < nr_cpu_ids; i++)
-                        if (net_dma->channels[i] == chan) {
-                                found = 1;
-                                break;
-                        } else if (net_dma->channels[i] == NULL && pos < 0)
-                                pos = i;
-
-                if (!found && pos >= 0) {
-                        ack = DMA_ACK;
-                        net_dma->channels[pos] = chan;
-                        cpu_set(pos, net_dma->channel_mask);
-                        net_dma_rebalance(net_dma);
-                }
-                break;
-        case DMA_RESOURCE_REMOVED:
-                for (i = 0; i < nr_cpu_ids; i++)
-                        if (net_dma->channels[i] == chan) {
-                                found = 1;
-                                pos = i;
-                                break;
-                        }
-
-                if (found) {
-                        ack = DMA_ACK;
-                        cpu_clear(pos, net_dma->channel_mask);
-                        net_dma->channels[i] = NULL;
-                        net_dma_rebalance(net_dma);
-                }
-                break;
-        default:
-                break;
-        }
-        spin_unlock(&net_dma->lock);
-
-        return ack;
-}
-
-/**
- * netdev_dma_register - register the networking subsystem as a DMA client
- */
-static int __init netdev_dma_register(void)
-{
-        net_dma.channels = kzalloc(nr_cpu_ids * sizeof(struct net_dma),
-                                                                GFP_KERNEL);
-        if (unlikely(!net_dma.channels)) {
-                printk(KERN_NOTICE
-                                "netdev_dma: no memory for net_dma.channels\n");
-                return -ENOMEM;
-        }
-        spin_lock_init(&net_dma.lock);
-        dma_cap_set(DMA_MEMCPY, net_dma.client.cap_mask);
-        dma_async_client_register(&net_dma.client);
-        dma_async_client_chan_request(&net_dma.client);
-        return 0;
-}
-
-#else
-static int __init netdev_dma_register(void) { return -ENODEV; }
-#endif /* CONFIG_NET_DMA */
 
 /**
  *      netdev_increment_features - increment feature set by one
@@ -5251,8 +5155,6 @@ static int __init net_dev_init(void)
         if (register_pernet_device(&default_device_ops))
                 goto out;
 
-        netdev_dma_register();
-
         open_softirq(NET_TX_SOFTIRQ, net_tx_action);
         open_softirq(NET_RX_SOFTIRQ, net_rx_action);