aboutsummaryrefslogtreecommitdiffstats
path: root/net/core/netpoll.c
diff options
context:
space:
mode:
Diffstat (limited to 'net/core/netpoll.c')
-rw-r--r--net/core/netpoll.c587
1 files changed, 59 insertions, 528 deletions
diff --git a/net/core/netpoll.c b/net/core/netpoll.c
index df9e6b1a9759..e33937fb32a0 100644
--- a/net/core/netpoll.c
+++ b/net/core/netpoll.c
@@ -46,13 +46,9 @@
46 46
47static struct sk_buff_head skb_pool; 47static struct sk_buff_head skb_pool;
48 48
49static atomic_t trapped;
50
51DEFINE_STATIC_SRCU(netpoll_srcu); 49DEFINE_STATIC_SRCU(netpoll_srcu);
52 50
53#define USEC_PER_POLL 50 51#define USEC_PER_POLL 50
54#define NETPOLL_RX_ENABLED 1
55#define NETPOLL_RX_DROP 2
56 52
57#define MAX_SKB_SIZE \ 53#define MAX_SKB_SIZE \
58 (sizeof(struct ethhdr) + \ 54 (sizeof(struct ethhdr) + \
@@ -61,7 +57,6 @@ DEFINE_STATIC_SRCU(netpoll_srcu);
61 MAX_UDP_CHUNK) 57 MAX_UDP_CHUNK)
62 58
63static void zap_completion_queue(void); 59static void zap_completion_queue(void);
64static void netpoll_neigh_reply(struct sk_buff *skb, struct netpoll_info *npinfo);
65static void netpoll_async_cleanup(struct work_struct *work); 60static void netpoll_async_cleanup(struct work_struct *work);
66 61
67static unsigned int carrier_timeout = 4; 62static unsigned int carrier_timeout = 4;
@@ -74,6 +69,37 @@ module_param(carrier_timeout, uint, 0644);
74#define np_notice(np, fmt, ...) \ 69#define np_notice(np, fmt, ...) \
75 pr_notice("%s: " fmt, np->name, ##__VA_ARGS__) 70 pr_notice("%s: " fmt, np->name, ##__VA_ARGS__)
76 71
72static int netpoll_start_xmit(struct sk_buff *skb, struct net_device *dev,
73 struct netdev_queue *txq)
74{
75 const struct net_device_ops *ops = dev->netdev_ops;
76 int status = NETDEV_TX_OK;
77 netdev_features_t features;
78
79 features = netif_skb_features(skb);
80
81 if (vlan_tx_tag_present(skb) &&
82 !vlan_hw_offload_capable(features, skb->vlan_proto)) {
83 skb = __vlan_put_tag(skb, skb->vlan_proto,
84 vlan_tx_tag_get(skb));
85 if (unlikely(!skb)) {
86 /* This is actually a packet drop, but we
87 * don't want the code that calls this
88 * function to try and operate on a NULL skb.
89 */
90 goto out;
91 }
92 skb->vlan_tci = 0;
93 }
94
95 status = ops->ndo_start_xmit(skb, dev);
96 if (status == NETDEV_TX_OK)
97 txq_trans_update(txq);
98
99out:
100 return status;
101}
102
77static void queue_process(struct work_struct *work) 103static void queue_process(struct work_struct *work)
78{ 104{
79 struct netpoll_info *npinfo = 105 struct netpoll_info *npinfo =
@@ -83,51 +109,31 @@ static void queue_process(struct work_struct *work)
83 109
84 while ((skb = skb_dequeue(&npinfo->txq))) { 110 while ((skb = skb_dequeue(&npinfo->txq))) {
85 struct net_device *dev = skb->dev; 111 struct net_device *dev = skb->dev;
86 const struct net_device_ops *ops = dev->netdev_ops;
87 struct netdev_queue *txq; 112 struct netdev_queue *txq;
88 113
89 if (!netif_device_present(dev) || !netif_running(dev)) { 114 if (!netif_device_present(dev) || !netif_running(dev)) {
90 __kfree_skb(skb); 115 kfree_skb(skb);
91 continue; 116 continue;
92 } 117 }
93 118
94 txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb)); 119 txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
95 120
96 local_irq_save(flags); 121 local_irq_save(flags);
97 __netif_tx_lock(txq, smp_processor_id()); 122 HARD_TX_LOCK(dev, txq, smp_processor_id());
98 if (netif_xmit_frozen_or_stopped(txq) || 123 if (netif_xmit_frozen_or_stopped(txq) ||
99 ops->ndo_start_xmit(skb, dev) != NETDEV_TX_OK) { 124 netpoll_start_xmit(skb, dev, txq) != NETDEV_TX_OK) {
100 skb_queue_head(&npinfo->txq, skb); 125 skb_queue_head(&npinfo->txq, skb);
101 __netif_tx_unlock(txq); 126 HARD_TX_UNLOCK(dev, txq);
102 local_irq_restore(flags); 127 local_irq_restore(flags);
103 128
104 schedule_delayed_work(&npinfo->tx_work, HZ/10); 129 schedule_delayed_work(&npinfo->tx_work, HZ/10);
105 return; 130 return;
106 } 131 }
107 __netif_tx_unlock(txq); 132 HARD_TX_UNLOCK(dev, txq);
108 local_irq_restore(flags); 133 local_irq_restore(flags);
109 } 134 }
110} 135}
111 136
112static __sum16 checksum_udp(struct sk_buff *skb, struct udphdr *uh,
113 unsigned short ulen, __be32 saddr, __be32 daddr)
114{
115 __wsum psum;
116
117 if (uh->check == 0 || skb_csum_unnecessary(skb))
118 return 0;
119
120 psum = csum_tcpudp_nofold(saddr, daddr, ulen, IPPROTO_UDP, 0);
121
122 if (skb->ip_summed == CHECKSUM_COMPLETE &&
123 !csum_fold(csum_add(psum, skb->csum)))
124 return 0;
125
126 skb->csum = psum;
127
128 return __skb_checksum_complete(skb);
129}
130
131/* 137/*
132 * Check whether delayed processing was scheduled for our NIC. If so, 138 * Check whether delayed processing was scheduled for our NIC. If so,
133 * we attempt to grab the poll lock and use ->poll() to pump the card. 139 * we attempt to grab the poll lock and use ->poll() to pump the card.
@@ -138,14 +144,8 @@ static __sum16 checksum_udp(struct sk_buff *skb, struct udphdr *uh,
138 * trylock here and interrupts are already disabled in the softirq 144 * trylock here and interrupts are already disabled in the softirq
139 * case. Further, we test the poll_owner to avoid recursion on UP 145 * case. Further, we test the poll_owner to avoid recursion on UP
140 * systems where the lock doesn't exist. 146 * systems where the lock doesn't exist.
141 *
142 * In cases where there is bi-directional communications, reading only
143 * one message at a time can lead to packets being dropped by the
144 * network adapter, forcing superfluous retries and possibly timeouts.
145 * Thus, we set our budget to greater than 1.
146 */ 147 */
147static int poll_one_napi(struct netpoll_info *npinfo, 148static int poll_one_napi(struct napi_struct *napi, int budget)
148 struct napi_struct *napi, int budget)
149{ 149{
150 int work; 150 int work;
151 151
@@ -156,52 +156,35 @@ static int poll_one_napi(struct netpoll_info *npinfo,
156 if (!test_bit(NAPI_STATE_SCHED, &napi->state)) 156 if (!test_bit(NAPI_STATE_SCHED, &napi->state))
157 return budget; 157 return budget;
158 158
159 npinfo->rx_flags |= NETPOLL_RX_DROP;
160 atomic_inc(&trapped);
161 set_bit(NAPI_STATE_NPSVC, &napi->state); 159 set_bit(NAPI_STATE_NPSVC, &napi->state);
162 160
163 work = napi->poll(napi, budget); 161 work = napi->poll(napi, budget);
162 WARN_ONCE(work > budget, "%pF exceeded budget in poll\n", napi->poll);
164 trace_napi_poll(napi); 163 trace_napi_poll(napi);
165 164
166 clear_bit(NAPI_STATE_NPSVC, &napi->state); 165 clear_bit(NAPI_STATE_NPSVC, &napi->state);
167 atomic_dec(&trapped);
168 npinfo->rx_flags &= ~NETPOLL_RX_DROP;
169 166
170 return budget - work; 167 return budget - work;
171} 168}
172 169
173static void poll_napi(struct net_device *dev) 170static void poll_napi(struct net_device *dev, int budget)
174{ 171{
175 struct napi_struct *napi; 172 struct napi_struct *napi;
176 int budget = 16;
177 173
178 list_for_each_entry(napi, &dev->napi_list, dev_list) { 174 list_for_each_entry(napi, &dev->napi_list, dev_list) {
179 if (napi->poll_owner != smp_processor_id() && 175 if (napi->poll_owner != smp_processor_id() &&
180 spin_trylock(&napi->poll_lock)) { 176 spin_trylock(&napi->poll_lock)) {
181 budget = poll_one_napi(rcu_dereference_bh(dev->npinfo), 177 budget = poll_one_napi(napi, budget);
182 napi, budget);
183 spin_unlock(&napi->poll_lock); 178 spin_unlock(&napi->poll_lock);
184
185 if (!budget)
186 break;
187 } 179 }
188 } 180 }
189} 181}
190 182
191static void service_neigh_queue(struct netpoll_info *npi)
192{
193 if (npi) {
194 struct sk_buff *skb;
195
196 while ((skb = skb_dequeue(&npi->neigh_tx)))
197 netpoll_neigh_reply(skb, npi);
198 }
199}
200
201static void netpoll_poll_dev(struct net_device *dev) 183static void netpoll_poll_dev(struct net_device *dev)
202{ 184{
203 const struct net_device_ops *ops; 185 const struct net_device_ops *ops;
204 struct netpoll_info *ni = rcu_dereference_bh(dev->npinfo); 186 struct netpoll_info *ni = rcu_dereference_bh(dev->npinfo);
187 int budget = 0;
205 188
206 /* Don't do any rx activity if the dev_lock mutex is held 189 /* Don't do any rx activity if the dev_lock mutex is held
207 * the dev_open/close paths use this to block netpoll activity 190 * the dev_open/close paths use this to block netpoll activity
@@ -224,31 +207,14 @@ static void netpoll_poll_dev(struct net_device *dev)
224 /* Process pending work on NIC */ 207 /* Process pending work on NIC */
225 ops->ndo_poll_controller(dev); 208 ops->ndo_poll_controller(dev);
226 209
227 poll_napi(dev); 210 poll_napi(dev, budget);
228 211
229 up(&ni->dev_lock); 212 up(&ni->dev_lock);
230 213
231 if (dev->flags & IFF_SLAVE) {
232 if (ni) {
233 struct net_device *bond_dev;
234 struct sk_buff *skb;
235 struct netpoll_info *bond_ni;
236
237 bond_dev = netdev_master_upper_dev_get_rcu(dev);
238 bond_ni = rcu_dereference_bh(bond_dev->npinfo);
239 while ((skb = skb_dequeue(&ni->neigh_tx))) {
240 skb->dev = bond_dev;
241 skb_queue_tail(&bond_ni->neigh_tx, skb);
242 }
243 }
244 }
245
246 service_neigh_queue(ni);
247
248 zap_completion_queue(); 214 zap_completion_queue();
249} 215}
250 216
251void netpoll_rx_disable(struct net_device *dev) 217void netpoll_poll_disable(struct net_device *dev)
252{ 218{
253 struct netpoll_info *ni; 219 struct netpoll_info *ni;
254 int idx; 220 int idx;
@@ -259,9 +225,9 @@ void netpoll_rx_disable(struct net_device *dev)
259 down(&ni->dev_lock); 225 down(&ni->dev_lock);
260 srcu_read_unlock(&netpoll_srcu, idx); 226 srcu_read_unlock(&netpoll_srcu, idx);
261} 227}
262EXPORT_SYMBOL(netpoll_rx_disable); 228EXPORT_SYMBOL(netpoll_poll_disable);
263 229
264void netpoll_rx_enable(struct net_device *dev) 230void netpoll_poll_enable(struct net_device *dev)
265{ 231{
266 struct netpoll_info *ni; 232 struct netpoll_info *ni;
267 rcu_read_lock(); 233 rcu_read_lock();
@@ -270,7 +236,7 @@ void netpoll_rx_enable(struct net_device *dev)
270 up(&ni->dev_lock); 236 up(&ni->dev_lock);
271 rcu_read_unlock(); 237 rcu_read_unlock();
272} 238}
273EXPORT_SYMBOL(netpoll_rx_enable); 239EXPORT_SYMBOL(netpoll_poll_enable);
274 240
275static void refill_skbs(void) 241static void refill_skbs(void)
276{ 242{
@@ -304,7 +270,7 @@ static void zap_completion_queue(void)
304 while (clist != NULL) { 270 while (clist != NULL) {
305 struct sk_buff *skb = clist; 271 struct sk_buff *skb = clist;
306 clist = clist->next; 272 clist = clist->next;
307 if (skb->destructor) { 273 if (!skb_irq_freeable(skb)) {
308 atomic_inc(&skb->users); 274 atomic_inc(&skb->users);
309 dev_kfree_skb_any(skb); /* put this one back */ 275 dev_kfree_skb_any(skb); /* put this one back */
310 } else { 276 } else {
@@ -359,7 +325,6 @@ void netpoll_send_skb_on_dev(struct netpoll *np, struct sk_buff *skb,
359{ 325{
360 int status = NETDEV_TX_BUSY; 326 int status = NETDEV_TX_BUSY;
361 unsigned long tries; 327 unsigned long tries;
362 const struct net_device_ops *ops = dev->netdev_ops;
363 /* It is up to the caller to keep npinfo alive. */ 328 /* It is up to the caller to keep npinfo alive. */
364 struct netpoll_info *npinfo; 329 struct netpoll_info *npinfo;
365 330
@@ -367,7 +332,7 @@ void netpoll_send_skb_on_dev(struct netpoll *np, struct sk_buff *skb,
367 332
368 npinfo = rcu_dereference_bh(np->dev->npinfo); 333 npinfo = rcu_dereference_bh(np->dev->npinfo);
369 if (!npinfo || !netif_running(dev) || !netif_device_present(dev)) { 334 if (!npinfo || !netif_running(dev) || !netif_device_present(dev)) {
370 __kfree_skb(skb); 335 dev_kfree_skb_irq(skb);
371 return; 336 return;
372 } 337 }
373 338
@@ -380,29 +345,11 @@ void netpoll_send_skb_on_dev(struct netpoll *np, struct sk_buff *skb,
380 /* try until next clock tick */ 345 /* try until next clock tick */
381 for (tries = jiffies_to_usecs(1)/USEC_PER_POLL; 346 for (tries = jiffies_to_usecs(1)/USEC_PER_POLL;
382 tries > 0; --tries) { 347 tries > 0; --tries) {
383 if (__netif_tx_trylock(txq)) { 348 if (HARD_TX_TRYLOCK(dev, txq)) {
384 if (!netif_xmit_stopped(txq)) { 349 if (!netif_xmit_stopped(txq))
385 if (vlan_tx_tag_present(skb) && 350 status = netpoll_start_xmit(skb, dev, txq);
386 !vlan_hw_offload_capable(netif_skb_features(skb), 351
387 skb->vlan_proto)) { 352 HARD_TX_UNLOCK(dev, txq);
388 skb = __vlan_put_tag(skb, skb->vlan_proto, vlan_tx_tag_get(skb));
389 if (unlikely(!skb)) {
390 /* This is actually a packet drop, but we
391 * don't want the code at the end of this
392 * function to try and re-queue a NULL skb.
393 */
394 status = NETDEV_TX_OK;
395 goto unlock_txq;
396 }
397 skb->vlan_tci = 0;
398 }
399
400 status = ops->ndo_start_xmit(skb, dev);
401 if (status == NETDEV_TX_OK)
402 txq_trans_update(txq);
403 }
404 unlock_txq:
405 __netif_tx_unlock(txq);
406 353
407 if (status == NETDEV_TX_OK) 354 if (status == NETDEV_TX_OK)
408 break; 355 break;
@@ -417,7 +364,7 @@ void netpoll_send_skb_on_dev(struct netpoll *np, struct sk_buff *skb,
417 364
418 WARN_ONCE(!irqs_disabled(), 365 WARN_ONCE(!irqs_disabled(),
419 "netpoll_send_skb_on_dev(): %s enabled interrupts in poll (%pF)\n", 366 "netpoll_send_skb_on_dev(): %s enabled interrupts in poll (%pF)\n",
420 dev->name, ops->ndo_start_xmit); 367 dev->name, dev->netdev_ops->ndo_start_xmit);
421 368
422 } 369 }
423 370
@@ -529,384 +476,6 @@ void netpoll_send_udp(struct netpoll *np, const char *msg, int len)
529} 476}
530EXPORT_SYMBOL(netpoll_send_udp); 477EXPORT_SYMBOL(netpoll_send_udp);
531 478
532static void netpoll_neigh_reply(struct sk_buff *skb, struct netpoll_info *npinfo)
533{
534 int size, type = ARPOP_REPLY;
535 __be32 sip, tip;
536 unsigned char *sha;
537 struct sk_buff *send_skb;
538 struct netpoll *np, *tmp;
539 unsigned long flags;
540 int hlen, tlen;
541 int hits = 0, proto;
542
543 if (list_empty(&npinfo->rx_np))
544 return;
545
546 /* Before checking the packet, we do some early
547 inspection whether this is interesting at all */
548 spin_lock_irqsave(&npinfo->rx_lock, flags);
549 list_for_each_entry_safe(np, tmp, &npinfo->rx_np, rx) {
550 if (np->dev == skb->dev)
551 hits++;
552 }
553 spin_unlock_irqrestore(&npinfo->rx_lock, flags);
554
555 /* No netpoll struct is using this dev */
556 if (!hits)
557 return;
558
559 proto = ntohs(eth_hdr(skb)->h_proto);
560 if (proto == ETH_P_ARP) {
561 struct arphdr *arp;
562 unsigned char *arp_ptr;
563 /* No arp on this interface */
564 if (skb->dev->flags & IFF_NOARP)
565 return;
566
567 if (!pskb_may_pull(skb, arp_hdr_len(skb->dev)))
568 return;
569
570 skb_reset_network_header(skb);
571 skb_reset_transport_header(skb);
572 arp = arp_hdr(skb);
573
574 if ((arp->ar_hrd != htons(ARPHRD_ETHER) &&
575 arp->ar_hrd != htons(ARPHRD_IEEE802)) ||
576 arp->ar_pro != htons(ETH_P_IP) ||
577 arp->ar_op != htons(ARPOP_REQUEST))
578 return;
579
580 arp_ptr = (unsigned char *)(arp+1);
581 /* save the location of the src hw addr */
582 sha = arp_ptr;
583 arp_ptr += skb->dev->addr_len;
584 memcpy(&sip, arp_ptr, 4);
585 arp_ptr += 4;
586 /* If we actually cared about dst hw addr,
587 it would get copied here */
588 arp_ptr += skb->dev->addr_len;
589 memcpy(&tip, arp_ptr, 4);
590
591 /* Should we ignore arp? */
592 if (ipv4_is_loopback(tip) || ipv4_is_multicast(tip))
593 return;
594
595 size = arp_hdr_len(skb->dev);
596
597 spin_lock_irqsave(&npinfo->rx_lock, flags);
598 list_for_each_entry_safe(np, tmp, &npinfo->rx_np, rx) {
599 if (tip != np->local_ip.ip)
600 continue;
601
602 hlen = LL_RESERVED_SPACE(np->dev);
603 tlen = np->dev->needed_tailroom;
604 send_skb = find_skb(np, size + hlen + tlen, hlen);
605 if (!send_skb)
606 continue;
607
608 skb_reset_network_header(send_skb);
609 arp = (struct arphdr *) skb_put(send_skb, size);
610 send_skb->dev = skb->dev;
611 send_skb->protocol = htons(ETH_P_ARP);
612
613 /* Fill the device header for the ARP frame */
614 if (dev_hard_header(send_skb, skb->dev, ETH_P_ARP,
615 sha, np->dev->dev_addr,
616 send_skb->len) < 0) {
617 kfree_skb(send_skb);
618 continue;
619 }
620
621 /*
622 * Fill out the arp protocol part.
623 *
624 * we only support ethernet device type,
625 * which (according to RFC 1390) should
626 * always equal 1 (Ethernet).
627 */
628
629 arp->ar_hrd = htons(np->dev->type);
630 arp->ar_pro = htons(ETH_P_IP);
631 arp->ar_hln = np->dev->addr_len;
632 arp->ar_pln = 4;
633 arp->ar_op = htons(type);
634
635 arp_ptr = (unsigned char *)(arp + 1);
636 memcpy(arp_ptr, np->dev->dev_addr, np->dev->addr_len);
637 arp_ptr += np->dev->addr_len;
638 memcpy(arp_ptr, &tip, 4);
639 arp_ptr += 4;
640 memcpy(arp_ptr, sha, np->dev->addr_len);
641 arp_ptr += np->dev->addr_len;
642 memcpy(arp_ptr, &sip, 4);
643
644 netpoll_send_skb(np, send_skb);
645
646 /* If there are several rx_skb_hooks for the same
647 * address we're fine by sending a single reply
648 */
649 break;
650 }
651 spin_unlock_irqrestore(&npinfo->rx_lock, flags);
652 } else if( proto == ETH_P_IPV6) {
653#if IS_ENABLED(CONFIG_IPV6)
654 struct nd_msg *msg;
655 u8 *lladdr = NULL;
656 struct ipv6hdr *hdr;
657 struct icmp6hdr *icmp6h;
658 const struct in6_addr *saddr;
659 const struct in6_addr *daddr;
660 struct inet6_dev *in6_dev = NULL;
661 struct in6_addr *target;
662
663 in6_dev = in6_dev_get(skb->dev);
664 if (!in6_dev || !in6_dev->cnf.accept_ra)
665 return;
666
667 if (!pskb_may_pull(skb, skb->len))
668 return;
669
670 msg = (struct nd_msg *)skb_transport_header(skb);
671
672 __skb_push(skb, skb->data - skb_transport_header(skb));
673
674 if (ipv6_hdr(skb)->hop_limit != 255)
675 return;
676 if (msg->icmph.icmp6_code != 0)
677 return;
678 if (msg->icmph.icmp6_type != NDISC_NEIGHBOUR_SOLICITATION)
679 return;
680
681 saddr = &ipv6_hdr(skb)->saddr;
682 daddr = &ipv6_hdr(skb)->daddr;
683
684 size = sizeof(struct icmp6hdr) + sizeof(struct in6_addr);
685
686 spin_lock_irqsave(&npinfo->rx_lock, flags);
687 list_for_each_entry_safe(np, tmp, &npinfo->rx_np, rx) {
688 if (!ipv6_addr_equal(daddr, &np->local_ip.in6))
689 continue;
690
691 hlen = LL_RESERVED_SPACE(np->dev);
692 tlen = np->dev->needed_tailroom;
693 send_skb = find_skb(np, size + hlen + tlen, hlen);
694 if (!send_skb)
695 continue;
696
697 send_skb->protocol = htons(ETH_P_IPV6);
698 send_skb->dev = skb->dev;
699
700 skb_reset_network_header(send_skb);
701 hdr = (struct ipv6hdr *) skb_put(send_skb, sizeof(struct ipv6hdr));
702 *(__be32*)hdr = htonl(0x60000000);
703 hdr->payload_len = htons(size);
704 hdr->nexthdr = IPPROTO_ICMPV6;
705 hdr->hop_limit = 255;
706 hdr->saddr = *saddr;
707 hdr->daddr = *daddr;
708
709 icmp6h = (struct icmp6hdr *) skb_put(send_skb, sizeof(struct icmp6hdr));
710 icmp6h->icmp6_type = NDISC_NEIGHBOUR_ADVERTISEMENT;
711 icmp6h->icmp6_router = 0;
712 icmp6h->icmp6_solicited = 1;
713
714 target = (struct in6_addr *) skb_put(send_skb, sizeof(struct in6_addr));
715 *target = msg->target;
716 icmp6h->icmp6_cksum = csum_ipv6_magic(saddr, daddr, size,
717 IPPROTO_ICMPV6,
718 csum_partial(icmp6h,
719 size, 0));
720
721 if (dev_hard_header(send_skb, skb->dev, ETH_P_IPV6,
722 lladdr, np->dev->dev_addr,
723 send_skb->len) < 0) {
724 kfree_skb(send_skb);
725 continue;
726 }
727
728 netpoll_send_skb(np, send_skb);
729
730 /* If there are several rx_skb_hooks for the same
731 * address, we're fine by sending a single reply
732 */
733 break;
734 }
735 spin_unlock_irqrestore(&npinfo->rx_lock, flags);
736#endif
737 }
738}
739
740static bool pkt_is_ns(struct sk_buff *skb)
741{
742 struct nd_msg *msg;
743 struct ipv6hdr *hdr;
744
745 if (skb->protocol != htons(ETH_P_IPV6))
746 return false;
747 if (!pskb_may_pull(skb, sizeof(struct ipv6hdr) + sizeof(struct nd_msg)))
748 return false;
749
750 msg = (struct nd_msg *)skb_transport_header(skb);
751 __skb_push(skb, skb->data - skb_transport_header(skb));
752 hdr = ipv6_hdr(skb);
753
754 if (hdr->nexthdr != IPPROTO_ICMPV6)
755 return false;
756 if (hdr->hop_limit != 255)
757 return false;
758 if (msg->icmph.icmp6_code != 0)
759 return false;
760 if (msg->icmph.icmp6_type != NDISC_NEIGHBOUR_SOLICITATION)
761 return false;
762
763 return true;
764}
765
766int __netpoll_rx(struct sk_buff *skb, struct netpoll_info *npinfo)
767{
768 int proto, len, ulen, data_len;
769 int hits = 0, offset;
770 const struct iphdr *iph;
771 struct udphdr *uh;
772 struct netpoll *np, *tmp;
773 uint16_t source;
774
775 if (list_empty(&npinfo->rx_np))
776 goto out;
777
778 if (skb->dev->type != ARPHRD_ETHER)
779 goto out;
780
781 /* check if netpoll clients need ARP */
782 if (skb->protocol == htons(ETH_P_ARP) && atomic_read(&trapped)) {
783 skb_queue_tail(&npinfo->neigh_tx, skb);
784 return 1;
785 } else if (pkt_is_ns(skb) && atomic_read(&trapped)) {
786 skb_queue_tail(&npinfo->neigh_tx, skb);
787 return 1;
788 }
789
790 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
791 skb = vlan_untag(skb);
792 if (unlikely(!skb))
793 goto out;
794 }
795
796 proto = ntohs(eth_hdr(skb)->h_proto);
797 if (proto != ETH_P_IP && proto != ETH_P_IPV6)
798 goto out;
799 if (skb->pkt_type == PACKET_OTHERHOST)
800 goto out;
801 if (skb_shared(skb))
802 goto out;
803
804 if (proto == ETH_P_IP) {
805 if (!pskb_may_pull(skb, sizeof(struct iphdr)))
806 goto out;
807 iph = (struct iphdr *)skb->data;
808 if (iph->ihl < 5 || iph->version != 4)
809 goto out;
810 if (!pskb_may_pull(skb, iph->ihl*4))
811 goto out;
812 iph = (struct iphdr *)skb->data;
813 if (ip_fast_csum((u8 *)iph, iph->ihl) != 0)
814 goto out;
815
816 len = ntohs(iph->tot_len);
817 if (skb->len < len || len < iph->ihl*4)
818 goto out;
819
820 /*
821 * Our transport medium may have padded the buffer out.
822 * Now We trim to the true length of the frame.
823 */
824 if (pskb_trim_rcsum(skb, len))
825 goto out;
826
827 iph = (struct iphdr *)skb->data;
828 if (iph->protocol != IPPROTO_UDP)
829 goto out;
830
831 len -= iph->ihl*4;
832 uh = (struct udphdr *)(((char *)iph) + iph->ihl*4);
833 offset = (unsigned char *)(uh + 1) - skb->data;
834 ulen = ntohs(uh->len);
835 data_len = skb->len - offset;
836 source = ntohs(uh->source);
837
838 if (ulen != len)
839 goto out;
840 if (checksum_udp(skb, uh, ulen, iph->saddr, iph->daddr))
841 goto out;
842 list_for_each_entry_safe(np, tmp, &npinfo->rx_np, rx) {
843 if (np->local_ip.ip && np->local_ip.ip != iph->daddr)
844 continue;
845 if (np->remote_ip.ip && np->remote_ip.ip != iph->saddr)
846 continue;
847 if (np->local_port && np->local_port != ntohs(uh->dest))
848 continue;
849
850 np->rx_skb_hook(np, source, skb, offset, data_len);
851 hits++;
852 }
853 } else {
854#if IS_ENABLED(CONFIG_IPV6)
855 const struct ipv6hdr *ip6h;
856
857 if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
858 goto out;
859 ip6h = (struct ipv6hdr *)skb->data;
860 if (ip6h->version != 6)
861 goto out;
862 len = ntohs(ip6h->payload_len);
863 if (!len)
864 goto out;
865 if (len + sizeof(struct ipv6hdr) > skb->len)
866 goto out;
867 if (pskb_trim_rcsum(skb, len + sizeof(struct ipv6hdr)))
868 goto out;
869 ip6h = ipv6_hdr(skb);
870 if (!pskb_may_pull(skb, sizeof(struct udphdr)))
871 goto out;
872 uh = udp_hdr(skb);
873 offset = (unsigned char *)(uh + 1) - skb->data;
874 ulen = ntohs(uh->len);
875 data_len = skb->len - offset;
876 source = ntohs(uh->source);
877 if (ulen != skb->len)
878 goto out;
879 if (udp6_csum_init(skb, uh, IPPROTO_UDP))
880 goto out;
881 list_for_each_entry_safe(np, tmp, &npinfo->rx_np, rx) {
882 if (!ipv6_addr_equal(&np->local_ip.in6, &ip6h->daddr))
883 continue;
884 if (!ipv6_addr_equal(&np->remote_ip.in6, &ip6h->saddr))
885 continue;
886 if (np->local_port && np->local_port != ntohs(uh->dest))
887 continue;
888
889 np->rx_skb_hook(np, source, skb, offset, data_len);
890 hits++;
891 }
892#endif
893 }
894
895 if (!hits)
896 goto out;
897
898 kfree_skb(skb);
899 return 1;
900
901out:
902 if (atomic_read(&trapped)) {
903 kfree_skb(skb);
904 return 1;
905 }
906
907 return 0;
908}
909
910void netpoll_print_options(struct netpoll *np) 479void netpoll_print_options(struct netpoll *np)
911{ 480{
912 np_info(np, "local port %d\n", np->local_port); 481 np_info(np, "local port %d\n", np->local_port);
@@ -1026,11 +595,10 @@ int netpoll_parse_options(struct netpoll *np, char *opt)
1026} 595}
1027EXPORT_SYMBOL(netpoll_parse_options); 596EXPORT_SYMBOL(netpoll_parse_options);
1028 597
1029int __netpoll_setup(struct netpoll *np, struct net_device *ndev, gfp_t gfp) 598int __netpoll_setup(struct netpoll *np, struct net_device *ndev)
1030{ 599{
1031 struct netpoll_info *npinfo; 600 struct netpoll_info *npinfo;
1032 const struct net_device_ops *ops; 601 const struct net_device_ops *ops;
1033 unsigned long flags;
1034 int err; 602 int err;
1035 603
1036 np->dev = ndev; 604 np->dev = ndev;
@@ -1046,18 +614,13 @@ int __netpoll_setup(struct netpoll *np, struct net_device *ndev, gfp_t gfp)
1046 } 614 }
1047 615
1048 if (!ndev->npinfo) { 616 if (!ndev->npinfo) {
1049 npinfo = kmalloc(sizeof(*npinfo), gfp); 617 npinfo = kmalloc(sizeof(*npinfo), GFP_KERNEL);
1050 if (!npinfo) { 618 if (!npinfo) {
1051 err = -ENOMEM; 619 err = -ENOMEM;
1052 goto out; 620 goto out;
1053 } 621 }
1054 622
1055 npinfo->rx_flags = 0;
1056 INIT_LIST_HEAD(&npinfo->rx_np);
1057
1058 spin_lock_init(&npinfo->rx_lock);
1059 sema_init(&npinfo->dev_lock, 1); 623 sema_init(&npinfo->dev_lock, 1);
1060 skb_queue_head_init(&npinfo->neigh_tx);
1061 skb_queue_head_init(&npinfo->txq); 624 skb_queue_head_init(&npinfo->txq);
1062 INIT_DELAYED_WORK(&npinfo->tx_work, queue_process); 625 INIT_DELAYED_WORK(&npinfo->tx_work, queue_process);
1063 626
@@ -1065,7 +628,7 @@ int __netpoll_setup(struct netpoll *np, struct net_device *ndev, gfp_t gfp)
1065 628
1066 ops = np->dev->netdev_ops; 629 ops = np->dev->netdev_ops;
1067 if (ops->ndo_netpoll_setup) { 630 if (ops->ndo_netpoll_setup) {
1068 err = ops->ndo_netpoll_setup(ndev, npinfo, gfp); 631 err = ops->ndo_netpoll_setup(ndev, npinfo);
1069 if (err) 632 if (err)
1070 goto free_npinfo; 633 goto free_npinfo;
1071 } 634 }
@@ -1076,13 +639,6 @@ int __netpoll_setup(struct netpoll *np, struct net_device *ndev, gfp_t gfp)
1076 639
1077 npinfo->netpoll = np; 640 npinfo->netpoll = np;
1078 641
1079 if (np->rx_skb_hook) {
1080 spin_lock_irqsave(&npinfo->rx_lock, flags);
1081 npinfo->rx_flags |= NETPOLL_RX_ENABLED;
1082 list_add_tail(&np->rx, &npinfo->rx_np);
1083 spin_unlock_irqrestore(&npinfo->rx_lock, flags);
1084 }
1085
1086 /* last thing to do is link it to the net device structure */ 642 /* last thing to do is link it to the net device structure */
1087 rcu_assign_pointer(ndev->npinfo, npinfo); 643 rcu_assign_pointer(ndev->npinfo, npinfo);
1088 644
@@ -1204,7 +760,7 @@ int netpoll_setup(struct netpoll *np)
1204 /* fill up the skb queue */ 760 /* fill up the skb queue */
1205 refill_skbs(); 761 refill_skbs();
1206 762
1207 err = __netpoll_setup(np, ndev, GFP_KERNEL); 763 err = __netpoll_setup(np, ndev);
1208 if (err) 764 if (err)
1209 goto put; 765 goto put;
1210 766
@@ -1231,7 +787,6 @@ static void rcu_cleanup_netpoll_info(struct rcu_head *rcu_head)
1231 struct netpoll_info *npinfo = 787 struct netpoll_info *npinfo =
1232 container_of(rcu_head, struct netpoll_info, rcu); 788 container_of(rcu_head, struct netpoll_info, rcu);
1233 789
1234 skb_queue_purge(&npinfo->neigh_tx);
1235 skb_queue_purge(&npinfo->txq); 790 skb_queue_purge(&npinfo->txq);
1236 791
1237 /* we can't call cancel_delayed_work_sync here, as we are in softirq */ 792 /* we can't call cancel_delayed_work_sync here, as we are in softirq */
@@ -1247,7 +802,6 @@ static void rcu_cleanup_netpoll_info(struct rcu_head *rcu_head)
1247void __netpoll_cleanup(struct netpoll *np) 802void __netpoll_cleanup(struct netpoll *np)
1248{ 803{
1249 struct netpoll_info *npinfo; 804 struct netpoll_info *npinfo;
1250 unsigned long flags;
1251 805
1252 /* rtnl_dereference would be preferable here but 806 /* rtnl_dereference would be preferable here but
1253 * rcu_cleanup_netpoll path can put us in here safely without 807 * rcu_cleanup_netpoll path can put us in here safely without
@@ -1257,14 +811,6 @@ void __netpoll_cleanup(struct netpoll *np)
1257 if (!npinfo) 811 if (!npinfo)
1258 return; 812 return;
1259 813
1260 if (!list_empty(&npinfo->rx_np)) {
1261 spin_lock_irqsave(&npinfo->rx_lock, flags);
1262 list_del(&np->rx);
1263 if (list_empty(&npinfo->rx_np))
1264 npinfo->rx_flags &= ~NETPOLL_RX_ENABLED;
1265 spin_unlock_irqrestore(&npinfo->rx_lock, flags);
1266 }
1267
1268 synchronize_srcu(&netpoll_srcu); 814 synchronize_srcu(&netpoll_srcu);
1269 815
1270 if (atomic_dec_and_test(&npinfo->refcnt)) { 816 if (atomic_dec_and_test(&npinfo->refcnt)) {
@@ -1274,7 +820,7 @@ void __netpoll_cleanup(struct netpoll *np)
1274 if (ops->ndo_netpoll_cleanup) 820 if (ops->ndo_netpoll_cleanup)
1275 ops->ndo_netpoll_cleanup(np->dev); 821 ops->ndo_netpoll_cleanup(np->dev);
1276 822
1277 rcu_assign_pointer(np->dev->npinfo, NULL); 823 RCU_INIT_POINTER(np->dev->npinfo, NULL);
1278 call_rcu_bh(&npinfo->rcu, rcu_cleanup_netpoll_info); 824 call_rcu_bh(&npinfo->rcu, rcu_cleanup_netpoll_info);
1279 } 825 }
1280} 826}
@@ -1308,18 +854,3 @@ out:
1308 rtnl_unlock(); 854 rtnl_unlock();
1309} 855}
1310EXPORT_SYMBOL(netpoll_cleanup); 856EXPORT_SYMBOL(netpoll_cleanup);
1311
1312int netpoll_trap(void)
1313{
1314 return atomic_read(&trapped);
1315}
1316EXPORT_SYMBOL(netpoll_trap);
1317
1318void netpoll_set_trap(int trap)
1319{
1320 if (trap)
1321 atomic_inc(&trapped);
1322 else
1323 atomic_dec(&trapped);
1324}
1325EXPORT_SYMBOL(netpoll_set_trap);
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-03 08:32:55 -0400