aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/netdevice.h14
-rw-r--r--net/core/dev.c28
2 files changed, 26 insertions, 16 deletions
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index a1bff6518166..256419583d9d 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1407,17 +1407,25 @@ struct softnet_data {
1407 struct softnet_data *rps_ipi_next; 1407 struct softnet_data *rps_ipi_next;
1408 unsigned int cpu; 1408 unsigned int cpu;
1409 unsigned int input_queue_head; 1409 unsigned int input_queue_head;
1410 unsigned int input_queue_tail;
1410#endif 1411#endif
1411 unsigned dropped; 1412 unsigned dropped;
1412 struct sk_buff_head input_pkt_queue; 1413 struct sk_buff_head input_pkt_queue;
1413 struct napi_struct backlog; 1414 struct napi_struct backlog;
1414}; 1415};
1415 1416
1416static inline void input_queue_head_add(struct softnet_data *sd, 1417static inline void input_queue_head_incr(struct softnet_data *sd)
1417 unsigned int len)
1418{ 1418{
1419#ifdef CONFIG_RPS 1419#ifdef CONFIG_RPS
1420 sd->input_queue_head += len; 1420 sd->input_queue_head++;
1421#endif
1422}
1423
1424static inline void input_queue_tail_incr_save(struct softnet_data *sd,
1425 unsigned int *qtail)
1426{
1427#ifdef CONFIG_RPS
1428 *qtail = ++sd->input_queue_tail;
1421#endif 1429#endif
1422} 1430}
1423 1431
diff --git a/net/core/dev.c b/net/core/dev.c
index 6c820650b80f..0aab66d68b19 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -2426,10 +2426,7 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
2426 if (skb_queue_len(&sd->input_pkt_queue)) { 2426 if (skb_queue_len(&sd->input_pkt_queue)) {
2427enqueue: 2427enqueue:
2428 __skb_queue_tail(&sd->input_pkt_queue, skb); 2428 __skb_queue_tail(&sd->input_pkt_queue, skb);
2429#ifdef CONFIG_RPS 2429 input_queue_tail_incr_save(sd, qtail);
2430 *qtail = sd->input_queue_head +
2431 skb_queue_len(&sd->input_pkt_queue);
2432#endif
2433 rps_unlock(sd); 2430 rps_unlock(sd);
2434 local_irq_restore(flags); 2431 local_irq_restore(flags);
2435 return NET_RX_SUCCESS; 2432 return NET_RX_SUCCESS;
@@ -2964,7 +2961,7 @@ static void flush_backlog(void *arg)
2964 if (skb->dev == dev) { 2961 if (skb->dev == dev) {
2965 __skb_unlink(skb, &sd->input_pkt_queue); 2962 __skb_unlink(skb, &sd->input_pkt_queue);
2966 kfree_skb(skb); 2963 kfree_skb(skb);
2967 input_queue_head_add(sd, 1); 2964 input_queue_head_incr(sd);
2968 } 2965 }
2969 } 2966 }
2970 rps_unlock(sd); 2967 rps_unlock(sd);
@@ -2973,6 +2970,7 @@ static void flush_backlog(void *arg)
2973 if (skb->dev == dev) { 2970 if (skb->dev == dev) {
2974 __skb_unlink(skb, &sd->process_queue); 2971 __skb_unlink(skb, &sd->process_queue);
2975 kfree_skb(skb); 2972 kfree_skb(skb);
2973 input_queue_head_incr(sd);
2976 } 2974 }
2977 } 2975 }
2978} 2976}
@@ -3328,18 +3326,20 @@ static int process_backlog(struct napi_struct *napi, int quota)
3328 while ((skb = __skb_dequeue(&sd->process_queue))) { 3326 while ((skb = __skb_dequeue(&sd->process_queue))) {
3329 local_irq_enable(); 3327 local_irq_enable();
3330 __netif_receive_skb(skb); 3328 __netif_receive_skb(skb);
3331 if (++work >= quota)
3332 return work;
3333 local_irq_disable(); 3329 local_irq_disable();
3330 input_queue_head_incr(sd);
3331 if (++work >= quota) {
3332 local_irq_enable();
3333 return work;
3334 }
3334 } 3335 }
3335 3336
3336 rps_lock(sd); 3337 rps_lock(sd);
3337 qlen = skb_queue_len(&sd->input_pkt_queue); 3338 qlen = skb_queue_len(&sd->input_pkt_queue);
3338 if (qlen) { 3339 if (qlen)
3339 input_queue_head_add(sd, qlen);
3340 skb_queue_splice_tail_init(&sd->input_pkt_queue, 3340 skb_queue_splice_tail_init(&sd->input_pkt_queue,
3341 &sd->process_queue); 3341 &sd->process_queue);
3342 } 3342
3343 if (qlen < quota - work) { 3343 if (qlen < quota - work) {
3344 /* 3344 /*
3345 * Inline a custom version of __napi_complete(). 3345 * Inline a custom version of __napi_complete().
@@ -5679,12 +5679,14 @@ static int dev_cpu_callback(struct notifier_block *nfb,
5679 local_irq_enable(); 5679 local_irq_enable();
5680 5680
5681 /* Process offline CPU's input_pkt_queue */ 5681 /* Process offline CPU's input_pkt_queue */
5682 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) { 5682 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
5683 netif_rx(skb); 5683 netif_rx(skb);
5684 input_queue_head_add(oldsd, 1); 5684 input_queue_head_incr(oldsd);
5685 } 5685 }
5686 while ((skb = __skb_dequeue(&oldsd->process_queue))) 5686 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
5687 netif_rx(skb); 5687 netif_rx(skb);
5688 input_queue_head_incr(oldsd);
5689 }
5688 5690
5689 return NOTIFY_OK; 5691 return NOTIFY_OK;
5690} 5692}