diff options
-rw-r--r-- | include/linux/netdevice.h | 14 | ||||
-rw-r--r-- | net/core/dev.c | 28 |
2 files changed, 26 insertions, 16 deletions
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index a1bff6518166..256419583d9d 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h | |||
@@ -1407,17 +1407,25 @@ struct softnet_data { | |||
1407 | struct softnet_data *rps_ipi_next; | 1407 | struct softnet_data *rps_ipi_next; |
1408 | unsigned int cpu; | 1408 | unsigned int cpu; |
1409 | unsigned int input_queue_head; | 1409 | unsigned int input_queue_head; |
1410 | unsigned int input_queue_tail; | ||
1410 | #endif | 1411 | #endif |
1411 | unsigned dropped; | 1412 | unsigned dropped; |
1412 | struct sk_buff_head input_pkt_queue; | 1413 | struct sk_buff_head input_pkt_queue; |
1413 | struct napi_struct backlog; | 1414 | struct napi_struct backlog; |
1414 | }; | 1415 | }; |
1415 | 1416 | ||
1416 | static inline void input_queue_head_add(struct softnet_data *sd, | 1417 | static inline void input_queue_head_incr(struct softnet_data *sd) |
1417 | unsigned int len) | ||
1418 | { | 1418 | { |
1419 | #ifdef CONFIG_RPS | 1419 | #ifdef CONFIG_RPS |
1420 | sd->input_queue_head += len; | 1420 | sd->input_queue_head++; |
1421 | #endif | ||
1422 | } | ||
1423 | |||
1424 | static inline void input_queue_tail_incr_save(struct softnet_data *sd, | ||
1425 | unsigned int *qtail) | ||
1426 | { | ||
1427 | #ifdef CONFIG_RPS | ||
1428 | *qtail = ++sd->input_queue_tail; | ||
1421 | #endif | 1429 | #endif |
1422 | } | 1430 | } |
1423 | 1431 | ||
diff --git a/net/core/dev.c b/net/core/dev.c index 6c820650b80f..0aab66d68b19 100644 --- a/net/core/dev.c +++ b/net/core/dev.c | |||
@@ -2426,10 +2426,7 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu, | |||
2426 | if (skb_queue_len(&sd->input_pkt_queue)) { | 2426 | if (skb_queue_len(&sd->input_pkt_queue)) { |
2427 | enqueue: | 2427 | enqueue: |
2428 | __skb_queue_tail(&sd->input_pkt_queue, skb); | 2428 | __skb_queue_tail(&sd->input_pkt_queue, skb); |
2429 | #ifdef CONFIG_RPS | 2429 | input_queue_tail_incr_save(sd, qtail); |
2430 | *qtail = sd->input_queue_head + | ||
2431 | skb_queue_len(&sd->input_pkt_queue); | ||
2432 | #endif | ||
2433 | rps_unlock(sd); | 2430 | rps_unlock(sd); |
2434 | local_irq_restore(flags); | 2431 | local_irq_restore(flags); |
2435 | return NET_RX_SUCCESS; | 2432 | return NET_RX_SUCCESS; |
@@ -2964,7 +2961,7 @@ static void flush_backlog(void *arg) | |||
2964 | if (skb->dev == dev) { | 2961 | if (skb->dev == dev) { |
2965 | __skb_unlink(skb, &sd->input_pkt_queue); | 2962 | __skb_unlink(skb, &sd->input_pkt_queue); |
2966 | kfree_skb(skb); | 2963 | kfree_skb(skb); |
2967 | input_queue_head_add(sd, 1); | 2964 | input_queue_head_incr(sd); |
2968 | } | 2965 | } |
2969 | } | 2966 | } |
2970 | rps_unlock(sd); | 2967 | rps_unlock(sd); |
@@ -2973,6 +2970,7 @@ static void flush_backlog(void *arg) | |||
2973 | if (skb->dev == dev) { | 2970 | if (skb->dev == dev) { |
2974 | __skb_unlink(skb, &sd->process_queue); | 2971 | __skb_unlink(skb, &sd->process_queue); |
2975 | kfree_skb(skb); | 2972 | kfree_skb(skb); |
2973 | input_queue_head_incr(sd); | ||
2976 | } | 2974 | } |
2977 | } | 2975 | } |
2978 | } | 2976 | } |
@@ -3328,18 +3326,20 @@ static int process_backlog(struct napi_struct *napi, int quota) | |||
3328 | while ((skb = __skb_dequeue(&sd->process_queue))) { | 3326 | while ((skb = __skb_dequeue(&sd->process_queue))) { |
3329 | local_irq_enable(); | 3327 | local_irq_enable(); |
3330 | __netif_receive_skb(skb); | 3328 | __netif_receive_skb(skb); |
3331 | if (++work >= quota) | ||
3332 | return work; | ||
3333 | local_irq_disable(); | 3329 | local_irq_disable(); |
3330 | input_queue_head_incr(sd); | ||
3331 | if (++work >= quota) { | ||
3332 | local_irq_enable(); | ||
3333 | return work; | ||
3334 | } | ||
3334 | } | 3335 | } |
3335 | 3336 | ||
3336 | rps_lock(sd); | 3337 | rps_lock(sd); |
3337 | qlen = skb_queue_len(&sd->input_pkt_queue); | 3338 | qlen = skb_queue_len(&sd->input_pkt_queue); |
3338 | if (qlen) { | 3339 | if (qlen) |
3339 | input_queue_head_add(sd, qlen); | ||
3340 | skb_queue_splice_tail_init(&sd->input_pkt_queue, | 3340 | skb_queue_splice_tail_init(&sd->input_pkt_queue, |
3341 | &sd->process_queue); | 3341 | &sd->process_queue); |
3342 | } | 3342 | |
3343 | if (qlen < quota - work) { | 3343 | if (qlen < quota - work) { |
3344 | /* | 3344 | /* |
3345 | * Inline a custom version of __napi_complete(). | 3345 | * Inline a custom version of __napi_complete(). |
@@ -5679,12 +5679,14 @@ static int dev_cpu_callback(struct notifier_block *nfb, | |||
5679 | local_irq_enable(); | 5679 | local_irq_enable(); |
5680 | 5680 | ||
5681 | /* Process offline CPU's input_pkt_queue */ | 5681 | /* Process offline CPU's input_pkt_queue */ |
5682 | while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) { | 5682 | while ((skb = __skb_dequeue(&oldsd->process_queue))) { |
5683 | netif_rx(skb); | 5683 | netif_rx(skb); |
5684 | input_queue_head_add(oldsd, 1); | 5684 | input_queue_head_incr(oldsd); |
5685 | } | 5685 | } |
5686 | while ((skb = __skb_dequeue(&oldsd->process_queue))) | 5686 | while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) { |
5687 | netif_rx(skb); | 5687 | netif_rx(skb); |
5688 | input_queue_head_incr(oldsd); | ||
5689 | } | ||
5688 | 5690 | ||
5689 | return NOTIFY_OK; | 5691 | return NOTIFY_OK; |
5690 | } | 5692 | } |