diff options
author | Ben Hutchings <bhutchings@solarflare.com> | 2011-01-19 06:03:53 -0500 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2011-01-24 17:53:01 -0500 |
commit | c445477d74ab3779d1386ab797fbb9b628eb9f64 (patch) | |
tree | 3ee70b7748c6c63a688f367e5ffd83fde21b87e3 /net/core | |
parent | c39649c331c70952700f99832b03f87e9d7f5b4b (diff) |
net: RPS: Enable hardware acceleration of RFS
Allow drivers for multiqueue hardware with flow filter tables to
accelerate RFS. The driver must:
1. Set net_device::rx_cpu_rmap to a cpu_rmap of the RX completion
IRQs (in queue order). This will provide a mapping from CPUs to the
queues for which completions are handled nearest to them.
2. Implement net_device_ops::ndo_rx_flow_steer. This operation adds
or replaces a filter steering the given flow to the given RX queue, if
possible.
3. Periodically remove filters for which rps_may_expire_flow() returns
true.
Signed-off-by: Ben Hutchings <bhutchings@solarflare.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'net/core')
-rw-r--r-- | net/core/dev.c | 97 |
1 files changed, 91 insertions, 6 deletions
diff --git a/net/core/dev.c b/net/core/dev.c index d162ba8d622..aa761472f9e 100644 --- a/net/core/dev.c +++ b/net/core/dev.c | |||
@@ -132,6 +132,7 @@ | |||
132 | #include <trace/events/skb.h> | 132 | #include <trace/events/skb.h> |
133 | #include <linux/pci.h> | 133 | #include <linux/pci.h> |
134 | #include <linux/inetdevice.h> | 134 | #include <linux/inetdevice.h> |
135 | #include <linux/cpu_rmap.h> | ||
135 | 136 | ||
136 | #include "net-sysfs.h" | 137 | #include "net-sysfs.h" |
137 | 138 | ||
@@ -2588,6 +2589,53 @@ EXPORT_SYMBOL(__skb_get_rxhash); | |||
2588 | struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; | 2589 | struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; |
2589 | EXPORT_SYMBOL(rps_sock_flow_table); | 2590 | EXPORT_SYMBOL(rps_sock_flow_table); |
2590 | 2591 | ||
2592 | static struct rps_dev_flow * | ||
2593 | set_rps_cpu(struct net_device *dev, struct sk_buff *skb, | ||
2594 | struct rps_dev_flow *rflow, u16 next_cpu) | ||
2595 | { | ||
2596 | u16 tcpu; | ||
2597 | |||
2598 | tcpu = rflow->cpu = next_cpu; | ||
2599 | if (tcpu != RPS_NO_CPU) { | ||
2600 | #ifdef CONFIG_RFS_ACCEL | ||
2601 | struct netdev_rx_queue *rxqueue; | ||
2602 | struct rps_dev_flow_table *flow_table; | ||
2603 | struct rps_dev_flow *old_rflow; | ||
2604 | u32 flow_id; | ||
2605 | u16 rxq_index; | ||
2606 | int rc; | ||
2607 | |||
2608 | /* Should we steer this flow to a different hardware queue? */ | ||
2609 | if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap) | ||
2610 | goto out; | ||
2611 | rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); | ||
2612 | if (rxq_index == skb_get_rx_queue(skb)) | ||
2613 | goto out; | ||
2614 | |||
2615 | rxqueue = dev->_rx + rxq_index; | ||
2616 | flow_table = rcu_dereference(rxqueue->rps_flow_table); | ||
2617 | if (!flow_table) | ||
2618 | goto out; | ||
2619 | flow_id = skb->rxhash & flow_table->mask; | ||
2620 | rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, | ||
2621 | rxq_index, flow_id); | ||
2622 | if (rc < 0) | ||
2623 | goto out; | ||
2624 | old_rflow = rflow; | ||
2625 | rflow = &flow_table->flows[flow_id]; | ||
2626 | rflow->cpu = next_cpu; | ||
2627 | rflow->filter = rc; | ||
2628 | if (old_rflow->filter == rflow->filter) | ||
2629 | old_rflow->filter = RPS_NO_FILTER; | ||
2630 | out: | ||
2631 | #endif | ||
2632 | rflow->last_qtail = | ||
2633 | per_cpu(softnet_data, tcpu).input_queue_head; | ||
2634 | } | ||
2635 | |||
2636 | return rflow; | ||
2637 | } | ||
2638 | |||
2591 | /* | 2639 | /* |
2592 | * get_rps_cpu is called from netif_receive_skb and returns the target | 2640 | * get_rps_cpu is called from netif_receive_skb and returns the target |
2593 | * CPU from the RPS map of the receiving queue for a given skb. | 2641 | * CPU from the RPS map of the receiving queue for a given skb. |
@@ -2658,12 +2706,9 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, | |||
2658 | if (unlikely(tcpu != next_cpu) && | 2706 | if (unlikely(tcpu != next_cpu) && |
2659 | (tcpu == RPS_NO_CPU || !cpu_online(tcpu) || | 2707 | (tcpu == RPS_NO_CPU || !cpu_online(tcpu) || |
2660 | ((int)(per_cpu(softnet_data, tcpu).input_queue_head - | 2708 | ((int)(per_cpu(softnet_data, tcpu).input_queue_head - |
2661 | rflow->last_qtail)) >= 0)) { | 2709 | rflow->last_qtail)) >= 0)) |
2662 | tcpu = rflow->cpu = next_cpu; | 2710 | rflow = set_rps_cpu(dev, skb, rflow, next_cpu); |
2663 | if (tcpu != RPS_NO_CPU) | 2711 | |
2664 | rflow->last_qtail = per_cpu(softnet_data, | ||
2665 | tcpu).input_queue_head; | ||
2666 | } | ||
2667 | if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) { | 2712 | if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) { |
2668 | *rflowp = rflow; | 2713 | *rflowp = rflow; |
2669 | cpu = tcpu; | 2714 | cpu = tcpu; |
@@ -2684,6 +2729,46 @@ done: | |||
2684 | return cpu; | 2729 | return cpu; |
2685 | } | 2730 | } |
2686 | 2731 | ||
2732 | #ifdef CONFIG_RFS_ACCEL | ||
2733 | |||
2734 | /** | ||
2735 | * rps_may_expire_flow - check whether an RFS hardware filter may be removed | ||
2736 | * @dev: Device on which the filter was set | ||
2737 | * @rxq_index: RX queue index | ||
2738 | * @flow_id: Flow ID passed to ndo_rx_flow_steer() | ||
2739 | * @filter_id: Filter ID returned by ndo_rx_flow_steer() | ||
2740 | * | ||
2741 | * Drivers that implement ndo_rx_flow_steer() should periodically call | ||
2742 | * this function for each installed filter and remove the filters for | ||
2743 | * which it returns %true. | ||
2744 | */ | ||
2745 | bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, | ||
2746 | u32 flow_id, u16 filter_id) | ||
2747 | { | ||
2748 | struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; | ||
2749 | struct rps_dev_flow_table *flow_table; | ||
2750 | struct rps_dev_flow *rflow; | ||
2751 | bool expire = true; | ||
2752 | int cpu; | ||
2753 | |||
2754 | rcu_read_lock(); | ||
2755 | flow_table = rcu_dereference(rxqueue->rps_flow_table); | ||
2756 | if (flow_table && flow_id <= flow_table->mask) { | ||
2757 | rflow = &flow_table->flows[flow_id]; | ||
2758 | cpu = ACCESS_ONCE(rflow->cpu); | ||
2759 | if (rflow->filter == filter_id && cpu != RPS_NO_CPU && | ||
2760 | ((int)(per_cpu(softnet_data, cpu).input_queue_head - | ||
2761 | rflow->last_qtail) < | ||
2762 | (int)(10 * flow_table->mask))) | ||
2763 | expire = false; | ||
2764 | } | ||
2765 | rcu_read_unlock(); | ||
2766 | return expire; | ||
2767 | } | ||
2768 | EXPORT_SYMBOL(rps_may_expire_flow); | ||
2769 | |||
2770 | #endif /* CONFIG_RFS_ACCEL */ | ||
2771 | |||
2687 | /* Called from hardirq (IPI) context */ | 2772 | /* Called from hardirq (IPI) context */ |
2688 | static void rps_trigger_softirq(void *data) | 2773 | static void rps_trigger_softirq(void *data) |
2689 | { | 2774 | { |