aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorEdward Cree <ecree@solarflare.com>2018-03-27 12:41:59 -0400
committerDavid S. Miller <davem@davemloft.net>2018-03-27 13:33:19 -0400
commit3af0f34290f6192756ee1d9c2d5fe27222267035 (patch)
tree84d74399e83cc23f71854b35a7606d966f2e1143
parentc709002c23f91d90eb6ee2d4efbb548a8fe3cc80 (diff)
sfc: replace asynchronous filter operations
Instead of having an efx->type->filter_rfs_insert() method, just use workitems with a worker function that calls efx->type->filter_insert(). The only user of this is efx_filter_rfs(), which now queues a call to efx_filter_rfs_work(). Similarly, efx_filter_rfs_expire() is now a worker function called on a new channel->filter_work work_struct, so the method efx->type->filter_rfs_expire_one() is no longer called in atomic context. We also add a new mutex efx->rps_mutex to protect the RPS state (efx-> rps_expire_channel, efx->rps_expire_index, and channel->rps_flow_id) so that the taking of efx->filter_lock can be moved to efx->type->filter_rfs_expire_one(). Thus, all filter table functions are now called in a sleepable context, allowing them to use sleeping locks in a future patch. Signed-off-by: Edward Cree <ecree@solarflare.com> Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r--drivers/net/ethernet/sfc/ef10.c161
-rw-r--r--drivers/net/ethernet/sfc/efx.c15
-rw-r--r--drivers/net/ethernet/sfc/efx.h7
-rw-r--r--drivers/net/ethernet/sfc/farch.c17
-rw-r--r--drivers/net/ethernet/sfc/net_driver.h10
-rw-r--r--drivers/net/ethernet/sfc/nic.h2
-rw-r--r--drivers/net/ethernet/sfc/rx.c119
-rw-r--r--drivers/net/ethernet/sfc/siena.c1
8 files changed, 124 insertions, 208 deletions
diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
index e100273b623d..bcbaba330fb5 100644
--- a/drivers/net/ethernet/sfc/ef10.c
+++ b/drivers/net/ethernet/sfc/ef10.c
@@ -4758,143 +4758,6 @@ static s32 efx_ef10_filter_get_rx_ids(struct efx_nic *efx,
4758 4758
4759#ifdef CONFIG_RFS_ACCEL 4759#ifdef CONFIG_RFS_ACCEL
4760 4760
4761static efx_mcdi_async_completer efx_ef10_filter_rfs_insert_complete;
4762
4763static s32 efx_ef10_filter_rfs_insert(struct efx_nic *efx,
4764 struct efx_filter_spec *spec)
4765{
4766 struct efx_ef10_filter_table *table = efx->filter_state;
4767 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
4768 struct efx_filter_spec *saved_spec;
4769 unsigned int hash, i, depth = 1;
4770 bool replacing = false;
4771 int ins_index = -1;
4772 u64 cookie;
4773 s32 rc;
4774
4775 /* Must be an RX filter without RSS and not for a multicast
4776 * destination address (RFS only works for connected sockets).
4777 * These restrictions allow us to pass only a tiny amount of
4778 * data through to the completion function.
4779 */
4780 EFX_WARN_ON_PARANOID(spec->flags !=
4781 (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_RX_SCATTER));
4782 EFX_WARN_ON_PARANOID(spec->priority != EFX_FILTER_PRI_HINT);
4783 EFX_WARN_ON_PARANOID(efx_filter_is_mc_recipient(spec));
4784
4785 hash = efx_ef10_filter_hash(spec);
4786
4787 spin_lock_bh(&efx->filter_lock);
4788
4789 /* Find any existing filter with the same match tuple or else
4790 * a free slot to insert at. If an existing filter is busy,
4791 * we have to give up.
4792 */
4793 for (;;) {
4794 i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1);
4795 saved_spec = efx_ef10_filter_entry_spec(table, i);
4796
4797 if (!saved_spec) {
4798 if (ins_index < 0)
4799 ins_index = i;
4800 } else if (efx_ef10_filter_equal(spec, saved_spec)) {
4801 if (table->entry[i].spec & EFX_EF10_FILTER_FLAG_BUSY) {
4802 rc = -EBUSY;
4803 goto fail_unlock;
4804 }
4805 if (spec->priority < saved_spec->priority) {
4806 rc = -EPERM;
4807 goto fail_unlock;
4808 }
4809 ins_index = i;
4810 break;
4811 }
4812
4813 /* Once we reach the maximum search depth, use the
4814 * first suitable slot or return -EBUSY if there was
4815 * none
4816 */
4817 if (depth == EFX_EF10_FILTER_SEARCH_LIMIT) {
4818 if (ins_index < 0) {
4819 rc = -EBUSY;
4820 goto fail_unlock;
4821 }
4822 break;
4823 }
4824
4825 ++depth;
4826 }
4827
4828 /* Create a software table entry if necessary, and mark it
4829 * busy. We might yet fail to insert, but any attempt to
4830 * insert a conflicting filter while we're waiting for the
4831 * firmware must find the busy entry.
4832 */
4833 saved_spec = efx_ef10_filter_entry_spec(table, ins_index);
4834 if (saved_spec) {
4835 replacing = true;
4836 } else {
4837 saved_spec = kmalloc(sizeof(*spec), GFP_ATOMIC);
4838 if (!saved_spec) {
4839 rc = -ENOMEM;
4840 goto fail_unlock;
4841 }
4842 *saved_spec = *spec;
4843 }
4844 efx_ef10_filter_set_entry(table, ins_index, saved_spec,
4845 EFX_EF10_FILTER_FLAG_BUSY);
4846
4847 spin_unlock_bh(&efx->filter_lock);
4848
4849 /* Pack up the variables needed on completion */
4850 cookie = replacing << 31 | ins_index << 16 | spec->dmaq_id;
4851
4852 efx_ef10_filter_push_prep(efx, spec, inbuf,
4853 table->entry[ins_index].handle, NULL,
4854 replacing);
4855 efx_mcdi_rpc_async(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf),
4856 MC_CMD_FILTER_OP_OUT_LEN,
4857 efx_ef10_filter_rfs_insert_complete, cookie);
4858
4859 return ins_index;
4860
4861fail_unlock:
4862 spin_unlock_bh(&efx->filter_lock);
4863 return rc;
4864}
4865
4866static void
4867efx_ef10_filter_rfs_insert_complete(struct efx_nic *efx, unsigned long cookie,
4868 int rc, efx_dword_t *outbuf,
4869 size_t outlen_actual)
4870{
4871 struct efx_ef10_filter_table *table = efx->filter_state;
4872 unsigned int ins_index, dmaq_id;
4873 struct efx_filter_spec *spec;
4874 bool replacing;
4875
4876 /* Unpack the cookie */
4877 replacing = cookie >> 31;
4878 ins_index = (cookie >> 16) & (HUNT_FILTER_TBL_ROWS - 1);
4879 dmaq_id = cookie & 0xffff;
4880
4881 spin_lock_bh(&efx->filter_lock);
4882 spec = efx_ef10_filter_entry_spec(table, ins_index);
4883 if (rc == 0) {
4884 table->entry[ins_index].handle =
4885 MCDI_QWORD(outbuf, FILTER_OP_OUT_HANDLE);
4886 if (replacing)
4887 spec->dmaq_id = dmaq_id;
4888 } else if (!replacing) {
4889 kfree(spec);
4890 spec = NULL;
4891 }
4892 efx_ef10_filter_set_entry(table, ins_index, spec, 0);
4893 spin_unlock_bh(&efx->filter_lock);
4894
4895 wake_up_all(&table->waitq);
4896}
4897
4898static void 4761static void
4899efx_ef10_filter_rfs_expire_complete(struct efx_nic *efx, 4762efx_ef10_filter_rfs_expire_complete(struct efx_nic *efx,
4900 unsigned long filter_idx, 4763 unsigned long filter_idx,
@@ -4905,18 +4768,22 @@ static bool efx_ef10_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,
4905 unsigned int filter_idx) 4768 unsigned int filter_idx)
4906{ 4769{
4907 struct efx_ef10_filter_table *table = efx->filter_state; 4770 struct efx_ef10_filter_table *table = efx->filter_state;
4908 struct efx_filter_spec *spec = 4771 struct efx_filter_spec *spec;
4909 efx_ef10_filter_entry_spec(table, filter_idx);
4910 MCDI_DECLARE_BUF(inbuf, 4772 MCDI_DECLARE_BUF(inbuf,
4911 MC_CMD_FILTER_OP_IN_HANDLE_OFST + 4773 MC_CMD_FILTER_OP_IN_HANDLE_OFST +
4912 MC_CMD_FILTER_OP_IN_HANDLE_LEN); 4774 MC_CMD_FILTER_OP_IN_HANDLE_LEN);
4775 bool ret = true;
4913 4776
4777 spin_lock_bh(&efx->filter_lock);
4778 spec = efx_ef10_filter_entry_spec(table, filter_idx);
4914 if (!spec || 4779 if (!spec ||
4915 (table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAG_BUSY) || 4780 (table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAG_BUSY) ||
4916 spec->priority != EFX_FILTER_PRI_HINT || 4781 spec->priority != EFX_FILTER_PRI_HINT ||
4917 !rps_may_expire_flow(efx->net_dev, spec->dmaq_id, 4782 !rps_may_expire_flow(efx->net_dev, spec->dmaq_id,
4918 flow_id, filter_idx)) 4783 flow_id, filter_idx)) {
4919 return false; 4784 ret = false;
4785 goto out_unlock;
4786 }
4920 4787
4921 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 4788 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
4922 MC_CMD_FILTER_OP_IN_OP_REMOVE); 4789 MC_CMD_FILTER_OP_IN_OP_REMOVE);
@@ -4924,10 +4791,12 @@ static bool efx_ef10_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,
4924 table->entry[filter_idx].handle); 4791 table->entry[filter_idx].handle);
4925 if (efx_mcdi_rpc_async(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), 0, 4792 if (efx_mcdi_rpc_async(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), 0,
4926 efx_ef10_filter_rfs_expire_complete, filter_idx)) 4793 efx_ef10_filter_rfs_expire_complete, filter_idx))
4927 return false; 4794 ret = false;
4928 4795 else
4929 table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY; 4796 table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY;
4930 return true; 4797out_unlock:
4798 spin_unlock_bh(&efx->filter_lock);
4799 return ret;
4931} 4800}
4932 4801
4933static void 4802static void
@@ -6784,7 +6653,6 @@ const struct efx_nic_type efx_hunt_a0_vf_nic_type = {
6784 .filter_get_rx_id_limit = efx_ef10_filter_get_rx_id_limit, 6653 .filter_get_rx_id_limit = efx_ef10_filter_get_rx_id_limit,
6785 .filter_get_rx_ids = efx_ef10_filter_get_rx_ids, 6654 .filter_get_rx_ids = efx_ef10_filter_get_rx_ids,
6786#ifdef CONFIG_RFS_ACCEL 6655#ifdef CONFIG_RFS_ACCEL
6787 .filter_rfs_insert = efx_ef10_filter_rfs_insert,
6788 .filter_rfs_expire_one = efx_ef10_filter_rfs_expire_one, 6656 .filter_rfs_expire_one = efx_ef10_filter_rfs_expire_one,
6789#endif 6657#endif
6790#ifdef CONFIG_SFC_MTD 6658#ifdef CONFIG_SFC_MTD
@@ -6897,7 +6765,6 @@ const struct efx_nic_type efx_hunt_a0_nic_type = {
6897 .filter_get_rx_id_limit = efx_ef10_filter_get_rx_id_limit, 6765 .filter_get_rx_id_limit = efx_ef10_filter_get_rx_id_limit,
6898 .filter_get_rx_ids = efx_ef10_filter_get_rx_ids, 6766 .filter_get_rx_ids = efx_ef10_filter_get_rx_ids,
6899#ifdef CONFIG_RFS_ACCEL 6767#ifdef CONFIG_RFS_ACCEL
6900 .filter_rfs_insert = efx_ef10_filter_rfs_insert,
6901 .filter_rfs_expire_one = efx_ef10_filter_rfs_expire_one, 6768 .filter_rfs_expire_one = efx_ef10_filter_rfs_expire_one,
6902#endif 6769#endif
6903#ifdef CONFIG_SFC_MTD 6770#ifdef CONFIG_SFC_MTD
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index 7321a4cf6f4d..0be93abdb2ad 100644
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -340,7 +340,10 @@ static int efx_poll(struct napi_struct *napi, int budget)
340 efx_update_irq_mod(efx, channel); 340 efx_update_irq_mod(efx, channel);
341 } 341 }
342 342
343 efx_filter_rfs_expire(channel); 343#ifdef CONFIG_RFS_ACCEL
344 /* Perhaps expire some ARFS filters */
345 schedule_work(&channel->filter_work);
346#endif
344 347
345 /* There is no race here; although napi_disable() will 348 /* There is no race here; although napi_disable() will
346 * only wait for napi_complete(), this isn't a problem 349 * only wait for napi_complete(), this isn't a problem
@@ -470,6 +473,10 @@ efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel)
470 tx_queue->channel = channel; 473 tx_queue->channel = channel;
471 } 474 }
472 475
476#ifdef CONFIG_RFS_ACCEL
477 INIT_WORK(&channel->filter_work, efx_filter_rfs_expire);
478#endif
479
473 rx_queue = &channel->rx_queue; 480 rx_queue = &channel->rx_queue;
474 rx_queue->efx = efx; 481 rx_queue->efx = efx;
475 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 482 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
@@ -512,6 +519,9 @@ efx_copy_channel(const struct efx_channel *old_channel)
512 rx_queue->buffer = NULL; 519 rx_queue->buffer = NULL;
513 memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 520 memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
514 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); 521 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
522#ifdef CONFIG_RFS_ACCEL
523 INIT_WORK(&channel->filter_work, efx_filter_rfs_expire);
524#endif
515 525
516 return channel; 526 return channel;
517} 527}
@@ -3012,6 +3022,9 @@ static int efx_init_struct(struct efx_nic *efx,
3012 efx->num_mac_stats = MC_CMD_MAC_NSTATS; 3022 efx->num_mac_stats = MC_CMD_MAC_NSTATS;
3013 BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END); 3023 BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END);
3014 mutex_init(&efx->mac_lock); 3024 mutex_init(&efx->mac_lock);
3025#ifdef CONFIG_RFS_ACCEL
3026 mutex_init(&efx->rps_mutex);
3027#endif
3015 efx->phy_op = &efx_dummy_phy_operations; 3028 efx->phy_op = &efx_dummy_phy_operations;
3016 efx->mdio.dev = net_dev; 3029 efx->mdio.dev = net_dev;
3017 INIT_WORK(&efx->mac_work, efx_mac_work); 3030 INIT_WORK(&efx->mac_work, efx_mac_work);
diff --git a/drivers/net/ethernet/sfc/efx.h b/drivers/net/ethernet/sfc/efx.h
index 3429ae3f3b08..545c2ea1622e 100644
--- a/drivers/net/ethernet/sfc/efx.h
+++ b/drivers/net/ethernet/sfc/efx.h
@@ -170,15 +170,18 @@ static inline s32 efx_filter_get_rx_ids(struct efx_nic *efx,
170int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, 170int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
171 u16 rxq_index, u32 flow_id); 171 u16 rxq_index, u32 flow_id);
172bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota); 172bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota);
173static inline void efx_filter_rfs_expire(struct efx_channel *channel) 173static inline void efx_filter_rfs_expire(struct work_struct *data)
174{ 174{
175 struct efx_channel *channel = container_of(data, struct efx_channel,
176 filter_work);
177
175 if (channel->rfs_filters_added >= 60 && 178 if (channel->rfs_filters_added >= 60 &&
176 __efx_filter_rfs_expire(channel->efx, 100)) 179 __efx_filter_rfs_expire(channel->efx, 100))
177 channel->rfs_filters_added -= 60; 180 channel->rfs_filters_added -= 60;
178} 181}
179#define efx_filter_rfs_enabled() 1 182#define efx_filter_rfs_enabled() 1
180#else 183#else
181static inline void efx_filter_rfs_expire(struct efx_channel *channel) {} 184static inline void efx_filter_rfs_expire(struct work_struct *data) {}
182#define efx_filter_rfs_enabled() 0 185#define efx_filter_rfs_enabled() 0
183#endif 186#endif
184bool efx_filter_is_mc_recipient(const struct efx_filter_spec *spec); 187bool efx_filter_is_mc_recipient(const struct efx_filter_spec *spec);
diff --git a/drivers/net/ethernet/sfc/farch.c b/drivers/net/ethernet/sfc/farch.c
index ad001e77d554..6b5ca569db5e 100644
--- a/drivers/net/ethernet/sfc/farch.c
+++ b/drivers/net/ethernet/sfc/farch.c
@@ -2901,28 +2901,25 @@ void efx_farch_filter_update_rx_scatter(struct efx_nic *efx)
2901 2901
2902#ifdef CONFIG_RFS_ACCEL 2902#ifdef CONFIG_RFS_ACCEL
2903 2903
2904s32 efx_farch_filter_rfs_insert(struct efx_nic *efx,
2905 struct efx_filter_spec *gen_spec)
2906{
2907 return efx_farch_filter_insert(efx, gen_spec, true);
2908}
2909
2910bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, 2904bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,
2911 unsigned int index) 2905 unsigned int index)
2912{ 2906{
2913 struct efx_farch_filter_state *state = efx->filter_state; 2907 struct efx_farch_filter_state *state = efx->filter_state;
2914 struct efx_farch_filter_table *table = 2908 struct efx_farch_filter_table *table;
2915 &state->table[EFX_FARCH_FILTER_TABLE_RX_IP]; 2909 bool ret = false;
2916 2910
2911 spin_lock_bh(&efx->filter_lock);
2912 table = &state->table[EFX_FARCH_FILTER_TABLE_RX_IP];
2917 if (test_bit(index, table->used_bitmap) && 2913 if (test_bit(index, table->used_bitmap) &&
2918 table->spec[index].priority == EFX_FILTER_PRI_HINT && 2914 table->spec[index].priority == EFX_FILTER_PRI_HINT &&
2919 rps_may_expire_flow(efx->net_dev, table->spec[index].dmaq_id, 2915 rps_may_expire_flow(efx->net_dev, table->spec[index].dmaq_id,
2920 flow_id, index)) { 2916 flow_id, index)) {
2921 efx_farch_filter_table_clear_entry(efx, table, index); 2917 efx_farch_filter_table_clear_entry(efx, table, index);
2922 return true; 2918 ret = true;
2923 } 2919 }
2924 2920
2925 return false; 2921 spin_unlock_bh(&efx->filter_lock);
2922 return ret;
2926} 2923}
2927 2924
2928#endif /* CONFIG_RFS_ACCEL */ 2925#endif /* CONFIG_RFS_ACCEL */
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index 2453f3849e72..dad78e3dde70 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -430,6 +430,7 @@ enum efx_sync_events_state {
430 * @event_test_cpu: Last CPU to handle interrupt or test event for this channel 430 * @event_test_cpu: Last CPU to handle interrupt or test event for this channel
431 * @irq_count: Number of IRQs since last adaptive moderation decision 431 * @irq_count: Number of IRQs since last adaptive moderation decision
432 * @irq_mod_score: IRQ moderation score 432 * @irq_mod_score: IRQ moderation score
433 * @filter_work: Work item for efx_filter_rfs_expire()
433 * @rps_flow_id: Flow IDs of filters allocated for accelerated RFS, 434 * @rps_flow_id: Flow IDs of filters allocated for accelerated RFS,
434 * indexed by filter ID 435 * indexed by filter ID
435 * @n_rx_tobe_disc: Count of RX_TOBE_DISC errors 436 * @n_rx_tobe_disc: Count of RX_TOBE_DISC errors
@@ -475,6 +476,7 @@ struct efx_channel {
475 unsigned int irq_mod_score; 476 unsigned int irq_mod_score;
476#ifdef CONFIG_RFS_ACCEL 477#ifdef CONFIG_RFS_ACCEL
477 unsigned int rfs_filters_added; 478 unsigned int rfs_filters_added;
479 struct work_struct filter_work;
478#define RPS_FLOW_ID_INVALID 0xFFFFFFFF 480#define RPS_FLOW_ID_INVALID 0xFFFFFFFF
479 u32 *rps_flow_id; 481 u32 *rps_flow_id;
480#endif 482#endif
@@ -844,6 +846,7 @@ struct efx_rss_context {
844 * @filter_sem: Filter table rw_semaphore, for freeing the table 846 * @filter_sem: Filter table rw_semaphore, for freeing the table
845 * @filter_lock: Filter table lock, for mere content changes 847 * @filter_lock: Filter table lock, for mere content changes
846 * @filter_state: Architecture-dependent filter table state 848 * @filter_state: Architecture-dependent filter table state
849 * @rps_mutex: Protects RPS state of all channels
847 * @rps_expire_channel: Next channel to check for expiry 850 * @rps_expire_channel: Next channel to check for expiry
848 * @rps_expire_index: Next index to check for expiry in 851 * @rps_expire_index: Next index to check for expiry in
849 * @rps_expire_channel's @rps_flow_id 852 * @rps_expire_channel's @rps_flow_id
@@ -998,6 +1001,7 @@ struct efx_nic {
998 spinlock_t filter_lock; 1001 spinlock_t filter_lock;
999 void *filter_state; 1002 void *filter_state;
1000#ifdef CONFIG_RFS_ACCEL 1003#ifdef CONFIG_RFS_ACCEL
1004 struct mutex rps_mutex;
1001 unsigned int rps_expire_channel; 1005 unsigned int rps_expire_channel;
1002 unsigned int rps_expire_index; 1006 unsigned int rps_expire_index;
1003#endif 1007#endif
@@ -1152,10 +1156,6 @@ struct efx_udp_tunnel {
1152 * @filter_count_rx_used: Get the number of filters in use at a given priority 1156 * @filter_count_rx_used: Get the number of filters in use at a given priority
1153 * @filter_get_rx_id_limit: Get maximum value of a filter id, plus 1 1157 * @filter_get_rx_id_limit: Get maximum value of a filter id, plus 1
1154 * @filter_get_rx_ids: Get list of RX filters at a given priority 1158 * @filter_get_rx_ids: Get list of RX filters at a given priority
1155 * @filter_rfs_insert: Add or replace a filter for RFS. This must be
1156 * atomic. The hardware change may be asynchronous but should
1157 * not be delayed for long. It may fail if this can't be done
1158 * atomically.
1159 * @filter_rfs_expire_one: Consider expiring a filter inserted for RFS. 1159 * @filter_rfs_expire_one: Consider expiring a filter inserted for RFS.
1160 * This must check whether the specified table entry is used by RFS 1160 * This must check whether the specified table entry is used by RFS
1161 * and that rps_may_expire_flow() returns true for it. 1161 * and that rps_may_expire_flow() returns true for it.
@@ -1306,8 +1306,6 @@ struct efx_nic_type {
1306 enum efx_filter_priority priority, 1306 enum efx_filter_priority priority,
1307 u32 *buf, u32 size); 1307 u32 *buf, u32 size);
1308#ifdef CONFIG_RFS_ACCEL 1308#ifdef CONFIG_RFS_ACCEL
1309 s32 (*filter_rfs_insert)(struct efx_nic *efx,
1310 struct efx_filter_spec *spec);
1311 bool (*filter_rfs_expire_one)(struct efx_nic *efx, u32 flow_id, 1309 bool (*filter_rfs_expire_one)(struct efx_nic *efx, u32 flow_id,
1312 unsigned int index); 1310 unsigned int index);
1313#endif 1311#endif
diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h
index d080a414e8f2..ac59afad6e5c 100644
--- a/drivers/net/ethernet/sfc/nic.h
+++ b/drivers/net/ethernet/sfc/nic.h
@@ -601,8 +601,6 @@ s32 efx_farch_filter_get_rx_ids(struct efx_nic *efx,
601 enum efx_filter_priority priority, u32 *buf, 601 enum efx_filter_priority priority, u32 *buf,
602 u32 size); 602 u32 size);
603#ifdef CONFIG_RFS_ACCEL 603#ifdef CONFIG_RFS_ACCEL
604s32 efx_farch_filter_rfs_insert(struct efx_nic *efx,
605 struct efx_filter_spec *spec);
606bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, 604bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,
607 unsigned int index); 605 unsigned int index);
608#endif 606#endif
diff --git a/drivers/net/ethernet/sfc/rx.c b/drivers/net/ethernet/sfc/rx.c
index cfe76aad79ee..95682831484e 100644
--- a/drivers/net/ethernet/sfc/rx.c
+++ b/drivers/net/ethernet/sfc/rx.c
@@ -827,14 +827,67 @@ MODULE_PARM_DESC(rx_refill_threshold,
827 827
828#ifdef CONFIG_RFS_ACCEL 828#ifdef CONFIG_RFS_ACCEL
829 829
830/**
831 * struct efx_async_filter_insertion - Request to asynchronously insert a filter
832 * @net_dev: Reference to the netdevice
833 * @spec: The filter to insert
834 * @work: Workitem for this request
835 * @rxq_index: Identifies the channel for which this request was made
836 * @flow_id: Identifies the kernel-side flow for which this request was made
837 */
838struct efx_async_filter_insertion {
839 struct net_device *net_dev;
840 struct efx_filter_spec spec;
841 struct work_struct work;
842 u16 rxq_index;
843 u32 flow_id;
844};
845
846static void efx_filter_rfs_work(struct work_struct *data)
847{
848 struct efx_async_filter_insertion *req = container_of(data, struct efx_async_filter_insertion,
849 work);
850 struct efx_nic *efx = netdev_priv(req->net_dev);
851 struct efx_channel *channel = efx_get_channel(efx, req->rxq_index);
852 int rc;
853
854 rc = efx->type->filter_insert(efx, &req->spec, false);
855 if (rc >= 0) {
856 /* Remember this so we can check whether to expire the filter
857 * later.
858 */
859 mutex_lock(&efx->rps_mutex);
860 channel->rps_flow_id[rc] = req->flow_id;
861 ++channel->rfs_filters_added;
862 mutex_unlock(&efx->rps_mutex);
863
864 if (req->spec.ether_type == htons(ETH_P_IP))
865 netif_info(efx, rx_status, efx->net_dev,
866 "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
867 (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
868 req->spec.rem_host, ntohs(req->spec.rem_port),
869 req->spec.loc_host, ntohs(req->spec.loc_port),
870 req->rxq_index, req->flow_id, rc);
871 else
872 netif_info(efx, rx_status, efx->net_dev,
873 "steering %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u filter %d]\n",
874 (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
875 req->spec.rem_host, ntohs(req->spec.rem_port),
876 req->spec.loc_host, ntohs(req->spec.loc_port),
877 req->rxq_index, req->flow_id, rc);
878 }
879
880 /* Release references */
881 dev_put(req->net_dev);
882 kfree(req);
883}
884
830int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, 885int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
831 u16 rxq_index, u32 flow_id) 886 u16 rxq_index, u32 flow_id)
832{ 887{
833 struct efx_nic *efx = netdev_priv(net_dev); 888 struct efx_nic *efx = netdev_priv(net_dev);
834 struct efx_channel *channel; 889 struct efx_async_filter_insertion *req;
835 struct efx_filter_spec spec;
836 struct flow_keys fk; 890 struct flow_keys fk;
837 int rc;
838 891
839 if (flow_id == RPS_FLOW_ID_INVALID) 892 if (flow_id == RPS_FLOW_ID_INVALID)
840 return -EINVAL; 893 return -EINVAL;
@@ -847,50 +900,39 @@ int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
847 if (fk.control.flags & FLOW_DIS_IS_FRAGMENT) 900 if (fk.control.flags & FLOW_DIS_IS_FRAGMENT)
848 return -EPROTONOSUPPORT; 901 return -EPROTONOSUPPORT;
849 902
850 efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, 903 req = kmalloc(sizeof(*req), GFP_ATOMIC);
904 if (!req)
905 return -ENOMEM;
906
907 efx_filter_init_rx(&req->spec, EFX_FILTER_PRI_HINT,
851 efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0, 908 efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0,
852 rxq_index); 909 rxq_index);
853 spec.match_flags = 910 req->spec.match_flags =
854 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | 911 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
855 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT | 912 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
856 EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT; 913 EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
857 spec.ether_type = fk.basic.n_proto; 914 req->spec.ether_type = fk.basic.n_proto;
858 spec.ip_proto = fk.basic.ip_proto; 915 req->spec.ip_proto = fk.basic.ip_proto;
859 916
860 if (fk.basic.n_proto == htons(ETH_P_IP)) { 917 if (fk.basic.n_proto == htons(ETH_P_IP)) {
861 spec.rem_host[0] = fk.addrs.v4addrs.src; 918 req->spec.rem_host[0] = fk.addrs.v4addrs.src;
862 spec.loc_host[0] = fk.addrs.v4addrs.dst; 919 req->spec.loc_host[0] = fk.addrs.v4addrs.dst;
863 } else { 920 } else {
864 memcpy(spec.rem_host, &fk.addrs.v6addrs.src, sizeof(struct in6_addr)); 921 memcpy(req->spec.rem_host, &fk.addrs.v6addrs.src,
865 memcpy(spec.loc_host, &fk.addrs.v6addrs.dst, sizeof(struct in6_addr)); 922 sizeof(struct in6_addr));
923 memcpy(req->spec.loc_host, &fk.addrs.v6addrs.dst,
924 sizeof(struct in6_addr));
866 } 925 }
867 926
868 spec.rem_port = fk.ports.src; 927 req->spec.rem_port = fk.ports.src;
869 spec.loc_port = fk.ports.dst; 928 req->spec.loc_port = fk.ports.dst;
870
871 rc = efx->type->filter_rfs_insert(efx, &spec);
872 if (rc < 0)
873 return rc;
874 929
875 /* Remember this so we can check whether to expire the filter later */ 930 dev_hold(req->net_dev = net_dev);
876 channel = efx_get_channel(efx, rxq_index); 931 INIT_WORK(&req->work, efx_filter_rfs_work);
877 channel->rps_flow_id[rc] = flow_id; 932 req->rxq_index = rxq_index;
878 ++channel->rfs_filters_added; 933 req->flow_id = flow_id;
879 934 schedule_work(&req->work);
880 if (spec.ether_type == htons(ETH_P_IP)) 935 return 0;
881 netif_info(efx, rx_status, efx->net_dev,
882 "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
883 (spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
884 spec.rem_host, ntohs(spec.rem_port), spec.loc_host,
885 ntohs(spec.loc_port), rxq_index, flow_id, rc);
886 else
887 netif_info(efx, rx_status, efx->net_dev,
888 "steering %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u filter %d]\n",
889 (spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
890 spec.rem_host, ntohs(spec.rem_port), spec.loc_host,
891 ntohs(spec.loc_port), rxq_index, flow_id, rc);
892
893 return rc;
894} 936}
895 937
896bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned int quota) 938bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned int quota)
@@ -899,9 +941,8 @@ bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned int quota)
899 unsigned int channel_idx, index, size; 941 unsigned int channel_idx, index, size;
900 u32 flow_id; 942 u32 flow_id;
901 943
902 if (!spin_trylock_bh(&efx->filter_lock)) 944 if (!mutex_trylock(&efx->rps_mutex))
903 return false; 945 return false;
904
905 expire_one = efx->type->filter_rfs_expire_one; 946 expire_one = efx->type->filter_rfs_expire_one;
906 channel_idx = efx->rps_expire_channel; 947 channel_idx = efx->rps_expire_channel;
907 index = efx->rps_expire_index; 948 index = efx->rps_expire_index;
@@ -926,7 +967,7 @@ bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned int quota)
926 efx->rps_expire_channel = channel_idx; 967 efx->rps_expire_channel = channel_idx;
927 efx->rps_expire_index = index; 968 efx->rps_expire_index = index;
928 969
929 spin_unlock_bh(&efx->filter_lock); 970 mutex_unlock(&efx->rps_mutex);
930 return true; 971 return true;
931} 972}
932 973
diff --git a/drivers/net/ethernet/sfc/siena.c b/drivers/net/ethernet/sfc/siena.c
index 18aab25234ba..65161f68265a 100644
--- a/drivers/net/ethernet/sfc/siena.c
+++ b/drivers/net/ethernet/sfc/siena.c
@@ -1035,7 +1035,6 @@ const struct efx_nic_type siena_a0_nic_type = {
1035 .filter_get_rx_id_limit = efx_farch_filter_get_rx_id_limit, 1035 .filter_get_rx_id_limit = efx_farch_filter_get_rx_id_limit,
1036 .filter_get_rx_ids = efx_farch_filter_get_rx_ids, 1036 .filter_get_rx_ids = efx_farch_filter_get_rx_ids,
1037#ifdef CONFIG_RFS_ACCEL 1037#ifdef CONFIG_RFS_ACCEL
1038 .filter_rfs_insert = efx_farch_filter_rfs_insert,
1039 .filter_rfs_expire_one = efx_farch_filter_rfs_expire_one, 1038 .filter_rfs_expire_one = efx_farch_filter_rfs_expire_one,
1040#endif 1039#endif
1041#ifdef CONFIG_SFC_MTD 1040#ifdef CONFIG_SFC_MTD