diff options
author | Edward Cree <ecree@solarflare.com> | 2018-03-27 12:41:59 -0400 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2018-03-27 13:33:19 -0400 |
commit | 3af0f34290f6192756ee1d9c2d5fe27222267035 (patch) | |
tree | 84d74399e83cc23f71854b35a7606d966f2e1143 | |
parent | c709002c23f91d90eb6ee2d4efbb548a8fe3cc80 (diff) |
sfc: replace asynchronous filter operations
Instead of having an efx->type->filter_rfs_insert() method, just use
workitems with a worker function that calls efx->type->filter_insert().
The only user of this is efx_filter_rfs(), which now queues a call to
efx_filter_rfs_work().
Similarly, efx_filter_rfs_expire() is now a worker function called on a
new channel->filter_work work_struct, so the method
efx->type->filter_rfs_expire_one() is no longer called in atomic context.
We also add a new mutex efx->rps_mutex to protect the RPS state (efx->
rps_expire_channel, efx->rps_expire_index, and channel->rps_flow_id) so
that the taking of efx->filter_lock can be moved to
efx->type->filter_rfs_expire_one().
Thus, all filter table functions are now called in a sleepable context,
allowing them to use sleeping locks in a future patch.
Signed-off-by: Edward Cree <ecree@solarflare.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r-- | drivers/net/ethernet/sfc/ef10.c | 161 | ||||
-rw-r--r-- | drivers/net/ethernet/sfc/efx.c | 15 | ||||
-rw-r--r-- | drivers/net/ethernet/sfc/efx.h | 7 | ||||
-rw-r--r-- | drivers/net/ethernet/sfc/farch.c | 17 | ||||
-rw-r--r-- | drivers/net/ethernet/sfc/net_driver.h | 10 | ||||
-rw-r--r-- | drivers/net/ethernet/sfc/nic.h | 2 | ||||
-rw-r--r-- | drivers/net/ethernet/sfc/rx.c | 119 | ||||
-rw-r--r-- | drivers/net/ethernet/sfc/siena.c | 1 |
8 files changed, 124 insertions, 208 deletions
diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c index e100273b623d..bcbaba330fb5 100644 --- a/drivers/net/ethernet/sfc/ef10.c +++ b/drivers/net/ethernet/sfc/ef10.c | |||
@@ -4758,143 +4758,6 @@ static s32 efx_ef10_filter_get_rx_ids(struct efx_nic *efx, | |||
4758 | 4758 | ||
4759 | #ifdef CONFIG_RFS_ACCEL | 4759 | #ifdef CONFIG_RFS_ACCEL |
4760 | 4760 | ||
4761 | static efx_mcdi_async_completer efx_ef10_filter_rfs_insert_complete; | ||
4762 | |||
4763 | static s32 efx_ef10_filter_rfs_insert(struct efx_nic *efx, | ||
4764 | struct efx_filter_spec *spec) | ||
4765 | { | ||
4766 | struct efx_ef10_filter_table *table = efx->filter_state; | ||
4767 | MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN); | ||
4768 | struct efx_filter_spec *saved_spec; | ||
4769 | unsigned int hash, i, depth = 1; | ||
4770 | bool replacing = false; | ||
4771 | int ins_index = -1; | ||
4772 | u64 cookie; | ||
4773 | s32 rc; | ||
4774 | |||
4775 | /* Must be an RX filter without RSS and not for a multicast | ||
4776 | * destination address (RFS only works for connected sockets). | ||
4777 | * These restrictions allow us to pass only a tiny amount of | ||
4778 | * data through to the completion function. | ||
4779 | */ | ||
4780 | EFX_WARN_ON_PARANOID(spec->flags != | ||
4781 | (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_RX_SCATTER)); | ||
4782 | EFX_WARN_ON_PARANOID(spec->priority != EFX_FILTER_PRI_HINT); | ||
4783 | EFX_WARN_ON_PARANOID(efx_filter_is_mc_recipient(spec)); | ||
4784 | |||
4785 | hash = efx_ef10_filter_hash(spec); | ||
4786 | |||
4787 | spin_lock_bh(&efx->filter_lock); | ||
4788 | |||
4789 | /* Find any existing filter with the same match tuple or else | ||
4790 | * a free slot to insert at. If an existing filter is busy, | ||
4791 | * we have to give up. | ||
4792 | */ | ||
4793 | for (;;) { | ||
4794 | i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1); | ||
4795 | saved_spec = efx_ef10_filter_entry_spec(table, i); | ||
4796 | |||
4797 | if (!saved_spec) { | ||
4798 | if (ins_index < 0) | ||
4799 | ins_index = i; | ||
4800 | } else if (efx_ef10_filter_equal(spec, saved_spec)) { | ||
4801 | if (table->entry[i].spec & EFX_EF10_FILTER_FLAG_BUSY) { | ||
4802 | rc = -EBUSY; | ||
4803 | goto fail_unlock; | ||
4804 | } | ||
4805 | if (spec->priority < saved_spec->priority) { | ||
4806 | rc = -EPERM; | ||
4807 | goto fail_unlock; | ||
4808 | } | ||
4809 | ins_index = i; | ||
4810 | break; | ||
4811 | } | ||
4812 | |||
4813 | /* Once we reach the maximum search depth, use the | ||
4814 | * first suitable slot or return -EBUSY if there was | ||
4815 | * none | ||
4816 | */ | ||
4817 | if (depth == EFX_EF10_FILTER_SEARCH_LIMIT) { | ||
4818 | if (ins_index < 0) { | ||
4819 | rc = -EBUSY; | ||
4820 | goto fail_unlock; | ||
4821 | } | ||
4822 | break; | ||
4823 | } | ||
4824 | |||
4825 | ++depth; | ||
4826 | } | ||
4827 | |||
4828 | /* Create a software table entry if necessary, and mark it | ||
4829 | * busy. We might yet fail to insert, but any attempt to | ||
4830 | * insert a conflicting filter while we're waiting for the | ||
4831 | * firmware must find the busy entry. | ||
4832 | */ | ||
4833 | saved_spec = efx_ef10_filter_entry_spec(table, ins_index); | ||
4834 | if (saved_spec) { | ||
4835 | replacing = true; | ||
4836 | } else { | ||
4837 | saved_spec = kmalloc(sizeof(*spec), GFP_ATOMIC); | ||
4838 | if (!saved_spec) { | ||
4839 | rc = -ENOMEM; | ||
4840 | goto fail_unlock; | ||
4841 | } | ||
4842 | *saved_spec = *spec; | ||
4843 | } | ||
4844 | efx_ef10_filter_set_entry(table, ins_index, saved_spec, | ||
4845 | EFX_EF10_FILTER_FLAG_BUSY); | ||
4846 | |||
4847 | spin_unlock_bh(&efx->filter_lock); | ||
4848 | |||
4849 | /* Pack up the variables needed on completion */ | ||
4850 | cookie = replacing << 31 | ins_index << 16 | spec->dmaq_id; | ||
4851 | |||
4852 | efx_ef10_filter_push_prep(efx, spec, inbuf, | ||
4853 | table->entry[ins_index].handle, NULL, | ||
4854 | replacing); | ||
4855 | efx_mcdi_rpc_async(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), | ||
4856 | MC_CMD_FILTER_OP_OUT_LEN, | ||
4857 | efx_ef10_filter_rfs_insert_complete, cookie); | ||
4858 | |||
4859 | return ins_index; | ||
4860 | |||
4861 | fail_unlock: | ||
4862 | spin_unlock_bh(&efx->filter_lock); | ||
4863 | return rc; | ||
4864 | } | ||
4865 | |||
4866 | static void | ||
4867 | efx_ef10_filter_rfs_insert_complete(struct efx_nic *efx, unsigned long cookie, | ||
4868 | int rc, efx_dword_t *outbuf, | ||
4869 | size_t outlen_actual) | ||
4870 | { | ||
4871 | struct efx_ef10_filter_table *table = efx->filter_state; | ||
4872 | unsigned int ins_index, dmaq_id; | ||
4873 | struct efx_filter_spec *spec; | ||
4874 | bool replacing; | ||
4875 | |||
4876 | /* Unpack the cookie */ | ||
4877 | replacing = cookie >> 31; | ||
4878 | ins_index = (cookie >> 16) & (HUNT_FILTER_TBL_ROWS - 1); | ||
4879 | dmaq_id = cookie & 0xffff; | ||
4880 | |||
4881 | spin_lock_bh(&efx->filter_lock); | ||
4882 | spec = efx_ef10_filter_entry_spec(table, ins_index); | ||
4883 | if (rc == 0) { | ||
4884 | table->entry[ins_index].handle = | ||
4885 | MCDI_QWORD(outbuf, FILTER_OP_OUT_HANDLE); | ||
4886 | if (replacing) | ||
4887 | spec->dmaq_id = dmaq_id; | ||
4888 | } else if (!replacing) { | ||
4889 | kfree(spec); | ||
4890 | spec = NULL; | ||
4891 | } | ||
4892 | efx_ef10_filter_set_entry(table, ins_index, spec, 0); | ||
4893 | spin_unlock_bh(&efx->filter_lock); | ||
4894 | |||
4895 | wake_up_all(&table->waitq); | ||
4896 | } | ||
4897 | |||
4898 | static void | 4761 | static void |
4899 | efx_ef10_filter_rfs_expire_complete(struct efx_nic *efx, | 4762 | efx_ef10_filter_rfs_expire_complete(struct efx_nic *efx, |
4900 | unsigned long filter_idx, | 4763 | unsigned long filter_idx, |
@@ -4905,18 +4768,22 @@ static bool efx_ef10_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, | |||
4905 | unsigned int filter_idx) | 4768 | unsigned int filter_idx) |
4906 | { | 4769 | { |
4907 | struct efx_ef10_filter_table *table = efx->filter_state; | 4770 | struct efx_ef10_filter_table *table = efx->filter_state; |
4908 | struct efx_filter_spec *spec = | 4771 | struct efx_filter_spec *spec; |
4909 | efx_ef10_filter_entry_spec(table, filter_idx); | ||
4910 | MCDI_DECLARE_BUF(inbuf, | 4772 | MCDI_DECLARE_BUF(inbuf, |
4911 | MC_CMD_FILTER_OP_IN_HANDLE_OFST + | 4773 | MC_CMD_FILTER_OP_IN_HANDLE_OFST + |
4912 | MC_CMD_FILTER_OP_IN_HANDLE_LEN); | 4774 | MC_CMD_FILTER_OP_IN_HANDLE_LEN); |
4775 | bool ret = true; | ||
4913 | 4776 | ||
4777 | spin_lock_bh(&efx->filter_lock); | ||
4778 | spec = efx_ef10_filter_entry_spec(table, filter_idx); | ||
4914 | if (!spec || | 4779 | if (!spec || |
4915 | (table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAG_BUSY) || | 4780 | (table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAG_BUSY) || |
4916 | spec->priority != EFX_FILTER_PRI_HINT || | 4781 | spec->priority != EFX_FILTER_PRI_HINT || |
4917 | !rps_may_expire_flow(efx->net_dev, spec->dmaq_id, | 4782 | !rps_may_expire_flow(efx->net_dev, spec->dmaq_id, |
4918 | flow_id, filter_idx)) | 4783 | flow_id, filter_idx)) { |
4919 | return false; | 4784 | ret = false; |
4785 | goto out_unlock; | ||
4786 | } | ||
4920 | 4787 | ||
4921 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, | 4788 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, |
4922 | MC_CMD_FILTER_OP_IN_OP_REMOVE); | 4789 | MC_CMD_FILTER_OP_IN_OP_REMOVE); |
@@ -4924,10 +4791,12 @@ static bool efx_ef10_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, | |||
4924 | table->entry[filter_idx].handle); | 4791 | table->entry[filter_idx].handle); |
4925 | if (efx_mcdi_rpc_async(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), 0, | 4792 | if (efx_mcdi_rpc_async(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), 0, |
4926 | efx_ef10_filter_rfs_expire_complete, filter_idx)) | 4793 | efx_ef10_filter_rfs_expire_complete, filter_idx)) |
4927 | return false; | 4794 | ret = false; |
4928 | 4795 | else | |
4929 | table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY; | 4796 | table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY; |
4930 | return true; | 4797 | out_unlock: |
4798 | spin_unlock_bh(&efx->filter_lock); | ||
4799 | return ret; | ||
4931 | } | 4800 | } |
4932 | 4801 | ||
4933 | static void | 4802 | static void |
@@ -6784,7 +6653,6 @@ const struct efx_nic_type efx_hunt_a0_vf_nic_type = { | |||
6784 | .filter_get_rx_id_limit = efx_ef10_filter_get_rx_id_limit, | 6653 | .filter_get_rx_id_limit = efx_ef10_filter_get_rx_id_limit, |
6785 | .filter_get_rx_ids = efx_ef10_filter_get_rx_ids, | 6654 | .filter_get_rx_ids = efx_ef10_filter_get_rx_ids, |
6786 | #ifdef CONFIG_RFS_ACCEL | 6655 | #ifdef CONFIG_RFS_ACCEL |
6787 | .filter_rfs_insert = efx_ef10_filter_rfs_insert, | ||
6788 | .filter_rfs_expire_one = efx_ef10_filter_rfs_expire_one, | 6656 | .filter_rfs_expire_one = efx_ef10_filter_rfs_expire_one, |
6789 | #endif | 6657 | #endif |
6790 | #ifdef CONFIG_SFC_MTD | 6658 | #ifdef CONFIG_SFC_MTD |
@@ -6897,7 +6765,6 @@ const struct efx_nic_type efx_hunt_a0_nic_type = { | |||
6897 | .filter_get_rx_id_limit = efx_ef10_filter_get_rx_id_limit, | 6765 | .filter_get_rx_id_limit = efx_ef10_filter_get_rx_id_limit, |
6898 | .filter_get_rx_ids = efx_ef10_filter_get_rx_ids, | 6766 | .filter_get_rx_ids = efx_ef10_filter_get_rx_ids, |
6899 | #ifdef CONFIG_RFS_ACCEL | 6767 | #ifdef CONFIG_RFS_ACCEL |
6900 | .filter_rfs_insert = efx_ef10_filter_rfs_insert, | ||
6901 | .filter_rfs_expire_one = efx_ef10_filter_rfs_expire_one, | 6768 | .filter_rfs_expire_one = efx_ef10_filter_rfs_expire_one, |
6902 | #endif | 6769 | #endif |
6903 | #ifdef CONFIG_SFC_MTD | 6770 | #ifdef CONFIG_SFC_MTD |
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index 7321a4cf6f4d..0be93abdb2ad 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c | |||
@@ -340,7 +340,10 @@ static int efx_poll(struct napi_struct *napi, int budget) | |||
340 | efx_update_irq_mod(efx, channel); | 340 | efx_update_irq_mod(efx, channel); |
341 | } | 341 | } |
342 | 342 | ||
343 | efx_filter_rfs_expire(channel); | 343 | #ifdef CONFIG_RFS_ACCEL |
344 | /* Perhaps expire some ARFS filters */ | ||
345 | schedule_work(&channel->filter_work); | ||
346 | #endif | ||
344 | 347 | ||
345 | /* There is no race here; although napi_disable() will | 348 | /* There is no race here; although napi_disable() will |
346 | * only wait for napi_complete(), this isn't a problem | 349 | * only wait for napi_complete(), this isn't a problem |
@@ -470,6 +473,10 @@ efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel) | |||
470 | tx_queue->channel = channel; | 473 | tx_queue->channel = channel; |
471 | } | 474 | } |
472 | 475 | ||
476 | #ifdef CONFIG_RFS_ACCEL | ||
477 | INIT_WORK(&channel->filter_work, efx_filter_rfs_expire); | ||
478 | #endif | ||
479 | |||
473 | rx_queue = &channel->rx_queue; | 480 | rx_queue = &channel->rx_queue; |
474 | rx_queue->efx = efx; | 481 | rx_queue->efx = efx; |
475 | timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); | 482 | timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); |
@@ -512,6 +519,9 @@ efx_copy_channel(const struct efx_channel *old_channel) | |||
512 | rx_queue->buffer = NULL; | 519 | rx_queue->buffer = NULL; |
513 | memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); | 520 | memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); |
514 | timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); | 521 | timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); |
522 | #ifdef CONFIG_RFS_ACCEL | ||
523 | INIT_WORK(&channel->filter_work, efx_filter_rfs_expire); | ||
524 | #endif | ||
515 | 525 | ||
516 | return channel; | 526 | return channel; |
517 | } | 527 | } |
@@ -3012,6 +3022,9 @@ static int efx_init_struct(struct efx_nic *efx, | |||
3012 | efx->num_mac_stats = MC_CMD_MAC_NSTATS; | 3022 | efx->num_mac_stats = MC_CMD_MAC_NSTATS; |
3013 | BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END); | 3023 | BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END); |
3014 | mutex_init(&efx->mac_lock); | 3024 | mutex_init(&efx->mac_lock); |
3025 | #ifdef CONFIG_RFS_ACCEL | ||
3026 | mutex_init(&efx->rps_mutex); | ||
3027 | #endif | ||
3015 | efx->phy_op = &efx_dummy_phy_operations; | 3028 | efx->phy_op = &efx_dummy_phy_operations; |
3016 | efx->mdio.dev = net_dev; | 3029 | efx->mdio.dev = net_dev; |
3017 | INIT_WORK(&efx->mac_work, efx_mac_work); | 3030 | INIT_WORK(&efx->mac_work, efx_mac_work); |
diff --git a/drivers/net/ethernet/sfc/efx.h b/drivers/net/ethernet/sfc/efx.h index 3429ae3f3b08..545c2ea1622e 100644 --- a/drivers/net/ethernet/sfc/efx.h +++ b/drivers/net/ethernet/sfc/efx.h | |||
@@ -170,15 +170,18 @@ static inline s32 efx_filter_get_rx_ids(struct efx_nic *efx, | |||
170 | int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, | 170 | int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, |
171 | u16 rxq_index, u32 flow_id); | 171 | u16 rxq_index, u32 flow_id); |
172 | bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota); | 172 | bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota); |
173 | static inline void efx_filter_rfs_expire(struct efx_channel *channel) | 173 | static inline void efx_filter_rfs_expire(struct work_struct *data) |
174 | { | 174 | { |
175 | struct efx_channel *channel = container_of(data, struct efx_channel, | ||
176 | filter_work); | ||
177 | |||
175 | if (channel->rfs_filters_added >= 60 && | 178 | if (channel->rfs_filters_added >= 60 && |
176 | __efx_filter_rfs_expire(channel->efx, 100)) | 179 | __efx_filter_rfs_expire(channel->efx, 100)) |
177 | channel->rfs_filters_added -= 60; | 180 | channel->rfs_filters_added -= 60; |
178 | } | 181 | } |
179 | #define efx_filter_rfs_enabled() 1 | 182 | #define efx_filter_rfs_enabled() 1 |
180 | #else | 183 | #else |
181 | static inline void efx_filter_rfs_expire(struct efx_channel *channel) {} | 184 | static inline void efx_filter_rfs_expire(struct work_struct *data) {} |
182 | #define efx_filter_rfs_enabled() 0 | 185 | #define efx_filter_rfs_enabled() 0 |
183 | #endif | 186 | #endif |
184 | bool efx_filter_is_mc_recipient(const struct efx_filter_spec *spec); | 187 | bool efx_filter_is_mc_recipient(const struct efx_filter_spec *spec); |
diff --git a/drivers/net/ethernet/sfc/farch.c b/drivers/net/ethernet/sfc/farch.c index ad001e77d554..6b5ca569db5e 100644 --- a/drivers/net/ethernet/sfc/farch.c +++ b/drivers/net/ethernet/sfc/farch.c | |||
@@ -2901,28 +2901,25 @@ void efx_farch_filter_update_rx_scatter(struct efx_nic *efx) | |||
2901 | 2901 | ||
2902 | #ifdef CONFIG_RFS_ACCEL | 2902 | #ifdef CONFIG_RFS_ACCEL |
2903 | 2903 | ||
2904 | s32 efx_farch_filter_rfs_insert(struct efx_nic *efx, | ||
2905 | struct efx_filter_spec *gen_spec) | ||
2906 | { | ||
2907 | return efx_farch_filter_insert(efx, gen_spec, true); | ||
2908 | } | ||
2909 | |||
2910 | bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, | 2904 | bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, |
2911 | unsigned int index) | 2905 | unsigned int index) |
2912 | { | 2906 | { |
2913 | struct efx_farch_filter_state *state = efx->filter_state; | 2907 | struct efx_farch_filter_state *state = efx->filter_state; |
2914 | struct efx_farch_filter_table *table = | 2908 | struct efx_farch_filter_table *table; |
2915 | &state->table[EFX_FARCH_FILTER_TABLE_RX_IP]; | 2909 | bool ret = false; |
2916 | 2910 | ||
2911 | spin_lock_bh(&efx->filter_lock); | ||
2912 | table = &state->table[EFX_FARCH_FILTER_TABLE_RX_IP]; | ||
2917 | if (test_bit(index, table->used_bitmap) && | 2913 | if (test_bit(index, table->used_bitmap) && |
2918 | table->spec[index].priority == EFX_FILTER_PRI_HINT && | 2914 | table->spec[index].priority == EFX_FILTER_PRI_HINT && |
2919 | rps_may_expire_flow(efx->net_dev, table->spec[index].dmaq_id, | 2915 | rps_may_expire_flow(efx->net_dev, table->spec[index].dmaq_id, |
2920 | flow_id, index)) { | 2916 | flow_id, index)) { |
2921 | efx_farch_filter_table_clear_entry(efx, table, index); | 2917 | efx_farch_filter_table_clear_entry(efx, table, index); |
2922 | return true; | 2918 | ret = true; |
2923 | } | 2919 | } |
2924 | 2920 | ||
2925 | return false; | 2921 | spin_unlock_bh(&efx->filter_lock); |
2922 | return ret; | ||
2926 | } | 2923 | } |
2927 | 2924 | ||
2928 | #endif /* CONFIG_RFS_ACCEL */ | 2925 | #endif /* CONFIG_RFS_ACCEL */ |
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h index 2453f3849e72..dad78e3dde70 100644 --- a/drivers/net/ethernet/sfc/net_driver.h +++ b/drivers/net/ethernet/sfc/net_driver.h | |||
@@ -430,6 +430,7 @@ enum efx_sync_events_state { | |||
430 | * @event_test_cpu: Last CPU to handle interrupt or test event for this channel | 430 | * @event_test_cpu: Last CPU to handle interrupt or test event for this channel |
431 | * @irq_count: Number of IRQs since last adaptive moderation decision | 431 | * @irq_count: Number of IRQs since last adaptive moderation decision |
432 | * @irq_mod_score: IRQ moderation score | 432 | * @irq_mod_score: IRQ moderation score |
433 | * @filter_work: Work item for efx_filter_rfs_expire() | ||
433 | * @rps_flow_id: Flow IDs of filters allocated for accelerated RFS, | 434 | * @rps_flow_id: Flow IDs of filters allocated for accelerated RFS, |
434 | * indexed by filter ID | 435 | * indexed by filter ID |
435 | * @n_rx_tobe_disc: Count of RX_TOBE_DISC errors | 436 | * @n_rx_tobe_disc: Count of RX_TOBE_DISC errors |
@@ -475,6 +476,7 @@ struct efx_channel { | |||
475 | unsigned int irq_mod_score; | 476 | unsigned int irq_mod_score; |
476 | #ifdef CONFIG_RFS_ACCEL | 477 | #ifdef CONFIG_RFS_ACCEL |
477 | unsigned int rfs_filters_added; | 478 | unsigned int rfs_filters_added; |
479 | struct work_struct filter_work; | ||
478 | #define RPS_FLOW_ID_INVALID 0xFFFFFFFF | 480 | #define RPS_FLOW_ID_INVALID 0xFFFFFFFF |
479 | u32 *rps_flow_id; | 481 | u32 *rps_flow_id; |
480 | #endif | 482 | #endif |
@@ -844,6 +846,7 @@ struct efx_rss_context { | |||
844 | * @filter_sem: Filter table rw_semaphore, for freeing the table | 846 | * @filter_sem: Filter table rw_semaphore, for freeing the table |
845 | * @filter_lock: Filter table lock, for mere content changes | 847 | * @filter_lock: Filter table lock, for mere content changes |
846 | * @filter_state: Architecture-dependent filter table state | 848 | * @filter_state: Architecture-dependent filter table state |
849 | * @rps_mutex: Protects RPS state of all channels | ||
847 | * @rps_expire_channel: Next channel to check for expiry | 850 | * @rps_expire_channel: Next channel to check for expiry |
848 | * @rps_expire_index: Next index to check for expiry in | 851 | * @rps_expire_index: Next index to check for expiry in |
849 | * @rps_expire_channel's @rps_flow_id | 852 | * @rps_expire_channel's @rps_flow_id |
@@ -998,6 +1001,7 @@ struct efx_nic { | |||
998 | spinlock_t filter_lock; | 1001 | spinlock_t filter_lock; |
999 | void *filter_state; | 1002 | void *filter_state; |
1000 | #ifdef CONFIG_RFS_ACCEL | 1003 | #ifdef CONFIG_RFS_ACCEL |
1004 | struct mutex rps_mutex; | ||
1001 | unsigned int rps_expire_channel; | 1005 | unsigned int rps_expire_channel; |
1002 | unsigned int rps_expire_index; | 1006 | unsigned int rps_expire_index; |
1003 | #endif | 1007 | #endif |
@@ -1152,10 +1156,6 @@ struct efx_udp_tunnel { | |||
1152 | * @filter_count_rx_used: Get the number of filters in use at a given priority | 1156 | * @filter_count_rx_used: Get the number of filters in use at a given priority |
1153 | * @filter_get_rx_id_limit: Get maximum value of a filter id, plus 1 | 1157 | * @filter_get_rx_id_limit: Get maximum value of a filter id, plus 1 |
1154 | * @filter_get_rx_ids: Get list of RX filters at a given priority | 1158 | * @filter_get_rx_ids: Get list of RX filters at a given priority |
1155 | * @filter_rfs_insert: Add or replace a filter for RFS. This must be | ||
1156 | * atomic. The hardware change may be asynchronous but should | ||
1157 | * not be delayed for long. It may fail if this can't be done | ||
1158 | * atomically. | ||
1159 | * @filter_rfs_expire_one: Consider expiring a filter inserted for RFS. | 1159 | * @filter_rfs_expire_one: Consider expiring a filter inserted for RFS. |
1160 | * This must check whether the specified table entry is used by RFS | 1160 | * This must check whether the specified table entry is used by RFS |
1161 | * and that rps_may_expire_flow() returns true for it. | 1161 | * and that rps_may_expire_flow() returns true for it. |
@@ -1306,8 +1306,6 @@ struct efx_nic_type { | |||
1306 | enum efx_filter_priority priority, | 1306 | enum efx_filter_priority priority, |
1307 | u32 *buf, u32 size); | 1307 | u32 *buf, u32 size); |
1308 | #ifdef CONFIG_RFS_ACCEL | 1308 | #ifdef CONFIG_RFS_ACCEL |
1309 | s32 (*filter_rfs_insert)(struct efx_nic *efx, | ||
1310 | struct efx_filter_spec *spec); | ||
1311 | bool (*filter_rfs_expire_one)(struct efx_nic *efx, u32 flow_id, | 1309 | bool (*filter_rfs_expire_one)(struct efx_nic *efx, u32 flow_id, |
1312 | unsigned int index); | 1310 | unsigned int index); |
1313 | #endif | 1311 | #endif |
diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h index d080a414e8f2..ac59afad6e5c 100644 --- a/drivers/net/ethernet/sfc/nic.h +++ b/drivers/net/ethernet/sfc/nic.h | |||
@@ -601,8 +601,6 @@ s32 efx_farch_filter_get_rx_ids(struct efx_nic *efx, | |||
601 | enum efx_filter_priority priority, u32 *buf, | 601 | enum efx_filter_priority priority, u32 *buf, |
602 | u32 size); | 602 | u32 size); |
603 | #ifdef CONFIG_RFS_ACCEL | 603 | #ifdef CONFIG_RFS_ACCEL |
604 | s32 efx_farch_filter_rfs_insert(struct efx_nic *efx, | ||
605 | struct efx_filter_spec *spec); | ||
606 | bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, | 604 | bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, |
607 | unsigned int index); | 605 | unsigned int index); |
608 | #endif | 606 | #endif |
diff --git a/drivers/net/ethernet/sfc/rx.c b/drivers/net/ethernet/sfc/rx.c index cfe76aad79ee..95682831484e 100644 --- a/drivers/net/ethernet/sfc/rx.c +++ b/drivers/net/ethernet/sfc/rx.c | |||
@@ -827,14 +827,67 @@ MODULE_PARM_DESC(rx_refill_threshold, | |||
827 | 827 | ||
828 | #ifdef CONFIG_RFS_ACCEL | 828 | #ifdef CONFIG_RFS_ACCEL |
829 | 829 | ||
830 | /** | ||
831 | * struct efx_async_filter_insertion - Request to asynchronously insert a filter | ||
832 | * @net_dev: Reference to the netdevice | ||
833 | * @spec: The filter to insert | ||
834 | * @work: Workitem for this request | ||
835 | * @rxq_index: Identifies the channel for which this request was made | ||
836 | * @flow_id: Identifies the kernel-side flow for which this request was made | ||
837 | */ | ||
838 | struct efx_async_filter_insertion { | ||
839 | struct net_device *net_dev; | ||
840 | struct efx_filter_spec spec; | ||
841 | struct work_struct work; | ||
842 | u16 rxq_index; | ||
843 | u32 flow_id; | ||
844 | }; | ||
845 | |||
846 | static void efx_filter_rfs_work(struct work_struct *data) | ||
847 | { | ||
848 | struct efx_async_filter_insertion *req = container_of(data, struct efx_async_filter_insertion, | ||
849 | work); | ||
850 | struct efx_nic *efx = netdev_priv(req->net_dev); | ||
851 | struct efx_channel *channel = efx_get_channel(efx, req->rxq_index); | ||
852 | int rc; | ||
853 | |||
854 | rc = efx->type->filter_insert(efx, &req->spec, false); | ||
855 | if (rc >= 0) { | ||
856 | /* Remember this so we can check whether to expire the filter | ||
857 | * later. | ||
858 | */ | ||
859 | mutex_lock(&efx->rps_mutex); | ||
860 | channel->rps_flow_id[rc] = req->flow_id; | ||
861 | ++channel->rfs_filters_added; | ||
862 | mutex_unlock(&efx->rps_mutex); | ||
863 | |||
864 | if (req->spec.ether_type == htons(ETH_P_IP)) | ||
865 | netif_info(efx, rx_status, efx->net_dev, | ||
866 | "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n", | ||
867 | (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP", | ||
868 | req->spec.rem_host, ntohs(req->spec.rem_port), | ||
869 | req->spec.loc_host, ntohs(req->spec.loc_port), | ||
870 | req->rxq_index, req->flow_id, rc); | ||
871 | else | ||
872 | netif_info(efx, rx_status, efx->net_dev, | ||
873 | "steering %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u filter %d]\n", | ||
874 | (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP", | ||
875 | req->spec.rem_host, ntohs(req->spec.rem_port), | ||
876 | req->spec.loc_host, ntohs(req->spec.loc_port), | ||
877 | req->rxq_index, req->flow_id, rc); | ||
878 | } | ||
879 | |||
880 | /* Release references */ | ||
881 | dev_put(req->net_dev); | ||
882 | kfree(req); | ||
883 | } | ||
884 | |||
830 | int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, | 885 | int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, |
831 | u16 rxq_index, u32 flow_id) | 886 | u16 rxq_index, u32 flow_id) |
832 | { | 887 | { |
833 | struct efx_nic *efx = netdev_priv(net_dev); | 888 | struct efx_nic *efx = netdev_priv(net_dev); |
834 | struct efx_channel *channel; | 889 | struct efx_async_filter_insertion *req; |
835 | struct efx_filter_spec spec; | ||
836 | struct flow_keys fk; | 890 | struct flow_keys fk; |
837 | int rc; | ||
838 | 891 | ||
839 | if (flow_id == RPS_FLOW_ID_INVALID) | 892 | if (flow_id == RPS_FLOW_ID_INVALID) |
840 | return -EINVAL; | 893 | return -EINVAL; |
@@ -847,50 +900,39 @@ int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, | |||
847 | if (fk.control.flags & FLOW_DIS_IS_FRAGMENT) | 900 | if (fk.control.flags & FLOW_DIS_IS_FRAGMENT) |
848 | return -EPROTONOSUPPORT; | 901 | return -EPROTONOSUPPORT; |
849 | 902 | ||
850 | efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, | 903 | req = kmalloc(sizeof(*req), GFP_ATOMIC); |
904 | if (!req) | ||
905 | return -ENOMEM; | ||
906 | |||
907 | efx_filter_init_rx(&req->spec, EFX_FILTER_PRI_HINT, | ||
851 | efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0, | 908 | efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0, |
852 | rxq_index); | 909 | rxq_index); |
853 | spec.match_flags = | 910 | req->spec.match_flags = |
854 | EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | | 911 | EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | |
855 | EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT | | 912 | EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT | |
856 | EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT; | 913 | EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT; |
857 | spec.ether_type = fk.basic.n_proto; | 914 | req->spec.ether_type = fk.basic.n_proto; |
858 | spec.ip_proto = fk.basic.ip_proto; | 915 | req->spec.ip_proto = fk.basic.ip_proto; |
859 | 916 | ||
860 | if (fk.basic.n_proto == htons(ETH_P_IP)) { | 917 | if (fk.basic.n_proto == htons(ETH_P_IP)) { |
861 | spec.rem_host[0] = fk.addrs.v4addrs.src; | 918 | req->spec.rem_host[0] = fk.addrs.v4addrs.src; |
862 | spec.loc_host[0] = fk.addrs.v4addrs.dst; | 919 | req->spec.loc_host[0] = fk.addrs.v4addrs.dst; |
863 | } else { | 920 | } else { |
864 | memcpy(spec.rem_host, &fk.addrs.v6addrs.src, sizeof(struct in6_addr)); | 921 | memcpy(req->spec.rem_host, &fk.addrs.v6addrs.src, |
865 | memcpy(spec.loc_host, &fk.addrs.v6addrs.dst, sizeof(struct in6_addr)); | 922 | sizeof(struct in6_addr)); |
923 | memcpy(req->spec.loc_host, &fk.addrs.v6addrs.dst, | ||
924 | sizeof(struct in6_addr)); | ||
866 | } | 925 | } |
867 | 926 | ||
868 | spec.rem_port = fk.ports.src; | 927 | req->spec.rem_port = fk.ports.src; |
869 | spec.loc_port = fk.ports.dst; | 928 | req->spec.loc_port = fk.ports.dst; |
870 | |||
871 | rc = efx->type->filter_rfs_insert(efx, &spec); | ||
872 | if (rc < 0) | ||
873 | return rc; | ||
874 | 929 | ||
875 | /* Remember this so we can check whether to expire the filter later */ | 930 | dev_hold(req->net_dev = net_dev); |
876 | channel = efx_get_channel(efx, rxq_index); | 931 | INIT_WORK(&req->work, efx_filter_rfs_work); |
877 | channel->rps_flow_id[rc] = flow_id; | 932 | req->rxq_index = rxq_index; |
878 | ++channel->rfs_filters_added; | 933 | req->flow_id = flow_id; |
879 | 934 | schedule_work(&req->work); | |
880 | if (spec.ether_type == htons(ETH_P_IP)) | 935 | return 0; |
881 | netif_info(efx, rx_status, efx->net_dev, | ||
882 | "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n", | ||
883 | (spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP", | ||
884 | spec.rem_host, ntohs(spec.rem_port), spec.loc_host, | ||
885 | ntohs(spec.loc_port), rxq_index, flow_id, rc); | ||
886 | else | ||
887 | netif_info(efx, rx_status, efx->net_dev, | ||
888 | "steering %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u filter %d]\n", | ||
889 | (spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP", | ||
890 | spec.rem_host, ntohs(spec.rem_port), spec.loc_host, | ||
891 | ntohs(spec.loc_port), rxq_index, flow_id, rc); | ||
892 | |||
893 | return rc; | ||
894 | } | 936 | } |
895 | 937 | ||
896 | bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned int quota) | 938 | bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned int quota) |
@@ -899,9 +941,8 @@ bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned int quota) | |||
899 | unsigned int channel_idx, index, size; | 941 | unsigned int channel_idx, index, size; |
900 | u32 flow_id; | 942 | u32 flow_id; |
901 | 943 | ||
902 | if (!spin_trylock_bh(&efx->filter_lock)) | 944 | if (!mutex_trylock(&efx->rps_mutex)) |
903 | return false; | 945 | return false; |
904 | |||
905 | expire_one = efx->type->filter_rfs_expire_one; | 946 | expire_one = efx->type->filter_rfs_expire_one; |
906 | channel_idx = efx->rps_expire_channel; | 947 | channel_idx = efx->rps_expire_channel; |
907 | index = efx->rps_expire_index; | 948 | index = efx->rps_expire_index; |
@@ -926,7 +967,7 @@ bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned int quota) | |||
926 | efx->rps_expire_channel = channel_idx; | 967 | efx->rps_expire_channel = channel_idx; |
927 | efx->rps_expire_index = index; | 968 | efx->rps_expire_index = index; |
928 | 969 | ||
929 | spin_unlock_bh(&efx->filter_lock); | 970 | mutex_unlock(&efx->rps_mutex); |
930 | return true; | 971 | return true; |
931 | } | 972 | } |
932 | 973 | ||
diff --git a/drivers/net/ethernet/sfc/siena.c b/drivers/net/ethernet/sfc/siena.c index 18aab25234ba..65161f68265a 100644 --- a/drivers/net/ethernet/sfc/siena.c +++ b/drivers/net/ethernet/sfc/siena.c | |||
@@ -1035,7 +1035,6 @@ const struct efx_nic_type siena_a0_nic_type = { | |||
1035 | .filter_get_rx_id_limit = efx_farch_filter_get_rx_id_limit, | 1035 | .filter_get_rx_id_limit = efx_farch_filter_get_rx_id_limit, |
1036 | .filter_get_rx_ids = efx_farch_filter_get_rx_ids, | 1036 | .filter_get_rx_ids = efx_farch_filter_get_rx_ids, |
1037 | #ifdef CONFIG_RFS_ACCEL | 1037 | #ifdef CONFIG_RFS_ACCEL |
1038 | .filter_rfs_insert = efx_farch_filter_rfs_insert, | ||
1039 | .filter_rfs_expire_one = efx_farch_filter_rfs_expire_one, | 1038 | .filter_rfs_expire_one = efx_farch_filter_rfs_expire_one, |
1040 | #endif | 1039 | #endif |
1041 | #ifdef CONFIG_SFC_MTD | 1040 | #ifdef CONFIG_SFC_MTD |