diff options
Diffstat (limited to 'drivers/net/sfc/filter.c')
-rw-r--r-- | drivers/net/sfc/filter.c | 727 |
1 files changed, 727 insertions, 0 deletions
diff --git a/drivers/net/sfc/filter.c b/drivers/net/sfc/filter.c new file mode 100644 index 00000000000..2b9636f96e0 --- /dev/null +++ b/drivers/net/sfc/filter.c | |||
@@ -0,0 +1,727 @@ | |||
1 | /**************************************************************************** | ||
2 | * Driver for Solarflare Solarstorm network controllers and boards | ||
3 | * Copyright 2005-2010 Solarflare Communications Inc. | ||
4 | * | ||
5 | * This program is free software; you can redistribute it and/or modify it | ||
6 | * under the terms of the GNU General Public License version 2 as published | ||
7 | * by the Free Software Foundation, incorporated herein by reference. | ||
8 | */ | ||
9 | |||
10 | #include <linux/in.h> | ||
11 | #include <net/ip.h> | ||
12 | #include "efx.h" | ||
13 | #include "filter.h" | ||
14 | #include "io.h" | ||
15 | #include "nic.h" | ||
16 | #include "regs.h" | ||
17 | |||
18 | /* "Fudge factors" - difference between programmed value and actual depth. | ||
19 | * Due to pipelined implementation we need to program H/W with a value that | ||
20 | * is larger than the hop limit we want. | ||
21 | */ | ||
22 | #define FILTER_CTL_SRCH_FUDGE_WILD 3 | ||
23 | #define FILTER_CTL_SRCH_FUDGE_FULL 1 | ||
24 | |||
25 | /* Hard maximum hop limit. Hardware will time-out beyond 200-something. | ||
26 | * We also need to avoid infinite loops in efx_filter_search() when the | ||
27 | * table is full. | ||
28 | */ | ||
29 | #define FILTER_CTL_SRCH_MAX 200 | ||
30 | |||
31 | /* Don't try very hard to find space for performance hints, as this is | ||
32 | * counter-productive. */ | ||
33 | #define FILTER_CTL_SRCH_HINT_MAX 5 | ||
34 | |||
35 | enum efx_filter_table_id { | ||
36 | EFX_FILTER_TABLE_RX_IP = 0, | ||
37 | EFX_FILTER_TABLE_RX_MAC, | ||
38 | EFX_FILTER_TABLE_COUNT, | ||
39 | }; | ||
40 | |||
41 | struct efx_filter_table { | ||
42 | enum efx_filter_table_id id; | ||
43 | u32 offset; /* address of table relative to BAR */ | ||
44 | unsigned size; /* number of entries */ | ||
45 | unsigned step; /* step between entries */ | ||
46 | unsigned used; /* number currently used */ | ||
47 | unsigned long *used_bitmap; | ||
48 | struct efx_filter_spec *spec; | ||
49 | unsigned search_depth[EFX_FILTER_TYPE_COUNT]; | ||
50 | }; | ||
51 | |||
52 | struct efx_filter_state { | ||
53 | spinlock_t lock; | ||
54 | struct efx_filter_table table[EFX_FILTER_TABLE_COUNT]; | ||
55 | #ifdef CONFIG_RFS_ACCEL | ||
56 | u32 *rps_flow_id; | ||
57 | unsigned rps_expire_index; | ||
58 | #endif | ||
59 | }; | ||
60 | |||
61 | /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit | ||
62 | * key derived from the n-tuple. The initial LFSR state is 0xffff. */ | ||
63 | static u16 efx_filter_hash(u32 key) | ||
64 | { | ||
65 | u16 tmp; | ||
66 | |||
67 | /* First 16 rounds */ | ||
68 | tmp = 0x1fff ^ key >> 16; | ||
69 | tmp = tmp ^ tmp >> 3 ^ tmp >> 6; | ||
70 | tmp = tmp ^ tmp >> 9; | ||
71 | /* Last 16 rounds */ | ||
72 | tmp = tmp ^ tmp << 13 ^ key; | ||
73 | tmp = tmp ^ tmp >> 3 ^ tmp >> 6; | ||
74 | return tmp ^ tmp >> 9; | ||
75 | } | ||
76 | |||
77 | /* To allow for hash collisions, filter search continues at these | ||
78 | * increments from the first possible entry selected by the hash. */ | ||
79 | static u16 efx_filter_increment(u32 key) | ||
80 | { | ||
81 | return key * 2 - 1; | ||
82 | } | ||
83 | |||
84 | static enum efx_filter_table_id | ||
85 | efx_filter_spec_table_id(const struct efx_filter_spec *spec) | ||
86 | { | ||
87 | BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2)); | ||
88 | BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2)); | ||
89 | BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2)); | ||
90 | BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2)); | ||
91 | BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2)); | ||
92 | BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2)); | ||
93 | EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC); | ||
94 | return spec->type >> 2; | ||
95 | } | ||
96 | |||
97 | static struct efx_filter_table * | ||
98 | efx_filter_spec_table(struct efx_filter_state *state, | ||
99 | const struct efx_filter_spec *spec) | ||
100 | { | ||
101 | if (spec->type == EFX_FILTER_UNSPEC) | ||
102 | return NULL; | ||
103 | else | ||
104 | return &state->table[efx_filter_spec_table_id(spec)]; | ||
105 | } | ||
106 | |||
107 | static void efx_filter_table_reset_search_depth(struct efx_filter_table *table) | ||
108 | { | ||
109 | memset(table->search_depth, 0, sizeof(table->search_depth)); | ||
110 | } | ||
111 | |||
112 | static void efx_filter_push_rx_limits(struct efx_nic *efx) | ||
113 | { | ||
114 | struct efx_filter_state *state = efx->filter_state; | ||
115 | struct efx_filter_table *table; | ||
116 | efx_oword_t filter_ctl; | ||
117 | |||
118 | efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL); | ||
119 | |||
120 | table = &state->table[EFX_FILTER_TABLE_RX_IP]; | ||
121 | EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT, | ||
122 | table->search_depth[EFX_FILTER_TCP_FULL] + | ||
123 | FILTER_CTL_SRCH_FUDGE_FULL); | ||
124 | EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT, | ||
125 | table->search_depth[EFX_FILTER_TCP_WILD] + | ||
126 | FILTER_CTL_SRCH_FUDGE_WILD); | ||
127 | EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT, | ||
128 | table->search_depth[EFX_FILTER_UDP_FULL] + | ||
129 | FILTER_CTL_SRCH_FUDGE_FULL); | ||
130 | EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT, | ||
131 | table->search_depth[EFX_FILTER_UDP_WILD] + | ||
132 | FILTER_CTL_SRCH_FUDGE_WILD); | ||
133 | |||
134 | table = &state->table[EFX_FILTER_TABLE_RX_MAC]; | ||
135 | if (table->size) { | ||
136 | EFX_SET_OWORD_FIELD( | ||
137 | filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT, | ||
138 | table->search_depth[EFX_FILTER_MAC_FULL] + | ||
139 | FILTER_CTL_SRCH_FUDGE_FULL); | ||
140 | EFX_SET_OWORD_FIELD( | ||
141 | filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT, | ||
142 | table->search_depth[EFX_FILTER_MAC_WILD] + | ||
143 | FILTER_CTL_SRCH_FUDGE_WILD); | ||
144 | } | ||
145 | |||
146 | efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL); | ||
147 | } | ||
148 | |||
149 | static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec, | ||
150 | __be32 host1, __be16 port1, | ||
151 | __be32 host2, __be16 port2) | ||
152 | { | ||
153 | spec->data[0] = ntohl(host1) << 16 | ntohs(port1); | ||
154 | spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16; | ||
155 | spec->data[2] = ntohl(host2); | ||
156 | } | ||
157 | |||
158 | /** | ||
159 | * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port | ||
160 | * @spec: Specification to initialise | ||
161 | * @proto: Transport layer protocol number | ||
162 | * @host: Local host address (network byte order) | ||
163 | * @port: Local port (network byte order) | ||
164 | */ | ||
165 | int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto, | ||
166 | __be32 host, __be16 port) | ||
167 | { | ||
168 | __be32 host1; | ||
169 | __be16 port1; | ||
170 | |||
171 | EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX)); | ||
172 | |||
173 | /* This cannot currently be combined with other filtering */ | ||
174 | if (spec->type != EFX_FILTER_UNSPEC) | ||
175 | return -EPROTONOSUPPORT; | ||
176 | |||
177 | if (port == 0) | ||
178 | return -EINVAL; | ||
179 | |||
180 | switch (proto) { | ||
181 | case IPPROTO_TCP: | ||
182 | spec->type = EFX_FILTER_TCP_WILD; | ||
183 | break; | ||
184 | case IPPROTO_UDP: | ||
185 | spec->type = EFX_FILTER_UDP_WILD; | ||
186 | break; | ||
187 | default: | ||
188 | return -EPROTONOSUPPORT; | ||
189 | } | ||
190 | |||
191 | /* Filter is constructed in terms of source and destination, | ||
192 | * with the odd wrinkle that the ports are swapped in a UDP | ||
193 | * wildcard filter. We need to convert from local and remote | ||
194 | * (= zero for wildcard) addresses. | ||
195 | */ | ||
196 | host1 = 0; | ||
197 | if (proto != IPPROTO_UDP) { | ||
198 | port1 = 0; | ||
199 | } else { | ||
200 | port1 = port; | ||
201 | port = 0; | ||
202 | } | ||
203 | |||
204 | __efx_filter_set_ipv4(spec, host1, port1, host, port); | ||
205 | return 0; | ||
206 | } | ||
207 | |||
208 | /** | ||
209 | * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports | ||
210 | * @spec: Specification to initialise | ||
211 | * @proto: Transport layer protocol number | ||
212 | * @host: Local host address (network byte order) | ||
213 | * @port: Local port (network byte order) | ||
214 | * @rhost: Remote host address (network byte order) | ||
215 | * @rport: Remote port (network byte order) | ||
216 | */ | ||
217 | int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto, | ||
218 | __be32 host, __be16 port, | ||
219 | __be32 rhost, __be16 rport) | ||
220 | { | ||
221 | EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX)); | ||
222 | |||
223 | /* This cannot currently be combined with other filtering */ | ||
224 | if (spec->type != EFX_FILTER_UNSPEC) | ||
225 | return -EPROTONOSUPPORT; | ||
226 | |||
227 | if (port == 0 || rport == 0) | ||
228 | return -EINVAL; | ||
229 | |||
230 | switch (proto) { | ||
231 | case IPPROTO_TCP: | ||
232 | spec->type = EFX_FILTER_TCP_FULL; | ||
233 | break; | ||
234 | case IPPROTO_UDP: | ||
235 | spec->type = EFX_FILTER_UDP_FULL; | ||
236 | break; | ||
237 | default: | ||
238 | return -EPROTONOSUPPORT; | ||
239 | } | ||
240 | |||
241 | __efx_filter_set_ipv4(spec, rhost, rport, host, port); | ||
242 | return 0; | ||
243 | } | ||
244 | |||
245 | /** | ||
246 | * efx_filter_set_eth_local - specify local Ethernet address and optional VID | ||
247 | * @spec: Specification to initialise | ||
248 | * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC | ||
249 | * @addr: Local Ethernet MAC address | ||
250 | */ | ||
251 | int efx_filter_set_eth_local(struct efx_filter_spec *spec, | ||
252 | u16 vid, const u8 *addr) | ||
253 | { | ||
254 | EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX)); | ||
255 | |||
256 | /* This cannot currently be combined with other filtering */ | ||
257 | if (spec->type != EFX_FILTER_UNSPEC) | ||
258 | return -EPROTONOSUPPORT; | ||
259 | |||
260 | if (vid == EFX_FILTER_VID_UNSPEC) { | ||
261 | spec->type = EFX_FILTER_MAC_WILD; | ||
262 | spec->data[0] = 0; | ||
263 | } else { | ||
264 | spec->type = EFX_FILTER_MAC_FULL; | ||
265 | spec->data[0] = vid; | ||
266 | } | ||
267 | |||
268 | spec->data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5]; | ||
269 | spec->data[2] = addr[0] << 8 | addr[1]; | ||
270 | return 0; | ||
271 | } | ||
272 | |||
273 | /* Build a filter entry and return its n-tuple key. */ | ||
274 | static u32 efx_filter_build(efx_oword_t *filter, struct efx_filter_spec *spec) | ||
275 | { | ||
276 | u32 data3; | ||
277 | |||
278 | switch (efx_filter_spec_table_id(spec)) { | ||
279 | case EFX_FILTER_TABLE_RX_IP: { | ||
280 | bool is_udp = (spec->type == EFX_FILTER_UDP_FULL || | ||
281 | spec->type == EFX_FILTER_UDP_WILD); | ||
282 | EFX_POPULATE_OWORD_7( | ||
283 | *filter, | ||
284 | FRF_BZ_RSS_EN, | ||
285 | !!(spec->flags & EFX_FILTER_FLAG_RX_RSS), | ||
286 | FRF_BZ_SCATTER_EN, | ||
287 | !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER), | ||
288 | FRF_BZ_TCP_UDP, is_udp, | ||
289 | FRF_BZ_RXQ_ID, spec->dmaq_id, | ||
290 | EFX_DWORD_2, spec->data[2], | ||
291 | EFX_DWORD_1, spec->data[1], | ||
292 | EFX_DWORD_0, spec->data[0]); | ||
293 | data3 = is_udp; | ||
294 | break; | ||
295 | } | ||
296 | |||
297 | case EFX_FILTER_TABLE_RX_MAC: { | ||
298 | bool is_wild = spec->type == EFX_FILTER_MAC_WILD; | ||
299 | EFX_POPULATE_OWORD_8( | ||
300 | *filter, | ||
301 | FRF_CZ_RMFT_RSS_EN, | ||
302 | !!(spec->flags & EFX_FILTER_FLAG_RX_RSS), | ||
303 | FRF_CZ_RMFT_SCATTER_EN, | ||
304 | !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER), | ||
305 | FRF_CZ_RMFT_IP_OVERRIDE, | ||
306 | !!(spec->flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP), | ||
307 | FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id, | ||
308 | FRF_CZ_RMFT_WILDCARD_MATCH, is_wild, | ||
309 | FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2], | ||
310 | FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1], | ||
311 | FRF_CZ_RMFT_VLAN_ID, spec->data[0]); | ||
312 | data3 = is_wild; | ||
313 | break; | ||
314 | } | ||
315 | |||
316 | default: | ||
317 | BUG(); | ||
318 | } | ||
319 | |||
320 | return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3; | ||
321 | } | ||
322 | |||
323 | static bool efx_filter_equal(const struct efx_filter_spec *left, | ||
324 | const struct efx_filter_spec *right) | ||
325 | { | ||
326 | if (left->type != right->type || | ||
327 | memcmp(left->data, right->data, sizeof(left->data))) | ||
328 | return false; | ||
329 | |||
330 | return true; | ||
331 | } | ||
332 | |||
333 | static int efx_filter_search(struct efx_filter_table *table, | ||
334 | struct efx_filter_spec *spec, u32 key, | ||
335 | bool for_insert, int *depth_required) | ||
336 | { | ||
337 | unsigned hash, incr, filter_idx, depth, depth_max; | ||
338 | |||
339 | hash = efx_filter_hash(key); | ||
340 | incr = efx_filter_increment(key); | ||
341 | |||
342 | filter_idx = hash & (table->size - 1); | ||
343 | depth = 1; | ||
344 | depth_max = (for_insert ? | ||
345 | (spec->priority <= EFX_FILTER_PRI_HINT ? | ||
346 | FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX) : | ||
347 | table->search_depth[spec->type]); | ||
348 | |||
349 | for (;;) { | ||
350 | /* Return success if entry is used and matches this spec | ||
351 | * or entry is unused and we are trying to insert. | ||
352 | */ | ||
353 | if (test_bit(filter_idx, table->used_bitmap) ? | ||
354 | efx_filter_equal(spec, &table->spec[filter_idx]) : | ||
355 | for_insert) { | ||
356 | *depth_required = depth; | ||
357 | return filter_idx; | ||
358 | } | ||
359 | |||
360 | /* Return failure if we reached the maximum search depth */ | ||
361 | if (depth == depth_max) | ||
362 | return for_insert ? -EBUSY : -ENOENT; | ||
363 | |||
364 | filter_idx = (filter_idx + incr) & (table->size - 1); | ||
365 | ++depth; | ||
366 | } | ||
367 | } | ||
368 | |||
369 | /* Construct/deconstruct external filter IDs */ | ||
370 | |||
371 | static inline int | ||
372 | efx_filter_make_id(enum efx_filter_table_id table_id, unsigned index) | ||
373 | { | ||
374 | return table_id << 16 | index; | ||
375 | } | ||
376 | |||
377 | /** | ||
378 | * efx_filter_insert_filter - add or replace a filter | ||
379 | * @efx: NIC in which to insert the filter | ||
380 | * @spec: Specification for the filter | ||
381 | * @replace: Flag for whether the specified filter may replace a filter | ||
382 | * with an identical match expression and equal or lower priority | ||
383 | * | ||
384 | * On success, return the filter ID. | ||
385 | * On failure, return a negative error code. | ||
386 | */ | ||
387 | int efx_filter_insert_filter(struct efx_nic *efx, struct efx_filter_spec *spec, | ||
388 | bool replace) | ||
389 | { | ||
390 | struct efx_filter_state *state = efx->filter_state; | ||
391 | struct efx_filter_table *table = efx_filter_spec_table(state, spec); | ||
392 | struct efx_filter_spec *saved_spec; | ||
393 | efx_oword_t filter; | ||
394 | int filter_idx, depth; | ||
395 | u32 key; | ||
396 | int rc; | ||
397 | |||
398 | if (!table || table->size == 0) | ||
399 | return -EINVAL; | ||
400 | |||
401 | key = efx_filter_build(&filter, spec); | ||
402 | |||
403 | netif_vdbg(efx, hw, efx->net_dev, | ||
404 | "%s: type %d search_depth=%d", __func__, spec->type, | ||
405 | table->search_depth[spec->type]); | ||
406 | |||
407 | spin_lock_bh(&state->lock); | ||
408 | |||
409 | rc = efx_filter_search(table, spec, key, true, &depth); | ||
410 | if (rc < 0) | ||
411 | goto out; | ||
412 | filter_idx = rc; | ||
413 | BUG_ON(filter_idx >= table->size); | ||
414 | saved_spec = &table->spec[filter_idx]; | ||
415 | |||
416 | if (test_bit(filter_idx, table->used_bitmap)) { | ||
417 | /* Should we replace the existing filter? */ | ||
418 | if (!replace) { | ||
419 | rc = -EEXIST; | ||
420 | goto out; | ||
421 | } | ||
422 | if (spec->priority < saved_spec->priority) { | ||
423 | rc = -EPERM; | ||
424 | goto out; | ||
425 | } | ||
426 | } else { | ||
427 | __set_bit(filter_idx, table->used_bitmap); | ||
428 | ++table->used; | ||
429 | } | ||
430 | *saved_spec = *spec; | ||
431 | |||
432 | if (table->search_depth[spec->type] < depth) { | ||
433 | table->search_depth[spec->type] = depth; | ||
434 | efx_filter_push_rx_limits(efx); | ||
435 | } | ||
436 | |||
437 | efx_writeo(efx, &filter, table->offset + table->step * filter_idx); | ||
438 | |||
439 | netif_vdbg(efx, hw, efx->net_dev, | ||
440 | "%s: filter type %d index %d rxq %u set", | ||
441 | __func__, spec->type, filter_idx, spec->dmaq_id); | ||
442 | rc = efx_filter_make_id(table->id, filter_idx); | ||
443 | |||
444 | out: | ||
445 | spin_unlock_bh(&state->lock); | ||
446 | return rc; | ||
447 | } | ||
448 | |||
449 | static void efx_filter_table_clear_entry(struct efx_nic *efx, | ||
450 | struct efx_filter_table *table, | ||
451 | int filter_idx) | ||
452 | { | ||
453 | static efx_oword_t filter; | ||
454 | |||
455 | if (test_bit(filter_idx, table->used_bitmap)) { | ||
456 | __clear_bit(filter_idx, table->used_bitmap); | ||
457 | --table->used; | ||
458 | memset(&table->spec[filter_idx], 0, sizeof(table->spec[0])); | ||
459 | |||
460 | efx_writeo(efx, &filter, | ||
461 | table->offset + table->step * filter_idx); | ||
462 | } | ||
463 | } | ||
464 | |||
465 | /** | ||
466 | * efx_filter_remove_filter - remove a filter by specification | ||
467 | * @efx: NIC from which to remove the filter | ||
468 | * @spec: Specification for the filter | ||
469 | * | ||
470 | * On success, return zero. | ||
471 | * On failure, return a negative error code. | ||
472 | */ | ||
473 | int efx_filter_remove_filter(struct efx_nic *efx, struct efx_filter_spec *spec) | ||
474 | { | ||
475 | struct efx_filter_state *state = efx->filter_state; | ||
476 | struct efx_filter_table *table = efx_filter_spec_table(state, spec); | ||
477 | struct efx_filter_spec *saved_spec; | ||
478 | efx_oword_t filter; | ||
479 | int filter_idx, depth; | ||
480 | u32 key; | ||
481 | int rc; | ||
482 | |||
483 | if (!table) | ||
484 | return -EINVAL; | ||
485 | |||
486 | key = efx_filter_build(&filter, spec); | ||
487 | |||
488 | spin_lock_bh(&state->lock); | ||
489 | |||
490 | rc = efx_filter_search(table, spec, key, false, &depth); | ||
491 | if (rc < 0) | ||
492 | goto out; | ||
493 | filter_idx = rc; | ||
494 | saved_spec = &table->spec[filter_idx]; | ||
495 | |||
496 | if (spec->priority < saved_spec->priority) { | ||
497 | rc = -EPERM; | ||
498 | goto out; | ||
499 | } | ||
500 | |||
501 | efx_filter_table_clear_entry(efx, table, filter_idx); | ||
502 | if (table->used == 0) | ||
503 | efx_filter_table_reset_search_depth(table); | ||
504 | rc = 0; | ||
505 | |||
506 | out: | ||
507 | spin_unlock_bh(&state->lock); | ||
508 | return rc; | ||
509 | } | ||
510 | |||
511 | static void efx_filter_table_clear(struct efx_nic *efx, | ||
512 | enum efx_filter_table_id table_id, | ||
513 | enum efx_filter_priority priority) | ||
514 | { | ||
515 | struct efx_filter_state *state = efx->filter_state; | ||
516 | struct efx_filter_table *table = &state->table[table_id]; | ||
517 | int filter_idx; | ||
518 | |||
519 | spin_lock_bh(&state->lock); | ||
520 | |||
521 | for (filter_idx = 0; filter_idx < table->size; ++filter_idx) | ||
522 | if (table->spec[filter_idx].priority <= priority) | ||
523 | efx_filter_table_clear_entry(efx, table, filter_idx); | ||
524 | if (table->used == 0) | ||
525 | efx_filter_table_reset_search_depth(table); | ||
526 | |||
527 | spin_unlock_bh(&state->lock); | ||
528 | } | ||
529 | |||
530 | /** | ||
531 | * efx_filter_clear_rx - remove RX filters by priority | ||
532 | * @efx: NIC from which to remove the filters | ||
533 | * @priority: Maximum priority to remove | ||
534 | */ | ||
535 | void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority) | ||
536 | { | ||
537 | efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority); | ||
538 | efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority); | ||
539 | } | ||
540 | |||
541 | /* Restore filter stater after reset */ | ||
542 | void efx_restore_filters(struct efx_nic *efx) | ||
543 | { | ||
544 | struct efx_filter_state *state = efx->filter_state; | ||
545 | enum efx_filter_table_id table_id; | ||
546 | struct efx_filter_table *table; | ||
547 | efx_oword_t filter; | ||
548 | int filter_idx; | ||
549 | |||
550 | spin_lock_bh(&state->lock); | ||
551 | |||
552 | for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) { | ||
553 | table = &state->table[table_id]; | ||
554 | for (filter_idx = 0; filter_idx < table->size; filter_idx++) { | ||
555 | if (!test_bit(filter_idx, table->used_bitmap)) | ||
556 | continue; | ||
557 | efx_filter_build(&filter, &table->spec[filter_idx]); | ||
558 | efx_writeo(efx, &filter, | ||
559 | table->offset + table->step * filter_idx); | ||
560 | } | ||
561 | } | ||
562 | |||
563 | efx_filter_push_rx_limits(efx); | ||
564 | |||
565 | spin_unlock_bh(&state->lock); | ||
566 | } | ||
567 | |||
568 | int efx_probe_filters(struct efx_nic *efx) | ||
569 | { | ||
570 | struct efx_filter_state *state; | ||
571 | struct efx_filter_table *table; | ||
572 | unsigned table_id; | ||
573 | |||
574 | state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL); | ||
575 | if (!state) | ||
576 | return -ENOMEM; | ||
577 | efx->filter_state = state; | ||
578 | |||
579 | spin_lock_init(&state->lock); | ||
580 | |||
581 | if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { | ||
582 | #ifdef CONFIG_RFS_ACCEL | ||
583 | state->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS, | ||
584 | sizeof(*state->rps_flow_id), | ||
585 | GFP_KERNEL); | ||
586 | if (!state->rps_flow_id) | ||
587 | goto fail; | ||
588 | #endif | ||
589 | table = &state->table[EFX_FILTER_TABLE_RX_IP]; | ||
590 | table->id = EFX_FILTER_TABLE_RX_IP; | ||
591 | table->offset = FR_BZ_RX_FILTER_TBL0; | ||
592 | table->size = FR_BZ_RX_FILTER_TBL0_ROWS; | ||
593 | table->step = FR_BZ_RX_FILTER_TBL0_STEP; | ||
594 | } | ||
595 | |||
596 | if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) { | ||
597 | table = &state->table[EFX_FILTER_TABLE_RX_MAC]; | ||
598 | table->id = EFX_FILTER_TABLE_RX_MAC; | ||
599 | table->offset = FR_CZ_RX_MAC_FILTER_TBL0; | ||
600 | table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS; | ||
601 | table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP; | ||
602 | } | ||
603 | |||
604 | for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) { | ||
605 | table = &state->table[table_id]; | ||
606 | if (table->size == 0) | ||
607 | continue; | ||
608 | table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size), | ||
609 | sizeof(unsigned long), | ||
610 | GFP_KERNEL); | ||
611 | if (!table->used_bitmap) | ||
612 | goto fail; | ||
613 | table->spec = vzalloc(table->size * sizeof(*table->spec)); | ||
614 | if (!table->spec) | ||
615 | goto fail; | ||
616 | } | ||
617 | |||
618 | return 0; | ||
619 | |||
620 | fail: | ||
621 | efx_remove_filters(efx); | ||
622 | return -ENOMEM; | ||
623 | } | ||
624 | |||
625 | void efx_remove_filters(struct efx_nic *efx) | ||
626 | { | ||
627 | struct efx_filter_state *state = efx->filter_state; | ||
628 | enum efx_filter_table_id table_id; | ||
629 | |||
630 | for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) { | ||
631 | kfree(state->table[table_id].used_bitmap); | ||
632 | vfree(state->table[table_id].spec); | ||
633 | } | ||
634 | #ifdef CONFIG_RFS_ACCEL | ||
635 | kfree(state->rps_flow_id); | ||
636 | #endif | ||
637 | kfree(state); | ||
638 | } | ||
639 | |||
640 | #ifdef CONFIG_RFS_ACCEL | ||
641 | |||
642 | int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, | ||
643 | u16 rxq_index, u32 flow_id) | ||
644 | { | ||
645 | struct efx_nic *efx = netdev_priv(net_dev); | ||
646 | struct efx_channel *channel; | ||
647 | struct efx_filter_state *state = efx->filter_state; | ||
648 | struct efx_filter_spec spec; | ||
649 | const struct iphdr *ip; | ||
650 | const __be16 *ports; | ||
651 | int nhoff; | ||
652 | int rc; | ||
653 | |||
654 | nhoff = skb_network_offset(skb); | ||
655 | |||
656 | if (skb->protocol != htons(ETH_P_IP)) | ||
657 | return -EPROTONOSUPPORT; | ||
658 | |||
659 | /* RFS must validate the IP header length before calling us */ | ||
660 | EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip)); | ||
661 | ip = (const struct iphdr *)(skb->data + nhoff); | ||
662 | if (ip_is_fragment(ip)) | ||
663 | return -EPROTONOSUPPORT; | ||
664 | EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + 4 * ip->ihl + 4); | ||
665 | ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl); | ||
666 | |||
667 | efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, 0, rxq_index); | ||
668 | rc = efx_filter_set_ipv4_full(&spec, ip->protocol, | ||
669 | ip->daddr, ports[1], ip->saddr, ports[0]); | ||
670 | if (rc) | ||
671 | return rc; | ||
672 | |||
673 | rc = efx_filter_insert_filter(efx, &spec, true); | ||
674 | if (rc < 0) | ||
675 | return rc; | ||
676 | |||
677 | /* Remember this so we can check whether to expire the filter later */ | ||
678 | state->rps_flow_id[rc] = flow_id; | ||
679 | channel = efx_get_channel(efx, skb_get_rx_queue(skb)); | ||
680 | ++channel->rfs_filters_added; | ||
681 | |||
682 | netif_info(efx, rx_status, efx->net_dev, | ||
683 | "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n", | ||
684 | (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP", | ||
685 | &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]), | ||
686 | rxq_index, flow_id, rc); | ||
687 | |||
688 | return rc; | ||
689 | } | ||
690 | |||
691 | bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota) | ||
692 | { | ||
693 | struct efx_filter_state *state = efx->filter_state; | ||
694 | struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_IP]; | ||
695 | unsigned mask = table->size - 1; | ||
696 | unsigned index; | ||
697 | unsigned stop; | ||
698 | |||
699 | if (!spin_trylock_bh(&state->lock)) | ||
700 | return false; | ||
701 | |||
702 | index = state->rps_expire_index; | ||
703 | stop = (index + quota) & mask; | ||
704 | |||
705 | while (index != stop) { | ||
706 | if (test_bit(index, table->used_bitmap) && | ||
707 | table->spec[index].priority == EFX_FILTER_PRI_HINT && | ||
708 | rps_may_expire_flow(efx->net_dev, | ||
709 | table->spec[index].dmaq_id, | ||
710 | state->rps_flow_id[index], index)) { | ||
711 | netif_info(efx, rx_status, efx->net_dev, | ||
712 | "expiring filter %d [flow %u]\n", | ||
713 | index, state->rps_flow_id[index]); | ||
714 | efx_filter_table_clear_entry(efx, table, index); | ||
715 | } | ||
716 | index = (index + 1) & mask; | ||
717 | } | ||
718 | |||
719 | state->rps_expire_index = stop; | ||
720 | if (table->used == 0) | ||
721 | efx_filter_table_reset_search_depth(table); | ||
722 | |||
723 | spin_unlock_bh(&state->lock); | ||
724 | return true; | ||
725 | } | ||
726 | |||
727 | #endif /* CONFIG_RFS_ACCEL */ | ||