aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--kernel/bpf/hashtab.c23
-rw-r--r--kernel/bpf/sockmap.c11
-rw-r--r--net/tls/tls_main.c9
-rw-r--r--net/xdp/xdp_umem.c4
-rw-r--r--tools/bpf/bpftool/map_perf_ring.c5
5 files changed, 35 insertions, 17 deletions
diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index 04b8eda94e7d..03cc59ee9c95 100644
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -15,6 +15,7 @@
15#include <linux/jhash.h> 15#include <linux/jhash.h>
16#include <linux/filter.h> 16#include <linux/filter.h>
17#include <linux/rculist_nulls.h> 17#include <linux/rculist_nulls.h>
18#include <linux/random.h>
18#include <uapi/linux/btf.h> 19#include <uapi/linux/btf.h>
19#include "percpu_freelist.h" 20#include "percpu_freelist.h"
20#include "bpf_lru_list.h" 21#include "bpf_lru_list.h"
@@ -41,6 +42,7 @@ struct bpf_htab {
41 atomic_t count; /* number of elements in this hashtable */ 42 atomic_t count; /* number of elements in this hashtable */
42 u32 n_buckets; /* number of hash buckets */ 43 u32 n_buckets; /* number of hash buckets */
43 u32 elem_size; /* size of each element in bytes */ 44 u32 elem_size; /* size of each element in bytes */
45 u32 hashrnd;
44}; 46};
45 47
46/* each htab element is struct htab_elem + key + value */ 48/* each htab element is struct htab_elem + key + value */
@@ -371,6 +373,7 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
371 if (!htab->buckets) 373 if (!htab->buckets)
372 goto free_htab; 374 goto free_htab;
373 375
376 htab->hashrnd = get_random_int();
374 for (i = 0; i < htab->n_buckets; i++) { 377 for (i = 0; i < htab->n_buckets; i++) {
375 INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i); 378 INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i);
376 raw_spin_lock_init(&htab->buckets[i].lock); 379 raw_spin_lock_init(&htab->buckets[i].lock);
@@ -402,9 +405,9 @@ free_htab:
402 return ERR_PTR(err); 405 return ERR_PTR(err);
403} 406}
404 407
405static inline u32 htab_map_hash(const void *key, u32 key_len) 408static inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd)
406{ 409{
407 return jhash(key, key_len, 0); 410 return jhash(key, key_len, hashrnd);
408} 411}
409 412
410static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash) 413static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash)
@@ -470,7 +473,7 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)
470 473
471 key_size = map->key_size; 474 key_size = map->key_size;
472 475
473 hash = htab_map_hash(key, key_size); 476 hash = htab_map_hash(key, key_size, htab->hashrnd);
474 477
475 head = select_bucket(htab, hash); 478 head = select_bucket(htab, hash);
476 479
@@ -597,7 +600,7 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
597 if (!key) 600 if (!key)
598 goto find_first_elem; 601 goto find_first_elem;
599 602
600 hash = htab_map_hash(key, key_size); 603 hash = htab_map_hash(key, key_size, htab->hashrnd);
601 604
602 head = select_bucket(htab, hash); 605 head = select_bucket(htab, hash);
603 606
@@ -824,7 +827,7 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value,
824 827
825 key_size = map->key_size; 828 key_size = map->key_size;
826 829
827 hash = htab_map_hash(key, key_size); 830 hash = htab_map_hash(key, key_size, htab->hashrnd);
828 831
829 b = __select_bucket(htab, hash); 832 b = __select_bucket(htab, hash);
830 head = &b->head; 833 head = &b->head;
@@ -880,7 +883,7 @@ static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value,
880 883
881 key_size = map->key_size; 884 key_size = map->key_size;
882 885
883 hash = htab_map_hash(key, key_size); 886 hash = htab_map_hash(key, key_size, htab->hashrnd);
884 887
885 b = __select_bucket(htab, hash); 888 b = __select_bucket(htab, hash);
886 head = &b->head; 889 head = &b->head;
@@ -945,7 +948,7 @@ static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key,
945 948
946 key_size = map->key_size; 949 key_size = map->key_size;
947 950
948 hash = htab_map_hash(key, key_size); 951 hash = htab_map_hash(key, key_size, htab->hashrnd);
949 952
950 b = __select_bucket(htab, hash); 953 b = __select_bucket(htab, hash);
951 head = &b->head; 954 head = &b->head;
@@ -998,7 +1001,7 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
998 1001
999 key_size = map->key_size; 1002 key_size = map->key_size;
1000 1003
1001 hash = htab_map_hash(key, key_size); 1004 hash = htab_map_hash(key, key_size, htab->hashrnd);
1002 1005
1003 b = __select_bucket(htab, hash); 1006 b = __select_bucket(htab, hash);
1004 head = &b->head; 1007 head = &b->head;
@@ -1071,7 +1074,7 @@ static int htab_map_delete_elem(struct bpf_map *map, void *key)
1071 1074
1072 key_size = map->key_size; 1075 key_size = map->key_size;
1073 1076
1074 hash = htab_map_hash(key, key_size); 1077 hash = htab_map_hash(key, key_size, htab->hashrnd);
1075 b = __select_bucket(htab, hash); 1078 b = __select_bucket(htab, hash);
1076 head = &b->head; 1079 head = &b->head;
1077 1080
@@ -1103,7 +1106,7 @@ static int htab_lru_map_delete_elem(struct bpf_map *map, void *key)
1103 1106
1104 key_size = map->key_size; 1107 key_size = map->key_size;
1105 1108
1106 hash = htab_map_hash(key, key_size); 1109 hash = htab_map_hash(key, key_size, htab->hashrnd);
1107 b = __select_bucket(htab, hash); 1110 b = __select_bucket(htab, hash);
1108 head = &b->head; 1111 head = &b->head;
1109 1112
diff --git a/kernel/bpf/sockmap.c b/kernel/bpf/sockmap.c
index 98e621a29e8e..cf5195c7c331 100644
--- a/kernel/bpf/sockmap.c
+++ b/kernel/bpf/sockmap.c
@@ -1427,12 +1427,15 @@ out:
1427static void smap_write_space(struct sock *sk) 1427static void smap_write_space(struct sock *sk)
1428{ 1428{
1429 struct smap_psock *psock; 1429 struct smap_psock *psock;
1430 void (*write_space)(struct sock *sk);
1430 1431
1431 rcu_read_lock(); 1432 rcu_read_lock();
1432 psock = smap_psock_sk(sk); 1433 psock = smap_psock_sk(sk);
1433 if (likely(psock && test_bit(SMAP_TX_RUNNING, &psock->state))) 1434 if (likely(psock && test_bit(SMAP_TX_RUNNING, &psock->state)))
1434 schedule_work(&psock->tx_work); 1435 schedule_work(&psock->tx_work);
1436 write_space = psock->save_write_space;
1435 rcu_read_unlock(); 1437 rcu_read_unlock();
1438 write_space(sk);
1436} 1439}
1437 1440
1438static void smap_stop_sock(struct smap_psock *psock, struct sock *sk) 1441static void smap_stop_sock(struct smap_psock *psock, struct sock *sk)
@@ -2140,7 +2143,9 @@ static struct bpf_map *sock_hash_alloc(union bpf_attr *attr)
2140 return ERR_PTR(-EPERM); 2143 return ERR_PTR(-EPERM);
2141 2144
2142 /* check sanity of attributes */ 2145 /* check sanity of attributes */
2143 if (attr->max_entries == 0 || attr->value_size != 4 || 2146 if (attr->max_entries == 0 ||
2147 attr->key_size == 0 ||
2148 attr->value_size != 4 ||
2144 attr->map_flags & ~SOCK_CREATE_FLAG_MASK) 2149 attr->map_flags & ~SOCK_CREATE_FLAG_MASK)
2145 return ERR_PTR(-EINVAL); 2150 return ERR_PTR(-EINVAL);
2146 2151
@@ -2267,8 +2272,10 @@ static struct htab_elem *alloc_sock_hash_elem(struct bpf_htab *htab,
2267 } 2272 }
2268 l_new = kmalloc_node(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN, 2273 l_new = kmalloc_node(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN,
2269 htab->map.numa_node); 2274 htab->map.numa_node);
2270 if (!l_new) 2275 if (!l_new) {
2276 atomic_dec(&htab->count);
2271 return ERR_PTR(-ENOMEM); 2277 return ERR_PTR(-ENOMEM);
2278 }
2272 2279
2273 memcpy(l_new->key, key, key_size); 2280 memcpy(l_new->key, key, key_size);
2274 l_new->sk = sk; 2281 l_new->sk = sk;
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index 93c0c225ab34..180b6640e531 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -213,9 +213,14 @@ static void tls_write_space(struct sock *sk)
213{ 213{
214 struct tls_context *ctx = tls_get_ctx(sk); 214 struct tls_context *ctx = tls_get_ctx(sk);
215 215
216 /* We are already sending pages, ignore notification */ 216 /* If in_tcp_sendpages call lower protocol write space handler
217 if (ctx->in_tcp_sendpages) 217 * to ensure we wake up any waiting operations there. For example
218 * if do_tcp_sendpages where to call sk_wait_event.
219 */
220 if (ctx->in_tcp_sendpages) {
221 ctx->sk_write_space(sk);
218 return; 222 return;
223 }
219 224
220 if (!sk->sk_write_pending && tls_is_pending_closed_record(ctx)) { 225 if (!sk->sk_write_pending && tls_is_pending_closed_record(ctx)) {
221 gfp_t sk_allocation = sk->sk_allocation; 226 gfp_t sk_allocation = sk->sk_allocation;
diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c
index 911ca6d3cb5a..bfe2dbea480b 100644
--- a/net/xdp/xdp_umem.c
+++ b/net/xdp/xdp_umem.c
@@ -74,14 +74,14 @@ int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev,
74 return 0;