diff options
| -rw-r--r-- | kernel/bpf/hashtab.c | 23 | ||||
| -rw-r--r-- | kernel/bpf/sockmap.c | 11 | ||||
| -rw-r--r-- | net/tls/tls_main.c | 9 | ||||
| -rw-r--r-- | net/xdp/xdp_umem.c | 4 | ||||
| -rw-r--r-- | tools/bpf/bpftool/map_perf_ring.c | 5 |
5 files changed, 35 insertions, 17 deletions
diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 04b8eda94e7d..03cc59ee9c95 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c | |||
| @@ -15,6 +15,7 @@ | |||
| 15 | #include <linux/jhash.h> | 15 | #include <linux/jhash.h> |
| 16 | #include <linux/filter.h> | 16 | #include <linux/filter.h> |
| 17 | #include <linux/rculist_nulls.h> | 17 | #include <linux/rculist_nulls.h> |
| 18 | #include <linux/random.h> | ||
| 18 | #include <uapi/linux/btf.h> | 19 | #include <uapi/linux/btf.h> |
| 19 | #include "percpu_freelist.h" | 20 | #include "percpu_freelist.h" |
| 20 | #include "bpf_lru_list.h" | 21 | #include "bpf_lru_list.h" |
| @@ -41,6 +42,7 @@ struct bpf_htab { | |||
| 41 | atomic_t count; /* number of elements in this hashtable */ | 42 | atomic_t count; /* number of elements in this hashtable */ |
| 42 | u32 n_buckets; /* number of hash buckets */ | 43 | u32 n_buckets; /* number of hash buckets */ |
| 43 | u32 elem_size; /* size of each element in bytes */ | 44 | u32 elem_size; /* size of each element in bytes */ |
| 45 | u32 hashrnd; | ||
| 44 | }; | 46 | }; |
| 45 | 47 | ||
| 46 | /* each htab element is struct htab_elem + key + value */ | 48 | /* each htab element is struct htab_elem + key + value */ |
| @@ -371,6 +373,7 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) | |||
| 371 | if (!htab->buckets) | 373 | if (!htab->buckets) |
| 372 | goto free_htab; | 374 | goto free_htab; |
| 373 | 375 | ||
| 376 | htab->hashrnd = get_random_int(); | ||
| 374 | for (i = 0; i < htab->n_buckets; i++) { | 377 | for (i = 0; i < htab->n_buckets; i++) { |
| 375 | INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i); | 378 | INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i); |
| 376 | raw_spin_lock_init(&htab->buckets[i].lock); | 379 | raw_spin_lock_init(&htab->buckets[i].lock); |
| @@ -402,9 +405,9 @@ free_htab: | |||
| 402 | return ERR_PTR(err); | 405 | return ERR_PTR(err); |
| 403 | } | 406 | } |
| 404 | 407 | ||
| 405 | static inline u32 htab_map_hash(const void *key, u32 key_len) | 408 | static inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd) |
| 406 | { | 409 | { |
| 407 | return jhash(key, key_len, 0); | 410 | return jhash(key, key_len, hashrnd); |
| 408 | } | 411 | } |
| 409 | 412 | ||
| 410 | static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash) | 413 | static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash) |
| @@ -470,7 +473,7 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) | |||
| 470 | 473 | ||
| 471 | key_size = map->key_size; | 474 | key_size = map->key_size; |
| 472 | 475 | ||
| 473 | hash = htab_map_hash(key, key_size); | 476 | hash = htab_map_hash(key, key_size, htab->hashrnd); |
| 474 | 477 | ||
| 475 | head = select_bucket(htab, hash); | 478 | head = select_bucket(htab, hash); |
| 476 | 479 | ||
| @@ -597,7 +600,7 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) | |||
| 597 | if (!key) | 600 | if (!key) |
| 598 | goto find_first_elem; | 601 | goto find_first_elem; |
| 599 | 602 | ||
| 600 | hash = htab_map_hash(key, key_size); | 603 | hash = htab_map_hash(key, key_size, htab->hashrnd); |
| 601 | 604 | ||
| 602 | head = select_bucket(htab, hash); | 605 | head = select_bucket(htab, hash); |
| 603 | 606 | ||
| @@ -824,7 +827,7 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value, | |||
| 824 | 827 | ||
| 825 | key_size = map->key_size; | 828 | key_size = map->key_size; |
| 826 | 829 | ||
| 827 | hash = htab_map_hash(key, key_size); | 830 | hash = htab_map_hash(key, key_size, htab->hashrnd); |
| 828 | 831 | ||
| 829 | b = __select_bucket(htab, hash); | 832 | b = __select_bucket(htab, hash); |
| 830 | head = &b->head; | 833 | head = &b->head; |
| @@ -880,7 +883,7 @@ static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value, | |||
| 880 | 883 | ||
| 881 | key_size = map->key_size; | 884 | key_size = map->key_size; |
| 882 | 885 | ||
| 883 | hash = htab_map_hash(key, key_size); | 886 | hash = htab_map_hash(key, key_size, htab->hashrnd); |
| 884 | 887 | ||
| 885 | b = __select_bucket(htab, hash); | 888 | b = __select_bucket(htab, hash); |
| 886 | head = &b->head; | 889 | head = &b->head; |
| @@ -945,7 +948,7 @@ static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key, | |||
| 945 | 948 | ||
| 946 | key_size = map->key_size; | 949 | key_size = map->key_size; |
| 947 | 950 | ||
| 948 | hash = htab_map_hash(key, key_size); | 951 | hash = htab_map_hash(key, key_size, htab->hashrnd); |
| 949 | 952 | ||
| 950 | b = __select_bucket(htab, hash); | 953 | b = __select_bucket(htab, hash); |
| 951 | head = &b->head; | 954 | head = &b->head; |
| @@ -998,7 +1001,7 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, | |||
| 998 | 1001 | ||
| 999 | key_size = map->key_size; | 1002 | key_size = map->key_size; |
| 1000 | 1003 | ||
| 1001 | hash = htab_map_hash(key, key_size); | 1004 | hash = htab_map_hash(key, key_size, htab->hashrnd); |
| 1002 | 1005 | ||
| 1003 | b = __select_bucket(htab, hash); | 1006 | b = __select_bucket(htab, hash); |
| 1004 | head = &b->head; | 1007 | head = &b->head; |
| @@ -1071,7 +1074,7 @@ static int htab_map_delete_elem(struct bpf_map *map, void *key) | |||
| 1071 | 1074 | ||
| 1072 | key_size = map->key_size; | 1075 | key_size = map->key_size; |
| 1073 | 1076 | ||
| 1074 | hash = htab_map_hash(key, key_size); | 1077 | hash = htab_map_hash(key, key_size, htab->hashrnd); |
| 1075 | b = __select_bucket(htab, hash); | 1078 | b = __select_bucket(htab, hash); |
| 1076 | head = &b->head; | 1079 | head = &b->head; |
| 1077 | 1080 | ||
| @@ -1103,7 +1106,7 @@ static int htab_lru_map_delete_elem(struct bpf_map *map, void *key) | |||
| 1103 | 1106 | ||
| 1104 | key_size = map->key_size; | 1107 | key_size = map->key_size; |
| 1105 | 1108 | ||
| 1106 | hash = htab_map_hash(key, key_size); | 1109 | hash = htab_map_hash(key, key_size, htab->hashrnd); |
| 1107 | b = __select_bucket(htab, hash); | 1110 | b = __select_bucket(htab, hash); |
| 1108 | head = &b->head; | 1111 | head = &b->head; |
| 1109 | 1112 | ||
diff --git a/kernel/bpf/sockmap.c b/kernel/bpf/sockmap.c index 98e621a29e8e..cf5195c7c331 100644 --- a/kernel/bpf/sockmap.c +++ b/kernel/bpf/sockmap.c | |||
| @@ -1427,12 +1427,15 @@ out: | |||
| 1427 | static void smap_write_space(struct sock *sk) | 1427 | static void smap_write_space(struct sock *sk) |
| 1428 | { | 1428 | { |
| 1429 | struct smap_psock *psock; | 1429 | struct smap_psock *psock; |
| 1430 | void (*write_space)(struct sock *sk); | ||
| 1430 | 1431 | ||
| 1431 | rcu_read_lock(); | 1432 | rcu_read_lock(); |
| 1432 | psock = smap_psock_sk(sk); | 1433 | psock = smap_psock_sk(sk); |
| 1433 | if (likely(psock && test_bit(SMAP_TX_RUNNING, &psock->state))) | 1434 | if (likely(psock && test_bit(SMAP_TX_RUNNING, &psock->state))) |
| 1434 | schedule_work(&psock->tx_work); | 1435 | schedule_work(&psock->tx_work); |
| 1436 | write_space = psock->save_write_space; | ||
| 1435 | rcu_read_unlock(); | 1437 | rcu_read_unlock(); |
| 1438 | write_space(sk); | ||
| 1436 | } | 1439 | } |
| 1437 | 1440 | ||
| 1438 | static void smap_stop_sock(struct smap_psock *psock, struct sock *sk) | 1441 | static void smap_stop_sock(struct smap_psock *psock, struct sock *sk) |
| @@ -2140,7 +2143,9 @@ static struct bpf_map *sock_hash_alloc(union bpf_attr *attr) | |||
| 2140 | return ERR_PTR(-EPERM); | 2143 | return ERR_PTR(-EPERM); |
| 2141 | 2144 | ||
| 2142 | /* check sanity of attributes */ | 2145 | /* check sanity of attributes */ |
| 2143 | if (attr->max_entries == 0 || attr->value_size != 4 || | 2146 | if (attr->max_entries == 0 || |
| 2147 | attr->key_size == 0 || | ||
| 2148 | attr->value_size != 4 || | ||
| 2144 | attr->map_flags & ~SOCK_CREATE_FLAG_MASK) | 2149 | attr->map_flags & ~SOCK_CREATE_FLAG_MASK) |
| 2145 | return ERR_PTR(-EINVAL); | 2150 | return ERR_PTR(-EINVAL); |
| 2146 | 2151 | ||
| @@ -2267,8 +2272,10 @@ static struct htab_elem *alloc_sock_hash_elem(struct bpf_htab *htab, | |||
| 2267 | } | 2272 | } |
| 2268 | l_new = kmalloc_node(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN, | 2273 | l_new = kmalloc_node(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN, |
| 2269 | htab->map.numa_node); | 2274 | htab->map.numa_node); |
| 2270 | if (!l_new) | 2275 | if (!l_new) { |
| 2276 | atomic_dec(&htab->count); | ||
| 2271 | return ERR_PTR(-ENOMEM); | 2277 | return ERR_PTR(-ENOMEM); |
| 2278 | } | ||
| 2272 | 2279 | ||
| 2273 | memcpy(l_new->key, key, key_size); | 2280 | memcpy(l_new->key, key, key_size); |
| 2274 | l_new->sk = sk; | 2281 | l_new->sk = sk; |
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c index 93c0c225ab34..180b6640e531 100644 --- a/net/tls/tls_main.c +++ b/net/tls/tls_main.c | |||
| @@ -213,9 +213,14 @@ static void tls_write_space(struct sock *sk) | |||
| 213 | { | 213 | { |
| 214 | struct tls_context *ctx = tls_get_ctx(sk); | 214 | struct tls_context *ctx = tls_get_ctx(sk); |
| 215 | 215 | ||
| 216 | /* We are already sending pages, ignore notification */ | 216 | /* If in_tcp_sendpages call lower protocol write space handler |
| 217 | if (ctx->in_tcp_sendpages) | 217 | * to ensure we wake up any waiting operations there. For example |
| 218 | * if do_tcp_sendpages where to call sk_wait_event. | ||
| 219 | */ | ||
| 220 | if (ctx->in_tcp_sendpages) { | ||
| 221 | ctx->sk_write_space(sk); | ||
| 218 | return; | 222 | return; |
| 223 | } | ||
| 219 | 224 | ||
| 220 | if (!sk->sk_write_pending && tls_is_pending_closed_record(ctx)) { | 225 | if (!sk->sk_write_pending && tls_is_pending_closed_record(ctx)) { |
| 221 | gfp_t sk_allocation = sk->sk_allocation; | 226 | gfp_t sk_allocation = sk->sk_allocation; |
diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c index 911ca6d3cb5a..bfe2dbea480b 100644 --- a/net/xdp/xdp_umem.c +++ b/net/xdp/xdp_umem.c | |||
| @@ -74,14 +74,14 @@ int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev, | |||
| 74 | return 0; | ||
