aboutsummaryrefslogtreecommitdiffstats
path: root/include/linux/skmsg.h
diff options
context:
space:
mode:
Diffstat (limited to 'include/linux/skmsg.h')
-rw-r--r--include/linux/skmsg.h434
1 files changed, 434 insertions, 0 deletions
diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h
new file mode 100644
index 000000000000..2a11e9d91dfa
--- /dev/null
+++ b/include/linux/skmsg.h
@@ -0,0 +1,434 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
3
4#ifndef _LINUX_SKMSG_H
5#define _LINUX_SKMSG_H
6
7#include <linux/bpf.h>
8#include <linux/filter.h>
9#include <linux/scatterlist.h>
10#include <linux/skbuff.h>
11
12#include <net/sock.h>
13#include <net/tcp.h>
14#include <net/strparser.h>
15
16#define MAX_MSG_FRAGS MAX_SKB_FRAGS
17
18enum __sk_action {
19 __SK_DROP = 0,
20 __SK_PASS,
21 __SK_REDIRECT,
22 __SK_NONE,
23};
24
25struct sk_msg_sg {
26 u32 start;
27 u32 curr;
28 u32 end;
29 u32 size;
30 u32 copybreak;
31 bool copy[MAX_MSG_FRAGS];
32 /* The extra element is used for chaining the front and sections when
33 * the list becomes partitioned (e.g. end < start). The crypto APIs
34 * require the chaining.
35 */
36 struct scatterlist data[MAX_MSG_FRAGS + 1];
37};
38
39struct sk_msg {
40 struct sk_msg_sg sg;
41 void *data;
42 void *data_end;
43 u32 apply_bytes;
44 u32 cork_bytes;
45 u32 flags;
46 struct sk_buff *skb;
47 struct sock *sk_redir;
48 struct sock *sk;
49 struct list_head list;
50};
51
52struct sk_psock_progs {
53 struct bpf_prog *msg_parser;
54 struct bpf_prog *skb_parser;
55 struct bpf_prog *skb_verdict;
56};
57
58enum sk_psock_state_bits {
59 SK_PSOCK_TX_ENABLED,
60};
61
62struct sk_psock_link {
63 struct list_head list;
64 struct bpf_map *map;
65 void *link_raw;
66};
67
68struct sk_psock_parser {
69 struct strparser strp;
70 bool enabled;
71 void (*saved_data_ready)(struct sock *sk);
72};
73
74struct sk_psock_work_state {
75 struct sk_buff *skb;
76 u32 len;
77 u32 off;
78};
79
80struct sk_psock {
81 struct sock *sk;
82 struct sock *sk_redir;
83 u32 apply_bytes;
84 u32 cork_bytes;
85 u32 eval;
86 struct sk_msg *cork;
87 struct sk_psock_progs progs;
88 struct sk_psock_parser parser;
89 struct sk_buff_head ingress_skb;
90 struct list_head ingress_msg;
91 unsigned long state;
92 struct list_head link;
93 spinlock_t link_lock;
94 refcount_t refcnt;
95 void (*saved_unhash)(struct sock *sk);
96 void (*saved_close)(struct sock *sk, long timeout);
97 void (*saved_write_space)(struct sock *sk);
98 struct proto *sk_proto;
99 struct sk_psock_work_state work_state;
100 struct work_struct work;
101 union {
102 struct rcu_head rcu;
103 struct work_struct gc;
104 };
105};
106
107int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
108 int elem_first_coalesce);
109int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src,
110 u32 off, u32 len);
111void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len);
112int sk_msg_free(struct sock *sk, struct sk_msg *msg);
113int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg);
114void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes);
115void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg,
116 u32 bytes);
117
118void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes);
119void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes);
120
121int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
122 struct sk_msg *msg, u32 bytes);
123int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from,
124 struct sk_msg *msg, u32 bytes);
125
126static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)
127{
128 WARN_ON(i == msg->sg.end && bytes);
129}
130
131static inline void sk_msg_apply_bytes(struct sk_psock *psock, u32 bytes)
132{
133 if (psock->apply_bytes) {
134 if (psock->apply_bytes < bytes)
135 psock->apply_bytes = 0;
136 else
137 psock->apply_bytes -= bytes;
138 }
139}
140
141#define sk_msg_iter_var_prev(var) \
142 do { \
143 if (var == 0) \
144 var = MAX_MSG_FRAGS - 1; \
145 else \
146 var--; \
147 } while (0)
148
149#define sk_msg_iter_var_next(var) \
150 do { \
151 var++; \
152 if (var == MAX_MSG_FRAGS) \
153 var = 0; \
154 } while (0)
155
156#define sk_msg_iter_prev(msg, which) \
157 sk_msg_iter_var_prev(msg->sg.which)
158
159#define sk_msg_iter_next(msg, which) \
160 sk_msg_iter_var_next(msg->sg.which)
161
162static inline void sk_msg_clear_meta(struct sk_msg *msg)
163{
164 memset(&msg->sg, 0, offsetofend(struct sk_msg_sg, copy));
165}
166
167static inline void sk_msg_init(struct sk_msg *msg)
168{
169 BUILD_BUG_ON(ARRAY_SIZE(msg->sg.data) - 1 != MAX_MSG_FRAGS);
170 memset(msg, 0, sizeof(*msg));
171 sg_init_marker(msg->sg.data, MAX_MSG_FRAGS);
172}
173
174static inline void sk_msg_xfer(struct sk_msg *dst, struct sk_msg *src,
175 int which, u32 size)
176{
177 dst->sg.data[which] = src->sg.data[which];
178 dst->sg.data[which].length = size;
179 dst->sg.size += size;
180 src->sg.data[which].length -= size;
181 src->sg.data[which].offset += size;
182}
183
184static inline void sk_msg_xfer_full(struct sk_msg *dst, struct sk_msg *src)
185{
186 memcpy(dst, src, sizeof(*src));
187 sk_msg_init(src);
188}
189
190static inline bool sk_msg_full(const struct sk_msg *msg)
191{
192 return (msg->sg.end == msg->sg.start) && msg->sg.size;
193}
194
195static inline u32 sk_msg_elem_used(const struct sk_msg *msg)
196{
197 if (sk_msg_full(msg))
198 return MAX_MSG_FRAGS;
199
200 return msg->sg.end >= msg->sg.start ?
201 msg->sg.end - msg->sg.start :
202 msg->sg.end + (MAX_MSG_FRAGS - msg->sg.start);
203}
204
205static inline struct scatterlist *sk_msg_elem(struct sk_msg *msg, int which)
206{
207 return &msg->sg.data[which];
208}
209
210static inline struct scatterlist sk_msg_elem_cpy(struct sk_msg *msg, int which)
211{
212 return msg->sg.data[which];
213}
214
215static inline struct page *sk_msg_page(struct sk_msg *msg, int which)
216{
217 return sg_page(sk_msg_elem(msg, which));
218}
219
220static inline bool sk_msg_to_ingress(const struct sk_msg *msg)
221{
222 return msg->flags & BPF_F_INGRESS;
223}
224
225static inline void sk_msg_compute_data_pointers(struct sk_msg *msg)
226{
227 struct scatterlist *sge = sk_msg_elem(msg, msg->sg.start);
228
229 if (msg->sg.copy[msg->sg.start]) {
230 msg->data = NULL;
231 msg->data_end = NULL;
232 } else {
233 msg->data = sg_virt(sge);
234 msg->data_end = msg->data + sge->length;
235 }
236}
237
238static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page,
239 u32 len, u32 offset)
240{
241 struct scatterlist *sge;
242
243 get_page(page);
244 sge = sk_msg_elem(msg, msg->sg.end);
245 sg_set_page(sge, page, len, offset);
246 sg_unmark_end(sge);
247
248 msg->sg.copy[msg->sg.end] = true;
249 msg->sg.size += len;
250 sk_msg_iter_next(msg, end);
251}
252
253static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state)
254{
255 do {
256 msg->sg.copy[i] = copy_state;
257 sk_msg_iter_var_next(i);
258 if (i == msg->sg.end)
259 break;
260 } while (1);
261}
262
263static inline void sk_msg_sg_copy_set(struct sk_msg *msg, u32 start)
264{
265 sk_msg_sg_copy(msg, start, true);
266}
267
268static inline void sk_msg_sg_copy_clear(struct sk_msg *msg, u32 start)
269{
270 sk_msg_sg_copy(msg, start, false);
271}
272
273static inline struct sk_psock *sk_psock(const struct sock *sk)
274{
275 return rcu_dereference_sk_user_data(sk);
276}
277
278static inline void sk_psock_queue_msg(struct sk_psock *psock,
279 struct sk_msg *msg)
280{
281 list_add_tail(&msg->list, &psock->ingress_msg);
282}
283
284static inline bool sk_psock_queue_empty(const struct sk_psock *psock)
285{
286 return psock ? list_empty(&psock->ingress_msg) : true;
287}
288
289static inline void sk_psock_report_error(struct sk_psock *psock, int err)
290{
291 struct sock *sk = psock->sk;
292
293 sk->sk_err = err;
294 sk->sk_error_report(sk);
295}
296
297struct sk_psock *sk_psock_init(struct sock *sk, int node);
298
299int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock);
300void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock);
301void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock);
302
303int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,
304 struct sk_msg *msg);
305
306static inline struct sk_psock_link *sk_psock_init_link(void)
307{
308 return kzalloc(sizeof(struct sk_psock_link),
309 GFP_ATOMIC | __GFP_NOWARN);
310}
311
312static inline void sk_psock_free_link(struct sk_psock_link *link)
313{
314 kfree(link);
315}
316
317struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock);
318#if defined(CONFIG_BPF_STREAM_PARSER)
319void sk_psock_unlink(struct sock *sk, struct sk_psock_link *link);
320#else
321static inline void sk_psock_unlink(struct sock *sk,
322 struct sk_psock_link *link)
323{
324}
325#endif
326
327void __sk_psock_purge_ingress_msg(struct sk_psock *psock);
328
329static inline void sk_psock_cork_free(struct sk_psock *psock)
330{
331 if (psock->cork) {
332 sk_msg_free(psock->sk, psock->cork);
333 kfree(psock->cork);
334 psock->cork = NULL;
335 }
336}
337
338static inline void sk_psock_update_proto(struct sock *sk,
339 struct sk_psock *psock,
340 struct proto *ops)
341{
342 psock->saved_unhash = sk->sk_prot->unhash;
343 psock->saved_close = sk->sk_prot->close;
344 psock->saved_write_space = sk->sk_write_space;
345
346 psock->sk_proto = sk->sk_prot;
347 sk->sk_prot = ops;
348}
349
350static inline void sk_psock_restore_proto(struct sock *sk,
351 struct sk_psock *psock)
352{
353 if (psock->sk_proto) {
354 sk->sk_prot = psock->sk_proto;
355 psock->sk_proto = NULL;
356 }
357}
358
359static inline void sk_psock_set_state(struct sk_psock *psock,
360 enum sk_psock_state_bits bit)
361{
362 set_bit(bit, &psock->state);
363}
364
365static inline void sk_psock_clear_state(struct sk_psock *psock,
366 enum sk_psock_state_bits bit)
367{
368 clear_bit(bit, &psock->state);
369}
370
371static inline bool sk_psock_test_state(const struct sk_psock *psock,
372 enum sk_psock_state_bits bit)
373{
374 return test_bit(bit, &psock->state);
375}
376
377static inline struct sk_psock *sk_psock_get_checked(struct sock *sk)
378{
379 struct sk_psock *psock;
380
381 rcu_read_lock();
382 psock = sk_psock(sk);
383 if (psock) {
384 if (sk->sk_prot->recvmsg != tcp_bpf_recvmsg) {
385 psock = ERR_PTR(-EBUSY);
386 goto out;
387 }
388
389 if (!refcount_inc_not_zero(&psock->refcnt))
390 psock = ERR_PTR(-EBUSY);
391 }
392out:
393 rcu_read_unlock();
394 return psock;
395}
396
397static inline struct sk_psock *sk_psock_get(struct sock *sk)
398{
399 struct sk_psock *psock;
400
401 rcu_read_lock();
402 psock = sk_psock(sk);
403 if (psock && !refcount_inc_not_zero(&psock->refcnt))
404 psock = NULL;
405 rcu_read_unlock();
406 return psock;
407}
408
409void sk_psock_stop(struct sock *sk, struct sk_psock *psock);
410void sk_psock_destroy(struct rcu_head *rcu);
411void sk_psock_drop(struct sock *sk, struct sk_psock *psock);
412
413static inline void sk_psock_put(struct sock *sk, struct sk_psock *psock)
414{
415 if (refcount_dec_and_test(&psock->refcnt))
416 sk_psock_drop(sk, psock);
417}
418
419static inline void psock_set_prog(struct bpf_prog **pprog,
420 struct bpf_prog *prog)
421{
422 prog = xchg(pprog, prog);
423 if (prog)
424 bpf_prog_put(prog);
425}
426
427static inline void psock_progs_drop(struct sk_psock_progs *progs)
428{
429 psock_set_prog(&progs->msg_parser, NULL);
430 psock_set_prog(&progs->skb_parser, NULL);
431 psock_set_prog(&progs->skb_verdict, NULL);
432}
433
434#endif /* _LINUX_SKMSG_H */
generated by cgit v1.2.2 (git 2.25.0) at 2026-04-27 14:15:38 -0400