aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/linux/bpf.h1
-rw-r--r--include/linux/bpf_types.h1
-rw-r--r--include/linux/skbuff.h7
-rw-r--r--include/net/net_namespace.h3
-rw-r--r--include/net/sch_generic.h12
-rw-r--r--include/uapi/linux/bpf.h26
-rw-r--r--kernel/bpf/syscall.c8
-rw-r--r--kernel/bpf/verifier.c32
-rw-r--r--net/core/filter.c70
-rw-r--r--net/core/flow_dissector.c134
10 files changed, 291 insertions, 3 deletions
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 523481a3471b..988a00797bcd 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -212,6 +212,7 @@ enum bpf_reg_type {
212 PTR_TO_PACKET_META, /* skb->data - meta_len */ 212 PTR_TO_PACKET_META, /* skb->data - meta_len */
213 PTR_TO_PACKET, /* reg points to skb->data */ 213 PTR_TO_PACKET, /* reg points to skb->data */
214 PTR_TO_PACKET_END, /* skb->data + headlen */ 214 PTR_TO_PACKET_END, /* skb->data + headlen */
215 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */
215}; 216};
216 217
217/* The information passed from prog-specific *_is_valid_access 218/* The information passed from prog-specific *_is_valid_access
diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h
index cd26c090e7c0..22083712dd18 100644
--- a/include/linux/bpf_types.h
+++ b/include/linux/bpf_types.h
@@ -32,6 +32,7 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_LIRC_MODE2, lirc_mode2)
32#ifdef CONFIG_INET 32#ifdef CONFIG_INET
33BPF_PROG_TYPE(BPF_PROG_TYPE_SK_REUSEPORT, sk_reuseport) 33BPF_PROG_TYPE(BPF_PROG_TYPE_SK_REUSEPORT, sk_reuseport)
34#endif 34#endif
35BPF_PROG_TYPE(BPF_PROG_TYPE_FLOW_DISSECTOR, flow_dissector)
35 36
36BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops) 37BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops)
37BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops) 38BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops)
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 17a13e4785fc..ce0e863f02a2 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -243,6 +243,8 @@ struct scatterlist;
243struct pipe_inode_info; 243struct pipe_inode_info;
244struct iov_iter; 244struct iov_iter;
245struct napi_struct; 245struct napi_struct;
246struct bpf_prog;
247union bpf_attr;
246 248
247#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 249#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
248struct nf_conntrack { 250struct nf_conntrack {
@@ -1192,6 +1194,11 @@ void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
1192 const struct flow_dissector_key *key, 1194 const struct flow_dissector_key *key,
1193 unsigned int key_count); 1195 unsigned int key_count);
1194 1196
1197int skb_flow_dissector_bpf_prog_attach(const union bpf_attr *attr,
1198 struct bpf_prog *prog);
1199
1200int skb_flow_dissector_bpf_prog_detach(const union bpf_attr *attr);
1201
1195bool __skb_flow_dissect(const struct sk_buff *skb, 1202bool __skb_flow_dissect(const struct sk_buff *skb,
1196 struct flow_dissector *flow_dissector, 1203 struct flow_dissector *flow_dissector,
1197 void *target_container, 1204 void *target_container,
diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h
index 9b5fdc50519a..99d4148e0f90 100644
--- a/include/net/net_namespace.h
+++ b/include/net/net_namespace.h
@@ -43,6 +43,7 @@ struct ctl_table_header;
43struct net_generic; 43struct net_generic;
44struct uevent_sock; 44struct uevent_sock;
45struct netns_ipvs; 45struct netns_ipvs;
46struct bpf_prog;
46 47
47 48
48#define NETDEV_HASHBITS 8 49#define NETDEV_HASHBITS 8
@@ -145,6 +146,8 @@ struct net {
145#endif 146#endif
146 struct net_generic __rcu *gen; 147 struct net_generic __rcu *gen;
147 148
149 struct bpf_prog __rcu *flow_dissector_prog;
150
148 /* Note : following structs are cache line aligned */ 151 /* Note : following structs are cache line aligned */
149#ifdef CONFIG_XFRM 152#ifdef CONFIG_XFRM
150 struct netns_xfrm xfrm; 153 struct netns_xfrm xfrm;
diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index a6d00093f35e..1b81ba85fd2d 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -19,6 +19,7 @@ struct Qdisc_ops;
19struct qdisc_walker; 19struct qdisc_walker;
20struct tcf_walker; 20struct tcf_walker;
21struct module; 21struct module;
22struct bpf_flow_keys;
22 23
23typedef int tc_setup_cb_t(enum tc_setup_type type, 24typedef int tc_setup_cb_t(enum tc_setup_type type,
24 void *type_data, void *cb_priv); 25 void *type_data, void *cb_priv);
@@ -307,9 +308,14 @@ struct tcf_proto {
307}; 308};
308 309
309struct qdisc_skb_cb { 310struct qdisc_skb_cb {
310 unsigned int pkt_len; 311 union {
311 u16 slave_dev_queue_mapping; 312 struct {
312 u16 tc_classid; 313 unsigned int pkt_len;
314 u16 slave_dev_queue_mapping;
315 u16 tc_classid;
316 };
317 struct bpf_flow_keys *flow_keys;
318 };
313#define QDISC_CB_PRIV_LEN 20 319#define QDISC_CB_PRIV_LEN 20
314 unsigned char data[QDISC_CB_PRIV_LEN]; 320 unsigned char data[QDISC_CB_PRIV_LEN];
315}; 321};
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 66917a4eba27..aa5ccd2385ed 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -152,6 +152,7 @@ enum bpf_prog_type {
152 BPF_PROG_TYPE_LWT_SEG6LOCAL, 152 BPF_PROG_TYPE_LWT_SEG6LOCAL,
153 BPF_PROG_TYPE_LIRC_MODE2, 153 BPF_PROG_TYPE_LIRC_MODE2,
154 BPF_PROG_TYPE_SK_REUSEPORT, 154 BPF_PROG_TYPE_SK_REUSEPORT,
155 BPF_PROG_TYPE_FLOW_DISSECTOR,
155}; 156};
156 157
157enum bpf_attach_type { 158enum bpf_attach_type {
@@ -172,6 +173,7 @@ enum bpf_attach_type {
172 BPF_CGROUP_UDP4_SENDMSG, 173 BPF_CGROUP_UDP4_SENDMSG,
173 BPF_CGROUP_UDP6_SENDMSG, 174 BPF_CGROUP_UDP6_SENDMSG,
174 BPF_LIRC_MODE2, 175 BPF_LIRC_MODE2,
176 BPF_FLOW_DISSECTOR,
175 __MAX_BPF_ATTACH_TYPE 177 __MAX_BPF_ATTACH_TYPE
176}; 178};
177 179
@@ -2333,6 +2335,7 @@ struct __sk_buff {
2333 /* ... here. */ 2335 /* ... here. */
2334 2336
2335 __u32 data_meta; 2337 __u32 data_meta;
2338 struct bpf_flow_keys *flow_keys;
2336}; 2339};
2337 2340
2338struct bpf_tunnel_key { 2341struct bpf_tunnel_key {
@@ -2778,4 +2781,27 @@ enum bpf_task_fd_type {
2778 BPF_FD_TYPE_URETPROBE, /* filename + offset */ 2781 BPF_FD_TYPE_URETPROBE, /* filename + offset */
2779}; 2782};
2780 2783
2784struct bpf_flow_keys {
2785 __u16 nhoff;
2786 __u16 thoff;
2787 __u16 addr_proto; /* ETH_P_* of valid addrs */
2788 __u8 is_frag;
2789 __u8 is_first_frag;
2790 __u8 is_encap;
2791 __u8 ip_proto;
2792 __be16 n_proto;
2793 __be16 sport;
2794 __be16 dport;
2795 union {
2796 struct {
2797 __be32 ipv4_src;
2798 __be32 ipv4_dst;
2799 };
2800 struct {
2801 __u32 ipv6_src[4]; /* in6_addr; network order */
2802 __u32 ipv6_dst[4]; /* in6_addr; network order */
2803 };
2804 };
2805};
2806
2781#endif /* _UAPI__LINUX_BPF_H__ */ 2807#endif /* _UAPI__LINUX_BPF_H__ */
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 3c9636f03bb2..b3c2d09bcf7a 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1615,6 +1615,9 @@ static int bpf_prog_attach(const union bpf_attr *attr)
1615 case BPF_LIRC_MODE2: 1615 case BPF_LIRC_MODE2:
1616 ptype = BPF_PROG_TYPE_LIRC_MODE2; 1616 ptype = BPF_PROG_TYPE_LIRC_MODE2;
1617 break; 1617 break;
1618 case BPF_FLOW_DISSECTOR:
1619 ptype = BPF_PROG_TYPE_FLOW_DISSECTOR;
1620 break;
1618 default: 1621 default:
1619 return -EINVAL; 1622 return -EINVAL;
1620 } 1623 }
@@ -1636,6 +1639,9 @@ static int bpf_prog_attach(const union bpf_attr *attr)
1636 case BPF_PROG_TYPE_LIRC_MODE2: 1639 case BPF_PROG_TYPE_LIRC_MODE2:
1637 ret = lirc_prog_attach(attr, prog); 1640 ret = lirc_prog_attach(attr, prog);
1638 break; 1641 break;
1642 case BPF_PROG_TYPE_FLOW_DISSECTOR:
1643 ret = skb_flow_dissector_bpf_prog_attach(attr, prog);
1644 break;
1639 default: 1645 default:
1640 ret = cgroup_bpf_prog_attach(attr, ptype, prog); 1646 ret = cgroup_bpf_prog_attach(attr, ptype, prog);
1641 } 1647 }
@@ -1688,6 +1694,8 @@ static int bpf_prog_detach(const union bpf_attr *attr)
1688 return sockmap_get_from_fd(attr, BPF_PROG_TYPE_SK_SKB, NULL); 1694 return sockmap_get_from_fd(attr, BPF_PROG_TYPE_SK_SKB, NULL);
1689 case BPF_LIRC_MODE2: 1695 case BPF_LIRC_MODE2:
1690 return lirc_prog_detach(attr); 1696 return lirc_prog_detach(attr);
1697 case BPF_FLOW_DISSECTOR:
1698 return skb_flow_dissector_bpf_prog_detach(attr);
1691 default: 1699 default:
1692 return -EINVAL; 1700 return -EINVAL;
1693 } 1701 }
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 6ff1bac1795d..8ccbff4fff93 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -261,6 +261,7 @@ static const char * const reg_type_str[] = {
261 [PTR_TO_PACKET] = "pkt", 261 [PTR_TO_PACKET] = "pkt",
262 [PTR_TO_PACKET_META] = "pkt_meta", 262 [PTR_TO_PACKET_META] = "pkt_meta",
263 [PTR_TO_PACKET_END] = "pkt_end", 263 [PTR_TO_PACKET_END] = "pkt_end",
264 [PTR_TO_FLOW_KEYS] = "flow_keys",
264}; 265};
265 266
266static char slot_type_char[] = { 267static char slot_type_char[] = {
@@ -965,6 +966,7 @@ static bool is_spillable_regtype(enum bpf_reg_type type)
965 case PTR_TO_PACKET: 966 case PTR_TO_PACKET:
966 case PTR_TO_PACKET_META: 967 case PTR_TO_PACKET_META:
967 case PTR_TO_PACKET_END: 968 case PTR_TO_PACKET_END:
969 case PTR_TO_FLOW_KEYS:
968 case CONST_PTR_TO_MAP: 970 case CONST_PTR_TO_MAP:
969 return true; 971 return true;
970 default: 972 default:
@@ -1238,6 +1240,7 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
1238 case BPF_PROG_TYPE_LWT_XMIT: 1240 case BPF_PROG_TYPE_LWT_XMIT:
1239 case BPF_PROG_TYPE_SK_SKB: 1241 case BPF_PROG_TYPE_SK_SKB:
1240 case BPF_PROG_TYPE_SK_MSG: 1242 case BPF_PROG_TYPE_SK_MSG:
1243 case BPF_PROG_TYPE_FLOW_DISSECTOR:
1241 if (meta) 1244 if (meta)
1242 return meta->pkt_access; 1245 return meta->pkt_access;
1243 1246
@@ -1321,6 +1324,18 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off,
1321 return -EACCES; 1324 return -EACCES;
1322} 1325}
1323 1326
1327static int check_flow_keys_access(struct bpf_verifier_env *env, int off,
1328 int size)
1329{
1330 if (size < 0 || off < 0 ||
1331 (u64)off + size > sizeof(struct bpf_flow_keys)) {
1332 verbose(env, "invalid access to flow keys off=%d size=%d\n",
1333 off, size);
1334 return -EACCES;
1335 }
1336 return 0;
1337}
1338
1324static bool __is_pointer_value(bool allow_ptr_leaks, 1339static bool __is_pointer_value(bool allow_ptr_leaks,
1325 const struct bpf_reg_state *reg) 1340 const struct bpf_reg_state *reg)
1326{ 1341{
@@ -1422,6 +1437,9 @@ static int check_ptr_alignment(struct bpf_verifier_env *env,
1422 * right in front, treat it the very same way. 1437 * right in front, treat it the very same way.
1423 */ 1438 */
1424 return check_pkt_ptr_alignment(env, reg, off, size, strict); 1439 return check_pkt_ptr_alignment(env, reg, off, size, strict);
1440 case PTR_TO_FLOW_KEYS:
1441 pointer_desc = "flow keys ";
1442 break;
1425 case PTR_TO_MAP_VALUE: 1443 case PTR_TO_MAP_VALUE:
1426 pointer_desc = "value "; 1444 pointer_desc = "value ";
1427 break; 1445 break;
@@ -1692,6 +1710,17 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
1692 err = check_packet_access(env, regno, off, size, false); 1710 err = check_packet_access(env, regno, off, size, false);
1693 if (!err && t == BPF_READ && value_regno >= 0) 1711 if (!err && t == BPF_READ && value_regno >= 0)
1694 mark_reg_unknown(env, regs, value_regno); 1712 mark_reg_unknown(env, regs, value_regno);
1713 } else if (reg->type == PTR_TO_FLOW_KEYS) {
1714 if (t == BPF_WRITE && value_regno >= 0 &&
1715 is_pointer_value(env, value_regno)) {
1716 verbose(env, "R%d leaks addr into flow keys\n",
1717 value_regno);
1718 return -EACCES;
1719 }
1720
1721 err = check_flow_keys_access(env, off, size);
1722 if (!err && t == BPF_READ && value_regno >= 0)
1723 mark_reg_unknown(env, regs, value_regno);
1695 } else { 1724 } else {
1696 verbose(env, "R%d invalid mem access '%s'\n", regno, 1725 verbose(env, "R%d invalid mem access '%s'\n", regno,
1697 reg_type_str[reg->type]); 1726 reg_type_str[reg->type]);
@@ -1839,6 +1868,8 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
1839 case PTR_TO_PACKET_META: 1868 case PTR_TO_PACKET_META:
1840 return check_packet_access(env, regno, reg->off, access_size, 1869 return check_packet_access(env, regno, reg->off, access_size,
1841 zero_size_allowed); 1870 zero_size_allowed);
1871 case PTR_TO_FLOW_KEYS:
1872 return check_flow_keys_access(env, reg->off, access_size);
1842 case PTR_TO_MAP_VALUE: 1873 case PTR_TO_MAP_VALUE:
1843 return check_map_access(env, regno, reg->off, access_size, 1874 return check_map_access(env, regno, reg->off, access_size,
1844 zero_size_allowed); 1875 zero_size_allowed);
@@ -4366,6 +4397,7 @@ static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur,
4366 case PTR_TO_CTX: 4397 case PTR_TO_CTX:
4367 case CONST_PTR_TO_MAP: 4398 case CONST_PTR_TO_MAP:
4368 case PTR_TO_PACKET_END: 4399 case PTR_TO_PACKET_END:
4400 case PTR_TO_FLOW_KEYS:
4369 /* Only valid matches are exact, which memcmp() above 4401 /* Only valid matches are exact, which memcmp() above
4370 * would have accepted 4402 * would have accepted
4371 */ 4403 */
diff --git a/net/core/filter.c b/net/core/filter.c
index bf5b6efd369a..9cc76f134ddb 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -5124,6 +5124,17 @@ sk_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
5124} 5124}
5125 5125
5126static const struct bpf_func_proto * 5126static const struct bpf_func_proto *
5127flow_dissector_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
5128{
5129 switch (func_id) {
5130 case BPF_FUNC_skb_load_bytes:
5131 return &bpf_skb_load_bytes_proto;
5132 default:
5133 return bpf_base_func_proto(func_id);
5134 }
5135}
5136
5137static const struct bpf_func_proto *
5127lwt_out_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 5138lwt_out_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
5128{ 5139{
5129 switch (func_id) { 5140 switch (func_id) {
@@ -5241,6 +5252,10 @@ static bool bpf_skb_is_valid_access(int off, int size, enum bpf_access_type type
5241 if (size != size_default) 5252 if (size != size_default)
5242 return false; 5253 return false;
5243 break; 5254 break;
5255 case bpf_ctx_range(struct __sk_buff, flow_keys):
5256 if (size != sizeof(struct bpf_flow_keys *))
5257 return false;
5258 break;
5244 default: 5259 default:
5245 /* Only narrow read access allowed for now. */ 5260 /* Only narrow read access allowed for now. */
5246 if (type == BPF_WRITE) { 5261 if (type == BPF_WRITE) {
@@ -5266,6 +5281,7 @@ static bool sk_filter_is_valid_access(int off, int size,
5266 case bpf_ctx_range(struct __sk_buff, data): 5281 case bpf_ctx_range(struct __sk_buff, data):
5267 case bpf_ctx_range(struct __sk_buff, data_meta): 5282 case bpf_ctx_range(struct __sk_buff, data_meta):
5268 case bpf_ctx_range(struct __sk_buff, data_end): 5283 case bpf_ctx_range(struct __sk_buff, data_end):
5284 case bpf_ctx_range(struct __sk_buff, flow_keys):
5269 case bpf_ctx_range_till(struct __sk_buff, family, local_port): 5285 case bpf_ctx_range_till(struct __sk_buff, family, local_port):
5270 return false; 5286 return false;
5271 } 5287 }
@@ -5291,6 +5307,7 @@ static bool lwt_is_valid_access(int off, int size,
5291 case bpf_ctx_range(struct __sk_buff, tc_classid): 5307 case bpf_ctx_range(struct __sk_buff, tc_classid):
5292 case bpf_ctx_range_till(struct __sk_buff, family, local_port): 5308 case bpf_ctx_range_till(struct __sk_buff, family, local_port):
5293 case bpf_ctx_range(struct __sk_buff, data_meta): 5309 case bpf_ctx_range(struct __sk_buff, data_meta):
5310 case bpf_ctx_range(struct __sk_buff, flow_keys):
5294 return false; 5311 return false;
5295 } 5312 }
5296 5313
@@ -5501,6 +5518,7 @@ static bool tc_cls_act_is_valid_access(int off, int size,
5501 case bpf_ctx_range(struct __sk_buff, data_end): 5518 case bpf_ctx_range(struct __sk_buff, data_end):
5502 info->reg_type = PTR_TO_PACKET_END; 5519 info->reg_type = PTR_TO_PACKET_END;
5503 break; 5520 break;
5521 case bpf_ctx_range(struct __sk_buff, flow_keys):
5504 case bpf_ctx_range_till(struct __sk_buff, family, local_port): 5522 case bpf_ctx_range_till(struct __sk_buff, family, local_port):
5505 return false; 5523 return false;
5506 } 5524 }
@@ -5702,6 +5720,7 @@ static bool sk_skb_is_valid_access(int off, int size,
5702 switch (off) { 5720 switch (off) {
5703 case bpf_ctx_range(struct __sk_buff, tc_classid): 5721 case bpf_ctx_range(struct __sk_buff, tc_classid):
5704 case bpf_ctx_range(struct __sk_buff, data_meta): 5722 case bpf_ctx_range(struct __sk_buff, data_meta):
5723 case bpf_ctx_range(struct __sk_buff, flow_keys):
5705 return false; 5724 return false;
5706 } 5725 }
5707 5726
@@ -5761,6 +5780,39 @@ static bool sk_msg_is_valid_access(int off, int size,
5761 return true; 5780 return true;
5762} 5781}
5763 5782
5783static bool flow_dissector_is_valid_access(int off, int size,
5784 enum bpf_access_type type,
5785 const struct bpf_prog *prog,
5786 struct bpf_insn_access_aux *info)
5787{
5788 if (type == BPF_WRITE) {
5789 switch (off) {
5790 case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
5791 break;
5792 default:
5793 return false;
5794 }
5795 }
5796
5797 switch (off) {
5798 case bpf_ctx_range(struct __sk_buff, data):
5799 info->reg_type = PTR_TO_PACKET;
5800 break;
5801 case bpf_ctx_range(struct __sk_buff, data_end):
5802 info->reg_type = PTR_TO_PACKET_END;
5803 break;
5804 case bpf_ctx_range(struct __sk_buff, flow_keys):
5805 info->reg_type = PTR_TO_FLOW_KEYS;
5806 break;
5807 case bpf_ctx_range(struct __sk_buff, tc_classid):
5808 case bpf_ctx_range(struct __sk_buff, data_meta):
5809 case bpf_ctx_range_till(struct __sk_buff, family, local_port):
5810 return false;
5811 }
5812
5813 return bpf_skb_is_valid_access(off, size, type, prog, info);
5814}
5815
5764static u32 bpf_convert_ctx_access(enum bpf_access_type type, 5816static u32 bpf_convert_ctx_access(enum bpf_access_type type,
5765 const struct bpf_insn *si, 5817 const struct bpf_insn *si,
5766 struct bpf_insn *insn_buf, 5818 struct bpf_insn *insn_buf,
@@ -6055,6 +6107,15 @@ static u32 bpf_convert_ctx_access(enum bpf_access_type type,
6055 bpf_target_off(struct sock_common, 6107 bpf_target_off(struct sock_common,
6056 skc_num, 2, target_size)); 6108 skc_num, 2, target_size));
6057 break; 6109 break;
6110
6111 case offsetof(struct __sk_buff, flow_keys):
6112 off = si->off;
6113 off -= offsetof(struct __sk_buff, flow_keys);
6114 off += offsetof(struct sk_buff, cb);
6115 off += offsetof(struct qdisc_skb_cb, flow_keys);
6116 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg,
6117 si->src_reg, off);
6118 break;
6058 } 6119 }
6059 6120
6060 return insn - insn_buf; 6121 return insn - insn_buf;
@@ -7018,6 +7079,15 @@ const struct bpf_verifier_ops sk_msg_verifier_ops = {
7018const struct bpf_prog_ops sk_msg_prog_ops = { 7079const struct bpf_prog_ops sk_msg_prog_ops = {
7019}; 7080};
7020 7081
7082const struct bpf_verifier_ops flow_dissector_verifier_ops = {
7083 .get_func_proto = flow_dissector_func_proto,
7084 .is_valid_access = flow_dissector_is_valid_access,
7085 .convert_ctx_access = bpf_convert_ctx_access,
7086};
7087
7088const struct bpf_prog_ops flow_dissector_prog_ops = {
7089};
7090
7021int sk_detach_filter(struct sock *sk) 7091int sk_detach_filter(struct sock *sk)
7022{ 7092{
7023 int ret = -ENOENT; 7093 int ret = -ENOENT;
diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c
index ce9eeeb7c024..5c5dd74b5b3b 100644
--- a/net/core/flow_dissector.c
+++ b/net/core/flow_dissector.c
@@ -25,6 +25,9 @@
25#include <net/flow_dissector.h> 25#include <net/flow_dissector.h>
26#include <scsi/fc/fc_fcoe.h> 26#include <scsi/fc/fc_fcoe.h>
27#include <uapi/linux/batadv_packet.h> 27#include <uapi/linux/batadv_packet.h>
28#include <linux/bpf.h>
29
30static DEFINE_MUTEX(flow_dissector_mutex);
28 31
29static void dissector_set_key(struct flow_dissector *flow_dissector, 32static void dissector_set_key(struct flow_dissector *flow_dissector,
30 enum flow_dissector_key_id key_id) 33 enum flow_dissector_key_id key_id)
@@ -62,6 +65,44 @@ void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
62} 65}
63EXPORT_SYMBOL(skb_flow_dissector_init); 66EXPORT_SYMBOL(skb_flow_dissector_init);
64 67
68int skb_flow_dissector_bpf_prog_attach(const union bpf_attr *attr,
69 struct bpf_prog *prog)
70{
71 struct bpf_prog *attached;
72 struct net *net;
73
74 net = current->nsproxy->net_ns;
75 mutex_lock(&flow_dissector_mutex);
76 attached = rcu_dereference_protected(net->flow_dissector_prog,
77 lockdep_is_held(&flow_dissector_mutex));
78 if (attached) {
79 /* Only one BPF program can be attached at a time */
80 mutex_unlock(&flow_dissector_mutex);
81 return -EEXIST;
82 }
83 rcu_assign_pointer(net->flow_dissector_prog, prog);
84 mutex_unlock(&flow_dissector_mutex);
85 return 0;
86}
87
88int skb_flow_dissector_bpf_prog_detach(const union bpf_attr *attr)
89{
90 struct bpf_prog *attached;
91 struct net *net;
92
93 net = current->nsproxy->net_ns;
94 mutex_lock(&flow_dissector_mutex);
95 attached = rcu_dereference_protected(net->flow_dissector_prog,
96 lockdep_is_held(&flow_dissector_mutex));
97 if (!attached) {
98 mutex_unlock(&flow_dissector_mutex);
99 return -ENOENT;
100 }
101 bpf_prog_put(attached);
102 RCU_INIT_POINTER(net->flow_dissector_prog, NULL);
103 mutex_unlock(&flow_dissector_mutex);
104 return 0;
105}
65/** 106/**
66 * skb_flow_get_be16 - extract be16 entity 107 * skb_flow_get_be16 - extract be16 entity
67 * @skb: sk_buff to extract from 108 * @skb: sk_buff to extract from
@@ -588,6 +629,60 @@ static bool skb_flow_dissect_allowed(int *num_hdrs)
588 return (*num_hdrs <= MAX_FLOW_DISSECT_HDRS); 629 return (*num_hdrs <= MAX_FLOW_DISSECT_HDRS);
589} 630}
590 631
632static void __skb_flow_bpf_to_target(const struct bpf_flow_keys *flow_keys,
633 struct flow_dissector *flow_dissector,
634 void *target_container)
635{
636 struct flow_dissector_key_control *key_control;
637 struct flow_dissector_key_basic *key_basic;
638 struct flow_dissector_key_addrs *key_addrs;
639 struct flow_dissector_key_ports *key_ports;
640
641 key_control = skb_flow_dissector_target(flow_dissector,
642 FLOW_DISSECTOR_KEY_CONTROL,
643 target_container);
644 key_control->thoff = flow_keys->thoff;
645 if (flow_keys->is_frag)
646 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
647 if (flow_keys->is_first_frag)
648 key_control->flags |= FLOW_DIS_FIRST_FRAG;
649 if (flow_keys->is_encap)
650 key_control->flags |= FLOW_DIS_ENCAPSULATION;
651
652 key_basic = skb_flow_dissector_target(flow_dissector,
653 FLOW_DISSECTOR_KEY_BASIC,
654 target_container);
655 key_basic->n_proto = flow_keys->n_proto;
656 key_basic->ip_proto = flow_keys->ip_proto;
657
658 if (flow_keys->addr_proto == ETH_P_IP &&
659 dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
660 key_addrs = skb_flow_dissector_target(flow_dissector,
661 FLOW_DISSECTOR_KEY_IPV4_ADDRS,
662 target_container);
663 key_addrs->v4addrs.src = flow_keys->ipv4_src;
664 key_addrs->v4addrs.dst = flow_keys->ipv4_dst;
665 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
666 } else if (flow_keys->addr_proto == ETH_P_IPV6 &&
667 dissector_uses_key(flow_dissector,
668 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
669 key_addrs = skb_flow_dissector_target(flow_dissector,
670 FLOW_DISSECTOR_KEY_IPV6_ADDRS,
671 target_container);
672 memcpy(&key_addrs->v6addrs, &flow_keys->ipv6_src,
673 sizeof(key_addrs->v6addrs));
674 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
675 }
676
677 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS)) {
678 key_ports = skb_flow_dissector_target(flow_dissector,
679 FLOW_DISSECTOR_KEY_PORTS,
680 target_container);
681 key_ports->src = flow_keys->sport;
682 key_ports->dst = flow_keys->dport;
683 }
684}
685
591/** 686/**
592 * __skb_flow_dissect - extract the flow_keys struct and return it 687 * __skb_flow_dissect - extract the flow_keys struct and return it
593 * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified 688 * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
@@ -619,6 +714,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb,
619 struct flow_dissector_key_vlan *key_vlan; 714 struct flow_dissector_key_vlan *key_vlan;
620 enum flow_dissect_ret fdret; 715 enum flow_dissect_ret fdret;
621 enum flow_dissector_key_id dissector_vlan = FLOW_DISSECTOR_KEY_MAX; 716 enum flow_dissector_key_id dissector_vlan = FLOW_DISSECTOR_KEY_MAX;
717 struct bpf_prog *attached;
622 int num_hdrs = 0; 718 int num_hdrs = 0;
623 u8 ip_proto = 0; 719 u8 ip_proto = 0;
624 bool ret; 720 bool ret;
@@ -658,6 +754,44 @@ bool __skb_flow_dissect(const struct sk_buff *skb,
658 FLOW_DISSECTOR_KEY_BASIC, 754 FLOW_DISSECTOR_KEY_BASIC,
659 target_container); 755 target_container);
660 756
757 rcu_read_lock();
758 attached = skb ? rcu_dereference(dev_net(skb->dev)->flow_dissector_prog)
759 : NULL;
760 if (attached) {
761 /* Note that even though the const qualifier is discarded
762 * throughout the execution of the BPF program, all changes(the
763 * control block) are reverted after the BPF program returns.
764 * Therefore, __skb_flow_dissect does not alter the skb.
765 */
766 struct bpf_flow_keys flow_keys = {};
767 struct bpf_skb_data_end cb_saved;
768 struct bpf_skb_data_end *cb;
769 u32 result;
770
771 cb = (struct bpf_skb_data_end *)skb->cb;
772
773 /* Save Control Block */
774 memcpy(&cb_saved, cb, sizeof(cb_saved));
775 memset(cb, 0, sizeof(cb_saved));
776
777 /* Pass parameters to the BPF program */
778 cb->qdisc_cb.flow_keys = &flow_keys;
779 flow_keys.nhoff = nhoff;
780
781 bpf_compute_data_pointers((struct sk_buff *)skb);
782 result = BPF_PROG_RUN(attached, skb);
783
784 /* Restore state */
785 memcpy(cb, &cb_saved, sizeof(cb_saved));
786
787 __skb_flow_bpf_to_target(&flow_keys, flow_dissector,
788 target_container);
789 key_control->thoff = min_t(u16, key_control->thoff, skb->len);
790 rcu_read_unlock();
791 return result == BPF_OK;
792 }
793 rcu_read_unlock();
794
661 if (dissector_uses_key(flow_dissector, 795 if (dissector_uses_key(flow_dissector,
662 FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 796 FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
663 struct ethhdr *eth = eth_hdr(skb); 797 struct ethhdr *eth = eth_hdr(skb);
generated by cgit v1.2.2 (git 2.25.0) at 2025-10-10 00:07:22 -0400