summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMagnus Karlsson <magnus.karlsson@intel.com>2019-02-21 04:21:27 -0500
committerDaniel Borkmann <daniel@iogearbox.net>2019-02-25 17:21:42 -0500
commit248c7f9c0e215fcfd847bd3a41cf0160a2359e1a (patch)
treeb15e2e5fd8bc2adf29c1061bd224031cc7e37215
parent1cad078842396f0047a796694b6130fc096d97e2 (diff)
samples/bpf: convert xdpsock to use libbpf for AF_XDP access
This commit converts the xdpsock sample application to use the AF_XDP functions present in libbpf. This cuts down the size of it by nearly 300 lines of code. The default ring sizes plus the batch size has been increased and the size of the umem area has decreased. This so that the sample application will provide higher throughput. Note also that the shared umem code has been removed from the sample as this is not supported by libbpf at this point in time. Tested-by: Björn Töpel <bjorn.topel@intel.com> Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com> Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Diffstat
-rw-r--r--samples/bpf/Makefile1
-rw-r--r--samples/bpf/xdpsock.h11
-rw-r--r--samples/bpf/xdpsock_kern.c56
-rw-r--r--samples/bpf/xdpsock_user.c841
4 files changed, 261 insertions, 648 deletions
diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile
index a0ef7eddd0b3..a333e258f319 100644
--- a/samples/bpf/Makefile
+++ b/samples/bpf/Makefile
@@ -163,7 +163,6 @@ always += xdp2skb_meta_kern.o
163always += syscall_tp_kern.o 163always += syscall_tp_kern.o
164always += cpustat_kern.o 164always += cpustat_kern.o
165always += xdp_adjust_tail_kern.o 165always += xdp_adjust_tail_kern.o
166always += xdpsock_kern.o
167always += xdp_fwd_kern.o 166always += xdp_fwd_kern.o
168always += task_fd_query_kern.o 167always += task_fd_query_kern.o
169always += xdp_sample_pkts_kern.o 168always += xdp_sample_pkts_kern.o
diff --git a/samples/bpf/xdpsock.h b/samples/bpf/xdpsock.h
deleted file mode 100644
index 533ab81adfa1..000000000000
--- a/samples/bpf/xdpsock.h
+++ /dev/null
@@ -1,11 +0,0 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef XDPSOCK_H_
3#define XDPSOCK_H_
4
5/* Power-of-2 number of sockets */
6#define MAX_SOCKS 4
7
8/* Round-robin receive */
9#define RR_LB 0
10
11#endif /* XDPSOCK_H_ */
diff --git a/samples/bpf/xdpsock_kern.c b/samples/bpf/xdpsock_kern.c
deleted file mode 100644
index b8ccd0802b3f..000000000000
--- a/samples/bpf/xdpsock_kern.c
+++ /dev/null
@@ -1,56 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0
2#define KBUILD_MODNAME "foo"
3#include <uapi/linux/bpf.h>
4#include "bpf_helpers.h"
5
6#include "xdpsock.h"
7
8struct bpf_map_def SEC("maps") qidconf_map = {
9 .type = BPF_MAP_TYPE_ARRAY,
10 .key_size = sizeof(int),
11 .value_size = sizeof(int),
12 .max_entries = 1,
13};
14
15struct bpf_map_def SEC("maps") xsks_map = {
16 .type = BPF_MAP_TYPE_XSKMAP,
17 .key_size = sizeof(int),
18 .value_size = sizeof(int),
19 .max_entries = MAX_SOCKS,
20};
21
22struct bpf_map_def SEC("maps") rr_map = {
23 .type = BPF_MAP_TYPE_PERCPU_ARRAY,
24 .key_size = sizeof(int),
25 .value_size = sizeof(unsigned int),
26 .max_entries = 1,
27};
28
29SEC("xdp_sock")
30int xdp_sock_prog(struct xdp_md *ctx)
31{
32 int *qidconf, key = 0, idx;
33 unsigned int *rr;
34
35 qidconf = bpf_map_lookup_elem(&qidconf_map, &key);
36 if (!qidconf)
37 return XDP_ABORTED;
38
39 if (*qidconf != ctx->rx_queue_index)
40 return XDP_PASS;
41
42#if RR_LB /* NB! RR_LB is configured in xdpsock.h */
43 rr = bpf_map_lookup_elem(&rr_map, &key);
44 if (!rr)
45 return XDP_ABORTED;
46
47 *rr = (*rr + 1) & (MAX_SOCKS - 1);
48 idx = *rr;
49#else
50 idx = 0;
51#endif
52
53 return bpf_redirect_map(&xsks_map, idx, 0);
54}
55
56char _license[] SEC("license") = "GPL";
diff --git a/samples/bpf/xdpsock_user.c b/samples/bpf/xdpsock_user.c
index f73055e0191f..9c76d6d43deb 100644
--- a/samples/bpf/xdpsock_user.c
+++ b/samples/bpf/xdpsock_user.c
@@ -1,37 +1,36 @@
1// SPDX-License-Identifier: GPL-2.0 1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2017 - 2018 Intel Corporation. */ 2/* Copyright(c) 2017 - 2018 Intel Corporation. */
3 3
4#include <assert.h> 4#include <asm/barrier.h>
5#include <errno.h> 5#include <errno.h>
6#include <getopt.h> 6#include <getopt.h>
7#include <libgen.h> 7#include <libgen.h>
8#include <linux/bpf.h> 8#include <linux/bpf.h>
9#include <linux/compiler.h>
9#include <linux/if_link.h> 10#include <linux/if_link.h>
10#include <linux/if_xdp.h> 11#include <linux/if_xdp.h>
11#include <linux/if_ether.h> 12#include <linux/if_ether.h>
13#include <locale.h>
14#include <net/ethernet.h>
12#include <net/if.h> 15#include <net/if.h>
16#include <poll.h>
17#include <pthread.h>
13#include <signal.h> 18#include <signal.h>
14#include <stdbool.h> 19#include <stdbool.h>
15#include <stdio.h> 20#include <stdio.h>
16#include <stdlib.h> 21#include <stdlib.h>
17#include <string.h> 22#include <string.h>
18#include <net/ethernet.h> 23#include <sys/mman.h>
19#include <sys/resource.h> 24#include <sys/resource.h>
20#include <sys/socket.h> 25#include <sys/socket.h>
21#include <sys/mman.h> 26#include <sys/types.h>
22#include <time.h> 27#include <time.h>
23#include <unistd.h> 28#include <unistd.h>
24#include <pthread.h>
25#include <locale.h>
26#include <sys/types.h>
27#include <poll.h>
28 29
29#include "bpf/libbpf.h" 30#include "bpf/libbpf.h"
30#include "bpf_util.h" 31#include "bpf/xsk.h"
31#include <bpf/bpf.h> 32#include <bpf/bpf.h>
32 33
33#include "xdpsock.h"
34
35#ifndef SOL_XDP 34#ifndef SOL_XDP
36#define SOL_XDP 283 35#define SOL_XDP 283
37#endif 36#endif
@@ -44,17 +43,11 @@
44#define PF_XDP AF_XDP 43#define PF_XDP AF_XDP
45#endif 44#endif
46 45
47#define NUM_FRAMES 131072 46#define NUM_FRAMES (4 * 1024)
48#define FRAME_HEADROOM 0 47#define BATCH_SIZE 64
49#define FRAME_SHIFT 11
50#define FRAME_SIZE 2048
51#define NUM_DESCS 1024
52#define BATCH_SIZE 16
53
54#define FQ_NUM_DESCS 1024
55#define CQ_NUM_DESCS 1024
56 48
57#define DEBUG_HEXDUMP 0 49#define DEBUG_HEXDUMP 0
50#define MAX_SOCKS 8
58 51
59typedef __u64 u64; 52typedef __u64 u64;
60typedef __u32 u32; 53typedef __u32 u32;
@@ -73,54 +66,31 @@ static const char *opt_if = "";
73static int opt_ifindex; 66static int opt_ifindex;
74static int opt_queue; 67static int opt_queue;
75static int opt_poll; 68static int opt_poll;
76static int opt_shared_packet_buffer;
77static int opt_interval = 1; 69static int opt_interval = 1;
78static u32 opt_xdp_bind_flags; 70static u32 opt_xdp_bind_flags;
79static __u32 prog_id; 71static __u32 prog_id;
80 72
81struct xdp_umem_uqueue { 73struct xsk_umem_info {
82 u32 cached_prod; 74 struct xsk_ring_prod fq;
83 u32 cached_cons; 75 struct xsk_ring_cons cq;
84 u32 mask; 76 struct xsk_umem *umem;
85 u32 size; 77 void *buffer;
86 u32 *producer;
87 u32 *consumer;
88 u64 *ring;
89 void *map;
90}; 78};
91 79
92struct xdp_umem { 80struct xsk_socket_info {
93 char *frames; 81 struct xsk_ring_cons rx;
94 struct xdp_umem_uqueue fq; 82 struct xsk_ring_prod tx;
95 struct xdp_umem_uqueue cq; 83 struct xsk_umem_info *umem;
96 int fd; 84 struct xsk_socket *xsk;
97};
98
99struct xdp_uqueue {
100 u32 cached_prod;
101 u32 cached_cons;
102 u32 mask;
103 u32 size;
104 u32 *producer;
105 u32 *consumer;
106 struct xdp_desc *ring;
107 void *map;
108};
109
110struct xdpsock {
111 struct xdp_uqueue rx;
112 struct xdp_uqueue tx;
113 int sfd;
114 struct xdp_umem *umem;
115 u32 outstanding_tx;
116 unsigned long rx_npkts; 85 unsigned long rx_npkts;
117 unsigned long tx_npkts; 86 unsigned long tx_npkts;
118 unsigned long prev_rx_npkts; 87 unsigned long prev_rx_npkts;
119 unsigned long prev_tx_npkts; 88 unsigned long prev_tx_npkts;
89 u32 outstanding_tx;
120}; 90};
121 91
122static int num_socks; 92static int num_socks;
123struct xdpsock *xsks[MAX_SOCKS]; 93struct xsk_socket_info *xsks[MAX_SOCKS];
124 94
125static unsigned long get_nsecs(void) 95static unsigned long get_nsecs(void)
126{ 96{
@@ -130,225 +100,124 @@ static unsigned long get_nsecs(void)
130 return ts.tv_sec * 1000000000UL + ts.tv_nsec; 100 return ts.tv_sec * 1000000000UL + ts.tv_nsec;
131} 101}
132 102
133static void dump_stats(void); 103static void print_benchmark(bool running)
134
135#define lassert(expr) \
136 do { \
137 if (!(expr)) { \
138 fprintf(stderr, "%s:%s:%i: Assertion failed: " \
139 #expr ": errno: %d/\"%s\"\n", \
140 __FILE__, __func__, __LINE__, \
141 errno, strerror(errno)); \
142 dump_stats(); \
143 exit(EXIT_FAILURE); \
144 } \
145 } while (0)
146
147#define barrier() __asm__ __volatile__("": : :"memory")
148#ifdef __aarch64__
149#define u_smp_rmb() __asm__ __volatile__("dmb ishld": : :"memory")
150#define u_smp_wmb() __asm__ __volatile__("dmb ishst": : :"memory")
151#else
152#define u_smp_rmb() barrier()
153#define u_smp_wmb() barrier()
154#endif
155#define likely(x) __builtin_expect(!!(x), 1)
156#define unlikely(x) __builtin_expect(!!(x), 0)
157
158static const char pkt_data[] =
159 "\x3c\xfd\xfe\x9e\x7f\x71\xec\xb1\xd7\x98\x3a\xc0\x08\x00\x45\x00"
160 "\x00\x2e\x00\x00\x00\x00\x40\x11\x88\x97\x05\x08\x07\x08\xc8\x14"
161 "\x1e\x04\x10\x92\x10\x92\x00\x1a\x6d\xa3\x34\x33\x1f\x69\x40\x6b"
162 "\x54\x59\xb6\x14\x2d\x11\x44\xbf\xaf\xd9\xbe\xaa";
163
164static inline u32 umem_nb_free(struct xdp_umem_uqueue *q, u32 nb)
165{
166 u32 free_entries = q->cached_cons - q->cached_prod;
167
168 if (free_entries >= nb)
169 return free_entries;
170
171 /* Refresh the local tail pointer */
172 q->cached_cons = *q->consumer + q->size;
173
174 return q->cached_cons - q->cached_prod;
175}
176
177static inline u32 xq_nb_free(struct xdp_uqueue *q, u32 ndescs)
178{ 104{
179 u32 free_entries = q->cached_cons - q->cached_prod; 105 const char *bench_str = "INVALID";
180 106
181 if (free_entries >= ndescs) 107 if (opt_bench == BENCH_RXDROP)
182 return free_entries; 108 bench_str = "rxdrop";
109 else if (opt_bench == BENCH_TXONLY)
110 bench_str = "txonly";
111 else if (opt_bench == BENCH_L2FWD)
112 bench_str = "l2fwd";
183 113
184 /* Refresh the local tail pointer */ 114 printf("%s:%d %s ", opt_if, opt_queue, bench_str);
185 q->cached_cons = *q->consumer + q->size; 115 if (opt_xdp_flags & XDP_FLAGS_SKB_MODE)
186 return q->cached_cons - q->cached_prod; 116 printf("xdp-skb ");
187} 117 else if (opt_xdp_flags & XDP_FLAGS_DRV_MODE)
118 printf("xdp-drv ");
119 else
120 printf(" ");
188 121
189static inline u32 umem_nb_avail(struct xdp_umem_uqueue *q, u32 nb) 122 if (opt_poll)
190{ 123 printf("poll() ");
191 u32 entries = q->cached_prod - q->cached_cons;
192 124
193 if (entries == 0) { 125 if (running) {
194 q->cached_prod = *q->producer; 126 printf("running...");
195 entries = q->cached_prod - q->cached_cons; 127 fflush(stdout);
196 } 128 }
197
198 return (entries > nb) ? nb : entries;
199} 129}
200 130
201static inline u32 xq_nb_avail(struct xdp_uqueue *q, u32 ndescs) 131static void dump_stats(void)
202{ 132{
203 u32 entries = q->cached_prod - q->cached_cons; 133 unsigned long now = get_nsecs();
134 long dt = now - prev_time;
135 int i;
204 136
205 if (entries == 0) { 137 prev_time = now;
206 q->cached_prod = *q->producer;
207 entries = q->cached_prod - q->cached_cons;
208 }
209 138
210 return (entries > ndescs) ? ndescs : entries; 139 for (i = 0; i < num_socks && xsks[i]; i++) {
211} 140 char *fmt = "%-15s %'-11.0f %'-11lu\n";
141 double rx_pps, tx_pps;
212 142
213static inline int umem_fill_to_kernel_ex(struct xdp_umem_uqueue *fq, 143 rx_pps = (xsks[i]->rx_npkts - xsks[i]->prev_rx_npkts) *
214 struct xdp_desc *d, 144 1000000000. / dt;
215 size_t nb) 145 tx_pps = (xsks[i]->tx_npkts - xsks[i]->prev_tx_npkts) *
216{ 146 1000000000. / dt;
217 u32 i;
218 147
219 if (umem_nb_free(fq, nb) < nb) 148 printf("\n sock%d@", i);
220 return -ENOSPC; 149 print_benchmark(false);
150 printf("\n");
221 151
222 for (i = 0; i < nb; i++) { 152 printf("%-15s %-11s %-11s %-11.2f\n", "", "pps", "pkts",
223 u32 idx = fq->cached_prod++ & fq->mask; 153 dt / 1000000000.);
154 printf(fmt, "rx", rx_pps, xsks[i]->rx_npkts);
155 printf(fmt, "tx", tx_pps, xsks[i]->tx_npkts);
224 156
225 fq->ring[idx] = d[i].addr; 157 xsks[i]->prev_rx_npkts = xsks[i]->rx_npkts;
158 xsks[i]->prev_tx_npkts = xsks[i]->tx_npkts;
226 } 159 }
227
228 u_smp_wmb();
229
230 *fq->producer = fq->cached_prod;
231
232 return 0;
233} 160}
234 161
235static inline int umem_fill_to_kernel(struct xdp_umem_uqueue *fq, u64 *d, 162static void *poller(void *arg)
236 size_t nb)
237{ 163{
238 u32 i; 164 (void)arg;
239 165 for (;;) {
240 if (umem_nb_free(fq, nb) < nb) 166 sleep(opt_interval);
241 return -ENOSPC; 167 dump_stats();
242
243 for (i = 0; i < nb; i++) {
244 u32 idx = fq->cached_prod++ & fq->mask;
245
246 fq->ring[idx] = d[i];
247 } 168 }
248 169
249 u_smp_wmb(); 170 return NULL;
250
251 *fq->producer = fq->cached_prod;
252
253 return 0;
254} 171}
255 172
256static inline size_t umem_complete_from_kernel(struct xdp_umem_uqueue *cq, 173static void remove_xdp_program(void)
257 u64 *d, size_t nb)
258{ 174{
259 u32 idx, i, entries = umem_nb_avail(cq, nb); 175 __u32 curr_prog_id = 0;
260
261 u_smp_rmb();
262
263 for (i = 0; i < entries; i++) {
264 idx = cq->cached_cons++ & cq->mask;
265 d[i] = cq->ring[idx];
266 }
267
268 if (entries > 0) {
269 u_smp_wmb();
270 176
271 *cq->consumer = cq->cached_cons; 177 if (bpf_get_link_xdp_id(opt_ifindex, &curr_prog_id, opt_xdp_flags)) {
178 printf("bpf_get_link_xdp_id failed\n");
179 exit(EXIT_FAILURE);
272 } 180 }
273 181 if (prog_id == curr_prog_id)
274 return entries; 182 bpf_set_link_xdp_fd(opt_ifindex, -1, opt_xdp_flags);
275} 183 else if (!curr_prog_id)
276 184 printf("couldn't find a prog id on a given interface\n");
277static inline void *xq_get_data(struct xdpsock *xsk, u64 addr) 185 else
278{ 186 printf("program on interface changed, not removing\n");
279 return &xsk->umem->frames[addr];
280} 187}
281 188
282static inline int xq_enq(struct xdp_uqueue *uq, 189static void int_exit(int sig)
283 const struct xdp_desc *descs,
284 unsigned int ndescs)
285{ 190{
286 struct xdp_desc *r = uq->ring; 191 struct xsk_umem *umem = xsks[0]->umem->umem;
287 unsigned int i;
288 192
289 if (xq_nb_free(uq, ndescs) < ndescs) 193 (void)sig;
290 return -ENOSPC;
291
292 for (i = 0; i < ndescs; i++) {
293 u32 idx = uq->cached_prod++ & uq->mask;
294
295 r[idx].addr = descs[i].addr;
296 r[idx].len = descs[i].len;
297 }
298 194
299 u_smp_wmb(); 195 dump_stats();
196 xsk_socket__delete(xsks[0]->xsk);
197 (void)xsk_umem__delete(umem);
198 remove_xdp_program();
300 199
301 *uq->producer = uq->cached_prod; 200 exit(EXIT_SUCCESS);
302 return 0;
303} 201}
304 202
305static inline int xq_enq_tx_only(struct xdp_uqueue *uq, 203static void __exit_with_error(int error, const char *file, const char *func,
306 unsigned int id, unsigned int ndescs) 204 int line)
307{ 205{
308 struct xdp_desc *r = uq->ring; 206 fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func,
309 unsigned int i; 207 line, error, strerror(error));
310 208 dump_stats();
311 if (xq_nb_free(uq, ndescs) < ndescs) 209 remove_xdp_program();
312 return -ENOSPC; 210 exit(EXIT_FAILURE);
313
314 for (i = 0; i < ndescs; i++) {
315 u32 idx = uq->cached_prod++ & uq->mask;
316
317 r[idx].addr = (id + i) << FRAME_SHIFT;
318 r[idx].len = sizeof(pkt_data) - 1;
319 }
320
321 u_smp_wmb();
322
323 *uq->producer = uq->cached_prod;
324 return 0;
325} 211}
326 212
327static inline int xq_deq(struct xdp_uqueue *uq, 213#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, \
328 struct xdp_desc *descs, 214 __LINE__)
329 int ndescs)
330{
331 struct xdp_desc *r = uq->ring;
332 unsigned int idx;
333 int i, entries;
334
335 entries = xq_nb_avail(uq, ndescs);
336
337 u_smp_rmb();
338
339 for (i = 0; i < entries; i++) {
340 idx = uq->cached_cons++ & uq->mask;
341 descs[i] = r[idx];
342 }
343
344 if (entries > 0) {
345 u_smp_wmb();
346 215
347 *uq->consumer = uq->cached_cons; 216static const char pkt_data[] =
348 } 217 "\x3c\xfd\xfe\x9e\x7f\x71\xec\xb1\xd7\x98\x3a\xc0\x08\x00\x45\x00"
349 218 "\x00\x2e\x00\x00\x00\x00\x40\x11\x88\x97\x05\x08\x07\x08\xc8\x14"
350 return entries; 219 "\x1e\x04\x10\x92\x10\x92\x00\x1a\x6d\xa3\x34\x33\x1f\x69\x40\x6b"
351} 220 "\x54\x59\xb6\x14\x2d\x11\x44\xbf\xaf\xd9\xbe\xaa";
352 221
353static void swap_mac_addresses(void *data) 222static void swap_mac_addresses(void *data)
354{ 223{
@@ -397,258 +266,74 @@ static void hex_dump(void *pkt, size_t length, u64 addr)
397 printf("\n"); 266 printf("\n");
398} 267}
399 268
400static size_t gen_eth_frame(char *frame) 269static size_t gen_eth_frame(struct xsk_umem_info *umem, u64 addr)
401{ 270{
402 memcpy(frame, pkt_data, sizeof(pkt_data) - 1); 271 memcpy(xsk_umem__get_data(umem->buffer, addr), pkt_data,
272 sizeof(pkt_data) - 1);
403 return sizeof(pkt_data) - 1; 273 return sizeof(pkt_data) - 1;
404} 274}
405 275
406static struct xdp_umem *xdp_umem_configure(int sfd) 276static struct xsk_umem_info *xsk_configure_umem(void *buffer, u64 size)
407{ 277{
408 int fq_size = FQ_NUM_DESCS, cq_size = CQ_NUM_DESCS; 278 struct xsk_umem_info *umem;
409 struct xdp_mmap_offsets off; 279 int ret;
410 struct xdp_umem_reg mr;
411 struct xdp_umem *umem;
412 socklen_t optlen;
413 void *bufs;
414 280
415 umem = calloc(1, sizeof(*umem)); 281 umem = calloc(1, sizeof(*umem));
416 lassert(umem); 282 if (!umem)
417 283 exit_with_error(errno);
418 lassert(posix_memalign(&bufs, getpagesize(), /* PAGE_SIZE aligned */
419 NUM_FRAMES * FRAME_SIZE) == 0);
420
421 mr.addr = (__u64)bufs;
422 mr.len = NUM_FRAMES * FRAME_SIZE;
423 mr.chunk_size = FRAME_SIZE;
424 mr.headroom = FRAME_HEADROOM;
425
426 lassert(setsockopt(sfd, SOL_XDP, XDP_UMEM_REG, &mr, sizeof(mr)) == 0);
427 lassert(setsockopt(sfd, SOL_XDP, XDP_UMEM_FILL_RING, &fq_size,
428 sizeof(int)) == 0);
429 lassert(setsockopt(sfd, SOL_XDP, XDP_UMEM_COMPLETION_RING, &cq_size,
430 sizeof(int)) == 0);
431
432 optlen = sizeof(off);
433 lassert(getsockopt(sfd, SOL_XDP, XDP_MMAP_OFFSETS, &off,
434 &optlen) == 0);
435
436 umem->fq.map = mmap(0, off.fr.desc +
437 FQ_NUM_DESCS * sizeof(u64),
438 PROT_READ | PROT_WRITE,
439 MAP_SHARED | MAP_POPULATE, sfd,
440 XDP_UMEM_PGOFF_FILL_RING);
441 lassert(umem->fq.map != MAP_FAILED);
442
443 umem->fq.mask = FQ_NUM_DESCS - 1;
444 umem->fq.size = FQ_NUM_DESCS;
445 umem->fq.producer = umem->fq.map + off.fr.producer;
446 umem->fq.consumer = umem->fq.map + off.fr.consumer;
447 umem->fq.ring = umem->fq.map + off.fr.desc;
448 umem->fq.cached_cons = FQ_NUM_DESCS;
449
450 umem->cq.map = mmap(0, off.cr.desc +
451 CQ_NUM_DESCS * sizeof(u64),
452 PROT_READ | PROT_WRITE,
453 MAP_SHARED | MAP_POPULATE, sfd,
454 XDP_UMEM_PGOFF_COMPLETION_RING);
455 lassert(umem->cq.map != MAP_FAILED);
456
457 umem->cq.mask = CQ_NUM_DESCS - 1;
458 umem->cq.size = CQ_NUM_DESCS;
459 umem->cq.producer = umem->cq.map + off.cr.producer;
460 umem->cq.consumer = umem->cq.map + off.cr.consumer;
461 umem->cq.ring = umem->cq.map + off.cr.desc;
462
463 umem->frames = bufs;
464 umem->fd = sfd;
465 284
466 if (opt_bench == BENCH_TXONLY) { 285 ret = xsk_umem__create(&umem->umem, buffer, size, &umem->fq, &umem->cq,
467 int i; 286 NULL);
468 287 if (ret)
469 for (i = 0; i < NUM_FRAMES * FRAME_SIZE; i += FRAME_SIZE) 288 exit_with_error(-ret);
470 (void)gen_eth_frame(&umem->frames[i]);
471 }
472 289
290 umem->buffer = buffer;
473 return umem; 291 return umem;
474} 292}
475 293
476static struct xdpsock *xsk_configure(struct xdp_umem *umem) 294static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem)
477{ 295{
478 struct sockaddr_xdp sxdp = {}; 296 struct xsk_socket_config cfg;
479 struct xdp_mmap_offsets off; 297 struct xsk_socket_info *xsk;
480 int sfd, ndescs = NUM_DESCS; 298 int ret;
481 struct xdpsock *xsk; 299 u32 idx;
482 bool shared = true; 300 int i;
483 socklen_t optlen;
484 u64 i;
485
486 sfd = socket(PF_XDP, SOCK_RAW, 0);
487 lassert(sfd >= 0);
488 301
489 xsk = calloc(1, sizeof(*xsk)); 302 xsk = calloc(1, sizeof(*xsk));
490 lassert(xsk); 303 if (!xsk)
491 304 exit_with_error(errno);
492 xsk->sfd = sfd; 305
493 xsk->outstanding_tx = 0; 306 xsk->umem = umem;
494 307 cfg.rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
495 if (!umem) { 308 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
496 shared = false; 309 cfg.libbpf_flags = 0;
497 xsk->umem = xdp_umem_configure(sfd); 310 cfg.xdp_flags = opt_xdp_flags;
498 } else { 311 cfg.bind_flags = opt_xdp_bind_flags;
499 xsk->umem = umem; 312 ret = xsk_socket__create(&xsk->xsk, opt_if, opt_queue, umem->umem,
500 } 313 &xsk->rx, &xsk->tx, &cfg);
501 314 if (ret)
502 lassert(setsockopt(sfd, SOL_XDP, XDP_RX_RING, 315 exit_with_error(-ret);
503 &ndescs, sizeof(int)) == 0); 316
504 lassert(setsockopt(sfd, SOL_XDP, XDP_TX_RING, 317 ret = bpf_get_link_xdp_id(opt_ifindex, &prog_id, opt_xdp_flags);
505 &ndescs, sizeof(int)) == 0); 318 if (ret)
506 optlen = sizeof(off); 319 exit_with_error(-ret);
507 lassert(getsockopt(sfd, SOL_XDP, XDP_MMAP_OFFSETS, &off, 320
508 &optlen) == 0); 321 ret = xsk_ring_prod__reserve(&xsk->umem->fq,
509 322 XSK_RING_PROD__DEFAULT_NUM_DESCS,
510 /* Rx */ 323 &idx);
511 xsk->rx.map = mmap(NULL, 324 if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS)
512 off.rx.desc + 325 exit_with_error(-ret);
513 NUM_DESCS * sizeof(struct xdp_desc), 326 for (i = 0;
514 PROT_READ | PROT_WRITE, 327 i < XSK_RING_PROD__DEFAULT_NUM_DESCS *
515 MAP_SHARED | MAP_POPULATE, sfd, 328 XSK_UMEM__DEFAULT_FRAME_SIZE;
516 XDP_PGOFF_RX_RING); 329 i += XSK_UMEM__DEFAULT_FRAME_SIZE)
517 lassert(xsk->rx.map != MAP_FAILED); 330 *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx++) = i;
518 331 xsk_ring_prod__submit(&xsk->umem->fq,
519 if (!shared) { 332 XSK_RING_PROD__DEFAULT_NUM_DESCS);
520 for (i = 0; i < NUM_DESCS * FRAME_SIZE; i += FRAME_SIZE)
521 lassert(umem_fill_to_kernel(&xsk->umem->fq, &i, 1)
522 == 0);
523 }
524
525 /* Tx */
526 xsk->tx.map = mmap(NULL,
527 off.tx.desc +
528 NUM_DESCS * sizeof(struct xdp_desc),
529 PROT_READ | PROT_WRITE,
530 MAP_SHARED | MAP_POPULATE, sfd,
531 XDP_PGOFF_TX_RING);
532 lassert(xsk->tx.map != MAP_FAILED);
533
534 xsk->rx.mask = NUM_DESCS - 1;
535 xsk->rx.size = NUM_DESCS;
536 xsk->rx.producer = xsk->rx.map + off.rx.producer;
537 xsk->rx.consumer = xsk->rx.map + off.rx.consumer;
538 xsk->rx.ring = xsk->rx.map + off.rx.desc;
539
540 xsk->tx.mask = NUM_DESCS - 1;
541 xsk->tx.size = NUM_DESCS;
542 xsk->tx.producer = xsk->tx.map + off.tx.producer;
543 xsk->tx.consumer = xsk->tx.map + off.tx.consumer;
544 xsk->tx.ring = xsk->tx.map + off.tx.desc;
545 xsk->tx.cached_cons = NUM_DESCS;
546
547 sxdp.sxdp_family = PF_XDP;
548 sxdp.sxdp_ifindex = opt_ifindex;
549 sxdp.sxdp_queue_id = opt_queue;
550
551 if (shared) {
552 sxdp.sxdp_flags = XDP_SHARED_UMEM;
553 sxdp.sxdp_shared_umem_fd = umem->fd;
554 } else {
555 sxdp.sxdp_flags = opt_xdp_bind_flags;
556 }
557
558 lassert(bind(sfd, (struct sockaddr *)&sxdp, sizeof(sxdp)) == 0);
559 333
560 return xsk; 334 return xsk;
561} 335}
562 336
563static void print_benchmark(bool running)
564{
565 const char *bench_str = "INVALID";
566
567 if (opt_bench == BENCH_RXDROP)
568 bench_str = "rxdrop";
569 else if (opt_bench == BENCH_TXONLY)
570 bench_str = "txonly";
571 else if (opt_bench == BENCH_L2FWD)
572 bench_str = "l2fwd";
573
574 printf("%s:%d %s ", opt_if, opt_queue, bench_str);
575 if (opt_xdp_flags & XDP_FLAGS_SKB_MODE)
576 printf("xdp-skb ");
577 else if (opt_xdp_flags & XDP_FLAGS_DRV_MODE)
578 printf("xdp-drv ");
579 else
580 printf(" ");
581
582 if (opt_poll)
583 printf("poll() ");
584
585 if (running) {
586 printf("running...");
587 fflush(stdout);
588 }
589}
590
591static void dump_stats(void)
592{
593 unsigned long now = get_nsecs();
594 long dt = now - prev_time;
595 int i;
596
597 prev_time = now;
598
599 for (i = 0; i < num_socks && xsks[i]; i++) {
600 char *fmt = "%-15s %'-11.0f %'-11lu\n";
601 double rx_pps, tx_pps;
602
603 rx_pps = (xsks[i]->rx_npkts - xsks[i]->prev_rx_npkts) *
604 1000000000. / dt;
605 tx_pps = (xsks[i]->tx_npkts - xsks[i]->prev_tx_npkts) *
606 1000000000. / dt;
607
608 printf("\n sock%d@", i);
609 print_benchmark(false);
610 printf("\n");
611
612 printf("%-15s %-11s %-11s %-11.2f\n", "", "pps", "pkts",
613 dt / 1000000000.);
614 printf(fmt, "rx", rx_pps, xsks[i]->rx_npkts);
615 printf(fmt, "tx", tx_pps, xsks[i]->tx_npkts);
616
617 xsks[i]->prev_rx_npkts = xsks[i]->rx_npkts;
618 xsks[i]->prev_tx_npkts = xsks[i]->tx_npkts;
619 }
620}
621
622static void *poller(void *arg)
623{
624 (void)arg;
625 for (;;) {
626 sleep(opt_interval);
627 dump_stats();
628 }
629
630 return NULL;
631}
632
633static void int_exit(int sig)
634{
635 __u32 curr_prog_id = 0;
636
637 (void)sig;
638 dump_stats();
639 if (bpf_get_link_xdp_id(opt_ifindex, &curr_prog_id, opt_xdp_flags)) {
640 printf("bpf_get_link_xdp_id failed\n");
641 exit(EXIT_FAILURE);
642 }
643 if (prog_id == curr_prog_id)
644 bpf_set_link_xdp_fd(opt_ifindex, -1, opt_xdp_flags);
645 else if (!curr_prog_id)
646 printf("couldn't find a prog id on a given interface\n");
647 else
648 printf("program on interface changed, not removing\n");
649 exit(EXIT_SUCCESS);
650}
651
652static struct option long_options[] = { 337static struct option long_options[] = {
653 {"rxdrop", no_argument, 0, 'r'}, 338 {"rxdrop", no_argument, 0, 'r'},
654 {"txonly", no_argument, 0, 't'}, 339 {"txonly", no_argument, 0, 't'},
@@ -656,7 +341,6 @@ static struct option long_options[] = {
656 {"interface", required_argument, 0, 'i'}, 341 {"interface", required_argument, 0, 'i'},
657 {"queue", required_argument, 0, 'q'}, 342 {"queue", required_argument, 0, 'q'},
658 {"poll", no_argument, 0, 'p'}, 343 {"poll", no_argument, 0, 'p'},
659 {"shared-buffer", no_argument, 0, 's'},
660 {"xdp-skb", no_argument, 0, 'S'}, 344 {"xdp-skb", no_argument, 0, 'S'},
661 {"xdp-native", no_argument, 0, 'N'}, 345 {"xdp-native", no_argument, 0, 'N'},
662 {"interval", required_argument, 0, 'n'}, 346 {"interval", required_argument, 0, 'n'},
@@ -676,7 +360,6 @@ static void usage(const char *prog)
676 " -i, --interface=n Run on interface n\n" 360 " -i, --interface=n Run on interface n\n"
677 " -q, --queue=n Use queue n (default 0)\n" 361 " -q, --queue=n Use queue n (default 0)\n"
678 " -p, --poll Use poll syscall\n" 362 " -p, --poll Use poll syscall\n"
679 " -s, --shared-buffer Use shared packet buffer\n"
680 " -S, --xdp-skb=n Use XDP skb-mod\n" 363 " -S, --xdp-skb=n Use XDP skb-mod\n"
681 " -N, --xdp-native=n Enfore XDP native mode\n" 364 " -N, --xdp-native=n Enfore XDP native mode\n"
682 " -n, --interval=n Specify statistics update interval (default 1 sec).\n" 365 " -n, --interval=n Specify statistics update interval (default 1 sec).\n"
@@ -715,9 +398,6 @@ static void parse_command_line(int argc, char **argv)
715 case 'q': 398 case 'q':
716 opt_queue = atoi(optarg); 399 opt_queue = atoi(optarg);
717 break; 400 break;
718 case 's':
719 opt_shared_packet_buffer = 1;
720 break;
721 case 'p': 401 case 'p':
722 opt_poll = 1; 402 opt_poll = 1;
723 break; 403 break;
@@ -751,75 +431,104 @@ static void parse_command_line(int argc, char **argv)
751 opt_if); 431 opt_if);
752 usage(basename(argv[0])); 432 usage(basename(argv[0]));
753 } 433 }
434
754} 435}
755 436
756static void kick_tx(int fd) 437static void kick_tx(struct xsk_socket_info *xsk)
757{ 438{
758 int ret; 439 int ret;
759 440
760 ret = sendto(fd, NULL, 0, MSG_DONTWAIT, NULL, 0); 441 ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
761 if (ret >= 0 || errno == ENOBUFS || errno == EAGAIN || errno == EBUSY) 442 if (ret >= 0 || errno == ENOBUFS || errno == EAGAIN || errno == EBUSY)
762 return; 443 return;
763 lassert(0); 444 exit_with_error(errno);
764} 445}
765 446
766static inline void complete_tx_l2fwd(struct xdpsock *xsk) 447static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk)
767{ 448{
768 u64 descs[BATCH_SIZE]; 449 u32 idx_cq, idx_fq;
769 unsigned int rcvd; 450 unsigned int rcvd;
770 size_t ndescs; 451 size_t ndescs;
771 452
772 if (!xsk->outstanding_tx) 453 if (!xsk->outstanding_tx)
773 return; 454 return;
774 455
775 kick_tx(xsk->sfd); 456 kick_tx(xsk);
776 ndescs = (xsk->outstanding_tx > BATCH_SIZE) ? BATCH_SIZE : 457 ndescs = (xsk->outstanding_tx > BATCH_SIZE) ? BATCH_SIZE :
777 xsk->outstanding_tx; 458 xsk->outstanding_tx;
778 459
779 /* re-add completed Tx buffers */ 460 /* re-add completed Tx buffers */
780 rcvd = umem_complete_from_kernel(&xsk->umem->cq, descs, ndescs); 461 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, ndescs, &idx_cq);
781 if (rcvd > 0) { 462 if (rcvd > 0) {
782 umem_fill_to_kernel(&xsk->umem->fq, descs, rcvd); 463 unsigned int i;
464 int ret;
465
466 ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);
467 while (ret != rcvd) {
468 if (ret < 0)
469 exit_with_error(-ret);
470 ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd,
471 &idx_fq);
472 }
473 for (i = 0; i < rcvd; i++)
474 *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) =
475 *xsk_ring_cons__comp_addr(&xsk->umem->cq,
476 idx_cq++);
477
478 xsk_ring_prod__submit(&xsk->umem->fq, rcvd);
479 xsk_ring_cons__release(&xsk->umem->cq, rcvd);
783 xsk->outstanding_tx -= rcvd; 480 xsk->outstanding_tx -= rcvd;
784 xsk->tx_npkts += rcvd; 481 xsk->tx_npkts += rcvd;
785 } 482 }
786} 483}
787 484
788static inline void complete_tx_only(struct xdpsock *xsk) 485static inline void complete_tx_only(struct xsk_socket_info *xsk)
789{ 486{
790 u64 descs[BATCH_SIZE];
791 unsigned int rcvd; 487 unsigned int rcvd;
488 u32 idx;
792 489
793 if (!xsk->outstanding_tx) 490 if (!xsk->outstanding_tx)
794 return; 491 return;
795 492
796 kick_tx(xsk->sfd); 493 kick_tx(xsk);
797 494
798 rcvd = umem_complete_from_kernel(&xsk->umem->cq, descs, BATCH_SIZE); 495 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, BATCH_SIZE, &idx);
799 if (rcvd > 0) { 496 if (rcvd > 0) {
497 xsk_ring_cons__release(&xsk->umem->cq, rcvd);
800 xsk->outstanding_tx -= rcvd; 498 xsk->outstanding_tx -= rcvd;
801 xsk->tx_npkts += rcvd; 499 xsk->tx_npkts += rcvd;
802 } 500 }
803} 501}
804 502
805static void rx_drop(struct xdpsock *xsk) 503static void rx_drop(struct xsk_socket_info *xsk)
806{ 504{
807 struct xdp_desc descs[BATCH_SIZE];
808 unsigned int rcvd, i; 505 unsigned int rcvd, i;
506 u32 idx_rx, idx_fq = 0;
507 int ret;
809 508
810 rcvd = xq_deq(&xsk->rx, descs, BATCH_SIZE); 509 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
811 if (!rcvd) 510 if (!rcvd)
812 return; 511 return;
813 512
513 ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);
514 while (ret != rcvd) {
515 if (ret < 0)
516 exit_with_error(-ret);
517 ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);
518 }
519
814 for (i = 0; i < rcvd; i++) { 520 for (i = 0; i < rcvd; i++) {
815 char *pkt = xq_get_data(xsk, descs[i].addr); 521 u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr;
522 u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len;
523 char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr);
816 524
817 hex_dump(pkt, descs[i].len, descs[i].addr); 525 hex_dump(pkt, len, addr);
526 *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = addr;
818 } 527 }
819 528
529 xsk_ring_prod__submit(&xsk->umem->fq, rcvd);
530 xsk_ring_cons__release(&xsk->rx, rcvd);
820 xsk->rx_npkts += rcvd; 531 xsk->rx_npkts += rcvd;
821
822 umem_fill_to_kernel_ex(&xsk->umem->fq, descs, rcvd);
823} 532}
824 533
825static void rx_drop_all(void) 534static void rx_drop_all(void)
@@ -830,7 +539,7 @@ static void rx_drop_all(void)
830 memset(fds, 0, sizeof(fds)); 539 memset(fds, 0, sizeof(fds));
831 540
832 for (i = 0; i < num_socks; i++) { 541 for (i = 0; i < num_socks; i++) {
833 fds[i].fd = xsks[i]->sfd; 542 fds[i].fd = xsk_socket__fd(xsks[i]->xsk);
834 fds[i].events = POLLIN; 543 fds[i].events = POLLIN;
835 timeout = 1000; /* 1sn */ 544 timeout = 1000; /* 1sn */
836 } 545 }
@@ -847,14 +556,14 @@ static void rx_drop_all(void)
847 } 556 }
848} 557}
849 558
850static void tx_only(struct xdpsock *xsk) 559static void tx_only(struct xsk_socket_info *xsk)
851{ 560{
852 int timeout, ret, nfds = 1; 561 int timeout, ret, nfds = 1;
853 struct pollfd fds[nfds + 1]; 562 struct pollfd fds[nfds + 1];
854 unsigned int idx = 0; 563 u32 idx, frame_nb = 0;
855 564
856 memset(fds, 0, sizeof(fds)); 565 memset(fds, 0, sizeof(fds));
857 fds[0].fd = xsk->sfd; 566 fds[0].fd = xsk_socket__fd(xsk->xsk);
858 fds[0].events = POLLOUT; 567 fds[0].events = POLLOUT;
859 timeout = 1000; /* 1sn */ 568 timeout = 1000; /* 1sn */
860 569
@@ -864,50 +573,73 @@ static void tx_only(struct xdpsock *xsk)
864 if (ret <= 0) 573 if (ret <= 0)
865 continue; 574 continue;
866 575
867 if (fds[0].fd != xsk->sfd || 576 if (!(fds[0].revents & POLLOUT))
868 !(fds[0].revents & POLLOUT))
869 continue; 577 continue;
870 } 578 }
871 579
872 if (xq_nb_free(&xsk->tx, BATCH_SIZE) >= BATCH_SIZE) { 580 if (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) ==
873 lassert(xq_enq_tx_only(&xsk->tx, idx, BATCH_SIZE) == 0); 581 BATCH_SIZE) {
582 unsigned int i;
874 583
584 for (i = 0; i < BATCH_SIZE; i++) {
585 xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->addr
586 = (frame_nb + i) <<
587 XSK_UMEM__DEFAULT_FRAME_SHIFT;
588 xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->len =
589 sizeof(pkt_data) - 1;
590 }
591
592 xsk_ring_prod__submit(&xsk->tx, BATCH_SIZE);
875 xsk->outstanding_tx += BATCH_SIZE; 593 xsk->outstanding_tx += BATCH_SIZE;
876 idx += BATCH_SIZE; 594 frame_nb += BATCH_SIZE;
877 idx %= NUM_FRAMES; 595 frame_nb %= NUM_FRAMES;
878 } 596 }
879 597
880 complete_tx_only(xsk); 598 complete_tx_only(xsk);
881 } 599 }
882} 600}
883 601
884static void l2fwd(struct xdpsock *xsk) 602static void l2fwd(struct xsk_socket_info *xsk)
885{ 603{
886 for (;;) { 604 for (;;) {
887 struct xdp_desc descs[BATCH_SIZE];
888 unsigned int rcvd, i; 605 unsigned int rcvd, i;
606 u32 idx_rx, idx_tx = 0;
889 int ret; 607 int ret;
890 608
891 for (;;) { 609 for (;;) {
892 complete_tx_l2fwd(xsk); 610 complete_tx_l2fwd(xsk);
893 611
894 rcvd = xq_deq(&xsk->rx, descs, BATCH_SIZE); 612 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE,
613 &idx_rx);
895 if (rcvd > 0) 614 if (rcvd > 0)
896 break; 615 break;
897 } 616 }
898 617
618 ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx);
619 while (ret != rcvd) {
620 if (ret < 0)
621 exit_with_error(-ret);
622 ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx);
623 }
624
899 for (i = 0; i < rcvd; i++) { 625 for (i = 0; i < rcvd; i++) {
900 char *pkt = xq_get_data(xsk, descs[i].addr); 626 u64 addr = xsk_ring_cons__rx_desc(&xsk->rx,
627 idx_rx)->addr;
628 u32 len = xsk_ring_cons__rx_desc(&xsk->rx,
629 idx_rx++)->len;
630 char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr);
901 631
902 swap_mac_addresses(pkt); 632 swap_mac_addresses(pkt);
903 633
904 hex_dump(pkt, descs[i].len, descs[i].addr); 634 hex_dump(pkt, len, addr);
635 xsk_ring_prod__tx_desc(&xsk->tx, idx_tx)->addr = addr;
636 xsk_ring_prod__tx_desc(&xsk->tx, idx_tx++)->len = len;
905 } 637 }
906 638
907 xsk->rx_npkts += rcvd; 639 xsk_ring_prod__submit(&xsk->tx, rcvd);
640 xsk_ring_cons__release(&xsk->rx, rcvd);
908 641
909 ret = xq_enq(&xsk->tx, descs, rcvd); 642 xsk->rx_npkts += rcvd;
910 lassert(ret == 0);
911 xsk->outstanding_tx += rcvd; 643 xsk->outstanding_tx += rcvd;
912 } 644 }
913} 645}
@@ -915,17 +647,10 @@ static void l2fwd(struct xdpsock *xsk)
915int main(int argc, char **argv) 647int main(int argc, char **argv)
916{ 648{
917 struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY}; 649 struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
918 struct bpf_prog_load_attr prog_load_attr = { 650 struct xsk_umem_info *umem;
919 .prog_type = BPF_PROG_TYPE_XDP,
920 };
921 int prog_fd, qidconf_map, xsks_map;
922 struct bpf_prog_info info = {};
923 __u32 info_len = sizeof(info);
924 struct bpf_object *obj;
925 char xdp_filename[256];
926 struct bpf_map *map;
927 int i, ret, key = 0;
928 pthread_t pt; 651 pthread_t pt;
652 void *bufs;
653 int ret;
929 654
930 parse_command_line(argc, argv); 655 parse_command_line(argc, argv);
931 656
@@ -935,67 +660,22 @@ int main(int argc, char **argv)
935 exit(EXIT_FAILURE); 660 exit(EXIT_FAILURE);
936 } 661 }
937 662
938 snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.o", argv[0]); 663 ret = posix_memalign(&bufs, getpagesize(), /* PAGE_SIZE aligned */
939 prog_load_attr.file = xdp_filename; 664 NUM_FRAMES * XSK_UMEM__DEFAULT_FRAME_SIZE);
940 665 if (ret)
941 if (bpf_prog_load_xattr(&prog_load_attr, &obj, &prog_fd)) 666 exit_with_error(ret);
942 exit(EXIT_FAILURE);
943 if (prog_fd < 0) {
944 fprintf(stderr, "ERROR: no program found: %s\n",
945 strerror(prog_fd));
946 exit(EXIT_FAILURE);
947 }
948
949 map = bpf_object__find_map_by_name(obj, "qidconf_map");
950 qidconf_map = bpf_map__fd(map);
951 if (qidconf_map < 0) {
952 fprintf(stderr, "ERROR: no qidconf map found: %s\n",
953 strerror(qidconf_map));
954 exit(EXIT_FAILURE);
955 }
956
957 map = bpf_object__find_map_by_name(obj, "xsks_map");
958 xsks_map = bpf_map__fd(map);
959 if (xsks_map < 0) {
960 fprintf(stderr, "ERROR: no xsks map found: %s\n",
961 strerror(xsks_map));
962 exit(EXIT_FAILURE);
963 }
964
965 if (bpf_set_link_xdp_fd(opt_ifindex, prog_fd, opt_xdp_flags) < 0) {
966 fprintf(stderr, "ERROR: link set xdp fd failed\n");
967 exit(EXIT_FAILURE);
968 }
969
970 ret = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len);
971 if (ret) {
972 printf("can't get prog info - %s\n", strerror(errno));
973 return 1;
974 }
975 prog_id = info.id;
976 667
977 ret = bpf_map_update_elem(qidconf_map, &key, &opt_queue, 0); 668 /* Create sockets... */
978 if (ret) { 669 umem = xsk_configure_umem(bufs,
979 fprintf(stderr, "ERROR: bpf_map_update_elem qidconf\n"); 670 NUM_FRAMES * XSK_UMEM__DEFAULT_FRAME_SIZE);
980 exit(EXIT_FAILURE); 671 xsks[num_socks++] = xsk_configure_socket(umem);
981 }
982 672
983 /* Create sockets... */ 673 if (opt_bench == BENCH_TXONLY) {
984 xsks[num_socks++] = xsk_configure(NULL); 674 int i;
985
986#if RR_LB
987 for (i = 0; i < MAX_SOCKS - 1; i++)
988 xsks[num_socks++] = xsk_configure(xsks[0]->umem);
989#endif
990 675
991 /* ...and insert them into the map. */ 676 for (i = 0; i < NUM_FRAMES * XSK_UMEM__DEFAULT_FRAME_SIZE;
992 for (i = 0; i < num_socks; i++) { 677 i += XSK_UMEM__DEFAULT_FRAME_SIZE)
993 key = i; 678 (void)gen_eth_frame(umem, i);
994 ret = bpf_map_update_elem(xsks_map, &key, &xsks[i]->sfd, 0);
995 if (ret) {
996 fprintf(stderr, "ERROR: bpf_map_update_elem %d\n", i);
997 exit(EXIT_FAILURE);
998 }
999 } 679 }
1000 680
1001 signal(SIGINT, int_exit); 681 signal(SIGINT, int_exit);
@@ -1005,7 +685,8 @@ int main(int argc, char **argv)
1005 setlocale(LC_ALL, ""); 685 setlocale(LC_ALL, "");
1006 686
1007 ret = pthread_create(&pt, NULL, poller, NULL); 687 ret = pthread_create(&pt, NULL, poller, NULL);
1008 lassert(ret == 0); 688 if (ret)
689 exit_with_error(ret);
1009 690
1010 prev_time = get_nsecs(); 691 prev_time = get_nsecs();
1011 692
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-29 09:53:50 -0400