aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMarkos Chandras <markos.chandras@imgtec.com>2014-04-08 07:47:14 -0400
committerRalf Baechle <ralf@linux-mips.org>2014-05-30 10:10:20 -0400
commitc6610de353da5ca6eee5b8960e838a87a90ead0c (patch)
tree856218411defc225858174e0e2384cffba68c86b
parent8248881835da58fa075299926c2994d12b56895b (diff)
MIPS: net: Add BPF JIT
This adds initial support for BPF-JIT on MIPS Tested on mips32 LE/BE and mips64 BE/n64 using dhcp, ping and various tcpdump filters. Benchmarking: Assuming the remote MIPS target uses 192.168.154.181 as its IP address, and the local host uses 192.168.154.136, the following results can be obtained using the following tcpdump filter (catches no frames) and a simple 'time ping -f -c 1000000' command. [root@(none) ~]# tcpdump -p -n -s 0 -i eth0 net 10.0.0.0/24 -d (000) ldh [12] (001) jeq #0x800 jt 2 jf 8 (002) ld [26] (003) and #0xffffff00 (004) jeq #0xa000000 jt 16 jf 5 (005) ld [30] (006) and #0xffffff00 (007) jeq #0xa000000 jt 16 jf 17 (008) jeq #0x806 jt 10 jf 9 (009) jeq #0x8035 jt 10 jf 17 (010) ld [28] (011) and #0xffffff00 (012) jeq #0xa000000 jt 16 jf 13 (013) ld [38] (014) and #0xffffff00 (015) jeq #0xa000000 jt 16 jf 17 (016) ret #65535 - BPF-JIT Disabled real 1m38.005s user 0m1.510s sys 0m6.710s - BPF-JIT Enabled real 1m35.215s user 0m1.200s sys 0m4.140s [ralf@linux-mips.org: Resolved conflict.] Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>
Diffstat
-rw-r--r--arch/mips/Kbuild1
-rw-r--r--arch/mips/net/Makefile3
-rw-r--r--arch/mips/net/bpf_jit.c1399
-rw-r--r--arch/mips/net/bpf_jit.h44
4 files changed, 1447 insertions, 0 deletions
diff --git a/arch/mips/Kbuild b/arch/mips/Kbuild
index cc39966ca63d..dd295335891a 100644
--- a/arch/mips/Kbuild
+++ b/arch/mips/Kbuild
@@ -16,6 +16,7 @@ obj- := $(platform-)
16 16
17obj-y += kernel/ 17obj-y += kernel/
18obj-y += mm/ 18obj-y += mm/
19obj-y += net/
19 20
20ifdef CONFIG_KVM 21ifdef CONFIG_KVM
21obj-y += kvm/ 22obj-y += kvm/
diff --git a/arch/mips/net/Makefile b/arch/mips/net/Makefile
new file mode 100644
index 000000000000..ae74b3a91f5c
--- /dev/null
+++ b/arch/mips/net/Makefile
@@ -0,0 +1,3 @@
1# MIPS networking code
2
3obj-$(CONFIG_BPF_JIT) += bpf_jit.o
diff --git a/arch/mips/net/bpf_jit.c b/arch/mips/net/bpf_jit.c
new file mode 100644
index 000000000000..98e9d2395598
--- /dev/null
+++ b/arch/mips/net/bpf_jit.c
@@ -0,0 +1,1399 @@
1/*
2 * Just-In-Time compiler for BPF filters on MIPS
3 *
4 * Copyright (c) 2014 Imagination Technologies Ltd.
5 * Author: Markos Chandras <markos.chandras@imgtec.com>
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; version 2 of the License.
10 */
11
12#include <linux/bitops.h>
13#include <linux/compiler.h>
14#include <linux/errno.h>
15#include <linux/filter.h>
16#include <linux/if_vlan.h>
17#include <linux/kconfig.h>
18#include <linux/moduleloader.h>
19#include <linux/netdevice.h>
20#include <linux/string.h>
21#include <linux/slab.h>
22#include <linux/types.h>
23#include <asm/bitops.h>
24#include <asm/cacheflush.h>
25#include <asm/cpu-features.h>
26#include <asm/uasm.h>
27
28#include "bpf_jit.h"
29
30/* ABI
31 *
32 * s0 1st scratch register
33 * s1 2nd scratch register
34 * s2 offset register
35 * s3 BPF register A
36 * s4 BPF register X
37 * s5 *skb
38 * s6 *scratch memory
39 *
40 * On entry (*bpf_func)(*skb, *filter)
41 * a0 = MIPS_R_A0 = skb;
42 * a1 = MIPS_R_A1 = filter;
43 *
44 * Stack
45 * ...
46 * M[15]
47 * M[14]
48 * M[13]
49 * ...
50 * M[0] <-- r_M
51 * saved reg k-1
52 * saved reg k-2
53 * ...
54 * saved reg 0 <-- r_sp
55 * <no argument area>
56 *
57 * Packet layout
58 *
59 * <--------------------- len ------------------------>
60 * <--skb-len(r_skb_hl)-->< ----- skb->data_len ------>
61 * ----------------------------------------------------
62 * | skb->data |
63 * ----------------------------------------------------
64 */
65
66#define RSIZE (sizeof(unsigned long))
67#define ptr typeof(unsigned long)
68
69/* ABI specific return values */
70#ifdef CONFIG_32BIT /* O32 */
71#ifdef CONFIG_CPU_LITTLE_ENDIAN
72#define r_err MIPS_R_V1
73#define r_val MIPS_R_V0
74#else /* CONFIG_CPU_LITTLE_ENDIAN */
75#define r_err MIPS_R_V0
76#define r_val MIPS_R_V1
77#endif
78#else /* N64 */
79#define r_err MIPS_R_V0
80#define r_val MIPS_R_V0
81#endif
82
83#define r_ret MIPS_R_V0
84
85/*
86 * Use 2 scratch registers to avoid pipeline interlocks.
87 * There is no overhead during epilogue and prologue since
88 * any of the $s0-$s6 registers will only be preserved if
89 * they are going to actually be used.
90 */
91#define r_s0 MIPS_R_S0 /* scratch reg 1 */
92#define r_s1 MIPS_R_S1 /* scratch reg 2 */
93#define r_off MIPS_R_S2
94#define r_A MIPS_R_S3
95#define r_X MIPS_R_S4
96#define r_skb MIPS_R_S5
97#define r_M MIPS_R_S6
98#define r_tmp_imm MIPS_R_T6 /* No need to preserve this */
99#define r_tmp MIPS_R_T7 /* No need to preserve this */
100#define r_zero MIPS_R_ZERO
101#define r_sp MIPS_R_SP
102#define r_ra MIPS_R_RA
103
104#define SCRATCH_OFF(k) (4 * (k))
105
106/* JIT flags */
107#define SEEN_CALL (1 << BPF_MEMWORDS)
108#define SEEN_SREG_SFT (BPF_MEMWORDS + 1)
109#define SEEN_SREG_BASE (1 << SEEN_SREG_SFT)
110#define SEEN_SREG(x) (SEEN_SREG_BASE << (x))
111#define SEEN_S0 SEEN_SREG(0)
112#define SEEN_S1 SEEN_SREG(1)
113#define SEEN_OFF SEEN_SREG(2)
114#define SEEN_A SEEN_SREG(3)
115#define SEEN_X SEEN_SREG(4)
116#define SEEN_SKB SEEN_SREG(5)
117#define SEEN_MEM SEEN_SREG(6)
118
119/* Arguments used by JIT */
120#define ARGS_USED_BY_JIT 2 /* only applicable to 64-bit */
121
122#define FLAG_NEED_X_RESET (1 << 0)
123
124#define SBIT(x) (1 << (x)) /* Signed version of BIT() */
125
126/**
127 * struct jit_ctx - JIT context
128 * @skf: The sk_filter
129 * @prologue_bytes: Number of bytes for prologue
130 * @idx: Instruction index
131 * @flags: JIT flags
132 * @offsets: Instruction offsets
133 * @target: Memory location for the compiled filter
134 */
135struct jit_ctx {
136 const struct sk_filter *skf;
137 unsigned int prologue_bytes;
138 u32 idx;
139 u32 flags;
140 u32 *offsets;
141 u32 *target;
142};
143
144
145static inline int optimize_div(u32 *k)
146{
147 /* power of 2 divides can be implemented with right shift */
148 if (!(*k & (*k-1))) {
149 *k = ilog2(*k);
150 return 1;
151 }
152
153 return 0;
154}
155
156/* Simply emit the instruction if the JIT memory space has been allocated */
157#define emit_instr(ctx, func, ...) \
158do { \
159 if ((ctx)->target != NULL) { \
160 u32 *p = &(ctx)->target[ctx->idx]; \
161 uasm_i_##func(&p, ##__VA_ARGS__); \
162 } \
163 (ctx)->idx++; \
164} while (0)
165
166/* Determine if immediate is within the 16-bit signed range */
167static inline bool is_range16(s32 imm)
168{
169 if (imm >= SBIT(15) || imm < -SBIT(15))
170 return true;
171 return false;
172}
173
174static inline void emit_addu(unsigned int dst, unsigned int src1,
175 unsigned int src2, struct jit_ctx *ctx)
176{
177 emit_instr(ctx, addu, dst, src1, src2);
178}
179
180static inline void emit_nop(struct jit_ctx *ctx)
181{
182 emit_instr(ctx, nop);
183}
184
185/* Load a u32 immediate to a register */
186static inline void emit_load_imm(unsigned int dst, u32 imm, struct jit_ctx *ctx)
187{
188 if (ctx->target != NULL) {
189 /* addiu can only handle s16 */
190 if (is_range16(imm)) {
191 u32 *p = &ctx->target[ctx->idx];
192 uasm_i_lui(&p, r_tmp_imm, (s32)imm >> 16);
193 p = &ctx->target[ctx->idx + 1];
194 uasm_i_ori(&p, dst, r_tmp_imm, imm & 0xffff);
195 } else {
196 u32 *p = &ctx->target[ctx->idx];
197 uasm_i_addiu(&p, dst, r_zero, imm);
198 }
199 }
200 ctx->idx++;
201
202 if (is_range16(imm))
203 ctx->idx++;
204}
205
206static inline void emit_or(unsigned int dst, unsigned int src1,
207 unsigned int src2, struct jit_ctx *ctx)
208{
209 emit_instr(ctx, or, dst, src1, src2);
210}
211
212static inline void emit_ori(unsigned int dst, unsigned src, u32 imm,
213 struct jit_ctx *ctx)
214{
215 if (imm >= BIT(16)) {
216 emit_load_imm(r_tmp, imm, ctx);
217 emit_or(dst, src, r_tmp, ctx);
218 } else {
219 emit_instr(ctx, ori, dst, src, imm);
220 }
221}
222
223
224static inline void emit_daddu(unsigned int dst, unsigned int src1,
225 unsigned int src2, struct jit_ctx *ctx)
226{
227 emit_instr(ctx, daddu, dst, src1, src2);
228}
229
230static inline void emit_daddiu(unsigned int dst, unsigned int src,
231 int imm, struct jit_ctx *ctx)
232{
233 /*
234 * Only used for stack, so the imm is relatively small
235 * and it fits in 15-bits
236 */
237 emit_instr(ctx, daddiu, dst, src, imm);
238}
239
240static inline void emit_addiu(unsigned int dst, unsigned int src,
241 u32 imm, struct jit_ctx *ctx)
242{
243 if (is_range16(imm)) {
244 emit_load_imm(r_tmp, imm, ctx);
245 emit_addu(dst, r_tmp, src, ctx);
246 } else {
247 emit_instr(ctx, addiu, dst, src, imm);
248 }
249}
250
251static inline void emit_and(unsigned int dst, unsigned int src1,
252 unsigned int src2, struct jit_ctx *ctx)
253{
254 emit_instr(ctx, and, dst, src1, src2);
255}
256
257static inline void emit_andi(unsigned int dst, unsigned int src,
258 u32 imm, struct jit_ctx *ctx)
259{
260 /* If imm does not fit in u16 then load it to register */
261 if (imm >= BIT(16)) {
262 emit_load_imm(r_tmp, imm, ctx);
263 emit_and(dst, src, r_tmp, ctx);
264 } else {
265 emit_instr(ctx, andi, dst, src, imm);
266 }
267}
268
269static inline void emit_xor(unsigned int dst, unsigned int src1,
270 unsigned int src2, struct jit_ctx *ctx)
271{
272 emit_instr(ctx, xor, dst, src1, src2);
273}
274
275static inline void emit_xori(ptr dst, ptr src, u32 imm, struct jit_ctx *ctx)
276{
277 /* If imm does not fit in u16 then load it to register */
278 if (imm >= BIT(16)) {
279 emit_load_imm(r_tmp, imm, ctx);
280 emit_xor(dst, src, r_tmp, ctx);
281 } else {
282 emit_instr(ctx, xori, dst, src, imm);
283 }
284}
285
286static inline void emit_stack_offset(int offset, struct jit_ctx *ctx)
287{
288 if (config_enabled(CONFIG_64BIT))
289 emit_instr(ctx, daddiu, r_sp, r_sp, offset);
290 else
291 emit_instr(ctx, addiu, r_sp, r_sp, offset);
292
293}
294
295static inline void emit_subu(unsigned int dst, unsigned int src1,
296 unsigned int src2, struct jit_ctx *ctx)
297{
298 emit_instr(ctx, subu, dst, src1, src2);
299}
300
301static inline void emit_neg(unsigned int reg, struct jit_ctx *ctx)
302{
303 emit_subu(reg, r_zero, reg, ctx);
304}
305
306static inline void emit_sllv(unsigned int dst, unsigned int src,
307 unsigned int sa, struct jit_ctx *ctx)
308{
309 emit_instr(ctx, sllv, dst, src, sa);
310}
311
312static inline void emit_sll(unsigned int dst, unsigned int src,
313 unsigned int sa, struct jit_ctx *ctx)
314{
315 /* sa is 5-bits long */
316 BUG_ON(sa >= BIT(5));
317 emit_instr(ctx, sll, dst, src, sa);
318}
319
320static inline void emit_srlv(unsigned int dst, unsigned int src,
321 unsigned int sa, struct jit_ctx *ctx)
322{
323 emit_instr(ctx, srlv, dst, src, sa);
324}
325
326static inline void emit_srl(unsigned int dst, unsigned int src,
327 unsigned int sa, struct jit_ctx *ctx)
328{
329 /* sa is 5-bits long */
330 BUG_ON(sa >= BIT(5));
331 emit_instr(ctx, srl, dst, src, sa);
332}
333
334static inline void emit_sltu(unsigned int dst, unsigned int src1,
335 unsigned int src2, struct jit_ctx *ctx)
336{
337 emit_instr(ctx, sltu, dst, src1, src2);
338}
339
340static inline void emit_sltiu(unsigned dst, unsigned int src,
341 unsigned int imm, struct jit_ctx *ctx)
342{
343 /* 16 bit immediate */
344 if (is_range16((s32)imm)) {
345 emit_load_imm(r_tmp, imm, ctx);
346 emit_sltu(dst, src, r_tmp, ctx);
347 } else {
348 emit_instr(ctx, sltiu, dst, src, imm);
349 }
350
351}
352
353/* Store register on the stack */
354static inline void emit_store_stack_reg(ptr reg, ptr base,
355 unsigned int offset,
356 struct jit_ctx *ctx)
357{
358 if (config_enabled(CONFIG_64BIT))
359 emit_instr(ctx, sd, reg, offset, base);
360 else
361 emit_instr(ctx, sw, reg, offset, base);
362}
363
364static inline void emit_store(ptr reg, ptr base, unsigned int offset,
365 struct jit_ctx *ctx)
366{
367 emit_instr(ctx, sw, reg, offset, base);
368}
369
370static inline void emit_load_stack_reg(ptr reg, ptr base,
371 unsigned int offset,
372 struct jit_ctx *ctx)
373{
374 if (config_enabled(CONFIG_64BIT))
375 emit_instr(ctx, ld, reg, offset, base);
376 else
377 emit_instr(ctx, lw, reg, offset, base);
378}
379
380static inline void emit_load(unsigned int reg, unsigned int base,
381 unsigned int offset, struct jit_ctx *ctx)
382{
383 emit_instr(ctx, lw, reg, offset, base);
384}
385
386static inline void emit_load_byte(unsigned int reg, unsigned int base,
387 unsigned int offset, struct jit_ctx *ctx)
388{
389 emit_instr(ctx, lb, reg, offset, base);
390}
391
392static inline void emit_half_load(unsigned int reg, unsigned int base,
393 unsigned int offset, struct jit_ctx *ctx)
394{
395 emit_instr(ctx, lh, reg, offset, base);
396}
397
398static inline void emit_mul(unsigned int dst, unsigned int src1,
399 unsigned int src2, struct jit_ctx *ctx)
400{
401 emit_instr(ctx, mul, dst, src1, src2);
402}
403
404static inline void emit_div(unsigned int dst, unsigned int src,
405 struct jit_ctx *ctx)
406{
407 if (ctx->target != NULL) {
408 u32 *p = &ctx->target[ctx->idx];
409 uasm_i_divu(&p, dst, src);
410 p = &ctx->target[ctx->idx + 1];
411 uasm_i_mfhi(&p, dst);
412 }
413 ctx->idx += 2; /* 2 insts */
414}
415
416static inline void emit_mod(unsigned int dst, unsigned int src,
417 struct jit_ctx *ctx)
418{
419 if (ctx->target != NULL) {
420 u32 *p = &ctx->target[ctx->idx];
421 uasm_i_divu(&p, dst, src);
422 p = &ctx->target[ctx->idx + 1];
423 uasm_i_mflo(&p, dst);
424 }
425 ctx->idx += 2; /* 2 insts */
426}
427
428static inline void emit_dsll(unsigned int dst, unsigned int src,
429 unsigned int sa, struct jit_ctx *ctx)
430{
431 emit_instr(ctx, dsll, dst, src, sa);
432}
433
434static inline void emit_dsrl32(unsigned int dst, unsigned int src,
435 unsigned int sa, struct jit_ctx *ctx)
436{
437 emit_instr(ctx, dsrl32, dst, src, sa);
438}
439
440static inline void emit_wsbh(unsigned int dst, unsigned int src,
441 struct jit_ctx *ctx)
442{
443 emit_instr(ctx, wsbh, dst, src);
444}
445
446/* load a function pointer to register */
447static inline void emit_load_func(unsigned int reg, ptr imm,
448 struct jit_ctx *ctx)
449{
450 if (config_enabled(CONFIG_64BIT)) {
451 /* At this point imm is always 64-bit */
452 emit_load_imm(r_tmp, (u64)imm >> 32, ctx);
453 emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */
454 emit_ori(r_tmp, r_tmp_imm, (imm >> 16) & 0xffff, ctx);
455 emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */
456 emit_ori(reg, r_tmp_imm, imm & 0xffff, ctx);
457 } else {
458 emit_load_imm(reg, imm, ctx);
459 }
460}
461
462/* Move to real MIPS register */
463static inline void emit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx)
464{
465 if (config_enabled(CONFIG_64BIT))
466 emit_daddu(dst, src, r_zero, ctx);
467 else
468 emit_addu(dst, src, r_zero, ctx);
469}
470
471/* Move to JIT (32-bit) register */
472static inline void emit_jit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx)
473{
474 emit_addu(dst, src, r_zero, ctx);
475}
476
477/* Compute the immediate value for PC-relative branches. */
478static inline u32 b_imm(unsigned int tgt, struct jit_ctx *ctx)
479{
480 if (ctx->target == NULL)
481 return 0;
482
483 /*
484 * We want a pc-relative branch. We only do forward branches
485 * so tgt is always after pc. tgt is the instruction offset
486 * we want to jump to.
487
488 * Branch on MIPS:
489 * I: target_offset <- sign_extend(offset)
490 * I+1: PC += target_offset (delay slot)
491 *
492 * ctx->idx currently points to the branch instruction
493 * but the offset is added to the delay slot so we need
494 * to subtract 4.
495 */
496 return ctx->offsets[tgt] -
497 (ctx->idx * 4 - ctx->prologue_bytes) - 4;
498}
499
500static inline void emit_bcond(int cond, unsigned int reg1, unsigned int reg2,
501 unsigned int imm, struct jit_ctx *ctx)
502{
503 if (ctx->target != NULL) {
504 u32 *p = &ctx->target[ctx->idx];
505
506 switch (cond) {
507 case MIPS_COND_EQ:
508 uasm_i_beq(&p, reg1, reg2, imm);
509 break;
510 case MIPS_COND_NE:
511 uasm_i_bne(&p, reg1, reg2, imm);
512 break;
513 case MIPS_COND_ALL:
514 uasm_i_b(&p, imm);
515 break;
516 default:
517 pr_warn("%s: Unhandled branch conditional: %d\n",
518 __func__, cond);
519 }
520 }
521 ctx->idx++;
522}
523
524static inline void emit_b(unsigned int imm, struct jit_ctx *ctx)
525{
526 emit_bcond(MIPS_COND_ALL, r_zero, r_zero, imm, ctx);
527}
528
529static inline void emit_jalr(unsigned int link, unsigned int reg,
530 struct jit_ctx *ctx)
531{
532 emit_instr(ctx, jalr, link, reg);
533}
534
535static inline void emit_jr(unsigned int reg, struct jit_ctx *ctx)
536{
537 emit_instr(ctx, jr, reg);
538}
539
540static inline u16 align_sp(unsigned int num)
541{
542 /* Double word alignment for 32-bit, quadword for 64-bit */
543 unsigned int align = config_enabled(CONFIG_64BIT) ? 16 : 8;
544 num = (num + (align - 1)) & -align;
545 return num;
546}
547
548static inline void update_on_xread(struct jit_ctx *ctx)
549{
550 if (!(ctx->flags & SEEN_X))
551 ctx->flags |= FLAG_NEED_X_RESET;
552
553 ctx->flags |= SEEN_X;
554}
555
556static bool is_load_to_a(u16 inst)
557{
558 switch (inst) {
559 case BPF_S_LD_W_LEN:
560 case BPF_S_LD_W_ABS:
561 case BPF_S_LD_H_ABS:
562 case BPF_S_LD_B_ABS:
563 case BPF_S_ANC_CPU:
564 case BPF_S_ANC_IFINDEX:
565 case BPF_S_ANC_MARK:
566 case BPF_S_ANC_PROTOCOL:
567 case BPF_S_ANC_RXHASH:
568 case BPF_S_ANC_VLAN_TAG:
569 case BPF_S_ANC_VLAN_TAG_PRESENT:
570 case BPF_S_ANC_QUEUE:
571 return true;
572 default:
573 return false;
574 }
575}
576
577static void save_bpf_jit_regs(struct jit_ctx *ctx, unsigned offset)
578{
579 int i = 0, real_off = 0;
580 u32 sflags, tmp_flags;
581
582 /* Adjust the stack pointer */
583 emit_stack_offset(-align_sp(offset), ctx);
584
585 if (ctx->flags & SEEN_CALL) {
586 /* Argument save area */
587 if (config_enabled(CONFIG_64BIT))
588 /* Bottom of current frame */
589 real_off = align_sp(offset) - RSIZE;
590 else
591 /* Top of previous frame */
592 real_off = align_sp(offset) + RSIZE;
593 emit_store_stack_reg(MIPS_R_A0, r_sp, real_off, ctx);
594 emit_store_stack_reg(MIPS_R_A1, r_sp, real_off + RSIZE, ctx);
595
596 real_off = 0;
597 }
598
599 tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT;
600 /* sflags is essentially a bitmap */
601 while (tmp_flags) {
602 if ((sflags >> i) & 0x1) {
603 emit_store_stack_reg(MIPS_R_S0 + i, r_sp, real_off,
604 ctx);
605 real_off += RSIZE;
606 }
607 i++;
608 tmp_flags >>= 1;
609 }
610
611 /* save return address */
612 if (ctx->flags & SEEN_CALL) {
613 emit_store_stack_reg(r_ra, r_sp, real_off, ctx);
614 real_off += RSIZE;
615 }
616
617 /* Setup r_M leaving the alignment gap if necessary */
618 if (ctx->flags & SEEN_MEM) {
619 if (real_off % (RSIZE * 2))
620 real_off += RSIZE;
621 emit_addiu(r_M, r_sp, real_off, ctx);
622 }
623}
624
625static void restore_bpf_jit_regs(struct jit_ctx *ctx,
626 unsigned int offset)
627{
628 int i, real_off = 0;
629 u32 sflags, tmp_flags;
630
631 if (ctx->flags & SEEN_CALL) {
632 if (config_enabled(CONFIG_64BIT))
633 /* Bottom of current frame */
634 real_off = align_sp(offset) - RSIZE;
635 else
636 /* Top of previous frame */
637 real_off = align_sp(offset) + RSIZE;
638 emit_load_stack_reg(MIPS_R_A0, r_sp, real_off, ctx);
639 emit_load_stack_reg(MIPS_R_A1, r_sp, real_off + RSIZE, ctx);
640
641 real_off = 0;
642 }
643
644 tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT;
645 /* sflags is a bitmap */
646 i = 0;
647 while (tmp_flags) {
648 if ((sflags >> i) & 0x1) {
649 emit_load_stack_reg(MIPS_R_S0 + i, r_sp, real_off,
650 ctx);
651 real_off += RSIZE;
652 }
653 i++;
654 tmp_flags >>= 1;
655 }
656
657 /* restore return address */
658 if (ctx->flags & SEEN_CALL)
659 emit_load_stack_reg(r_ra, r_sp, real_off, ctx);
660
661 /* Restore the sp and discard the scrach memory */
662 emit_stack_offset(align_sp(offset), ctx);
663}
664
665static unsigned int get_stack_depth(struct jit_ctx *ctx)
666{
667 int sp_off = 0;
668
669
670 /* How may s* regs do we need to preserved? */
671 sp_off += hweight32(ctx->flags >> SEEN_SREG_SFT) * RSIZE;
672
673 if (ctx->flags & SEEN_MEM)
674 sp_off += 4 * BPF_MEMWORDS; /* BPF_MEMWORDS are 32-bit */
675
676 if (ctx->flags & SEEN_CALL)
677 /*
678 * The JIT code make calls to external functions using 2
679 * arguments. Therefore, for o32 we don't need to allocate
680 * space because we don't care if the argumetns are lost
681 * across calls. We do need however to preserve incoming
682 * arguments but the space is already allocated for us by
683 * the caller. On the other hand, for n64, we need to allocate
684 * this space ourselves. We need to preserve $ra as well.
685 */
686 sp_off += config_enabled(CONFIG_64BIT) ?
687 (ARGS_USED_BY_JIT + 1) * RSIZE : RSIZE;
688
689 /*
690 * Subtract the bytes for the last registers since we only care about
691 * the location on the stack pointer.
692 */
693 return sp_off - RSIZE;
694}
695
696static void build_prologue(struct jit_ctx *ctx)
697{
698 u16 first_inst = ctx->skf->insns[0].code;
699 int sp_off;
700
701 /* Calculate the total offset for the stack pointer */
702 sp_off = get_stack_depth(ctx);
703 save_bpf_jit_regs(ctx, sp_off);
704
705 if (ctx->flags & SEEN_SKB)
706 emit_reg_move(r_skb, MIPS_R_A0, ctx);
707
708 if (ctx->flags & FLAG_NEED_X_RESET)
709 emit_jit_reg_move(r_X, r_zero, ctx);
710
711 /* Do not leak kernel data to userspace */
712 if ((first_inst != BPF_S_RET_K) && !(is_load_to_a(first_inst)))
713 emit_jit_reg_move(r_A, r_zero, ctx);
714}
715
716static void build_epilogue(struct jit_ctx *ctx)
717{
718 unsigned int sp_off;
719
720 /* Calculate the total offset for the stack pointer */
721
722 sp_off = get_stack_depth(ctx);
723 restore_bpf_jit_regs(ctx, sp_off);
724
725 /* Return */
726 emit_jr(r_ra, ctx);
727 emit_nop(ctx);
728}
729
730static u64 jit_get_skb_b(struct sk_buff *skb, unsigned offset)
731{
732 u8 ret;
733 int err;
734
735 err = skb_copy_bits(skb, offset, &ret, 1);
736
737 return (u64)err << 32 | ret;
738}
739
740static u64 jit_get_skb_h(struct sk_buff *skb, unsigned offset)
741{
742 u16 ret;
743 int err;
744
745 err = skb_copy_bits(skb, offset, &ret, 2);
746
747 return (u64)err << 32 | ntohs(ret);
748}
749
750static u64 jit_get_skb_w(struct sk_buff *skb, unsigned offset)
751{
752 u32 ret;
753 int err;
754
755 err = skb_copy_bits(skb, offset, &ret, 4);
756
757 return (u64)err << 32 | ntohl(ret);
758}
759
760#define PKT_TYPE_MAX 7
761static int pkt_type_offset(void)
762{
763 struct sk_buff skb_probe = {
764 .pkt_type = ~0,
765 };
766 char *ct = (char *)&skb_probe;
767 unsigned int off;
768
769 for (off = 0; off < sizeof(struct sk_buff); off++) {
770 if (ct[off] == PKT_TYPE_MAX)
771 return off;
772 }
773 pr_err_once("Please fix pkt_type_offset(), as pkt_type couldn't be found\n");
774 return -1;
775}
776
777static int build_body(struct jit_ctx *ctx)
778{
779 void *load_func[] = {jit_get_skb_b, jit_get_skb_h, jit_get_skb_w};
780 const struct sk_filter *prog = ctx->skf;
781 const struct sock_filter *inst;
782 unsigned int i, off, load_order, condt;
783 u32 k, b_off __maybe_unused;
784
785 for (i = 0; i < prog->len; i++) {
786 inst = &(prog->insns[i]);
787 pr_debug("%s: code->0x%02x, jt->0x%x, jf->0x%x, k->0x%x\n",
788 __func__, inst->code, inst->jt, inst->jf, inst->k);
789 k = inst->k;
790
791 if (ctx->target == NULL)
792 ctx->offsets[i] = ctx->idx * 4;
793
794 switch (inst->code) {
795 case BPF_S_LD_IMM:
796 /* A <- k ==> li r_A, k */
797 ctx->flags |= SEEN_A;
798 emit_load_imm(r_A, k, ctx);
799 break;
800 case BPF_S_LD_W_LEN:
801 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4);
802 /* A <- len ==> lw r_A, offset(skb) */
803 ctx->flags |= SEEN_SKB | SEEN_A;
804 off = offsetof(struct sk_buff, len);
805 emit_load(r_A, r_skb, off, ctx);
806 break;
807 case BPF_S_LD_MEM:
808 /* A <- M[k] ==> lw r_A, offset(M) */
809 ctx->flags |= SEEN_MEM | SEEN_A;
810 emit_load(r_A, r_M, SCRATCH_OFF(k), ctx);
811 break;
812 case BPF_S_LD_W_ABS:
813 /* A <- P[k:4] */
814 load_order = 2;
815 goto load;
816 case BPF_S_LD_H_ABS:
817 /* A <- P[k:2] */
818 load_order = 1;
819 goto load;
820 case BPF_S_LD_B_ABS:
821 /* A <- P[k:1] */
822 load_order = 0;
823load:
824 emit_load_imm(r_off, k, ctx);
825load_common:
826 ctx->flags |= SEEN_CALL | SEEN_OFF | SEEN_S0 |
827 SEEN_SKB | SEEN_A;
828
829 emit_load_func(r_s0, (ptr)load_func[load_order],
830 ctx);
831 emit_reg_move(MIPS_R_A0, r_skb, ctx);
832 emit_jalr(MIPS_R_RA, r_s0, ctx);
833 /* Load second argument to delay slot */
834 emit_reg_move(MIPS_R_A1, r_off, ctx);
835 /* Check the error value */
836 if (config_enabled(CONFIG_64BIT)) {
837 /* Get error code from the top 32-bits */
838 emit_dsrl32(r_s0, r_val, 0, ctx);
839 /* Branch to 3 instructions ahead */
840 emit_bcond(MIPS_COND_NE, r_s0, r_zero, 3 << 2,
841 ctx);
842 } else {
843 /* Branch to 3 instructions ahead */
844 emit_bcond(MIPS_COND_NE, r_err, r_zero, 3 << 2,
845 ctx);
846 }
847 emit_nop(ctx);
848 /* We are good */
849 emit_b(b_imm(i + 1, ctx), ctx);
850 emit_jit_reg_move(r_A, r_val, ctx);
851 /* Return with error */
852 emit_b(b_imm(prog->len, ctx), ctx);
853 emit_reg_move(r_ret, r_zero, ctx);
854 break;
855 case BPF_S_LD_W_IND:
856 /* A <- P[X + k:4] */
857 load_order = 2;
858 goto load_ind;
859 case BPF_S_LD_H_IND:
860 /* A <- P[X + k:2] */
861 load_order = 1;
862 goto load_ind;
863 case BPF_S_LD_B_IND:
864 /* A <- P[X + k:1] */
865 load_order = 0;
866load_ind:
867 update_on_xread(ctx);
868 ctx->flags |= SEEN_OFF | SEEN_X;
869 emit_addiu(r_off, r_X, k, ctx);
870 goto load_common;
871 case BPF_S_LDX_IMM:
872 /* X <- k */
873 ctx->flags |= SEEN_X;
874 emit_load_imm(r_X, k, ctx);
875 break;
876 case BPF_S_LDX_MEM:
877 /* X <- M[k] */
878 ctx->flags |= SEEN_X | SEEN_MEM;
879 emit_load(r_X, r_M, SCRATCH_OFF(k), ctx);
880 break;
881 case BPF_S_LDX_W_LEN:
882 /* X <- len */
883 ctx->flags |= SEEN_X | SEEN_SKB;
884 off = offsetof(struct sk_buff, len);
885 emit_load(r_X, r_skb, off, ctx);
886 break;
887 case BPF_S_LDX_B_MSH:
888 /* X <- 4 * (P[k:1] & 0xf) */
889 ctx->flags |= SEEN_X | SEEN_CALL | SEEN_S0 | SEEN_SKB;
890 /* Load offset to a1 */
891 emit_load_func(r_s0, (ptr)jit_get_skb_b, ctx);
892 /*
893 * This may emit two instructions so it may not fit
894 * in the delay slot. So use a0 in the delay slot.
895 */
896 emit_load_imm(MIPS_R_A1, k, ctx);
897 emit_jalr(MIPS_R_RA, r_s0, ctx);
898 emit_reg_move(MIPS_R_A0, r_skb, ctx); /* delay slot */
899 /* Check the error value */
900 if (config_enabled(CONFIG_64BIT)) {
901 /* Top 32-bits of $v0 on 64-bit */
902 emit_dsrl32(r_s0, r_val, 0, ctx);
903 emit_bcond(MIPS_COND_NE, r_s0, r_zero,
904 3 << 2, ctx);
905 } else {
906 emit_bcond(MIPS_COND_NE, r_err, r_zero,
907 3 << 2, ctx);
908 }
909 /* No need for delay slot */
910 /* We are good */
911 /* X <- P[1:K] & 0xf */
912 emit_andi(r_X, r_val, 0xf, ctx);
913 /* X << 2 */
914 emit_b(b_imm(i + 1, ctx), ctx);
915 emit_sll(r_X, r_X, 2, ctx); /* delay slot */
916 /* Return with error */
917 emit_b(b_imm(prog->len, ctx), ctx);
918 emit_load_imm(r_ret, 0, ctx); /* delay slot */
919 break;
920 case BPF_S_ST:
921 /* M[k] <- A */
922 ctx->flags |= SEEN_MEM | SEEN_A;
923 emit_store(r_A, r_M, SCRATCH_OFF(k), ctx);
924 break;
925 case BPF_S_STX:
926 /* M[k] <- X */
927 ctx->flags |= SEEN_MEM | SEEN_X;
928 emit_store(r_X, r_M, SCRATCH_OFF(k), ctx);
929 break;
930 case BPF_S_ALU_ADD_K:
931 /* A += K */
932 ctx->flags |= SEEN_A;
933 emit_addiu(r_A, r_A, k, ctx);
934 break;
935 case BPF_S_ALU_ADD_X:
936 /* A += X */
937 ctx->flags |= SEEN_A | SEEN_X;
938 emit_addu(r_A, r_A, r_X, ctx);
939 break;
940 case BPF_S_ALU_SUB_K:
941 /* A -= K */
942 ctx->flags |= SEEN_A;
943 emit_addiu(r_A, r_A, -k, ctx);
944 break;
945 case BPF_S_ALU_SUB_X:
946 /* A -= X */
947 ctx->flags |= SEEN_A | SEEN_X;
948 emit_subu(r_A, r_A, r_X, ctx);
949 break;
950 case BPF_S_ALU_MUL_K:
951 /* A *= K */
952 /* Load K to scratch register before MUL */
953 ctx->flags |= SEEN_A | SEEN_S0;
954 emit_load_imm(r_s0, k, ctx);
955 emit_mul(r_A, r_A, r_s0, ctx);
956 break;
957 case BPF_S_ALU_MUL_X:
958 /* A *= X */
959 update_on_xread(ctx);
960 ctx->flags |= SEEN_A | SEEN_X;
961 emit_mul(r_A, r_A, r_X, ctx);
962 break;
963 case BPF_S_ALU_DIV_K:
964 /* A /= k */
965 if (k == 1)
966 break;
967 if (optimize_div(&k)) {
968 ctx->flags |= SEEN_A;
969 emit_srl(r_A, r_A, k, ctx);
970 break;
971 }
972 ctx->flags |= SEEN_A | SEEN_S0;
973 emit_load_imm(r_s0, k, ctx);
974 emit_div(r_A, r_s0, ctx);
975 break;
976 case BPF_S_ALU_MOD_K:
977 /* A %= k */
978 if (k == 1 || optimize_div(&k)) {
979 ctx->flags |= SEEN_A;
980 emit_jit_reg_move(r_A, r_zero, ctx);
981 } else {
982 ctx->flags |= SEEN_A | SEEN_S0;
983 emit_load_imm(r_s0, k, ctx);
984 emit_mod(r_A, r_s0, ctx);
985 }
986 break;
987 case BPF_S_ALU_DIV_X:
988 /* A /= X */
989 update_on_xread(ctx);
990 ctx->flags |= SEEN_X | SEEN_A;
991 /* Check if r_X is zero */
992 emit_bcond(MIPS_COND_EQ, r_X, r_zero,
993 b_imm(prog->len, ctx), ctx);
994 emit_load_imm(r_val, 0, ctx); /* delay slot */
995 emit_div(r_A, r_X, ctx);
996 break;
997 case BPF_S_ALU_MOD_X:
998 /* A %= X */
999 update_on_xread(ctx);
1000 ctx->flags |= SEEN_X | SEEN_A;
1001 /* Check if r_X is zero */
1002 emit_bcond(MIPS_COND_EQ, r_X, r_zero,
1003 b_imm(prog->len, ctx), ctx);
1004 emit_load_imm(r_val, 0, ctx); /* delay slot */
1005 emit_mod(r_A, r_X, ctx);
1006 break;
1007 case BPF_S_ALU_OR_K:
1008 /* A |= K */
1009 ctx->flags |= SEEN_A;
1010 emit_ori(r_A, r_A, k, ctx);
1011 break;
1012 case BPF_S_ALU_OR_X:
1013 /* A |= X */
1014 update_on_xread(ctx);
1015 ctx->flags |= SEEN_A;
1016 emit_ori(r_A, r_A, r_X, ctx);
1017 break;
1018 case BPF_S_ALU_XOR_K:
1019 /* A ^= k */
1020 ctx->flags |= SEEN_A;
1021 emit_xori(r_A, r_A, k, ctx);
1022 break;
1023 case BPF_S_ANC_ALU_XOR_X:
1024 case BPF_S_ALU_XOR_X:
1025 /* A ^= X */
1026 update_on_xread(ctx);
1027 ctx->flags |= SEEN_A;
1028 emit_xor(r_A, r_A, r_X, ctx);
1029 break;
1030 case BPF_S_ALU_AND_K:
1031 /* A &= K */
1032 ctx->flags |= SEEN_A;
1033 emit_andi(r_A, r_A, k, ctx);
1034 break;
1035 case BPF_S_ALU_AND_X:
1036 /* A &= X */
1037 update_on_xread(ctx);
1038 ctx->flags |= SEEN_A | SEEN_X;
1039 emit_and(r_A, r_A, r_X, ctx);
1040 break;
1041 case BPF_S_ALU_LSH_K:
1042 /* A <<= K */
1043 ctx->flags |= SEEN_A;
1044 emit_sll(r_A, r_A, k, ctx);
1045 break;
1046 case BPF_S_ALU_LSH_X:
1047 /* A <<= X */
1048 ctx->flags |= SEEN_A | SEEN_X;
1049 update_on_xread(ctx);
1050 emit_sllv(r_A, r_A, r_X, ctx);
1051 break;
1052 case BPF_S_ALU_RSH_K:
1053 /* A >>= K */
1054 ctx->flags |= SEEN_A;
1055 emit_srl(r_A, r_A, k, ctx);
1056 break;
1057 case BPF_S_ALU_RSH_X:
1058 ctx->flags |= SEEN_A | SEEN_X;
1059 update_on_xread(ctx);
1060 emit_srlv(r_A, r_A, r_X, ctx);
1061 break;
1062 case BPF_S_ALU_NEG:
1063 /* A = -A */
1064 ctx->flags |= SEEN_A;
1065 emit_neg(r_A, ctx);
1066 break;
1067 case BPF_S_JMP_JA:
1068 /* pc += K */
1069 emit_b(b_imm(i + k + 1, ctx), ctx);
1070 emit_nop(ctx);
1071 break;
1072 case BPF_S_JMP_JEQ_K:
1073 /* pc += ( A == K ) ? pc->jt : pc->jf */
1074 condt = MIPS_COND_EQ | MIPS_COND_K;
1075 goto jmp_cmp;
1076 case BPF_S_JMP_JEQ_X:
1077 ctx->flags |= SEEN_X;
1078 /* pc += ( A == X ) ? pc->jt : pc->jf */
1079 condt = MIPS_COND_EQ | MIPS_COND_X;
1080 goto jmp_cmp;
1081 case BPF_S_JMP_JGE_K:
1082 /* pc += ( A >= K ) ? pc->jt : pc->jf */
1083 condt = MIPS_COND_GE | MIPS_COND_K;
1084 goto jmp_cmp;
1085 case BPF_S_JMP_JGE_X:
1086 ctx->flags |= SEEN_X;
1087 /* pc += ( A >= X ) ? pc->jt : pc->jf */
1088 condt = MIPS_COND_GE | MIPS_COND_X;
1089 goto jmp_cmp;
1090 case BPF_S_JMP_JGT_K:
1091 /* pc += ( A > K ) ? pc->jt : pc->jf */
1092 condt = MIPS_COND_GT | MIPS_COND_K;
1093 goto jmp_cmp;
1094 case BPF_S_JMP_JGT_X:
1095 ctx->flags |= SEEN_X;
1096 /* pc += ( A > X ) ? pc->jt : pc->jf */
1097 condt = MIPS_COND_GT | MIPS_COND_X;
1098jmp_cmp:
1099 /* Greater or Equal */
1100 if ((condt & MIPS_COND_GE) ||
1101 (condt & MIPS_COND_GT)) {
1102 if (condt & MIPS_COND_K) { /* K */
1103 ctx->flags |= SEEN_S0 | SEEN_A;
1104 emit_sltiu(r_s0, r_A, k, ctx);
1105 } else { /* X */
1106 ctx->flags |= SEEN_S0 | SEEN_A |
1107 SEEN_X;
1108 emit_sltu(r_s0, r_A, r_X, ctx);
1109 }
1110 /* A < (K|X) ? r_scrach = 1 */
1111 b_off = b_imm(i + inst->jf + 1, ctx);
1112 emit_bcond(MIPS_COND_GT, r_s0, r_zero, b_off,
1113 ctx);
1114 emit_nop(ctx);
1115 /* A > (K|X) ? scratch = 0 */
1116 if (condt & MIPS_COND_GT) {
1117 /* Checking for equality */
1118 ctx->flags |= SEEN_S0 | SEEN_A | SEEN_X;
1119 if (condt & MIPS_COND_K)
1120 emit_load_imm(r_s0, k, ctx);
1121 else
1122 emit_jit_reg_move(r_s0, r_X,
1123 ctx);
1124 b_off = b_imm(i + inst->jf + 1, ctx);
1125 emit_bcond(MIPS_COND_EQ, r_A, r_s0,
1126 b_off, ctx);
1127 emit_nop(ctx);
1128 /* Finally, A > K|X */
1129 b_off = b_imm(i + inst->jt + 1, ctx);
1130 emit_b(b_off, ctx);
1131 emit_nop(ctx);
1132 } else {
1133 /* A >= (K|X) so jump */
1134 b_off = b_imm(i + inst->jt + 1, ctx);
1135 emit_b(b_off, ctx);
1136 emit_nop(ctx);
1137 }
1138 } else {
1139 /* A == K|X */
1140 if (condt & MIPS_COND_K) { /* K */
1141 ctx->flags |= SEEN_S0 | SEEN_A;
1142 emit_load_imm(r_s0, k, ctx);
1143 /* jump true */
1144 b_off = b_imm(i + inst->jt + 1, ctx);
1145 emit_bcond(MIPS_COND_EQ, r_A, r_s0,
1146 b_off, ctx);
1147 emit_nop(ctx);
1148 /* jump false */
1149 b_off = b_imm(i + inst->jf + 1,
1150 ctx);
1151 emit_bcond(MIPS_COND_NE, r_A, r_s0,
1152 b_off, ctx);
1153 emit_nop(ctx);
1154 } else { /* X */
1155 /* jump true */
1156 ctx->flags |= SEEN_A | SEEN_X;
1157 b_off = b_imm(i + inst->jt + 1,
1158 ctx);
1159 emit_bcond(MIPS_COND_EQ, r_A, r_X,
1160 b_off, ctx);
1161 emit_nop(ctx);
1162 /* jump false */
1163 b_off = b_imm(i + inst->jf + 1, ctx);
1164 emit_bcond(MIPS_COND_NE, r_A, r_X,
1165 b_off, ctx);
1166 emit_nop(ctx);
1167 }
1168 }
1169 break;
1170 case BPF_S_JMP_JSET_K:
1171 ctx->flags |= SEEN_S0 | SEEN_S1 | SEEN_A;
1172 /* pc += (A & K) ? pc -> jt : pc -> jf */
1173 emit_load_imm(r_s1, k, ctx);
1174 emit_and(r_s0, r_A, r_s1, ctx);
1175 /* jump true */
1176 b_off = b_imm(i + inst->jt + 1, ctx);
1177 emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx);
1178 emit_nop(ctx);
1179 /* jump false */
1180 b_off = b_imm(i + inst->jf + 1, ctx);
1181 emit_b(b_off, ctx);
1182 emit_nop(ctx);
1183 break;
1184 case BPF_S_JMP_JSET_X:
1185 ctx->flags |= SEEN_S0 | SEEN_X | SEEN_A;
1186 /* pc += (A & X) ? pc -> jt : pc -> jf */
1187 emit_and(r_s0, r_A, r_X, ctx);
1188 /* jump true */
1189 b_off = b_imm(i + inst->jt + 1, ctx);
1190 emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx);
1191 emit_nop(ctx);
1192 /* jump false */
1193 b_off = b_imm(i + inst->jf + 1, ctx);
1194 emit_b(b_off, ctx);
1195 emit_nop(ctx);
1196 break;
1197 case BPF_S_RET_A:
1198 ctx->flags |= SEEN_A;
1199 if (i != prog->len - 1)
1200 /*
1201 * If this is not the last instruction
1202 * then jump to the epilogue
1203 */
1204 emit_b(b_imm(prog->len, ctx), ctx);
1205 emit_reg_move(r_ret, r_A, ctx); /* delay slot */
1206 break;
1207 case BPF_S_RET_K:
1208 /*
1209 * It can emit two instructions so it does not fit on
1210 * the delay slot.
1211 */
1212 emit_load_imm(r_ret, k, ctx);
1213 if (i != prog->len - 1) {
1214 /*
1215 * If this is not the last instruction
1216 * then jump to the epilogue
1217 */
1218 emit_b(b_imm(prog->len, ctx), ctx);
1219 emit_nop(ctx);
1220 }
1221 break;
1222 case BPF_S_MISC_TAX:
1223 /* X = A */
1224 ctx->flags |= SEEN_X | SEEN_A;
1225 emit_jit_reg_move(r_X, r_A, ctx);
1226 break;
1227 case BPF_S_MISC_TXA:
1228 /* A = X */
1229 ctx->flags |= SEEN_A | SEEN_X;
1230 update_on_xread(ctx);
1231 emit_jit_reg_move(r_A, r_X, ctx);
1232 break;
1233 /* AUX */
1234 case BPF_S_ANC_PROTOCOL:
1235 /* A = ntohs(skb->protocol */
1236 ctx->flags |= SEEN_SKB | SEEN_OFF | SEEN_A;
1237 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff,
1238 protocol) != 2);
1239 off = offsetof(struct sk_buff, protocol);
1240 emit_half_load(r_A, r_skb, off, ctx);
1241#ifdef CONFIG_CPU_LITTLE_ENDIAN
1242 /* This needs little endian fixup */
1243 if (!cpu_has_mips_r2) {
1244 /* Get first byte */
1245 emit_andi(r_tmp_imm, r_A, 0xff, ctx);
1246 /* Shift it */
1247 emit_sll(r_tmp, r_tmp_imm, 8, ctx);
1248 /* Get second byte */
1249 emit_srl(r_tmp_imm, r_A, 8, ctx);
1250 emit_andi(r_tmp_imm, r_tmp_imm, 0xff, ctx);
1251 /* Put everyting together in r_A */
1252 emit_or(r_A, r_tmp, r_tmp_imm, ctx);
1253 } else {
1254 /* R2 and later have the wsbh instruction */
1255 emit_wsbh(r_A, r_A, ctx);
1256 }
1257#endif
1258 break;
1259 case BPF_S_ANC_CPU:
1260 ctx->flags |= SEEN_A | SEEN_OFF;
1261 /* A = current_thread_info()->cpu */
1262 BUILD_BUG_ON(FIELD_SIZEOF(struct thread_info,
1263 cpu) != 4);
1264 off = offsetof(struct thread_info, cpu);
1265 /* $28/gp points to the thread_info struct */
1266 emit_load(r_A, 28, off, ctx);
1267 break;
1268 case BPF_S_ANC_IFINDEX:
1269 /* A = skb->dev->ifindex */
1270 ctx->flags |= SEEN_SKB | SEEN_A | SEEN_S0;
1271 off = offsetof(struct sk_buff, dev);
1272 emit_load(r_s0, r_skb, off, ctx);
1273 /* error (0) in the delay slot */
1274 emit_bcond(MIPS_COND_EQ, r_s0, r_zero,
1275 b_imm(prog->len, ctx), ctx);
1276 emit_reg_move(r_ret, r_zero, ctx);
1277 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device,
1278 ifindex) != 4);
1279 off = offsetof(struct net_device, ifindex);
1280 emit_load(r_A, r_s0, off, ctx);
1281 break;
1282 case BPF_S_ANC_MARK:
1283 ctx->flags |= SEEN_SKB | SEEN_A;
1284 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
1285 off = offsetof(struct sk_buff, mark);
1286 emit_load(r_A, r_skb, off, ctx);
1287 break;
1288 case BPF_S_ANC_RXHASH:
1289 ctx->flags |= SEEN_SKB | SEEN_A;
1290 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4);
1291 off = offsetof(struct sk_buff, hash);
1292 emit_load(r_A, r_skb, off, ctx);
1293 break;
1294 case BPF_S_ANC_VLAN_TAG:
1295 case BPF_S_ANC_VLAN_TAG_PRESENT:
1296 ctx->flags |= SEEN_SKB | SEEN_S0 | SEEN_A;
1297 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff,
1298 vlan_tci) != 2);
1299 off = offsetof(struct sk_buff, vlan_tci);
1300 emit_half_load(r_s0, r_skb, off, ctx);
1301 if (inst->code == BPF_S_ANC_VLAN_TAG)
1302 emit_and(r_A, r_s0, VLAN_VID_MASK, ctx);
1303 else
1304 emit_and(r_A, r_s0, VLAN_TAG_PRESENT, ctx);
1305 break;
1306 case BPF_S_ANC_PKTTYPE:
1307 off = pkt_type_offset();
1308
1309 if (off < 0)
1310 return -1;
1311 emit_load_byte(r_tmp, r_skb, off, ctx);
1312 /* Keep only the last 3 bits */
1313 emit_andi(r_A, r_tmp, PKT_TYPE_MAX, ctx);
1314 break;
1315 case BPF_S_ANC_QUEUE:
1316 ctx->flags |= SEEN_SKB | SEEN_A;
1317 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff,
1318 queue_mapping) != 2);
1319 BUILD_BUG_ON(offsetof(struct sk_buff,
1320 queue_mapping) > 0xff);
1321 off = offsetof(struct sk_buff, queue_mapping);
1322 emit_half_load(r_A, r_skb, off, ctx);
1323 break;
1324 default:
1325 pr_warn("%s: Unhandled opcode: 0x%02x\n", __FILE__,
1326 inst->code);
1327 return -1;
1328 }
1329 }
1330
1331 /* compute offsets only during the first pass */
1332 if (ctx->target == NULL)
1333 ctx->offsets[i] = ctx->idx * 4;
1334
1335 return 0;
1336}
1337
1338int bpf_jit_enable __read_mostly;
1339
1340void bpf_jit_compile(struct sk_filter *fp)
1341{
1342 struct jit_ctx ctx;
1343 unsigned int alloc_size, tmp_idx;
1344
1345 if (!bpf_jit_enable)
1346 return;
1347
1348 memset(&ctx, 0, sizeof(ctx));
1349
1350 ctx.offsets = kcalloc(fp->len, sizeof(*ctx.offsets), GFP_KERNEL);
1351 if (ctx.offsets == NULL)
1352 return;
1353
1354 ctx.skf = fp;
1355
1356 if (build_body(&ctx))
1357 goto out;
1358
1359 tmp_idx = ctx.idx;
1360 build_prologue(&ctx);
1361 ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4;
1362 /* just to complete the ctx.idx count */
1363 build_epilogue(&ctx);
1364
1365 alloc_size = 4 * ctx.idx;
1366 ctx.target = module_alloc(alloc_size);
1367 if (ctx.target == NULL)
1368 goto out;
1369
1370 /* Clean it */
1371 memset(ctx.target, 0, alloc_size);
1372
1373 ctx.idx = 0;
1374
1375 /* Generate the actual JIT code */
1376 build_prologue(&ctx);
1377 build_body(&ctx);
1378 build_epilogue(&ctx);
1379
1380 /* Update the icache */
1381 flush_icache_range((ptr)ctx.target, (ptr)(ctx.target + ctx.idx));
1382
1383 if (bpf_jit_enable > 1)
1384 /* Dump JIT code */
1385 bpf_jit_dump(fp->len, alloc_size, 2, ctx.target);
1386
1387 fp->bpf_func = (void *)ctx.target;
1388 fp->jited = 1;
1389
1390out:
1391 kfree(ctx.offsets);
1392}
1393
1394void bpf_jit_free(struct sk_filter *fp)
1395{
1396 if (fp->jited)
1397 module_free(NULL, fp->bpf_func);
1398 kfree(fp);
1399}
diff --git a/arch/mips/net/bpf_jit.h b/arch/mips/net/bpf_jit.h
new file mode 100644
index 000000000000..3a5751b4335a
--- /dev/null
+++ b/arch/mips/net/bpf_jit.h
@@ -0,0 +1,44 @@
1/*
2 * Just-In-Time compiler for BPF filters on MIPS
3 *
4 * Copyright (c) 2014 Imagination Technologies Ltd.
5 * Author: Markos Chandras <markos.chandras@imgtec.com>
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; version 2 of the License.
10 */
11
12#ifndef BPF_JIT_MIPS_OP_H
13#define BPF_JIT_MIPS_OP_H
14
15/* Registers used by JIT */
16#define MIPS_R_ZERO 0
17#define MIPS_R_V0 2
18#define MIPS_R_V1 3
19#define MIPS_R_A0 4
20#define MIPS_R_A1 5
21#define MIPS_R_T6 14
22#define MIPS_R_T7 15
23#define MIPS_R_S0 16
24#define MIPS_R_S1 17
25#define MIPS_R_S2 18
26#define MIPS_R_S3 19
27#define MIPS_R_S4 20
28#define MIPS_R_S5 21
29#define MIPS_R_S6 22
30#define MIPS_R_S7 23
31#define MIPS_R_SP 29
32#define MIPS_R_RA 31
33
34/* Conditional codes */
35#define MIPS_COND_EQ 0x1
36#define MIPS_COND_GE (0x1 << 1)
37#define MIPS_COND_GT (0x1 << 2)
38#define MIPS_COND_NE (0x1 << 3)
39#define MIPS_COND_ALL (0x1 << 4)
40/* Conditionals on X register or K immediate */
41#define MIPS_COND_X (0x1 << 5)
42#define MIPS_COND_K (0x1 << 6)
43
44#endif /* BPF_JIT_MIPS_OP_H */
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-18 04:24:02 -0500