diff options
Diffstat (limited to 'arch/s390/net/bpf_jit_comp.c')
-rw-r--r-- | arch/s390/net/bpf_jit_comp.c | 776 |
1 files changed, 776 insertions, 0 deletions
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c new file mode 100644 index 00000000000..9b355b406af --- /dev/null +++ b/arch/s390/net/bpf_jit_comp.c | |||
@@ -0,0 +1,776 @@ | |||
1 | /* | ||
2 | * BPF Jit compiler for s390. | ||
3 | * | ||
4 | * Copyright IBM Corp. 2012 | ||
5 | * | ||
6 | * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> | ||
7 | */ | ||
8 | #include <linux/moduleloader.h> | ||
9 | #include <linux/netdevice.h> | ||
10 | #include <linux/filter.h> | ||
11 | #include <asm/cacheflush.h> | ||
12 | #include <asm/processor.h> | ||
13 | #include <asm/facility.h> | ||
14 | |||
15 | /* | ||
16 | * Conventions: | ||
17 | * %r2 = skb pointer | ||
18 | * %r3 = offset parameter | ||
19 | * %r4 = scratch register / length parameter | ||
20 | * %r5 = BPF A accumulator | ||
21 | * %r8 = return address | ||
22 | * %r9 = save register for skb pointer | ||
23 | * %r10 = skb->data | ||
24 | * %r11 = skb->len - skb->data_len (headlen) | ||
25 | * %r12 = BPF X accumulator | ||
26 | * %r13 = literal pool pointer | ||
27 | * 0(%r15) - 63(%r15) scratch memory array with BPF_MEMWORDS | ||
28 | */ | ||
29 | int bpf_jit_enable __read_mostly; | ||
30 | |||
31 | /* | ||
32 | * assembly code in arch/x86/net/bpf_jit.S | ||
33 | */ | ||
34 | extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[]; | ||
35 | extern u8 sk_load_word_ind[], sk_load_half_ind[], sk_load_byte_ind[]; | ||
36 | |||
37 | struct bpf_jit { | ||
38 | unsigned int seen; | ||
39 | u8 *start; | ||
40 | u8 *prg; | ||
41 | u8 *mid; | ||
42 | u8 *lit; | ||
43 | u8 *end; | ||
44 | u8 *base_ip; | ||
45 | u8 *ret0_ip; | ||
46 | u8 *exit_ip; | ||
47 | unsigned int off_load_word; | ||
48 | unsigned int off_load_half; | ||
49 | unsigned int off_load_byte; | ||
50 | unsigned int off_load_bmsh; | ||
51 | unsigned int off_load_iword; | ||
52 | unsigned int off_load_ihalf; | ||
53 | unsigned int off_load_ibyte; | ||
54 | }; | ||
55 | |||
56 | #define BPF_SIZE_MAX 4096 /* Max size for program */ | ||
57 | |||
58 | #define SEEN_DATAREF 1 /* might call external helpers */ | ||
59 | #define SEEN_XREG 2 /* ebx is used */ | ||
60 | #define SEEN_MEM 4 /* use mem[] for temporary storage */ | ||
61 | #define SEEN_RET0 8 /* pc_ret0 points to a valid return 0 */ | ||
62 | #define SEEN_LITERAL 16 /* code uses literals */ | ||
63 | #define SEEN_LOAD_WORD 32 /* code uses sk_load_word */ | ||
64 | #define SEEN_LOAD_HALF 64 /* code uses sk_load_half */ | ||
65 | #define SEEN_LOAD_BYTE 128 /* code uses sk_load_byte */ | ||
66 | #define SEEN_LOAD_BMSH 256 /* code uses sk_load_byte_msh */ | ||
67 | #define SEEN_LOAD_IWORD 512 /* code uses sk_load_word_ind */ | ||
68 | #define SEEN_LOAD_IHALF 1024 /* code uses sk_load_half_ind */ | ||
69 | #define SEEN_LOAD_IBYTE 2048 /* code uses sk_load_byte_ind */ | ||
70 | |||
71 | #define EMIT2(op) \ | ||
72 | ({ \ | ||
73 | if (jit->prg + 2 <= jit->mid) \ | ||
74 | *(u16 *) jit->prg = op; \ | ||
75 | jit->prg += 2; \ | ||
76 | }) | ||
77 | |||
78 | #define EMIT4(op) \ | ||
79 | ({ \ | ||
80 | if (jit->prg + 4 <= jit->mid) \ | ||
81 | *(u32 *) jit->prg = op; \ | ||
82 | jit->prg += 4; \ | ||
83 | }) | ||
84 | |||
85 | #define EMIT4_DISP(op, disp) \ | ||
86 | ({ \ | ||
87 | unsigned int __disp = (disp) & 0xfff; \ | ||
88 | EMIT4(op | __disp); \ | ||
89 | }) | ||
90 | |||
91 | #define EMIT4_IMM(op, imm) \ | ||
92 | ({ \ | ||
93 | unsigned int __imm = (imm) & 0xffff; \ | ||
94 | EMIT4(op | __imm); \ | ||
95 | }) | ||
96 | |||
97 | #define EMIT4_PCREL(op, pcrel) \ | ||
98 | ({ \ | ||
99 | long __pcrel = ((pcrel) >> 1) & 0xffff; \ | ||
100 | EMIT4(op | __pcrel); \ | ||
101 | }) | ||
102 | |||
103 | #define EMIT6(op1, op2) \ | ||
104 | ({ \ | ||
105 | if (jit->prg + 6 <= jit->mid) { \ | ||
106 | *(u32 *) jit->prg = op1; \ | ||
107 | *(u16 *) (jit->prg + 4) = op2; \ | ||
108 | } \ | ||
109 | jit->prg += 6; \ | ||
110 | }) | ||
111 | |||
112 | #define EMIT6_DISP(op1, op2, disp) \ | ||
113 | ({ \ | ||
114 | unsigned int __disp = (disp) & 0xfff; \ | ||
115 | EMIT6(op1 | __disp, op2); \ | ||
116 | }) | ||
117 | |||
118 | #define EMIT6_IMM(op, imm) \ | ||
119 | ({ \ | ||
120 | unsigned int __imm = (imm); \ | ||
121 | EMIT6(op | (__imm >> 16), __imm & 0xffff); \ | ||
122 | }) | ||
123 | |||
124 | #define EMIT_CONST(val) \ | ||
125 | ({ \ | ||
126 | unsigned int ret; \ | ||
127 | ret = (unsigned int) (jit->lit - jit->base_ip); \ | ||
128 | jit->seen |= SEEN_LITERAL; \ | ||
129 | if (jit->lit + 4 <= jit->end) \ | ||
130 | *(u32 *) jit->lit = val; \ | ||
131 | jit->lit += 4; \ | ||
132 | ret; \ | ||
133 | }) | ||
134 | |||
135 | #define EMIT_FN_CONST(bit, fn) \ | ||
136 | ({ \ | ||
137 | unsigned int ret; \ | ||
138 | ret = (unsigned int) (jit->lit - jit->base_ip); \ | ||
139 | if (jit->seen & bit) { \ | ||
140 | jit->seen |= SEEN_LITERAL; \ | ||
141 | if (jit->lit + 8 <= jit->end) \ | ||
142 | *(void **) jit->lit = fn; \ | ||
143 | jit->lit += 8; \ | ||
144 | } \ | ||
145 | ret; \ | ||
146 | }) | ||
147 | |||
148 | static void bpf_jit_prologue(struct bpf_jit *jit) | ||
149 | { | ||
150 | /* Save registers and create stack frame if necessary */ | ||
151 | if (jit->seen & SEEN_DATAREF) { | ||
152 | /* stmg %r8,%r15,88(%r15) */ | ||
153 | EMIT6(0xeb8ff058, 0x0024); | ||
154 | /* lgr %r14,%r15 */ | ||
155 | EMIT4(0xb90400ef); | ||
156 | /* ahi %r15,<offset> */ | ||
157 | EMIT4_IMM(0xa7fa0000, (jit->seen & SEEN_MEM) ? -112 : -80); | ||
158 | /* stg %r14,152(%r15) */ | ||
159 | EMIT6(0xe3e0f098, 0x0024); | ||
160 | } else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) | ||
161 | /* stmg %r12,%r13,120(%r15) */ | ||
162 | EMIT6(0xebcdf078, 0x0024); | ||
163 | else if (jit->seen & SEEN_XREG) | ||
164 | /* stg %r12,120(%r15) */ | ||
165 | EMIT6(0xe3c0f078, 0x0024); | ||
166 | else if (jit->seen & SEEN_LITERAL) | ||
167 | /* stg %r13,128(%r15) */ | ||
168 | EMIT6(0xe3d0f080, 0x0024); | ||
169 | |||
170 | /* Setup literal pool */ | ||
171 | if (jit->seen & SEEN_LITERAL) { | ||
172 | /* basr %r13,0 */ | ||
173 | EMIT2(0x0dd0); | ||
174 | jit->base_ip = jit->prg; | ||
175 | } | ||
176 | jit->off_load_word = EMIT_FN_CONST(SEEN_LOAD_WORD, sk_load_word); | ||
177 | jit->off_load_half = EMIT_FN_CONST(SEEN_LOAD_HALF, sk_load_half); | ||
178 | jit->off_load_byte = EMIT_FN_CONST(SEEN_LOAD_BYTE, sk_load_byte); | ||
179 | jit->off_load_bmsh = EMIT_FN_CONST(SEEN_LOAD_BMSH, sk_load_byte_msh); | ||
180 | jit->off_load_iword = EMIT_FN_CONST(SEEN_LOAD_IWORD, sk_load_word_ind); | ||
181 | jit->off_load_ihalf = EMIT_FN_CONST(SEEN_LOAD_IHALF, sk_load_half_ind); | ||
182 | jit->off_load_ibyte = EMIT_FN_CONST(SEEN_LOAD_IBYTE, sk_load_byte_ind); | ||
183 | |||
184 | /* Filter needs to access skb data */ | ||
185 | if (jit->seen & SEEN_DATAREF) { | ||
186 | /* l %r11,<len>(%r2) */ | ||
187 | EMIT4_DISP(0x58b02000, offsetof(struct sk_buff, len)); | ||
188 | /* s %r11,<data_len>(%r2) */ | ||
189 | EMIT4_DISP(0x5bb02000, offsetof(struct sk_buff, data_len)); | ||
190 | /* lg %r10,<data>(%r2) */ | ||
191 | EMIT6_DISP(0xe3a02000, 0x0004, | ||
192 | offsetof(struct sk_buff, data)); | ||
193 | } | ||
194 | } | ||
195 | |||
196 | static void bpf_jit_epilogue(struct bpf_jit *jit) | ||
197 | { | ||
198 | /* Return 0 */ | ||
199 | if (jit->seen & SEEN_RET0) { | ||
200 | jit->ret0_ip = jit->prg; | ||
201 | /* lghi %r2,0 */ | ||
202 | EMIT4(0xa7290000); | ||
203 | } | ||
204 | jit->exit_ip = jit->prg; | ||
205 | /* Restore registers */ | ||
206 | if (jit->seen & SEEN_DATAREF) | ||
207 | /* lmg %r8,%r15,<offset>(%r15) */ | ||
208 | EMIT6_DISP(0xeb8ff000, 0x0004, | ||
209 | (jit->seen & SEEN_MEM) ? 200 : 168); | ||
210 | else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) | ||
211 | /* lmg %r12,%r13,120(%r15) */ | ||
212 | EMIT6(0xebcdf078, 0x0004); | ||
213 | else if (jit->seen & SEEN_XREG) | ||
214 | /* lg %r12,120(%r15) */ | ||
215 | EMIT6(0xe3c0f078, 0x0004); | ||
216 | else if (jit->seen & SEEN_LITERAL) | ||
217 | /* lg %r13,128(%r15) */ | ||
218 | EMIT6(0xe3d0f080, 0x0004); | ||
219 | /* br %r14 */ | ||
220 | EMIT2(0x07fe); | ||
221 | } | ||
222 | |||
223 | /* | ||
224 | * make sure we dont leak kernel information to user | ||
225 | */ | ||
226 | static void bpf_jit_noleaks(struct bpf_jit *jit, struct sock_filter *filter) | ||
227 | { | ||
228 | /* Clear temporary memory if (seen & SEEN_MEM) */ | ||
229 | if (jit->seen & SEEN_MEM) | ||
230 | /* xc 0(64,%r15),0(%r15) */ | ||
231 | EMIT6(0xd73ff000, 0xf000); | ||
232 | /* Clear X if (seen & SEEN_XREG) */ | ||
233 | if (jit->seen & SEEN_XREG) | ||
234 | /* lhi %r12,0 */ | ||
235 | EMIT4(0xa7c80000); | ||
236 | /* Clear A if the first register does not set it. */ | ||
237 | switch (filter[0].code) { | ||
238 | case BPF_S_LD_W_ABS: | ||
239 | case BPF_S_LD_H_ABS: | ||
240 | case BPF_S_LD_B_ABS: | ||
241 | case BPF_S_LD_W_LEN: | ||
242 | case BPF_S_LD_W_IND: | ||
243 | case BPF_S_LD_H_IND: | ||
244 | case BPF_S_LD_B_IND: | ||
245 | case BPF_S_LDX_B_MSH: | ||
246 | case BPF_S_LD_IMM: | ||
247 | case BPF_S_LD_MEM: | ||
248 | case BPF_S_MISC_TXA: | ||
249 | case BPF_S_ANC_PROTOCOL: | ||
250 | case BPF_S_ANC_PKTTYPE: | ||
251 | case BPF_S_ANC_IFINDEX: | ||
252 | case BPF_S_ANC_MARK: | ||
253 | case BPF_S_ANC_QUEUE: | ||
254 | case BPF_S_ANC_HATYPE: | ||
255 | case BPF_S_ANC_RXHASH: | ||
256 | case BPF_S_ANC_CPU: | ||
257 | case BPF_S_RET_K: | ||
258 | /* first instruction sets A register */ | ||
259 | break; | ||
260 | default: /* A = 0 */ | ||
261 | /* lhi %r5,0 */ | ||
262 | EMIT4(0xa7580000); | ||
263 | } | ||
264 | } | ||
265 | |||
266 | static int bpf_jit_insn(struct bpf_jit *jit, struct sock_filter *filter, | ||
267 | unsigned int *addrs, int i, int last) | ||
268 | { | ||
269 | unsigned int K; | ||
270 | int offset; | ||
271 | unsigned int mask; | ||
272 | |||
273 | K = filter->k; | ||
274 | switch (filter->code) { | ||
275 | case BPF_S_ALU_ADD_X: /* A += X */ | ||
276 | jit->seen |= SEEN_XREG; | ||
277 | /* ar %r5,%r12 */ | ||
278 | EMIT2(0x1a5c); | ||
279 | break; | ||
280 | case BPF_S_ALU_ADD_K: /* A += K */ | ||
281 | if (!K) | ||
282 | break; | ||
283 | if (K <= 16383) | ||
284 | /* ahi %r5,<K> */ | ||
285 | EMIT4_IMM(0xa75a0000, K); | ||
286 | else if (test_facility(21)) | ||
287 | /* alfi %r5,<K> */ | ||
288 | EMIT6_IMM(0xc25b0000, K); | ||
289 | else | ||
290 | /* a %r5,<d(K)>(%r13) */ | ||
291 | EMIT4_DISP(0x5a50d000, EMIT_CONST(K)); | ||
292 | break; | ||
293 | case BPF_S_ALU_SUB_X: /* A -= X */ | ||
294 | jit->seen |= SEEN_XREG; | ||
295 | /* sr %r5,%r12 */ | ||
296 | EMIT2(0x1b5c); | ||
297 | break; | ||
298 | case BPF_S_ALU_SUB_K: /* A -= K */ | ||
299 | if (!K) | ||
300 | break; | ||
301 | if (K <= 16384) | ||
302 | /* ahi %r5,-K */ | ||
303 | EMIT4_IMM(0xa75a0000, -K); | ||
304 | else if (test_facility(21)) | ||
305 | /* alfi %r5,-K */ | ||
306 | EMIT6_IMM(0xc25b0000, -K); | ||
307 | else | ||
308 | /* s %r5,<d(K)>(%r13) */ | ||
309 | EMIT4_DISP(0x5b50d000, EMIT_CONST(K)); | ||
310 | break; | ||
311 | case BPF_S_ALU_MUL_X: /* A *= X */ | ||
312 | jit->seen |= SEEN_XREG; | ||
313 | /* msr %r5,%r12 */ | ||
314 | EMIT4(0xb252005c); | ||
315 | break; | ||
316 | case BPF_S_ALU_MUL_K: /* A *= K */ | ||
317 | if (K <= 16383) | ||
318 | /* mhi %r5,K */ | ||
319 | EMIT4_IMM(0xa75c0000, K); | ||
320 | else if (test_facility(34)) | ||
321 | /* msfi %r5,<K> */ | ||
322 | EMIT6_IMM(0xc2510000, K); | ||
323 | else | ||
324 | /* ms %r5,<d(K)>(%r13) */ | ||
325 | EMIT4_DISP(0x7150d000, EMIT_CONST(K)); | ||
326 | break; | ||
327 | case BPF_S_ALU_DIV_X: /* A /= X */ | ||
328 | jit->seen |= SEEN_XREG | SEEN_RET0; | ||
329 | /* ltr %r12,%r12 */ | ||
330 | EMIT2(0x12cc); | ||
331 | /* jz <ret0> */ | ||
332 | EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg)); | ||
333 | /* lhi %r4,0 */ | ||
334 | EMIT4(0xa7480000); | ||
335 | /* dr %r4,%r12 */ | ||
336 | EMIT2(0x1d4c); | ||
337 | break; | ||
338 | case BPF_S_ALU_DIV_K: /* A = reciprocal_divide(A, K) */ | ||
339 | /* m %r4,<d(K)>(%r13) */ | ||
340 | EMIT4_DISP(0x5c40d000, EMIT_CONST(K)); | ||
341 | /* lr %r5,%r4 */ | ||
342 | EMIT2(0x1854); | ||
343 | break; | ||
344 | case BPF_S_ALU_AND_X: /* A &= X */ | ||
345 | jit->seen |= SEEN_XREG; | ||
346 | /* nr %r5,%r12 */ | ||
347 | EMIT2(0x145c); | ||
348 | break; | ||
349 | case BPF_S_ALU_AND_K: /* A &= K */ | ||
350 | if (test_facility(21)) | ||
351 | /* nilf %r5,<K> */ | ||
352 | EMIT6_IMM(0xc05b0000, K); | ||
353 | else | ||
354 | /* n %r5,<d(K)>(%r13) */ | ||
355 | EMIT4_DISP(0x5450d000, EMIT_CONST(K)); | ||
356 | break; | ||
357 | case BPF_S_ALU_OR_X: /* A |= X */ | ||
358 | jit->seen |= SEEN_XREG; | ||
359 | /* or %r5,%r12 */ | ||
360 | EMIT2(0x165c); | ||
361 | break; | ||
362 | case BPF_S_ALU_OR_K: /* A |= K */ | ||
363 | if (test_facility(21)) | ||
364 | /* oilf %r5,<K> */ | ||
365 | EMIT6_IMM(0xc05d0000, K); | ||
366 | else | ||
367 | /* o %r5,<d(K)>(%r13) */ | ||
368 | EMIT4_DISP(0x5650d000, EMIT_CONST(K)); | ||
369 | break; | ||
370 | case BPF_S_ANC_ALU_XOR_X: /* A ^= X; */ | ||
371 | jit->seen |= SEEN_XREG; | ||
372 | /* xr %r5,%r12 */ | ||
373 | EMIT2(0x175c); | ||
374 | break; | ||
375 | case BPF_S_ALU_LSH_X: /* A <<= X; */ | ||
376 | jit->seen |= SEEN_XREG; | ||
377 | /* sll %r5,0(%r12) */ | ||
378 | EMIT4(0x8950c000); | ||
379 | break; | ||
380 | case BPF_S_ALU_LSH_K: /* A <<= K */ | ||
381 | if (K == 0) | ||
382 | break; | ||
383 | /* sll %r5,K */ | ||
384 | EMIT4_DISP(0x89500000, K); | ||
385 | break; | ||
386 | case BPF_S_ALU_RSH_X: /* A >>= X; */ | ||
387 | jit->seen |= SEEN_XREG; | ||
388 | /* srl %r5,0(%r12) */ | ||
389 | EMIT4(0x8850c000); | ||
390 | break; | ||
391 | case BPF_S_ALU_RSH_K: /* A >>= K; */ | ||
392 | if (K == 0) | ||
393 | break; | ||
394 | /* srl %r5,K */ | ||
395 | EMIT4_DISP(0x88500000, K); | ||
396 | break; | ||
397 | case BPF_S_ALU_NEG: /* A = -A */ | ||
398 | /* lnr %r5,%r5 */ | ||
399 | EMIT2(0x1155); | ||
400 | break; | ||
401 | case BPF_S_JMP_JA: /* ip += K */ | ||
402 | offset = addrs[i + K] + jit->start - jit->prg; | ||
403 | EMIT4_PCREL(0xa7f40000, offset); | ||
404 | break; | ||
405 | case BPF_S_JMP_JGT_K: /* ip += (A > K) ? jt : jf */ | ||
406 | mask = 0x200000; /* jh */ | ||
407 | goto kbranch; | ||
408 | case BPF_S_JMP_JGE_K: /* ip += (A >= K) ? jt : jf */ | ||
409 | mask = 0xa00000; /* jhe */ | ||
410 | goto kbranch; | ||
411 | case BPF_S_JMP_JEQ_K: /* ip += (A == K) ? jt : jf */ | ||
412 | mask = 0x800000; /* je */ | ||
413 | kbranch: /* Emit compare if the branch targets are different */ | ||
414 | if (filter->jt != filter->jf) { | ||
415 | if (K <= 16383) | ||
416 | /* chi %r5,<K> */ | ||
417 | EMIT4_IMM(0xa75e0000, K); | ||
418 | else if (test_facility(21)) | ||
419 | /* clfi %r5,<K> */ | ||
420 | EMIT6_IMM(0xc25f0000, K); | ||
421 | else | ||
422 | /* c %r5,<d(K)>(%r13) */ | ||
423 | EMIT4_DISP(0x5950d000, EMIT_CONST(K)); | ||
424 | } | ||
425 | branch: if (filter->jt == filter->jf) { | ||
426 | if (filter->jt == 0) | ||
427 | break; | ||
428 | /* j <jt> */ | ||
429 | offset = addrs[i + filter->jt] + jit->start - jit->prg; | ||
430 | EMIT4_PCREL(0xa7f40000, offset); | ||
431 | break; | ||
432 | } | ||
433 | if (filter->jt != 0) { | ||
434 | /* brc <mask>,<jt> */ | ||
435 | offset = addrs[i + filter->jt] + jit->start - jit->prg; | ||
436 | EMIT4_PCREL(0xa7040000 | mask, offset); | ||
437 | } | ||
438 | if (filter->jf != 0) { | ||
439 | /* brc <mask^15>,<jf> */ | ||
440 | offset = addrs[i + filter->jf] + jit->start - jit->prg; | ||
441 | EMIT4_PCREL(0xa7040000 | (mask ^ 0xf00000), offset); | ||
442 | } | ||
443 | break; | ||
444 | case BPF_S_JMP_JSET_K: /* ip += (A & K) ? jt : jf */ | ||
445 | mask = 0x700000; /* jnz */ | ||
446 | /* Emit test if the branch targets are different */ | ||
447 | if (filter->jt != filter->jf) { | ||
448 | if (K > 65535) { | ||
449 | /* lr %r4,%r5 */ | ||
450 | EMIT2(0x1845); | ||
451 | /* n %r4,<d(K)>(%r13) */ | ||
452 | EMIT4_DISP(0x5440d000, EMIT_CONST(K)); | ||
453 | } else | ||
454 | /* tmll %r5,K */ | ||
455 | EMIT4_IMM(0xa7510000, K); | ||
456 | } | ||
457 | goto branch; | ||
458 | case BPF_S_JMP_JGT_X: /* ip += (A > X) ? jt : jf */ | ||
459 | mask = 0x200000; /* jh */ | ||
460 | goto xbranch; | ||
461 | case BPF_S_JMP_JGE_X: /* ip += (A >= X) ? jt : jf */ | ||
462 | mask = 0xa00000; /* jhe */ | ||
463 | goto xbranch; | ||
464 | case BPF_S_JMP_JEQ_X: /* ip += (A == X) ? jt : jf */ | ||
465 | mask = 0x800000; /* je */ | ||
466 | xbranch: /* Emit compare if the branch targets are different */ | ||
467 | if (filter->jt != filter->jf) { | ||
468 | jit->seen |= SEEN_XREG; | ||
469 | /* cr %r5,%r12 */ | ||
470 | EMIT2(0x195c); | ||
471 | } | ||
472 | goto branch; | ||
473 | case BPF_S_JMP_JSET_X: /* ip += (A & X) ? jt : jf */ | ||
474 | mask = 0x700000; /* jnz */ | ||
475 | /* Emit test if the branch targets are different */ | ||
476 | if (filter->jt != filter->jf) { | ||
477 | jit->seen |= SEEN_XREG; | ||
478 | /* lr %r4,%r5 */ | ||
479 | EMIT2(0x1845); | ||
480 | /* nr %r4,%r12 */ | ||
481 | EMIT2(0x144c); | ||
482 | } | ||
483 | goto branch; | ||
484 | case BPF_S_LD_W_ABS: /* A = *(u32 *) (skb->data+K) */ | ||
485 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_WORD; | ||
486 | offset = jit->off_load_word; | ||
487 | goto load_abs; | ||
488 | case BPF_S_LD_H_ABS: /* A = *(u16 *) (skb->data+K) */ | ||
489 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_HALF; | ||
490 | offset = jit->off_load_half; | ||
491 | goto load_abs; | ||
492 | case BPF_S_LD_B_ABS: /* A = *(u8 *) (skb->data+K) */ | ||
493 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_BYTE; | ||
494 | offset = jit->off_load_byte; | ||
495 | load_abs: if ((int) K < 0) | ||
496 | goto out; | ||
497 | call_fn: /* lg %r1,<d(function)>(%r13) */ | ||
498 | EMIT6_DISP(0xe310d000, 0x0004, offset); | ||
499 | /* l %r3,<d(K)>(%r13) */ | ||
500 | EMIT4_DISP(0x5830d000, EMIT_CONST(K)); | ||
501 | /* basr %r8,%r1 */ | ||
502 | EMIT2(0x0d81); | ||
503 | /* jnz <ret0> */ | ||
504 | EMIT4_PCREL(0xa7740000, (jit->ret0_ip - jit->prg)); | ||
505 | break; | ||
506 | case BPF_S_LD_W_IND: /* A = *(u32 *) (skb->data+K+X) */ | ||
507 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IWORD; | ||
508 | offset = jit->off_load_iword; | ||
509 | goto call_fn; | ||
510 | case BPF_S_LD_H_IND: /* A = *(u16 *) (skb->data+K+X) */ | ||
511 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IHALF; | ||
512 | offset = jit->off_load_ihalf; | ||
513 | goto call_fn; | ||
514 | case BPF_S_LD_B_IND: /* A = *(u8 *) (skb->data+K+X) */ | ||
515 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IBYTE; | ||
516 | offset = jit->off_load_ibyte; | ||
517 | goto call_fn; | ||
518 | case BPF_S_LDX_B_MSH: | ||
519 | /* X = (*(u8 *)(skb->data+K) & 0xf) << 2 */ | ||
520 | jit->seen |= SEEN_RET0; | ||
521 | if ((int) K < 0) { | ||
522 | /* j <ret0> */ | ||
523 | EMIT4_PCREL(0xa7f40000, (jit->ret0_ip - jit->prg)); | ||
524 | break; | ||
525 | } | ||
526 | jit->seen |= SEEN_DATAREF | SEEN_LOAD_BMSH; | ||
527 | offset = jit->off_load_bmsh; | ||
528 | goto call_fn; | ||
529 | case BPF_S_LD_W_LEN: /* A = skb->len; */ | ||
530 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); | ||
531 | /* l %r5,<d(len)>(%r2) */ | ||
532 | EMIT4_DISP(0x58502000, offsetof(struct sk_buff, len)); | ||
533 | break; | ||
534 | case BPF_S_LDX_W_LEN: /* X = skb->len; */ | ||
535 | jit->seen |= SEEN_XREG; | ||
536 | /* l %r12,<d(len)>(%r2) */ | ||
537 | EMIT4_DISP(0x58c02000, offsetof(struct sk_buff, len)); | ||
538 | break; | ||
539 | case BPF_S_LD_IMM: /* A = K */ | ||
540 | if (K <= 16383) | ||
541 | /* lhi %r5,K */ | ||
542 | EMIT4_IMM(0xa7580000, K); | ||
543 | else if (test_facility(21)) | ||
544 | /* llilf %r5,<K> */ | ||
545 | EMIT6_IMM(0xc05f0000, K); | ||
546 | else | ||
547 | /* l %r5,<d(K)>(%r13) */ | ||
548 | EMIT4_DISP(0x5850d000, EMIT_CONST(K)); | ||
549 | break; | ||
550 | case BPF_S_LDX_IMM: /* X = K */ | ||
551 | jit->seen |= SEEN_XREG; | ||
552 | if (K <= 16383) | ||
553 | /* lhi %r12,<K> */ | ||
554 | EMIT4_IMM(0xa7c80000, K); | ||
555 | else if (test_facility(21)) | ||
556 | /* llilf %r12,<K> */ | ||
557 | EMIT6_IMM(0xc0cf0000, K); | ||
558 | else | ||
559 | /* l %r12,<d(K)>(%r13) */ | ||
560 | EMIT4_DISP(0x58c0d000, EMIT_CONST(K)); | ||
561 | break; | ||
562 | case BPF_S_LD_MEM: /* A = mem[K] */ | ||
563 | jit->seen |= SEEN_MEM; | ||
564 | /* l %r5,<K>(%r15) */ | ||
565 | EMIT4_DISP(0x5850f000, | ||
566 | (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); | ||
567 | break; | ||
568 | case BPF_S_LDX_MEM: /* X = mem[K] */ | ||
569 | jit->seen |= SEEN_XREG | SEEN_MEM; | ||
570 | /* l %r12,<K>(%r15) */ | ||
571 | EMIT4_DISP(0x58c0f000, | ||
572 | (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); | ||
573 | break; | ||
574 | case BPF_S_MISC_TAX: /* X = A */ | ||
575 | jit->seen |= SEEN_XREG; | ||
576 | /* lr %r12,%r5 */ | ||
577 | EMIT2(0x18c5); | ||
578 | break; | ||
579 | case BPF_S_MISC_TXA: /* A = X */ | ||
580 | jit->seen |= SEEN_XREG; | ||
581 | /* lr %r5,%r12 */ | ||
582 | EMIT2(0x185c); | ||
583 | break; | ||
584 | case BPF_S_RET_K: | ||
585 | if (K == 0) { | ||
586 | jit->seen |= SEEN_RET0; | ||
587 | if (last) | ||
588 | break; | ||
589 | /* j <ret0> */ | ||
590 | EMIT4_PCREL(0xa7f40000, jit->ret0_ip - jit->prg); | ||
591 | } else { | ||
592 | if (K <= 16383) | ||
593 | /* lghi %r2,K */ | ||
594 | EMIT4_IMM(0xa7290000, K); | ||
595 | else | ||
596 | /* llgf %r2,<K>(%r13) */ | ||
597 | EMIT6_DISP(0xe320d000, 0x0016, EMIT_CONST(K)); | ||
598 | /* j <exit> */ | ||
599 | if (last && !(jit->seen & SEEN_RET0)) | ||
600 | break; | ||
601 | EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); | ||
602 | } | ||
603 | break; | ||
604 | case BPF_S_RET_A: | ||
605 | /* llgfr %r2,%r5 */ | ||
606 | EMIT4(0xb9160025); | ||
607 | /* j <exit> */ | ||
608 | EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); | ||
609 | break; | ||
610 | case BPF_S_ST: /* mem[K] = A */ | ||
611 | jit->seen |= SEEN_MEM; | ||
612 | /* st %r5,<K>(%r15) */ | ||
613 | EMIT4_DISP(0x5050f000, | ||
614 | (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); | ||
615 | break; | ||
616 | case BPF_S_STX: /* mem[K] = X : mov %ebx,off8(%rbp) */ | ||
617 | jit->seen |= SEEN_XREG | SEEN_MEM; | ||
618 | /* st %r12,<K>(%r15) */ | ||
619 | EMIT4_DISP(0x50c0f000, | ||
620 | (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); | ||
621 | break; | ||
622 | case BPF_S_ANC_PROTOCOL: /* A = ntohs(skb->protocol); */ | ||
623 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2); | ||
624 | /* lhi %r5,0 */ | ||
625 | EMIT4(0xa7580000); | ||
626 | /* icm %r5,3,<d(protocol)>(%r2) */ | ||
627 | EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, protocol)); | ||
628 | break; | ||
629 | case BPF_S_ANC_IFINDEX: /* if (!skb->dev) return 0; | ||
630 | * A = skb->dev->ifindex */ | ||
631 | BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); | ||
632 | jit->seen |= SEEN_RET0; | ||
633 | /* lg %r1,<d(dev)>(%r2) */ | ||
634 | EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); | ||
635 | /* ltgr %r1,%r1 */ | ||
636 | EMIT4(0xb9020011); | ||
637 | /* jz <ret0> */ | ||
638 | EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); | ||
639 | /* l %r5,<d(ifindex)>(%r1) */ | ||
640 | EMIT4_DISP(0x58501000, offsetof(struct net_device, ifindex)); | ||
641 | break; | ||
642 | case BPF_S_ANC_MARK: /* A = skb->mark */ | ||
643 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); | ||
644 | /* l %r5,<d(mark)>(%r2) */ | ||
645 | EMIT4_DISP(0x58502000, offsetof(struct sk_buff, mark)); | ||
646 | break; | ||
647 | case BPF_S_ANC_QUEUE: /* A = skb->queue_mapping */ | ||
648 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2); | ||
649 | /* lhi %r5,0 */ | ||
650 | EMIT4(0xa7580000); | ||
651 | /* icm %r5,3,<d(queue_mapping)>(%r2) */ | ||
652 | EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, queue_mapping)); | ||
653 | break; | ||
654 | case BPF_S_ANC_HATYPE: /* if (!skb->dev) return 0; | ||
655 | * A = skb->dev->type */ | ||
656 | BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2); | ||
657 | jit->seen |= SEEN_RET0; | ||
658 | /* lg %r1,<d(dev)>(%r2) */ | ||
659 | EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); | ||
660 | /* ltgr %r1,%r1 */ | ||
661 | EMIT4(0xb9020011); | ||
662 | /* jz <ret0> */ | ||
663 | EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); | ||
664 | /* lhi %r5,0 */ | ||
665 | EMIT4(0xa7580000); | ||
666 | /* icm %r5,3,<d(type)>(%r1) */ | ||
667 | EMIT4_DISP(0xbf531000, offsetof(struct net_device, type)); | ||
668 | break; | ||
669 | case BPF_S_ANC_RXHASH: /* A = skb->rxhash */ | ||
670 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, rxhash) != 4); | ||
671 | /* l %r5,<d(rxhash)>(%r2) */ | ||
672 | EMIT4_DISP(0x58502000, offsetof(struct sk_buff, rxhash)); | ||
673 | break; | ||
674 | case BPF_S_ANC_CPU: /* A = smp_processor_id() */ | ||
675 | #ifdef CONFIG_SMP | ||
676 | /* l %r5,<d(cpu_nr)> */ | ||
677 | EMIT4_DISP(0x58500000, offsetof(struct _lowcore, cpu_nr)); | ||
678 | #else | ||
679 | /* lhi %r5,0 */ | ||
680 | EMIT4(0xa7580000); | ||
681 | #endif | ||
682 | break; | ||
683 | default: /* too complex, give up */ | ||
684 | goto out; | ||
685 | } | ||
686 | addrs[i] = jit->prg - jit->start; | ||
687 | return 0; | ||
688 | out: | ||
689 | return -1; | ||
690 | } | ||
691 | |||
692 | void bpf_jit_compile(struct sk_filter *fp) | ||
693 | { | ||
694 | unsigned long size, prg_len, lit_len; | ||
695 | struct bpf_jit jit, cjit; | ||
696 | unsigned int *addrs; | ||
697 | int pass, i; | ||
698 | |||
699 | if (!bpf_jit_enable) | ||
700 | return; | ||
701 | addrs = kmalloc(fp->len * sizeof(*addrs), GFP_KERNEL); | ||
702 | if (addrs == NULL) | ||
703 | return; | ||
704 | memset(addrs, 0, fp->len * sizeof(*addrs)); | ||
705 | memset(&jit, 0, sizeof(cjit)); | ||
706 | memset(&cjit, 0, sizeof(cjit)); | ||
707 | |||
708 | for (pass = 0; pass < 10; pass++) { | ||
709 | jit.prg = jit.start; | ||
710 | jit.lit = jit.mid; | ||
711 | |||
712 | bpf_jit_prologue(&jit); | ||
713 | bpf_jit_noleaks(&jit, fp->insns); | ||
714 | for (i = 0; i < fp->len; i++) { | ||
715 | if (bpf_jit_insn(&jit, fp->insns + i, addrs, i, | ||
716 | i == fp->len - 1)) | ||
717 | goto out; | ||
718 | } | ||
719 | bpf_jit_epilogue(&jit); | ||
720 | if (jit.start) { | ||
721 | WARN_ON(jit.prg > cjit.prg || jit.lit > cjit.lit); | ||
722 | if (memcmp(&jit, &cjit, sizeof(jit)) == 0) | ||
723 | break; | ||
724 | } else if (jit.prg == cjit.prg && jit.lit == cjit.lit) { | ||
725 | prg_len = jit.prg - jit.start; | ||
726 | lit_len = jit.lit - jit.mid; | ||
727 | size = max_t(unsigned long, prg_len + lit_len, | ||
728 | sizeof(struct work_struct)); | ||
729 | if (size >= BPF_SIZE_MAX) | ||
730 | goto out; | ||
731 | jit.start = module_alloc(size); | ||
732 | if (!jit.start) | ||
733 | goto out; | ||
734 | jit.prg = jit.mid = jit.start + prg_len; | ||
735 | jit.lit = jit.end = jit.start + prg_len + lit_len; | ||
736 | jit.base_ip += (unsigned long) jit.start; | ||
737 | jit.exit_ip += (unsigned long) jit.start; | ||
738 | jit.ret0_ip += (unsigned long) jit.start; | ||
739 | } | ||
740 | cjit = jit; | ||
741 | } | ||
742 | if (bpf_jit_enable > 1) { | ||
743 | pr_err("flen=%d proglen=%lu pass=%d image=%p\n", | ||
744 | fp->len, jit.end - jit.start, pass, jit.start); | ||
745 | if (jit.start) { | ||
746 | printk(KERN_ERR "JIT code:\n"); | ||
747 | print_fn_code(jit.start, jit.mid - jit.start); | ||
748 | print_hex_dump(KERN_ERR, "JIT literals:\n", | ||
749 | DUMP_PREFIX_ADDRESS, 16, 1, | ||
750 | jit.mid, jit.end - jit.mid, false); | ||
751 | } | ||
752 | } | ||
753 | if (jit.start) | ||
754 | fp->bpf_func = (void *) jit.start; | ||
755 | out: | ||
756 | kfree(addrs); | ||
757 | } | ||
758 | |||
759 | static void jit_free_defer(struct work_struct *arg) | ||
760 | { | ||
761 | module_free(NULL, arg); | ||
762 | } | ||
763 | |||
764 | /* run from softirq, we must use a work_struct to call | ||
765 | * module_free() from process context | ||
766 | */ | ||
767 | void bpf_jit_free(struct sk_filter *fp) | ||
768 | { | ||
769 | struct work_struct *work; | ||
770 | |||
771 | if (fp->bpf_func == sk_run_filter) | ||
772 | return; | ||
773 | work = (struct work_struct *)fp->bpf_func; | ||
774 | INIT_WORK(work, jit_free_defer); | ||
775 | schedule_work(work); | ||
776 | } | ||