diff options
Diffstat (limited to 'arch/s390/net/bpf_jit_comp.c')
-rw-r--r-- | arch/s390/net/bpf_jit_comp.c | 127 |
1 files changed, 11 insertions, 116 deletions
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 78a19c93b380..b020bea040b7 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c | |||
@@ -47,23 +47,21 @@ struct bpf_jit { | |||
47 | 47 | ||
48 | #define BPF_SIZE_MAX 0xffff /* Max size for program (16 bit branches) */ | 48 | #define BPF_SIZE_MAX 0xffff /* Max size for program (16 bit branches) */ |
49 | 49 | ||
50 | #define SEEN_SKB 1 /* skb access */ | 50 | #define SEEN_MEM (1 << 0) /* use mem[] for temporary storage */ |
51 | #define SEEN_MEM 2 /* use mem[] for temporary storage */ | 51 | #define SEEN_RET0 (1 << 1) /* ret0_ip points to a valid return 0 */ |
52 | #define SEEN_RET0 4 /* ret0_ip points to a valid return 0 */ | 52 | #define SEEN_LITERAL (1 << 2) /* code uses literals */ |
53 | #define SEEN_LITERAL 8 /* code uses literals */ | 53 | #define SEEN_FUNC (1 << 3) /* calls C functions */ |
54 | #define SEEN_FUNC 16 /* calls C functions */ | 54 | #define SEEN_TAIL_CALL (1 << 4) /* code uses tail calls */ |
55 | #define SEEN_TAIL_CALL 32 /* code uses tail calls */ | 55 | #define SEEN_REG_AX (1 << 5) /* code uses constant blinding */ |
56 | #define SEEN_REG_AX 64 /* code uses constant blinding */ | 56 | #define SEEN_STACK (SEEN_FUNC | SEEN_MEM) |
57 | #define SEEN_STACK (SEEN_FUNC | SEEN_MEM | SEEN_SKB) | ||
58 | 57 | ||
59 | /* | 58 | /* |
60 | * s390 registers | 59 | * s390 registers |
61 | */ | 60 | */ |
62 | #define REG_W0 (MAX_BPF_JIT_REG + 0) /* Work register 1 (even) */ | 61 | #define REG_W0 (MAX_BPF_JIT_REG + 0) /* Work register 1 (even) */ |
63 | #define REG_W1 (MAX_BPF_JIT_REG + 1) /* Work register 2 (odd) */ | 62 | #define REG_W1 (MAX_BPF_JIT_REG + 1) /* Work register 2 (odd) */ |
64 | #define REG_SKB_DATA (MAX_BPF_JIT_REG + 2) /* SKB data register */ | 63 | #define REG_L (MAX_BPF_JIT_REG + 2) /* Literal pool register */ |
65 | #define REG_L (MAX_BPF_JIT_REG + 3) /* Literal pool register */ | 64 | #define REG_15 (MAX_BPF_JIT_REG + 3) /* Register 15 */ |
66 | #define REG_15 (MAX_BPF_JIT_REG + 4) /* Register 15 */ | ||
67 | #define REG_0 REG_W0 /* Register 0 */ | 65 | #define REG_0 REG_W0 /* Register 0 */ |
68 | #define REG_1 REG_W1 /* Register 1 */ | 66 | #define REG_1 REG_W1 /* Register 1 */ |
69 | #define REG_2 BPF_REG_1 /* Register 2 */ | 67 | #define REG_2 BPF_REG_1 /* Register 2 */ |
@@ -88,10 +86,8 @@ static const int reg2hex[] = { | |||
88 | [BPF_REG_9] = 10, | 86 | [BPF_REG_9] = 10, |
89 | /* BPF stack pointer */ | 87 | /* BPF stack pointer */ |
90 | [BPF_REG_FP] = 13, | 88 | [BPF_REG_FP] = 13, |
91 | /* Register for blinding (shared with REG_SKB_DATA) */ | 89 | /* Register for blinding */ |
92 | [BPF_REG_AX] = 12, | 90 | [BPF_REG_AX] = 12, |
93 | /* SKB data pointer */ | ||
94 | [REG_SKB_DATA] = 12, | ||
95 | /* Work registers for s390x backend */ | 91 | /* Work registers for s390x backend */ |
96 | [REG_W0] = 0, | 92 | [REG_W0] = 0, |
97 | [REG_W1] = 1, | 93 | [REG_W1] = 1, |
@@ -385,27 +381,6 @@ static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth) | |||
385 | } | 381 | } |
386 | 382 | ||
387 | /* | 383 | /* |
388 | * For SKB access %b1 contains the SKB pointer. For "bpf_jit.S" | ||
389 | * we store the SKB header length on the stack and the SKB data | ||
390 | * pointer in REG_SKB_DATA if BPF_REG_AX is not used. | ||
391 | */ | ||
392 | static void emit_load_skb_data_hlen(struct bpf_jit *jit) | ||
393 | { | ||
394 | /* Header length: llgf %w1,<len>(%b1) */ | ||
395 | EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_1, | ||
396 | offsetof(struct sk_buff, len)); | ||
397 | /* s %w1,<data_len>(%b1) */ | ||
398 | EMIT4_DISP(0x5b000000, REG_W1, BPF_REG_1, | ||
399 | offsetof(struct sk_buff, data_len)); | ||
400 | /* stg %w1,ST_OFF_HLEN(%r0,%r15) */ | ||
401 | EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15, STK_OFF_HLEN); | ||
402 | if (!(jit->seen & SEEN_REG_AX)) | ||
403 | /* lg %skb_data,data_off(%b1) */ | ||
404 | EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0, | ||
405 | BPF_REG_1, offsetof(struct sk_buff, data)); | ||
406 | } | ||
407 | |||
408 | /* | ||
409 | * Emit function prologue | 384 | * Emit function prologue |
410 | * | 385 | * |
411 | * Save registers and create stack frame if necessary. | 386 | * Save registers and create stack frame if necessary. |
@@ -445,12 +420,6 @@ static void bpf_jit_prologue(struct bpf_jit *jit, u32 stack_depth) | |||
445 | EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, | 420 | EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, |
446 | REG_15, 152); | 421 | REG_15, 152); |
447 | } | 422 | } |
448 | if (jit->seen & SEEN_SKB) { | ||
449 | emit_load_skb_data_hlen(jit); | ||
450 | /* stg %b1,ST_OFF_SKBP(%r0,%r15) */ | ||
451 | EMIT6_DISP_LH(0xe3000000, 0x0024, BPF_REG_1, REG_0, REG_15, | ||
452 | STK_OFF_SKBP); | ||
453 | } | ||
454 | } | 423 | } |
455 | 424 | ||
456 | /* | 425 | /* |
@@ -483,12 +452,12 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i | |||
483 | { | 452 | { |
484 | struct bpf_insn *insn = &fp->insnsi[i]; | 453 | struct bpf_insn *insn = &fp->insnsi[i]; |
485 | int jmp_off, last, insn_count = 1; | 454 | int jmp_off, last, insn_count = 1; |
486 | unsigned int func_addr, mask; | ||
487 | u32 dst_reg = insn->dst_reg; | 455 | u32 dst_reg = insn->dst_reg; |
488 | u32 src_reg = insn->src_reg; | 456 | u32 src_reg = insn->src_reg; |
489 | u32 *addrs = jit->addrs; | 457 | u32 *addrs = jit->addrs; |
490 | s32 imm = insn->imm; | 458 | s32 imm = insn->imm; |
491 | s16 off = insn->off; | 459 | s16 off = insn->off; |
460 | unsigned int mask; | ||
492 | 461 | ||
493 | if (dst_reg == BPF_REG_AX || src_reg == BPF_REG_AX) | 462 | if (dst_reg == BPF_REG_AX || src_reg == BPF_REG_AX) |
494 | jit->seen |= SEEN_REG_AX; | 463 | jit->seen |= SEEN_REG_AX; |
@@ -970,13 +939,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i | |||
970 | EMIT2(0x0d00, REG_14, REG_W1); | 939 | EMIT2(0x0d00, REG_14, REG_W1); |
971 | /* lgr %b0,%r2: load return value into %b0 */ | 940 | /* lgr %b0,%r2: load return value into %b0 */ |
972 | EMIT4(0xb9040000, BPF_REG_0, REG_2); | 941 | EMIT4(0xb9040000, BPF_REG_0, REG_2); |
973 | if ((jit->seen & SEEN_SKB) && | ||
974 | bpf_helper_changes_pkt_data((void *)func)) { | ||
975 | /* lg %b1,ST_OFF_SKBP(%r15) */ | ||
976 | EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_1, REG_0, | ||
977 | REG_15, STK_OFF_SKBP); | ||
978 | emit_load_skb_data_hlen(jit); | ||
979 | } | ||
980 | break; | 942 | break; |
981 | } | 943 | } |
982 | case BPF_JMP | BPF_TAIL_CALL: | 944 | case BPF_JMP | BPF_TAIL_CALL: |
@@ -1176,73 +1138,6 @@ branch_oc: | |||
1176 | jmp_off = addrs[i + off + 1] - (addrs[i + 1] - 4); | 1138 | jmp_off = addrs[i + off + 1] - (addrs[i + 1] - 4); |
1177 | EMIT4_PCREL(0xa7040000 | mask << 8, jmp_off); | 1139 | EMIT4_PCREL(0xa7040000 | mask << 8, jmp_off); |
1178 | break; | 1140 | break; |
1179 | /* | ||
1180 | * BPF_LD | ||
1181 | */ | ||
1182 | case BPF_LD | BPF_ABS | BPF_B: /* b0 = *(u8 *) (skb->data+imm) */ | ||
1183 | case BPF_LD | BPF_IND | BPF_B: /* b0 = *(u8 *) (skb->data+imm+src) */ | ||
1184 | if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0)) | ||
1185 | func_addr = __pa(sk_load_byte_pos); | ||
1186 | else | ||
1187 | func_addr = __pa(sk_load_byte); | ||
1188 | goto call_fn; | ||
1189 | case BPF_LD | BPF_ABS | BPF_H: /* b0 = *(u16 *) (skb->data+imm) */ | ||
1190 | case BPF_LD | BPF_IND | BPF_H: /* b0 = *(u16 *) (skb->data+imm+src) */ | ||
1191 | if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0)) | ||
1192 | func_addr = __pa(sk_load_half_pos); | ||
1193 | else | ||
1194 | func_addr = __pa(sk_load_half); | ||
1195 | goto call_fn; | ||
1196 | case BPF_LD | BPF_ABS | BPF_W: /* b0 = *(u32 *) (skb->data+imm) */ | ||
1197 | case BPF_LD | BPF_IND | BPF_W: /* b0 = *(u32 *) (skb->data+imm+src) */ | ||
1198 | if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0)) | ||
1199 | func_addr = __pa(sk_load_word_pos); | ||
1200 | else | ||
1201 | func_addr = __pa(sk_load_word); | ||
1202 | goto call_fn; | ||
1203 | call_fn: | ||
1204 | jit->seen |= SEEN_SKB | SEEN_RET0 | SEEN_FUNC; | ||
1205 | REG_SET_SEEN(REG_14); /* Return address of possible func call */ | ||
1206 | |||
1207 | /* | ||
1208 | * Implicit input: | ||
1209 | * BPF_REG_6 (R7) : skb pointer | ||
1210 | * REG_SKB_DATA (R12): skb data pointer (if no BPF_REG_AX) | ||
1211 | * | ||
1212 | * Calculated input: | ||
1213 | * BPF_REG_2 (R3) : offset of byte(s) to fetch in skb | ||
1214 | * BPF_REG_5 (R6) : return address | ||
1215 | * | ||
1216 | * Output: | ||
1217 | * BPF_REG_0 (R14): data read from skb | ||
1218 | * | ||
1219 | * Scratch registers (BPF_REG_1-5) | ||
1220 | */ | ||
1221 | |||
1222 | /* Call function: llilf %w1,func_addr */ | ||
1223 | EMIT6_IMM(0xc00f0000, REG_W1, func_addr); | ||
1224 | |||
1225 | /* Offset: lgfi %b2,imm */ | ||
1226 | EMIT6_IMM(0xc0010000, BPF_REG_2, imm); | ||
1227 | if (BPF_MODE(insn->code) == BPF_IND) | ||
1228 | /* agfr %b2,%src (%src is s32 here) */ | ||
1229 | EMIT4(0xb9180000, BPF_REG_2, src_reg); | ||
1230 | |||
1231 | /* Reload REG_SKB_DATA if BPF_REG_AX is used */ | ||
1232 | if (jit->seen & SEEN_REG_AX) | ||
1233 | /* lg %skb_data,data_off(%b6) */ | ||
1234 | EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0, | ||
1235 | BPF_REG_6, offsetof(struct sk_buff, data)); | ||
1236 | /* basr %b5,%w1 (%b5 is call saved) */ | ||
1237 | EMIT2(0x0d00, BPF_REG_5, REG_W1); | ||
1238 | |||
1239 | /* | ||
1240 | * Note: For fast access we jump directly after the | ||
1241 | * jnz instruction from bpf_jit.S | ||
1242 | */ | ||
1243 | /* jnz <ret0> */ | ||
1244 | EMIT4_PCREL(0xa7740000, jit->ret0_ip - jit->prg); | ||
1245 | break; | ||
1246 | default: /* too complex, give up */ | 1141 | default: /* too complex, give up */ |
1247 | pr_err("Unknown opcode %02x\n", insn->code); | 1142 | pr_err("Unknown opcode %02x\n", insn->code); |
1248 | return -1; | 1143 | return -1; |