aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/net/bpf_jit_comp32.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/net/bpf_jit_comp32.c')
-rw-r--r--arch/x86/net/bpf_jit_comp32.c83
1 files changed, 56 insertions, 27 deletions
diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c
index b29e82f190c7..133433d181ba 100644
--- a/arch/x86/net/bpf_jit_comp32.c
+++ b/arch/x86/net/bpf_jit_comp32.c
@@ -253,13 +253,14 @@ static inline void emit_ia32_mov_r(const u8 dst, const u8 src, bool dstk,
253/* dst = src */ 253/* dst = src */
254static inline void emit_ia32_mov_r64(const bool is64, const u8 dst[], 254static inline void emit_ia32_mov_r64(const bool is64, const u8 dst[],
255 const u8 src[], bool dstk, 255 const u8 src[], bool dstk,
256 bool sstk, u8 **pprog) 256 bool sstk, u8 **pprog,
257 const struct bpf_prog_aux *aux)
257{ 258{
258 emit_ia32_mov_r(dst_lo, src_lo, dstk, sstk, pprog); 259 emit_ia32_mov_r(dst_lo, src_lo, dstk, sstk, pprog);
259 if (is64) 260 if (is64)
260 /* complete 8 byte move */ 261 /* complete 8 byte move */
261 emit_ia32_mov_r(dst_hi, src_hi, dstk, sstk, pprog); 262 emit_ia32_mov_r(dst_hi, src_hi, dstk, sstk, pprog);
262 else 263 else if (!aux->verifier_zext)
263 /* zero out high 4 bytes */ 264 /* zero out high 4 bytes */
264 emit_ia32_mov_i(dst_hi, 0, dstk, pprog); 265 emit_ia32_mov_i(dst_hi, 0, dstk, pprog);
265} 266}
@@ -313,7 +314,8 @@ static inline void emit_ia32_mul_r(const u8 dst, const u8 src, bool dstk,
313} 314}
314 315
315static inline void emit_ia32_to_le_r64(const u8 dst[], s32 val, 316static inline void emit_ia32_to_le_r64(const u8 dst[], s32 val,
316 bool dstk, u8 **pprog) 317 bool dstk, u8 **pprog,
318 const struct bpf_prog_aux *aux)
317{ 319{
318 u8 *prog = *pprog; 320 u8 *prog = *pprog;
319 int cnt = 0; 321 int cnt = 0;
@@ -334,12 +336,14 @@ static inline void emit_ia32_to_le_r64(const u8 dst[], s32 val,
334 */ 336 */
335 EMIT2(0x0F, 0xB7); 337 EMIT2(0x0F, 0xB7);
336 EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo)); 338 EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo));
337 /* xor dreg_hi,dreg_hi */ 339 if (!aux->verifier_zext)
338 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); 340 /* xor dreg_hi,dreg_hi */
341 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
339 break; 342 break;
340 case 32: 343 case 32:
341 /* xor dreg_hi,dreg_hi */ 344 if (!aux->verifier_zext)
342 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); 345 /* xor dreg_hi,dreg_hi */
346 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
343 break; 347 break;
344 case 64: 348 case 64:
345 /* nop */ 349 /* nop */
@@ -358,7 +362,8 @@ static inline void emit_ia32_to_le_r64(const u8 dst[], s32 val,
358} 362}
359 363
360static inline void emit_ia32_to_be_r64(const u8 dst[], s32 val, 364static inline void emit_ia32_to_be_r64(const u8 dst[], s32 val,
361 bool dstk, u8 **pprog) 365 bool dstk, u8 **pprog,
366 const struct bpf_prog_aux *aux)
362{ 367{
363 u8 *prog = *pprog; 368 u8 *prog = *pprog;
364 int cnt = 0; 369 int cnt = 0;
@@ -380,16 +385,18 @@ static inline void emit_ia32_to_be_r64(const u8 dst[], s32 val,
380 EMIT2(0x0F, 0xB7); 385 EMIT2(0x0F, 0xB7);
381 EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo)); 386 EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo));
382 387
383 /* xor dreg_hi,dreg_hi */ 388 if (!aux->verifier_zext)
384 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); 389 /* xor dreg_hi,dreg_hi */
390 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
385 break; 391 break;
386 case 32: 392 case 32:
387 /* Emit 'bswap eax' to swap lower 4 bytes */ 393 /* Emit 'bswap eax' to swap lower 4 bytes */
388 EMIT1(0x0F); 394 EMIT1(0x0F);
389 EMIT1(add_1reg(0xC8, dreg_lo)); 395 EMIT1(add_1reg(0xC8, dreg_lo));
390 396
391 /* xor dreg_hi,dreg_hi */ 397 if (!aux->verifier_zext)
392 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); 398 /* xor dreg_hi,dreg_hi */
399 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
393 break; 400 break;
394 case 64: 401 case 64:
395 /* Emit 'bswap eax' to swap lower 4 bytes */ 402 /* Emit 'bswap eax' to swap lower 4 bytes */
@@ -569,7 +576,7 @@ static inline void emit_ia32_alu_r(const bool is64, const bool hi, const u8 op,
569static inline void emit_ia32_alu_r64(const bool is64, const u8 op, 576static inline void emit_ia32_alu_r64(const bool is64, const u8 op,
570 const u8 dst[], const u8 src[], 577 const u8 dst[], const u8 src[],
571 bool dstk, bool sstk, 578 bool dstk, bool sstk,
572 u8 **pprog) 579 u8 **pprog, const struct bpf_prog_aux *aux)
573{ 580{
574 u8 *prog = *pprog; 581 u8 *prog = *pprog;
575 582
@@ -577,7 +584,7 @@ static inline void emit_ia32_alu_r64(const bool is64, const u8 op,
577 if (is64) 584 if (is64)
578 emit_ia32_alu_r(is64, true, op, dst_hi, src_hi, dstk, sstk, 585 emit_ia32_alu_r(is64, true, op, dst_hi, src_hi, dstk, sstk,
579 &prog); 586 &prog);
580 else 587 else if (!aux->verifier_zext)
581 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); 588 emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
582 *pprog = prog; 589 *pprog = prog;
583} 590}
@@ -668,7 +675,8 @@ static inline void emit_ia32_alu_i(const bool is64, const bool hi, const u8 op,
668/* ALU operation (64 bit) */ 675/* ALU operation (64 bit) */
669static inline void emit_ia32_alu_i64(const bool is64, const u8 op, 676static inline void emit_ia32_alu_i64(const bool is64, const u8 op,
670 const u8 dst[], const u32 val, 677 const u8 dst[], const u32 val,
671 bool dstk, u8 **pprog) 678 bool dstk, u8 **pprog,
679 const struct bpf_prog_aux *aux)
672{ 680{
673 u8 *prog = *pprog; 681 u8 *prog = *pprog;
674 u32 hi = 0; 682 u32 hi = 0;
@@ -679,7 +687,7 @@ static inline void emit_ia32_alu_i64(const bool is64, const u8 op,
679 emit_ia32_alu_i(is64, false, op, dst_lo, val, dstk, &prog); 687 emit_ia32_alu_i(is64, false, op, dst_lo, val, dstk, &prog);
680 if (is64) 688 if (is64)
681 emit_ia32_alu_i(is64, true, op, dst_hi, hi, dstk, &prog); 689 emit_ia32_alu_i(is64, true, op, dst_hi, hi, dstk, &prog);
682 else 690 else if (!aux->verifier_zext)
683 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); 691 emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
684 692
685 *pprog = prog; 693 *pprog = prog;
@@ -1713,8 +1721,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
1713 case BPF_ALU64 | BPF_MOV | BPF_X: 1721 case BPF_ALU64 | BPF_MOV | BPF_X:
1714 switch (BPF_SRC(code)) { 1722 switch (BPF_SRC(code)) {
1715 case BPF_X: 1723 case BPF_X:
1716 emit_ia32_mov_r64(is64, dst, src, dstk, 1724 if (imm32 == 1) {
1717 sstk, &prog); 1725 /* Special mov32 for zext. */
1726 emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1727 break;
1728 }
1729 emit_ia32_mov_r64(is64, dst, src, dstk, sstk,
1730 &prog, bpf_prog->aux);
1718 break; 1731 break;
1719 case BPF_K: 1732 case BPF_K:
1720 /* Sign-extend immediate value to dst reg */ 1733 /* Sign-extend immediate value to dst reg */
@@ -1754,11 +1767,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
1754 switch (BPF_SRC(code)) { 1767 switch (BPF_SRC(code)) {
1755 case BPF_X: 1768 case BPF_X:
1756 emit_ia32_alu_r64(is64, BPF_OP(code), dst, 1769 emit_ia32_alu_r64(is64, BPF_OP(code), dst,
1757 src, dstk, sstk, &prog); 1770 src, dstk, sstk, &prog,
1771 bpf_prog->aux);
1758 break; 1772 break;
1759 case BPF_K: 1773 case BPF_K:
1760 emit_ia32_alu_i64(is64, BPF_OP(code), dst, 1774 emit_ia32_alu_i64(is64, BPF_OP(code), dst,
1761 imm32, dstk, &prog); 1775 imm32, dstk, &prog,
1776 bpf_prog->aux);
1762 break; 1777 break;
1763 } 1778 }
1764 break; 1779 break;
@@ -1777,7 +1792,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
1777 false, &prog); 1792 false, &prog);
1778 break; 1793 break;
1779 } 1794 }
1780 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); 1795 if (!bpf_prog->aux->verifier_zext)
1796 emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1781 break; 1797 break;
1782 case BPF_ALU | BPF_LSH | BPF_X: 1798 case BPF_ALU | BPF_LSH | BPF_X:
1783 case BPF_ALU | BPF_RSH | BPF_X: 1799 case BPF_ALU | BPF_RSH | BPF_X:
@@ -1797,7 +1813,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
1797 &prog); 1813 &prog);
1798 break; 1814 break;
1799 } 1815 }
1800 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); 1816 if (!bpf_prog->aux->verifier_zext)
1817 emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1801 break; 1818 break;
1802 /* dst = dst / src(imm) */ 1819 /* dst = dst / src(imm) */
1803 /* dst = dst % src(imm) */ 1820 /* dst = dst % src(imm) */
@@ -1819,7 +1836,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
1819 &prog); 1836 &prog);
1820 break; 1837 break;
1821 } 1838 }
1822 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); 1839 if (!bpf_prog->aux->verifier_zext)
1840 emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1823 break; 1841 break;
1824 case BPF_ALU64 | BPF_DIV | BPF_K: 1842 case BPF_ALU64 | BPF_DIV | BPF_K:
1825 case BPF_ALU64 | BPF_DIV | BPF_X: 1843 case BPF_ALU64 | BPF_DIV | BPF_X:
@@ -1836,7 +1854,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
1836 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32); 1854 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32);
1837 emit_ia32_shift_r(BPF_OP(code), dst_lo, IA32_ECX, dstk, 1855 emit_ia32_shift_r(BPF_OP(code), dst_lo, IA32_ECX, dstk,
1838 false, &prog); 1856 false, &prog);
1839 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); 1857 if (!bpf_prog->aux->verifier_zext)
1858 emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1840 break; 1859 break;
1841 /* dst = dst << imm */ 1860 /* dst = dst << imm */
1842 case BPF_ALU64 | BPF_LSH | BPF_K: 1861 case BPF_ALU64 | BPF_LSH | BPF_K:
@@ -1872,7 +1891,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
1872 case BPF_ALU | BPF_NEG: 1891 case BPF_ALU | BPF_NEG:
1873 emit_ia32_alu_i(is64, false, BPF_OP(code), 1892 emit_ia32_alu_i(is64, false, BPF_OP(code),
1874 dst_lo, 0, dstk, &prog); 1893 dst_lo, 0, dstk, &prog);
1875 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); 1894 if (!bpf_prog->aux->verifier_zext)
1895 emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1876 break; 1896 break;
1877 /* dst = ~dst (64 bit) */ 1897 /* dst = ~dst (64 bit) */
1878 case BPF_ALU64 | BPF_NEG: 1898 case BPF_ALU64 | BPF_NEG:
@@ -1892,11 +1912,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
1892 break; 1912 break;
1893 /* dst = htole(dst) */ 1913 /* dst = htole(dst) */
1894 case BPF_ALU | BPF_END | BPF_FROM_LE: 1914 case BPF_ALU | BPF_END | BPF_FROM_LE:
1895 emit_ia32_to_le_r64(dst, imm32, dstk, &prog); 1915 emit_ia32_to_le_r64(dst, imm32, dstk, &prog,
1916 bpf_prog->aux);
1896 break; 1917 break;
1897 /* dst = htobe(dst) */ 1918 /* dst = htobe(dst) */
1898 case BPF_ALU | BPF_END | BPF_FROM_BE: 1919 case BPF_ALU | BPF_END | BPF_FROM_BE:
1899 emit_ia32_to_be_r64(dst, imm32, dstk, &prog); 1920 emit_ia32_to_be_r64(dst, imm32, dstk, &prog,
1921 bpf_prog->aux);
1900 break; 1922 break;
1901 /* dst = imm64 */ 1923 /* dst = imm64 */
1902 case BPF_LD | BPF_IMM | BPF_DW: { 1924 case BPF_LD | BPF_IMM | BPF_DW: {
@@ -2051,6 +2073,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
2051 case BPF_B: 2073 case BPF_B:
2052 case BPF_H: 2074 case BPF_H:
2053 case BPF_W: 2075 case BPF_W:
2076 if (!bpf_prog->aux->verifier_zext)
2077 break;
2054 if (dstk) { 2078 if (dstk) {
2055 EMIT3(0xC7, add_1reg(0x40, IA32_EBP), 2079 EMIT3(0xC7, add_1reg(0x40, IA32_EBP),
2056 STACK_VAR(dst_hi)); 2080 STACK_VAR(dst_hi));
@@ -2475,6 +2499,11 @@ notyet:
2475 return proglen; 2499 return proglen;
2476} 2500}
2477 2501
2502bool bpf_jit_needs_zext(void)
2503{
2504 return true;
2505}
2506
2478struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 2507struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
2479{ 2508{
2480 struct bpf_binary_header *header = NULL; 2509 struct bpf_binary_header *header = NULL;
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-02 14:23:07 -0400