aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/powerpc/net/Makefile2
-rw-r--r--arch/powerpc/net/bpf_jit64.h37
-rw-r--r--arch/powerpc/net/bpf_jit_asm64.S180
-rw-r--r--arch/powerpc/net/bpf_jit_comp64.c109
4 files changed, 11 insertions, 317 deletions
diff --git a/arch/powerpc/net/Makefile b/arch/powerpc/net/Makefile
index 02d369ca6a53..809f019d3cba 100644
--- a/arch/powerpc/net/Makefile
+++ b/arch/powerpc/net/Makefile
@@ -3,7 +3,7 @@
3# Arch-specific network modules 3# Arch-specific network modules
4# 4#
5ifeq ($(CONFIG_PPC64),y) 5ifeq ($(CONFIG_PPC64),y)
6obj-$(CONFIG_BPF_JIT) += bpf_jit_asm64.o bpf_jit_comp64.o 6obj-$(CONFIG_BPF_JIT) += bpf_jit_comp64.o
7else 7else
8obj-$(CONFIG_BPF_JIT) += bpf_jit_asm.o bpf_jit_comp.o 8obj-$(CONFIG_BPF_JIT) += bpf_jit_asm.o bpf_jit_comp.o
9endif 9endif
diff --git a/arch/powerpc/net/bpf_jit64.h b/arch/powerpc/net/bpf_jit64.h
index 8bdef7ed28a8..3609be4692b3 100644
--- a/arch/powerpc/net/bpf_jit64.h
+++ b/arch/powerpc/net/bpf_jit64.h
@@ -20,7 +20,7 @@
20 * with our redzone usage. 20 * with our redzone usage.
21 * 21 *
22 * [ prev sp ] <------------- 22 * [ prev sp ] <-------------
23 * [ nv gpr save area ] 8*8 | 23 * [ nv gpr save area ] 6*8 |
24 * [ tail_call_cnt ] 8 | 24 * [ tail_call_cnt ] 8 |
25 * [ local_tmp_var ] 8 | 25 * [ local_tmp_var ] 8 |
26 * fp (r31) --> [ ebpf stack space ] upto 512 | 26 * fp (r31) --> [ ebpf stack space ] upto 512 |
@@ -28,8 +28,8 @@
28 * sp (r1) ---> [ stack pointer ] -------------- 28 * sp (r1) ---> [ stack pointer ] --------------
29 */ 29 */
30 30
31/* for gpr non volatile registers BPG_REG_6 to 10, plus skb cache registers */ 31/* for gpr non volatile registers BPG_REG_6 to 10 */
32#define BPF_PPC_STACK_SAVE (8*8) 32#define BPF_PPC_STACK_SAVE (6*8)
33/* for bpf JIT code internal usage */ 33/* for bpf JIT code internal usage */
34#define BPF_PPC_STACK_LOCALS 16 34#define BPF_PPC_STACK_LOCALS 16
35/* stack frame excluding BPF stack, ensure this is quadword aligned */ 35/* stack frame excluding BPF stack, ensure this is quadword aligned */
@@ -39,10 +39,8 @@
39#ifndef __ASSEMBLY__ 39#ifndef __ASSEMBLY__
40 40
41/* BPF register usage */ 41/* BPF register usage */
42#define SKB_HLEN_REG (MAX_BPF_JIT_REG + 0) 42#define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
43#define SKB_DATA_REG (MAX_BPF_JIT_REG + 1) 43#define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
44#define TMP_REG_1 (MAX_BPF_JIT_REG + 2)
45#define TMP_REG_2 (MAX_BPF_JIT_REG + 3)
46 44
47/* BPF to ppc register mappings */ 45/* BPF to ppc register mappings */
48static const int b2p[] = { 46static const int b2p[] = {
@@ -63,40 +61,23 @@ static const int b2p[] = {
63 [BPF_REG_FP] = 31, 61 [BPF_REG_FP] = 31,
64 /* eBPF jit internal registers */ 62 /* eBPF jit internal registers */
65 [BPF_REG_AX] = 2, 63 [BPF_REG_AX] = 2,
66 [SKB_HLEN_REG] = 25,
67 [SKB_DATA_REG] = 26,
68 [TMP_REG_1] = 9, 64 [TMP_REG_1] = 9,
69 [TMP_REG_2] = 10 65 [TMP_REG_2] = 10
70}; 66};
71 67
72/* PPC NVR range -- update this if we ever use NVRs below r24 */ 68/* PPC NVR range -- update this if we ever use NVRs below r27 */
73#define BPF_PPC_NVR_MIN 24 69#define BPF_PPC_NVR_MIN 27
74
75/* Assembly helpers */
76#define DECLARE_LOAD_FUNC(func) u64 func(u64 r3, u64 r4); \
77 u64 func##_negative_offset(u64 r3, u64 r4); \
78 u64 func##_positive_offset(u64 r3, u64 r4);
79
80DECLARE_LOAD_FUNC(sk_load_word);
81DECLARE_LOAD_FUNC(sk_load_half);
82DECLARE_LOAD_FUNC(sk_load_byte);
83
84#define CHOOSE_LOAD_FUNC(imm, func) \
85 (imm < 0 ? \
86 (imm >= SKF_LL_OFF ? func##_negative_offset : func) : \
87 func##_positive_offset)
88 70
89#define SEEN_FUNC 0x1000 /* might call external helpers */ 71#define SEEN_FUNC 0x1000 /* might call external helpers */
90#define SEEN_STACK 0x2000 /* uses BPF stack */ 72#define SEEN_STACK 0x2000 /* uses BPF stack */
91#define SEEN_SKB 0x4000 /* uses sk_buff */ 73#define SEEN_TAILCALL 0x4000 /* uses tail calls */
92#define SEEN_TAILCALL 0x8000 /* uses tail calls */
93 74
94struct codegen_context { 75struct codegen_context {
95 /* 76 /*
96 * This is used to track register usage as well 77 * This is used to track register usage as well
97 * as calls to external helpers. 78 * as calls to external helpers.
98 * - register usage is tracked with corresponding 79 * - register usage is tracked with corresponding
99 * bits (r3-r10 and r25-r31) 80 * bits (r3-r10 and r27-r31)
100 * - rest of the bits can be used to track other 81 * - rest of the bits can be used to track other
101 * things -- for now, we use bits 16 to 23 82 * things -- for now, we use bits 16 to 23
102 * encoded in SEEN_* macros above 83 * encoded in SEEN_* macros above
diff --git a/arch/powerpc/net/bpf_jit_asm64.S b/arch/powerpc/net/bpf_jit_asm64.S
deleted file mode 100644
index 7e4c51430b84..000000000000
--- a/arch/powerpc/net/bpf_jit_asm64.S
+++ /dev/null
@@ -1,180 +0,0 @@
1/*
2 * bpf_jit_asm64.S: Packet/header access helper functions
3 * for PPC64 BPF compiler.
4 *
5 * Copyright 2016, Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
6 * IBM Corporation
7 *
8 * Based on bpf_jit_asm.S by Matt Evans
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; version 2
13 * of the License.
14 */
15
16#include <asm/ppc_asm.h>
17#include <asm/ptrace.h>
18#include "bpf_jit64.h"
19
20/*
21 * All of these routines are called directly from generated code,
22 * with the below register usage:
23 * r27 skb pointer (ctx)
24 * r25 skb header length
25 * r26 skb->data pointer
26 * r4 offset
27 *
28 * Result is passed back in:
29 * r8 data read in host endian format (accumulator)
30 *
31 * r9 is used as a temporary register
32 */
33
34#define r_skb r27
35#define r_hlen r25
36#define r_data r26
37#define r_off r4
38#define r_val r8
39#define r_tmp r9
40
41_GLOBAL_TOC(sk_load_word)
42 cmpdi r_off, 0
43 blt bpf_slow_path_word_neg
44 b sk_load_word_positive_offset
45
46_GLOBAL_TOC(sk_load_word_positive_offset)
47 /* Are we accessing past headlen? */
48 subi r_tmp, r_hlen, 4
49 cmpd r_tmp, r_off
50 blt bpf_slow_path_word
51 /* Nope, just hitting the header. cr0 here is eq or gt! */
52 LWZX_BE r_val, r_data, r_off
53 blr /* Return success, cr0 != LT */
54
55_GLOBAL_TOC(sk_load_half)
56 cmpdi r_off, 0
57 blt bpf_slow_path_half_neg
58 b sk_load_half_positive_offset
59
60_GLOBAL_TOC(sk_load_half_positive_offset)
61 subi r_tmp, r_hlen, 2
62 cmpd r_tmp, r_off
63 blt bpf_slow_path_half
64 LHZX_BE r_val, r_data, r_off
65 blr
66
67_GLOBAL_TOC(sk_load_byte)
68 cmpdi r_off, 0
69 blt bpf_slow_path_byte_neg
70 b sk_load_byte_positive_offset
71
72_GLOBAL_TOC(sk_load_byte_positive_offset)
73 cmpd r_hlen, r_off
74 ble bpf_slow_path_byte
75 lbzx r_val, r_data, r_off
76 blr
77
78/*
79 * Call out to skb_copy_bits:
80 * Allocate a new stack frame here to remain ABI-compliant in
81 * stashing LR.
82 */
83#define bpf_slow_path_common(SIZE) \
84 mflr r0; \
85 std r0, PPC_LR_STKOFF(r1); \
86 stdu r1, -(STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_LOCALS)(r1); \
87 mr r3, r_skb; \
88 /* r4 = r_off as passed */ \
89 addi r5, r1, STACK_FRAME_MIN_SIZE; \
90 li r6, SIZE; \
91 bl skb_copy_bits; \
92 nop; \
93 /* save r5 */ \
94 addi r5, r1, STACK_FRAME_MIN_SIZE; \
95 /* r3 = 0 on success */ \
96 addi r1, r1, STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_LOCALS; \
97 ld r0, PPC_LR_STKOFF(r1); \
98 mtlr r0; \
99 cmpdi r3, 0; \
100 blt bpf_error; /* cr0 = LT */
101
102bpf_slow_path_word:
103 bpf_slow_path_common(4)
104 /* Data value is on stack, and cr0 != LT */
105 LWZX_BE r_val, 0, r5
106 blr
107
108bpf_slow_path_half:
109 bpf_slow_path_common(2)
110 LHZX_BE r_val, 0, r5
111 blr
112
113bpf_slow_path_byte:
114 bpf_slow_path_common(1)
115 lbzx r_val, 0, r5
116 blr
117
118/*
119 * Call out to bpf_internal_load_pointer_neg_helper
120 */
121#define sk_negative_common(SIZE) \
122 mflr r0; \
123 std r0, PPC_LR_STKOFF(r1); \
124 stdu r1, -STACK_FRAME_MIN_SIZE(r1); \
125 mr r3, r_skb; \
126 /* r4 = r_off, as passed */ \
127 li r5, SIZE; \
128 bl bpf_internal_load_pointer_neg_helper; \
129 nop; \
130 addi r1, r1, STACK_FRAME_MIN_SIZE; \
131 ld r0, PPC_LR_STKOFF(r1); \
132 mtlr r0; \
133 /* R3 != 0 on success */ \
134 cmpldi r3, 0; \
135 beq bpf_error_slow; /* cr0 = EQ */
136
137bpf_slow_path_word_neg:
138 lis r_tmp, -32 /* SKF_LL_OFF */
139 cmpd r_off, r_tmp /* addr < SKF_* */
140 blt bpf_error /* cr0 = LT */
141 b sk_load_word_negative_offset
142
143_GLOBAL_TOC(sk_load_word_negative_offset)
144 sk_negative_common(4)
145 LWZX_BE r_val, 0, r3
146 blr
147
148bpf_slow_path_half_neg:
149 lis r_tmp, -32 /* SKF_LL_OFF */
150 cmpd r_off, r_tmp /* addr < SKF_* */
151 blt bpf_error /* cr0 = LT */
152 b sk_load_half_negative_offset
153
154_GLOBAL_TOC(sk_load_half_negative_offset)
155 sk_negative_common(2)
156 LHZX_BE r_val, 0, r3
157 blr
158
159bpf_slow_path_byte_neg:
160 lis r_tmp, -32 /* SKF_LL_OFF */
161 cmpd r_off, r_tmp /* addr < SKF_* */
162 blt bpf_error /* cr0 = LT */
163 b sk_load_byte_negative_offset
164
165_GLOBAL_TOC(sk_load_byte_negative_offset)
166 sk_negative_common(1)
167 lbzx r_val, 0, r3
168 blr
169
170bpf_error_slow:
171 /* fabricate a cr0 = lt */
172 li r_tmp, -1
173 cmpdi r_tmp, 0
174bpf_error:
175 /*
176 * Entered with cr0 = lt
177 * Generated code will 'blt epilogue', returning 0.
178 */
179 li r_val, 0
180 blr
diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
index 0ef3d9580e98..1bdb1aff0619 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -59,7 +59,7 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx)
59 * [ prev sp ] <------------- 59 * [ prev sp ] <-------------
60 * [ ... ] | 60 * [ ... ] |
61 * sp (r1) ---> [ stack pointer ] -------------- 61 * sp (r1) ---> [ stack pointer ] --------------
62 * [ nv gpr save area ] 8*8 62 * [ nv gpr save area ] 6*8
63 * [ tail_call_cnt ] 8 63 * [ tail_call_cnt ] 8
64 * [ local_tmp_var ] 8 64 * [ local_tmp_var ] 8
65 * [ unused red zone ] 208 bytes protected 65 * [ unused red zone ] 208 bytes protected
@@ -88,21 +88,6 @@ static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
88 BUG(); 88 BUG();
89} 89}
90 90
91static void bpf_jit_emit_skb_loads(u32 *image, struct codegen_context *ctx)
92{
93 /*
94 * Load skb->len and skb->data_len
95 * r3 points to skb
96 */
97 PPC_LWZ(b2p[SKB_HLEN_REG], 3, offsetof(struct sk_buff, len));
98 PPC_LWZ(b2p[TMP_REG_1], 3, offsetof(struct sk_buff, data_len));
99 /* header_len = len - data_len */
100 PPC_SUB(b2p[SKB_HLEN_REG], b2p[SKB_HLEN_REG], b2p[TMP_REG_1]);
101
102 /* skb->data pointer */
103 PPC_BPF_LL(b2p[SKB_DATA_REG], 3, offsetof(struct sk_buff, data));
104}
105
106static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) 91static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
107{ 92{
108 int i; 93 int i;
@@ -145,18 +130,6 @@ static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
145 if (bpf_is_seen_register(ctx, i)) 130 if (bpf_is_seen_register(ctx, i))
146 PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i])); 131 PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
147 132
148 /*
149 * Save additional non-volatile regs if we cache skb
150 * Also, setup skb data
151 */
152 if (ctx->seen & SEEN_SKB) {
153 PPC_BPF_STL(b2p[SKB_HLEN_REG], 1,
154 bpf_jit_stack_offsetof(ctx, b2p[SKB_HLEN_REG]));
155 PPC_BPF_STL(b2p[SKB_DATA_REG], 1,
156 bpf_jit_stack_offsetof(ctx, b2p[SKB_DATA_REG]));
157 bpf_jit_emit_skb_loads(image, ctx);
158 }
159
160 /* Setup frame pointer to point to the bpf stack area */ 133 /* Setup frame pointer to point to the bpf stack area */
161 if (bpf_is_seen_register(ctx, BPF_REG_FP)) 134 if (bpf_is_seen_register(ctx, BPF_REG_FP))
162 PPC_ADDI(b2p[BPF_REG_FP], 1, 135 PPC_ADDI(b2p[BPF_REG_FP], 1,
@@ -172,14 +145,6 @@ static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx
172 if (bpf_is_seen_register(ctx, i)) 145 if (bpf_is_seen_register(ctx, i))
173 PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i])); 146 PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
174 147
175 /* Restore non-volatile registers used for skb cache */
176 if (ctx->seen & SEEN_SKB) {
177 PPC_BPF_LL(b2p[SKB_HLEN_REG], 1,
178 bpf_jit_stack_offsetof(ctx, b2p[SKB_HLEN_REG]));
179 PPC_BPF_LL(b2p[SKB_DATA_REG], 1,
180 bpf_jit_stack_offsetof(ctx, b2p[SKB_DATA_REG]));
181 }
182
183 /* Tear down our stack frame */ 148 /* Tear down our stack frame */
184 if (bpf_has_stack_frame(ctx)) { 149 if (bpf_has_stack_frame(ctx)) {
185 PPC_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size); 150 PPC_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size);
@@ -753,23 +718,10 @@ emit_clear:
753 ctx->seen |= SEEN_FUNC; 718 ctx->seen |= SEEN_FUNC;
754 func = (u8 *) __bpf_call_base + imm; 719 func = (u8 *) __bpf_call_base + imm;
755 720
756 /* Save skb pointer if we need to re-cache skb data */
757 if ((ctx->seen & SEEN_SKB) &&
758 bpf_helper_changes_pkt_data(func))
759 PPC_BPF_STL(3, 1, bpf_jit_stack_local(ctx));
760
761 bpf_jit_emit_func_call(image, ctx, (u64)func); 721 bpf_jit_emit_func_call(image, ctx, (u64)func);
762 722
763 /* move return value from r3 to BPF_REG_0 */ 723 /* move return value from r3 to BPF_REG_0 */
764 PPC_MR(b2p[BPF_REG_0], 3); 724 PPC_MR(b2p[BPF_REG_0], 3);
765
766 /* refresh skb cache */
767 if ((ctx->seen & SEEN_SKB) &&
768 bpf_helper_changes_pkt_data(func)) {
769 /* reload skb pointer to r3 */
770 PPC_BPF_LL(3, 1, bpf_jit_stack_local(ctx));
771 bpf_jit_emit_skb_loads(image, ctx);
772 }
773 break; 725 break;
774 726
775 /* 727 /*
@@ -887,65 +839,6 @@ cond_branch:
887 break; 839 break;
888 840
889 /* 841 /*
890 * Loads from packet header/data
891 * Assume 32-bit input value in imm and X (src_reg)
892 */
893
894 /* Absolute loads */
895 case BPF_LD | BPF_W | BPF_ABS:
896 func = (u8 *)CHOOSE_LOAD_FUNC(imm, sk_load_word);
897 goto common_load_abs;
898 case BPF_LD | BPF_H | BPF_ABS:
899 func = (u8 *)CHOOSE_LOAD_FUNC(imm, sk_load_half);
900 goto common_load_abs;
901 case BPF_LD | BPF_B | BPF_ABS:
902 func = (u8 *)CHOOSE_LOAD_FUNC(imm, sk_load_byte);
903common_load_abs:
904 /*
905 * Load from [imm]
906 * Load into r4, which can just be passed onto
907 * skb load helpers as the second parameter
908 */
909 PPC_LI32(4, imm);
910 goto common_load;
911
912 /* Indirect loads */
913 case BPF_LD | BPF_W | BPF_IND:
914 func = (u8 *)sk_load_word;
915 goto common_load_ind;
916 case BPF_LD | BPF_H | BPF_IND:
917 func = (u8 *)sk_load_half;
918 goto common_load_ind;
919 case BPF_LD | BPF_B | BPF_IND:
920 func = (u8 *)sk_load_byte;
921common_load_ind:
922 /*
923 * Load from [src_reg + imm]
924 * Treat src_reg as a 32-bit value
925 */
926 PPC_EXTSW(4, src_reg);
927 if (imm) {
928 if (imm >= -32768 && imm < 32768)
929 PPC_ADDI(4, 4, IMM_L(imm));
930 else {
931 PPC_LI32(b2p[TMP_REG_1], imm);
932 PPC_ADD(4, 4, b2p[TMP_REG_1]);
933 }
934 }
935
936common_load:
937 ctx->seen |= SEEN_SKB;
938 ctx->seen |= SEEN_FUNC;
939 bpf_jit_emit_func_call(image, ctx, (u64)func);
940
941 /*
942 * Helper returns 'lt' condition on error, and an
943 * appropriate return value in BPF_REG_0
944 */
945 PPC_BCC(COND_LT, exit_addr);
946 break;
947
948 /*
949 * Tail call 842 * Tail call
950 */ 843 */
951 case BPF_JMP | BPF_TAIL_CALL: 844 case BPF_JMP | BPF_TAIL_CALL:
generated by cgit v1.2.2 (git 2.25.0) at 2025-11-02 19:22:47 -0500