aboutsummaryrefslogtreecommitdiffstats
path: root/arch/sparc
diff options
context:
space:
mode:
Diffstat (limited to 'arch/sparc')
-rw-r--r--arch/sparc/Kconfig7
-rw-r--r--arch/sparc/include/asm/page_64.h3
-rw-r--r--arch/sparc/include/asm/pgtable_64.h16
-rw-r--r--arch/sparc/include/asm/processor_32.h6
-rw-r--r--arch/sparc/include/asm/processor_64.h4
-rw-r--r--arch/sparc/include/asm/ptrace.h3
-rw-r--r--arch/sparc/include/asm/uaccess.h2
-rw-r--r--arch/sparc/include/asm/uaccess_32.h44
-rw-r--r--arch/sparc/include/asm/uaccess_64.h44
-rw-r--r--arch/sparc/include/uapi/asm/socket.h6
-rw-r--r--arch/sparc/include/uapi/asm/unistd.h8
-rw-r--r--arch/sparc/kernel/head_32.S7
-rw-r--r--arch/sparc/kernel/head_64.S4
-rw-r--r--arch/sparc/kernel/misctrap.S1
-rw-r--r--arch/sparc/kernel/ptrace_64.c38
-rw-r--r--arch/sparc/kernel/rtrap_64.S1
-rw-r--r--arch/sparc/kernel/spiterrs.S1
-rw-r--r--arch/sparc/kernel/sun4v_tlb_miss.S1
-rw-r--r--arch/sparc/kernel/sysfs.c39
-rw-r--r--arch/sparc/kernel/systbls_32.S1
-rw-r--r--arch/sparc/kernel/systbls_64.S2
-rw-r--r--arch/sparc/kernel/time_32.c2
-rw-r--r--arch/sparc/kernel/time_64.c2
-rw-r--r--arch/sparc/kernel/urtt_fill.S1
-rw-r--r--arch/sparc/kernel/winfixup.S2
-rw-r--r--arch/sparc/lib/GENcopy_from_user.S2
-rw-r--r--arch/sparc/lib/GENcopy_to_user.S2
-rw-r--r--arch/sparc/lib/GENpatch.S4
-rw-r--r--arch/sparc/lib/NG2copy_from_user.S2
-rw-r--r--arch/sparc/lib/NG2copy_to_user.S2
-rw-r--r--arch/sparc/lib/NG2memcpy.S4
-rw-r--r--arch/sparc/lib/NG2patch.S4
-rw-r--r--arch/sparc/lib/NG4copy_from_user.S2
-rw-r--r--arch/sparc/lib/NG4copy_to_user.S2
-rw-r--r--arch/sparc/lib/NG4memcpy.S1
-rw-r--r--arch/sparc/lib/NG4memset.S1
-rw-r--r--arch/sparc/lib/NG4patch.S4
-rw-r--r--arch/sparc/lib/NGcopy_from_user.S2
-rw-r--r--arch/sparc/lib/NGcopy_to_user.S2
-rw-r--r--arch/sparc/lib/NGmemcpy.S1
-rw-r--r--arch/sparc/lib/NGpatch.S4
-rw-r--r--arch/sparc/lib/U1copy_from_user.S4
-rw-r--r--arch/sparc/lib/U1copy_to_user.S4
-rw-r--r--arch/sparc/lib/U3copy_to_user.S2
-rw-r--r--arch/sparc/lib/U3patch.S4
-rw-r--r--arch/sparc/lib/copy_in_user.S6
-rw-r--r--arch/sparc/lib/copy_user.S16
-rw-r--r--arch/sparc/mm/hugetlbpage.c25
-rw-r--r--arch/sparc/mm/init_64.c6
-rw-r--r--arch/sparc/mm/srmmu.c1
-rw-r--r--arch/sparc/mm/tlb.c6
-rw-r--r--arch/sparc/mm/tsb.c4
-rw-r--r--arch/sparc/net/Makefile2
-rw-r--r--arch/sparc/net/bpf_jit_32.h (renamed from arch/sparc/net/bpf_jit.h)2
-rw-r--r--arch/sparc/net/bpf_jit_64.h66
-rw-r--r--arch/sparc/net/bpf_jit_asm_32.S (renamed from arch/sparc/net/bpf_jit_asm.S)9
-rw-r--r--arch/sparc/net/bpf_jit_asm_64.S161
-rw-r--r--arch/sparc/net/bpf_jit_comp_32.c (renamed from arch/sparc/net/bpf_jit_comp.c)51
-rw-r--r--arch/sparc/net/bpf_jit_comp_64.c1566
59 files changed, 1973 insertions, 246 deletions
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index 68ac5c7cd982..58243b0d21c0 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -31,7 +31,8 @@ config SPARC
31 select ARCH_WANT_IPC_PARSE_VERSION 31 select ARCH_WANT_IPC_PARSE_VERSION
32 select GENERIC_PCI_IOMAP 32 select GENERIC_PCI_IOMAP
33 select HAVE_NMI_WATCHDOG if SPARC64 33 select HAVE_NMI_WATCHDOG if SPARC64
34 select HAVE_CBPF_JIT 34 select HAVE_CBPF_JIT if SPARC32
35 select HAVE_EBPF_JIT if SPARC64
35 select HAVE_DEBUG_BUGVERBOSE 36 select HAVE_DEBUG_BUGVERBOSE
36 select GENERIC_SMP_IDLE_THREAD 37 select GENERIC_SMP_IDLE_THREAD
37 select GENERIC_CLOCKEVENTS 38 select GENERIC_CLOCKEVENTS
@@ -42,8 +43,7 @@ config SPARC
42 select OLD_SIGSUSPEND 43 select OLD_SIGSUSPEND
43 select ARCH_HAS_SG_CHAIN 44 select ARCH_HAS_SG_CHAIN
44 select CPU_NO_EFFICIENT_FFS 45 select CPU_NO_EFFICIENT_FFS
45 select HAVE_ARCH_HARDENED_USERCOPY 46 select LOCKDEP_SMALL if LOCKDEP
46 select PROVE_LOCKING_SMALL if PROVE_LOCKING
47 select ARCH_WANT_RELAX_ORDER 47 select ARCH_WANT_RELAX_ORDER
48 48
49config SPARC32 49config SPARC32
@@ -82,6 +82,7 @@ config SPARC64
82 select HAVE_ARCH_AUDITSYSCALL 82 select HAVE_ARCH_AUDITSYSCALL
83 select ARCH_SUPPORTS_ATOMIC_RMW 83 select ARCH_SUPPORTS_ATOMIC_RMW
84 select HAVE_NMI 84 select HAVE_NMI
85 select HAVE_REGS_AND_STACK_ACCESS_API
85 86
86config ARCH_DEFCONFIG 87config ARCH_DEFCONFIG
87 string 88 string
diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h
index f294dd42fc7d..5961b2d8398a 100644
--- a/arch/sparc/include/asm/page_64.h
+++ b/arch/sparc/include/asm/page_64.h
@@ -17,6 +17,7 @@
17 17
18#define HPAGE_SHIFT 23 18#define HPAGE_SHIFT 23
19#define REAL_HPAGE_SHIFT 22 19#define REAL_HPAGE_SHIFT 22
20#define HPAGE_2GB_SHIFT 31
20#define HPAGE_256MB_SHIFT 28 21#define HPAGE_256MB_SHIFT 28
21#define HPAGE_64K_SHIFT 16 22#define HPAGE_64K_SHIFT 16
22#define REAL_HPAGE_SIZE (_AC(1,UL) << REAL_HPAGE_SHIFT) 23#define REAL_HPAGE_SIZE (_AC(1,UL) << REAL_HPAGE_SHIFT)
@@ -27,7 +28,7 @@
27#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) 28#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)
28#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA 29#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA
29#define REAL_HPAGE_PER_HPAGE (_AC(1,UL) << (HPAGE_SHIFT - REAL_HPAGE_SHIFT)) 30#define REAL_HPAGE_PER_HPAGE (_AC(1,UL) << (HPAGE_SHIFT - REAL_HPAGE_SHIFT))
30#define HUGE_MAX_HSTATE 3 31#define HUGE_MAX_HSTATE 4
31#endif 32#endif
32 33
33#ifndef __ASSEMBLY__ 34#ifndef __ASSEMBLY__
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 56e49c8f770d..6fbd931f0570 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -12,6 +12,7 @@
12 * the SpitFire page tables. 12 * the SpitFire page tables.
13 */ 13 */
14 14
15#include <asm-generic/5level-fixup.h>
15#include <linux/compiler.h> 16#include <linux/compiler.h>
16#include <linux/const.h> 17#include <linux/const.h>
17#include <asm/types.h> 18#include <asm/types.h>
@@ -678,26 +679,27 @@ static inline unsigned long pmd_pfn(pmd_t pmd)
678 return pte_pfn(pte); 679 return pte_pfn(pte);
679} 680}
680 681
681#ifdef CONFIG_TRANSPARENT_HUGEPAGE 682#define __HAVE_ARCH_PMD_WRITE
682static inline unsigned long pmd_dirty(pmd_t pmd) 683static inline unsigned long pmd_write(pmd_t pmd)
683{ 684{
684 pte_t pte = __pte(pmd_val(pmd)); 685 pte_t pte = __pte(pmd_val(pmd));
685 686
686 return pte_dirty(pte); 687 return pte_write(pte);
687} 688}
688 689
689static inline unsigned long pmd_young(pmd_t pmd) 690#ifdef CONFIG_TRANSPARENT_HUGEPAGE
691static inline unsigned long pmd_dirty(pmd_t pmd)
690{ 692{
691 pte_t pte = __pte(pmd_val(pmd)); 693 pte_t pte = __pte(pmd_val(pmd));
692 694
693 return pte_young(pte); 695 return pte_dirty(pte);
694} 696}
695 697
696static inline unsigned long pmd_write(pmd_t pmd) 698static inline unsigned long pmd_young(pmd_t pmd)
697{ 699{
698 pte_t pte = __pte(pmd_val(pmd)); 700 pte_t pte = __pte(pmd_val(pmd));
699 701
700 return pte_write(pte); 702 return pte_young(pte);
701} 703}
702 704
703static inline unsigned long pmd_trans_huge(pmd_t pmd) 705static inline unsigned long pmd_trans_huge(pmd_t pmd)
diff --git a/arch/sparc/include/asm/processor_32.h b/arch/sparc/include/asm/processor_32.h
index 365d4cb267b4..dd27159819eb 100644
--- a/arch/sparc/include/asm/processor_32.h
+++ b/arch/sparc/include/asm/processor_32.h
@@ -18,12 +18,6 @@
18#include <asm/signal.h> 18#include <asm/signal.h>
19#include <asm/page.h> 19#include <asm/page.h>
20 20
21/*
22 * The sparc has no problems with write protection
23 */
24#define wp_works_ok 1
25#define wp_works_ok__is_a_macro /* for versions in ksyms.c */
26
27/* Whee, this is STACK_TOP + PAGE_SIZE and the lowest kernel address too... 21/* Whee, this is STACK_TOP + PAGE_SIZE and the lowest kernel address too...
28 * That one page is used to protect kernel from intruders, so that 22 * That one page is used to protect kernel from intruders, so that
29 * we can make our access_ok test faster 23 * we can make our access_ok test faster
diff --git a/arch/sparc/include/asm/processor_64.h b/arch/sparc/include/asm/processor_64.h
index 6448cfc8292f..b58ee9018433 100644
--- a/arch/sparc/include/asm/processor_64.h
+++ b/arch/sparc/include/asm/processor_64.h
@@ -18,10 +18,6 @@
18#include <asm/ptrace.h> 18#include <asm/ptrace.h>
19#include <asm/page.h> 19#include <asm/page.h>
20 20
21/* The sparc has no problems with write protection */
22#define wp_works_ok 1
23#define wp_works_ok__is_a_macro /* for versions in ksyms.c */
24
25/* 21/*
26 * User lives in his very own context, and cannot reference us. Note 22 * User lives in his very own context, and cannot reference us. Note
27 * that TASK_SIZE is a misnomer, it really gives maximum user virtual 23 * that TASK_SIZE is a misnomer, it really gives maximum user virtual
diff --git a/arch/sparc/include/asm/ptrace.h b/arch/sparc/include/asm/ptrace.h
index ca57f08bd3db..d73428e4333c 100644
--- a/arch/sparc/include/asm/ptrace.h
+++ b/arch/sparc/include/asm/ptrace.h
@@ -83,7 +83,8 @@ unsigned long profile_pc(struct pt_regs *);
83 83
84#define MAX_REG_OFFSET (offsetof(struct pt_regs, magic)) 84#define MAX_REG_OFFSET (offsetof(struct pt_regs, magic))
85 85
86extern int regs_query_register_offset(const char *name); 86int regs_query_register_offset(const char *name);
87unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n);
87 88
88/** 89/**
89 * regs_get_register() - get register value from its offset 90 * regs_get_register() - get register value from its offset
diff --git a/arch/sparc/include/asm/uaccess.h b/arch/sparc/include/asm/uaccess.h
index bd56c28fff9f..9e068bf9060a 100644
--- a/arch/sparc/include/asm/uaccess.h
+++ b/arch/sparc/include/asm/uaccess.h
@@ -7,7 +7,7 @@
7#endif 7#endif
8 8
9#define user_addr_max() \ 9#define user_addr_max() \
10 (segment_eq(get_fs(), USER_DS) ? TASK_SIZE : ~0UL) 10 (uaccess_kernel() ? ~0UL : TASK_SIZE)
11 11
12long strncpy_from_user(char *dest, const char __user *src, long count); 12long strncpy_from_user(char *dest, const char __user *src, long count);
13 13
diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h
index ea55f86d7ccd..12ebee2d97c7 100644
--- a/arch/sparc/include/asm/uaccess_32.h
+++ b/arch/sparc/include/asm/uaccess_32.h
@@ -7,14 +7,8 @@
7#ifndef _ASM_UACCESS_H 7#ifndef _ASM_UACCESS_H
8#define _ASM_UACCESS_H 8#define _ASM_UACCESS_H
9 9
10#ifdef __KERNEL__
11#include <linux/compiler.h> 10#include <linux/compiler.h>
12#include <linux/sched.h>
13#include <linux/string.h> 11#include <linux/string.h>
14#include <linux/errno.h>
15#endif
16
17#ifndef __ASSEMBLY__
18 12
19#include <asm/processor.h> 13#include <asm/processor.h>
20 14
@@ -30,9 +24,6 @@
30#define KERNEL_DS ((mm_segment_t) { 0 }) 24#define KERNEL_DS ((mm_segment_t) { 0 })
31#define USER_DS ((mm_segment_t) { -1 }) 25#define USER_DS ((mm_segment_t) { -1 })
32 26
33#define VERIFY_READ 0
34#define VERIFY_WRITE 1
35
36#define get_ds() (KERNEL_DS) 27#define get_ds() (KERNEL_DS)
37#define get_fs() (current->thread.current_ds) 28#define get_fs() (current->thread.current_ds)
38#define set_fs(val) ((current->thread.current_ds) = (val)) 29#define set_fs(val) ((current->thread.current_ds) = (val))
@@ -45,7 +36,7 @@
45 * large size and address near to PAGE_OFFSET - a fault will break his intentions. 36 * large size and address near to PAGE_OFFSET - a fault will break his intentions.
46 */ 37 */
47#define __user_ok(addr, size) ({ (void)(size); (addr) < STACK_TOP; }) 38#define __user_ok(addr, size) ({ (void)(size); (addr) < STACK_TOP; })
48#define __kernel_ok (segment_eq(get_fs(), KERNEL_DS)) 39#define __kernel_ok (uaccess_kernel())
49#define __access_ok(addr, size) (__user_ok((addr) & get_fs().seg, (size))) 40#define __access_ok(addr, size) (__user_ok((addr) & get_fs().seg, (size)))
50#define access_ok(type, addr, size) \ 41#define access_ok(type, addr, size) \
51 ({ (void)(type); __access_ok((unsigned long)(addr), size); }) 42 ({ (void)(type); __access_ok((unsigned long)(addr), size); })
@@ -80,8 +71,6 @@ struct exception_table_entry
80/* Returns 0 if exception not found and fixup otherwise. */ 71/* Returns 0 if exception not found and fixup otherwise. */
81unsigned long search_extables_range(unsigned long addr, unsigned long *g2); 72unsigned long search_extables_range(unsigned long addr, unsigned long *g2);
82 73
83void __ret_efault(void);
84
85/* Uh, these should become the main single-value transfer routines.. 74/* Uh, these should become the main single-value transfer routines..
86 * They automatically use the right size if we just have the right 75 * They automatically use the right size if we just have the right
87 * pointer type.. 76 * pointer type..
@@ -246,39 +235,18 @@ int __get_user_bad(void);
246 235
247unsigned long __copy_user(void __user *to, const void __user *from, unsigned long size); 236unsigned long __copy_user(void __user *to, const void __user *from, unsigned long size);
248 237
249static inline unsigned long copy_to_user(void __user *to, const void *from, unsigned long n) 238static inline unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
250{
251 if (n && __access_ok((unsigned long) to, n)) {
252 check_object_size(from, n, true);
253 return __copy_user(to, (__force void __user *) from, n);
254 } else
255 return n;
256}
257
258static inline unsigned long __copy_to_user(void __user *to, const void *from, unsigned long n)
259{ 239{
260 check_object_size(from, n, true);
261 return __copy_user(to, (__force void __user *) from, n); 240 return __copy_user(to, (__force void __user *) from, n);
262} 241}
263 242
264static inline unsigned long copy_from_user(void *to, const void __user *from, unsigned long n) 243static inline unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
265{
266 if (n && __access_ok((unsigned long) from, n)) {
267 check_object_size(to, n, false);
268 return __copy_user((__force void __user *) to, from, n);
269 } else {
270 memset(to, 0, n);
271 return n;
272 }
273}
274
275static inline unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n)
276{ 244{
277 return __copy_user((__force void __user *) to, from, n); 245 return __copy_user((__force void __user *) to, from, n);
278} 246}
279 247
280#define __copy_to_user_inatomic __copy_to_user 248#define INLINE_COPY_FROM_USER
281#define __copy_from_user_inatomic __copy_from_user 249#define INLINE_COPY_TO_USER
282 250
283static inline unsigned long __clear_user(void __user *addr, unsigned long size) 251static inline unsigned long __clear_user(void __user *addr, unsigned long size)
284{ 252{
@@ -312,6 +280,4 @@ static inline unsigned long clear_user(void __user *addr, unsigned long n)
312__must_check long strlen_user(const char __user *str); 280__must_check long strlen_user(const char __user *str);
313__must_check long strnlen_user(const char __user *str, long n); 281__must_check long strnlen_user(const char __user *str, long n);
314 282
315#endif /* __ASSEMBLY__ */
316
317#endif /* _ASM_UACCESS_H */ 283#endif /* _ASM_UACCESS_H */
diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h
index 5373136c412b..6096d671aa63 100644
--- a/arch/sparc/include/asm/uaccess_64.h
+++ b/arch/sparc/include/asm/uaccess_64.h
@@ -5,18 +5,12 @@
5 * User space memory access functions 5 * User space memory access functions
6 */ 6 */
7 7
8#ifdef __KERNEL__
9#include <linux/errno.h>
10#include <linux/compiler.h> 8#include <linux/compiler.h>
11#include <linux/string.h> 9#include <linux/string.h>
12#include <linux/thread_info.h>
13#include <asm/asi.h> 10#include <asm/asi.h>
14#include <asm/spitfire.h> 11#include <asm/spitfire.h>
15#include <asm-generic/uaccess-unaligned.h> 12#include <asm-generic/uaccess-unaligned.h>
16#include <asm/extable_64.h> 13#include <asm/extable_64.h>
17#endif
18
19#ifndef __ASSEMBLY__
20 14
21#include <asm/processor.h> 15#include <asm/processor.h>
22 16
@@ -36,9 +30,6 @@
36#define KERNEL_DS ((mm_segment_t) { ASI_P }) 30#define KERNEL_DS ((mm_segment_t) { ASI_P })
37#define USER_DS ((mm_segment_t) { ASI_AIUS }) /* har har har */ 31#define USER_DS ((mm_segment_t) { ASI_AIUS }) /* har har har */
38 32
39#define VERIFY_READ 0
40#define VERIFY_WRITE 1
41
42#define get_fs() ((mm_segment_t){(current_thread_info()->current_ds)}) 33#define get_fs() ((mm_segment_t){(current_thread_info()->current_ds)})
43#define get_ds() (KERNEL_DS) 34#define get_ds() (KERNEL_DS)
44 35
@@ -185,39 +176,19 @@ __asm__ __volatile__( \
185 176
186int __get_user_bad(void); 177int __get_user_bad(void);
187 178
188unsigned long __must_check ___copy_from_user(void *to, 179unsigned long __must_check raw_copy_from_user(void *to,
189 const void __user *from, 180 const void __user *from,
190 unsigned long size); 181 unsigned long size);
191static inline unsigned long __must_check
192copy_from_user(void *to, const void __user *from, unsigned long size)
193{
194 check_object_size(to, size, false);
195 182
196 return ___copy_from_user(to, from, size); 183unsigned long __must_check raw_copy_to_user(void __user *to,
197}
198#define __copy_from_user copy_from_user
199
200unsigned long __must_check ___copy_to_user(void __user *to,
201 const void *from, 184 const void *from,
202 unsigned long size); 185 unsigned long size);
203static inline unsigned long __must_check 186#define INLINE_COPY_FROM_USER
204copy_to_user(void __user *to, const void *from, unsigned long size) 187#define INLINE_COPY_TO_USER
205{
206 check_object_size(from, size, true);
207 188
208 return ___copy_to_user(to, from, size); 189unsigned long __must_check raw_copy_in_user(void __user *to,
209}
210#define __copy_to_user copy_to_user
211
212unsigned long __must_check ___copy_in_user(void __user *to,
213 const void __user *from, 190 const void __user *from,
214 unsigned long size); 191 unsigned long size);
215static inline unsigned long __must_check
216copy_in_user(void __user *to, void __user *from, unsigned long size)
217{
218 return ___copy_in_user(to, from, size);
219}
220#define __copy_in_user copy_in_user
221 192
222unsigned long __must_check __clear_user(void __user *, unsigned long); 193unsigned long __must_check __clear_user(void __user *, unsigned long);
223 194
@@ -226,14 +197,9 @@ unsigned long __must_check __clear_user(void __user *, unsigned long);
226__must_check long strlen_user(const char __user *str); 197__must_check long strlen_user(const char __user *str);
227__must_check long strnlen_user(const char __user *str, long n); 198__must_check long strnlen_user(const char __user *str, long n);
228 199
229#define __copy_to_user_inatomic __copy_to_user
230#define __copy_from_user_inatomic __copy_from_user
231
232struct pt_regs; 200struct pt_regs;
233unsigned long compute_effective_address(struct pt_regs *, 201unsigned long compute_effective_address(struct pt_regs *,
234 unsigned int insn, 202 unsigned int insn,
235 unsigned int rd); 203 unsigned int rd);
236 204
237#endif /* __ASSEMBLY__ */
238
239#endif /* _ASM_UACCESS_H */ 205#endif /* _ASM_UACCESS_H */
diff --git a/arch/sparc/include/uapi/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h
index a25dc32f5d6a..3f4ad19d9ec7 100644
--- a/arch/sparc/include/uapi/asm/socket.h
+++ b/arch/sparc/include/uapi/asm/socket.h
@@ -88,6 +88,12 @@
88 88
89#define SCM_TIMESTAMPING_OPT_STATS 0x0038 89#define SCM_TIMESTAMPING_OPT_STATS 0x0038
90 90
91#define SO_MEMINFO 0x0039
92
93#define SO_INCOMING_NAPI_ID 0x003a
94
95#define SO_COOKIE 0x003b
96
91/* Security levels - as per NRL IPv6 - don't actually do anything */ 97/* Security levels - as per NRL IPv6 - don't actually do anything */
92#define SO_SECURITY_AUTHENTICATION 0x5001 98#define SO_SECURITY_AUTHENTICATION 0x5001
93#define SO_SECURITY_ENCRYPTION_TRANSPORT 0x5002 99#define SO_SECURITY_ENCRYPTION_TRANSPORT 0x5002
diff --git a/arch/sparc/include/uapi/asm/unistd.h b/arch/sparc/include/uapi/asm/unistd.h
index 36eee8132c22..ae77df75bffa 100644
--- a/arch/sparc/include/uapi/asm/unistd.h
+++ b/arch/sparc/include/uapi/asm/unistd.h
@@ -425,8 +425,9 @@
425#define __NR_copy_file_range 357 425#define __NR_copy_file_range 357
426#define __NR_preadv2 358 426#define __NR_preadv2 358
427#define __NR_pwritev2 359 427#define __NR_pwritev2 359
428#define __NR_statx 360
428 429
429#define NR_syscalls 360 430#define NR_syscalls 361
430 431
431/* Bitmask values returned from kern_features system call. */ 432/* Bitmask values returned from kern_features system call. */
432#define KERN_FEATURE_MIXED_MODE_STACK 0x00000001 433#define KERN_FEATURE_MIXED_MODE_STACK 0x00000001
@@ -442,4 +443,9 @@
442#define __IGNORE_getresgid 443#define __IGNORE_getresgid
443#endif 444#endif
444 445
446/* Sparc doesn't have protection keys. */
447#define __IGNORE_pkey_mprotect
448#define __IGNORE_pkey_alloc
449#define __IGNORE_pkey_free
450
445#endif /* _UAPI_SPARC_UNISTD_H */ 451#endif /* _UAPI_SPARC_UNISTD_H */
diff --git a/arch/sparc/kernel/head_32.S b/arch/sparc/kernel/head_32.S
index 7bb317b87dde..7274e43ff9be 100644
--- a/arch/sparc/kernel/head_32.S
+++ b/arch/sparc/kernel/head_32.S
@@ -809,10 +809,3 @@ lvl14_save:
809 .word 0 809 .word 0
810 .word 0 810 .word 0
811 .word t_irq14 811 .word t_irq14
812
813 .section ".fixup",#alloc,#execinstr
814 .globl __ret_efault
815__ret_efault:
816 ret
817 restore %g0, -EFAULT, %o0
818EXPORT_SYMBOL(__ret_efault)
diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S
index 6aa3da152c20..44101196d02b 100644
--- a/arch/sparc/kernel/head_64.S
+++ b/arch/sparc/kernel/head_64.S
@@ -96,6 +96,7 @@ sparc64_boot:
96 andn %g1, PSTATE_AM, %g1 96 andn %g1, PSTATE_AM, %g1
97 wrpr %g1, 0x0, %pstate 97 wrpr %g1, 0x0, %pstate
98 ba,a,pt %xcc, 1f 98 ba,a,pt %xcc, 1f
99 nop
99 100
100 .globl prom_finddev_name, prom_chosen_path, prom_root_node 101 .globl prom_finddev_name, prom_chosen_path, prom_root_node
101 .globl prom_getprop_name, prom_mmu_name, prom_peer_name 102 .globl prom_getprop_name, prom_mmu_name, prom_peer_name
@@ -613,6 +614,7 @@ niagara_tlb_fixup:
613 nop 614 nop
614 615
615 ba,a,pt %xcc, 80f 616 ba,a,pt %xcc, 80f
617 nop
616niagara4_patch: 618niagara4_patch:
617 call niagara4_patch_copyops 619 call niagara4_patch_copyops
618 nop 620 nop
@@ -622,6 +624,7 @@ niagara4_patch:
622 nop 624 nop
623 625
624 ba,a,pt %xcc, 80f 626 ba,a,pt %xcc, 80f
627 nop
625 628
626niagara2_patch: 629niagara2_patch:
627 call niagara2_patch_copyops 630 call niagara2_patch_copyops
@@ -632,6 +635,7 @@ niagara2_patch:
632 nop 635 nop
633 636
634 ba,a,pt %xcc, 80f 637 ba,a,pt %xcc, 80f
638 nop
635 639
636niagara_patch: 640niagara_patch:
637 call niagara_patch_copyops 641 call niagara_patch_copyops
diff --git a/arch/sparc/kernel/misctrap.S b/arch/sparc/kernel/misctrap.S
index 34b4933900bf..9276d2f0dd86 100644
--- a/arch/sparc/kernel/misctrap.S
+++ b/arch/sparc/kernel/misctrap.S
@@ -82,6 +82,7 @@ do_stdfmna:
82 call handle_stdfmna 82 call handle_stdfmna
83 add %sp, PTREGS_OFF, %o0 83 add %sp, PTREGS_OFF, %o0
84 ba,a,pt %xcc, rtrap 84 ba,a,pt %xcc, rtrap
85 nop
85 .size do_stdfmna,.-do_stdfmna 86 .size do_stdfmna,.-do_stdfmna
86 87
87 .type breakpoint_trap,#function 88 .type breakpoint_trap,#function
diff --git a/arch/sparc/kernel/ptrace_64.c b/arch/sparc/kernel/ptrace_64.c
index df9e731a76f5..e1d965e90e16 100644
--- a/arch/sparc/kernel/ptrace_64.c
+++ b/arch/sparc/kernel/ptrace_64.c
@@ -351,7 +351,7 @@ static int genregs64_set(struct task_struct *target,
351 } 351 }
352 352
353 if (!ret) { 353 if (!ret) {
354 unsigned long y; 354 unsigned long y = regs->y;
355 355
356 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, 356 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
357 &y, 357 &y,
@@ -1162,3 +1162,39 @@ int regs_query_register_offset(const char *name)
1162 return roff->offset; 1162 return roff->offset;
1163 return -EINVAL; 1163 return -EINVAL;
1164} 1164}
1165
1166/**
1167 * regs_within_kernel_stack() - check the address in the stack
1168 * @regs: pt_regs which contains kernel stack pointer.
1169 * @addr: address which is checked.
1170 *
1171 * regs_within_kernel_stack() checks @addr is within the kernel stack page(s).
1172 * If @addr is within the kernel stack, it returns true. If not, returns false.
1173 */
1174static inline int regs_within_kernel_stack(struct pt_regs *regs,
1175 unsigned long addr)
1176{
1177 unsigned long ksp = kernel_stack_pointer(regs) + STACK_BIAS;
1178 return ((addr & ~(THREAD_SIZE - 1)) ==
1179 (ksp & ~(THREAD_SIZE - 1)));
1180}
1181
1182/**
1183 * regs_get_kernel_stack_nth() - get Nth entry of the stack
1184 * @regs: pt_regs which contains kernel stack pointer.
1185 * @n: stack entry number.
1186 *
1187 * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which
1188 * is specified by @regs. If the @n th entry is NOT in the kernel stack,
1189 * this returns 0.
1190 */
1191unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n)
1192{
1193 unsigned long ksp = kernel_stack_pointer(regs) + STACK_BIAS;
1194 unsigned long *addr = (unsigned long *)ksp;
1195 addr += n;
1196 if (regs_within_kernel_stack(regs, (unsigned long)addr))
1197 return *addr;
1198 else
1199 return 0;
1200}
diff --git a/arch/sparc/kernel/rtrap_64.S b/arch/sparc/kernel/rtrap_64.S
index 216948ca4382..709a82ebd294 100644
--- a/arch/sparc/kernel/rtrap_64.S
+++ b/arch/sparc/kernel/rtrap_64.S
@@ -237,6 +237,7 @@ rt_continue: ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1
237 bne,pt %xcc, user_rtt_fill_32bit 237 bne,pt %xcc, user_rtt_fill_32bit
238 wrpr %g1, %cwp 238 wrpr %g1, %cwp
239 ba,a,pt %xcc, user_rtt_fill_64bit 239 ba,a,pt %xcc, user_rtt_fill_64bit
240 nop
240 241
241user_rtt_fill_fixup_dax: 242user_rtt_fill_fixup_dax:
242 ba,pt %xcc, user_rtt_fill_fixup_common 243 ba,pt %xcc, user_rtt_fill_fixup_common
diff --git a/arch/sparc/kernel/spiterrs.S b/arch/sparc/kernel/spiterrs.S
index 4a73009f66a5..d7e540842809 100644
--- a/arch/sparc/kernel/spiterrs.S
+++ b/arch/sparc/kernel/spiterrs.S
@@ -86,6 +86,7 @@ __spitfire_cee_trap_continue:
86 rd %pc, %g7 86 rd %pc, %g7
87 87
88 ba,a,pt %xcc, 2f 88 ba,a,pt %xcc, 2f
89 nop
89 90
901: ba,pt %xcc, etrap_irq 911: ba,pt %xcc, etrap_irq
91 rd %pc, %g7 92 rd %pc, %g7
diff --git a/arch/sparc/kernel/sun4v_tlb_miss.S b/arch/sparc/kernel/sun4v_tlb_miss.S
index 6179e19bc9b9..c19f352f46c7 100644
--- a/arch/sparc/kernel/sun4v_tlb_miss.S
+++ b/arch/sparc/kernel/sun4v_tlb_miss.S
@@ -352,6 +352,7 @@ sun4v_mna:
352 call sun4v_do_mna 352 call sun4v_do_mna
353 add %sp, PTREGS_OFF, %o0 353 add %sp, PTREGS_OFF, %o0
354 ba,a,pt %xcc, rtrap 354 ba,a,pt %xcc, rtrap
355 nop
355 356
356 /* Privileged Action. */ 357 /* Privileged Action. */
357sun4v_privact: 358sun4v_privact:
diff --git a/arch/sparc/kernel/sysfs.c b/arch/sparc/kernel/sysfs.c
index d63fc613e7a9..5fd352b759af 100644
--- a/arch/sparc/kernel/sysfs.c
+++ b/arch/sparc/kernel/sysfs.c
@@ -98,27 +98,7 @@ static struct attribute_group mmu_stat_group = {
98 .name = "mmu_stats", 98 .name = "mmu_stats",
99}; 99};
100 100
101/* XXX convert to rusty's on_one_cpu */ 101static long read_mmustat_enable(void *data __maybe_unused)
102static unsigned long run_on_cpu(unsigned long cpu,
103 unsigned long (*func)(unsigned long),
104 unsigned long arg)
105{
106 cpumask_t old_affinity;
107 unsigned long ret;
108
109 cpumask_copy(&old_affinity, &current->cpus_allowed);
110 /* should return -EINVAL to userspace */
111 if (set_cpus_allowed_ptr(current, cpumask_of(cpu)))
112 return 0;
113
114 ret = func(arg);
115
116 set_cpus_allowed_ptr(current, &old_affinity);
117
118 return ret;
119}
120
121static unsigned long read_mmustat_enable(unsigned long junk)
122{ 102{
123 unsigned long ra = 0; 103 unsigned long ra = 0;
124 104
@@ -127,11 +107,11 @@ static unsigned long read_mmustat_enable(unsigned long junk)
127 return ra != 0; 107 return ra != 0;
128} 108}
129 109
130static unsigned long write_mmustat_enable(unsigned long val) 110static long write_mmustat_enable(void *data)
131{ 111{
132 unsigned long ra, orig_ra; 112 unsigned long ra, orig_ra, *val = data;
133 113
134 if (val) 114 if (*val)
135 ra = __pa(&per_cpu(mmu_stats, smp_processor_id())); 115 ra = __pa(&per_cpu(mmu_stats, smp_processor_id()));
136 else 116 else
137 ra = 0UL; 117 ra = 0UL;
@@ -142,7 +122,8 @@ static unsigned long write_mmustat_enable(unsigned long val)
142static ssize_t show_mmustat_enable(struct device *s, 122static ssize_t show_mmustat_enable(struct device *s,
143 struct device_attribute *attr, char *buf) 123 struct device_attribute *attr, char *buf)
144{ 124{
145 unsigned long val = run_on_cpu(s->id, read_mmustat_enable, 0); 125 long val = work_on_cpu(s->id, read_mmustat_enable, NULL);
126
146 return sprintf(buf, "%lx\n", val); 127 return sprintf(buf, "%lx\n", val);
147} 128}
148 129
@@ -150,13 +131,15 @@ static ssize_t store_mmustat_enable(struct device *s,
150 struct device_attribute *attr, const char *buf, 131 struct device_attribute *attr, const char *buf,
151 size_t count) 132 size_t count)
152{ 133{
153 unsigned long val, err; 134 unsigned long val;
154 int ret = sscanf(buf, "%lu", &val); 135 long err;
136 int ret;
155 137
138 ret = sscanf(buf, "%lu", &val);
156 if (ret != 1) 139 if (ret != 1)
157 return -EINVAL; 140 return -EINVAL;
158 141
159 err = run_on_cpu(s->id, write_mmustat_enable, val); 142 err = work_on_cpu(s->id, write_mmustat_enable, &val);
160 if (err) 143 if (err)
161 return -EIO; 144 return -EIO;
162 145
diff --git a/arch/sparc/kernel/systbls_32.S b/arch/sparc/kernel/systbls_32.S
index eac7f0db5c8c..5253e895b81b 100644
--- a/arch/sparc/kernel/systbls_32.S
+++ b/arch/sparc/kernel/systbls_32.S
@@ -89,3 +89,4 @@ sys_call_table:
89/*345*/ .long sys_renameat2, sys_seccomp, sys_getrandom, sys_memfd_create, sys_bpf 89/*345*/ .long sys_renameat2, sys_seccomp, sys_getrandom, sys_memfd_create, sys_bpf
90/*350*/ .long sys_execveat, sys_membarrier, sys_userfaultfd, sys_bind, sys_listen 90/*350*/ .long sys_execveat, sys_membarrier, sys_userfaultfd, sys_bind, sys_listen
91/*355*/ .long sys_setsockopt, sys_mlock2, sys_copy_file_range, sys_preadv2, sys_pwritev2 91/*355*/ .long sys_setsockopt, sys_mlock2, sys_copy_file_range, sys_preadv2, sys_pwritev2
92/*360*/ .long sys_statx
diff --git a/arch/sparc/kernel/systbls_64.S b/arch/sparc/kernel/systbls_64.S
index b0f17ff2ddba..82339f6be0b2 100644
--- a/arch/sparc/kernel/systbls_64.S
+++ b/arch/sparc/kernel/systbls_64.S
@@ -90,6 +90,7 @@ sys_call_table32:
90 .word sys32_renameat2, sys_seccomp, sys_getrandom, sys_memfd_create, sys_bpf 90 .word sys32_renameat2, sys_seccomp, sys_getrandom, sys_memfd_create, sys_bpf
91/*350*/ .word sys32_execveat, sys_membarrier, sys_userfaultfd, sys_bind, sys_listen 91/*350*/ .word sys32_execveat, sys_membarrier, sys_userfaultfd, sys_bind, sys_listen
92 .word compat_sys_setsockopt, sys_mlock2, sys_copy_file_range, compat_sys_preadv2, compat_sys_pwritev2 92 .word compat_sys_setsockopt, sys_mlock2, sys_copy_file_range, compat_sys_preadv2, compat_sys_pwritev2
93/*360*/ .word sys_statx
93 94
94#endif /* CONFIG_COMPAT */ 95#endif /* CONFIG_COMPAT */
95 96
@@ -171,3 +172,4 @@ sys_call_table:
171 .word sys_renameat2, sys_seccomp, sys_getrandom, sys_memfd_create, sys_bpf 172 .word sys_renameat2, sys_seccomp, sys_getrandom, sys_memfd_create, sys_bpf
172/*350*/ .word sys64_execveat, sys_membarrier, sys_userfaultfd, sys_bind, sys_listen 173/*350*/ .word sys64_execveat, sys_membarrier, sys_userfaultfd, sys_bind, sys_listen
173 .word sys_setsockopt, sys_mlock2, sys_copy_file_range, sys_preadv2, sys_pwritev2 174 .word sys_setsockopt, sys_mlock2, sys_copy_file_range, sys_preadv2, sys_pwritev2
175/*360*/ .word sys_statx
diff --git a/arch/sparc/kernel/time_32.c b/arch/sparc/kernel/time_32.c
index 244062bdaa56..9f575dfc2e41 100644
--- a/arch/sparc/kernel/time_32.c
+++ b/arch/sparc/kernel/time_32.c
@@ -228,7 +228,9 @@ void register_percpu_ce(int cpu)
228 ce->mult = div_sc(sparc_config.clock_rate, NSEC_PER_SEC, 228 ce->mult = div_sc(sparc_config.clock_rate, NSEC_PER_SEC,
229 ce->shift); 229 ce->shift);
230 ce->max_delta_ns = clockevent_delta2ns(sparc_config.clock_rate, ce); 230 ce->max_delta_ns = clockevent_delta2ns(sparc_config.clock_rate, ce);
231 ce->max_delta_ticks = (unsigned long)sparc_config.clock_rate;
231 ce->min_delta_ns = clockevent_delta2ns(100, ce); 232 ce->min_delta_ns = clockevent_delta2ns(100, ce);
233 ce->min_delta_ticks = 100;
232 234
233 clockevents_register_device(ce); 235 clockevents_register_device(ce);
234} 236}
diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c
index 12a6d3555cb8..98d05de8da66 100644
--- a/arch/sparc/kernel/time_64.c
+++ b/arch/sparc/kernel/time_64.c
@@ -796,8 +796,10 @@ void __init time_init(void)
796 796
797 sparc64_clockevent.max_delta_ns = 797 sparc64_clockevent.max_delta_ns =
798 clockevent_delta2ns(0x7fffffffffffffffUL, &sparc64_clockevent); 798 clockevent_delta2ns(0x7fffffffffffffffUL, &sparc64_clockevent);
799 sparc64_clockevent.max_delta_ticks = 0x7fffffffffffffffUL;
799 sparc64_clockevent.min_delta_ns = 800 sparc64_clockevent.min_delta_ns =
800 clockevent_delta2ns(0xF, &sparc64_clockevent); 801 clockevent_delta2ns(0xF, &sparc64_clockevent);
802 sparc64_clockevent.min_delta_ticks = 0xF;
801 803
802 printk("clockevent: mult[%x] shift[%d]\n", 804 printk("clockevent: mult[%x] shift[%d]\n",
803 sparc64_clockevent.mult, sparc64_clockevent.shift); 805 sparc64_clockevent.mult, sparc64_clockevent.shift);
diff --git a/arch/sparc/kernel/urtt_fill.S b/arch/sparc/kernel/urtt_fill.S
index 5604a2b051d4..364af3250646 100644
--- a/arch/sparc/kernel/urtt_fill.S
+++ b/arch/sparc/kernel/urtt_fill.S
@@ -92,6 +92,7 @@ user_rtt_fill_fixup_common:
92 call sun4v_data_access_exception 92 call sun4v_data_access_exception
93 nop 93 nop
94 ba,a,pt %xcc, rtrap 94 ba,a,pt %xcc, rtrap
95 nop
95 96
961: call spitfire_data_access_exception 971: call spitfire_data_access_exception
97 nop 98 nop
diff --git a/arch/sparc/kernel/winfixup.S b/arch/sparc/kernel/winfixup.S
index 855019a8590e..1ee173cc3c39 100644
--- a/arch/sparc/kernel/winfixup.S
+++ b/arch/sparc/kernel/winfixup.S
@@ -152,6 +152,8 @@ fill_fixup_dax:
152 call sun4v_data_access_exception 152 call sun4v_data_access_exception
153 nop 153 nop
154 ba,a,pt %xcc, rtrap 154 ba,a,pt %xcc, rtrap
155 nop
1551: call spitfire_data_access_exception 1561: call spitfire_data_access_exception
156 nop 157 nop
157 ba,a,pt %xcc, rtrap 158 ba,a,pt %xcc, rtrap
159 nop
diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S
index 69a439fa2fc1..8aa16ef113f2 100644
--- a/arch/sparc/lib/GENcopy_from_user.S
+++ b/arch/sparc/lib/GENcopy_from_user.S
@@ -23,7 +23,7 @@
23#define PREAMBLE \ 23#define PREAMBLE \
24 rd %asi, %g1; \ 24 rd %asi, %g1; \
25 cmp %g1, ASI_AIUS; \ 25 cmp %g1, ASI_AIUS; \
26 bne,pn %icc, ___copy_in_user; \ 26 bne,pn %icc, raw_copy_in_user; \
27 nop 27 nop
28#endif 28#endif
29 29
diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S
index 9947427ce354..311c8fa5e98e 100644
--- a/arch/sparc/lib/GENcopy_to_user.S
+++ b/arch/sparc/lib/GENcopy_to_user.S
@@ -27,7 +27,7 @@
27#define PREAMBLE \ 27#define PREAMBLE \
28 rd %asi, %g1; \ 28 rd %asi, %g1; \
29 cmp %g1, ASI_AIUS; \ 29 cmp %g1, ASI_AIUS; \
30 bne,pn %icc, ___copy_in_user; \ 30 bne,pn %icc, raw_copy_in_user; \
31 nop 31 nop
32#endif 32#endif
33 33
diff --git a/arch/sparc/lib/GENpatch.S b/arch/sparc/lib/GENpatch.S
index fab9e89f16bd..95e2f1f9e477 100644
--- a/arch/sparc/lib/GENpatch.S
+++ b/arch/sparc/lib/GENpatch.S
@@ -26,8 +26,8 @@
26 .type generic_patch_copyops,#function 26 .type generic_patch_copyops,#function
27generic_patch_copyops: 27generic_patch_copyops:
28 GEN_DO_PATCH(memcpy, GENmemcpy) 28 GEN_DO_PATCH(memcpy, GENmemcpy)
29 GEN_DO_PATCH(___copy_from_user, GENcopy_from_user) 29 GEN_DO_PATCH(raw_copy_from_user, GENcopy_from_user)
30 GEN_DO_PATCH(___copy_to_user, GENcopy_to_user) 30 GEN_DO_PATCH(raw_copy_to_user, GENcopy_to_user)
31 retl 31 retl
32 nop 32 nop
33 .size generic_patch_copyops,.-generic_patch_copyops 33 .size generic_patch_copyops,.-generic_patch_copyops
diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S
index b79a6998d87c..0d8a018118c2 100644
--- a/arch/sparc/lib/NG2copy_from_user.S
+++ b/arch/sparc/lib/NG2copy_from_user.S
@@ -36,7 +36,7 @@
36#define PREAMBLE \ 36#define PREAMBLE \
37 rd %asi, %g1; \ 37 rd %asi, %g1; \
38 cmp %g1, ASI_AIUS; \ 38 cmp %g1, ASI_AIUS; \
39 bne,pn %icc, ___copy_in_user; \ 39 bne,pn %icc, raw_copy_in_user; \
40 nop 40 nop
41#endif 41#endif
42 42
diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S
index dcec55f254ab..a7a0ea0d8a0b 100644
--- a/arch/sparc/lib/NG2copy_to_user.S
+++ b/arch/sparc/lib/NG2copy_to_user.S
@@ -45,7 +45,7 @@
45#define PREAMBLE \ 45#define PREAMBLE \
46 rd %asi, %g1; \ 46 rd %asi, %g1; \
47 cmp %g1, ASI_AIUS; \ 47 cmp %g1, ASI_AIUS; \
48 bne,pn %icc, ___copy_in_user; \ 48 bne,pn %icc, raw_copy_in_user; \
49 nop 49 nop
50#endif 50#endif
51 51
diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S
index c629dbd121b6..64dcd6cdb606 100644
--- a/arch/sparc/lib/NG2memcpy.S
+++ b/arch/sparc/lib/NG2memcpy.S
@@ -326,11 +326,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
326 blu 170f 326 blu 170f
327 nop 327 nop
328 ba,a,pt %xcc, 180f 328 ba,a,pt %xcc, 180f
329 nop
329 330
3304: /* 32 <= low bits < 48 */ 3314: /* 32 <= low bits < 48 */
331 blu 150f 332 blu 150f
332 nop 333 nop
333 ba,a,pt %xcc, 160f 334 ba,a,pt %xcc, 160f
335 nop
3345: /* 0 < low bits < 32 */ 3365: /* 0 < low bits < 32 */
335 blu,a 6f 337 blu,a 6f
336 cmp %g2, 8 338 cmp %g2, 8
@@ -338,6 +340,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
338 blu 130f 340 blu 130f
339 nop 341 nop
340 ba,a,pt %xcc, 140f 342 ba,a,pt %xcc, 140f
343 nop
3416: /* 0 < low bits < 16 */ 3446: /* 0 < low bits < 16 */
342 bgeu 120f 345 bgeu 120f
343 nop 346 nop
@@ -475,6 +478,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
475 brz,pt %o2, 85f 478 brz,pt %o2, 85f
476 sub %o0, %o1, GLOBAL_SPARE 479 sub %o0, %o1, GLOBAL_SPARE
477 ba,a,pt %XCC, 90f 480 ba,a,pt %XCC, 90f
481 nop
478 482
479 .align 64 483 .align 64
48075: /* 16 < len <= 64 */ 48475: /* 16 < len <= 64 */
diff --git a/arch/sparc/lib/NG2patch.S b/arch/sparc/lib/NG2patch.S
index 28c36f06a6d1..56ccc19adde8 100644
--- a/arch/sparc/lib/NG2patch.S
+++ b/arch/sparc/lib/NG2patch.S
@@ -26,8 +26,8 @@
26 .type niagara2_patch_copyops,#function 26 .type niagara2_patch_copyops,#function
27niagara2_patch_copyops: 27niagara2_patch_copyops:
28 NG_DO_PATCH(memcpy, NG2memcpy) 28 NG_DO_PATCH(memcpy, NG2memcpy)
29 NG_DO_PATCH(___copy_from_user, NG2copy_from_user) 29 NG_DO_PATCH(raw_copy_from_user, NG2copy_from_user)
30 NG_DO_PATCH(___copy_to_user, NG2copy_to_user) 30 NG_DO_PATCH(raw_copy_to_user, NG2copy_to_user)
31 retl 31 retl
32 nop 32 nop
33 .size niagara2_patch_copyops,.-niagara2_patch_copyops 33 .size niagara2_patch_copyops,.-niagara2_patch_copyops
diff --git a/arch/sparc/lib/NG4copy_from_user.S b/arch/sparc/lib/NG4copy_from_user.S
index 16a286c1a528..5bb506bd61fa 100644
--- a/arch/sparc/lib/NG4copy_from_user.S
+++ b/arch/sparc/lib/NG4copy_from_user.S
@@ -31,7 +31,7 @@
31#define PREAMBLE \ 31#define PREAMBLE \
32 rd %asi, %g1; \ 32 rd %asi, %g1; \
33 cmp %g1, ASI_AIUS; \ 33 cmp %g1, ASI_AIUS; \
34 bne,pn %icc, ___copy_in_user; \ 34 bne,pn %icc, raw_copy_in_user; \
35 nop 35 nop
36#endif 36#endif
37 37
diff --git a/arch/sparc/lib/NG4copy_to_user.S b/arch/sparc/lib/NG4copy_to_user.S
index 6b0276ffc858..a82d4d45fc1c 100644
--- a/arch/sparc/lib/NG4copy_to_user.S
+++ b/arch/sparc/lib/NG4copy_to_user.S
@@ -40,7 +40,7 @@
40#define PREAMBLE \ 40#define PREAMBLE \
41 rd %asi, %g1; \ 41 rd %asi, %g1; \
42 cmp %g1, ASI_AIUS; \ 42 cmp %g1, ASI_AIUS; \
43 bne,pn %icc, ___copy_in_user; \ 43 bne,pn %icc, raw_copy_in_user; \
44 nop 44 nop
45#endif 45#endif
46 46
diff --git a/arch/sparc/lib/NG4memcpy.S b/arch/sparc/lib/NG4memcpy.S
index 75bb93b1437f..78ea962edcbe 100644
--- a/arch/sparc/lib/NG4memcpy.S
+++ b/arch/sparc/lib/NG4memcpy.S
@@ -530,4 +530,5 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
530 bne,pt %icc, 1b 530 bne,pt %icc, 1b
531 EX_ST(STORE(stb, %g1, %o0 - 0x01), NG4_retl_o2_plus_1) 531 EX_ST(STORE(stb, %g1, %o0 - 0x01), NG4_retl_o2_plus_1)
532 ba,a,pt %icc, .Lexit 532 ba,a,pt %icc, .Lexit
533 nop
533 .size FUNC_NAME, .-FUNC_NAME 534 .size FUNC_NAME, .-FUNC_NAME
diff --git a/arch/sparc/lib/NG4memset.S b/arch/sparc/lib/NG4memset.S
index 41da4bdd95cb..7c0c81f18837 100644
--- a/arch/sparc/lib/NG4memset.S
+++ b/arch/sparc/lib/NG4memset.S
@@ -102,4 +102,5 @@ NG4bzero:
102 bne,pt %icc, 1b 102 bne,pt %icc, 1b
103 add %o0, 0x30, %o0 103 add %o0, 0x30, %o0
104 ba,a,pt %icc, .Lpostloop 104 ba,a,pt %icc, .Lpostloop
105 nop
105 .size NG4bzero,.-NG4bzero 106 .size NG4bzero,.-NG4bzero
diff --git a/arch/sparc/lib/NG4patch.S b/arch/sparc/lib/NG4patch.S
index a114cbcf2a48..3cc0f8cc95df 100644
--- a/arch/sparc/lib/NG4patch.S
+++ b/arch/sparc/lib/NG4patch.S
@@ -26,8 +26,8 @@
26 .type niagara4_patch_copyops,#function 26 .type niagara4_patch_copyops,#function
27niagara4_patch_copyops: 27niagara4_patch_copyops:
28 NG_DO_PATCH(memcpy, NG4memcpy) 28 NG_DO_PATCH(memcpy, NG4memcpy)
29 NG_DO_PATCH(___copy_from_user, NG4copy_from_user) 29 NG_DO_PATCH(raw_copy_from_user, NG4copy_from_user)
30 NG_DO_PATCH(___copy_to_user, NG4copy_to_user) 30 NG_DO_PATCH(raw_copy_to_user, NG4copy_to_user)
31 retl 31 retl
32 nop 32 nop
33 .size niagara4_patch_copyops,.-niagara4_patch_copyops 33 .size niagara4_patch_copyops,.-niagara4_patch_copyops
diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S
index 9cd42fcbc781..2333b6f3e824 100644
--- a/arch/sparc/lib/NGcopy_from_user.S
+++ b/arch/sparc/lib/NGcopy_from_user.S
@@ -25,7 +25,7 @@
25#define PREAMBLE \ 25#define PREAMBLE \
26 rd %asi, %g1; \ 26 rd %asi, %g1; \
27 cmp %g1, ASI_AIUS; \ 27 cmp %g1, ASI_AIUS; \
28 bne,pn %icc, ___copy_in_user; \ 28 bne,pn %icc, raw_copy_in_user; \
29 nop 29 nop
30#endif 30#endif
31 31
diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S
index 5c358afd464e..07ba20bc4ea4 100644
--- a/arch/sparc/lib/NGcopy_to_user.S
+++ b/arch/sparc/lib/NGcopy_to_user.S
@@ -28,7 +28,7 @@
28#define PREAMBLE \ 28#define PREAMBLE \
29 rd %asi, %g1; \ 29 rd %asi, %g1; \
30 cmp %g1, ASI_AIUS; \ 30 cmp %g1, ASI_AIUS; \
31 bne,pn %icc, ___copy_in_user; \ 31 bne,pn %icc, raw_copy_in_user; \
32 nop 32 nop
33#endif 33#endif
34 34
diff --git a/arch/sparc/lib/NGmemcpy.S b/arch/sparc/lib/NGmemcpy.S
index d88c4ed50a00..cd654a719b27 100644
--- a/arch/sparc/lib/NGmemcpy.S
+++ b/arch/sparc/lib/NGmemcpy.S
@@ -394,6 +394,7 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
394 brz,pt %i2, 85f 394 brz,pt %i2, 85f
395 sub %o0, %i1, %i3 395 sub %o0, %i1, %i3
396 ba,a,pt %XCC, 90f 396 ba,a,pt %XCC, 90f
397 nop
397 398
398 .align 64 399 .align 64
39970: /* 16 < len <= 64 */ 40070: /* 16 < len <= 64 */
diff --git a/arch/sparc/lib/NGpatch.S b/arch/sparc/lib/NGpatch.S
index 3b0674fc3366..62ccda7e7b38 100644
--- a/arch/sparc/lib/NGpatch.S
+++ b/arch/sparc/lib/NGpatch.S
@@ -26,8 +26,8 @@
26 .type niagara_patch_copyops,#function 26 .type niagara_patch_copyops,#function
27niagara_patch_copyops: 27niagara_patch_copyops:
28 NG_DO_PATCH(memcpy, NGmemcpy) 28 NG_DO_PATCH(memcpy, NGmemcpy)
29 NG_DO_PATCH(___copy_from_user, NGcopy_from_user) 29 NG_DO_PATCH(raw_copy_from_user, NGcopy_from_user)
30 NG_DO_PATCH(___copy_to_user, NGcopy_to_user) 30 NG_DO_PATCH(raw_copy_to_user, NGcopy_to_user)
31 retl 31 retl
32 nop 32 nop
33 .size niagara_patch_copyops,.-niagara_patch_copyops 33 .size niagara_patch_copyops,.-niagara_patch_copyops
diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S
index bb6ff73229e3..9a6e68a9bf4a 100644
--- a/arch/sparc/lib/U1copy_from_user.S
+++ b/arch/sparc/lib/U1copy_from_user.S
@@ -19,7 +19,7 @@
19 .text; \ 19 .text; \
20 .align 4; 20 .align 4;
21 21
22#define FUNC_NAME ___copy_from_user 22#define FUNC_NAME raw_copy_from_user
23#define LOAD(type,addr,dest) type##a [addr] %asi, dest 23#define LOAD(type,addr,dest) type##a [addr] %asi, dest
24#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_AIUS, dest 24#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_AIUS, dest
25#define EX_RETVAL(x) 0 25#define EX_RETVAL(x) 0
@@ -31,7 +31,7 @@
31#define PREAMBLE \ 31#define PREAMBLE \
32 rd %asi, %g1; \ 32 rd %asi, %g1; \
33 cmp %g1, ASI_AIUS; \ 33 cmp %g1, ASI_AIUS; \
34 bne,pn %icc, ___copy_in_user; \ 34 bne,pn %icc, raw_copy_in_user; \
35 nop; \ 35 nop; \
36 36
37#include "U1memcpy.S" 37#include "U1memcpy.S"
diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S
index ed92ce739558..d7b28491eddf 100644
--- a/arch/sparc/lib/U1copy_to_user.S
+++ b/arch/sparc/lib/U1copy_to_user.S
@@ -19,7 +19,7 @@
19 .text; \ 19 .text; \
20 .align 4; 20 .align 4;
21 21
22#define FUNC_NAME ___copy_to_user 22#define FUNC_NAME raw_copy_to_user
23#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS 23#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS
24#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS 24#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS
25#define EX_RETVAL(x) 0 25#define EX_RETVAL(x) 0
@@ -31,7 +31,7 @@
31#define PREAMBLE \ 31#define PREAMBLE \
32 rd %asi, %g1; \ 32 rd %asi, %g1; \
33 cmp %g1, ASI_AIUS; \ 33 cmp %g1, ASI_AIUS; \
34 bne,pn %icc, ___copy_in_user; \ 34 bne,pn %icc, raw_copy_in_user; \
35 nop; \ 35 nop; \
36 36
37#include "U1memcpy.S" 37#include "U1memcpy.S"
diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S
index c4ee858e352a..f48fb87fe9f2 100644
--- a/arch/sparc/lib/U3copy_to_user.S
+++ b/arch/sparc/lib/U3copy_to_user.S
@@ -31,7 +31,7 @@
31#define PREAMBLE \ 31#define PREAMBLE \
32 rd %asi, %g1; \ 32 rd %asi, %g1; \
33 cmp %g1, ASI_AIUS; \ 33 cmp %g1, ASI_AIUS; \
34 bne,pn %icc, ___copy_in_user; \ 34 bne,pn %icc, raw_copy_in_user; \
35 nop; \ 35 nop; \
36 36
37#include "U3memcpy.S" 37#include "U3memcpy.S"
diff --git a/arch/sparc/lib/U3patch.S b/arch/sparc/lib/U3patch.S
index ecc302619a6e..91cd6539b6e1 100644
--- a/arch/sparc/lib/U3patch.S
+++ b/arch/sparc/lib/U3patch.S
@@ -26,8 +26,8 @@
26 .type cheetah_patch_copyops,#function 26 .type cheetah_patch_copyops,#function
27cheetah_patch_copyops: 27cheetah_patch_copyops:
28 ULTRA3_DO_PATCH(memcpy, U3memcpy) 28 ULTRA3_DO_PATCH(memcpy, U3memcpy)
29 ULTRA3_DO_PATCH(___copy_from_user, U3copy_from_user) 29 ULTRA3_DO_PATCH(raw_copy_from_user, U3copy_from_user)
30 ULTRA3_DO_PATCH(___copy_to_user, U3copy_to_user) 30 ULTRA3_DO_PATCH(raw_copy_to_user, U3copy_to_user)
31 retl 31 retl
32 nop 32 nop
33 .size cheetah_patch_copyops,.-cheetah_patch_copyops 33 .size cheetah_patch_copyops,.-cheetah_patch_copyops
diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S
index 0252b218de45..1b73bb80aeb0 100644
--- a/arch/sparc/lib/copy_in_user.S
+++ b/arch/sparc/lib/copy_in_user.S
@@ -44,7 +44,7 @@ __retl_o2_plus_1:
44 * to copy register windows around during thread cloning. 44 * to copy register windows around during thread cloning.
45 */ 45 */
46 46
47ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ 47ENTRY(raw_copy_in_user) /* %o0=dst, %o1=src, %o2=len */
48 cmp %o2, 0 48 cmp %o2, 0
49 be,pn %XCC, 85f 49 be,pn %XCC, 85f
50 or %o0, %o1, %o3 50 or %o0, %o1, %o3
@@ -105,5 +105,5 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
105 add %o0, 1, %o0 105 add %o0, 1, %o0
106 retl 106 retl
107 clr %o0 107 clr %o0
108ENDPROC(___copy_in_user) 108ENDPROC(raw_copy_in_user)
109EXPORT_SYMBOL(___copy_in_user) 109EXPORT_SYMBOL(raw_copy_in_user)
diff --git a/arch/sparc/lib/copy_user.S b/arch/sparc/lib/copy_user.S
index cea644dc67a6..bc243ee807cc 100644
--- a/arch/sparc/lib/copy_user.S
+++ b/arch/sparc/lib/copy_user.S
@@ -364,21 +364,7 @@ short_aligned_end:
36497: 36497:
365 mov %o2, %g3 365 mov %o2, %g3
366fixupretl: 366fixupretl:
367 sethi %hi(PAGE_OFFSET), %g1 367 retl
368 cmp %o0, %g1
369 blu 1f
370 cmp %o1, %g1
371 bgeu 1f
372 ld [%g6 + TI_PREEMPT], %g1
373 cmp %g1, 0
374 bne 1f
375 nop
376 save %sp, -64, %sp
377 mov %i0, %o0
378 call __bzero
379 mov %g3, %o1
380 restore
3811: retl
382 mov %g3, %o0 368 mov %g3, %o0
383 369
384/* exception routine sets %g2 to (broken_insn - first_insn)>>2 */ 370/* exception routine sets %g2 to (broken_insn - first_insn)>>2 */
diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
index 323bc6b6e3ad..7c29d38e6b99 100644
--- a/arch/sparc/mm/hugetlbpage.c
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -143,6 +143,10 @@ static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
143 pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V; 143 pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V;
144 144
145 switch (shift) { 145 switch (shift) {
146 case HPAGE_2GB_SHIFT:
147 hugepage_size = _PAGE_SZ2GB_4V;
148 pte_val(entry) |= _PAGE_PMD_HUGE;
149 break;
146 case HPAGE_256MB_SHIFT: 150 case HPAGE_256MB_SHIFT:
147 hugepage_size = _PAGE_SZ256MB_4V; 151 hugepage_size = _PAGE_SZ256MB_4V;
148 pte_val(entry) |= _PAGE_PMD_HUGE; 152 pte_val(entry) |= _PAGE_PMD_HUGE;
@@ -183,6 +187,9 @@ static unsigned int sun4v_huge_tte_to_shift(pte_t entry)
183 unsigned int shift; 187 unsigned int shift;
184 188
185 switch (tte_szbits) { 189 switch (tte_szbits) {
190 case _PAGE_SZ2GB_4V:
191 shift = HPAGE_2GB_SHIFT;
192 break;
186 case _PAGE_SZ256MB_4V: 193 case _PAGE_SZ256MB_4V:
187 shift = HPAGE_256MB_SHIFT; 194 shift = HPAGE_256MB_SHIFT;
188 break; 195 break;
@@ -261,7 +268,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
261 if (!pmd) 268 if (!pmd)
262 return NULL; 269 return NULL;
263 270
264 if (sz == PMD_SHIFT) 271 if (sz >= PMD_SIZE)
265 pte = (pte_t *)pmd; 272 pte = (pte_t *)pmd;
266 else 273 else
267 pte = pte_alloc_map(mm, pmd, addr); 274 pte = pte_alloc_map(mm, pmd, addr);
@@ -454,6 +461,22 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,
454 pgd_t *pgd; 461 pgd_t *pgd;
455 unsigned long next; 462 unsigned long next;
456 463
464 addr &= PMD_MASK;
465 if (addr < floor) {
466 addr += PMD_SIZE;
467 if (!addr)
468 return;
469 }
470 if (ceiling) {
471 ceiling &= PMD_MASK;
472 if (!ceiling)
473 return;
474 }
475 if (end - 1 > ceiling - 1)
476 end -= PMD_SIZE;
477 if (addr > end - 1)
478 return;
479
457 pgd = pgd_offset(tlb->mm, addr); 480 pgd = pgd_offset(tlb->mm, addr);
458 do { 481 do {
459 next = pgd_addr_end(addr, end); 482 next = pgd_addr_end(addr, end);
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index ccd455328989..0cda653ae007 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -337,6 +337,10 @@ static int __init setup_hugepagesz(char *string)
337 hugepage_shift = ilog2(hugepage_size); 337 hugepage_shift = ilog2(hugepage_size);
338 338
339 switch (hugepage_shift) { 339 switch (hugepage_shift) {
340 case HPAGE_2GB_SHIFT:
341 hv_pgsz_mask = HV_PGSZ_MASK_2GB;
342 hv_pgsz_idx = HV_PGSZ_IDX_2GB;
343 break;
340 case HPAGE_256MB_SHIFT: 344 case HPAGE_256MB_SHIFT:
341 hv_pgsz_mask = HV_PGSZ_MASK_256MB; 345 hv_pgsz_mask = HV_PGSZ_MASK_256MB;
342 hv_pgsz_idx = HV_PGSZ_IDX_256MB; 346 hv_pgsz_idx = HV_PGSZ_IDX_256MB;
@@ -1563,7 +1567,7 @@ bool kern_addr_valid(unsigned long addr)
1563 if ((long)addr < 0L) { 1567 if ((long)addr < 0L) {
1564 unsigned long pa = __pa(addr); 1568 unsigned long pa = __pa(addr);
1565 1569
1566 if ((addr >> max_phys_bits) != 0UL) 1570 if ((pa >> max_phys_bits) != 0UL)
1567 return false; 1571 return false;
1568 1572
1569 return pfn_valid(pa >> PAGE_SHIFT); 1573 return pfn_valid(pa >> PAGE_SHIFT);
diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index def82f6d626f..8e76ebba2986 100644
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -54,6 +54,7 @@
54enum mbus_module srmmu_modtype; 54enum mbus_module srmmu_modtype;
55static unsigned int hwbug_bitmask; 55static unsigned int hwbug_bitmask;
56int vac_cache_size; 56int vac_cache_size;
57EXPORT_SYMBOL(vac_cache_size);
57int vac_line_size; 58int vac_line_size;
58 59
59extern struct resource sparc_iomap; 60extern struct resource sparc_iomap;
diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
index afda3bbf7854..ee8066c3d96c 100644
--- a/arch/sparc/mm/tlb.c
+++ b/arch/sparc/mm/tlb.c
@@ -154,7 +154,7 @@ static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr,
154 if (pte_val(*pte) & _PAGE_VALID) { 154 if (pte_val(*pte) & _PAGE_VALID) {
155 bool exec = pte_exec(*pte); 155 bool exec = pte_exec(*pte);
156 156
157 tlb_batch_add_one(mm, vaddr, exec, false); 157 tlb_batch_add_one(mm, vaddr, exec, PAGE_SHIFT);
158 } 158 }
159 pte++; 159 pte++;
160 vaddr += PAGE_SIZE; 160 vaddr += PAGE_SIZE;
@@ -209,9 +209,9 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
209 pte_t orig_pte = __pte(pmd_val(orig)); 209 pte_t orig_pte = __pte(pmd_val(orig));
210 bool exec = pte_exec(orig_pte); 210 bool exec = pte_exec(orig_pte);
211 211
212 tlb_batch_add_one(mm, addr, exec, true); 212 tlb_batch_add_one(mm, addr, exec, REAL_HPAGE_SHIFT);
213 tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec, 213 tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec,
214 true); 214 REAL_HPAGE_SHIFT);
215 } else { 215 } else {
216 tlb_batch_pmd_scan(mm, addr, orig); 216 tlb_batch_pmd_scan(mm, addr, orig);
217 } 217 }
diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
index 0a04811f06b7..bedf08b22a47 100644
--- a/arch/sparc/mm/tsb.c
+++ b/arch/sparc/mm/tsb.c
@@ -122,7 +122,7 @@ void flush_tsb_user(struct tlb_batch *tb)
122 122
123 spin_lock_irqsave(&mm->context.lock, flags); 123 spin_lock_irqsave(&mm->context.lock, flags);
124 124
125 if (tb->hugepage_shift < HPAGE_SHIFT) { 125 if (tb->hugepage_shift < REAL_HPAGE_SHIFT) {
126 base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; 126 base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
127 nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; 127 nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
128 if (tlb_type == cheetah_plus || tlb_type == hypervisor) 128 if (tlb_type == cheetah_plus || tlb_type == hypervisor)
@@ -155,7 +155,7 @@ void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr,
155 155
156 spin_lock_irqsave(&mm->context.lock, flags); 156 spin_lock_irqsave(&mm->context.lock, flags);
157 157
158 if (hugepage_shift < HPAGE_SHIFT) { 158 if (hugepage_shift < REAL_HPAGE_SHIFT) {
159 base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; 159 base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
160 nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; 160 nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
161 if (tlb_type == cheetah_plus || tlb_type == hypervisor) 161 if (tlb_type == cheetah_plus || tlb_type == hypervisor)
diff --git a/arch/sparc/net/Makefile b/arch/sparc/net/Makefile
index 1306a58ac541..76fa8e95b721 100644
--- a/arch/sparc/net/Makefile
+++ b/arch/sparc/net/Makefile
@@ -1,4 +1,4 @@
1# 1#
2# Arch-specific network modules 2# Arch-specific network modules
3# 3#
4obj-$(CONFIG_BPF_JIT) += bpf_jit_asm.o bpf_jit_comp.o 4obj-$(CONFIG_BPF_JIT) += bpf_jit_asm_$(BITS).o bpf_jit_comp_$(BITS).o
diff --git a/arch/sparc/net/bpf_jit.h b/arch/sparc/net/bpf_jit_32.h
index 33d6b375ff12..d5c069bff5f9 100644
--- a/arch/sparc/net/bpf_jit.h
+++ b/arch/sparc/net/bpf_jit_32.h
@@ -39,7 +39,7 @@
39#define r_TMP2 G2 39#define r_TMP2 G2
40#define r_OFF G3 40#define r_OFF G3
41 41
42/* assembly code in arch/sparc/net/bpf_jit_asm.S */ 42/* assembly code in arch/sparc/net/bpf_jit_asm_32.S */
43extern u32 bpf_jit_load_word[]; 43extern u32 bpf_jit_load_word[];
44extern u32 bpf_jit_load_half[]; 44extern u32 bpf_jit_load_half[];
45extern u32 bpf_jit_load_byte[]; 45extern u32 bpf_jit_load_byte[];
diff --git a/arch/sparc/net/bpf_jit_64.h b/arch/sparc/net/bpf_jit_64.h
new file mode 100644
index 000000000000..74abd45796ea
--- /dev/null
+++ b/arch/sparc/net/bpf_jit_64.h
@@ -0,0 +1,66 @@
1#ifndef _BPF_JIT_H
2#define _BPF_JIT_H
3
4#ifndef __ASSEMBLER__
5#define G0 0x00
6#define G1 0x01
7#define G2 0x02
8#define G3 0x03
9#define G6 0x06
10#define G7 0x07
11#define O0 0x08
12#define O1 0x09
13#define O2 0x0a
14#define O3 0x0b
15#define O4 0x0c
16#define O5 0x0d
17#define SP 0x0e
18#define O7 0x0f
19#define L0 0x10
20#define L1 0x11
21#define L2 0x12
22#define L3 0x13
23#define L4 0x14
24#define L5 0x15
25#define L6 0x16
26#define L7 0x17
27#define I0 0x18
28#define I1 0x19
29#define I2 0x1a
30#define I3 0x1b
31#define I4 0x1c
32#define I5 0x1d
33#define FP 0x1e
34#define I7 0x1f
35
36#define r_SKB L0
37#define r_HEADLEN L4
38#define r_SKB_DATA L5
39#define r_TMP G1
40#define r_TMP2 G3
41
42/* assembly code in arch/sparc/net/bpf_jit_asm_64.S */
43extern u32 bpf_jit_load_word[];
44extern u32 bpf_jit_load_half[];
45extern u32 bpf_jit_load_byte[];
46extern u32 bpf_jit_load_byte_msh[];
47extern u32 bpf_jit_load_word_positive_offset[];
48extern u32 bpf_jit_load_half_positive_offset[];
49extern u32 bpf_jit_load_byte_positive_offset[];
50extern u32 bpf_jit_load_byte_msh_positive_offset[];
51extern u32 bpf_jit_load_word_negative_offset[];
52extern u32 bpf_jit_load_half_negative_offset[];
53extern u32 bpf_jit_load_byte_negative_offset[];
54extern u32 bpf_jit_load_byte_msh_negative_offset[];
55
56#else
57#define r_RESULT %o0
58#define r_SKB %o0
59#define r_OFF %o1
60#define r_HEADLEN %l4
61#define r_SKB_DATA %l5
62#define r_TMP %g1
63#define r_TMP2 %g3
64#endif
65
66#endif /* _BPF_JIT_H */
diff --git a/arch/sparc/net/bpf_jit_asm.S b/arch/sparc/net/bpf_jit_asm_32.S
index 8c83f4b8eb15..dcc402f5738a 100644
--- a/arch/sparc/net/bpf_jit_asm.S
+++ b/arch/sparc/net/bpf_jit_asm_32.S
@@ -1,18 +1,11 @@
1#include <asm/ptrace.h> 1#include <asm/ptrace.h>
2 2
3#include "bpf_jit.h" 3#include "bpf_jit_32.h"
4 4
5#ifdef CONFIG_SPARC64
6#define SAVE_SZ 176
7#define SCRATCH_OFF STACK_BIAS + 128
8#define BE_PTR(label) be,pn %xcc, label
9#define SIGN_EXTEND(reg) sra reg, 0, reg
10#else
11#define SAVE_SZ 96 5#define SAVE_SZ 96
12#define SCRATCH_OFF 72 6#define SCRATCH_OFF 72
13#define BE_PTR(label) be label 7#define BE_PTR(label) be label
14#define SIGN_EXTEND(reg) 8#define SIGN_EXTEND(reg)
15#endif
16 9
17#define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */ 10#define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */
18 11
diff --git a/arch/sparc/net/bpf_jit_asm_64.S b/arch/sparc/net/bpf_jit_asm_64.S
new file mode 100644
index 000000000000..3b3f14655f81
--- /dev/null
+++ b/arch/sparc/net/bpf_jit_asm_64.S
@@ -0,0 +1,161 @@
1#include <asm/ptrace.h>
2
3#include "bpf_jit_64.h"
4
5#define SAVE_SZ 176
6#define SCRATCH_OFF STACK_BIAS + 128
7#define BE_PTR(label) be,pn %xcc, label
8#define SIGN_EXTEND(reg) sra reg, 0, reg
9
10#define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */
11
12 .text
13 .globl bpf_jit_load_word
14bpf_jit_load_word:
15 cmp r_OFF, 0
16 bl bpf_slow_path_word_neg
17 nop
18 .globl bpf_jit_load_word_positive_offset
19bpf_jit_load_word_positive_offset:
20 sub r_HEADLEN, r_OFF, r_TMP
21 cmp r_TMP, 3
22 ble bpf_slow_path_word
23 add r_SKB_DATA, r_OFF, r_TMP
24 andcc r_TMP, 3, %g0
25 bne load_word_unaligned
26 nop
27 retl
28 ld [r_TMP], r_RESULT
29load_word_unaligned:
30 ldub [r_TMP + 0x0], r_OFF
31 ldub [r_TMP + 0x1], r_TMP2
32 sll r_OFF, 8, r_OFF
33 or r_OFF, r_TMP2, r_OFF
34 ldub [r_TMP + 0x2], r_TMP2
35 sll r_OFF, 8, r_OFF
36 or r_OFF, r_TMP2, r_OFF
37 ldub [r_TMP + 0x3], r_TMP2
38 sll r_OFF, 8, r_OFF
39 retl
40 or r_OFF, r_TMP2, r_RESULT
41
42 .globl bpf_jit_load_half
43bpf_jit_load_half:
44 cmp r_OFF, 0
45 bl bpf_slow_path_half_neg
46 nop
47 .globl bpf_jit_load_half_positive_offset
48bpf_jit_load_half_positive_offset:
49 sub r_HEADLEN, r_OFF, r_TMP
50 cmp r_TMP, 1
51 ble bpf_slow_path_half
52 add r_SKB_DATA, r_OFF, r_TMP
53 andcc r_TMP, 1, %g0
54 bne load_half_unaligned
55 nop
56 retl
57 lduh [r_TMP], r_RESULT
58load_half_unaligned:
59 ldub [r_TMP + 0x0], r_OFF
60 ldub [r_TMP + 0x1], r_TMP2
61 sll r_OFF, 8, r_OFF
62 retl
63 or r_OFF, r_TMP2, r_RESULT
64
65 .globl bpf_jit_load_byte
66bpf_jit_load_byte:
67 cmp r_OFF, 0
68 bl bpf_slow_path_byte_neg
69 nop
70 .globl bpf_jit_load_byte_positive_offset
71bpf_jit_load_byte_positive_offset:
72 cmp r_OFF, r_HEADLEN
73 bge bpf_slow_path_byte
74 nop
75 retl
76 ldub [r_SKB_DATA + r_OFF], r_RESULT
77
78#define bpf_slow_path_common(LEN) \
79 save %sp, -SAVE_SZ, %sp; \
80 mov %i0, %o0; \
81 mov %i1, %o1; \
82 add %fp, SCRATCH_OFF, %o2; \
83 call skb_copy_bits; \
84 mov (LEN), %o3; \
85 cmp %o0, 0; \
86 restore;
87
88bpf_slow_path_word:
89 bpf_slow_path_common(4)
90 bl bpf_error
91 ld [%sp + SCRATCH_OFF], r_RESULT
92 retl
93 nop
94bpf_slow_path_half:
95 bpf_slow_path_common(2)
96 bl bpf_error
97 lduh [%sp + SCRATCH_OFF], r_RESULT
98 retl
99 nop
100bpf_slow_path_byte:
101 bpf_slow_path_common(1)
102 bl bpf_error
103 ldub [%sp + SCRATCH_OFF], r_RESULT
104 retl
105 nop
106
107#define bpf_negative_common(LEN) \
108 save %sp, -SAVE_SZ, %sp; \
109 mov %i0, %o0; \
110 mov %i1, %o1; \
111 SIGN_EXTEND(%o1); \
112 call bpf_internal_load_pointer_neg_helper; \
113 mov (LEN), %o2; \
114 mov %o0, r_TMP; \
115 cmp %o0, 0; \
116 BE_PTR(bpf_error); \
117 restore;
118
119bpf_slow_path_word_neg:
120 sethi %hi(SKF_MAX_NEG_OFF), r_TMP
121 cmp r_OFF, r_TMP
122 bl bpf_error
123 nop
124 .globl bpf_jit_load_word_negative_offset
125bpf_jit_load_word_negative_offset:
126 bpf_negative_common(4)
127 andcc r_TMP, 3, %g0
128 bne load_word_unaligned
129 nop
130 retl
131 ld [r_TMP], r_RESULT
132
133bpf_slow_path_half_neg:
134 sethi %hi(SKF_MAX_NEG_OFF), r_TMP
135 cmp r_OFF, r_TMP
136 bl bpf_error
137 nop
138 .globl bpf_jit_load_half_negative_offset
139bpf_jit_load_half_negative_offset:
140 bpf_negative_common(2)
141 andcc r_TMP, 1, %g0
142 bne load_half_unaligned
143 nop
144 retl
145 lduh [r_TMP], r_RESULT
146
147bpf_slow_path_byte_neg:
148 sethi %hi(SKF_MAX_NEG_OFF), r_TMP
149 cmp r_OFF, r_TMP
150 bl bpf_error
151 nop
152 .globl bpf_jit_load_byte_negative_offset
153bpf_jit_load_byte_negative_offset:
154 bpf_negative_common(1)
155 retl
156 ldub [r_TMP], r_RESULT
157
158bpf_error:
159 /* Make the JIT program itself return zero. */
160 ret
161 restore %g0, %g0, %o0
diff --git a/arch/sparc/net/bpf_jit_comp.c b/arch/sparc/net/bpf_jit_comp_32.c
index a6d9204a6a0b..d193748548e2 100644
--- a/arch/sparc/net/bpf_jit_comp.c
+++ b/arch/sparc/net/bpf_jit_comp_32.c
@@ -8,7 +8,7 @@
8#include <asm/cacheflush.h> 8#include <asm/cacheflush.h>
9#include <asm/ptrace.h> 9#include <asm/ptrace.h>
10 10
11#include "bpf_jit.h" 11#include "bpf_jit_32.h"
12 12
13int bpf_jit_enable __read_mostly; 13int bpf_jit_enable __read_mostly;
14 14
@@ -17,24 +17,6 @@ static inline bool is_simm13(unsigned int value)
17 return value + 0x1000 < 0x2000; 17 return value + 0x1000 < 0x2000;
18} 18}
19 19
20static void bpf_flush_icache(void *start_, void *end_)
21{
22#ifdef CONFIG_SPARC64
23 /* Cheetah's I-cache is fully coherent. */
24 if (tlb_type == spitfire) {
25 unsigned long start = (unsigned long) start_;
26 unsigned long end = (unsigned long) end_;
27
28 start &= ~7UL;
29 end = (end + 7UL) & ~7UL;
30 while (start < end) {
31 flushi(start);
32 start += 32;
33 }
34 }
35#endif
36}
37
38#define SEEN_DATAREF 1 /* might call external helpers */ 20#define SEEN_DATAREF 1 /* might call external helpers */
39#define SEEN_XREG 2 /* ebx is used */ 21#define SEEN_XREG 2 /* ebx is used */
40#define SEEN_MEM 4 /* use mem[] for temporary storage */ 22#define SEEN_MEM 4 /* use mem[] for temporary storage */
@@ -82,11 +64,7 @@ static void bpf_flush_icache(void *start_, void *end_)
82#define BE (F2(0, 2) | CONDE) 64#define BE (F2(0, 2) | CONDE)
83#define BNE (F2(0, 2) | CONDNE) 65#define BNE (F2(0, 2) | CONDNE)
84 66
85#ifdef CONFIG_SPARC64
86#define BE_PTR (F2(0, 1) | CONDE | (2 << 20))
87#else
88#define BE_PTR BE 67#define BE_PTR BE
89#endif
90 68
91#define SETHI(K, REG) \ 69#define SETHI(K, REG) \
92 (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff)) 70 (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff))
@@ -116,13 +94,8 @@ static void bpf_flush_icache(void *start_, void *end_)
116#define LD64 F3(3, 0x0b) 94#define LD64 F3(3, 0x0b)
117#define ST32 F3(3, 0x04) 95#define ST32 F3(3, 0x04)
118 96
119#ifdef CONFIG_SPARC64
120#define LDPTR LD64
121#define BASE_STACKFRAME 176
122#else
123#define LDPTR LD32 97#define LDPTR LD32
124#define BASE_STACKFRAME 96 98#define BASE_STACKFRAME 96
125#endif
126 99
127#define LD32I (LD32 | IMMED) 100#define LD32I (LD32 | IMMED)
128#define LD8I (LD8 | IMMED) 101#define LD8I (LD8 | IMMED)
@@ -234,11 +207,7 @@ do { BUILD_BUG_ON(FIELD_SIZEOF(STRUCT, FIELD) != sizeof(u8)); \
234 __emit_load8(BASE, STRUCT, FIELD, DEST); \ 207 __emit_load8(BASE, STRUCT, FIELD, DEST); \
235} while (0) 208} while (0)
236 209
237#ifdef CONFIG_SPARC64
238#define BIAS (STACK_BIAS - 4)
239#else
240#define BIAS (-4) 210#define BIAS (-4)
241#endif
242 211
243#define emit_ldmem(OFF, DEST) \ 212#define emit_ldmem(OFF, DEST) \
244do { *prog++ = LD32I | RS1(SP) | S13(BIAS - (OFF)) | RD(DEST); \ 213do { *prog++ = LD32I | RS1(SP) | S13(BIAS - (OFF)) | RD(DEST); \
@@ -249,13 +218,8 @@ do { *prog++ = ST32I | RS1(SP) | S13(BIAS - (OFF)) | RD(SRC); \
249} while (0) 218} while (0)
250 219
251#ifdef CONFIG_SMP 220#ifdef CONFIG_SMP
252#ifdef CONFIG_SPARC64
253#define emit_load_cpu(REG) \
254 emit_load16(G6, struct thread_info, cpu, REG)
255#else
256#define emit_load_cpu(REG) \ 221#define emit_load_cpu(REG) \
257 emit_load32(G6, struct thread_info, cpu, REG) 222 emit_load32(G6, struct thread_info, cpu, REG)
258#endif
259#else 223#else
260#define emit_load_cpu(REG) emit_clear(REG) 224#define emit_load_cpu(REG) emit_clear(REG)
261#endif 225#endif
@@ -486,7 +450,6 @@ void bpf_jit_compile(struct bpf_prog *fp)
486 if (K == 1) 450 if (K == 1)
487 break; 451 break;
488 emit_write_y(G0); 452 emit_write_y(G0);
489#ifdef CONFIG_SPARC32
490 /* The Sparc v8 architecture requires 453 /* The Sparc v8 architecture requires
491 * three instructions between a %y 454 * three instructions between a %y
492 * register write and the first use. 455 * register write and the first use.
@@ -494,31 +457,21 @@ void bpf_jit_compile(struct bpf_prog *fp)
494 emit_nop(); 457 emit_nop();
495 emit_nop(); 458 emit_nop();
496 emit_nop(); 459 emit_nop();
497#endif
498 emit_alu_K(DIV, K); 460 emit_alu_K(DIV, K);
499 break; 461 break;
500 case BPF_ALU | BPF_DIV | BPF_X: /* A /= X; */ 462 case BPF_ALU | BPF_DIV | BPF_X: /* A /= X; */
501 emit_cmpi(r_X, 0); 463 emit_cmpi(r_X, 0);
502 if (pc_ret0 > 0) { 464 if (pc_ret0 > 0) {
503 t_offset = addrs[pc_ret0 - 1]; 465 t_offset = addrs[pc_ret0 - 1];
504#ifdef CONFIG_SPARC32
505 emit_branch(BE, t_offset + 20); 466 emit_branch(BE, t_offset + 20);
506#else
507 emit_branch(BE, t_offset + 8);
508#endif
509 emit_nop(); /* delay slot */ 467 emit_nop(); /* delay slot */
510 } else { 468 } else {
511 emit_branch_off(BNE, 16); 469 emit_branch_off(BNE, 16);
512 emit_nop(); 470 emit_nop();
513#ifdef CONFIG_SPARC32
514 emit_jump(cleanup_addr + 20); 471 emit_jump(cleanup_addr + 20);
515#else
516 emit_jump(cleanup_addr + 8);
517#endif
518 emit_clear(r_A); 472 emit_clear(r_A);
519 } 473 }
520 emit_write_y(G0); 474 emit_write_y(G0);
521#ifdef CONFIG_SPARC32
522 /* The Sparc v8 architecture requires 475 /* The Sparc v8 architecture requires
523 * three instructions between a %y 476 * three instructions between a %y
524 * register write and the first use. 477 * register write and the first use.
@@ -526,7 +479,6 @@ void bpf_jit_compile(struct bpf_prog *fp)
526 emit_nop(); 479 emit_nop();
527 emit_nop(); 480 emit_nop();
528 emit_nop(); 481 emit_nop();
529#endif
530 emit_alu_X(DIV); 482 emit_alu_X(DIV);
531 break; 483 break;
532 case BPF_ALU | BPF_NEG: 484 case BPF_ALU | BPF_NEG:
@@ -797,7 +749,6 @@ cond_branch: f_offset = addrs[i + filter[i].jf];
797 bpf_jit_dump(flen, proglen, pass + 1, image); 749 bpf_jit_dump(flen, proglen, pass + 1, image);
798 750
799 if (image) { 751 if (image) {
800 bpf_flush_icache(image, image + proglen);
801 fp->bpf_func = (void *)image; 752 fp->bpf_func = (void *)image;
802 fp->jited = 1; 753 fp->jited = 1;
803 } 754 }
diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c
new file mode 100644
index 000000000000..21de77419f48
--- /dev/null
+++ b/arch/sparc/net/bpf_jit_comp_64.c
@@ -0,0 +1,1566 @@
1#include <linux/moduleloader.h>
2#include <linux/workqueue.h>
3#include <linux/netdevice.h>
4#include <linux/filter.h>
5#include <linux/bpf.h>
6#include <linux/cache.h>
7#include <linux/if_vlan.h>
8
9#include <asm/cacheflush.h>
10#include <asm/ptrace.h>
11
12#include "bpf_jit_64.h"
13
14int bpf_jit_enable __read_mostly;
15
16static inline bool is_simm13(unsigned int value)
17{
18 return value + 0x1000 < 0x2000;
19}
20
21static inline bool is_simm10(unsigned int value)
22{
23 return value + 0x200 < 0x400;
24}
25
26static inline bool is_simm5(unsigned int value)
27{
28 return value + 0x10 < 0x20;
29}
30
31static inline bool is_sethi(unsigned int value)
32{
33 return (value & ~0x3fffff) == 0;
34}
35
36static void bpf_flush_icache(void *start_, void *end_)
37{
38 /* Cheetah's I-cache is fully coherent. */
39 if (tlb_type == spitfire) {
40 unsigned long start = (unsigned long) start_;
41 unsigned long end = (unsigned long) end_;
42
43 start &= ~7UL;
44 end = (end + 7UL) & ~7UL;
45 while (start < end) {
46 flushi(start);
47 start += 32;
48 }
49 }
50}
51
52#define SEEN_DATAREF 1 /* might call external helpers */
53#define SEEN_XREG 2 /* ebx is used */
54#define SEEN_MEM 4 /* use mem[] for temporary storage */
55
56#define S13(X) ((X) & 0x1fff)
57#define S5(X) ((X) & 0x1f)
58#define IMMED 0x00002000
59#define RD(X) ((X) << 25)
60#define RS1(X) ((X) << 14)
61#define RS2(X) ((X))
62#define OP(X) ((X) << 30)
63#define OP2(X) ((X) << 22)
64#define OP3(X) ((X) << 19)
65#define COND(X) (((X) & 0xf) << 25)
66#define CBCOND(X) (((X) & 0x1f) << 25)
67#define F1(X) OP(X)
68#define F2(X, Y) (OP(X) | OP2(Y))
69#define F3(X, Y) (OP(X) | OP3(Y))
70#define ASI(X) (((X) & 0xff) << 5)
71
72#define CONDN COND(0x0)
73#define CONDE COND(0x1)
74#define CONDLE COND(0x2)
75#define CONDL COND(0x3)
76#define CONDLEU COND(0x4)
77#define CONDCS COND(0x5)
78#define CONDNEG COND(0x6)
79#define CONDVC COND(0x7)
80#define CONDA COND(0x8)
81#define CONDNE COND(0x9)
82#define CONDG COND(0xa)
83#define CONDGE COND(0xb)
84#define CONDGU COND(0xc)
85#define CONDCC COND(0xd)
86#define CONDPOS COND(0xe)
87#define CONDVS COND(0xf)
88
89#define CONDGEU CONDCC
90#define CONDLU CONDCS
91
92#define WDISP22(X) (((X) >> 2) & 0x3fffff)
93#define WDISP19(X) (((X) >> 2) & 0x7ffff)
94
95/* The 10-bit branch displacement for CBCOND is split into two fields */
96static u32 WDISP10(u32 off)
97{
98 u32 ret = ((off >> 2) & 0xff) << 5;
99
100 ret |= ((off >> (2 + 8)) & 0x03) << 19;
101
102 return ret;
103}
104
105#define CBCONDE CBCOND(0x09)
106#define CBCONDLE CBCOND(0x0a)
107#define CBCONDL CBCOND(0x0b)
108#define CBCONDLEU CBCOND(0x0c)
109#define CBCONDCS CBCOND(0x0d)
110#define CBCONDN CBCOND(0x0e)
111#define CBCONDVS CBCOND(0x0f)
112#define CBCONDNE CBCOND(0x19)
113#define CBCONDG CBCOND(0x1a)
114#define CBCONDGE CBCOND(0x1b)
115#define CBCONDGU CBCOND(0x1c)
116#define CBCONDCC CBCOND(0x1d)
117#define CBCONDPOS CBCOND(0x1e)
118#define CBCONDVC CBCOND(0x1f)
119
120#define CBCONDGEU CBCONDCC
121#define CBCONDLU CBCONDCS
122
123#define ANNUL (1 << 29)
124#define XCC (1 << 21)
125
126#define BRANCH (F2(0, 1) | XCC)
127#define CBCOND_OP (F2(0, 3) | XCC)
128
129#define BA (BRANCH | CONDA)
130#define BG (BRANCH | CONDG)
131#define BGU (BRANCH | CONDGU)
132#define BLEU (BRANCH | CONDLEU)
133#define BGE (BRANCH | CONDGE)
134#define BGEU (BRANCH | CONDGEU)
135#define BLU (BRANCH | CONDLU)
136#define BE (BRANCH | CONDE)
137#define BNE (BRANCH | CONDNE)
138
139#define SETHI(K, REG) \
140 (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff))
141#define OR_LO(K, REG) \
142 (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG))
143
144#define ADD F3(2, 0x00)
145#define AND F3(2, 0x01)
146#define ANDCC F3(2, 0x11)
147#define OR F3(2, 0x02)
148#define XOR F3(2, 0x03)
149#define SUB F3(2, 0x04)
150#define SUBCC F3(2, 0x14)
151#define MUL F3(2, 0x0a)
152#define MULX F3(2, 0x09)
153#define UDIVX F3(2, 0x0d)
154#define DIV F3(2, 0x0e)
155#define SLL F3(2, 0x25)
156#define SLLX (F3(2, 0x25)|(1<<12))
157#define SRA F3(2, 0x27)
158#define SRAX (F3(2, 0x27)|(1<<12))
159#define SRL F3(2, 0x26)
160#define SRLX (F3(2, 0x26)|(1<<12))
161#define JMPL F3(2, 0x38)
162#define SAVE F3(2, 0x3c)
163#define RESTORE F3(2, 0x3d)
164#define CALL F1(1)
165#define BR F2(0, 0x01)
166#define RD_Y F3(2, 0x28)
167#define WR_Y F3(2, 0x30)
168
169#define LD32 F3(3, 0x00)
170#define LD8 F3(3, 0x01)
171#define LD16 F3(3, 0x02)
172#define LD64 F3(3, 0x0b)
173#define LD64A F3(3, 0x1b)
174#define ST8 F3(3, 0x05)
175#define ST16 F3(3, 0x06)
176#define ST32 F3(3, 0x04)
177#define ST64 F3(3, 0x0e)
178
179#define CAS F3(3, 0x3c)
180#define CASX F3(3, 0x3e)
181
182#define LDPTR LD64
183#define BASE_STACKFRAME 176
184
185#define LD32I (LD32 | IMMED)
186#define LD8I (LD8 | IMMED)
187#define LD16I (LD16 | IMMED)
188#define LD64I (LD64 | IMMED)
189#define LDPTRI (LDPTR | IMMED)
190#define ST32I (ST32 | IMMED)
191
192struct jit_ctx {
193 struct bpf_prog *prog;
194 unsigned int *offset;
195 int idx;
196 int epilogue_offset;
197 bool tmp_1_used;
198 bool tmp_2_used;
199 bool tmp_3_used;
200 bool saw_ld_abs_ind;
201 bool saw_frame_pointer;
202 bool saw_call;
203 bool saw_tail_call;
204 u32 *image;
205};
206
207#define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
208#define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
209#define SKB_HLEN_REG (MAX_BPF_JIT_REG + 2)
210#define SKB_DATA_REG (MAX_BPF_JIT_REG + 3)
211#define TMP_REG_3 (MAX_BPF_JIT_REG + 4)
212
213/* Map BPF registers to SPARC registers */
214static const int bpf2sparc[] = {
215 /* return value from in-kernel function, and exit value from eBPF */
216 [BPF_REG_0] = O5,
217
218 /* arguments from eBPF program to in-kernel function */
219 [BPF_REG_1] = O0,
220 [BPF_REG_2] = O1,
221 [BPF_REG_3] = O2,
222 [BPF_REG_4] = O3,
223 [BPF_REG_5] = O4,
224
225 /* callee saved registers that in-kernel function will preserve */
226 [BPF_REG_6] = L0,
227 [BPF_REG_7] = L1,
228 [BPF_REG_8] = L2,
229 [BPF_REG_9] = L3,
230
231 /* read-only frame pointer to access stack */
232 [BPF_REG_FP] = L6,
233
234 [BPF_REG_AX] = G7,
235
236 /* temporary register for internal BPF JIT */
237 [TMP_REG_1] = G1,
238 [TMP_REG_2] = G2,
239 [TMP_REG_3] = G3,
240
241 [SKB_HLEN_REG] = L4,
242 [SKB_DATA_REG] = L5,
243};
244
245static void emit(const u32 insn, struct jit_ctx *ctx)
246{
247 if (ctx->image != NULL)
248 ctx->image[ctx->idx] = insn;
249
250 ctx->idx++;
251}
252
253static void emit_call(u32 *func, struct jit_ctx *ctx)
254{
255 if (ctx->image != NULL) {
256 void *here = &ctx->image[ctx->idx];
257 unsigned int off;
258
259 off = (void *)func - here;
260 ctx->image[ctx->idx] = CALL | ((off >> 2) & 0x3fffffff);
261 }
262 ctx->idx++;
263}
264
265static void emit_nop(struct jit_ctx *ctx)
266{
267 emit(SETHI(0, G0), ctx);
268}
269
270static void emit_reg_move(u32 from, u32 to, struct jit_ctx *ctx)
271{
272 emit(OR | RS1(G0) | RS2(from) | RD(to), ctx);
273}
274
275/* Emit 32-bit constant, zero extended. */
276static void emit_set_const(s32 K, u32 reg, struct jit_ctx *ctx)
277{
278 emit(SETHI(K, reg), ctx);
279 emit(OR_LO(K, reg), ctx);
280}
281
282/* Emit 32-bit constant, sign extended. */
283static void emit_set_const_sext(s32 K, u32 reg, struct jit_ctx *ctx)
284{
285 if (K >= 0) {
286 emit(SETHI(K, reg), ctx);
287 emit(OR_LO(K, reg), ctx);
288 } else {
289 u32 hbits = ~(u32) K;
290 u32 lbits = -0x400 | (u32) K;
291
292 emit(SETHI(hbits, reg), ctx);
293 emit(XOR | IMMED | RS1(reg) | S13(lbits) | RD(reg), ctx);
294 }
295}
296
297static void emit_alu(u32 opcode, u32 src, u32 dst, struct jit_ctx *ctx)
298{
299 emit(opcode | RS1(dst) | RS2(src) | RD(dst), ctx);
300}
301
302static void emit_alu3(u32 opcode, u32 a, u32 b, u32 c, struct jit_ctx *ctx)
303{
304 emit(opcode | RS1(a) | RS2(b) | RD(c), ctx);
305}
306
307static void emit_alu_K(unsigned int opcode, unsigned int dst, unsigned int imm,
308 struct jit_ctx *ctx)
309{
310 bool small_immed = is_simm13(imm);
311 unsigned int insn = opcode;
312
313 insn |= RS1(dst) | RD(dst);
314 if (small_immed) {
315 emit(insn | IMMED | S13(imm), ctx);
316 } else {
317 unsigned int tmp = bpf2sparc[TMP_REG_1];
318
319 ctx->tmp_1_used = true;
320
321 emit_set_const_sext(imm, tmp, ctx);
322 emit(insn | RS2(tmp), ctx);
323 }
324}
325
326static void emit_alu3_K(unsigned int opcode, unsigned int src, unsigned int imm,
327 unsigned int dst, struct jit_ctx *ctx)
328{
329 bool small_immed = is_simm13(imm);
330 unsigned int insn = opcode;
331
332 insn |= RS1(src) | RD(dst);
333 if (small_immed) {
334 emit(insn | IMMED | S13(imm), ctx);
335 } else {
336 unsigned int tmp = bpf2sparc[TMP_REG_1];
337
338 ctx->tmp_1_used = true;
339
340 emit_set_const_sext(imm, tmp, ctx);
341 emit(insn | RS2(tmp), ctx);
342 }
343}
344
345static void emit_loadimm32(s32 K, unsigned int dest, struct jit_ctx *ctx)
346{
347 if (K >= 0 && is_simm13(K)) {
348 /* or %g0, K, DEST */
349 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
350 } else {
351 emit_set_const(K, dest, ctx);
352 }
353}
354
355static void emit_loadimm(s32 K, unsigned int dest, struct jit_ctx *ctx)
356{
357 if (is_simm13(K)) {
358 /* or %g0, K, DEST */
359 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
360 } else {
361 emit_set_const(K, dest, ctx);
362 }
363}
364
365static void emit_loadimm_sext(s32 K, unsigned int dest, struct jit_ctx *ctx)
366{
367 if (is_simm13(K)) {
368 /* or %g0, K, DEST */
369 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
370 } else {
371 emit_set_const_sext(K, dest, ctx);
372 }
373}
374
375static void analyze_64bit_constant(u32 high_bits, u32 low_bits,
376 int *hbsp, int *lbsp, int *abbasp)
377{
378 int lowest_bit_set, highest_bit_set, all_bits_between_are_set;
379 int i;
380
381 lowest_bit_set = highest_bit_set = -1;
382 i = 0;
383 do {
384 if ((lowest_bit_set == -1) && ((low_bits >> i) & 1))
385 lowest_bit_set = i;
386 if ((highest_bit_set == -1) && ((high_bits >> (32 - i - 1)) & 1))
387 highest_bit_set = (64 - i - 1);
388 } while (++i < 32 && (highest_bit_set == -1 ||
389 lowest_bit_set == -1));
390 if (i == 32) {
391 i = 0;
392 do {
393 if (lowest_bit_set == -1 && ((high_bits >> i) & 1))
394 lowest_bit_set = i + 32;
395 if (highest_bit_set == -1 &&
396 ((low_bits >> (32 - i - 1)) & 1))
397 highest_bit_set = 32 - i - 1;
398 } while (++i < 32 && (highest_bit_set == -1 ||
399 lowest_bit_set == -1));
400 }
401
402 all_bits_between_are_set = 1;
403 for (i = lowest_bit_set; i <= highest_bit_set; i++) {
404 if (i < 32) {
405 if ((low_bits & (1 << i)) != 0)
406 continue;
407 } else {
408 if ((high_bits & (1 << (i - 32))) != 0)
409 continue;
410 }
411 all_bits_between_are_set = 0;
412 break;
413 }
414 *hbsp = highest_bit_set;
415 *lbsp = lowest_bit_set;
416 *abbasp = all_bits_between_are_set;
417}
418
419static unsigned long create_simple_focus_bits(unsigned long high_bits,
420 unsigned long low_bits,
421 int lowest_bit_set, int shift)
422{
423 long hi, lo;
424
425 if (lowest_bit_set < 32) {
426 lo = (low_bits >> lowest_bit_set) << shift;
427 hi = ((high_bits << (32 - lowest_bit_set)) << shift);
428 } else {
429 lo = 0;
430 hi = ((high_bits >> (lowest_bit_set - 32)) << shift);
431 }
432 return hi | lo;
433}
434
435static bool const64_is_2insns(unsigned long high_bits,
436 unsigned long low_bits)
437{
438 int highest_bit_set, lowest_bit_set, all_bits_between_are_set;
439
440 if (high_bits == 0 || high_bits == 0xffffffff)
441 return true;
442
443 analyze_64bit_constant(high_bits, low_bits,
444 &highest_bit_set, &lowest_bit_set,
445 &all_bits_between_are_set);
446
447 if ((highest_bit_set == 63 || lowest_bit_set == 0) &&
448 all_bits_between_are_set != 0)
449 return true;
450
451 if (highest_bit_set - lowest_bit_set < 21)
452 return true;
453
454 return false;
455}
456
457static void sparc_emit_set_const64_quick2(unsigned long high_bits,
458 unsigned long low_imm,
459 unsigned int dest,
460 int shift_count, struct jit_ctx *ctx)
461{
462 emit_loadimm32(high_bits, dest, ctx);
463
464 /* Now shift it up into place. */
465 emit_alu_K(SLLX, dest, shift_count, ctx);
466
467 /* If there is a low immediate part piece, finish up by
468 * putting that in as well.
469 */
470 if (low_imm != 0)
471 emit(OR | IMMED | RS1(dest) | S13(low_imm) | RD(dest), ctx);
472}
473
474static void emit_loadimm64(u64 K, unsigned int dest, struct jit_ctx *ctx)
475{
476 int all_bits_between_are_set, lowest_bit_set, highest_bit_set;
477 unsigned int tmp = bpf2sparc[TMP_REG_1];
478 u32 low_bits = (K & 0xffffffff);
479 u32 high_bits = (K >> 32);
480
481 /* These two tests also take care of all of the one
482 * instruction cases.
483 */
484 if (high_bits == 0xffffffff && (low_bits & 0x80000000))
485 return emit_loadimm_sext(K, dest, ctx);
486 if (high_bits == 0x00000000)
487 return emit_loadimm32(K, dest, ctx);
488
489 analyze_64bit_constant(high_bits, low_bits, &highest_bit_set,
490 &lowest_bit_set, &all_bits_between_are_set);
491
492 /* 1) mov -1, %reg
493 * sllx %reg, shift, %reg
494 * 2) mov -1, %reg
495 * srlx %reg, shift, %reg
496 * 3) mov some_small_const, %reg
497 * sllx %reg, shift, %reg
498 */
499 if (((highest_bit_set == 63 || lowest_bit_set == 0) &&
500 all_bits_between_are_set != 0) ||
501 ((highest_bit_set - lowest_bit_set) < 12)) {
502 int shift = lowest_bit_set;
503 long the_const = -1;
504
505 if ((highest_bit_set != 63 && lowest_bit_set != 0) ||
506 all_bits_between_are_set == 0) {
507 the_const =
508 create_simple_focus_bits(high_bits, low_bits,
509 lowest_bit_set, 0);
510 } else if (lowest_bit_set == 0)
511 shift = -(63 - highest_bit_set);
512
513 emit(OR | IMMED | RS1(G0) | S13(the_const) | RD(dest), ctx);
514 if (shift > 0)
515 emit_alu_K(SLLX, dest, shift, ctx);
516 else if (shift < 0)
517 emit_alu_K(SRLX, dest, -shift, ctx);
518
519 return;
520 }
521
522 /* Now a range of 22 or less bits set somewhere.
523 * 1) sethi %hi(focus_bits), %reg
524 * sllx %reg, shift, %reg
525 * 2) sethi %hi(focus_bits), %reg
526 * srlx %reg, shift, %reg
527 */
528 if ((highest_bit_set - lowest_bit_set) < 21) {
529 unsigned long focus_bits =
530 create_simple_focus_bits(high_bits, low_bits,
531 lowest_bit_set, 10);
532
533 emit(SETHI(focus_bits, dest), ctx);
534
535 /* If lowest_bit_set == 10 then a sethi alone could
536 * have done it.
537 */
538 if (lowest_bit_set < 10)
539 emit_alu_K(SRLX, dest, 10 - lowest_bit_set, ctx);
540 else if (lowest_bit_set > 10)
541 emit_alu_K(SLLX, dest, lowest_bit_set - 10, ctx);
542 return;
543 }
544
545 /* Ok, now 3 instruction sequences. */
546 if (low_bits == 0) {
547 emit_loadimm32(high_bits, dest, ctx);
548 emit_alu_K(SLLX, dest, 32, ctx);
549 return;
550 }
551
552 /* We may be able to do something quick
553 * when the constant is negated, so try that.
554 */
555 if (const64_is_2insns((~high_bits) & 0xffffffff,
556 (~low_bits) & 0xfffffc00)) {
557 /* NOTE: The trailing bits get XOR'd so we need the
558 * non-negated bits, not the negated ones.
559 */
560 unsigned long trailing_bits = low_bits & 0x3ff;
561
562 if ((((~high_bits) & 0xffffffff) == 0 &&
563 ((~low_bits) & 0x80000000) == 0) ||
564 (((~high_bits) & 0xffffffff) == 0xffffffff &&
565 ((~low_bits) & 0x80000000) != 0)) {
566 unsigned long fast_int = (~low_bits & 0xffffffff);
567
568 if ((is_sethi(fast_int) &&
569 (~high_bits & 0xffffffff) == 0)) {
570 emit(SETHI(fast_int, dest), ctx);
571 } else if (is_simm13(fast_int)) {
572 emit(OR | IMMED | RS1(G0) | S13(fast_int) | RD(dest), ctx);
573 } else {
574 emit_loadimm64(fast_int, dest, ctx);
575 }
576 } else {
577 u64 n = ((~low_bits) & 0xfffffc00) |
578 (((unsigned long)((~high_bits) & 0xffffffff))<<32);
579 emit_loadimm64(n, dest, ctx);
580 }
581
582 low_bits = -0x400 | trailing_bits;
583
584 emit(XOR | IMMED | RS1(dest) | S13(low_bits) | RD(dest), ctx);
585 return;
586 }
587
588 /* 1) sethi %hi(xxx), %reg
589 * or %reg, %lo(xxx), %reg
590 * sllx %reg, yyy, %reg
591 */
592 if ((highest_bit_set - lowest_bit_set) < 32) {
593 unsigned long focus_bits =
594 create_simple_focus_bits(high_bits, low_bits,
595 lowest_bit_set, 0);
596
597 /* So what we know is that the set bits straddle the
598 * middle of the 64-bit word.
599 */
600 sparc_emit_set_const64_quick2(focus_bits, 0, dest,
601 lowest_bit_set, ctx);
602 return;
603 }
604
605 /* 1) sethi %hi(high_bits), %reg
606 * or %reg, %lo(high_bits), %reg
607 * sllx %reg, 32, %reg
608 * or %reg, low_bits, %reg
609 */
610 if (is_simm13(low_bits) && ((int)low_bits > 0)) {
611 sparc_emit_set_const64_quick2(high_bits, low_bits,
612 dest, 32, ctx);
613 return;
614 }
615
616 /* Oh well, we tried... Do a full 64-bit decomposition. */
617 ctx->tmp_1_used = true;
618
619 emit_loadimm32(high_bits, tmp, ctx);
620 emit_loadimm32(low_bits, dest, ctx);
621 emit_alu_K(SLLX, tmp, 32, ctx);
622 emit(OR | RS1(dest) | RS2(tmp) | RD(dest), ctx);
623}
624
625static void emit_branch(unsigned int br_opc, unsigned int from_idx, unsigned int to_idx,
626 struct jit_ctx *ctx)
627{
628 unsigned int off = to_idx - from_idx;
629
630 if (br_opc & XCC)
631 emit(br_opc | WDISP19(off << 2), ctx);
632 else
633 emit(br_opc | WDISP22(off << 2), ctx);
634}
635
636static void emit_cbcond(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
637 const u8 dst, const u8 src, struct jit_ctx *ctx)
638{
639 unsigned int off = to_idx - from_idx;
640
641 emit(cb_opc | WDISP10(off << 2) | RS1(dst) | RS2(src), ctx);
642}
643
644static void emit_cbcondi(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
645 const u8 dst, s32 imm, struct jit_ctx *ctx)
646{
647 unsigned int off = to_idx - from_idx;
648
649 emit(cb_opc | IMMED | WDISP10(off << 2) | RS1(dst) | S5(imm), ctx);
650}
651
652#define emit_read_y(REG, CTX) emit(RD_Y | RD(REG), CTX)
653#define emit_write_y(REG, CTX) emit(WR_Y | IMMED | RS1(REG) | S13(0), CTX)
654
655#define emit_cmp(R1, R2, CTX) \
656 emit(SUBCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
657
658#define emit_cmpi(R1, IMM, CTX) \
659 emit(SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
660
661#define emit_btst(R1, R2, CTX) \
662 emit(ANDCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
663
664#define emit_btsti(R1, IMM, CTX) \
665 emit(ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
666
667static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src,
668 const s32 imm, bool is_imm, int branch_dst,
669 struct jit_ctx *ctx)
670{
671 bool use_cbcond = (sparc64_elf_hwcap & AV_SPARC_CBCOND) != 0;
672 const u8 tmp = bpf2sparc[TMP_REG_1];
673
674 branch_dst = ctx->offset[branch_dst];
675
676 if (!is_simm10(branch_dst - ctx->idx) ||
677 BPF_OP(code) == BPF_JSET)
678 use_cbcond = false;
679
680 if (is_imm) {
681 bool fits = true;
682
683 if (use_cbcond) {
684 if (!is_simm5(imm))
685 fits = false;
686 } else if (!is_simm13(imm)) {
687 fits = false;
688 }
689 if (!fits) {
690 ctx->tmp_1_used = true;
691 emit_loadimm_sext(imm, tmp, ctx);
692 src = tmp;
693 is_imm = false;
694 }
695 }
696
697 if (!use_cbcond) {
698 u32 br_opcode;
699
700 if (BPF_OP(code) == BPF_JSET) {
701 if (is_imm)
702 emit_btsti(dst, imm, ctx);
703 else
704 emit_btst(dst, src, ctx);
705 } else {
706 if (is_imm)
707 emit_cmpi(dst, imm, ctx);
708 else
709 emit_cmp(dst, src, ctx);
710 }
711 switch (BPF_OP(code)) {
712 case BPF_JEQ:
713 br_opcode = BE;
714 break;
715 case BPF_JGT:
716 br_opcode = BGU;
717 break;
718 case BPF_JGE:
719 br_opcode = BGEU;
720 break;
721 case BPF_JSET:
722 case BPF_JNE:
723 br_opcode = BNE;
724 break;
725 case BPF_JSGT:
726 br_opcode = BG;
727 break;
728 case BPF_JSGE:
729 br_opcode = BGE;
730 break;
731 default:
732 /* Make sure we dont leak kernel information to the
733 * user.
734 */
735 return -EFAULT;
736 }
737 emit_branch(br_opcode, ctx->idx, branch_dst, ctx);
738 emit_nop(ctx);
739 } else {
740 u32 cbcond_opcode;
741
742 switch (BPF_OP(code)) {
743 case BPF_JEQ:
744 cbcond_opcode = CBCONDE;
745 break;
746 case BPF_JGT:
747 cbcond_opcode = CBCONDGU;
748 break;
749 case BPF_JGE:
750 cbcond_opcode = CBCONDGEU;
751 break;
752 case BPF_JNE:
753 cbcond_opcode = CBCONDNE;
754 break;
755 case BPF_JSGT:
756 cbcond_opcode = CBCONDG;
757 break;
758 case BPF_JSGE:
759 cbcond_opcode = CBCONDGE;
760 break;
761 default:
762 /* Make sure we dont leak kernel information to the
763 * user.
764 */
765 return -EFAULT;
766 }
767 cbcond_opcode |= CBCOND_OP;
768 if (is_imm)
769 emit_cbcondi(cbcond_opcode, ctx->idx, branch_dst,
770 dst, imm, ctx);
771 else
772 emit_cbcond(cbcond_opcode, ctx->idx, branch_dst,
773 dst, src, ctx);
774 }
775 return 0;
776}
777
778static void load_skb_regs(struct jit_ctx *ctx, u8 r_skb)
779{
780 const u8 r_headlen = bpf2sparc[SKB_HLEN_REG];
781 const u8 r_data = bpf2sparc[SKB_DATA_REG];
782 const u8 r_tmp = bpf2sparc[TMP_REG_1];
783 unsigned int off;
784
785 off = offsetof(struct sk_buff, len);
786 emit(LD32I | RS1(r_skb) | S13(off) | RD(r_headlen), ctx);
787
788 off = offsetof(struct sk_buff, data_len);
789 emit(LD32I | RS1(r_skb) | S13(off) | RD(r_tmp), ctx);
790
791 emit(SUB | RS1(r_headlen) | RS2(r_tmp) | RD(r_headlen), ctx);
792
793 off = offsetof(struct sk_buff, data);
794 emit(LDPTRI | RS1(r_skb) | S13(off) | RD(r_data), ctx);
795}
796
797/* Just skip the save instruction and the ctx register move. */
798#define BPF_TAILCALL_PROLOGUE_SKIP 16
799#define BPF_TAILCALL_CNT_SP_OFF (STACK_BIAS + 128)
800
801static void build_prologue(struct jit_ctx *ctx)
802{
803 s32 stack_needed = BASE_STACKFRAME;
804
805 if (ctx->saw_frame_pointer || ctx->saw_tail_call)
806 stack_needed += MAX_BPF_STACK;
807
808 if (ctx->saw_tail_call)
809 stack_needed += 8;
810
811 /* save %sp, -176, %sp */
812 emit(SAVE | IMMED | RS1(SP) | S13(-stack_needed) | RD(SP), ctx);
813
814 /* tail_call_cnt = 0 */
815 if (ctx->saw_tail_call) {
816 u32 off = BPF_TAILCALL_CNT_SP_OFF;
817
818 emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(G0), ctx);
819 } else {
820 emit_nop(ctx);
821 }
822 if (ctx->saw_frame_pointer) {
823 const u8 vfp = bpf2sparc[BPF_REG_FP];
824
825 emit(ADD | IMMED | RS1(FP) | S13(STACK_BIAS) | RD(vfp), ctx);
826 }
827
828 emit_reg_move(I0, O0, ctx);
829 /* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */
830
831 if (ctx->saw_ld_abs_ind)
832 load_skb_regs(ctx, bpf2sparc[BPF_REG_1]);
833}
834
835static void build_epilogue(struct jit_ctx *ctx)
836{
837 ctx->epilogue_offset = ctx->idx;
838
839 /* ret (jmpl %i7 + 8, %g0) */
840 emit(JMPL | IMMED | RS1(I7) | S13(8) | RD(G0), ctx);
841
842 /* restore %i5, %g0, %o0 */
843 emit(RESTORE | RS1(bpf2sparc[BPF_REG_0]) | RS2(G0) | RD(O0), ctx);
844}
845
846static void emit_tail_call(struct jit_ctx *ctx)
847{
848 const u8 bpf_array = bpf2sparc[BPF_REG_2];
849 const u8 bpf_index = bpf2sparc[BPF_REG_3];
850 const u8 tmp = bpf2sparc[TMP_REG_1];
851 u32 off;
852
853 ctx->saw_tail_call = true;
854
855 off = offsetof(struct bpf_array, map.max_entries);
856 emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx);
857 emit_cmp(bpf_index, tmp, ctx);
858#define OFFSET1 17
859 emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET1, ctx);
860 emit_nop(ctx);
861
862 off = BPF_TAILCALL_CNT_SP_OFF;
863 emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
864 emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx);
865#define OFFSET2 13
866 emit_branch(BGU, ctx->idx, ctx->idx + OFFSET2, ctx);
867 emit_nop(ctx);
868
869 emit_alu_K(ADD, tmp, 1, ctx);
870 off = BPF_TAILCALL_CNT_SP_OFF;
871 emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
872
873 emit_alu3_K(SLL, bpf_index, 3, tmp, ctx);
874 emit_alu(ADD, bpf_array, tmp, ctx);
875 off = offsetof(struct bpf_array, ptrs);
876 emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
877
878 emit_cmpi(tmp, 0, ctx);
879#define OFFSET3 5
880 emit_branch(BE, ctx->idx, ctx->idx + OFFSET3, ctx);
881 emit_nop(ctx);
882
883 off = offsetof(struct bpf_prog, bpf_func);
884 emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
885
886 off = BPF_TAILCALL_PROLOGUE_SKIP;
887 emit(JMPL | IMMED | RS1(tmp) | S13(off) | RD(G0), ctx);
888 emit_nop(ctx);
889}
890
891static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
892{
893 const u8 code = insn->code;
894 const u8 dst = bpf2sparc[insn->dst_reg];
895 const u8 src = bpf2sparc[insn->src_reg];
896 const int i = insn - ctx->prog->insnsi;
897 const s16 off = insn->off;
898 const s32 imm = insn->imm;
899 u32 *func;
900
901 if (insn->src_reg == BPF_REG_FP)
902 ctx->saw_frame_pointer = true;
903
904 switch (code) {
905 /* dst = src */
906 case BPF_ALU | BPF_MOV | BPF_X:
907 emit_alu3_K(SRL, src, 0, dst, ctx);
908 break;
909 case BPF_ALU64 | BPF_MOV | BPF_X:
910 emit_reg_move(src, dst, ctx);
911 break;
912 /* dst = dst OP src */
913 case BPF_ALU | BPF_ADD | BPF_X:
914 case BPF_ALU64 | BPF_ADD | BPF_X:
915 emit_alu(ADD, src, dst, ctx);
916 goto do_alu32_trunc;
917 case BPF_ALU | BPF_SUB | BPF_X:
918 case BPF_ALU64 | BPF_SUB | BPF_X:
919 emit_alu(SUB, src, dst, ctx);
920 goto do_alu32_trunc;
921 case BPF_ALU | BPF_AND | BPF_X:
922 case BPF_ALU64 | BPF_AND | BPF_X:
923 emit_alu(AND, src, dst, ctx);
924 goto do_alu32_trunc;
925 case BPF_ALU | BPF_OR | BPF_X:
926 case BPF_ALU64 | BPF_OR | BPF_X:
927 emit_alu(OR, src, dst, ctx);
928 goto do_alu32_trunc;
929 case BPF_ALU | BPF_XOR | BPF_X:
930 case BPF_ALU64 | BPF_XOR | BPF_X:
931 emit_alu(XOR, src, dst, ctx);
932 goto do_alu32_trunc;
933 case BPF_ALU | BPF_MUL | BPF_X:
934 emit_alu(MUL, src, dst, ctx);
935 goto do_alu32_trunc;
936 case BPF_ALU64 | BPF_MUL | BPF_X:
937 emit_alu(MULX, src, dst, ctx);
938 break;
939 case BPF_ALU | BPF_DIV | BPF_X:
940 emit_cmp(src, G0, ctx);
941 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
942 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
943
944 emit_write_y(G0, ctx);
945 emit_alu(DIV, src, dst, ctx);
946 break;
947
948 case BPF_ALU64 | BPF_DIV | BPF_X:
949 emit_cmp(src, G0, ctx);
950 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
951 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
952
953 emit_alu(UDIVX, src, dst, ctx);
954 break;
955
956 case BPF_ALU | BPF_MOD | BPF_X: {
957 const u8 tmp = bpf2sparc[TMP_REG_1];
958
959 ctx->tmp_1_used = true;
960
961 emit_cmp(src, G0, ctx);
962 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
963 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
964
965 emit_write_y(G0, ctx);
966 emit_alu3(DIV, dst, src, tmp, ctx);
967 emit_alu3(MULX, tmp, src, tmp, ctx);
968 emit_alu3(SUB, dst, tmp, dst, ctx);
969 goto do_alu32_trunc;
970 }
971 case BPF_ALU64 | BPF_MOD | BPF_X: {
972 const u8 tmp = bpf2sparc[TMP_REG_1];
973
974 ctx->tmp_1_used = true;
975
976 emit_cmp(src, G0, ctx);
977 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
978 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
979
980 emit_alu3(UDIVX, dst, src, tmp, ctx);
981 emit_alu3(MULX, tmp, src, tmp, ctx);
982 emit_alu3(SUB, dst, tmp, dst, ctx);
983 break;
984 }
985 case BPF_ALU | BPF_LSH | BPF_X:
986 emit_alu(SLL, src, dst, ctx);
987 goto do_alu32_trunc;
988 case BPF_ALU64 | BPF_LSH | BPF_X:
989 emit_alu(SLLX, src, dst, ctx);
990 break;
991 case BPF_ALU | BPF_RSH | BPF_X:
992 emit_alu(SRL, src, dst, ctx);
993 break;
994 case BPF_ALU64 | BPF_RSH | BPF_X:
995 emit_alu(SRLX, src, dst, ctx);
996 break;
997 case BPF_ALU | BPF_ARSH | BPF_X:
998 emit_alu(SRA, src, dst, ctx);
999 goto do_alu32_trunc;
1000 case BPF_ALU64 | BPF_ARSH | BPF_X:
1001 emit_alu(SRAX, src, dst, ctx);
1002 break;
1003
1004 /* dst = -dst */
1005 case BPF_ALU | BPF_NEG:
1006 case BPF_ALU64 | BPF_NEG:
1007 emit(SUB | RS1(0) | RS2(dst) | RD(dst), ctx);
1008 goto do_alu32_trunc;
1009
1010 case BPF_ALU | BPF_END | BPF_FROM_BE:
1011 switch (imm) {
1012 case 16:
1013 emit_alu_K(SLL, dst, 16, ctx);
1014 emit_alu_K(SRL, dst, 16, ctx);
1015 break;
1016 case 32:
1017 emit_alu_K(SRL, dst, 0, ctx);
1018 break;
1019 case 64:
1020 /* nop */
1021 break;
1022
1023 }
1024 break;
1025
1026 /* dst = BSWAP##imm(dst) */
1027 case BPF_ALU | BPF_END | BPF_FROM_LE: {
1028 const u8 tmp = bpf2sparc[TMP_REG_1];
1029 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1030
1031 ctx->tmp_1_used = true;
1032 switch (imm) {
1033 case 16:
1034 emit_alu3_K(AND, dst, 0xff, tmp, ctx);
1035 emit_alu3_K(SRL, dst, 8, dst, ctx);
1036 emit_alu3_K(AND, dst, 0xff, dst, ctx);
1037 emit_alu3_K(SLL, tmp, 8, tmp, ctx);
1038 emit_alu(OR, tmp, dst, ctx);
1039 break;
1040
1041 case 32:
1042 ctx->tmp_2_used = true;
1043 emit_alu3_K(SRL, dst, 24, tmp, ctx); /* tmp = dst >> 24 */
1044 emit_alu3_K(SRL, dst, 16, tmp2, ctx); /* tmp2 = dst >> 16 */
1045 emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1046 emit_alu3_K(SLL, tmp2, 8, tmp2, ctx); /* tmp2 = tmp2 << 8 */
1047 emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */
1048 emit_alu3_K(SRL, dst, 8, tmp2, ctx); /* tmp2 = dst >> 8 */
1049 emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1050 emit_alu3_K(SLL, tmp2, 16, tmp2, ctx); /* tmp2 = tmp2 << 16 */
1051 emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */
1052 emit_alu3_K(AND, dst, 0xff, dst, ctx); /* dst = dst & 0xff */
1053 emit_alu3_K(SLL, dst, 24, dst, ctx); /* dst = dst << 24 */
1054 emit_alu(OR, tmp, dst, ctx); /* dst = dst | tmp */
1055 break;
1056
1057 case 64:
1058 emit_alu3_K(ADD, SP, STACK_BIAS + 128, tmp, ctx);
1059 emit(ST64 | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1060 emit(LD64A | ASI(ASI_PL) | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1061 break;
1062 }
1063 break;
1064 }
1065 /* dst = imm */
1066 case BPF_ALU | BPF_MOV | BPF_K:
1067 emit_loadimm32(imm, dst, ctx);
1068 break;
1069 case BPF_ALU64 | BPF_MOV | BPF_K:
1070 emit_loadimm_sext(imm, dst, ctx);
1071 break;
1072 /* dst = dst OP imm */
1073 case BPF_ALU | BPF_ADD | BPF_K:
1074 case BPF_ALU64 | BPF_ADD | BPF_K:
1075 emit_alu_K(ADD, dst, imm, ctx);
1076 goto do_alu32_trunc;
1077 case BPF_ALU | BPF_SUB | BPF_K:
1078 case BPF_ALU64 | BPF_SUB | BPF_K:
1079 emit_alu_K(SUB, dst, imm, ctx);
1080 goto do_alu32_trunc;
1081 case BPF_ALU | BPF_AND | BPF_K:
1082 case BPF_ALU64 | BPF_AND | BPF_K:
1083 emit_alu_K(AND, dst, imm, ctx);
1084 goto do_alu32_trunc;
1085 case BPF_ALU | BPF_OR | BPF_K:
1086 case BPF_ALU64 | BPF_OR | BPF_K:
1087 emit_alu_K(OR, dst, imm, ctx);
1088 goto do_alu32_trunc;
1089 case BPF_ALU | BPF_XOR | BPF_K:
1090 case BPF_ALU64 | BPF_XOR | BPF_K:
1091 emit_alu_K(XOR, dst, imm, ctx);
1092 goto do_alu32_trunc;
1093 case BPF_ALU | BPF_MUL | BPF_K:
1094 emit_alu_K(MUL, dst, imm, ctx);
1095 goto do_alu32_trunc;
1096 case BPF_ALU64 | BPF_MUL | BPF_K:
1097 emit_alu_K(MULX, dst, imm, ctx);
1098 break;
1099 case BPF_ALU | BPF_DIV | BPF_K:
1100 if (imm == 0)
1101 return -EINVAL;
1102
1103 emit_write_y(G0, ctx);
1104 emit_alu_K(DIV, dst, imm, ctx);
1105 goto do_alu32_trunc;
1106 case BPF_ALU64 | BPF_DIV | BPF_K:
1107 if (imm == 0)
1108 return -EINVAL;
1109
1110 emit_alu_K(UDIVX, dst, imm, ctx);
1111 break;
1112 case BPF_ALU64 | BPF_MOD | BPF_K:
1113 case BPF_ALU | BPF_MOD | BPF_K: {
1114 const u8 tmp = bpf2sparc[TMP_REG_2];
1115 unsigned int div;
1116
1117 if (imm == 0)
1118 return -EINVAL;
1119
1120 div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV;
1121
1122 ctx->tmp_2_used = true;
1123
1124 if (BPF_CLASS(code) != BPF_ALU64)
1125 emit_write_y(G0, ctx);
1126 if (is_simm13(imm)) {
1127 emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx);
1128 emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx);
1129 emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1130 } else {
1131 const u8 tmp1 = bpf2sparc[TMP_REG_1];
1132
1133 ctx->tmp_1_used = true;
1134
1135 emit_set_const_sext(imm, tmp1, ctx);
1136 emit(div | RS1(dst) | RS2(tmp1) | RD(tmp), ctx);
1137 emit(MULX | RS1(tmp) | RS2(tmp1) | RD(tmp), ctx);
1138 emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1139 }
1140 goto do_alu32_trunc;
1141 }
1142 case BPF_ALU | BPF_LSH | BPF_K:
1143 emit_alu_K(SLL, dst, imm, ctx);
1144 goto do_alu32_trunc;
1145 case BPF_ALU64 | BPF_LSH | BPF_K:
1146 emit_alu_K(SLLX, dst, imm, ctx);
1147 break;
1148 case BPF_ALU | BPF_RSH | BPF_K:
1149 emit_alu_K(SRL, dst, imm, ctx);
1150 break;
1151 case BPF_ALU64 | BPF_RSH | BPF_K:
1152 emit_alu_K(SRLX, dst, imm, ctx);
1153 break;
1154 case BPF_ALU | BPF_ARSH | BPF_K:
1155 emit_alu_K(SRA, dst, imm, ctx);
1156 goto do_alu32_trunc;
1157 case BPF_ALU64 | BPF_ARSH | BPF_K:
1158 emit_alu_K(SRAX, dst, imm, ctx);
1159 break;
1160
1161 do_alu32_trunc:
1162 if (BPF_CLASS(code) == BPF_ALU)
1163 emit_alu_K(SRL, dst, 0, ctx);
1164 break;
1165
1166 /* JUMP off */
1167 case BPF_JMP | BPF_JA:
1168 emit_branch(BA, ctx->idx, ctx->offset[i + off], ctx);
1169 emit_nop(ctx);
1170 break;
1171 /* IF (dst COND src) JUMP off */
1172 case BPF_JMP | BPF_JEQ | BPF_X:
1173 case BPF_JMP | BPF_JGT | BPF_X:
1174 case BPF_JMP | BPF_JGE | BPF_X:
1175 case BPF_JMP | BPF_JNE | BPF_X:
1176 case BPF_JMP | BPF_JSGT | BPF_X:
1177 case BPF_JMP | BPF_JSGE | BPF_X:
1178 case BPF_JMP | BPF_JSET | BPF_X: {
1179 int err;
1180
1181 err = emit_compare_and_branch(code, dst, src, 0, false, i + off, ctx);
1182 if (err)
1183 return err;
1184 break;
1185 }
1186 /* IF (dst COND imm) JUMP off */
1187 case BPF_JMP | BPF_JEQ | BPF_K:
1188 case BPF_JMP | BPF_JGT | BPF_K:
1189 case BPF_JMP | BPF_JGE | BPF_K:
1190 case BPF_JMP | BPF_JNE | BPF_K:
1191 case BPF_JMP | BPF_JSGT | BPF_K:
1192 case BPF_JMP | BPF_JSGE | BPF_K:
1193 case BPF_JMP | BPF_JSET | BPF_K: {
1194 int err;
1195
1196 err = emit_compare_and_branch(code, dst, 0, imm, true, i + off, ctx);
1197 if (err)
1198 return err;
1199 break;
1200 }
1201
1202 /* function call */
1203 case BPF_JMP | BPF_CALL:
1204 {
1205 u8 *func = ((u8 *)__bpf_call_base) + imm;
1206
1207 ctx->saw_call = true;
1208
1209 emit_call((u32 *)func, ctx);
1210 emit_nop(ctx);
1211
1212 emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
1213
1214 if (bpf_helper_changes_pkt_data(func) && ctx->saw_ld_abs_ind)
1215 load_skb_regs(ctx, bpf2sparc[BPF_REG_6]);
1216 break;
1217 }
1218
1219 /* tail call */
1220 case BPF_JMP | BPF_CALL |BPF_X:
1221 emit_tail_call(ctx);
1222 break;
1223
1224 /* function return */
1225 case BPF_JMP | BPF_EXIT:
1226 /* Optimization: when last instruction is EXIT,
1227 simply fallthrough to epilogue. */
1228 if (i == ctx->prog->len - 1)
1229 break;
1230 emit_branch(BA, ctx->idx, ctx->epilogue_offset, ctx);
1231 emit_nop(ctx);
1232 break;
1233
1234 /* dst = imm64 */
1235 case BPF_LD | BPF_IMM | BPF_DW:
1236 {
1237 const struct bpf_insn insn1 = insn[1];
1238 u64 imm64;
1239
1240 imm64 = (u64)insn1.imm << 32 | (u32)imm;
1241 emit_loadimm64(imm64, dst, ctx);
1242
1243 return 1;
1244 }
1245
1246 /* LDX: dst = *(size *)(src + off) */
1247 case BPF_LDX | BPF_MEM | BPF_W:
1248 case BPF_LDX | BPF_MEM | BPF_H:
1249 case BPF_LDX | BPF_MEM | BPF_B:
1250 case BPF_LDX | BPF_MEM | BPF_DW: {
1251 const u8 tmp = bpf2sparc[TMP_REG_1];
1252 u32 opcode = 0, rs2;
1253
1254 ctx->tmp_1_used = true;
1255 switch (BPF_SIZE(code)) {
1256 case BPF_W:
1257 opcode = LD32;
1258 break;
1259 case BPF_H:
1260 opcode = LD16;
1261 break;
1262 case BPF_B:
1263 opcode = LD8;
1264 break;
1265 case BPF_DW:
1266 opcode = LD64;
1267 break;
1268 }
1269
1270 if (is_simm13(off)) {
1271 opcode |= IMMED;
1272 rs2 = S13(off);
1273 } else {
1274 emit_loadimm(off, tmp, ctx);
1275 rs2 = RS2(tmp);
1276 }
1277 emit(opcode | RS1(src) | rs2 | RD(dst), ctx);
1278 break;
1279 }
1280 /* ST: *(size *)(dst + off) = imm */
1281 case BPF_ST | BPF_MEM | BPF_W:
1282 case BPF_ST | BPF_MEM | BPF_H:
1283 case BPF_ST | BPF_MEM | BPF_B:
1284 case BPF_ST | BPF_MEM | BPF_DW: {
1285 const u8 tmp = bpf2sparc[TMP_REG_1];
1286 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1287 u32 opcode = 0, rs2;
1288
1289 ctx->tmp_2_used = true;
1290 emit_loadimm(imm, tmp2, ctx);
1291
1292 switch (BPF_SIZE(code)) {
1293 case BPF_W:
1294 opcode = ST32;
1295 break;
1296 case BPF_H:
1297 opcode = ST16;
1298 break;
1299 case BPF_B:
1300 opcode = ST8;
1301 break;
1302 case BPF_DW:
1303 opcode = ST64;
1304 break;
1305 }
1306
1307 if (is_simm13(off)) {
1308 opcode |= IMMED;
1309 rs2 = S13(off);
1310 } else {
1311 ctx->tmp_1_used = true;
1312 emit_loadimm(off, tmp, ctx);
1313 rs2 = RS2(tmp);
1314 }
1315 emit(opcode | RS1(dst) | rs2 | RD(tmp2), ctx);
1316 break;
1317 }
1318
1319 /* STX: *(size *)(dst + off) = src */
1320 case BPF_STX | BPF_MEM | BPF_W:
1321 case BPF_STX | BPF_MEM | BPF_H:
1322 case BPF_STX | BPF_MEM | BPF_B:
1323 case BPF_STX | BPF_MEM | BPF_DW: {
1324 const u8 tmp = bpf2sparc[TMP_REG_1];
1325 u32 opcode = 0, rs2;
1326
1327 switch (BPF_SIZE(code)) {
1328 case BPF_W:
1329 opcode = ST32;
1330 break;
1331 case BPF_H:
1332 opcode = ST16;
1333 break;
1334 case BPF_B:
1335 opcode = ST8;
1336 break;
1337 case BPF_DW:
1338 opcode = ST64;
1339 break;
1340 }
1341 if (is_simm13(off)) {
1342 opcode |= IMMED;
1343 rs2 = S13(off);
1344 } else {
1345 ctx->tmp_1_used = true;
1346 emit_loadimm(off, tmp, ctx);
1347 rs2 = RS2(tmp);
1348 }
1349 emit(opcode | RS1(dst) | rs2 | RD(src), ctx);
1350 break;
1351 }
1352
1353 /* STX XADD: lock *(u32 *)(dst + off) += src */
1354 case BPF_STX | BPF_XADD | BPF_W: {
1355 const u8 tmp = bpf2sparc[TMP_REG_1];
1356 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1357 const u8 tmp3 = bpf2sparc[TMP_REG_3];
1358
1359 ctx->tmp_1_used = true;
1360 ctx->tmp_2_used = true;
1361 ctx->tmp_3_used = true;
1362 emit_loadimm(off, tmp, ctx);
1363 emit_alu3(ADD, dst, tmp, tmp, ctx);
1364
1365 emit(LD32 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1366 emit_alu3(ADD, tmp2, src, tmp3, ctx);
1367 emit(CAS | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1368 emit_cmp(tmp2, tmp3, ctx);
1369 emit_branch(BNE, 4, 0, ctx);
1370 emit_nop(ctx);
1371 break;
1372 }
1373 /* STX XADD: lock *(u64 *)(dst + off) += src */
1374 case BPF_STX | BPF_XADD | BPF_DW: {
1375 const u8 tmp = bpf2sparc[TMP_REG_1];
1376 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1377 const u8 tmp3 = bpf2sparc[TMP_REG_3];
1378
1379 ctx->tmp_1_used = true;
1380 ctx->tmp_2_used = true;
1381 ctx->tmp_3_used = true;
1382 emit_loadimm(off, tmp, ctx);
1383 emit_alu3(ADD, dst, tmp, tmp, ctx);
1384
1385 emit(LD64 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1386 emit_alu3(ADD, tmp2, src, tmp3, ctx);
1387 emit(CASX | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1388 emit_cmp(tmp2, tmp3, ctx);
1389 emit_branch(BNE, 4, 0, ctx);
1390 emit_nop(ctx);
1391 break;
1392 }
1393#define CHOOSE_LOAD_FUNC(K, func) \
1394 ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
1395
1396 /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */
1397 case BPF_LD | BPF_ABS | BPF_W:
1398 func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_word);
1399 goto common_load;
1400 case BPF_LD | BPF_ABS | BPF_H:
1401 func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_half);
1402 goto common_load;
1403 case BPF_LD | BPF_ABS | BPF_B:
1404 func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_byte);
1405 goto common_load;
1406 /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */
1407 case BPF_LD | BPF_IND | BPF_W:
1408 func = bpf_jit_load_word;
1409 goto common_load;
1410 case BPF_LD | BPF_IND | BPF_H:
1411 func = bpf_jit_load_half;
1412 goto common_load;
1413
1414 case BPF_LD | BPF_IND | BPF_B:
1415 func = bpf_jit_load_byte;
1416 common_load:
1417 ctx->saw_ld_abs_ind = true;
1418
1419 emit_reg_move(bpf2sparc[BPF_REG_6], O0, ctx);
1420 emit_loadimm(imm, O1, ctx);
1421
1422 if (BPF_MODE(code) == BPF_IND)
1423 emit_alu(ADD, src, O1, ctx);
1424
1425 emit_call(func, ctx);
1426 emit_alu_K(SRA, O1, 0, ctx);
1427
1428 emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
1429 break;
1430
1431 default:
1432 pr_err_once("unknown opcode %02x\n", code);
1433 return -EINVAL;
1434 }
1435
1436 return 0;
1437}
1438
1439static int build_body(struct jit_ctx *ctx)
1440{
1441 const struct bpf_prog *prog = ctx->prog;
1442 int i;
1443
1444 for (i = 0; i < prog->len; i++) {
1445 const struct bpf_insn *insn = &prog->insnsi[i];
1446 int ret;
1447
1448 ret = build_insn(insn, ctx);
1449
1450 if (ret > 0) {
1451 i++;
1452 ctx->offset[i] = ctx->idx;
1453 continue;
1454 }
1455 ctx->offset[i] = ctx->idx;
1456 if (ret)
1457 return ret;
1458 }
1459 return 0;
1460}
1461
1462static void jit_fill_hole(void *area, unsigned int size)
1463{
1464 u32 *ptr;
1465 /* We are guaranteed to have aligned memory. */
1466 for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
1467 *ptr++ = 0x91d02005; /* ta 5 */
1468}
1469
1470struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1471{
1472 struct bpf_prog *tmp, *orig_prog = prog;
1473 struct bpf_binary_header *header;
1474 bool tmp_blinded = false;
1475 struct jit_ctx ctx;
1476 u32 image_size;
1477 u8 *image_ptr;
1478 int pass;
1479
1480 if (!bpf_jit_enable)
1481 return orig_prog;
1482
1483 tmp = bpf_jit_blind_constants(prog);
1484 /* If blinding was requested and we failed during blinding,
1485 * we must fall back to the interpreter.
1486 */
1487 if (IS_ERR(tmp))
1488 return orig_prog;
1489 if (tmp != prog) {
1490 tmp_blinded = true;
1491 prog = tmp;
1492 }
1493
1494 memset(&ctx, 0, sizeof(ctx));
1495 ctx.prog = prog;
1496
1497 ctx.offset = kcalloc(prog->len, sizeof(unsigned int), GFP_KERNEL);
1498 if (ctx.offset == NULL) {
1499 prog = orig_prog;
1500 goto out;
1501 }
1502
1503 /* Fake pass to detect features used, and get an accurate assessment
1504 * of what the final image size will be.
1505 */
1506 if (build_body(&ctx)) {
1507 prog = orig_prog;
1508 goto out_off;
1509 }
1510 build_prologue(&ctx);
1511 build_epilogue(&ctx);
1512
1513 /* Now we know the actual image size. */
1514 image_size = sizeof(u32) * ctx.idx;
1515 header = bpf_jit_binary_alloc(image_size, &image_ptr,
1516 sizeof(u32), jit_fill_hole);
1517 if (header == NULL) {
1518 prog = orig_prog;
1519 goto out_off;
1520 }
1521
1522 ctx.image = (u32 *)image_ptr;
1523
1524 for (pass = 1; pass < 3; pass++) {
1525 ctx.idx = 0;
1526
1527 build_prologue(&ctx);
1528
1529 if (build_body(&ctx)) {
1530 bpf_jit_binary_free(header);
1531 prog = orig_prog;
1532 goto out_off;
1533 }
1534
1535 build_epilogue(&ctx);
1536
1537 if (bpf_jit_enable > 1)
1538 pr_info("Pass %d: shrink = %d, seen = [%c%c%c%c%c%c%c]\n", pass,
1539 image_size - (ctx.idx * 4),
1540 ctx.tmp_1_used ? '1' : ' ',
1541 ctx.tmp_2_used ? '2' : ' ',
1542 ctx.tmp_3_used ? '3' : ' ',
1543 ctx.saw_ld_abs_ind ? 'L' : ' ',
1544 ctx.saw_frame_pointer ? 'F' : ' ',
1545 ctx.saw_call ? 'C' : ' ',
1546 ctx.saw_tail_call ? 'T' : ' ');
1547 }
1548
1549 if (bpf_jit_enable > 1)
1550 bpf_jit_dump(prog->len, image_size, pass, ctx.image);
1551
1552 bpf_flush_icache(header, (u8 *)header + (header->pages * PAGE_SIZE));
1553
1554 bpf_jit_binary_lock_ro(header);
1555
1556 prog->bpf_func = (void *)ctx.image;
1557 prog->jited = 1;
1558
1559out_off:
1560 kfree(ctx.offset);
1561out:
1562 if (tmp_blinded)
1563 bpf_jit_prog_release_other(prog, prog == orig_prog ?
1564 tmp : orig_prog);
1565 return prog;
1566}
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-26 12:07:20 -0400