aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/riscv/Kconfig12
-rw-r--r--arch/riscv/Makefile8
-rw-r--r--arch/riscv/configs/defconfig2
-rw-r--r--arch/riscv/include/asm/atomic.h417
-rw-r--r--arch/riscv/include/asm/barrier.h15
-rw-r--r--arch/riscv/include/asm/cmpxchg.h391
-rw-r--r--arch/riscv/include/asm/fence.h12
-rw-r--r--arch/riscv/include/asm/ftrace.h56
-rw-r--r--arch/riscv/include/asm/module.h113
-rw-r--r--arch/riscv/include/asm/spinlock.h29
-rw-r--r--arch/riscv/include/uapi/asm/elf.h7
-rw-r--r--arch/riscv/kernel/Makefile6
-rw-r--r--arch/riscv/kernel/ftrace.c175
-rw-r--r--arch/riscv/kernel/mcount-dyn.S239
-rw-r--r--arch/riscv/kernel/mcount.S22
-rw-r--r--arch/riscv/kernel/module-sections.c156
-rw-r--r--arch/riscv/kernel/module.c179
-rw-r--r--arch/riscv/kernel/module.lds8
-rw-r--r--arch/riscv/kernel/stacktrace.c6
-rwxr-xr-xscripts/recordmcount.pl5
20 files changed, 1603 insertions, 255 deletions
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 148865de1692..23d8acca5c90 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -115,6 +115,9 @@ config ARCH_RV64I
115 select 64BIT 115 select 64BIT
116 select HAVE_FUNCTION_TRACER 116 select HAVE_FUNCTION_TRACER
117 select HAVE_FUNCTION_GRAPH_TRACER 117 select HAVE_FUNCTION_GRAPH_TRACER
118 select HAVE_FTRACE_MCOUNT_RECORD
119 select HAVE_DYNAMIC_FTRACE
120 select HAVE_DYNAMIC_FTRACE_WITH_REGS
118 121
119endchoice 122endchoice
120 123
@@ -132,6 +135,10 @@ choice
132 bool "medium any code model" 135 bool "medium any code model"
133endchoice 136endchoice
134 137
138config MODULE_SECTIONS
139 bool
140 select HAVE_MOD_ARCH_SPECIFIC
141
135choice 142choice
136 prompt "Maximum Physical Memory" 143 prompt "Maximum Physical Memory"
137 default MAXPHYSMEM_2GB if 32BIT 144 default MAXPHYSMEM_2GB if 32BIT
@@ -142,6 +149,7 @@ choice
142 bool "2GiB" 149 bool "2GiB"
143 config MAXPHYSMEM_128GB 150 config MAXPHYSMEM_128GB
144 depends on 64BIT && CMODEL_MEDANY 151 depends on 64BIT && CMODEL_MEDANY
152 select MODULE_SECTIONS if MODULES
145 bool "128GiB" 153 bool "128GiB"
146endchoice 154endchoice
147 155
@@ -282,7 +290,7 @@ config CMDLINE_BOOL
282 in CONFIG_CMDLINE. 290 in CONFIG_CMDLINE.
283 291
284 The built-in options will be concatenated to the default command 292 The built-in options will be concatenated to the default command
285 line if CMDLINE_OVERRIDE is set to 'N'. Otherwise, the default 293 line if CMDLINE_FORCE is set to 'N'. Otherwise, the default
286 command line will be ignored and replaced by the built-in string. 294 command line will be ignored and replaced by the built-in string.
287 295
288config CMDLINE 296config CMDLINE
@@ -292,7 +300,7 @@ config CMDLINE
292 help 300 help
293 Supply command-line options at build time by entering them here. 301 Supply command-line options at build time by entering them here.
294 302
295config CMDLINE_OVERRIDE 303config CMDLINE_FORCE
296 bool "Built-in command line overrides bootloader arguments" 304 bool "Built-in command line overrides bootloader arguments"
297 depends on CMDLINE_BOOL 305 depends on CMDLINE_BOOL
298 help 306 help
diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile
index 6719dd30ec5b..76e958a5414a 100644
--- a/arch/riscv/Makefile
+++ b/arch/riscv/Makefile
@@ -11,6 +11,9 @@
11LDFLAGS := 11LDFLAGS :=
12OBJCOPYFLAGS := -O binary 12OBJCOPYFLAGS := -O binary
13LDFLAGS_vmlinux := 13LDFLAGS_vmlinux :=
14ifeq ($(CONFIG_DYNAMIC_FTRACE),y)
15 LDFLAGS_vmlinux := --no-relax
16endif
14KBUILD_AFLAGS_MODULE += -fPIC 17KBUILD_AFLAGS_MODULE += -fPIC
15KBUILD_CFLAGS_MODULE += -fPIC 18KBUILD_CFLAGS_MODULE += -fPIC
16 19
@@ -56,6 +59,11 @@ endif
56ifeq ($(CONFIG_CMODEL_MEDANY),y) 59ifeq ($(CONFIG_CMODEL_MEDANY),y)
57 KBUILD_CFLAGS += -mcmodel=medany 60 KBUILD_CFLAGS += -mcmodel=medany
58endif 61endif
62ifeq ($(CONFIG_MODULE_SECTIONS),y)
63 KBUILD_LDFLAGS_MODULE += -T $(srctree)/arch/riscv/kernel/module.lds
64endif
65
66KBUILD_CFLAGS_MODULE += $(call cc-option,-mno-relax)
59 67
60# GCC versions that support the "-mstrict-align" option default to allowing 68# GCC versions that support the "-mstrict-align" option default to allowing
61# unaligned accesses. While unaligned accesses are explicitly allowed in the 69# unaligned accesses. While unaligned accesses are explicitly allowed in the
diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig
index 47dacf06c679..bca0eee733b0 100644
--- a/arch/riscv/configs/defconfig
+++ b/arch/riscv/configs/defconfig
@@ -73,3 +73,5 @@ CONFIG_NFS_V4_2=y
73CONFIG_ROOT_NFS=y 73CONFIG_ROOT_NFS=y
74# CONFIG_RCU_TRACE is not set 74# CONFIG_RCU_TRACE is not set
75CONFIG_CRYPTO_USER_API_HASH=y 75CONFIG_CRYPTO_USER_API_HASH=y
76CONFIG_MODULES=y
77CONFIG_MODULE_UNLOAD=y
diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h
index e65d1cd89e28..855115ace98c 100644
--- a/arch/riscv/include/asm/atomic.h
+++ b/arch/riscv/include/asm/atomic.h
@@ -24,6 +24,20 @@
24#include <asm/barrier.h> 24#include <asm/barrier.h>
25 25
26#define ATOMIC_INIT(i) { (i) } 26#define ATOMIC_INIT(i) { (i) }
27
28#define __atomic_op_acquire(op, args...) \
29({ \
30 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \
31 __asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory"); \
32 __ret; \
33})
34
35#define __atomic_op_release(op, args...) \
36({ \
37 __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory"); \
38 op##_relaxed(args); \
39})
40
27static __always_inline int atomic_read(const atomic_t *v) 41static __always_inline int atomic_read(const atomic_t *v)
28{ 42{
29 return READ_ONCE(v->counter); 43 return READ_ONCE(v->counter);
@@ -50,22 +64,23 @@ static __always_inline void atomic64_set(atomic64_t *v, long i)
50 * have the AQ or RL bits set. These don't return anything, so there's only 64 * have the AQ or RL bits set. These don't return anything, so there's only
51 * one version to worry about. 65 * one version to worry about.
52 */ 66 */
53#define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \ 67#define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \
54static __always_inline void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ 68static __always_inline \
55{ \ 69void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
56 __asm__ __volatile__ ( \ 70{ \
57 "amo" #asm_op "." #asm_type " zero, %1, %0" \ 71 __asm__ __volatile__ ( \
58 : "+A" (v->counter) \ 72 " amo" #asm_op "." #asm_type " zero, %1, %0" \
59 : "r" (I) \ 73 : "+A" (v->counter) \
60 : "memory"); \ 74 : "r" (I) \
61} 75 : "memory"); \
76} \
62 77
63#ifdef CONFIG_GENERIC_ATOMIC64 78#ifdef CONFIG_GENERIC_ATOMIC64
64#define ATOMIC_OPS(op, asm_op, I) \ 79#define ATOMIC_OPS(op, asm_op, I) \
65 ATOMIC_OP (op, asm_op, I, w, int, ) 80 ATOMIC_OP (op, asm_op, I, w, int, )
66#else 81#else
67#define ATOMIC_OPS(op, asm_op, I) \ 82#define ATOMIC_OPS(op, asm_op, I) \
68 ATOMIC_OP (op, asm_op, I, w, int, ) \ 83 ATOMIC_OP (op, asm_op, I, w, int, ) \
69 ATOMIC_OP (op, asm_op, I, d, long, 64) 84 ATOMIC_OP (op, asm_op, I, d, long, 64)
70#endif 85#endif
71 86
@@ -79,75 +94,115 @@ ATOMIC_OPS(xor, xor, i)
79#undef ATOMIC_OPS 94#undef ATOMIC_OPS
80 95
81/* 96/*
82 * Atomic ops that have ordered, relaxed, acquire, and relese variants. 97 * Atomic ops that have ordered, relaxed, acquire, and release variants.
83 * There's two flavors of these: the arithmatic ops have both fetch and return 98 * There's two flavors of these: the arithmatic ops have both fetch and return
84 * versions, while the logical ops only have fetch versions. 99 * versions, while the logical ops only have fetch versions.
85 */ 100 */
86#define ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, asm_type, c_type, prefix) \ 101#define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \
87static __always_inline c_type atomic##prefix##_fetch_##op##c_or(c_type i, atomic##prefix##_t *v) \ 102static __always_inline \
88{ \ 103c_type atomic##prefix##_fetch_##op##_relaxed(c_type i, \
89 register c_type ret; \ 104 atomic##prefix##_t *v) \
90 __asm__ __volatile__ ( \ 105{ \
91 "amo" #asm_op "." #asm_type #asm_or " %1, %2, %0" \ 106 register c_type ret; \
92 : "+A" (v->counter), "=r" (ret) \ 107 __asm__ __volatile__ ( \
93 : "r" (I) \ 108 " amo" #asm_op "." #asm_type " %1, %2, %0" \
94 : "memory"); \ 109 : "+A" (v->counter), "=r" (ret) \
95 return ret; \ 110 : "r" (I) \
111 : "memory"); \
112 return ret; \
113} \
114static __always_inline \
115c_type atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \
116{ \
117 register c_type ret; \
118 __asm__ __volatile__ ( \
119 " amo" #asm_op "." #asm_type ".aqrl %1, %2, %0" \
120 : "+A" (v->counter), "=r" (ret) \
121 : "r" (I) \
122 : "memory"); \
123 return ret; \
96} 124}
97 125
98#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, asm_type, c_type, prefix) \ 126#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix) \
99static __always_inline c_type atomic##prefix##_##op##_return##c_or(c_type i, atomic##prefix##_t *v) \ 127static __always_inline \
100{ \ 128c_type atomic##prefix##_##op##_return_relaxed(c_type i, \
101 return atomic##prefix##_fetch_##op##c_or(i, v) c_op I; \ 129 atomic##prefix##_t *v) \
130{ \
131 return atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \
132} \
133static __always_inline \
134c_type atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \
135{ \
136 return atomic##prefix##_fetch_##op(i, v) c_op I; \
102} 137}
103 138
104#ifdef CONFIG_GENERIC_ATOMIC64 139#ifdef CONFIG_GENERIC_ATOMIC64
105#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or) \ 140#define ATOMIC_OPS(op, asm_op, c_op, I) \
106 ATOMIC_FETCH_OP (op, asm_op, I, asm_or, c_or, w, int, ) \ 141 ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \
107 ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, w, int, ) 142 ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, )
108#else 143#else
109#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or) \ 144#define ATOMIC_OPS(op, asm_op, c_op, I) \
110 ATOMIC_FETCH_OP (op, asm_op, I, asm_or, c_or, w, int, ) \ 145 ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \
111 ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, w, int, ) \ 146 ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, ) \
112 ATOMIC_FETCH_OP (op, asm_op, I, asm_or, c_or, d, long, 64) \ 147 ATOMIC_FETCH_OP( op, asm_op, I, d, long, 64) \
113 ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, d, long, 64) 148 ATOMIC_OP_RETURN(op, asm_op, c_op, I, d, long, 64)
114#endif 149#endif
115 150
116ATOMIC_OPS(add, add, +, i, , _relaxed) 151ATOMIC_OPS(add, add, +, i)
117ATOMIC_OPS(add, add, +, i, .aq , _acquire) 152ATOMIC_OPS(sub, add, +, -i)
118ATOMIC_OPS(add, add, +, i, .rl , _release) 153
119ATOMIC_OPS(add, add, +, i, .aqrl, ) 154#define atomic_add_return_relaxed atomic_add_return_relaxed
155#define atomic_sub_return_relaxed atomic_sub_return_relaxed
156#define atomic_add_return atomic_add_return
157#define atomic_sub_return atomic_sub_return
120 158
121ATOMIC_OPS(sub, add, +, -i, , _relaxed) 159#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed
122ATOMIC_OPS(sub, add, +, -i, .aq , _acquire) 160#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed
123ATOMIC_OPS(sub, add, +, -i, .rl , _release) 161#define atomic_fetch_add atomic_fetch_add
124ATOMIC_OPS(sub, add, +, -i, .aqrl, ) 162#define atomic_fetch_sub atomic_fetch_sub
163
164#ifndef CONFIG_GENERIC_ATOMIC64
165#define atomic64_add_return_relaxed atomic64_add_return_relaxed
166#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
167#define atomic64_add_return atomic64_add_return
168#define atomic64_sub_return atomic64_sub_return
169
170#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed
171#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed
172#define atomic64_fetch_add atomic64_fetch_add
173#define atomic64_fetch_sub atomic64_fetch_sub
174#endif
125 175
126#undef ATOMIC_OPS 176#undef ATOMIC_OPS
127 177
128#ifdef CONFIG_GENERIC_ATOMIC64 178#ifdef CONFIG_GENERIC_ATOMIC64
129#define ATOMIC_OPS(op, asm_op, I, asm_or, c_or) \ 179#define ATOMIC_OPS(op, asm_op, I) \
130 ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, w, int, ) 180 ATOMIC_FETCH_OP(op, asm_op, I, w, int, )
131#else 181#else
132#define ATOMIC_OPS(op, asm_op, I, asm_or, c_or) \ 182#define ATOMIC_OPS(op, asm_op, I) \
133 ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, w, int, ) \ 183 ATOMIC_FETCH_OP(op, asm_op, I, w, int, ) \
134 ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, d, long, 64) 184 ATOMIC_FETCH_OP(op, asm_op, I, d, long, 64)
135#endif 185#endif
136 186
137ATOMIC_OPS(and, and, i, , _relaxed) 187ATOMIC_OPS(and, and, i)
138ATOMIC_OPS(and, and, i, .aq , _acquire) 188ATOMIC_OPS( or, or, i)
139ATOMIC_OPS(and, and, i, .rl , _release) 189ATOMIC_OPS(xor, xor, i)
140ATOMIC_OPS(and, and, i, .aqrl, )
141 190
142ATOMIC_OPS( or, or, i, , _relaxed) 191#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed
143ATOMIC_OPS( or, or, i, .aq , _acquire) 192#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed
144ATOMIC_OPS( or, or, i, .rl , _release) 193#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed
145ATOMIC_OPS( or, or, i, .aqrl, ) 194#define atomic_fetch_and atomic_fetch_and
195#define atomic_fetch_or atomic_fetch_or
196#define atomic_fetch_xor atomic_fetch_xor
146 197
147ATOMIC_OPS(xor, xor, i, , _relaxed) 198#ifndef CONFIG_GENERIC_ATOMIC64
148ATOMIC_OPS(xor, xor, i, .aq , _acquire) 199#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed
149ATOMIC_OPS(xor, xor, i, .rl , _release) 200#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed
150ATOMIC_OPS(xor, xor, i, .aqrl, ) 201#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed
202#define atomic64_fetch_and atomic64_fetch_and
203#define atomic64_fetch_or atomic64_fetch_or
204#define atomic64_fetch_xor atomic64_fetch_xor
205#endif
151 206
152#undef ATOMIC_OPS 207#undef ATOMIC_OPS
153 208
@@ -157,22 +212,24 @@ ATOMIC_OPS(xor, xor, i, .aqrl, )
157/* 212/*
158 * The extra atomic operations that are constructed from one of the core 213 * The extra atomic operations that are constructed from one of the core
159 * AMO-based operations above (aside from sub, which is easier to fit above). 214 * AMO-based operations above (aside from sub, which is easier to fit above).
160 * These are required to perform a barrier, but they're OK this way because 215 * These are required to perform a full barrier, but they're OK this way
161 * atomic_*_return is also required to perform a barrier. 216 * because atomic_*_return is also required to perform a full barrier.
217 *
162 */ 218 */
163#define ATOMIC_OP(op, func_op, comp_op, I, c_type, prefix) \ 219#define ATOMIC_OP(op, func_op, comp_op, I, c_type, prefix) \
164static __always_inline bool atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ 220static __always_inline \
165{ \ 221bool atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
166 return atomic##prefix##_##func_op##_return(i, v) comp_op I; \ 222{ \
223 return atomic##prefix##_##func_op##_return(i, v) comp_op I; \
167} 224}
168 225
169#ifdef CONFIG_GENERIC_ATOMIC64 226#ifdef CONFIG_GENERIC_ATOMIC64
170#define ATOMIC_OPS(op, func_op, comp_op, I) \ 227#define ATOMIC_OPS(op, func_op, comp_op, I) \
171 ATOMIC_OP (op, func_op, comp_op, I, int, ) 228 ATOMIC_OP(op, func_op, comp_op, I, int, )
172#else 229#else
173#define ATOMIC_OPS(op, func_op, comp_op, I) \ 230#define ATOMIC_OPS(op, func_op, comp_op, I) \
174 ATOMIC_OP (op, func_op, comp_op, I, int, ) \ 231 ATOMIC_OP(op, func_op, comp_op, I, int, ) \
175 ATOMIC_OP (op, func_op, comp_op, I, long, 64) 232 ATOMIC_OP(op, func_op, comp_op, I, long, 64)
176#endif 233#endif
177 234
178ATOMIC_OPS(add_and_test, add, ==, 0) 235ATOMIC_OPS(add_and_test, add, ==, 0)
@@ -182,51 +239,87 @@ ATOMIC_OPS(add_negative, add, <, 0)
182#undef ATOMIC_OP 239#undef ATOMIC_OP
183#undef ATOMIC_OPS 240#undef ATOMIC_OPS
184 241
185#define ATOMIC_OP(op, func_op, I, c_type, prefix) \ 242#define ATOMIC_OP(op, func_op, I, c_type, prefix) \
186static __always_inline void atomic##prefix##_##op(atomic##prefix##_t *v) \ 243static __always_inline \
187{ \ 244void atomic##prefix##_##op(atomic##prefix##_t *v) \
188 atomic##prefix##_##func_op(I, v); \ 245{ \
246 atomic##prefix##_##func_op(I, v); \
189} 247}
190 248
191#define ATOMIC_FETCH_OP(op, func_op, I, c_type, prefix) \ 249#define ATOMIC_FETCH_OP(op, func_op, I, c_type, prefix) \
192static __always_inline c_type atomic##prefix##_fetch_##op(atomic##prefix##_t *v) \ 250static __always_inline \
193{ \ 251c_type atomic##prefix##_fetch_##op##_relaxed(atomic##prefix##_t *v) \
194 return atomic##prefix##_fetch_##func_op(I, v); \ 252{ \
253 return atomic##prefix##_fetch_##func_op##_relaxed(I, v); \
254} \
255static __always_inline \
256c_type atomic##prefix##_fetch_##op(atomic##prefix##_t *v) \
257{ \
258 return atomic##prefix##_fetch_##func_op(I, v); \
195} 259}
196 260
197#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, c_type, prefix) \ 261#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, c_type, prefix) \
198static __always_inline c_type atomic##prefix##_##op##_return(atomic##prefix##_t *v) \ 262static __always_inline \
199{ \ 263c_type atomic##prefix##_##op##_return_relaxed(atomic##prefix##_t *v) \
200 return atomic##prefix##_fetch_##op(v) c_op I; \ 264{ \
265 return atomic##prefix##_fetch_##op##_relaxed(v) c_op I; \
266} \
267static __always_inline \
268c_type atomic##prefix##_##op##_return(atomic##prefix##_t *v) \
269{ \
270 return atomic##prefix##_fetch_##op(v) c_op I; \
201} 271}
202 272
203#ifdef CONFIG_GENERIC_ATOMIC64 273#ifdef CONFIG_GENERIC_ATOMIC64
204#define ATOMIC_OPS(op, asm_op, c_op, I) \ 274#define ATOMIC_OPS(op, asm_op, c_op, I) \
205 ATOMIC_OP (op, asm_op, I, int, ) \ 275 ATOMIC_OP( op, asm_op, I, int, ) \
206 ATOMIC_FETCH_OP (op, asm_op, I, int, ) \ 276 ATOMIC_FETCH_OP( op, asm_op, I, int, ) \
207 ATOMIC_OP_RETURN(op, asm_op, c_op, I, int, ) 277 ATOMIC_OP_RETURN(op, asm_op, c_op, I, int, )
208#else 278#else
209#define ATOMIC_OPS(op, asm_op, c_op, I) \ 279#define ATOMIC_OPS(op, asm_op, c_op, I) \
210 ATOMIC_OP (op, asm_op, I, int, ) \ 280 ATOMIC_OP( op, asm_op, I, int, ) \
211 ATOMIC_FETCH_OP (op, asm_op, I, int, ) \ 281 ATOMIC_FETCH_OP( op, asm_op, I, int, ) \
212 ATOMIC_OP_RETURN(op, asm_op, c_op, I, int, ) \ 282 ATOMIC_OP_RETURN(op, asm_op, c_op, I, int, ) \
213 ATOMIC_OP (op, asm_op, I, long, 64) \ 283 ATOMIC_OP( op, asm_op, I, long, 64) \
214 ATOMIC_FETCH_OP (op, asm_op, I, long, 64) \ 284 ATOMIC_FETCH_OP( op, asm_op, I, long, 64) \
215 ATOMIC_OP_RETURN(op, asm_op, c_op, I, long, 64) 285 ATOMIC_OP_RETURN(op, asm_op, c_op, I, long, 64)
216#endif 286#endif
217 287
218ATOMIC_OPS(inc, add, +, 1) 288ATOMIC_OPS(inc, add, +, 1)
219ATOMIC_OPS(dec, add, +, -1) 289ATOMIC_OPS(dec, add, +, -1)
220 290
291#define atomic_inc_return_relaxed atomic_inc_return_relaxed
292#define atomic_dec_return_relaxed atomic_dec_return_relaxed
293#define atomic_inc_return atomic_inc_return
294#define atomic_dec_return atomic_dec_return
295
296#define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed
297#define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed
298#define atomic_fetch_inc atomic_fetch_inc
299#define atomic_fetch_dec atomic_fetch_dec
300
301#ifndef CONFIG_GENERIC_ATOMIC64
302#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed
303#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed
304#define atomic64_inc_return atomic64_inc_return
305#define atomic64_dec_return atomic64_dec_return
306
307#define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed
308#define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed
309#define atomic64_fetch_inc atomic64_fetch_inc
310#define atomic64_fetch_dec atomic64_fetch_dec
311#endif
312
221#undef ATOMIC_OPS 313#undef ATOMIC_OPS
222#undef ATOMIC_OP 314#undef ATOMIC_OP
223#undef ATOMIC_FETCH_OP 315#undef ATOMIC_FETCH_OP
224#undef ATOMIC_OP_RETURN 316#undef ATOMIC_OP_RETURN
225 317
226#define ATOMIC_OP(op, func_op, comp_op, I, prefix) \ 318#define ATOMIC_OP(op, func_op, comp_op, I, prefix) \
227static __always_inline bool atomic##prefix##_##op(atomic##prefix##_t *v) \ 319static __always_inline \
228{ \ 320bool atomic##prefix##_##op(atomic##prefix##_t *v) \
229 return atomic##prefix##_##func_op##_return(v) comp_op I; \ 321{ \
322 return atomic##prefix##_##func_op##_return(v) comp_op I; \
230} 323}
231 324
232ATOMIC_OP(inc_and_test, inc, ==, 0, ) 325ATOMIC_OP(inc_and_test, inc, ==, 0, )
@@ -238,19 +331,19 @@ ATOMIC_OP(dec_and_test, dec, ==, 0, 64)
238 331
239#undef ATOMIC_OP 332#undef ATOMIC_OP
240 333
241/* This is required to provide a barrier on success. */ 334/* This is required to provide a full barrier on success. */
242static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u) 335static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u)
243{ 336{
244 int prev, rc; 337 int prev, rc;
245 338
246 __asm__ __volatile__ ( 339 __asm__ __volatile__ (
247 "0:\n\t" 340 "0: lr.w %[p], %[c]\n"
248 "lr.w.aqrl %[p], %[c]\n\t" 341 " beq %[p], %[u], 1f\n"
249 "beq %[p], %[u], 1f\n\t" 342 " add %[rc], %[p], %[a]\n"
250 "add %[rc], %[p], %[a]\n\t" 343 " sc.w.rl %[rc], %[rc], %[c]\n"
251 "sc.w.aqrl %[rc], %[rc], %[c]\n\t" 344 " bnez %[rc], 0b\n"
252 "bnez %[rc], 0b\n\t" 345 " fence rw, rw\n"
253 "1:" 346 "1:\n"
254 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 347 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
255 : [a]"r" (a), [u]"r" (u) 348 : [a]"r" (a), [u]"r" (u)
256 : "memory"); 349 : "memory");
@@ -263,13 +356,13 @@ static __always_inline long __atomic64_add_unless(atomic64_t *v, long a, long u)
263 long prev, rc; 356 long prev, rc;
264 357
265 __asm__ __volatile__ ( 358 __asm__ __volatile__ (
266 "0:\n\t" 359 "0: lr.d %[p], %[c]\n"
267 "lr.d.aqrl %[p], %[c]\n\t" 360 " beq %[p], %[u], 1f\n"
268 "beq %[p], %[u], 1f\n\t" 361 " add %[rc], %[p], %[a]\n"
269 "add %[rc], %[p], %[a]\n\t" 362 " sc.d.rl %[rc], %[rc], %[c]\n"
270 "sc.d.aqrl %[rc], %[rc], %[c]\n\t" 363 " bnez %[rc], 0b\n"
271 "bnez %[rc], 0b\n\t" 364 " fence rw, rw\n"
272 "1:" 365 "1:\n"
273 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 366 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
274 : [a]"r" (a), [u]"r" (u) 367 : [a]"r" (a), [u]"r" (u)
275 : "memory"); 368 : "memory");
@@ -300,37 +393,63 @@ static __always_inline long atomic64_inc_not_zero(atomic64_t *v)
300 393
301/* 394/*
302 * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as 395 * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as
303 * {cmp,}xchg and the operations that return, so they need a barrier. 396 * {cmp,}xchg and the operations that return, so they need a full barrier.
304 */
305/*
306 * FIXME: atomic_cmpxchg_{acquire,release,relaxed} are all implemented by
307 * assigning the same barrier to both the LR and SC operations, but that might
308 * not make any sense. We're waiting on a memory model specification to
309 * determine exactly what the right thing to do is here.
310 */ 397 */
311#define ATOMIC_OP(c_t, prefix, c_or, size, asm_or) \ 398#define ATOMIC_OP(c_t, prefix, size) \
312static __always_inline c_t atomic##prefix##_cmpxchg##c_or(atomic##prefix##_t *v, c_t o, c_t n) \ 399static __always_inline \
313{ \ 400c_t atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n) \
314 return __cmpxchg(&(v->counter), o, n, size, asm_or, asm_or); \ 401{ \
315} \ 402 return __xchg_relaxed(&(v->counter), n, size); \
316static __always_inline c_t atomic##prefix##_xchg##c_or(atomic##prefix##_t *v, c_t n) \ 403} \
317{ \ 404static __always_inline \
318 return __xchg(n, &(v->counter), size, asm_or); \ 405c_t atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n) \
406{ \
407 return __xchg_acquire(&(v->counter), n, size); \
408} \
409static __always_inline \
410c_t atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n) \
411{ \
412 return __xchg_release(&(v->counter), n, size); \
413} \
414static __always_inline \
415c_t atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n) \
416{ \
417 return __xchg(&(v->counter), n, size); \
418} \
419static __always_inline \
420c_t atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v, \
421 c_t o, c_t n) \
422{ \
423 return __cmpxchg_relaxed(&(v->counter), o, n, size); \
424} \
425static __always_inline \
426c_t atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v, \
427 c_t o, c_t n) \
428{ \
429 return __cmpxchg_acquire(&(v->counter), o, n, size); \
430} \
431static __always_inline \
432c_t atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v, \
433 c_t o, c_t n) \
434{ \
435 return __cmpxchg_release(&(v->counter), o, n, size); \
436} \
437static __always_inline \
438c_t atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n) \
439{ \
440 return __cmpxchg(&(v->counter), o, n, size); \
319} 441}
320 442
321#ifdef CONFIG_GENERIC_ATOMIC64 443#ifdef CONFIG_GENERIC_ATOMIC64
322#define ATOMIC_OPS(c_or, asm_or) \ 444#define ATOMIC_OPS() \
323 ATOMIC_OP( int, , c_or, 4, asm_or) 445 ATOMIC_OP( int, , 4)
324#else 446#else
325#define ATOMIC_OPS(c_or, asm_or) \ 447#define ATOMIC_OPS() \
326 ATOMIC_OP( int, , c_or, 4, asm_or) \ 448 ATOMIC_OP( int, , 4) \
327 ATOMIC_OP(long, 64, c_or, 8, asm_or) 449 ATOMIC_OP(long, 64, 8)
328#endif 450#endif
329 451
330ATOMIC_OPS( , .aqrl) 452ATOMIC_OPS()
331ATOMIC_OPS(_acquire, .aq)
332ATOMIC_OPS(_release, .rl)
333ATOMIC_OPS(_relaxed, )
334 453
335#undef ATOMIC_OPS 454#undef ATOMIC_OPS
336#undef ATOMIC_OP 455#undef ATOMIC_OP
@@ -340,13 +459,13 @@ static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset)
340 int prev, rc; 459 int prev, rc;
341 460
342 __asm__ __volatile__ ( 461 __asm__ __volatile__ (
343 "0:\n\t" 462 "0: lr.w %[p], %[c]\n"
344 "lr.w.aqrl %[p], %[c]\n\t" 463 " sub %[rc], %[p], %[o]\n"
345 "sub %[rc], %[p], %[o]\n\t" 464 " bltz %[rc], 1f\n"
346 "bltz %[rc], 1f\n\t" 465 " sc.w.rl %[rc], %[rc], %[c]\n"
347 "sc.w.aqrl %[rc], %[rc], %[c]\n\t" 466 " bnez %[rc], 0b\n"
348 "bnez %[rc], 0b\n\t" 467 " fence rw, rw\n"
349 "1:" 468 "1:\n"
350 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 469 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
351 : [o]"r" (offset) 470 : [o]"r" (offset)
352 : "memory"); 471 : "memory");
@@ -361,13 +480,13 @@ static __always_inline long atomic64_sub_if_positive(atomic64_t *v, int offset)
361 long prev, rc; 480 long prev, rc;
362 481
363 __asm__ __volatile__ ( 482 __asm__ __volatile__ (
364 "0:\n\t" 483 "0: lr.d %[p], %[c]\n"
365 "lr.d.aqrl %[p], %[c]\n\t" 484 " sub %[rc], %[p], %[o]\n"
366 "sub %[rc], %[p], %[o]\n\t" 485 " bltz %[rc], 1f\n"
367 "bltz %[rc], 1f\n\t" 486 " sc.d.rl %[rc], %[rc], %[c]\n"
368 "sc.d.aqrl %[rc], %[rc], %[c]\n\t" 487 " bnez %[rc], 0b\n"
369 "bnez %[rc], 0b\n\t" 488 " fence rw, rw\n"
370 "1:" 489 "1:\n"
371 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 490 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
372 : [o]"r" (offset) 491 : [o]"r" (offset)
373 : "memory"); 492 : "memory");
diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h
index 5510366d169a..d4628e4b3a5e 100644
--- a/arch/riscv/include/asm/barrier.h
+++ b/arch/riscv/include/asm/barrier.h
@@ -38,6 +38,21 @@
38#define __smp_rmb() RISCV_FENCE(r,r) 38#define __smp_rmb() RISCV_FENCE(r,r)
39#define __smp_wmb() RISCV_FENCE(w,w) 39#define __smp_wmb() RISCV_FENCE(w,w)
40 40
41#define __smp_store_release(p, v) \
42do { \
43 compiletime_assert_atomic_type(*p); \
44 RISCV_FENCE(rw,w); \
45 WRITE_ONCE(*p, v); \
46} while (0)
47
48#define __smp_load_acquire(p) \
49({ \
50 typeof(*p) ___p1 = READ_ONCE(*p); \
51 compiletime_assert_atomic_type(*p); \
52 RISCV_FENCE(r,rw); \
53 ___p1; \
54})
55
41/* 56/*
42 * This is a very specific barrier: it's currently only used in two places in 57 * This is a very specific barrier: it's currently only used in two places in
43 * the kernel, both in the scheduler. See include/linux/spinlock.h for the two 58 * the kernel, both in the scheduler. See include/linux/spinlock.h for the two
diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h
index db249dbc7b97..c12833f7b6bd 100644
--- a/arch/riscv/include/asm/cmpxchg.h
+++ b/arch/riscv/include/asm/cmpxchg.h
@@ -17,45 +17,153 @@
17#include <linux/bug.h> 17#include <linux/bug.h>
18 18
19#include <asm/barrier.h> 19#include <asm/barrier.h>
20#include <asm/fence.h>
20 21
21#define __xchg(new, ptr, size, asm_or) \ 22#define __xchg_relaxed(ptr, new, size) \
22({ \ 23({ \
23 __typeof__(ptr) __ptr = (ptr); \ 24 __typeof__(ptr) __ptr = (ptr); \
24 __typeof__(new) __new = (new); \ 25 __typeof__(new) __new = (new); \
25 __typeof__(*(ptr)) __ret; \ 26 __typeof__(*(ptr)) __ret; \
26 switch (size) { \ 27 switch (size) { \
27 case 4: \ 28 case 4: \
28 __asm__ __volatile__ ( \ 29 __asm__ __volatile__ ( \
29 "amoswap.w" #asm_or " %0, %2, %1" \ 30 " amoswap.w %0, %2, %1\n" \
30 : "=r" (__ret), "+A" (*__ptr) \ 31 : "=r" (__ret), "+A" (*__ptr) \
31 : "r" (__new) \ 32 : "r" (__new) \
32 : "memory"); \ 33 : "memory"); \
33 break; \ 34 break; \
34 case 8: \ 35 case 8: \
35 __asm__ __volatile__ ( \ 36 __asm__ __volatile__ ( \
36 "amoswap.d" #asm_or " %0, %2, %1" \ 37 " amoswap.d %0, %2, %1\n" \
37 : "=r" (__ret), "+A" (*__ptr) \ 38 : "=r" (__ret), "+A" (*__ptr) \
38 : "r" (__new) \ 39 : "r" (__new) \
39 : "memory"); \ 40 : "memory"); \
40 break; \ 41 break; \
41 default: \ 42 default: \
42 BUILD_BUG(); \ 43 BUILD_BUG(); \
43 } \ 44 } \
44 __ret; \ 45 __ret; \
45}) 46})
46 47
47#define xchg(ptr, x) (__xchg((x), (ptr), sizeof(*(ptr)), .aqrl)) 48#define xchg_relaxed(ptr, x) \
48 49({ \
49#define xchg32(ptr, x) \ 50 __typeof__(*(ptr)) _x_ = (x); \
50({ \ 51 (__typeof__(*(ptr))) __xchg_relaxed((ptr), \
51 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ 52 _x_, sizeof(*(ptr))); \
52 xchg((ptr), (x)); \ 53})
53}) 54
54 55#define __xchg_acquire(ptr, new, size) \
55#define xchg64(ptr, x) \ 56({ \
56({ \ 57 __typeof__(ptr) __ptr = (ptr); \
57 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 58 __typeof__(new) __new = (new); \
58 xchg((ptr), (x)); \ 59 __typeof__(*(ptr)) __ret; \
60 switch (size) { \
61 case 4: \
62 __asm__ __volatile__ ( \
63 " amoswap.w %0, %2, %1\n" \
64 RISCV_ACQUIRE_BARRIER \
65 : "=r" (__ret), "+A" (*__ptr) \
66 : "r" (__new) \
67 : "memory"); \
68 break; \
69 case 8: \
70 __asm__ __volatile__ ( \
71 " amoswap.d %0, %2, %1\n" \
72 RISCV_ACQUIRE_BARRIER \
73 : "=r" (__ret), "+A" (*__ptr) \
74 : "r" (__new) \
75 : "memory"); \
76 break; \
77 default: \
78 BUILD_BUG(); \
79 } \
80 __ret; \
81})
82
83#define xchg_acquire(ptr, x) \
84({ \
85 __typeof__(*(ptr)) _x_ = (x); \
86 (__typeof__(*(ptr))) __xchg_acquire((ptr), \
87 _x_, sizeof(*(ptr))); \
88})
89
90#define __xchg_release(ptr, new, size) \
91({ \
92 __typeof__(ptr) __ptr = (ptr); \
93 __typeof__(new) __new = (new); \
94 __typeof__(*(ptr)) __ret; \
95 switch (size) { \
96 case 4: \
97 __asm__ __volatile__ ( \
98 RISCV_RELEASE_BARRIER \
99 " amoswap.w %0, %2, %1\n" \
100 : "=r" (__ret), "+A" (*__ptr) \
101 : "r" (__new) \
102 : "memory"); \
103 break; \
104 case 8: \
105 __asm__ __volatile__ ( \
106 RISCV_RELEASE_BARRIER \
107 " amoswap.d %0, %2, %1\n" \
108 : "=r" (__ret), "+A" (*__ptr) \
109 : "r" (__new) \
110 : "memory"); \
111 break; \
112 default: \
113 BUILD_BUG(); \
114 } \
115 __ret; \
116})
117
118#define xchg_release(ptr, x) \
119({ \
120 __typeof__(*(ptr)) _x_ = (x); \
121 (__typeof__(*(ptr))) __xchg_release((ptr), \
122 _x_, sizeof(*(ptr))); \
123})
124
125#define __xchg(ptr, new, size) \
126({ \
127 __typeof__(ptr) __ptr = (ptr); \
128 __typeof__(new) __new = (new); \
129 __typeof__(*(ptr)) __ret; \
130 switch (size) { \
131 case 4: \
132 __asm__ __volatile__ ( \
133 " amoswap.w.aqrl %0, %2, %1\n" \
134 : "=r" (__ret), "+A" (*__ptr) \
135 : "r" (__new) \
136 : "memory"); \
137 break; \
138 case 8: \
139 __asm__ __volatile__ ( \
140 " amoswap.d.aqrl %0, %2, %1\n" \
141 : "=r" (__ret), "+A" (*__ptr) \
142 : "r" (__new) \
143 : "memory"); \
144 break; \
145 default: \
146 BUILD_BUG(); \
147 } \
148 __ret; \
149})
150
151#define xchg(ptr, x) \
152({ \
153 __typeof__(*(ptr)) _x_ = (x); \
154 (__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \
155})
156
157#define xchg32(ptr, x) \
158({ \
159 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
160 xchg((ptr), (x)); \
161})
162
163#define xchg64(ptr, x) \
164({ \
165 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
166 xchg((ptr), (x)); \
59}) 167})
60 168
61/* 169/*
@@ -63,7 +171,51 @@
63 * store NEW in MEM. Return the initial value in MEM. Success is 171 * store NEW in MEM. Return the initial value in MEM. Success is
64 * indicated by comparing RETURN with OLD. 172 * indicated by comparing RETURN with OLD.
65 */ 173 */
66#define __cmpxchg(ptr, old, new, size, lrb, scb) \ 174#define __cmpxchg_relaxed(ptr, old, new, size) \
175({ \
176 __typeof__(ptr) __ptr = (ptr); \
177 __typeof__(*(ptr)) __old = (old); \
178 __typeof__(*(ptr)) __new = (new); \
179 __typeof__(*(ptr)) __ret; \
180 register unsigned int __rc; \
181 switch (size) { \
182 case 4: \
183 __asm__ __volatile__ ( \
184 "0: lr.w %0, %2\n" \
185 " bne %0, %z3, 1f\n" \
186 " sc.w %1, %z4, %2\n" \
187 " bnez %1, 0b\n" \
188 "1:\n" \
189 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
190 : "rJ" (__old), "rJ" (__new) \
191 : "memory"); \
192 break; \
193 case 8: \
194 __asm__ __volatile__ ( \
195 "0: lr.d %0, %2\n" \
196 " bne %0, %z3, 1f\n" \
197 " sc.d %1, %z4, %2\n" \
198 " bnez %1, 0b\n" \
199 "1:\n" \
200 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
201 : "rJ" (__old), "rJ" (__new) \
202 : "memory"); \
203 break; \
204 default: \
205 BUILD_BUG(); \
206 } \
207 __ret; \
208})
209
210#define cmpxchg_relaxed(ptr, o, n) \
211({ \
212 __typeof__(*(ptr)) _o_ = (o); \
213 __typeof__(*(ptr)) _n_ = (n); \
214 (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \
215 _o_, _n_, sizeof(*(ptr))); \
216})
217
218#define __cmpxchg_acquire(ptr, old, new, size) \
67({ \ 219({ \
68 __typeof__(ptr) __ptr = (ptr); \ 220 __typeof__(ptr) __ptr = (ptr); \
69 __typeof__(*(ptr)) __old = (old); \ 221 __typeof__(*(ptr)) __old = (old); \
@@ -73,24 +225,24 @@
73 switch (size) { \ 225 switch (size) { \
74 case 4: \ 226 case 4: \
75 __asm__ __volatile__ ( \ 227 __asm__ __volatile__ ( \
76 "0:" \ 228 "0: lr.w %0, %2\n" \
77 "lr.w" #scb " %0, %2\n" \ 229 " bne %0, %z3, 1f\n" \
78 "bne %0, %z3, 1f\n" \ 230 " sc.w %1, %z4, %2\n" \
79 "sc.w" #lrb " %1, %z4, %2\n" \ 231 " bnez %1, 0b\n" \
80 "bnez %1, 0b\n" \ 232 RISCV_ACQUIRE_BARRIER \
81 "1:" \ 233 "1:\n" \
82 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 234 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
83 : "rJ" (__old), "rJ" (__new) \ 235 : "rJ" (__old), "rJ" (__new) \
84 : "memory"); \ 236 : "memory"); \
85 break; \ 237 break; \
86 case 8: \ 238 case 8: \
87 __asm__ __volatile__ ( \ 239 __asm__ __volatile__ ( \
88 "0:" \ 240 "0: lr.d %0, %2\n" \
89 "lr.d" #scb " %0, %2\n" \ 241 " bne %0, %z3, 1f\n" \
90 "bne %0, %z3, 1f\n" \ 242 " sc.d %1, %z4, %2\n" \
91 "sc.d" #lrb " %1, %z4, %2\n" \ 243 " bnez %1, 0b\n" \
92 "bnez %1, 0b\n" \ 244 RISCV_ACQUIRE_BARRIER \
93 "1:" \ 245 "1:\n" \
94 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 246 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
95 : "rJ" (__old), "rJ" (__new) \ 247 : "rJ" (__old), "rJ" (__new) \
96 : "memory"); \ 248 : "memory"); \
@@ -101,34 +253,131 @@
101 __ret; \ 253 __ret; \
102}) 254})
103 255
104#define cmpxchg(ptr, o, n) \ 256#define cmpxchg_acquire(ptr, o, n) \
105 (__cmpxchg((ptr), (o), (n), sizeof(*(ptr)), .aqrl, .aqrl)) 257({ \
258 __typeof__(*(ptr)) _o_ = (o); \
259 __typeof__(*(ptr)) _n_ = (n); \
260 (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \
261 _o_, _n_, sizeof(*(ptr))); \
262})
106 263
107#define cmpxchg_local(ptr, o, n) \ 264#define __cmpxchg_release(ptr, old, new, size) \
108 (__cmpxchg((ptr), (o), (n), sizeof(*(ptr)), , )) 265({ \
266 __typeof__(ptr) __ptr = (ptr); \
267 __typeof__(*(ptr)) __old = (old); \
268 __typeof__(*(ptr)) __new = (new); \
269 __typeof__(*(ptr)) __ret; \
270 register unsigned int __rc; \
271 switch (size) { \
272 case 4: \
273 __asm__ __volatile__ ( \
274 RISCV_RELEASE_BARRIER \
275 "0: lr.w %0, %2\n" \
276 " bne %0, %z3, 1f\n" \
277 " sc.w %1, %z4, %2\n" \
278 " bnez %1, 0b\n" \
279 "1:\n" \
280 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
281 : "rJ" (__old), "rJ" (__new) \
282 : "memory"); \
283 break; \
284 case 8: \
285 __asm__ __volatile__ ( \
286 RISCV_RELEASE_BARRIER \
287 "0: lr.d %0, %2\n" \
288 " bne %0, %z3, 1f\n" \
289 " sc.d %1, %z4, %2\n" \
290 " bnez %1, 0b\n" \
291 "1:\n" \
292 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
293 : "rJ" (__old), "rJ" (__new) \
294 : "memory"); \
295 break; \
296 default: \
297 BUILD_BUG(); \
298 } \
299 __ret; \
300})
301
302#define cmpxchg_release(ptr, o, n) \
303({ \
304 __typeof__(*(ptr)) _o_ = (o); \
305 __typeof__(*(ptr)) _n_ = (n); \
306 (__typeof__(*(ptr))) __cmpxchg_release((ptr), \
307 _o_, _n_, sizeof(*(ptr))); \
308})
309
310#define __cmpxchg(ptr, old, new, size) \
311({ \
312 __typeof__(ptr) __ptr = (ptr); \
313 __typeof__(*(ptr)) __old = (old); \
314 __typeof__(*(ptr)) __new = (new); \
315 __typeof__(*(ptr)) __ret; \
316 register unsigned int __rc; \
317 switch (size) { \
318 case 4: \
319 __asm__ __volatile__ ( \
320 "0: lr.w %0, %2\n" \
321 " bne %0, %z3, 1f\n" \
322 " sc.w.rl %1, %z4, %2\n" \
323 " bnez %1, 0b\n" \
324 " fence rw, rw\n" \
325 "1:\n" \
326 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
327 : "rJ" (__old), "rJ" (__new) \
328 : "memory"); \
329 break; \
330 case 8: \
331 __asm__ __volatile__ ( \
332 "0: lr.d %0, %2\n" \
333 " bne %0, %z3, 1f\n" \
334 " sc.d.rl %1, %z4, %2\n" \
335 " bnez %1, 0b\n" \
336 " fence rw, rw\n" \
337 "1:\n" \
338 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
339 : "rJ" (__old), "rJ" (__new) \
340 : "memory"); \
341 break; \
342 default: \
343 BUILD_BUG(); \
344 } \
345 __ret; \
346})
109 347
110#define cmpxchg32(ptr, o, n) \ 348#define cmpxchg(ptr, o, n) \
111({ \ 349({ \
112 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ 350 __typeof__(*(ptr)) _o_ = (o); \
113 cmpxchg((ptr), (o), (n)); \ 351 __typeof__(*(ptr)) _n_ = (n); \
352 (__typeof__(*(ptr))) __cmpxchg((ptr), \
353 _o_, _n_, sizeof(*(ptr))); \
114}) 354})
115 355
116#define cmpxchg32_local(ptr, o, n) \ 356#define cmpxchg_local(ptr, o, n) \
117({ \ 357 (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr))))
118 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ 358
119 cmpxchg_local((ptr), (o), (n)); \ 359#define cmpxchg32(ptr, o, n) \
360({ \
361 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
362 cmpxchg((ptr), (o), (n)); \
120}) 363})
121 364
122#define cmpxchg64(ptr, o, n) \ 365#define cmpxchg32_local(ptr, o, n) \
123({ \ 366({ \
124 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 367 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
125 cmpxchg((ptr), (o), (n)); \ 368 cmpxchg_relaxed((ptr), (o), (n)) \
126}) 369})
127 370
128#define cmpxchg64_local(ptr, o, n) \ 371#define cmpxchg64(ptr, o, n) \
129({ \ 372({ \
130 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 373 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
131 cmpxchg_local((ptr), (o), (n)); \ 374 cmpxchg((ptr), (o), (n)); \
375})
376
377#define cmpxchg64_local(ptr, o, n) \
378({ \
379 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
380 cmpxchg_relaxed((ptr), (o), (n)); \
132}) 381})
133 382
134#endif /* _ASM_RISCV_CMPXCHG_H */ 383#endif /* _ASM_RISCV_CMPXCHG_H */
diff --git a/arch/riscv/include/asm/fence.h b/arch/riscv/include/asm/fence.h
new file mode 100644
index 000000000000..2b443a3a487f
--- /dev/null
+++ b/arch/riscv/include/asm/fence.h
@@ -0,0 +1,12 @@
1#ifndef _ASM_RISCV_FENCE_H
2#define _ASM_RISCV_FENCE_H
3
4#ifdef CONFIG_SMP
5#define RISCV_ACQUIRE_BARRIER "\tfence r , rw\n"
6#define RISCV_RELEASE_BARRIER "\tfence rw, w\n"
7#else
8#define RISCV_ACQUIRE_BARRIER
9#define RISCV_RELEASE_BARRIER
10#endif
11
12#endif /* _ASM_RISCV_FENCE_H */
diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h
index 66d4175eb13e..c6dcc5291f97 100644
--- a/arch/riscv/include/asm/ftrace.h
+++ b/arch/riscv/include/asm/ftrace.h
@@ -8,3 +8,59 @@
8#if defined(CONFIG_FUNCTION_GRAPH_TRACER) && defined(CONFIG_FRAME_POINTER) 8#if defined(CONFIG_FUNCTION_GRAPH_TRACER) && defined(CONFIG_FRAME_POINTER)
9#define HAVE_FUNCTION_GRAPH_FP_TEST 9#define HAVE_FUNCTION_GRAPH_FP_TEST
10#endif 10#endif
11#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
12
13#define ARCH_SUPPORTS_FTRACE_OPS 1
14#ifndef __ASSEMBLY__
15void _mcount(void);
16static inline unsigned long ftrace_call_adjust(unsigned long addr)
17{
18 return addr;
19}
20
21struct dyn_arch_ftrace {
22};
23#endif
24
25#ifdef CONFIG_DYNAMIC_FTRACE
26/*
27 * A general call in RISC-V is a pair of insts:
28 * 1) auipc: setting high-20 pc-related bits to ra register
29 * 2) jalr: setting low-12 offset to ra, jump to ra, and set ra to
30 * return address (original pc + 4)
31 *
32 * Dynamic ftrace generates probes to call sites, so we must deal with
33 * both auipc and jalr at the same time.
34 */
35
36#define MCOUNT_ADDR ((unsigned long)_mcount)
37#define JALR_SIGN_MASK (0x00000800)
38#define JALR_OFFSET_MASK (0x00000fff)
39#define AUIPC_OFFSET_MASK (0xfffff000)
40#define AUIPC_PAD (0x00001000)
41#define JALR_SHIFT 20
42#define JALR_BASIC (0x000080e7)
43#define AUIPC_BASIC (0x00000097)
44#define NOP4 (0x00000013)
45
46#define make_call(caller, callee, call) \
47do { \
48 call[0] = to_auipc_insn((unsigned int)((unsigned long)callee - \
49 (unsigned long)caller)); \
50 call[1] = to_jalr_insn((unsigned int)((unsigned long)callee - \
51 (unsigned long)caller)); \
52} while (0)
53
54#define to_jalr_insn(offset) \
55 (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_BASIC)
56
57#define to_auipc_insn(offset) \
58 ((offset & JALR_SIGN_MASK) ? \
59 (((offset & AUIPC_OFFSET_MASK) + AUIPC_PAD) | AUIPC_BASIC) : \
60 ((offset & AUIPC_OFFSET_MASK) | AUIPC_BASIC))
61
62/*
63 * Let auipc+jalr be the basic *mcount unit*, so we make it 8 bytes here.
64 */
65#define MCOUNT_INSN_SIZE 8
66#endif
diff --git a/arch/riscv/include/asm/module.h b/arch/riscv/include/asm/module.h
new file mode 100644
index 000000000000..349df33808c4
--- /dev/null
+++ b/arch/riscv/include/asm/module.h
@@ -0,0 +1,113 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (C) 2017 Andes Technology Corporation */
3
4#ifndef _ASM_RISCV_MODULE_H
5#define _ASM_RISCV_MODULE_H
6
7#include <asm-generic/module.h>
8
9#define MODULE_ARCH_VERMAGIC "riscv"
10
11u64 module_emit_got_entry(struct module *mod, u64 val);
12u64 module_emit_plt_entry(struct module *mod, u64 val);
13
14#ifdef CONFIG_MODULE_SECTIONS
15struct mod_section {
16 struct elf64_shdr *shdr;
17 int num_entries;
18 int max_entries;
19};
20
21struct mod_arch_specific {
22 struct mod_section got;
23 struct mod_section plt;
24 struct mod_section got_plt;
25};
26
27struct got_entry {
28 u64 symbol_addr; /* the real variable address */
29};
30
31static inline struct got_entry emit_got_entry(u64 val)
32{
33 return (struct got_entry) {val};
34}
35
36static inline struct got_entry *get_got_entry(u64 val,
37 const struct mod_section *sec)
38{
39 struct got_entry *got = (struct got_entry *)sec->shdr->sh_addr;
40 int i;
41 for (i = 0; i < sec->num_entries; i++) {
42 if (got[i].symbol_addr == val)
43 return &got[i];
44 }
45 return NULL;
46}
47
48struct plt_entry {
49 /*
50 * Trampoline code to real target address. The return address
51 * should be the original (pc+4) before entring plt entry.
52 */
53 u32 insn_auipc; /* auipc t0, 0x0 */
54 u32 insn_ld; /* ld t1, 0x10(t0) */
55 u32 insn_jr; /* jr t1 */
56};
57
58#define OPC_AUIPC 0x0017
59#define OPC_LD 0x3003
60#define OPC_JALR 0x0067
61#define REG_T0 0x5
62#define REG_T1 0x6
63
64static inline struct plt_entry emit_plt_entry(u64 val, u64 plt, u64 got_plt)
65{
66 /*
67 * U-Type encoding:
68 * +------------+----------+----------+
69 * | imm[31:12] | rd[11:7] | opc[6:0] |
70 * +------------+----------+----------+
71 *
72 * I-Type encoding:
73 * +------------+------------+--------+----------+----------+
74 * | imm[31:20] | rs1[19:15] | funct3 | rd[11:7] | opc[6:0] |
75 * +------------+------------+--------+----------+----------+
76 *
77 */
78 u64 offset = got_plt - plt;
79 u32 hi20 = (offset + 0x800) & 0xfffff000;
80 u32 lo12 = (offset - hi20);
81 return (struct plt_entry) {
82 OPC_AUIPC | (REG_T0 << 7) | hi20,
83 OPC_LD | (lo12 << 20) | (REG_T0 << 15) | (REG_T1 << 7),
84 OPC_JALR | (REG_T1 << 15)
85 };
86}
87
88static inline int get_got_plt_idx(u64 val, const struct mod_section *sec)
89{
90 struct got_entry *got_plt = (struct got_entry *)sec->shdr->sh_addr;
91 int i;
92 for (i = 0; i < sec->num_entries; i++) {
93 if (got_plt[i].symbol_addr == val)
94 return i;
95 }
96 return -1;
97}
98
99static inline struct plt_entry *get_plt_entry(u64 val,
100 const struct mod_section *sec_plt,
101 const struct mod_section *sec_got_plt)
102{
103 struct plt_entry *plt = (struct plt_entry *)sec_plt->shdr->sh_addr;
104 int got_plt_idx = get_got_plt_idx(val, sec_got_plt);
105 if (got_plt_idx >= 0)
106 return plt + got_plt_idx;
107 else
108 return NULL;
109}
110
111#endif /* CONFIG_MODULE_SECTIONS */
112
113#endif /* _ASM_RISCV_MODULE_H */
diff --git a/arch/riscv/include/asm/spinlock.h b/arch/riscv/include/asm/spinlock.h
index 2fd27e8ef1fd..8eb26d1ede81 100644
--- a/arch/riscv/include/asm/spinlock.h
+++ b/arch/riscv/include/asm/spinlock.h
@@ -17,6 +17,7 @@
17 17
18#include <linux/kernel.h> 18#include <linux/kernel.h>
19#include <asm/current.h> 19#include <asm/current.h>
20#include <asm/fence.h>
20 21
21/* 22/*
22 * Simple spin lock operations. These provide no fairness guarantees. 23 * Simple spin lock operations. These provide no fairness guarantees.
@@ -28,10 +29,7 @@
28 29
29static inline void arch_spin_unlock(arch_spinlock_t *lock) 30static inline void arch_spin_unlock(arch_spinlock_t *lock)
30{ 31{
31 __asm__ __volatile__ ( 32 smp_store_release(&lock->lock, 0);
32 "amoswap.w.rl x0, x0, %0"
33 : "=A" (lock->lock)
34 :: "memory");
35} 33}
36 34
37static inline int arch_spin_trylock(arch_spinlock_t *lock) 35static inline int arch_spin_trylock(arch_spinlock_t *lock)
@@ -39,7 +37,8 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock)
39 int tmp = 1, busy; 37 int tmp = 1, busy;
40 38
41 __asm__ __volatile__ ( 39 __asm__ __volatile__ (
42 "amoswap.w.aq %0, %2, %1" 40 " amoswap.w %0, %2, %1\n"
41 RISCV_ACQUIRE_BARRIER
43 : "=r" (busy), "+A" (lock->lock) 42 : "=r" (busy), "+A" (lock->lock)
44 : "r" (tmp) 43 : "r" (tmp)
45 : "memory"); 44 : "memory");
@@ -68,8 +67,9 @@ static inline void arch_read_lock(arch_rwlock_t *lock)
68 "1: lr.w %1, %0\n" 67 "1: lr.w %1, %0\n"
69 " bltz %1, 1b\n" 68 " bltz %1, 1b\n"
70 " addi %1, %1, 1\n" 69 " addi %1, %1, 1\n"
71 " sc.w.aq %1, %1, %0\n" 70 " sc.w %1, %1, %0\n"
72 " bnez %1, 1b\n" 71 " bnez %1, 1b\n"
72 RISCV_ACQUIRE_BARRIER
73 : "+A" (lock->lock), "=&r" (tmp) 73 : "+A" (lock->lock), "=&r" (tmp)
74 :: "memory"); 74 :: "memory");
75} 75}
@@ -82,8 +82,9 @@ static inline void arch_write_lock(arch_rwlock_t *lock)
82 "1: lr.w %1, %0\n" 82 "1: lr.w %1, %0\n"
83 " bnez %1, 1b\n" 83 " bnez %1, 1b\n"
84 " li %1, -1\n" 84 " li %1, -1\n"
85 " sc.w.aq %1, %1, %0\n" 85 " sc.w %1, %1, %0\n"
86 " bnez %1, 1b\n" 86 " bnez %1, 1b\n"
87 RISCV_ACQUIRE_BARRIER
87 : "+A" (lock->lock), "=&r" (tmp) 88 : "+A" (lock->lock), "=&r" (tmp)
88 :: "memory"); 89 :: "memory");
89} 90}
@@ -96,8 +97,9 @@ static inline int arch_read_trylock(arch_rwlock_t *lock)
96 "1: lr.w %1, %0\n" 97 "1: lr.w %1, %0\n"
97 " bltz %1, 1f\n" 98 " bltz %1, 1f\n"
98 " addi %1, %1, 1\n" 99 " addi %1, %1, 1\n"
99 " sc.w.aq %1, %1, %0\n" 100 " sc.w %1, %1, %0\n"
100 " bnez %1, 1b\n" 101 " bnez %1, 1b\n"
102 RISCV_ACQUIRE_BARRIER
101 "1:\n" 103 "1:\n"
102 : "+A" (lock->lock), "=&r" (busy) 104 : "+A" (lock->lock), "=&r" (busy)
103 :: "memory"); 105 :: "memory");
@@ -113,8 +115,9 @@ static inline int arch_write_trylock(arch_rwlock_t *lock)
113 "1: lr.w %1, %0\n" 115 "1: lr.w %1, %0\n"
114 " bnez %1, 1f\n" 116 " bnez %1, 1f\n"
115 " li %1, -1\n" 117 " li %1, -1\n"
116 " sc.w.aq %1, %1, %0\n" 118 " sc.w %1, %1, %0\n"
117 " bnez %1, 1b\n" 119 " bnez %1, 1b\n"
120 RISCV_ACQUIRE_BARRIER
118 "1:\n" 121 "1:\n"
119 : "+A" (lock->lock), "=&r" (busy) 122 : "+A" (lock->lock), "=&r" (busy)
120 :: "memory"); 123 :: "memory");
@@ -125,7 +128,8 @@ static inline int arch_write_trylock(arch_rwlock_t *lock)
125static inline void arch_read_unlock(arch_rwlock_t *lock) 128static inline void arch_read_unlock(arch_rwlock_t *lock)
126{ 129{
127 __asm__ __volatile__( 130 __asm__ __volatile__(
128 "amoadd.w.rl x0, %1, %0" 131 RISCV_RELEASE_BARRIER
132 " amoadd.w x0, %1, %0\n"
129 : "+A" (lock->lock) 133 : "+A" (lock->lock)
130 : "r" (-1) 134 : "r" (-1)
131 : "memory"); 135 : "memory");
@@ -133,10 +137,7 @@ static inline void arch_read_unlock(arch_rwlock_t *lock)
133 137
134static inline void arch_write_unlock(arch_rwlock_t *lock) 138static inline void arch_write_unlock(arch_rwlock_t *lock)
135{ 139{
136 __asm__ __volatile__ ( 140 smp_store_release(&lock->lock, 0);
137 "amoswap.w.rl x0, x0, %0"
138 : "=A" (lock->lock)
139 :: "memory");
140} 141}
141 142
142#endif /* _ASM_RISCV_SPINLOCK_H */ 143#endif /* _ASM_RISCV_SPINLOCK_H */
diff --git a/arch/riscv/include/uapi/asm/elf.h b/arch/riscv/include/uapi/asm/elf.h
index a510edfa8226..5cae4c30cd8e 100644
--- a/arch/riscv/include/uapi/asm/elf.h
+++ b/arch/riscv/include/uapi/asm/elf.h
@@ -79,5 +79,12 @@ typedef union __riscv_fp_state elf_fpregset_t;
79#define R_RISCV_TPREL_I 49 79#define R_RISCV_TPREL_I 49
80#define R_RISCV_TPREL_S 50 80#define R_RISCV_TPREL_S 50
81#define R_RISCV_RELAX 51 81#define R_RISCV_RELAX 51
82#define R_RISCV_SUB6 52
83#define R_RISCV_SET6 53
84#define R_RISCV_SET8 54
85#define R_RISCV_SET16 55
86#define R_RISCV_SET32 56
87#define R_RISCV_32_PCREL 57
88
82 89
83#endif /* _UAPI_ASM_ELF_H */ 90#endif /* _UAPI_ASM_ELF_H */
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index 196f62ffc428..8586dd96c2f0 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -34,7 +34,9 @@ CFLAGS_setup.o := -mcmodel=medany
34obj-$(CONFIG_SMP) += smpboot.o 34obj-$(CONFIG_SMP) += smpboot.o
35obj-$(CONFIG_SMP) += smp.o 35obj-$(CONFIG_SMP) += smp.o
36obj-$(CONFIG_MODULES) += module.o 36obj-$(CONFIG_MODULES) += module.o
37obj-$(CONFIG_FUNCTION_TRACER) += mcount.o 37obj-$(CONFIG_MODULE_SECTIONS) += module-sections.o
38obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o 38
39obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o
40obj-$(CONFIG_DYNAMIC_FTRACE) += mcount-dyn.o
39 41
40clean: 42clean:
diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c
index d0de68d144cb..1157b6b52d25 100644
--- a/arch/riscv/kernel/ftrace.c
+++ b/arch/riscv/kernel/ftrace.c
@@ -6,9 +6,126 @@
6 */ 6 */
7 7
8#include <linux/ftrace.h> 8#include <linux/ftrace.h>
9#include <linux/uaccess.h>
10#include <asm/cacheflush.h>
9 11
12#ifdef CONFIG_DYNAMIC_FTRACE
13static int ftrace_check_current_call(unsigned long hook_pos,
14 unsigned int *expected)
15{
16 unsigned int replaced[2];
17 unsigned int nops[2] = {NOP4, NOP4};
18
19 /* we expect nops at the hook position */
20 if (!expected)
21 expected = nops;
22
23 /*
24 * Read the text we want to modify;
25 * return must be -EFAULT on read error
26 */
27 if (probe_kernel_read(replaced, (void *)hook_pos, MCOUNT_INSN_SIZE))
28 return -EFAULT;
29
30 /*
31 * Make sure it is what we expect it to be;
32 * return must be -EINVAL on failed comparison
33 */
34 if (memcmp(expected, replaced, sizeof(replaced))) {
35 pr_err("%p: expected (%08x %08x) but get (%08x %08x)",
36 (void *)hook_pos, expected[0], expected[1], replaced[0],
37 replaced[1]);
38 return -EINVAL;
39 }
40
41 return 0;
42}
43
44static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target,
45 bool enable)
46{
47 unsigned int call[2];
48 unsigned int nops[2] = {NOP4, NOP4};
49 int ret = 0;
50
51 make_call(hook_pos, target, call);
52
53 /* replace the auipc-jalr pair at once */
54 ret = probe_kernel_write((void *)hook_pos, enable ? call : nops,
55 MCOUNT_INSN_SIZE);
56 /* return must be -EPERM on write error */
57 if (ret)
58 return -EPERM;
59
60 smp_mb();
61 flush_icache_range((void *)hook_pos, (void *)hook_pos + MCOUNT_INSN_SIZE);
62
63 return 0;
64}
65
66int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
67{
68 int ret = ftrace_check_current_call(rec->ip, NULL);
69
70 if (ret)
71 return ret;
72
73 return __ftrace_modify_call(rec->ip, addr, true);
74}
75
76int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
77 unsigned long addr)
78{
79 unsigned int call[2];
80 int ret;
81
82 make_call(rec->ip, addr, call);
83 ret = ftrace_check_current_call(rec->ip, call);
84
85 if (ret)
86 return ret;
87
88 return __ftrace_modify_call(rec->ip, addr, false);
89}
90
91int ftrace_update_ftrace_func(ftrace_func_t func)
92{
93 int ret = __ftrace_modify_call((unsigned long)&ftrace_call,
94 (unsigned long)func, true);
95 if (!ret) {
96 ret = __ftrace_modify_call((unsigned long)&ftrace_regs_call,
97 (unsigned long)func, true);
98 }
99
100 return ret;
101}
102
103int __init ftrace_dyn_arch_init(void)
104{
105 return 0;
106}
107#endif
108
109#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
110int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
111 unsigned long addr)
112{
113 unsigned int call[2];
114 int ret;
115
116 make_call(rec->ip, old_addr, call);
117 ret = ftrace_check_current_call(rec->ip, call);
118
119 if (ret)
120 return ret;
121
122 return __ftrace_modify_call(rec->ip, addr, true);
123}
124#endif
125
126#ifdef CONFIG_FUNCTION_GRAPH_TRACER
10/* 127/*
11 * Most of this file is copied from arm64. 128 * Most of this function is copied from arm64.
12 */ 129 */
13void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, 130void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
14 unsigned long frame_pointer) 131 unsigned long frame_pointer)
@@ -34,8 +151,62 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
34 return; 151 return;
35 152
36 err = ftrace_push_return_trace(old, self_addr, &trace.depth, 153 err = ftrace_push_return_trace(old, self_addr, &trace.depth,
37 frame_pointer, NULL); 154 frame_pointer, parent);
38 if (err == -EBUSY) 155 if (err == -EBUSY)
39 return; 156 return;
40 *parent = return_hooker; 157 *parent = return_hooker;
41} 158}
159
160#ifdef CONFIG_DYNAMIC_FTRACE
161extern void ftrace_graph_call(void);
162int ftrace_enable_ftrace_graph_caller(void)
163{
164 unsigned int call[2];
165 static int init_graph = 1;
166 int ret;
167
168 make_call(&ftrace_graph_call, &ftrace_stub, call);
169
170 /*
171 * When enabling graph tracer for the first time, ftrace_graph_call
172 * should contains a call to ftrace_stub. Once it has been disabled,
173 * the 8-bytes at the position becomes NOPs.
174 */
175 if (init_graph) {
176 ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call,
177 call);
178 init_graph = 0;
179 } else {
180 ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call,
181 NULL);
182 }
183
184 if (ret)
185 return ret;
186
187 return __ftrace_modify_call((unsigned long)&ftrace_graph_call,
188 (unsigned long)&prepare_ftrace_return, true);
189}
190
191int ftrace_disable_ftrace_graph_caller(void)
192{
193 unsigned int call[2];
194 int ret;
195
196 make_call(&ftrace_graph_call, &prepare_ftrace_return, call);
197
198 /*
199 * This is to make sure that ftrace_enable_ftrace_graph_caller
200 * did the right thing.
201 */
202 ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call,
203 call);
204
205 if (ret)
206 return ret;
207
208 return __ftrace_modify_call((unsigned long)&ftrace_graph_call,
209 (unsigned long)&prepare_ftrace_return, false);
210}
211#endif /* CONFIG_DYNAMIC_FTRACE */
212#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
diff --git a/arch/riscv/kernel/mcount-dyn.S b/arch/riscv/kernel/mcount-dyn.S
new file mode 100644
index 000000000000..35a6ed76cb8b
--- /dev/null
+++ b/arch/riscv/kernel/mcount-dyn.S
@@ -0,0 +1,239 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (C) 2017 Andes Technology Corporation */
3
4#include <linux/init.h>
5#include <linux/linkage.h>
6#include <asm/asm.h>
7#include <asm/csr.h>
8#include <asm/unistd.h>
9#include <asm/thread_info.h>
10#include <asm/asm-offsets.h>
11#include <asm-generic/export.h>
12#include <asm/ftrace.h>
13
14 .text
15
16 .macro SAVE_ABI_STATE
17#ifdef CONFIG_FUNCTION_GRAPH_TRACER
18 addi sp, sp, -48
19 sd s0, 32(sp)
20 sd ra, 40(sp)
21 addi s0, sp, 48
22 sd t0, 24(sp)
23 sd t1, 16(sp)
24#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
25 sd t2, 8(sp)
26#endif
27#else
28 addi sp, sp, -16
29 sd s0, 0(sp)
30 sd ra, 8(sp)
31 addi s0, sp, 16
32#endif
33 .endm
34
35 .macro RESTORE_ABI_STATE
36#ifdef CONFIG_FUNCTION_GRAPH_TRACER
37 ld s0, 32(sp)
38 ld ra, 40(sp)
39 addi sp, sp, 48
40#else
41 ld ra, 8(sp)
42 ld s0, 0(sp)
43 addi sp, sp, 16
44#endif
45 .endm
46
47 .macro RESTORE_GRAPH_ARGS
48 ld a0, 24(sp)
49 ld a1, 16(sp)
50#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
51 ld a2, 8(sp)
52#endif
53 .endm
54
55ENTRY(ftrace_graph_caller)
56 addi sp, sp, -16
57 sd s0, 0(sp)
58 sd ra, 8(sp)
59 addi s0, sp, 16
60ftrace_graph_call:
61 .global ftrace_graph_call
62 /*
63 * Calling ftrace_enable/disable_ftrace_graph_caller would overwrite the
64 * call below. Check ftrace_modify_all_code for details.
65 */
66 call ftrace_stub
67 ld ra, 8(sp)
68 ld s0, 0(sp)
69 addi sp, sp, 16
70 ret
71ENDPROC(ftrace_graph_caller)
72
73ENTRY(ftrace_caller)
74 /*
75 * a0: the address in the caller when calling ftrace_caller
76 * a1: the caller's return address
77 * a2: the address of global variable function_trace_op
78 */
79 ld a1, -8(s0)
80 addi a0, ra, -MCOUNT_INSN_SIZE
81 la t5, function_trace_op
82 ld a2, 0(t5)
83
84#ifdef CONFIG_FUNCTION_GRAPH_TRACER
85 /*
86 * the graph tracer (specifically, prepare_ftrace_return) needs these
87 * arguments but for now the function tracer occupies the regs, so we
88 * save them in temporary regs to recover later.
89 */
90 addi t0, s0, -8
91 mv t1, a0
92#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
93 ld t2, -16(s0)
94#endif
95#endif
96
97 SAVE_ABI_STATE
98ftrace_call:
99 .global ftrace_call
100 /*
101 * For the dynamic ftrace to work, here we should reserve at least
102 * 8 bytes for a functional auipc-jalr pair. The following call
103 * serves this purpose.
104 *
105 * Calling ftrace_update_ftrace_func would overwrite the nops below.
106 * Check ftrace_modify_all_code for details.
107 */
108 call ftrace_stub
109
110#ifdef CONFIG_FUNCTION_GRAPH_TRACER
111 RESTORE_GRAPH_ARGS
112 call ftrace_graph_caller
113#endif
114
115 RESTORE_ABI_STATE
116 ret
117ENDPROC(ftrace_caller)
118
119#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
120 .macro SAVE_ALL
121 addi sp, sp, -(PT_SIZE_ON_STACK+16)
122 sd s0, (PT_SIZE_ON_STACK)(sp)
123 sd ra, (PT_SIZE_ON_STACK+8)(sp)
124 addi s0, sp, (PT_SIZE_ON_STACK+16)
125
126 sd x1, PT_RA(sp)
127 sd x2, PT_SP(sp)
128 sd x3, PT_GP(sp)
129 sd x4, PT_TP(sp)
130 sd x5, PT_T0(sp)
131 sd x6, PT_T1(sp)
132 sd x7, PT_T2(sp)
133 sd x8, PT_S0(sp)
134 sd x9, PT_S1(sp)
135 sd x10, PT_A0(sp)
136 sd x11, PT_A1(sp)
137 sd x12, PT_A2(sp)
138 sd x13, PT_A3(sp)
139 sd x14, PT_A4(sp)
140 sd x15, PT_A5(sp)
141 sd x16, PT_A6(sp)
142 sd x17, PT_A7(sp)
143 sd x18, PT_S2(sp)
144 sd x19, PT_S3(sp)
145 sd x20, PT_S4(sp)
146 sd x21, PT_S5(sp)
147 sd x22, PT_S6(sp)
148 sd x23, PT_S7(sp)
149 sd x24, PT_S8(sp)
150 sd x25, PT_S9(sp)
151 sd x26, PT_S10(sp)
152 sd x27, PT_S11(sp)
153 sd x28, PT_T3(sp)
154 sd x29, PT_T4(sp)
155 sd x30, PT_T5(sp)
156 sd x31, PT_T6(sp)
157 .endm
158
159 .macro RESTORE_ALL
160 ld x1, PT_RA(sp)
161 ld x2, PT_SP(sp)
162 ld x3, PT_GP(sp)
163 ld x4, PT_TP(sp)
164 ld x5, PT_T0(sp)
165 ld x6, PT_T1(sp)
166 ld x7, PT_T2(sp)
167 ld x8, PT_S0(sp)
168 ld x9, PT_S1(sp)
169 ld x10, PT_A0(sp)
170 ld x11, PT_A1(sp)
171 ld x12, PT_A2(sp)
172 ld x13, PT_A3(sp)
173 ld x14, PT_A4(sp)
174 ld x15, PT_A5(sp)
175 ld x16, PT_A6(sp)
176 ld x17, PT_A7(sp)
177 ld x18, PT_S2(sp)
178 ld x19, PT_S3(sp)
179 ld x20, PT_S4(sp)
180 ld x21, PT_S5(sp)
181 ld x22, PT_S6(sp)
182 ld x23, PT_S7(sp)
183 ld x24, PT_S8(sp)
184 ld x25, PT_S9(sp)
185 ld x26, PT_S10(sp)
186 ld x27, PT_S11(sp)
187 ld x28, PT_T3(sp)
188 ld x29, PT_T4(sp)
189 ld x30, PT_T5(sp)
190 ld x31, PT_T6(sp)
191
192 ld s0, (PT_SIZE_ON_STACK)(sp)
193 ld ra, (PT_SIZE_ON_STACK+8)(sp)
194 addi sp, sp, (PT_SIZE_ON_STACK+16)
195 .endm
196
197 .macro RESTORE_GRAPH_REG_ARGS
198 ld a0, PT_T0(sp)
199 ld a1, PT_T1(sp)
200#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
201 ld a2, PT_T2(sp)
202#endif
203 .endm
204
205/*
206 * Most of the contents are the same as ftrace_caller.
207 */
208ENTRY(ftrace_regs_caller)
209 /*
210 * a3: the address of all registers in the stack
211 */
212 ld a1, -8(s0)
213 addi a0, ra, -MCOUNT_INSN_SIZE
214 la t5, function_trace_op
215 ld a2, 0(t5)
216 addi a3, sp, -(PT_SIZE_ON_STACK+16)
217
218#ifdef CONFIG_FUNCTION_GRAPH_TRACER
219 addi t0, s0, -8
220 mv t1, a0
221#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
222 ld t2, -16(s0)
223#endif
224#endif
225 SAVE_ALL
226
227ftrace_regs_call:
228 .global ftrace_regs_call
229 call ftrace_stub
230
231#ifdef CONFIG_FUNCTION_GRAPH_TRACER
232 RESTORE_GRAPH_REG_ARGS
233 call ftrace_graph_caller
234#endif
235
236 RESTORE_ALL
237 ret
238ENDPROC(ftrace_regs_caller)
239#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
diff --git a/arch/riscv/kernel/mcount.S b/arch/riscv/kernel/mcount.S
index c46a778627be..ce9bdc57a2a1 100644
--- a/arch/riscv/kernel/mcount.S
+++ b/arch/riscv/kernel/mcount.S
@@ -32,13 +32,13 @@
32 addi s0, sp, 32 32 addi s0, sp, 32
33 .endm 33 .endm
34 34
35 .macro STORE_ABI_STATE 35 .macro RESTORE_ABI_STATE
36 ld ra, 8(sp) 36 ld ra, 8(sp)
37 ld s0, 0(sp) 37 ld s0, 0(sp)
38 addi sp, sp, 16 38 addi sp, sp, 16
39 .endm 39 .endm
40 40
41 .macro STORE_RET_ABI_STATE 41 .macro RESTORE_RET_ABI_STATE
42 ld ra, 24(sp) 42 ld ra, 24(sp)
43 ld s0, 16(sp) 43 ld s0, 16(sp)
44 ld a0, 8(sp) 44 ld a0, 8(sp)
@@ -46,6 +46,10 @@
46 .endm 46 .endm
47 47
48ENTRY(ftrace_stub) 48ENTRY(ftrace_stub)
49#ifdef CONFIG_DYNAMIC_FTRACE
50 .global _mcount
51 .set _mcount, ftrace_stub
52#endif
49 ret 53 ret
50ENDPROC(ftrace_stub) 54ENDPROC(ftrace_stub)
51 55
@@ -66,15 +70,15 @@ ENTRY(return_to_handler)
66#ifdef HAVE_FUNCTION_GRAPH_FP_TEST 70#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
67 mv a0, t6 71 mv a0, t6
68#endif 72#endif
69 la t0, ftrace_return_to_handler 73 call ftrace_return_to_handler
70 jalr t0
71 mv a1, a0 74 mv a1, a0
72 STORE_RET_ABI_STATE 75 RESTORE_RET_ABI_STATE
73 jalr a1 76 jalr a1
74ENDPROC(return_to_handler) 77ENDPROC(return_to_handler)
75EXPORT_SYMBOL(return_to_handler) 78EXPORT_SYMBOL(return_to_handler)
76#endif 79#endif
77 80
81#ifndef CONFIG_DYNAMIC_FTRACE
78ENTRY(_mcount) 82ENTRY(_mcount)
79 la t4, ftrace_stub 83 la t4, ftrace_stub
80#ifdef CONFIG_FUNCTION_GRAPH_TRACER 84#ifdef CONFIG_FUNCTION_GRAPH_TRACER
@@ -104,9 +108,8 @@ do_ftrace_graph_caller:
104 ld a2, -16(s0) 108 ld a2, -16(s0)
105#endif 109#endif
106 SAVE_ABI_STATE 110 SAVE_ABI_STATE
107 la t0, prepare_ftrace_return 111 call prepare_ftrace_return
108 jalr t0 112 RESTORE_ABI_STATE
109 STORE_ABI_STATE
110 ret 113 ret
111#endif 114#endif
112 115
@@ -120,7 +123,8 @@ do_trace:
120 123
121 SAVE_ABI_STATE 124 SAVE_ABI_STATE
122 jalr t5 125 jalr t5
123 STORE_ABI_STATE 126 RESTORE_ABI_STATE
124 ret 127 ret
125ENDPROC(_mcount) 128ENDPROC(_mcount)
126EXPORT_SYMBOL(_mcount) 129EXPORT_SYMBOL(_mcount)
130#endif
diff --git a/arch/riscv/kernel/module-sections.c b/arch/riscv/kernel/module-sections.c
new file mode 100644
index 000000000000..bbbd26e19bfd
--- /dev/null
+++ b/arch/riscv/kernel/module-sections.c
@@ -0,0 +1,156 @@
1/* SPDX-License-Identifier: GPL-2.0
2 *
3 * Copyright (C) 2014-2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
4 *
5 * Copyright (C) 2018 Andes Technology Corporation <zong@andestech.com>
6 */
7
8#include <linux/elf.h>
9#include <linux/kernel.h>
10#include <linux/module.h>
11
12u64 module_emit_got_entry(struct module *mod, u64 val)
13{
14 struct mod_section *got_sec = &mod->arch.got;
15 int i = got_sec->num_entries;
16 struct got_entry *got = get_got_entry(val, got_sec);
17
18 if (got)
19 return (u64)got;
20
21 /* There is no duplicate entry, create a new one */
22 got = (struct got_entry *)got_sec->shdr->sh_addr;
23 got[i] = emit_got_entry(val);
24
25 got_sec->num_entries++;
26 BUG_ON(got_sec->num_entries > got_sec->max_entries);
27
28 return (u64)&got[i];
29}
30
31u64 module_emit_plt_entry(struct module *mod, u64 val)
32{
33 struct mod_section *got_plt_sec = &mod->arch.got_plt;
34 struct got_entry *got_plt;
35 struct mod_section *plt_sec = &mod->arch.plt;
36 struct plt_entry *plt = get_plt_entry(val, plt_sec, got_plt_sec);
37 int i = plt_sec->num_entries;
38
39 if (plt)
40 return (u64)plt;
41
42 /* There is no duplicate entry, create a new one */
43 got_plt = (struct got_entry *)got_plt_sec->shdr->sh_addr;
44 got_plt[i] = emit_got_entry(val);
45 plt = (struct plt_entry *)plt_sec->shdr->sh_addr;
46 plt[i] = emit_plt_entry(val, (u64)&plt[i], (u64)&got_plt[i]);
47
48 plt_sec->num_entries++;
49 got_plt_sec->num_entries++;
50 BUG_ON(plt_sec->num_entries > plt_sec->max_entries);
51
52 return (u64)&plt[i];
53}
54
55static int is_rela_equal(const Elf64_Rela *x, const Elf64_Rela *y)
56{
57 return x->r_info == y->r_info && x->r_addend == y->r_addend;
58}
59
60static bool duplicate_rela(const Elf64_Rela *rela, int idx)
61{
62 int i;
63 for (i = 0; i < idx; i++) {
64 if (is_rela_equal(&rela[i], &rela[idx]))
65 return true;
66 }
67 return false;
68}
69
70static void count_max_entries(Elf64_Rela *relas, int num,
71 unsigned int *plts, unsigned int *gots)
72{
73 unsigned int type, i;
74
75 for (i = 0; i < num; i++) {
76 type = ELF64_R_TYPE(relas[i].r_info);
77 if (type == R_RISCV_CALL_PLT) {
78 if (!duplicate_rela(relas, i))
79 (*plts)++;
80 } else if (type == R_RISCV_GOT_HI20) {
81 if (!duplicate_rela(relas, i))
82 (*gots)++;
83 }
84 }
85}
86
87int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
88 char *secstrings, struct module *mod)
89{
90 unsigned int num_plts = 0;
91 unsigned int num_gots = 0;
92 int i;
93
94 /*
95 * Find the empty .got and .plt sections.
96 */
97 for (i = 0; i < ehdr->e_shnum; i++) {
98 if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt"))
99 mod->arch.plt.shdr = sechdrs + i;
100 else if (!strcmp(secstrings + sechdrs[i].sh_name, ".got"))
101 mod->arch.got.shdr = sechdrs + i;
102 else if (!strcmp(secstrings + sechdrs[i].sh_name, ".got.plt"))
103 mod->arch.got_plt.shdr = sechdrs + i;
104 }
105
106 if (!mod->arch.plt.shdr) {
107 pr_err("%s: module PLT section(s) missing\n", mod->name);
108 return -ENOEXEC;
109 }
110 if (!mod->arch.got.shdr) {
111 pr_err("%s: module GOT section(s) missing\n", mod->name);
112 return -ENOEXEC;
113 }
114 if (!mod->arch.got_plt.shdr) {
115 pr_err("%s: module GOT.PLT section(s) missing\n", mod->name);
116 return -ENOEXEC;
117 }
118
119 /* Calculate the maxinum number of entries */
120 for (i = 0; i < ehdr->e_shnum; i++) {
121 Elf64_Rela *relas = (void *)ehdr + sechdrs[i].sh_offset;
122 int num_rela = sechdrs[i].sh_size / sizeof(Elf64_Rela);
123 Elf64_Shdr *dst_sec = sechdrs + sechdrs[i].sh_info;
124
125 if (sechdrs[i].sh_type != SHT_RELA)
126 continue;
127
128 /* ignore relocations that operate on non-exec sections */
129 if (!(dst_sec->sh_flags & SHF_EXECINSTR))
130 continue;
131
132 count_max_entries(relas, num_rela, &num_plts, &num_gots);
133 }
134
135 mod->arch.plt.shdr->sh_type = SHT_NOBITS;
136 mod->arch.plt.shdr->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
137 mod->arch.plt.shdr->sh_addralign = L1_CACHE_BYTES;
138 mod->arch.plt.shdr->sh_size = (num_plts + 1) * sizeof(struct plt_entry);
139 mod->arch.plt.num_entries = 0;
140 mod->arch.plt.max_entries = num_plts;
141
142 mod->arch.got.shdr->sh_type = SHT_NOBITS;
143 mod->arch.got.shdr->sh_flags = SHF_ALLOC;
144 mod->arch.got.shdr->sh_addralign = L1_CACHE_BYTES;
145 mod->arch.got.shdr->sh_size = (num_gots + 1) * sizeof(struct got_entry);
146 mod->arch.got.num_entries = 0;
147 mod->arch.got.max_entries = num_gots;
148
149 mod->arch.got_plt.shdr->sh_type = SHT_NOBITS;
150 mod->arch.got_plt.shdr->sh_flags = SHF_ALLOC;
151 mod->arch.got_plt.shdr->sh_addralign = L1_CACHE_BYTES;
152 mod->arch.got_plt.shdr->sh_size = (num_plts + 1) * sizeof(struct got_entry);
153 mod->arch.got_plt.num_entries = 0;
154 mod->arch.got_plt.max_entries = num_plts;
155 return 0;
156}
diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c
index e0f05034fc21..5dddba301d0a 100644
--- a/arch/riscv/kernel/module.c
+++ b/arch/riscv/kernel/module.c
@@ -49,6 +49,39 @@ static int apply_r_riscv_jal_rela(struct module *me, u32 *location,
49 return 0; 49 return 0;
50} 50}
51 51
52static int apply_r_riscv_rcv_branch_rela(struct module *me, u32 *location,
53 Elf_Addr v)
54{
55 s64 offset = (void *)v - (void *)location;
56 u16 imm8 = (offset & 0x100) << (12 - 8);
57 u16 imm7_6 = (offset & 0xc0) >> (6 - 5);
58 u16 imm5 = (offset & 0x20) >> (5 - 2);
59 u16 imm4_3 = (offset & 0x18) << (12 - 5);
60 u16 imm2_1 = (offset & 0x6) << (12 - 10);
61
62 *(u16 *)location = (*(u16 *)location & 0xe383) |
63 imm8 | imm7_6 | imm5 | imm4_3 | imm2_1;
64 return 0;
65}
66
67static int apply_r_riscv_rvc_jump_rela(struct module *me, u32 *location,
68 Elf_Addr v)
69{
70 s64 offset = (void *)v - (void *)location;
71 u16 imm11 = (offset & 0x800) << (12 - 11);
72 u16 imm10 = (offset & 0x400) >> (10 - 8);
73 u16 imm9_8 = (offset & 0x300) << (12 - 11);
74 u16 imm7 = (offset & 0x80) >> (7 - 6);
75 u16 imm6 = (offset & 0x40) << (12 - 11);
76 u16 imm5 = (offset & 0x20) >> (5 - 2);
77 u16 imm4 = (offset & 0x10) << (12 - 5);
78 u16 imm3_1 = (offset & 0xe) << (12 - 10);
79
80 *(u16 *)location = (*(u16 *)location & 0xe003) |
81 imm11 | imm10 | imm9_8 | imm7 | imm6 | imm5 | imm4 | imm3_1;
82 return 0;
83}
84
52static int apply_r_riscv_pcrel_hi20_rela(struct module *me, u32 *location, 85static int apply_r_riscv_pcrel_hi20_rela(struct module *me, u32 *location,
53 Elf_Addr v) 86 Elf_Addr v)
54{ 87{
@@ -92,6 +125,67 @@ static int apply_r_riscv_pcrel_lo12_s_rela(struct module *me, u32 *location,
92 return 0; 125 return 0;
93} 126}
94 127
128static int apply_r_riscv_hi20_rela(struct module *me, u32 *location,
129 Elf_Addr v)
130{
131 s32 hi20;
132
133 if (IS_ENABLED(CMODEL_MEDLOW)) {
134 pr_err(
135 "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
136 me->name, v, location);
137 return -EINVAL;
138 }
139
140 hi20 = ((s32)v + 0x800) & 0xfffff000;
141 *location = (*location & 0xfff) | hi20;
142 return 0;
143}
144
145static int apply_r_riscv_lo12_i_rela(struct module *me, u32 *location,
146 Elf_Addr v)
147{
148 /* Skip medlow checking because of filtering by HI20 already */
149 s32 hi20 = ((s32)v + 0x800) & 0xfffff000;
150 s32 lo12 = ((s32)v - hi20);
151 *location = (*location & 0xfffff) | ((lo12 & 0xfff) << 20);
152 return 0;
153}
154
155static int apply_r_riscv_lo12_s_rela(struct module *me, u32 *location,
156 Elf_Addr v)
157{
158 /* Skip medlow checking because of filtering by HI20 already */
159 s32 hi20 = ((s32)v + 0x800) & 0xfffff000;
160 s32 lo12 = ((s32)v - hi20);
161 u32 imm11_5 = (lo12 & 0xfe0) << (31 - 11);
162 u32 imm4_0 = (lo12 & 0x1f) << (11 - 4);
163 *location = (*location & 0x1fff07f) | imm11_5 | imm4_0;
164 return 0;
165}
166
167static int apply_r_riscv_got_hi20_rela(struct module *me, u32 *location,
168 Elf_Addr v)
169{
170 s64 offset = (void *)v - (void *)location;
171 s32 hi20;
172
173 /* Always emit the got entry */
174 if (IS_ENABLED(CONFIG_MODULE_SECTIONS)) {
175 offset = module_emit_got_entry(me, v);
176 offset = (void *)offset - (void *)location;
177 } else {
178 pr_err(
179 "%s: can not generate the GOT entry for symbol = %016llx from PC = %p\n",
180 me->name, v, location);
181 return -EINVAL;
182 }
183
184 hi20 = (offset + 0x800) & 0xfffff000;
185 *location = (*location & 0xfff) | hi20;
186 return 0;
187}
188
95static int apply_r_riscv_call_plt_rela(struct module *me, u32 *location, 189static int apply_r_riscv_call_plt_rela(struct module *me, u32 *location,
96 Elf_Addr v) 190 Elf_Addr v)
97{ 191{
@@ -100,6 +194,33 @@ static int apply_r_riscv_call_plt_rela(struct module *me, u32 *location,
100 u32 hi20, lo12; 194 u32 hi20, lo12;
101 195
102 if (offset != fill_v) { 196 if (offset != fill_v) {
197 /* Only emit the plt entry if offset over 32-bit range */
198 if (IS_ENABLED(CONFIG_MODULE_SECTIONS)) {
199 offset = module_emit_plt_entry(me, v);
200 offset = (void *)offset - (void *)location;
201 } else {
202 pr_err(
203 "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
204 me->name, v, location);
205 return -EINVAL;
206 }
207 }
208
209 hi20 = (offset + 0x800) & 0xfffff000;
210 lo12 = (offset - hi20) & 0xfff;
211 *location = (*location & 0xfff) | hi20;
212 *(location + 1) = (*(location + 1) & 0xfffff) | (lo12 << 20);
213 return 0;
214}
215
216static int apply_r_riscv_call_rela(struct module *me, u32 *location,
217 Elf_Addr v)
218{
219 s64 offset = (void *)v - (void *)location;
220 s32 fill_v = offset;
221 u32 hi20, lo12;
222
223 if (offset != fill_v) {
103 pr_err( 224 pr_err(
104 "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n", 225 "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
105 me->name, v, location); 226 me->name, v, location);
@@ -119,16 +240,49 @@ static int apply_r_riscv_relax_rela(struct module *me, u32 *location,
119 return 0; 240 return 0;
120} 241}
121 242
243static int apply_r_riscv_align_rela(struct module *me, u32 *location,
244 Elf_Addr v)
245{
246 pr_err(
247 "%s: The unexpected relocation type 'R_RISCV_ALIGN' from PC = %p\n",
248 me->name, location);
249 return -EINVAL;
250}
251
252static int apply_r_riscv_add32_rela(struct module *me, u32 *location,
253 Elf_Addr v)
254{
255 *(u32 *)location += (*(u32 *)v);
256 return 0;
257}
258
259static int apply_r_riscv_sub32_rela(struct module *me, u32 *location,
260 Elf_Addr v)
261{
262 *(u32 *)location -= (*(u32 *)v);
263 return 0;
264}
265
122static int (*reloc_handlers_rela[]) (struct module *me, u32 *location, 266static int (*reloc_handlers_rela[]) (struct module *me, u32 *location,
123 Elf_Addr v) = { 267 Elf_Addr v) = {
124 [R_RISCV_64] = apply_r_riscv_64_rela, 268 [R_RISCV_64] = apply_r_riscv_64_rela,
125 [R_RISCV_BRANCH] = apply_r_riscv_branch_rela, 269 [R_RISCV_BRANCH] = apply_r_riscv_branch_rela,
126 [R_RISCV_JAL] = apply_r_riscv_jal_rela, 270 [R_RISCV_JAL] = apply_r_riscv_jal_rela,
271 [R_RISCV_RVC_BRANCH] = apply_r_riscv_rcv_branch_rela,
272 [R_RISCV_RVC_JUMP] = apply_r_riscv_rvc_jump_rela,
127 [R_RISCV_PCREL_HI20] = apply_r_riscv_pcrel_hi20_rela, 273 [R_RISCV_PCREL_HI20] = apply_r_riscv_pcrel_hi20_rela,
128 [R_RISCV_PCREL_LO12_I] = apply_r_riscv_pcrel_lo12_i_rela, 274 [R_RISCV_PCREL_LO12_I] = apply_r_riscv_pcrel_lo12_i_rela,
129 [R_RISCV_PCREL_LO12_S] = apply_r_riscv_pcrel_lo12_s_rela, 275 [R_RISCV_PCREL_LO12_S] = apply_r_riscv_pcrel_lo12_s_rela,
276 [R_RISCV_HI20] = apply_r_riscv_hi20_rela,
277 [R_RISCV_LO12_I] = apply_r_riscv_lo12_i_rela,
278 [R_RISCV_LO12_S] = apply_r_riscv_lo12_s_rela,
279 [R_RISCV_GOT_HI20] = apply_r_riscv_got_hi20_rela,
130 [R_RISCV_CALL_PLT] = apply_r_riscv_call_plt_rela, 280 [R_RISCV_CALL_PLT] = apply_r_riscv_call_plt_rela,
281 [R_RISCV_CALL] = apply_r_riscv_call_rela,
131 [R_RISCV_RELAX] = apply_r_riscv_relax_rela, 282 [R_RISCV_RELAX] = apply_r_riscv_relax_rela,
283 [R_RISCV_ALIGN] = apply_r_riscv_align_rela,
284 [R_RISCV_ADD32] = apply_r_riscv_add32_rela,
285 [R_RISCV_SUB32] = apply_r_riscv_sub32_rela,
132}; 286};
133 287
134int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, 288int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
@@ -184,25 +338,38 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
184 u64 hi20_loc = 338 u64 hi20_loc =
185 sechdrs[sechdrs[relsec].sh_info].sh_addr 339 sechdrs[sechdrs[relsec].sh_info].sh_addr
186 + rel[j].r_offset; 340 + rel[j].r_offset;
187 /* Find the corresponding HI20 PC-relative relocation entry */ 341 u32 hi20_type = ELF_RISCV_R_TYPE(rel[j].r_info);
188 if (hi20_loc == sym->st_value) { 342
343 /* Find the corresponding HI20 relocation entry */
344 if (hi20_loc == sym->st_value
345 && (hi20_type == R_RISCV_PCREL_HI20
346 || hi20_type == R_RISCV_GOT_HI20)) {
347 s32 hi20, lo12;
189 Elf_Sym *hi20_sym = 348 Elf_Sym *hi20_sym =
190 (Elf_Sym *)sechdrs[symindex].sh_addr 349 (Elf_Sym *)sechdrs[symindex].sh_addr
191 + ELF_RISCV_R_SYM(rel[j].r_info); 350 + ELF_RISCV_R_SYM(rel[j].r_info);
192 u64 hi20_sym_val = 351 u64 hi20_sym_val =
193 hi20_sym->st_value 352 hi20_sym->st_value
194 + rel[j].r_addend; 353 + rel[j].r_addend;
354
195 /* Calculate lo12 */ 355 /* Calculate lo12 */
196 s64 offset = hi20_sym_val - hi20_loc; 356 u64 offset = hi20_sym_val - hi20_loc;
197 s32 hi20 = (offset + 0x800) & 0xfffff000; 357 if (IS_ENABLED(CONFIG_MODULE_SECTIONS)
198 s32 lo12 = offset - hi20; 358 && hi20_type == R_RISCV_GOT_HI20) {
359 offset = module_emit_got_entry(
360 me, hi20_sym_val);
361 offset = offset - hi20_loc;
362 }
363 hi20 = (offset + 0x800) & 0xfffff000;
364 lo12 = offset - hi20;
199 v = lo12; 365 v = lo12;
366
200 break; 367 break;
201 } 368 }
202 } 369 }
203 if (j == sechdrs[relsec].sh_size / sizeof(*rel)) { 370 if (j == sechdrs[relsec].sh_size / sizeof(*rel)) {
204 pr_err( 371 pr_err(
205 "%s: Can not find HI20 PC-relative relocation information\n", 372 "%s: Can not find HI20 relocation information\n",
206 me->name); 373 me->name);
207 return -EINVAL; 374 return -EINVAL;
208 } 375 }
diff --git a/arch/riscv/kernel/module.lds b/arch/riscv/kernel/module.lds
new file mode 100644
index 000000000000..295ecfb341a2
--- /dev/null
+++ b/arch/riscv/kernel/module.lds
@@ -0,0 +1,8 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (C) 2017 Andes Technology Corporation */
3
4SECTIONS {
5 .plt (NOLOAD) : { BYTE(0) }
6 .got (NOLOAD) : { BYTE(0) }
7 .got.plt (NOLOAD) : { BYTE(0) }
8}
diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c
index 559aae781154..a4b1d94371a0 100644
--- a/arch/riscv/kernel/stacktrace.c
+++ b/arch/riscv/kernel/stacktrace.c
@@ -18,6 +18,7 @@
18#include <linux/sched/debug.h> 18#include <linux/sched/debug.h>
19#include <linux/sched/task_stack.h> 19#include <linux/sched/task_stack.h>
20#include <linux/stacktrace.h> 20#include <linux/stacktrace.h>
21#include <linux/ftrace.h>
21 22
22#ifdef CONFIG_FRAME_POINTER 23#ifdef CONFIG_FRAME_POINTER
23 24
@@ -63,7 +64,12 @@ static void notrace walk_stackframe(struct task_struct *task,
63 frame = (struct stackframe *)fp - 1; 64 frame = (struct stackframe *)fp - 1;
64 sp = fp; 65 sp = fp;
65 fp = frame->fp; 66 fp = frame->fp;
67#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
68 pc = ftrace_graph_ret_addr(current, NULL, frame->ra,
69 (unsigned long *)(fp - 8));
70#else
66 pc = frame->ra - 0x4; 71 pc = frame->ra - 0x4;
72#endif
67 } 73 }
68} 74}
69 75
diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl
index c74ecc6504e8..191eb949d52c 100755
--- a/scripts/recordmcount.pl
+++ b/scripts/recordmcount.pl
@@ -368,6 +368,11 @@ if ($arch eq "x86_64") {
368} elsif ($arch eq "microblaze") { 368} elsif ($arch eq "microblaze") {
369 # Microblaze calls '_mcount' instead of plain 'mcount'. 369 # Microblaze calls '_mcount' instead of plain 'mcount'.
370 $mcount_regex = "^\\s*([0-9a-fA-F]+):.*\\s_mcount\$"; 370 $mcount_regex = "^\\s*([0-9a-fA-F]+):.*\\s_mcount\$";
371} elsif ($arch eq "riscv") {
372 $function_regex = "^([0-9a-fA-F]+)\\s+<([^.0-9][0-9a-zA-Z_\\.]+)>:";
373 $mcount_regex = "^\\s*([0-9a-fA-F]+):\\sR_RISCV_CALL\\s_mcount\$";
374 $type = ".quad";
375 $alignment = 2;
371} else { 376} else {
372 die "Arch $arch is not supported with CONFIG_FTRACE_MCOUNT_RECORD"; 377 die "Arch $arch is not supported with CONFIG_FTRACE_MCOUNT_RECORD";
373} 378}
generated by cgit v1.2.2 (git 2.25.0) at 2025-06-01 00:36:33 -0400