aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2017-12-01 19:39:12 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2017-12-01 19:39:12 -0500
commite1ba1c99dad92c5917b22b1047cf36e4426b124a (patch)
treee812f55a2442ad85f810b6877bbd4f5193156b84
parent4b1967c90af473e3a8bec00024758a3e676cea2d (diff)
parent3b62de26cf5ef17340a0e986d3e53eb4f74f96d5 (diff)
Merge tag 'riscv-for-linus-4.15-rc2_cleanups' of git://git.kernel.org/pub/scm/linux/kernel/git/palmer/linux
Pull RISC-V cleanups and ABI fixes from Palmer Dabbelt: "This contains a handful of small cleanups that are a result of feedback that didn't make it into our original patch set, either because the feedback hadn't been given yet, I missed the original emails, or we weren't ready to submit the changes yet. I've been maintaining the various cleanup patch sets I have as their own branches, which I then merged together and signed. Each merge commit has a short summary of the changes, and each branch is based on your latest tag (4.15-rc1, in this case). If this isn't the right way to do this then feel free to suggest something else, but it seems sane to me. Here's a short summary of the changes, roughly in order of how interesting they are. - libgcc.h has been moved from include/lib, where it's the only member, to include/linux. This is meant to avoid tab completion conflicts. - VDSO entries for clock_get/gettimeofday/getcpu have been added. These are simple syscalls now, but we want to let glibc use them from the start so we can make them faster later. - A VDSO entry for instruction cache flushing has been added so userspace can flush the instruction cache. - The VDSO symbol versions for __vdso_cmpxchg{32,64} have been removed, as those VDSO entries don't actually exist. - __io_writes has been corrected to respect the given type. - A new READ_ONCE in arch_spin_is_locked(). - __test_and_op_bit_ord() is now actually ordered. - Various small fixes throughout the tree to enable allmodconfig to build cleanly. - Removal of some dead code in our atomic support headers. - Improvements to various comments in our atomic support headers" * tag 'riscv-for-linus-4.15-rc2_cleanups' of git://git.kernel.org/pub/scm/linux/kernel/git/palmer/linux: (23 commits) RISC-V: __io_writes should respect the length argument move libgcc.h to include/linux RISC-V: Clean up an unused include RISC-V: Allow userspace to flush the instruction cache RISC-V: Flush I$ when making a dirty page executable RISC-V: Add missing include RISC-V: Use define for get_cycles like other architectures RISC-V: Provide stub of setup_profiling_timer() RISC-V: Export some expected symbols for modules RISC-V: move empty_zero_page definition to C and export it RISC-V: io.h: type fixes for warnings RISC-V: use RISCV_{INT,SHORT} instead of {INT,SHORT} for asm macros RISC-V: use generic serial.h RISC-V: remove spin_unlock_wait() RISC-V: `sfence.vma` orderes the instruction cache RISC-V: Add READ_ONCE in arch_spin_is_locked() RISC-V: __test_and_op_bit_ord should be strongly ordered RISC-V: Remove smb_mb__{before,after}_spinlock() RISC-V: Remove __smp_bp__{before,after}_atomic RISC-V: Comment on why {,cmp}xchg is ordered how it is ...
-rw-r--r--arch/riscv/include/asm/Kbuild1
-rw-r--r--arch/riscv/include/asm/asm.h12
-rw-r--r--arch/riscv/include/asm/atomic.h103
-rw-r--r--arch/riscv/include/asm/barrier.h23
-rw-r--r--arch/riscv/include/asm/bitops.h2
-rw-r--r--arch/riscv/include/asm/bug.h6
-rw-r--r--arch/riscv/include/asm/cacheflush.h30
-rw-r--r--arch/riscv/include/asm/io.h18
-rw-r--r--arch/riscv/include/asm/mmu.h4
-rw-r--r--arch/riscv/include/asm/mmu_context.h45
-rw-r--r--arch/riscv/include/asm/pgtable.h58
-rw-r--r--arch/riscv/include/asm/spinlock.h11
-rw-r--r--arch/riscv/include/asm/timex.h3
-rw-r--r--arch/riscv/include/asm/tlbflush.h7
-rw-r--r--arch/riscv/include/asm/vdso-syscalls.h28
-rw-r--r--arch/riscv/include/asm/vdso.h4
-rw-r--r--arch/riscv/kernel/head.S3
-rw-r--r--arch/riscv/kernel/riscv_ksyms.c3
-rw-r--r--arch/riscv/kernel/setup.c5
-rw-r--r--arch/riscv/kernel/smp.c55
-rw-r--r--arch/riscv/kernel/sys_riscv.c33
-rw-r--r--arch/riscv/kernel/syscall_table.c2
-rw-r--r--arch/riscv/kernel/vdso/Makefile7
-rw-r--r--arch/riscv/kernel/vdso/clock_getres.S26
-rw-r--r--arch/riscv/kernel/vdso/clock_gettime.S26
-rw-r--r--arch/riscv/kernel/vdso/flush_icache.S31
-rw-r--r--arch/riscv/kernel/vdso/getcpu.S26
-rw-r--r--arch/riscv/kernel/vdso/gettimeofday.S26
-rw-r--r--arch/riscv/kernel/vdso/vdso.lds.S7
-rw-r--r--arch/riscv/lib/delay.c1
-rw-r--r--arch/riscv/mm/Makefile1
-rw-r--r--arch/riscv/mm/cacheflush.c23
-rw-r--r--arch/riscv/mm/ioremap.c2
-rw-r--r--include/linux/libgcc.h (renamed from include/lib/libgcc.h)0
-rw-r--r--lib/ashldi3.c2
-rw-r--r--lib/ashrdi3.c2
-rw-r--r--lib/cmpdi2.c2
-rw-r--r--lib/lshrdi3.c2
-rw-r--r--lib/muldi3.c2
-rw-r--r--lib/ucmpdi2.c2
40 files changed, 498 insertions, 146 deletions
diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild
index 18158be62a2b..970460a0b492 100644
--- a/arch/riscv/include/asm/Kbuild
+++ b/arch/riscv/include/asm/Kbuild
@@ -40,6 +40,7 @@ generic-y += resource.h
40generic-y += scatterlist.h 40generic-y += scatterlist.h
41generic-y += sections.h 41generic-y += sections.h
42generic-y += sembuf.h 42generic-y += sembuf.h
43generic-y += serial.h
43generic-y += setup.h 44generic-y += setup.h
44generic-y += shmbuf.h 45generic-y += shmbuf.h
45generic-y += shmparam.h 46generic-y += shmparam.h
diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h
index 6cbbb6a68d76..5ad4cb622bed 100644
--- a/arch/riscv/include/asm/asm.h
+++ b/arch/riscv/include/asm/asm.h
@@ -58,17 +58,17 @@
58#endif 58#endif
59 59
60#if (__SIZEOF_INT__ == 4) 60#if (__SIZEOF_INT__ == 4)
61#define INT __ASM_STR(.word) 61#define RISCV_INT __ASM_STR(.word)
62#define SZINT __ASM_STR(4) 62#define RISCV_SZINT __ASM_STR(4)
63#define LGINT __ASM_STR(2) 63#define RISCV_LGINT __ASM_STR(2)
64#else 64#else
65#error "Unexpected __SIZEOF_INT__" 65#error "Unexpected __SIZEOF_INT__"
66#endif 66#endif
67 67
68#if (__SIZEOF_SHORT__ == 2) 68#if (__SIZEOF_SHORT__ == 2)
69#define SHORT __ASM_STR(.half) 69#define RISCV_SHORT __ASM_STR(.half)
70#define SZSHORT __ASM_STR(2) 70#define RISCV_SZSHORT __ASM_STR(2)
71#define LGSHORT __ASM_STR(1) 71#define RISCV_LGSHORT __ASM_STR(1)
72#else 72#else
73#error "Unexpected __SIZEOF_SHORT__" 73#error "Unexpected __SIZEOF_SHORT__"
74#endif 74#endif
diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h
index e2e37c57cbeb..e65d1cd89e28 100644
--- a/arch/riscv/include/asm/atomic.h
+++ b/arch/riscv/include/asm/atomic.h
@@ -50,30 +50,30 @@ static __always_inline void atomic64_set(atomic64_t *v, long i)
50 * have the AQ or RL bits set. These don't return anything, so there's only 50 * have the AQ or RL bits set. These don't return anything, so there's only
51 * one version to worry about. 51 * one version to worry about.
52 */ 52 */
53#define ATOMIC_OP(op, asm_op, c_op, I, asm_type, c_type, prefix) \ 53#define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \
54static __always_inline void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ 54static __always_inline void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
55{ \ 55{ \
56 __asm__ __volatile__ ( \ 56 __asm__ __volatile__ ( \
57 "amo" #asm_op "." #asm_type " zero, %1, %0" \ 57 "amo" #asm_op "." #asm_type " zero, %1, %0" \
58 : "+A" (v->counter) \ 58 : "+A" (v->counter) \
59 : "r" (I) \ 59 : "r" (I) \
60 : "memory"); \ 60 : "memory"); \
61} 61}
62 62
63#ifdef CONFIG_GENERIC_ATOMIC64 63#ifdef CONFIG_GENERIC_ATOMIC64
64#define ATOMIC_OPS(op, asm_op, c_op, I) \ 64#define ATOMIC_OPS(op, asm_op, I) \
65 ATOMIC_OP (op, asm_op, c_op, I, w, int, ) 65 ATOMIC_OP (op, asm_op, I, w, int, )
66#else 66#else
67#define ATOMIC_OPS(op, asm_op, c_op, I) \ 67#define ATOMIC_OPS(op, asm_op, I) \
68 ATOMIC_OP (op, asm_op, c_op, I, w, int, ) \ 68 ATOMIC_OP (op, asm_op, I, w, int, ) \
69 ATOMIC_OP (op, asm_op, c_op, I, d, long, 64) 69 ATOMIC_OP (op, asm_op, I, d, long, 64)
70#endif 70#endif
71 71
72ATOMIC_OPS(add, add, +, i) 72ATOMIC_OPS(add, add, i)
73ATOMIC_OPS(sub, add, +, -i) 73ATOMIC_OPS(sub, add, -i)
74ATOMIC_OPS(and, and, &, i) 74ATOMIC_OPS(and, and, i)
75ATOMIC_OPS( or, or, |, i) 75ATOMIC_OPS( or, or, i)
76ATOMIC_OPS(xor, xor, ^, i) 76ATOMIC_OPS(xor, xor, i)
77 77
78#undef ATOMIC_OP 78#undef ATOMIC_OP
79#undef ATOMIC_OPS 79#undef ATOMIC_OPS
@@ -83,7 +83,7 @@ ATOMIC_OPS(xor, xor, ^, i)
83 * There's two flavors of these: the arithmatic ops have both fetch and return 83 * There's two flavors of these: the arithmatic ops have both fetch and return
84 * versions, while the logical ops only have fetch versions. 84 * versions, while the logical ops only have fetch versions.
85 */ 85 */
86#define ATOMIC_FETCH_OP(op, asm_op, c_op, I, asm_or, c_or, asm_type, c_type, prefix) \ 86#define ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, asm_type, c_type, prefix) \
87static __always_inline c_type atomic##prefix##_fetch_##op##c_or(c_type i, atomic##prefix##_t *v) \ 87static __always_inline c_type atomic##prefix##_fetch_##op##c_or(c_type i, atomic##prefix##_t *v) \
88{ \ 88{ \
89 register c_type ret; \ 89 register c_type ret; \
@@ -103,13 +103,13 @@ static __always_inline c_type atomic##prefix##_##op##_return##c_or(c_type i, ato
103 103
104#ifdef CONFIG_GENERIC_ATOMIC64 104#ifdef CONFIG_GENERIC_ATOMIC64
105#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or) \ 105#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or) \
106 ATOMIC_FETCH_OP (op, asm_op, c_op, I, asm_or, c_or, w, int, ) \ 106 ATOMIC_FETCH_OP (op, asm_op, I, asm_or, c_or, w, int, ) \
107 ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, w, int, ) 107 ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, w, int, )
108#else 108#else
109#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or) \ 109#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or) \
110 ATOMIC_FETCH_OP (op, asm_op, c_op, I, asm_or, c_or, w, int, ) \ 110 ATOMIC_FETCH_OP (op, asm_op, I, asm_or, c_or, w, int, ) \
111 ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, w, int, ) \ 111 ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, w, int, ) \
112 ATOMIC_FETCH_OP (op, asm_op, c_op, I, asm_or, c_or, d, long, 64) \ 112 ATOMIC_FETCH_OP (op, asm_op, I, asm_or, c_or, d, long, 64) \
113 ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, d, long, 64) 113 ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, d, long, 64)
114#endif 114#endif
115 115
@@ -126,28 +126,28 @@ ATOMIC_OPS(sub, add, +, -i, .aqrl, )
126#undef ATOMIC_OPS 126#undef ATOMIC_OPS
127 127
128#ifdef CONFIG_GENERIC_ATOMIC64 128#ifdef CONFIG_GENERIC_ATOMIC64
129#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or) \ 129#define ATOMIC_OPS(op, asm_op, I, asm_or, c_or) \
130 ATOMIC_FETCH_OP(op, asm_op, c_op, I, asm_or, c_or, w, int, ) 130 ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, w, int, )
131#else 131#else
132#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or) \ 132#define ATOMIC_OPS(op, asm_op, I, asm_or, c_or) \
133 ATOMIC_FETCH_OP(op, asm_op, c_op, I, asm_or, c_or, w, int, ) \ 133 ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, w, int, ) \
134 ATOMIC_FETCH_OP(op, asm_op, c_op, I, asm_or, c_or, d, long, 64) 134 ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, d, long, 64)
135#endif 135#endif
136 136
137ATOMIC_OPS(and, and, &, i, , _relaxed) 137ATOMIC_OPS(and, and, i, , _relaxed)
138ATOMIC_OPS(and, and, &, i, .aq , _acquire) 138ATOMIC_OPS(and, and, i, .aq , _acquire)
139ATOMIC_OPS(and, and, &, i, .rl , _release) 139ATOMIC_OPS(and, and, i, .rl , _release)
140ATOMIC_OPS(and, and, &, i, .aqrl, ) 140ATOMIC_OPS(and, and, i, .aqrl, )
141 141
142ATOMIC_OPS( or, or, |, i, , _relaxed) 142ATOMIC_OPS( or, or, i, , _relaxed)
143ATOMIC_OPS( or, or, |, i, .aq , _acquire) 143ATOMIC_OPS( or, or, i, .aq , _acquire)
144ATOMIC_OPS( or, or, |, i, .rl , _release) 144ATOMIC_OPS( or, or, i, .rl , _release)
145ATOMIC_OPS( or, or, |, i, .aqrl, ) 145ATOMIC_OPS( or, or, i, .aqrl, )
146 146
147ATOMIC_OPS(xor, xor, ^, i, , _relaxed) 147ATOMIC_OPS(xor, xor, i, , _relaxed)
148ATOMIC_OPS(xor, xor, ^, i, .aq , _acquire) 148ATOMIC_OPS(xor, xor, i, .aq , _acquire)
149ATOMIC_OPS(xor, xor, ^, i, .rl , _release) 149ATOMIC_OPS(xor, xor, i, .rl , _release)
150ATOMIC_OPS(xor, xor, ^, i, .aqrl, ) 150ATOMIC_OPS(xor, xor, i, .aqrl, )
151 151
152#undef ATOMIC_OPS 152#undef ATOMIC_OPS
153 153
@@ -182,13 +182,13 @@ ATOMIC_OPS(add_negative, add, <, 0)
182#undef ATOMIC_OP 182#undef ATOMIC_OP
183#undef ATOMIC_OPS 183#undef ATOMIC_OPS
184 184
185#define ATOMIC_OP(op, func_op, c_op, I, c_type, prefix) \ 185#define ATOMIC_OP(op, func_op, I, c_type, prefix) \
186static __always_inline void atomic##prefix##_##op(atomic##prefix##_t *v) \ 186static __always_inline void atomic##prefix##_##op(atomic##prefix##_t *v) \
187{ \ 187{ \
188 atomic##prefix##_##func_op(I, v); \ 188 atomic##prefix##_##func_op(I, v); \
189} 189}
190 190
191#define ATOMIC_FETCH_OP(op, func_op, c_op, I, c_type, prefix) \ 191#define ATOMIC_FETCH_OP(op, func_op, I, c_type, prefix) \
192static __always_inline c_type atomic##prefix##_fetch_##op(atomic##prefix##_t *v) \ 192static __always_inline c_type atomic##prefix##_fetch_##op(atomic##prefix##_t *v) \
193{ \ 193{ \
194 return atomic##prefix##_fetch_##func_op(I, v); \ 194 return atomic##prefix##_fetch_##func_op(I, v); \
@@ -202,16 +202,16 @@ static __always_inline c_type atomic##prefix##_##op##_return(atomic##prefix##_t
202 202
203#ifdef CONFIG_GENERIC_ATOMIC64 203#ifdef CONFIG_GENERIC_ATOMIC64
204#define ATOMIC_OPS(op, asm_op, c_op, I) \ 204#define ATOMIC_OPS(op, asm_op, c_op, I) \
205 ATOMIC_OP (op, asm_op, c_op, I, int, ) \ 205 ATOMIC_OP (op, asm_op, I, int, ) \
206 ATOMIC_FETCH_OP (op, asm_op, c_op, I, int, ) \ 206 ATOMIC_FETCH_OP (op, asm_op, I, int, ) \
207 ATOMIC_OP_RETURN(op, asm_op, c_op, I, int, ) 207 ATOMIC_OP_RETURN(op, asm_op, c_op, I, int, )
208#else 208#else
209#define ATOMIC_OPS(op, asm_op, c_op, I) \ 209#define ATOMIC_OPS(op, asm_op, c_op, I) \
210 ATOMIC_OP (op, asm_op, c_op, I, int, ) \ 210 ATOMIC_OP (op, asm_op, I, int, ) \
211 ATOMIC_FETCH_OP (op, asm_op, c_op, I, int, ) \ 211 ATOMIC_FETCH_OP (op, asm_op, I, int, ) \
212 ATOMIC_OP_RETURN(op, asm_op, c_op, I, int, ) \ 212 ATOMIC_OP_RETURN(op, asm_op, c_op, I, int, ) \
213 ATOMIC_OP (op, asm_op, c_op, I, long, 64) \ 213 ATOMIC_OP (op, asm_op, I, long, 64) \
214 ATOMIC_FETCH_OP (op, asm_op, c_op, I, long, 64) \ 214 ATOMIC_FETCH_OP (op, asm_op, I, long, 64) \
215 ATOMIC_OP_RETURN(op, asm_op, c_op, I, long, 64) 215 ATOMIC_OP_RETURN(op, asm_op, c_op, I, long, 64)
216#endif 216#endif
217 217
@@ -300,8 +300,13 @@ static __always_inline long atomic64_inc_not_zero(atomic64_t *v)
300 300
301/* 301/*
302 * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as 302 * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as
303 * {cmp,}xchg and the operations that return, so they need a barrier. We just 303 * {cmp,}xchg and the operations that return, so they need a barrier.
304 * use the other implementations directly. 304 */
305/*
306 * FIXME: atomic_cmpxchg_{acquire,release,relaxed} are all implemented by
307 * assigning the same barrier to both the LR and SC operations, but that might
308 * not make any sense. We're waiting on a memory model specification to
309 * determine exactly what the right thing to do is here.
305 */ 310 */
306#define ATOMIC_OP(c_t, prefix, c_or, size, asm_or) \ 311#define ATOMIC_OP(c_t, prefix, c_or, size, asm_or) \
307static __always_inline c_t atomic##prefix##_cmpxchg##c_or(atomic##prefix##_t *v, c_t o, c_t n) \ 312static __always_inline c_t atomic##prefix##_cmpxchg##c_or(atomic##prefix##_t *v, c_t o, c_t n) \
diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h
index 183534b7c39b..773c4e039cd7 100644
--- a/arch/riscv/include/asm/barrier.h
+++ b/arch/riscv/include/asm/barrier.h
@@ -38,29 +38,6 @@
38#define smp_rmb() RISCV_FENCE(r,r) 38#define smp_rmb() RISCV_FENCE(r,r)
39#define smp_wmb() RISCV_FENCE(w,w) 39#define smp_wmb() RISCV_FENCE(w,w)
40 40
41/*
42 * These fences exist to enforce ordering around the relaxed AMOs. The
43 * documentation defines that
44 * "
45 * atomic_fetch_add();
46 * is equivalent to:
47 * smp_mb__before_atomic();
48 * atomic_fetch_add_relaxed();
49 * smp_mb__after_atomic();
50 * "
51 * So we emit full fences on both sides.
52 */
53#define __smb_mb__before_atomic() smp_mb()
54#define __smb_mb__after_atomic() smp_mb()
55
56/*
57 * These barriers prevent accesses performed outside a spinlock from being moved
58 * inside a spinlock. Since RISC-V sets the aq/rl bits on our spinlock only
59 * enforce release consistency, we need full fences here.
60 */
61#define smb_mb__before_spinlock() smp_mb()
62#define smb_mb__after_spinlock() smp_mb()
63
64#include <asm-generic/barrier.h> 41#include <asm-generic/barrier.h>
65 42
66#endif /* __ASSEMBLY__ */ 43#endif /* __ASSEMBLY__ */
diff --git a/arch/riscv/include/asm/bitops.h b/arch/riscv/include/asm/bitops.h
index 7c281ef1d583..f30daf26f08f 100644
--- a/arch/riscv/include/asm/bitops.h
+++ b/arch/riscv/include/asm/bitops.h
@@ -67,7 +67,7 @@
67 : "memory"); 67 : "memory");
68 68
69#define __test_and_op_bit(op, mod, nr, addr) \ 69#define __test_and_op_bit(op, mod, nr, addr) \
70 __test_and_op_bit_ord(op, mod, nr, addr, ) 70 __test_and_op_bit_ord(op, mod, nr, addr, .aqrl)
71#define __op_bit(op, mod, nr, addr) \ 71#define __op_bit(op, mod, nr, addr) \
72 __op_bit_ord(op, mod, nr, addr, ) 72 __op_bit_ord(op, mod, nr, addr, )
73 73
diff --git a/arch/riscv/include/asm/bug.h b/arch/riscv/include/asm/bug.h
index c3e13764a943..bfc7f099ab1f 100644
--- a/arch/riscv/include/asm/bug.h
+++ b/arch/riscv/include/asm/bug.h
@@ -27,8 +27,8 @@
27typedef u32 bug_insn_t; 27typedef u32 bug_insn_t;
28 28
29#ifdef CONFIG_GENERIC_BUG_RELATIVE_POINTERS 29#ifdef CONFIG_GENERIC_BUG_RELATIVE_POINTERS
30#define __BUG_ENTRY_ADDR INT " 1b - 2b" 30#define __BUG_ENTRY_ADDR RISCV_INT " 1b - 2b"
31#define __BUG_ENTRY_FILE INT " %0 - 2b" 31#define __BUG_ENTRY_FILE RISCV_INT " %0 - 2b"
32#else 32#else
33#define __BUG_ENTRY_ADDR RISCV_PTR " 1b" 33#define __BUG_ENTRY_ADDR RISCV_PTR " 1b"
34#define __BUG_ENTRY_FILE RISCV_PTR " %0" 34#define __BUG_ENTRY_FILE RISCV_PTR " %0"
@@ -38,7 +38,7 @@ typedef u32 bug_insn_t;
38#define __BUG_ENTRY \ 38#define __BUG_ENTRY \
39 __BUG_ENTRY_ADDR "\n\t" \ 39 __BUG_ENTRY_ADDR "\n\t" \
40 __BUG_ENTRY_FILE "\n\t" \ 40 __BUG_ENTRY_FILE "\n\t" \
41 SHORT " %1" 41 RISCV_SHORT " %1"
42#else 42#else
43#define __BUG_ENTRY \ 43#define __BUG_ENTRY \
44 __BUG_ENTRY_ADDR 44 __BUG_ENTRY_ADDR
diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h
index 0595585013b0..efd89a88d2d0 100644
--- a/arch/riscv/include/asm/cacheflush.h
+++ b/arch/riscv/include/asm/cacheflush.h
@@ -18,22 +18,44 @@
18 18
19#undef flush_icache_range 19#undef flush_icache_range
20#undef flush_icache_user_range 20#undef flush_icache_user_range
21#undef flush_dcache_page
21 22
22static inline void local_flush_icache_all(void) 23static inline void local_flush_icache_all(void)
23{ 24{
24 asm volatile ("fence.i" ::: "memory"); 25 asm volatile ("fence.i" ::: "memory");
25} 26}
26 27
28#define PG_dcache_clean PG_arch_1
29
30static inline void flush_dcache_page(struct page *page)
31{
32 if (test_bit(PG_dcache_clean, &page->flags))
33 clear_bit(PG_dcache_clean, &page->flags);
34}
35
36/*
37 * RISC-V doesn't have an instruction to flush parts of the instruction cache,
38 * so instead we just flush the whole thing.
39 */
40#define flush_icache_range(start, end) flush_icache_all()
41#define flush_icache_user_range(vma, pg, addr, len) flush_icache_all()
42
27#ifndef CONFIG_SMP 43#ifndef CONFIG_SMP
28 44
29#define flush_icache_range(start, end) local_flush_icache_all() 45#define flush_icache_all() local_flush_icache_all()
30#define flush_icache_user_range(vma, pg, addr, len) local_flush_icache_all() 46#define flush_icache_mm(mm, local) flush_icache_all()
31 47
32#else /* CONFIG_SMP */ 48#else /* CONFIG_SMP */
33 49
34#define flush_icache_range(start, end) sbi_remote_fence_i(0) 50#define flush_icache_all() sbi_remote_fence_i(0)
35#define flush_icache_user_range(vma, pg, addr, len) sbi_remote_fence_i(0) 51void flush_icache_mm(struct mm_struct *mm, bool local);
36 52
37#endif /* CONFIG_SMP */ 53#endif /* CONFIG_SMP */
38 54
55/*
56 * Bits in sys_riscv_flush_icache()'s flags argument.
57 */
58#define SYS_RISCV_FLUSH_ICACHE_LOCAL 1UL
59#define SYS_RISCV_FLUSH_ICACHE_ALL (SYS_RISCV_FLUSH_ICACHE_LOCAL)
60
39#endif /* _ASM_RISCV_CACHEFLUSH_H */ 61#endif /* _ASM_RISCV_CACHEFLUSH_H */
diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h
index c1f32cfcc79b..a82ce599b639 100644
--- a/arch/riscv/include/asm/io.h
+++ b/arch/riscv/include/asm/io.h
@@ -19,6 +19,8 @@
19#ifndef _ASM_RISCV_IO_H 19#ifndef _ASM_RISCV_IO_H
20#define _ASM_RISCV_IO_H 20#define _ASM_RISCV_IO_H
21 21
22#include <linux/types.h>
23
22#ifdef CONFIG_MMU 24#ifdef CONFIG_MMU
23 25
24extern void __iomem *ioremap(phys_addr_t offset, unsigned long size); 26extern void __iomem *ioremap(phys_addr_t offset, unsigned long size);
@@ -32,7 +34,7 @@ extern void __iomem *ioremap(phys_addr_t offset, unsigned long size);
32#define ioremap_wc(addr, size) ioremap((addr), (size)) 34#define ioremap_wc(addr, size) ioremap((addr), (size))
33#define ioremap_wt(addr, size) ioremap((addr), (size)) 35#define ioremap_wt(addr, size) ioremap((addr), (size))
34 36
35extern void iounmap(void __iomem *addr); 37extern void iounmap(volatile void __iomem *addr);
36 38
37#endif /* CONFIG_MMU */ 39#endif /* CONFIG_MMU */
38 40
@@ -250,7 +252,7 @@ static inline u64 __raw_readq(const volatile void __iomem *addr)
250 const ctype *buf = buffer; \ 252 const ctype *buf = buffer; \
251 \ 253 \
252 do { \ 254 do { \
253 __raw_writeq(*buf++, addr); \ 255 __raw_write ## len(*buf++, addr); \
254 } while (--count); \ 256 } while (--count); \
255 } \ 257 } \
256 afence; \ 258 afence; \
@@ -266,9 +268,9 @@ __io_reads_ins(reads, u32, l, __io_br(), __io_ar())
266__io_reads_ins(ins, u8, b, __io_pbr(), __io_par()) 268__io_reads_ins(ins, u8, b, __io_pbr(), __io_par())
267__io_reads_ins(ins, u16, w, __io_pbr(), __io_par()) 269__io_reads_ins(ins, u16, w, __io_pbr(), __io_par())
268__io_reads_ins(ins, u32, l, __io_pbr(), __io_par()) 270__io_reads_ins(ins, u32, l, __io_pbr(), __io_par())
269#define insb(addr, buffer, count) __insb((void __iomem *)addr, buffer, count) 271#define insb(addr, buffer, count) __insb((void __iomem *)(long)addr, buffer, count)
270#define insw(addr, buffer, count) __insw((void __iomem *)addr, buffer, count) 272#define insw(addr, buffer, count) __insw((void __iomem *)(long)addr, buffer, count)
271#define insl(addr, buffer, count) __insl((void __iomem *)addr, buffer, count) 273#define insl(addr, buffer, count) __insl((void __iomem *)(long)addr, buffer, count)
272 274
273__io_writes_outs(writes, u8, b, __io_bw(), __io_aw()) 275__io_writes_outs(writes, u8, b, __io_bw(), __io_aw())
274__io_writes_outs(writes, u16, w, __io_bw(), __io_aw()) 276__io_writes_outs(writes, u16, w, __io_bw(), __io_aw())
@@ -280,9 +282,9 @@ __io_writes_outs(writes, u32, l, __io_bw(), __io_aw())
280__io_writes_outs(outs, u8, b, __io_pbw(), __io_paw()) 282__io_writes_outs(outs, u8, b, __io_pbw(), __io_paw())
281__io_writes_outs(outs, u16, w, __io_pbw(), __io_paw()) 283__io_writes_outs(outs, u16, w, __io_pbw(), __io_paw())
282__io_writes_outs(outs, u32, l, __io_pbw(), __io_paw()) 284__io_writes_outs(outs, u32, l, __io_pbw(), __io_paw())
283#define outsb(addr, buffer, count) __outsb((void __iomem *)addr, buffer, count) 285#define outsb(addr, buffer, count) __outsb((void __iomem *)(long)addr, buffer, count)
284#define outsw(addr, buffer, count) __outsw((void __iomem *)addr, buffer, count) 286#define outsw(addr, buffer, count) __outsw((void __iomem *)(long)addr, buffer, count)
285#define outsl(addr, buffer, count) __outsl((void __iomem *)addr, buffer, count) 287#define outsl(addr, buffer, count) __outsl((void __iomem *)(long)addr, buffer, count)
286 288
287#ifdef CONFIG_64BIT 289#ifdef CONFIG_64BIT
288__io_reads_ins(reads, u64, q, __io_br(), __io_ar()) 290__io_reads_ins(reads, u64, q, __io_br(), __io_ar())
diff --git a/arch/riscv/include/asm/mmu.h b/arch/riscv/include/asm/mmu.h
index 66805cba9a27..5df2dccdba12 100644
--- a/arch/riscv/include/asm/mmu.h
+++ b/arch/riscv/include/asm/mmu.h
@@ -19,6 +19,10 @@
19 19
20typedef struct { 20typedef struct {
21 void *vdso; 21 void *vdso;
22#ifdef CONFIG_SMP
23 /* A local icache flush is needed before user execution can resume. */
24 cpumask_t icache_stale_mask;
25#endif
22} mm_context_t; 26} mm_context_t;
23 27
24#endif /* __ASSEMBLY__ */ 28#endif /* __ASSEMBLY__ */
diff --git a/arch/riscv/include/asm/mmu_context.h b/arch/riscv/include/asm/mmu_context.h
index de1fc1631fc4..97424834dce2 100644
--- a/arch/riscv/include/asm/mmu_context.h
+++ b/arch/riscv/include/asm/mmu_context.h
@@ -1,5 +1,6 @@
1/* 1/*
2 * Copyright (C) 2012 Regents of the University of California 2 * Copyright (C) 2012 Regents of the University of California
3 * Copyright (C) 2017 SiFive
3 * 4 *
4 * This program is free software; you can redistribute it and/or 5 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License 6 * modify it under the terms of the GNU General Public License
@@ -14,11 +15,13 @@
14#ifndef _ASM_RISCV_MMU_CONTEXT_H 15#ifndef _ASM_RISCV_MMU_CONTEXT_H
15#define _ASM_RISCV_MMU_CONTEXT_H 16#define _ASM_RISCV_MMU_CONTEXT_H
16 17
18#include <linux/mm_types.h>
17#include <asm-generic/mm_hooks.h> 19#include <asm-generic/mm_hooks.h>
18 20
19#include <linux/mm.h> 21#include <linux/mm.h>
20#include <linux/sched.h> 22#include <linux/sched.h>
21#include <asm/tlbflush.h> 23#include <asm/tlbflush.h>
24#include <asm/cacheflush.h>
22 25
23static inline void enter_lazy_tlb(struct mm_struct *mm, 26static inline void enter_lazy_tlb(struct mm_struct *mm,
24 struct task_struct *task) 27 struct task_struct *task)
@@ -46,12 +49,54 @@ static inline void set_pgdir(pgd_t *pgd)
46 csr_write(sptbr, virt_to_pfn(pgd) | SPTBR_MODE); 49 csr_write(sptbr, virt_to_pfn(pgd) | SPTBR_MODE);
47} 50}
48 51
52/*
53 * When necessary, performs a deferred icache flush for the given MM context,
54 * on the local CPU. RISC-V has no direct mechanism for instruction cache
55 * shoot downs, so instead we send an IPI that informs the remote harts they
56 * need to flush their local instruction caches. To avoid pathologically slow
57 * behavior in a common case (a bunch of single-hart processes on a many-hart
58 * machine, ie 'make -j') we avoid the IPIs for harts that are not currently
59 * executing a MM context and instead schedule a deferred local instruction
60 * cache flush to be performed before execution resumes on each hart. This
61 * actually performs that local instruction cache flush, which implicitly only
62 * refers to the current hart.
63 */
64static inline void flush_icache_deferred(struct mm_struct *mm)
65{
66#ifdef CONFIG_SMP
67 unsigned int cpu = smp_processor_id();
68 cpumask_t *mask = &mm->context.icache_stale_mask;
69
70 if (cpumask_test_cpu(cpu, mask)) {
71 cpumask_clear_cpu(cpu, mask);
72 /*
73 * Ensure the remote hart's writes are visible to this hart.
74 * This pairs with a barrier in flush_icache_mm.
75 */
76 smp_mb();
77 local_flush_icache_all();
78 }
79#endif
80}
81
49static inline void switch_mm(struct mm_struct *prev, 82static inline void switch_mm(struct mm_struct *prev,
50 struct mm_struct *next, struct task_struct *task) 83 struct mm_struct *next, struct task_struct *task)
51{ 84{
52 if (likely(prev != next)) { 85 if (likely(prev != next)) {
86 /*
87 * Mark the current MM context as inactive, and the next as
88 * active. This is at least used by the icache flushing
89 * routines in order to determine who should
90 */
91 unsigned int cpu = smp_processor_id();
92
93 cpumask_clear_cpu(cpu, mm_cpumask(prev));
94 cpumask_set_cpu(cpu, mm_cpumask(next));
95
53 set_pgdir(next->pgd); 96 set_pgdir(next->pgd);
54 local_flush_tlb_all(); 97 local_flush_tlb_all();
98
99 flush_icache_deferred(next);
55 } 100 }
56} 101}
57 102
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 3399257780b2..2cbd92ed1629 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -178,28 +178,6 @@ static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long addr)
178#define pte_offset_map(dir, addr) pte_offset_kernel((dir), (addr)) 178#define pte_offset_map(dir, addr) pte_offset_kernel((dir), (addr))
179#define pte_unmap(pte) ((void)(pte)) 179#define pte_unmap(pte) ((void)(pte))
180 180
181/*
182 * Certain architectures need to do special things when PTEs within
183 * a page table are directly modified. Thus, the following hook is
184 * made available.
185 */
186static inline void set_pte(pte_t *ptep, pte_t pteval)
187{
188 *ptep = pteval;
189}
190
191static inline void set_pte_at(struct mm_struct *mm,
192 unsigned long addr, pte_t *ptep, pte_t pteval)
193{
194 set_pte(ptep, pteval);
195}
196
197static inline void pte_clear(struct mm_struct *mm,
198 unsigned long addr, pte_t *ptep)
199{
200 set_pte_at(mm, addr, ptep, __pte(0));
201}
202
203static inline int pte_present(pte_t pte) 181static inline int pte_present(pte_t pte)
204{ 182{
205 return (pte_val(pte) & _PAGE_PRESENT); 183 return (pte_val(pte) & _PAGE_PRESENT);
@@ -210,21 +188,22 @@ static inline int pte_none(pte_t pte)
210 return (pte_val(pte) == 0); 188 return (pte_val(pte) == 0);
211} 189}
212 190
213/* static inline int pte_read(pte_t pte) */
214
215static inline int pte_write(pte_t pte) 191static inline int pte_write(pte_t pte)
216{ 192{
217 return pte_val(pte) & _PAGE_WRITE; 193 return pte_val(pte) & _PAGE_WRITE;
218} 194}
219 195
196static inline int pte_exec(pte_t pte)
197{
198 return pte_val(pte) & _PAGE_EXEC;
199}
200
220static inline int pte_huge(pte_t pte) 201static inline int pte_huge(pte_t pte)
221{ 202{
222 return pte_present(pte) 203 return pte_present(pte)
223 && (pte_val(pte) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)); 204 && (pte_val(pte) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC));
224} 205}
225 206
226/* static inline int pte_exec(pte_t pte) */
227
228static inline int pte_dirty(pte_t pte) 207static inline int pte_dirty(pte_t pte)
229{ 208{
230 return pte_val(pte) & _PAGE_DIRTY; 209 return pte_val(pte) & _PAGE_DIRTY;
@@ -311,6 +290,33 @@ static inline int pte_same(pte_t pte_a, pte_t pte_b)
311 return pte_val(pte_a) == pte_val(pte_b); 290 return pte_val(pte_a) == pte_val(pte_b);
312} 291}
313 292
293/*
294 * Certain architectures need to do special things when PTEs within
295 * a page table are directly modified. Thus, the following hook is
296 * made available.
297 */
298static inline void set_pte(pte_t *ptep, pte_t pteval)
299{
300 *ptep = pteval;
301}
302
303void flush_icache_pte(pte_t pte);
304
305static inline void set_pte_at(struct mm_struct *mm,
306 unsigned long addr, pte_t *ptep, pte_t pteval)
307{
308 if (pte_present(pteval) && pte_exec(pteval))
309 flush_icache_pte(pteval);
310
311 set_pte(ptep, pteval);
312}
313
314static inline void pte_clear(struct mm_struct *mm,
315 unsigned long addr, pte_t *ptep)
316{
317 set_pte_at(mm, addr, ptep, __pte(0));
318}
319
314#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS 320#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
315static inline int ptep_set_access_flags(struct vm_area_struct *vma, 321static inline int ptep_set_access_flags(struct vm_area_struct *vma,
316 unsigned long address, pte_t *ptep, 322 unsigned long address, pte_t *ptep,
diff --git a/arch/riscv/include/asm/spinlock.h b/arch/riscv/include/asm/spinlock.h
index 04c71d938afd..2fd27e8ef1fd 100644
--- a/arch/riscv/include/asm/spinlock.h
+++ b/arch/riscv/include/asm/spinlock.h
@@ -24,7 +24,7 @@
24 24
25/* FIXME: Replace this with a ticket lock, like MIPS. */ 25/* FIXME: Replace this with a ticket lock, like MIPS. */
26 26
27#define arch_spin_is_locked(x) ((x)->lock != 0) 27#define arch_spin_is_locked(x) (READ_ONCE((x)->lock) != 0)
28 28
29static inline void arch_spin_unlock(arch_spinlock_t *lock) 29static inline void arch_spin_unlock(arch_spinlock_t *lock)
30{ 30{
@@ -58,15 +58,6 @@ static inline void arch_spin_lock(arch_spinlock_t *lock)
58 } 58 }
59} 59}
60 60
61static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
62{
63 smp_rmb();
64 do {
65 cpu_relax();
66 } while (arch_spin_is_locked(lock));
67 smp_acquire__after_ctrl_dep();
68}
69
70/***********************************************************/ 61/***********************************************************/
71 62
72static inline void arch_read_lock(arch_rwlock_t *lock) 63static inline void arch_read_lock(arch_rwlock_t *lock)
diff --git a/arch/riscv/include/asm/timex.h b/arch/riscv/include/asm/timex.h
index 3df4932d8964..2f26989cb864 100644
--- a/arch/riscv/include/asm/timex.h
+++ b/arch/riscv/include/asm/timex.h
@@ -18,7 +18,7 @@
18 18
19typedef unsigned long cycles_t; 19typedef unsigned long cycles_t;
20 20
21static inline cycles_t get_cycles(void) 21static inline cycles_t get_cycles_inline(void)
22{ 22{
23 cycles_t n; 23 cycles_t n;
24 24
@@ -27,6 +27,7 @@ static inline cycles_t get_cycles(void)
27 : "=r" (n)); 27 : "=r" (n));
28 return n; 28 return n;
29} 29}
30#define get_cycles get_cycles_inline
30 31
31#ifdef CONFIG_64BIT 32#ifdef CONFIG_64BIT
32static inline uint64_t get_cycles64(void) 33static inline uint64_t get_cycles64(void)
diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h
index 5ee4ae370b5e..715b0f10af58 100644
--- a/arch/riscv/include/asm/tlbflush.h
+++ b/arch/riscv/include/asm/tlbflush.h
@@ -17,7 +17,12 @@
17 17
18#ifdef CONFIG_MMU 18#ifdef CONFIG_MMU
19 19
20/* Flush entire local TLB */ 20#include <linux/mm_types.h>
21
22/*
23 * Flush entire local TLB. 'sfence.vma' implicitly fences with the instruction
24 * cache as well, so a 'fence.i' is not necessary.
25 */
21static inline void local_flush_tlb_all(void) 26static inline void local_flush_tlb_all(void)
22{ 27{
23 __asm__ __volatile__ ("sfence.vma" : : : "memory"); 28 __asm__ __volatile__ ("sfence.vma" : : : "memory");
diff --git a/arch/riscv/include/asm/vdso-syscalls.h b/arch/riscv/include/asm/vdso-syscalls.h
new file mode 100644
index 000000000000..a2ccf1894929
--- /dev/null
+++ b/arch/riscv/include/asm/vdso-syscalls.h
@@ -0,0 +1,28 @@
1/*
2 * Copyright (C) 2017 SiFive
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16
17#ifndef _ASM_RISCV_VDSO_SYSCALLS_H
18#define _ASM_RISCV_VDSO_SYSCALLS_H
19
20#ifdef CONFIG_SMP
21
22/* These syscalls are only used by the vDSO and are not in the uapi. */
23#define __NR_riscv_flush_icache (__NR_arch_specific_syscall + 15)
24__SYSCALL(__NR_riscv_flush_icache, sys_riscv_flush_icache)
25
26#endif
27
28#endif /* _ASM_RISCV_VDSO_H */
diff --git a/arch/riscv/include/asm/vdso.h b/arch/riscv/include/asm/vdso.h
index 602f61257553..541544d64c33 100644
--- a/arch/riscv/include/asm/vdso.h
+++ b/arch/riscv/include/asm/vdso.h
@@ -38,4 +38,8 @@ struct vdso_data {
38 (void __user *)((unsigned long)(base) + __vdso_##name); \ 38 (void __user *)((unsigned long)(base) + __vdso_##name); \
39}) 39})
40 40
41#ifdef CONFIG_SMP
42asmlinkage long sys_riscv_flush_icache(uintptr_t, uintptr_t, uintptr_t);
43#endif
44
41#endif /* _ASM_RISCV_VDSO_H */ 45#endif /* _ASM_RISCV_VDSO_H */
diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S
index 76af908f87c1..78f670d70133 100644
--- a/arch/riscv/kernel/head.S
+++ b/arch/riscv/kernel/head.S
@@ -152,6 +152,3 @@ END(_start)
152__PAGE_ALIGNED_BSS 152__PAGE_ALIGNED_BSS
153 /* Empty zero page */ 153 /* Empty zero page */
154 .balign PAGE_SIZE 154 .balign PAGE_SIZE
155ENTRY(empty_zero_page)
156 .fill (empty_zero_page + PAGE_SIZE) - ., 1, 0x00
157END(empty_zero_page)
diff --git a/arch/riscv/kernel/riscv_ksyms.c b/arch/riscv/kernel/riscv_ksyms.c
index 23cc81ec9e94..551734248748 100644
--- a/arch/riscv/kernel/riscv_ksyms.c
+++ b/arch/riscv/kernel/riscv_ksyms.c
@@ -12,4 +12,7 @@
12/* 12/*
13 * Assembly functions that may be used (directly or indirectly) by modules 13 * Assembly functions that may be used (directly or indirectly) by modules
14 */ 14 */
15EXPORT_SYMBOL(__clear_user);
15EXPORT_SYMBOL(__copy_user); 16EXPORT_SYMBOL(__copy_user);
17EXPORT_SYMBOL(memset);
18EXPORT_SYMBOL(memcpy);
diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
index de7db114c315..8fbb6749910d 100644
--- a/arch/riscv/kernel/setup.c
+++ b/arch/riscv/kernel/setup.c
@@ -58,7 +58,12 @@ static char __initdata builtin_cmdline[COMMAND_LINE_SIZE] = CONFIG_CMDLINE;
58#endif /* CONFIG_CMDLINE_BOOL */ 58#endif /* CONFIG_CMDLINE_BOOL */
59 59
60unsigned long va_pa_offset; 60unsigned long va_pa_offset;
61EXPORT_SYMBOL(va_pa_offset);
61unsigned long pfn_base; 62unsigned long pfn_base;
63EXPORT_SYMBOL(pfn_base);
64
65unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
66EXPORT_SYMBOL(empty_zero_page);
62 67
63/* The lucky hart to first increment this variable will boot the other cores */ 68/* The lucky hart to first increment this variable will boot the other cores */
64atomic_t hart_lottery; 69atomic_t hart_lottery;
diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c
index b4a71ec5906f..6d3962435720 100644
--- a/arch/riscv/kernel/smp.c
+++ b/arch/riscv/kernel/smp.c
@@ -38,6 +38,13 @@ enum ipi_message_type {
38 IPI_MAX 38 IPI_MAX
39}; 39};
40 40
41
42/* Unsupported */
43int setup_profiling_timer(unsigned int multiplier)
44{
45 return -EINVAL;
46}
47
41irqreturn_t handle_ipi(void) 48irqreturn_t handle_ipi(void)
42{ 49{
43 unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits; 50 unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits;
@@ -108,3 +115,51 @@ void smp_send_reschedule(int cpu)
108{ 115{
109 send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); 116 send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
110} 117}
118
119/*
120 * Performs an icache flush for the given MM context. RISC-V has no direct
121 * mechanism for instruction cache shoot downs, so instead we send an IPI that
122 * informs the remote harts they need to flush their local instruction caches.
123 * To avoid pathologically slow behavior in a common case (a bunch of
124 * single-hart processes on a many-hart machine, ie 'make -j') we avoid the
125 * IPIs for harts that are not currently executing a MM context and instead
126 * schedule a deferred local instruction cache flush to be performed before
127 * execution resumes on each hart.
128 */
129void flush_icache_mm(struct mm_struct *mm, bool local)
130{
131 unsigned int cpu;
132 cpumask_t others, *mask;
133
134 preempt_disable();
135
136 /* Mark every hart's icache as needing a flush for this MM. */
137 mask = &mm->context.icache_stale_mask;
138 cpumask_setall(mask);
139 /* Flush this hart's I$ now, and mark it as flushed. */
140 cpu = smp_processor_id();
141 cpumask_clear_cpu(cpu, mask);
142 local_flush_icache_all();
143
144 /*
145 * Flush the I$ of other harts concurrently executing, and mark them as
146 * flushed.
147 */
148 cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu));
149 local |= cpumask_empty(&others);
150 if (mm != current->active_mm || !local)
151 sbi_remote_fence_i(others.bits);
152 else {
153 /*
154 * It's assumed that at least one strongly ordered operation is
155 * performed on this hart between setting a hart's cpumask bit
156 * and scheduling this MM context on that hart. Sending an SBI
157 * remote message will do this, but in the case where no
158 * messages are sent we still need to order this hart's writes
159 * with flush_icache_deferred().
160 */
161 smp_mb();
162 }
163
164 preempt_enable();
165}
diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c
index 4351be7d0533..a2ae936a093e 100644
--- a/arch/riscv/kernel/sys_riscv.c
+++ b/arch/riscv/kernel/sys_riscv.c
@@ -14,8 +14,8 @@
14 */ 14 */
15 15
16#include <linux/syscalls.h> 16#include <linux/syscalls.h>
17#include <asm/cmpxchg.h>
18#include <asm/unistd.h> 17#include <asm/unistd.h>
18#include <asm/cacheflush.h>
19 19
20static long riscv_sys_mmap(unsigned long addr, unsigned long len, 20static long riscv_sys_mmap(unsigned long addr, unsigned long len,
21 unsigned long prot, unsigned long flags, 21 unsigned long prot, unsigned long flags,
@@ -47,3 +47,34 @@ SYSCALL_DEFINE6(mmap2, unsigned long, addr, unsigned long, len,
47 return riscv_sys_mmap(addr, len, prot, flags, fd, offset, 12); 47 return riscv_sys_mmap(addr, len, prot, flags, fd, offset, 12);
48} 48}
49#endif /* !CONFIG_64BIT */ 49#endif /* !CONFIG_64BIT */
50
51#ifdef CONFIG_SMP
52/*
53 * Allows the instruction cache to be flushed from userspace. Despite RISC-V
54 * having a direct 'fence.i' instruction available to userspace (which we
55 * can't trap!), that's not actually viable when running on Linux because the
56 * kernel might schedule a process on another hart. There is no way for
57 * userspace to handle this without invoking the kernel (as it doesn't know the
58 * thread->hart mappings), so we've defined a RISC-V specific system call to
59 * flush the instruction cache.
60 *
61 * sys_riscv_flush_icache() is defined to flush the instruction cache over an
62 * address range, with the flush applying to either all threads or just the
63 * caller. We don't currently do anything with the address range, that's just
64 * in there for forwards compatibility.
65 */
66SYSCALL_DEFINE3(riscv_flush_icache, uintptr_t, start, uintptr_t, end,
67 uintptr_t, flags)
68{
69 struct mm_struct *mm = current->mm;
70 bool local = (flags & SYS_RISCV_FLUSH_ICACHE_LOCAL) != 0;
71
72 /* Check the reserved flags. */
73 if (unlikely(flags & !SYS_RISCV_FLUSH_ICACHE_ALL))
74 return -EINVAL;
75
76 flush_icache_mm(mm, local);
77
78 return 0;
79}
80#endif
diff --git a/arch/riscv/kernel/syscall_table.c b/arch/riscv/kernel/syscall_table.c
index 4e30dc5fb593..a5bd6401f95e 100644
--- a/arch/riscv/kernel/syscall_table.c
+++ b/arch/riscv/kernel/syscall_table.c
@@ -15,6 +15,7 @@
15#include <linux/linkage.h> 15#include <linux/linkage.h>
16#include <linux/syscalls.h> 16#include <linux/syscalls.h>
17#include <asm-generic/syscalls.h> 17#include <asm-generic/syscalls.h>
18#include <asm/vdso.h>
18 19
19#undef __SYSCALL 20#undef __SYSCALL
20#define __SYSCALL(nr, call) [nr] = (call), 21#define __SYSCALL(nr, call) [nr] = (call),
@@ -22,4 +23,5 @@
22void *sys_call_table[__NR_syscalls] = { 23void *sys_call_table[__NR_syscalls] = {
23 [0 ... __NR_syscalls - 1] = sys_ni_syscall, 24 [0 ... __NR_syscalls - 1] = sys_ni_syscall,
24#include <asm/unistd.h> 25#include <asm/unistd.h>
26#include <asm/vdso-syscalls.h>
25}; 27};
diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile
index 523d0a8ac8db..324568d33921 100644
--- a/arch/riscv/kernel/vdso/Makefile
+++ b/arch/riscv/kernel/vdso/Makefile
@@ -1,7 +1,12 @@
1# Copied from arch/tile/kernel/vdso/Makefile 1# Copied from arch/tile/kernel/vdso/Makefile
2 2
3# Symbols present in the vdso 3# Symbols present in the vdso
4vdso-syms = rt_sigreturn 4vdso-syms = rt_sigreturn
5vdso-syms += gettimeofday
6vdso-syms += clock_gettime
7vdso-syms += clock_getres
8vdso-syms += getcpu
9vdso-syms += flush_icache
5 10
6# Files to link into the vdso 11# Files to link into the vdso
7obj-vdso = $(patsubst %, %.o, $(vdso-syms)) 12obj-vdso = $(patsubst %, %.o, $(vdso-syms))
diff --git a/arch/riscv/kernel/vdso/clock_getres.S b/arch/riscv/kernel/vdso/clock_getres.S
new file mode 100644
index 000000000000..edf7e2339648
--- /dev/null
+++ b/arch/riscv/kernel/vdso/clock_getres.S
@@ -0,0 +1,26 @@
1/*
2 * Copyright (C) 2017 SiFive
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 */
13
14#include <linux/linkage.h>
15#include <asm/unistd.h>
16
17 .text
18/* int __vdso_clock_getres(clockid_t clock_id, struct timespec *res); */
19ENTRY(__vdso_clock_getres)
20 .cfi_startproc
21 /* For now, just do the syscall. */
22 li a7, __NR_clock_getres
23 ecall
24 ret
25 .cfi_endproc
26ENDPROC(__vdso_clock_getres)
diff --git a/arch/riscv/kernel/vdso/clock_gettime.S b/arch/riscv/kernel/vdso/clock_gettime.S
new file mode 100644
index 000000000000..aac65676c6d5
--- /dev/null
+++ b/arch/riscv/kernel/vdso/clock_gettime.S
@@ -0,0 +1,26 @@
1/*
2 * Copyright (C) 2017 SiFive
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 */
13
14#include <linux/linkage.h>
15#include <asm/unistd.h>
16
17 .text
18/* int __vdso_clock_gettime(clockid_t clock_id, struct timespec *tp); */
19ENTRY(__vdso_clock_gettime)
20 .cfi_startproc
21 /* For now, just do the syscall. */
22 li a7, __NR_clock_gettime
23 ecall
24 ret
25 .cfi_endproc
26ENDPROC(__vdso_clock_gettime)
diff --git a/arch/riscv/kernel/vdso/flush_icache.S b/arch/riscv/kernel/vdso/flush_icache.S
new file mode 100644
index 000000000000..b0fbad74e873
--- /dev/null
+++ b/arch/riscv/kernel/vdso/flush_icache.S
@@ -0,0 +1,31 @@
1/*
2 * Copyright (C) 2017 SiFive
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 */
13
14#include <linux/linkage.h>
15#include <asm/unistd.h>
16#include <asm/vdso-syscalls.h>
17
18 .text
19/* int __vdso_flush_icache(void *start, void *end, unsigned long flags); */
20ENTRY(__vdso_flush_icache)
21 .cfi_startproc
22#ifdef CONFIG_SMP
23 li a7, __NR_riscv_flush_icache
24 ecall
25#else
26 fence.i
27 li a0, 0
28#endif
29 ret
30 .cfi_endproc
31ENDPROC(__vdso_flush_icache)
diff --git a/arch/riscv/kernel/vdso/getcpu.S b/arch/riscv/kernel/vdso/getcpu.S
new file mode 100644
index 000000000000..cc7e98924484
--- /dev/null
+++ b/arch/riscv/kernel/vdso/getcpu.S
@@ -0,0 +1,26 @@
1/*
2 * Copyright (C) 2017 SiFive
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 */
13
14#include <linux/linkage.h>
15#include <asm/unistd.h>
16
17 .text
18/* int __vdso_getcpu(unsigned *cpu, unsigned *node, void *unused); */
19ENTRY(__vdso_getcpu)
20 .cfi_startproc
21 /* For now, just do the syscall. */
22 li a7, __NR_getcpu
23 ecall
24 ret
25 .cfi_endproc
26ENDPROC(__vdso_getcpu)
diff --git a/arch/riscv/kernel/vdso/gettimeofday.S b/arch/riscv/kernel/vdso/gettimeofday.S
new file mode 100644
index 000000000000..da85d33e8990
--- /dev/null
+++ b/arch/riscv/kernel/vdso/gettimeofday.S
@@ -0,0 +1,26 @@
1/*
2 * Copyright (C) 2017 SiFive
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 */
13
14#include <linux/linkage.h>
15#include <asm/unistd.h>
16
17 .text
18/* int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz); */
19ENTRY(__vdso_gettimeofday)
20 .cfi_startproc
21 /* For now, just do the syscall. */
22 li a7, __NR_gettimeofday
23 ecall
24 ret
25 .cfi_endproc
26ENDPROC(__vdso_gettimeofday)
diff --git a/arch/riscv/kernel/vdso/vdso.lds.S b/arch/riscv/kernel/vdso/vdso.lds.S
index 8c9dce95c11d..cd1d47e0724b 100644
--- a/arch/riscv/kernel/vdso/vdso.lds.S
+++ b/arch/riscv/kernel/vdso/vdso.lds.S
@@ -70,8 +70,11 @@ VERSION
70 LINUX_4.15 { 70 LINUX_4.15 {
71 global: 71 global:
72 __vdso_rt_sigreturn; 72 __vdso_rt_sigreturn;
73 __vdso_cmpxchg32; 73 __vdso_gettimeofday;
74 __vdso_cmpxchg64; 74 __vdso_clock_gettime;
75 __vdso_clock_getres;
76 __vdso_getcpu;
77 __vdso_flush_icache;
75 local: *; 78 local: *;
76 }; 79 };
77} 80}
diff --git a/arch/riscv/lib/delay.c b/arch/riscv/lib/delay.c
index 1cc4ac3964b4..dce8ae24c6d3 100644
--- a/arch/riscv/lib/delay.c
+++ b/arch/riscv/lib/delay.c
@@ -84,6 +84,7 @@ void __delay(unsigned long cycles)
84 while ((unsigned long)(get_cycles() - t0) < cycles) 84 while ((unsigned long)(get_cycles() - t0) < cycles)
85 cpu_relax(); 85 cpu_relax();
86} 86}
87EXPORT_SYMBOL(__delay);
87 88
88void udelay(unsigned long usecs) 89void udelay(unsigned long usecs)
89{ 90{
diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile
index 81f7d9ce6d88..eb22ab49b3e0 100644
--- a/arch/riscv/mm/Makefile
+++ b/arch/riscv/mm/Makefile
@@ -2,3 +2,4 @@ obj-y += init.o
2obj-y += fault.o 2obj-y += fault.o
3obj-y += extable.o 3obj-y += extable.o
4obj-y += ioremap.o 4obj-y += ioremap.o
5obj-y += cacheflush.o
diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c
new file mode 100644
index 000000000000..498c0a0814fe
--- /dev/null
+++ b/arch/riscv/mm/cacheflush.c
@@ -0,0 +1,23 @@
1/*
2 * Copyright (C) 2017 SiFive
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 */
13
14#include <asm/pgtable.h>
15#include <asm/cacheflush.h>
16
17void flush_icache_pte(pte_t pte)
18{
19 struct page *page = pte_page(pte);
20
21 if (!test_and_set_bit(PG_dcache_clean, &page->flags))
22 flush_icache_all();
23}
diff --git a/arch/riscv/mm/ioremap.c b/arch/riscv/mm/ioremap.c
index e99194a4077e..70ef2724cdf6 100644
--- a/arch/riscv/mm/ioremap.c
+++ b/arch/riscv/mm/ioremap.c
@@ -85,7 +85,7 @@ EXPORT_SYMBOL(ioremap);
85 * 85 *
86 * Caller must ensure there is only one unmapping for the same pointer. 86 * Caller must ensure there is only one unmapping for the same pointer.
87 */ 87 */
88void iounmap(void __iomem *addr) 88void iounmap(volatile void __iomem *addr)
89{ 89{
90 vunmap((void *)((unsigned long)addr & PAGE_MASK)); 90 vunmap((void *)((unsigned long)addr & PAGE_MASK));
91} 91}
diff --git a/include/lib/libgcc.h b/include/linux/libgcc.h
index 32e1e0f4b2d0..32e1e0f4b2d0 100644
--- a/include/lib/libgcc.h
+++ b/include/linux/libgcc.h
diff --git a/lib/ashldi3.c b/lib/ashldi3.c
index 1b6087db95a5..3ffc46e3bb6c 100644
--- a/lib/ashldi3.c
+++ b/lib/ashldi3.c
@@ -16,7 +16,7 @@
16 16
17#include <linux/export.h> 17#include <linux/export.h>
18 18
19#include <lib/libgcc.h> 19#include <linux/libgcc.h>
20 20
21long long notrace __ashldi3(long long u, word_type b) 21long long notrace __ashldi3(long long u, word_type b)
22{ 22{
diff --git a/lib/ashrdi3.c b/lib/ashrdi3.c
index 2e67c97ac65a..ea054550f0e8 100644
--- a/lib/ashrdi3.c
+++ b/lib/ashrdi3.c
@@ -16,7 +16,7 @@
16 16
17#include <linux/export.h> 17#include <linux/export.h>
18 18
19#include <lib/libgcc.h> 19#include <linux/libgcc.h>
20 20
21long long notrace __ashrdi3(long long u, word_type b) 21long long notrace __ashrdi3(long long u, word_type b)
22{ 22{
diff --git a/lib/cmpdi2.c b/lib/cmpdi2.c
index 6d7ebf6c2b86..2250da7e503e 100644
--- a/lib/cmpdi2.c
+++ b/lib/cmpdi2.c
@@ -16,7 +16,7 @@
16 16
17#include <linux/export.h> 17#include <linux/export.h>
18 18
19#include <lib/libgcc.h> 19#include <linux/libgcc.h>
20 20
21word_type notrace __cmpdi2(long long a, long long b) 21word_type notrace __cmpdi2(long long a, long long b)
22{ 22{
diff --git a/lib/lshrdi3.c b/lib/lshrdi3.c
index 8e845f4bb65f..99cfa5721f2d 100644
--- a/lib/lshrdi3.c
+++ b/lib/lshrdi3.c
@@ -17,7 +17,7 @@
17 */ 17 */
18 18
19#include <linux/module.h> 19#include <linux/module.h>
20#include <lib/libgcc.h> 20#include <linux/libgcc.h>
21 21
22long long notrace __lshrdi3(long long u, word_type b) 22long long notrace __lshrdi3(long long u, word_type b)
23{ 23{
diff --git a/lib/muldi3.c b/lib/muldi3.c
index 88938543e10a..54c8b3123376 100644
--- a/lib/muldi3.c
+++ b/lib/muldi3.c
@@ -15,7 +15,7 @@
15 */ 15 */
16 16
17#include <linux/export.h> 17#include <linux/export.h>
18#include <lib/libgcc.h> 18#include <linux/libgcc.h>
19 19
20#define W_TYPE_SIZE 32 20#define W_TYPE_SIZE 32
21 21
diff --git a/lib/ucmpdi2.c b/lib/ucmpdi2.c
index 49a53505c8e3..25ca2d4c1e19 100644
--- a/lib/ucmpdi2.c
+++ b/lib/ucmpdi2.c
@@ -15,7 +15,7 @@
15 */ 15 */
16 16
17#include <linux/module.h> 17#include <linux/module.h>
18#include <lib/libgcc.h> 18#include <linux/libgcc.h>
19 19
20word_type __ucmpdi2(unsigned long long a, unsigned long long b) 20word_type __ucmpdi2(unsigned long long a, unsigned long long b)
21{ 21{