aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/locking/lockstat.txt2
-rw-r--r--Documentation/memory-barriers.txt3
-rw-r--r--Makefile9
-rw-r--r--arch/Kconfig3
-rw-r--r--arch/arm64/Kconfig1
-rw-r--r--arch/arm64/include/asm/jump_label.h38
-rw-r--r--arch/arm64/kernel/jump_label.c6
-rw-r--r--arch/s390/Kconfig1
-rw-r--r--arch/s390/include/asm/jump_label.h40
-rw-r--r--arch/s390/kernel/jump_label.c11
-rw-r--r--arch/s390/kernel/vmlinux.lds.S1
-rw-r--r--arch/x86/Kconfig1
-rw-r--r--arch/x86/Makefile8
-rw-r--r--arch/x86/configs/i386_defconfig1
-rw-r--r--arch/x86/configs/x86_64_defconfig1
-rw-r--r--arch/x86/entry/calling.h2
-rw-r--r--arch/x86/include/asm/alternative-asm.h20
-rw-r--r--arch/x86/include/asm/alternative.h11
-rw-r--r--arch/x86/include/asm/asm.h57
-rw-r--r--arch/x86/include/asm/atomic.h8
-rw-r--r--arch/x86/include/asm/atomic64_64.h8
-rw-r--r--arch/x86/include/asm/bitops.h9
-rw-r--r--arch/x86/include/asm/bug.h98
-rw-r--r--arch/x86/include/asm/cpufeature.h82
-rw-r--r--arch/x86/include/asm/elf.h3
-rw-r--r--arch/x86/include/asm/extable.h3
-rw-r--r--arch/x86/include/asm/fpu/internal.h2
-rw-r--r--arch/x86/include/asm/futex.h6
-rw-r--r--arch/x86/include/asm/jump_label.h80
-rw-r--r--arch/x86/include/asm/local.h8
-rw-r--r--arch/x86/include/asm/paravirt_types.h56
-rw-r--r--arch/x86/include/asm/preempt.h2
-rw-r--r--arch/x86/include/asm/ptrace.h2
-rw-r--r--arch/x86/include/asm/qspinlock.h15
-rw-r--r--arch/x86/include/asm/refcount.h79
-rw-r--r--arch/x86/include/asm/rmwcc.h69
-rw-r--r--arch/x86/include/asm/uaccess.h22
-rw-r--r--arch/x86/kernel/cpu/mcheck/mce.c2
-rw-r--r--arch/x86/kernel/jump_label.c62
-rw-r--r--arch/x86/kernel/kprobes/core.c38
-rw-r--r--arch/x86/kernel/macros.S16
-rw-r--r--arch/x86/kernel/module.c6
-rw-r--r--arch/x86/kernel/traps.c16
-rw-r--r--arch/x86/lib/checksum_32.S4
-rw-r--r--arch/x86/lib/copy_user_64.S90
-rw-r--r--arch/x86/lib/csum-copy_64.S8
-rw-r--r--arch/x86/lib/getuser.S12
-rw-r--r--arch/x86/lib/putuser.S10
-rw-r--r--arch/x86/lib/usercopy_32.c126
-rw-r--r--arch/x86/lib/usercopy_64.c4
-rw-r--r--arch/x86/mm/extable.c114
-rw-r--r--arch/x86/mm/fault.c26
-rw-r--r--arch/x86/tools/relocs.c10
-rw-r--r--arch/x86/um/asm/elf.h3
-rw-r--r--arch/xtensa/kernel/Makefile4
-rw-r--r--drivers/misc/lkdtm/core.c1
-rw-r--r--drivers/misc/lkdtm/lkdtm.h1
-rw-r--r--drivers/misc/lkdtm/usercopy.c13
-rw-r--r--fs/namespace.c2
-rw-r--r--include/asm-generic/bug.h8
-rw-r--r--include/asm-generic/qrwlock.h7
-rw-r--r--include/asm-generic/qspinlock.h16
-rw-r--r--include/asm-generic/vmlinux.lds.h11
-rw-r--r--include/linux/compiler.h56
-rw-r--r--include/linux/debug_locks.h4
-rw-r--r--include/linux/jump_label.h65
-rw-r--r--include/linux/lockdep.h7
-rw-r--r--include/linux/rwsem.h4
-rw-r--r--include/linux/sched.h6
-rw-r--r--init/main.c1
-rw-r--r--kernel/cpu.c28
-rw-r--r--kernel/futex.c4
-rw-r--r--kernel/jump_label.c107
-rw-r--r--kernel/locking/lockdep.c116
-rw-r--r--kernel/locking/lockdep_internals.h27
-rw-r--r--kernel/locking/lockdep_proc.c2
-rw-r--r--kernel/locking/qspinlock.c143
-rw-r--r--kernel/locking/qspinlock_paravirt.h4
-rw-r--r--kernel/locking/qspinlock_stat.h6
-rw-r--r--kernel/locking/rtmutex.c4
-rw-r--r--kernel/locking/rwsem-xadd.c15
-rw-r--r--kernel/locking/rwsem.c7
-rw-r--r--kernel/locking/rwsem.h95
-rw-r--r--kernel/module.c9
-rw-r--r--lib/debug_locks.c6
-rw-r--r--mm/maccess.c6
-rw-r--r--scripts/Kbuild.include4
-rw-r--r--scripts/mod/Makefile2
-rw-r--r--tools/memory-model/Documentation/explanation.txt186
-rw-r--r--tools/memory-model/Documentation/recipes.txt2
-rw-r--r--tools/memory-model/README39
-rw-r--r--tools/memory-model/linux-kernel.cat8
-rw-r--r--tools/memory-model/litmus-tests/ISA2+pooncelock+pooncelock+pombonce.litmus7
-rw-r--r--tools/memory-model/litmus-tests/README104
-rw-r--r--tools/objtool/special.c4
95 files changed, 1522 insertions, 934 deletions
diff --git a/Documentation/locking/lockstat.txt b/Documentation/locking/lockstat.txt
index 5786ad2cd5e6..fdbeb0c45ef3 100644
--- a/Documentation/locking/lockstat.txt
+++ b/Documentation/locking/lockstat.txt
@@ -91,7 +91,7 @@ Look at the current lock statistics:
9107 &mm->mmap_sem-R: 37 100 1.31 299502.61 325629.52 3256.30 212344 34316685 0.10 7744.91 95016910.20 2.77 9107 &mm->mmap_sem-R: 37 100 1.31 299502.61 325629.52 3256.30 212344 34316685 0.10 7744.91 95016910.20 2.77
9208 --------------- 9208 ---------------
9309 &mm->mmap_sem 1 [<ffffffff811502a7>] khugepaged_scan_mm_slot+0x57/0x280 9309 &mm->mmap_sem 1 [<ffffffff811502a7>] khugepaged_scan_mm_slot+0x57/0x280
9419 &mm->mmap_sem 96 [<ffffffff815351c4>] __do_page_fault+0x1d4/0x510 9410 &mm->mmap_sem 96 [<ffffffff815351c4>] __do_page_fault+0x1d4/0x510
9511 &mm->mmap_sem 34 [<ffffffff81113d77>] vm_mmap_pgoff+0x87/0xd0 9511 &mm->mmap_sem 34 [<ffffffff81113d77>] vm_mmap_pgoff+0x87/0xd0
9612 &mm->mmap_sem 17 [<ffffffff81127e71>] vm_munmap+0x41/0x80 9612 &mm->mmap_sem 17 [<ffffffff81127e71>] vm_munmap+0x41/0x80
9713 --------------- 9713 ---------------
diff --git a/Documentation/memory-barriers.txt b/Documentation/memory-barriers.txt
index 0d8d7ef131e9..c1d913944ad8 100644
--- a/Documentation/memory-barriers.txt
+++ b/Documentation/memory-barriers.txt
@@ -471,8 +471,7 @@ And a couple of implicit varieties:
471 operations after the ACQUIRE operation will appear to happen after the 471 operations after the ACQUIRE operation will appear to happen after the
472 ACQUIRE operation with respect to the other components of the system. 472 ACQUIRE operation with respect to the other components of the system.
473 ACQUIRE operations include LOCK operations and both smp_load_acquire() 473 ACQUIRE operations include LOCK operations and both smp_load_acquire()
474 and smp_cond_acquire() operations. The later builds the necessary ACQUIRE 474 and smp_cond_load_acquire() operations.
475 semantics from relying on a control dependency and smp_rmb().
476 475
477 Memory operations that occur before an ACQUIRE operation may appear to 476 Memory operations that occur before an ACQUIRE operation may appear to
478 happen after it completes. 477 happen after it completes.
diff --git a/Makefile b/Makefile
index 69fa5c0310d8..2fc5732a4f9e 100644
--- a/Makefile
+++ b/Makefile
@@ -1073,7 +1073,7 @@ scripts: scripts_basic asm-generic gcc-plugins $(autoksyms_h)
1073# version.h and scripts_basic is processed / created. 1073# version.h and scripts_basic is processed / created.
1074 1074
1075# Listed in dependency order 1075# Listed in dependency order
1076PHONY += prepare archprepare prepare0 prepare1 prepare2 prepare3 1076PHONY += prepare archprepare macroprepare prepare0 prepare1 prepare2 prepare3
1077 1077
1078# prepare3 is used to check if we are building in a separate output directory, 1078# prepare3 is used to check if we are building in a separate output directory,
1079# and if so do: 1079# and if so do:
@@ -1096,7 +1096,9 @@ prepare2: prepare3 outputmakefile asm-generic
1096prepare1: prepare2 $(version_h) $(autoksyms_h) include/generated/utsrelease.h 1096prepare1: prepare2 $(version_h) $(autoksyms_h) include/generated/utsrelease.h
1097 $(cmd_crmodverdir) 1097 $(cmd_crmodverdir)
1098 1098
1099archprepare: archheaders archscripts prepare1 scripts_basic 1099macroprepare: prepare1 archmacros
1100
1101archprepare: archheaders archscripts macroprepare scripts_basic
1100 1102
1101prepare0: archprepare gcc-plugins 1103prepare0: archprepare gcc-plugins
1102 $(Q)$(MAKE) $(build)=. 1104 $(Q)$(MAKE) $(build)=.
@@ -1164,6 +1166,9 @@ archheaders:
1164PHONY += archscripts 1166PHONY += archscripts
1165archscripts: 1167archscripts:
1166 1168
1169PHONY += archmacros
1170archmacros:
1171
1167PHONY += __headers 1172PHONY += __headers
1168__headers: $(version_h) scripts_basic uapi-asm-generic archheaders archscripts 1173__headers: $(version_h) scripts_basic uapi-asm-generic archheaders archscripts
1169 $(Q)$(MAKE) $(build)=scripts build_unifdef 1174 $(Q)$(MAKE) $(build)=scripts build_unifdef
diff --git a/arch/Kconfig b/arch/Kconfig
index 6801123932a5..9d329608913e 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -359,6 +359,9 @@ config HAVE_PERF_USER_STACK_DUMP
359config HAVE_ARCH_JUMP_LABEL 359config HAVE_ARCH_JUMP_LABEL
360 bool 360 bool
361 361
362config HAVE_ARCH_JUMP_LABEL_RELATIVE
363 bool
364
362config HAVE_RCU_TABLE_FREE 365config HAVE_RCU_TABLE_FREE
363 bool 366 bool
364 367
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index a8ae30fab508..c03cd0d765d3 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -105,6 +105,7 @@ config ARM64
105 select HAVE_ARCH_BITREVERSE 105 select HAVE_ARCH_BITREVERSE
106 select HAVE_ARCH_HUGE_VMAP 106 select HAVE_ARCH_HUGE_VMAP
107 select HAVE_ARCH_JUMP_LABEL 107 select HAVE_ARCH_JUMP_LABEL
108 select HAVE_ARCH_JUMP_LABEL_RELATIVE
108 select HAVE_ARCH_KASAN if !(ARM64_16K_PAGES && ARM64_VA_BITS_48) 109 select HAVE_ARCH_KASAN if !(ARM64_16K_PAGES && ARM64_VA_BITS_48)
109 select HAVE_ARCH_KGDB 110 select HAVE_ARCH_KGDB
110 select HAVE_ARCH_MMAP_RND_BITS 111 select HAVE_ARCH_MMAP_RND_BITS
diff --git a/arch/arm64/include/asm/jump_label.h b/arch/arm64/include/asm/jump_label.h
index 7e2b3e360086..472023498d71 100644
--- a/arch/arm64/include/asm/jump_label.h
+++ b/arch/arm64/include/asm/jump_label.h
@@ -26,13 +26,16 @@
26 26
27#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE 27#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE
28 28
29static __always_inline bool arch_static_branch(struct static_key *key, bool branch) 29static __always_inline bool arch_static_branch(struct static_key *key,
30 bool branch)
30{ 31{
31 asm_volatile_goto("1: nop\n\t" 32 asm_volatile_goto(
32 ".pushsection __jump_table, \"aw\"\n\t" 33 "1: nop \n\t"
33 ".align 3\n\t" 34 " .pushsection __jump_table, \"aw\" \n\t"
34 ".quad 1b, %l[l_yes], %c0\n\t" 35 " .align 3 \n\t"
35 ".popsection\n\t" 36 " .long 1b - ., %l[l_yes] - . \n\t"
37 " .quad %c0 - . \n\t"
38 " .popsection \n\t"
36 : : "i"(&((char *)key)[branch]) : : l_yes); 39 : : "i"(&((char *)key)[branch]) : : l_yes);
37 40
38 return false; 41 return false;
@@ -40,13 +43,16 @@ l_yes:
40 return true; 43 return true;
41} 44}
42 45
43static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) 46static __always_inline bool arch_static_branch_jump(struct static_key *key,
47 bool branch)
44{ 48{
45 asm_volatile_goto("1: b %l[l_yes]\n\t" 49 asm_volatile_goto(
46 ".pushsection __jump_table, \"aw\"\n\t" 50 "1: b %l[l_yes] \n\t"
47 ".align 3\n\t" 51 " .pushsection __jump_table, \"aw\" \n\t"
48 ".quad 1b, %l[l_yes], %c0\n\t" 52 " .align 3 \n\t"
49 ".popsection\n\t" 53 " .long 1b - ., %l[l_yes] - . \n\t"
54 " .quad %c0 - . \n\t"
55 " .popsection \n\t"
50 : : "i"(&((char *)key)[branch]) : : l_yes); 56 : : "i"(&((char *)key)[branch]) : : l_yes);
51 57
52 return false; 58 return false;
@@ -54,13 +60,5 @@ l_yes:
54 return true; 60 return true;
55} 61}
56 62
57typedef u64 jump_label_t;
58
59struct jump_entry {
60 jump_label_t code;
61 jump_label_t target;
62 jump_label_t key;
63};
64
65#endif /* __ASSEMBLY__ */ 63#endif /* __ASSEMBLY__ */
66#endif /* __ASM_JUMP_LABEL_H */ 64#endif /* __ASM_JUMP_LABEL_H */
diff --git a/arch/arm64/kernel/jump_label.c b/arch/arm64/kernel/jump_label.c
index e0756416e567..646b9562ee64 100644
--- a/arch/arm64/kernel/jump_label.c
+++ b/arch/arm64/kernel/jump_label.c
@@ -25,12 +25,12 @@
25void arch_jump_label_transform(struct jump_entry *entry, 25void arch_jump_label_transform(struct jump_entry *entry,
26 enum jump_label_type type) 26 enum jump_label_type type)
27{ 27{
28 void *addr = (void *)entry->code; 28 void *addr = (void *)jump_entry_code(entry);
29 u32 insn; 29 u32 insn;
30 30
31 if (type == JUMP_LABEL_JMP) { 31 if (type == JUMP_LABEL_JMP) {
32 insn = aarch64_insn_gen_branch_imm(entry->code, 32 insn = aarch64_insn_gen_branch_imm(jump_entry_code(entry),
33 entry->target, 33 jump_entry_target(entry),
34 AARCH64_INSN_BRANCH_NOLINK); 34 AARCH64_INSN_BRANCH_NOLINK);
35 } else { 35 } else {
36 insn = aarch64_insn_gen_nop(); 36 insn = aarch64_insn_gen_nop();
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index cc8313550493..039a3417dfc4 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -126,6 +126,7 @@ config S390
126 select HAVE_ALIGNED_STRUCT_PAGE if SLUB 126 select HAVE_ALIGNED_STRUCT_PAGE if SLUB
127 select HAVE_ARCH_AUDITSYSCALL 127 select HAVE_ARCH_AUDITSYSCALL
128 select HAVE_ARCH_JUMP_LABEL 128 select HAVE_ARCH_JUMP_LABEL
129 select HAVE_ARCH_JUMP_LABEL_RELATIVE
129 select HAVE_ARCH_KASAN 130 select HAVE_ARCH_KASAN
130 select CPU_NO_EFFICIENT_FFS if !HAVE_MARCH_Z9_109_FEATURES 131 select CPU_NO_EFFICIENT_FFS if !HAVE_MARCH_Z9_109_FEATURES
131 select HAVE_ARCH_SECCOMP_FILTER 132 select HAVE_ARCH_SECCOMP_FILTER
diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h
index 40f651292aa7..e2d3e6c43395 100644
--- a/arch/s390/include/asm/jump_label.h
+++ b/arch/s390/include/asm/jump_label.h
@@ -14,41 +14,33 @@
14 * We use a brcl 0,2 instruction for jump labels at compile time so it 14 * We use a brcl 0,2 instruction for jump labels at compile time so it
15 * can be easily distinguished from a hotpatch generated instruction. 15 * can be easily distinguished from a hotpatch generated instruction.
16 */ 16 */
17static __always_inline bool arch_static_branch(struct static_key *key, bool branch) 17static inline bool arch_static_branch(struct static_key *key, bool branch)
18{ 18{
19 asm_volatile_goto("0: brcl 0,"__stringify(JUMP_LABEL_NOP_OFFSET)"\n" 19 asm_volatile_goto("0: brcl 0,"__stringify(JUMP_LABEL_NOP_OFFSET)"\n"
20 ".pushsection __jump_table, \"aw\"\n" 20 ".pushsection __jump_table,\"aw\"\n"
21 ".balign 8\n" 21 ".balign 8\n"
22 ".quad 0b, %l[label], %0\n" 22 ".long 0b-.,%l[label]-.\n"
23 ".popsection\n" 23 ".quad %0-.\n"
24 : : "X" (&((char *)key)[branch]) : : label); 24 ".popsection\n"
25 25 : : "X" (&((char *)key)[branch]) : : label);
26 return false; 26 return false;
27label: 27label:
28 return true; 28 return true;
29} 29}
30 30
31static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) 31static inline bool arch_static_branch_jump(struct static_key *key, bool branch)
32{ 32{
33 asm_volatile_goto("0: brcl 15, %l[label]\n" 33 asm_volatile_goto("0: brcl 15,%l[label]\n"
34 ".pushsection __jump_table, \"aw\"\n" 34 ".pushsection __jump_table,\"aw\"\n"
35 ".balign 8\n" 35 ".balign 8\n"
36 ".quad 0b, %l[label], %0\n" 36 ".long 0b-.,%l[label]-.\n"
37 ".popsection\n" 37 ".quad %0-.\n"
38 : : "X" (&((char *)key)[branch]) : : label); 38 ".popsection\n"
39 39 : : "X" (&((char *)key)[branch]) : : label);
40 return false; 40 return false;
41label: 41label:
42 return true; 42 return true;
43} 43}
44 44
45typedef unsigned long jump_label_t;
46
47struct jump_entry {
48 jump_label_t code;
49 jump_label_t target;
50 jump_label_t key;
51};
52
53#endif /* __ASSEMBLY__ */ 45#endif /* __ASSEMBLY__ */
54#endif 46#endif
diff --git a/arch/s390/kernel/jump_label.c b/arch/s390/kernel/jump_label.c
index 43f8430fb67d..50a1798604a8 100644
--- a/arch/s390/kernel/jump_label.c
+++ b/arch/s390/kernel/jump_label.c
@@ -33,13 +33,13 @@ static void jump_label_make_branch(struct jump_entry *entry, struct insn *insn)
33{ 33{
34 /* brcl 15,offset */ 34 /* brcl 15,offset */
35 insn->opcode = 0xc0f4; 35 insn->opcode = 0xc0f4;
36 insn->offset = (entry->target - entry->code) >> 1; 36 insn->offset = (jump_entry_target(entry) - jump_entry_code(entry)) >> 1;
37} 37}
38 38
39static void jump_label_bug(struct jump_entry *entry, struct insn *expected, 39static void jump_label_bug(struct jump_entry *entry, struct insn *expected,
40 struct insn *new) 40 struct insn *new)
41{ 41{
42 unsigned char *ipc = (unsigned char *)entry->code; 42 unsigned char *ipc = (unsigned char *)jump_entry_code(entry);
43 unsigned char *ipe = (unsigned char *)expected; 43 unsigned char *ipe = (unsigned char *)expected;
44 unsigned char *ipn = (unsigned char *)new; 44 unsigned char *ipn = (unsigned char *)new;
45 45
@@ -59,6 +59,7 @@ static void __jump_label_transform(struct jump_entry *entry,
59 enum jump_label_type type, 59 enum jump_label_type type,
60 int init) 60 int init)
61{ 61{
62 void *code = (void *)jump_entry_code(entry);
62 struct insn old, new; 63 struct insn old, new;
63 64
64 if (type == JUMP_LABEL_JMP) { 65 if (type == JUMP_LABEL_JMP) {
@@ -69,13 +70,13 @@ static void __jump_label_transform(struct jump_entry *entry,
69 jump_label_make_nop(entry, &new); 70 jump_label_make_nop(entry, &new);
70 } 71 }
71 if (init) { 72 if (init) {
72 if (memcmp((void *)entry->code, &orignop, sizeof(orignop))) 73 if (memcmp(code, &orignop, sizeof(orignop)))
73 jump_label_bug(entry, &orignop, &new); 74 jump_label_bug(entry, &orignop, &new);
74 } else { 75 } else {
75 if (memcmp((void *)entry->code, &old, sizeof(old))) 76 if (memcmp(code, &old, sizeof(old)))
76 jump_label_bug(entry, &old, &new); 77 jump_label_bug(entry, &old, &new);
77 } 78 }
78 s390_kernel_write((void *)entry->code, &new, sizeof(new)); 79 s390_kernel_write(code, &new, sizeof(new));
79} 80}
80 81
81static int __sm_arch_jump_label_transform(void *data) 82static int __sm_arch_jump_label_transform(void *data)
diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S
index cc3cbdc93d35..21eb7407d51b 100644
--- a/arch/s390/kernel/vmlinux.lds.S
+++ b/arch/s390/kernel/vmlinux.lds.S
@@ -65,6 +65,7 @@ SECTIONS
65 __start_ro_after_init = .; 65 __start_ro_after_init = .;
66 .data..ro_after_init : { 66 .data..ro_after_init : {
67 *(.data..ro_after_init) 67 *(.data..ro_after_init)
68 JUMP_TABLE_DATA
68 } 69 }
69 EXCEPTION_TABLE(16) 70 EXCEPTION_TABLE(16)
70 . = ALIGN(PAGE_SIZE); 71 . = ALIGN(PAGE_SIZE);
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e8de5de1057f..45b94fa9e98c 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -119,6 +119,7 @@ config X86
119 select HAVE_ARCH_AUDITSYSCALL 119 select HAVE_ARCH_AUDITSYSCALL
120 select HAVE_ARCH_HUGE_VMAP if X86_64 || X86_PAE 120 select HAVE_ARCH_HUGE_VMAP if X86_64 || X86_PAE
121 select HAVE_ARCH_JUMP_LABEL 121 select HAVE_ARCH_JUMP_LABEL
122 select HAVE_ARCH_JUMP_LABEL_RELATIVE
122 select HAVE_ARCH_KASAN if X86_64 123 select HAVE_ARCH_KASAN if X86_64
123 select HAVE_ARCH_KGDB 124 select HAVE_ARCH_KGDB
124 select HAVE_ARCH_MMAP_RND_BITS if MMU 125 select HAVE_ARCH_MMAP_RND_BITS if MMU
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 8f6e7eb8ae9f..5b562e464009 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -193,7 +193,6 @@ cfi-sections := $(call as-instr,.cfi_sections .debug_frame,-DCONFIG_AS_CFI_SECTI
193# does binutils support specific instructions? 193# does binutils support specific instructions?
194asinstr := $(call as-instr,fxsaveq (%rax),-DCONFIG_AS_FXSAVEQ=1) 194asinstr := $(call as-instr,fxsaveq (%rax),-DCONFIG_AS_FXSAVEQ=1)
195asinstr += $(call as-instr,pshufb %xmm0$(comma)%xmm0,-DCONFIG_AS_SSSE3=1) 195asinstr += $(call as-instr,pshufb %xmm0$(comma)%xmm0,-DCONFIG_AS_SSSE3=1)
196asinstr += $(call as-instr,crc32l %eax$(comma)%eax,-DCONFIG_AS_CRC32=1)
197avx_instr := $(call as-instr,vxorps %ymm0$(comma)%ymm1$(comma)%ymm2,-DCONFIG_AS_AVX=1) 196avx_instr := $(call as-instr,vxorps %ymm0$(comma)%ymm1$(comma)%ymm2,-DCONFIG_AS_AVX=1)
198avx2_instr :=$(call as-instr,vpbroadcastb %xmm0$(comma)%ymm1,-DCONFIG_AS_AVX2=1) 197avx2_instr :=$(call as-instr,vpbroadcastb %xmm0$(comma)%ymm1,-DCONFIG_AS_AVX2=1)
199avx512_instr :=$(call as-instr,vpmovm2b %k1$(comma)%zmm5,-DCONFIG_AS_AVX512=1) 198avx512_instr :=$(call as-instr,vpmovm2b %k1$(comma)%zmm5,-DCONFIG_AS_AVX512=1)
@@ -237,6 +236,13 @@ archscripts: scripts_basic
237archheaders: 236archheaders:
238 $(Q)$(MAKE) $(build)=arch/x86/entry/syscalls all 237 $(Q)$(MAKE) $(build)=arch/x86/entry/syscalls all
239 238
239archmacros:
240 $(Q)$(MAKE) $(build)=arch/x86/kernel arch/x86/kernel/macros.s
241
242ASM_MACRO_FLAGS = -Wa,arch/x86/kernel/macros.s -Wa,-
243export ASM_MACRO_FLAGS
244KBUILD_CFLAGS += $(ASM_MACRO_FLAGS)
245
240### 246###
241# Kernel objects 247# Kernel objects
242 248
diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig
index 0eb9f92f3717..6c3ab05c231d 100644
--- a/arch/x86/configs/i386_defconfig
+++ b/arch/x86/configs/i386_defconfig
@@ -247,6 +247,7 @@ CONFIG_USB_HIDDEV=y
247CONFIG_USB=y 247CONFIG_USB=y
248CONFIG_USB_ANNOUNCE_NEW_DEVICES=y 248CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
249CONFIG_USB_MON=y 249CONFIG_USB_MON=y
250CONFIG_USB_XHCI_HCD=y
250CONFIG_USB_EHCI_HCD=y 251CONFIG_USB_EHCI_HCD=y
251CONFIG_USB_EHCI_TT_NEWSCHED=y 252CONFIG_USB_EHCI_TT_NEWSCHED=y
252CONFIG_USB_OHCI_HCD=y 253CONFIG_USB_OHCI_HCD=y
diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig
index e32fc1f274d8..ac9ae487cfeb 100644
--- a/arch/x86/configs/x86_64_defconfig
+++ b/arch/x86/configs/x86_64_defconfig
@@ -243,6 +243,7 @@ CONFIG_USB_HIDDEV=y
243CONFIG_USB=y 243CONFIG_USB=y
244CONFIG_USB_ANNOUNCE_NEW_DEVICES=y 244CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
245CONFIG_USB_MON=y 245CONFIG_USB_MON=y
246CONFIG_USB_XHCI_HCD=y
246CONFIG_USB_EHCI_HCD=y 247CONFIG_USB_EHCI_HCD=y
247CONFIG_USB_EHCI_TT_NEWSCHED=y 248CONFIG_USB_EHCI_TT_NEWSCHED=y
248CONFIG_USB_OHCI_HCD=y 249CONFIG_USB_OHCI_HCD=y
diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
index 352e70cd33e8..708b46a54578 100644
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -338,7 +338,7 @@ For 32-bit we have the following conventions - kernel is built with
338.macro CALL_enter_from_user_mode 338.macro CALL_enter_from_user_mode
339#ifdef CONFIG_CONTEXT_TRACKING 339#ifdef CONFIG_CONTEXT_TRACKING
340#ifdef HAVE_JUMP_LABEL 340#ifdef HAVE_JUMP_LABEL
341 STATIC_JUMP_IF_FALSE .Lafter_call_\@, context_tracking_enabled, def=0 341 STATIC_BRANCH_JMP l_yes=.Lafter_call_\@, key=context_tracking_enabled, branch=1
342#endif 342#endif
343 call enter_from_user_mode 343 call enter_from_user_mode
344.Lafter_call_\@: 344.Lafter_call_\@:
diff --git a/arch/x86/include/asm/alternative-asm.h b/arch/x86/include/asm/alternative-asm.h
index 31b627b43a8e..8e4ea39e55d0 100644
--- a/arch/x86/include/asm/alternative-asm.h
+++ b/arch/x86/include/asm/alternative-asm.h
@@ -7,16 +7,24 @@
7#include <asm/asm.h> 7#include <asm/asm.h>
8 8
9#ifdef CONFIG_SMP 9#ifdef CONFIG_SMP
10 .macro LOCK_PREFIX 10.macro LOCK_PREFIX_HERE
11672: lock
12 .pushsection .smp_locks,"a" 11 .pushsection .smp_locks,"a"
13 .balign 4 12 .balign 4
14 .long 672b - . 13 .long 671f - . # offset
15 .popsection 14 .popsection
16 .endm 15671:
16.endm
17
18.macro LOCK_PREFIX insn:vararg
19 LOCK_PREFIX_HERE
20 lock \insn
21.endm
17#else 22#else
18 .macro LOCK_PREFIX 23.macro LOCK_PREFIX_HERE
19 .endm 24.endm
25
26.macro LOCK_PREFIX insn:vararg
27.endm
20#endif 28#endif
21 29
22/* 30/*
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 4cd6a3b71824..d7faa16622d8 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -31,15 +31,8 @@
31 */ 31 */
32 32
33#ifdef CONFIG_SMP 33#ifdef CONFIG_SMP
34#define LOCK_PREFIX_HERE \ 34#define LOCK_PREFIX_HERE "LOCK_PREFIX_HERE\n\t"
35 ".pushsection .smp_locks,\"a\"\n" \ 35#define LOCK_PREFIX "LOCK_PREFIX "
36 ".balign 4\n" \
37 ".long 671f - .\n" /* offset */ \
38 ".popsection\n" \
39 "671:"
40
41#define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; "
42
43#else /* ! CONFIG_SMP */ 36#else /* ! CONFIG_SMP */
44#define LOCK_PREFIX_HERE "" 37#define LOCK_PREFIX_HERE ""
45#define LOCK_PREFIX "" 38#define LOCK_PREFIX ""
diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h
index 990770f9e76b..21b086786404 100644
--- a/arch/x86/include/asm/asm.h
+++ b/arch/x86/include/asm/asm.h
@@ -120,16 +120,32 @@
120/* Exception table entry */ 120/* Exception table entry */
121#ifdef __ASSEMBLY__ 121#ifdef __ASSEMBLY__
122# define _ASM_EXTABLE_HANDLE(from, to, handler) \ 122# define _ASM_EXTABLE_HANDLE(from, to, handler) \
123 .pushsection "__ex_table","a" ; \ 123 ASM_EXTABLE_HANDLE from to handler
124 .balign 4 ; \ 124
125 .long (from) - . ; \ 125.macro ASM_EXTABLE_HANDLE from:req to:req handler:req
126 .long (to) - . ; \ 126 .pushsection "__ex_table","a"
127 .long (handler) - . ; \ 127 .balign 4
128 .long (\from) - .
129 .long (\to) - .
130 .long (\handler) - .
128 .popsection 131 .popsection
132.endm
133#else /* __ASSEMBLY__ */
134
135# define _ASM_EXTABLE_HANDLE(from, to, handler) \
136 "ASM_EXTABLE_HANDLE from=" #from " to=" #to \
137 " handler=\"" #handler "\"\n\t"
138
139/* For C file, we already have NOKPROBE_SYMBOL macro */
140
141#endif /* __ASSEMBLY__ */
129 142
130# define _ASM_EXTABLE(from, to) \ 143# define _ASM_EXTABLE(from, to) \
131 _ASM_EXTABLE_HANDLE(from, to, ex_handler_default) 144 _ASM_EXTABLE_HANDLE(from, to, ex_handler_default)
132 145
146# define _ASM_EXTABLE_UA(from, to) \
147 _ASM_EXTABLE_HANDLE(from, to, ex_handler_uaccess)
148
133# define _ASM_EXTABLE_FAULT(from, to) \ 149# define _ASM_EXTABLE_FAULT(from, to) \
134 _ASM_EXTABLE_HANDLE(from, to, ex_handler_fault) 150 _ASM_EXTABLE_HANDLE(from, to, ex_handler_fault)
135 151
@@ -145,6 +161,7 @@
145 _ASM_PTR (entry); \ 161 _ASM_PTR (entry); \
146 .popsection 162 .popsection
147 163
164#ifdef __ASSEMBLY__
148.macro ALIGN_DESTINATION 165.macro ALIGN_DESTINATION
149 /* check for bad alignment of destination */ 166 /* check for bad alignment of destination */
150 movl %edi,%ecx 167 movl %edi,%ecx
@@ -165,34 +182,10 @@
165 jmp copy_user_handle_tail 182 jmp copy_user_handle_tail
166 .previous 183 .previous
167 184
168 _ASM_EXTABLE(100b,103b) 185 _ASM_EXTABLE_UA(100b, 103b)
169 _ASM_EXTABLE(101b,103b) 186 _ASM_EXTABLE_UA(101b, 103b)
170 .endm 187 .endm
171 188#endif /* __ASSEMBLY__ */
172#else
173# define _EXPAND_EXTABLE_HANDLE(x) #x
174# define _ASM_EXTABLE_HANDLE(from, to, handler) \
175 " .pushsection \"__ex_table\",\"a\"\n" \
176 " .balign 4\n" \
177 " .long (" #from ") - .\n" \
178 " .long (" #to ") - .\n" \
179 " .long (" _EXPAND_EXTABLE_HANDLE(handler) ") - .\n" \
180 " .popsection\n"
181
182# define _ASM_EXTABLE(from, to) \
183 _ASM_EXTABLE_HANDLE(from, to, ex_handler_default)
184
185# define _ASM_EXTABLE_FAULT(from, to) \
186 _ASM_EXTABLE_HANDLE(from, to, ex_handler_fault)
187
188# define _ASM_EXTABLE_EX(from, to) \
189 _ASM_EXTABLE_HANDLE(from, to, ex_handler_ext)
190
191# define _ASM_EXTABLE_REFCOUNT(from, to) \
192 _ASM_EXTABLE_HANDLE(from, to, ex_handler_refcount)
193
194/* For C file, we already have NOKPROBE_SYMBOL macro */
195#endif
196 189
197#ifndef __ASSEMBLY__ 190#ifndef __ASSEMBLY__
198/* 191/*
diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h
index ce84388e540c..ea3d95275b43 100644
--- a/arch/x86/include/asm/atomic.h
+++ b/arch/x86/include/asm/atomic.h
@@ -82,7 +82,7 @@ static __always_inline void arch_atomic_sub(int i, atomic_t *v)
82 */ 82 */
83static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v) 83static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v)
84{ 84{
85 GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, "er", i, "%0", e); 85 return GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, e, "er", i);
86} 86}
87#define arch_atomic_sub_and_test arch_atomic_sub_and_test 87#define arch_atomic_sub_and_test arch_atomic_sub_and_test
88 88
@@ -122,7 +122,7 @@ static __always_inline void arch_atomic_dec(atomic_t *v)
122 */ 122 */
123static __always_inline bool arch_atomic_dec_and_test(atomic_t *v) 123static __always_inline bool arch_atomic_dec_and_test(atomic_t *v)
124{ 124{
125 GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, "%0", e); 125 return GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, e);
126} 126}
127#define arch_atomic_dec_and_test arch_atomic_dec_and_test 127#define arch_atomic_dec_and_test arch_atomic_dec_and_test
128 128
@@ -136,7 +136,7 @@ static __always_inline bool arch_atomic_dec_and_test(atomic_t *v)
136 */ 136 */
137static __always_inline bool arch_atomic_inc_and_test(atomic_t *v) 137static __always_inline bool arch_atomic_inc_and_test(atomic_t *v)
138{ 138{
139 GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, "%0", e); 139 return GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, e);
140} 140}
141#define arch_atomic_inc_and_test arch_atomic_inc_and_test 141#define arch_atomic_inc_and_test arch_atomic_inc_and_test
142 142
@@ -151,7 +151,7 @@ static __always_inline bool arch_atomic_inc_and_test(atomic_t *v)
151 */ 151 */
152static __always_inline bool arch_atomic_add_negative(int i, atomic_t *v) 152static __always_inline bool arch_atomic_add_negative(int i, atomic_t *v)
153{ 153{
154 GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, "er", i, "%0", s); 154 return GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, s, "er", i);
155} 155}
156#define arch_atomic_add_negative arch_atomic_add_negative 156#define arch_atomic_add_negative arch_atomic_add_negative
157 157
diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atomic64_64.h
index 5f851d92eecd..dadc20adba21 100644
--- a/arch/x86/include/asm/atomic64_64.h
+++ b/arch/x86/include/asm/atomic64_64.h
@@ -73,7 +73,7 @@ static inline void arch_atomic64_sub(long i, atomic64_t *v)
73 */ 73 */
74static inline bool arch_atomic64_sub_and_test(long i, atomic64_t *v) 74static inline bool arch_atomic64_sub_and_test(long i, atomic64_t *v)
75{ 75{
76 GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, "er", i, "%0", e); 76 return GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, e, "er", i);
77} 77}
78#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test 78#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
79 79
@@ -115,7 +115,7 @@ static __always_inline void arch_atomic64_dec(atomic64_t *v)
115 */ 115 */
116static inline bool arch_atomic64_dec_and_test(atomic64_t *v) 116static inline bool arch_atomic64_dec_and_test(atomic64_t *v)
117{ 117{
118 GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, "%0", e); 118 return GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, e);
119} 119}
120#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test 120#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
121 121
@@ -129,7 +129,7 @@ static inline bool arch_atomic64_dec_and_test(atomic64_t *v)
129 */ 129 */
130static inline bool arch_atomic64_inc_and_test(atomic64_t *v) 130static inline bool arch_atomic64_inc_and_test(atomic64_t *v)
131{ 131{
132 GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, "%0", e); 132 return GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, e);
133} 133}
134#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test 134#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
135 135
@@ -144,7 +144,7 @@ static inline bool arch_atomic64_inc_and_test(atomic64_t *v)
144 */ 144 */
145static inline bool arch_atomic64_add_negative(long i, atomic64_t *v) 145static inline bool arch_atomic64_add_negative(long i, atomic64_t *v)
146{ 146{
147 GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, "er", i, "%0", s); 147 return GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, s, "er", i);
148} 148}
149#define arch_atomic64_add_negative arch_atomic64_add_negative 149#define arch_atomic64_add_negative arch_atomic64_add_negative
150 150
diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index 9f645ba57dbb..124f9195eb3e 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -217,8 +217,7 @@ static __always_inline void change_bit(long nr, volatile unsigned long *addr)
217 */ 217 */
218static __always_inline bool test_and_set_bit(long nr, volatile unsigned long *addr) 218static __always_inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
219{ 219{
220 GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(bts), 220 return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(bts), *addr, c, "Ir", nr);
221 *addr, "Ir", nr, "%0", c);
222} 221}
223 222
224/** 223/**
@@ -264,8 +263,7 @@ static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long *
264 */ 263 */
265static __always_inline bool test_and_clear_bit(long nr, volatile unsigned long *addr) 264static __always_inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
266{ 265{
267 GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btr), 266 return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btr), *addr, c, "Ir", nr);
268 *addr, "Ir", nr, "%0", c);
269} 267}
270 268
271/** 269/**
@@ -318,8 +316,7 @@ static __always_inline bool __test_and_change_bit(long nr, volatile unsigned lon
318 */ 316 */
319static __always_inline bool test_and_change_bit(long nr, volatile unsigned long *addr) 317static __always_inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
320{ 318{
321 GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btc), 319 return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btc), *addr, c, "Ir", nr);
322 *addr, "Ir", nr, "%0", c);
323} 320}
324 321
325static __always_inline bool constant_test_bit(long nr, const volatile unsigned long *addr) 322static __always_inline bool constant_test_bit(long nr, const volatile unsigned long *addr)
diff --git a/arch/x86/include/asm/bug.h b/arch/x86/include/asm/bug.h
index 6804d6642767..5090035e6d16 100644
--- a/arch/x86/include/asm/bug.h
+++ b/arch/x86/include/asm/bug.h
@@ -4,6 +4,8 @@
4 4
5#include <linux/stringify.h> 5#include <linux/stringify.h>
6 6
7#ifndef __ASSEMBLY__
8
7/* 9/*
8 * Despite that some emulators terminate on UD2, we use it for WARN(). 10 * Despite that some emulators terminate on UD2, we use it for WARN().
9 * 11 *
@@ -20,53 +22,15 @@
20 22
21#define LEN_UD2 2 23#define LEN_UD2 2
22 24
23#ifdef CONFIG_GENERIC_BUG
24
25#ifdef CONFIG_X86_32
26# define __BUG_REL(val) ".long " __stringify(val)
27#else
28# define __BUG_REL(val) ".long " __stringify(val) " - 2b"
29#endif
30
31#ifdef CONFIG_DEBUG_BUGVERBOSE
32
33#define _BUG_FLAGS(ins, flags) \
34do { \
35 asm volatile("1:\t" ins "\n" \
36 ".pushsection __bug_table,\"aw\"\n" \
37 "2:\t" __BUG_REL(1b) "\t# bug_entry::bug_addr\n" \
38 "\t" __BUG_REL(%c0) "\t# bug_entry::file\n" \
39 "\t.word %c1" "\t# bug_entry::line\n" \
40 "\t.word %c2" "\t# bug_entry::flags\n" \
41 "\t.org 2b+%c3\n" \
42 ".popsection" \
43 : : "i" (__FILE__), "i" (__LINE__), \
44 "i" (flags), \
45 "i" (sizeof(struct bug_entry))); \
46} while (0)
47
48#else /* !CONFIG_DEBUG_BUGVERBOSE */
49
50#define _BUG_FLAGS(ins, flags) \ 25#define _BUG_FLAGS(ins, flags) \
51do { \ 26do { \
52 asm volatile("1:\t" ins "\n" \ 27 asm volatile("ASM_BUG ins=\"" ins "\" file=%c0 line=%c1 " \
53 ".pushsection __bug_table,\"aw\"\n" \ 28 "flags=%c2 size=%c3" \
54 "2:\t" __BUG_REL(1b) "\t# bug_entry::bug_addr\n" \ 29 : : "i" (__FILE__), "i" (__LINE__), \
55 "\t.word %c0" "\t# bug_entry::flags\n" \ 30 "i" (flags), \
56 "\t.org 2b+%c1\n" \
57 ".popsection" \
58 : : "i" (flags), \
59 "i" (sizeof(struct bug_entry))); \ 31 "i" (sizeof(struct bug_entry))); \
60} while (0) 32} while (0)
61 33
62#endif /* CONFIG_DEBUG_BUGVERBOSE */
63
64#else
65
66#define _BUG_FLAGS(ins, flags) asm volatile(ins)
67
68#endif /* CONFIG_GENERIC_BUG */
69
70#define HAVE_ARCH_BUG 34#define HAVE_ARCH_BUG
71#define BUG() \ 35#define BUG() \
72do { \ 36do { \
@@ -82,4 +46,54 @@ do { \
82 46
83#include <asm-generic/bug.h> 47#include <asm-generic/bug.h>
84 48
49#else /* __ASSEMBLY__ */
50
51#ifdef CONFIG_GENERIC_BUG
52
53#ifdef CONFIG_X86_32
54.macro __BUG_REL val:req
55 .long \val
56.endm
57#else
58.macro __BUG_REL val:req
59 .long \val - 2b
60.endm
61#endif
62
63#ifdef CONFIG_DEBUG_BUGVERBOSE
64
65.macro ASM_BUG ins:req file:req line:req flags:req size:req
661: \ins
67 .pushsection __bug_table,"aw"
682: __BUG_REL val=1b # bug_entry::bug_addr
69 __BUG_REL val=\file # bug_entry::file
70 .word \line # bug_entry::line
71 .word \flags # bug_entry::flags
72 .org 2b+\size
73 .popsection
74.endm
75
76#else /* !CONFIG_DEBUG_BUGVERBOSE */
77
78.macro ASM_BUG ins:req file:req line:req flags:req size:req
791: \ins
80 .pushsection __bug_table,"aw"
812: __BUG_REL val=1b # bug_entry::bug_addr
82 .word \flags # bug_entry::flags
83 .org 2b+\size
84 .popsection
85.endm
86
87#endif /* CONFIG_DEBUG_BUGVERBOSE */
88
89#else /* CONFIG_GENERIC_BUG */
90
91.macro ASM_BUG ins:req file:req line:req flags:req size:req
92 \ins
93.endm
94
95#endif /* CONFIG_GENERIC_BUG */
96
97#endif /* __ASSEMBLY__ */
98
85#endif /* _ASM_X86_BUG_H */ 99#endif /* _ASM_X86_BUG_H */
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index aced6c9290d6..7d442722ef24 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -2,10 +2,10 @@
2#ifndef _ASM_X86_CPUFEATURE_H 2#ifndef _ASM_X86_CPUFEATURE_H
3#define _ASM_X86_CPUFEATURE_H 3#define _ASM_X86_CPUFEATURE_H
4 4
5#include <asm/processor.h> 5#ifdef __KERNEL__
6 6#ifndef __ASSEMBLY__
7#if defined(__KERNEL__) && !defined(__ASSEMBLY__)
8 7
8#include <asm/processor.h>
9#include <asm/asm.h> 9#include <asm/asm.h>
10#include <linux/bitops.h> 10#include <linux/bitops.h>
11 11
@@ -161,37 +161,10 @@ extern void clear_cpu_cap(struct cpuinfo_x86 *c, unsigned int bit);
161 */ 161 */
162static __always_inline __pure bool _static_cpu_has(u16 bit) 162static __always_inline __pure bool _static_cpu_has(u16 bit)
163{ 163{
164 asm_volatile_goto("1: jmp 6f\n" 164 asm_volatile_goto("STATIC_CPU_HAS bitnum=%[bitnum] "
165 "2:\n" 165 "cap_byte=\"%[cap_byte]\" "
166 ".skip -(((5f-4f) - (2b-1b)) > 0) * " 166 "feature=%P[feature] t_yes=%l[t_yes] "
167 "((5f-4f) - (2b-1b)),0x90\n" 167 "t_no=%l[t_no] always=%P[always]"
168 "3:\n"
169 ".section .altinstructions,\"a\"\n"
170 " .long 1b - .\n" /* src offset */
171 " .long 4f - .\n" /* repl offset */
172 " .word %P[always]\n" /* always replace */
173 " .byte 3b - 1b\n" /* src len */
174 " .byte 5f - 4f\n" /* repl len */
175 " .byte 3b - 2b\n" /* pad len */
176 ".previous\n"
177 ".section .altinstr_replacement,\"ax\"\n"
178 "4: jmp %l[t_no]\n"
179 "5:\n"
180 ".previous\n"
181 ".section .altinstructions,\"a\"\n"
182 " .long 1b - .\n" /* src offset */
183 " .long 0\n" /* no replacement */
184 " .word %P[feature]\n" /* feature bit */
185 " .byte 3b - 1b\n" /* src len */
186 " .byte 0\n" /* repl len */
187 " .byte 0\n" /* pad len */
188 ".previous\n"
189 ".section .altinstr_aux,\"ax\"\n"
190 "6:\n"
191 " testb %[bitnum],%[cap_byte]\n"
192 " jnz %l[t_yes]\n"
193 " jmp %l[t_no]\n"
194 ".previous\n"
195 : : [feature] "i" (bit), 168 : : [feature] "i" (bit),
196 [always] "i" (X86_FEATURE_ALWAYS), 169 [always] "i" (X86_FEATURE_ALWAYS),
197 [bitnum] "i" (1 << (bit & 7)), 170 [bitnum] "i" (1 << (bit & 7)),
@@ -226,5 +199,44 @@ t_no:
226#define CPU_FEATURE_TYPEVAL boot_cpu_data.x86_vendor, boot_cpu_data.x86, \ 199#define CPU_FEATURE_TYPEVAL boot_cpu_data.x86_vendor, boot_cpu_data.x86, \
227 boot_cpu_data.x86_model 200 boot_cpu_data.x86_model
228 201
229#endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */ 202#else /* __ASSEMBLY__ */
203
204.macro STATIC_CPU_HAS bitnum:req cap_byte:req feature:req t_yes:req t_no:req always:req
2051:
206 jmp 6f
2072:
208 .skip -(((5f-4f) - (2b-1b)) > 0) * ((5f-4f) - (2b-1b)),0x90
2093:
210 .section .altinstructions,"a"
211 .long 1b - . /* src offset */
212 .long 4f - . /* repl offset */
213 .word \always /* always replace */
214 .byte 3b - 1b /* src len */
215 .byte 5f - 4f /* repl len */
216 .byte 3b - 2b /* pad len */
217 .previous
218 .section .altinstr_replacement,"ax"
2194:
220 jmp \t_no
2215:
222 .previous
223 .section .altinstructions,"a"
224 .long 1b - . /* src offset */
225 .long 0 /* no replacement */
226 .word \feature /* feature bit */
227 .byte 3b - 1b /* src len */
228 .byte 0 /* repl len */
229 .byte 0 /* pad len */
230 .previous
231 .section .altinstr_aux,"ax"
2326:
233 testb \bitnum,\cap_byte
234 jnz \t_yes
235 jmp \t_no
236 .previous
237.endm
238
239#endif /* __ASSEMBLY__ */
240
241#endif /* __KERNEL__ */
230#endif /* _ASM_X86_CPUFEATURE_H */ 242#endif /* _ASM_X86_CPUFEATURE_H */
diff --git a/arch/x86/include/asm/elf.h b/arch/x86/include/asm/elf.h
index 0d157d2a1e2a..a357031d85b5 100644
--- a/arch/x86/include/asm/elf.h
+++ b/arch/x86/include/asm/elf.h
@@ -62,8 +62,7 @@ typedef struct user_fxsr_struct elf_fpxregset_t;
62#define R_X86_64_PC16 13 /* 16 bit sign extended pc relative */ 62#define R_X86_64_PC16 13 /* 16 bit sign extended pc relative */
63#define R_X86_64_8 14 /* Direct 8 bit sign extended */ 63#define R_X86_64_8 14 /* Direct 8 bit sign extended */
64#define R_X86_64_PC8 15 /* 8 bit sign extended pc relative */ 64#define R_X86_64_PC8 15 /* 8 bit sign extended pc relative */
65 65#define R_X86_64_PC64 24 /* Place relative 64-bit signed */
66#define R_X86_64_NUM 16
67 66
68/* 67/*
69 * These are used to set parameters in the core dumps. 68 * These are used to set parameters in the core dumps.
diff --git a/arch/x86/include/asm/extable.h b/arch/x86/include/asm/extable.h
index f9c3a5d502f4..d8c2198d543b 100644
--- a/arch/x86/include/asm/extable.h
+++ b/arch/x86/include/asm/extable.h
@@ -29,7 +29,8 @@ struct pt_regs;
29 (b)->handler = (tmp).handler - (delta); \ 29 (b)->handler = (tmp).handler - (delta); \
30 } while (0) 30 } while (0)
31 31
32extern int fixup_exception(struct pt_regs *regs, int trapnr); 32extern int fixup_exception(struct pt_regs *regs, int trapnr,
33 unsigned long error_code, unsigned long fault_addr);
33extern int fixup_bug(struct pt_regs *regs, int trapnr); 34extern int fixup_bug(struct pt_regs *regs, int trapnr);
34extern bool ex_has_fault_handler(unsigned long ip); 35extern bool ex_has_fault_handler(unsigned long ip);
35extern void early_fixup_exception(struct pt_regs *regs, int trapnr); 36extern void early_fixup_exception(struct pt_regs *regs, int trapnr);
diff --git a/arch/x86/include/asm/fpu/internal.h b/arch/x86/include/asm/fpu/internal.h
index 69dcdf195b61..5f7290e6e954 100644
--- a/arch/x86/include/asm/fpu/internal.h
+++ b/arch/x86/include/asm/fpu/internal.h
@@ -226,7 +226,7 @@ static inline void copy_fxregs_to_kernel(struct fpu *fpu)
226 "3: movl $-2,%[err]\n\t" \ 226 "3: movl $-2,%[err]\n\t" \
227 "jmp 2b\n\t" \ 227 "jmp 2b\n\t" \
228 ".popsection\n\t" \ 228 ".popsection\n\t" \
229 _ASM_EXTABLE(1b, 3b) \ 229 _ASM_EXTABLE_UA(1b, 3b) \
230 : [err] "=r" (err) \ 230 : [err] "=r" (err) \
231 : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ 231 : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \
232 : "memory") 232 : "memory")
diff --git a/arch/x86/include/asm/futex.h b/arch/x86/include/asm/futex.h
index de4d68852d3a..13c83fe97988 100644
--- a/arch/x86/include/asm/futex.h
+++ b/arch/x86/include/asm/futex.h
@@ -20,7 +20,7 @@
20 "3:\tmov\t%3, %1\n" \ 20 "3:\tmov\t%3, %1\n" \
21 "\tjmp\t2b\n" \ 21 "\tjmp\t2b\n" \
22 "\t.previous\n" \ 22 "\t.previous\n" \
23 _ASM_EXTABLE(1b, 3b) \ 23 _ASM_EXTABLE_UA(1b, 3b) \
24 : "=r" (oldval), "=r" (ret), "+m" (*uaddr) \ 24 : "=r" (oldval), "=r" (ret), "+m" (*uaddr) \
25 : "i" (-EFAULT), "0" (oparg), "1" (0)) 25 : "i" (-EFAULT), "0" (oparg), "1" (0))
26 26
@@ -36,8 +36,8 @@
36 "4:\tmov\t%5, %1\n" \ 36 "4:\tmov\t%5, %1\n" \
37 "\tjmp\t3b\n" \ 37 "\tjmp\t3b\n" \
38 "\t.previous\n" \ 38 "\t.previous\n" \
39 _ASM_EXTABLE(1b, 4b) \ 39 _ASM_EXTABLE_UA(1b, 4b) \
40 _ASM_EXTABLE(2b, 4b) \ 40 _ASM_EXTABLE_UA(2b, 4b) \
41 : "=&a" (oldval), "=&r" (ret), \ 41 : "=&a" (oldval), "=&r" (ret), \
42 "+m" (*uaddr), "=&r" (tem) \ 42 "+m" (*uaddr), "=&r" (tem) \
43 : "r" (oparg), "i" (-EFAULT), "1" (0)) 43 : "r" (oparg), "i" (-EFAULT), "1" (0))
diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
index 8c0de4282659..a5fb34fe56a4 100644
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -2,19 +2,6 @@
2#ifndef _ASM_X86_JUMP_LABEL_H 2#ifndef _ASM_X86_JUMP_LABEL_H
3#define _ASM_X86_JUMP_LABEL_H 3#define _ASM_X86_JUMP_LABEL_H
4 4
5#ifndef HAVE_JUMP_LABEL
6/*
7 * For better or for worse, if jump labels (the gcc extension) are missing,
8 * then the entire static branch patching infrastructure is compiled out.
9 * If that happens, the code in here will malfunction. Raise a compiler
10 * error instead.
11 *
12 * In theory, jump labels and the static branch patching infrastructure
13 * could be decoupled to fix this.
14 */
15#error asm/jump_label.h included on a non-jump-label kernel
16#endif
17
18#define JUMP_LABEL_NOP_SIZE 5 5#define JUMP_LABEL_NOP_SIZE 5
19 6
20#ifdef CONFIG_X86_64 7#ifdef CONFIG_X86_64
@@ -33,14 +20,9 @@
33 20
34static __always_inline bool arch_static_branch(struct static_key *key, bool branch) 21static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
35{ 22{
36 asm_volatile_goto("1:" 23 asm_volatile_goto("STATIC_BRANCH_NOP l_yes=\"%l[l_yes]\" key=\"%c0\" "
37 ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" 24 "branch=\"%c1\""
38 ".pushsection __jump_table, \"aw\" \n\t" 25 : : "i" (key), "i" (branch) : : l_yes);
39 _ASM_ALIGN "\n\t"
40 _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
41 ".popsection \n\t"
42 : : "i" (key), "i" (branch) : : l_yes);
43
44 return false; 26 return false;
45l_yes: 27l_yes:
46 return true; 28 return true;
@@ -48,13 +30,8 @@ l_yes:
48 30
49static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) 31static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
50{ 32{
51 asm_volatile_goto("1:" 33 asm_volatile_goto("STATIC_BRANCH_JMP l_yes=\"%l[l_yes]\" key=\"%c0\" "
52 ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t" 34 "branch=\"%c1\""
53 "2:\n\t"
54 ".pushsection __jump_table, \"aw\" \n\t"
55 _ASM_ALIGN "\n\t"
56 _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
57 ".popsection \n\t"
58 : : "i" (key), "i" (branch) : : l_yes); 35 : : "i" (key), "i" (branch) : : l_yes);
59 36
60 return false; 37 return false;
@@ -62,49 +39,28 @@ l_yes:
62 return true; 39 return true;
63} 40}
64 41
65#ifdef CONFIG_X86_64
66typedef u64 jump_label_t;
67#else
68typedef u32 jump_label_t;
69#endif
70
71struct jump_entry {
72 jump_label_t code;
73 jump_label_t target;
74 jump_label_t key;
75};
76
77#else /* __ASSEMBLY__ */ 42#else /* __ASSEMBLY__ */
78 43
79.macro STATIC_JUMP_IF_TRUE target, key, def 44.macro STATIC_BRANCH_NOP l_yes:req key:req branch:req
80.Lstatic_jump_\@: 45.Lstatic_branch_nop_\@:
81 .if \def 46 .byte STATIC_KEY_INIT_NOP
82 /* Equivalent to "jmp.d32 \target" */ 47.Lstatic_branch_no_after_\@:
83 .byte 0xe9
84 .long \target - .Lstatic_jump_after_\@
85.Lstatic_jump_after_\@:
86 .else
87 .byte STATIC_KEY_INIT_NOP
88 .endif
89 .pushsection __jump_table, "aw" 48 .pushsection __jump_table, "aw"
90 _ASM_ALIGN 49 _ASM_ALIGN
91 _ASM_PTR .Lstatic_jump_\@, \target, \key 50 .long .Lstatic_branch_nop_\@ - ., \l_yes - .
51 _ASM_PTR \key + \branch - .
92 .popsection 52 .popsection
93.endm 53.endm
94 54
95.macro STATIC_JUMP_IF_FALSE target, key, def 55.macro STATIC_BRANCH_JMP l_yes:req key:req branch:req
96.Lstatic_jump_\@: 56.Lstatic_branch_jmp_\@:
97 .if \def 57 .byte 0xe9
98 .byte STATIC_KEY_INIT_NOP 58 .long \l_yes - .Lstatic_branch_jmp_after_\@
99 .else 59.Lstatic_branch_jmp_after_\@:
100 /* Equivalent to "jmp.d32 \target" */
101 .byte 0xe9
102 .long \target - .Lstatic_jump_after_\@
103.Lstatic_jump_after_\@:
104 .endif
105 .pushsection __jump_table, "aw" 60 .pushsection __jump_table, "aw"
106 _ASM_ALIGN 61 _ASM_ALIGN
107 _ASM_PTR .Lstatic_jump_\@, \target, \key + 1 62 .long .Lstatic_branch_jmp_\@ - ., \l_yes - .
63 _ASM_PTR \key + \branch - .
108 .popsection 64 .popsection
109.endm 65.endm
110 66
diff --git a/arch/x86/include/asm/local.h b/arch/x86/include/asm/local.h
index c91083c59845..349a47acaa4a 100644
--- a/arch/x86/include/asm/local.h
+++ b/arch/x86/include/asm/local.h
@@ -53,7 +53,7 @@ static inline void local_sub(long i, local_t *l)
53 */ 53 */
54static inline bool local_sub_and_test(long i, local_t *l) 54static inline bool local_sub_and_test(long i, local_t *l)
55{ 55{
56 GEN_BINARY_RMWcc(_ASM_SUB, l->a.counter, "er", i, "%0", e); 56 return GEN_BINARY_RMWcc(_ASM_SUB, l->a.counter, e, "er", i);
57} 57}
58 58
59/** 59/**
@@ -66,7 +66,7 @@ static inline bool local_sub_and_test(long i, local_t *l)
66 */ 66 */
67static inline bool local_dec_and_test(local_t *l) 67static inline bool local_dec_and_test(local_t *l)
68{ 68{
69 GEN_UNARY_RMWcc(_ASM_DEC, l->a.counter, "%0", e); 69 return GEN_UNARY_RMWcc(_ASM_DEC, l->a.counter, e);
70} 70}
71 71
72/** 72/**
@@ -79,7 +79,7 @@ static inline bool local_dec_and_test(local_t *l)
79 */ 79 */
80static inline bool local_inc_and_test(local_t *l) 80static inline bool local_inc_and_test(local_t *l)
81{ 81{
82 GEN_UNARY_RMWcc(_ASM_INC, l->a.counter, "%0", e); 82 return GEN_UNARY_RMWcc(_ASM_INC, l->a.counter, e);
83} 83}
84 84
85/** 85/**
@@ -93,7 +93,7 @@ static inline bool local_inc_and_test(local_t *l)
93 */ 93 */
94static inline bool local_add_negative(long i, local_t *l) 94static inline bool local_add_negative(long i, local_t *l)
95{ 95{
96 GEN_BINARY_RMWcc(_ASM_ADD, l->a.counter, "er", i, "%0", s); 96 return GEN_BINARY_RMWcc(_ASM_ADD, l->a.counter, s, "er", i);
97} 97}
98 98
99/** 99/**
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 4b75acc23b30..83ce282eed0a 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -346,23 +346,11 @@ extern struct pv_lock_ops pv_lock_ops;
346#define paravirt_clobber(clobber) \ 346#define paravirt_clobber(clobber) \
347 [paravirt_clobber] "i" (clobber) 347 [paravirt_clobber] "i" (clobber)
348 348
349/*
350 * Generate some code, and mark it as patchable by the
351 * apply_paravirt() alternate instruction patcher.
352 */
353#define _paravirt_alt(insn_string, type, clobber) \
354 "771:\n\t" insn_string "\n" "772:\n" \
355 ".pushsection .parainstructions,\"a\"\n" \
356 _ASM_ALIGN "\n" \
357 _ASM_PTR " 771b\n" \
358 " .byte " type "\n" \
359 " .byte 772b-771b\n" \
360 " .short " clobber "\n" \
361 ".popsection\n"
362
363/* Generate patchable code, with the default asm parameters. */ 349/* Generate patchable code, with the default asm parameters. */
364#define paravirt_alt(insn_string) \ 350#define paravirt_call \
365 _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]") 351 "PARAVIRT_CALL type=\"%c[paravirt_typenum]\"" \
352 " clobber=\"%c[paravirt_clobber]\"" \
353 " pv_opptr=\"%c[paravirt_opptr]\";"
366 354
367/* Simple instruction patching code. */ 355/* Simple instruction patching code. */
368#define NATIVE_LABEL(a,x,b) "\n\t.globl " a #x "_" #b "\n" a #x "_" #b ":\n\t" 356#define NATIVE_LABEL(a,x,b) "\n\t.globl " a #x "_" #b "\n" a #x "_" #b ":\n\t"
@@ -391,16 +379,6 @@ unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
391int paravirt_disable_iospace(void); 379int paravirt_disable_iospace(void);
392 380
393/* 381/*
394 * This generates an indirect call based on the operation type number.
395 * The type number, computed in PARAVIRT_PATCH, is derived from the
396 * offset into the paravirt_patch_template structure, and can therefore be
397 * freely converted back into a structure offset.
398 */
399#define PARAVIRT_CALL \
400 ANNOTATE_RETPOLINE_SAFE \
401 "call *%c[paravirt_opptr];"
402
403/*
404 * These macros are intended to wrap calls through one of the paravirt 382 * These macros are intended to wrap calls through one of the paravirt
405 * ops structs, so that they can be later identified and patched at 383 * ops structs, so that they can be later identified and patched at
406 * runtime. 384 * runtime.
@@ -537,7 +515,7 @@ int paravirt_disable_iospace(void);
537 /* since this condition will never hold */ \ 515 /* since this condition will never hold */ \
538 if (sizeof(rettype) > sizeof(unsigned long)) { \ 516 if (sizeof(rettype) > sizeof(unsigned long)) { \
539 asm volatile(pre \ 517 asm volatile(pre \
540 paravirt_alt(PARAVIRT_CALL) \ 518 paravirt_call \
541 post \ 519 post \
542 : call_clbr, ASM_CALL_CONSTRAINT \ 520 : call_clbr, ASM_CALL_CONSTRAINT \
543 : paravirt_type(op), \ 521 : paravirt_type(op), \
@@ -547,7 +525,7 @@ int paravirt_disable_iospace(void);
547 __ret = (rettype)((((u64)__edx) << 32) | __eax); \ 525 __ret = (rettype)((((u64)__edx) << 32) | __eax); \
548 } else { \ 526 } else { \
549 asm volatile(pre \ 527 asm volatile(pre \
550 paravirt_alt(PARAVIRT_CALL) \ 528 paravirt_call \
551 post \ 529 post \
552 : call_clbr, ASM_CALL_CONSTRAINT \ 530 : call_clbr, ASM_CALL_CONSTRAINT \
553 : paravirt_type(op), \ 531 : paravirt_type(op), \
@@ -574,7 +552,7 @@ int paravirt_disable_iospace(void);
574 PVOP_VCALL_ARGS; \ 552 PVOP_VCALL_ARGS; \
575 PVOP_TEST_NULL(op); \ 553 PVOP_TEST_NULL(op); \
576 asm volatile(pre \ 554 asm volatile(pre \
577 paravirt_alt(PARAVIRT_CALL) \ 555 paravirt_call \
578 post \ 556 post \
579 : call_clbr, ASM_CALL_CONSTRAINT \ 557 : call_clbr, ASM_CALL_CONSTRAINT \
580 : paravirt_type(op), \ 558 : paravirt_type(op), \
@@ -694,6 +672,26 @@ struct paravirt_patch_site {
694extern struct paravirt_patch_site __parainstructions[], 672extern struct paravirt_patch_site __parainstructions[],
695 __parainstructions_end[]; 673 __parainstructions_end[];
696 674
675#else /* __ASSEMBLY__ */
676
677/*
678 * This generates an indirect call based on the operation type number.
679 * The type number, computed in PARAVIRT_PATCH, is derived from the
680 * offset into the paravirt_patch_template structure, and can therefore be
681 * freely converted back into a structure offset.
682 */
683.macro PARAVIRT_CALL type:req clobber:req pv_opptr:req
684771: ANNOTATE_RETPOLINE_SAFE
685 call *\pv_opptr
686772: .pushsection .parainstructions,"a"
687 _ASM_ALIGN
688 _ASM_PTR 771b
689 .byte \type
690 .byte 772b-771b
691 .short \clobber
692 .popsection
693.endm
694
697#endif /* __ASSEMBLY__ */ 695#endif /* __ASSEMBLY__ */
698 696
699#endif /* _ASM_X86_PARAVIRT_TYPES_H */ 697#endif /* _ASM_X86_PARAVIRT_TYPES_H */
diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 7f2dbd91fc74..90cb2f36c042 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -88,7 +88,7 @@ static __always_inline void __preempt_count_sub(int val)
88 */ 88 */
89static __always_inline bool __preempt_count_dec_and_test(void) 89static __always_inline bool __preempt_count_dec_and_test(void)
90{ 90{
91 GEN_UNARY_RMWcc("decl", __preempt_count, __percpu_arg(0), e); 91 return GEN_UNARY_RMWcc("decl", __preempt_count, e, __percpu_arg([var]));
92} 92}
93 93
94/* 94/*
diff --git a/arch/x86/include/asm/ptrace.h b/arch/x86/include/asm/ptrace.h
index 6de1fd3d0097..5e58a74bfd3a 100644
--- a/arch/x86/include/asm/ptrace.h
+++ b/arch/x86/include/asm/ptrace.h
@@ -37,8 +37,10 @@ struct pt_regs {
37 unsigned short __esh; 37 unsigned short __esh;
38 unsigned short fs; 38 unsigned short fs;
39 unsigned short __fsh; 39 unsigned short __fsh;
40 /* On interrupt, gs and __gsh store the vector number. */
40 unsigned short gs; 41 unsigned short gs;
41 unsigned short __gsh; 42 unsigned short __gsh;
43 /* On interrupt, this is the error code. */
42 unsigned long orig_ax; 44 unsigned long orig_ax;
43 unsigned long ip; 45 unsigned long ip;
44 unsigned short cs; 46 unsigned short cs;
diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h
index 3e70bed8a978..87623c6b13db 100644
--- a/arch/x86/include/asm/qspinlock.h
+++ b/arch/x86/include/asm/qspinlock.h
@@ -6,9 +6,24 @@
6#include <asm/cpufeature.h> 6#include <asm/cpufeature.h>
7#include <asm-generic/qspinlock_types.h> 7#include <asm-generic/qspinlock_types.h>
8#include <asm/paravirt.h> 8#include <asm/paravirt.h>
9#include <asm/rmwcc.h>
9 10
10#define _Q_PENDING_LOOPS (1 << 9) 11#define _Q_PENDING_LOOPS (1 << 9)
11 12
13#define queued_fetch_set_pending_acquire queued_fetch_set_pending_acquire
14static __always_inline u32 queued_fetch_set_pending_acquire(struct qspinlock *lock)
15{
16 u32 val = 0;
17
18 if (GEN_BINARY_RMWcc(LOCK_PREFIX "btsl", lock->val.counter, c,
19 "I", _Q_PENDING_OFFSET))
20 val |= _Q_PENDING_VAL;
21
22 val |= atomic_read(&lock->val) & ~_Q_PENDING_MASK;
23
24 return val;
25}
26
12#ifdef CONFIG_PARAVIRT_SPINLOCKS 27#ifdef CONFIG_PARAVIRT_SPINLOCKS
13extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); 28extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
14extern void __pv_init_lock_hash(void); 29extern void __pv_init_lock_hash(void);
diff --git a/arch/x86/include/asm/refcount.h b/arch/x86/include/asm/refcount.h
index 19b90521954c..a8b5e1e13319 100644
--- a/arch/x86/include/asm/refcount.h
+++ b/arch/x86/include/asm/refcount.h
@@ -4,6 +4,41 @@
4 * x86-specific implementation of refcount_t. Based on PAX_REFCOUNT from 4 * x86-specific implementation of refcount_t. Based on PAX_REFCOUNT from
5 * PaX/grsecurity. 5 * PaX/grsecurity.
6 */ 6 */
7
8#ifdef __ASSEMBLY__
9
10#include <asm/asm.h>
11#include <asm/bug.h>
12
13.macro REFCOUNT_EXCEPTION counter:req
14 .pushsection .text..refcount
15111: lea \counter, %_ASM_CX
16112: ud2
17 ASM_UNREACHABLE
18 .popsection
19113: _ASM_EXTABLE_REFCOUNT(112b, 113b)
20.endm
21
22/* Trigger refcount exception if refcount result is negative. */
23.macro REFCOUNT_CHECK_LT_ZERO counter:req
24 js 111f
25 REFCOUNT_EXCEPTION counter="\counter"
26.endm
27
28/* Trigger refcount exception if refcount result is zero or negative. */
29.macro REFCOUNT_CHECK_LE_ZERO counter:req
30 jz 111f
31 REFCOUNT_CHECK_LT_ZERO counter="\counter"
32.endm
33
34/* Trigger refcount exception unconditionally. */
35.macro REFCOUNT_ERROR counter:req
36 jmp 111f
37 REFCOUNT_EXCEPTION counter="\counter"
38.endm
39
40#else /* __ASSEMBLY__ */
41
7#include <linux/refcount.h> 42#include <linux/refcount.h>
8#include <asm/bug.h> 43#include <asm/bug.h>
9 44
@@ -15,34 +50,11 @@
15 * central refcount exception. The fixup address for the exception points 50 * central refcount exception. The fixup address for the exception points
16 * back to the regular execution flow in .text. 51 * back to the regular execution flow in .text.
17 */ 52 */
18#define _REFCOUNT_EXCEPTION \
19 ".pushsection .text..refcount\n" \
20 "111:\tlea %[counter], %%" _ASM_CX "\n" \
21 "112:\t" ASM_UD2 "\n" \
22 ASM_UNREACHABLE \
23 ".popsection\n" \
24 "113:\n" \
25 _ASM_EXTABLE_REFCOUNT(112b, 113b)
26
27/* Trigger refcount exception if refcount result is negative. */
28#define REFCOUNT_CHECK_LT_ZERO \
29 "js 111f\n\t" \
30 _REFCOUNT_EXCEPTION
31
32/* Trigger refcount exception if refcount result is zero or negative. */
33#define REFCOUNT_CHECK_LE_ZERO \
34 "jz 111f\n\t" \
35 REFCOUNT_CHECK_LT_ZERO
36
37/* Trigger refcount exception unconditionally. */
38#define REFCOUNT_ERROR \
39 "jmp 111f\n\t" \
40 _REFCOUNT_EXCEPTION
41 53
42static __always_inline void refcount_add(unsigned int i, refcount_t *r) 54static __always_inline void refcount_add(unsigned int i, refcount_t *r)
43{ 55{
44 asm volatile(LOCK_PREFIX "addl %1,%0\n\t" 56 asm volatile(LOCK_PREFIX "addl %1,%0\n\t"
45 REFCOUNT_CHECK_LT_ZERO 57 "REFCOUNT_CHECK_LT_ZERO counter=\"%[counter]\""
46 : [counter] "+m" (r->refs.counter) 58 : [counter] "+m" (r->refs.counter)
47 : "ir" (i) 59 : "ir" (i)
48 : "cc", "cx"); 60 : "cc", "cx");
@@ -51,7 +63,7 @@ static __always_inline void refcount_add(unsigned int i, refcount_t *r)
51static __always_inline void refcount_inc(refcount_t *r) 63static __always_inline void refcount_inc(refcount_t *r)
52{ 64{
53 asm volatile(LOCK_PREFIX "incl %0\n\t" 65 asm volatile(LOCK_PREFIX "incl %0\n\t"
54 REFCOUNT_CHECK_LT_ZERO 66 "REFCOUNT_CHECK_LT_ZERO counter=\"%[counter]\""
55 : [counter] "+m" (r->refs.counter) 67 : [counter] "+m" (r->refs.counter)
56 : : "cc", "cx"); 68 : : "cc", "cx");
57} 69}
@@ -59,7 +71,7 @@ static __always_inline void refcount_inc(refcount_t *r)
59static __always_inline void refcount_dec(refcount_t *r) 71static __always_inline void refcount_dec(refcount_t *r)
60{ 72{
61 asm volatile(LOCK_PREFIX "decl %0\n\t" 73 asm volatile(LOCK_PREFIX "decl %0\n\t"
62 REFCOUNT_CHECK_LE_ZERO 74 "REFCOUNT_CHECK_LE_ZERO counter=\"%[counter]\""
63 : [counter] "+m" (r->refs.counter) 75 : [counter] "+m" (r->refs.counter)
64 : : "cc", "cx"); 76 : : "cc", "cx");
65} 77}
@@ -67,14 +79,17 @@ static __always_inline void refcount_dec(refcount_t *r)
67static __always_inline __must_check 79static __always_inline __must_check
68bool refcount_sub_and_test(unsigned int i, refcount_t *r) 80bool refcount_sub_and_test(unsigned int i, refcount_t *r)
69{ 81{
70 GEN_BINARY_SUFFIXED_RMWcc(LOCK_PREFIX "subl", REFCOUNT_CHECK_LT_ZERO, 82
71 r->refs.counter, "er", i, "%0", e, "cx"); 83 return GEN_BINARY_SUFFIXED_RMWcc(LOCK_PREFIX "subl",
84 "REFCOUNT_CHECK_LT_ZERO counter=\"%[var]\"",
85 r->refs.counter, e, "er", i, "cx");
72} 86}
73 87
74static __always_inline __must_check bool refcount_dec_and_test(refcount_t *r) 88static __always_inline __must_check bool refcount_dec_and_test(refcount_t *r)
75{ 89{
76 GEN_UNARY_SUFFIXED_RMWcc(LOCK_PREFIX "decl", REFCOUNT_CHECK_LT_ZERO, 90 return GEN_UNARY_SUFFIXED_RMWcc(LOCK_PREFIX "decl",
77 r->refs.counter, "%0", e, "cx"); 91 "REFCOUNT_CHECK_LT_ZERO counter=\"%[var]\"",
92 r->refs.counter, e, "cx");
78} 93}
79 94
80static __always_inline __must_check 95static __always_inline __must_check
@@ -91,7 +106,7 @@ bool refcount_add_not_zero(unsigned int i, refcount_t *r)
91 106
92 /* Did we try to increment from/to an undesirable state? */ 107 /* Did we try to increment from/to an undesirable state? */
93 if (unlikely(c < 0 || c == INT_MAX || result < c)) { 108 if (unlikely(c < 0 || c == INT_MAX || result < c)) {
94 asm volatile(REFCOUNT_ERROR 109 asm volatile("REFCOUNT_ERROR counter=\"%[counter]\""
95 : : [counter] "m" (r->refs.counter) 110 : : [counter] "m" (r->refs.counter)
96 : "cc", "cx"); 111 : "cc", "cx");
97 break; 112 break;
@@ -107,4 +122,6 @@ static __always_inline __must_check bool refcount_inc_not_zero(refcount_t *r)
107 return refcount_add_not_zero(1, r); 122 return refcount_add_not_zero(1, r);
108} 123}
109 124
125#endif /* __ASSEMBLY__ */
126
110#endif 127#endif
diff --git a/arch/x86/include/asm/rmwcc.h b/arch/x86/include/asm/rmwcc.h
index 4914a3e7c803..46ac84b506f5 100644
--- a/arch/x86/include/asm/rmwcc.h
+++ b/arch/x86/include/asm/rmwcc.h
@@ -2,56 +2,69 @@
2#ifndef _ASM_X86_RMWcc 2#ifndef _ASM_X86_RMWcc
3#define _ASM_X86_RMWcc 3#define _ASM_X86_RMWcc
4 4
5/* This counts to 12. Any more, it will return 13th argument. */
6#define __RMWcc_ARGS(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _n, X...) _n
7#define RMWcc_ARGS(X...) __RMWcc_ARGS(, ##X, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
8
9#define __RMWcc_CONCAT(a, b) a ## b
10#define RMWcc_CONCAT(a, b) __RMWcc_CONCAT(a, b)
11
5#define __CLOBBERS_MEM(clb...) "memory", ## clb 12#define __CLOBBERS_MEM(clb...) "memory", ## clb
6 13
7#if !defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(CC_HAVE_ASM_GOTO) 14#if !defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(CC_HAVE_ASM_GOTO)
8 15
9/* Use asm goto */ 16/* Use asm goto */
10 17
11#define __GEN_RMWcc(fullop, var, cc, clobbers, ...) \ 18#define __GEN_RMWcc(fullop, _var, cc, clobbers, ...) \
12do { \ 19({ \
20 bool c = false; \
13 asm_volatile_goto (fullop "; j" #cc " %l[cc_label]" \ 21 asm_volatile_goto (fullop "; j" #cc " %l[cc_label]" \
14 : : [counter] "m" (var), ## __VA_ARGS__ \ 22 : : [var] "m" (_var), ## __VA_ARGS__ \
15 : clobbers : cc_label); \ 23 : clobbers : cc_label); \
16 return 0; \ 24 if (0) { \
17cc_label: \ 25cc_label: c = true; \
18 return 1; \ 26 } \
19} while (0) 27 c; \
20 28})
21#define __BINARY_RMWcc_ARG " %1, "
22
23 29
24#else /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */ 30#else /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */
25 31
26/* Use flags output or a set instruction */ 32/* Use flags output or a set instruction */
27 33
28#define __GEN_RMWcc(fullop, var, cc, clobbers, ...) \ 34#define __GEN_RMWcc(fullop, _var, cc, clobbers, ...) \
29do { \ 35({ \
30 bool c; \ 36 bool c; \
31 asm volatile (fullop CC_SET(cc) \ 37 asm volatile (fullop CC_SET(cc) \
32 : [counter] "+m" (var), CC_OUT(cc) (c) \ 38 : [var] "+m" (_var), CC_OUT(cc) (c) \
33 : __VA_ARGS__ : clobbers); \ 39 : __VA_ARGS__ : clobbers); \
34 return c; \ 40 c; \
35} while (0) 41})
36
37#define __BINARY_RMWcc_ARG " %2, "
38 42
39#endif /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */ 43#endif /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */
40 44
41#define GEN_UNARY_RMWcc(op, var, arg0, cc) \ 45#define GEN_UNARY_RMWcc_4(op, var, cc, arg0) \
42 __GEN_RMWcc(op " " arg0, var, cc, __CLOBBERS_MEM()) 46 __GEN_RMWcc(op " " arg0, var, cc, __CLOBBERS_MEM())
43 47
44#define GEN_UNARY_SUFFIXED_RMWcc(op, suffix, var, arg0, cc, clobbers...)\ 48#define GEN_UNARY_RMWcc_3(op, var, cc) \
45 __GEN_RMWcc(op " " arg0 "\n\t" suffix, var, cc, \ 49 GEN_UNARY_RMWcc_4(op, var, cc, "%[var]")
46 __CLOBBERS_MEM(clobbers))
47 50
48#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \ 51#define GEN_UNARY_RMWcc(X...) RMWcc_CONCAT(GEN_UNARY_RMWcc_, RMWcc_ARGS(X))(X)
49 __GEN_RMWcc(op __BINARY_RMWcc_ARG arg0, var, cc, \ 52
50 __CLOBBERS_MEM(), vcon (val)) 53#define GEN_BINARY_RMWcc_6(op, var, cc, vcon, _val, arg0) \
54 __GEN_RMWcc(op " %[val], " arg0, var, cc, \
55 __CLOBBERS_MEM(), [val] vcon (_val))
56
57#define GEN_BINARY_RMWcc_5(op, var, cc, vcon, val) \
58 GEN_BINARY_RMWcc_6(op, var, cc, vcon, val, "%[var]")
59
60#define GEN_BINARY_RMWcc(X...) RMWcc_CONCAT(GEN_BINARY_RMWcc_, RMWcc_ARGS(X))(X)
61
62#define GEN_UNARY_SUFFIXED_RMWcc(op, suffix, var, cc, clobbers...) \
63 __GEN_RMWcc(op " %[var]\n\t" suffix, var, cc, \
64 __CLOBBERS_MEM(clobbers))
51 65
52#define GEN_BINARY_SUFFIXED_RMWcc(op, suffix, var, vcon, val, arg0, cc, \ 66#define GEN_BINARY_SUFFIXED_RMWcc(op, suffix, var, cc, vcon, _val, clobbers...)\
53 clobbers...) \ 67 __GEN_RMWcc(op " %[val], %[var]\n\t" suffix, var, cc, \
54 __GEN_RMWcc(op __BINARY_RMWcc_ARG arg0 "\n\t" suffix, var, cc, \ 68 __CLOBBERS_MEM(clobbers), [val] vcon (_val))
55 __CLOBBERS_MEM(clobbers), vcon (val))
56 69
57#endif /* _ASM_X86_RMWcc */ 70#endif /* _ASM_X86_RMWcc */
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index aae77eb8491c..b5e58cc0c5e7 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -198,8 +198,8 @@ __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
198 "4: movl %3,%0\n" \ 198 "4: movl %3,%0\n" \
199 " jmp 3b\n" \ 199 " jmp 3b\n" \
200 ".previous\n" \ 200 ".previous\n" \
201 _ASM_EXTABLE(1b, 4b) \ 201 _ASM_EXTABLE_UA(1b, 4b) \
202 _ASM_EXTABLE(2b, 4b) \ 202 _ASM_EXTABLE_UA(2b, 4b) \
203 : "=r" (err) \ 203 : "=r" (err) \
204 : "A" (x), "r" (addr), "i" (errret), "0" (err)) 204 : "A" (x), "r" (addr), "i" (errret), "0" (err))
205 205
@@ -340,8 +340,8 @@ do { \
340 " xorl %%edx,%%edx\n" \ 340 " xorl %%edx,%%edx\n" \
341 " jmp 3b\n" \ 341 " jmp 3b\n" \
342 ".previous\n" \ 342 ".previous\n" \
343 _ASM_EXTABLE(1b, 4b) \ 343 _ASM_EXTABLE_UA(1b, 4b) \
344 _ASM_EXTABLE(2b, 4b) \ 344 _ASM_EXTABLE_UA(2b, 4b) \
345 : "=r" (retval), "=&A"(x) \ 345 : "=r" (retval), "=&A"(x) \
346 : "m" (__m(__ptr)), "m" __m(((u32 __user *)(__ptr)) + 1), \ 346 : "m" (__m(__ptr)), "m" __m(((u32 __user *)(__ptr)) + 1), \
347 "i" (errret), "0" (retval)); \ 347 "i" (errret), "0" (retval)); \
@@ -386,7 +386,7 @@ do { \
386 " xor"itype" %"rtype"1,%"rtype"1\n" \ 386 " xor"itype" %"rtype"1,%"rtype"1\n" \
387 " jmp 2b\n" \ 387 " jmp 2b\n" \
388 ".previous\n" \ 388 ".previous\n" \
389 _ASM_EXTABLE(1b, 3b) \ 389 _ASM_EXTABLE_UA(1b, 3b) \
390 : "=r" (err), ltype(x) \ 390 : "=r" (err), ltype(x) \
391 : "m" (__m(addr)), "i" (errret), "0" (err)) 391 : "m" (__m(addr)), "i" (errret), "0" (err))
392 392
@@ -398,7 +398,7 @@ do { \
398 "3: mov %3,%0\n" \ 398 "3: mov %3,%0\n" \
399 " jmp 2b\n" \ 399 " jmp 2b\n" \
400 ".previous\n" \ 400 ".previous\n" \
401 _ASM_EXTABLE(1b, 3b) \ 401 _ASM_EXTABLE_UA(1b, 3b) \
402 : "=r" (err), ltype(x) \ 402 : "=r" (err), ltype(x) \
403 : "m" (__m(addr)), "i" (errret), "0" (err)) 403 : "m" (__m(addr)), "i" (errret), "0" (err))
404 404
@@ -474,7 +474,7 @@ struct __large_struct { unsigned long buf[100]; };
474 "3: mov %3,%0\n" \ 474 "3: mov %3,%0\n" \
475 " jmp 2b\n" \ 475 " jmp 2b\n" \
476 ".previous\n" \ 476 ".previous\n" \
477 _ASM_EXTABLE(1b, 3b) \ 477 _ASM_EXTABLE_UA(1b, 3b) \
478 : "=r"(err) \ 478 : "=r"(err) \
479 : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err)) 479 : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))
480 480
@@ -602,7 +602,7 @@ extern void __cmpxchg_wrong_size(void)
602 "3:\tmov %3, %0\n" \ 602 "3:\tmov %3, %0\n" \
603 "\tjmp 2b\n" \ 603 "\tjmp 2b\n" \
604 "\t.previous\n" \ 604 "\t.previous\n" \
605 _ASM_EXTABLE(1b, 3b) \ 605 _ASM_EXTABLE_UA(1b, 3b) \
606 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ 606 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
607 : "i" (-EFAULT), "q" (__new), "1" (__old) \ 607 : "i" (-EFAULT), "q" (__new), "1" (__old) \
608 : "memory" \ 608 : "memory" \
@@ -618,7 +618,7 @@ extern void __cmpxchg_wrong_size(void)
618 "3:\tmov %3, %0\n" \ 618 "3:\tmov %3, %0\n" \
619 "\tjmp 2b\n" \ 619 "\tjmp 2b\n" \
620 "\t.previous\n" \ 620 "\t.previous\n" \
621 _ASM_EXTABLE(1b, 3b) \ 621 _ASM_EXTABLE_UA(1b, 3b) \
622 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ 622 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
623 : "i" (-EFAULT), "r" (__new), "1" (__old) \ 623 : "i" (-EFAULT), "r" (__new), "1" (__old) \
624 : "memory" \ 624 : "memory" \
@@ -634,7 +634,7 @@ extern void __cmpxchg_wrong_size(void)
634 "3:\tmov %3, %0\n" \ 634 "3:\tmov %3, %0\n" \
635 "\tjmp 2b\n" \ 635 "\tjmp 2b\n" \
636 "\t.previous\n" \ 636 "\t.previous\n" \
637 _ASM_EXTABLE(1b, 3b) \ 637 _ASM_EXTABLE_UA(1b, 3b) \
638 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ 638 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
639 : "i" (-EFAULT), "r" (__new), "1" (__old) \ 639 : "i" (-EFAULT), "r" (__new), "1" (__old) \
640 : "memory" \ 640 : "memory" \
@@ -653,7 +653,7 @@ extern void __cmpxchg_wrong_size(void)
653 "3:\tmov %3, %0\n" \ 653 "3:\tmov %3, %0\n" \
654 "\tjmp 2b\n" \ 654 "\tjmp 2b\n" \
655 "\t.previous\n" \ 655 "\t.previous\n" \
656 _ASM_EXTABLE(1b, 3b) \ 656 _ASM_EXTABLE_UA(1b, 3b) \
657 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ 657 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
658 : "i" (-EFAULT), "r" (__new), "1" (__old) \ 658 : "i" (-EFAULT), "r" (__new), "1" (__old) \
659 : "memory" \ 659 : "memory" \
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 953b3ce92dcc..ef8fd1f2ede0 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -1315,7 +1315,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
1315 local_irq_disable(); 1315 local_irq_disable();
1316 ist_end_non_atomic(); 1316 ist_end_non_atomic();
1317 } else { 1317 } else {
1318 if (!fixup_exception(regs, X86_TRAP_MC)) 1318 if (!fixup_exception(regs, X86_TRAP_MC, error_code, 0))
1319 mce_panic("Failed kernel mode recovery", &m, NULL); 1319 mce_panic("Failed kernel mode recovery", &m, NULL);
1320 } 1320 }
1321 1321
diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c
index eeea935e9bb5..aac0c1f7e354 100644
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -42,55 +42,40 @@ static void __ref __jump_label_transform(struct jump_entry *entry,
42 void *(*poker)(void *, const void *, size_t), 42 void *(*poker)(void *, const void *, size_t),
43 int init) 43 int init)
44{ 44{
45 union jump_code_union code; 45 union jump_code_union jmp;
46 const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; 46 const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
47 const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5]; 47 const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
48 const void *expect, *code;
49 int line;
50
51 jmp.jump = 0xe9;
52 jmp.offset = jump_entry_target(entry) -
53 (jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
48 54
49 if (early_boot_irqs_disabled) 55 if (early_boot_irqs_disabled)
50 poker = text_poke_early; 56 poker = text_poke_early;
51 57
52 if (type == JUMP_LABEL_JMP) { 58 if (type == JUMP_LABEL_JMP) {
53 if (init) { 59 if (init) {
54 /* 60 expect = default_nop; line = __LINE__;
55 * Jump label is enabled for the first time.
56 * So we expect a default_nop...
57 */
58 if (unlikely(memcmp((void *)entry->code, default_nop, 5)
59 != 0))
60 bug_at((void *)entry->code, __LINE__);
61 } else { 61 } else {
62 /* 62 expect = ideal_nop; line = __LINE__;
63 * ...otherwise expect an ideal_nop. Otherwise
64 * something went horribly wrong.
65 */
66 if (unlikely(memcmp((void *)entry->code, ideal_nop, 5)
67 != 0))
68 bug_at((void *)entry->code, __LINE__);
69 } 63 }
70 64
71 code.jump = 0xe9; 65 code = &jmp.code;
72 code.offset = entry->target -
73 (entry->code + JUMP_LABEL_NOP_SIZE);
74 } else { 66 } else {
75 /*
76 * We are disabling this jump label. If it is not what
77 * we think it is, then something must have gone wrong.
78 * If this is the first initialization call, then we
79 * are converting the default nop to the ideal nop.
80 */
81 if (init) { 67 if (init) {
82 if (unlikely(memcmp((void *)entry->code, default_nop, 5) != 0)) 68 expect = default_nop; line = __LINE__;
83 bug_at((void *)entry->code, __LINE__);
84 } else { 69 } else {
85 code.jump = 0xe9; 70 expect = &jmp.code; line = __LINE__;
86 code.offset = entry->target -
87 (entry->code + JUMP_LABEL_NOP_SIZE);
88 if (unlikely(memcmp((void *)entry->code, &code, 5) != 0))
89 bug_at((void *)entry->code, __LINE__);
90 } 71 }
91 memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE); 72
73 code = ideal_nop;
92 } 74 }
93 75
76 if (memcmp((void *)jump_entry_code(entry), expect, JUMP_LABEL_NOP_SIZE))
77 bug_at((void *)jump_entry_code(entry), line);
78
94 /* 79 /*
95 * Make text_poke_bp() a default fallback poker. 80 * Make text_poke_bp() a default fallback poker.
96 * 81 *
@@ -99,11 +84,14 @@ static void __ref __jump_label_transform(struct jump_entry *entry,
99 * always nop being the 'currently valid' instruction 84 * always nop being the 'currently valid' instruction
100 * 85 *
101 */ 86 */
102 if (poker) 87 if (poker) {
103 (*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE); 88 (*poker)((void *)jump_entry_code(entry), code,
104 else 89 JUMP_LABEL_NOP_SIZE);
105 text_poke_bp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE, 90 return;
106 (void *)entry->code + JUMP_LABEL_NOP_SIZE); 91 }
92
93 text_poke_bp((void *)jump_entry_code(entry), code, JUMP_LABEL_NOP_SIZE,
94 (void *)jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
107} 95}
108 96
109void arch_jump_label_transform(struct jump_entry *entry, 97void arch_jump_label_transform(struct jump_entry *entry,
diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c
index b0d1e81c96bb..f72a47b602e2 100644
--- a/arch/x86/kernel/kprobes/core.c
+++ b/arch/x86/kernel/kprobes/core.c
@@ -1020,50 +1020,12 @@ int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
1020 */ 1020 */
1021 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) 1021 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
1022 return 1; 1022 return 1;
1023
1024 /*
1025 * In case the user-specified fault handler returned
1026 * zero, try to fix up.
1027 */
1028 if (fixup_exception(regs, trapnr))
1029 return 1;
1030
1031 /*
1032 * fixup routine could not handle it,
1033 * Let do_page_fault() fix it.
1034 */
1035 } 1023 }
1036 1024
1037 return 0; 1025 return 0;
1038} 1026}
1039NOKPROBE_SYMBOL(kprobe_fault_handler); 1027NOKPROBE_SYMBOL(kprobe_fault_handler);
1040 1028
1041/*
1042 * Wrapper routine for handling exceptions.
1043 */
1044int kprobe_exceptions_notify(struct notifier_block *self, unsigned long val,
1045 void *data)
1046{
1047 struct die_args *args = data;
1048 int ret = NOTIFY_DONE;
1049
1050 if (args->regs && user_mode(args->regs))
1051 return ret;
1052
1053 if (val == DIE_GPF) {
1054 /*
1055 * To be potentially processing a kprobe fault and to
1056 * trust the result from kprobe_running(), we have
1057 * be non-preemptible.
1058 */
1059 if (!preemptible() && kprobe_running() &&
1060 kprobe_fault_handler(args->regs, args->trapnr))
1061 ret = NOTIFY_STOP;
1062 }
1063 return ret;
1064}
1065NOKPROBE_SYMBOL(kprobe_exceptions_notify);
1066
1067bool arch_within_kprobe_blacklist(unsigned long addr) 1029bool arch_within_kprobe_blacklist(unsigned long addr)
1068{ 1030{
1069 bool is_in_entry_trampoline_section = false; 1031 bool is_in_entry_trampoline_section = false;
diff --git a/arch/x86/kernel/macros.S b/arch/x86/kernel/macros.S
new file mode 100644
index 000000000000..161c95059044
--- /dev/null
+++ b/arch/x86/kernel/macros.S
@@ -0,0 +1,16 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2
3/*
4 * This file includes headers whose assembly part includes macros which are
5 * commonly used. The macros are precompiled into assmebly file which is later
6 * assembled together with each compiled file.
7 */
8
9#include <linux/compiler.h>
10#include <asm/refcount.h>
11#include <asm/alternative-asm.h>
12#include <asm/bug.h>
13#include <asm/paravirt.h>
14#include <asm/asm.h>
15#include <asm/cpufeature.h>
16#include <asm/jump_label.h>
diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
index f58336af095c..b052e883dd8c 100644
--- a/arch/x86/kernel/module.c
+++ b/arch/x86/kernel/module.c
@@ -201,6 +201,12 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
201 goto overflow; 201 goto overflow;
202#endif 202#endif
203 break; 203 break;
204 case R_X86_64_PC64:
205 if (*(u64 *)loc != 0)
206 goto invalid_relocation;
207 val -= (u64)loc;
208 *(u64 *)loc = val;
209 break;
204 default: 210 default:
205 pr_err("%s: Unknown rela relocation: %llu\n", 211 pr_err("%s: Unknown rela relocation: %llu\n",
206 me->name, ELF64_R_TYPE(rel[i].r_info)); 212 me->name, ELF64_R_TYPE(rel[i].r_info));
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index e6db475164ed..16c95cb90496 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -206,7 +206,7 @@ do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str,
206 } 206 }
207 207
208 if (!user_mode(regs)) { 208 if (!user_mode(regs)) {
209 if (fixup_exception(regs, trapnr)) 209 if (fixup_exception(regs, trapnr, error_code, 0))
210 return 0; 210 return 0;
211 211
212 tsk->thread.error_code = error_code; 212 tsk->thread.error_code = error_code;
@@ -551,11 +551,21 @@ do_general_protection(struct pt_regs *regs, long error_code)
551 551
552 tsk = current; 552 tsk = current;
553 if (!user_mode(regs)) { 553 if (!user_mode(regs)) {
554 if (fixup_exception(regs, X86_TRAP_GP)) 554 if (fixup_exception(regs, X86_TRAP_GP, error_code, 0))
555 return; 555 return;
556 556
557 tsk->thread.error_code = error_code; 557 tsk->thread.error_code = error_code;
558 tsk->thread.trap_nr = X86_TRAP_GP; 558 tsk->thread.trap_nr = X86_TRAP_GP;
559
560 /*
561 * To be potentially processing a kprobe fault and to
562 * trust the result from kprobe_running(), we have to
563 * be non-preemptible.
564 */
565 if (!preemptible() && kprobe_running() &&
566 kprobe_fault_handler(regs, X86_TRAP_GP))
567 return;
568
559 if (notify_die(DIE_GPF, "general protection fault", regs, error_code, 569 if (notify_die(DIE_GPF, "general protection fault", regs, error_code,
560 X86_TRAP_GP, SIGSEGV) != NOTIFY_STOP) 570 X86_TRAP_GP, SIGSEGV) != NOTIFY_STOP)
561 die("general protection fault", regs, error_code); 571 die("general protection fault", regs, error_code);
@@ -838,7 +848,7 @@ static void math_error(struct pt_regs *regs, int error_code, int trapnr)
838 cond_local_irq_enable(regs); 848 cond_local_irq_enable(regs);
839 849
840 if (!user_mode(regs)) { 850 if (!user_mode(regs)) {
841 if (fixup_exception(regs, trapnr)) 851 if (fixup_exception(regs, trapnr, error_code, 0))
842 return; 852 return;
843 853
844 task->thread.error_code = error_code; 854 task->thread.error_code = error_code;
diff --git a/arch/x86/lib/checksum_32.S b/arch/x86/lib/checksum_32.S
index 46e71a74e612..ad8e0906d1ea 100644
--- a/arch/x86/lib/checksum_32.S
+++ b/arch/x86/lib/checksum_32.S
@@ -273,11 +273,11 @@ unsigned int csum_partial_copy_generic (const char *src, char *dst,
273 273
274#define SRC(y...) \ 274#define SRC(y...) \
275 9999: y; \ 275 9999: y; \
276 _ASM_EXTABLE(9999b, 6001f) 276 _ASM_EXTABLE_UA(9999b, 6001f)
277 277
278#define DST(y...) \ 278#define DST(y...) \
279 9999: y; \ 279 9999: y; \
280 _ASM_EXTABLE(9999b, 6002f) 280 _ASM_EXTABLE_UA(9999b, 6002f)
281 281
282#ifndef CONFIG_X86_USE_PPRO_CHECKSUM 282#ifndef CONFIG_X86_USE_PPRO_CHECKSUM
283 283
diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S
index 020f75cc8cf6..db4e5aa0858b 100644
--- a/arch/x86/lib/copy_user_64.S
+++ b/arch/x86/lib/copy_user_64.S
@@ -92,26 +92,26 @@ ENTRY(copy_user_generic_unrolled)
9260: jmp copy_user_handle_tail /* ecx is zerorest also */ 9260: jmp copy_user_handle_tail /* ecx is zerorest also */
93 .previous 93 .previous
94 94
95 _ASM_EXTABLE(1b,30b) 95 _ASM_EXTABLE_UA(1b, 30b)
96 _ASM_EXTABLE(2b,30b) 96 _ASM_EXTABLE_UA(2b, 30b)
97 _ASM_EXTABLE(3b,30b) 97 _ASM_EXTABLE_UA(3b, 30b)
98 _ASM_EXTABLE(4b,30b) 98 _ASM_EXTABLE_UA(4b, 30b)
99 _ASM_EXTABLE(5b,30b) 99 _ASM_EXTABLE_UA(5b, 30b)
100 _ASM_EXTABLE(6b,30b) 100 _ASM_EXTABLE_UA(6b, 30b)
101 _ASM_EXTABLE(7b,30b) 101 _ASM_EXTABLE_UA(7b, 30b)
102 _ASM_EXTABLE(8b,30b) 102 _ASM_EXTABLE_UA(8b, 30b)
103 _ASM_EXTABLE(9b,30b) 103 _ASM_EXTABLE_UA(9b, 30b)
104 _ASM_EXTABLE(10b,30b) 104 _ASM_EXTABLE_UA(10b, 30b)
105 _ASM_EXTABLE(11b,30b) 105 _ASM_EXTABLE_UA(11b, 30b)
106 _ASM_EXTABLE(12b,30b) 106 _ASM_EXTABLE_UA(12b, 30b)
107 _ASM_EXTABLE(13b,30b) 107 _ASM_EXTABLE_UA(13b, 30b)
108 _ASM_EXTABLE(14b,30b) 108 _ASM_EXTABLE_UA(14b, 30b)
109 _ASM_EXTABLE(15b,30b) 109 _ASM_EXTABLE_UA(15b, 30b)
110 _ASM_EXTABLE(16b,30b) 110 _ASM_EXTABLE_UA(16b, 30b)
111 _ASM_EXTABLE(18b,40b) 111 _ASM_EXTABLE_UA(18b, 40b)
112 _ASM_EXTABLE(19b,40b) 112 _ASM_EXTABLE_UA(19b, 40b)
113 _ASM_EXTABLE(21b,50b) 113 _ASM_EXTABLE_UA(21b, 50b)
114 _ASM_EXTABLE(22b,50b) 114 _ASM_EXTABLE_UA(22b, 50b)
115ENDPROC(copy_user_generic_unrolled) 115ENDPROC(copy_user_generic_unrolled)
116EXPORT_SYMBOL(copy_user_generic_unrolled) 116EXPORT_SYMBOL(copy_user_generic_unrolled)
117 117
@@ -156,8 +156,8 @@ ENTRY(copy_user_generic_string)
156 jmp copy_user_handle_tail 156 jmp copy_user_handle_tail
157 .previous 157 .previous
158 158
159 _ASM_EXTABLE(1b,11b) 159 _ASM_EXTABLE_UA(1b, 11b)
160 _ASM_EXTABLE(3b,12b) 160 _ASM_EXTABLE_UA(3b, 12b)
161ENDPROC(copy_user_generic_string) 161ENDPROC(copy_user_generic_string)
162EXPORT_SYMBOL(copy_user_generic_string) 162EXPORT_SYMBOL(copy_user_generic_string)
163 163
@@ -189,7 +189,7 @@ ENTRY(copy_user_enhanced_fast_string)
189 jmp copy_user_handle_tail 189 jmp copy_user_handle_tail
190 .previous 190 .previous
191 191
192 _ASM_EXTABLE(1b,12b) 192 _ASM_EXTABLE_UA(1b, 12b)
193ENDPROC(copy_user_enhanced_fast_string) 193ENDPROC(copy_user_enhanced_fast_string)
194EXPORT_SYMBOL(copy_user_enhanced_fast_string) 194EXPORT_SYMBOL(copy_user_enhanced_fast_string)
195 195
@@ -319,27 +319,27 @@ ENTRY(__copy_user_nocache)
319 jmp copy_user_handle_tail 319 jmp copy_user_handle_tail
320 .previous 320 .previous
321 321
322 _ASM_EXTABLE(1b,.L_fixup_4x8b_copy) 322 _ASM_EXTABLE_UA(1b, .L_fixup_4x8b_copy)
323 _ASM_EXTABLE(2b,.L_fixup_4x8b_copy) 323 _ASM_EXTABLE_UA(2b, .L_fixup_4x8b_copy)
324 _ASM_EXTABLE(3b,.L_fixup_4x8b_copy) 324 _ASM_EXTABLE_UA(3b, .L_fixup_4x8b_copy)
325 _ASM_EXTABLE(4b,.L_fixup_4x8b_copy) 325 _ASM_EXTABLE_UA(4b, .L_fixup_4x8b_copy)
326 _ASM_EXTABLE(5b,.L_fixup_4x8b_copy) 326 _ASM_EXTABLE_UA(5b, .L_fixup_4x8b_copy)
327 _ASM_EXTABLE(6b,.L_fixup_4x8b_copy) 327 _ASM_EXTABLE_UA(6b, .L_fixup_4x8b_copy)
328 _ASM_EXTABLE(7b,.L_fixup_4x8b_copy) 328 _ASM_EXTABLE_UA(7b, .L_fixup_4x8b_copy)
329 _ASM_EXTABLE(8b,.L_fixup_4x8b_copy) 329 _ASM_EXTABLE_UA(8b, .L_fixup_4x8b_copy)
330 _ASM_EXTABLE(9b,.L_fixup_4x8b_copy) 330 _ASM_EXTABLE_UA(9b, .L_fixup_4x8b_copy)
331 _ASM_EXTABLE(10b,.L_fixup_4x8b_copy) 331 _ASM_EXTABLE_UA(10b, .L_fixup_4x8b_copy)
332 _ASM_EXTABLE(11b,.L_fixup_4x8b_copy) 332 _ASM_EXTABLE_UA(11b, .L_fixup_4x8b_copy)
333 _ASM_EXTABLE(12b,.L_fixup_4x8b_copy) 333 _ASM_EXTABLE_UA(12b, .L_fixup_4x8b_copy)
334 _ASM_EXTABLE(13b,.L_fixup_4x8b_copy) 334 _ASM_EXTABLE_UA(13b, .L_fixup_4x8b_copy)
335 _ASM_EXTABLE(14b,.L_fixup_4x8b_copy) 335 _ASM_EXTABLE_UA(14b, .L_fixup_4x8b_copy)
336 _ASM_EXTABLE(15b,.L_fixup_4x8b_copy) 336 _ASM_EXTABLE_UA(15b, .L_fixup_4x8b_copy)
337 _ASM_EXTABLE(16b,.L_fixup_4x8b_copy) 337 _ASM_EXTABLE_UA(16b, .L_fixup_4x8b_copy)
338 _ASM_EXTABLE(20b,.L_fixup_8b_copy) 338 _ASM_EXTABLE_UA(20b, .L_fixup_8b_copy)
339 _ASM_EXTABLE(21b,.L_fixup_8b_copy) 339 _ASM_EXTABLE_UA(21b, .L_fixup_8b_copy)
340 _ASM_EXTABLE(30b,.L_fixup_4b_copy) 340 _ASM_EXTABLE_UA(30b, .L_fixup_4b_copy)
341 _ASM_EXTABLE(31b,.L_fixup_4b_copy) 341 _ASM_EXTABLE_UA(31b, .L_fixup_4b_copy)
342 _ASM_EXTABLE(40b,.L_fixup_1b_copy) 342 _ASM_EXTABLE_UA(40b, .L_fixup_1b_copy)
343 _ASM_EXTABLE(41b,.L_fixup_1b_copy) 343 _ASM_EXTABLE_UA(41b, .L_fixup_1b_copy)
344ENDPROC(__copy_user_nocache) 344ENDPROC(__copy_user_nocache)
345EXPORT_SYMBOL(__copy_user_nocache) 345EXPORT_SYMBOL(__copy_user_nocache)
diff --git a/arch/x86/lib/csum-copy_64.S b/arch/x86/lib/csum-copy_64.S
index 45a53dfe1859..a4a379e79259 100644
--- a/arch/x86/lib/csum-copy_64.S
+++ b/arch/x86/lib/csum-copy_64.S
@@ -31,14 +31,18 @@
31 31
32 .macro source 32 .macro source
3310: 3310:
34 _ASM_EXTABLE(10b, .Lbad_source) 34 _ASM_EXTABLE_UA(10b, .Lbad_source)
35 .endm 35 .endm
36 36
37 .macro dest 37 .macro dest
3820: 3820:
39 _ASM_EXTABLE(20b, .Lbad_dest) 39 _ASM_EXTABLE_UA(20b, .Lbad_dest)
40 .endm 40 .endm
41 41
42 /*
43 * No _ASM_EXTABLE_UA; this is used for intentional prefetch on a
44 * potentially unmapped kernel address.
45 */
42 .macro ignore L=.Lignore 46 .macro ignore L=.Lignore
4330: 4730:
44 _ASM_EXTABLE(30b, \L) 48 _ASM_EXTABLE(30b, \L)
diff --git a/arch/x86/lib/getuser.S b/arch/x86/lib/getuser.S
index 49b167f73215..74fdff968ea3 100644
--- a/arch/x86/lib/getuser.S
+++ b/arch/x86/lib/getuser.S
@@ -132,12 +132,12 @@ bad_get_user_8:
132END(bad_get_user_8) 132END(bad_get_user_8)
133#endif 133#endif
134 134
135 _ASM_EXTABLE(1b,bad_get_user) 135 _ASM_EXTABLE_UA(1b, bad_get_user)
136 _ASM_EXTABLE(2b,bad_get_user) 136 _ASM_EXTABLE_UA(2b, bad_get_user)
137 _ASM_EXTABLE(3b,bad_get_user) 137 _ASM_EXTABLE_UA(3b, bad_get_user)
138#ifdef CONFIG_X86_64 138#ifdef CONFIG_X86_64
139 _ASM_EXTABLE(4b,bad_get_user) 139 _ASM_EXTABLE_UA(4b, bad_get_user)
140#else 140#else
141 _ASM_EXTABLE(4b,bad_get_user_8) 141 _ASM_EXTABLE_UA(4b, bad_get_user_8)
142 _ASM_EXTABLE(5b,bad_get_user_8) 142 _ASM_EXTABLE_UA(5b, bad_get_user_8)
143#endif 143#endif
diff --git a/arch/x86/lib/putuser.S b/arch/x86/lib/putuser.S
index 96dce5fe2a35..d2e5c9c39601 100644
--- a/arch/x86/lib/putuser.S
+++ b/arch/x86/lib/putuser.S
@@ -94,10 +94,10 @@ bad_put_user:
94 EXIT 94 EXIT
95END(bad_put_user) 95END(bad_put_user)
96 96
97 _ASM_EXTABLE(1b,bad_put_user) 97 _ASM_EXTABLE_UA(1b, bad_put_user)
98 _ASM_EXTABLE(2b,bad_put_user) 98 _ASM_EXTABLE_UA(2b, bad_put_user)
99 _ASM_EXTABLE(3b,bad_put_user) 99 _ASM_EXTABLE_UA(3b, bad_put_user)
100 _ASM_EXTABLE(4b,bad_put_user) 100 _ASM_EXTABLE_UA(4b, bad_put_user)
101#ifdef CONFIG_X86_32 101#ifdef CONFIG_X86_32
102 _ASM_EXTABLE(5b,bad_put_user) 102 _ASM_EXTABLE_UA(5b, bad_put_user)
103#endif 103#endif
diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c
index 7add8ba06887..71fb58d44d58 100644
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -47,8 +47,8 @@ do { \
47 "3: lea 0(%2,%0,4),%0\n" \ 47 "3: lea 0(%2,%0,4),%0\n" \
48 " jmp 2b\n" \ 48 " jmp 2b\n" \
49 ".previous\n" \ 49 ".previous\n" \
50 _ASM_EXTABLE(0b,3b) \ 50 _ASM_EXTABLE_UA(0b, 3b) \
51 _ASM_EXTABLE(1b,2b) \ 51 _ASM_EXTABLE_UA(1b, 2b) \
52 : "=&c"(size), "=&D" (__d0) \ 52 : "=&c"(size), "=&D" (__d0) \
53 : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \ 53 : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \
54} while (0) 54} while (0)
@@ -153,44 +153,44 @@ __copy_user_intel(void __user *to, const void *from, unsigned long size)
153 "101: lea 0(%%eax,%0,4),%0\n" 153 "101: lea 0(%%eax,%0,4),%0\n"
154 " jmp 100b\n" 154 " jmp 100b\n"
155 ".previous\n" 155 ".previous\n"
156 _ASM_EXTABLE(1b,100b) 156 _ASM_EXTABLE_UA(1b, 100b)
157 _ASM_EXTABLE(2b,100b) 157 _ASM_EXTABLE_UA(2b, 100b)
158 _ASM_EXTABLE(3b,100b) 158 _ASM_EXTABLE_UA(3b, 100b)
159 _ASM_EXTABLE(4b,100b) 159 _ASM_EXTABLE_UA(4b, 100b)
160 _ASM_EXTABLE(5b,100b) 160 _ASM_EXTABLE_UA(5b, 100b)
161 _ASM_EXTABLE(6b,100b) 161 _ASM_EXTABLE_UA(6b, 100b)
162 _ASM_EXTABLE(7b,100b) 162 _ASM_EXTABLE_UA(7b, 100b)
163 _ASM_EXTABLE(8b,100b) 163 _ASM_EXTABLE_UA(8b, 100b)
164 _ASM_EXTABLE(9b,100b) 164 _ASM_EXTABLE_UA(9b, 100b)
165 _ASM_EXTABLE(10b,100b) 165 _ASM_EXTABLE_UA(10b, 100b)
166 _ASM_EXTABLE(11b,100b) 166 _ASM_EXTABLE_UA(11b, 100b)
167 _ASM_EXTABLE(12b,100b) 167 _ASM_EXTABLE_UA(12b, 100b)
168 _ASM_EXTABLE(13b,100b) 168 _ASM_EXTABLE_UA(13b, 100b)
169 _ASM_EXTABLE(14b,100b) 169 _ASM_EXTABLE_UA(14b, 100b)
170 _ASM_EXTABLE(15b,100b) 170 _ASM_EXTABLE_UA(15b, 100b)
171 _ASM_EXTABLE(16b,100b) 171 _ASM_EXTABLE_UA(16b, 100b)
172 _ASM_EXTABLE(17b,100b) 172 _ASM_EXTABLE_UA(17b, 100b)
173 _ASM_EXTABLE(18b,100b) 173 _ASM_EXTABLE_UA(18b, 100b)
174 _ASM_EXTABLE(19b,100b) 174 _ASM_EXTABLE_UA(19b, 100b)
175 _ASM_EXTABLE(20b,100b) 175 _ASM_EXTABLE_UA(20b, 100b)
176 _ASM_EXTABLE(21b,100b) 176 _ASM_EXTABLE_UA(21b, 100b)
177 _ASM_EXTABLE(22b,100b) 177 _ASM_EXTABLE_UA(22b, 100b)
178 _ASM_EXTABLE(23b,100b) 178 _ASM_EXTABLE_UA(23b, 100b)
179 _ASM_EXTABLE(24b,100b) 179 _ASM_EXTABLE_UA(24b, 100b)
180 _ASM_EXTABLE(25b,100b) 180 _ASM_EXTABLE_UA(25b, 100b)
181 _ASM_EXTABLE(26b,100b) 181 _ASM_EXTABLE_UA(26b, 100b)
182 _ASM_EXTABLE(27b,100b) 182 _ASM_EXTABLE_UA(27b, 100b)
183 _ASM_EXTABLE(28b,100b) 183 _ASM_EXTABLE_UA(28b, 100b)
184 _ASM_EXTABLE(29b,100b) 184 _ASM_EXTABLE_UA(29b, 100b)
185 _ASM_EXTABLE(30b,100b) 185 _ASM_EXTABLE_UA(30b, 100b)
186 _ASM_EXTABLE(31b,100b) 186 _ASM_EXTABLE_UA(31b, 100b)
187 _ASM_EXTABLE(32b,100b) 187 _ASM_EXTABLE_UA(32b, 100b)
188 _ASM_EXTABLE(33b,100b) 188 _ASM_EXTABLE_UA(33b, 100b)
189 _ASM_EXTABLE(34b,100b) 189 _ASM_EXTABLE_UA(34b, 100b)
190 _ASM_EXTABLE(35b,100b) 190 _ASM_EXTABLE_UA(35b, 100b)
191 _ASM_EXTABLE(36b,100b) 191 _ASM_EXTABLE_UA(36b, 100b)
192 _ASM_EXTABLE(37b,100b) 192 _ASM_EXTABLE_UA(37b, 100b)
193 _ASM_EXTABLE(99b,101b) 193 _ASM_EXTABLE_UA(99b, 101b)
194 : "=&c"(size), "=&D" (d0), "=&S" (d1) 194 : "=&c"(size), "=&D" (d0), "=&S" (d1)
195 : "1"(to), "2"(from), "0"(size) 195 : "1"(to), "2"(from), "0"(size)
196 : "eax", "edx", "memory"); 196 : "eax", "edx", "memory");
@@ -259,26 +259,26 @@ static unsigned long __copy_user_intel_nocache(void *to,
259 "9: lea 0(%%eax,%0,4),%0\n" 259 "9: lea 0(%%eax,%0,4),%0\n"
260 "16: jmp 8b\n" 260 "16: jmp 8b\n"
261 ".previous\n" 261 ".previous\n"
262 _ASM_EXTABLE(0b,16b) 262 _ASM_EXTABLE_UA(0b, 16b)
263 _ASM_EXTABLE(1b,16b) 263 _ASM_EXTABLE_UA(1b, 16b)
264 _ASM_EXTABLE(2b,16b) 264 _ASM_EXTABLE_UA(2b, 16b)
265 _ASM_EXTABLE(21b,16b) 265 _ASM_EXTABLE_UA(21b, 16b)
266 _ASM_EXTABLE(3b,16b) 266 _ASM_EXTABLE_UA(3b, 16b)
267 _ASM_EXTABLE(31b,16b) 267 _ASM_EXTABLE_UA(31b, 16b)
268 _ASM_EXTABLE(4b,16b) 268 _ASM_EXTABLE_UA(4b, 16b)
269 _ASM_EXTABLE(41b,16b) 269 _ASM_EXTABLE_UA(41b, 16b)
270 _ASM_EXTABLE(10b,16b) 270 _ASM_EXTABLE_UA(10b, 16b)
271 _ASM_EXTABLE(51b,16b) 271 _ASM_EXTABLE_UA(51b, 16b)
272 _ASM_EXTABLE(11b,16b) 272 _ASM_EXTABLE_UA(11b, 16b)
273 _ASM_EXTABLE(61b,16b) 273 _ASM_EXTABLE_UA(61b, 16b)
274 _ASM_EXTABLE(12b,16b) 274 _ASM_EXTABLE_UA(12b, 16b)
275 _ASM_EXTABLE(71b,16b) 275 _ASM_EXTABLE_UA(71b, 16b)
276 _ASM_EXTABLE(13b,16b) 276 _ASM_EXTABLE_UA(13b, 16b)
277 _ASM_EXTABLE(81b,16b) 277 _ASM_EXTABLE_UA(81b, 16b)
278 _ASM_EXTABLE(14b,16b) 278 _ASM_EXTABLE_UA(14b, 16b)
279 _ASM_EXTABLE(91b,16b) 279 _ASM_EXTABLE_UA(91b, 16b)
280 _ASM_EXTABLE(6b,9b) 280 _ASM_EXTABLE_UA(6b, 9b)
281 _ASM_EXTABLE(7b,16b) 281 _ASM_EXTABLE_UA(7b, 16b)
282 : "=&c"(size), "=&D" (d0), "=&S" (d1) 282 : "=&c"(size), "=&D" (d0), "=&S" (d1)
283 : "1"(to), "2"(from), "0"(size) 283 : "1"(to), "2"(from), "0"(size)
284 : "eax", "edx", "memory"); 284 : "eax", "edx", "memory");
@@ -321,9 +321,9 @@ do { \
321 "3: lea 0(%3,%0,4),%0\n" \ 321 "3: lea 0(%3,%0,4),%0\n" \
322 " jmp 2b\n" \ 322 " jmp 2b\n" \
323 ".previous\n" \ 323 ".previous\n" \
324 _ASM_EXTABLE(4b,5b) \ 324 _ASM_EXTABLE_UA(4b, 5b) \
325 _ASM_EXTABLE(0b,3b) \ 325 _ASM_EXTABLE_UA(0b, 3b) \
326 _ASM_EXTABLE(1b,2b) \ 326 _ASM_EXTABLE_UA(1b, 2b) \
327 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \ 327 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \
328 : "3"(size), "0"(size), "1"(to), "2"(from) \ 328 : "3"(size), "0"(size), "1"(to), "2"(from) \
329 : "memory"); \ 329 : "memory"); \
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index 9c5606d88f61..fefe64436398 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -37,8 +37,8 @@ unsigned long __clear_user(void __user *addr, unsigned long size)
37 "3: lea 0(%[size1],%[size8],8),%[size8]\n" 37 "3: lea 0(%[size1],%[size8],8),%[size8]\n"
38 " jmp 2b\n" 38 " jmp 2b\n"
39 ".previous\n" 39 ".previous\n"
40 _ASM_EXTABLE(0b,3b) 40 _ASM_EXTABLE_UA(0b, 3b)
41 _ASM_EXTABLE(1b,2b) 41 _ASM_EXTABLE_UA(1b, 2b)
42 : [size8] "=&c"(size), [dst] "=&D" (__d0) 42 : [size8] "=&c"(size), [dst] "=&D" (__d0)
43 : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr)); 43 : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr));
44 clac(); 44 clac();
diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c
index 45f5d6cf65ae..6521134057e8 100644
--- a/arch/x86/mm/extable.c
+++ b/arch/x86/mm/extable.c
@@ -8,7 +8,8 @@
8#include <asm/kdebug.h> 8#include <asm/kdebug.h>
9 9
10typedef bool (*ex_handler_t)(const struct exception_table_entry *, 10typedef bool (*ex_handler_t)(const struct exception_table_entry *,
11 struct pt_regs *, int); 11 struct pt_regs *, int, unsigned long,
12 unsigned long);
12 13
13static inline unsigned long 14static inline unsigned long
14ex_fixup_addr(const struct exception_table_entry *x) 15ex_fixup_addr(const struct exception_table_entry *x)
@@ -22,7 +23,9 @@ ex_fixup_handler(const struct exception_table_entry *x)
22} 23}
23 24
24__visible bool ex_handler_default(const struct exception_table_entry *fixup, 25__visible bool ex_handler_default(const struct exception_table_entry *fixup,
25 struct pt_regs *regs, int trapnr) 26 struct pt_regs *regs, int trapnr,
27 unsigned long error_code,
28 unsigned long fault_addr)
26{ 29{
27 regs->ip = ex_fixup_addr(fixup); 30 regs->ip = ex_fixup_addr(fixup);
28 return true; 31 return true;
@@ -30,7 +33,9 @@ __visible bool ex_handler_default(const struct exception_table_entry *fixup,
30EXPORT_SYMBOL(ex_handler_default); 33EXPORT_SYMBOL(ex_handler_default);
31 34
32__visible bool ex_handler_fault(const struct exception_table_entry *fixup, 35__visible bool ex_handler_fault(const struct exception_table_entry *fixup,
33 struct pt_regs *regs, int trapnr) 36 struct pt_regs *regs, int trapnr,
37 unsigned long error_code,
38 unsigned long fault_addr)
34{ 39{
35 regs->ip = ex_fixup_addr(fixup); 40 regs->ip = ex_fixup_addr(fixup);
36 regs->ax = trapnr; 41 regs->ax = trapnr;
@@ -43,7 +48,9 @@ EXPORT_SYMBOL_GPL(ex_handler_fault);
43 * result of a refcount inc/dec/add/sub. 48 * result of a refcount inc/dec/add/sub.
44 */ 49 */
45__visible bool ex_handler_refcount(const struct exception_table_entry *fixup, 50__visible bool ex_handler_refcount(const struct exception_table_entry *fixup,
46 struct pt_regs *regs, int trapnr) 51 struct pt_regs *regs, int trapnr,
52 unsigned long error_code,
53 unsigned long fault_addr)
47{ 54{
48 /* First unconditionally saturate the refcount. */ 55 /* First unconditionally saturate the refcount. */
49 *(int *)regs->cx = INT_MIN / 2; 56 *(int *)regs->cx = INT_MIN / 2;
@@ -96,7 +103,9 @@ EXPORT_SYMBOL(ex_handler_refcount);
96 * out all the FPU registers) if we can't restore from the task's FPU state. 103 * out all the FPU registers) if we can't restore from the task's FPU state.
97 */ 104 */
98__visible bool ex_handler_fprestore(const struct exception_table_entry *fixup, 105__visible bool ex_handler_fprestore(const struct exception_table_entry *fixup,
99 struct pt_regs *regs, int trapnr) 106 struct pt_regs *regs, int trapnr,
107 unsigned long error_code,
108 unsigned long fault_addr)
100{ 109{
101 regs->ip = ex_fixup_addr(fixup); 110 regs->ip = ex_fixup_addr(fixup);
102 111
@@ -108,9 +117,79 @@ __visible bool ex_handler_fprestore(const struct exception_table_entry *fixup,
108} 117}
109EXPORT_SYMBOL_GPL(ex_handler_fprestore); 118EXPORT_SYMBOL_GPL(ex_handler_fprestore);
110 119
120/* Helper to check whether a uaccess fault indicates a kernel bug. */
121static bool bogus_uaccess(struct pt_regs *regs, int trapnr,
122 unsigned long fault_addr)
123{
124 /* This is the normal case: #PF with a fault address in userspace. */
125 if (trapnr == X86_TRAP_PF && fault_addr < TASK_SIZE_MAX)
126 return false;
127
128 /*
129 * This code can be reached for machine checks, but only if the #MC
130 * handler has already decided that it looks like a candidate for fixup.
131 * This e.g. happens when attempting to access userspace memory which
132 * the CPU can't access because of uncorrectable bad memory.
133 */
134 if (trapnr == X86_TRAP_MC)
135 return false;
136
137 /*
138 * There are two remaining exception types we might encounter here:
139 * - #PF for faulting accesses to kernel addresses
140 * - #GP for faulting accesses to noncanonical addresses
141 * Complain about anything else.
142 */
143 if (trapnr != X86_TRAP_PF && trapnr != X86_TRAP_GP) {
144 WARN(1, "unexpected trap %d in uaccess\n", trapnr);
145 return false;
146 }
147
148 /*
149 * This is a faulting memory access in kernel space, on a kernel
150 * address, in a usercopy function. This can e.g. be caused by improper
151 * use of helpers like __put_user and by improper attempts to access
152 * userspace addresses in KERNEL_DS regions.
153 * The one (semi-)legitimate exception are probe_kernel_{read,write}(),
154 * which can be invoked from places like kgdb, /dev/mem (for reading)
155 * and privileged BPF code (for reading).
156 * The probe_kernel_*() functions set the kernel_uaccess_faults_ok flag
157 * to tell us that faulting on kernel addresses, and even noncanonical
158 * addresses, in a userspace accessor does not necessarily imply a
159 * kernel bug, root might just be doing weird stuff.
160 */
161 if (current->kernel_uaccess_faults_ok)
162 return false;
163
164 /* This is bad. Refuse the fixup so that we go into die(). */
165 if (trapnr == X86_TRAP_PF) {
166 pr_emerg("BUG: pagefault on kernel address 0x%lx in non-whitelisted uaccess\n",
167 fault_addr);
168 } else {
169 pr_emerg("BUG: GPF in non-whitelisted uaccess (non-canonical address?)\n");
170 }
171 return true;
172}
173
174__visible bool ex_handler_uaccess(const struct exception_table_entry *fixup,
175 struct pt_regs *regs, int trapnr,
176 unsigned long error_code,
177 unsigned long fault_addr)
178{
179 if (bogus_uaccess(regs, trapnr, fault_addr))
180 return false;
181 regs->ip = ex_fixup_addr(fixup);
182 return true;
183}
184EXPORT_SYMBOL(ex_handler_uaccess);
185
111__visible bool ex_handler_ext(const struct exception_table_entry *fixup, 186__visible bool ex_handler_ext(const struct exception_table_entry *fixup,
112 struct pt_regs *regs, int trapnr) 187 struct pt_regs *regs, int trapnr,
188 unsigned long error_code,
189 unsigned long fault_addr)
113{ 190{
191 if (bogus_uaccess(regs, trapnr, fault_addr))
192 return false;
114 /* Special hack for uaccess_err */ 193 /* Special hack for uaccess_err */
115 current->thread.uaccess_err = 1; 194 current->thread.uaccess_err = 1;
116 regs->ip = ex_fixup_addr(fixup); 195 regs->ip = ex_fixup_addr(fixup);
@@ -119,7 +198,9 @@ __visible bool ex_handler_ext(const struct exception_table_entry *fixup,
119EXPORT_SYMBOL(ex_handler_ext); 198EXPORT_SYMBOL(ex_handler_ext);
120 199
121__visible bool ex_handler_rdmsr_unsafe(const struct exception_table_entry *fixup, 200__visible bool ex_handler_rdmsr_unsafe(const struct exception_table_entry *fixup,
122 struct pt_regs *regs, int trapnr) 201 struct pt_regs *regs, int trapnr,
202 unsigned long error_code,
203 unsigned long fault_addr)
123{ 204{
124 if (pr_warn_once("unchecked MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pF)\n", 205 if (pr_warn_once("unchecked MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pF)\n",
125 (unsigned int)regs->cx, regs->ip, (void *)regs->ip)) 206 (unsigned int)regs->cx, regs->ip, (void *)regs->ip))
@@ -134,7 +215,9 @@ __visible bool ex_handler_rdmsr_unsafe(const struct exception_table_entry *fixup
134EXPORT_SYMBOL(ex_handler_rdmsr_unsafe); 215EXPORT_SYMBOL(ex_handler_rdmsr_unsafe);
135 216
136__visible bool ex_handler_wrmsr_unsafe(const struct exception_table_entry *fixup, 217__visible bool ex_handler_wrmsr_unsafe(const struct exception_table_entry *fixup,
137 struct pt_regs *regs, int trapnr) 218 struct pt_regs *regs, int trapnr,
219 unsigned long error_code,
220 unsigned long fault_addr)
138{ 221{
139 if (pr_warn_once("unchecked MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pF)\n", 222 if (pr_warn_once("unchecked MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pF)\n",
140 (unsigned int)regs->cx, (unsigned int)regs->dx, 223 (unsigned int)regs->cx, (unsigned int)regs->dx,
@@ -148,12 +231,14 @@ __visible bool ex_handler_wrmsr_unsafe(const struct exception_table_entry *fixup
148EXPORT_SYMBOL(ex_handler_wrmsr_unsafe); 231EXPORT_SYMBOL(ex_handler_wrmsr_unsafe);
149 232
150__visible bool ex_handler_clear_fs(const struct exception_table_entry *fixup, 233__visible bool ex_handler_clear_fs(const struct exception_table_entry *fixup,
151 struct pt_regs *regs, int trapnr) 234 struct pt_regs *regs, int trapnr,
235 unsigned long error_code,
236 unsigned long fault_addr)
152{ 237{
153 if (static_cpu_has(X86_BUG_NULL_SEG)) 238 if (static_cpu_has(X86_BUG_NULL_SEG))
154 asm volatile ("mov %0, %%fs" : : "rm" (__USER_DS)); 239 asm volatile ("mov %0, %%fs" : : "rm" (__USER_DS));
155 asm volatile ("mov %0, %%fs" : : "rm" (0)); 240 asm volatile ("mov %0, %%fs" : : "rm" (0));
156 return ex_handler_default(fixup, regs, trapnr); 241 return ex_handler_default(fixup, regs, trapnr, error_code, fault_addr);
157} 242}
158EXPORT_SYMBOL(ex_handler_clear_fs); 243EXPORT_SYMBOL(ex_handler_clear_fs);
159 244
@@ -170,7 +255,8 @@ __visible bool ex_has_fault_handler(unsigned long ip)
170 return handler == ex_handler_fault; 255 return handler == ex_handler_fault;
171} 256}
172 257
173int fixup_exception(struct pt_regs *regs, int trapnr) 258int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code,
259 unsigned long fault_addr)
174{ 260{
175 const struct exception_table_entry *e; 261 const struct exception_table_entry *e;
176 ex_handler_t handler; 262 ex_handler_t handler;
@@ -194,7 +280,7 @@ int fixup_exception(struct pt_regs *regs, int trapnr)
194 return 0; 280 return 0;
195 281
196 handler = ex_fixup_handler(e); 282 handler = ex_fixup_handler(e);
197 return handler(e, regs, trapnr); 283 return handler(e, regs, trapnr, error_code, fault_addr);
198} 284}
199 285
200extern unsigned int early_recursion_flag; 286extern unsigned int early_recursion_flag;
@@ -230,9 +316,9 @@ void __init early_fixup_exception(struct pt_regs *regs, int trapnr)
230 * result in a hard-to-debug panic. 316 * result in a hard-to-debug panic.
231 * 317 *
232 * Keep in mind that not all vectors actually get here. Early 318 * Keep in mind that not all vectors actually get here. Early
233 * fage faults, for example, are special. 319 * page faults, for example, are special.
234 */ 320 */
235 if (fixup_exception(regs, trapnr)) 321 if (fixup_exception(regs, trapnr, regs->orig_ax, 0))
236 return; 322 return;
237 323
238 if (fixup_bug(regs, trapnr)) 324 if (fixup_bug(regs, trapnr))
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index a5b9ddb0f1fe..0d45f6debb3a 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -46,17 +46,19 @@ kmmio_fault(struct pt_regs *regs, unsigned long addr)
46 46
47static nokprobe_inline int kprobes_fault(struct pt_regs *regs) 47static nokprobe_inline int kprobes_fault(struct pt_regs *regs)
48{ 48{
49 int ret = 0; 49 if (!kprobes_built_in())
50 50 return 0;
51 /* kprobe_running() needs smp_processor_id() */ 51 if (user_mode(regs))
52 if (kprobes_built_in() && !user_mode(regs)) { 52 return 0;
53 preempt_disable(); 53 /*
54 if (kprobe_running() && kprobe_fault_handler(regs, 14)) 54 * To be potentially processing a kprobe fault and to be allowed to call
55 ret = 1; 55 * kprobe_running(), we have to be non-preemptible.
56 preempt_enable(); 56 */
57 } 57 if (preemptible())
58 58 return 0;
59 return ret; 59 if (!kprobe_running())
60 return 0;
61 return kprobe_fault_handler(regs, X86_TRAP_PF);
60} 62}
61 63
62/* 64/*
@@ -711,7 +713,7 @@ no_context(struct pt_regs *regs, unsigned long error_code,
711 int sig; 713 int sig;
712 714
713 /* Are we prepared to handle this kernel fault? */ 715 /* Are we prepared to handle this kernel fault? */
714 if (fixup_exception(regs, X86_TRAP_PF)) { 716 if (fixup_exception(regs, X86_TRAP_PF, error_code, address)) {
715 /* 717 /*
716 * Any interrupt that takes a fault gets the fixup. This makes 718 * Any interrupt that takes a fault gets the fixup. This makes
717 * the below recursive fault logic only apply to a faults from 719 * the below recursive fault logic only apply to a faults from
diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c
index 3a6c8ebc8032..0b08067c45f3 100644
--- a/arch/x86/tools/relocs.c
+++ b/arch/x86/tools/relocs.c
@@ -196,6 +196,7 @@ static const char *rel_type(unsigned type)
196#if ELF_BITS == 64 196#if ELF_BITS == 64
197 REL_TYPE(R_X86_64_NONE), 197 REL_TYPE(R_X86_64_NONE),
198 REL_TYPE(R_X86_64_64), 198 REL_TYPE(R_X86_64_64),
199 REL_TYPE(R_X86_64_PC64),
199 REL_TYPE(R_X86_64_PC32), 200 REL_TYPE(R_X86_64_PC32),
200 REL_TYPE(R_X86_64_GOT32), 201 REL_TYPE(R_X86_64_GOT32),
201 REL_TYPE(R_X86_64_PLT32), 202 REL_TYPE(R_X86_64_PLT32),
@@ -782,6 +783,15 @@ static int do_reloc64(struct section *sec, Elf_Rel *rel, ElfW(Sym) *sym,
782 add_reloc(&relocs32neg, offset); 783 add_reloc(&relocs32neg, offset);
783 break; 784 break;
784 785
786 case R_X86_64_PC64:
787 /*
788 * Only used by jump labels
789 */
790 if (is_percpu_sym(sym, symname))
791 die("Invalid R_X86_64_PC64 relocation against per-CPU symbol %s\n",
792 symname);
793 break;
794
785 case R_X86_64_32: 795 case R_X86_64_32:
786 case R_X86_64_32S: 796 case R_X86_64_32S:
787 case R_X86_64_64: 797 case R_X86_64_64:
diff --git a/arch/x86/um/asm/elf.h b/arch/x86/um/asm/elf.h
index 548197212a45..413f3519d9a1 100644
--- a/arch/x86/um/asm/elf.h
+++ b/arch/x86/um/asm/elf.h
@@ -116,8 +116,7 @@ do { \
116#define R_X86_64_PC16 13 /* 16 bit sign extended pc relative */ 116#define R_X86_64_PC16 13 /* 16 bit sign extended pc relative */
117#define R_X86_64_8 14 /* Direct 8 bit sign extended */ 117#define R_X86_64_8 14 /* Direct 8 bit sign extended */
118#define R_X86_64_PC8 15 /* 8 bit sign extended pc relative */ 118#define R_X86_64_PC8 15 /* 8 bit sign extended pc relative */
119 119#define R_X86_64_PC64 24 /* Place relative 64-bit signed */
120#define R_X86_64_NUM 16
121 120
122/* 121/*
123 * This is used to ensure we don't load something for the wrong architecture. 122 * This is used to ensure we don't load something for the wrong architecture.
diff --git a/arch/xtensa/kernel/Makefile b/arch/xtensa/kernel/Makefile
index 91907590d183..8dff506caf07 100644
--- a/arch/xtensa/kernel/Makefile
+++ b/arch/xtensa/kernel/Makefile
@@ -35,8 +35,8 @@ sed-y = -e ':a; s/\*(\([^)]*\)\.text\.unlikely/*(\1.literal.unlikely .{text}.unl
35 -e 's/\.{text}/.text/g' 35 -e 's/\.{text}/.text/g'
36 36
37quiet_cmd__cpp_lds_S = LDS $@ 37quiet_cmd__cpp_lds_S = LDS $@
38cmd__cpp_lds_S = $(CPP) $(cpp_flags) -P -C -Uxtensa -D__ASSEMBLY__ $< \ 38cmd__cpp_lds_S = $(CPP) $(cpp_flags) -P -C -Uxtensa -D__ASSEMBLY__ \
39 | sed $(sed-y) >$@ 39 -DLINKER_SCRIPT $< | sed $(sed-y) >$@
40 40
41$(obj)/vmlinux.lds: $(src)/vmlinux.lds.S FORCE 41$(obj)/vmlinux.lds: $(src)/vmlinux.lds.S FORCE
42 $(call if_changed_dep,_cpp_lds_S) 42 $(call if_changed_dep,_cpp_lds_S)
diff --git a/drivers/misc/lkdtm/core.c b/drivers/misc/lkdtm/core.c
index 2154d1bfd18b..5a755590d3dc 100644
--- a/drivers/misc/lkdtm/core.c
+++ b/drivers/misc/lkdtm/core.c
@@ -183,6 +183,7 @@ static const struct crashtype crashtypes[] = {
183 CRASHTYPE(USERCOPY_STACK_FRAME_FROM), 183 CRASHTYPE(USERCOPY_STACK_FRAME_FROM),
184 CRASHTYPE(USERCOPY_STACK_BEYOND), 184 CRASHTYPE(USERCOPY_STACK_BEYOND),
185 CRASHTYPE(USERCOPY_KERNEL), 185 CRASHTYPE(USERCOPY_KERNEL),
186 CRASHTYPE(USERCOPY_KERNEL_DS),
186}; 187};
187 188
188 189
diff --git a/drivers/misc/lkdtm/lkdtm.h b/drivers/misc/lkdtm/lkdtm.h
index 9e513dcfd809..07db641d71d0 100644
--- a/drivers/misc/lkdtm/lkdtm.h
+++ b/drivers/misc/lkdtm/lkdtm.h
@@ -82,5 +82,6 @@ void lkdtm_USERCOPY_STACK_FRAME_TO(void);
82void lkdtm_USERCOPY_STACK_FRAME_FROM(void); 82void lkdtm_USERCOPY_STACK_FRAME_FROM(void);
83void lkdtm_USERCOPY_STACK_BEYOND(void); 83void lkdtm_USERCOPY_STACK_BEYOND(void);
84void lkdtm_USERCOPY_KERNEL(void); 84void lkdtm_USERCOPY_KERNEL(void);
85void lkdtm_USERCOPY_KERNEL_DS(void);
85 86
86#endif 87#endif
diff --git a/drivers/misc/lkdtm/usercopy.c b/drivers/misc/lkdtm/usercopy.c
index 9725aed305bb..389475b25bb7 100644
--- a/drivers/misc/lkdtm/usercopy.c
+++ b/drivers/misc/lkdtm/usercopy.c
@@ -322,6 +322,19 @@ free_user:
322 vm_munmap(user_addr, PAGE_SIZE); 322 vm_munmap(user_addr, PAGE_SIZE);
323} 323}
324 324
325void lkdtm_USERCOPY_KERNEL_DS(void)
326{
327 char __user *user_ptr = (char __user *)ERR_PTR(-EINVAL);
328 mm_segment_t old_fs = get_fs();
329 char buf[10] = {0};
330
331 pr_info("attempting copy_to_user on unmapped kernel address\n");
332 set_fs(KERNEL_DS);
333 if (copy_to_user(user_ptr, buf, sizeof(buf)))
334 pr_info("copy_to_user un unmapped kernel address failed\n");
335 set_fs(old_fs);
336}
337
325void __init lkdtm_usercopy_init(void) 338void __init lkdtm_usercopy_init(void)
326{ 339{
327 /* Prepare cache that lacks SLAB_USERCOPY flag. */ 340 /* Prepare cache that lacks SLAB_USERCOPY flag. */
diff --git a/fs/namespace.c b/fs/namespace.c
index 99186556f8d3..d86830c86ce8 100644
--- a/fs/namespace.c
+++ b/fs/namespace.c
@@ -2642,6 +2642,7 @@ static long exact_copy_from_user(void *to, const void __user * from,
2642 if (!access_ok(VERIFY_READ, from, n)) 2642 if (!access_ok(VERIFY_READ, from, n))
2643 return n; 2643 return n;
2644 2644
2645 current->kernel_uaccess_faults_ok++;
2645 while (n) { 2646 while (n) {
2646 if (__get_user(c, f)) { 2647 if (__get_user(c, f)) {
2647 memset(t, 0, n); 2648 memset(t, 0, n);
@@ -2651,6 +2652,7 @@ static long exact_copy_from_user(void *to, const void __user * from,
2651 f++; 2652 f++;
2652 n--; 2653 n--;
2653 } 2654 }
2655 current->kernel_uaccess_faults_ok--;
2654 return n; 2656 return n;
2655} 2657}
2656 2658
diff --git a/include/asm-generic/bug.h b/include/asm-generic/bug.h
index 20561a60db9c..cdafa5edea49 100644
--- a/include/asm-generic/bug.h
+++ b/include/asm-generic/bug.h
@@ -17,10 +17,8 @@
17#ifndef __ASSEMBLY__ 17#ifndef __ASSEMBLY__
18#include <linux/kernel.h> 18#include <linux/kernel.h>
19 19
20#ifdef CONFIG_BUG
21
22#ifdef CONFIG_GENERIC_BUG
23struct bug_entry { 20struct bug_entry {
21#ifdef CONFIG_GENERIC_BUG
24#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS 22#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS
25 unsigned long bug_addr; 23 unsigned long bug_addr;
26#else 24#else
@@ -35,8 +33,10 @@ struct bug_entry {
35 unsigned short line; 33 unsigned short line;
36#endif 34#endif
37 unsigned short flags; 35 unsigned short flags;
38};
39#endif /* CONFIG_GENERIC_BUG */ 36#endif /* CONFIG_GENERIC_BUG */
37};
38
39#ifdef CONFIG_BUG
40 40
41/* 41/*
42 * Don't use BUG() or BUG_ON() unless there's really no way out; one 42 * Don't use BUG() or BUG_ON() unless there's really no way out; one
diff --git a/include/asm-generic/qrwlock.h b/include/asm-generic/qrwlock.h
index 0f7062bd55e5..36254d2da8e0 100644
--- a/include/asm-generic/qrwlock.h
+++ b/include/asm-generic/qrwlock.h
@@ -71,8 +71,8 @@ static inline int queued_write_trylock(struct qrwlock *lock)
71 if (unlikely(cnts)) 71 if (unlikely(cnts))
72 return 0; 72 return 0;
73 73
74 return likely(atomic_cmpxchg_acquire(&lock->cnts, 74 return likely(atomic_try_cmpxchg_acquire(&lock->cnts, &cnts,
75 cnts, cnts | _QW_LOCKED) == cnts); 75 _QW_LOCKED));
76} 76}
77/** 77/**
78 * queued_read_lock - acquire read lock of a queue rwlock 78 * queued_read_lock - acquire read lock of a queue rwlock
@@ -96,8 +96,9 @@ static inline void queued_read_lock(struct qrwlock *lock)
96 */ 96 */
97static inline void queued_write_lock(struct qrwlock *lock) 97static inline void queued_write_lock(struct qrwlock *lock)
98{ 98{
99 u32 cnts = 0;
99 /* Optimize for the unfair lock case where the fair flag is 0. */ 100 /* Optimize for the unfair lock case where the fair flag is 0. */
100 if (atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0) 101 if (likely(atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED)))
101 return; 102 return;
102 103
103 queued_write_lock_slowpath(lock); 104 queued_write_lock_slowpath(lock);
diff --git a/include/asm-generic/qspinlock.h b/include/asm-generic/qspinlock.h
index 9cc457597ddf..7541fa707f5b 100644
--- a/include/asm-generic/qspinlock.h
+++ b/include/asm-generic/qspinlock.h
@@ -66,10 +66,12 @@ static __always_inline int queued_spin_is_contended(struct qspinlock *lock)
66 */ 66 */
67static __always_inline int queued_spin_trylock(struct qspinlock *lock) 67static __always_inline int queued_spin_trylock(struct qspinlock *lock)
68{ 68{
69 if (!atomic_read(&lock->val) && 69 u32 val = atomic_read(&lock->val);
70 (atomic_cmpxchg_acquire(&lock->val, 0, _Q_LOCKED_VAL) == 0)) 70
71 return 1; 71 if (unlikely(val))
72 return 0; 72 return 0;
73
74 return likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL));
73} 75}
74 76
75extern void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); 77extern void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
@@ -80,11 +82,11 @@ extern void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
80 */ 82 */
81static __always_inline void queued_spin_lock(struct qspinlock *lock) 83static __always_inline void queued_spin_lock(struct qspinlock *lock)
82{ 84{
83 u32 val; 85 u32 val = 0;
84 86
85 val = atomic_cmpxchg_acquire(&lock->val, 0, _Q_LOCKED_VAL); 87 if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL)))
86 if (likely(val == 0))
87 return; 88 return;
89
88 queued_spin_lock_slowpath(lock, val); 90 queued_spin_lock_slowpath(lock, val);
89} 91}
90 92
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index d7701d466b60..d0bcea7c8f84 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -253,10 +253,6 @@
253 STRUCT_ALIGN(); \ 253 STRUCT_ALIGN(); \
254 *(__tracepoints) \ 254 *(__tracepoints) \
255 /* implement dynamic printk debug */ \ 255 /* implement dynamic printk debug */ \
256 . = ALIGN(8); \
257 __start___jump_table = .; \
258 KEEP(*(__jump_table)) \
259 __stop___jump_table = .; \
260 . = ALIGN(8); \ 256 . = ALIGN(8); \
261 __start___verbose = .; \ 257 __start___verbose = .; \
262 KEEP(*(__verbose)) \ 258 KEEP(*(__verbose)) \
@@ -300,6 +296,12 @@
300 . = __start_init_task + THREAD_SIZE; \ 296 . = __start_init_task + THREAD_SIZE; \
301 __end_init_task = .; 297 __end_init_task = .;
302 298
299#define JUMP_TABLE_DATA \
300 . = ALIGN(8); \
301 __start___jump_table = .; \
302 KEEP(*(__jump_table)) \
303 __stop___jump_table = .;
304
303/* 305/*
304 * Allow architectures to handle ro_after_init data on their 306 * Allow architectures to handle ro_after_init data on their
305 * own by defining an empty RO_AFTER_INIT_DATA. 307 * own by defining an empty RO_AFTER_INIT_DATA.
@@ -308,6 +310,7 @@
308#define RO_AFTER_INIT_DATA \ 310#define RO_AFTER_INIT_DATA \
309 __start_ro_after_init = .; \ 311 __start_ro_after_init = .; \
310 *(.data..ro_after_init) \ 312 *(.data..ro_after_init) \
313 JUMP_TABLE_DATA \
311 __end_ro_after_init = .; 314 __end_ro_after_init = .;
312#endif 315#endif
313 316
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index 681d866efb1e..1921545c6351 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -99,22 +99,13 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
99 * unique, to convince GCC not to merge duplicate inline asm statements. 99 * unique, to convince GCC not to merge duplicate inline asm statements.
100 */ 100 */
101#define annotate_reachable() ({ \ 101#define annotate_reachable() ({ \
102 asm volatile("%c0:\n\t" \ 102 asm volatile("ANNOTATE_REACHABLE counter=%c0" \
103 ".pushsection .discard.reachable\n\t" \ 103 : : "i" (__COUNTER__)); \
104 ".long %c0b - .\n\t" \
105 ".popsection\n\t" : : "i" (__COUNTER__)); \
106}) 104})
107#define annotate_unreachable() ({ \ 105#define annotate_unreachable() ({ \
108 asm volatile("%c0:\n\t" \ 106 asm volatile("ANNOTATE_UNREACHABLE counter=%c0" \
109 ".pushsection .discard.unreachable\n\t" \ 107 : : "i" (__COUNTER__)); \
110 ".long %c0b - .\n\t" \
111 ".popsection\n\t" : : "i" (__COUNTER__)); \
112}) 108})
113#define ASM_UNREACHABLE \
114 "999:\n\t" \
115 ".pushsection .discard.unreachable\n\t" \
116 ".long 999b - .\n\t" \
117 ".popsection\n\t"
118#else 109#else
119#define annotate_reachable() 110#define annotate_reachable()
120#define annotate_unreachable() 111#define annotate_unreachable()
@@ -299,6 +290,45 @@ static inline void *offset_to_ptr(const int *off)
299 return (void *)((unsigned long)off + *off); 290 return (void *)((unsigned long)off + *off);
300} 291}
301 292
293#else /* __ASSEMBLY__ */
294
295#ifdef __KERNEL__
296#ifndef LINKER_SCRIPT
297
298#ifdef CONFIG_STACK_VALIDATION
299.macro ANNOTATE_UNREACHABLE counter:req
300\counter:
301 .pushsection .discard.unreachable
302 .long \counter\()b -.
303 .popsection
304.endm
305
306.macro ANNOTATE_REACHABLE counter:req
307\counter:
308 .pushsection .discard.reachable
309 .long \counter\()b -.
310 .popsection
311.endm
312
313.macro ASM_UNREACHABLE
314999:
315 .pushsection .discard.unreachable
316 .long 999b - .
317 .popsection
318.endm
319#else /* CONFIG_STACK_VALIDATION */
320.macro ANNOTATE_UNREACHABLE counter:req
321.endm
322
323.macro ANNOTATE_REACHABLE counter:req
324.endm
325
326.macro ASM_UNREACHABLE
327.endm
328#endif /* CONFIG_STACK_VALIDATION */
329
330#endif /* LINKER_SCRIPT */
331#endif /* __KERNEL__ */
302#endif /* __ASSEMBLY__ */ 332#endif /* __ASSEMBLY__ */
303 333
304#ifndef __optimize 334#ifndef __optimize
diff --git a/include/linux/debug_locks.h b/include/linux/debug_locks.h
index 120225e9a366..257ab3c92cb8 100644
--- a/include/linux/debug_locks.h
+++ b/include/linux/debug_locks.h
@@ -8,8 +8,8 @@
8 8
9struct task_struct; 9struct task_struct;
10 10
11extern int debug_locks; 11extern int debug_locks __read_mostly;
12extern int debug_locks_silent; 12extern int debug_locks_silent __read_mostly;
13 13
14 14
15static inline int __debug_locks_off(void) 15static inline int __debug_locks_off(void)
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 1a0b6f17a5d6..5df6a621e464 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -119,6 +119,68 @@ struct static_key {
119 119
120#ifdef HAVE_JUMP_LABEL 120#ifdef HAVE_JUMP_LABEL
121#include <asm/jump_label.h> 121#include <asm/jump_label.h>
122
123#ifndef __ASSEMBLY__
124#ifdef CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE
125
126struct jump_entry {
127 s32 code;
128 s32 target;
129 long key; // key may be far away from the core kernel under KASLR
130};
131
132static inline unsigned long jump_entry_code(const struct jump_entry *entry)
133{
134 return (unsigned long)&entry->code + entry->code;
135}
136
137static inline unsigned long jump_entry_target(const struct jump_entry *entry)
138{
139 return (unsigned long)&entry->target + entry->target;
140}
141
142static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
143{
144 long offset = entry->key & ~3L;
145
146 return (struct static_key *)((unsigned long)&entry->key + offset);
147}
148
149#else
150
151static inline unsigned long jump_entry_code(const struct jump_entry *entry)
152{
153 return entry->code;
154}
155
156static inline unsigned long jump_entry_target(const struct jump_entry *entry)
157{
158 return entry->target;
159}
160
161static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
162{
163 return (struct static_key *)((unsigned long)entry->key & ~3UL);
164}
165
166#endif
167
168static inline bool jump_entry_is_branch(const struct jump_entry *entry)
169{
170 return (unsigned long)entry->key & 1UL;
171}
172
173static inline bool jump_entry_is_init(const struct jump_entry *entry)
174{
175 return (unsigned long)entry->key & 2UL;
176}
177
178static inline void jump_entry_set_init(struct jump_entry *entry)
179{
180 entry->key |= 2;
181}
182
183#endif
122#endif 184#endif
123 185
124#ifndef __ASSEMBLY__ 186#ifndef __ASSEMBLY__
@@ -151,7 +213,6 @@ extern struct jump_entry __start___jump_table[];
151extern struct jump_entry __stop___jump_table[]; 213extern struct jump_entry __stop___jump_table[];
152 214
153extern void jump_label_init(void); 215extern void jump_label_init(void);
154extern void jump_label_invalidate_initmem(void);
155extern void jump_label_lock(void); 216extern void jump_label_lock(void);
156extern void jump_label_unlock(void); 217extern void jump_label_unlock(void);
157extern void arch_jump_label_transform(struct jump_entry *entry, 218extern void arch_jump_label_transform(struct jump_entry *entry,
@@ -199,8 +260,6 @@ static __always_inline void jump_label_init(void)
199 static_key_initialized = true; 260 static_key_initialized = true;
200} 261}
201 262
202static inline void jump_label_invalidate_initmem(void) {}
203
204static __always_inline bool static_key_false(struct static_key *key) 263static __always_inline bool static_key_false(struct static_key *key)
205{ 264{
206 if (unlikely(static_key_count(key) > 0)) 265 if (unlikely(static_key_count(key) > 0))
diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h
index b0d0b51c4d85..1fd82ff99c65 100644
--- a/include/linux/lockdep.h
+++ b/include/linux/lockdep.h
@@ -99,13 +99,8 @@ struct lock_class {
99 */ 99 */
100 unsigned int version; 100 unsigned int version;
101 101
102 /*
103 * Statistics counter:
104 */
105 unsigned long ops;
106
107 const char *name;
108 int name_version; 102 int name_version;
103 const char *name;
109 104
110#ifdef CONFIG_LOCK_STAT 105#ifdef CONFIG_LOCK_STAT
111 unsigned long contention_point[LOCKSTAT_POINTS]; 106 unsigned long contention_point[LOCKSTAT_POINTS];
diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h
index ab93b6eae696..67dbb57508b1 100644
--- a/include/linux/rwsem.h
+++ b/include/linux/rwsem.h
@@ -45,10 +45,10 @@ struct rw_semaphore {
45}; 45};
46 46
47/* 47/*
48 * Setting bit 0 of the owner field with other non-zero bits will indicate 48 * Setting bit 1 of the owner field but not bit 0 will indicate
49 * that the rwsem is writer-owned with an unknown owner. 49 * that the rwsem is writer-owned with an unknown owner.
50 */ 50 */
51#define RWSEM_OWNER_UNKNOWN ((struct task_struct *)-1L) 51#define RWSEM_OWNER_UNKNOWN ((struct task_struct *)-2L)
52 52
53extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem); 53extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem);
54extern struct rw_semaphore *rwsem_down_read_failed_killable(struct rw_semaphore *sem); 54extern struct rw_semaphore *rwsem_down_read_failed_killable(struct rw_semaphore *sem);
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 004ca21f7e80..58e2af8d064b 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -735,6 +735,12 @@ struct task_struct {
735 unsigned use_memdelay:1; 735 unsigned use_memdelay:1;
736#endif 736#endif
737 737
738 /*
739 * May usercopy functions fault on kernel addresses?
740 * This is not just a single bit because this can potentially nest.
741 */
742 unsigned int kernel_uaccess_faults_ok;
743
738 unsigned long atomic_flags; /* Flags requiring atomic access. */ 744 unsigned long atomic_flags; /* Flags requiring atomic access. */
739 745
740 struct restart_block restart_block; 746 struct restart_block restart_block;
diff --git a/init/main.c b/init/main.c
index 78b714a5fa94..1c3f90264280 100644
--- a/init/main.c
+++ b/init/main.c
@@ -1069,7 +1069,6 @@ static int __ref kernel_init(void *unused)
1069 /* need to finish all async __init code before freeing the memory */ 1069 /* need to finish all async __init code before freeing the memory */
1070 async_synchronize_full(); 1070 async_synchronize_full();
1071 ftrace_free_init_mem(); 1071 ftrace_free_init_mem();
1072 jump_label_invalidate_initmem();
1073 free_initmem(); 1072 free_initmem();
1074 mark_readonly(); 1073 mark_readonly();
1075 1074
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 0097acec1c71..be4859f07153 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -315,6 +315,16 @@ void lockdep_assert_cpus_held(void)
315 percpu_rwsem_assert_held(&cpu_hotplug_lock); 315 percpu_rwsem_assert_held(&cpu_hotplug_lock);
316} 316}
317 317
318static void lockdep_acquire_cpus_lock(void)
319{
320 rwsem_acquire(&cpu_hotplug_lock.rw_sem.dep_map, 0, 0, _THIS_IP_);
321}
322
323static void lockdep_release_cpus_lock(void)
324{
325 rwsem_release(&cpu_hotplug_lock.rw_sem.dep_map, 1, _THIS_IP_);
326}
327
318/* 328/*
319 * Wait for currently running CPU hotplug operations to complete (if any) and 329 * Wait for currently running CPU hotplug operations to complete (if any) and
320 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects 330 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
@@ -344,6 +354,17 @@ void cpu_hotplug_enable(void)
344 cpu_maps_update_done(); 354 cpu_maps_update_done();
345} 355}
346EXPORT_SYMBOL_GPL(cpu_hotplug_enable); 356EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
357
358#else
359
360static void lockdep_acquire_cpus_lock(void)
361{
362}
363
364static void lockdep_release_cpus_lock(void)
365{
366}
367
347#endif /* CONFIG_HOTPLUG_CPU */ 368#endif /* CONFIG_HOTPLUG_CPU */
348 369
349#ifdef CONFIG_HOTPLUG_SMT 370#ifdef CONFIG_HOTPLUG_SMT
@@ -616,6 +637,12 @@ static void cpuhp_thread_fun(unsigned int cpu)
616 */ 637 */
617 smp_mb(); 638 smp_mb();
618 639
640 /*
641 * The BP holds the hotplug lock, but we're now running on the AP,
642 * ensure that anybody asserting the lock is held, will actually find
643 * it so.
644 */
645 lockdep_acquire_cpus_lock();
619 cpuhp_lock_acquire(bringup); 646 cpuhp_lock_acquire(bringup);
620 647
621 if (st->single) { 648 if (st->single) {
@@ -661,6 +688,7 @@ static void cpuhp_thread_fun(unsigned int cpu)
661 } 688 }
662 689
663 cpuhp_lock_release(bringup); 690 cpuhp_lock_release(bringup);
691 lockdep_release_cpus_lock();
664 692
665 if (!st->should_run) 693 if (!st->should_run)
666 complete_ap_thread(st, bringup); 694 complete_ap_thread(st, bringup);
diff --git a/kernel/futex.c b/kernel/futex.c
index 11fc3bb456d6..3e2de8fc1891 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -1365,9 +1365,9 @@ static void __unqueue_futex(struct futex_q *q)
1365{ 1365{
1366 struct futex_hash_bucket *hb; 1366 struct futex_hash_bucket *hb;
1367 1367
1368 if (WARN_ON_SMP(!q->lock_ptr || !spin_is_locked(q->lock_ptr)) 1368 if (WARN_ON_SMP(!q->lock_ptr) || WARN_ON(plist_node_empty(&q->list)))
1369 || WARN_ON(plist_node_empty(&q->list)))
1370 return; 1369 return;
1370 lockdep_assert_held(q->lock_ptr);
1371 1371
1372 hb = container_of(q->lock_ptr, struct futex_hash_bucket, lock); 1372 hb = container_of(q->lock_ptr, struct futex_hash_bucket, lock);
1373 plist_del(&q->list, &hb->chain); 1373 plist_del(&q->list, &hb->chain);
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 2e62503bea0d..b28028b08d44 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -38,23 +38,43 @@ static int jump_label_cmp(const void *a, const void *b)
38 const struct jump_entry *jea = a; 38 const struct jump_entry *jea = a;
39 const struct jump_entry *jeb = b; 39 const struct jump_entry *jeb = b;
40 40
41 if (jea->key < jeb->key) 41 if (jump_entry_key(jea) < jump_entry_key(jeb))
42 return -1; 42 return -1;
43 43
44 if (jea->key > jeb->key) 44 if (jump_entry_key(jea) > jump_entry_key(jeb))
45 return 1; 45 return 1;
46 46
47 return 0; 47 return 0;
48} 48}
49 49
50static void jump_label_swap(void *a, void *b, int size)
51{
52 long delta = (unsigned long)a - (unsigned long)b;
53 struct jump_entry *jea = a;
54 struct jump_entry *jeb = b;
55 struct jump_entry tmp = *jea;
56
57 jea->code = jeb->code - delta;
58 jea->target = jeb->target - delta;
59 jea->key = jeb->key - delta;
60
61 jeb->code = tmp.code + delta;
62 jeb->target = tmp.target + delta;
63 jeb->key = tmp.key + delta;
64}
65
50static void 66static void
51jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) 67jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
52{ 68{
53 unsigned long size; 69 unsigned long size;
70 void *swapfn = NULL;
71
72 if (IS_ENABLED(CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE))
73 swapfn = jump_label_swap;
54 74
55 size = (((unsigned long)stop - (unsigned long)start) 75 size = (((unsigned long)stop - (unsigned long)start)
56 / sizeof(struct jump_entry)); 76 / sizeof(struct jump_entry));
57 sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); 77 sort(start, size, sizeof(struct jump_entry), jump_label_cmp, swapfn);
58} 78}
59 79
60static void jump_label_update(struct static_key *key); 80static void jump_label_update(struct static_key *key);
@@ -85,6 +105,7 @@ void static_key_slow_inc_cpuslocked(struct static_key *key)
85 int v, v1; 105 int v, v1;
86 106
87 STATIC_KEY_CHECK_USE(key); 107 STATIC_KEY_CHECK_USE(key);
108 lockdep_assert_cpus_held();
88 109
89 /* 110 /*
90 * Careful if we get concurrent static_key_slow_inc() calls; 111 * Careful if we get concurrent static_key_slow_inc() calls;
@@ -130,6 +151,7 @@ EXPORT_SYMBOL_GPL(static_key_slow_inc);
130void static_key_enable_cpuslocked(struct static_key *key) 151void static_key_enable_cpuslocked(struct static_key *key)
131{ 152{
132 STATIC_KEY_CHECK_USE(key); 153 STATIC_KEY_CHECK_USE(key);
154 lockdep_assert_cpus_held();
133 155
134 if (atomic_read(&key->enabled) > 0) { 156 if (atomic_read(&key->enabled) > 0) {
135 WARN_ON_ONCE(atomic_read(&key->enabled) != 1); 157 WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
@@ -160,6 +182,7 @@ EXPORT_SYMBOL_GPL(static_key_enable);
160void static_key_disable_cpuslocked(struct static_key *key) 182void static_key_disable_cpuslocked(struct static_key *key)
161{ 183{
162 STATIC_KEY_CHECK_USE(key); 184 STATIC_KEY_CHECK_USE(key);
185 lockdep_assert_cpus_held();
163 186
164 if (atomic_read(&key->enabled) != 1) { 187 if (atomic_read(&key->enabled) != 1) {
165 WARN_ON_ONCE(atomic_read(&key->enabled) != 0); 188 WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
@@ -185,6 +208,8 @@ static void __static_key_slow_dec_cpuslocked(struct static_key *key,
185 unsigned long rate_limit, 208 unsigned long rate_limit,
186 struct delayed_work *work) 209 struct delayed_work *work)
187{ 210{
211 lockdep_assert_cpus_held();
212
188 /* 213 /*
189 * The negative count check is valid even when a negative 214 * The negative count check is valid even when a negative
190 * key->enabled is in use by static_key_slow_inc(); a 215 * key->enabled is in use by static_key_slow_inc(); a
@@ -261,8 +286,8 @@ EXPORT_SYMBOL_GPL(jump_label_rate_limit);
261 286
262static int addr_conflict(struct jump_entry *entry, void *start, void *end) 287static int addr_conflict(struct jump_entry *entry, void *start, void *end)
263{ 288{
264 if (entry->code <= (unsigned long)end && 289 if (jump_entry_code(entry) <= (unsigned long)end &&
265 entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start) 290 jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
266 return 1; 291 return 1;
267 292
268 return 0; 293 return 0;
@@ -321,16 +346,6 @@ static inline void static_key_set_linked(struct static_key *key)
321 key->type |= JUMP_TYPE_LINKED; 346 key->type |= JUMP_TYPE_LINKED;
322} 347}
323 348
324static inline struct static_key *jump_entry_key(struct jump_entry *entry)
325{
326 return (struct static_key *)((unsigned long)entry->key & ~1UL);
327}
328
329static bool jump_entry_branch(struct jump_entry *entry)
330{
331 return (unsigned long)entry->key & 1UL;
332}
333
334/*** 349/***
335 * A 'struct static_key' uses a union such that it either points directly 350 * A 'struct static_key' uses a union such that it either points directly
336 * to a table of 'struct jump_entry' or to a linked list of modules which in 351 * to a table of 'struct jump_entry' or to a linked list of modules which in
@@ -355,7 +370,7 @@ static enum jump_label_type jump_label_type(struct jump_entry *entry)
355{ 370{
356 struct static_key *key = jump_entry_key(entry); 371 struct static_key *key = jump_entry_key(entry);
357 bool enabled = static_key_enabled(key); 372 bool enabled = static_key_enabled(key);
358 bool branch = jump_entry_branch(entry); 373 bool branch = jump_entry_is_branch(entry);
359 374
360 /* See the comment in linux/jump_label.h */ 375 /* See the comment in linux/jump_label.h */
361 return enabled ^ branch; 376 return enabled ^ branch;
@@ -363,19 +378,20 @@ static enum jump_label_type jump_label_type(struct jump_entry *entry)
363 378
364static void __jump_label_update(struct static_key *key, 379static void __jump_label_update(struct static_key *key,
365 struct jump_entry *entry, 380 struct jump_entry *entry,
366 struct jump_entry *stop) 381 struct jump_entry *stop,
382 bool init)
367{ 383{
368 for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) { 384 for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
369 /* 385 /*
370 * An entry->code of 0 indicates an entry which has been 386 * An entry->code of 0 indicates an entry which has been
371 * disabled because it was in an init text area. 387 * disabled because it was in an init text area.
372 */ 388 */
373 if (entry->code) { 389 if (init || !jump_entry_is_init(entry)) {
374 if (kernel_text_address(entry->code)) 390 if (kernel_text_address(jump_entry_code(entry)))
375 arch_jump_label_transform(entry, jump_label_type(entry)); 391 arch_jump_label_transform(entry, jump_label_type(entry));
376 else 392 else
377 WARN_ONCE(1, "can't patch jump_label at %pS", 393 WARN_ONCE(1, "can't patch jump_label at %pS",
378 (void *)(unsigned long)entry->code); 394 (void *)jump_entry_code(entry));
379 } 395 }
380 } 396 }
381} 397}
@@ -410,6 +426,9 @@ void __init jump_label_init(void)
410 if (jump_label_type(iter) == JUMP_LABEL_NOP) 426 if (jump_label_type(iter) == JUMP_LABEL_NOP)
411 arch_jump_label_transform_static(iter, JUMP_LABEL_NOP); 427 arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
412 428
429 if (init_section_contains((void *)jump_entry_code(iter), 1))
430 jump_entry_set_init(iter);
431
413 iterk = jump_entry_key(iter); 432 iterk = jump_entry_key(iter);
414 if (iterk == key) 433 if (iterk == key)
415 continue; 434 continue;
@@ -422,26 +441,13 @@ void __init jump_label_init(void)
422 cpus_read_unlock(); 441 cpus_read_unlock();
423} 442}
424 443
425/* Disable any jump label entries in __init/__exit code */
426void __init jump_label_invalidate_initmem(void)
427{
428 struct jump_entry *iter_start = __start___jump_table;
429 struct jump_entry *iter_stop = __stop___jump_table;
430 struct jump_entry *iter;
431
432 for (iter = iter_start; iter < iter_stop; iter++) {
433 if (init_section_contains((void *)(unsigned long)iter->code, 1))
434 iter->code = 0;
435 }
436}
437
438#ifdef CONFIG_MODULES 444#ifdef CONFIG_MODULES
439 445
440static enum jump_label_type jump_label_init_type(struct jump_entry *entry) 446static enum jump_label_type jump_label_init_type(struct jump_entry *entry)
441{ 447{
442 struct static_key *key = jump_entry_key(entry); 448 struct static_key *key = jump_entry_key(entry);
443 bool type = static_key_type(key); 449 bool type = static_key_type(key);
444 bool branch = jump_entry_branch(entry); 450 bool branch = jump_entry_is_branch(entry);
445 451
446 /* See the comment in linux/jump_label.h */ 452 /* See the comment in linux/jump_label.h */
447 return type ^ branch; 453 return type ^ branch;
@@ -455,7 +461,7 @@ struct static_key_mod {
455 461
456static inline struct static_key_mod *static_key_mod(struct static_key *key) 462static inline struct static_key_mod *static_key_mod(struct static_key *key)
457{ 463{
458 WARN_ON_ONCE(!(key->type & JUMP_TYPE_LINKED)); 464 WARN_ON_ONCE(!static_key_linked(key));
459 return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK); 465 return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK);
460} 466}
461 467
@@ -514,7 +520,8 @@ static void __jump_label_mod_update(struct static_key *key)
514 stop = __stop___jump_table; 520 stop = __stop___jump_table;
515 else 521 else
516 stop = m->jump_entries + m->num_jump_entries; 522 stop = m->jump_entries + m->num_jump_entries;
517 __jump_label_update(key, mod->entries, stop); 523 __jump_label_update(key, mod->entries, stop,
524 m && m->state == MODULE_STATE_COMING);
518 } 525 }
519} 526}
520 527
@@ -560,12 +567,15 @@ static int jump_label_add_module(struct module *mod)
560 for (iter = iter_start; iter < iter_stop; iter++) { 567 for (iter = iter_start; iter < iter_stop; iter++) {
561 struct static_key *iterk; 568 struct static_key *iterk;
562 569
570 if (within_module_init(jump_entry_code(iter), mod))
571 jump_entry_set_init(iter);
572
563 iterk = jump_entry_key(iter); 573 iterk = jump_entry_key(iter);
564 if (iterk == key) 574 if (iterk == key)
565 continue; 575 continue;
566 576
567 key = iterk; 577 key = iterk;
568 if (within_module(iter->key, mod)) { 578 if (within_module((unsigned long)key, mod)) {
569 static_key_set_entries(key, iter); 579 static_key_set_entries(key, iter);
570 continue; 580 continue;
571 } 581 }
@@ -595,7 +605,7 @@ static int jump_label_add_module(struct module *mod)
595 605
596 /* Only update if we've changed from our initial state */ 606 /* Only update if we've changed from our initial state */
597 if (jump_label_type(iter) != jump_label_init_type(iter)) 607 if (jump_label_type(iter) != jump_label_init_type(iter))
598 __jump_label_update(key, iter, iter_stop); 608 __jump_label_update(key, iter, iter_stop, true);
599 } 609 }
600 610
601 return 0; 611 return 0;
@@ -615,7 +625,7 @@ static void jump_label_del_module(struct module *mod)
615 625
616 key = jump_entry_key(iter); 626 key = jump_entry_key(iter);
617 627
618 if (within_module(iter->key, mod)) 628 if (within_module((unsigned long)key, mod))
619 continue; 629 continue;
620 630
621 /* No memory during module load */ 631 /* No memory during module load */
@@ -651,19 +661,6 @@ static void jump_label_del_module(struct module *mod)
651 } 661 }
652} 662}
653 663
654/* Disable any jump label entries in module init code */
655static void jump_label_invalidate_module_init(struct module *mod)
656{
657 struct jump_entry *iter_start = mod->jump_entries;
658 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
659 struct jump_entry *iter;
660
661 for (iter = iter_start; iter < iter_stop; iter++) {
662 if (within_module_init(iter->code, mod))
663 iter->code = 0;
664 }
665}
666
667static int 664static int
668jump_label_module_notify(struct notifier_block *self, unsigned long val, 665jump_label_module_notify(struct notifier_block *self, unsigned long val,
669 void *data) 666 void *data)
@@ -685,9 +682,6 @@ jump_label_module_notify(struct notifier_block *self, unsigned long val,
685 case MODULE_STATE_GOING: 682 case MODULE_STATE_GOING:
686 jump_label_del_module(mod); 683 jump_label_del_module(mod);
687 break; 684 break;
688 case MODULE_STATE_LIVE:
689 jump_label_invalidate_module_init(mod);
690 break;
691 } 685 }
692 686
693 jump_label_unlock(); 687 jump_label_unlock();
@@ -757,7 +751,8 @@ static void jump_label_update(struct static_key *key)
757 entry = static_key_entries(key); 751 entry = static_key_entries(key);
758 /* if there are no users, entry can be NULL */ 752 /* if there are no users, entry can be NULL */
759 if (entry) 753 if (entry)
760 __jump_label_update(key, entry, stop); 754 __jump_label_update(key, entry, stop,
755 system_state < SYSTEM_RUNNING);
761} 756}
762 757
763#ifdef CONFIG_STATIC_KEYS_SELFTEST 758#ifdef CONFIG_STATIC_KEYS_SELFTEST
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index dd13f865ad40..1efada2dd9dd 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -138,7 +138,7 @@ static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES];
138 * get freed - this significantly simplifies the debugging code. 138 * get freed - this significantly simplifies the debugging code.
139 */ 139 */
140unsigned long nr_lock_classes; 140unsigned long nr_lock_classes;
141static struct lock_class lock_classes[MAX_LOCKDEP_KEYS]; 141struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
142 142
143static inline struct lock_class *hlock_class(struct held_lock *hlock) 143static inline struct lock_class *hlock_class(struct held_lock *hlock)
144{ 144{
@@ -1391,7 +1391,9 @@ static void print_lock_class_header(struct lock_class *class, int depth)
1391 1391
1392 printk("%*s->", depth, ""); 1392 printk("%*s->", depth, "");
1393 print_lock_name(class); 1393 print_lock_name(class);
1394 printk(KERN_CONT " ops: %lu", class->ops); 1394#ifdef CONFIG_DEBUG_LOCKDEP
1395 printk(KERN_CONT " ops: %lu", debug_class_ops_read(class));
1396#endif
1395 printk(KERN_CONT " {\n"); 1397 printk(KERN_CONT " {\n");
1396 1398
1397 for (bit = 0; bit < LOCK_USAGE_STATES; bit++) { 1399 for (bit = 0; bit < LOCK_USAGE_STATES; bit++) {
@@ -2148,76 +2150,6 @@ static int check_no_collision(struct task_struct *curr,
2148} 2150}
2149 2151
2150/* 2152/*
2151 * This is for building a chain between just two different classes,
2152 * instead of adding a new hlock upon current, which is done by
2153 * add_chain_cache().
2154 *
2155 * This can be called in any context with two classes, while
2156 * add_chain_cache() must be done within the lock owener's context
2157 * since it uses hlock which might be racy in another context.
2158 */
2159static inline int add_chain_cache_classes(unsigned int prev,
2160 unsigned int next,
2161 unsigned int irq_context,
2162 u64 chain_key)
2163{
2164 struct hlist_head *hash_head = chainhashentry(chain_key);
2165 struct lock_chain *chain;
2166
2167 /*
2168 * Allocate a new chain entry from the static array, and add
2169 * it to the hash:
2170 */
2171
2172 /*
2173 * We might need to take the graph lock, ensure we've got IRQs
2174 * disabled to make this an IRQ-safe lock.. for recursion reasons
2175 * lockdep won't complain about its own locking errors.
2176 */
2177 if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
2178 return 0;
2179
2180 if (unlikely(nr_lock_chains >= MAX_LOCKDEP_CHAINS)) {
2181 if (!debug_locks_off_graph_unlock())
2182 return 0;
2183
2184 print_lockdep_off("BUG: MAX_LOCKDEP_CHAINS too low!");
2185 dump_stack();
2186 return 0;
2187 }
2188
2189 chain = lock_chains + nr_lock_chains++;
2190 chain->chain_key = chain_key;
2191 chain->irq_context = irq_context;
2192 chain->depth = 2;
2193 if (likely(nr_chain_hlocks + chain->depth <= MAX_LOCKDEP_CHAIN_HLOCKS)) {
2194 chain->base = nr_chain_hlocks;
2195 nr_chain_hlocks += chain->depth;
2196 chain_hlocks[chain->base] = prev - 1;
2197 chain_hlocks[chain->base + 1] = next -1;
2198 }
2199#ifdef CONFIG_DEBUG_LOCKDEP
2200 /*
2201 * Important for check_no_collision().
2202 */
2203 else {
2204 if (!debug_locks_off_graph_unlock())
2205 return 0;
2206
2207 print_lockdep_off("BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!");
2208 dump_stack();
2209 return 0;
2210 }
2211#endif
2212
2213 hlist_add_head_rcu(&chain->entry, hash_head);
2214 debug_atomic_inc(chain_lookup_misses);
2215 inc_chains();
2216
2217 return 1;
2218}
2219
2220/*
2221 * Adds a dependency chain into chain hashtable. And must be called with 2153 * Adds a dependency chain into chain hashtable. And must be called with
2222 * graph_lock held. 2154 * graph_lock held.
2223 * 2155 *
@@ -3262,6 +3194,10 @@ static int __lock_is_held(const struct lockdep_map *lock, int read);
3262/* 3194/*
3263 * This gets called for every mutex_lock*()/spin_lock*() operation. 3195 * This gets called for every mutex_lock*()/spin_lock*() operation.
3264 * We maintain the dependency maps and validate the locking attempt: 3196 * We maintain the dependency maps and validate the locking attempt:
3197 *
3198 * The callers must make sure that IRQs are disabled before calling it,
3199 * otherwise we could get an interrupt which would want to take locks,
3200 * which would end up in lockdep again.
3265 */ 3201 */
3266static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, 3202static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
3267 int trylock, int read, int check, int hardirqs_off, 3203 int trylock, int read, int check, int hardirqs_off,
@@ -3279,14 +3215,6 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
3279 if (unlikely(!debug_locks)) 3215 if (unlikely(!debug_locks))
3280 return 0; 3216 return 0;
3281 3217
3282 /*
3283 * Lockdep should run with IRQs disabled, otherwise we could
3284 * get an interrupt which would want to take locks, which would
3285 * end up in lockdep and have you got a head-ache already?
3286 */
3287 if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
3288 return 0;
3289
3290 if (!prove_locking || lock->key == &__lockdep_no_validate__) 3218 if (!prove_locking || lock->key == &__lockdep_no_validate__)
3291 check = 0; 3219 check = 0;
3292 3220
@@ -3300,7 +3228,9 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
3300 if (!class) 3228 if (!class)
3301 return 0; 3229 return 0;
3302 } 3230 }
3303 atomic_inc((atomic_t *)&class->ops); 3231
3232 debug_class_ops_inc(class);
3233
3304 if (very_verbose(class)) { 3234 if (very_verbose(class)) {
3305 printk("\nacquire class [%px] %s", class->key, class->name); 3235 printk("\nacquire class [%px] %s", class->key, class->name);
3306 if (class->name_version > 1) 3236 if (class->name_version > 1)
@@ -3543,6 +3473,9 @@ static int reacquire_held_locks(struct task_struct *curr, unsigned int depth,
3543{ 3473{
3544 struct held_lock *hlock; 3474 struct held_lock *hlock;
3545 3475
3476 if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
3477 return 0;
3478
3546 for (hlock = curr->held_locks + idx; idx < depth; idx++, hlock++) { 3479 for (hlock = curr->held_locks + idx; idx < depth; idx++, hlock++) {
3547 if (!__lock_acquire(hlock->instance, 3480 if (!__lock_acquire(hlock->instance,
3548 hlock_class(hlock)->subclass, 3481 hlock_class(hlock)->subclass,
@@ -3696,6 +3629,13 @@ __lock_release(struct lockdep_map *lock, int nested, unsigned long ip)
3696 curr->lockdep_depth = i; 3629 curr->lockdep_depth = i;
3697 curr->curr_chain_key = hlock->prev_chain_key; 3630 curr->curr_chain_key = hlock->prev_chain_key;
3698 3631
3632 /*
3633 * The most likely case is when the unlock is on the innermost
3634 * lock. In this case, we are done!
3635 */
3636 if (i == depth-1)
3637 return 1;
3638
3699 if (reacquire_held_locks(curr, depth, i + 1)) 3639 if (reacquire_held_locks(curr, depth, i + 1))
3700 return 0; 3640 return 0;
3701 3641
@@ -3703,10 +3643,14 @@ __lock_release(struct lockdep_map *lock, int nested, unsigned long ip)
3703 * We had N bottles of beer on the wall, we drank one, but now 3643 * We had N bottles of beer on the wall, we drank one, but now
3704 * there's not N-1 bottles of beer left on the wall... 3644 * there's not N-1 bottles of beer left on the wall...
3705 */ 3645 */
3706 if (DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth - 1)) 3646 DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth-1);
3707 return 0;
3708 3647
3709 return 1; 3648 /*
3649 * Since reacquire_held_locks() would have called check_chain_key()
3650 * indirectly via __lock_acquire(), we don't need to do it again
3651 * on return.
3652 */
3653 return 0;
3710} 3654}
3711 3655
3712static int __lock_is_held(const struct lockdep_map *lock, int read) 3656static int __lock_is_held(const struct lockdep_map *lock, int read)
@@ -4122,7 +4066,7 @@ void lock_contended(struct lockdep_map *lock, unsigned long ip)
4122{ 4066{
4123 unsigned long flags; 4067 unsigned long flags;
4124 4068
4125 if (unlikely(!lock_stat)) 4069 if (unlikely(!lock_stat || !debug_locks))
4126 return; 4070 return;
4127 4071
4128 if (unlikely(current->lockdep_recursion)) 4072 if (unlikely(current->lockdep_recursion))
@@ -4142,7 +4086,7 @@ void lock_acquired(struct lockdep_map *lock, unsigned long ip)
4142{ 4086{
4143 unsigned long flags; 4087 unsigned long flags;
4144 4088
4145 if (unlikely(!lock_stat)) 4089 if (unlikely(!lock_stat || !debug_locks))
4146 return; 4090 return;
4147 4091
4148 if (unlikely(current->lockdep_recursion)) 4092 if (unlikely(current->lockdep_recursion))
diff --git a/kernel/locking/lockdep_internals.h b/kernel/locking/lockdep_internals.h
index d459d624ba2a..88c847a41c8a 100644
--- a/kernel/locking/lockdep_internals.h
+++ b/kernel/locking/lockdep_internals.h
@@ -152,9 +152,15 @@ struct lockdep_stats {
152 int nr_find_usage_forwards_recursions; 152 int nr_find_usage_forwards_recursions;
153 int nr_find_usage_backwards_checks; 153 int nr_find_usage_backwards_checks;
154 int nr_find_usage_backwards_recursions; 154 int nr_find_usage_backwards_recursions;
155
156 /*
157 * Per lock class locking operation stat counts
158 */
159 unsigned long lock_class_ops[MAX_LOCKDEP_KEYS];
155}; 160};
156 161
157DECLARE_PER_CPU(struct lockdep_stats, lockdep_stats); 162DECLARE_PER_CPU(struct lockdep_stats, lockdep_stats);
163extern struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
158 164
159#define __debug_atomic_inc(ptr) \ 165#define __debug_atomic_inc(ptr) \
160 this_cpu_inc(lockdep_stats.ptr); 166 this_cpu_inc(lockdep_stats.ptr);
@@ -179,9 +185,30 @@ DECLARE_PER_CPU(struct lockdep_stats, lockdep_stats);
179 } \ 185 } \
180 __total; \ 186 __total; \
181}) 187})
188
189static inline void debug_class_ops_inc(struct lock_class *class)
190{
191 int idx;
192
193 idx = class - lock_classes;
194 __debug_atomic_inc(lock_class_ops[idx]);
195}
196
197static inline unsigned long debug_class_ops_read(struct lock_class *class)
198{
199 int idx, cpu;
200 unsigned long ops = 0;
201
202 idx = class - lock_classes;
203 for_each_possible_cpu(cpu)
204 ops += per_cpu(lockdep_stats.lock_class_ops[idx], cpu);
205 return ops;
206}
207
182#else 208#else
183# define __debug_atomic_inc(ptr) do { } while (0) 209# define __debug_atomic_inc(ptr) do { } while (0)
184# define debug_atomic_inc(ptr) do { } while (0) 210# define debug_atomic_inc(ptr) do { } while (0)
185# define debug_atomic_dec(ptr) do { } while (0) 211# define debug_atomic_dec(ptr) do { } while (0)
186# define debug_atomic_read(ptr) 0 212# define debug_atomic_read(ptr) 0
213# define debug_class_ops_inc(ptr) do { } while (0)
187#endif 214#endif
diff --git a/kernel/locking/lockdep_proc.c b/kernel/locking/lockdep_proc.c
index 3dd980dfba2d..3d31f9b0059e 100644
--- a/kernel/locking/lockdep_proc.c
+++ b/kernel/locking/lockdep_proc.c
@@ -68,7 +68,7 @@ static int l_show(struct seq_file *m, void *v)
68 68
69 seq_printf(m, "%p", class->key); 69 seq_printf(m, "%p", class->key);
70#ifdef CONFIG_DEBUG_LOCKDEP 70#ifdef CONFIG_DEBUG_LOCKDEP
71 seq_printf(m, " OPS:%8ld", class->ops); 71 seq_printf(m, " OPS:%8ld", debug_class_ops_read(class));
72#endif 72#endif
73#ifdef CONFIG_PROVE_LOCKING 73#ifdef CONFIG_PROVE_LOCKING
74 seq_printf(m, " FD:%5ld", lockdep_count_forward_deps(class)); 74 seq_printf(m, " FD:%5ld", lockdep_count_forward_deps(class));
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index bfaeb05123ff..8a8c3c208c5e 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -74,12 +74,24 @@
74 */ 74 */
75 75
76#include "mcs_spinlock.h" 76#include "mcs_spinlock.h"
77#define MAX_NODES 4
77 78
79/*
80 * On 64-bit architectures, the mcs_spinlock structure will be 16 bytes in
81 * size and four of them will fit nicely in one 64-byte cacheline. For
82 * pvqspinlock, however, we need more space for extra data. To accommodate
83 * that, we insert two more long words to pad it up to 32 bytes. IOW, only
84 * two of them can fit in a cacheline in this case. That is OK as it is rare
85 * to have more than 2 levels of slowpath nesting in actual use. We don't
86 * want to penalize pvqspinlocks to optimize for a rare case in native
87 * qspinlocks.
88 */
89struct qnode {
90 struct mcs_spinlock mcs;
78#ifdef CONFIG_PARAVIRT_SPINLOCKS 91#ifdef CONFIG_PARAVIRT_SPINLOCKS
79#define MAX_NODES 8 92 long reserved[2];
80#else
81#define MAX_NODES 4
82#endif 93#endif
94};
83 95
84/* 96/*
85 * The pending bit spinning loop count. 97 * The pending bit spinning loop count.
@@ -101,7 +113,7 @@
101 * 113 *
102 * PV doubles the storage and uses the second cacheline for PV state. 114 * PV doubles the storage and uses the second cacheline for PV state.
103 */ 115 */
104static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[MAX_NODES]); 116static DEFINE_PER_CPU_ALIGNED(struct qnode, qnodes[MAX_NODES]);
105 117
106/* 118/*
107 * We must be able to distinguish between no-tail and the tail at 0:0, 119 * We must be able to distinguish between no-tail and the tail at 0:0,
@@ -126,7 +138,13 @@ static inline __pure struct mcs_spinlock *decode_tail(u32 tail)
126 int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1; 138 int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
127 int idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET; 139 int idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
128 140
129 return per_cpu_ptr(&mcs_nodes[idx], cpu); 141 return per_cpu_ptr(&qnodes[idx].mcs, cpu);
142}
143
144static inline __pure
145struct mcs_spinlock *grab_mcs_node(struct mcs_spinlock *base, int idx)
146{
147 return &((struct qnode *)base + idx)->mcs;
130} 148}
131 149
132#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK) 150#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
@@ -232,6 +250,20 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
232#endif /* _Q_PENDING_BITS == 8 */ 250#endif /* _Q_PENDING_BITS == 8 */
233 251
234/** 252/**
253 * queued_fetch_set_pending_acquire - fetch the whole lock value and set pending
254 * @lock : Pointer to queued spinlock structure
255 * Return: The previous lock value
256 *
257 * *,*,* -> *,1,*
258 */
259#ifndef queued_fetch_set_pending_acquire
260static __always_inline u32 queued_fetch_set_pending_acquire(struct qspinlock *lock)
261{
262 return atomic_fetch_or_acquire(_Q_PENDING_VAL, &lock->val);
263}
264#endif
265
266/**
235 * set_locked - Set the lock bit and own the lock 267 * set_locked - Set the lock bit and own the lock
236 * @lock: Pointer to queued spinlock structure 268 * @lock: Pointer to queued spinlock structure
237 * 269 *
@@ -326,43 +358,48 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
326 /* 358 /*
327 * trylock || pending 359 * trylock || pending
328 * 360 *
329 * 0,0,0 -> 0,0,1 ; trylock 361 * 0,0,* -> 0,1,* -> 0,0,1 pending, trylock
330 * 0,0,1 -> 0,1,1 ; pending
331 */ 362 */
332 val = atomic_fetch_or_acquire(_Q_PENDING_VAL, &lock->val); 363 val = queued_fetch_set_pending_acquire(lock);
333 if (!(val & ~_Q_LOCKED_MASK)) {
334 /*
335 * We're pending, wait for the owner to go away.
336 *
337 * *,1,1 -> *,1,0
338 *
339 * this wait loop must be a load-acquire such that we match the
340 * store-release that clears the locked bit and create lock
341 * sequentiality; this is because not all
342 * clear_pending_set_locked() implementations imply full
343 * barriers.
344 */
345 if (val & _Q_LOCKED_MASK) {
346 atomic_cond_read_acquire(&lock->val,
347 !(VAL & _Q_LOCKED_MASK));
348 }
349 364
350 /* 365 /*
351 * take ownership and clear the pending bit. 366 * If we observe contention, there is a concurrent locker.
352 * 367 *
353 * *,1,0 -> *,0,1 368 * Undo and queue; our setting of PENDING might have made the
354 */ 369 * n,0,0 -> 0,0,0 transition fail and it will now be waiting
355 clear_pending_set_locked(lock); 370 * on @next to become !NULL.
356 qstat_inc(qstat_lock_pending, true); 371 */
357 return; 372 if (unlikely(val & ~_Q_LOCKED_MASK)) {
373
374 /* Undo PENDING if we set it. */
375 if (!(val & _Q_PENDING_MASK))
376 clear_pending(lock);
377
378 goto queue;
358 } 379 }
359 380
360 /* 381 /*
361 * If pending was clear but there are waiters in the queue, then 382 * We're pending, wait for the owner to go away.
362 * we need to undo our setting of pending before we queue ourselves. 383 *
384 * 0,1,1 -> 0,1,0
385 *
386 * this wait loop must be a load-acquire such that we match the
387 * store-release that clears the locked bit and create lock
388 * sequentiality; this is because not all
389 * clear_pending_set_locked() implementations imply full
390 * barriers.
391 */
392 if (val & _Q_LOCKED_MASK)
393 atomic_cond_read_acquire(&lock->val, !(VAL & _Q_LOCKED_MASK));
394
395 /*
396 * take ownership and clear the pending bit.
397 *
398 * 0,1,0 -> 0,0,1
363 */ 399 */
364 if (!(val & _Q_PENDING_MASK)) 400 clear_pending_set_locked(lock);
365 clear_pending(lock); 401 qstat_inc(qstat_lock_pending, true);
402 return;
366 403
367 /* 404 /*
368 * End of pending bit optimistic spinning and beginning of MCS 405 * End of pending bit optimistic spinning and beginning of MCS
@@ -371,11 +408,16 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
371queue: 408queue:
372 qstat_inc(qstat_lock_slowpath, true); 409 qstat_inc(qstat_lock_slowpath, true);
373pv_queue: 410pv_queue:
374 node = this_cpu_ptr(&mcs_nodes[0]); 411 node = this_cpu_ptr(&qnodes[0].mcs);
375 idx = node->count++; 412 idx = node->count++;
376 tail = encode_tail(smp_processor_id(), idx); 413 tail = encode_tail(smp_processor_id(), idx);
377 414
378 node += idx; 415 node = grab_mcs_node(node, idx);
416
417 /*
418 * Keep counts of non-zero index values:
419 */
420 qstat_inc(qstat_lock_idx1 + idx - 1, idx);
379 421
380 /* 422 /*
381 * Ensure that we increment the head node->count before initialising 423 * Ensure that we increment the head node->count before initialising
@@ -476,16 +518,25 @@ locked:
476 */ 518 */
477 519
478 /* 520 /*
479 * In the PV case we might already have _Q_LOCKED_VAL set. 521 * In the PV case we might already have _Q_LOCKED_VAL set, because
522 * of lock stealing; therefore we must also allow:
523 *
524 * n,0,1 -> 0,0,1
480 * 525 *
481 * The atomic_cond_read_acquire() call above has provided the 526 * Note: at this point: (val & _Q_PENDING_MASK) == 0, because of the
482 * necessary acquire semantics required for locking. 527 * above wait condition, therefore any concurrent setting of
528 * PENDING will make the uncontended transition fail.
483 */ 529 */
484 if (((val & _Q_TAIL_MASK) == tail) && 530 if ((val & _Q_TAIL_MASK) == tail) {
485 atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL)) 531 if (atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL))
486 goto release; /* No contention */ 532 goto release; /* No contention */
533 }
487 534
488 /* Either somebody is queued behind us or _Q_PENDING_VAL is set */ 535 /*
536 * Either somebody is queued behind us or _Q_PENDING_VAL got set
537 * which will then detect the remaining tail and queue behind us
538 * ensuring we'll see a @next.
539 */
489 set_locked(lock); 540 set_locked(lock);
490 541
491 /* 542 /*
@@ -501,7 +552,7 @@ release:
501 /* 552 /*
502 * release the node 553 * release the node
503 */ 554 */
504 __this_cpu_dec(mcs_nodes[0].count); 555 __this_cpu_dec(qnodes[0].mcs.count);
505} 556}
506EXPORT_SYMBOL(queued_spin_lock_slowpath); 557EXPORT_SYMBOL(queued_spin_lock_slowpath);
507 558
diff --git a/kernel/locking/qspinlock_paravirt.h b/kernel/locking/qspinlock_paravirt.h
index 5a0cf5f9008c..0130e488ebfe 100644
--- a/kernel/locking/qspinlock_paravirt.h
+++ b/kernel/locking/qspinlock_paravirt.h
@@ -49,8 +49,6 @@ enum vcpu_state {
49 49
50struct pv_node { 50struct pv_node {
51 struct mcs_spinlock mcs; 51 struct mcs_spinlock mcs;
52 struct mcs_spinlock __res[3];
53
54 int cpu; 52 int cpu;
55 u8 state; 53 u8 state;
56}; 54};
@@ -281,7 +279,7 @@ static void pv_init_node(struct mcs_spinlock *node)
281{ 279{
282 struct pv_node *pn = (struct pv_node *)node; 280 struct pv_node *pn = (struct pv_node *)node;
283 281
284 BUILD_BUG_ON(sizeof(struct pv_node) > 5*sizeof(struct mcs_spinlock)); 282 BUILD_BUG_ON(sizeof(struct pv_node) > sizeof(struct qnode));
285 283
286 pn->cpu = smp_processor_id(); 284 pn->cpu = smp_processor_id();
287 pn->state = vcpu_running; 285 pn->state = vcpu_running;
diff --git a/kernel/locking/qspinlock_stat.h b/kernel/locking/qspinlock_stat.h
index 6bd78c0740fc..42d3d8dc8f49 100644
--- a/kernel/locking/qspinlock_stat.h
+++ b/kernel/locking/qspinlock_stat.h
@@ -55,6 +55,9 @@ enum qlock_stats {
55 qstat_pv_wait_node, 55 qstat_pv_wait_node,
56 qstat_lock_pending, 56 qstat_lock_pending,
57 qstat_lock_slowpath, 57 qstat_lock_slowpath,
58 qstat_lock_idx1,
59 qstat_lock_idx2,
60 qstat_lock_idx3,
58 qstat_num, /* Total number of statistical counters */ 61 qstat_num, /* Total number of statistical counters */
59 qstat_reset_cnts = qstat_num, 62 qstat_reset_cnts = qstat_num,
60}; 63};
@@ -82,6 +85,9 @@ static const char * const qstat_names[qstat_num + 1] = {
82 [qstat_pv_wait_node] = "pv_wait_node", 85 [qstat_pv_wait_node] = "pv_wait_node",
83 [qstat_lock_pending] = "lock_pending", 86 [qstat_lock_pending] = "lock_pending",
84 [qstat_lock_slowpath] = "lock_slowpath", 87 [qstat_lock_slowpath] = "lock_slowpath",
88 [qstat_lock_idx1] = "lock_index1",
89 [qstat_lock_idx2] = "lock_index2",
90 [qstat_lock_idx3] = "lock_index3",
85 [qstat_reset_cnts] = "reset_counters", 91 [qstat_reset_cnts] = "reset_counters",
86}; 92};
87 93
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index 2823d4163a37..581edcc63c26 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -1485,9 +1485,9 @@ void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass)
1485 __rt_mutex_lock(lock, subclass); 1485 __rt_mutex_lock(lock, subclass);
1486} 1486}
1487EXPORT_SYMBOL_GPL(rt_mutex_lock_nested); 1487EXPORT_SYMBOL_GPL(rt_mutex_lock_nested);
1488#endif
1489 1488
1490#ifndef CONFIG_DEBUG_LOCK_ALLOC 1489#else /* !CONFIG_DEBUG_LOCK_ALLOC */
1490
1491/** 1491/**
1492 * rt_mutex_lock - lock a rt_mutex 1492 * rt_mutex_lock - lock a rt_mutex
1493 * 1493 *
diff --git a/kernel/locking/rwsem-xadd.c b/kernel/locking/rwsem-xadd.c
index 3064c50e181e..09b180063ee1 100644
--- a/kernel/locking/rwsem-xadd.c
+++ b/kernel/locking/rwsem-xadd.c
@@ -180,7 +180,7 @@ static void __rwsem_mark_wake(struct rw_semaphore *sem,
180 * but it gives the spinners an early indication that the 180 * but it gives the spinners an early indication that the
181 * readers now have the lock. 181 * readers now have the lock.
182 */ 182 */
183 rwsem_set_reader_owned(sem); 183 __rwsem_set_reader_owned(sem, waiter->task);
184 } 184 }
185 185
186 /* 186 /*
@@ -233,8 +233,19 @@ __rwsem_down_read_failed_common(struct rw_semaphore *sem, int state)
233 waiter.type = RWSEM_WAITING_FOR_READ; 233 waiter.type = RWSEM_WAITING_FOR_READ;
234 234
235 raw_spin_lock_irq(&sem->wait_lock); 235 raw_spin_lock_irq(&sem->wait_lock);
236 if (list_empty(&sem->wait_list)) 236 if (list_empty(&sem->wait_list)) {
237 /*
238 * In case the wait queue is empty and the lock isn't owned
239 * by a writer, this reader can exit the slowpath and return
240 * immediately as its RWSEM_ACTIVE_READ_BIAS has already
241 * been set in the count.
242 */
243 if (atomic_long_read(&sem->count) >= 0) {
244 raw_spin_unlock_irq(&sem->wait_lock);
245 return sem;
246 }
237 adjustment += RWSEM_WAITING_BIAS; 247 adjustment += RWSEM_WAITING_BIAS;
248 }
238 list_add_tail(&waiter.list, &sem->wait_list); 249 list_add_tail(&waiter.list, &sem->wait_list);
239 250
240 /* we're now waiting on the lock, but no longer actively locking */ 251 /* we're now waiting on the lock, but no longer actively locking */
diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
index 776308d2fa9e..e586f0d03ad3 100644
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -117,8 +117,9 @@ EXPORT_SYMBOL(down_write_trylock);
117void up_read(struct rw_semaphore *sem) 117void up_read(struct rw_semaphore *sem)
118{ 118{
119 rwsem_release(&sem->dep_map, 1, _RET_IP_); 119 rwsem_release(&sem->dep_map, 1, _RET_IP_);
120 DEBUG_RWSEMS_WARN_ON(sem->owner != RWSEM_READER_OWNED); 120 DEBUG_RWSEMS_WARN_ON(!((unsigned long)sem->owner & RWSEM_READER_OWNED));
121 121
122 rwsem_clear_reader_owned(sem);
122 __up_read(sem); 123 __up_read(sem);
123} 124}
124 125
@@ -181,7 +182,7 @@ void down_read_non_owner(struct rw_semaphore *sem)
181 might_sleep(); 182 might_sleep();
182 183
183 __down_read(sem); 184 __down_read(sem);
184 rwsem_set_reader_owned(sem); 185 __rwsem_set_reader_owned(sem, NULL);
185} 186}
186 187
187EXPORT_SYMBOL(down_read_non_owner); 188EXPORT_SYMBOL(down_read_non_owner);
@@ -215,7 +216,7 @@ EXPORT_SYMBOL(down_write_killable_nested);
215 216
216void up_read_non_owner(struct rw_semaphore *sem) 217void up_read_non_owner(struct rw_semaphore *sem)
217{ 218{
218 DEBUG_RWSEMS_WARN_ON(sem->owner != RWSEM_READER_OWNED); 219 DEBUG_RWSEMS_WARN_ON(!((unsigned long)sem->owner & RWSEM_READER_OWNED));
219 __up_read(sem); 220 __up_read(sem);
220} 221}
221 222
diff --git a/kernel/locking/rwsem.h b/kernel/locking/rwsem.h
index b9d0e72aa80f..bad2bca0268b 100644
--- a/kernel/locking/rwsem.h
+++ b/kernel/locking/rwsem.h
@@ -1,24 +1,30 @@
1/* SPDX-License-Identifier: GPL-2.0 */ 1/* SPDX-License-Identifier: GPL-2.0 */
2/* 2/*
3 * The owner field of the rw_semaphore structure will be set to 3 * The least significant 2 bits of the owner value has the following
4 * RWSEM_READER_OWNED when a reader grabs the lock. A writer will clear 4 * meanings when set.
5 * the owner field when it unlocks. A reader, on the other hand, will 5 * - RWSEM_READER_OWNED (bit 0): The rwsem is owned by readers
6 * not touch the owner field when it unlocks. 6 * - RWSEM_ANONYMOUSLY_OWNED (bit 1): The rwsem is anonymously owned,
7 * i.e. the owner(s) cannot be readily determined. It can be reader
8 * owned or the owning writer is indeterminate.
7 * 9 *
8 * In essence, the owner field now has the following 4 states: 10 * When a writer acquires a rwsem, it puts its task_struct pointer
9 * 1) 0 11 * into the owner field. It is cleared after an unlock.
10 * - lock is free or the owner hasn't set the field yet 12 *
11 * 2) RWSEM_READER_OWNED 13 * When a reader acquires a rwsem, it will also puts its task_struct
12 * - lock is currently or previously owned by readers (lock is free 14 * pointer into the owner field with both the RWSEM_READER_OWNED and
13 * or not set by owner yet) 15 * RWSEM_ANONYMOUSLY_OWNED bits set. On unlock, the owner field will
14 * 3) RWSEM_ANONYMOUSLY_OWNED bit set with some other bits set as well 16 * largely be left untouched. So for a free or reader-owned rwsem,
15 * - lock is owned by an anonymous writer, so spinning on the lock 17 * the owner value may contain information about the last reader that
16 * owner should be disabled. 18 * acquires the rwsem. The anonymous bit is set because that particular
17 * 4) Other non-zero value 19 * reader may or may not still own the lock.
18 * - a writer owns the lock and other writers can spin on the lock owner. 20 *
21 * That information may be helpful in debugging cases where the system
22 * seems to hang on a reader owned rwsem especially if only one reader
23 * is involved. Ideally we would like to track all the readers that own
24 * a rwsem, but the overhead is simply too big.
19 */ 25 */
20#define RWSEM_ANONYMOUSLY_OWNED (1UL << 0) 26#define RWSEM_READER_OWNED (1UL << 0)
21#define RWSEM_READER_OWNED ((struct task_struct *)RWSEM_ANONYMOUSLY_OWNED) 27#define RWSEM_ANONYMOUSLY_OWNED (1UL << 1)
22 28
23#ifdef CONFIG_DEBUG_RWSEMS 29#ifdef CONFIG_DEBUG_RWSEMS
24# define DEBUG_RWSEMS_WARN_ON(c) DEBUG_LOCKS_WARN_ON(c) 30# define DEBUG_RWSEMS_WARN_ON(c) DEBUG_LOCKS_WARN_ON(c)
@@ -44,15 +50,26 @@ static inline void rwsem_clear_owner(struct rw_semaphore *sem)
44 WRITE_ONCE(sem->owner, NULL); 50 WRITE_ONCE(sem->owner, NULL);
45} 51}
46 52
53/*
54 * The task_struct pointer of the last owning reader will be left in
55 * the owner field.
56 *
57 * Note that the owner value just indicates the task has owned the rwsem
58 * previously, it may not be the real owner or one of the real owners
59 * anymore when that field is examined, so take it with a grain of salt.
60 */
61static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem,
62 struct task_struct *owner)
63{
64 unsigned long val = (unsigned long)owner | RWSEM_READER_OWNED
65 | RWSEM_ANONYMOUSLY_OWNED;
66
67 WRITE_ONCE(sem->owner, (struct task_struct *)val);
68}
69
47static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) 70static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
48{ 71{
49 /* 72 __rwsem_set_reader_owned(sem, current);
50 * We check the owner value first to make sure that we will only
51 * do a write to the rwsem cacheline when it is really necessary
52 * to minimize cacheline contention.
53 */
54 if (READ_ONCE(sem->owner) != RWSEM_READER_OWNED)
55 WRITE_ONCE(sem->owner, RWSEM_READER_OWNED);
56} 73}
57 74
58/* 75/*
@@ -72,6 +89,25 @@ static inline bool rwsem_has_anonymous_owner(struct task_struct *owner)
72{ 89{
73 return (unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED; 90 return (unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED;
74} 91}
92
93#ifdef CONFIG_DEBUG_RWSEMS
94/*
95 * With CONFIG_DEBUG_RWSEMS configured, it will make sure that if there
96 * is a task pointer in owner of a reader-owned rwsem, it will be the
97 * real owner or one of the real owners. The only exception is when the
98 * unlock is done by up_read_non_owner().
99 */
100#define rwsem_clear_reader_owned rwsem_clear_reader_owned
101static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)
102{
103 unsigned long val = (unsigned long)current | RWSEM_READER_OWNED
104 | RWSEM_ANONYMOUSLY_OWNED;
105 if (READ_ONCE(sem->owner) == (struct task_struct *)val)
106 cmpxchg_relaxed((unsigned long *)&sem->owner, val,
107 RWSEM_READER_OWNED | RWSEM_ANONYMOUSLY_OWNED);
108}
109#endif
110
75#else 111#else
76static inline void rwsem_set_owner(struct rw_semaphore *sem) 112static inline void rwsem_set_owner(struct rw_semaphore *sem)
77{ 113{
@@ -81,7 +117,18 @@ static inline void rwsem_clear_owner(struct rw_semaphore *sem)
81{ 117{
82} 118}
83 119
120static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem,
121 struct task_struct *owner)
122{
123}
124
84static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) 125static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
85{ 126{
86} 127}
87#endif 128#endif
129
130#ifndef rwsem_clear_reader_owned
131static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)
132{
133}
134#endif
diff --git a/kernel/module.c b/kernel/module.c
index 6746c85511fe..49a405891587 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -3317,6 +3317,15 @@ static struct module *layout_and_allocate(struct load_info *info, int flags)
3317 ndx = find_sec(info, ".data..ro_after_init"); 3317 ndx = find_sec(info, ".data..ro_after_init");
3318 if (ndx) 3318 if (ndx)
3319 info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT; 3319 info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT;
3320 /*
3321 * Mark the __jump_table section as ro_after_init as well: these data
3322 * structures are never modified, with the exception of entries that
3323 * refer to code in the __init section, which are annotated as such
3324 * at module load time.
3325 */
3326 ndx = find_sec(info, "__jump_table");
3327 if (ndx)
3328 info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT;
3320 3329
3321 /* Determine total sizes, and put offsets in sh_entsize. For now 3330 /* Determine total sizes, and put offsets in sh_entsize. For now
3322 this is done generically; there doesn't appear to be any 3331 this is done generically; there doesn't appear to be any
diff --git a/lib/debug_locks.c b/lib/debug_locks.c
index 96c4c633d95e..ce51749cc145 100644
--- a/lib/debug_locks.c
+++ b/lib/debug_locks.c
@@ -21,7 +21,7 @@
21 * that would just muddy the log. So we report the first one and 21 * that would just muddy the log. So we report the first one and
22 * shut up after that. 22 * shut up after that.
23 */ 23 */
24int debug_locks = 1; 24int debug_locks __read_mostly = 1;
25EXPORT_SYMBOL_GPL(debug_locks); 25EXPORT_SYMBOL_GPL(debug_locks);
26 26
27/* 27/*
@@ -29,7 +29,7 @@ EXPORT_SYMBOL_GPL(debug_locks);
29 * 'silent failure': nothing is printed to the console when 29 * 'silent failure': nothing is printed to the console when
30 * a locking bug is detected. 30 * a locking bug is detected.
31 */ 31 */
32int debug_locks_silent; 32int debug_locks_silent __read_mostly;
33EXPORT_SYMBOL_GPL(debug_locks_silent); 33EXPORT_SYMBOL_GPL(debug_locks_silent);
34 34
35/* 35/*
@@ -37,7 +37,7 @@ EXPORT_SYMBOL_GPL(debug_locks_silent);
37 */ 37 */
38int debug_locks_off(void) 38int debug_locks_off(void)
39{ 39{
40 if (__debug_locks_off()) { 40 if (debug_locks && __debug_locks_off()) {
41 if (!debug_locks_silent) { 41 if (!debug_locks_silent) {
42 console_verbose(); 42 console_verbose();
43 return 1; 43 return 1;
diff --git a/mm/maccess.c b/mm/maccess.c
index ec00be51a24f..f3416632e5a4 100644
--- a/mm/maccess.c
+++ b/mm/maccess.c
@@ -30,8 +30,10 @@ long __probe_kernel_read(void *dst, const void *src, size_t size)
30 30
31 set_fs(KERNEL_DS); 31 set_fs(KERNEL_DS);
32 pagefault_disable(); 32 pagefault_disable();
33 current->kernel_uaccess_faults_ok++;
33 ret = __copy_from_user_inatomic(dst, 34 ret = __copy_from_user_inatomic(dst,
34 (__force const void __user *)src, size); 35 (__force const void __user *)src, size);
36 current->kernel_uaccess_faults_ok--;
35 pagefault_enable(); 37 pagefault_enable();
36 set_fs(old_fs); 38 set_fs(old_fs);
37 39
@@ -58,7 +60,9 @@ long __probe_kernel_write(void *dst, const void *src, size_t size)
58 60
59 set_fs(KERNEL_DS); 61 set_fs(KERNEL_DS);
60 pagefault_disable(); 62 pagefault_disable();
63 current->kernel_uaccess_faults_ok++;
61 ret = __copy_to_user_inatomic((__force void __user *)dst, src, size); 64 ret = __copy_to_user_inatomic((__force void __user *)dst, src, size);
65 current->kernel_uaccess_faults_ok--;
62 pagefault_enable(); 66 pagefault_enable();
63 set_fs(old_fs); 67 set_fs(old_fs);
64 68
@@ -94,11 +98,13 @@ long strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count)
94 98
95 set_fs(KERNEL_DS); 99 set_fs(KERNEL_DS);
96 pagefault_disable(); 100 pagefault_disable();
101 current->kernel_uaccess_faults_ok++;
97 102
98 do { 103 do {
99 ret = __get_user(*dst++, (const char __user __force *)src++); 104 ret = __get_user(*dst++, (const char __user __force *)src++);
100 } while (dst[-1] && ret == 0 && src - unsafe_addr < count); 105 } while (dst[-1] && ret == 0 && src - unsafe_addr < count);
101 106
107 current->kernel_uaccess_faults_ok--;
102 dst[-1] = '\0'; 108 dst[-1] = '\0';
103 pagefault_enable(); 109 pagefault_enable();
104 set_fs(old_fs); 110 set_fs(old_fs);
diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
index ce53639a864a..8aeb60eb6ee3 100644
--- a/scripts/Kbuild.include
+++ b/scripts/Kbuild.include
@@ -115,7 +115,9 @@ __cc-option = $(call try-run,\
115 115
116# Do not attempt to build with gcc plugins during cc-option tests. 116# Do not attempt to build with gcc plugins during cc-option tests.
117# (And this uses delayed resolution so the flags will be up to date.) 117# (And this uses delayed resolution so the flags will be up to date.)
118CC_OPTION_CFLAGS = $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS)) 118# In addition, do not include the asm macros which are built later.
119CC_OPTION_FILTERED = $(GCC_PLUGINS_CFLAGS) $(ASM_MACRO_FLAGS)
120CC_OPTION_CFLAGS = $(filter-out $(CC_OPTION_FILTERED),$(KBUILD_CFLAGS))
119 121
120# cc-option 122# cc-option
121# Usage: cflags-y += $(call cc-option,-march=winchip-c6,-march=i586) 123# Usage: cflags-y += $(call cc-option,-march=winchip-c6,-march=i586)
diff --git a/scripts/mod/Makefile b/scripts/mod/Makefile
index 42c5d50f2bcc..a5b4af47987a 100644
--- a/scripts/mod/Makefile
+++ b/scripts/mod/Makefile
@@ -4,6 +4,8 @@ OBJECT_FILES_NON_STANDARD := y
4hostprogs-y := modpost mk_elfconfig 4hostprogs-y := modpost mk_elfconfig
5always := $(hostprogs-y) empty.o 5always := $(hostprogs-y) empty.o
6 6
7CFLAGS_REMOVE_empty.o := $(ASM_MACRO_FLAGS)
8
7modpost-objs := modpost.o file2alias.o sumversion.o 9modpost-objs := modpost.o file2alias.o sumversion.o
8 10
9devicetable-offsets-file := devicetable-offsets.h 11devicetable-offsets-file := devicetable-offsets.h
diff --git a/tools/memory-model/Documentation/explanation.txt b/tools/memory-model/Documentation/explanation.txt
index 0cbd1ef8f86d..35bff92cc773 100644
--- a/tools/memory-model/Documentation/explanation.txt
+++ b/tools/memory-model/Documentation/explanation.txt
@@ -28,7 +28,8 @@ Explanation of the Linux-Kernel Memory Consistency Model
28 20. THE HAPPENS-BEFORE RELATION: hb 28 20. THE HAPPENS-BEFORE RELATION: hb
29 21. THE PROPAGATES-BEFORE RELATION: pb 29 21. THE PROPAGATES-BEFORE RELATION: pb
30 22. RCU RELATIONS: rcu-link, gp, rscs, rcu-fence, and rb 30 22. RCU RELATIONS: rcu-link, gp, rscs, rcu-fence, and rb
31 23. ODDS AND ENDS 31 23. LOCKING
32 24. ODDS AND ENDS
32 33
33 34
34 35
@@ -1067,28 +1068,6 @@ allowing out-of-order writes like this to occur. The model avoided
1067violating the write-write coherence rule by requiring the CPU not to 1068violating the write-write coherence rule by requiring the CPU not to
1068send the W write to the memory subsystem at all!) 1069send the W write to the memory subsystem at all!)
1069 1070
1070There is one last example of preserved program order in the LKMM: when
1071a load-acquire reads from an earlier store-release. For example:
1072
1073 smp_store_release(&x, 123);
1074 r1 = smp_load_acquire(&x);
1075
1076If the smp_load_acquire() ends up obtaining the 123 value that was
1077stored by the smp_store_release(), the LKMM says that the load must be
1078executed after the store; the store cannot be forwarded to the load.
1079This requirement does not arise from the operational model, but it
1080yields correct predictions on all architectures supported by the Linux
1081kernel, although for differing reasons.
1082
1083On some architectures, including x86 and ARMv8, it is true that the
1084store cannot be forwarded to the load. On others, including PowerPC
1085and ARMv7, smp_store_release() generates object code that starts with
1086a fence and smp_load_acquire() generates object code that ends with a
1087fence. The upshot is that even though the store may be forwarded to
1088the load, it is still true that any instruction preceding the store
1089will be executed before the load or any following instructions, and
1090the store will be executed before any instruction following the load.
1091
1092 1071
1093AND THEN THERE WAS ALPHA 1072AND THEN THERE WAS ALPHA
1094------------------------ 1073------------------------
@@ -1766,6 +1745,147 @@ before it does, and the critical section in P2 both starts after P1's
1766grace period does and ends after it does. 1745grace period does and ends after it does.
1767 1746
1768 1747
1748LOCKING
1749-------
1750
1751The LKMM includes locking. In fact, there is special code for locking
1752in the formal model, added in order to make tools run faster.
1753However, this special code is intended to be more or less equivalent
1754to concepts we have already covered. A spinlock_t variable is treated
1755the same as an int, and spin_lock(&s) is treated almost the same as:
1756
1757 while (cmpxchg_acquire(&s, 0, 1) != 0)
1758 cpu_relax();
1759
1760This waits until s is equal to 0 and then atomically sets it to 1,
1761and the read part of the cmpxchg operation acts as an acquire fence.
1762An alternate way to express the same thing would be:
1763
1764 r = xchg_acquire(&s, 1);
1765
1766along with a requirement that at the end, r = 0. Similarly,
1767spin_trylock(&s) is treated almost the same as:
1768
1769 return !cmpxchg_acquire(&s, 0, 1);
1770
1771which atomically sets s to 1 if it is currently equal to 0 and returns
1772true if it succeeds (the read part of the cmpxchg operation acts as an
1773acquire fence only if the operation is successful). spin_unlock(&s)
1774is treated almost the same as:
1775
1776 smp_store_release(&s, 0);
1777
1778The "almost" qualifiers above need some explanation. In the LKMM, the
1779store-release in a spin_unlock() and the load-acquire which forms the
1780first half of the atomic rmw update in a spin_lock() or a successful
1781spin_trylock() -- we can call these things lock-releases and
1782lock-acquires -- have two properties beyond those of ordinary releases
1783and acquires.
1784
1785First, when a lock-acquire reads from a lock-release, the LKMM
1786requires that every instruction po-before the lock-release must
1787execute before any instruction po-after the lock-acquire. This would
1788naturally hold if the release and acquire operations were on different
1789CPUs, but the LKMM says it holds even when they are on the same CPU.
1790For example:
1791
1792 int x, y;
1793 spinlock_t s;
1794
1795 P0()
1796 {
1797 int r1, r2;
1798
1799 spin_lock(&s);
1800 r1 = READ_ONCE(x);
1801 spin_unlock(&s);
1802 spin_lock(&s);
1803 r2 = READ_ONCE(y);
1804 spin_unlock(&s);
1805 }
1806
1807 P1()
1808 {
1809 WRITE_ONCE(y, 1);
1810 smp_wmb();
1811 WRITE_ONCE(x, 1);
1812 }
1813
1814Here the second spin_lock() reads from the first spin_unlock(), and
1815therefore the load of x must execute before the load of y. Thus we
1816cannot have r1 = 1 and r2 = 0 at the end (this is an instance of the
1817MP pattern).
1818
1819This requirement does not apply to ordinary release and acquire
1820fences, only to lock-related operations. For instance, suppose P0()
1821in the example had been written as:
1822
1823 P0()
1824 {
1825 int r1, r2, r3;
1826
1827 r1 = READ_ONCE(x);
1828 smp_store_release(&s, 1);
1829 r3 = smp_load_acquire(&s);
1830 r2 = READ_ONCE(y);
1831 }
1832
1833Then the CPU would be allowed to forward the s = 1 value from the
1834smp_store_release() to the smp_load_acquire(), executing the
1835instructions in the following order:
1836
1837 r3 = smp_load_acquire(&s); // Obtains r3 = 1
1838 r2 = READ_ONCE(y);
1839 r1 = READ_ONCE(x);
1840 smp_store_release(&s, 1); // Value is forwarded
1841
1842and thus it could load y before x, obtaining r2 = 0 and r1 = 1.
1843
1844Second, when a lock-acquire reads from a lock-release, and some other
1845stores W and W' occur po-before the lock-release and po-after the
1846lock-acquire respectively, the LKMM requires that W must propagate to
1847each CPU before W' does. For example, consider:
1848
1849 int x, y;
1850 spinlock_t x;
1851
1852 P0()
1853 {
1854 spin_lock(&s);
1855 WRITE_ONCE(x, 1);
1856 spin_unlock(&s);
1857 }
1858
1859 P1()
1860 {
1861 int r1;
1862
1863 spin_lock(&s);
1864 r1 = READ_ONCE(x);
1865 WRITE_ONCE(y, 1);
1866 spin_unlock(&s);
1867 }
1868
1869 P2()
1870 {
1871 int r2, r3;
1872
1873 r2 = READ_ONCE(y);
1874 smp_rmb();
1875 r3 = READ_ONCE(x);
1876 }
1877
1878If r1 = 1 at the end then the spin_lock() in P1 must have read from
1879the spin_unlock() in P0. Hence the store to x must propagate to P2
1880before the store to y does, so we cannot have r2 = 1 and r3 = 0.
1881
1882These two special requirements for lock-release and lock-acquire do
1883not arise from the operational model. Nevertheless, kernel developers
1884have come to expect and rely on them because they do hold on all
1885architectures supported by the Linux kernel, albeit for various
1886differing reasons.
1887
1888
1769ODDS AND ENDS 1889ODDS AND ENDS
1770------------- 1890-------------
1771 1891
@@ -1831,26 +1951,6 @@ they behave as follows:
1831 events and the events preceding them against all po-later 1951 events and the events preceding them against all po-later
1832 events. 1952 events.
1833 1953
1834The LKMM includes locking. In fact, there is special code for locking
1835in the formal model, added in order to make tools run faster.
1836However, this special code is intended to be exactly equivalent to
1837concepts we have already covered. A spinlock_t variable is treated
1838the same as an int, and spin_lock(&s) is treated the same as:
1839
1840 while (cmpxchg_acquire(&s, 0, 1) != 0)
1841 cpu_relax();
1842
1843which waits until s is equal to 0 and then atomically sets it to 1,
1844and where the read part of the atomic update is also an acquire fence.
1845An alternate way to express the same thing would be:
1846
1847 r = xchg_acquire(&s, 1);
1848
1849along with a requirement that at the end, r = 0. spin_unlock(&s) is
1850treated the same as:
1851
1852 smp_store_release(&s, 0);
1853
1854Interestingly, RCU and locking each introduce the possibility of 1954Interestingly, RCU and locking each introduce the possibility of
1855deadlock. When faced with code sequences such as: 1955deadlock. When faced with code sequences such as:
1856 1956
diff --git a/tools/memory-model/Documentation/recipes.txt b/tools/memory-model/Documentation/recipes.txt
index af72700cc20a..7fe8d7aa3029 100644
--- a/tools/memory-model/Documentation/recipes.txt
+++ b/tools/memory-model/Documentation/recipes.txt
@@ -311,7 +311,7 @@ The smp_wmb() macro orders prior stores against later stores, and the
311smp_rmb() macro orders prior loads against later loads. Therefore, if 311smp_rmb() macro orders prior loads against later loads. Therefore, if
312the final value of r0 is 1, the final value of r1 must also be 1. 312the final value of r0 is 1, the final value of r1 must also be 1.
313 313
314The the xlog_state_switch_iclogs() function in fs/xfs/xfs_log.c contains 314The xlog_state_switch_iclogs() function in fs/xfs/xfs_log.c contains
315the following write-side code fragment: 315the following write-side code fragment:
316 316
317 log->l_curr_block -= log->l_logBBsize; 317 log->l_curr_block -= log->l_logBBsize;
diff --git a/tools/memory-model/README b/tools/memory-model/README
index ee987ce20aae..acf9077cffaa 100644
--- a/tools/memory-model/README
+++ b/tools/memory-model/README
@@ -171,6 +171,12 @@ The Linux-kernel memory model has the following limitations:
171 particular, the "THE PROGRAM ORDER RELATION: po AND po-loc" 171 particular, the "THE PROGRAM ORDER RELATION: po AND po-loc"
172 and "A WARNING" sections). 172 and "A WARNING" sections).
173 173
174 Note that this limitation in turn limits LKMM's ability to
175 accurately model address, control, and data dependencies.
176 For example, if the compiler can deduce the value of some variable
177 carrying a dependency, then the compiler can break that dependency
178 by substituting a constant of that value.
179
1742. Multiple access sizes for a single variable are not supported, 1802. Multiple access sizes for a single variable are not supported,
175 and neither are misaligned or partially overlapping accesses. 181 and neither are misaligned or partially overlapping accesses.
176 182
@@ -190,6 +196,36 @@ The Linux-kernel memory model has the following limitations:
190 However, a substantial amount of support is provided for these 196 However, a substantial amount of support is provided for these
191 operations, as shown in the linux-kernel.def file. 197 operations, as shown in the linux-kernel.def file.
192 198
199 a. When rcu_assign_pointer() is passed NULL, the Linux
200 kernel provides no ordering, but LKMM models this
201 case as a store release.
202
203 b. The "unless" RMW operations are not currently modeled:
204 atomic_long_add_unless(), atomic_add_unless(),
205 atomic_inc_unless_negative(), and
206 atomic_dec_unless_positive(). These can be emulated
207 in litmus tests, for example, by using atomic_cmpxchg().
208
209 c. The call_rcu() function is not modeled. It can be
210 emulated in litmus tests by adding another process that
211 invokes synchronize_rcu() and the body of the callback
212 function, with (for example) a release-acquire from
213 the site of the emulated call_rcu() to the beginning
214 of the additional process.
215
216 d. The rcu_barrier() function is not modeled. It can be
217 emulated in litmus tests emulating call_rcu() via
218 (for example) a release-acquire from the end of each
219 additional call_rcu() process to the site of the
220 emulated rcu-barrier().
221
222 e. Sleepable RCU (SRCU) is not modeled. It can be
223 emulated, but perhaps not simply.
224
225 f. Reader-writer locking is not modeled. It can be
226 emulated in litmus tests using atomic read-modify-write
227 operations.
228
193The "herd7" tool has some additional limitations of its own, apart from 229The "herd7" tool has some additional limitations of its own, apart from
194the memory model: 230the memory model:
195 231
@@ -204,3 +240,6 @@ the memory model:
204Some of these limitations may be overcome in the future, but others are 240Some of these limitations may be overcome in the future, but others are
205more likely to be addressed by incorporating the Linux-kernel memory model 241more likely to be addressed by incorporating the Linux-kernel memory model
206into other tools. 242into other tools.
243
244Finally, please note that LKMM is subject to change as hardware, use cases,
245and compilers evolve.
diff --git a/tools/memory-model/linux-kernel.cat b/tools/memory-model/linux-kernel.cat
index 59b5cbe6b624..882fc33274ac 100644
--- a/tools/memory-model/linux-kernel.cat
+++ b/tools/memory-model/linux-kernel.cat
@@ -38,7 +38,7 @@ let strong-fence = mb | gp
38(* Release Acquire *) 38(* Release Acquire *)
39let acq-po = [Acquire] ; po ; [M] 39let acq-po = [Acquire] ; po ; [M]
40let po-rel = [M] ; po ; [Release] 40let po-rel = [M] ; po ; [Release]
41let rfi-rel-acq = [Release] ; rfi ; [Acquire] 41let po-unlock-rf-lock-po = po ; [UL] ; rf ; [LKR] ; po
42 42
43(**********************************) 43(**********************************)
44(* Fundamental coherence ordering *) 44(* Fundamental coherence ordering *)
@@ -60,13 +60,13 @@ let dep = addr | data
60let rwdep = (dep | ctrl) ; [W] 60let rwdep = (dep | ctrl) ; [W]
61let overwrite = co | fr 61let overwrite = co | fr
62let to-w = rwdep | (overwrite & int) 62let to-w = rwdep | (overwrite & int)
63let to-r = addr | (dep ; rfi) | rfi-rel-acq 63let to-r = addr | (dep ; rfi)
64let fence = strong-fence | wmb | po-rel | rmb | acq-po 64let fence = strong-fence | wmb | po-rel | rmb | acq-po
65let ppo = to-r | to-w | fence 65let ppo = to-r | to-w | fence | (po-unlock-rf-lock-po & int)
66 66
67(* Propagation: Ordering from release operations and strong fences. *) 67(* Propagation: Ordering from release operations and strong fences. *)
68let A-cumul(r) = rfe? ; r 68let A-cumul(r) = rfe? ; r
69let cumul-fence = A-cumul(strong-fence | po-rel) | wmb 69let cumul-fence = A-cumul(strong-fence | po-rel) | wmb | po-unlock-rf-lock-po
70let prop = (overwrite & ext)? ; cumul-fence* ; rfe? 70let prop = (overwrite & ext)? ; cumul-fence* ; rfe?
71 71
72(* 72(*
diff --git a/tools/memory-model/litmus-tests/ISA2+pooncelock+pooncelock+pombonce.litmus b/tools/memory-model/litmus-tests/ISA2+pooncelock+pooncelock+pombonce.litmus
index 0f749e419b34..094d58df7789 100644
--- a/tools/memory-model/litmus-tests/ISA2+pooncelock+pooncelock+pombonce.litmus
+++ b/tools/memory-model/litmus-tests/ISA2+pooncelock+pooncelock+pombonce.litmus
@@ -1,11 +1,10 @@
1C ISA2+pooncelock+pooncelock+pombonce 1C ISA2+pooncelock+pooncelock+pombonce
2 2
3(* 3(*
4 * Result: Sometimes 4 * Result: Never
5 * 5 *
6 * This test shows that the ordering provided by a lock-protected S 6 * This test shows that write-write ordering provided by locks
7 * litmus test (P0() and P1()) are not visible to external process P2(). 7 * (in P0() and P1()) is visible to external process P2().
8 * This is likely to change soon.
9 *) 8 *)
10 9
11{} 10{}
diff --git a/tools/memory-model/litmus-tests/README b/tools/memory-model/litmus-tests/README
index 4581ec2d3c57..5ee08f129094 100644
--- a/tools/memory-model/litmus-tests/README
+++ b/tools/memory-model/litmus-tests/README
@@ -1,4 +1,6 @@
1This directory contains the following litmus tests: 1============
2LITMUS TESTS
3============
2 4
3CoRR+poonceonce+Once.litmus 5CoRR+poonceonce+Once.litmus
4 Test of read-read coherence, that is, whether or not two 6 Test of read-read coherence, that is, whether or not two
@@ -36,7 +38,7 @@ IRIW+poonceonces+OnceOnce.litmus
36ISA2+pooncelock+pooncelock+pombonce.litmus 38ISA2+pooncelock+pooncelock+pombonce.litmus
37 Tests whether the ordering provided by a lock-protected S 39 Tests whether the ordering provided by a lock-protected S
38 litmus test is visible to an external process whose accesses are 40 litmus test is visible to an external process whose accesses are
39 separated by smp_mb(). This addition of an external process to 41 separated by smp_mb(). This addition of an external process to
40 S is otherwise known as ISA2. 42 S is otherwise known as ISA2.
41 43
42ISA2+poonceonces.litmus 44ISA2+poonceonces.litmus
@@ -151,3 +153,101 @@ Z6.0+pooncerelease+poacquirerelease+fencembonceonce.litmus
151A great many more litmus tests are available here: 153A great many more litmus tests are available here:
152 154
153 https://github.com/paulmckrcu/litmus 155 https://github.com/paulmckrcu/litmus
156
157==================
158LITMUS TEST NAMING
159==================
160
161Litmus tests are usually named based on their contents, which means that
162looking at the name tells you what the litmus test does. The naming
163scheme covers litmus tests having a single cycle that passes through
164each process exactly once, so litmus tests not fitting this description
165are named on an ad-hoc basis.
166
167The structure of a litmus-test name is the litmus-test class, a plus
168sign ("+"), and one string for each process, separated by plus signs.
169The end of the name is ".litmus".
170
171The litmus-test classes may be found in the infamous test6.pdf:
172https://www.cl.cam.ac.uk/~pes20/ppc-supplemental/test6.pdf
173Each class defines the pattern of accesses and of the variables accessed.
174For example, if the one process writes to a pair of variables, and
175the other process reads from these same variables, the corresponding
176litmus-test class is "MP" (message passing), which may be found on the
177left-hand end of the second row of tests on page one of test6.pdf.
178
179The strings used to identify the actions carried out by each process are
180complex due to a desire to have short(er) names. Thus, there is a tool to
181generate these strings from a given litmus test's actions. For example,
182consider the processes from SB+rfionceonce-poonceonces.litmus:
183
184 P0(int *x, int *y)
185 {
186 int r1;
187 int r2;
188
189 WRITE_ONCE(*x, 1);
190 r1 = READ_ONCE(*x);
191 r2 = READ_ONCE(*y);
192 }
193
194 P1(int *x, int *y)
195 {
196 int r3;
197 int r4;
198
199 WRITE_ONCE(*y, 1);
200 r3 = READ_ONCE(*y);
201 r4 = READ_ONCE(*x);
202 }
203
204The next step is to construct a space-separated list of descriptors,
205interleaving descriptions of the relation between a pair of consecutive
206accesses with descriptions of the second access in the pair.
207
208P0()'s WRITE_ONCE() is read by its first READ_ONCE(), which is a
209reads-from link (rf) and internal to the P0() process. This is
210"rfi", which is an abbreviation for "reads-from internal". Because
211some of the tools string these abbreviations together with space
212characters separating processes, the first character is capitalized,
213resulting in "Rfi".
214
215P0()'s second access is a READ_ONCE(), as opposed to (for example)
216smp_load_acquire(), so next is "Once". Thus far, we have "Rfi Once".
217
218P0()'s third access is also a READ_ONCE(), but to y rather than x.
219This is related to P0()'s second access by program order ("po"),
220to a different variable ("d"), and both accesses are reads ("RR").
221The resulting descriptor is "PodRR". Because P0()'s third access is
222READ_ONCE(), we add another "Once" descriptor.
223
224A from-read ("fre") relation links P0()'s third to P1()'s first
225access, and the resulting descriptor is "Fre". P1()'s first access is
226WRITE_ONCE(), which as before gives the descriptor "Once". The string
227thus far is thus "Rfi Once PodRR Once Fre Once".
228
229The remainder of P1() is similar to P0(), which means we add
230"Rfi Once PodRR Once". Another fre links P1()'s last access to
231P0()'s first access, which is WRITE_ONCE(), so we add "Fre Once".
232The full string is thus:
233
234 Rfi Once PodRR Once Fre Once Rfi Once PodRR Once Fre Once
235
236This string can be given to the "norm7" and "classify7" tools to
237produce the name:
238
239 $ norm7 -bell linux-kernel.bell \
240 Rfi Once PodRR Once Fre Once Rfi Once PodRR Once Fre Once | \
241 sed -e 's/:.*//g'
242 SB+rfionceonce-poonceonces
243
244Adding the ".litmus" suffix: SB+rfionceonce-poonceonces.litmus
245
246The descriptors that describe connections between consecutive accesses
247within the cycle through a given litmus test can be provided by the herd
248tool (Rfi, Po, Fre, and so on) or by the linux-kernel.bell file (Once,
249Release, Acquire, and so on).
250
251To see the full list of descriptors, execute the following command:
252
253 $ diyone7 -bell linux-kernel.bell -show edges
diff --git a/tools/objtool/special.c b/tools/objtool/special.c
index 84f001d52322..50af4e1274b3 100644
--- a/tools/objtool/special.c
+++ b/tools/objtool/special.c
@@ -30,9 +30,9 @@
30#define EX_ORIG_OFFSET 0 30#define EX_ORIG_OFFSET 0
31#define EX_NEW_OFFSET 4 31#define EX_NEW_OFFSET 4
32 32
33#define JUMP_ENTRY_SIZE 24 33#define JUMP_ENTRY_SIZE 16
34#define JUMP_ORIG_OFFSET 0 34#define JUMP_ORIG_OFFSET 0
35#define JUMP_NEW_OFFSET 8 35#define JUMP_NEW_OFFSET 4
36 36
37#define ALT_ENTRY_SIZE 13 37#define ALT_ENTRY_SIZE 13
38#define ALT_ORIG_OFFSET 0 38#define ALT_ORIG_OFFSET 0
generated by cgit v1.2.2 (git 2.25.0) at 2026-04-27 07:38:24 -0400