aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexander Potapenko <glider@google.com>2016-03-25 17:22:05 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2016-03-25 19:37:42 -0400
commitbe7635e7287e0e8013af3c89a6354a9e0182594c (patch)
tree2c88dacfaa9d9b6a0479264c871142cf80e480de
parent505f5dcb1c419e55a9621a01f83eb5745d8d7398 (diff)
arch, ftrace: for KASAN put hard/soft IRQ entries into separate sections
KASAN needs to know whether the allocation happens in an IRQ handler. This lets us strip everything below the IRQ entry point to reduce the number of unique stack traces needed to be stored. Move the definition of __irq_entry to <linux/interrupt.h> so that the users don't need to pull in <linux/ftrace.h>. Also introduce the __softirq_entry macro which is similar to __irq_entry, but puts the corresponding functions to the .softirqentry.text section. Signed-off-by: Alexander Potapenko <glider@google.com> Acked-by: Steven Rostedt <rostedt@goodmis.org> Cc: Christoph Lameter <cl@linux.com> Cc: Pekka Enberg <penberg@kernel.org> Cc: David Rientjes <rientjes@google.com> Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com> Cc: Andrey Konovalov <adech.fo@gmail.com> Cc: Dmitry Vyukov <dvyukov@google.com> Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com> Cc: Konstantin Serebryany <kcc@google.com> Cc: Dmitry Chernenkov <dmitryc@google.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat
-rw-r--r--arch/arm/include/asm/exception.h2
-rw-r--r--arch/arm/kernel/vmlinux.lds.S1
-rw-r--r--arch/arm64/include/asm/exception.h2
-rw-r--r--arch/arm64/kernel/vmlinux.lds.S1
-rw-r--r--arch/blackfin/kernel/vmlinux.lds.S1
-rw-r--r--arch/c6x/kernel/vmlinux.lds.S1
-rw-r--r--arch/metag/kernel/vmlinux.lds.S1
-rw-r--r--arch/microblaze/kernel/vmlinux.lds.S1
-rw-r--r--arch/mips/kernel/vmlinux.lds.S1
-rw-r--r--arch/nios2/kernel/vmlinux.lds.S1
-rw-r--r--arch/openrisc/kernel/vmlinux.lds.S1
-rw-r--r--arch/parisc/kernel/vmlinux.lds.S1
-rw-r--r--arch/powerpc/kernel/vmlinux.lds.S1
-rw-r--r--arch/s390/kernel/vmlinux.lds.S1
-rw-r--r--arch/sh/kernel/vmlinux.lds.S1
-rw-r--r--arch/sparc/kernel/vmlinux.lds.S1
-rw-r--r--arch/tile/kernel/vmlinux.lds.S1
-rw-r--r--arch/x86/kernel/vmlinux.lds.S1
-rw-r--r--include/asm-generic/vmlinux.lds.h12
-rw-r--r--include/linux/ftrace.h11
-rw-r--r--include/linux/interrupt.h20
-rw-r--r--kernel/softirq.c2
-rw-r--r--kernel/trace/trace_functions_graph.c1
23 files changed, 51 insertions, 15 deletions
diff --git a/arch/arm/include/asm/exception.h b/arch/arm/include/asm/exception.h
index 5abaf5bbd985..bf1991263d2d 100644
--- a/arch/arm/include/asm/exception.h
+++ b/arch/arm/include/asm/exception.h
@@ -7,7 +7,7 @@
7#ifndef __ASM_ARM_EXCEPTION_H 7#ifndef __ASM_ARM_EXCEPTION_H
8#define __ASM_ARM_EXCEPTION_H 8#define __ASM_ARM_EXCEPTION_H
9 9
10#include <linux/ftrace.h> 10#include <linux/interrupt.h>
11 11
12#define __exception __attribute__((section(".exception.text"))) 12#define __exception __attribute__((section(".exception.text")))
13#ifdef CONFIG_FUNCTION_GRAPH_TRACER 13#ifdef CONFIG_FUNCTION_GRAPH_TRACER
diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S
index 1fab979daeaf..e2c6da096cef 100644
--- a/arch/arm/kernel/vmlinux.lds.S
+++ b/arch/arm/kernel/vmlinux.lds.S
@@ -108,6 +108,7 @@ SECTIONS
108 *(.exception.text) 108 *(.exception.text)
109 __exception_text_end = .; 109 __exception_text_end = .;
110 IRQENTRY_TEXT 110 IRQENTRY_TEXT
111 SOFTIRQENTRY_TEXT
111 TEXT_TEXT 112 TEXT_TEXT
112 SCHED_TEXT 113 SCHED_TEXT
113 LOCK_TEXT 114 LOCK_TEXT
diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h
index 6cb7e1a6bc02..0c2eec490abf 100644
--- a/arch/arm64/include/asm/exception.h
+++ b/arch/arm64/include/asm/exception.h
@@ -18,7 +18,7 @@
18#ifndef __ASM_EXCEPTION_H 18#ifndef __ASM_EXCEPTION_H
19#define __ASM_EXCEPTION_H 19#define __ASM_EXCEPTION_H
20 20
21#include <linux/ftrace.h> 21#include <linux/interrupt.h>
22 22
23#define __exception __attribute__((section(".exception.text"))) 23#define __exception __attribute__((section(".exception.text")))
24#ifdef CONFIG_FUNCTION_GRAPH_TRACER 24#ifdef CONFIG_FUNCTION_GRAPH_TRACER
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 37f624df68fa..5a1939a74ff3 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -103,6 +103,7 @@ SECTIONS
103 *(.exception.text) 103 *(.exception.text)
104 __exception_text_end = .; 104 __exception_text_end = .;
105 IRQENTRY_TEXT 105 IRQENTRY_TEXT
106 SOFTIRQENTRY_TEXT
106 TEXT_TEXT 107 TEXT_TEXT
107 SCHED_TEXT 108 SCHED_TEXT
108 LOCK_TEXT 109 LOCK_TEXT
diff --git a/arch/blackfin/kernel/vmlinux.lds.S b/arch/blackfin/kernel/vmlinux.lds.S
index c9eec84aa258..d920b959ff3a 100644
--- a/arch/blackfin/kernel/vmlinux.lds.S
+++ b/arch/blackfin/kernel/vmlinux.lds.S
@@ -35,6 +35,7 @@ SECTIONS
35#endif 35#endif
36 LOCK_TEXT 36 LOCK_TEXT
37 IRQENTRY_TEXT 37 IRQENTRY_TEXT
38 SOFTIRQENTRY_TEXT
38 KPROBES_TEXT 39 KPROBES_TEXT
39#ifdef CONFIG_ROMKERNEL 40#ifdef CONFIG_ROMKERNEL
40 __sinittext = .; 41 __sinittext = .;
diff --git a/arch/c6x/kernel/vmlinux.lds.S b/arch/c6x/kernel/vmlinux.lds.S
index 5a6e141d1641..50bc10f97bcb 100644
--- a/arch/c6x/kernel/vmlinux.lds.S
+++ b/arch/c6x/kernel/vmlinux.lds.S
@@ -72,6 +72,7 @@ SECTIONS
72 SCHED_TEXT 72 SCHED_TEXT
73 LOCK_TEXT 73 LOCK_TEXT
74 IRQENTRY_TEXT 74 IRQENTRY_TEXT
75 SOFTIRQENTRY_TEXT
75 KPROBES_TEXT 76 KPROBES_TEXT
76 *(.fixup) 77 *(.fixup)
77 *(.gnu.warning) 78 *(.gnu.warning)
diff --git a/arch/metag/kernel/vmlinux.lds.S b/arch/metag/kernel/vmlinux.lds.S
index e12055e88bfe..150ace92c7ad 100644
--- a/arch/metag/kernel/vmlinux.lds.S
+++ b/arch/metag/kernel/vmlinux.lds.S
@@ -24,6 +24,7 @@ SECTIONS
24 LOCK_TEXT 24 LOCK_TEXT
25 KPROBES_TEXT 25 KPROBES_TEXT
26 IRQENTRY_TEXT 26 IRQENTRY_TEXT
27 SOFTIRQENTRY_TEXT
27 *(.text.*) 28 *(.text.*)
28 *(.gnu.warning) 29 *(.gnu.warning)
29 } 30 }
diff --git a/arch/microblaze/kernel/vmlinux.lds.S b/arch/microblaze/kernel/vmlinux.lds.S
index be9488d69734..0a47f0410554 100644
--- a/arch/microblaze/kernel/vmlinux.lds.S
+++ b/arch/microblaze/kernel/vmlinux.lds.S
@@ -36,6 +36,7 @@ SECTIONS {
36 LOCK_TEXT 36 LOCK_TEXT
37 KPROBES_TEXT 37 KPROBES_TEXT
38 IRQENTRY_TEXT 38 IRQENTRY_TEXT
39 SOFTIRQENTRY_TEXT
39 . = ALIGN (4) ; 40 . = ALIGN (4) ;
40 _etext = . ; 41 _etext = . ;
41 } 42 }
diff --git a/arch/mips/kernel/vmlinux.lds.S b/arch/mips/kernel/vmlinux.lds.S
index 0a93e83cd014..54d653ee17e1 100644
--- a/arch/mips/kernel/vmlinux.lds.S
+++ b/arch/mips/kernel/vmlinux.lds.S
@@ -58,6 +58,7 @@ SECTIONS
58 LOCK_TEXT 58 LOCK_TEXT
59 KPROBES_TEXT 59 KPROBES_TEXT
60 IRQENTRY_TEXT 60 IRQENTRY_TEXT
61 SOFTIRQENTRY_TEXT
61 *(.text.*) 62 *(.text.*)
62 *(.fixup) 63 *(.fixup)
63 *(.gnu.warning) 64 *(.gnu.warning)
diff --git a/arch/nios2/kernel/vmlinux.lds.S b/arch/nios2/kernel/vmlinux.lds.S
index 326fab40a9de..e23e89539967 100644
--- a/arch/nios2/kernel/vmlinux.lds.S
+++ b/arch/nios2/kernel/vmlinux.lds.S
@@ -39,6 +39,7 @@ SECTIONS
39 SCHED_TEXT 39 SCHED_TEXT
40 LOCK_TEXT 40 LOCK_TEXT
41 IRQENTRY_TEXT 41 IRQENTRY_TEXT
42 SOFTIRQENTRY_TEXT
42 KPROBES_TEXT 43 KPROBES_TEXT
43 } =0 44 } =0
44 _etext = .; 45 _etext = .;
diff --git a/arch/openrisc/kernel/vmlinux.lds.S b/arch/openrisc/kernel/vmlinux.lds.S
index 2d69a853b742..d936de4c07ca 100644
--- a/arch/openrisc/kernel/vmlinux.lds.S
+++ b/arch/openrisc/kernel/vmlinux.lds.S
@@ -50,6 +50,7 @@ SECTIONS
50 LOCK_TEXT 50 LOCK_TEXT
51 KPROBES_TEXT 51 KPROBES_TEXT
52 IRQENTRY_TEXT 52 IRQENTRY_TEXT
53 SOFTIRQENTRY_TEXT
53 *(.fixup) 54 *(.fixup)
54 *(.text.__*) 55 *(.text.__*)
55 _etext = .; 56 _etext = .;
diff --git a/arch/parisc/kernel/vmlinux.lds.S b/arch/parisc/kernel/vmlinux.lds.S
index 308f29081d46..f3ead0b6ce46 100644
--- a/arch/parisc/kernel/vmlinux.lds.S
+++ b/arch/parisc/kernel/vmlinux.lds.S
@@ -72,6 +72,7 @@ SECTIONS
72 LOCK_TEXT 72 LOCK_TEXT
73 KPROBES_TEXT 73 KPROBES_TEXT
74 IRQENTRY_TEXT 74 IRQENTRY_TEXT
75 SOFTIRQENTRY_TEXT
75 *(.text.do_softirq) 76 *(.text.do_softirq)
76 *(.text.sys_exit) 77 *(.text.sys_exit)
77 *(.text.do_sigaltstack) 78 *(.text.do_sigaltstack)
diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S
index d41fd0af8980..2dd91f79de05 100644
--- a/arch/powerpc/kernel/vmlinux.lds.S
+++ b/arch/powerpc/kernel/vmlinux.lds.S
@@ -55,6 +55,7 @@ SECTIONS
55 LOCK_TEXT 55 LOCK_TEXT
56 KPROBES_TEXT 56 KPROBES_TEXT
57 IRQENTRY_TEXT 57 IRQENTRY_TEXT
58 SOFTIRQENTRY_TEXT
58 59
59#ifdef CONFIG_PPC32 60#ifdef CONFIG_PPC32
60 *(.got1) 61 *(.got1)
diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S
index 445657fe658c..0f41a8286378 100644
--- a/arch/s390/kernel/vmlinux.lds.S
+++ b/arch/s390/kernel/vmlinux.lds.S
@@ -28,6 +28,7 @@ SECTIONS
28 LOCK_TEXT 28 LOCK_TEXT
29 KPROBES_TEXT 29 KPROBES_TEXT
30 IRQENTRY_TEXT 30 IRQENTRY_TEXT
31 SOFTIRQENTRY_TEXT
31 *(.fixup) 32 *(.fixup)
32 *(.gnu.warning) 33 *(.gnu.warning)
33 } :text = 0x0700 34 } :text = 0x0700
diff --git a/arch/sh/kernel/vmlinux.lds.S b/arch/sh/kernel/vmlinux.lds.S
index db88cbf9eafd..235a4101999f 100644
--- a/arch/sh/kernel/vmlinux.lds.S
+++ b/arch/sh/kernel/vmlinux.lds.S
@@ -39,6 +39,7 @@ SECTIONS
39 LOCK_TEXT 39 LOCK_TEXT
40 KPROBES_TEXT 40 KPROBES_TEXT
41 IRQENTRY_TEXT 41 IRQENTRY_TEXT
42 SOFTIRQENTRY_TEXT
42 *(.fixup) 43 *(.fixup)
43 *(.gnu.warning) 44 *(.gnu.warning)
44 _etext = .; /* End of text section */ 45 _etext = .; /* End of text section */
diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S
index f1a2f688b28a..aadd321aa05d 100644
--- a/arch/sparc/kernel/vmlinux.lds.S
+++ b/arch/sparc/kernel/vmlinux.lds.S
@@ -48,6 +48,7 @@ SECTIONS
48 LOCK_TEXT 48 LOCK_TEXT
49 KPROBES_TEXT 49 KPROBES_TEXT
50 IRQENTRY_TEXT 50 IRQENTRY_TEXT
51 SOFTIRQENTRY_TEXT
51 *(.gnu.warning) 52 *(.gnu.warning)
52 } = 0 53 } = 0
53 _etext = .; 54 _etext = .;
diff --git a/arch/tile/kernel/vmlinux.lds.S b/arch/tile/kernel/vmlinux.lds.S
index 0e059a0101ea..378f5d8d1ec8 100644
--- a/arch/tile/kernel/vmlinux.lds.S
+++ b/arch/tile/kernel/vmlinux.lds.S
@@ -45,6 +45,7 @@ SECTIONS
45 LOCK_TEXT 45 LOCK_TEXT
46 KPROBES_TEXT 46 KPROBES_TEXT
47 IRQENTRY_TEXT 47 IRQENTRY_TEXT
48 SOFTIRQENTRY_TEXT
48 __fix_text_end = .; /* tile-cpack won't rearrange before this */ 49 __fix_text_end = .; /* tile-cpack won't rearrange before this */
49 ALIGN_FUNCTION(); 50 ALIGN_FUNCTION();
50 *(.hottext*) 51 *(.hottext*)
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index d239639e0c1d..4c941f88d405 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -101,6 +101,7 @@ SECTIONS
101 KPROBES_TEXT 101 KPROBES_TEXT
102 ENTRY_TEXT 102 ENTRY_TEXT
103 IRQENTRY_TEXT 103 IRQENTRY_TEXT
104 SOFTIRQENTRY_TEXT
104 *(.fixup) 105 *(.fixup)
105 *(.gnu.warning) 106 *(.gnu.warning)
106 /* End of text section */ 107 /* End of text section */
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 8f5a12ab2f2b..339125bb4d2c 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -456,7 +456,7 @@
456 *(.entry.text) \ 456 *(.entry.text) \
457 VMLINUX_SYMBOL(__entry_text_end) = .; 457 VMLINUX_SYMBOL(__entry_text_end) = .;
458 458
459#ifdef CONFIG_FUNCTION_GRAPH_TRACER 459#if defined(CONFIG_FUNCTION_GRAPH_TRACER) || defined(CONFIG_KASAN)
460#define IRQENTRY_TEXT \ 460#define IRQENTRY_TEXT \
461 ALIGN_FUNCTION(); \ 461 ALIGN_FUNCTION(); \
462 VMLINUX_SYMBOL(__irqentry_text_start) = .; \ 462 VMLINUX_SYMBOL(__irqentry_text_start) = .; \
@@ -466,6 +466,16 @@
466#define IRQENTRY_TEXT 466#define IRQENTRY_TEXT
467#endif 467#endif
468 468
469#if defined(CONFIG_FUNCTION_GRAPH_TRACER) || defined(CONFIG_KASAN)
470#define SOFTIRQENTRY_TEXT \
471 ALIGN_FUNCTION(); \
472 VMLINUX_SYMBOL(__softirqentry_text_start) = .; \
473 *(.softirqentry.text) \
474 VMLINUX_SYMBOL(__softirqentry_text_end) = .;
475#else
476#define SOFTIRQENTRY_TEXT
477#endif
478
469/* Section used for early init (in .S files) */ 479/* Section used for early init (in .S files) */
470#define HEAD_TEXT *(.head.text) 480#define HEAD_TEXT *(.head.text)
471 481
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 6d9df3f7e334..dea12a6e413b 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -811,16 +811,6 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth,
811 */ 811 */
812#define __notrace_funcgraph notrace 812#define __notrace_funcgraph notrace
813 813
814/*
815 * We want to which function is an entrypoint of a hardirq.
816 * That will help us to put a signal on output.
817 */
818#define __irq_entry __attribute__((__section__(".irqentry.text")))
819
820/* Limits of hardirq entrypoints */
821extern char __irqentry_text_start[];
822extern char __irqentry_text_end[];
823
824#define FTRACE_NOTRACE_DEPTH 65536 814#define FTRACE_NOTRACE_DEPTH 65536
825#define FTRACE_RETFUNC_DEPTH 50 815#define FTRACE_RETFUNC_DEPTH 50
826#define FTRACE_RETSTACK_ALLOC_SIZE 32 816#define FTRACE_RETSTACK_ALLOC_SIZE 32
@@ -857,7 +847,6 @@ static inline void unpause_graph_tracing(void)
857#else /* !CONFIG_FUNCTION_GRAPH_TRACER */ 847#else /* !CONFIG_FUNCTION_GRAPH_TRACER */
858 848
859#define __notrace_funcgraph 849#define __notrace_funcgraph
860#define __irq_entry
861#define INIT_FTRACE_GRAPH 850#define INIT_FTRACE_GRAPH
862 851
863static inline void ftrace_graph_init_task(struct task_struct *t) { } 852static inline void ftrace_graph_init_task(struct task_struct *t) { }
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 358076eda364..9fcabeb07787 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -683,4 +683,24 @@ extern int early_irq_init(void);
683extern int arch_probe_nr_irqs(void); 683extern int arch_probe_nr_irqs(void);
684extern int arch_early_irq_init(void); 684extern int arch_early_irq_init(void);
685 685
686#if defined(CONFIG_FUNCTION_GRAPH_TRACER) || defined(CONFIG_KASAN)
687/*
688 * We want to know which function is an entrypoint of a hardirq or a softirq.
689 */
690#define __irq_entry __attribute__((__section__(".irqentry.text")))
691#define __softirq_entry \
692 __attribute__((__section__(".softirqentry.text")))
693
694/* Limits of hardirq entrypoints */
695extern char __irqentry_text_start[];
696extern char __irqentry_text_end[];
697/* Limits of softirq entrypoints */
698extern char __softirqentry_text_start[];
699extern char __softirqentry_text_end[];
700
701#else
702#define __irq_entry
703#define __softirq_entry
704#endif
705
686#endif 706#endif
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 8aae49dd7da8..17caf4b63342 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -227,7 +227,7 @@ static inline bool lockdep_softirq_start(void) { return false; }
227static inline void lockdep_softirq_end(bool in_hardirq) { } 227static inline void lockdep_softirq_end(bool in_hardirq) { }
228#endif 228#endif
229 229
230asmlinkage __visible void __do_softirq(void) 230asmlinkage __visible void __softirq_entry __do_softirq(void)
231{ 231{
232 unsigned long end = jiffies + MAX_SOFTIRQ_TIME; 232 unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
233 unsigned long old_flags = current->flags; 233 unsigned long old_flags = current->flags;
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 91d6a63a2ea7..3a0244ff7ea8 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -8,6 +8,7 @@
8 */ 8 */
9#include <linux/uaccess.h> 9#include <linux/uaccess.h>
10#include <linux/ftrace.h> 10#include <linux/ftrace.h>
11#include <linux/interrupt.h>
11#include <linux/slab.h> 12#include <linux/slab.h>
12#include <linux/fs.h> 13#include <linux/fs.h>
13 14
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-13 02:52:05 -0400