diff options
-rw-r--r-- | include/linux/slab_def.h | 10 | ||||
-rw-r--r-- | include/linux/slub_def.h | 12 | ||||
-rw-r--r-- | include/trace/kmemtrace.h | 92 | ||||
-rw-r--r-- | kernel/trace/kmemtrace.c | 206 | ||||
-rw-r--r-- | kernel/trace/trace.h | 6 | ||||
-rw-r--r-- | mm/slab.c | 24 | ||||
-rw-r--r-- | mm/slob.c | 28 | ||||
-rw-r--r-- | mm/slub.c | 30 | ||||
-rw-r--r-- | mm/util.c | 16 |
9 files changed, 251 insertions, 173 deletions
diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index f4523651fa42..5ac9b0bcaf9a 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h | |||
@@ -73,8 +73,8 @@ found: | |||
73 | 73 | ||
74 | ret = kmem_cache_alloc_notrace(cachep, flags); | 74 | ret = kmem_cache_alloc_notrace(cachep, flags); |
75 | 75 | ||
76 | kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, _THIS_IP_, ret, | 76 | trace_kmalloc(_THIS_IP_, ret, |
77 | size, slab_buffer_size(cachep), flags); | 77 | size, slab_buffer_size(cachep), flags); |
78 | 78 | ||
79 | return ret; | 79 | return ret; |
80 | } | 80 | } |
@@ -128,9 +128,9 @@ found: | |||
128 | 128 | ||
129 | ret = kmem_cache_alloc_node_notrace(cachep, flags, node); | 129 | ret = kmem_cache_alloc_node_notrace(cachep, flags, node); |
130 | 130 | ||
131 | kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, _THIS_IP_, | 131 | trace_kmalloc_node(_THIS_IP_, ret, |
132 | ret, size, slab_buffer_size(cachep), | 132 | size, slab_buffer_size(cachep), |
133 | flags, node); | 133 | flags, node); |
134 | 134 | ||
135 | return ret; | 135 | return ret; |
136 | } | 136 | } |
diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index a1f90528e70b..5046f90c1171 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h | |||
@@ -233,8 +233,7 @@ static __always_inline void *kmalloc_large(size_t size, gfp_t flags) | |||
233 | unsigned int order = get_order(size); | 233 | unsigned int order = get_order(size); |
234 | void *ret = (void *) __get_free_pages(flags | __GFP_COMP, order); | 234 | void *ret = (void *) __get_free_pages(flags | __GFP_COMP, order); |
235 | 235 | ||
236 | kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, _THIS_IP_, ret, | 236 | trace_kmalloc(_THIS_IP_, ret, size, PAGE_SIZE << order, flags); |
237 | size, PAGE_SIZE << order, flags); | ||
238 | 237 | ||
239 | return ret; | 238 | return ret; |
240 | } | 239 | } |
@@ -255,9 +254,7 @@ static __always_inline void *kmalloc(size_t size, gfp_t flags) | |||
255 | 254 | ||
256 | ret = kmem_cache_alloc_notrace(s, flags); | 255 | ret = kmem_cache_alloc_notrace(s, flags); |
257 | 256 | ||
258 | kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, | 257 | trace_kmalloc(_THIS_IP_, ret, size, s->size, flags); |
259 | _THIS_IP_, ret, | ||
260 | size, s->size, flags); | ||
261 | 258 | ||
262 | return ret; | 259 | return ret; |
263 | } | 260 | } |
@@ -296,9 +293,8 @@ static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) | |||
296 | 293 | ||
297 | ret = kmem_cache_alloc_node_notrace(s, flags, node); | 294 | ret = kmem_cache_alloc_node_notrace(s, flags, node); |
298 | 295 | ||
299 | kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, | 296 | trace_kmalloc_node(_THIS_IP_, ret, |
300 | _THIS_IP_, ret, | 297 | size, s->size, flags, node); |
301 | size, s->size, flags, node); | ||
302 | 298 | ||
303 | return ret; | 299 | return ret; |
304 | } | 300 | } |
diff --git a/include/trace/kmemtrace.h b/include/trace/kmemtrace.h index ad8b7857855a..28ee69f9cd46 100644 --- a/include/trace/kmemtrace.h +++ b/include/trace/kmemtrace.h | |||
@@ -9,65 +9,53 @@ | |||
9 | 9 | ||
10 | #ifdef __KERNEL__ | 10 | #ifdef __KERNEL__ |
11 | 11 | ||
12 | #include <linux/tracepoint.h> | ||
12 | #include <linux/types.h> | 13 | #include <linux/types.h> |
13 | #include <linux/marker.h> | ||
14 | |||
15 | enum kmemtrace_type_id { | ||
16 | KMEMTRACE_TYPE_KMALLOC = 0, /* kmalloc() or kfree(). */ | ||
17 | KMEMTRACE_TYPE_CACHE, /* kmem_cache_*(). */ | ||
18 | KMEMTRACE_TYPE_PAGES, /* __get_free_pages() and friends. */ | ||
19 | }; | ||
20 | 14 | ||
21 | #ifdef CONFIG_KMEMTRACE | 15 | #ifdef CONFIG_KMEMTRACE |
22 | |||
23 | extern void kmemtrace_init(void); | 16 | extern void kmemtrace_init(void); |
24 | 17 | #else | |
25 | extern void kmemtrace_mark_alloc_node(enum kmemtrace_type_id type_id, | ||
26 | unsigned long call_site, | ||
27 | const void *ptr, | ||
28 | size_t bytes_req, | ||
29 | size_t bytes_alloc, | ||
30 | gfp_t gfp_flags, | ||
31 | int node); | ||
32 | |||
33 | extern void kmemtrace_mark_free(enum kmemtrace_type_id type_id, | ||
34 | unsigned long call_site, | ||
35 | const void *ptr); | ||
36 | |||
37 | #else /* CONFIG_KMEMTRACE */ | ||
38 | |||
39 | static inline void kmemtrace_init(void) | 18 | static inline void kmemtrace_init(void) |
40 | { | 19 | { |
41 | } | 20 | } |
42 | 21 | #endif | |
43 | static inline void kmemtrace_mark_alloc_node(enum kmemtrace_type_id type_id, | 22 | |
44 | unsigned long call_site, | 23 | DECLARE_TRACE(kmalloc, |
45 | const void *ptr, | 24 | TP_PROTO(unsigned long call_site, |
46 | size_t bytes_req, | 25 | const void *ptr, |
47 | size_t bytes_alloc, | 26 | size_t bytes_req, |
48 | gfp_t gfp_flags, | 27 | size_t bytes_alloc, |
49 | int node) | 28 | gfp_t gfp_flags), |
50 | { | 29 | TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags)); |
51 | } | 30 | DECLARE_TRACE(kmem_cache_alloc, |
52 | 31 | TP_PROTO(unsigned long call_site, | |
53 | static inline void kmemtrace_mark_free(enum kmemtrace_type_id type_id, | 32 | const void *ptr, |
54 | unsigned long call_site, | 33 | size_t bytes_req, |
55 | const void *ptr) | 34 | size_t bytes_alloc, |
56 | { | 35 | gfp_t gfp_flags), |
57 | } | 36 | TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags)); |
58 | 37 | DECLARE_TRACE(kmalloc_node, | |
59 | #endif /* CONFIG_KMEMTRACE */ | 38 | TP_PROTO(unsigned long call_site, |
60 | 39 | const void *ptr, | |
61 | static inline void kmemtrace_mark_alloc(enum kmemtrace_type_id type_id, | 40 | size_t bytes_req, |
62 | unsigned long call_site, | 41 | size_t bytes_alloc, |
63 | const void *ptr, | 42 | gfp_t gfp_flags, |
64 | size_t bytes_req, | 43 | int node), |
65 | size_t bytes_alloc, | 44 | TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node)); |
66 | gfp_t gfp_flags) | 45 | DECLARE_TRACE(kmem_cache_alloc_node, |
67 | { | 46 | TP_PROTO(unsigned long call_site, |
68 | kmemtrace_mark_alloc_node(type_id, call_site, ptr, | 47 | const void *ptr, |
69 | bytes_req, bytes_alloc, gfp_flags, -1); | 48 | size_t bytes_req, |
70 | } | 49 | size_t bytes_alloc, |
50 | gfp_t gfp_flags, | ||
51 | int node), | ||
52 | TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node)); | ||
53 | DECLARE_TRACE(kfree, | ||
54 | TP_PROTO(unsigned long call_site, const void *ptr), | ||
55 | TP_ARGS(call_site, ptr)); | ||
56 | DECLARE_TRACE(kmem_cache_free, | ||
57 | TP_PROTO(unsigned long call_site, const void *ptr), | ||
58 | TP_ARGS(call_site, ptr)); | ||
71 | 59 | ||
72 | #endif /* __KERNEL__ */ | 60 | #endif /* __KERNEL__ */ |
73 | 61 | ||
diff --git a/kernel/trace/kmemtrace.c b/kernel/trace/kmemtrace.c index ae201b3eda89..4f7b5db5dd06 100644 --- a/kernel/trace/kmemtrace.c +++ b/kernel/trace/kmemtrace.c | |||
@@ -10,6 +10,7 @@ | |||
10 | #include <linux/debugfs.h> | 10 | #include <linux/debugfs.h> |
11 | #include <linux/fs.h> | 11 | #include <linux/fs.h> |
12 | #include <linux/seq_file.h> | 12 | #include <linux/seq_file.h> |
13 | #include <linux/tracepoint.h> | ||
13 | #include <trace/kmemtrace.h> | 14 | #include <trace/kmemtrace.h> |
14 | 15 | ||
15 | #include "trace.h" | 16 | #include "trace.h" |
@@ -29,10 +30,150 @@ static struct tracer_flags kmem_tracer_flags = { | |||
29 | .opts = kmem_opts | 30 | .opts = kmem_opts |
30 | }; | 31 | }; |
31 | 32 | ||
32 | |||
33 | static bool kmem_tracing_enabled __read_mostly; | ||
34 | static struct trace_array *kmemtrace_array; | 33 | static struct trace_array *kmemtrace_array; |
35 | 34 | ||
35 | /* Trace allocations */ | ||
36 | static inline void kmemtrace_alloc(enum kmemtrace_type_id type_id, | ||
37 | unsigned long call_site, | ||
38 | const void *ptr, | ||
39 | size_t bytes_req, | ||
40 | size_t bytes_alloc, | ||
41 | gfp_t gfp_flags, | ||
42 | int node) | ||
43 | { | ||
44 | struct ring_buffer_event *event; | ||
45 | struct kmemtrace_alloc_entry *entry; | ||
46 | struct trace_array *tr = kmemtrace_array; | ||
47 | |||
48 | event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry)); | ||
49 | if (!event) | ||
50 | return; | ||
51 | entry = ring_buffer_event_data(event); | ||
52 | tracing_generic_entry_update(&entry->ent, 0, 0); | ||
53 | |||
54 | entry->ent.type = TRACE_KMEM_ALLOC; | ||
55 | entry->call_site = call_site; | ||
56 | entry->ptr = ptr; | ||
57 | entry->bytes_req = bytes_req; | ||
58 | entry->bytes_alloc = bytes_alloc; | ||
59 | entry->gfp_flags = gfp_flags; | ||
60 | entry->node = node; | ||
61 | |||
62 | ring_buffer_unlock_commit(tr->buffer, event); | ||
63 | |||
64 | trace_wake_up(); | ||
65 | } | ||
66 | |||
67 | static inline void kmemtrace_free(enum kmemtrace_type_id type_id, | ||
68 | unsigned long call_site, | ||
69 | const void *ptr) | ||
70 | { | ||
71 | struct ring_buffer_event *event; | ||
72 | struct kmemtrace_free_entry *entry; | ||
73 | struct trace_array *tr = kmemtrace_array; | ||
74 | |||
75 | event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry)); | ||
76 | if (!event) | ||
77 | return; | ||
78 | entry = ring_buffer_event_data(event); | ||
79 | tracing_generic_entry_update(&entry->ent, 0, 0); | ||
80 | |||
81 | entry->ent.type = TRACE_KMEM_FREE; | ||
82 | entry->type_id = type_id; | ||
83 | entry->call_site = call_site; | ||
84 | entry->ptr = ptr; | ||
85 | |||
86 | ring_buffer_unlock_commit(tr->buffer, event); | ||
87 | |||
88 | trace_wake_up(); | ||
89 | } | ||
90 | |||
91 | static void kmemtrace_kmalloc(unsigned long call_site, | ||
92 | const void *ptr, | ||
93 | size_t bytes_req, | ||
94 | size_t bytes_alloc, | ||
95 | gfp_t gfp_flags) | ||
96 | { | ||
97 | kmemtrace_alloc(KMEMTRACE_TYPE_KMALLOC, call_site, ptr, | ||
98 | bytes_req, bytes_alloc, gfp_flags, -1); | ||
99 | } | ||
100 | |||
101 | static void kmemtrace_kmem_cache_alloc(unsigned long call_site, | ||
102 | const void *ptr, | ||
103 | size_t bytes_req, | ||
104 | size_t bytes_alloc, | ||
105 | gfp_t gfp_flags) | ||
106 | { | ||
107 | kmemtrace_alloc(KMEMTRACE_TYPE_CACHE, call_site, ptr, | ||
108 | bytes_req, bytes_alloc, gfp_flags, -1); | ||
109 | } | ||
110 | |||
111 | static void kmemtrace_kmalloc_node(unsigned long call_site, | ||
112 | const void *ptr, | ||
113 | size_t bytes_req, | ||
114 | size_t bytes_alloc, | ||
115 | gfp_t gfp_flags, | ||
116 | int node) | ||
117 | { | ||
118 | kmemtrace_alloc(KMEMTRACE_TYPE_KMALLOC, call_site, ptr, | ||
119 | bytes_req, bytes_alloc, gfp_flags, node); | ||
120 | } | ||
121 | |||
122 | static void kmemtrace_kmem_cache_alloc_node(unsigned long call_site, | ||
123 | const void *ptr, | ||
124 | size_t bytes_req, | ||
125 | size_t bytes_alloc, | ||
126 | gfp_t gfp_flags, | ||
127 | int node) | ||
128 | { | ||
129 | kmemtrace_alloc(KMEMTRACE_TYPE_CACHE, call_site, ptr, | ||
130 | bytes_req, bytes_alloc, gfp_flags, node); | ||
131 | } | ||
132 | |||
133 | static void kmemtrace_kfree(unsigned long call_site, const void *ptr) | ||
134 | { | ||
135 | kmemtrace_free(KMEMTRACE_TYPE_KMALLOC, call_site, ptr); | ||
136 | } | ||
137 | |||
138 | static void kmemtrace_kmem_cache_free(unsigned long call_site, const void *ptr) | ||
139 | { | ||
140 | kmemtrace_free(KMEMTRACE_TYPE_CACHE, call_site, ptr); | ||
141 | } | ||
142 | |||
143 | static int kmemtrace_start_probes(void) | ||
144 | { | ||
145 | int err; | ||
146 | |||
147 | err = register_trace_kmalloc(kmemtrace_kmalloc); | ||
148 | if (err) | ||
149 | return err; | ||
150 | err = register_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc); | ||
151 | if (err) | ||
152 | return err; | ||
153 | err = register_trace_kmalloc_node(kmemtrace_kmalloc_node); | ||
154 | if (err) | ||
155 | return err; | ||
156 | err = register_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node); | ||
157 | if (err) | ||
158 | return err; | ||
159 | err = register_trace_kfree(kmemtrace_kfree); | ||
160 | if (err) | ||
161 | return err; | ||
162 | err = register_trace_kmem_cache_free(kmemtrace_kmem_cache_free); | ||
163 | |||
164 | return err; | ||
165 | } | ||
166 | |||
167 | static void kmemtrace_stop_probes(void) | ||
168 | { | ||
169 | unregister_trace_kmalloc(kmemtrace_kmalloc); | ||
170 | unregister_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc); | ||
171 | unregister_trace_kmalloc_node(kmemtrace_kmalloc_node); | ||
172 | unregister_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node); | ||
173 | unregister_trace_kfree(kmemtrace_kfree); | ||
174 | unregister_trace_kmem_cache_free(kmemtrace_kmem_cache_free); | ||
175 | } | ||
176 | |||
36 | static int kmem_trace_init(struct trace_array *tr) | 177 | static int kmem_trace_init(struct trace_array *tr) |
37 | { | 178 | { |
38 | int cpu; | 179 | int cpu; |
@@ -41,14 +182,14 @@ static int kmem_trace_init(struct trace_array *tr) | |||
41 | for_each_cpu_mask(cpu, cpu_possible_map) | 182 | for_each_cpu_mask(cpu, cpu_possible_map) |
42 | tracing_reset(tr, cpu); | 183 | tracing_reset(tr, cpu); |
43 | 184 | ||
44 | kmem_tracing_enabled = true; | 185 | kmemtrace_start_probes(); |
45 | 186 | ||
46 | return 0; | 187 | return 0; |
47 | } | 188 | } |
48 | 189 | ||
49 | static void kmem_trace_reset(struct trace_array *tr) | 190 | static void kmem_trace_reset(struct trace_array *tr) |
50 | { | 191 | { |
51 | kmem_tracing_enabled = false; | 192 | kmemtrace_stop_probes(); |
52 | } | 193 | } |
53 | 194 | ||
54 | static void kmemtrace_headers(struct seq_file *s) | 195 | static void kmemtrace_headers(struct seq_file *s) |
@@ -260,63 +401,6 @@ static enum print_line_t kmemtrace_print_line(struct trace_iterator *iter) | |||
260 | } | 401 | } |
261 | } | 402 | } |
262 | 403 | ||
263 | /* Trace allocations */ | ||
264 | void kmemtrace_mark_alloc_node(enum kmemtrace_type_id type_id, | ||
265 | unsigned long call_site, | ||
266 | const void *ptr, | ||
267 | size_t bytes_req, | ||
268 | size_t bytes_alloc, | ||
269 | gfp_t gfp_flags, | ||
270 | int node) | ||
271 | { | ||
272 | struct ring_buffer_event *event; | ||
273 | struct kmemtrace_alloc_entry *entry; | ||
274 | struct trace_array *tr = kmemtrace_array; | ||
275 | |||
276 | if (!kmem_tracing_enabled) | ||
277 | return; | ||
278 | |||
279 | event = trace_buffer_lock_reserve(tr, TRACE_KMEM_ALLOC, | ||
280 | sizeof(*entry), 0, 0); | ||
281 | if (!event) | ||
282 | return; | ||
283 | entry = ring_buffer_event_data(event); | ||
284 | |||
285 | entry->call_site = call_site; | ||
286 | entry->ptr = ptr; | ||
287 | entry->bytes_req = bytes_req; | ||
288 | entry->bytes_alloc = bytes_alloc; | ||
289 | entry->gfp_flags = gfp_flags; | ||
290 | entry->node = node; | ||
291 | |||
292 | trace_buffer_unlock_commit(tr, event, 0, 0); | ||
293 | } | ||
294 | EXPORT_SYMBOL(kmemtrace_mark_alloc_node); | ||
295 | |||
296 | void kmemtrace_mark_free(enum kmemtrace_type_id type_id, | ||
297 | unsigned long call_site, | ||
298 | const void *ptr) | ||
299 | { | ||
300 | struct ring_buffer_event *event; | ||
301 | struct kmemtrace_free_entry *entry; | ||
302 | struct trace_array *tr = kmemtrace_array; | ||
303 | |||
304 | if (!kmem_tracing_enabled) | ||
305 | return; | ||
306 | |||
307 | event = trace_buffer_lock_reserve(tr, TRACE_KMEM_FREE, | ||
308 | sizeof(*entry), 0, 0); | ||
309 | if (!event) | ||
310 | return; | ||
311 | entry = ring_buffer_event_data(event); | ||
312 | entry->type_id = type_id; | ||
313 | entry->call_site = call_site; | ||
314 | entry->ptr = ptr; | ||
315 | |||
316 | trace_buffer_unlock_commit(tr, event, 0, 0); | ||
317 | } | ||
318 | EXPORT_SYMBOL(kmemtrace_mark_free); | ||
319 | |||
320 | static struct tracer kmem_tracer __read_mostly = { | 404 | static struct tracer kmem_tracer __read_mostly = { |
321 | .name = "kmemtrace", | 405 | .name = "kmemtrace", |
322 | .init = kmem_trace_init, | 406 | .init = kmem_trace_init, |
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index cb0ce3fc36d3..cbc168f1e43d 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h | |||
@@ -182,6 +182,12 @@ struct trace_power { | |||
182 | struct power_trace state_data; | 182 | struct power_trace state_data; |
183 | }; | 183 | }; |
184 | 184 | ||
185 | enum kmemtrace_type_id { | ||
186 | KMEMTRACE_TYPE_KMALLOC = 0, /* kmalloc() or kfree(). */ | ||
187 | KMEMTRACE_TYPE_CACHE, /* kmem_cache_*(). */ | ||
188 | KMEMTRACE_TYPE_PAGES, /* __get_free_pages() and friends. */ | ||
189 | }; | ||
190 | |||
185 | struct kmemtrace_alloc_entry { | 191 | struct kmemtrace_alloc_entry { |
186 | struct trace_entry ent; | 192 | struct trace_entry ent; |
187 | enum kmemtrace_type_id type_id; | 193 | enum kmemtrace_type_id type_id; |
@@ -3565,8 +3565,8 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) | |||
3565 | { | 3565 | { |
3566 | void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0)); | 3566 | void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0)); |
3567 | 3567 | ||
3568 | kmemtrace_mark_alloc(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret, | 3568 | trace_kmem_cache_alloc(_RET_IP_, ret, |
3569 | obj_size(cachep), cachep->buffer_size, flags); | 3569 | obj_size(cachep), cachep->buffer_size, flags); |
3570 | 3570 | ||
3571 | return ret; | 3571 | return ret; |
3572 | } | 3572 | } |
@@ -3627,9 +3627,9 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) | |||
3627 | void *ret = __cache_alloc_node(cachep, flags, nodeid, | 3627 | void *ret = __cache_alloc_node(cachep, flags, nodeid, |
3628 | __builtin_return_address(0)); | 3628 | __builtin_return_address(0)); |
3629 | 3629 | ||
3630 | kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret, | 3630 | trace_kmem_cache_alloc_node(_RET_IP_, ret, |
3631 | obj_size(cachep), cachep->buffer_size, | 3631 | obj_size(cachep), cachep->buffer_size, |
3632 | flags, nodeid); | 3632 | flags, nodeid); |
3633 | 3633 | ||
3634 | return ret; | 3634 | return ret; |
3635 | } | 3635 | } |
@@ -3657,9 +3657,8 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, void *caller) | |||
3657 | return cachep; | 3657 | return cachep; |
3658 | ret = kmem_cache_alloc_node_notrace(cachep, flags, node); | 3658 | ret = kmem_cache_alloc_node_notrace(cachep, flags, node); |
3659 | 3659 | ||
3660 | kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, | 3660 | trace_kmalloc_node((unsigned long) caller, ret, |
3661 | (unsigned long) caller, ret, | 3661 | size, cachep->buffer_size, flags, node); |
3662 | size, cachep->buffer_size, flags, node); | ||
3663 | 3662 | ||
3664 | return ret; | 3663 | return ret; |
3665 | } | 3664 | } |
@@ -3709,9 +3708,8 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, | |||
3709 | return cachep; | 3708 | return cachep; |
3710 | ret = __cache_alloc(cachep, flags, caller); | 3709 | ret = __cache_alloc(cachep, flags, caller); |
3711 | 3710 | ||
3712 | kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, | 3711 | trace_kmalloc((unsigned long) caller, ret, |
3713 | (unsigned long) caller, ret, | 3712 | size, cachep->buffer_size, flags); |
3714 | size, cachep->buffer_size, flags); | ||
3715 | 3713 | ||
3716 | return ret; | 3714 | return ret; |
3717 | } | 3715 | } |
@@ -3757,7 +3755,7 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) | |||
3757 | __cache_free(cachep, objp); | 3755 | __cache_free(cachep, objp); |
3758 | local_irq_restore(flags); | 3756 | local_irq_restore(flags); |
3759 | 3757 | ||
3760 | kmemtrace_mark_free(KMEMTRACE_TYPE_CACHE, _RET_IP_, objp); | 3758 | trace_kmem_cache_free(_RET_IP_, objp); |
3761 | } | 3759 | } |
3762 | EXPORT_SYMBOL(kmem_cache_free); | 3760 | EXPORT_SYMBOL(kmem_cache_free); |
3763 | 3761 | ||
@@ -3785,7 +3783,7 @@ void kfree(const void *objp) | |||
3785 | __cache_free(c, (void *)objp); | 3783 | __cache_free(c, (void *)objp); |
3786 | local_irq_restore(flags); | 3784 | local_irq_restore(flags); |
3787 | 3785 | ||
3788 | kmemtrace_mark_free(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, objp); | 3786 | trace_kfree(_RET_IP_, objp); |
3789 | } | 3787 | } |
3790 | EXPORT_SYMBOL(kfree); | 3788 | EXPORT_SYMBOL(kfree); |
3791 | 3789 | ||
@@ -490,9 +490,8 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node) | |||
490 | *m = size; | 490 | *m = size; |
491 | ret = (void *)m + align; | 491 | ret = (void *)m + align; |
492 | 492 | ||
493 | kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, | 493 | trace_kmalloc_node(_RET_IP_, ret, |
494 | _RET_IP_, ret, | 494 | size, size + align, gfp, node); |
495 | size, size + align, gfp, node); | ||
496 | } else { | 495 | } else { |
497 | unsigned int order = get_order(size); | 496 | unsigned int order = get_order(size); |
498 | 497 | ||
@@ -503,9 +502,8 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node) | |||
503 | page->private = size; | 502 | page->private = size; |
504 | } | 503 | } |
505 | 504 | ||
506 | kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, | 505 | trace_kmalloc_node(_RET_IP_, ret, |
507 | _RET_IP_, ret, | 506 | size, PAGE_SIZE << order, gfp, node); |
508 | size, PAGE_SIZE << order, gfp, node); | ||
509 | } | 507 | } |
510 | 508 | ||
511 | return ret; | 509 | return ret; |
@@ -527,7 +525,7 @@ void kfree(const void *block) | |||
527 | } else | 525 | } else |
528 | put_page(&sp->page); | 526 | put_page(&sp->page); |
529 | 527 | ||
530 | kmemtrace_mark_free(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, block); | 528 | trace_kfree(_RET_IP_, block); |
531 | } | 529 | } |
532 | EXPORT_SYMBOL(kfree); | 530 | EXPORT_SYMBOL(kfree); |
533 | 531 | ||
@@ -599,16 +597,14 @@ void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node) | |||
599 | 597 | ||
600 | if (c->size < PAGE_SIZE) { | 598 | if (c->size < PAGE_SIZE) { |
601 | b = slob_alloc(c->size, flags, c->align, node); | 599 | b = slob_alloc(c->size, flags, c->align, node); |
602 | kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE, | 600 | trace_kmem_cache_alloc_node(_RET_IP_, b, c->size, |
603 | _RET_IP_, b, c->size, | 601 | SLOB_UNITS(c->size) * SLOB_UNIT, |
604 | SLOB_UNITS(c->size) * SLOB_UNIT, | 602 | flags, node); |
605 | flags, node); | ||
606 | } else { | 603 | } else { |
607 | b = slob_new_pages(flags, get_order(c->size), node); | 604 | b = slob_new_pages(flags, get_order(c->size), node); |
608 | kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE, | 605 | trace_kmem_cache_alloc_node(_RET_IP_, b, c->size, |
609 | _RET_IP_, b, c->size, | 606 | PAGE_SIZE << get_order(c->size), |
610 | PAGE_SIZE << get_order(c->size), | 607 | flags, node); |
611 | flags, node); | ||
612 | } | 608 | } |
613 | 609 | ||
614 | if (c->ctor) | 610 | if (c->ctor) |
@@ -646,7 +642,7 @@ void kmem_cache_free(struct kmem_cache *c, void *b) | |||
646 | __kmem_cache_free(b, c->size); | 642 | __kmem_cache_free(b, c->size); |
647 | } | 643 | } |
648 | 644 | ||
649 | kmemtrace_mark_free(KMEMTRACE_TYPE_CACHE, _RET_IP_, b); | 645 | trace_kmem_cache_free(_RET_IP_, b); |
650 | } | 646 | } |
651 | EXPORT_SYMBOL(kmem_cache_free); | 647 | EXPORT_SYMBOL(kmem_cache_free); |
652 | 648 | ||
@@ -1621,8 +1621,7 @@ void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) | |||
1621 | { | 1621 | { |
1622 | void *ret = slab_alloc(s, gfpflags, -1, _RET_IP_); | 1622 | void *ret = slab_alloc(s, gfpflags, -1, _RET_IP_); |
1623 | 1623 | ||
1624 | kmemtrace_mark_alloc(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret, | 1624 | trace_kmem_cache_alloc(_RET_IP_, ret, s->objsize, s->size, gfpflags); |
1625 | s->objsize, s->size, gfpflags); | ||
1626 | 1625 | ||
1627 | return ret; | 1626 | return ret; |
1628 | } | 1627 | } |
@@ -1641,8 +1640,8 @@ void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) | |||
1641 | { | 1640 | { |
1642 | void *ret = slab_alloc(s, gfpflags, node, _RET_IP_); | 1641 | void *ret = slab_alloc(s, gfpflags, node, _RET_IP_); |
1643 | 1642 | ||
1644 | kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret, | 1643 | trace_kmem_cache_alloc_node(_RET_IP_, ret, |
1645 | s->objsize, s->size, gfpflags, node); | 1644 | s->objsize, s->size, gfpflags, node); |
1646 | 1645 | ||
1647 | return ret; | 1646 | return ret; |
1648 | } | 1647 | } |
@@ -1767,7 +1766,7 @@ void kmem_cache_free(struct kmem_cache *s, void *x) | |||
1767 | 1766 | ||
1768 | slab_free(s, page, x, _RET_IP_); | 1767 | slab_free(s, page, x, _RET_IP_); |
1769 | 1768 | ||
1770 | kmemtrace_mark_free(KMEMTRACE_TYPE_CACHE, _RET_IP_, x); | 1769 | trace_kmem_cache_free(_RET_IP_, x); |
1771 | } | 1770 | } |
1772 | EXPORT_SYMBOL(kmem_cache_free); | 1771 | EXPORT_SYMBOL(kmem_cache_free); |
1773 | 1772 | ||
@@ -2702,8 +2701,7 @@ void *__kmalloc(size_t size, gfp_t flags) | |||
2702 | 2701 | ||
2703 | ret = slab_alloc(s, flags, -1, _RET_IP_); | 2702 | ret = slab_alloc(s, flags, -1, _RET_IP_); |
2704 | 2703 | ||
2705 | kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, ret, | 2704 | trace_kmalloc(_RET_IP_, ret, size, s->size, flags); |
2706 | size, s->size, flags); | ||
2707 | 2705 | ||
2708 | return ret; | 2706 | return ret; |
2709 | } | 2707 | } |
@@ -2729,10 +2727,9 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node) | |||
2729 | if (unlikely(size > SLUB_MAX_SIZE)) { | 2727 | if (unlikely(size > SLUB_MAX_SIZE)) { |
2730 | ret = kmalloc_large_node(size, flags, node); | 2728 | ret = kmalloc_large_node(size, flags, node); |
2731 | 2729 | ||
2732 | kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, | 2730 | trace_kmalloc_node(_RET_IP_, ret, |
2733 | _RET_IP_, ret, | 2731 | size, PAGE_SIZE << get_order(size), |
2734 | size, PAGE_SIZE << get_order(size), | 2732 | flags, node); |
2735 | flags, node); | ||
2736 | 2733 | ||
2737 | return ret; | 2734 | return ret; |
2738 | } | 2735 | } |
@@ -2744,8 +2741,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node) | |||
2744 | 2741 | ||
2745 | ret = slab_alloc(s, flags, node, _RET_IP_); | 2742 | ret = slab_alloc(s, flags, node, _RET_IP_); |
2746 | 2743 | ||
2747 | kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, ret, | 2744 | trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node); |
2748 | size, s->size, flags, node); | ||
2749 | 2745 | ||
2750 | return ret; | 2746 | return ret; |
2751 | } | 2747 | } |
@@ -2807,7 +2803,7 @@ void kfree(const void *x) | |||
2807 | } | 2803 | } |
2808 | slab_free(page->slab, page, object, _RET_IP_); | 2804 | slab_free(page->slab, page, object, _RET_IP_); |
2809 | 2805 | ||
2810 | kmemtrace_mark_free(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, x); | 2806 | trace_kfree(_RET_IP_, x); |
2811 | } | 2807 | } |
2812 | EXPORT_SYMBOL(kfree); | 2808 | EXPORT_SYMBOL(kfree); |
2813 | 2809 | ||
@@ -3290,8 +3286,7 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller) | |||
3290 | ret = slab_alloc(s, gfpflags, -1, caller); | 3286 | ret = slab_alloc(s, gfpflags, -1, caller); |
3291 | 3287 | ||
3292 | /* Honor the call site pointer we recieved. */ | 3288 | /* Honor the call site pointer we recieved. */ |
3293 | kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, caller, ret, size, | 3289 | trace_kmalloc(caller, ret, size, s->size, gfpflags); |
3294 | s->size, gfpflags); | ||
3295 | 3290 | ||
3296 | return ret; | 3291 | return ret; |
3297 | } | 3292 | } |
@@ -3313,8 +3308,7 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, | |||
3313 | ret = slab_alloc(s, gfpflags, node, caller); | 3308 | ret = slab_alloc(s, gfpflags, node, caller); |
3314 | 3309 | ||
3315 | /* Honor the call site pointer we recieved. */ | 3310 | /* Honor the call site pointer we recieved. */ |
3316 | kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, caller, ret, | 3311 | trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node); |
3317 | size, s->size, gfpflags, node); | ||
3318 | 3312 | ||
3319 | return ret; | 3313 | return ret; |
3320 | } | 3314 | } |
@@ -4,6 +4,7 @@ | |||
4 | #include <linux/module.h> | 4 | #include <linux/module.h> |
5 | #include <linux/err.h> | 5 | #include <linux/err.h> |
6 | #include <linux/sched.h> | 6 | #include <linux/sched.h> |
7 | #include <linux/tracepoint.h> | ||
7 | #include <asm/uaccess.h> | 8 | #include <asm/uaccess.h> |
8 | 9 | ||
9 | /** | 10 | /** |
@@ -236,3 +237,18 @@ int __attribute__((weak)) get_user_pages_fast(unsigned long start, | |||
236 | return ret; | 237 | return ret; |
237 | } | 238 | } |
238 | EXPORT_SYMBOL_GPL(get_user_pages_fast); | 239 | EXPORT_SYMBOL_GPL(get_user_pages_fast); |
240 | |||
241 | /* Tracepoints definitions. */ | ||
242 | DEFINE_TRACE(kmalloc); | ||
243 | DEFINE_TRACE(kmem_cache_alloc); | ||
244 | DEFINE_TRACE(kmalloc_node); | ||
245 | DEFINE_TRACE(kmem_cache_alloc_node); | ||
246 | DEFINE_TRACE(kfree); | ||
247 | DEFINE_TRACE(kmem_cache_free); | ||
248 | |||
249 | EXPORT_TRACEPOINT_SYMBOL(kmalloc); | ||
250 | EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc); | ||
251 | EXPORT_TRACEPOINT_SYMBOL(kmalloc_node); | ||
252 | EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node); | ||
253 | EXPORT_TRACEPOINT_SYMBOL(kfree); | ||
254 | EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); | ||