diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/perf_event.c | 389 | ||||
| -rw-r--r-- | kernel/trace/blktrace.c | 138 | ||||
| -rw-r--r-- | kernel/trace/ftrace.c | 7 | ||||
| -rw-r--r-- | kernel/trace/kmemtrace.c | 70 | ||||
| -rw-r--r-- | kernel/trace/trace.c | 9 | ||||
| -rw-r--r-- | kernel/trace/trace.h | 9 | ||||
| -rw-r--r-- | kernel/trace/trace_branch.c | 8 | ||||
| -rw-r--r-- | kernel/trace/trace_event_perf.c | 185 | ||||
| -rw-r--r-- | kernel/trace/trace_events.c | 139 | ||||
| -rw-r--r-- | kernel/trace/trace_events_filter.c | 28 | ||||
| -rw-r--r-- | kernel/trace/trace_export.c | 16 | ||||
| -rw-r--r-- | kernel/trace/trace_functions_graph.c | 13 | ||||
| -rw-r--r-- | kernel/trace/trace_kprobe.c | 113 | ||||
| -rw-r--r-- | kernel/trace/trace_output.c | 137 | ||||
| -rw-r--r-- | kernel/trace/trace_output.h | 2 | ||||
| -rw-r--r-- | kernel/trace/trace_sched_switch.c | 20 | ||||
| -rw-r--r-- | kernel/trace/trace_sched_wakeup.c | 28 | ||||
| -rw-r--r-- | kernel/trace/trace_syscalls.c | 146 | ||||
| -rw-r--r-- | kernel/trace/trace_workqueue.c | 26 | ||||
| -rw-r--r-- | kernel/tracepoint.c | 91 |
20 files changed, 951 insertions, 623 deletions
diff --git a/kernel/perf_event.c b/kernel/perf_event.c index a4fa381db3c2..e099650cd249 100644 --- a/kernel/perf_event.c +++ b/kernel/perf_event.c | |||
| @@ -2297,11 +2297,6 @@ unlock: | |||
| 2297 | rcu_read_unlock(); | 2297 | rcu_read_unlock(); |
| 2298 | } | 2298 | } |
| 2299 | 2299 | ||
| 2300 | static unsigned long perf_data_size(struct perf_mmap_data *data) | ||
| 2301 | { | ||
| 2302 | return data->nr_pages << (PAGE_SHIFT + data->data_order); | ||
| 2303 | } | ||
| 2304 | |||
| 2305 | #ifndef CONFIG_PERF_USE_VMALLOC | 2300 | #ifndef CONFIG_PERF_USE_VMALLOC |
| 2306 | 2301 | ||
| 2307 | /* | 2302 | /* |
| @@ -2320,6 +2315,19 @@ perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff) | |||
| 2320 | return virt_to_page(data->data_pages[pgoff - 1]); | 2315 | return virt_to_page(data->data_pages[pgoff - 1]); |
| 2321 | } | 2316 | } |
| 2322 | 2317 | ||
| 2318 | static void *perf_mmap_alloc_page(int cpu) | ||
| 2319 | { | ||
| 2320 | struct page *page; | ||
| 2321 | int node; | ||
| 2322 | |||
| 2323 | node = (cpu == -1) ? cpu : cpu_to_node(cpu); | ||
| 2324 | page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); | ||
| 2325 | if (!page) | ||
| 2326 | return NULL; | ||
| 2327 | |||
| 2328 | return page_address(page); | ||
| 2329 | } | ||
| 2330 | |||
| 2323 | static struct perf_mmap_data * | 2331 | static struct perf_mmap_data * |
| 2324 | perf_mmap_data_alloc(struct perf_event *event, int nr_pages) | 2332 | perf_mmap_data_alloc(struct perf_event *event, int nr_pages) |
| 2325 | { | 2333 | { |
| @@ -2336,17 +2344,16 @@ perf_mmap_data_alloc(struct perf_event *event, int nr_pages) | |||
| 2336 | if (!data) | 2344 | if (!data) |
| 2337 | goto fail; | 2345 | goto fail; |
| 2338 | 2346 | ||
| 2339 | data->user_page = (void *)get_zeroed_page(GFP_KERNEL); | 2347 | data->user_page = perf_mmap_alloc_page(event->cpu); |
| 2340 | if (!data->user_page) | 2348 | if (!data->user_page) |
| 2341 | goto fail_user_page; | 2349 | goto fail_user_page; |
| 2342 | 2350 | ||
| 2343 | for (i = 0; i < nr_pages; i++) { | 2351 | for (i = 0; i < nr_pages; i++) { |
| 2344 | data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL); | 2352 | data->data_pages[i] = perf_mmap_alloc_page(event->cpu); |
| 2345 | if (!data->data_pages[i]) | 2353 | if (!data->data_pages[i]) |
| 2346 | goto fail_data_pages; | 2354 | goto fail_data_pages; |
| 2347 | } | 2355 | } |
| 2348 | 2356 | ||
| 2349 | data->data_order = 0; | ||
| 2350 | data->nr_pages = nr_pages; | 2357 | data->nr_pages = nr_pages; |
| 2351 | 2358 | ||
| 2352 | return data; | 2359 | return data; |
| @@ -2382,6 +2389,11 @@ static void perf_mmap_data_free(struct perf_mmap_data *data) | |||
| 2382 | kfree(data); | 2389 | kfree(data); |
| 2383 | } | 2390 | } |
| 2384 | 2391 | ||
| 2392 | static inline int page_order(struct perf_mmap_data *data) | ||
| 2393 | { | ||
| 2394 | return 0; | ||
| 2395 | } | ||
| 2396 | |||
| 2385 | #else | 2397 | #else |
| 2386 | 2398 | ||
| 2387 | /* | 2399 | /* |
| @@ -2390,10 +2402,15 @@ static void perf_mmap_data_free(struct perf_mmap_data *data) | |||
| 2390 | * Required for architectures that have d-cache aliasing issues. | 2402 | * Required for architectures that have d-cache aliasing issues. |
| 2391 | */ | 2403 | */ |
| 2392 | 2404 | ||
| 2405 | static inline int page_order(struct perf_mmap_data *data) | ||
| 2406 | { | ||
| 2407 | return data->page_order; | ||
| 2408 | } | ||
| 2409 | |||
| 2393 | static struct page * | 2410 | static struct page * |
| 2394 | perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff) | 2411 | perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff) |
| 2395 | { | 2412 | { |
| 2396 | if (pgoff > (1UL << data->data_order)) | 2413 | if (pgoff > (1UL << page_order(data))) |
| 2397 | return NULL; | 2414 | return NULL; |
| 2398 | 2415 | ||
| 2399 | return vmalloc_to_page((void *)data->user_page + pgoff * PAGE_SIZE); | 2416 | return vmalloc_to_page((void *)data->user_page + pgoff * PAGE_SIZE); |
| @@ -2413,7 +2430,7 @@ static void perf_mmap_data_free_work(struct work_struct *work) | |||
| 2413 | int i, nr; | 2430 | int i, nr; |
| 2414 | 2431 | ||
| 2415 | data = container_of(work, struct perf_mmap_data, work); | 2432 | data = container_of(work, struct perf_mmap_data, work); |
| 2416 | nr = 1 << data->data_order; | 2433 | nr = 1 << page_order(data); |
| 2417 | 2434 | ||
| 2418 | base = data->user_page; | 2435 | base = data->user_page; |
| 2419 | for (i = 0; i < nr + 1; i++) | 2436 | for (i = 0; i < nr + 1; i++) |
| @@ -2452,7 +2469,7 @@ perf_mmap_data_alloc(struct perf_event *event, int nr_pages) | |||
| 2452 | 2469 | ||
| 2453 | data->user_page = all_buf; | 2470 | data->user_page = all_buf; |
| 2454 | data->data_pages[0] = all_buf + PAGE_SIZE; | 2471 | data->data_pages[0] = all_buf + PAGE_SIZE; |
| 2455 | data->data_order = ilog2(nr_pages); | 2472 | data->page_order = ilog2(nr_pages); |
| 2456 | data->nr_pages = 1; | 2473 | data->nr_pages = 1; |
| 2457 | 2474 | ||
| 2458 | return data; | 2475 | return data; |
| @@ -2466,6 +2483,11 @@ fail: | |||
| 2466 | 2483 | ||
| 2467 | #endif | 2484 | #endif |
| 2468 | 2485 | ||
| 2486 | static unsigned long perf_data_size(struct perf_mmap_data *data) | ||
| 2487 | { | ||
| 2488 | return data->nr_pages << (PAGE_SHIFT + page_order(data)); | ||
| 2489 | } | ||
| 2490 | |||
| 2469 | static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | 2491 | static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) |
| 2470 | { | 2492 | { |
| 2471 | struct perf_event *event = vma->vm_file->private_data; | 2493 | struct perf_event *event = vma->vm_file->private_data; |
| @@ -2506,8 +2528,6 @@ perf_mmap_data_init(struct perf_event *event, struct perf_mmap_data *data) | |||
| 2506 | { | 2528 | { |
| 2507 | long max_size = perf_data_size(data); | 2529 | long max_size = perf_data_size(data); |
| 2508 | 2530 | ||
| 2509 | atomic_set(&data->lock, -1); | ||
| 2510 | |||
| 2511 | if (event->attr.watermark) { | 2531 | if (event->attr.watermark) { |
| 2512 | data->watermark = min_t(long, max_size, | 2532 | data->watermark = min_t(long, max_size, |
| 2513 | event->attr.wakeup_watermark); | 2533 | event->attr.wakeup_watermark); |
| @@ -2580,6 +2600,14 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) | |||
| 2580 | long user_extra, extra; | 2600 | long user_extra, extra; |
| 2581 | int ret = 0; | 2601 | int ret = 0; |
| 2582 | 2602 | ||
| 2603 | /* | ||
| 2604 | * Don't allow mmap() of inherited per-task counters. This would | ||
| 2605 | * create a performance issue due to all children writing to the | ||
| 2606 | * same buffer. | ||
| 2607 | */ | ||
| 2608 | if (event->cpu == -1 && event->attr.inherit) | ||
| 2609 | return -EINVAL; | ||
| 2610 | |||
| 2583 | if (!(vma->vm_flags & VM_SHARED)) | 2611 | if (!(vma->vm_flags & VM_SHARED)) |
| 2584 | return -EINVAL; | 2612 | return -EINVAL; |
| 2585 | 2613 | ||
| @@ -2885,120 +2913,80 @@ static void perf_output_wakeup(struct perf_output_handle *handle) | |||
| 2885 | } | 2913 | } |
| 2886 | 2914 | ||
| 2887 | /* | 2915 | /* |
| 2888 | * Curious locking construct. | ||
| 2889 | * | ||
| 2890 | * We need to ensure a later event_id doesn't publish a head when a former | 2916 | * We need to ensure a later event_id doesn't publish a head when a former |
| 2891 | * event_id isn't done writing. However since we need to deal with NMIs we | 2917 | * event isn't done writing. However since we need to deal with NMIs we |
| 2892 | * cannot fully serialize things. | 2918 | * cannot fully serialize things. |
| 2893 | * | 2919 | * |
| 2894 | * What we do is serialize between CPUs so we only have to deal with NMI | ||
| 2895 | * nesting on a single CPU. | ||
| 2896 | * | ||
| 2897 | * We only publish the head (and generate a wakeup) when the outer-most | 2920 | * We only publish the head (and generate a wakeup) when the outer-most |
| 2898 | * event_id completes. | 2921 | * event completes. |
| 2899 | */ | 2922 | */ |
| 2900 | static void perf_output_lock(struct perf_output_handle *handle) | 2923 | static void perf_output_get_handle(struct perf_output_handle *handle) |
| 2901 | { | 2924 | { |
| 2902 | struct perf_mmap_data *data = handle->data; | 2925 | struct perf_mmap_data *data = handle->data; |
| 2903 | int cur, cpu = get_cpu(); | ||
| 2904 | |||
| 2905 | handle->locked = 0; | ||
| 2906 | |||
| 2907 | for (;;) { | ||
| 2908 | cur = atomic_cmpxchg(&data->lock, -1, cpu); | ||
| 2909 | if (cur == -1) { | ||
| 2910 | handle->locked = 1; | ||
| 2911 | break; | ||
| 2912 | } | ||
| 2913 | if (cur == cpu) | ||
| 2914 | break; | ||
| 2915 | 2926 | ||
| 2916 | cpu_relax(); | 2927 | preempt_disable(); |
| 2917 | } | 2928 | local_inc(&data->nest); |
| 2929 | handle->wakeup = local_read(&data->wakeup); | ||
| 2918 | } | 2930 | } |
| 2919 | 2931 | ||
| 2920 | static void perf_output_unlock(struct perf_output_handle *handle) | 2932 | static void perf_output_put_handle(struct perf_output_handle *handle) |
| 2921 | { | 2933 | { |
| 2922 | struct perf_mmap_data *data = handle->data; | 2934 | struct perf_mmap_data *data = handle->data; |
| 2923 | unsigned long head; | 2935 | unsigned long head; |
| 2924 | int cpu; | ||
| 2925 | |||
| 2926 | data->done_head = data->head; | ||
| 2927 | |||
| 2928 | if (!handle->locked) | ||
| 2929 | goto out; | ||
| 2930 | 2936 | ||
| 2931 | again: | 2937 | again: |
| 2932 | /* | 2938 | head = local_read(&data->head); |
| 2933 | * The xchg implies a full barrier that ensures all writes are done | ||
| 2934 | * before we publish the new head, matched by a rmb() in userspace when | ||
| 2935 | * reading this position. | ||
| 2936 | */ | ||
| 2937 | while ((head = atomic_long_xchg(&data->done_head, 0))) | ||
| 2938 | data->user_page->data_head = head; | ||
| 2939 | 2939 | ||
| 2940 | /* | 2940 | /* |
| 2941 | * NMI can happen here, which means we can miss a done_head update. | 2941 | * IRQ/NMI can happen here, which means we can miss a head update. |
| 2942 | */ | 2942 | */ |
| 2943 | 2943 | ||
| 2944 | cpu = atomic_xchg(&data->lock, -1); | 2944 | if (!local_dec_and_test(&data->nest)) |
| 2945 | WARN_ON_ONCE(cpu != smp_processor_id()); | 2945 | goto out; |
| 2946 | 2946 | ||
| 2947 | /* | 2947 | /* |
| 2948 | * Therefore we have to validate we did not indeed do so. | 2948 | * Publish the known good head. Rely on the full barrier implied |
| 2949 | * by atomic_dec_and_test() order the data->head read and this | ||
| 2950 | * write. | ||
| 2949 | */ | 2951 | */ |
| 2950 | if (unlikely(atomic_long_read(&data->done_head))) { | 2952 | data->user_page->data_head = head; |
| 2951 | /* | ||
| 2952 | * Since we had it locked, we can lock it again. | ||
| 2953 | */ | ||
| 2954 | while (atomic_cmpxchg(&data->lock, -1, cpu) != -1) | ||
| 2955 | cpu_relax(); | ||
| 2956 | 2953 | ||
| 2954 | /* | ||
| 2955 | * Now check if we missed an update, rely on the (compiler) | ||
| 2956 | * barrier in atomic_dec_and_test() to re-read data->head. | ||
| 2957 | */ | ||
| 2958 | if (unlikely(head != local_read(&data->head))) { | ||
| 2959 | local_inc(&data->nest); | ||
| 2957 | goto again; | 2960 | goto again; |
| 2958 | } | 2961 | } |
| 2959 | 2962 | ||
| 2960 | if (atomic_xchg(&data->wakeup, 0)) | 2963 | if (handle->wakeup != local_read(&data->wakeup)) |
| 2961 | perf_output_wakeup(handle); | 2964 | perf_output_wakeup(handle); |
| 2962 | out: | 2965 | |
| 2963 | put_cpu(); | 2966 | out: |
| 2967 | preempt_enable(); | ||
| 2964 | } | 2968 | } |
| 2965 | 2969 | ||
| 2966 | void perf_output_copy(struct perf_output_handle *handle, | 2970 | __always_inline void perf_output_copy(struct perf_output_handle *handle, |
| 2967 | const void *buf, unsigned int len) | 2971 | const void *buf, unsigned int len) |
| 2968 | { | 2972 | { |
| 2969 | unsigned int pages_mask; | ||
| 2970 | unsigned long offset; | ||
| 2971 | unsigned int size; | ||
| 2972 | void **pages; | ||
| 2973 | |||
| 2974 | offset = handle->offset; | ||
| 2975 | pages_mask = handle->data->nr_pages - 1; | ||
| 2976 | pages = handle->data->data_pages; | ||
| 2977 | |||
| 2978 | do { | 2973 | do { |
| 2979 | unsigned long page_offset; | 2974 | unsigned long size = min_t(unsigned long, handle->size, len); |
| 2980 | unsigned long page_size; | ||
| 2981 | int nr; | ||
| 2982 | 2975 | ||
| 2983 | nr = (offset >> PAGE_SHIFT) & pages_mask; | 2976 | memcpy(handle->addr, buf, size); |
| 2984 | page_size = 1UL << (handle->data->data_order + PAGE_SHIFT); | ||
| 2985 | page_offset = offset & (page_size - 1); | ||
| 2986 | size = min_t(unsigned int, page_size - page_offset, len); | ||
| 2987 | 2977 | ||
| 2988 | memcpy(pages[nr] + page_offset, buf, size); | 2978 | len -= size; |
| 2979 | handle->addr += size; | ||
| 2980 | handle->size -= size; | ||
| 2981 | if (!handle->size) { | ||
| 2982 | struct perf_mmap_data *data = handle->data; | ||
| 2989 | 2983 | ||
| 2990 | len -= size; | 2984 | handle->page++; |
| 2991 | buf += size; | 2985 | handle->page &= data->nr_pages - 1; |
| 2992 | offset += size; | 2986 | handle->addr = data->data_pages[handle->page]; |
| 2987 | handle->size = PAGE_SIZE << page_order(data); | ||
| 2988 | } | ||
| 2993 | } while (len); | 2989 | } while (len); |
| 2994 | |||
| 2995 | handle->offset = offset; | ||
| 2996 | |||
| 2997 | /* | ||
| 2998 | * Check we didn't copy past our reservation window, taking the | ||
| 2999 | * possible unsigned int wrap into account. | ||
| 3000 | */ | ||
| 3001 | WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0); | ||
| 3002 | } | 2990 | } |
| 3003 | 2991 | ||
| 3004 | int perf_output_begin(struct perf_output_handle *handle, | 2992 | int perf_output_begin(struct perf_output_handle *handle, |
| @@ -3036,13 +3024,13 @@ int perf_output_begin(struct perf_output_handle *handle, | |||
| 3036 | handle->sample = sample; | 3024 | handle->sample = sample; |
| 3037 | 3025 | ||
| 3038 | if (!data->nr_pages) | 3026 | if (!data->nr_pages) |
| 3039 | goto fail; | 3027 | goto out; |
| 3040 | 3028 | ||
| 3041 | have_lost = atomic_read(&data->lost); | 3029 | have_lost = local_read(&data->lost); |
| 3042 | if (have_lost) | 3030 | if (have_lost) |
| 3043 | size += sizeof(lost_event); | 3031 | size += sizeof(lost_event); |
| 3044 | 3032 | ||
| 3045 | perf_output_lock(handle); | 3033 | perf_output_get_handle(handle); |
| 3046 | 3034 | ||
| 3047 | do { | 3035 | do { |
| 3048 | /* | 3036 | /* |
| @@ -3052,24 +3040,28 @@ int perf_output_begin(struct perf_output_handle *handle, | |||
| 3052 | */ | 3040 | */ |
| 3053 | tail = ACCESS_ONCE(data->user_page->data_tail); | 3041 | tail = ACCESS_ONCE(data->user_page->data_tail); |
| 3054 | smp_rmb(); | 3042 | smp_rmb(); |
| 3055 | offset = head = atomic_long_read(&data->head); | 3043 | offset = head = local_read(&data->head); |
| 3056 | head += size; | 3044 | head += size; |
| 3057 | if (unlikely(!perf_output_space(data, tail, offset, head))) | 3045 | if (unlikely(!perf_output_space(data, tail, offset, head))) |
| 3058 | goto fail; | 3046 | goto fail; |
| 3059 | } while (atomic_long_cmpxchg(&data->head, offset, head) != offset); | 3047 | } while (local_cmpxchg(&data->head, offset, head) != offset); |
| 3060 | 3048 | ||
| 3061 | handle->offset = offset; | 3049 | if (head - local_read(&data->wakeup) > data->watermark) |
| 3062 | handle->head = head; | 3050 | local_add(data->watermark, &data->wakeup); |
| 3063 | 3051 | ||
| 3064 | if (head - tail > data->watermark) | 3052 | handle->page = offset >> (PAGE_SHIFT + page_order(data)); |
| 3065 | atomic_set(&data->wakeup, 1); | 3053 | handle->page &= data->nr_pages - 1; |
| 3054 | handle->size = offset & ((PAGE_SIZE << page_order(data)) - 1); | ||
| 3055 | handle->addr = data->data_pages[handle->page]; | ||
| 3056 | handle->addr += handle->size; | ||
| 3057 | handle->size = (PAGE_SIZE << page_order(data)) - handle->size; | ||
| 3066 | 3058 | ||
| 3067 | if (have_lost) { | 3059 | if (have_lost) { |
| 3068 | lost_event.header.type = PERF_RECORD_LOST; | 3060 | lost_event.header.type = PERF_RECORD_LOST; |
| 3069 | lost_event.header.misc = 0; | 3061 | lost_event.header.misc = 0; |
| 3070 | lost_event.header.size = sizeof(lost_event); | 3062 | lost_event.header.size = sizeof(lost_event); |
| 3071 | lost_event.id = event->id; | 3063 | lost_event.id = event->id; |
| 3072 | lost_event.lost = atomic_xchg(&data->lost, 0); | 3064 | lost_event.lost = local_xchg(&data->lost, 0); |
| 3073 | 3065 | ||
| 3074 | perf_output_put(handle, lost_event); | 3066 | perf_output_put(handle, lost_event); |
| 3075 | } | 3067 | } |
| @@ -3077,8 +3069,8 @@ int perf_output_begin(struct perf_output_handle *handle, | |||
| 3077 | return 0; | 3069 | return 0; |
| 3078 | 3070 | ||
| 3079 | fail: | 3071 | fail: |
| 3080 | atomic_inc(&data->lost); | 3072 | local_inc(&data->lost); |
| 3081 | perf_output_unlock(handle); | 3073 | perf_output_put_handle(handle); |
| 3082 | out: | 3074 | out: |
| 3083 | rcu_read_unlock(); | 3075 | rcu_read_unlock(); |
| 3084 | 3076 | ||
| @@ -3093,14 +3085,14 @@ void perf_output_end(struct perf_output_handle *handle) | |||
| 3093 | int wakeup_events = event->attr.wakeup_events; | 3085 | int wakeup_events = event->attr.wakeup_events; |
| 3094 | 3086 | ||
| 3095 | if (handle->sample && wakeup_events) { | 3087 | if (handle->sample && wakeup_events) { |
| 3096 | int events = atomic_inc_return(&data->events); | 3088 | int events = local_inc_return(&data->events); |
| 3097 | if (events >= wakeup_events) { | 3089 | if (events >= wakeup_events) { |
| 3098 | atomic_sub(wakeup_events, &data->events); | 3090 | local_sub(wakeup_events, &data->events); |
| 3099 | atomic_set(&data->wakeup, 1); | 3091 | local_inc(&data->wakeup); |
| 3100 | } | 3092 | } |
| 3101 | } | 3093 | } |
| 3102 | 3094 | ||
| 3103 | perf_output_unlock(handle); | 3095 | perf_output_put_handle(handle); |
| 3104 | rcu_read_unlock(); | 3096 | rcu_read_unlock(); |
| 3105 | } | 3097 | } |
| 3106 | 3098 | ||
| @@ -3436,22 +3428,13 @@ static void perf_event_task_output(struct perf_event *event, | |||
| 3436 | { | 3428 | { |
| 3437 | struct perf_output_handle handle; | 3429 | struct perf_output_handle handle; |
| 3438 | struct task_struct *task = task_event->task; | 3430 | struct task_struct *task = task_event->task; |
| 3439 | unsigned long flags; | ||
| 3440 | int size, ret; | 3431 | int size, ret; |
| 3441 | 3432 | ||
| 3442 | /* | ||
| 3443 | * If this CPU attempts to acquire an rq lock held by a CPU spinning | ||
| 3444 | * in perf_output_lock() from interrupt context, it's game over. | ||
| 3445 | */ | ||
| 3446 | local_irq_save(flags); | ||
| 3447 | |||
| 3448 | size = task_event->event_id.header.size; | 3433 | size = task_event->event_id.header.size; |
| 3449 | ret = perf_output_begin(&handle, event, size, 0, 0); | 3434 | ret = perf_output_begin(&handle, event, size, 0, 0); |
| 3450 | 3435 | ||
| 3451 | if (ret) { | 3436 | if (ret) |
| 3452 | local_irq_restore(flags); | ||
| 3453 | return; | 3437 | return; |
| 3454 | } | ||
| 3455 | 3438 | ||
| 3456 | task_event->event_id.pid = perf_event_pid(event, task); | 3439 | task_event->event_id.pid = perf_event_pid(event, task); |
| 3457 | task_event->event_id.ppid = perf_event_pid(event, current); | 3440 | task_event->event_id.ppid = perf_event_pid(event, current); |
| @@ -3462,7 +3445,6 @@ static void perf_event_task_output(struct perf_event *event, | |||
| 3462 | perf_output_put(&handle, task_event->event_id); | 3445 | perf_output_put(&handle, task_event->event_id); |
| 3463 | 3446 | ||
| 3464 | perf_output_end(&handle); | 3447 | perf_output_end(&handle); |
| 3465 | local_irq_restore(flags); | ||
| 3466 | } | 3448 | } |
| 3467 | 3449 | ||
| 3468 | static int perf_event_task_match(struct perf_event *event) | 3450 | static int perf_event_task_match(struct perf_event *event) |
| @@ -4020,9 +4002,6 @@ static void perf_swevent_add(struct perf_event *event, u64 nr, | |||
| 4020 | perf_swevent_overflow(event, 0, nmi, data, regs); | 4002 | perf_swevent_overflow(event, 0, nmi, data, regs); |
| 4021 | } | 4003 | } |
| 4022 | 4004 | ||
| 4023 | static int perf_tp_event_match(struct perf_event *event, | ||
| 4024 | struct perf_sample_data *data); | ||
| 4025 | |||
| 4026 | static int perf_exclude_event(struct perf_event *event, | 4005 | static int perf_exclude_event(struct perf_event *event, |
| 4027 | struct pt_regs *regs) | 4006 | struct pt_regs *regs) |
| 4028 | { | 4007 | { |
| @@ -4052,10 +4031,6 @@ static int perf_swevent_match(struct perf_event *event, | |||
| 4052 | if (perf_exclude_event(event, regs)) | 4031 | if (perf_exclude_event(event, regs)) |
| 4053 | return 0; | 4032 | return 0; |
| 4054 | 4033 | ||
| 4055 | if (event->attr.type == PERF_TYPE_TRACEPOINT && | ||
| 4056 | !perf_tp_event_match(event, data)) | ||
| 4057 | return 0; | ||
| 4058 | |||
| 4059 | return 1; | 4034 | return 1; |
| 4060 | } | 4035 | } |
| 4061 | 4036 | ||
| @@ -4066,19 +4041,46 @@ static inline u64 swevent_hash(u64 type, u32 event_id) | |||
| 4066 | return hash_64(val, SWEVENT_HLIST_BITS); | 4041 | return hash_64(val, SWEVENT_HLIST_BITS); |
| 4067 | } | 4042 | } |
| 4068 | 4043 | ||
| 4069 | static struct hlist_head * | 4044 | static inline struct hlist_head * |
| 4070 | find_swevent_head(struct perf_cpu_context *ctx, u64 type, u32 event_id) | 4045 | __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id) |
| 4071 | { | 4046 | { |
| 4072 | u64 hash; | 4047 | u64 hash = swevent_hash(type, event_id); |
| 4073 | struct swevent_hlist *hlist; | 4048 | |
| 4049 | return &hlist->heads[hash]; | ||
| 4050 | } | ||
| 4074 | 4051 | ||
| 4075 | hash = swevent_hash(type, event_id); | 4052 | /* For the read side: events when they trigger */ |
| 4053 | static inline struct hlist_head * | ||
| 4054 | find_swevent_head_rcu(struct perf_cpu_context *ctx, u64 type, u32 event_id) | ||
| 4055 | { | ||
| 4056 | struct swevent_hlist *hlist; | ||
| 4076 | 4057 | ||
| 4077 | hlist = rcu_dereference(ctx->swevent_hlist); | 4058 | hlist = rcu_dereference(ctx->swevent_hlist); |
| 4078 | if (!hlist) | 4059 | if (!hlist) |
| 4079 | return NULL; | 4060 | return NULL; |
| 4080 | 4061 | ||
| 4081 | return &hlist->heads[hash]; | 4062 | return __find_swevent_head(hlist, type, event_id); |
| 4063 | } | ||
| 4064 | |||
| 4065 | /* For the event head insertion and removal in the hlist */ | ||
| 4066 | static inline struct hlist_head * | ||
| 4067 | find_swevent_head(struct perf_cpu_context *ctx, struct perf_event *event) | ||
| 4068 | { | ||
| 4069 | struct swevent_hlist *hlist; | ||
| 4070 | u32 event_id = event->attr.config; | ||
| 4071 | u64 type = event->attr.type; | ||
| 4072 | |||
| 4073 | /* | ||
| 4074 | * Event scheduling is always serialized against hlist allocation | ||
| 4075 | * and release. Which makes the protected version suitable here. | ||
| 4076 | * The context lock guarantees that. | ||
| 4077 | */ | ||
| 4078 | hlist = rcu_dereference_protected(ctx->swevent_hlist, | ||
| 4079 | lockdep_is_held(&event->ctx->lock)); | ||
| 4080 | if (!hlist) | ||
| 4081 | return NULL; | ||
| 4082 | |||
| 4083 | return __find_swevent_head(hlist, type, event_id); | ||
| 4082 | } | 4084 | } |
| 4083 | 4085 | ||
| 4084 | static void do_perf_sw_event(enum perf_type_id type, u32 event_id, | 4086 | static void do_perf_sw_event(enum perf_type_id type, u32 event_id, |
| @@ -4095,7 +4097,7 @@ static void do_perf_sw_event(enum perf_type_id type, u32 event_id, | |||
| 4095 | 4097 | ||
| 4096 | rcu_read_lock(); | 4098 | rcu_read_lock(); |
| 4097 | 4099 | ||
| 4098 | head = find_swevent_head(cpuctx, type, event_id); | 4100 | head = find_swevent_head_rcu(cpuctx, type, event_id); |
| 4099 | 4101 | ||
| 4100 | if (!head) | 4102 | if (!head) |
| 4101 | goto end; | 4103 | goto end; |
| @@ -4110,7 +4112,7 @@ end: | |||
| 4110 | 4112 | ||
| 4111 | int perf_swevent_get_recursion_context(void) | 4113 | int perf_swevent_get_recursion_context(void) |
| 4112 | { | 4114 | { |
| 4113 | struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context); | 4115 | struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context); |
| 4114 | int rctx; | 4116 | int rctx; |
| 4115 | 4117 | ||
| 4116 | if (in_nmi()) | 4118 | if (in_nmi()) |
| @@ -4122,10 +4124,8 @@ int perf_swevent_get_recursion_context(void) | |||
| 4122 | else | 4124 | else |
| 4123 | rctx = 0; | 4125 | rctx = 0; |
| 4124 | 4126 | ||
| 4125 | if (cpuctx->recursion[rctx]) { | 4127 | if (cpuctx->recursion[rctx]) |
| 4126 | put_cpu_var(perf_cpu_context); | ||
| 4127 | return -1; | 4128 | return -1; |
| 4128 | } | ||
| 4129 | 4129 | ||
| 4130 | cpuctx->recursion[rctx]++; | 4130 | cpuctx->recursion[rctx]++; |
| 4131 | barrier(); | 4131 | barrier(); |
| @@ -4139,7 +4139,6 @@ void perf_swevent_put_recursion_context(int rctx) | |||
| 4139 | struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context); | 4139 | struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context); |
| 4140 | barrier(); | 4140 | barrier(); |
| 4141 | cpuctx->recursion[rctx]--; | 4141 | cpuctx->recursion[rctx]--; |
| 4142 | put_cpu_var(perf_cpu_context); | ||
| 4143 | } | 4142 | } |
| 4144 | EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context); | 4143 | EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context); |
| 4145 | 4144 | ||
| @@ -4150,6 +4149,7 @@ void __perf_sw_event(u32 event_id, u64 nr, int nmi, | |||
| 4150 | struct perf_sample_data data; | 4149 | struct perf_sample_data data; |
| 4151 | int rctx; | 4150 | int rctx; |
| 4152 | 4151 | ||
| 4152 | preempt_disable_notrace(); | ||
| 4153 | rctx = perf_swevent_get_recursion_context(); | 4153 | rctx = perf_swevent_get_recursion_context(); |
| 4154 | if (rctx < 0) | 4154 | if (rctx < 0) |
| 4155 | return; | 4155 | return; |
| @@ -4159,6 +4159,7 @@ void __perf_sw_event(u32 event_id, u64 nr, int nmi, | |||
| 4159 | do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs); | 4159 | do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs); |
| 4160 | 4160 | ||
| 4161 | perf_swevent_put_recursion_context(rctx); | 4161 | perf_swevent_put_recursion_context(rctx); |
| 4162 | preempt_enable_notrace(); | ||
| 4162 | } | 4163 | } |
| 4163 | 4164 | ||
| 4164 | static void perf_swevent_read(struct perf_event *event) | 4165 | static void perf_swevent_read(struct perf_event *event) |
| @@ -4178,7 +4179,7 @@ static int perf_swevent_enable(struct perf_event *event) | |||
| 4178 | perf_swevent_set_period(event); | 4179 | perf_swevent_set_period(event); |
| 4179 | } | 4180 | } |
| 4180 | 4181 | ||
| 4181 | head = find_swevent_head(cpuctx, event->attr.type, event->attr.config); | 4182 | head = find_swevent_head(cpuctx, event); |
| 4182 | if (WARN_ON_ONCE(!head)) | 4183 | if (WARN_ON_ONCE(!head)) |
| 4183 | return -EINVAL; | 4184 | return -EINVAL; |
| 4184 | 4185 | ||
| @@ -4366,6 +4367,14 @@ static const struct pmu perf_ops_task_clock = { | |||
| 4366 | .read = task_clock_perf_event_read, | 4367 | .read = task_clock_perf_event_read, |
| 4367 | }; | 4368 | }; |
| 4368 | 4369 | ||
| 4370 | /* Deref the hlist from the update side */ | ||
| 4371 | static inline struct swevent_hlist * | ||
| 4372 | swevent_hlist_deref(struct perf_cpu_context *cpuctx) | ||
| 4373 | { | ||
| 4374 | return rcu_dereference_protected(cpuctx->swevent_hlist, | ||
| 4375 | lockdep_is_held(&cpuctx->hlist_mutex)); | ||
| 4376 | } | ||
| 4377 | |||
| 4369 | static void swevent_hlist_release_rcu(struct rcu_head *rcu_head) | 4378 | static void swevent_hlist_release_rcu(struct rcu_head *rcu_head) |
| 4370 | { | 4379 | { |
| 4371 | struct swevent_hlist *hlist; | 4380 | struct swevent_hlist *hlist; |
| @@ -4376,12 +4385,11 @@ static void swevent_hlist_release_rcu(struct rcu_head *rcu_head) | |||
| 4376 | 4385 | ||
| 4377 | static void swevent_hlist_release(struct perf_cpu_context *cpuctx) | 4386 | static void swevent_hlist_release(struct perf_cpu_context *cpuctx) |
| 4378 | { | 4387 | { |
| 4379 | struct swevent_hlist *hlist; | 4388 | struct swevent_hlist *hlist = swevent_hlist_deref(cpuctx); |
| 4380 | 4389 | ||
| 4381 | if (!cpuctx->swevent_hlist) | 4390 | if (!hlist) |
| 4382 | return; | 4391 | return; |
| 4383 | 4392 | ||
| 4384 | hlist = cpuctx->swevent_hlist; | ||
| 4385 | rcu_assign_pointer(cpuctx->swevent_hlist, NULL); | 4393 | rcu_assign_pointer(cpuctx->swevent_hlist, NULL); |
| 4386 | call_rcu(&hlist->rcu_head, swevent_hlist_release_rcu); | 4394 | call_rcu(&hlist->rcu_head, swevent_hlist_release_rcu); |
| 4387 | } | 4395 | } |
| @@ -4418,7 +4426,7 @@ static int swevent_hlist_get_cpu(struct perf_event *event, int cpu) | |||
| 4418 | 4426 | ||
| 4419 | mutex_lock(&cpuctx->hlist_mutex); | 4427 | mutex_lock(&cpuctx->hlist_mutex); |
| 4420 | 4428 | ||
| 4421 | if (!cpuctx->swevent_hlist && cpu_online(cpu)) { | 4429 | if (!swevent_hlist_deref(cpuctx) && cpu_online(cpu)) { |
| 4422 | struct swevent_hlist *hlist; | 4430 | struct swevent_hlist *hlist; |
| 4423 | 4431 | ||
| 4424 | hlist = kzalloc(sizeof(*hlist), GFP_KERNEL); | 4432 | hlist = kzalloc(sizeof(*hlist), GFP_KERNEL); |
| @@ -4467,10 +4475,46 @@ static int swevent_hlist_get(struct perf_event *event) | |||
| 4467 | 4475 | ||
| 4468 | #ifdef CONFIG_EVENT_TRACING | 4476 | #ifdef CONFIG_EVENT_TRACING |
| 4469 | 4477 | ||
| 4470 | void perf_tp_event(int event_id, u64 addr, u64 count, void *record, | 4478 | static const struct pmu perf_ops_tracepoint = { |
| 4471 | int entry_size, struct pt_regs *regs) | 4479 | .enable = perf_trace_enable, |
| 4480 | .disable = perf_trace_disable, | ||
| 4481 | .read = perf_swevent_read, | ||
| 4482 | .unthrottle = perf_swevent_unthrottle, | ||
| 4483 | }; | ||
| 4484 | |||
| 4485 | static int perf_tp_filter_match(struct perf_event *event, | ||
| 4486 | struct perf_sample_data *data) | ||
| 4487 | { | ||
| 4488 | void *record = data->raw->data; | ||
| 4489 | |||
| 4490 | if (likely(!event->filter) || filter_match_preds(event->filter, record)) | ||
| 4491 | return 1; | ||
| 4492 | return 0; | ||
| 4493 | } | ||
| 4494 | |||
| 4495 | static int perf_tp_event_match(struct perf_event *event, | ||
| 4496 | struct perf_sample_data *data, | ||
| 4497 | struct pt_regs *regs) | ||
| 4498 | { | ||
| 4499 | /* | ||
| 4500 | * All tracepoints are from kernel-space. | ||
| 4501 | */ | ||
| 4502 | if (event->attr.exclude_kernel) | ||
| 4503 | return 0; | ||
| 4504 | |||
| 4505 | if (!perf_tp_filter_match(event, data)) | ||
| 4506 | return 0; | ||
| 4507 | |||
| 4508 | return 1; | ||
| 4509 | } | ||
| 4510 | |||
| 4511 | void perf_tp_event(u64 addr, u64 count, void *record, int entry_size, | ||
| 4512 | struct pt_regs *regs, struct hlist_head *head) | ||
| 4472 | { | 4513 | { |
| 4473 | struct perf_sample_data data; | 4514 | struct perf_sample_data data; |
| 4515 | struct perf_event *event; | ||
| 4516 | struct hlist_node *node; | ||
| 4517 | |||
| 4474 | struct perf_raw_record raw = { | 4518 | struct perf_raw_record raw = { |
| 4475 | .size = entry_size, | 4519 | .size = entry_size, |
| 4476 | .data = record, | 4520 | .data = record, |
| @@ -4479,26 +4523,18 @@ void perf_tp_event(int event_id, u64 addr, u64 count, void *record, | |||
| 4479 | perf_sample_data_init(&data, addr); | 4523 | perf_sample_data_init(&data, addr); |
| 4480 | data.raw = &raw; | 4524 | data.raw = &raw; |
| 4481 | 4525 | ||
| 4482 | /* Trace events already protected against recursion */ | 4526 | rcu_read_lock(); |
| 4483 | do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1, | 4527 | hlist_for_each_entry_rcu(event, node, head, hlist_entry) { |
| 4484 | &data, regs); | 4528 | if (perf_tp_event_match(event, &data, regs)) |
| 4529 | perf_swevent_add(event, count, 1, &data, regs); | ||
| 4530 | } | ||
| 4531 | rcu_read_unlock(); | ||
| 4485 | } | 4532 | } |
| 4486 | EXPORT_SYMBOL_GPL(perf_tp_event); | 4533 | EXPORT_SYMBOL_GPL(perf_tp_event); |
| 4487 | 4534 | ||
| 4488 | static int perf_tp_event_match(struct perf_event *event, | ||
| 4489 | struct perf_sample_data *data) | ||
| 4490 | { | ||
| 4491 | void *record = data->raw->data; | ||
| 4492 | |||
| 4493 | if (likely(!event->filter) || filter_match_preds(event->filter, record)) | ||
| 4494 | return 1; | ||
| 4495 | return 0; | ||
| 4496 | } | ||
| 4497 | |||
| 4498 | static void tp_perf_event_destroy(struct perf_event *event) | 4535 | static void tp_perf_event_destroy(struct perf_event *event) |
| 4499 | { | 4536 | { |
| 4500 | perf_trace_disable(event->attr.config); | 4537 | perf_trace_destroy(event); |
| 4501 | swevent_hlist_put(event); | ||
| 4502 | } | 4538 | } |
| 4503 | 4539 | ||
| 4504 | static const struct pmu *tp_perf_event_init(struct perf_event *event) | 4540 | static const struct pmu *tp_perf_event_init(struct perf_event *event) |
| @@ -4514,17 +4550,13 @@ static const struct pmu *tp_perf_event_init(struct perf_event *event) | |||
| 4514 | !capable(CAP_SYS_ADMIN)) | 4550 | !capable(CAP_SYS_ADMIN)) |
| 4515 | return ERR_PTR(-EPERM); | 4551 | return ERR_PTR(-EPERM); |
| 4516 | 4552 | ||
| 4517 | if (perf_trace_enable(event->attr.config)) | 4553 | err = perf_trace_init(event); |
| 4554 | if (err) | ||
| 4518 | return NULL; | 4555 | return NULL; |
| 4519 | 4556 | ||
| 4520 | event->destroy = tp_perf_event_destroy; | 4557 | event->destroy = tp_perf_event_destroy; |
| 4521 | err = swevent_hlist_get(event); | ||
| 4522 | if (err) { | ||
| 4523 | perf_trace_disable(event->attr.config); | ||
| 4524 | return ERR_PTR(err); | ||
| 4525 | } | ||
| 4526 | 4558 | ||
| 4527 | return &perf_ops_generic; | 4559 | return &perf_ops_tracepoint; |
| 4528 | } | 4560 | } |
| 4529 | 4561 | ||
| 4530 | static int perf_event_set_filter(struct perf_event *event, void __user *arg) | 4562 | static int perf_event_set_filter(struct perf_event *event, void __user *arg) |
| @@ -4552,12 +4584,6 @@ static void perf_event_free_filter(struct perf_event *event) | |||
| 4552 | 4584 | ||
| 4553 | #else | 4585 | #else |
| 4554 | 4586 | ||
| 4555 | static int perf_tp_event_match(struct perf_event *event, | ||
| 4556 | struct perf_sample_data *data) | ||
| 4557 | { | ||
| 4558 | return 1; | ||
| 4559 | } | ||
| 4560 | |||
| 4561 | static const struct pmu *tp_perf_event_init(struct perf_event *event) | 4587 | static const struct pmu *tp_perf_event_init(struct perf_event *event) |
| 4562 | { | 4588 | { |
| 4563 | return NULL; | 4589 | return NULL; |
| @@ -4894,6 +4920,13 @@ static int perf_event_set_output(struct perf_event *event, int output_fd) | |||
| 4894 | int fput_needed = 0; | 4920 | int fput_needed = 0; |
| 4895 | int ret = -EINVAL; | 4921 | int ret = -EINVAL; |
| 4896 | 4922 | ||
| 4923 | /* | ||
| 4924 | * Don't allow output of inherited per-task events. This would | ||
| 4925 | * create performance issues due to cross cpu access. | ||
| 4926 | */ | ||
| 4927 | if (event->cpu == -1 && event->attr.inherit) | ||
| 4928 | return -EINVAL; | ||
| 4929 | |||
| 4897 | if (!output_fd) | 4930 | if (!output_fd) |
| 4898 | goto set; | 4931 | goto set; |
| 4899 | 4932 | ||
| @@ -4914,6 +4947,18 @@ static int perf_event_set_output(struct perf_event *event, int output_fd) | |||
| 4914 | if (event->data) | 4947 | if (event->data) |
| 4915 | goto out; | 4948 | goto out; |
| 4916 | 4949 | ||
| 4950 | /* | ||
| 4951 | * Don't allow cross-cpu buffers | ||
| 4952 | */ | ||
| 4953 | if (output_event->cpu != event->cpu) | ||
| 4954 | goto out; | ||
| 4955 | |||
| 4956 | /* | ||
| 4957 | * If its not a per-cpu buffer, it must be the same task. | ||
| 4958 | */ | ||
| 4959 | if (output_event->cpu == -1 && output_event->ctx != event->ctx) | ||
| 4960 | goto out; | ||
| 4961 | |||
| 4917 | atomic_long_inc(&output_file->f_count); | 4962 | atomic_long_inc(&output_file->f_count); |
| 4918 | 4963 | ||
| 4919 | set: | 4964 | set: |
diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index b3bc91a3f510..36ea2b65dcdc 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c | |||
| @@ -675,28 +675,33 @@ static void blk_add_trace_rq(struct request_queue *q, struct request *rq, | |||
| 675 | } | 675 | } |
| 676 | } | 676 | } |
| 677 | 677 | ||
| 678 | static void blk_add_trace_rq_abort(struct request_queue *q, struct request *rq) | 678 | static void blk_add_trace_rq_abort(void *ignore, |
| 679 | struct request_queue *q, struct request *rq) | ||
| 679 | { | 680 | { |
| 680 | blk_add_trace_rq(q, rq, BLK_TA_ABORT); | 681 | blk_add_trace_rq(q, rq, BLK_TA_ABORT); |
| 681 | } | 682 | } |
| 682 | 683 | ||
| 683 | static void blk_add_trace_rq_insert(struct request_queue *q, struct request *rq) | 684 | static void blk_add_trace_rq_insert(void *ignore, |
| 685 | struct request_queue *q, struct request *rq) | ||
| 684 | { | 686 | { |
| 685 | blk_add_trace_rq(q, rq, BLK_TA_INSERT); | 687 | blk_add_trace_rq(q, rq, BLK_TA_INSERT); |
| 686 | } | 688 | } |
| 687 | 689 | ||
| 688 | static void blk_add_trace_rq_issue(struct request_queue *q, struct request *rq) | 690 | static void blk_add_trace_rq_issue(void *ignore, |
| 691 | struct request_queue *q, struct request *rq) | ||
| 689 | { | 692 | { |
| 690 | blk_add_trace_rq(q, rq, BLK_TA_ISSUE); | 693 | blk_add_trace_rq(q, rq, BLK_TA_ISSUE); |
| 691 | } | 694 | } |
| 692 | 695 | ||
| 693 | static void blk_add_trace_rq_requeue(struct request_queue *q, | 696 | static void blk_add_trace_rq_requeue(void *ignore, |
| 697 | struct request_queue *q, | ||
| 694 | struct request *rq) | 698 | struct request *rq) |
| 695 | { | 699 | { |
| 696 | blk_add_trace_rq(q, rq, BLK_TA_REQUEUE); | 700 | blk_add_trace_rq(q, rq, BLK_TA_REQUEUE); |
| 697 | } | 701 | } |
| 698 | 702 | ||
| 699 | static void blk_add_trace_rq_complete(struct request_queue *q, | 703 | static void blk_add_trace_rq_complete(void *ignore, |
| 704 | struct request_queue *q, | ||
| 700 | struct request *rq) | 705 | struct request *rq) |
| 701 | { | 706 | { |
| 702 | blk_add_trace_rq(q, rq, BLK_TA_COMPLETE); | 707 | blk_add_trace_rq(q, rq, BLK_TA_COMPLETE); |
| @@ -724,34 +729,40 @@ static void blk_add_trace_bio(struct request_queue *q, struct bio *bio, | |||
| 724 | !bio_flagged(bio, BIO_UPTODATE), 0, NULL); | 729 | !bio_flagged(bio, BIO_UPTODATE), 0, NULL); |
| 725 | } | 730 | } |
| 726 | 731 | ||
| 727 | static void blk_add_trace_bio_bounce(struct request_queue *q, struct bio *bio) | 732 | static void blk_add_trace_bio_bounce(void *ignore, |
| 733 | struct request_queue *q, struct bio *bio) | ||
| 728 | { | 734 | { |
| 729 | blk_add_trace_bio(q, bio, BLK_TA_BOUNCE); | 735 | blk_add_trace_bio(q, bio, BLK_TA_BOUNCE); |
| 730 | } | 736 | } |
| 731 | 737 | ||
| 732 | static void blk_add_trace_bio_complete(struct request_queue *q, struct bio *bio) | 738 | static void blk_add_trace_bio_complete(void *ignore, |
| 739 | struct request_queue *q, struct bio *bio) | ||
| 733 | { | 740 | { |
| 734 | blk_add_trace_bio(q, bio, BLK_TA_COMPLETE); | 741 | blk_add_trace_bio(q, bio, BLK_TA_COMPLETE); |
| 735 | } | 742 | } |
| 736 | 743 | ||
| 737 | static void blk_add_trace_bio_backmerge(struct request_queue *q, | 744 | static void blk_add_trace_bio_backmerge(void *ignore, |
| 745 | struct request_queue *q, | ||
| 738 | struct bio *bio) | 746 | struct bio *bio) |
| 739 | { | 747 | { |
| 740 | blk_add_trace_bio(q, bio, BLK_TA_BACKMERGE); | 748 | blk_add_trace_bio(q, bio, BLK_TA_BACKMERGE); |
| 741 | } | 749 | } |
| 742 | 750 | ||
| 743 | static void blk_add_trace_bio_frontmerge(struct request_queue *q, | 751 | static void blk_add_trace_bio_frontmerge(void *ignore, |
| 752 | struct request_queue *q, | ||
| 744 | struct bio *bio) | 753 | struct bio *bio) |
| 745 | { | 754 | { |
| 746 | blk_add_trace_bio(q, bio, BLK_TA_FRONTMERGE); | 755 | blk_add_trace_bio(q, bio, BLK_TA_FRONTMERGE); |
| 747 | } | 756 | } |
| 748 | 757 | ||
| 749 | static void blk_add_trace_bio_queue(struct request_queue *q, struct bio *bio) | 758 | static void blk_add_trace_bio_queue(void *ignore, |
| 759 | struct request_queue *q, struct bio *bio) | ||
| 750 | { | 760 | { |
| 751 | blk_add_trace_bio(q, bio, BLK_TA_QUEUE); | 761 | blk_add_trace_bio(q, bio, BLK_TA_QUEUE); |
| 752 | } | 762 | } |
| 753 | 763 | ||
| 754 | static void blk_add_trace_getrq(struct request_queue *q, | 764 | static void blk_add_trace_getrq(void *ignore, |
| 765 | struct request_queue *q, | ||
| 755 | struct bio *bio, int rw) | 766 | struct bio *bio, int rw) |
| 756 | { | 767 | { |
| 757 | if (bio) | 768 | if (bio) |
| @@ -765,7 +776,8 @@ static void blk_add_trace_getrq(struct request_queue *q, | |||
| 765 | } | 776 | } |
| 766 | 777 | ||
| 767 | 778 | ||
| 768 | static void blk_add_trace_sleeprq(struct request_queue *q, | 779 | static void blk_add_trace_sleeprq(void *ignore, |
| 780 | struct request_queue *q, | ||
| 769 | struct bio *bio, int rw) | 781 | struct bio *bio, int rw) |
| 770 | { | 782 | { |
| 771 | if (bio) | 783 | if (bio) |
| @@ -779,7 +791,7 @@ static void blk_add_trace_sleeprq(struct request_queue *q, | |||
| 779 | } | 791 | } |
| 780 | } | 792 | } |
| 781 | 793 | ||
| 782 | static void blk_add_trace_plug(struct request_queue *q) | 794 | static void blk_add_trace_plug(void *ignore, struct request_queue *q) |
| 783 | { | 795 | { |
| 784 | struct blk_trace *bt = q->blk_trace; | 796 | struct blk_trace *bt = q->blk_trace; |
| 785 | 797 | ||
| @@ -787,7 +799,7 @@ static void blk_add_trace_plug(struct request_queue *q) | |||
| 787 | __blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL); | 799 | __blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL); |
| 788 | } | 800 | } |
| 789 | 801 | ||
| 790 | static void blk_add_trace_unplug_io(struct request_queue *q) | 802 | static void blk_add_trace_unplug_io(void *ignore, struct request_queue *q) |
| 791 | { | 803 | { |
| 792 | struct blk_trace *bt = q->blk_trace; | 804 | struct blk_trace *bt = q->blk_trace; |
| 793 | 805 | ||
| @@ -800,7 +812,7 @@ static void blk_add_trace_unplug_io(struct request_queue *q) | |||
| 800 | } | 812 | } |
| 801 | } | 813 | } |
| 802 | 814 | ||
| 803 | static void blk_add_trace_unplug_timer(struct request_queue *q) | 815 | static void blk_add_trace_unplug_timer(void *ignore, struct request_queue *q) |
| 804 | { | 816 | { |
| 805 | struct blk_trace *bt = q->blk_trace; | 817 | struct blk_trace *bt = q->blk_trace; |
| 806 | 818 | ||
| @@ -813,7 +825,8 @@ static void blk_add_trace_unplug_timer(struct request_queue *q) | |||
| 813 | } | 825 | } |
| 814 | } | 826 | } |
| 815 | 827 | ||
| 816 | static void blk_add_trace_split(struct request_queue *q, struct bio *bio, | 828 | static void blk_add_trace_split(void *ignore, |
| 829 | struct request_queue *q, struct bio *bio, | ||
| 817 | unsigned int pdu) | 830 | unsigned int pdu) |
| 818 | { | 831 | { |
| 819 | struct blk_trace *bt = q->blk_trace; | 832 | struct blk_trace *bt = q->blk_trace; |
| @@ -839,8 +852,9 @@ static void blk_add_trace_split(struct request_queue *q, struct bio *bio, | |||
| 839 | * it spans a stripe (or similar). Add a trace for that action. | 852 | * it spans a stripe (or similar). Add a trace for that action. |
| 840 | * | 853 | * |
| 841 | **/ | 854 | **/ |
| 842 | static void blk_add_trace_remap(struct request_queue *q, struct bio *bio, | 855 | static void blk_add_trace_remap(void *ignore, |
| 843 | dev_t dev, sector_t from) | 856 | struct request_queue *q, struct bio *bio, |
| 857 | dev_t dev, sector_t from) | ||
| 844 | { | 858 | { |
| 845 | struct blk_trace *bt = q->blk_trace; | 859 | struct blk_trace *bt = q->blk_trace; |
| 846 | struct blk_io_trace_remap r; | 860 | struct blk_io_trace_remap r; |
| @@ -869,7 +883,8 @@ static void blk_add_trace_remap(struct request_queue *q, struct bio *bio, | |||
| 869 | * Add a trace for that action. | 883 | * Add a trace for that action. |
| 870 | * | 884 | * |
| 871 | **/ | 885 | **/ |
| 872 | static void blk_add_trace_rq_remap(struct request_queue *q, | 886 | static void blk_add_trace_rq_remap(void *ignore, |
| 887 | struct request_queue *q, | ||
| 873 | struct request *rq, dev_t dev, | 888 | struct request *rq, dev_t dev, |
| 874 | sector_t from) | 889 | sector_t from) |
| 875 | { | 890 | { |
| @@ -921,64 +936,64 @@ static void blk_register_tracepoints(void) | |||
| 921 | { | 936 | { |
| 922 | int ret; | 937 | int ret; |
| 923 | 938 | ||
| 924 | ret = register_trace_block_rq_abort(blk_add_trace_rq_abort); | 939 | ret = register_trace_block_rq_abort(blk_add_trace_rq_abort, NULL); |
| 925 | WARN_ON(ret); | 940 | WARN_ON(ret); |
| 926 | ret = register_trace_block_rq_insert(blk_add_trace_rq_insert); | 941 | ret = register_trace_block_rq_insert(blk_add_trace_rq_insert, NULL); |
| 927 | WARN_ON(ret); | 942 | WARN_ON(ret); |
| 928 | ret = register_trace_block_rq_issue(blk_add_trace_rq_issue); | 943 | ret = register_trace_block_rq_issue(blk_add_trace_rq_issue, NULL); |
| 929 | WARN_ON(ret); | 944 | WARN_ON(ret); |
| 930 | ret = register_trace_block_rq_requeue(blk_add_trace_rq_requeue); | 945 | ret = register_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL); |
| 931 | WARN_ON(ret); | 946 | WARN_ON(ret); |
| 932 | ret = register_trace_block_rq_complete(blk_add_trace_rq_complete); | 947 | ret = register_trace_block_rq_complete(blk_add_trace_rq_complete, NULL); |
| 933 | WARN_ON(ret); | 948 | WARN_ON(ret); |
| 934 | ret = register_trace_block_bio_bounce(blk_add_trace_bio_bounce); | 949 | ret = register_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL); |
| 935 | WARN_ON(ret); | 950 | WARN_ON(ret); |
| 936 | ret = register_trace_block_bio_complete(blk_add_trace_bio_complete); | 951 | ret = register_trace_block_bio_complete(blk_add_trace_bio_complete, NULL); |
| 937 | WARN_ON(ret); | 952 | WARN_ON(ret); |
| 938 | ret = register_trace_block_bio_backmerge(blk_add_trace_bio_backmerge); | 953 | ret = register_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL); |
| 939 | WARN_ON(ret); | 954 | WARN_ON(ret); |
| 940 | ret = register_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge); | 955 | ret = register_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL); |
| 941 | WARN_ON(ret); | 956 | WARN_ON(ret); |
| 942 | ret = register_trace_block_bio_queue(blk_add_trace_bio_queue); | 957 | ret = register_trace_block_bio_queue(blk_add_trace_bio_queue, NULL); |
| 943 | WARN_ON(ret); | 958 | WARN_ON(ret); |
| 944 | ret = register_trace_block_getrq(blk_add_trace_getrq); | 959 | ret = register_trace_block_getrq(blk_add_trace_getrq, NULL); |
| 945 | WARN_ON(ret); | 960 | WARN_ON(ret); |
| 946 | ret = register_trace_block_sleeprq(blk_add_trace_sleeprq); | 961 | ret = register_trace_block_sleeprq(blk_add_trace_sleeprq, NULL); |
| 947 | WARN_ON(ret); | 962 | WARN_ON(ret); |
| 948 | ret = register_trace_block_plug(blk_add_trace_plug); | 963 | ret = register_trace_block_plug(blk_add_trace_plug, NULL); |
| 949 | WARN_ON(ret); | 964 | WARN_ON(ret); |
| 950 | ret = register_trace_block_unplug_timer(blk_add_trace_unplug_timer); | 965 | ret = register_trace_block_unplug_timer(blk_add_trace_unplug_timer, NULL); |
| 951 | WARN_ON(ret); | 966 | WARN_ON(ret); |
| 952 | ret = register_trace_block_unplug_io(blk_add_trace_unplug_io); | 967 | ret = register_trace_block_unplug_io(blk_add_trace_unplug_io, NULL); |
| 953 | WARN_ON(ret); | 968 | WARN_ON(ret); |
| 954 | ret = register_trace_block_split(blk_add_trace_split); | 969 | ret = register_trace_block_split(blk_add_trace_split, NULL); |
| 955 | WARN_ON(ret); | 970 | WARN_ON(ret); |
| 956 | ret = register_trace_block_remap(blk_add_trace_remap); | 971 | ret = register_trace_block_remap(blk_add_trace_remap, NULL); |
| 957 | WARN_ON(ret); | 972 | WARN_ON(ret); |
| 958 | ret = register_trace_block_rq_remap(blk_add_trace_rq_remap); | 973 | ret = register_trace_block_rq_remap(blk_add_trace_rq_remap, NULL); |
| 959 | WARN_ON(ret); | 974 | WARN_ON(ret); |
| 960 | } | 975 | } |
| 961 | 976 | ||
| 962 | static void blk_unregister_tracepoints(void) | 977 | static void blk_unregister_tracepoints(void) |
| 963 | { | 978 | { |
| 964 | unregister_trace_block_rq_remap(blk_add_trace_rq_remap); | 979 | unregister_trace_block_rq_remap(blk_add_trace_rq_remap, NULL); |
| 965 | unregister_trace_block_remap(blk_add_trace_remap); | 980 | unregister_trace_block_remap(blk_add_trace_remap, NULL); |
| 966 | unregister_trace_block_split(blk_add_trace_split); | 981 | unregister_trace_block_split(blk_add_trace_split, NULL); |
| 967 | unregister_trace_block_unplug_io(blk_add_trace_unplug_io); | 982 | unregister_trace_block_unplug_io(blk_add_trace_unplug_io, NULL); |
| 968 | unregister_trace_block_unplug_timer(blk_add_trace_unplug_timer); | 983 | unregister_trace_block_unplug_timer(blk_add_trace_unplug_timer, NULL); |
| 969 | unregister_trace_block_plug(blk_add_trace_plug); | 984 | unregister_trace_block_plug(blk_add_trace_plug, NULL); |
| 970 | unregister_trace_block_sleeprq(blk_add_trace_sleeprq); | 985 | unregister_trace_block_sleeprq(blk_add_trace_sleeprq, NULL); |
| 971 | unregister_trace_block_getrq(blk_add_trace_getrq); | 986 | unregister_trace_block_getrq(blk_add_trace_getrq, NULL); |
| 972 | unregister_trace_block_bio_queue(blk_add_trace_bio_queue); | 987 | unregister_trace_block_bio_queue(blk_add_trace_bio_queue, NULL); |
| 973 | unregister_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge); | 988 | unregister_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL); |
| 974 | unregister_trace_block_bio_backmerge(blk_add_trace_bio_backmerge); | 989 | unregister_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL); |
| 975 | unregister_trace_block_bio_complete(blk_add_trace_bio_complete); | 990 | unregister_trace_block_bio_complete(blk_add_trace_bio_complete, NULL); |
| 976 | unregister_trace_block_bio_bounce(blk_add_trace_bio_bounce); | 991 | unregister_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL); |
| 977 | unregister_trace_block_rq_complete(blk_add_trace_rq_complete); | 992 | unregister_trace_block_rq_complete(blk_add_trace_rq_complete, NULL); |
| 978 | unregister_trace_block_rq_requeue(blk_add_trace_rq_requeue); | 993 | unregister_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL); |
| 979 | unregister_trace_block_rq_issue(blk_add_trace_rq_issue); | 994 | unregister_trace_block_rq_issue(blk_add_trace_rq_issue, NULL); |
| 980 | unregister_trace_block_rq_insert(blk_add_trace_rq_insert); | 995 | unregister_trace_block_rq_insert(blk_add_trace_rq_insert, NULL); |
| 981 | unregister_trace_block_rq_abort(blk_add_trace_rq_abort); | 996 | unregister_trace_block_rq_abort(blk_add_trace_rq_abort, NULL); |
| 982 | 997 | ||
| 983 | tracepoint_synchronize_unregister(); | 998 | tracepoint_synchronize_unregister(); |
| 984 | } | 999 | } |
| @@ -1321,7 +1336,7 @@ out: | |||
| 1321 | } | 1336 | } |
| 1322 | 1337 | ||
| 1323 | static enum print_line_t blk_trace_event_print(struct trace_iterator *iter, | 1338 | static enum print_line_t blk_trace_event_print(struct trace_iterator *iter, |
| 1324 | int flags) | 1339 | int flags, struct trace_event *event) |
| 1325 | { | 1340 | { |
| 1326 | return print_one_line(iter, false); | 1341 | return print_one_line(iter, false); |
| 1327 | } | 1342 | } |
| @@ -1343,7 +1358,8 @@ static int blk_trace_synthesize_old_trace(struct trace_iterator *iter) | |||
| 1343 | } | 1358 | } |
| 1344 | 1359 | ||
| 1345 | static enum print_line_t | 1360 | static enum print_line_t |
| 1346 | blk_trace_event_print_binary(struct trace_iterator *iter, int flags) | 1361 | blk_trace_event_print_binary(struct trace_iterator *iter, int flags, |
| 1362 | struct trace_event *event) | ||
| 1347 | { | 1363 | { |
| 1348 | return blk_trace_synthesize_old_trace(iter) ? | 1364 | return blk_trace_synthesize_old_trace(iter) ? |
| 1349 | TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE; | 1365 | TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE; |
| @@ -1381,12 +1397,16 @@ static struct tracer blk_tracer __read_mostly = { | |||
| 1381 | .set_flag = blk_tracer_set_flag, | 1397 | .set_flag = blk_tracer_set_flag, |
| 1382 | }; | 1398 | }; |
| 1383 | 1399 | ||
| 1384 | static struct trace_event trace_blk_event = { | 1400 | static struct trace_event_functions trace_blk_event_funcs = { |
| 1385 | .type = TRACE_BLK, | ||
| 1386 | .trace = blk_trace_event_print, | 1401 | .trace = blk_trace_event_print, |
| 1387 | .binary = blk_trace_event_print_binary, | 1402 | .binary = blk_trace_event_print_binary, |
| 1388 | }; | 1403 | }; |
| 1389 | 1404 | ||
| 1405 | static struct trace_event trace_blk_event = { | ||
| 1406 | .type = TRACE_BLK, | ||
| 1407 | .funcs = &trace_blk_event_funcs, | ||
| 1408 | }; | ||
| 1409 | |||
| 1390 | static int __init init_blk_tracer(void) | 1410 | static int __init init_blk_tracer(void) |
| 1391 | { | 1411 | { |
| 1392 | if (!register_ftrace_event(&trace_blk_event)) { | 1412 | if (!register_ftrace_event(&trace_blk_event)) { |
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 32837e19e3bd..6d2cb14f9449 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c | |||
| @@ -3234,7 +3234,8 @@ free: | |||
| 3234 | } | 3234 | } |
| 3235 | 3235 | ||
| 3236 | static void | 3236 | static void |
| 3237 | ftrace_graph_probe_sched_switch(struct task_struct *prev, struct task_struct *next) | 3237 | ftrace_graph_probe_sched_switch(void *ignore, |
| 3238 | struct task_struct *prev, struct task_struct *next) | ||
| 3238 | { | 3239 | { |
| 3239 | unsigned long long timestamp; | 3240 | unsigned long long timestamp; |
| 3240 | int index; | 3241 | int index; |
| @@ -3288,7 +3289,7 @@ static int start_graph_tracing(void) | |||
| 3288 | } while (ret == -EAGAIN); | 3289 | } while (ret == -EAGAIN); |
| 3289 | 3290 | ||
| 3290 | if (!ret) { | 3291 | if (!ret) { |
| 3291 | ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch); | 3292 | ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); |
| 3292 | if (ret) | 3293 | if (ret) |
| 3293 | pr_info("ftrace_graph: Couldn't activate tracepoint" | 3294 | pr_info("ftrace_graph: Couldn't activate tracepoint" |
| 3294 | " probe to kernel_sched_switch\n"); | 3295 | " probe to kernel_sched_switch\n"); |
| @@ -3364,7 +3365,7 @@ void unregister_ftrace_graph(void) | |||
| 3364 | ftrace_graph_entry = ftrace_graph_entry_stub; | 3365 | ftrace_graph_entry = ftrace_graph_entry_stub; |
| 3365 | ftrace_shutdown(FTRACE_STOP_FUNC_RET); | 3366 | ftrace_shutdown(FTRACE_STOP_FUNC_RET); |
| 3366 | unregister_pm_notifier(&ftrace_suspend_notifier); | 3367 | unregister_pm_notifier(&ftrace_suspend_notifier); |
| 3367 | unregister_trace_sched_switch(ftrace_graph_probe_sched_switch); | 3368 | unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); |
| 3368 | 3369 | ||
| 3369 | out: | 3370 | out: |
| 3370 | mutex_unlock(&ftrace_lock); | 3371 | mutex_unlock(&ftrace_lock); |
diff --git a/kernel/trace/kmemtrace.c b/kernel/trace/kmemtrace.c index a91da69f153a..bbfc1bb1660b 100644 --- a/kernel/trace/kmemtrace.c +++ b/kernel/trace/kmemtrace.c | |||
| @@ -95,7 +95,8 @@ static inline void kmemtrace_free(enum kmemtrace_type_id type_id, | |||
| 95 | trace_wake_up(); | 95 | trace_wake_up(); |
| 96 | } | 96 | } |
| 97 | 97 | ||
| 98 | static void kmemtrace_kmalloc(unsigned long call_site, | 98 | static void kmemtrace_kmalloc(void *ignore, |
| 99 | unsigned long call_site, | ||
| 99 | const void *ptr, | 100 | const void *ptr, |
| 100 | size_t bytes_req, | 101 | size_t bytes_req, |
| 101 | size_t bytes_alloc, | 102 | size_t bytes_alloc, |
| @@ -105,7 +106,8 @@ static void kmemtrace_kmalloc(unsigned long call_site, | |||
| 105 | bytes_req, bytes_alloc, gfp_flags, -1); | 106 | bytes_req, bytes_alloc, gfp_flags, -1); |
| 106 | } | 107 | } |
| 107 | 108 | ||
| 108 | static void kmemtrace_kmem_cache_alloc(unsigned long call_site, | 109 | static void kmemtrace_kmem_cache_alloc(void *ignore, |
| 110 | unsigned long call_site, | ||
| 109 | const void *ptr, | 111 | const void *ptr, |
| 110 | size_t bytes_req, | 112 | size_t bytes_req, |
| 111 | size_t bytes_alloc, | 113 | size_t bytes_alloc, |
| @@ -115,7 +117,8 @@ static void kmemtrace_kmem_cache_alloc(unsigned long call_site, | |||
| 115 | bytes_req, bytes_alloc, gfp_flags, -1); | 117 | bytes_req, bytes_alloc, gfp_flags, -1); |
| 116 | } | 118 | } |
| 117 | 119 | ||
| 118 | static void kmemtrace_kmalloc_node(unsigned long call_site, | 120 | static void kmemtrace_kmalloc_node(void *ignore, |
| 121 | unsigned long call_site, | ||
| 119 | const void *ptr, | 122 | const void *ptr, |
| 120 | size_t bytes_req, | 123 | size_t bytes_req, |
| 121 | size_t bytes_alloc, | 124 | size_t bytes_alloc, |
| @@ -126,7 +129,8 @@ static void kmemtrace_kmalloc_node(unsigned long call_site, | |||
| 126 | bytes_req, bytes_alloc, gfp_flags, node); | 129 | bytes_req, bytes_alloc, gfp_flags, node); |
| 127 | } | 130 | } |
| 128 | 131 | ||
| 129 | static void kmemtrace_kmem_cache_alloc_node(unsigned long call_site, | 132 | static void kmemtrace_kmem_cache_alloc_node(void *ignore, |
| 133 | unsigned long call_site, | ||
| 130 | const void *ptr, | 134 | const void *ptr, |
| 131 | size_t bytes_req, | 135 | size_t bytes_req, |
| 132 | size_t bytes_alloc, | 136 | size_t bytes_alloc, |
| @@ -137,12 +141,14 @@ static void kmemtrace_kmem_cache_alloc_node(unsigned long call_site, | |||
| 137 | bytes_req, bytes_alloc, gfp_flags, node); | 141 | bytes_req, bytes_alloc, gfp_flags, node); |
| 138 | } | 142 | } |
| 139 | 143 | ||
| 140 | static void kmemtrace_kfree(unsigned long call_site, const void *ptr) | 144 | static void |
| 145 | kmemtrace_kfree(void *ignore, unsigned long call_site, const void *ptr) | ||
| 141 | { | 146 | { |
| 142 | kmemtrace_free(KMEMTRACE_TYPE_KMALLOC, call_site, ptr); | 147 | kmemtrace_free(KMEMTRACE_TYPE_KMALLOC, call_site, ptr); |
| 143 | } | 148 | } |
| 144 | 149 | ||
| 145 | static void kmemtrace_kmem_cache_free(unsigned long call_site, const void *ptr) | 150 | static void kmemtrace_kmem_cache_free(void *ignore, |
| 151 | unsigned long call_site, const void *ptr) | ||
| 146 | { | 152 | { |
| 147 | kmemtrace_free(KMEMTRACE_TYPE_CACHE, call_site, ptr); | 153 | kmemtrace_free(KMEMTRACE_TYPE_CACHE, call_site, ptr); |
| 148 | } | 154 | } |
| @@ -151,34 +157,34 @@ static int kmemtrace_start_probes(void) | |||
| 151 | { | 157 | { |
| 152 | int err; | 158 | int err; |
| 153 | 159 | ||
| 154 | err = register_trace_kmalloc(kmemtrace_kmalloc); | 160 | err = register_trace_kmalloc(kmemtrace_kmalloc, NULL); |
| 155 | if (err) | 161 | if (err) |
| 156 | return err; | 162 | return err; |
| 157 | err = register_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc); | 163 | err = register_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc, NULL); |
| 158 | if (err) | 164 | if (err) |
| 159 | return err; | 165 | return err; |
| 160 | err = register_trace_kmalloc_node(kmemtrace_kmalloc_node); | 166 | err = register_trace_kmalloc_node(kmemtrace_kmalloc_node, NULL); |
| 161 | if (err) | 167 | if (err) |
| 162 | return err; | 168 | return err; |
| 163 | err = register_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node); | 169 | err = register_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node, NULL); |
| 164 | if (err) | 170 | if (err) |
| 165 | return err; | 171 | return err; |
| 166 | err = register_trace_kfree(kmemtrace_kfree); | 172 | err = register_trace_kfree(kmemtrace_kfree, NULL); |
| 167 | if (err) | 173 | if (err) |
| 168 | return err; | 174 | return err; |
| 169 | err = register_trace_kmem_cache_free(kmemtrace_kmem_cache_free); | 175 | err = register_trace_kmem_cache_free(kmemtrace_kmem_cache_free, NULL); |
| 170 | 176 | ||
| 171 | return err; | 177 | return err; |
| 172 | } | 178 | } |
| 173 | 179 | ||
| 174 | static void kmemtrace_stop_probes(void) | 180 | static void kmemtrace_stop_probes(void) |
| 175 | { | 181 | { |
| 176 | unregister_trace_kmalloc(kmemtrace_kmalloc); | 182 | unregister_trace_kmalloc(kmemtrace_kmalloc, NULL); |
| 177 | unregister_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc); | 183 | unregister_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc, NULL); |
| 178 | unregister_trace_kmalloc_node(kmemtrace_kmalloc_node); | 184 | unregister_trace_kmalloc_node(kmemtrace_kmalloc_node, NULL); |
| 179 | unregister_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node); | 185 | unregister_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node, NULL); |
| 180 | unregister_trace_kfree(kmemtrace_kfree); | 186 | unregister_trace_kfree(kmemtrace_kfree, NULL); |
| 181 | unregister_trace_kmem_cache_free(kmemtrace_kmem_cache_free); | 187 | unregister_trace_kmem_cache_free(kmemtrace_kmem_cache_free, NULL); |
| 182 | } | 188 | } |
| 183 | 189 | ||
| 184 | static int kmem_trace_init(struct trace_array *tr) | 190 | static int kmem_trace_init(struct trace_array *tr) |
| @@ -237,7 +243,8 @@ struct kmemtrace_user_event_alloc { | |||
| 237 | }; | 243 | }; |
| 238 | 244 | ||
| 239 | static enum print_line_t | 245 | static enum print_line_t |
| 240 | kmemtrace_print_alloc(struct trace_iterator *iter, int flags) | 246 | kmemtrace_print_alloc(struct trace_iterator *iter, int flags, |
| 247 | struct trace_event *event) | ||
| 241 | { | 248 | { |
| 242 | struct trace_seq *s = &iter->seq; | 249 | struct trace_seq *s = &iter->seq; |
| 243 | struct kmemtrace_alloc_entry *entry; | 250 | struct kmemtrace_alloc_entry *entry; |
| @@ -257,7 +264,8 @@ kmemtrace_print_alloc(struct trace_iterator *iter, int flags) | |||
| 257 | } | 264 | } |
| 258 | 265 | ||
| 259 | static enum print_line_t | 266 | static enum print_line_t |
| 260 | kmemtrace_print_free(struct trace_iterator *iter, int flags) | 267 | kmemtrace_print_free(struct trace_iterator *iter, int flags, |
| 268 | struct trace_event *event) | ||
| 261 | { | 269 | { |
| 262 | struct trace_seq *s = &iter->seq; | 270 | struct trace_seq *s = &iter->seq; |
| 263 | struct kmemtrace_free_entry *entry; | 271 | struct kmemtrace_free_entry *entry; |
| @@ -275,7 +283,8 @@ kmemtrace_print_free(struct trace_iterator *iter, int flags) | |||
| 275 | } | 283 | } |
| 276 | 284 | ||
| 277 | static enum print_line_t | 285 | static enum print_line_t |
| 278 | kmemtrace_print_alloc_user(struct trace_iterator *iter, int flags) | 286 | kmemtrace_print_alloc_user(struct trace_iterator *iter, int flags, |
| 287 | struct trace_event *event) | ||
| 279 | { | 288 | { |
| 280 | struct trace_seq *s = &iter->seq; | 289 | struct trace_seq *s = &iter->seq; |
| 281 | struct kmemtrace_alloc_entry *entry; | 290 | struct kmemtrace_alloc_entry *entry; |
| @@ -309,7 +318,8 @@ kmemtrace_print_alloc_user(struct trace_iterator *iter, int flags) | |||
| 309 | } | 318 | } |
| 310 | 319 | ||
| 311 | static enum print_line_t | 320 | static enum print_line_t |
| 312 | kmemtrace_print_free_user(struct trace_iterator *iter, int flags) | 321 | kmemtrace_print_free_user(struct trace_iterator *iter, int flags, |
| 322 | struct trace_event *event) | ||
| 313 | { | 323 | { |
| 314 | struct trace_seq *s = &iter->seq; | 324 | struct trace_seq *s = &iter->seq; |
| 315 | struct kmemtrace_free_entry *entry; | 325 | struct kmemtrace_free_entry *entry; |
| @@ -463,18 +473,26 @@ static enum print_line_t kmemtrace_print_line(struct trace_iterator *iter) | |||
| 463 | } | 473 | } |
| 464 | } | 474 | } |
| 465 | 475 | ||
| 466 | static struct trace_event kmem_trace_alloc = { | 476 | static struct trace_event_functions kmem_trace_alloc_funcs = { |
| 467 | .type = TRACE_KMEM_ALLOC, | ||
| 468 | .trace = kmemtrace_print_alloc, | 477 | .trace = kmemtrace_print_alloc, |
| 469 | .binary = kmemtrace_print_alloc_user, | 478 | .binary = kmemtrace_print_alloc_user, |
| 470 | }; | 479 | }; |
| 471 | 480 | ||
| 472 | static struct trace_event kmem_trace_free = { | 481 | static struct trace_event kmem_trace_alloc = { |
| 473 | .type = TRACE_KMEM_FREE, | 482 | .type = TRACE_KMEM_ALLOC, |
| 483 | .funcs = &kmem_trace_alloc_funcs, | ||
| 484 | }; | ||
| 485 | |||
| 486 | static struct trace_event_functions kmem_trace_free_funcs = { | ||
| 474 | .trace = kmemtrace_print_free, | 487 | .trace = kmemtrace_print_free, |
| 475 | .binary = kmemtrace_print_free_user, | 488 | .binary = kmemtrace_print_free_user, |
| 476 | }; | 489 | }; |
| 477 | 490 | ||
| 491 | static struct trace_event kmem_trace_free = { | ||
| 492 | .type = TRACE_KMEM_FREE, | ||
| 493 | .funcs = &kmem_trace_free_funcs, | ||
| 494 | }; | ||
| 495 | |||
| 478 | static struct tracer kmem_tracer __read_mostly = { | 496 | static struct tracer kmem_tracer __read_mostly = { |
| 479 | .name = "kmemtrace", | 497 | .name = "kmemtrace", |
| 480 | .init = kmem_trace_init, | 498 | .init = kmem_trace_init, |
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 8a76339a9e65..55e48511d7c8 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c | |||
| @@ -1936,7 +1936,7 @@ static enum print_line_t print_trace_fmt(struct trace_iterator *iter) | |||
| 1936 | } | 1936 | } |
| 1937 | 1937 | ||
| 1938 | if (event) | 1938 | if (event) |
| 1939 | return event->trace(iter, sym_flags); | 1939 | return event->funcs->trace(iter, sym_flags, event); |
| 1940 | 1940 | ||
| 1941 | if (!trace_seq_printf(s, "Unknown type %d\n", entry->type)) | 1941 | if (!trace_seq_printf(s, "Unknown type %d\n", entry->type)) |
| 1942 | goto partial; | 1942 | goto partial; |
| @@ -1962,7 +1962,7 @@ static enum print_line_t print_raw_fmt(struct trace_iterator *iter) | |||
| 1962 | 1962 | ||
| 1963 | event = ftrace_find_event(entry->type); | 1963 | event = ftrace_find_event(entry->type); |
| 1964 | if (event) | 1964 | if (event) |
| 1965 | return event->raw(iter, 0); | 1965 | return event->funcs->raw(iter, 0, event); |
| 1966 | 1966 | ||
| 1967 | if (!trace_seq_printf(s, "%d ?\n", entry->type)) | 1967 | if (!trace_seq_printf(s, "%d ?\n", entry->type)) |
| 1968 | goto partial; | 1968 | goto partial; |
| @@ -1989,7 +1989,7 @@ static enum print_line_t print_hex_fmt(struct trace_iterator *iter) | |||
| 1989 | 1989 | ||
| 1990 | event = ftrace_find_event(entry->type); | 1990 | event = ftrace_find_event(entry->type); |
| 1991 | if (event) { | 1991 | if (event) { |
| 1992 | enum print_line_t ret = event->hex(iter, 0); | 1992 | enum print_line_t ret = event->funcs->hex(iter, 0, event); |
| 1993 | if (ret != TRACE_TYPE_HANDLED) | 1993 | if (ret != TRACE_TYPE_HANDLED) |
| 1994 | return ret; | 1994 | return ret; |
| 1995 | } | 1995 | } |
| @@ -2014,7 +2014,8 @@ static enum print_line_t print_bin_fmt(struct trace_iterator *iter) | |||
| 2014 | } | 2014 | } |
| 2015 | 2015 | ||
| 2016 | event = ftrace_find_event(entry->type); | 2016 | event = ftrace_find_event(entry->type); |
| 2017 | return event ? event->binary(iter, 0) : TRACE_TYPE_HANDLED; | 2017 | return event ? event->funcs->binary(iter, 0, event) : |
| 2018 | TRACE_TYPE_HANDLED; | ||
| 2018 | } | 2019 | } |
| 2019 | 2020 | ||
| 2020 | int trace_empty(struct trace_iterator *iter) | 2021 | int trace_empty(struct trace_iterator *iter) |
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index d1ce0bec1b3f..2cd96399463f 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h | |||
| @@ -405,12 +405,12 @@ void ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, | |||
| 405 | void __trace_stack(struct trace_array *tr, unsigned long flags, int skip, | 405 | void __trace_stack(struct trace_array *tr, unsigned long flags, int skip, |
| 406 | int pc); | 406 | int pc); |
| 407 | #else | 407 | #else |
| 408 | static inline void ftrace_trace_stack(struct trace_array *tr, | 408 | static inline void ftrace_trace_stack(struct ring_buffer *buffer, |
| 409 | unsigned long flags, int skip, int pc) | 409 | unsigned long flags, int skip, int pc) |
| 410 | { | 410 | { |
| 411 | } | 411 | } |
| 412 | 412 | ||
| 413 | static inline void ftrace_trace_userstack(struct trace_array *tr, | 413 | static inline void ftrace_trace_userstack(struct ring_buffer *buffer, |
| 414 | unsigned long flags, int pc) | 414 | unsigned long flags, int pc) |
| 415 | { | 415 | { |
| 416 | } | 416 | } |
| @@ -778,12 +778,15 @@ extern void print_subsystem_event_filter(struct event_subsystem *system, | |||
| 778 | struct trace_seq *s); | 778 | struct trace_seq *s); |
| 779 | extern int filter_assign_type(const char *type); | 779 | extern int filter_assign_type(const char *type); |
| 780 | 780 | ||
| 781 | struct list_head * | ||
| 782 | trace_get_fields(struct ftrace_event_call *event_call); | ||
| 783 | |||
| 781 | static inline int | 784 | static inline int |
| 782 | filter_check_discard(struct ftrace_event_call *call, void *rec, | 785 | filter_check_discard(struct ftrace_event_call *call, void *rec, |
| 783 | struct ring_buffer *buffer, | 786 | struct ring_buffer *buffer, |
| 784 | struct ring_buffer_event *event) | 787 | struct ring_buffer_event *event) |
| 785 | { | 788 | { |
| 786 | if (unlikely(call->filter_active) && | 789 | if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) && |
| 787 | !filter_match_preds(call->filter, rec)) { | 790 | !filter_match_preds(call->filter, rec)) { |
| 788 | ring_buffer_discard_commit(buffer, event); | 791 | ring_buffer_discard_commit(buffer, event); |
| 789 | return 1; | 792 | return 1; |
diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c index b9bc4d470177..8d3538b4ea5f 100644 --- a/kernel/trace/trace_branch.c +++ b/kernel/trace/trace_branch.c | |||
| @@ -143,7 +143,7 @@ static void branch_trace_reset(struct trace_array *tr) | |||
| 143 | } | 143 | } |
| 144 | 144 | ||
| 145 | static enum print_line_t trace_branch_print(struct trace_iterator *iter, | 145 | static enum print_line_t trace_branch_print(struct trace_iterator *iter, |
| 146 | int flags) | 146 | int flags, struct trace_event *event) |
| 147 | { | 147 | { |
| 148 | struct trace_branch *field; | 148 | struct trace_branch *field; |
| 149 | 149 | ||
| @@ -167,9 +167,13 @@ static void branch_print_header(struct seq_file *s) | |||
| 167 | " |\n"); | 167 | " |\n"); |
| 168 | } | 168 | } |
| 169 | 169 | ||
| 170 | static struct trace_event_functions trace_branch_funcs = { | ||
| 171 | .trace = trace_branch_print, | ||
| 172 | }; | ||
| 173 | |||
| 170 | static struct trace_event trace_branch_event = { | 174 | static struct trace_event trace_branch_event = { |
| 171 | .type = TRACE_BRANCH, | 175 | .type = TRACE_BRANCH, |
| 172 | .trace = trace_branch_print, | 176 | .funcs = &trace_branch_funcs, |
| 173 | }; | 177 | }; |
| 174 | 178 | ||
| 175 | static struct tracer branch_trace __read_mostly = | 179 | static struct tracer branch_trace __read_mostly = |
diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c index 0565bb42566f..cb6f365016e4 100644 --- a/kernel/trace/trace_event_perf.c +++ b/kernel/trace/trace_event_perf.c | |||
| @@ -9,13 +9,9 @@ | |||
| 9 | #include <linux/kprobes.h> | 9 | #include <linux/kprobes.h> |
| 10 | #include "trace.h" | 10 | #include "trace.h" |
| 11 | 11 | ||
| 12 | DEFINE_PER_CPU(struct pt_regs, perf_trace_regs); | ||
| 13 | EXPORT_PER_CPU_SYMBOL_GPL(perf_trace_regs); | ||
| 14 | |||
| 15 | EXPORT_SYMBOL_GPL(perf_arch_fetch_caller_regs); | 12 | EXPORT_SYMBOL_GPL(perf_arch_fetch_caller_regs); |
| 16 | 13 | ||
| 17 | static char *perf_trace_buf; | 14 | static char *perf_trace_buf[4]; |
| 18 | static char *perf_trace_buf_nmi; | ||
| 19 | 15 | ||
| 20 | /* | 16 | /* |
| 21 | * Force it to be aligned to unsigned long to avoid misaligned accesses | 17 | * Force it to be aligned to unsigned long to avoid misaligned accesses |
| @@ -27,57 +23,82 @@ typedef typeof(unsigned long [PERF_MAX_TRACE_SIZE / sizeof(unsigned long)]) | |||
| 27 | /* Count the events in use (per event id, not per instance) */ | 23 | /* Count the events in use (per event id, not per instance) */ |
| 28 | static int total_ref_count; | 24 | static int total_ref_count; |
| 29 | 25 | ||
| 30 | static int perf_trace_event_enable(struct ftrace_event_call *event) | 26 | static int perf_trace_event_init(struct ftrace_event_call *tp_event, |
| 27 | struct perf_event *p_event) | ||
| 31 | { | 28 | { |
| 32 | char *buf; | 29 | struct hlist_head *list; |
| 33 | int ret = -ENOMEM; | 30 | int ret = -ENOMEM; |
| 31 | int cpu; | ||
| 34 | 32 | ||
| 35 | if (event->perf_refcount++ > 0) | 33 | p_event->tp_event = tp_event; |
| 34 | if (tp_event->perf_refcount++ > 0) | ||
| 36 | return 0; | 35 | return 0; |
| 37 | 36 | ||
| 38 | if (!total_ref_count) { | 37 | list = alloc_percpu(struct hlist_head); |
| 39 | buf = (char *)alloc_percpu(perf_trace_t); | 38 | if (!list) |
| 40 | if (!buf) | 39 | goto fail; |
| 41 | goto fail_buf; | ||
| 42 | 40 | ||
| 43 | rcu_assign_pointer(perf_trace_buf, buf); | 41 | for_each_possible_cpu(cpu) |
| 42 | INIT_HLIST_HEAD(per_cpu_ptr(list, cpu)); | ||
| 44 | 43 | ||
| 45 | buf = (char *)alloc_percpu(perf_trace_t); | 44 | tp_event->perf_events = list; |
| 46 | if (!buf) | ||
| 47 | goto fail_buf_nmi; | ||
| 48 | 45 | ||
| 49 | rcu_assign_pointer(perf_trace_buf_nmi, buf); | 46 | if (!total_ref_count) { |
| 50 | } | 47 | char *buf; |
| 48 | int i; | ||
| 51 | 49 | ||
| 52 | ret = event->perf_event_enable(event); | 50 | for (i = 0; i < 4; i++) { |
| 53 | if (!ret) { | 51 | buf = (char *)alloc_percpu(perf_trace_t); |
| 54 | total_ref_count++; | 52 | if (!buf) |
| 55 | return 0; | 53 | goto fail; |
| 54 | |||
| 55 | perf_trace_buf[i] = buf; | ||
| 56 | } | ||
| 56 | } | 57 | } |
| 57 | 58 | ||
| 58 | fail_buf_nmi: | 59 | if (tp_event->class->reg) |
| 60 | ret = tp_event->class->reg(tp_event, TRACE_REG_PERF_REGISTER); | ||
| 61 | else | ||
| 62 | ret = tracepoint_probe_register(tp_event->name, | ||
| 63 | tp_event->class->perf_probe, | ||
| 64 | tp_event); | ||
| 65 | |||
| 66 | if (ret) | ||
| 67 | goto fail; | ||
| 68 | |||
| 69 | total_ref_count++; | ||
| 70 | return 0; | ||
| 71 | |||
| 72 | fail: | ||
| 59 | if (!total_ref_count) { | 73 | if (!total_ref_count) { |
| 60 | free_percpu(perf_trace_buf_nmi); | 74 | int i; |
| 61 | free_percpu(perf_trace_buf); | 75 | |
| 62 | perf_trace_buf_nmi = NULL; | 76 | for (i = 0; i < 4; i++) { |
| 63 | perf_trace_buf = NULL; | 77 | free_percpu(perf_trace_buf[i]); |
| 78 | perf_trace_buf[i] = NULL; | ||
| 79 | } | ||
| 80 | } | ||
| 81 | |||
| 82 | if (!--tp_event->perf_refcount) { | ||
| 83 | free_percpu(tp_event->perf_events); | ||
| 84 | tp_event->perf_events = NULL; | ||
| 64 | } | 85 | } |
| 65 | fail_buf: | ||
| 66 | event->perf_refcount--; | ||
| 67 | 86 | ||
| 68 | return ret; | 87 | return ret; |
| 69 | } | 88 | } |
| 70 | 89 | ||
| 71 | int perf_trace_enable(int event_id) | 90 | int perf_trace_init(struct perf_event *p_event) |
| 72 | { | 91 | { |
| 73 | struct ftrace_event_call *event; | 92 | struct ftrace_event_call *tp_event; |
| 93 | int event_id = p_event->attr.config; | ||
| 74 | int ret = -EINVAL; | 94 | int ret = -EINVAL; |
| 75 | 95 | ||
| 76 | mutex_lock(&event_mutex); | 96 | mutex_lock(&event_mutex); |
| 77 | list_for_each_entry(event, &ftrace_events, list) { | 97 | list_for_each_entry(tp_event, &ftrace_events, list) { |
| 78 | if (event->id == event_id && event->perf_event_enable && | 98 | if (tp_event->event.type == event_id && |
| 79 | try_module_get(event->mod)) { | 99 | tp_event->class && tp_event->class->perf_probe && |
| 80 | ret = perf_trace_event_enable(event); | 100 | try_module_get(tp_event->mod)) { |
| 101 | ret = perf_trace_event_init(tp_event, p_event); | ||
| 81 | break; | 102 | break; |
| 82 | } | 103 | } |
| 83 | } | 104 | } |
| @@ -86,90 +107,78 @@ int perf_trace_enable(int event_id) | |||
| 86 | return ret; | 107 | return ret; |
| 87 | } | 108 | } |
| 88 | 109 | ||
| 89 | static void perf_trace_event_disable(struct ftrace_event_call *event) | 110 | int perf_trace_enable(struct perf_event *p_event) |
| 90 | { | 111 | { |
| 91 | char *buf, *nmi_buf; | 112 | struct ftrace_event_call *tp_event = p_event->tp_event; |
| 92 | 113 | struct hlist_head *list; | |
| 93 | if (--event->perf_refcount > 0) | ||
| 94 | return; | ||
| 95 | |||
| 96 | event->perf_event_disable(event); | ||
| 97 | 114 | ||
| 98 | if (!--total_ref_count) { | 115 | list = tp_event->perf_events; |
| 99 | buf = perf_trace_buf; | 116 | if (WARN_ON_ONCE(!list)) |
| 100 | rcu_assign_pointer(perf_trace_buf, NULL); | 117 | return -EINVAL; |
| 101 | 118 | ||
| 102 | nmi_buf = perf_trace_buf_nmi; | 119 | list = per_cpu_ptr(list, smp_processor_id()); |
| 103 | rcu_assign_pointer(perf_trace_buf_nmi, NULL); | 120 | hlist_add_head_rcu(&p_event->hlist_entry, list); |
| 104 | 121 | ||
| 105 | /* | 122 | return 0; |
| 106 | * Ensure every events in profiling have finished before | 123 | } |
| 107 | * releasing the buffers | ||
| 108 | */ | ||
| 109 | synchronize_sched(); | ||
| 110 | 124 | ||
| 111 | free_percpu(buf); | 125 | void perf_trace_disable(struct perf_event *p_event) |
| 112 | free_percpu(nmi_buf); | 126 | { |
| 113 | } | 127 | hlist_del_rcu(&p_event->hlist_entry); |
| 114 | } | 128 | } |
| 115 | 129 | ||
| 116 | void perf_trace_disable(int event_id) | 130 | void perf_trace_destroy(struct perf_event *p_event) |
| 117 | { | 131 | { |
| 118 | struct ftrace_event_call *event; | 132 | struct ftrace_event_call *tp_event = p_event->tp_event; |
| 133 | int i; | ||
| 119 | 134 | ||
| 120 | mutex_lock(&event_mutex); | 135 | if (--tp_event->perf_refcount > 0) |
| 121 | list_for_each_entry(event, &ftrace_events, list) { | 136 | return; |
| 122 | if (event->id == event_id) { | 137 | |
| 123 | perf_trace_event_disable(event); | 138 | if (tp_event->class->reg) |
| 124 | module_put(event->mod); | 139 | tp_event->class->reg(tp_event, TRACE_REG_PERF_UNREGISTER); |
| 125 | break; | 140 | else |
| 141 | tracepoint_probe_unregister(tp_event->name, | ||
| 142 | tp_event->class->perf_probe, | ||
| 143 | tp_event); | ||
| 144 | |||
| 145 | free_percpu(tp_event->perf_events); | ||
| 146 | tp_event->perf_events = NULL; | ||
| 147 | |||
| 148 | if (!--total_ref_count) { | ||
| 149 | for (i = 0; i < 4; i++) { | ||
| 150 | free_percpu(perf_trace_buf[i]); | ||
| 151 | perf_trace_buf[i] = NULL; | ||
| 126 | } | 152 | } |
| 127 | } | 153 | } |
| 128 | mutex_unlock(&event_mutex); | ||
| 129 | } | 154 | } |
| 130 | 155 | ||
| 131 | __kprobes void *perf_trace_buf_prepare(int size, unsigned short type, | 156 | __kprobes void *perf_trace_buf_prepare(int size, unsigned short type, |
| 132 | int *rctxp, unsigned long *irq_flags) | 157 | struct pt_regs *regs, int *rctxp) |
| 133 | { | 158 | { |
| 134 | struct trace_entry *entry; | 159 | struct trace_entry *entry; |
| 135 | char *trace_buf, *raw_data; | 160 | unsigned long flags; |
| 136 | int pc, cpu; | 161 | char *raw_data; |
| 162 | int pc; | ||
| 137 | 163 | ||
| 138 | BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long)); | 164 | BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long)); |
| 139 | 165 | ||
| 140 | pc = preempt_count(); | 166 | pc = preempt_count(); |
| 141 | 167 | ||
| 142 | /* Protect the per cpu buffer, begin the rcu read side */ | ||
| 143 | local_irq_save(*irq_flags); | ||
| 144 | |||
| 145 | *rctxp = perf_swevent_get_recursion_context(); | 168 | *rctxp = perf_swevent_get_recursion_context(); |
| 146 | if (*rctxp < 0) | 169 | if (*rctxp < 0) |
| 147 | goto err_recursion; | 170 | return NULL; |
| 148 | |||
| 149 | cpu = smp_processor_id(); | ||
| 150 | |||
| 151 | if (in_nmi()) | ||
| 152 | trace_buf = rcu_dereference_sched(perf_trace_buf_nmi); | ||
| 153 | else | ||
| 154 | trace_buf = rcu_dereference_sched(perf_trace_buf); | ||
| 155 | |||
| 156 | if (!trace_buf) | ||
| 157 | goto err; | ||
| 158 | 171 | ||
| 159 | raw_data = per_cpu_ptr(trace_buf, cpu); | 172 | raw_data = per_cpu_ptr(perf_trace_buf[*rctxp], smp_processor_id()); |
| 160 | 173 | ||
| 161 | /* zero the dead bytes from align to not leak stack to user */ | 174 | /* zero the dead bytes from align to not leak stack to user */ |
| 162 | memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64)); | 175 | memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64)); |
| 163 | 176 | ||
| 164 | entry = (struct trace_entry *)raw_data; | 177 | entry = (struct trace_entry *)raw_data; |
| 165 | tracing_generic_entry_update(entry, *irq_flags, pc); | 178 | local_save_flags(flags); |
| 179 | tracing_generic_entry_update(entry, flags, pc); | ||
| 166 | entry->type = type; | 180 | entry->type = type; |
| 167 | 181 | ||
| 168 | return raw_data; | 182 | return raw_data; |
| 169 | err: | ||
| 170 | perf_swevent_put_recursion_context(*rctxp); | ||
| 171 | err_recursion: | ||
| 172 | local_irq_restore(*irq_flags); | ||
| 173 | return NULL; | ||
| 174 | } | 183 | } |
| 175 | EXPORT_SYMBOL_GPL(perf_trace_buf_prepare); | 184 | EXPORT_SYMBOL_GPL(perf_trace_buf_prepare); |
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index c697c7043349..53cffc0b0801 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c | |||
| @@ -29,11 +29,23 @@ DEFINE_MUTEX(event_mutex); | |||
| 29 | 29 | ||
| 30 | LIST_HEAD(ftrace_events); | 30 | LIST_HEAD(ftrace_events); |
| 31 | 31 | ||
| 32 | struct list_head * | ||
| 33 | trace_get_fields(struct ftrace_event_call *event_call) | ||
| 34 | { | ||
| 35 | if (!event_call->class->get_fields) | ||
| 36 | return &event_call->class->fields; | ||
| 37 | return event_call->class->get_fields(event_call); | ||
| 38 | } | ||
| 39 | |||
| 32 | int trace_define_field(struct ftrace_event_call *call, const char *type, | 40 | int trace_define_field(struct ftrace_event_call *call, const char *type, |
| 33 | const char *name, int offset, int size, int is_signed, | 41 | const char *name, int offset, int size, int is_signed, |
| 34 | int filter_type) | 42 | int filter_type) |
| 35 | { | 43 | { |
| 36 | struct ftrace_event_field *field; | 44 | struct ftrace_event_field *field; |
| 45 | struct list_head *head; | ||
| 46 | |||
| 47 | if (WARN_ON(!call->class)) | ||
| 48 | return 0; | ||
| 37 | 49 | ||
| 38 | field = kzalloc(sizeof(*field), GFP_KERNEL); | 50 | field = kzalloc(sizeof(*field), GFP_KERNEL); |
| 39 | if (!field) | 51 | if (!field) |
| @@ -56,7 +68,8 @@ int trace_define_field(struct ftrace_event_call *call, const char *type, | |||
| 56 | field->size = size; | 68 | field->size = size; |
| 57 | field->is_signed = is_signed; | 69 | field->is_signed = is_signed; |
| 58 | 70 | ||
| 59 | list_add(&field->link, &call->fields); | 71 | head = trace_get_fields(call); |
| 72 | list_add(&field->link, head); | ||
| 60 | 73 | ||
| 61 | return 0; | 74 | return 0; |
| 62 | 75 | ||
| @@ -94,8 +107,10 @@ static int trace_define_common_fields(struct ftrace_event_call *call) | |||
| 94 | void trace_destroy_fields(struct ftrace_event_call *call) | 107 | void trace_destroy_fields(struct ftrace_event_call *call) |
| 95 | { | 108 | { |
| 96 | struct ftrace_event_field *field, *next; | 109 | struct ftrace_event_field *field, *next; |
| 110 | struct list_head *head; | ||
| 97 | 111 | ||
| 98 | list_for_each_entry_safe(field, next, &call->fields, link) { | 112 | head = trace_get_fields(call); |
| 113 | list_for_each_entry_safe(field, next, head, link) { | ||
| 99 | list_del(&field->link); | 114 | list_del(&field->link); |
| 100 | kfree(field->type); | 115 | kfree(field->type); |
| 101 | kfree(field->name); | 116 | kfree(field->name); |
| @@ -107,11 +122,9 @@ int trace_event_raw_init(struct ftrace_event_call *call) | |||
| 107 | { | 122 | { |
| 108 | int id; | 123 | int id; |
| 109 | 124 | ||
| 110 | id = register_ftrace_event(call->event); | 125 | id = register_ftrace_event(&call->event); |
| 111 | if (!id) | 126 | if (!id) |
| 112 | return -ENODEV; | 127 | return -ENODEV; |
| 113 | call->id = id; | ||
| 114 | INIT_LIST_HEAD(&call->fields); | ||
| 115 | 128 | ||
| 116 | return 0; | 129 | return 0; |
| 117 | } | 130 | } |
| @@ -124,23 +137,33 @@ static int ftrace_event_enable_disable(struct ftrace_event_call *call, | |||
| 124 | 137 | ||
| 125 | switch (enable) { | 138 | switch (enable) { |
| 126 | case 0: | 139 | case 0: |
| 127 | if (call->enabled) { | 140 | if (call->flags & TRACE_EVENT_FL_ENABLED) { |
| 128 | call->enabled = 0; | 141 | call->flags &= ~TRACE_EVENT_FL_ENABLED; |
| 129 | tracing_stop_cmdline_record(); | 142 | tracing_stop_cmdline_record(); |
| 130 | call->unregfunc(call); | 143 | if (call->class->reg) |
| 144 | call->class->reg(call, TRACE_REG_UNREGISTER); | ||
| 145 | else | ||
| 146 | tracepoint_probe_unregister(call->name, | ||
| 147 | call->class->probe, | ||
| 148 | call); | ||
| 131 | } | 149 | } |
| 132 | break; | 150 | break; |
| 133 | case 1: | 151 | case 1: |
| 134 | if (!call->enabled) { | 152 | if (!(call->flags & TRACE_EVENT_FL_ENABLED)) { |
| 135 | tracing_start_cmdline_record(); | 153 | tracing_start_cmdline_record(); |
| 136 | ret = call->regfunc(call); | 154 | if (call->class->reg) |
| 155 | ret = call->class->reg(call, TRACE_REG_REGISTER); | ||
| 156 | else | ||
| 157 | ret = tracepoint_probe_register(call->name, | ||
| 158 | call->class->probe, | ||
| 159 | call); | ||
| 137 | if (ret) { | 160 | if (ret) { |
| 138 | tracing_stop_cmdline_record(); | 161 | tracing_stop_cmdline_record(); |
| 139 | pr_info("event trace: Could not enable event " | 162 | pr_info("event trace: Could not enable event " |
| 140 | "%s\n", call->name); | 163 | "%s\n", call->name); |
| 141 | break; | 164 | break; |
| 142 | } | 165 | } |
| 143 | call->enabled = 1; | 166 | call->flags |= TRACE_EVENT_FL_ENABLED; |
| 144 | } | 167 | } |
| 145 | break; | 168 | break; |
| 146 | } | 169 | } |
| @@ -171,15 +194,16 @@ static int __ftrace_set_clr_event(const char *match, const char *sub, | |||
| 171 | mutex_lock(&event_mutex); | 194 | mutex_lock(&event_mutex); |
| 172 | list_for_each_entry(call, &ftrace_events, list) { | 195 | list_for_each_entry(call, &ftrace_events, list) { |
| 173 | 196 | ||
| 174 | if (!call->name || !call->regfunc) | 197 | if (!call->name || !call->class || |
| 198 | (!call->class->probe && !call->class->reg)) | ||
| 175 | continue; | 199 | continue; |
| 176 | 200 | ||
| 177 | if (match && | 201 | if (match && |
| 178 | strcmp(match, call->name) != 0 && | 202 | strcmp(match, call->name) != 0 && |
| 179 | strcmp(match, call->system) != 0) | 203 | strcmp(match, call->class->system) != 0) |
| 180 | continue; | 204 | continue; |
| 181 | 205 | ||
| 182 | if (sub && strcmp(sub, call->system) != 0) | 206 | if (sub && strcmp(sub, call->class->system) != 0) |
| 183 | continue; | 207 | continue; |
| 184 | 208 | ||
| 185 | if (event && strcmp(event, call->name) != 0) | 209 | if (event && strcmp(event, call->name) != 0) |
| @@ -297,7 +321,7 @@ t_next(struct seq_file *m, void *v, loff_t *pos) | |||
| 297 | * The ftrace subsystem is for showing formats only. | 321 | * The ftrace subsystem is for showing formats only. |
| 298 | * They can not be enabled or disabled via the event files. | 322 | * They can not be enabled or disabled via the event files. |
| 299 | */ | 323 | */ |
| 300 | if (call->regfunc) | 324 | if (call->class && (call->class->probe || call->class->reg)) |
| 301 | return call; | 325 | return call; |
| 302 | } | 326 | } |
| 303 | 327 | ||
| @@ -328,7 +352,7 @@ s_next(struct seq_file *m, void *v, loff_t *pos) | |||
| 328 | (*pos)++; | 352 | (*pos)++; |
| 329 | 353 | ||
| 330 | list_for_each_entry_continue(call, &ftrace_events, list) { | 354 | list_for_each_entry_continue(call, &ftrace_events, list) { |
| 331 | if (call->enabled) | 355 | if (call->flags & TRACE_EVENT_FL_ENABLED) |
| 332 | return call; | 356 | return call; |
| 333 | } | 357 | } |
| 334 | 358 | ||
| @@ -355,8 +379,8 @@ static int t_show(struct seq_file *m, void *v) | |||
| 355 | { | 379 | { |
| 356 | struct ftrace_event_call *call = v; | 380 | struct ftrace_event_call *call = v; |
| 357 | 381 | ||
| 358 | if (strcmp(call->system, TRACE_SYSTEM) != 0) | 382 | if (strcmp(call->class->system, TRACE_SYSTEM) != 0) |
| 359 | seq_printf(m, "%s:", call->system); | 383 | seq_printf(m, "%s:", call->class->system); |
| 360 | seq_printf(m, "%s\n", call->name); | 384 | seq_printf(m, "%s\n", call->name); |
| 361 | 385 | ||
| 362 | return 0; | 386 | return 0; |
| @@ -387,7 +411,7 @@ event_enable_read(struct file *filp, char __user *ubuf, size_t cnt, | |||
| 387 | struct ftrace_event_call *call = filp->private_data; | 411 | struct ftrace_event_call *call = filp->private_data; |
| 388 | char *buf; | 412 | char *buf; |
| 389 | 413 | ||
| 390 | if (call->enabled) | 414 | if (call->flags & TRACE_EVENT_FL_ENABLED) |
| 391 | buf = "1\n"; | 415 | buf = "1\n"; |
| 392 | else | 416 | else |
| 393 | buf = "0\n"; | 417 | buf = "0\n"; |
| @@ -450,10 +474,11 @@ system_enable_read(struct file *filp, char __user *ubuf, size_t cnt, | |||
| 450 | 474 | ||
| 451 | mutex_lock(&event_mutex); | 475 | mutex_lock(&event_mutex); |
| 452 | list_for_each_entry(call, &ftrace_events, list) { | 476 | list_for_each_entry(call, &ftrace_events, list) { |
| 453 | if (!call->name || !call->regfunc) | 477 | if (!call->name || !call->class || |
| 478 | (!call->class->probe && !call->class->reg)) | ||
| 454 | continue; | 479 | continue; |
| 455 | 480 | ||
| 456 | if (system && strcmp(call->system, system) != 0) | 481 | if (system && strcmp(call->class->system, system) != 0) |
| 457 | continue; | 482 | continue; |
| 458 | 483 | ||
| 459 | /* | 484 | /* |
| @@ -461,7 +486,7 @@ system_enable_read(struct file *filp, char __user *ubuf, size_t cnt, | |||
| 461 | * or if all events or cleared, or if we have | 486 | * or if all events or cleared, or if we have |
| 462 | * a mixture. | 487 | * a mixture. |
| 463 | */ | 488 | */ |
| 464 | set |= (1 << !!call->enabled); | 489 | set |= (1 << !!(call->flags & TRACE_EVENT_FL_ENABLED)); |
| 465 | 490 | ||
| 466 | /* | 491 | /* |
| 467 | * If we have a mixture, no need to look further. | 492 | * If we have a mixture, no need to look further. |
| @@ -525,6 +550,7 @@ event_format_read(struct file *filp, char __user *ubuf, size_t cnt, | |||
| 525 | { | 550 | { |
| 526 | struct ftrace_event_call *call = filp->private_data; | 551 | struct ftrace_event_call *call = filp->private_data; |
| 527 | struct ftrace_event_field *field; | 552 | struct ftrace_event_field *field; |
| 553 | struct list_head *head; | ||
| 528 | struct trace_seq *s; | 554 | struct trace_seq *s; |
| 529 | int common_field_count = 5; | 555 | int common_field_count = 5; |
| 530 | char *buf; | 556 | char *buf; |
| @@ -540,10 +566,11 @@ event_format_read(struct file *filp, char __user *ubuf, size_t cnt, | |||
| 540 | trace_seq_init(s); | 566 | trace_seq_init(s); |
| 541 | 567 | ||
| 542 | trace_seq_printf(s, "name: %s\n", call->name); | 568 | trace_seq_printf(s, "name: %s\n", call->name); |
| 543 | trace_seq_printf(s, "ID: %d\n", call->id); | 569 | trace_seq_printf(s, "ID: %d\n", call->event.type); |
| 544 | trace_seq_printf(s, "format:\n"); | 570 | trace_seq_printf(s, "format:\n"); |
| 545 | 571 | ||
| 546 | list_for_each_entry_reverse(field, &call->fields, link) { | 572 | head = trace_get_fields(call); |
| 573 | list_for_each_entry_reverse(field, head, link) { | ||
| 547 | /* | 574 | /* |
| 548 | * Smartly shows the array type(except dynamic array). | 575 | * Smartly shows the array type(except dynamic array). |
| 549 | * Normal: | 576 | * Normal: |
| @@ -613,7 +640,7 @@ event_id_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) | |||
| 613 | return -ENOMEM; | 640 | return -ENOMEM; |
| 614 | 641 | ||
| 615 | trace_seq_init(s); | 642 | trace_seq_init(s); |
| 616 | trace_seq_printf(s, "%d\n", call->id); | 643 | trace_seq_printf(s, "%d\n", call->event.type); |
| 617 | 644 | ||
| 618 | r = simple_read_from_buffer(ubuf, cnt, ppos, | 645 | r = simple_read_from_buffer(ubuf, cnt, ppos, |
| 619 | s->buffer, s->len); | 646 | s->buffer, s->len); |
| @@ -919,14 +946,15 @@ event_create_dir(struct ftrace_event_call *call, struct dentry *d_events, | |||
| 919 | const struct file_operations *filter, | 946 | const struct file_operations *filter, |
| 920 | const struct file_operations *format) | 947 | const struct file_operations *format) |
| 921 | { | 948 | { |
| 949 | struct list_head *head; | ||
| 922 | int ret; | 950 | int ret; |
| 923 | 951 | ||
| 924 | /* | 952 | /* |
| 925 | * If the trace point header did not define TRACE_SYSTEM | 953 | * If the trace point header did not define TRACE_SYSTEM |
| 926 | * then the system would be called "TRACE_SYSTEM". | 954 | * then the system would be called "TRACE_SYSTEM". |
| 927 | */ | 955 | */ |
| 928 | if (strcmp(call->system, TRACE_SYSTEM) != 0) | 956 | if (strcmp(call->class->system, TRACE_SYSTEM) != 0) |
| 929 | d_events = event_subsystem_dir(call->system, d_events); | 957 | d_events = event_subsystem_dir(call->class->system, d_events); |
| 930 | 958 | ||
| 931 | call->dir = debugfs_create_dir(call->name, d_events); | 959 | call->dir = debugfs_create_dir(call->name, d_events); |
| 932 | if (!call->dir) { | 960 | if (!call->dir) { |
| @@ -935,22 +963,31 @@ event_create_dir(struct ftrace_event_call *call, struct dentry *d_events, | |||
| 935 | return -1; | 963 | return -1; |
| 936 | } | 964 | } |
| 937 | 965 | ||
| 938 | if (call->regfunc) | 966 | if (call->class->probe || call->class->reg) |
| 939 | trace_create_file("enable", 0644, call->dir, call, | 967 | trace_create_file("enable", 0644, call->dir, call, |
| 940 | enable); | 968 | enable); |
| 941 | 969 | ||
| 942 | if (call->id && call->perf_event_enable) | 970 | #ifdef CONFIG_PERF_EVENTS |
| 971 | if (call->event.type && (call->class->perf_probe || call->class->reg)) | ||
| 943 | trace_create_file("id", 0444, call->dir, call, | 972 | trace_create_file("id", 0444, call->dir, call, |
| 944 | id); | 973 | id); |
| 974 | #endif | ||
| 945 | 975 | ||
| 946 | if (call->define_fields) { | 976 | if (call->class->define_fields) { |
| 947 | ret = trace_define_common_fields(call); | 977 | /* |
| 948 | if (!ret) | 978 | * Other events may have the same class. Only update |
| 949 | ret = call->define_fields(call); | 979 | * the fields if they are not already defined. |
| 950 | if (ret < 0) { | 980 | */ |
| 951 | pr_warning("Could not initialize trace point" | 981 | head = trace_get_fields(call); |
| 952 | " events/%s\n", call->name); | 982 | if (list_empty(head)) { |
| 953 | return ret; | 983 | ret = trace_define_common_fields(call); |
| 984 | if (!ret) | ||
| 985 | ret = call->class->define_fields(call); | ||
| 986 | if (ret < 0) { | ||
| 987 | pr_warning("Could not initialize trace point" | ||
| 988 | " events/%s\n", call->name); | ||
| 989 | return ret; | ||
| 990 | } | ||
| 954 | } | 991 | } |
| 955 | trace_create_file("filter", 0644, call->dir, call, | 992 | trace_create_file("filter", 0644, call->dir, call, |
| 956 | filter); | 993 | filter); |
| @@ -970,8 +1007,8 @@ static int __trace_add_event_call(struct ftrace_event_call *call) | |||
| 970 | if (!call->name) | 1007 | if (!call->name) |
| 971 | return -EINVAL; | 1008 | return -EINVAL; |
| 972 | 1009 | ||
| 973 | if (call->raw_init) { | 1010 | if (call->class->raw_init) { |
| 974 | ret = call->raw_init(call); | 1011 | ret = call->class->raw_init(call); |
| 975 | if (ret < 0) { | 1012 | if (ret < 0) { |
| 976 | if (ret != -ENOSYS) | 1013 | if (ret != -ENOSYS) |
| 977 | pr_warning("Could not initialize trace " | 1014 | pr_warning("Could not initialize trace " |
| @@ -1035,13 +1072,13 @@ static void remove_subsystem_dir(const char *name) | |||
| 1035 | static void __trace_remove_event_call(struct ftrace_event_call *call) | 1072 | static void __trace_remove_event_call(struct ftrace_event_call *call) |
| 1036 | { | 1073 | { |
| 1037 | ftrace_event_enable_disable(call, 0); | 1074 | ftrace_event_enable_disable(call, 0); |
| 1038 | if (call->event) | 1075 | if (call->event.funcs) |
| 1039 | __unregister_ftrace_event(call->event); | 1076 | __unregister_ftrace_event(&call->event); |
| 1040 | debugfs_remove_recursive(call->dir); | 1077 | debugfs_remove_recursive(call->dir); |
| 1041 | list_del(&call->list); | 1078 | list_del(&call->list); |
| 1042 | trace_destroy_fields(call); | 1079 | trace_destroy_fields(call); |
| 1043 | destroy_preds(call); | 1080 | destroy_preds(call); |
| 1044 | remove_subsystem_dir(call->system); | 1081 | remove_subsystem_dir(call->class->system); |
| 1045 | } | 1082 | } |
| 1046 | 1083 | ||
| 1047 | /* Remove an event_call */ | 1084 | /* Remove an event_call */ |
| @@ -1132,8 +1169,8 @@ static void trace_module_add_events(struct module *mod) | |||
| 1132 | /* The linker may leave blanks */ | 1169 | /* The linker may leave blanks */ |
| 1133 | if (!call->name) | 1170 | if (!call->name) |
| 1134 | continue; | 1171 | continue; |
| 1135 | if (call->raw_init) { | 1172 | if (call->class->raw_init) { |
| 1136 | ret = call->raw_init(call); | 1173 | ret = call->class->raw_init(call); |
| 1137 | if (ret < 0) { | 1174 | if (ret < 0) { |
| 1138 | if (ret != -ENOSYS) | 1175 | if (ret != -ENOSYS) |
| 1139 | pr_warning("Could not initialize trace " | 1176 | pr_warning("Could not initialize trace " |
| @@ -1286,8 +1323,8 @@ static __init int event_trace_init(void) | |||
| 1286 | /* The linker may leave blanks */ | 1323 | /* The linker may leave blanks */ |
| 1287 | if (!call->name) | 1324 | if (!call->name) |
| 1288 | continue; | 1325 | continue; |
| 1289 | if (call->raw_init) { | 1326 | if (call->class->raw_init) { |
| 1290 | ret = call->raw_init(call); | 1327 | ret = call->class->raw_init(call); |
| 1291 | if (ret < 0) { | 1328 | if (ret < 0) { |
| 1292 | if (ret != -ENOSYS) | 1329 | if (ret != -ENOSYS) |
| 1293 | pr_warning("Could not initialize trace " | 1330 | pr_warning("Could not initialize trace " |
| @@ -1388,8 +1425,8 @@ static __init void event_trace_self_tests(void) | |||
| 1388 | 1425 | ||
| 1389 | list_for_each_entry(call, &ftrace_events, list) { | 1426 | list_for_each_entry(call, &ftrace_events, list) { |
| 1390 | 1427 | ||
| 1391 | /* Only test those that have a regfunc */ | 1428 | /* Only test those that have a probe */ |
| 1392 | if (!call->regfunc) | 1429 | if (!call->class || !call->class->probe) |
| 1393 | continue; | 1430 | continue; |
| 1394 | 1431 | ||
| 1395 | /* | 1432 | /* |
| @@ -1399,8 +1436,8 @@ static __init void event_trace_self_tests(void) | |||
| 1399 | * syscalls as we test. | 1436 | * syscalls as we test. |
| 1400 | */ | 1437 | */ |
| 1401 | #ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS | 1438 | #ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS |
| 1402 | if (call->system && | 1439 | if (call->class->system && |
| 1403 | strcmp(call->system, "syscalls") == 0) | 1440 | strcmp(call->class->system, "syscalls") == 0) |
| 1404 | continue; | 1441 | continue; |
| 1405 | #endif | 1442 | #endif |
| 1406 | 1443 | ||
| @@ -1410,7 +1447,7 @@ static __init void event_trace_self_tests(void) | |||
| 1410 | * If an event is already enabled, someone is using | 1447 | * If an event is already enabled, someone is using |
| 1411 | * it and the self test should not be on. | 1448 | * it and the self test should not be on. |
| 1412 | */ | 1449 | */ |
| 1413 | if (call->enabled) { | 1450 | if (call->flags & TRACE_EVENT_FL_ENABLED) { |
| 1414 | pr_warning("Enabled event during self test!\n"); | 1451 | pr_warning("Enabled event during self test!\n"); |
| 1415 | WARN_ON_ONCE(1); | 1452 | WARN_ON_ONCE(1); |
| 1416 | continue; | 1453 | continue; |
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 58092d844a1f..57bb1bb32999 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c | |||
| @@ -500,8 +500,10 @@ static struct ftrace_event_field * | |||
| 500 | find_event_field(struct ftrace_event_call *call, char *name) | 500 | find_event_field(struct ftrace_event_call *call, char *name) |
| 501 | { | 501 | { |
| 502 | struct ftrace_event_field *field; | 502 | struct ftrace_event_field *field; |
| 503 | struct list_head *head; | ||
| 503 | 504 | ||
| 504 | list_for_each_entry(field, &call->fields, link) { | 505 | head = trace_get_fields(call); |
| 506 | list_for_each_entry(field, head, link) { | ||
| 505 | if (!strcmp(field->name, name)) | 507 | if (!strcmp(field->name, name)) |
| 506 | return field; | 508 | return field; |
| 507 | } | 509 | } |
| @@ -545,7 +547,7 @@ static void filter_disable_preds(struct ftrace_event_call *call) | |||
| 545 | struct event_filter *filter = call->filter; | 547 | struct event_filter *filter = call->filter; |
| 546 | int i; | 548 | int i; |
| 547 | 549 | ||
| 548 | call->filter_active = 0; | 550 | call->flags &= ~TRACE_EVENT_FL_FILTERED; |
| 549 | filter->n_preds = 0; | 551 | filter->n_preds = 0; |
| 550 | 552 | ||
| 551 | for (i = 0; i < MAX_FILTER_PRED; i++) | 553 | for (i = 0; i < MAX_FILTER_PRED; i++) |
| @@ -572,7 +574,7 @@ void destroy_preds(struct ftrace_event_call *call) | |||
| 572 | { | 574 | { |
| 573 | __free_preds(call->filter); | 575 | __free_preds(call->filter); |
| 574 | call->filter = NULL; | 576 | call->filter = NULL; |
| 575 | call->filter_active = 0; | 577 | call->flags &= ~TRACE_EVENT_FL_FILTERED; |
| 576 | } | 578 | } |
| 577 | 579 | ||
| 578 | static struct event_filter *__alloc_preds(void) | 580 | static struct event_filter *__alloc_preds(void) |
| @@ -611,7 +613,7 @@ static int init_preds(struct ftrace_event_call *call) | |||
| 611 | if (call->filter) | 613 | if (call->filter) |
| 612 | return 0; | 614 | return 0; |
| 613 | 615 | ||
| 614 | call->filter_active = 0; | 616 | call->flags &= ~TRACE_EVENT_FL_FILTERED; |
| 615 | call->filter = __alloc_preds(); | 617 | call->filter = __alloc_preds(); |
| 616 | if (IS_ERR(call->filter)) | 618 | if (IS_ERR(call->filter)) |
| 617 | return PTR_ERR(call->filter); | 619 | return PTR_ERR(call->filter); |
| @@ -625,10 +627,10 @@ static int init_subsystem_preds(struct event_subsystem *system) | |||
| 625 | int err; | 627 | int err; |
| 626 | 628 | ||
| 627 | list_for_each_entry(call, &ftrace_events, list) { | 629 | list_for_each_entry(call, &ftrace_events, list) { |
| 628 | if (!call->define_fields) | 630 | if (!call->class || !call->class->define_fields) |
| 629 | continue; | 631 | continue; |
| 630 | 632 | ||
| 631 | if (strcmp(call->system, system->name) != 0) | 633 | if (strcmp(call->class->system, system->name) != 0) |
| 632 | continue; | 634 | continue; |
| 633 | 635 | ||
| 634 | err = init_preds(call); | 636 | err = init_preds(call); |
| @@ -644,10 +646,10 @@ static void filter_free_subsystem_preds(struct event_subsystem *system) | |||
| 644 | struct ftrace_event_call *call; | 646 | struct ftrace_event_call *call; |
| 645 | 647 | ||
| 646 | list_for_each_entry(call, &ftrace_events, list) { | 648 | list_for_each_entry(call, &ftrace_events, list) { |
| 647 | if (!call->define_fields) | 649 | if (!call->class || !call->class->define_fields) |
| 648 | continue; | 650 | continue; |
| 649 | 651 | ||
| 650 | if (strcmp(call->system, system->name) != 0) | 652 | if (strcmp(call->class->system, system->name) != 0) |
| 651 | continue; | 653 | continue; |
| 652 | 654 | ||
| 653 | filter_disable_preds(call); | 655 | filter_disable_preds(call); |
| @@ -1249,10 +1251,10 @@ static int replace_system_preds(struct event_subsystem *system, | |||
| 1249 | list_for_each_entry(call, &ftrace_events, list) { | 1251 | list_for_each_entry(call, &ftrace_events, list) { |
| 1250 | struct event_filter *filter = call->filter; | 1252 | struct event_filter *filter = call->filter; |
| 1251 | 1253 | ||
| 1252 | if (!call->define_fields) | 1254 | if (!call->class || !call->class->define_fields) |
| 1253 | continue; | 1255 | continue; |
| 1254 | 1256 | ||
| 1255 | if (strcmp(call->system, system->name) != 0) | 1257 | if (strcmp(call->class->system, system->name) != 0) |
| 1256 | continue; | 1258 | continue; |
| 1257 | 1259 | ||
| 1258 | /* try to see if the filter can be applied */ | 1260 | /* try to see if the filter can be applied */ |
| @@ -1266,7 +1268,7 @@ static int replace_system_preds(struct event_subsystem *system, | |||
| 1266 | if (err) | 1268 | if (err) |
| 1267 | filter_disable_preds(call); | 1269 | filter_disable_preds(call); |
| 1268 | else { | 1270 | else { |
| 1269 | call->filter_active = 1; | 1271 | call->flags |= TRACE_EVENT_FL_FILTERED; |
| 1270 | replace_filter_string(filter, filter_string); | 1272 | replace_filter_string(filter, filter_string); |
| 1271 | } | 1273 | } |
| 1272 | fail = false; | 1274 | fail = false; |
| @@ -1315,7 +1317,7 @@ int apply_event_filter(struct ftrace_event_call *call, char *filter_string) | |||
| 1315 | if (err) | 1317 | if (err) |
| 1316 | append_filter_err(ps, call->filter); | 1318 | append_filter_err(ps, call->filter); |
| 1317 | else | 1319 | else |
| 1318 | call->filter_active = 1; | 1320 | call->flags |= TRACE_EVENT_FL_FILTERED; |
| 1319 | out: | 1321 | out: |
| 1320 | filter_opstack_clear(ps); | 1322 | filter_opstack_clear(ps); |
| 1321 | postfix_clear(ps); | 1323 | postfix_clear(ps); |
| @@ -1393,7 +1395,7 @@ int ftrace_profile_set_filter(struct perf_event *event, int event_id, | |||
| 1393 | mutex_lock(&event_mutex); | 1395 | mutex_lock(&event_mutex); |
| 1394 | 1396 | ||
| 1395 | list_for_each_entry(call, &ftrace_events, list) { | 1397 | list_for_each_entry(call, &ftrace_events, list) { |
| 1396 | if (call->id == event_id) | 1398 | if (call->event.type == event_id) |
| 1397 | break; | 1399 | break; |
| 1398 | } | 1400 | } |
| 1399 | 1401 | ||
diff --git a/kernel/trace/trace_export.c b/kernel/trace/trace_export.c index e091f64ba6ce..8536e2a65969 100644 --- a/kernel/trace/trace_export.c +++ b/kernel/trace/trace_export.c | |||
| @@ -127,7 +127,7 @@ ftrace_define_fields_##name(struct ftrace_event_call *event_call) \ | |||
| 127 | 127 | ||
| 128 | static int ftrace_raw_init_event(struct ftrace_event_call *call) | 128 | static int ftrace_raw_init_event(struct ftrace_event_call *call) |
| 129 | { | 129 | { |
| 130 | INIT_LIST_HEAD(&call->fields); | 130 | INIT_LIST_HEAD(&call->class->fields); |
| 131 | return 0; | 131 | return 0; |
| 132 | } | 132 | } |
| 133 | 133 | ||
| @@ -153,17 +153,21 @@ static int ftrace_raw_init_event(struct ftrace_event_call *call) | |||
| 153 | #define F_printk(fmt, args...) #fmt ", " __stringify(args) | 153 | #define F_printk(fmt, args...) #fmt ", " __stringify(args) |
| 154 | 154 | ||
| 155 | #undef FTRACE_ENTRY | 155 | #undef FTRACE_ENTRY |
| 156 | #define FTRACE_ENTRY(call, struct_name, type, tstruct, print) \ | 156 | #define FTRACE_ENTRY(call, struct_name, etype, tstruct, print) \ |
| 157 | \ | ||
| 158 | struct ftrace_event_class event_class_ftrace_##call = { \ | ||
| 159 | .system = __stringify(TRACE_SYSTEM), \ | ||
| 160 | .define_fields = ftrace_define_fields_##call, \ | ||
| 161 | .raw_init = ftrace_raw_init_event, \ | ||
| 162 | }; \ | ||
| 157 | \ | 163 | \ |
| 158 | struct ftrace_event_call __used \ | 164 | struct ftrace_event_call __used \ |
| 159 | __attribute__((__aligned__(4))) \ | 165 | __attribute__((__aligned__(4))) \ |
| 160 | __attribute__((section("_ftrace_events"))) event_##call = { \ | 166 | __attribute__((section("_ftrace_events"))) event_##call = { \ |
| 161 | .name = #call, \ | 167 | .name = #call, \ |
| 162 | .id = type, \ | 168 | .event.type = etype, \ |
| 163 | .system = __stringify(TRACE_SYSTEM), \ | 169 | .class = &event_class_ftrace_##call, \ |
| 164 | .raw_init = ftrace_raw_init_event, \ | ||
| 165 | .print_fmt = print, \ | 170 | .print_fmt = print, \ |
| 166 | .define_fields = ftrace_define_fields_##call, \ | ||
| 167 | }; \ | 171 | }; \ |
| 168 | 172 | ||
| 169 | #include "trace_entries.h" | 173 | #include "trace_entries.h" |
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index dd11c830eb84..79f4bac99a94 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c | |||
| @@ -1025,7 +1025,7 @@ print_graph_comment(struct trace_seq *s, struct trace_entry *ent, | |||
| 1025 | if (!event) | 1025 | if (!event) |
| 1026 | return TRACE_TYPE_UNHANDLED; | 1026 | return TRACE_TYPE_UNHANDLED; |
| 1027 | 1027 | ||
| 1028 | ret = event->trace(iter, sym_flags); | 1028 | ret = event->funcs->trace(iter, sym_flags, event); |
| 1029 | if (ret != TRACE_TYPE_HANDLED) | 1029 | if (ret != TRACE_TYPE_HANDLED) |
| 1030 | return ret; | 1030 | return ret; |
| 1031 | } | 1031 | } |
| @@ -1112,7 +1112,8 @@ print_graph_function(struct trace_iterator *iter) | |||
| 1112 | } | 1112 | } |
| 1113 | 1113 | ||
| 1114 | static enum print_line_t | 1114 | static enum print_line_t |
| 1115 | print_graph_function_event(struct trace_iterator *iter, int flags) | 1115 | print_graph_function_event(struct trace_iterator *iter, int flags, |
| 1116 | struct trace_event *event) | ||
| 1116 | { | 1117 | { |
| 1117 | return print_graph_function(iter); | 1118 | return print_graph_function(iter); |
| 1118 | } | 1119 | } |
| @@ -1225,14 +1226,18 @@ void graph_trace_close(struct trace_iterator *iter) | |||
| 1225 | } | 1226 | } |
| 1226 | } | 1227 | } |
| 1227 | 1228 | ||
| 1229 | static struct trace_event_functions graph_functions = { | ||
| 1230 | .trace = print_graph_function_event, | ||
| 1231 | }; | ||
| 1232 | |||
| 1228 | static struct trace_event graph_trace_entry_event = { | 1233 | static struct trace_event graph_trace_entry_event = { |
| 1229 | .type = TRACE_GRAPH_ENT, | 1234 | .type = TRACE_GRAPH_ENT, |
| 1230 | .trace = print_graph_function_event, | 1235 | .funcs = &graph_functions, |
| 1231 | }; | 1236 | }; |
| 1232 | 1237 | ||
| 1233 | static struct trace_event graph_trace_ret_event = { | 1238 | static struct trace_event graph_trace_ret_event = { |
| 1234 | .type = TRACE_GRAPH_RET, | 1239 | .type = TRACE_GRAPH_RET, |
| 1235 | .trace = print_graph_function_event, | 1240 | .funcs = &graph_functions |
| 1236 | }; | 1241 | }; |
| 1237 | 1242 | ||
| 1238 | static struct tracer graph_trace __read_mostly = { | 1243 | static struct tracer graph_trace __read_mostly = { |
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index a7514326052b..faf7cefd15da 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c | |||
| @@ -324,8 +324,8 @@ struct trace_probe { | |||
| 324 | unsigned long nhit; | 324 | unsigned long nhit; |
| 325 | unsigned int flags; /* For TP_FLAG_* */ | 325 | unsigned int flags; /* For TP_FLAG_* */ |
| 326 | const char *symbol; /* symbol name */ | 326 | const char *symbol; /* symbol name */ |
| 327 | struct ftrace_event_class class; | ||
| 327 | struct ftrace_event_call call; | 328 | struct ftrace_event_call call; |
| 328 | struct trace_event event; | ||
| 329 | ssize_t size; /* trace entry size */ | 329 | ssize_t size; /* trace entry size */ |
| 330 | unsigned int nr_args; | 330 | unsigned int nr_args; |
| 331 | struct probe_arg args[]; | 331 | struct probe_arg args[]; |
| @@ -404,6 +404,7 @@ static struct trace_probe *alloc_trace_probe(const char *group, | |||
| 404 | goto error; | 404 | goto error; |
| 405 | } | 405 | } |
| 406 | 406 | ||
| 407 | tp->call.class = &tp->class; | ||
| 407 | tp->call.name = kstrdup(event, GFP_KERNEL); | 408 | tp->call.name = kstrdup(event, GFP_KERNEL); |
| 408 | if (!tp->call.name) | 409 | if (!tp->call.name) |
| 409 | goto error; | 410 | goto error; |
| @@ -413,8 +414,8 @@ static struct trace_probe *alloc_trace_probe(const char *group, | |||
| 413 | goto error; | 414 | goto error; |
| 414 | } | 415 | } |
| 415 | 416 | ||
| 416 | tp->call.system = kstrdup(group, GFP_KERNEL); | 417 | tp->class.system = kstrdup(group, GFP_KERNEL); |
| 417 | if (!tp->call.system) | 418 | if (!tp->class.system) |
| 418 | goto error; | 419 | goto error; |
| 419 | 420 | ||
| 420 | INIT_LIST_HEAD(&tp->list); | 421 | INIT_LIST_HEAD(&tp->list); |
| @@ -443,7 +444,7 @@ static void free_trace_probe(struct trace_probe *tp) | |||
| 443 | for (i = 0; i < tp->nr_args; i++) | 444 | for (i = 0; i < tp->nr_args; i++) |
| 444 | free_probe_arg(&tp->args[i]); | 445 | free_probe_arg(&tp->args[i]); |
| 445 | 446 | ||
| 446 | kfree(tp->call.system); | 447 | kfree(tp->call.class->system); |
| 447 | kfree(tp->call.name); | 448 | kfree(tp->call.name); |
| 448 | kfree(tp->symbol); | 449 | kfree(tp->symbol); |
| 449 | kfree(tp); | 450 | kfree(tp); |
| @@ -456,7 +457,7 @@ static struct trace_probe *find_probe_event(const char *event, | |||
| 456 | 457 | ||
| 457 | list_for_each_entry(tp, &probe_list, list) | 458 | list_for_each_entry(tp, &probe_list, list) |
| 458 | if (strcmp(tp->call.name, event) == 0 && | 459 | if (strcmp(tp->call.name, event) == 0 && |
| 459 | strcmp(tp->call.system, group) == 0) | 460 | strcmp(tp->call.class->system, group) == 0) |
| 460 | return tp; | 461 | return tp; |
| 461 | return NULL; | 462 | return NULL; |
| 462 | } | 463 | } |
| @@ -481,7 +482,7 @@ static int register_trace_probe(struct trace_probe *tp) | |||
| 481 | mutex_lock(&probe_lock); | 482 | mutex_lock(&probe_lock); |
| 482 | 483 | ||
| 483 | /* register as an event */ | 484 | /* register as an event */ |
| 484 | old_tp = find_probe_event(tp->call.name, tp->call.system); | 485 | old_tp = find_probe_event(tp->call.name, tp->call.class->system); |
| 485 | if (old_tp) { | 486 | if (old_tp) { |
| 486 | /* delete old event */ | 487 | /* delete old event */ |
| 487 | unregister_trace_probe(old_tp); | 488 | unregister_trace_probe(old_tp); |
| @@ -904,7 +905,7 @@ static int probes_seq_show(struct seq_file *m, void *v) | |||
| 904 | int i; | 905 | int i; |
| 905 | 906 | ||
| 906 | seq_printf(m, "%c", probe_is_return(tp) ? 'r' : 'p'); | 907 | seq_printf(m, "%c", probe_is_return(tp) ? 'r' : 'p'); |
| 907 | seq_printf(m, ":%s/%s", tp->call.system, tp->call.name); | 908 | seq_printf(m, ":%s/%s", tp->call.class->system, tp->call.name); |
| 908 | 909 | ||
| 909 | if (!tp->symbol) | 910 | if (!tp->symbol) |
| 910 | seq_printf(m, " 0x%p", tp->rp.kp.addr); | 911 | seq_printf(m, " 0x%p", tp->rp.kp.addr); |
| @@ -1061,8 +1062,8 @@ static __kprobes void kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs) | |||
| 1061 | 1062 | ||
| 1062 | size = sizeof(*entry) + tp->size; | 1063 | size = sizeof(*entry) + tp->size; |
| 1063 | 1064 | ||
| 1064 | event = trace_current_buffer_lock_reserve(&buffer, call->id, size, | 1065 | event = trace_current_buffer_lock_reserve(&buffer, call->event.type, |
| 1065 | irq_flags, pc); | 1066 | size, irq_flags, pc); |
| 1066 | if (!event) | 1067 | if (!event) |
| 1067 | return; | 1068 | return; |
| 1068 | 1069 | ||
| @@ -1094,8 +1095,8 @@ static __kprobes void kretprobe_trace_func(struct kretprobe_instance *ri, | |||
| 1094 | 1095 | ||
| 1095 | size = sizeof(*entry) + tp->size; | 1096 | size = sizeof(*entry) + tp->size; |
| 1096 | 1097 | ||
| 1097 | event = trace_current_buffer_lock_reserve(&buffer, call->id, size, | 1098 | event = trace_current_buffer_lock_reserve(&buffer, call->event.type, |
| 1098 | irq_flags, pc); | 1099 | size, irq_flags, pc); |
| 1099 | if (!event) | 1100 | if (!event) |
| 1100 | return; | 1101 | return; |
| 1101 | 1102 | ||
| @@ -1112,18 +1113,17 @@ static __kprobes void kretprobe_trace_func(struct kretprobe_instance *ri, | |||
| 1112 | 1113 | ||
| 1113 | /* Event entry printers */ | 1114 | /* Event entry printers */ |
| 1114 | enum print_line_t | 1115 | enum print_line_t |
| 1115 | print_kprobe_event(struct trace_iterator *iter, int flags) | 1116 | print_kprobe_event(struct trace_iterator *iter, int flags, |
| 1117 | struct trace_event *event) | ||
| 1116 | { | 1118 | { |
| 1117 | struct kprobe_trace_entry_head *field; | 1119 | struct kprobe_trace_entry_head *field; |
| 1118 | struct trace_seq *s = &iter->seq; | 1120 | struct trace_seq *s = &iter->seq; |
| 1119 | struct trace_event *event; | ||
| 1120 | struct trace_probe *tp; | 1121 | struct trace_probe *tp; |
| 1121 | u8 *data; | 1122 | u8 *data; |
| 1122 | int i; | 1123 | int i; |
| 1123 | 1124 | ||
| 1124 | field = (struct kprobe_trace_entry_head *)iter->ent; | 1125 | field = (struct kprobe_trace_entry_head *)iter->ent; |
| 1125 | event = ftrace_find_event(field->ent.type); | 1126 | tp = container_of(event, struct trace_probe, call.event); |
| 1126 | tp = container_of(event, struct trace_probe, event); | ||
| 1127 | 1127 | ||
| 1128 | if (!trace_seq_printf(s, "%s: (", tp->call.name)) | 1128 | if (!trace_seq_printf(s, "%s: (", tp->call.name)) |
| 1129 | goto partial; | 1129 | goto partial; |
| @@ -1149,18 +1149,17 @@ partial: | |||
| 1149 | } | 1149 | } |
| 1150 | 1150 | ||
| 1151 | enum print_line_t | 1151 | enum print_line_t |
| 1152 | print_kretprobe_event(struct trace_iterator *iter, int flags) | 1152 | print_kretprobe_event(struct trace_iterator *iter, int flags, |
| 1153 | struct trace_event *event) | ||
| 1153 | { | 1154 | { |
| 1154 | struct kretprobe_trace_entry_head *field; | 1155 | struct kretprobe_trace_entry_head *field; |
| 1155 | struct trace_seq *s = &iter->seq; | 1156 | struct trace_seq *s = &iter->seq; |
| 1156 | struct trace_event *event; | ||
| 1157 | struct trace_probe *tp; | 1157 | struct trace_probe *tp; |
| 1158 | u8 *data; | 1158 | u8 *data; |
| 1159 | int i; | 1159 | int i; |
| 1160 | 1160 | ||
| 1161 | field = (struct kretprobe_trace_entry_head *)iter->ent; | 1161 | field = (struct kretprobe_trace_entry_head *)iter->ent; |
| 1162 | event = ftrace_find_event(field->ent.type); | 1162 | tp = container_of(event, struct trace_probe, call.event); |
| 1163 | tp = container_of(event, struct trace_probe, event); | ||
| 1164 | 1163 | ||
| 1165 | if (!trace_seq_printf(s, "%s: (", tp->call.name)) | 1164 | if (!trace_seq_printf(s, "%s: (", tp->call.name)) |
| 1166 | goto partial; | 1165 | goto partial; |
| @@ -1217,8 +1216,6 @@ static void probe_event_disable(struct ftrace_event_call *call) | |||
| 1217 | 1216 | ||
| 1218 | static int probe_event_raw_init(struct ftrace_event_call *event_call) | 1217 | static int probe_event_raw_init(struct ftrace_event_call *event_call) |
| 1219 | { | 1218 | { |
| 1220 | INIT_LIST_HEAD(&event_call->fields); | ||
| 1221 | |||
| 1222 | return 0; | 1219 | return 0; |
| 1223 | } | 1220 | } |
| 1224 | 1221 | ||
| @@ -1341,9 +1338,9 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp, | |||
| 1341 | struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); | 1338 | struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); |
| 1342 | struct ftrace_event_call *call = &tp->call; | 1339 | struct ftrace_event_call *call = &tp->call; |
| 1343 | struct kprobe_trace_entry_head *entry; | 1340 | struct kprobe_trace_entry_head *entry; |
| 1341 | struct hlist_head *head; | ||
| 1344 | u8 *data; | 1342 | u8 *data; |
| 1345 | int size, __size, i; | 1343 | int size, __size, i; |
| 1346 | unsigned long irq_flags; | ||
| 1347 | int rctx; | 1344 | int rctx; |
| 1348 | 1345 | ||
| 1349 | __size = sizeof(*entry) + tp->size; | 1346 | __size = sizeof(*entry) + tp->size; |
| @@ -1353,7 +1350,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp, | |||
| 1353 | "profile buffer not large enough")) | 1350 | "profile buffer not large enough")) |
| 1354 | return; | 1351 | return; |
| 1355 | 1352 | ||
| 1356 | entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags); | 1353 | entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx); |
| 1357 | if (!entry) | 1354 | if (!entry) |
| 1358 | return; | 1355 | return; |
| 1359 | 1356 | ||
| @@ -1362,7 +1359,8 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp, | |||
| 1362 | for (i = 0; i < tp->nr_args; i++) | 1359 | for (i = 0; i < tp->nr_args; i++) |
| 1363 | call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset); | 1360 | call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset); |
| 1364 | 1361 | ||
| 1365 | perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags, regs); | 1362 | head = per_cpu_ptr(call->perf_events, smp_processor_id()); |
| 1363 | perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head); | ||
| 1366 | } | 1364 | } |
| 1367 | 1365 | ||
| 1368 | /* Kretprobe profile handler */ | 1366 | /* Kretprobe profile handler */ |
| @@ -1372,9 +1370,9 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, | |||
| 1372 | struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); | 1370 | struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); |
| 1373 | struct ftrace_event_call *call = &tp->call; | 1371 | struct ftrace_event_call *call = &tp->call; |
| 1374 | struct kretprobe_trace_entry_head *entry; | 1372 | struct kretprobe_trace_entry_head *entry; |
| 1373 | struct hlist_head *head; | ||
| 1375 | u8 *data; | 1374 | u8 *data; |
| 1376 | int size, __size, i; | 1375 | int size, __size, i; |
| 1377 | unsigned long irq_flags; | ||
| 1378 | int rctx; | 1376 | int rctx; |
| 1379 | 1377 | ||
| 1380 | __size = sizeof(*entry) + tp->size; | 1378 | __size = sizeof(*entry) + tp->size; |
| @@ -1384,7 +1382,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, | |||
| 1384 | "profile buffer not large enough")) | 1382 | "profile buffer not large enough")) |
| 1385 | return; | 1383 | return; |
| 1386 | 1384 | ||
| 1387 | entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags); | 1385 | entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx); |
| 1388 | if (!entry) | 1386 | if (!entry) |
| 1389 | return; | 1387 | return; |
| 1390 | 1388 | ||
| @@ -1394,8 +1392,8 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, | |||
| 1394 | for (i = 0; i < tp->nr_args; i++) | 1392 | for (i = 0; i < tp->nr_args; i++) |
| 1395 | call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset); | 1393 | call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset); |
| 1396 | 1394 | ||
| 1397 | perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, | 1395 | head = per_cpu_ptr(call->perf_events, smp_processor_id()); |
| 1398 | irq_flags, regs); | 1396 | perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, regs, head); |
| 1399 | } | 1397 | } |
| 1400 | 1398 | ||
| 1401 | static int probe_perf_enable(struct ftrace_event_call *call) | 1399 | static int probe_perf_enable(struct ftrace_event_call *call) |
| @@ -1425,6 +1423,26 @@ static void probe_perf_disable(struct ftrace_event_call *call) | |||
| 1425 | } | 1423 | } |
| 1426 | #endif /* CONFIG_PERF_EVENTS */ | 1424 | #endif /* CONFIG_PERF_EVENTS */ |
| 1427 | 1425 | ||
| 1426 | static __kprobes | ||
| 1427 | int kprobe_register(struct ftrace_event_call *event, enum trace_reg type) | ||
| 1428 | { | ||
| 1429 | switch (type) { | ||
| 1430 | case TRACE_REG_REGISTER: | ||
| 1431 | return probe_event_enable(event); | ||
| 1432 | case TRACE_REG_UNREGISTER: | ||
| 1433 | probe_event_disable(event); | ||
| 1434 | return 0; | ||
| 1435 | |||
| 1436 | #ifdef CONFIG_PERF_EVENTS | ||
| 1437 | case TRACE_REG_PERF_REGISTER: | ||
| 1438 | return probe_perf_enable(event); | ||
| 1439 | case TRACE_REG_PERF_UNREGISTER: | ||
| 1440 | probe_perf_disable(event); | ||
| 1441 | return 0; | ||
| 1442 | #endif | ||
| 1443 | } | ||
| 1444 | return 0; | ||
| 1445 | } | ||
| 1428 | 1446 | ||
| 1429 | static __kprobes | 1447 | static __kprobes |
| 1430 | int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) | 1448 | int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) |
| @@ -1454,6 +1472,14 @@ int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) | |||
| 1454 | return 0; /* We don't tweek kernel, so just return 0 */ | 1472 | return 0; /* We don't tweek kernel, so just return 0 */ |
| 1455 | } | 1473 | } |
| 1456 | 1474 | ||
| 1475 | static struct trace_event_functions kretprobe_funcs = { | ||
| 1476 | .trace = print_kretprobe_event | ||
| 1477 | }; | ||
| 1478 | |||
| 1479 | static struct trace_event_functions kprobe_funcs = { | ||
| 1480 | .trace = print_kprobe_event | ||
| 1481 | }; | ||
| 1482 | |||
| 1457 | static int register_probe_event(struct trace_probe *tp) | 1483 | static int register_probe_event(struct trace_probe *tp) |
| 1458 | { | 1484 | { |
| 1459 | struct ftrace_event_call *call = &tp->call; | 1485 | struct ftrace_event_call *call = &tp->call; |
| @@ -1461,36 +1487,31 @@ static int register_probe_event(struct trace_probe *tp) | |||
| 1461 | 1487 | ||
| 1462 | /* Initialize ftrace_event_call */ | 1488 | /* Initialize ftrace_event_call */ |
| 1463 | if (probe_is_return(tp)) { | 1489 | if (probe_is_return(tp)) { |
| 1464 | tp->event.trace = print_kretprobe_event; | 1490 | INIT_LIST_HEAD(&call->class->fields); |
| 1465 | call->raw_init = probe_event_raw_init; | 1491 | call->event.funcs = &kretprobe_funcs; |
| 1466 | call->define_fields = kretprobe_event_define_fields; | 1492 | call->class->raw_init = probe_event_raw_init; |
| 1493 | call->class->define_fields = kretprobe_event_define_fields; | ||
| 1467 | } else { | 1494 | } else { |
| 1468 | tp->event.trace = print_kprobe_event; | 1495 | INIT_LIST_HEAD(&call->class->fields); |
| 1469 | call->raw_init = probe_event_raw_init; | 1496 | call->event.funcs = &kprobe_funcs; |
| 1470 | call->define_fields = kprobe_event_define_fields; | 1497 | call->class->raw_init = probe_event_raw_init; |
| 1498 | call->class->define_fields = kprobe_event_define_fields; | ||
| 1471 | } | 1499 | } |
| 1472 | if (set_print_fmt(tp) < 0) | 1500 | if (set_print_fmt(tp) < 0) |
| 1473 | return -ENOMEM; | 1501 | return -ENOMEM; |
| 1474 | call->event = &tp->event; | 1502 | ret = register_ftrace_event(&call->event); |
| 1475 | call->id = register_ftrace_event(&tp->event); | 1503 | if (!ret) { |
| 1476 | if (!call->id) { | ||
| 1477 | kfree(call->print_fmt); | 1504 | kfree(call->print_fmt); |
| 1478 | return -ENODEV; | 1505 | return -ENODEV; |
| 1479 | } | 1506 | } |
| 1480 | call->enabled = 0; | 1507 | call->flags = 0; |
| 1481 | call->regfunc = probe_event_enable; | 1508 | call->class->reg = kprobe_register; |
| 1482 | call->unregfunc = probe_event_disable; | ||
| 1483 | |||
| 1484 | #ifdef CONFIG_PERF_EVENTS | ||
| 1485 | call->perf_event_enable = probe_perf_enable; | ||
| 1486 | call->perf_event_disable = probe_perf_disable; | ||
| 1487 | #endif | ||
| 1488 | call->data = tp; | 1509 | call->data = tp; |
| 1489 | ret = trace_add_event_call(call); | 1510 | ret = trace_add_event_call(call); |
| 1490 | if (ret) { | 1511 | if (ret) { |
| 1491 | pr_info("Failed to register kprobe event: %s\n", call->name); | 1512 | pr_info("Failed to register kprobe event: %s\n", call->name); |
| 1492 | kfree(call->print_fmt); | 1513 | kfree(call->print_fmt); |
| 1493 | unregister_ftrace_event(&tp->event); | 1514 | unregister_ftrace_event(&call->event); |
| 1494 | } | 1515 | } |
| 1495 | return ret; | 1516 | return ret; |
| 1496 | } | 1517 | } |
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c index ab13d7008061..57c1b4596470 100644 --- a/kernel/trace/trace_output.c +++ b/kernel/trace/trace_output.c | |||
| @@ -742,6 +742,9 @@ int register_ftrace_event(struct trace_event *event) | |||
| 742 | if (WARN_ON(!event)) | 742 | if (WARN_ON(!event)) |
| 743 | goto out; | 743 | goto out; |
| 744 | 744 | ||
| 745 | if (WARN_ON(!event->funcs)) | ||
| 746 | goto out; | ||
| 747 | |||
| 745 | INIT_LIST_HEAD(&event->list); | 748 | INIT_LIST_HEAD(&event->list); |
| 746 | 749 | ||
| 747 | if (!event->type) { | 750 | if (!event->type) { |
| @@ -774,14 +777,14 @@ int register_ftrace_event(struct trace_event *event) | |||
| 774 | goto out; | 777 | goto out; |
| 775 | } | 778 | } |
| 776 | 779 | ||
| 777 | if (event->trace == NULL) | 780 | if (event->funcs->trace == NULL) |
| 778 | event->trace = trace_nop_print; | 781 | event->funcs->trace = trace_nop_print; |
| 779 | if (event->raw == NULL) | 782 | if (event->funcs->raw == NULL) |
| 780 | event->raw = trace_nop_print; | 783 | event->funcs->raw = trace_nop_print; |
| 781 | if (event->hex == NULL) | 784 | if (event->funcs->hex == NULL) |
| 782 | event->hex = trace_nop_print; | 785 | event->funcs->hex = trace_nop_print; |
| 783 | if (event->binary == NULL) | 786 | if (event->funcs->binary == NULL) |
| 784 | event->binary = trace_nop_print; | 787 | event->funcs->binary = trace_nop_print; |
| 785 | 788 | ||
| 786 | key = event->type & (EVENT_HASHSIZE - 1); | 789 | key = event->type & (EVENT_HASHSIZE - 1); |
| 787 | 790 | ||
| @@ -823,13 +826,15 @@ EXPORT_SYMBOL_GPL(unregister_ftrace_event); | |||
| 823 | * Standard events | 826 | * Standard events |
| 824 | */ | 827 | */ |
| 825 | 828 | ||
| 826 | enum print_line_t trace_nop_print(struct trace_iterator *iter, int flags) | 829 | enum print_line_t trace_nop_print(struct trace_iterator *iter, int flags, |
| 830 | struct trace_event *event) | ||
| 827 | { | 831 | { |
| 828 | return TRACE_TYPE_HANDLED; | 832 | return TRACE_TYPE_HANDLED; |
| 829 | } | 833 | } |
| 830 | 834 | ||
| 831 | /* TRACE_FN */ | 835 | /* TRACE_FN */ |
| 832 | static enum print_line_t trace_fn_trace(struct trace_iterator *iter, int flags) | 836 | static enum print_line_t trace_fn_trace(struct trace_iterator *iter, int flags, |
| 837 | struct trace_event *event) | ||
| 833 | { | 838 | { |
| 834 | struct ftrace_entry *field; | 839 | struct ftrace_entry *field; |
| 835 | struct trace_seq *s = &iter->seq; | 840 | struct trace_seq *s = &iter->seq; |
| @@ -856,7 +861,8 @@ static enum print_line_t trace_fn_trace(struct trace_iterator *iter, int flags) | |||
| 856 | return TRACE_TYPE_PARTIAL_LINE; | 861 | return TRACE_TYPE_PARTIAL_LINE; |
| 857 | } | 862 | } |
| 858 | 863 | ||
| 859 | static enum print_line_t trace_fn_raw(struct trace_iterator *iter, int flags) | 864 | static enum print_line_t trace_fn_raw(struct trace_iterator *iter, int flags, |
| 865 | struct trace_event *event) | ||
| 860 | { | 866 | { |
| 861 | struct ftrace_entry *field; | 867 | struct ftrace_entry *field; |
| 862 | 868 | ||
| @@ -870,7 +876,8 @@ static enum print_line_t trace_fn_raw(struct trace_iterator *iter, int flags) | |||
| 870 | return TRACE_TYPE_HANDLED; | 876 | return TRACE_TYPE_HANDLED; |
| 871 | } | 877 | } |
| 872 | 878 | ||
| 873 | static enum print_line_t trace_fn_hex(struct trace_iterator *iter, int flags) | 879 | static enum print_line_t trace_fn_hex(struct trace_iterator *iter, int flags, |
| 880 | struct trace_event *event) | ||
| 874 | { | 881 | { |
| 875 | struct ftrace_entry *field; | 882 | struct ftrace_entry *field; |
| 876 | struct trace_seq *s = &iter->seq; | 883 | struct trace_seq *s = &iter->seq; |
| @@ -883,7 +890,8 @@ static enum print_line_t trace_fn_hex(struct trace_iterator *iter, int flags) | |||
| 883 | return TRACE_TYPE_HANDLED; | 890 | return TRACE_TYPE_HANDLED; |
| 884 | } | 891 | } |
| 885 | 892 | ||
| 886 | static enum print_line_t trace_fn_bin(struct trace_iterator *iter, int flags) | 893 | static enum print_line_t trace_fn_bin(struct trace_iterator *iter, int flags, |
| 894 | struct trace_event *event) | ||
| 887 | { | 895 | { |
| 888 | struct ftrace_entry *field; | 896 | struct ftrace_entry *field; |
| 889 | struct trace_seq *s = &iter->seq; | 897 | struct trace_seq *s = &iter->seq; |
| @@ -896,14 +904,18 @@ static enum print_line_t trace_fn_bin(struct trace_iterator *iter, int flags) | |||
| 896 | return TRACE_TYPE_HANDLED; | 904 | return TRACE_TYPE_HANDLED; |
| 897 | } | 905 | } |
| 898 | 906 | ||
| 899 | static struct trace_event trace_fn_event = { | 907 | static struct trace_event_functions trace_fn_funcs = { |
| 900 | .type = TRACE_FN, | ||
| 901 | .trace = trace_fn_trace, | 908 | .trace = trace_fn_trace, |
| 902 | .raw = trace_fn_raw, | 909 | .raw = trace_fn_raw, |
| 903 | .hex = trace_fn_hex, | 910 | .hex = trace_fn_hex, |
| 904 | .binary = trace_fn_bin, | 911 | .binary = trace_fn_bin, |
| 905 | }; | 912 | }; |
| 906 | 913 | ||
| 914 | static struct trace_event trace_fn_event = { | ||
| 915 | .type = TRACE_FN, | ||
| 916 | .funcs = &trace_fn_funcs, | ||
| 917 | }; | ||
| 918 | |||
| 907 | /* TRACE_CTX an TRACE_WAKE */ | 919 | /* TRACE_CTX an TRACE_WAKE */ |
| 908 | static enum print_line_t trace_ctxwake_print(struct trace_iterator *iter, | 920 | static enum print_line_t trace_ctxwake_print(struct trace_iterator *iter, |
| 909 | char *delim) | 921 | char *delim) |
| @@ -932,13 +944,14 @@ static enum print_line_t trace_ctxwake_print(struct trace_iterator *iter, | |||
| 932 | return TRACE_TYPE_HANDLED; | 944 | return TRACE_TYPE_HANDLED; |
| 933 | } | 945 | } |
| 934 | 946 | ||
| 935 | static enum print_line_t trace_ctx_print(struct trace_iterator *iter, int flags) | 947 | static enum print_line_t trace_ctx_print(struct trace_iterator *iter, int flags, |
| 948 | struct trace_event *event) | ||
| 936 | { | 949 | { |
| 937 | return trace_ctxwake_print(iter, "==>"); | 950 | return trace_ctxwake_print(iter, "==>"); |
| 938 | } | 951 | } |
| 939 | 952 | ||
| 940 | static enum print_line_t trace_wake_print(struct trace_iterator *iter, | 953 | static enum print_line_t trace_wake_print(struct trace_iterator *iter, |
| 941 | int flags) | 954 | int flags, struct trace_event *event) |
| 942 | { | 955 | { |
| 943 | return trace_ctxwake_print(iter, " +"); | 956 | return trace_ctxwake_print(iter, " +"); |
| 944 | } | 957 | } |
| @@ -966,12 +979,14 @@ static int trace_ctxwake_raw(struct trace_iterator *iter, char S) | |||
| 966 | return TRACE_TYPE_HANDLED; | 979 | return TRACE_TYPE_HANDLED; |
| 967 | } | 980 | } |
| 968 | 981 | ||
| 969 | static enum print_line_t trace_ctx_raw(struct trace_iterator *iter, int flags) | 982 | static enum print_line_t trace_ctx_raw(struct trace_iterator *iter, int flags, |
| 983 | struct trace_event *event) | ||
| 970 | { | 984 | { |
| 971 | return trace_ctxwake_raw(iter, 0); | 985 | return trace_ctxwake_raw(iter, 0); |
| 972 | } | 986 | } |
| 973 | 987 | ||
| 974 | static enum print_line_t trace_wake_raw(struct trace_iterator *iter, int flags) | 988 | static enum print_line_t trace_wake_raw(struct trace_iterator *iter, int flags, |
| 989 | struct trace_event *event) | ||
| 975 | { | 990 | { |
| 976 | return trace_ctxwake_raw(iter, '+'); | 991 | return trace_ctxwake_raw(iter, '+'); |
| 977 | } | 992 | } |
| @@ -1000,18 +1015,20 @@ static int trace_ctxwake_hex(struct trace_iterator *iter, char S) | |||
| 1000 | return TRACE_TYPE_HANDLED; | 1015 | return TRACE_TYPE_HANDLED; |
| 1001 | } | 1016 | } |
| 1002 | 1017 | ||
| 1003 | static enum print_line_t trace_ctx_hex(struct trace_iterator *iter, int flags) | 1018 | static enum print_line_t trace_ctx_hex(struct trace_iterator *iter, int flags, |
| 1019 | struct trace_event *event) | ||
| 1004 | { | 1020 | { |
| 1005 | return trace_ctxwake_hex(iter, 0); | 1021 | return trace_ctxwake_hex(iter, 0); |
| 1006 | } | 1022 | } |
| 1007 | 1023 | ||
| 1008 | static enum print_line_t trace_wake_hex(struct trace_iterator *iter, int flags) | 1024 | static enum print_line_t trace_wake_hex(struct trace_iterator *iter, int flags, |
| 1025 | struct trace_event *event) | ||
| 1009 | { | 1026 | { |
| 1010 | return trace_ctxwake_hex(iter, '+'); | 1027 | return trace_ctxwake_hex(iter, '+'); |
| 1011 | } | 1028 | } |
| 1012 | 1029 | ||
| 1013 | static enum print_line_t trace_ctxwake_bin(struct trace_iterator *iter, | 1030 | static enum print_line_t trace_ctxwake_bin(struct trace_iterator *iter, |
| 1014 | int flags) | 1031 | int flags, struct trace_event *event) |
| 1015 | { | 1032 | { |
| 1016 | struct ctx_switch_entry *field; | 1033 | struct ctx_switch_entry *field; |
| 1017 | struct trace_seq *s = &iter->seq; | 1034 | struct trace_seq *s = &iter->seq; |
| @@ -1028,25 +1045,33 @@ static enum print_line_t trace_ctxwake_bin(struct trace_iterator *iter, | |||
| 1028 | return TRACE_TYPE_HANDLED; | 1045 | return TRACE_TYPE_HANDLED; |
| 1029 | } | 1046 | } |
| 1030 | 1047 | ||
| 1031 | static struct trace_event trace_ctx_event = { | 1048 | static struct trace_event_functions trace_ctx_funcs = { |
| 1032 | .type = TRACE_CTX, | ||
| 1033 | .trace = trace_ctx_print, | 1049 | .trace = trace_ctx_print, |
| 1034 | .raw = trace_ctx_raw, | 1050 | .raw = trace_ctx_raw, |
| 1035 | .hex = trace_ctx_hex, | 1051 | .hex = trace_ctx_hex, |
| 1036 | .binary = trace_ctxwake_bin, | 1052 | .binary = trace_ctxwake_bin, |
| 1037 | }; | 1053 | }; |
| 1038 | 1054 | ||
| 1039 | static struct trace_event trace_wake_event = { | 1055 | static struct trace_event trace_ctx_event = { |
| 1040 | .type = TRACE_WAKE, | 1056 | .type = TRACE_CTX, |
| 1057 | .funcs = &trace_ctx_funcs, | ||
| 1058 | }; | ||
| 1059 | |||
| 1060 | static struct trace_event_functions trace_wake_funcs = { | ||
| 1041 | .trace = trace_wake_print, | 1061 | .trace = trace_wake_print, |
| 1042 | .raw = trace_wake_raw, | 1062 | .raw = trace_wake_raw, |
| 1043 | .hex = trace_wake_hex, | 1063 | .hex = trace_wake_hex, |
| 1044 | .binary = trace_ctxwake_bin, | 1064 | .binary = trace_ctxwake_bin, |
| 1045 | }; | 1065 | }; |
| 1046 | 1066 | ||
| 1067 | static struct trace_event trace_wake_event = { | ||
| 1068 | .type = TRACE_WAKE, | ||
| 1069 | .funcs = &trace_wake_funcs, | ||
| 1070 | }; | ||
| 1071 | |||
| 1047 | /* TRACE_SPECIAL */ | 1072 | /* TRACE_SPECIAL */ |
| 1048 | static enum print_line_t trace_special_print(struct trace_iterator *iter, | 1073 | static enum print_line_t trace_special_print(struct trace_iterator *iter, |
| 1049 | int flags) | 1074 | int flags, struct trace_event *event) |
| 1050 | { | 1075 | { |
| 1051 | struct special_entry *field; | 1076 | struct special_entry *field; |
| 1052 | 1077 | ||
| @@ -1062,7 +1087,7 @@ static enum print_line_t trace_special_print(struct trace_iterator *iter, | |||
| 1062 | } | 1087 | } |
| 1063 | 1088 | ||
| 1064 | static enum print_line_t trace_special_hex(struct trace_iterator *iter, | 1089 | static enum print_line_t trace_special_hex(struct trace_iterator *iter, |
| 1065 | int flags) | 1090 | int flags, struct trace_event *event) |
| 1066 | { | 1091 | { |
| 1067 | struct special_entry *field; | 1092 | struct special_entry *field; |
| 1068 | struct trace_seq *s = &iter->seq; | 1093 | struct trace_seq *s = &iter->seq; |
| @@ -1077,7 +1102,7 @@ static enum print_line_t trace_special_hex(struct trace_iterator *iter, | |||
| 1077 | } | 1102 | } |
| 1078 | 1103 | ||
| 1079 | static enum print_line_t trace_special_bin(struct trace_iterator *iter, | 1104 | static enum print_line_t trace_special_bin(struct trace_iterator *iter, |
| 1080 | int flags) | 1105 | int flags, struct trace_event *event) |
| 1081 | { | 1106 | { |
| 1082 | struct special_entry *field; | 1107 | struct special_entry *field; |
| 1083 | struct trace_seq *s = &iter->seq; | 1108 | struct trace_seq *s = &iter->seq; |
| @@ -1091,18 +1116,22 @@ static enum print_line_t trace_special_bin(struct trace_iterator *iter, | |||
| 1091 | return TRACE_TYPE_HANDLED; | 1116 | return TRACE_TYPE_HANDLED; |
| 1092 | } | 1117 | } |
| 1093 | 1118 | ||
| 1094 | static struct trace_event trace_special_event = { | 1119 | static struct trace_event_functions trace_special_funcs = { |
| 1095 | .type = TRACE_SPECIAL, | ||
| 1096 | .trace = trace_special_print, | 1120 | .trace = trace_special_print, |
| 1097 | .raw = trace_special_print, | 1121 | .raw = trace_special_print, |
| 1098 | .hex = trace_special_hex, | 1122 | .hex = trace_special_hex, |
| 1099 | .binary = trace_special_bin, | 1123 | .binary = trace_special_bin, |
| 1100 | }; | 1124 | }; |
| 1101 | 1125 | ||
| 1126 | static struct trace_event trace_special_event = { | ||
| 1127 | .type = TRACE_SPECIAL, | ||
| 1128 | .funcs = &trace_special_funcs, | ||
| 1129 | }; | ||
| 1130 | |||
| 1102 | /* TRACE_STACK */ | 1131 | /* TRACE_STACK */ |
| 1103 | 1132 | ||
| 1104 | static enum print_line_t trace_stack_print(struct trace_iterator *iter, | 1133 | static enum print_line_t trace_stack_print(struct trace_iterator *iter, |
| 1105 | int flags) | 1134 | int flags, struct trace_event *event) |
| 1106 | { | 1135 | { |
| 1107 | struct stack_entry *field; | 1136 | struct stack_entry *field; |
| 1108 | struct trace_seq *s = &iter->seq; | 1137 | struct trace_seq *s = &iter->seq; |
| @@ -1130,17 +1159,21 @@ static enum print_line_t trace_stack_print(struct trace_iterator *iter, | |||
| 1130 | return TRACE_TYPE_PARTIAL_LINE; | 1159 | return TRACE_TYPE_PARTIAL_LINE; |
| 1131 | } | 1160 | } |
| 1132 | 1161 | ||
| 1133 | static struct trace_event trace_stack_event = { | 1162 | static struct trace_event_functions trace_stack_funcs = { |
| 1134 | .type = TRACE_STACK, | ||
| 1135 | .trace = trace_stack_print, | 1163 | .trace = trace_stack_print, |
| 1136 | .raw = trace_special_print, | 1164 | .raw = trace_special_print, |
| 1137 | .hex = trace_special_hex, | 1165 | .hex = trace_special_hex, |
| 1138 | .binary = trace_special_bin, | 1166 | .binary = trace_special_bin, |
| 1139 | }; | 1167 | }; |
| 1140 | 1168 | ||
| 1169 | static struct trace_event trace_stack_event = { | ||
| 1170 | .type = TRACE_STACK, | ||
| 1171 | .funcs = &trace_stack_funcs, | ||
| 1172 | }; | ||
| 1173 | |||
| 1141 | /* TRACE_USER_STACK */ | 1174 | /* TRACE_USER_STACK */ |
| 1142 | static enum print_line_t trace_user_stack_print(struct trace_iterator *iter, | 1175 | static enum print_line_t trace_user_stack_print(struct trace_iterator *iter, |
| 1143 | int flags) | 1176 | int flags, struct trace_event *event) |
| 1144 | { | 1177 | { |
| 1145 | struct userstack_entry *field; | 1178 | struct userstack_entry *field; |
| 1146 | struct trace_seq *s = &iter->seq; | 1179 | struct trace_seq *s = &iter->seq; |
| @@ -1159,17 +1192,22 @@ static enum print_line_t trace_user_stack_print(struct trace_iterator *iter, | |||
| 1159 | return TRACE_TYPE_PARTIAL_LINE; | 1192 | return TRACE_TYPE_PARTIAL_LINE; |
| 1160 | } | 1193 | } |
| 1161 | 1194 | ||
| 1162 | static struct trace_event trace_user_stack_event = { | 1195 | static struct trace_event_functions trace_user_stack_funcs = { |
| 1163 | .type = TRACE_USER_STACK, | ||
| 1164 | .trace = trace_user_stack_print, | 1196 | .trace = trace_user_stack_print, |
| 1165 | .raw = trace_special_print, | 1197 | .raw = trace_special_print, |
| 1166 | .hex = trace_special_hex, | 1198 | .hex = trace_special_hex, |
| 1167 | .binary = trace_special_bin, | 1199 | .binary = trace_special_bin, |
| 1168 | }; | 1200 | }; |
| 1169 | 1201 | ||
| 1202 | static struct trace_event trace_user_stack_event = { | ||
| 1203 | .type = TRACE_USER_STACK, | ||
| 1204 | .funcs = &trace_user_stack_funcs, | ||
| 1205 | }; | ||
| 1206 | |||
| 1170 | /* TRACE_BPRINT */ | 1207 | /* TRACE_BPRINT */ |
| 1171 | static enum print_line_t | 1208 | static enum print_line_t |
| 1172 | trace_bprint_print(struct trace_iterator *iter, int flags) | 1209 | trace_bprint_print(struct trace_iterator *iter, int flags, |
| 1210 | struct trace_event *event) | ||
| 1173 | { | 1211 | { |
| 1174 | struct trace_entry *entry = iter->ent; | 1212 | struct trace_entry *entry = iter->ent; |
| 1175 | struct trace_seq *s = &iter->seq; | 1213 | struct trace_seq *s = &iter->seq; |
| @@ -1194,7 +1232,8 @@ trace_bprint_print(struct trace_iterator *iter, int flags) | |||
| 1194 | 1232 | ||
| 1195 | 1233 | ||
| 1196 | static enum print_line_t | 1234 | static enum print_line_t |
| 1197 | trace_bprint_raw(struct trace_iterator *iter, int flags) | 1235 | trace_bprint_raw(struct trace_iterator *iter, int flags, |
| 1236 | struct trace_event *event) | ||
| 1198 | { | 1237 | { |
| 1199 | struct bprint_entry *field; | 1238 | struct bprint_entry *field; |
| 1200 | struct trace_seq *s = &iter->seq; | 1239 | struct trace_seq *s = &iter->seq; |
| @@ -1213,16 +1252,19 @@ trace_bprint_raw(struct trace_iterator *iter, int flags) | |||
| 1213 | return TRACE_TYPE_PARTIAL_LINE; | 1252 | return TRACE_TYPE_PARTIAL_LINE; |
| 1214 | } | 1253 | } |
| 1215 | 1254 | ||
| 1255 | static struct trace_event_functions trace_bprint_funcs = { | ||
| 1256 | .trace = trace_bprint_print, | ||
| 1257 | .raw = trace_bprint_raw, | ||
| 1258 | }; | ||
| 1216 | 1259 | ||
| 1217 | static struct trace_event trace_bprint_event = { | 1260 | static struct trace_event trace_bprint_event = { |
| 1218 | .type = TRACE_BPRINT, | 1261 | .type = TRACE_BPRINT, |
| 1219 | .trace = trace_bprint_print, | 1262 | .funcs = &trace_bprint_funcs, |
| 1220 | .raw = trace_bprint_raw, | ||
| 1221 | }; | 1263 | }; |
| 1222 | 1264 | ||
| 1223 | /* TRACE_PRINT */ | 1265 | /* TRACE_PRINT */ |
| 1224 | static enum print_line_t trace_print_print(struct trace_iterator *iter, | 1266 | static enum print_line_t trace_print_print(struct trace_iterator *iter, |
| 1225 | int flags) | 1267 | int flags, struct trace_event *event) |
| 1226 | { | 1268 | { |
| 1227 | struct print_entry *field; | 1269 | struct print_entry *field; |
| 1228 | struct trace_seq *s = &iter->seq; | 1270 | struct trace_seq *s = &iter->seq; |
| @@ -1241,7 +1283,8 @@ static enum print_line_t trace_print_print(struct trace_iterator *iter, | |||
| 1241 | return TRACE_TYPE_PARTIAL_LINE; | 1283 | return TRACE_TYPE_PARTIAL_LINE; |
| 1242 | } | 1284 | } |
| 1243 | 1285 | ||
| 1244 | static enum print_line_t trace_print_raw(struct trace_iterator *iter, int flags) | 1286 | static enum print_line_t trace_print_raw(struct trace_iterator *iter, int flags, |
| 1287 | struct trace_event *event) | ||
| 1245 | { | 1288 | { |
| 1246 | struct print_entry *field; | 1289 | struct print_entry *field; |
| 1247 | 1290 | ||
| @@ -1256,12 +1299,16 @@ static enum print_line_t trace_print_raw(struct trace_iterator *iter, int flags) | |||
| 1256 | return TRACE_TYPE_PARTIAL_LINE; | 1299 | return TRACE_TYPE_PARTIAL_LINE; |
| 1257 | } | 1300 | } |
| 1258 | 1301 | ||
| 1259 | static struct trace_event trace_print_event = { | 1302 | static struct trace_event_functions trace_print_funcs = { |
| 1260 | .type = TRACE_PRINT, | ||
| 1261 | .trace = trace_print_print, | 1303 | .trace = trace_print_print, |
| 1262 | .raw = trace_print_raw, | 1304 | .raw = trace_print_raw, |
| 1263 | }; | 1305 | }; |
| 1264 | 1306 | ||
| 1307 | static struct trace_event trace_print_event = { | ||
| 1308 | .type = TRACE_PRINT, | ||
| 1309 | .funcs = &trace_print_funcs, | ||
| 1310 | }; | ||
| 1311 | |||
| 1265 | 1312 | ||
| 1266 | static struct trace_event *events[] __initdata = { | 1313 | static struct trace_event *events[] __initdata = { |
| 1267 | &trace_fn_event, | 1314 | &trace_fn_event, |
diff --git a/kernel/trace/trace_output.h b/kernel/trace/trace_output.h index 9d91c72ba38b..c038eba0492b 100644 --- a/kernel/trace/trace_output.h +++ b/kernel/trace/trace_output.h | |||
| @@ -25,7 +25,7 @@ extern void trace_event_read_unlock(void); | |||
| 25 | extern struct trace_event *ftrace_find_event(int type); | 25 | extern struct trace_event *ftrace_find_event(int type); |
| 26 | 26 | ||
| 27 | extern enum print_line_t trace_nop_print(struct trace_iterator *iter, | 27 | extern enum print_line_t trace_nop_print(struct trace_iterator *iter, |
| 28 | int flags); | 28 | int flags, struct trace_event *event); |
| 29 | extern int | 29 | extern int |
| 30 | trace_print_lat_fmt(struct trace_seq *s, struct trace_entry *entry); | 30 | trace_print_lat_fmt(struct trace_seq *s, struct trace_entry *entry); |
| 31 | 31 | ||
diff --git a/kernel/trace/trace_sched_switch.c b/kernel/trace/trace_sched_switch.c index a55fccfede5d..8f758d070c43 100644 --- a/kernel/trace/trace_sched_switch.c +++ b/kernel/trace/trace_sched_switch.c | |||
| @@ -50,7 +50,7 @@ tracing_sched_switch_trace(struct trace_array *tr, | |||
| 50 | } | 50 | } |
| 51 | 51 | ||
| 52 | static void | 52 | static void |
| 53 | probe_sched_switch(struct task_struct *prev, struct task_struct *next) | 53 | probe_sched_switch(void *ignore, struct task_struct *prev, struct task_struct *next) |
| 54 | { | 54 | { |
| 55 | struct trace_array_cpu *data; | 55 | struct trace_array_cpu *data; |
| 56 | unsigned long flags; | 56 | unsigned long flags; |
| @@ -108,7 +108,7 @@ tracing_sched_wakeup_trace(struct trace_array *tr, | |||
| 108 | } | 108 | } |
| 109 | 109 | ||
| 110 | static void | 110 | static void |
| 111 | probe_sched_wakeup(struct task_struct *wakee, int success) | 111 | probe_sched_wakeup(void *ignore, struct task_struct *wakee, int success) |
| 112 | { | 112 | { |
| 113 | struct trace_array_cpu *data; | 113 | struct trace_array_cpu *data; |
| 114 | unsigned long flags; | 114 | unsigned long flags; |
| @@ -138,21 +138,21 @@ static int tracing_sched_register(void) | |||
| 138 | { | 138 | { |
| 139 | int ret; | 139 | int ret; |
| 140 | 140 | ||
| 141 | ret = register_trace_sched_wakeup(probe_sched_wakeup); | 141 | ret = register_trace_sched_wakeup(probe_sched_wakeup, NULL); |
| 142 | if (ret) { | 142 | if (ret) { |
| 143 | pr_info("wakeup trace: Couldn't activate tracepoint" | 143 | pr_info("wakeup trace: Couldn't activate tracepoint" |
| 144 | " probe to kernel_sched_wakeup\n"); | 144 | " probe to kernel_sched_wakeup\n"); |
| 145 | return ret; | 145 | return ret; |
| 146 | } | 146 | } |
| 147 | 147 | ||
| 148 | ret = register_trace_sched_wakeup_new(probe_sched_wakeup); | 148 | ret = register_trace_sched_wakeup_new(probe_sched_wakeup, NULL); |
| 149 | if (ret) { | 149 | if (ret) { |
| 150 | pr_info("wakeup trace: Couldn't activate tracepoint" | 150 | pr_info("wakeup trace: Couldn't activate tracepoint" |
| 151 | " probe to kernel_sched_wakeup_new\n"); | 151 | " probe to kernel_sched_wakeup_new\n"); |
| 152 | goto fail_deprobe; | 152 | goto fail_deprobe; |
| 153 | } | 153 | } |
| 154 | 154 | ||
| 155 | ret = register_trace_sched_switch(probe_sched_switch); | 155 | ret = register_trace_sched_switch(probe_sched_switch, NULL); |
| 156 | if (ret) { | 156 | if (ret) { |
| 157 | pr_info("sched trace: Couldn't activate tracepoint" | 157 | pr_info("sched trace: Couldn't activate tracepoint" |
| 158 | " probe to kernel_sched_switch\n"); | 158 | " probe to kernel_sched_switch\n"); |
| @@ -161,17 +161,17 @@ static int tracing_sched_register(void) | |||
| 161 | 161 | ||
| 162 | return ret; | 162 | return ret; |
| 163 | fail_deprobe_wake_new: | 163 | fail_deprobe_wake_new: |
| 164 | unregister_trace_sched_wakeup_new(probe_sched_wakeup); | 164 | unregister_trace_sched_wakeup_new(probe_sched_wakeup, NULL); |
| 165 | fail_deprobe: | 165 | fail_deprobe: |
| 166 | unregister_trace_sched_wakeup(probe_sched_wakeup); | 166 | unregister_trace_sched_wakeup(probe_sched_wakeup, NULL); |
| 167 | return ret; | 167 | return ret; |
| 168 | } | 168 | } |
| 169 | 169 | ||
| 170 | static void tracing_sched_unregister(void) | 170 | static void tracing_sched_unregister(void) |
| 171 | { | 171 | { |
| 172 | unregister_trace_sched_switch(probe_sched_switch); | 172 | unregister_trace_sched_switch(probe_sched_switch, NULL); |
| 173 | unregister_trace_sched_wakeup_new(probe_sched_wakeup); | 173 | unregister_trace_sched_wakeup_new(probe_sched_wakeup, NULL); |
| 174 | unregister_trace_sched_wakeup(probe_sched_wakeup); | 174 | unregister_trace_sched_wakeup(probe_sched_wakeup, NULL); |
| 175 | } | 175 | } |
| 176 | 176 | ||
| 177 | static void tracing_start_sched_switch(void) | 177 | static void tracing_start_sched_switch(void) |
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c index 8052446ceeaa..0e73bc2ef8c5 100644 --- a/kernel/trace/trace_sched_wakeup.c +++ b/kernel/trace/trace_sched_wakeup.c | |||
| @@ -98,7 +98,8 @@ static int report_latency(cycle_t delta) | |||
| 98 | return 1; | 98 | return 1; |
| 99 | } | 99 | } |
| 100 | 100 | ||
| 101 | static void probe_wakeup_migrate_task(struct task_struct *task, int cpu) | 101 | static void |
| 102 | probe_wakeup_migrate_task(void *ignore, struct task_struct *task, int cpu) | ||
| 102 | { | 103 | { |
| 103 | if (task != wakeup_task) | 104 | if (task != wakeup_task) |
| 104 | return; | 105 | return; |
| @@ -107,7 +108,8 @@ static void probe_wakeup_migrate_task(struct task_struct *task, int cpu) | |||
| 107 | } | 108 | } |
| 108 | 109 | ||
| 109 | static void notrace | 110 | static void notrace |
| 110 | probe_wakeup_sched_switch(struct task_struct *prev, struct task_struct *next) | 111 | probe_wakeup_sched_switch(void *ignore, |
| 112 | struct task_struct *prev, struct task_struct *next) | ||
| 111 | { | 113 | { |
| 112 | struct trace_array_cpu *data; | 114 | struct trace_array_cpu *data; |
| 113 | cycle_t T0, T1, delta; | 115 | cycle_t T0, T1, delta; |
| @@ -199,7 +201,7 @@ static void wakeup_reset(struct trace_array *tr) | |||
| 199 | } | 201 | } |
| 200 | 202 | ||
| 201 | static void | 203 | static void |
| 202 | probe_wakeup(struct task_struct *p, int success) | 204 | probe_wakeup(void *ignore, struct task_struct *p, int success) |
| 203 | { | 205 | { |
| 204 | struct trace_array_cpu *data; | 206 | struct trace_array_cpu *data; |
| 205 | int cpu = smp_processor_id(); | 207 | int cpu = smp_processor_id(); |
| @@ -263,28 +265,28 @@ static void start_wakeup_tracer(struct trace_array *tr) | |||
| 263 | { | 265 | { |
| 264 | int ret; | 266 | int ret; |
| 265 | 267 | ||
| 266 | ret = register_trace_sched_wakeup(probe_wakeup); | 268 | ret = register_trace_sched_wakeup(probe_wakeup, NULL); |
| 267 | if (ret) { | 269 | if (ret) { |
| 268 | pr_info("wakeup trace: Couldn't activate tracepoint" | 270 | pr_info("wakeup trace: Couldn't activate tracepoint" |
| 269 | " probe to kernel_sched_wakeup\n"); | 271 | " probe to kernel_sched_wakeup\n"); |
| 270 | return; | 272 | return; |
| 271 | } | 273 | } |
| 272 | 274 | ||
| 273 | ret = register_trace_sched_wakeup_new(probe_wakeup); | 275 | ret = register_trace_sched_wakeup_new(probe_wakeup, NULL); |
| 274 | if (ret) { | 276 | if (ret) { |
| 275 | pr_info("wakeup trace: Couldn't activate tracepoint" | 277 | pr_info("wakeup trace: Couldn't activate tracepoint" |
| 276 | " probe to kernel_sched_wakeup_new\n"); | 278 | " probe to kernel_sched_wakeup_new\n"); |
| 277 | goto fail_deprobe; | 279 | goto fail_deprobe; |
| 278 | } | 280 | } |
| 279 | 281 | ||
| 280 | ret = register_trace_sched_switch(probe_wakeup_sched_switch); | 282 | ret = register_trace_sched_switch(probe_wakeup_sched_switch, NULL); |
| 281 | if (ret) { | 283 | if (ret) { |
| 282 | pr_info("sched trace: Couldn't activate tracepoint" | 284 | pr_info("sched trace: Couldn't activate tracepoint" |
| 283 | " probe to kernel_sched_switch\n"); | 285 | " probe to kernel_sched_switch\n"); |
| 284 | goto fail_deprobe_wake_new; | 286 | goto fail_deprobe_wake_new; |
| 285 | } | 287 | } |
| 286 | 288 | ||
| 287 | ret = register_trace_sched_migrate_task(probe_wakeup_migrate_task); | 289 | ret = register_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL); |
| 288 | if (ret) { | 290 | if (ret) { |
| 289 | pr_info("wakeup trace: Couldn't activate tracepoint" | 291 | pr_info("wakeup trace: Couldn't activate tracepoint" |
| 290 | " probe to kernel_sched_migrate_task\n"); | 292 | " probe to kernel_sched_migrate_task\n"); |
| @@ -311,19 +313,19 @@ static void start_wakeup_tracer(struct trace_array *tr) | |||
| 311 | 313 | ||
| 312 | return; | 314 | return; |
| 313 | fail_deprobe_wake_new: | 315 | fail_deprobe_wake_new: |
| 314 | unregister_trace_sched_wakeup_new(probe_wakeup); | 316 | unregister_trace_sched_wakeup_new(probe_wakeup, NULL); |
| 315 | fail_deprobe: | 317 | fail_deprobe: |
| 316 | unregister_trace_sched_wakeup(probe_wakeup); | 318 | unregister_trace_sched_wakeup(probe_wakeup, NULL); |
| 317 | } | 319 | } |
| 318 | 320 | ||
| 319 | static void stop_wakeup_tracer(struct trace_array *tr) | 321 | static void stop_wakeup_tracer(struct trace_array *tr) |
| 320 | { | 322 | { |
| 321 | tracer_enabled = 0; | 323 | tracer_enabled = 0; |
| 322 | unregister_ftrace_function(&trace_ops); | 324 | unregister_ftrace_function(&trace_ops); |
| 323 | unregister_trace_sched_switch(probe_wakeup_sched_switch); | 325 | unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL); |
| 324 | unregister_trace_sched_wakeup_new(probe_wakeup); | 326 | unregister_trace_sched_wakeup_new(probe_wakeup, NULL); |
| 325 | unregister_trace_sched_wakeup(probe_wakeup); | 327 | unregister_trace_sched_wakeup(probe_wakeup, NULL); |
| 326 | unregister_trace_sched_migrate_task(probe_wakeup_migrate_task); | 328 | unregister_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL); |
| 327 | } | 329 | } |
| 328 | 330 | ||
| 329 | static int __wakeup_tracer_init(struct trace_array *tr) | 331 | static int __wakeup_tracer_init(struct trace_array *tr) |
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index 4d6d711717f2..d2c859cec9ea 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c | |||
| @@ -15,6 +15,54 @@ static int sys_refcount_exit; | |||
| 15 | static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls); | 15 | static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls); |
| 16 | static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls); | 16 | static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls); |
| 17 | 17 | ||
| 18 | static int syscall_enter_register(struct ftrace_event_call *event, | ||
| 19 | enum trace_reg type); | ||
| 20 | static int syscall_exit_register(struct ftrace_event_call *event, | ||
| 21 | enum trace_reg type); | ||
| 22 | |||
| 23 | static int syscall_enter_define_fields(struct ftrace_event_call *call); | ||
| 24 | static int syscall_exit_define_fields(struct ftrace_event_call *call); | ||
| 25 | |||
| 26 | static struct list_head * | ||
| 27 | syscall_get_enter_fields(struct ftrace_event_call *call) | ||
| 28 | { | ||
| 29 | struct syscall_metadata *entry = call->data; | ||
| 30 | |||
| 31 | return &entry->enter_fields; | ||
| 32 | } | ||
| 33 | |||
| 34 | static struct list_head * | ||
| 35 | syscall_get_exit_fields(struct ftrace_event_call *call) | ||
| 36 | { | ||
| 37 | struct syscall_metadata *entry = call->data; | ||
| 38 | |||
| 39 | return &entry->exit_fields; | ||
| 40 | } | ||
| 41 | |||
| 42 | struct trace_event_functions enter_syscall_print_funcs = { | ||
| 43 | .trace = print_syscall_enter, | ||
| 44 | }; | ||
| 45 | |||
| 46 | struct trace_event_functions exit_syscall_print_funcs = { | ||
| 47 | .trace = print_syscall_exit, | ||
| 48 | }; | ||
| 49 | |||
| 50 | struct ftrace_event_class event_class_syscall_enter = { | ||
| 51 | .system = "syscalls", | ||
| 52 | .reg = syscall_enter_register, | ||
| 53 | .define_fields = syscall_enter_define_fields, | ||
| 54 | .get_fields = syscall_get_enter_fields, | ||
| 55 | .raw_init = init_syscall_trace, | ||
| 56 | }; | ||
| 57 | |||
| 58 | struct ftrace_event_class event_class_syscall_exit = { | ||
| 59 | .system = "syscalls", | ||
| 60 | .reg = syscall_exit_register, | ||
| 61 | .define_fields = syscall_exit_define_fields, | ||
| 62 | .get_fields = syscall_get_exit_fields, | ||
| 63 | .raw_init = init_syscall_trace, | ||
| 64 | }; | ||
| 65 | |||
| 18 | extern unsigned long __start_syscalls_metadata[]; | 66 | extern unsigned long __start_syscalls_metadata[]; |
| 19 | extern unsigned long __stop_syscalls_metadata[]; | 67 | extern unsigned long __stop_syscalls_metadata[]; |
| 20 | 68 | ||
| @@ -53,7 +101,8 @@ static struct syscall_metadata *syscall_nr_to_meta(int nr) | |||
| 53 | } | 101 | } |
| 54 | 102 | ||
| 55 | enum print_line_t | 103 | enum print_line_t |
| 56 | print_syscall_enter(struct trace_iterator *iter, int flags) | 104 | print_syscall_enter(struct trace_iterator *iter, int flags, |
| 105 | struct trace_event *event) | ||
| 57 | { | 106 | { |
| 58 | struct trace_seq *s = &iter->seq; | 107 | struct trace_seq *s = &iter->seq; |
| 59 | struct trace_entry *ent = iter->ent; | 108 | struct trace_entry *ent = iter->ent; |
| @@ -68,7 +117,7 @@ print_syscall_enter(struct trace_iterator *iter, int flags) | |||
| 68 | if (!entry) | 117 | if (!entry) |
| 69 | goto end; | 118 | goto end; |
| 70 | 119 | ||
| 71 | if (entry->enter_event->id != ent->type) { | 120 | if (entry->enter_event->event.type != ent->type) { |
| 72 | WARN_ON_ONCE(1); | 121 | WARN_ON_ONCE(1); |
| 73 | goto end; | 122 | goto end; |
| 74 | } | 123 | } |
| @@ -105,7 +154,8 @@ end: | |||
| 105 | } | 154 | } |
| 106 | 155 | ||
| 107 | enum print_line_t | 156 | enum print_line_t |
| 108 | print_syscall_exit(struct trace_iterator *iter, int flags) | 157 | print_syscall_exit(struct trace_iterator *iter, int flags, |
| 158 | struct trace_event *event) | ||
| 109 | { | 159 | { |
| 110 | struct trace_seq *s = &iter->seq; | 160 | struct trace_seq *s = &iter->seq; |
| 111 | struct trace_entry *ent = iter->ent; | 161 | struct trace_entry *ent = iter->ent; |
| @@ -123,7 +173,7 @@ print_syscall_exit(struct trace_iterator *iter, int flags) | |||
| 123 | return TRACE_TYPE_HANDLED; | 173 | return TRACE_TYPE_HANDLED; |
| 124 | } | 174 | } |
| 125 | 175 | ||
| 126 | if (entry->exit_event->id != ent->type) { | 176 | if (entry->exit_event->event.type != ent->type) { |
| 127 | WARN_ON_ONCE(1); | 177 | WARN_ON_ONCE(1); |
| 128 | return TRACE_TYPE_UNHANDLED; | 178 | return TRACE_TYPE_UNHANDLED; |
| 129 | } | 179 | } |
| @@ -205,7 +255,7 @@ static void free_syscall_print_fmt(struct ftrace_event_call *call) | |||
| 205 | kfree(call->print_fmt); | 255 | kfree(call->print_fmt); |
| 206 | } | 256 | } |
| 207 | 257 | ||
| 208 | int syscall_enter_define_fields(struct ftrace_event_call *call) | 258 | static int syscall_enter_define_fields(struct ftrace_event_call *call) |
| 209 | { | 259 | { |
| 210 | struct syscall_trace_enter trace; | 260 | struct syscall_trace_enter trace; |
| 211 | struct syscall_metadata *meta = call->data; | 261 | struct syscall_metadata *meta = call->data; |
| @@ -228,7 +278,7 @@ int syscall_enter_define_fields(struct ftrace_event_call *call) | |||
| 228 | return ret; | 278 | return ret; |
| 229 | } | 279 | } |
| 230 | 280 | ||
| 231 | int syscall_exit_define_fields(struct ftrace_event_call *call) | 281 | static int syscall_exit_define_fields(struct ftrace_event_call *call) |
| 232 | { | 282 | { |
| 233 | struct syscall_trace_exit trace; | 283 | struct syscall_trace_exit trace; |
| 234 | int ret; | 284 | int ret; |
| @@ -243,7 +293,7 @@ int syscall_exit_define_fields(struct ftrace_event_call *call) | |||
| 243 | return ret; | 293 | return ret; |
| 244 | } | 294 | } |
| 245 | 295 | ||
| 246 | void ftrace_syscall_enter(struct pt_regs *regs, long id) | 296 | void ftrace_syscall_enter(void *ignore, struct pt_regs *regs, long id) |
| 247 | { | 297 | { |
| 248 | struct syscall_trace_enter *entry; | 298 | struct syscall_trace_enter *entry; |
| 249 | struct syscall_metadata *sys_data; | 299 | struct syscall_metadata *sys_data; |
| @@ -265,7 +315,7 @@ void ftrace_syscall_enter(struct pt_regs *regs, long id) | |||
| 265 | size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args; | 315 | size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args; |
| 266 | 316 | ||
| 267 | event = trace_current_buffer_lock_reserve(&buffer, | 317 | event = trace_current_buffer_lock_reserve(&buffer, |
| 268 | sys_data->enter_event->id, size, 0, 0); | 318 | sys_data->enter_event->event.type, size, 0, 0); |
| 269 | if (!event) | 319 | if (!event) |
| 270 | return; | 320 | return; |
| 271 | 321 | ||
| @@ -278,7 +328,7 @@ void ftrace_syscall_enter(struct pt_regs *regs, long id) | |||
| 278 | trace_current_buffer_unlock_commit(buffer, event, 0, 0); | 328 | trace_current_buffer_unlock_commit(buffer, event, 0, 0); |
| 279 | } | 329 | } |
| 280 | 330 | ||
| 281 | void ftrace_syscall_exit(struct pt_regs *regs, long ret) | 331 | void ftrace_syscall_exit(void *ignore, struct pt_regs *regs, long ret) |
| 282 | { | 332 | { |
| 283 | struct syscall_trace_exit *entry; | 333 | struct syscall_trace_exit *entry; |
| 284 | struct syscall_metadata *sys_data; | 334 | struct syscall_metadata *sys_data; |
| @@ -297,7 +347,7 @@ void ftrace_syscall_exit(struct pt_regs *regs, long ret) | |||
| 297 | return; | 347 | return; |
| 298 | 348 | ||
| 299 | event = trace_current_buffer_lock_reserve(&buffer, | 349 | event = trace_current_buffer_lock_reserve(&buffer, |
| 300 | sys_data->exit_event->id, sizeof(*entry), 0, 0); | 350 | sys_data->exit_event->event.type, sizeof(*entry), 0, 0); |
| 301 | if (!event) | 351 | if (!event) |
| 302 | return; | 352 | return; |
| 303 | 353 | ||
| @@ -320,7 +370,7 @@ int reg_event_syscall_enter(struct ftrace_event_call *call) | |||
| 320 | return -ENOSYS; | 370 | return -ENOSYS; |
| 321 | mutex_lock(&syscall_trace_lock); | 371 | mutex_lock(&syscall_trace_lock); |
| 322 | if (!sys_refcount_enter) | 372 | if (!sys_refcount_enter) |
| 323 | ret = register_trace_sys_enter(ftrace_syscall_enter); | 373 | ret = register_trace_sys_enter(ftrace_syscall_enter, NULL); |
| 324 | if (!ret) { | 374 | if (!ret) { |
| 325 | set_bit(num, enabled_enter_syscalls); | 375 | set_bit(num, enabled_enter_syscalls); |
| 326 | sys_refcount_enter++; | 376 | sys_refcount_enter++; |
| @@ -340,7 +390,7 @@ void unreg_event_syscall_enter(struct ftrace_event_call *call) | |||
| 340 | sys_refcount_enter--; | 390 | sys_refcount_enter--; |
| 341 | clear_bit(num, enabled_enter_syscalls); | 391 | clear_bit(num, enabled_enter_syscalls); |
| 342 | if (!sys_refcount_enter) | 392 | if (!sys_refcount_enter) |
| 343 | unregister_trace_sys_enter(ftrace_syscall_enter); | 393 | unregister_trace_sys_enter(ftrace_syscall_enter, NULL); |
| 344 | mutex_unlock(&syscall_trace_lock); | 394 | mutex_unlock(&syscall_trace_lock); |
| 345 | } | 395 | } |
| 346 | 396 | ||
| @@ -354,7 +404,7 @@ int reg_event_syscall_exit(struct ftrace_event_call *call) | |||
| 354 | return -ENOSYS; | 404 | return -ENOSYS; |
| 355 | mutex_lock(&syscall_trace_lock); | 405 | mutex_lock(&syscall_trace_lock); |
| 356 | if (!sys_refcount_exit) | 406 | if (!sys_refcount_exit) |
| 357 | ret = register_trace_sys_exit(ftrace_syscall_exit); | 407 | ret = register_trace_sys_exit(ftrace_syscall_exit, NULL); |
| 358 | if (!ret) { | 408 | if (!ret) { |
| 359 | set_bit(num, enabled_exit_syscalls); | 409 | set_bit(num, enabled_exit_syscalls); |
| 360 | sys_refcount_exit++; | 410 | sys_refcount_exit++; |
| @@ -374,7 +424,7 @@ void unreg_event_syscall_exit(struct ftrace_event_call *call) | |||
| 374 | sys_refcount_exit--; | 424 | sys_refcount_exit--; |
| 375 | clear_bit(num, enabled_exit_syscalls); | 425 | clear_bit(num, enabled_exit_syscalls); |
| 376 | if (!sys_refcount_exit) | 426 | if (!sys_refcount_exit) |
| 377 | unregister_trace_sys_exit(ftrace_syscall_exit); | 427 | unregister_trace_sys_exit(ftrace_syscall_exit, NULL); |
| 378 | mutex_unlock(&syscall_trace_lock); | 428 | mutex_unlock(&syscall_trace_lock); |
| 379 | } | 429 | } |
| 380 | 430 | ||
| @@ -434,11 +484,11 @@ static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls); | |||
| 434 | static int sys_perf_refcount_enter; | 484 | static int sys_perf_refcount_enter; |
| 435 | static int sys_perf_refcount_exit; | 485 | static int sys_perf_refcount_exit; |
| 436 | 486 | ||
| 437 | static void perf_syscall_enter(struct pt_regs *regs, long id) | 487 | static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) |
| 438 | { | 488 | { |
| 439 | struct syscall_metadata *sys_data; | 489 | struct syscall_metadata *sys_data; |
| 440 | struct syscall_trace_enter *rec; | 490 | struct syscall_trace_enter *rec; |
| 441 | unsigned long flags; | 491 | struct hlist_head *head; |
| 442 | int syscall_nr; | 492 | int syscall_nr; |
| 443 | int rctx; | 493 | int rctx; |
| 444 | int size; | 494 | int size; |
| @@ -461,14 +511,16 @@ static void perf_syscall_enter(struct pt_regs *regs, long id) | |||
| 461 | return; | 511 | return; |
| 462 | 512 | ||
| 463 | rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size, | 513 | rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size, |
| 464 | sys_data->enter_event->id, &rctx, &flags); | 514 | sys_data->enter_event->event.type, regs, &rctx); |
| 465 | if (!rec) | 515 | if (!rec) |
| 466 | return; | 516 | return; |
| 467 | 517 | ||
| 468 | rec->nr = syscall_nr; | 518 | rec->nr = syscall_nr; |
| 469 | syscall_get_arguments(current, regs, 0, sys_data->nb_args, | 519 | syscall_get_arguments(current, regs, 0, sys_data->nb_args, |
| 470 | (unsigned long *)&rec->args); | 520 | (unsigned long *)&rec->args); |
| 471 | perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs); | 521 | |
| 522 | head = per_cpu_ptr(sys_data->enter_event->perf_events, smp_processor_id()); | ||
| 523 | perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head); | ||
| 472 | } | 524 | } |
| 473 | 525 | ||
| 474 | int perf_sysenter_enable(struct ftrace_event_call *call) | 526 | int perf_sysenter_enable(struct ftrace_event_call *call) |
| @@ -480,7 +532,7 @@ int perf_sysenter_enable(struct ftrace_event_call *call) | |||
| 480 | 532 | ||
| 481 | mutex_lock(&syscall_trace_lock); | 533 | mutex_lock(&syscall_trace_lock); |
| 482 | if (!sys_perf_refcount_enter) | 534 | if (!sys_perf_refcount_enter) |
| 483 | ret = register_trace_sys_enter(perf_syscall_enter); | 535 | ret = register_trace_sys_enter(perf_syscall_enter, NULL); |
| 484 | if (ret) { | 536 | if (ret) { |
| 485 | pr_info("event trace: Could not activate" | 537 | pr_info("event trace: Could not activate" |
| 486 | "syscall entry trace point"); | 538 | "syscall entry trace point"); |
| @@ -502,15 +554,15 @@ void perf_sysenter_disable(struct ftrace_event_call *call) | |||
| 502 | sys_perf_refcount_enter--; | 554 | sys_perf_refcount_enter--; |
| 503 | clear_bit(num, enabled_perf_enter_syscalls); | 555 | clear_bit(num, enabled_perf_enter_syscalls); |
| 504 | if (!sys_perf_refcount_enter) | 556 | if (!sys_perf_refcount_enter) |
| 505 | unregister_trace_sys_enter(perf_syscall_enter); | 557 | unregister_trace_sys_enter(perf_syscall_enter, NULL); |
| 506 | mutex_unlock(&syscall_trace_lock); | 558 | mutex_unlock(&syscall_trace_lock); |
| 507 | } | 559 | } |
| 508 | 560 | ||
| 509 | static void perf_syscall_exit(struct pt_regs *regs, long ret) | 561 | static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret) |
| 510 | { | 562 | { |
| 511 | struct syscall_metadata *sys_data; | 563 | struct syscall_metadata *sys_data; |
| 512 | struct syscall_trace_exit *rec; | 564 | struct syscall_trace_exit *rec; |
| 513 | unsigned long flags; | 565 | struct hlist_head *head; |
| 514 | int syscall_nr; | 566 | int syscall_nr; |
| 515 | int rctx; | 567 | int rctx; |
| 516 | int size; | 568 | int size; |
| @@ -536,14 +588,15 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret) | |||
| 536 | return; | 588 | return; |
| 537 | 589 | ||
| 538 | rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size, | 590 | rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size, |
| 539 | sys_data->exit_event->id, &rctx, &flags); | 591 | sys_data->exit_event->event.type, regs, &rctx); |
| 540 | if (!rec) | 592 | if (!rec) |
| 541 | return; | 593 | return; |
| 542 | 594 | ||
| 543 | rec->nr = syscall_nr; | 595 | rec->nr = syscall_nr; |
| 544 | rec->ret = syscall_get_return_value(current, regs); | 596 | rec->ret = syscall_get_return_value(current, regs); |
| 545 | 597 | ||
| 546 | perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs); | 598 | head = per_cpu_ptr(sys_data->exit_event->perf_events, smp_processor_id()); |
| 599 | perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head); | ||
| 547 | } | 600 | } |
| 548 | 601 | ||
| 549 | int perf_sysexit_enable(struct ftrace_event_call *call) | 602 | int perf_sysexit_enable(struct ftrace_event_call *call) |
| @@ -555,7 +608,7 @@ int perf_sysexit_enable(struct ftrace_event_call *call) | |||
| 555 | 608 | ||
| 556 | mutex_lock(&syscall_trace_lock); | 609 | mutex_lock(&syscall_trace_lock); |
| 557 | if (!sys_perf_refcount_exit) | 610 | if (!sys_perf_refcount_exit) |
| 558 | ret = register_trace_sys_exit(perf_syscall_exit); | 611 | ret = register_trace_sys_exit(perf_syscall_exit, NULL); |
| 559 | if (ret) { | 612 | if (ret) { |
| 560 | pr_info("event trace: Could not activate" | 613 | pr_info("event trace: Could not activate" |
| 561 | "syscall exit trace point"); | 614 | "syscall exit trace point"); |
| @@ -577,9 +630,50 @@ void perf_sysexit_disable(struct ftrace_event_call *call) | |||
| 577 | sys_perf_refcount_exit--; | 630 | sys_perf_refcount_exit--; |
| 578 | clear_bit(num, enabled_perf_exit_syscalls); | 631 | clear_bit(num, enabled_perf_exit_syscalls); |
| 579 | if (!sys_perf_refcount_exit) | 632 | if (!sys_perf_refcount_exit) |
| 580 | unregister_trace_sys_exit(perf_syscall_exit); | 633 | unregister_trace_sys_exit(perf_syscall_exit, NULL); |
| 581 | mutex_unlock(&syscall_trace_lock); | 634 | mutex_unlock(&syscall_trace_lock); |
| 582 | } | 635 | } |
| 583 | 636 | ||
| 584 | #endif /* CONFIG_PERF_EVENTS */ | 637 | #endif /* CONFIG_PERF_EVENTS */ |
| 585 | 638 | ||
| 639 | static int syscall_enter_register(struct ftrace_event_call *event, | ||
| 640 | enum trace_reg type) | ||
| 641 | { | ||
| 642 | switch (type) { | ||
| 643 | case TRACE_REG_REGISTER: | ||
| 644 | return reg_event_syscall_enter(event); | ||
| 645 | case TRACE_REG_UNREGISTER: | ||
| 646 | unreg_event_syscall_enter(event); | ||
| 647 | return 0; | ||
| 648 | |||
| 649 | #ifdef CONFIG_PERF_EVENTS | ||
| 650 | case TRACE_REG_PERF_REGISTER: | ||
| 651 | return perf_sysenter_enable(event); | ||
| 652 | case TRACE_REG_PERF_UNREGISTER: | ||
| 653 | perf_sysenter_disable(event); | ||
| 654 | return 0; | ||
| 655 | #endif | ||
| 656 | } | ||
| 657 | return 0; | ||
| 658 | } | ||
| 659 | |||
| 660 | static int syscall_exit_register(struct ftrace_event_call *event, | ||
| 661 | enum trace_reg type) | ||
| 662 | { | ||
| 663 | switch (type) { | ||
| 664 | case TRACE_REG_REGISTER: | ||
| 665 | return reg_event_syscall_exit(event); | ||
| 666 | case TRACE_REG_UNREGISTER: | ||
| 667 | unreg_event_syscall_exit(event); | ||
| 668 | return 0; | ||
| 669 | |||
| 670 | #ifdef CONFIG_PERF_EVENTS | ||
| 671 | case TRACE_REG_PERF_REGISTER: | ||
| 672 | return perf_sysexit_enable(event); | ||
| 673 | case TRACE_REG_PERF_UNREGISTER: | ||
| 674 | perf_sysexit_disable(event); | ||
| 675 | return 0; | ||
| 676 | #endif | ||
| 677 | } | ||
| 678 | return 0; | ||
| 679 | } | ||
diff --git a/kernel/trace/trace_workqueue.c b/kernel/trace/trace_workqueue.c index cc2d2faa7d9e..a7cc3793baf6 100644 --- a/kernel/trace/trace_workqueue.c +++ b/kernel/trace/trace_workqueue.c | |||
| @@ -49,7 +49,8 @@ static void cpu_workqueue_stat_free(struct kref *kref) | |||
| 49 | 49 | ||
| 50 | /* Insertion of a work */ | 50 | /* Insertion of a work */ |
| 51 | static void | 51 | static void |
| 52 | probe_workqueue_insertion(struct task_struct *wq_thread, | 52 | probe_workqueue_insertion(void *ignore, |
| 53 | struct task_struct *wq_thread, | ||
| 53 | struct work_struct *work) | 54 | struct work_struct *work) |
| 54 | { | 55 | { |
| 55 | int cpu = cpumask_first(&wq_thread->cpus_allowed); | 56 | int cpu = cpumask_first(&wq_thread->cpus_allowed); |
| @@ -70,7 +71,8 @@ found: | |||
| 70 | 71 | ||
| 71 | /* Execution of a work */ | 72 | /* Execution of a work */ |
| 72 | static void | 73 | static void |
| 73 | probe_workqueue_execution(struct task_struct *wq_thread, | 74 | probe_workqueue_execution(void *ignore, |
| 75 | struct task_struct *wq_thread, | ||
| 74 | struct work_struct *work) | 76 | struct work_struct *work) |
| 75 | { | 77 | { |
| 76 | int cpu = cpumask_first(&wq_thread->cpus_allowed); | 78 | int cpu = cpumask_first(&wq_thread->cpus_allowed); |
| @@ -90,7 +92,8 @@ found: | |||
| 90 | } | 92 | } |
| 91 | 93 | ||
| 92 | /* Creation of a cpu workqueue thread */ | 94 | /* Creation of a cpu workqueue thread */ |
| 93 | static void probe_workqueue_creation(struct task_struct *wq_thread, int cpu) | 95 | static void probe_workqueue_creation(void *ignore, |
| 96 | struct task_struct *wq_thread, int cpu) | ||
| 94 | { | 97 | { |
| 95 | struct cpu_workqueue_stats *cws; | 98 | struct cpu_workqueue_stats *cws; |
| 96 | unsigned long flags; | 99 | unsigned long flags; |
| @@ -114,7 +117,8 @@ static void probe_workqueue_creation(struct task_struct *wq_thread, int cpu) | |||
| 114 | } | 117 | } |
| 115 | 118 | ||
| 116 | /* Destruction of a cpu workqueue thread */ | 119 | /* Destruction of a cpu workqueue thread */ |
| 117 | static void probe_workqueue_destruction(struct task_struct *wq_thread) | 120 | static void |
| 121 | probe_workqueue_destruction(void *ignore, struct task_struct *wq_thread) | ||
| 118 | { | 122 | { |
| 119 | /* Workqueue only execute on one cpu */ | 123 | /* Workqueue only execute on one cpu */ |
| 120 | int cpu = cpumask_first(&wq_thread->cpus_allowed); | 124 | int cpu = cpumask_first(&wq_thread->cpus_allowed); |
| @@ -259,19 +263,19 @@ int __init trace_workqueue_early_init(void) | |||
| 259 | { | 263 | { |
| 260 | int ret, cpu; | 264 | int ret, cpu; |
| 261 | 265 | ||
| 262 | ret = register_trace_workqueue_insertion(probe_workqueue_insertion); | 266 | ret = register_trace_workqueue_insertion(probe_workqueue_insertion, NULL); |
| 263 | if (ret) | 267 | if (ret) |
| 264 | goto out; | 268 | goto out; |
| 265 | 269 | ||
| 266 | ret = register_trace_workqueue_execution(probe_workqueue_execution); | 270 | ret = register_trace_workqueue_execution(probe_workqueue_execution, NULL); |
| 267 | if (ret) | 271 | if (ret) |
| 268 | goto no_insertion; | 272 | goto no_insertion; |
| 269 | 273 | ||
| 270 | ret = register_trace_workqueue_creation(probe_workqueue_creation); | 274 | ret = register_trace_workqueue_creation(probe_workqueue_creation, NULL); |
| 271 | if (ret) | 275 | if (ret) |
| 272 | goto no_execution; | 276 | goto no_execution; |
| 273 | 277 | ||
| 274 | ret = register_trace_workqueue_destruction(probe_workqueue_destruction); | 278 | ret = register_trace_workqueue_destruction(probe_workqueue_destruction, NULL); |
| 275 | if (ret) | 279 | if (ret) |
| 276 | goto no_creation; | 280 | goto no_creation; |
| 277 | 281 | ||
| @@ -283,11 +287,11 @@ int __init trace_workqueue_early_init(void) | |||
| 283 | return 0; | 287 | return 0; |
| 284 | 288 | ||
| 285 | no_creation: | 289 | no_creation: |
| 286 | unregister_trace_workqueue_creation(probe_workqueue_creation); | 290 | unregister_trace_workqueue_creation(probe_workqueue_creation, NULL); |
| 287 | no_execution: | 291 | no_execution: |
| 288 | unregister_trace_workqueue_execution(probe_workqueue_execution); | 292 | unregister_trace_workqueue_execution(probe_workqueue_execution, NULL); |
| 289 | no_insertion: | 293 | no_insertion: |
| 290 | unregister_trace_workqueue_insertion(probe_workqueue_insertion); | 294 | unregister_trace_workqueue_insertion(probe_workqueue_insertion, NULL); |
| 291 | out: | 295 | out: |
| 292 | pr_warning("trace_workqueue: unable to trace workqueues\n"); | 296 | pr_warning("trace_workqueue: unable to trace workqueues\n"); |
| 293 | 297 | ||
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c index cc89be5bc0f8..c77f3eceea25 100644 --- a/kernel/tracepoint.c +++ b/kernel/tracepoint.c | |||
| @@ -54,7 +54,7 @@ static struct hlist_head tracepoint_table[TRACEPOINT_TABLE_SIZE]; | |||
| 54 | */ | 54 | */ |
| 55 | struct tracepoint_entry { | 55 | struct tracepoint_entry { |
| 56 | struct hlist_node hlist; | 56 | struct hlist_node hlist; |
| 57 | void **funcs; | 57 | struct tracepoint_func *funcs; |
| 58 | int refcount; /* Number of times armed. 0 if disarmed. */ | 58 | int refcount; /* Number of times armed. 0 if disarmed. */ |
| 59 | char name[0]; | 59 | char name[0]; |
| 60 | }; | 60 | }; |
| @@ -64,12 +64,12 @@ struct tp_probes { | |||
| 64 | struct rcu_head rcu; | 64 | struct rcu_head rcu; |
| 65 | struct list_head list; | 65 | struct list_head list; |
| 66 | } u; | 66 | } u; |
| 67 | void *probes[0]; | 67 | struct tracepoint_func probes[0]; |
| 68 | }; | 68 | }; |
| 69 | 69 | ||
| 70 | static inline void *allocate_probes(int count) | 70 | static inline void *allocate_probes(int count) |
| 71 | { | 71 | { |
| 72 | struct tp_probes *p = kmalloc(count * sizeof(void *) | 72 | struct tp_probes *p = kmalloc(count * sizeof(struct tracepoint_func) |
| 73 | + sizeof(struct tp_probes), GFP_KERNEL); | 73 | + sizeof(struct tp_probes), GFP_KERNEL); |
| 74 | return p == NULL ? NULL : p->probes; | 74 | return p == NULL ? NULL : p->probes; |
| 75 | } | 75 | } |
| @@ -79,7 +79,7 @@ static void rcu_free_old_probes(struct rcu_head *head) | |||
| 79 | kfree(container_of(head, struct tp_probes, u.rcu)); | 79 | kfree(container_of(head, struct tp_probes, u.rcu)); |
| 80 | } | 80 | } |
| 81 | 81 | ||
| 82 | static inline void release_probes(void *old) | 82 | static inline void release_probes(struct tracepoint_func *old) |
| 83 | { | 83 | { |
| 84 | if (old) { | 84 | if (old) { |
| 85 | struct tp_probes *tp_probes = container_of(old, | 85 | struct tp_probes *tp_probes = container_of(old, |
| @@ -95,15 +95,16 @@ static void debug_print_probes(struct tracepoint_entry *entry) | |||
| 95 | if (!tracepoint_debug || !entry->funcs) | 95 | if (!tracepoint_debug || !entry->funcs) |
| 96 | return; | 96 | return; |
| 97 | 97 | ||
| 98 | for (i = 0; entry->funcs[i]; i++) | 98 | for (i = 0; entry->funcs[i].func; i++) |
| 99 | printk(KERN_DEBUG "Probe %d : %p\n", i, entry->funcs[i]); | 99 | printk(KERN_DEBUG "Probe %d : %p\n", i, entry->funcs[i].func); |
| 100 | } | 100 | } |
| 101 | 101 | ||
| 102 | static void * | 102 | static struct tracepoint_func * |
| 103 | tracepoint_entry_add_probe(struct tracepoint_entry *entry, void *probe) | 103 | tracepoint_entry_add_probe(struct tracepoint_entry *entry, |
| 104 | void *probe, void *data) | ||
| 104 | { | 105 | { |
| 105 | int nr_probes = 0; | 106 | int nr_probes = 0; |
| 106 | void **old, **new; | 107 | struct tracepoint_func *old, *new; |
| 107 | 108 | ||
| 108 | WARN_ON(!probe); | 109 | WARN_ON(!probe); |
| 109 | 110 | ||
| @@ -111,8 +112,9 @@ tracepoint_entry_add_probe(struct tracepoint_entry *entry, void *probe) | |||
| 111 | old = entry->funcs; | 112 | old = entry->funcs; |
| 112 | if (old) { | 113 | if (old) { |
| 113 | /* (N -> N+1), (N != 0, 1) probes */ | 114 | /* (N -> N+1), (N != 0, 1) probes */ |
| 114 | for (nr_probes = 0; old[nr_probes]; nr_probes++) | 115 | for (nr_probes = 0; old[nr_probes].func; nr_probes++) |
| 115 | if (old[nr_probes] == probe) | 116 | if (old[nr_probes].func == probe && |
| 117 | old[nr_probes].data == data) | ||
| 116 | return ERR_PTR(-EEXIST); | 118 | return ERR_PTR(-EEXIST); |
| 117 | } | 119 | } |
| 118 | /* + 2 : one for new probe, one for NULL func */ | 120 | /* + 2 : one for new probe, one for NULL func */ |
| @@ -120,9 +122,10 @@ tracepoint_entry_add_probe(struct tracepoint_entry *entry, void *probe) | |||
| 120 | if (new == NULL) | 122 | if (new == NULL) |
| 121 | return ERR_PTR(-ENOMEM); | 123 | return ERR_PTR(-ENOMEM); |
| 122 | if (old) | 124 | if (old) |
| 123 | memcpy(new, old, nr_probes * sizeof(void *)); | 125 | memcpy(new, old, nr_probes * sizeof(struct tracepoint_func)); |
| 124 | new[nr_probes] = probe; | 126 | new[nr_probes].func = probe; |
| 125 | new[nr_probes + 1] = NULL; | 127 | new[nr_probes].data = data; |
| 128 | new[nr_probes + 1].func = NULL; | ||
| 126 | entry->refcount = nr_probes + 1; | 129 | entry->refcount = nr_probes + 1; |
| 127 | entry->funcs = new; | 130 | entry->funcs = new; |
| 128 | debug_print_probes(entry); | 131 | debug_print_probes(entry); |
| @@ -130,10 +133,11 @@ tracepoint_entry_add_probe(struct tracepoint_entry *entry, void *probe) | |||
| 130 | } | 133 | } |
| 131 | 134 | ||
| 132 | static void * | 135 | static void * |
| 133 | tracepoint_entry_remove_probe(struct tracepoint_entry *entry, void *probe) | 136 | tracepoint_entry_remove_probe(struct tracepoint_entry *entry, |
| 137 | void *probe, void *data) | ||
| 134 | { | 138 | { |
| 135 | int nr_probes = 0, nr_del = 0, i; | 139 | int nr_probes = 0, nr_del = 0, i; |
| 136 | void **old, **new; | 140 | struct tracepoint_func *old, *new; |
| 137 | 141 | ||
| 138 | old = entry->funcs; | 142 | old = entry->funcs; |
| 139 | 143 | ||
| @@ -142,8 +146,10 @@ tracepoint_entry_remove_probe(struct tracepoint_entry *entry, void *probe) | |||
| 142 | 146 | ||
| 143 | debug_print_probes(entry); | 147 | debug_print_probes(entry); |
| 144 | /* (N -> M), (N > 1, M >= 0) probes */ | 148 | /* (N -> M), (N > 1, M >= 0) probes */ |
| 145 | for (nr_probes = 0; old[nr_probes]; nr_probes++) { | 149 | for (nr_probes = 0; old[nr_probes].func; nr_probes++) { |
| 146 | if ((!probe || old[nr_probes] == probe)) | 150 | if (!probe || |
| 151 | (old[nr_probes].func == probe && | ||
| 152 | old[nr_probes].data == data)) | ||
| 147 | nr_del++; | 153 | nr_del++; |
| 148 | } | 154 | } |
| 149 | 155 | ||
| @@ -160,10 +166,11 @@ tracepoint_entry_remove_probe(struct tracepoint_entry *entry, void *probe) | |||
| 160 | new = allocate_probes(nr_probes - nr_del + 1); | 166 | new = allocate_probes(nr_probes - nr_del + 1); |
| 161 | if (new == NULL) | 167 | if (new == NULL) |
| 162 | return ERR_PTR(-ENOMEM); | 168 | return ERR_PTR(-ENOMEM); |
| 163 | for (i = 0; old[i]; i++) | 169 | for (i = 0; old[i].func; i++) |
| 164 | if ((probe && old[i] != probe)) | 170 | if (probe && |
| 171 | (old[i].func != probe || old[i].data != data)) | ||
| 165 | new[j++] = old[i]; | 172 | new[j++] = old[i]; |
| 166 | new[nr_probes - nr_del] = NULL; | 173 | new[nr_probes - nr_del].func = NULL; |
| 167 | entry->refcount = nr_probes - nr_del; | 174 | entry->refcount = nr_probes - nr_del; |
| 168 | entry->funcs = new; | 175 | entry->funcs = new; |
| 169 | } | 176 | } |
| @@ -315,18 +322,19 @@ static void tracepoint_update_probes(void) | |||
| 315 | module_update_tracepoints(); | 322 | module_update_tracepoints(); |
| 316 | } | 323 | } |
| 317 | 324 | ||
| 318 | static void *tracepoint_add_probe(const char *name, void *probe) | 325 | static struct tracepoint_func * |
| 326 | tracepoint_add_probe(const char *name, void *probe, void *data) | ||
| 319 | { | 327 | { |
| 320 | struct tracepoint_entry *entry; | 328 | struct tracepoint_entry *entry; |
| 321 | void *old; | 329 | struct tracepoint_func *old; |
| 322 | 330 | ||
| 323 | entry = get_tracepoint(name); | 331 | entry = get_tracepoint(name); |
| 324 | if (!entry) { | 332 | if (!entry) { |
| 325 | entry = add_tracepoint(name); | 333 | entry = add_tracepoint(name); |
| 326 | if (IS_ERR(entry)) | 334 | if (IS_ERR(entry)) |
| 327 | return entry; | 335 | return (struct tracepoint_func *)entry; |
| 328 | } | 336 | } |
| 329 | old = tracepoint_entry_add_probe(entry, probe); | 337 | old = tracepoint_entry_add_probe(entry, probe, data); |
| 330 | if (IS_ERR(old) && !entry->refcount) | 338 | if (IS_ERR(old) && !entry->refcount) |
| 331 | remove_tracepoint(entry); | 339 | remove_tracepoint(entry); |
| 332 | return old; | 340 | return old; |
| @@ -340,12 +348,12 @@ static void *tracepoint_add_probe(const char *name, void *probe) | |||
| 340 | * Returns 0 if ok, error value on error. | 348 | * Returns 0 if ok, error value on error. |
| 341 | * The probe address must at least be aligned on the architecture pointer size. | 349 | * The probe address must at least be aligned on the architecture pointer size. |
| 342 | */ | 350 | */ |
| 343 | int tracepoint_probe_register(const char *name, void *probe) | 351 | int tracepoint_probe_register(const char *name, void *probe, void *data) |
| 344 | { | 352 | { |
| 345 | void *old; | 353 | struct tracepoint_func *old; |
| 346 | 354 | ||
| 347 | mutex_lock(&tracepoints_mutex); | 355 | mutex_lock(&tracepoints_mutex); |
| 348 | old = tracepoint_add_probe(name, probe); | 356 | old = tracepoint_add_probe(name, probe, data); |
| 349 | mutex_unlock(&tracepoints_mutex); | 357 | mutex_unlock(&tracepoints_mutex); |
| 350 | if (IS_ERR(old)) | 358 | if (IS_ERR(old)) |
| 351 | return PTR_ERR(old); | 359 | return PTR_ERR(old); |
| @@ -356,15 +364,16 @@ int tracepoint_probe_register(const char *name, void *probe) | |||
| 356 | } | 364 | } |
| 357 | EXPORT_SYMBOL_GPL(tracepoint_probe_register); | 365 | EXPORT_SYMBOL_GPL(tracepoint_probe_register); |
| 358 | 366 | ||
| 359 | static void *tracepoint_remove_probe(const char *name, void *probe) | 367 | static struct tracepoint_func * |
| 368 | tracepoint_remove_probe(const char *name, void *probe, void *data) | ||
| 360 | { | 369 | { |
| 361 | struct tracepoint_entry *entry; | 370 | struct tracepoint_entry *entry; |
| 362 | void *old; | 371 | struct tracepoint_func *old; |
| 363 | 372 | ||
| 364 | entry = get_tracepoint(name); | 373 | entry = get_tracepoint(name); |
| 365 | if (!entry) | 374 | if (!entry) |
| 366 | return ERR_PTR(-ENOENT); | 375 | return ERR_PTR(-ENOENT); |
| 367 | old = tracepoint_entry_remove_probe(entry, probe); | 376 | old = tracepoint_entry_remove_probe(entry, probe, data); |
| 368 | if (IS_ERR(old)) | 377 | if (IS_ERR(old)) |
| 369 | return old; | 378 | return old; |
| 370 | if (!entry->refcount) | 379 | if (!entry->refcount) |
| @@ -382,12 +391,12 @@ static void *tracepoint_remove_probe(const char *name, void *probe) | |||
| 382 | * itself uses stop_machine(), which insures that every preempt disabled section | 391 | * itself uses stop_machine(), which insures that every preempt disabled section |
| 383 | * have finished. | 392 | * have finished. |
| 384 | */ | 393 | */ |
| 385 | int tracepoint_probe_unregister(const char *name, void *probe) | 394 | int tracepoint_probe_unregister(const char *name, void *probe, void *data) |
| 386 | { | 395 | { |
| 387 | void *old; | 396 | struct tracepoint_func *old; |
| 388 | 397 | ||
| 389 | mutex_lock(&tracepoints_mutex); | 398 | mutex_lock(&tracepoints_mutex); |
| 390 | old = tracepoint_remove_probe(name, probe); | 399 | old = tracepoint_remove_probe(name, probe, data); |
| 391 | mutex_unlock(&tracepoints_mutex); | 400 | mutex_unlock(&tracepoints_mutex); |
| 392 | if (IS_ERR(old)) | 401 | if (IS_ERR(old)) |
| 393 | return PTR_ERR(old); | 402 | return PTR_ERR(old); |
| @@ -418,12 +427,13 @@ static void tracepoint_add_old_probes(void *old) | |||
| 418 | * | 427 | * |
| 419 | * caller must call tracepoint_probe_update_all() | 428 | * caller must call tracepoint_probe_update_all() |
| 420 | */ | 429 | */ |
| 421 | int tracepoint_probe_register_noupdate(const char *name, void *probe) | 430 | int tracepoint_probe_register_noupdate(const char *name, void *probe, |
| 431 | void *data) | ||
| 422 | { | 432 | { |
| 423 | void *old; | 433 | struct tracepoint_func *old; |
| 424 | 434 | ||
| 425 | mutex_lock(&tracepoints_mutex); | 435 | mutex_lock(&tracepoints_mutex); |
| 426 | old = tracepoint_add_probe(name, probe); | 436 | old = tracepoint_add_probe(name, probe, data); |
| 427 | if (IS_ERR(old)) { | 437 | if (IS_ERR(old)) { |
| 428 | mutex_unlock(&tracepoints_mutex); | 438 | mutex_unlock(&tracepoints_mutex); |
| 429 | return PTR_ERR(old); | 439 | return PTR_ERR(old); |
| @@ -441,12 +451,13 @@ EXPORT_SYMBOL_GPL(tracepoint_probe_register_noupdate); | |||
| 441 | * | 451 | * |
| 442 | * caller must call tracepoint_probe_update_all() | 452 | * caller must call tracepoint_probe_update_all() |
| 443 | */ | 453 | */ |
| 444 | int tracepoint_probe_unregister_noupdate(const char *name, void *probe) | 454 | int tracepoint_probe_unregister_noupdate(const char *name, void *probe, |
| 455 | void *data) | ||
| 445 | { | 456 | { |
| 446 | void *old; | 457 | struct tracepoint_func *old; |
| 447 | 458 | ||
| 448 | mutex_lock(&tracepoints_mutex); | 459 | mutex_lock(&tracepoints_mutex); |
| 449 | old = tracepoint_remove_probe(name, probe); | 460 | old = tracepoint_remove_probe(name, probe, data); |
| 450 | if (IS_ERR(old)) { | 461 | if (IS_ERR(old)) { |
| 451 | mutex_unlock(&tracepoints_mutex); | 462 | mutex_unlock(&tracepoints_mutex); |
| 452 | return PTR_ERR(old); | 463 | return PTR_ERR(old); |
