diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2009-10-08 15:05:50 -0400 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2009-10-08 15:05:50 -0400 |
| commit | b924f9599dfd4a604761e84b1e920e480fb57f66 (patch) | |
| tree | a1456ef8aea8beb8415d8258a978e072467d8ff6 | |
| parent | b9d40b7b1e349bdc5c174b4ef1a333e62f7d749c (diff) | |
| parent | 2dca6999eed58d44b67e9de7d6ec230f6250553d (diff) | |
Merge branch 'sparc-perf-events-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip
* 'sparc-perf-events-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip:
mm, perf_event: Make vmalloc_user() align base kernel virtual address to SHMLBA
perf_event: Provide vmalloc() based mmap() backing
| -rw-r--r-- | arch/sparc/Kconfig | 2 | ||||
| -rw-r--r-- | include/linux/perf_event.h | 5 | ||||
| -rw-r--r-- | init/Kconfig | 18 | ||||
| -rw-r--r-- | kernel/perf_event.c | 248 | ||||
| -rw-r--r-- | mm/vmalloc.c | 48 | ||||
| -rw-r--r-- | tools/perf/design.txt | 3 |
6 files changed, 240 insertions, 84 deletions
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index ac45aab741a5..05ef5380a687 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig | |||
| @@ -26,6 +26,7 @@ config SPARC | |||
| 26 | select RTC_CLASS | 26 | select RTC_CLASS |
| 27 | select RTC_DRV_M48T59 | 27 | select RTC_DRV_M48T59 |
| 28 | select HAVE_PERF_EVENTS | 28 | select HAVE_PERF_EVENTS |
| 29 | select PERF_USE_VMALLOC | ||
| 29 | select HAVE_DMA_ATTRS | 30 | select HAVE_DMA_ATTRS |
| 30 | select HAVE_DMA_API_DEBUG | 31 | select HAVE_DMA_API_DEBUG |
| 31 | 32 | ||
| @@ -48,6 +49,7 @@ config SPARC64 | |||
| 48 | select RTC_DRV_SUN4V | 49 | select RTC_DRV_SUN4V |
| 49 | select RTC_DRV_STARFIRE | 50 | select RTC_DRV_STARFIRE |
| 50 | select HAVE_PERF_EVENTS | 51 | select HAVE_PERF_EVENTS |
| 52 | select PERF_USE_VMALLOC | ||
| 51 | 53 | ||
| 52 | config ARCH_DEFCONFIG | 54 | config ARCH_DEFCONFIG |
| 53 | string | 55 | string |
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 3a9d36d1e92a..2e6d95f97419 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h | |||
| @@ -442,6 +442,7 @@ enum perf_callchain_context { | |||
| 442 | #include <linux/hrtimer.h> | 442 | #include <linux/hrtimer.h> |
| 443 | #include <linux/fs.h> | 443 | #include <linux/fs.h> |
| 444 | #include <linux/pid_namespace.h> | 444 | #include <linux/pid_namespace.h> |
| 445 | #include <linux/workqueue.h> | ||
| 445 | #include <asm/atomic.h> | 446 | #include <asm/atomic.h> |
| 446 | 447 | ||
| 447 | #define PERF_MAX_STACK_DEPTH 255 | 448 | #define PERF_MAX_STACK_DEPTH 255 |
| @@ -513,6 +514,10 @@ struct file; | |||
| 513 | 514 | ||
| 514 | struct perf_mmap_data { | 515 | struct perf_mmap_data { |
| 515 | struct rcu_head rcu_head; | 516 | struct rcu_head rcu_head; |
| 517 | #ifdef CONFIG_PERF_USE_VMALLOC | ||
| 518 | struct work_struct work; | ||
| 519 | #endif | ||
| 520 | int data_order; | ||
| 516 | int nr_pages; /* nr of data pages */ | 521 | int nr_pages; /* nr of data pages */ |
| 517 | int writable; /* are we writable */ | 522 | int writable; /* are we writable */ |
| 518 | int nr_locked; /* nr pages mlocked */ | 523 | int nr_locked; /* nr pages mlocked */ |
diff --git a/init/Kconfig b/init/Kconfig index c7bac39d6c61..09c5c6431f42 100644 --- a/init/Kconfig +++ b/init/Kconfig | |||
| @@ -921,6 +921,11 @@ config HAVE_PERF_EVENTS | |||
| 921 | help | 921 | help |
| 922 | See tools/perf/design.txt for details. | 922 | See tools/perf/design.txt for details. |
| 923 | 923 | ||
| 924 | config PERF_USE_VMALLOC | ||
| 925 | bool | ||
| 926 | help | ||
| 927 | See tools/perf/design.txt for details | ||
| 928 | |||
| 924 | menu "Kernel Performance Events And Counters" | 929 | menu "Kernel Performance Events And Counters" |
| 925 | 930 | ||
| 926 | config PERF_EVENTS | 931 | config PERF_EVENTS |
| @@ -976,6 +981,19 @@ config PERF_COUNTERS | |||
| 976 | 981 | ||
| 977 | Say N if unsure. | 982 | Say N if unsure. |
| 978 | 983 | ||
| 984 | config DEBUG_PERF_USE_VMALLOC | ||
| 985 | default n | ||
| 986 | bool "Debug: use vmalloc to back perf mmap() buffers" | ||
| 987 | depends on PERF_EVENTS && DEBUG_KERNEL | ||
| 988 | select PERF_USE_VMALLOC | ||
| 989 | help | ||
| 990 | Use vmalloc memory to back perf mmap() buffers. | ||
| 991 | |||
| 992 | Mostly useful for debugging the vmalloc code on platforms | ||
| 993 | that don't require it. | ||
| 994 | |||
| 995 | Say N if unsure. | ||
| 996 | |||
| 979 | endmenu | 997 | endmenu |
| 980 | 998 | ||
| 981 | config VM_EVENT_COUNTERS | 999 | config VM_EVENT_COUNTERS |
diff --git a/kernel/perf_event.c b/kernel/perf_event.c index e491fb087939..9d0b5c665883 100644 --- a/kernel/perf_event.c +++ b/kernel/perf_event.c | |||
| @@ -20,6 +20,7 @@ | |||
| 20 | #include <linux/percpu.h> | 20 | #include <linux/percpu.h> |
| 21 | #include <linux/ptrace.h> | 21 | #include <linux/ptrace.h> |
| 22 | #include <linux/vmstat.h> | 22 | #include <linux/vmstat.h> |
| 23 | #include <linux/vmalloc.h> | ||
| 23 | #include <linux/hardirq.h> | 24 | #include <linux/hardirq.h> |
| 24 | #include <linux/rculist.h> | 25 | #include <linux/rculist.h> |
| 25 | #include <linux/uaccess.h> | 26 | #include <linux/uaccess.h> |
| @@ -2091,49 +2092,31 @@ unlock: | |||
| 2091 | rcu_read_unlock(); | 2092 | rcu_read_unlock(); |
| 2092 | } | 2093 | } |
| 2093 | 2094 | ||
| 2094 | static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | 2095 | static unsigned long perf_data_size(struct perf_mmap_data *data) |
| 2095 | { | 2096 | { |
| 2096 | struct perf_event *event = vma->vm_file->private_data; | 2097 | return data->nr_pages << (PAGE_SHIFT + data->data_order); |
| 2097 | struct perf_mmap_data *data; | 2098 | } |
| 2098 | int ret = VM_FAULT_SIGBUS; | ||
| 2099 | |||
| 2100 | if (vmf->flags & FAULT_FLAG_MKWRITE) { | ||
| 2101 | if (vmf->pgoff == 0) | ||
| 2102 | ret = 0; | ||
| 2103 | return ret; | ||
| 2104 | } | ||
| 2105 | |||
| 2106 | rcu_read_lock(); | ||
| 2107 | data = rcu_dereference(event->data); | ||
| 2108 | if (!data) | ||
| 2109 | goto unlock; | ||
| 2110 | |||
| 2111 | if (vmf->pgoff == 0) { | ||
| 2112 | vmf->page = virt_to_page(data->user_page); | ||
| 2113 | } else { | ||
| 2114 | int nr = vmf->pgoff - 1; | ||
| 2115 | |||
| 2116 | if ((unsigned)nr > data->nr_pages) | ||
| 2117 | goto unlock; | ||
| 2118 | 2099 | ||
| 2119 | if (vmf->flags & FAULT_FLAG_WRITE) | 2100 | #ifndef CONFIG_PERF_USE_VMALLOC |
| 2120 | goto unlock; | ||
| 2121 | 2101 | ||
| 2122 | vmf->page = virt_to_page(data->data_pages[nr]); | 2102 | /* |
| 2123 | } | 2103 | * Back perf_mmap() with regular GFP_KERNEL-0 pages. |
| 2104 | */ | ||
| 2124 | 2105 | ||
| 2125 | get_page(vmf->page); | 2106 | static struct page * |
| 2126 | vmf->page->mapping = vma->vm_file->f_mapping; | 2107 | perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff) |
| 2127 | vmf->page->index = vmf->pgoff; | 2108 | { |
| 2109 | if (pgoff > data->nr_pages) | ||
| 2110 | return NULL; | ||
| 2128 | 2111 | ||
| 2129 | ret = 0; | 2112 | if (pgoff == 0) |
| 2130 | unlock: | 2113 | return virt_to_page(data->user_page); |
| 2131 | rcu_read_unlock(); | ||
| 2132 | 2114 | ||
| 2133 | return ret; | 2115 | return virt_to_page(data->data_pages[pgoff - 1]); |
| 2134 | } | 2116 | } |
| 2135 | 2117 | ||
| 2136 | static int perf_mmap_data_alloc(struct perf_event *event, int nr_pages) | 2118 | static struct perf_mmap_data * |
| 2119 | perf_mmap_data_alloc(struct perf_event *event, int nr_pages) | ||
| 2137 | { | 2120 | { |
| 2138 | struct perf_mmap_data *data; | 2121 | struct perf_mmap_data *data; |
| 2139 | unsigned long size; | 2122 | unsigned long size; |
| @@ -2158,19 +2141,10 @@ static int perf_mmap_data_alloc(struct perf_event *event, int nr_pages) | |||
| 2158 | goto fail_data_pages; | 2141 | goto fail_data_pages; |
| 2159 | } | 2142 | } |
| 2160 | 2143 | ||
| 2144 | data->data_order = 0; | ||
| 2161 | data->nr_pages = nr_pages; | 2145 | data->nr_pages = nr_pages; |
| 2162 | atomic_set(&data->lock, -1); | ||
| 2163 | |||
| 2164 | if (event->attr.watermark) { | ||
| 2165 | data->watermark = min_t(long, PAGE_SIZE * nr_pages, | ||
| 2166 | event->attr.wakeup_watermark); | ||
| 2167 | } | ||
| 2168 | if (!data->watermark) | ||
| 2169 | data->watermark = max(PAGE_SIZE, PAGE_SIZE * nr_pages / 4); | ||
| 2170 | 2146 | ||
| 2171 | rcu_assign_pointer(event->data, data); | 2147 | return data; |
| 2172 | |||
| 2173 | return 0; | ||
| 2174 | 2148 | ||
| 2175 | fail_data_pages: | 2149 | fail_data_pages: |
| 2176 | for (i--; i >= 0; i--) | 2150 | for (i--; i >= 0; i--) |
| @@ -2182,7 +2156,7 @@ fail_user_page: | |||
| 2182 | kfree(data); | 2156 | kfree(data); |
| 2183 | 2157 | ||
| 2184 | fail: | 2158 | fail: |
| 2185 | return -ENOMEM; | 2159 | return NULL; |
| 2186 | } | 2160 | } |
| 2187 | 2161 | ||
| 2188 | static void perf_mmap_free_page(unsigned long addr) | 2162 | static void perf_mmap_free_page(unsigned long addr) |
| @@ -2193,28 +2167,169 @@ static void perf_mmap_free_page(unsigned long addr) | |||
| 2193 | __free_page(page); | 2167 | __free_page(page); |
| 2194 | } | 2168 | } |
| 2195 | 2169 | ||
| 2196 | static void __perf_mmap_data_free(struct rcu_head *rcu_head) | 2170 | static void perf_mmap_data_free(struct perf_mmap_data *data) |
| 2197 | { | 2171 | { |
| 2198 | struct perf_mmap_data *data; | ||
| 2199 | int i; | 2172 | int i; |
| 2200 | 2173 | ||
| 2201 | data = container_of(rcu_head, struct perf_mmap_data, rcu_head); | ||
| 2202 | |||
| 2203 | perf_mmap_free_page((unsigned long)data->user_page); | 2174 | perf_mmap_free_page((unsigned long)data->user_page); |
| 2204 | for (i = 0; i < data->nr_pages; i++) | 2175 | for (i = 0; i < data->nr_pages; i++) |
| 2205 | perf_mmap_free_page((unsigned long)data->data_pages[i]); | 2176 | perf_mmap_free_page((unsigned long)data->data_pages[i]); |
| 2177 | } | ||
| 2178 | |||
| 2179 | #else | ||
| 2180 | |||
| 2181 | /* | ||
| 2182 | * Back perf_mmap() with vmalloc memory. | ||
| 2183 | * | ||
| 2184 | * Required for architectures that have d-cache aliasing issues. | ||
| 2185 | */ | ||
| 2186 | |||
| 2187 | static struct page * | ||
| 2188 | perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff) | ||
| 2189 | { | ||
| 2190 | if (pgoff > (1UL << data->data_order)) | ||
| 2191 | return NULL; | ||
| 2192 | |||
| 2193 | return vmalloc_to_page((void *)data->user_page + pgoff * PAGE_SIZE); | ||
| 2194 | } | ||
| 2195 | |||
| 2196 | static void perf_mmap_unmark_page(void *addr) | ||
| 2197 | { | ||
| 2198 | struct page *page = vmalloc_to_page(addr); | ||
| 2199 | |||
| 2200 | page->mapping = NULL; | ||
| 2201 | } | ||
| 2202 | |||
| 2203 | static void perf_mmap_data_free_work(struct work_struct *work) | ||
| 2204 | { | ||
| 2205 | struct perf_mmap_data *data; | ||
| 2206 | void *base; | ||
| 2207 | int i, nr; | ||
| 2208 | |||
| 2209 | data = container_of(work, struct perf_mmap_data, work); | ||
| 2210 | nr = 1 << data->data_order; | ||
| 2211 | |||
| 2212 | base = data->user_page; | ||
| 2213 | for (i = 0; i < nr + 1; i++) | ||
| 2214 | perf_mmap_unmark_page(base + (i * PAGE_SIZE)); | ||
| 2215 | |||
| 2216 | vfree(base); | ||
| 2217 | } | ||
| 2218 | |||
| 2219 | static void perf_mmap_data_free(struct perf_mmap_data *data) | ||
| 2220 | { | ||
| 2221 | schedule_work(&data->work); | ||
| 2222 | } | ||
| 2223 | |||
| 2224 | static struct perf_mmap_data * | ||
| 2225 | perf_mmap_data_alloc(struct perf_event *event, int nr_pages) | ||
| 2226 | { | ||
| 2227 | struct perf_mmap_data *data; | ||
| 2228 | unsigned long size; | ||
| 2229 | void *all_buf; | ||
| 2206 | 2230 | ||
| 2231 | WARN_ON(atomic_read(&event->mmap_count)); | ||
| 2232 | |||
| 2233 | size = sizeof(struct perf_mmap_data); | ||
| 2234 | size += sizeof(void *); | ||
| 2235 | |||
| 2236 | data = kzalloc(size, GFP_KERNEL); | ||
| 2237 | if (!data) | ||
| 2238 | goto fail; | ||
| 2239 | |||
| 2240 | INIT_WORK(&data->work, perf_mmap_data_free_work); | ||
| 2241 | |||
| 2242 | all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE); | ||
| 2243 | if (!all_buf) | ||
| 2244 | goto fail_all_buf; | ||
| 2245 | |||
| 2246 | data->user_page = all_buf; | ||
| 2247 | data->data_pages[0] = all_buf + PAGE_SIZE; | ||
| 2248 | data->data_order = ilog2(nr_pages); | ||
| 2249 | data->nr_pages = 1; | ||
| 2250 | |||
| 2251 | return data; | ||
| 2252 | |||
| 2253 | fail_all_buf: | ||
| 2254 | kfree(data); | ||
| 2255 | |||
| 2256 | fail: | ||
| 2257 | return NULL; | ||
| 2258 | } | ||
| 2259 | |||
| 2260 | #endif | ||
| 2261 | |||
| 2262 | static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | ||
| 2263 | { | ||
| 2264 | struct perf_event *event = vma->vm_file->private_data; | ||
| 2265 | struct perf_mmap_data *data; | ||
| 2266 | int ret = VM_FAULT_SIGBUS; | ||
| 2267 | |||
| 2268 | if (vmf->flags & FAULT_FLAG_MKWRITE) { | ||
| 2269 | if (vmf->pgoff == 0) | ||
| 2270 | ret = 0; | ||
| 2271 | return ret; | ||
| 2272 | } | ||
| 2273 | |||
| 2274 | rcu_read_lock(); | ||
| 2275 | data = rcu_dereference(event->data); | ||
| 2276 | if (!data) | ||
| 2277 | goto unlock; | ||
| 2278 | |||
| 2279 | if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE)) | ||
| 2280 | goto unlock; | ||
| 2281 | |||
| 2282 | vmf->page = perf_mmap_to_page(data, vmf->pgoff); | ||
| 2283 | if (!vmf->page) | ||
| 2284 | goto unlock; | ||
| 2285 | |||
| 2286 | get_page(vmf->page); | ||
| 2287 | vmf->page->mapping = vma->vm_file->f_mapping; | ||
| 2288 | vmf->page->index = vmf->pgoff; | ||
| 2289 | |||
| 2290 | ret = 0; | ||
| 2291 | unlock: | ||
| 2292 | rcu_read_unlock(); | ||
| 2293 | |||
| 2294 | return ret; | ||
| 2295 | } | ||
| 2296 | |||
| 2297 | static void | ||
| 2298 | perf_mmap_data_init(struct perf_event *event, struct perf_mmap_data *data) | ||
| 2299 | { | ||
| 2300 | long max_size = perf_data_size(data); | ||
| 2301 | |||
| 2302 | atomic_set(&data->lock, -1); | ||
| 2303 | |||
| 2304 | if (event->attr.watermark) { | ||
| 2305 | data->watermark = min_t(long, max_size, | ||
| 2306 | event->attr.wakeup_watermark); | ||
| 2307 | } | ||
| 2308 | |||
| 2309 | if (!data->watermark) | ||
| 2310 | data->watermark = max_t(long, PAGE_SIZE, max_size / 2); | ||
| 2311 | |||
| 2312 | |||
| 2313 | rcu_assign_pointer(event->data, data); | ||
| 2314 | } | ||
| 2315 | |||
| 2316 | static void perf_mmap_data_free_rcu(struct rcu_head *rcu_head) | ||
| 2317 | { | ||
| 2318 | struct perf_mmap_data *data; | ||
| 2319 | |||
| 2320 | data = container_of(rcu_head, struct perf_mmap_data, rcu_head); | ||
| 2321 | perf_mmap_data_free(data); | ||
| 2207 | kfree(data); | 2322 | kfree(data); |
| 2208 | } | 2323 | } |
| 2209 | 2324 | ||
| 2210 | static void perf_mmap_data_free(struct perf_event *event) | 2325 | static void perf_mmap_data_release(struct perf_event *event) |
| 2211 | { | 2326 | { |
| 2212 | struct perf_mmap_data *data = event->data; | 2327 | struct perf_mmap_data *data = event->data; |
| 2213 | 2328 | ||
| 2214 | WARN_ON(atomic_read(&event->mmap_count)); | 2329 | WARN_ON(atomic_read(&event->mmap_count)); |
| 2215 | 2330 | ||
| 2216 | rcu_assign_pointer(event->data, NULL); | 2331 | rcu_assign_pointer(event->data, NULL); |
| 2217 | call_rcu(&data->rcu_head, __perf_mmap_data_free); | 2332 | call_rcu(&data->rcu_head, perf_mmap_data_free_rcu); |
| 2218 | } | 2333 | } |
| 2219 | 2334 | ||
| 2220 | static void perf_mmap_open(struct vm_area_struct *vma) | 2335 | static void perf_mmap_open(struct vm_area_struct *vma) |
| @@ -2230,11 +2345,12 @@ static void perf_mmap_close(struct vm_area_struct *vma) | |||
| 2230 | 2345 | ||
| 2231 | WARN_ON_ONCE(event->ctx->parent_ctx); | 2346 | WARN_ON_ONCE(event->ctx->parent_ctx); |
| 2232 | if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) { | 2347 | if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) { |
| 2348 | unsigned long size = perf_data_size(event->data); | ||
| 2233 | struct user_struct *user = current_user(); | 2349 | struct user_struct *user = current_user(); |
| 2234 | 2350 | ||
| 2235 | atomic_long_sub(event->data->nr_pages + 1, &user->locked_vm); | 2351 | atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm); |
| 2236 | vma->vm_mm->locked_vm -= event->data->nr_locked; | 2352 | vma->vm_mm->locked_vm -= event->data->nr_locked; |
| 2237 | perf_mmap_data_free(event); | 2353 | perf_mmap_data_release(event); |
| 2238 | mutex_unlock(&event->mmap_mutex); | 2354 | mutex_unlock(&event->mmap_mutex); |
| 2239 | } | 2355 | } |
| 2240 | } | 2356 | } |
| @@ -2252,6 +2368,7 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) | |||
| 2252 | unsigned long user_locked, user_lock_limit; | 2368 | unsigned long user_locked, user_lock_limit; |
| 2253 | struct user_struct *user = current_user(); | 2369 | struct user_struct *user = current_user(); |
| 2254 | unsigned long locked, lock_limit; | 2370 | unsigned long locked, lock_limit; |
| 2371 | struct perf_mmap_data *data; | ||
| 2255 | unsigned long vma_size; | 2372 | unsigned long vma_size; |
| 2256 | unsigned long nr_pages; | 2373 | unsigned long nr_pages; |
| 2257 | long user_extra, extra; | 2374 | long user_extra, extra; |
| @@ -2314,10 +2431,15 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) | |||
| 2314 | } | 2431 | } |
| 2315 | 2432 | ||
| 2316 | WARN_ON(event->data); | 2433 | WARN_ON(event->data); |
| 2317 | ret = perf_mmap_data_alloc(event, nr_pages); | 2434 | |
| 2318 | if (ret) | 2435 | data = perf_mmap_data_alloc(event, nr_pages); |
| 2436 | ret = -ENOMEM; | ||
| 2437 | if (!data) | ||
| 2319 | goto unlock; | 2438 | goto unlock; |
| 2320 | 2439 | ||
| 2440 | ret = 0; | ||
| 2441 | perf_mmap_data_init(event, data); | ||
| 2442 | |||
| 2321 | atomic_set(&event->mmap_count, 1); | 2443 | atomic_set(&event->mmap_count, 1); |
| 2322 | atomic_long_add(user_extra, &user->locked_vm); | 2444 | atomic_long_add(user_extra, &user->locked_vm); |
| 2323 | vma->vm_mm->locked_vm += extra; | 2445 | vma->vm_mm->locked_vm += extra; |
| @@ -2505,7 +2627,7 @@ static bool perf_output_space(struct perf_mmap_data *data, unsigned long tail, | |||
| 2505 | if (!data->writable) | 2627 | if (!data->writable) |
| 2506 | return true; | 2628 | return true; |
| 2507 | 2629 | ||
| 2508 | mask = (data->nr_pages << PAGE_SHIFT) - 1; | 2630 | mask = perf_data_size(data) - 1; |
| 2509 | 2631 | ||
| 2510 | offset = (offset - tail) & mask; | 2632 | offset = (offset - tail) & mask; |
| 2511 | head = (head - tail) & mask; | 2633 | head = (head - tail) & mask; |
| @@ -2610,7 +2732,7 @@ void perf_output_copy(struct perf_output_handle *handle, | |||
| 2610 | const void *buf, unsigned int len) | 2732 | const void *buf, unsigned int len) |
| 2611 | { | 2733 | { |
| 2612 | unsigned int pages_mask; | 2734 | unsigned int pages_mask; |
| 2613 | unsigned int offset; | 2735 | unsigned long offset; |
| 2614 | unsigned int size; | 2736 | unsigned int size; |
| 2615 | void **pages; | 2737 | void **pages; |
| 2616 | 2738 | ||
| @@ -2619,12 +2741,14 @@ void perf_output_copy(struct perf_output_handle *handle, | |||
| 2619 | pages = handle->data->data_pages; | 2741 | pages = handle->data->data_pages; |
| 2620 | 2742 | ||
| 2621 | do { | 2743 | do { |
| 2622 | unsigned int page_offset; | 2744 | unsigned long page_offset; |
| 2745 | unsigned long page_size; | ||
| 2623 | int nr; | 2746 | int nr; |
| 2624 | 2747 | ||
| 2625 | nr = (offset >> PAGE_SHIFT) & pages_mask; | 2748 | nr = (offset >> PAGE_SHIFT) & pages_mask; |
| 2626 | page_offset = offset & (PAGE_SIZE - 1); | 2749 | page_size = 1UL << (handle->data->data_order + PAGE_SHIFT); |
| 2627 | size = min_t(unsigned int, PAGE_SIZE - page_offset, len); | 2750 | page_offset = offset & (page_size - 1); |
| 2751 | size = min_t(unsigned int, page_size - page_offset, len); | ||
| 2628 | 2752 | ||
| 2629 | memcpy(pages[nr] + page_offset, buf, size); | 2753 | memcpy(pages[nr] + page_offset, buf, size); |
| 2630 | 2754 | ||
diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 2f7c9d75c552..5e7aed0802bf 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c | |||
| @@ -28,6 +28,7 @@ | |||
| 28 | #include <asm/atomic.h> | 28 | #include <asm/atomic.h> |
| 29 | #include <asm/uaccess.h> | 29 | #include <asm/uaccess.h> |
| 30 | #include <asm/tlbflush.h> | 30 | #include <asm/tlbflush.h> |
| 31 | #include <asm/shmparam.h> | ||
| 31 | 32 | ||
| 32 | 33 | ||
| 33 | /*** Page table manipulation functions ***/ | 34 | /*** Page table manipulation functions ***/ |
| @@ -1155,12 +1156,11 @@ static void insert_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va, | |||
| 1155 | } | 1156 | } |
| 1156 | 1157 | ||
| 1157 | static struct vm_struct *__get_vm_area_node(unsigned long size, | 1158 | static struct vm_struct *__get_vm_area_node(unsigned long size, |
| 1158 | unsigned long flags, unsigned long start, unsigned long end, | 1159 | unsigned long align, unsigned long flags, unsigned long start, |
| 1159 | int node, gfp_t gfp_mask, void *caller) | 1160 | unsigned long end, int node, gfp_t gfp_mask, void *caller) |
| 1160 | { | 1161 | { |
| 1161 | static struct vmap_area *va; | 1162 | static struct vmap_area *va; |
| 1162 | struct vm_struct *area; | 1163 | struct vm_struct *area; |
| 1163 | unsigned long align = 1; | ||
| 1164 | 1164 | ||
| 1165 | BUG_ON(in_interrupt()); | 1165 | BUG_ON(in_interrupt()); |
| 1166 | if (flags & VM_IOREMAP) { | 1166 | if (flags & VM_IOREMAP) { |
| @@ -1200,7 +1200,7 @@ static struct vm_struct *__get_vm_area_node(unsigned long size, | |||
| 1200 | struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags, | 1200 | struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags, |
| 1201 | unsigned long start, unsigned long end) | 1201 | unsigned long start, unsigned long end) |
| 1202 | { | 1202 | { |
| 1203 | return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL, | 1203 | return __get_vm_area_node(size, 1, flags, start, end, -1, GFP_KERNEL, |
| 1204 | __builtin_return_address(0)); | 1204 | __builtin_return_address(0)); |
| 1205 | } | 1205 | } |
| 1206 | EXPORT_SYMBOL_GPL(__get_vm_area); | 1206 | EXPORT_SYMBOL_GPL(__get_vm_area); |
| @@ -1209,7 +1209,7 @@ struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, | |||
| 1209 | unsigned long start, unsigned long end, | 1209 | unsigned long start, unsigned long end, |
| 1210 | void *caller) | 1210 | void *caller) |
| 1211 | { | 1211 | { |
| 1212 | return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL, | 1212 | return __get_vm_area_node(size, 1, flags, start, end, -1, GFP_KERNEL, |
| 1213 | caller); | 1213 | caller); |
| 1214 | } | 1214 | } |
| 1215 | 1215 | ||
| @@ -1224,22 +1224,22 @@ struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, | |||
| 1224 | */ | 1224 | */ |
| 1225 | struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) | 1225 | struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) |
| 1226 | { | 1226 | { |
| 1227 | return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, | 1227 | return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, |
| 1228 | -1, GFP_KERNEL, __builtin_return_address(0)); | 1228 | -1, GFP_KERNEL, __builtin_return_address(0)); |
| 1229 | } | 1229 | } |
| 1230 | 1230 | ||
| 1231 | struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, | 1231 | struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, |
| 1232 | void *caller) | 1232 | void *caller) |
| 1233 | { | 1233 | { |
| 1234 | return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, | 1234 | return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, |
| 1235 | -1, GFP_KERNEL, caller); | 1235 | -1, GFP_KERNEL, caller); |
| 1236 | } | 1236 | } |
| 1237 | 1237 | ||
| 1238 | struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags, | 1238 | struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags, |
| 1239 | int node, gfp_t gfp_mask) | 1239 | int node, gfp_t gfp_mask) |
| 1240 | { | 1240 | { |
| 1241 | return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node, | 1241 | return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, |
| 1242 | gfp_mask, __builtin_return_address(0)); | 1242 | node, gfp_mask, __builtin_return_address(0)); |
| 1243 | } | 1243 | } |
| 1244 | 1244 | ||
| 1245 | static struct vm_struct *find_vm_area(const void *addr) | 1245 | static struct vm_struct *find_vm_area(const void *addr) |
| @@ -1402,7 +1402,8 @@ void *vmap(struct page **pages, unsigned int count, | |||
| 1402 | } | 1402 | } |
| 1403 | EXPORT_SYMBOL(vmap); | 1403 | EXPORT_SYMBOL(vmap); |
| 1404 | 1404 | ||
| 1405 | static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot, | 1405 | static void *__vmalloc_node(unsigned long size, unsigned long align, |
| 1406 | gfp_t gfp_mask, pgprot_t prot, | ||
| 1406 | int node, void *caller); | 1407 | int node, void *caller); |
| 1407 | static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, | 1408 | static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, |
| 1408 | pgprot_t prot, int node, void *caller) | 1409 | pgprot_t prot, int node, void *caller) |
| @@ -1416,7 +1417,7 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, | |||
| 1416 | area->nr_pages = nr_pages; | 1417 | area->nr_pages = nr_pages; |
| 1417 | /* Please note that the recursion is strictly bounded. */ | 1418 | /* Please note that the recursion is strictly bounded. */ |
| 1418 | if (array_size > PAGE_SIZE) { | 1419 | if (array_size > PAGE_SIZE) { |
| 1419 | pages = __vmalloc_node(array_size, gfp_mask | __GFP_ZERO, | 1420 | pages = __vmalloc_node(array_size, 1, gfp_mask | __GFP_ZERO, |
| 1420 | PAGE_KERNEL, node, caller); | 1421 | PAGE_KERNEL, node, caller); |
| 1421 | area->flags |= VM_VPAGES; | 1422 | area->flags |= VM_VPAGES; |
| 1422 | } else { | 1423 | } else { |
| @@ -1475,6 +1476,7 @@ void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot) | |||
| 1475 | /** | 1476 | /** |
| 1476 | * __vmalloc_node - allocate virtually contiguous memory | 1477 | * __vmalloc_node - allocate virtually contiguous memory |
| 1477 | * @size: allocation size | 1478 | * @size: allocation size |
| 1479 | * @align: desired alignment | ||
| 1478 | * @gfp_mask: flags for the page level allocator | 1480 | * @gfp_mask: flags for the page level allocator |
| 1479 | * @prot: protection mask for the allocated pages | 1481 | * @prot: protection mask for the allocated pages |
| 1480 | * @node: node to use for allocation or -1 | 1482 | * @node: node to use for allocation or -1 |
| @@ -1484,8 +1486,9 @@ void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot) | |||
| 1484 | * allocator with @gfp_mask flags. Map them into contiguous | 1486 | * allocator with @gfp_mask flags. Map them into contiguous |
| 1485 | * kernel virtual space, using a pagetable protection of @prot. | 1487 | * kernel virtual space, using a pagetable protection of @prot. |
| 1486 | */ | 1488 | */ |
| 1487 | static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot, | 1489 | static void *__vmalloc_node(unsigned long size, unsigned long align, |
| 1488 | int node, void *caller) | 1490 | gfp_t gfp_mask, pgprot_t prot, |
| 1491 | int node, void *caller) | ||
| 1489 | { | 1492 | { |
| 1490 | struct vm_struct *area; | 1493 | struct vm_struct *area; |
| 1491 | void *addr; | 1494 | void *addr; |
| @@ -1495,8 +1498,8 @@ static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot, | |||
| 1495 | if (!size || (size >> PAGE_SHIFT) > totalram_pages) | 1498 | if (!size || (size >> PAGE_SHIFT) > totalram_pages) |
| 1496 | return NULL; | 1499 | return NULL; |
| 1497 | 1500 | ||
| 1498 | area = __get_vm_area_node(size, VM_ALLOC, VMALLOC_START, VMALLOC_END, | 1501 | area = __get_vm_area_node(size, align, VM_ALLOC, VMALLOC_START, |
| 1499 | node, gfp_mask, caller); | 1502 | VMALLOC_END, node, gfp_mask, caller); |
| 1500 | 1503 | ||
| 1501 | if (!area) | 1504 | if (!area) |
| 1502 | return NULL; | 1505 | return NULL; |
| @@ -1515,7 +1518,7 @@ static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot, | |||
| 1515 | 1518 | ||
| 1516 | void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) | 1519 | void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) |
| 1517 | { | 1520 | { |
| 1518 | return __vmalloc_node(size, gfp_mask, prot, -1, | 1521 | return __vmalloc_node(size, 1, gfp_mask, prot, -1, |
| 1519 | __builtin_return_address(0)); | 1522 | __builtin_return_address(0)); |
| 1520 | } | 1523 | } |
| 1521 | EXPORT_SYMBOL(__vmalloc); | 1524 | EXPORT_SYMBOL(__vmalloc); |
| @@ -1531,7 +1534,7 @@ EXPORT_SYMBOL(__vmalloc); | |||
| 1531 | */ | 1534 | */ |
| 1532 | void *vmalloc(unsigned long size) | 1535 | void *vmalloc(unsigned long size) |
| 1533 | { | 1536 | { |
| 1534 | return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL, | 1537 | return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL, |
| 1535 | -1, __builtin_return_address(0)); | 1538 | -1, __builtin_return_address(0)); |
| 1536 | } | 1539 | } |
| 1537 | EXPORT_SYMBOL(vmalloc); | 1540 | EXPORT_SYMBOL(vmalloc); |
| @@ -1548,7 +1551,8 @@ void *vmalloc_user(unsigned long size) | |||
| 1548 | struct vm_struct *area; | 1551 | struct vm_struct *area; |
| 1549 | void *ret; | 1552 | void *ret; |
| 1550 | 1553 | ||
| 1551 | ret = __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, | 1554 | ret = __vmalloc_node(size, SHMLBA, |
| 1555 | GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, | ||
| 1552 | PAGE_KERNEL, -1, __builtin_return_address(0)); | 1556 | PAGE_KERNEL, -1, __builtin_return_address(0)); |
| 1553 | if (ret) { | 1557 | if (ret) { |
| 1554 | area = find_vm_area(ret); | 1558 | area = find_vm_area(ret); |
| @@ -1571,7 +1575,7 @@ EXPORT_SYMBOL(vmalloc_user); | |||
| 1571 | */ | 1575 | */ |
| 1572 | void *vmalloc_node(unsigned long size, int node) | 1576 | void *vmalloc_node(unsigned long size, int node) |
| 1573 | { | 1577 | { |
| 1574 | return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL, | 1578 | return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL, |
| 1575 | node, __builtin_return_address(0)); | 1579 | node, __builtin_return_address(0)); |
| 1576 | } | 1580 | } |
| 1577 | EXPORT_SYMBOL(vmalloc_node); | 1581 | EXPORT_SYMBOL(vmalloc_node); |
| @@ -1594,7 +1598,7 @@ EXPORT_SYMBOL(vmalloc_node); | |||
| 1594 | 1598 | ||
| 1595 | void *vmalloc_exec(unsigned long size) | 1599 | void *vmalloc_exec(unsigned long size) |
| 1596 | { | 1600 | { |
| 1597 | return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC, | 1601 | return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC, |
| 1598 | -1, __builtin_return_address(0)); | 1602 | -1, __builtin_return_address(0)); |
| 1599 | } | 1603 | } |
| 1600 | 1604 | ||
| @@ -1615,7 +1619,7 @@ void *vmalloc_exec(unsigned long size) | |||
| 1615 | */ | 1619 | */ |
| 1616 | void *vmalloc_32(unsigned long size) | 1620 | void *vmalloc_32(unsigned long size) |
| 1617 | { | 1621 | { |
| 1618 | return __vmalloc_node(size, GFP_VMALLOC32, PAGE_KERNEL, | 1622 | return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL, |
| 1619 | -1, __builtin_return_address(0)); | 1623 | -1, __builtin_return_address(0)); |
| 1620 | } | 1624 | } |
| 1621 | EXPORT_SYMBOL(vmalloc_32); | 1625 | EXPORT_SYMBOL(vmalloc_32); |
| @@ -1632,7 +1636,7 @@ void *vmalloc_32_user(unsigned long size) | |||
| 1632 | struct vm_struct *area; | 1636 | struct vm_struct *area; |
| 1633 | void *ret; | 1637 | void *ret; |
| 1634 | 1638 | ||
| 1635 | ret = __vmalloc_node(size, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, | 1639 | ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, |
| 1636 | -1, __builtin_return_address(0)); | 1640 | -1, __builtin_return_address(0)); |
| 1637 | if (ret) { | 1641 | if (ret) { |
| 1638 | area = find_vm_area(ret); | 1642 | area = find_vm_area(ret); |
diff --git a/tools/perf/design.txt b/tools/perf/design.txt index f1946d107b10..fdd42a824c98 100644 --- a/tools/perf/design.txt +++ b/tools/perf/design.txt | |||
| @@ -455,3 +455,6 @@ will need at least this: | |||
| 455 | 455 | ||
| 456 | If your architecture does have hardware capabilities, you can override the | 456 | If your architecture does have hardware capabilities, you can override the |
| 457 | weak stub hw_perf_event_init() to register hardware counters. | 457 | weak stub hw_perf_event_init() to register hardware counters. |
| 458 | |||
| 459 | Architectures that have d-cache aliassing issues, such as Sparc and ARM, | ||
| 460 | should select PERF_USE_VMALLOC in order to avoid these for perf mmap(). | ||
