diff options
| author | Ingo Molnar <mingo@elte.hu> | 2012-02-17 03:27:41 -0500 |
|---|---|---|
| committer | Ingo Molnar <mingo@elte.hu> | 2012-02-17 04:18:07 -0500 |
| commit | 7b2d81d48a2d8e37efb6ce7b4d5ef58822b30d89 (patch) | |
| tree | 23987f194dcd91b0ba6d27f7f6e08c178797488e /kernel | |
| parent | 2b144498350860b6ee9dc57ff27a93ad488de5dc (diff) | |
uprobes/core: Clean up, refactor and improve the code
Make the uprobes code readable to me:
- improve the Kconfig text so that a mere mortal gets some idea
what CONFIG_UPROBES=y is really about
- do trivial renames to standardize around the uprobes_*() namespace
- clean up and simplify various code flow details
- separate basic blocks of functionality
- line break artifact and white space related removal
- use standard local varible definition blocks
- use vertical spacing to make things more readable
- remove unnecessary volatile
- restructure comment blocks to make them more uniform and
more readable in general
Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Cc: Jim Keniston <jkenisto@us.ibm.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Arnaldo Carvalho de Melo <acme@infradead.org>
Cc: Anton Arapov <anton@redhat.com>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
Link: http://lkml.kernel.org/n/tip-ewbwhb8o6navvllsauu7k07p@git.kernel.org
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/uprobes.c | 219 |
1 files changed, 127 insertions, 92 deletions
diff --git a/kernel/uprobes.c b/kernel/uprobes.c index 72e8bb3b52cd..884817f1b0d3 100644 --- a/kernel/uprobes.c +++ b/kernel/uprobes.c | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | /* | 1 | /* |
| 2 | * Userspace Probes (UProbes) | 2 | * User-space Probes (UProbes) |
| 3 | * | 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify | 4 | * This program is free software; you can redistribute it and/or modify |
| 5 | * it under the terms of the GNU General Public License as published by | 5 | * it under the terms of the GNU General Public License as published by |
| @@ -29,24 +29,26 @@ | |||
| 29 | #include <linux/rmap.h> /* anon_vma_prepare */ | 29 | #include <linux/rmap.h> /* anon_vma_prepare */ |
| 30 | #include <linux/mmu_notifier.h> /* set_pte_at_notify */ | 30 | #include <linux/mmu_notifier.h> /* set_pte_at_notify */ |
| 31 | #include <linux/swap.h> /* try_to_free_swap */ | 31 | #include <linux/swap.h> /* try_to_free_swap */ |
| 32 | |||
| 32 | #include <linux/uprobes.h> | 33 | #include <linux/uprobes.h> |
| 33 | 34 | ||
| 34 | static struct rb_root uprobes_tree = RB_ROOT; | 35 | static struct rb_root uprobes_tree = RB_ROOT; |
| 36 | |||
| 35 | static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */ | 37 | static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */ |
| 36 | 38 | ||
| 37 | #define UPROBES_HASH_SZ 13 | 39 | #define UPROBES_HASH_SZ 13 |
| 40 | |||
| 38 | /* serialize (un)register */ | 41 | /* serialize (un)register */ |
| 39 | static struct mutex uprobes_mutex[UPROBES_HASH_SZ]; | 42 | static struct mutex uprobes_mutex[UPROBES_HASH_SZ]; |
| 40 | #define uprobes_hash(v) (&uprobes_mutex[((unsigned long)(v)) %\ | 43 | |
| 41 | UPROBES_HASH_SZ]) | 44 | #define uprobes_hash(v) (&uprobes_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ]) |
| 42 | 45 | ||
| 43 | /* serialize uprobe->pending_list */ | 46 | /* serialize uprobe->pending_list */ |
| 44 | static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ]; | 47 | static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ]; |
| 45 | #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) %\ | 48 | #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ]) |
| 46 | UPROBES_HASH_SZ]) | ||
| 47 | 49 | ||
| 48 | /* | 50 | /* |
| 49 | * uprobe_events allows us to skip the mmap_uprobe if there are no uprobe | 51 | * uprobe_events allows us to skip the uprobe_mmap if there are no uprobe |
| 50 | * events active at this time. Probably a fine grained per inode count is | 52 | * events active at this time. Probably a fine grained per inode count is |
| 51 | * better? | 53 | * better? |
| 52 | */ | 54 | */ |
| @@ -58,9 +60,9 @@ static atomic_t uprobe_events = ATOMIC_INIT(0); | |||
| 58 | * vm_area_struct wasnt recommended. | 60 | * vm_area_struct wasnt recommended. |
| 59 | */ | 61 | */ |
| 60 | struct vma_info { | 62 | struct vma_info { |
| 61 | struct list_head probe_list; | 63 | struct list_head probe_list; |
| 62 | struct mm_struct *mm; | 64 | struct mm_struct *mm; |
| 63 | loff_t vaddr; | 65 | loff_t vaddr; |
| 64 | }; | 66 | }; |
| 65 | 67 | ||
| 66 | /* | 68 | /* |
| @@ -79,8 +81,7 @@ static bool valid_vma(struct vm_area_struct *vma, bool is_register) | |||
| 79 | if (!is_register) | 81 | if (!is_register) |
| 80 | return true; | 82 | return true; |
| 81 | 83 | ||
| 82 | if ((vma->vm_flags & (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)) == | 84 | if ((vma->vm_flags & (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)) == (VM_READ|VM_EXEC)) |
| 83 | (VM_READ|VM_EXEC)) | ||
| 84 | return true; | 85 | return true; |
| 85 | 86 | ||
| 86 | return false; | 87 | return false; |
| @@ -92,6 +93,7 @@ static loff_t vma_address(struct vm_area_struct *vma, loff_t offset) | |||
| 92 | 93 | ||
| 93 | vaddr = vma->vm_start + offset; | 94 | vaddr = vma->vm_start + offset; |
| 94 | vaddr -= vma->vm_pgoff << PAGE_SHIFT; | 95 | vaddr -= vma->vm_pgoff << PAGE_SHIFT; |
| 96 | |||
| 95 | return vaddr; | 97 | return vaddr; |
| 96 | } | 98 | } |
| 97 | 99 | ||
| @@ -105,8 +107,7 @@ static loff_t vma_address(struct vm_area_struct *vma, loff_t offset) | |||
| 105 | * | 107 | * |
| 106 | * Returns 0 on success, -EFAULT on failure. | 108 | * Returns 0 on success, -EFAULT on failure. |
| 107 | */ | 109 | */ |
| 108 | static int __replace_page(struct vm_area_struct *vma, struct page *page, | 110 | static int __replace_page(struct vm_area_struct *vma, struct page *page, struct page *kpage) |
| 109 | struct page *kpage) | ||
| 110 | { | 111 | { |
| 111 | struct mm_struct *mm = vma->vm_mm; | 112 | struct mm_struct *mm = vma->vm_mm; |
| 112 | pgd_t *pgd; | 113 | pgd_t *pgd; |
| @@ -163,7 +164,7 @@ out: | |||
| 163 | */ | 164 | */ |
| 164 | bool __weak is_bkpt_insn(uprobe_opcode_t *insn) | 165 | bool __weak is_bkpt_insn(uprobe_opcode_t *insn) |
| 165 | { | 166 | { |
| 166 | return (*insn == UPROBES_BKPT_INSN); | 167 | return *insn == UPROBES_BKPT_INSN; |
| 167 | } | 168 | } |
| 168 | 169 | ||
| 169 | /* | 170 | /* |
| @@ -203,6 +204,7 @@ static int write_opcode(struct mm_struct *mm, struct uprobe *uprobe, | |||
| 203 | ret = get_user_pages(NULL, mm, vaddr, 1, 0, 0, &old_page, &vma); | 204 | ret = get_user_pages(NULL, mm, vaddr, 1, 0, 0, &old_page, &vma); |
| 204 | if (ret <= 0) | 205 | if (ret <= 0) |
| 205 | return ret; | 206 | return ret; |
| 207 | |||
| 206 | ret = -EINVAL; | 208 | ret = -EINVAL; |
| 207 | 209 | ||
| 208 | /* | 210 | /* |
| @@ -239,6 +241,7 @@ static int write_opcode(struct mm_struct *mm, struct uprobe *uprobe, | |||
| 239 | vaddr_new = kmap_atomic(new_page); | 241 | vaddr_new = kmap_atomic(new_page); |
| 240 | 242 | ||
| 241 | memcpy(vaddr_new, vaddr_old, PAGE_SIZE); | 243 | memcpy(vaddr_new, vaddr_old, PAGE_SIZE); |
| 244 | |||
| 242 | /* poke the new insn in, ASSUMES we don't cross page boundary */ | 245 | /* poke the new insn in, ASSUMES we don't cross page boundary */ |
| 243 | vaddr &= ~PAGE_MASK; | 246 | vaddr &= ~PAGE_MASK; |
| 244 | BUG_ON(vaddr + uprobe_opcode_sz > PAGE_SIZE); | 247 | BUG_ON(vaddr + uprobe_opcode_sz > PAGE_SIZE); |
| @@ -260,7 +263,8 @@ unlock_out: | |||
| 260 | page_cache_release(new_page); | 263 | page_cache_release(new_page); |
| 261 | 264 | ||
| 262 | put_out: | 265 | put_out: |
| 263 | put_page(old_page); /* we did a get_page in the beginning */ | 266 | put_page(old_page); |
| 267 | |||
| 264 | return ret; | 268 | return ret; |
| 265 | } | 269 | } |
| 266 | 270 | ||
| @@ -276,8 +280,7 @@ put_out: | |||
| 276 | * For mm @mm, read the opcode at @vaddr and store it in @opcode. | 280 | * For mm @mm, read the opcode at @vaddr and store it in @opcode. |
| 277 | * Return 0 (success) or a negative errno. | 281 | * Return 0 (success) or a negative errno. |
| 278 | */ | 282 | */ |
| 279 | static int read_opcode(struct mm_struct *mm, unsigned long vaddr, | 283 | static int read_opcode(struct mm_struct *mm, unsigned long vaddr, uprobe_opcode_t *opcode) |
| 280 | uprobe_opcode_t *opcode) | ||
| 281 | { | 284 | { |
| 282 | struct page *page; | 285 | struct page *page; |
| 283 | void *vaddr_new; | 286 | void *vaddr_new; |
| @@ -293,15 +296,18 @@ static int read_opcode(struct mm_struct *mm, unsigned long vaddr, | |||
| 293 | memcpy(opcode, vaddr_new + vaddr, uprobe_opcode_sz); | 296 | memcpy(opcode, vaddr_new + vaddr, uprobe_opcode_sz); |
| 294 | kunmap_atomic(vaddr_new); | 297 | kunmap_atomic(vaddr_new); |
| 295 | unlock_page(page); | 298 | unlock_page(page); |
| 296 | put_page(page); /* we did a get_user_pages in the beginning */ | 299 | |
| 300 | put_page(page); | ||
| 301 | |||
| 297 | return 0; | 302 | return 0; |
| 298 | } | 303 | } |
| 299 | 304 | ||
| 300 | static int is_bkpt_at_addr(struct mm_struct *mm, unsigned long vaddr) | 305 | static int is_bkpt_at_addr(struct mm_struct *mm, unsigned long vaddr) |
| 301 | { | 306 | { |
| 302 | uprobe_opcode_t opcode; | 307 | uprobe_opcode_t opcode; |
| 303 | int result = read_opcode(mm, vaddr, &opcode); | 308 | int result; |
| 304 | 309 | ||
| 310 | result = read_opcode(mm, vaddr, &opcode); | ||
| 305 | if (result) | 311 | if (result) |
| 306 | return result; | 312 | return result; |
| 307 | 313 | ||
| @@ -320,11 +326,11 @@ static int is_bkpt_at_addr(struct mm_struct *mm, unsigned long vaddr) | |||
| 320 | * For mm @mm, store the breakpoint instruction at @vaddr. | 326 | * For mm @mm, store the breakpoint instruction at @vaddr. |
| 321 | * Return 0 (success) or a negative errno. | 327 | * Return 0 (success) or a negative errno. |
| 322 | */ | 328 | */ |
| 323 | int __weak set_bkpt(struct mm_struct *mm, struct uprobe *uprobe, | 329 | int __weak set_bkpt(struct mm_struct *mm, struct uprobe *uprobe, unsigned long vaddr) |
| 324 | unsigned long vaddr) | ||
| 325 | { | 330 | { |
| 326 | int result = is_bkpt_at_addr(mm, vaddr); | 331 | int result; |
| 327 | 332 | ||
| 333 | result = is_bkpt_at_addr(mm, vaddr); | ||
| 328 | if (result == 1) | 334 | if (result == 1) |
| 329 | return -EEXIST; | 335 | return -EEXIST; |
| 330 | 336 | ||
| @@ -344,35 +350,35 @@ int __weak set_bkpt(struct mm_struct *mm, struct uprobe *uprobe, | |||
| 344 | * For mm @mm, restore the original opcode (opcode) at @vaddr. | 350 | * For mm @mm, restore the original opcode (opcode) at @vaddr. |
| 345 | * Return 0 (success) or a negative errno. | 351 | * Return 0 (success) or a negative errno. |
| 346 | */ | 352 | */ |
| 347 | int __weak set_orig_insn(struct mm_struct *mm, struct uprobe *uprobe, | 353 | int __weak |
| 348 | unsigned long vaddr, bool verify) | 354 | set_orig_insn(struct mm_struct *mm, struct uprobe *uprobe, unsigned long vaddr, bool verify) |
| 349 | { | 355 | { |
| 350 | if (verify) { | 356 | if (verify) { |
| 351 | int result = is_bkpt_at_addr(mm, vaddr); | 357 | int result; |
| 352 | 358 | ||
| 359 | result = is_bkpt_at_addr(mm, vaddr); | ||
| 353 | if (!result) | 360 | if (!result) |
| 354 | return -EINVAL; | 361 | return -EINVAL; |
| 355 | 362 | ||
| 356 | if (result != 1) | 363 | if (result != 1) |
| 357 | return result; | 364 | return result; |
| 358 | } | 365 | } |
| 359 | return write_opcode(mm, uprobe, vaddr, | 366 | return write_opcode(mm, uprobe, vaddr, *(uprobe_opcode_t *)uprobe->insn); |
| 360 | *(uprobe_opcode_t *)uprobe->insn); | ||
| 361 | } | 367 | } |
| 362 | 368 | ||
| 363 | static int match_uprobe(struct uprobe *l, struct uprobe *r) | 369 | static int match_uprobe(struct uprobe *l, struct uprobe *r) |
| 364 | { | 370 | { |
| 365 | if (l->inode < r->inode) | 371 | if (l->inode < r->inode) |
| 366 | return -1; | 372 | return -1; |
| 373 | |||
| 367 | if (l->inode > r->inode) | 374 | if (l->inode > r->inode) |
| 368 | return 1; | 375 | return 1; |
| 369 | else { | ||
| 370 | if (l->offset < r->offset) | ||
| 371 | return -1; | ||
| 372 | 376 | ||
| 373 | if (l->offset > r->offset) | 377 | if (l->offset < r->offset) |
| 374 | return 1; | 378 | return -1; |
| 375 | } | 379 | |
| 380 | if (l->offset > r->offset) | ||
| 381 | return 1; | ||
| 376 | 382 | ||
| 377 | return 0; | 383 | return 0; |
| 378 | } | 384 | } |
| @@ -391,6 +397,7 @@ static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset) | |||
| 391 | atomic_inc(&uprobe->ref); | 397 | atomic_inc(&uprobe->ref); |
| 392 | return uprobe; | 398 | return uprobe; |
| 393 | } | 399 | } |
| 400 | |||
| 394 | if (match < 0) | 401 | if (match < 0) |
| 395 | n = n->rb_left; | 402 | n = n->rb_left; |
| 396 | else | 403 | else |
| @@ -411,6 +418,7 @@ static struct uprobe *find_uprobe(struct inode *inode, loff_t offset) | |||
| 411 | spin_lock_irqsave(&uprobes_treelock, flags); | 418 | spin_lock_irqsave(&uprobes_treelock, flags); |
| 412 | uprobe = __find_uprobe(inode, offset); | 419 | uprobe = __find_uprobe(inode, offset); |
| 413 | spin_unlock_irqrestore(&uprobes_treelock, flags); | 420 | spin_unlock_irqrestore(&uprobes_treelock, flags); |
| 421 | |||
| 414 | return uprobe; | 422 | return uprobe; |
| 415 | } | 423 | } |
| 416 | 424 | ||
| @@ -436,16 +444,18 @@ static struct uprobe *__insert_uprobe(struct uprobe *uprobe) | |||
| 436 | p = &parent->rb_right; | 444 | p = &parent->rb_right; |
| 437 | 445 | ||
| 438 | } | 446 | } |
| 447 | |||
| 439 | u = NULL; | 448 | u = NULL; |
| 440 | rb_link_node(&uprobe->rb_node, parent, p); | 449 | rb_link_node(&uprobe->rb_node, parent, p); |
| 441 | rb_insert_color(&uprobe->rb_node, &uprobes_tree); | 450 | rb_insert_color(&uprobe->rb_node, &uprobes_tree); |
| 442 | /* get access + creation ref */ | 451 | /* get access + creation ref */ |
| 443 | atomic_set(&uprobe->ref, 2); | 452 | atomic_set(&uprobe->ref, 2); |
| 453 | |||
| 444 | return u; | 454 | return u; |
| 445 | } | 455 | } |
| 446 | 456 | ||
| 447 | /* | 457 | /* |
| 448 | * Acquires uprobes_treelock. | 458 | * Acquire uprobes_treelock. |
| 449 | * Matching uprobe already exists in rbtree; | 459 | * Matching uprobe already exists in rbtree; |
| 450 | * increment (access refcount) and return the matching uprobe. | 460 | * increment (access refcount) and return the matching uprobe. |
| 451 | * | 461 | * |
| @@ -460,6 +470,7 @@ static struct uprobe *insert_uprobe(struct uprobe *uprobe) | |||
| 460 | spin_lock_irqsave(&uprobes_treelock, flags); | 470 | spin_lock_irqsave(&uprobes_treelock, flags); |
| 461 | u = __insert_uprobe(uprobe); | 471 | u = __insert_uprobe(uprobe); |
| 462 | spin_unlock_irqrestore(&uprobes_treelock, flags); | 472 | spin_unlock_irqrestore(&uprobes_treelock, flags); |
| 473 | |||
| 463 | return u; | 474 | return u; |
| 464 | } | 475 | } |
| 465 | 476 | ||
| @@ -490,19 +501,22 @@ static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset) | |||
| 490 | kfree(uprobe); | 501 | kfree(uprobe); |
| 491 | uprobe = cur_uprobe; | 502 | uprobe = cur_uprobe; |
| 492 | iput(inode); | 503 | iput(inode); |
| 493 | } else | 504 | } else { |
| 494 | atomic_inc(&uprobe_events); | 505 | atomic_inc(&uprobe_events); |
| 506 | } | ||
| 507 | |||
| 495 | return uprobe; | 508 | return uprobe; |
| 496 | } | 509 | } |
| 497 | 510 | ||
| 498 | /* Returns the previous consumer */ | 511 | /* Returns the previous consumer */ |
| 499 | static struct uprobe_consumer *add_consumer(struct uprobe *uprobe, | 512 | static struct uprobe_consumer * |
| 500 | struct uprobe_consumer *consumer) | 513 | consumer_add(struct uprobe *uprobe, struct uprobe_consumer *consumer) |
| 501 | { | 514 | { |
| 502 | down_write(&uprobe->consumer_rwsem); | 515 | down_write(&uprobe->consumer_rwsem); |
| 503 | consumer->next = uprobe->consumers; | 516 | consumer->next = uprobe->consumers; |
| 504 | uprobe->consumers = consumer; | 517 | uprobe->consumers = consumer; |
| 505 | up_write(&uprobe->consumer_rwsem); | 518 | up_write(&uprobe->consumer_rwsem); |
| 519 | |||
| 506 | return consumer->next; | 520 | return consumer->next; |
| 507 | } | 521 | } |
| 508 | 522 | ||
| @@ -511,8 +525,7 @@ static struct uprobe_consumer *add_consumer(struct uprobe *uprobe, | |||
| 511 | * Return true if the @consumer is deleted successfully | 525 | * Return true if the @consumer is deleted successfully |
| 512 | * or return false. | 526 | * or return false. |
| 513 | */ | 527 | */ |
| 514 | static bool del_consumer(struct uprobe *uprobe, | 528 | static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *consumer) |
| 515 | struct uprobe_consumer *consumer) | ||
| 516 | { | 529 | { |
| 517 | struct uprobe_consumer **con; | 530 | struct uprobe_consumer **con; |
| 518 | bool ret = false; | 531 | bool ret = false; |
| @@ -526,6 +539,7 @@ static bool del_consumer(struct uprobe *uprobe, | |||
| 526 | } | 539 | } |
| 527 | } | 540 | } |
| 528 | up_write(&uprobe->consumer_rwsem); | 541 | up_write(&uprobe->consumer_rwsem); |
| 542 | |||
| 529 | return ret; | 543 | return ret; |
| 530 | } | 544 | } |
| 531 | 545 | ||
| @@ -557,15 +571,15 @@ static int __copy_insn(struct address_space *mapping, | |||
| 557 | memcpy(insn, vaddr + off1, nbytes); | 571 | memcpy(insn, vaddr + off1, nbytes); |
| 558 | kunmap_atomic(vaddr); | 572 | kunmap_atomic(vaddr); |
| 559 | page_cache_release(page); | 573 | page_cache_release(page); |
| 574 | |||
| 560 | return 0; | 575 | return 0; |
| 561 | } | 576 | } |
| 562 | 577 | ||
| 563 | static int copy_insn(struct uprobe *uprobe, struct vm_area_struct *vma, | 578 | static int copy_insn(struct uprobe *uprobe, struct vm_area_struct *vma, unsigned long addr) |
| 564 | unsigned long addr) | ||
| 565 | { | 579 | { |
| 566 | struct address_space *mapping; | 580 | struct address_space *mapping; |
| 567 | int bytes; | ||
| 568 | unsigned long nbytes; | 581 | unsigned long nbytes; |
| 582 | int bytes; | ||
| 569 | 583 | ||
| 570 | addr &= ~PAGE_MASK; | 584 | addr &= ~PAGE_MASK; |
| 571 | nbytes = PAGE_SIZE - addr; | 585 | nbytes = PAGE_SIZE - addr; |
| @@ -605,6 +619,7 @@ static int install_breakpoint(struct mm_struct *mm, struct uprobe *uprobe, | |||
| 605 | return -EEXIST; | 619 | return -EEXIST; |
| 606 | 620 | ||
| 607 | addr = (unsigned long)vaddr; | 621 | addr = (unsigned long)vaddr; |
| 622 | |||
| 608 | if (!(uprobe->flags & UPROBES_COPY_INSN)) { | 623 | if (!(uprobe->flags & UPROBES_COPY_INSN)) { |
| 609 | ret = copy_insn(uprobe, vma, addr); | 624 | ret = copy_insn(uprobe, vma, addr); |
| 610 | if (ret) | 625 | if (ret) |
| @@ -613,7 +628,7 @@ static int install_breakpoint(struct mm_struct *mm, struct uprobe *uprobe, | |||
| 613 | if (is_bkpt_insn((uprobe_opcode_t *)uprobe->insn)) | 628 | if (is_bkpt_insn((uprobe_opcode_t *)uprobe->insn)) |
| 614 | return -EEXIST; | 629 | return -EEXIST; |
| 615 | 630 | ||
| 616 | ret = analyze_insn(mm, uprobe); | 631 | ret = arch_uprobes_analyze_insn(mm, uprobe); |
| 617 | if (ret) | 632 | if (ret) |
| 618 | return ret; | 633 | return ret; |
| 619 | 634 | ||
| @@ -624,8 +639,7 @@ static int install_breakpoint(struct mm_struct *mm, struct uprobe *uprobe, | |||
| 624 | return ret; | 639 | return ret; |
| 625 | } | 640 | } |
| 626 | 641 | ||
| 627 | static void remove_breakpoint(struct mm_struct *mm, struct uprobe *uprobe, | 642 | static void remove_breakpoint(struct mm_struct *mm, struct uprobe *uprobe, loff_t vaddr) |
| 628 | loff_t vaddr) | ||
| 629 | { | 643 | { |
| 630 | set_orig_insn(mm, uprobe, (unsigned long)vaddr, true); | 644 | set_orig_insn(mm, uprobe, (unsigned long)vaddr, true); |
| 631 | } | 645 | } |
| @@ -649,9 +663,11 @@ static struct vma_info *__find_next_vma_info(struct list_head *head, | |||
| 649 | struct prio_tree_iter iter; | 663 | struct prio_tree_iter iter; |
| 650 | struct vm_area_struct *vma; | 664 | struct vm_area_struct *vma; |
| 651 | struct vma_info *tmpvi; | 665 | struct vma_info *tmpvi; |
| 652 | loff_t vaddr; | 666 | unsigned long pgoff; |
| 653 | unsigned long pgoff = offset >> PAGE_SHIFT; | ||
| 654 | int existing_vma; | 667 | int existing_vma; |
| 668 | loff_t vaddr; | ||
| 669 | |||
| 670 | pgoff = offset >> PAGE_SHIFT; | ||
| 655 | 671 | ||
| 656 | vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) { | 672 | vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) { |
| 657 | if (!valid_vma(vma, is_register)) | 673 | if (!valid_vma(vma, is_register)) |
| @@ -659,6 +675,7 @@ static struct vma_info *__find_next_vma_info(struct list_head *head, | |||
| 659 | 675 | ||
| 660 | existing_vma = 0; | 676 | existing_vma = 0; |
| 661 | vaddr = vma_address(vma, offset); | 677 | vaddr = vma_address(vma, offset); |
| 678 | |||
| 662 | list_for_each_entry(tmpvi, head, probe_list) { | 679 | list_for_each_entry(tmpvi, head, probe_list) { |
| 663 | if (tmpvi->mm == vma->vm_mm && tmpvi->vaddr == vaddr) { | 680 | if (tmpvi->mm == vma->vm_mm && tmpvi->vaddr == vaddr) { |
| 664 | existing_vma = 1; | 681 | existing_vma = 1; |
| @@ -670,14 +687,15 @@ static struct vma_info *__find_next_vma_info(struct list_head *head, | |||
| 670 | * Another vma needs a probe to be installed. However skip | 687 | * Another vma needs a probe to be installed. However skip |
| 671 | * installing the probe if the vma is about to be unlinked. | 688 | * installing the probe if the vma is about to be unlinked. |
| 672 | */ | 689 | */ |
| 673 | if (!existing_vma && | 690 | if (!existing_vma && atomic_inc_not_zero(&vma->vm_mm->mm_users)) { |
| 674 | atomic_inc_not_zero(&vma->vm_mm->mm_users)) { | ||
| 675 | vi->mm = vma->vm_mm; | 691 | vi->mm = vma->vm_mm; |
| 676 | vi->vaddr = vaddr; | 692 | vi->vaddr = vaddr; |
| 677 | list_add(&vi->probe_list, head); | 693 | list_add(&vi->probe_list, head); |
| 694 | |||
| 678 | return vi; | 695 | return vi; |
| 679 | } | 696 | } |
| 680 | } | 697 | } |
| 698 | |||
| 681 | return NULL; | 699 | return NULL; |
| 682 | } | 700 | } |
| 683 | 701 | ||
| @@ -685,11 +703,12 @@ static struct vma_info *__find_next_vma_info(struct list_head *head, | |||
| 685 | * Iterate in the rmap prio tree and find a vma where a probe has not | 703 | * Iterate in the rmap prio tree and find a vma where a probe has not |
| 686 | * yet been inserted. | 704 | * yet been inserted. |
| 687 | */ | 705 | */ |
| 688 | static struct vma_info *find_next_vma_info(struct list_head *head, | 706 | static struct vma_info * |
| 689 | loff_t offset, struct address_space *mapping, | 707 | find_next_vma_info(struct list_head *head, loff_t offset, struct address_space *mapping, |
| 690 | bool is_register) | 708 | bool is_register) |
| 691 | { | 709 | { |
| 692 | struct vma_info *vi, *retvi; | 710 | struct vma_info *vi, *retvi; |
| 711 | |||
| 693 | vi = kzalloc(sizeof(struct vma_info), GFP_KERNEL); | 712 | vi = kzalloc(sizeof(struct vma_info), GFP_KERNEL); |
| 694 | if (!vi) | 713 | if (!vi) |
| 695 | return ERR_PTR(-ENOMEM); | 714 | return ERR_PTR(-ENOMEM); |
| @@ -700,6 +719,7 @@ static struct vma_info *find_next_vma_info(struct list_head *head, | |||
| 700 | 719 | ||
| 701 | if (!retvi) | 720 | if (!retvi) |
| 702 | kfree(vi); | 721 | kfree(vi); |
| 722 | |||
| 703 | return retvi; | 723 | return retvi; |
| 704 | } | 724 | } |
| 705 | 725 | ||
| @@ -711,16 +731,23 @@ static int register_for_each_vma(struct uprobe *uprobe, bool is_register) | |||
| 711 | struct vma_info *vi, *tmpvi; | 731 | struct vma_info *vi, *tmpvi; |
| 712 | struct mm_struct *mm; | 732 | struct mm_struct *mm; |
| 713 | loff_t vaddr; | 733 | loff_t vaddr; |
| 714 | int ret = 0; | 734 | int ret; |
| 715 | 735 | ||
| 716 | mapping = uprobe->inode->i_mapping; | 736 | mapping = uprobe->inode->i_mapping; |
| 717 | INIT_LIST_HEAD(&try_list); | 737 | INIT_LIST_HEAD(&try_list); |
| 718 | while ((vi = find_next_vma_info(&try_list, uprobe->offset, | 738 | |
| 719 | mapping, is_register)) != NULL) { | 739 | ret = 0; |
| 740 | |||
| 741 | for (;;) { | ||
| 742 | vi = find_next_vma_info(&try_list, uprobe->offset, mapping, is_register); | ||
| 743 | if (!vi) | ||
| 744 | break; | ||
| 745 | |||
| 720 | if (IS_ERR(vi)) { | 746 | if (IS_ERR(vi)) { |
| 721 | ret = PTR_ERR(vi); | 747 | ret = PTR_ERR(vi); |
| 722 | break; | 748 | break; |
| 723 | } | 749 | } |
| 750 | |||
| 724 | mm = vi->mm; | 751 | mm = vi->mm; |
| 725 | down_read(&mm->mmap_sem); | 752 | down_read(&mm->mmap_sem); |
| 726 | vma = find_vma(mm, (unsigned long)vi->vaddr); | 753 | vma = find_vma(mm, (unsigned long)vi->vaddr); |
| @@ -755,19 +782,21 @@ static int register_for_each_vma(struct uprobe *uprobe, bool is_register) | |||
| 755 | break; | 782 | break; |
| 756 | } | 783 | } |
| 757 | } | 784 | } |
| 785 | |||
| 758 | list_for_each_entry_safe(vi, tmpvi, &try_list, probe_list) { | 786 | list_for_each_entry_safe(vi, tmpvi, &try_list, probe_list) { |
| 759 | list_del(&vi->probe_list); | 787 | list_del(&vi->probe_list); |
| 760 | kfree(vi); | 788 | kfree(vi); |
| 761 | } | 789 | } |
| 790 | |||
| 762 | return ret; | 791 | return ret; |
| 763 | } | 792 | } |
| 764 | 793 | ||
| 765 | static int __register_uprobe(struct uprobe *uprobe) | 794 | static int __uprobe_register(struct uprobe *uprobe) |
| 766 | { | 795 | { |
| 767 | return register_for_each_vma(uprobe, true); | 796 | return register_for_each_vma(uprobe, true); |
| 768 | } | 797 | } |
| 769 | 798 | ||
| 770 | static void __unregister_uprobe(struct uprobe *uprobe) | 799 | static void __uprobe_unregister(struct uprobe *uprobe) |
| 771 | { | 800 | { |
| 772 | if (!register_for_each_vma(uprobe, false)) | 801 | if (!register_for_each_vma(uprobe, false)) |
| 773 | delete_uprobe(uprobe); | 802 | delete_uprobe(uprobe); |
| @@ -776,15 +805,15 @@ static void __unregister_uprobe(struct uprobe *uprobe) | |||
| 776 | } | 805 | } |
| 777 | 806 | ||
| 778 | /* | 807 | /* |
| 779 | * register_uprobe - register a probe | 808 | * uprobe_register - register a probe |
| 780 | * @inode: the file in which the probe has to be placed. | 809 | * @inode: the file in which the probe has to be placed. |
| 781 | * @offset: offset from the start of the file. | 810 | * @offset: offset from the start of the file. |
| 782 | * @consumer: information on howto handle the probe.. | 811 | * @consumer: information on howto handle the probe.. |
| 783 | * | 812 | * |
| 784 | * Apart from the access refcount, register_uprobe() takes a creation | 813 | * Apart from the access refcount, uprobe_register() takes a creation |
| 785 | * refcount (thro alloc_uprobe) if and only if this @uprobe is getting | 814 | * refcount (thro alloc_uprobe) if and only if this @uprobe is getting |
| 786 | * inserted into the rbtree (i.e first consumer for a @inode:@offset | 815 | * inserted into the rbtree (i.e first consumer for a @inode:@offset |
| 787 | * tuple). Creation refcount stops unregister_uprobe from freeing the | 816 | * tuple). Creation refcount stops uprobe_unregister from freeing the |
| 788 | * @uprobe even before the register operation is complete. Creation | 817 | * @uprobe even before the register operation is complete. Creation |
| 789 | * refcount is released when the last @consumer for the @uprobe | 818 | * refcount is released when the last @consumer for the @uprobe |
| 790 | * unregisters. | 819 | * unregisters. |
| @@ -792,28 +821,29 @@ static void __unregister_uprobe(struct uprobe *uprobe) | |||
| 792 | * Return errno if it cannot successully install probes | 821 | * Return errno if it cannot successully install probes |
| 793 | * else return 0 (success) | 822 | * else return 0 (success) |
| 794 | */ | 823 | */ |
| 795 | int register_uprobe(struct inode *inode, loff_t offset, | 824 | int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *consumer) |
| 796 | struct uprobe_consumer *consumer) | ||
| 797 | { | 825 | { |
| 798 | struct uprobe *uprobe; | 826 | struct uprobe *uprobe; |
| 799 | int ret = -EINVAL; | 827 | int ret; |
| 800 | 828 | ||
| 801 | if (!inode || !consumer || consumer->next) | 829 | if (!inode || !consumer || consumer->next) |
| 802 | return ret; | 830 | return -EINVAL; |
| 803 | 831 | ||
| 804 | if (offset > i_size_read(inode)) | 832 | if (offset > i_size_read(inode)) |
| 805 | return ret; | 833 | return -EINVAL; |
| 806 | 834 | ||
| 807 | ret = 0; | 835 | ret = 0; |
| 808 | mutex_lock(uprobes_hash(inode)); | 836 | mutex_lock(uprobes_hash(inode)); |
| 809 | uprobe = alloc_uprobe(inode, offset); | 837 | uprobe = alloc_uprobe(inode, offset); |
| 810 | if (uprobe && !add_consumer(uprobe, consumer)) { | 838 | |
| 811 | ret = __register_uprobe(uprobe); | 839 | if (uprobe && !consumer_add(uprobe, consumer)) { |
| 840 | ret = __uprobe_register(uprobe); | ||
| 812 | if (ret) { | 841 | if (ret) { |
| 813 | uprobe->consumers = NULL; | 842 | uprobe->consumers = NULL; |
| 814 | __unregister_uprobe(uprobe); | 843 | __uprobe_unregister(uprobe); |
| 815 | } else | 844 | } else { |
| 816 | uprobe->flags |= UPROBES_RUN_HANDLER; | 845 | uprobe->flags |= UPROBES_RUN_HANDLER; |
| 846 | } | ||
| 817 | } | 847 | } |
| 818 | 848 | ||
| 819 | mutex_unlock(uprobes_hash(inode)); | 849 | mutex_unlock(uprobes_hash(inode)); |
| @@ -823,15 +853,14 @@ int register_uprobe(struct inode *inode, loff_t offset, | |||
| 823 | } | 853 | } |
| 824 | 854 | ||
| 825 | /* | 855 | /* |
| 826 | * unregister_uprobe - unregister a already registered probe. | 856 | * uprobe_unregister - unregister a already registered probe. |
| 827 | * @inode: the file in which the probe has to be removed. | 857 | * @inode: the file in which the probe has to be removed. |
| 828 | * @offset: offset from the start of the file. | 858 | * @offset: offset from the start of the file. |
| 829 | * @consumer: identify which probe if multiple probes are colocated. | 859 | * @consumer: identify which probe if multiple probes are colocated. |
| 830 | */ | 860 | */ |
| 831 | void unregister_uprobe(struct inode *inode, loff_t offset, | 861 | void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *consumer) |
| 832 | struct uprobe_consumer *consumer) | ||
| 833 | { | 862 | { |
| 834 | struct uprobe *uprobe = NULL; | 863 | struct uprobe *uprobe; |
| 835 | 864 | ||
| 836 | if (!inode || !consumer) | 865 | if (!inode || !consumer) |
| 837 | return; | 866 | return; |
| @@ -841,15 +870,14 @@ void unregister_uprobe(struct inode *inode, loff_t offset, | |||
| 841 | return; | 870 | return; |
| 842 | 871 | ||
| 843 | mutex_lock(uprobes_hash(inode)); | 872 | mutex_lock(uprobes_hash(inode)); |
| 844 | if (!del_consumer(uprobe, consumer)) | ||
| 845 | goto unreg_out; | ||
| 846 | 873 | ||
| 847 | if (!uprobe->consumers) { | 874 | if (consumer_del(uprobe, consumer)) { |
| 848 | __unregister_uprobe(uprobe); | 875 | if (!uprobe->consumers) { |
| 849 | uprobe->flags &= ~UPROBES_RUN_HANDLER; | 876 | __uprobe_unregister(uprobe); |
| 877 | uprobe->flags &= ~UPROBES_RUN_HANDLER; | ||
| 878 | } | ||
| 850 | } | 879 | } |
| 851 | 880 | ||
| 852 | unreg_out: | ||
| 853 | mutex_unlock(uprobes_hash(inode)); | 881 | mutex_unlock(uprobes_hash(inode)); |
| 854 | if (uprobe) | 882 | if (uprobe) |
| 855 | put_uprobe(uprobe); | 883 | put_uprobe(uprobe); |
| @@ -870,6 +898,7 @@ static struct rb_node *find_least_offset_node(struct inode *inode) | |||
| 870 | while (n) { | 898 | while (n) { |
| 871 | uprobe = rb_entry(n, struct uprobe, rb_node); | 899 | uprobe = rb_entry(n, struct uprobe, rb_node); |
| 872 | match = match_uprobe(&u, uprobe); | 900 | match = match_uprobe(&u, uprobe); |
| 901 | |||
| 873 | if (uprobe->inode == inode) | 902 | if (uprobe->inode == inode) |
| 874 | close_node = n; | 903 | close_node = n; |
| 875 | 904 | ||
| @@ -881,6 +910,7 @@ static struct rb_node *find_least_offset_node(struct inode *inode) | |||
| 881 | else | 910 | else |
| 882 | n = n->rb_right; | 911 | n = n->rb_right; |
| 883 | } | 912 | } |
| 913 | |||
| 884 | return close_node; | 914 | return close_node; |
| 885 | } | 915 | } |
| 886 | 916 | ||
| @@ -890,11 +920,13 @@ static struct rb_node *find_least_offset_node(struct inode *inode) | |||
| 890 | static void build_probe_list(struct inode *inode, struct list_head *head) | 920 | static void build_probe_list(struct inode *inode, struct list_head *head) |
| 891 | { | 921 | { |
| 892 | struct uprobe *uprobe; | 922 | struct uprobe *uprobe; |
| 893 | struct rb_node *n; | ||
| 894 | unsigned long flags; | 923 | unsigned long flags; |
| 924 | struct rb_node *n; | ||
| 895 | 925 | ||
| 896 | spin_lock_irqsave(&uprobes_treelock, flags); | 926 | spin_lock_irqsave(&uprobes_treelock, flags); |
| 927 | |||
| 897 | n = find_least_offset_node(inode); | 928 | n = find_least_offset_node(inode); |
| 929 | |||
| 898 | for (; n; n = rb_next(n)) { | 930 | for (; n; n = rb_next(n)) { |
| 899 | uprobe = rb_entry(n, struct uprobe, rb_node); | 931 | uprobe = rb_entry(n, struct uprobe, rb_node); |
| 900 | if (uprobe->inode != inode) | 932 | if (uprobe->inode != inode) |
| @@ -903,6 +935,7 @@ static void build_probe_list(struct inode *inode, struct list_head *head) | |||
| 903 | list_add(&uprobe->pending_list, head); | 935 | list_add(&uprobe->pending_list, head); |
| 904 | atomic_inc(&uprobe->ref); | 936 | atomic_inc(&uprobe->ref); |
| 905 | } | 937 | } |
| 938 | |||
| 906 | spin_unlock_irqrestore(&uprobes_treelock, flags); | 939 | spin_unlock_irqrestore(&uprobes_treelock, flags); |
| 907 | } | 940 | } |
| 908 | 941 | ||
| @@ -912,42 +945,44 @@ static void build_probe_list(struct inode *inode, struct list_head *head) | |||
| 912 | * | 945 | * |
| 913 | * Return -ve no if we fail to insert probes and we cannot | 946 | * Return -ve no if we fail to insert probes and we cannot |
| 914 | * bail-out. | 947 | * bail-out. |
| 915 | * Return 0 otherwise. i.e : | 948 | * Return 0 otherwise. i.e: |
| 949 | * | ||
| 916 | * - successful insertion of probes | 950 | * - successful insertion of probes |
| 917 | * - (or) no possible probes to be inserted. | 951 | * - (or) no possible probes to be inserted. |
| 918 | * - (or) insertion of probes failed but we can bail-out. | 952 | * - (or) insertion of probes failed but we can bail-out. |
| 919 | */ | 953 | */ |
| 920 | int mmap_uprobe(struct vm_area_struct *vma) | 954 | int uprobe_mmap(struct vm_area_struct *vma) |
| 921 | { | 955 | { |
| 922 | struct list_head tmp_list; | 956 | struct list_head tmp_list; |
| 923 | struct uprobe *uprobe, *u; | 957 | struct uprobe *uprobe, *u; |
| 924 | struct inode *inode; | 958 | struct inode *inode; |
| 925 | int ret = 0; | 959 | int ret; |
| 926 | 960 | ||
| 927 | if (!atomic_read(&uprobe_events) || !valid_vma(vma, true)) | 961 | if (!atomic_read(&uprobe_events) || !valid_vma(vma, true)) |
| 928 | return ret; /* Bail-out */ | 962 | return 0; |
| 929 | 963 | ||
| 930 | inode = vma->vm_file->f_mapping->host; | 964 | inode = vma->vm_file->f_mapping->host; |
| 931 | if (!inode) | 965 | if (!inode) |
| 932 | return ret; | 966 | return 0; |
| 933 | 967 | ||
| 934 | INIT_LIST_HEAD(&tmp_list); | 968 | INIT_LIST_HEAD(&tmp_list); |
| 935 | mutex_lock(uprobes_mmap_hash(inode)); | 969 | mutex_lock(uprobes_mmap_hash(inode)); |
| 936 | build_probe_list(inode, &tmp_list); | 970 | build_probe_list(inode, &tmp_list); |
| 971 | |||
| 972 | ret = 0; | ||
| 973 | |||
| 937 | list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) { | 974 | list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) { |
| 938 | loff_t vaddr; | 975 | loff_t vaddr; |
| 939 | 976 | ||
| 940 | list_del(&uprobe->pending_list); | 977 | list_del(&uprobe->pending_list); |
| 941 | if (!ret) { | 978 | if (!ret) { |
| 942 | vaddr = vma_address(vma, uprobe->offset); | 979 | vaddr = vma_address(vma, uprobe->offset); |
| 943 | if (vaddr < vma->vm_start || vaddr >= vma->vm_end) { | 980 | if (vaddr >= vma->vm_start && vaddr < vma->vm_end) { |
| 944 | put_uprobe(uprobe); | 981 | ret = install_breakpoint(vma->vm_mm, uprobe, vma, vaddr); |
| 945 | continue; | 982 | /* Ignore double add: */ |
| 983 | if (ret == -EEXIST) | ||
| 984 | ret = 0; | ||
| 946 | } | 985 | } |
| 947 | ret = install_breakpoint(vma->vm_mm, uprobe, vma, | ||
| 948 | vaddr); | ||
| 949 | if (ret == -EEXIST) | ||
| 950 | ret = 0; | ||
| 951 | } | 986 | } |
| 952 | put_uprobe(uprobe); | 987 | put_uprobe(uprobe); |
| 953 | } | 988 | } |
