diff options
author | Ingo Molnar <mingo@elte.hu> | 2006-07-28 23:17:57 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@g5.osdl.org> | 2006-07-29 00:02:00 -0400 |
commit | e3f2ddeac718c768fdac4b7fe69d465172f788a8 (patch) | |
tree | 5428532dc6c87710c35a71858425f6d726f0e44c /kernel/futex.c | |
parent | 627371d73cdd04ed23fe098755b4f855138ad9e0 (diff) |
[PATCH] pi-futex: robust-futex exit
Fix robust PI-futexes to be properly unlocked on unexpected exit.
For this to work the kernel has to know whether a futex is a PI or a
non-PI one, because the semantics are different. Since the space in
relevant glibc data structures is extremely scarce, the best solution is
to encode the 'PI' information in bit 0 of the robust list pointer.
Existing (non-PI) glibc robust futexes have this bit always zero, so the
ABI is kept. New glibc with PI-robust-futexes will set this bit.
Further fixes from Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Ulrich Drepper <drepper@redhat.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Diffstat (limited to 'kernel/futex.c')
-rw-r--r-- | kernel/futex.c | 91 |
1 files changed, 62 insertions, 29 deletions
diff --git a/kernel/futex.c b/kernel/futex.c index f59003b1d8f9..dda2049692a2 100644 --- a/kernel/futex.c +++ b/kernel/futex.c | |||
@@ -495,10 +495,13 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb, struct futex_q *me) | |||
495 | } | 495 | } |
496 | 496 | ||
497 | /* | 497 | /* |
498 | * We are the first waiter - try to look up the real owner and | 498 | * We are the first waiter - try to look up the real owner and attach |
499 | * attach the new pi_state to it: | 499 | * the new pi_state to it, but bail out when the owner died bit is set |
500 | * and TID = 0: | ||
500 | */ | 501 | */ |
501 | pid = uval & FUTEX_TID_MASK; | 502 | pid = uval & FUTEX_TID_MASK; |
503 | if (!pid && (uval & FUTEX_OWNER_DIED)) | ||
504 | return -ESRCH; | ||
502 | p = futex_find_get_task(pid); | 505 | p = futex_find_get_task(pid); |
503 | if (!p) | 506 | if (!p) |
504 | return -ESRCH; | 507 | return -ESRCH; |
@@ -579,16 +582,17 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this) | |||
579 | * kept enabled while there is PI state around. We must also | 582 | * kept enabled while there is PI state around. We must also |
580 | * preserve the owner died bit.) | 583 | * preserve the owner died bit.) |
581 | */ | 584 | */ |
582 | newval = (uval & FUTEX_OWNER_DIED) | FUTEX_WAITERS | new_owner->pid; | 585 | if (!(uval & FUTEX_OWNER_DIED)) { |
583 | 586 | newval = FUTEX_WAITERS | new_owner->pid; | |
584 | inc_preempt_count(); | ||
585 | curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval); | ||
586 | dec_preempt_count(); | ||
587 | 587 | ||
588 | if (curval == -EFAULT) | 588 | inc_preempt_count(); |
589 | return -EFAULT; | 589 | curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval); |
590 | if (curval != uval) | 590 | dec_preempt_count(); |
591 | return -EINVAL; | 591 | if (curval == -EFAULT) |
592 | return -EFAULT; | ||
593 | if (curval != uval) | ||
594 | return -EINVAL; | ||
595 | } | ||
592 | 596 | ||
593 | spin_lock_irq(&pi_state->owner->pi_lock); | 597 | spin_lock_irq(&pi_state->owner->pi_lock); |
594 | WARN_ON(list_empty(&pi_state->list)); | 598 | WARN_ON(list_empty(&pi_state->list)); |
@@ -1443,9 +1447,11 @@ retry_locked: | |||
1443 | * again. If it succeeds then we can return without waking | 1447 | * again. If it succeeds then we can return without waking |
1444 | * anyone else up: | 1448 | * anyone else up: |
1445 | */ | 1449 | */ |
1446 | inc_preempt_count(); | 1450 | if (!(uval & FUTEX_OWNER_DIED)) { |
1447 | uval = futex_atomic_cmpxchg_inatomic(uaddr, current->pid, 0); | 1451 | inc_preempt_count(); |
1448 | dec_preempt_count(); | 1452 | uval = futex_atomic_cmpxchg_inatomic(uaddr, current->pid, 0); |
1453 | dec_preempt_count(); | ||
1454 | } | ||
1449 | 1455 | ||
1450 | if (unlikely(uval == -EFAULT)) | 1456 | if (unlikely(uval == -EFAULT)) |
1451 | goto pi_faulted; | 1457 | goto pi_faulted; |
@@ -1478,9 +1484,11 @@ retry_locked: | |||
1478 | /* | 1484 | /* |
1479 | * No waiters - kernel unlocks the futex: | 1485 | * No waiters - kernel unlocks the futex: |
1480 | */ | 1486 | */ |
1481 | ret = unlock_futex_pi(uaddr, uval); | 1487 | if (!(uval & FUTEX_OWNER_DIED)) { |
1482 | if (ret == -EFAULT) | 1488 | ret = unlock_futex_pi(uaddr, uval); |
1483 | goto pi_faulted; | 1489 | if (ret == -EFAULT) |
1490 | goto pi_faulted; | ||
1491 | } | ||
1484 | 1492 | ||
1485 | out_unlock: | 1493 | out_unlock: |
1486 | spin_unlock(&hb->lock); | 1494 | spin_unlock(&hb->lock); |
@@ -1699,9 +1707,9 @@ err_unlock: | |||
1699 | * Process a futex-list entry, check whether it's owned by the | 1707 | * Process a futex-list entry, check whether it's owned by the |
1700 | * dying task, and do notification if so: | 1708 | * dying task, and do notification if so: |
1701 | */ | 1709 | */ |
1702 | int handle_futex_death(u32 __user *uaddr, struct task_struct *curr) | 1710 | int handle_futex_death(u32 __user *uaddr, struct task_struct *curr, int pi) |
1703 | { | 1711 | { |
1704 | u32 uval, nval; | 1712 | u32 uval, nval, mval; |
1705 | 1713 | ||
1706 | retry: | 1714 | retry: |
1707 | if (get_user(uval, uaddr)) | 1715 | if (get_user(uval, uaddr)) |
@@ -1718,21 +1726,45 @@ retry: | |||
1718 | * thread-death.) The rest of the cleanup is done in | 1726 | * thread-death.) The rest of the cleanup is done in |
1719 | * userspace. | 1727 | * userspace. |
1720 | */ | 1728 | */ |
1721 | nval = futex_atomic_cmpxchg_inatomic(uaddr, uval, | 1729 | mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED; |
1722 | uval | FUTEX_OWNER_DIED); | 1730 | nval = futex_atomic_cmpxchg_inatomic(uaddr, uval, mval); |
1731 | |||
1723 | if (nval == -EFAULT) | 1732 | if (nval == -EFAULT) |
1724 | return -1; | 1733 | return -1; |
1725 | 1734 | ||
1726 | if (nval != uval) | 1735 | if (nval != uval) |
1727 | goto retry; | 1736 | goto retry; |
1728 | 1737 | ||
1729 | if (uval & FUTEX_WAITERS) | 1738 | /* |
1730 | futex_wake(uaddr, 1); | 1739 | * Wake robust non-PI futexes here. The wakeup of |
1740 | * PI futexes happens in exit_pi_state(): | ||
1741 | */ | ||
1742 | if (!pi) { | ||
1743 | if (uval & FUTEX_WAITERS) | ||
1744 | futex_wake(uaddr, 1); | ||
1745 | } | ||
1731 | } | 1746 | } |
1732 | return 0; | 1747 | return 0; |
1733 | } | 1748 | } |
1734 | 1749 | ||
1735 | /* | 1750 | /* |
1751 | * Fetch a robust-list pointer. Bit 0 signals PI futexes: | ||
1752 | */ | ||
1753 | static inline int fetch_robust_entry(struct robust_list __user **entry, | ||
1754 | struct robust_list __user **head, int *pi) | ||
1755 | { | ||
1756 | unsigned long uentry; | ||
1757 | |||
1758 | if (get_user(uentry, (unsigned long *)head)) | ||
1759 | return -EFAULT; | ||
1760 | |||
1761 | *entry = (void *)(uentry & ~1UL); | ||
1762 | *pi = uentry & 1; | ||
1763 | |||
1764 | return 0; | ||
1765 | } | ||
1766 | |||
1767 | /* | ||
1736 | * Walk curr->robust_list (very carefully, it's a userspace list!) | 1768 | * Walk curr->robust_list (very carefully, it's a userspace list!) |
1737 | * and mark any locks found there dead, and notify any waiters. | 1769 | * and mark any locks found there dead, and notify any waiters. |
1738 | * | 1770 | * |
@@ -1742,14 +1774,14 @@ void exit_robust_list(struct task_struct *curr) | |||
1742 | { | 1774 | { |
1743 | struct robust_list_head __user *head = curr->robust_list; | 1775 | struct robust_list_head __user *head = curr->robust_list; |
1744 | struct robust_list __user *entry, *pending; | 1776 | struct robust_list __user *entry, *pending; |
1745 | unsigned int limit = ROBUST_LIST_LIMIT; | 1777 | unsigned int limit = ROBUST_LIST_LIMIT, pi, pip; |
1746 | unsigned long futex_offset; | 1778 | unsigned long futex_offset; |
1747 | 1779 | ||
1748 | /* | 1780 | /* |
1749 | * Fetch the list head (which was registered earlier, via | 1781 | * Fetch the list head (which was registered earlier, via |
1750 | * sys_set_robust_list()): | 1782 | * sys_set_robust_list()): |
1751 | */ | 1783 | */ |
1752 | if (get_user(entry, &head->list.next)) | 1784 | if (fetch_robust_entry(&entry, &head->list.next, &pi)) |
1753 | return; | 1785 | return; |
1754 | /* | 1786 | /* |
1755 | * Fetch the relative futex offset: | 1787 | * Fetch the relative futex offset: |
@@ -1760,10 +1792,11 @@ void exit_robust_list(struct task_struct *curr) | |||
1760 | * Fetch any possibly pending lock-add first, and handle it | 1792 | * Fetch any possibly pending lock-add first, and handle it |
1761 | * if it exists: | 1793 | * if it exists: |
1762 | */ | 1794 | */ |
1763 | if (get_user(pending, &head->list_op_pending)) | 1795 | if (fetch_robust_entry(&pending, &head->list_op_pending, &pip)) |
1764 | return; | 1796 | return; |
1797 | |||
1765 | if (pending) | 1798 | if (pending) |
1766 | handle_futex_death((void *)pending + futex_offset, curr); | 1799 | handle_futex_death((void *)pending + futex_offset, curr, pip); |
1767 | 1800 | ||
1768 | while (entry != &head->list) { | 1801 | while (entry != &head->list) { |
1769 | /* | 1802 | /* |
@@ -1772,12 +1805,12 @@ void exit_robust_list(struct task_struct *curr) | |||
1772 | */ | 1805 | */ |
1773 | if (entry != pending) | 1806 | if (entry != pending) |
1774 | if (handle_futex_death((void *)entry + futex_offset, | 1807 | if (handle_futex_death((void *)entry + futex_offset, |
1775 | curr)) | 1808 | curr, pi)) |
1776 | return; | 1809 | return; |
1777 | /* | 1810 | /* |
1778 | * Fetch the next entry in the list: | 1811 | * Fetch the next entry in the list: |
1779 | */ | 1812 | */ |
1780 | if (get_user(entry, &entry->next)) | 1813 | if (fetch_robust_entry(&entry, &entry->next, &pi)) |
1781 | return; | 1814 | return; |
1782 | /* | 1815 | /* |
1783 | * Avoid excessively long or circular lists: | 1816 | * Avoid excessively long or circular lists: |