aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
authorDarren Hart <dvhltc@us.ibm.com>2009-03-12 03:56:13 -0400
committerIngo Molnar <mingo@elte.hu>2009-03-12 06:20:57 -0400
commite4dc5b7a36a49eff97050894cf1b3a9a02523717 (patch)
tree91a0aceb21b1869385507c2631bf97fbf180697b /kernel
parente8f6386c01a5699c115bdad10271a24076364c97 (diff)
futex: clean up fault logic
Impact: cleanup Older versions of the futex code held the mmap_sem which had to be dropped in order to call get_user(), so a two-pronged fault handling mechanism was employed to handle faults of the atomic operations. The mmap_sem is no longer held, so get_user() should be adequate. This patch greatly simplifies the logic and improves legibility. Build and boot tested on a 4 way Intel x86_64 workstation. Passes basic pthread_mutex and PI tests out of ltp/testcases/realtime. Signed-off-by: Darren Hart <dvhltc@us.ibm.com> Acked-by: Peter Zijlstra <peterz@infradead.org> Cc: Rusty Russell <rusty@rustcorp.com.au> LKML-Reference: <20090312075612.9856.48612.stgit@Aeon> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'kernel')
-rw-r--r--kernel/futex.c126
1 files changed, 36 insertions, 90 deletions
diff --git a/kernel/futex.c b/kernel/futex.c
index c980a556f82c..9c97f67d298e 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -298,41 +298,6 @@ static int get_futex_value_locked(u32 *dest, u32 __user *from)
298 return ret ? -EFAULT : 0; 298 return ret ? -EFAULT : 0;
299} 299}
300 300
301/*
302 * Fault handling.
303 */
304static int futex_handle_fault(unsigned long address, int attempt)
305{
306 struct vm_area_struct * vma;
307 struct mm_struct *mm = current->mm;
308 int ret = -EFAULT;
309
310 if (attempt > 2)
311 return ret;
312
313 down_read(&mm->mmap_sem);
314 vma = find_vma(mm, address);
315 if (vma && address >= vma->vm_start &&
316 (vma->vm_flags & VM_WRITE)) {
317 int fault;
318 fault = handle_mm_fault(mm, vma, address, 1);
319 if (unlikely((fault & VM_FAULT_ERROR))) {
320#if 0
321 /* XXX: let's do this when we verify it is OK */
322 if (ret & VM_FAULT_OOM)
323 ret = -ENOMEM;
324#endif
325 } else {
326 ret = 0;
327 if (fault & VM_FAULT_MAJOR)
328 current->maj_flt++;
329 else
330 current->min_flt++;
331 }
332 }
333 up_read(&mm->mmap_sem);
334 return ret;
335}
336 301
337/* 302/*
338 * PI code: 303 * PI code:
@@ -760,9 +725,9 @@ futex_wake_op(u32 __user *uaddr1, int fshared, u32 __user *uaddr2,
760 struct futex_hash_bucket *hb1, *hb2; 725 struct futex_hash_bucket *hb1, *hb2;
761 struct plist_head *head; 726 struct plist_head *head;
762 struct futex_q *this, *next; 727 struct futex_q *this, *next;
763 int ret, op_ret, attempt = 0; 728 int ret, op_ret;
764 729
765retryfull: 730retry:
766 ret = get_futex_key(uaddr1, fshared, &key1); 731 ret = get_futex_key(uaddr1, fshared, &key1);
767 if (unlikely(ret != 0)) 732 if (unlikely(ret != 0))
768 goto out; 733 goto out;
@@ -773,9 +738,8 @@ retryfull:
773 hb1 = hash_futex(&key1); 738 hb1 = hash_futex(&key1);
774 hb2 = hash_futex(&key2); 739 hb2 = hash_futex(&key2);
775 740
776retry:
777 double_lock_hb(hb1, hb2); 741 double_lock_hb(hb1, hb2);
778 742retry_private:
779 op_ret = futex_atomic_op_inuser(op, uaddr2); 743 op_ret = futex_atomic_op_inuser(op, uaddr2);
780 if (unlikely(op_ret < 0)) { 744 if (unlikely(op_ret < 0)) {
781 u32 dummy; 745 u32 dummy;
@@ -796,28 +760,16 @@ retry:
796 goto out_put_keys; 760 goto out_put_keys;
797 } 761 }
798 762
799 /*
800 * futex_atomic_op_inuser needs to both read and write
801 * *(int __user *)uaddr2, but we can't modify it
802 * non-atomically. Therefore, if get_user below is not
803 * enough, we need to handle the fault ourselves, while
804 * still holding the mmap_sem.
805 */
806 if (attempt++) {
807 ret = futex_handle_fault((unsigned long)uaddr2,
808 attempt);
809 if (ret)
810 goto out_put_keys;
811 goto retry;
812 }
813
814 ret = get_user(dummy, uaddr2); 763 ret = get_user(dummy, uaddr2);
815 if (ret) 764 if (ret)
816 goto out_put_keys; 765 goto out_put_keys;
817 766
767 if (!fshared)
768 goto retry_private;
769
818 put_futex_key(fshared, &key2); 770 put_futex_key(fshared, &key2);
819 put_futex_key(fshared, &key1); 771 put_futex_key(fshared, &key1);
820 goto retryfull; 772 goto retry;
821 } 773 }
822 774
823 head = &hb1->chain; 775 head = &hb1->chain;
@@ -877,6 +829,7 @@ retry:
877 hb1 = hash_futex(&key1); 829 hb1 = hash_futex(&key1);
878 hb2 = hash_futex(&key2); 830 hb2 = hash_futex(&key2);
879 831
832retry_private:
880 double_lock_hb(hb1, hb2); 833 double_lock_hb(hb1, hb2);
881 834
882 if (likely(cmpval != NULL)) { 835 if (likely(cmpval != NULL)) {
@@ -887,15 +840,16 @@ retry:
887 if (unlikely(ret)) { 840 if (unlikely(ret)) {
888 double_unlock_hb(hb1, hb2); 841 double_unlock_hb(hb1, hb2);
889 842
890 put_futex_key(fshared, &key2);
891 put_futex_key(fshared, &key1);
892
893 ret = get_user(curval, uaddr1); 843 ret = get_user(curval, uaddr1);
844 if (ret)
845 goto out_put_keys;
894 846
895 if (!ret) 847 if (!fshared)
896 goto retry; 848 goto retry_private;
897 849
898 goto out_put_keys; 850 put_futex_key(fshared, &key2);
851 put_futex_key(fshared, &key1);
852 goto retry;
899 } 853 }
900 if (curval != *cmpval) { 854 if (curval != *cmpval) {
901 ret = -EAGAIN; 855 ret = -EAGAIN;
@@ -1070,7 +1024,7 @@ static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,
1070 struct futex_pi_state *pi_state = q->pi_state; 1024 struct futex_pi_state *pi_state = q->pi_state;
1071 struct task_struct *oldowner = pi_state->owner; 1025 struct task_struct *oldowner = pi_state->owner;
1072 u32 uval, curval, newval; 1026 u32 uval, curval, newval;
1073 int ret, attempt = 0; 1027 int ret;
1074 1028
1075 /* Owner died? */ 1029 /* Owner died? */
1076 if (!pi_state->owner) 1030 if (!pi_state->owner)
@@ -1141,7 +1095,7 @@ retry:
1141handle_fault: 1095handle_fault:
1142 spin_unlock(q->lock_ptr); 1096 spin_unlock(q->lock_ptr);
1143 1097
1144 ret = futex_handle_fault((unsigned long)uaddr, attempt++); 1098 ret = get_user(uval, uaddr);
1145 1099
1146 spin_lock(q->lock_ptr); 1100 spin_lock(q->lock_ptr);
1147 1101
@@ -1190,6 +1144,7 @@ retry:
1190 if (unlikely(ret != 0)) 1144 if (unlikely(ret != 0))
1191 goto out; 1145 goto out;
1192 1146
1147retry_private:
1193 hb = queue_lock(&q); 1148 hb = queue_lock(&q);
1194 1149
1195 /* 1150 /*
@@ -1216,13 +1171,16 @@ retry:
1216 1171
1217 if (unlikely(ret)) { 1172 if (unlikely(ret)) {
1218 queue_unlock(&q, hb); 1173 queue_unlock(&q, hb);
1219 put_futex_key(fshared, &q.key);
1220 1174
1221 ret = get_user(uval, uaddr); 1175 ret = get_user(uval, uaddr);
1176 if (ret)
1177 goto out_put_key;
1222 1178
1223 if (!ret) 1179 if (!fshared)
1224 goto retry; 1180 goto retry_private;
1225 goto out; 1181
1182 put_futex_key(fshared, &q.key);
1183 goto retry;
1226 } 1184 }
1227 ret = -EWOULDBLOCK; 1185 ret = -EWOULDBLOCK;
1228 if (unlikely(uval != val)) { 1186 if (unlikely(uval != val)) {
@@ -1356,7 +1314,7 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared,
1356 struct futex_hash_bucket *hb; 1314 struct futex_hash_bucket *hb;
1357 u32 uval, newval, curval; 1315 u32 uval, newval, curval;
1358 struct futex_q q; 1316 struct futex_q q;
1359 int ret, lock_taken, ownerdied = 0, attempt = 0; 1317 int ret, lock_taken, ownerdied = 0;
1360 1318
1361 if (refill_pi_state_cache()) 1319 if (refill_pi_state_cache())
1362 return -ENOMEM; 1320 return -ENOMEM;
@@ -1376,7 +1334,7 @@ retry:
1376 if (unlikely(ret != 0)) 1334 if (unlikely(ret != 0))
1377 goto out; 1335 goto out;
1378 1336
1379retry_unlocked: 1337retry_private:
1380 hb = queue_lock(&q); 1338 hb = queue_lock(&q);
1381 1339
1382retry_locked: 1340retry_locked:
@@ -1601,18 +1559,15 @@ uaddr_faulted:
1601 */ 1559 */
1602 queue_unlock(&q, hb); 1560 queue_unlock(&q, hb);
1603 1561
1604 if (attempt++) {
1605 ret = futex_handle_fault((unsigned long)uaddr, attempt);
1606 if (ret)
1607 goto out_put_key;
1608 goto retry_unlocked;
1609 }
1610
1611 ret = get_user(uval, uaddr); 1562 ret = get_user(uval, uaddr);
1612 if (!ret) 1563 if (ret)
1613 goto retry_unlocked; 1564 goto out_put_key;
1614 1565
1615 goto out_put_key; 1566 if (!fshared)
1567 goto retry_private;
1568
1569 put_futex_key(fshared, &q.key);
1570 goto retry;
1616} 1571}
1617 1572
1618 1573
@@ -1628,7 +1583,7 @@ static int futex_unlock_pi(u32 __user *uaddr, int fshared)
1628 u32 uval; 1583 u32 uval;
1629 struct plist_head *head; 1584 struct plist_head *head;
1630 union futex_key key = FUTEX_KEY_INIT; 1585 union futex_key key = FUTEX_KEY_INIT;
1631 int ret, attempt = 0; 1586 int ret;
1632 1587
1633retry: 1588retry:
1634 if (get_user(uval, uaddr)) 1589 if (get_user(uval, uaddr))
@@ -1644,7 +1599,6 @@ retry:
1644 goto out; 1599 goto out;
1645 1600
1646 hb = hash_futex(&key); 1601 hb = hash_futex(&key);
1647retry_unlocked:
1648 spin_lock(&hb->lock); 1602 spin_lock(&hb->lock);
1649 1603
1650 /* 1604 /*
@@ -1709,17 +1663,9 @@ pi_faulted:
1709 * we have to drop the mmap_sem in order to call get_user(). 1663 * we have to drop the mmap_sem in order to call get_user().
1710 */ 1664 */
1711 spin_unlock(&hb->lock); 1665 spin_unlock(&hb->lock);
1712 1666 put_futex_key(fshared, &key);
1713 if (attempt++) {
1714 ret = futex_handle_fault((unsigned long)uaddr, attempt);
1715 if (ret)
1716 goto out;
1717 uval = 0;
1718 goto retry_unlocked;
1719 }
1720 1667
1721 ret = get_user(uval, uaddr); 1668 ret = get_user(uval, uaddr);
1722 put_futex_key(fshared, &key);
1723 if (!ret) 1669 if (!ret)
1724 goto retry; 1670 goto retry;
1725 1671