aboutsummaryrefslogtreecommitdiffstats
path: root/ipc
diff options
context:
space:
mode:
authorManfred Spraul <manfred@colorfullife.com>2010-05-26 17:43:41 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2010-05-27 12:12:49 -0400
commit0a2b9d4c79671b05956806ede5d054e03ae56280 (patch)
tree28431a1dc1e21528c0075c7f4ac345bda40ce21b /ipc
parentfd5db42254518fbf241dc454e918598fbe494fa2 (diff)
ipc/sem.c: move wake_up_process out of the spinlock section
The wake-up part of semtimedop() consists out of two steps: - the right tasks must be identified. - they must be woken up. Right now, both steps run while the array spinlock is held. This patch reorders the code and moves the actual wake_up_process() behind the point where the spinlock is dropped. The code also moves setting sem->sem_otime to one place: It does not make sense to set the last modify time multiple times. [akpm@linux-foundation.org: repair kerneldoc] [akpm@linux-foundation.org: fix uninitialised retval] Signed-off-by: Manfred Spraul <manfred@colorfullife.com> Cc: Chris Mason <chris.mason@oracle.com> Cc: Zach Brown <zach.brown@oracle.com> Cc: Jens Axboe <jens.axboe@oracle.com> Cc: Nick Piggin <npiggin@suse.de> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'ipc')
-rw-r--r--ipc/sem.c123
1 files changed, 91 insertions, 32 deletions
diff --git a/ipc/sem.c b/ipc/sem.c
index 81a9c74ab64c..a744eb579f07 100644
--- a/ipc/sem.c
+++ b/ipc/sem.c
@@ -381,7 +381,6 @@ static int try_atomic_semop (struct sem_array * sma, struct sembuf * sops,
381 sop--; 381 sop--;
382 } 382 }
383 383
384 sma->sem_otime = get_seconds();
385 return 0; 384 return 0;
386 385
387out_of_range: 386out_of_range:
@@ -404,25 +403,51 @@ undo:
404 return result; 403 return result;
405} 404}
406 405
407/* 406/** wake_up_sem_queue_prepare(q, error): Prepare wake-up
408 * Wake up a process waiting on the sem queue with a given error. 407 * @q: queue entry that must be signaled
409 * The queue is invalid (may not be accessed) after the function returns. 408 * @error: Error value for the signal
409 *
410 * Prepare the wake-up of the queue entry q.
410 */ 411 */
411static void wake_up_sem_queue(struct sem_queue *q, int error) 412static void wake_up_sem_queue_prepare(struct list_head *pt,
413 struct sem_queue *q, int error)
412{ 414{
413 /* 415 if (list_empty(pt)) {
414 * Hold preempt off so that we don't get preempted and have the 416 /*
415 * wakee busy-wait until we're scheduled back on. We're holding 417 * Hold preempt off so that we don't get preempted and have the
416 * locks here so it may not strictly be needed, however if the 418 * wakee busy-wait until we're scheduled back on.
417 * locks become preemptible then this prevents such a problem. 419 */
418 */ 420 preempt_disable();
419 preempt_disable(); 421 }
420 q->status = IN_WAKEUP; 422 q->status = IN_WAKEUP;
421 wake_up_process(q->sleeper); 423 q->pid = error;
422 /* hands-off: q can disappear immediately after writing q->status. */ 424
423 smp_wmb(); 425 list_add_tail(&q->simple_list, pt);
424 q->status = error; 426}
425 preempt_enable(); 427
428/**
429 * wake_up_sem_queue_do(pt) - do the actual wake-up
430 * @pt: list of tasks to be woken up
431 *
432 * Do the actual wake-up.
433 * The function is called without any locks held, thus the semaphore array
434 * could be destroyed already and the tasks can disappear as soon as the
435 * status is set to the actual return code.
436 */
437static void wake_up_sem_queue_do(struct list_head *pt)
438{
439 struct sem_queue *q, *t;
440 int did_something;
441
442 did_something = !list_empty(pt);
443 list_for_each_entry_safe(q, t, pt, simple_list) {
444 wake_up_process(q->sleeper);
445 /* q can disappear immediately after writing q->status. */
446 smp_wmb();
447 q->status = q->pid;
448 }
449 if (did_something)
450 preempt_enable();
426} 451}
427 452
428static void unlink_queue(struct sem_array *sma, struct sem_queue *q) 453static void unlink_queue(struct sem_array *sma, struct sem_queue *q)
@@ -502,17 +527,22 @@ static int check_restart(struct sem_array *sma, struct sem_queue *q)
502 * update_queue(sma, semnum): Look for tasks that can be completed. 527 * update_queue(sma, semnum): Look for tasks that can be completed.
503 * @sma: semaphore array. 528 * @sma: semaphore array.
504 * @semnum: semaphore that was modified. 529 * @semnum: semaphore that was modified.
530 * @pt: list head for the tasks that must be woken up.
505 * 531 *
506 * update_queue must be called after a semaphore in a semaphore array 532 * update_queue must be called after a semaphore in a semaphore array
507 * was modified. If multiple semaphore were modified, then @semnum 533 * was modified. If multiple semaphore were modified, then @semnum
508 * must be set to -1. 534 * must be set to -1.
535 * The tasks that must be woken up are added to @pt. The return code
536 * is stored in q->pid.
537 * The function return 1 if at least one semop was completed successfully.
509 */ 538 */
510static void update_queue(struct sem_array *sma, int semnum) 539static int update_queue(struct sem_array *sma, int semnum, struct list_head *pt)
511{ 540{
512 struct sem_queue *q; 541 struct sem_queue *q;
513 struct list_head *walk; 542 struct list_head *walk;
514 struct list_head *pending_list; 543 struct list_head *pending_list;
515 int offset; 544 int offset;
545 int semop_completed = 0;
516 546
517 /* if there are complex operations around, then knowing the semaphore 547 /* if there are complex operations around, then knowing the semaphore
518 * that was modified doesn't help us. Assume that multiple semaphores 548 * that was modified doesn't help us. Assume that multiple semaphores
@@ -557,40 +587,55 @@ again:
557 587
558 unlink_queue(sma, q); 588 unlink_queue(sma, q);
559 589
560 if (error) 590 if (error) {
561 restart = 0; 591 restart = 0;
562 else 592 } else {
593 semop_completed = 1;
563 restart = check_restart(sma, q); 594 restart = check_restart(sma, q);
595 }
564 596
565 wake_up_sem_queue(q, error); 597 wake_up_sem_queue_prepare(pt, q, error);
566 if (restart) 598 if (restart)
567 goto again; 599 goto again;
568 } 600 }
601 return semop_completed;
569} 602}
570 603
571/** do_smart_update(sma, sops, nsops): Optimized update_queue 604/**
605 * do_smart_update(sma, sops, nsops, otime, pt) - optimized update_queue
572 * @sma: semaphore array 606 * @sma: semaphore array
573 * @sops: operations that were performed 607 * @sops: operations that were performed
574 * @nsops: number of operations 608 * @nsops: number of operations
609 * @otime: force setting otime
610 * @pt: list head of the tasks that must be woken up.
575 * 611 *
576 * do_smart_update() does the required called to update_queue, based on the 612 * do_smart_update() does the required called to update_queue, based on the
577 * actual changes that were performed on the semaphore array. 613 * actual changes that were performed on the semaphore array.
614 * Note that the function does not do the actual wake-up: the caller is
615 * responsible for calling wake_up_sem_queue_do(@pt).
616 * It is safe to perform this call after dropping all locks.
578 */ 617 */
579static void do_smart_update(struct sem_array *sma, struct sembuf *sops, int nsops) 618static void do_smart_update(struct sem_array *sma, struct sembuf *sops, int nsops,
619 int otime, struct list_head *pt)
580{ 620{
581 int i; 621 int i;
582 622
583 if (sma->complex_count || sops == NULL) { 623 if (sma->complex_count || sops == NULL) {
584 update_queue(sma, -1); 624 if (update_queue(sma, -1, pt))
585 return; 625 otime = 1;
626 goto done;
586 } 627 }
587 628
588 for (i = 0; i < nsops; i++) { 629 for (i = 0; i < nsops; i++) {
589 if (sops[i].sem_op > 0 || 630 if (sops[i].sem_op > 0 ||
590 (sops[i].sem_op < 0 && 631 (sops[i].sem_op < 0 &&
591 sma->sem_base[sops[i].sem_num].semval == 0)) 632 sma->sem_base[sops[i].sem_num].semval == 0))
592 update_queue(sma, sops[i].sem_num); 633 if (update_queue(sma, sops[i].sem_num, pt))
634 otime = 1;
593 } 635 }
636done:
637 if (otime)
638 sma->sem_otime = get_seconds();
594} 639}
595 640
596 641
@@ -656,6 +701,7 @@ static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
656 struct sem_undo *un, *tu; 701 struct sem_undo *un, *tu;
657 struct sem_queue *q, *tq; 702 struct sem_queue *q, *tq;
658 struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm); 703 struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm);
704 struct list_head tasks;
659 705
660 /* Free the existing undo structures for this semaphore set. */ 706 /* Free the existing undo structures for this semaphore set. */
661 assert_spin_locked(&sma->sem_perm.lock); 707 assert_spin_locked(&sma->sem_perm.lock);
@@ -669,15 +715,17 @@ static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
669 } 715 }
670 716
671 /* Wake up all pending processes and let them fail with EIDRM. */ 717 /* Wake up all pending processes and let them fail with EIDRM. */
718 INIT_LIST_HEAD(&tasks);
672 list_for_each_entry_safe(q, tq, &sma->sem_pending, list) { 719 list_for_each_entry_safe(q, tq, &sma->sem_pending, list) {
673 unlink_queue(sma, q); 720 unlink_queue(sma, q);
674 wake_up_sem_queue(q, -EIDRM); 721 wake_up_sem_queue_prepare(&tasks, q, -EIDRM);
675 } 722 }
676 723
677 /* Remove the semaphore set from the IDR */ 724 /* Remove the semaphore set from the IDR */
678 sem_rmid(ns, sma); 725 sem_rmid(ns, sma);
679 sem_unlock(sma); 726 sem_unlock(sma);
680 727
728 wake_up_sem_queue_do(&tasks);
681 ns->used_sems -= sma->sem_nsems; 729 ns->used_sems -= sma->sem_nsems;
682 security_sem_free(sma); 730 security_sem_free(sma);
683 ipc_rcu_putref(sma); 731 ipc_rcu_putref(sma);
@@ -799,11 +847,13 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
799 ushort fast_sem_io[SEMMSL_FAST]; 847 ushort fast_sem_io[SEMMSL_FAST];
800 ushort* sem_io = fast_sem_io; 848 ushort* sem_io = fast_sem_io;
801 int nsems; 849 int nsems;
850 struct list_head tasks;
802 851
803 sma = sem_lock_check(ns, semid); 852 sma = sem_lock_check(ns, semid);
804 if (IS_ERR(sma)) 853 if (IS_ERR(sma))
805 return PTR_ERR(sma); 854 return PTR_ERR(sma);
806 855
856 INIT_LIST_HEAD(&tasks);
807 nsems = sma->sem_nsems; 857 nsems = sma->sem_nsems;
808 858
809 err = -EACCES; 859 err = -EACCES;
@@ -891,7 +941,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
891 } 941 }
892 sma->sem_ctime = get_seconds(); 942 sma->sem_ctime = get_seconds();
893 /* maybe some queued-up processes were waiting for this */ 943 /* maybe some queued-up processes were waiting for this */
894 update_queue(sma, -1); 944 do_smart_update(sma, NULL, 0, 0, &tasks);
895 err = 0; 945 err = 0;
896 goto out_unlock; 946 goto out_unlock;
897 } 947 }
@@ -933,13 +983,15 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
933 curr->sempid = task_tgid_vnr(current); 983 curr->sempid = task_tgid_vnr(current);
934 sma->sem_ctime = get_seconds(); 984 sma->sem_ctime = get_seconds();
935 /* maybe some queued-up processes were waiting for this */ 985 /* maybe some queued-up processes were waiting for this */
936 update_queue(sma, semnum); 986 do_smart_update(sma, NULL, 0, 0, &tasks);
937 err = 0; 987 err = 0;
938 goto out_unlock; 988 goto out_unlock;
939 } 989 }
940 } 990 }
941out_unlock: 991out_unlock:
942 sem_unlock(sma); 992 sem_unlock(sma);
993 wake_up_sem_queue_do(&tasks);
994
943out_free: 995out_free:
944 if(sem_io != fast_sem_io) 996 if(sem_io != fast_sem_io)
945 ipc_free(sem_io, sizeof(ushort)*nsems); 997 ipc_free(sem_io, sizeof(ushort)*nsems);
@@ -1213,6 +1265,7 @@ SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
1213 struct sem_queue queue; 1265 struct sem_queue queue;
1214 unsigned long jiffies_left = 0; 1266 unsigned long jiffies_left = 0;
1215 struct ipc_namespace *ns; 1267 struct ipc_namespace *ns;
1268 struct list_head tasks;
1216 1269
1217 ns = current->nsproxy->ipc_ns; 1270 ns = current->nsproxy->ipc_ns;
1218 1271
@@ -1261,6 +1314,8 @@ SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
1261 } else 1314 } else
1262 un = NULL; 1315 un = NULL;
1263 1316
1317 INIT_LIST_HEAD(&tasks);
1318
1264 sma = sem_lock_check(ns, semid); 1319 sma = sem_lock_check(ns, semid);
1265 if (IS_ERR(sma)) { 1320 if (IS_ERR(sma)) {
1266 if (un) 1321 if (un)
@@ -1309,7 +1364,7 @@ SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
1309 error = try_atomic_semop (sma, sops, nsops, un, task_tgid_vnr(current)); 1364 error = try_atomic_semop (sma, sops, nsops, un, task_tgid_vnr(current));
1310 if (error <= 0) { 1365 if (error <= 0) {
1311 if (alter && error == 0) 1366 if (alter && error == 0)
1312 do_smart_update(sma, sops, nsops); 1367 do_smart_update(sma, sops, nsops, 1, &tasks);
1313 1368
1314 goto out_unlock_free; 1369 goto out_unlock_free;
1315 } 1370 }
@@ -1386,6 +1441,8 @@ SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
1386 1441
1387out_unlock_free: 1442out_unlock_free:
1388 sem_unlock(sma); 1443 sem_unlock(sma);
1444
1445 wake_up_sem_queue_do(&tasks);
1389out_free: 1446out_free:
1390 if(sops != fast_sops) 1447 if(sops != fast_sops)
1391 kfree(sops); 1448 kfree(sops);
@@ -1446,6 +1503,7 @@ void exit_sem(struct task_struct *tsk)
1446 for (;;) { 1503 for (;;) {
1447 struct sem_array *sma; 1504 struct sem_array *sma;
1448 struct sem_undo *un; 1505 struct sem_undo *un;
1506 struct list_head tasks;
1449 int semid; 1507 int semid;
1450 int i; 1508 int i;
1451 1509
@@ -1509,10 +1567,11 @@ void exit_sem(struct task_struct *tsk)
1509 semaphore->sempid = task_tgid_vnr(current); 1567 semaphore->sempid = task_tgid_vnr(current);
1510 } 1568 }
1511 } 1569 }
1512 sma->sem_otime = get_seconds();
1513 /* maybe some queued-up processes were waiting for this */ 1570 /* maybe some queued-up processes were waiting for this */
1514 update_queue(sma, -1); 1571 INIT_LIST_HEAD(&tasks);
1572 do_smart_update(sma, NULL, 0, 1, &tasks);
1515 sem_unlock(sma); 1573 sem_unlock(sma);
1574 wake_up_sem_queue_do(&tasks);
1516 1575
1517 call_rcu(&un->rcu, free_un); 1576 call_rcu(&un->rcu, free_un);
1518 } 1577 }