[PATCH] RCU signal handling

RCU tasklist_lock and RCU signal handling: send signals RCU-read-locked
instead of tasklist_lock read-locked.  This is a scalability improvement on
SMP and a preemption-latency improvement under PREEMPT_RCU.

Signed-off-by: Paul E. McKenney <paulmck@us.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Acked-by: William Irwin <wli@holomorphy.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

6 files changed, 111 insertions, 27 deletions

diff --git a/kernel/exit.c b/kernel/exit.c
index ee515683b9..c73a7eb26d 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -72,7 +72,6 @@ repeat:
                 __ptrace_unlink(p);
         BUG_ON(!list_empty(&p->ptrace_list) || !list_empty(&p->ptrace_children));
         __exit_signal(p);
-        __exit_sighand(p);
         /*
          * Note that the fastpath in sys_times depends on __exit_signal having
          * updated the counters before a task is removed from the tasklist of
diff --git a/kernel/fork.c b/kernel/fork.c
index fb8572a422..7fe3adfa65 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -743,6 +743,14 @@ int unshare_files(void)
 
 EXPORT_SYMBOL(unshare_files);
 
+void sighand_free_cb(struct rcu_head *rhp)
+{
+        struct sighand_struct *sp;
+
+        sp = container_of(rhp, struct sighand_struct, rcu);
+        kmem_cache_free(sighand_cachep, sp);
+}
+
 static inline int copy_sighand(unsigned long clone_flags, struct task_struct * tsk)
 {
         struct sighand_struct *sig;
@@ -752,7 +760,7 @@ static inline int copy_sighand(unsigned long clone_flags, struct task_struct * t
                 return 0;
         }
         sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
-        tsk->sighand = sig;
+        rcu_assign_pointer(tsk->sighand, sig);
         if (!sig)
                 return -ENOMEM;
         spin_lock_init(&sig->siglock);
diff --git a/kernel/pid.c b/kernel/pid.c
index edba31c681..1acc072469 100644
--- a/kernel/pid.c
+++ b/kernel/pid.c
@@ -136,7 +136,7 @@ struct pid * fastcall find_pid(enum pid_type type, int nr)
         struct hlist_node *elem;
         struct pid *pid;
 
-        hlist_for_each_entry(pid, elem,
+        hlist_for_each_entry_rcu(pid, elem,
                         &pid_hash[type][pid_hashfn(nr)], pid_chain) {
                 if (pid->nr == nr)
                         return pid;
@@ -150,15 +150,15 @@ int fastcall attach_pid(task_t *task, enum pid_type type, int nr)
 
         task_pid = &task->pids[type];
         pid = find_pid(type, nr);
+        task_pid->nr = nr;
         if (pid == NULL) {
-                hlist_add_head(&task_pid->pid_chain,
-                                &pid_hash[type][pid_hashfn(nr)]);
                 INIT_LIST_HEAD(&task_pid->pid_list);
+                hlist_add_head_rcu(&task_pid->pid_chain,
+                                   &pid_hash[type][pid_hashfn(nr)]);
         } else {
                 INIT_HLIST_NODE(&task_pid->pid_chain);
-                list_add_tail(&task_pid->pid_list, &pid->pid_list);
+                list_add_tail_rcu(&task_pid->pid_list, &pid->pid_list);
         }
-        task_pid->nr = nr;
 
         return 0;
 }
@@ -170,20 +170,20 @@ static fastcall int __detach_pid(task_t *task, enum pid_type type)
 
         pid = &task->pids[type];
         if (!hlist_unhashed(&pid->pid_chain)) {
-                hlist_del(&pid->pid_chain);
 
-                if (list_empty(&pid->pid_list))
+                if (list_empty(&pid->pid_list)) {
                         nr = pid->nr;
-                else {
+                        hlist_del_rcu(&pid->pid_chain);
+                } else {
                         pid_next = list_entry(pid->pid_list.next,
                                                 struct pid, pid_list);
                         /* insert next pid from pid_list to hash */
-                        hlist_add_head(&pid_next->pid_chain,
-                                &pid_hash[type][pid_hashfn(pid_next->nr)]);
+                        hlist_replace_rcu(&pid->pid_chain,
+                                          &pid_next->pid_chain);
                 }
         }
 
-        list_del(&pid->pid_list);
+        list_del_rcu(&pid->pid_list);
         pid->nr = 0;
 
         return nr;
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index c9afc61240..0a669bd2f6 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -35,6 +35,7 @@
 #include <linux/init.h>
 #include <linux/spinlock.h>
 #include <linux/smp.h>
+#include <linux/rcupdate.h>
 #include <linux/interrupt.h>
 #include <linux/sched.h>
 #include <asm/atomic.h>
diff --git a/kernel/sched.c b/kernel/sched.c
index 6f46c94cc2..9273309115 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -176,6 +176,13 @@ static unsigned int task_timeslice(task_t *p)
 #define task_hot(p, now, sd) ((long long) ((now) - (p)->last_ran)       \
                                 < (long long) (sd)->cache_hot_time)
 
+void __put_task_struct_cb(struct rcu_head *rhp)
+{
+        __put_task_struct(container_of(rhp, struct task_struct, rcu));
+}
+
+EXPORT_SYMBOL_GPL(__put_task_struct_cb);
+
 /*
  * These are the runqueue data structures:
  */
diff --git a/kernel/signal.c b/kernel/signal.c
index d7611f189e..64737c72da 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -329,13 +329,20 @@ void __exit_sighand(struct task_struct *tsk)
         /* Ok, we're done with the signal handlers */
         tsk->sighand = NULL;
         if (atomic_dec_and_test(&sighand->count))
-                kmem_cache_free(sighand_cachep, sighand);
+                sighand_free(sighand);
 }
 
 void exit_sighand(struct task_struct *tsk)
 {
         write_lock_irq(&tasklist_lock);
-        __exit_sighand(tsk);
+        rcu_read_lock();
+        if (tsk->sighand != NULL) {
+                struct sighand_struct *sighand = rcu_dereference(tsk->sighand);
+                spin_lock(&sighand->siglock);
+                __exit_sighand(tsk);
+                spin_unlock(&sighand->siglock);
+        }
+        rcu_read_unlock();
         write_unlock_irq(&tasklist_lock);
 }
 
@@ -345,12 +352,14 @@ void exit_sighand(struct task_struct *tsk)
 void __exit_signal(struct task_struct *tsk)
 {
         struct signal_struct * sig = tsk->signal;
-        struct sighand_struct * sighand = tsk->sighand;
+        struct sighand_struct * sighand;
 
         if (!sig)
                 BUG();
         if (!atomic_read(&sig->count))
                 BUG();
+        rcu_read_lock();
+        sighand = rcu_dereference(tsk->sighand);
         spin_lock(&sighand->siglock);
         posix_cpu_timers_exit(tsk);
         if (atomic_dec_and_test(&sig->count)) {
@@ -358,6 +367,7 @@ void __exit_signal(struct task_struct *tsk)
                 if (tsk == sig->curr_target)
                         sig->curr_target = next_thread(tsk);
                 tsk->signal = NULL;
+                __exit_sighand(tsk);
                 spin_unlock(&sighand->siglock);
                 flush_sigqueue(&sig->shared_pending);
         } else {
@@ -389,9 +399,11 @@ void __exit_signal(struct task_struct *tsk)
                 sig->nvcsw += tsk->nvcsw;
                 sig->nivcsw += tsk->nivcsw;
                 sig->sched_time += tsk->sched_time;
+                __exit_sighand(tsk);
                 spin_unlock(&sighand->siglock);