1 files changed, 180 insertions, 1 deletions

diff --git a/kernel/fork.c b/kernel/fork.c
index f88bd984df35..340cbc85f694 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -38,6 +38,7 @@
 #include <linux/syscalls.h>
 #include <linux/jiffies.h>
 #include <linux/tracehook.h>
+#include <linux/interrupt.h>
 #include <linux/futex.h>
 #include <linux/compat.h>
 #include <linux/task_io_accounting_ops.h>
@@ -65,6 +66,8 @@
 #include <linux/perf_event.h>
 #include <linux/posix-timers.h>
 #include <linux/user-return-notifier.h>
+#include <linux/kthread.h>
+#include <linux/notifier.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -85,7 +88,19 @@ int max_threads;		/* tunable limit on nr_threads */
 
 DEFINE_PER_CPU(unsigned long, process_counts) = 0;
 
+#ifdef CONFIG_PREEMPT_RT
+DEFINE_RWLOCK(tasklist_lock);  /* outer */
+#else
 __cacheline_aligned DEFINE_RWLOCK(tasklist_lock);  /* outer */
+#endif
+
+/*
+ * Delayed mmdrop. In the PREEMPT_RT case we
+ * dont want to do this from the scheduling
+ * context.
+ */
+static DEFINE_PER_CPU(struct task_struct *, desched_task);
+static DEFINE_PER_CPU(struct list_head, delayed_drop_list);
 
 int nr_processes(void)
 {
@@ -170,6 +185,16 @@ void __put_task_struct(struct task_struct *tsk)
                 free_task(tsk);
 }
 
+#ifdef CONFIG_PREEMPT_RT
+void __put_task_struct_cb(struct rcu_head *rhp)
+{
+        struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
+
+        __put_task_struct(tsk);
+
+}
+#endif
+
 /*
  * macro override instead of weak attribute alias, to workaround
  * gcc 4.1.0 and 4.1.1 bugs with weak attribute and empty functions.
@@ -180,6 +205,8 @@ void __put_task_struct(struct task_struct *tsk)
 
 void __init fork_init(unsigned long mempages)
 {
+        int i;
+
 #ifndef __HAVE_ARCH_TASK_STRUCT_ALLOCATOR
 #ifndef ARCH_MIN_TASKALIGN
 #define ARCH_MIN_TASKALIGN      L1_CACHE_BYTES
@@ -210,6 +237,9 @@ void __init fork_init(unsigned long mempages)
         init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
         init_task.signal->rlim[RLIMIT_SIGPENDING] =
                 init_task.signal->rlim[RLIMIT_NPROC];
+
+        for (i = 0; i < NR_CPUS; i++)
+                INIT_LIST_HEAD(&per_cpu(delayed_drop_list, i));
 }
 
 int __attribute__((weak)) arch_dup_task_struct(struct task_struct *dst,
@@ -295,6 +325,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
         mm->locked_vm = 0;
         mm->mmap = NULL;
         mm->mmap_cache = NULL;
+        INIT_LIST_HEAD(&mm->delayed_drop);
         mm->free_area_cache = oldmm->mmap_base;
         mm->cached_hole_size = ~0UL;
         mm->map_count = 0;
@@ -943,6 +974,9 @@ static void rt_mutex_init_task(struct task_struct *p)
 #ifdef CONFIG_RT_MUTEXES
         plist_head_init_raw(&p->pi_waiters, &p->pi_lock);
         p->pi_blocked_on = NULL;
+# ifdef CONFIG_DEBUG_RT_MUTEXES
+        p->last_kernel_lock = NULL;
+# endif
 #endif
 }
 
@@ -1065,6 +1099,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
         spin_lock_init(&p->alloc_lock);
 
         init_sigpending(&p->pending);
+        p->sigqueue_cache = NULL;
 
         p->utime = cputime_zero;
         p->stime = cputime_zero;
@@ -1082,7 +1117,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
         acct_clear_integrals(p);
 
         posix_cpu_timers_init(p);
-
+        p->posix_timer_list = NULL;
         p->lock_depth = -1;             /* -1 = no lock */
         do_posix_clock_monotonic_gettime(&p->start_time);
         p->real_start_time = p->start_time;
@@ -1118,6 +1153,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
         p->hardirq_context = 0;
         p->softirq_context = 0;
 #endif
+        p->pagefault_disabled = 0;
 #ifdef CONFIG_LOCKDEP
         p->lockdep_depth = 0; /* no locks held yet */
         p->curr_chain_key = 0;
@@ -1165,6 +1201,9 @@ static struct task_struct *copy_process(unsigned long clone_flags,
         retval = copy_thread(clone_flags, stack_start, stack_size, p, regs);
         if (retval)
                 goto bad_fork_cleanup_io;
+#ifdef CONFIG_DEBUG_PREEMPT
+        atomic_set(&p->lock_count, 0);
+#endif
 
         if (pid != &init_struct_pid) {
                 retval = -ENOMEM;
@@ -1289,7 +1328,9 @@ static struct task_struct *copy_process(unsigned long clone_flags,
                         attach_pid(p, PIDTYPE_SID, task_session(current));
                         list_add_tail(&p->sibling, &p->real_parent->children);
                         list_add_tail_rcu(&p->tasks, &init_task.tasks);
+                        preempt_disable();
                         __get_cpu_var(process_counts)++;
+                        preempt_enable();
                 }
                 attach_pid(p, PIDTYPE_PID, pid);
                 nr_threads++;
@@ -1763,3 +1804,141 @@ int unshare_files(struct files_struct **displaced)
         task_unlock(task);
         return 0;
 }
+
+static int mmdrop_complete(void)
+{
+        struct list_head *head;
+        int ret = 0;
+
+        head = &get_cpu_var(delayed_drop_list);
+        while (!list_empty(head)) {
+                struct mm_struct *mm = list_entry(head->next,
+                                        struct mm_struct, delayed_drop);
+                list_del(&mm->delayed_drop);
+                put_cpu_var(delayed_drop_list);
+
+                __mmdrop(mm);
+                ret = 1;
+
+                head = &get_cpu_var(delayed_drop_list);
+        }
+        put_cpu_var(delayed_drop_list);
+
+        return ret;
+}
+
+/*
+ * We dont want to do complex work from the scheduler, thus
+ * we delay the work to a per-CPU worker thread:
+ */
+void  __mmdrop_delayed(struct mm_struct *mm)
+{
+        struct task_struct *desched_task;
+        struct list_head *head;
+
+        head = &get_cpu_var(delayed_drop_list);
+        list_add_tail(&mm->delayed_drop, head);
+        desched_task = __get_cpu_var(desched_task);
+        if (desched_task)
+                wake_up_process(desched_task);
+        put_cpu_var(delayed_drop_list);
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+static void takeover_delayed_drop(int hotcpu)
+{
+        struct list_head *head = &per_cpu(delayed_drop_list, hotcpu);
+
+        while (!list_empty(head)) {
+                struct mm_struct *mm = list_entry(head->next,
+                                struct mm_struct, delayed_drop);
+
+                list_del(&mm->delayed_drop);
+                __mmdrop_delayed(mm);
+        }
+}
+#endif
+
+static int desched_thread(void * __bind_cpu)
+{
+        set_user_nice(current, -10);
+        current->flags |= PF_NOFREEZE;
+        current->extra_flags |= PFE_SOFTIRQ;
+
+        set_current_state(TASK_INTERRUPTIBLE);
+
+        while (!kthread_should_stop()) {
+
+                if (mmdrop_complete())
+                        continue;
+                schedule();
+
+                /*
+                 * This must be called from time to time on ia64, and is a
+                 * no-op on other archs. Used to be in cpu_idle(), but with
+                 * the new -rt semantics it can't stay there.
+                 */
+                check_pgt_cache();
+
+                set_current_state(TASK_INTERRUPTIBLE);
+        }
+        __set_current_state(TASK_RUNNING);
+        return 0;
+}
+
+static int __devinit cpu_callback(struct notifier_block *nfb,
+                                  unsigned long action,
+                                  void *hcpu)
+{
+        int hotcpu = (unsigned long)hcpu;
+        struct task_struct *p;
+
+        switch (action) {
+        case CPU_UP_PREPARE:
+
+                BUG_ON(per_cpu(desched_task, hotcpu));
+                INIT_LIST_HEAD(&per_cpu(delayed_drop_list, hotcpu));
+                p = kthread_create(desched_thread, hcpu, "desched/%d", hotcpu);
+                if (IS_ERR(p)) {
+                        printk("desched_thread for %i failed\n", hotcpu);
+                        return NOTIFY_BAD;
+                }
+                per_cpu(desched_task, hotcpu) = p;
+                kthread_bind(p, hotcpu);
+                break;
+        case CPU_ONLINE:
+
+                wake_up_process(per_cpu(desched_task, hotcpu));
+                break;
+#ifdef CONFIG_HOTPLUG_CPU
+        case CPU_UP_CANCELED:
+
+                /* Unbind so it can run.  Fall thru. */
+                kthread_bind(per_cpu(desched_task, hotcpu), smp_processor_id());
+        case CPU_DEAD:
+
+                p = per_cpu(desched_task, hotcpu);
+                per_cpu(desched_task, hotcpu) = NULL;
+                kthread_stop(p);
+                takeover_delayed_drop(hotcpu);
+                takeover_tasklets(hotcpu);
+                break;
+#endif /* CONFIG_HOTPLUG_CPU */
+        }
+        return NOTIFY_OK;
+}
+
+static struct notifier_block __devinitdata cpu_nfb = {
+        .notifier_call = cpu_callback
+};
+
+__init int spawn_desched_task(void)
+{
+        void *cpu = (void *)(long)smp_processor_id();
+
+        cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
+        cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
+        register_cpu_notifier(&cpu_nfb);
+        return 0;
+}
+