69 files changed, 11387 insertions, 1037 deletions

diff --git a/kernel/Makefile b/kernel/Makefile
index 1c9938addb9d..0a7ed838984b 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -3,7 +3,7 @@
 #
 
 obj-y     = sched.o fork.o exec_domain.o panic.o printk.o profile.o \
-            exit.o itimer.o time.o softirq.o resource.o \
+            cpu.o exit.o itimer.o time.o softirq.o resource.o \
             sysctl.o capability.o ptrace.o timer.o user.o \
             signal.o sys.o kmod.o workqueue.o pid.o \
             rcupdate.o extable.o params.o posix-timers.o \
@@ -11,6 +11,18 @@ obj-y     = sched.o fork.o exec_domain.o panic.o printk.o profile.o \
             hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
             notifier.o ksysfs.o pm_qos_params.o sched_clock.o
 
+CFLAGS_REMOVE_sched.o = -mno-spe
+
+ifdef CONFIG_FTRACE
+# Do not trace debug files and internal ftrace files
+CFLAGS_REMOVE_lockdep.o = -pg
+CFLAGS_REMOVE_lockdep_proc.o = -pg
+CFLAGS_REMOVE_mutex-debug.o = -pg
+CFLAGS_REMOVE_rtmutex-debug.o = -pg
+CFLAGS_REMOVE_cgroup-debug.o = -pg
+CFLAGS_REMOVE_sched_clock.o = -pg
+endif
+
 obj-$(CONFIG_SYSCTL_SYSCALL_CHECK) += sysctl_check.o
 obj-$(CONFIG_STACKTRACE) += stacktrace.o
 obj-y += time/
@@ -27,7 +39,8 @@ obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
 obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
 obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
 obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o
-obj-$(CONFIG_SMP) += cpu.o spinlock.o
+obj-$(CONFIG_USE_GENERIC_SMP_HELPERS) += smp.o
+obj-$(CONFIG_SMP) += spinlock.o
 obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o
 obj-$(CONFIG_PROVE_LOCKING) += spinlock.o
 obj-$(CONFIG_UID16) += uid16.o
@@ -69,6 +82,9 @@ obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
 obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
 obj-$(CONFIG_MARKERS) += marker.o
 obj-$(CONFIG_LATENCYTOP) += latencytop.o
+obj-$(CONFIG_FTRACE) += trace/
+obj-$(CONFIG_TRACING) += trace/
+obj-$(CONFIG_SMP) += sched_cpupri.o
 
 ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y)
 # According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is
diff --git a/kernel/audit.c b/kernel/audit.c
index e8692a5748c2..e092f1c0ce30 100644
--- a/kernel/audit.c
+++ b/kernel/audit.c
@@ -738,7 +738,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh)
                 if (!audit_enabled && msg_type != AUDIT_USER_AVC)
                         return 0;
 
-                err = audit_filter_user(&NETLINK_CB(skb), msg_type);
+                err = audit_filter_user(&NETLINK_CB(skb));
                 if (err == 1) {
                         err = 0;
                         if (msg_type == AUDIT_USER_TTY) {
@@ -779,7 +779,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh)
                 }
                 /* fallthrough */
         case AUDIT_LIST:
-                err = audit_receive_filter(nlh->nlmsg_type, NETLINK_CB(skb).pid,
+                err = audit_receive_filter(msg_type, NETLINK_CB(skb).pid,
                                            uid, seq, data, nlmsg_len(nlh),
                                            loginuid, sessionid, sid);
                 break;
@@ -798,7 +798,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh)
                 }
                 /* fallthrough */
         case AUDIT_LIST_RULES:
-                err = audit_receive_filter(nlh->nlmsg_type, NETLINK_CB(skb).pid,
+                err = audit_receive_filter(msg_type, NETLINK_CB(skb).pid,
                                            uid, seq, data, nlmsg_len(nlh),
                                            loginuid, sessionid, sid);
                 break;
diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c
index 0e0bd27e6512..98c50cc671bb 100644
--- a/kernel/auditfilter.c
+++ b/kernel/auditfilter.c
@@ -1544,6 +1544,7 @@ static void audit_log_rule_change(uid_t loginuid, u32 sessionid, u32 sid,
  * @data: payload data
  * @datasz: size of payload data
  * @loginuid: loginuid of sender
+ * @sessionid: sessionid for netlink audit message
  * @sid: SE Linux Security ID of sender
  */
 int audit_receive_filter(int type, int pid, int uid, int seq, void *data,
@@ -1720,7 +1721,7 @@ static int audit_filter_user_rules(struct netlink_skb_parms *cb,
         return 1;
 }
 
-int audit_filter_user(struct netlink_skb_parms *cb, int type)
+int audit_filter_user(struct netlink_skb_parms *cb)
 {
         enum audit_state state = AUDIT_DISABLED;
         struct audit_entry *e;
diff --git a/kernel/backtracetest.c b/kernel/backtracetest.c
index d1a7605c5b8f..a5e026bc45c4 100644
--- a/kernel/backtracetest.c
+++ b/kernel/backtracetest.c
@@ -10,30 +10,73 @@
  * of the License.
  */
 
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/sched.h>
-#include <linux/delay.h>
+#include <linux/stacktrace.h>
+
+static void backtrace_test_normal(void)
+{
+        printk("Testing a backtrace from process context.\n");
+        printk("The following trace is a kernel self test and not a bug!\n");
 
-static struct timer_list backtrace_timer;
+        dump_stack();
+}
 
-static void backtrace_test_timer(unsigned long data)
+static DECLARE_COMPLETION(backtrace_work);
+
+static void backtrace_test_irq_callback(unsigned long data)
+{
+        dump_stack();
+        complete(&backtrace_work);
+}
+
+static DECLARE_TASKLET(backtrace_tasklet, &backtrace_test_irq_callback, 0);
+
+static void backtrace_test_irq(void)
 {
         printk("Testing a backtrace from irq context.\n");
         printk("The following trace is a kernel self test and not a bug!\n");
-        dump_stack();
+
+        init_completion(&backtrace_work);
+        tasklet_schedule(&backtrace_tasklet);
+        wait_for_completion(&backtrace_work);
+}
+
+#ifdef CONFIG_STACKTRACE
+static void backtrace_test_saved(void)
+{
+        struct stack_trace trace;
+        unsigned long entries[8];
+
+        printk("Testing a saved backtrace.\n");
+        printk("The following trace is a kernel self test and not a bug!\n");
+
+        trace.nr_entries = 0;
+        trace.max_entries = ARRAY_SIZE(entries);
+        trace.entries = entries;
+        trace.skip = 0;
+
+        save_stack_trace(&trace);
+        print_stack_trace(&trace, 0);
+}
+#else
+static void backtrace_test_saved(void)
+{
+        printk("Saved backtrace test skipped.\n");
 }
+#endif
+
 static int backtrace_regression_test(void)
 {
         printk("====[ backtrace testing ]===========\n");
-        printk("Testing a backtrace from process context.\n");
-        printk("The following trace is a kernel self test and not a bug!\n");
-        dump_stack();
 
-        init_timer(&backtrace_timer);
-        backtrace_timer.function = backtrace_test_timer;
-        mod_timer(&backtrace_timer, jiffies + 10);
+        backtrace_test_normal();
+        backtrace_test_irq();
+        backtrace_test_saved();
 
-        msleep(10);
         printk("====[ end of backtrace testing ]====\n");
         return 0;
 }
diff --git a/kernel/capability.c b/kernel/capability.c
index cfbe44299488..901e0fdc3fff 100644
--- a/kernel/capability.c
+++ b/kernel/capability.c
@@ -121,6 +121,27 @@ static int cap_validate_magic(cap_user_header_t header, unsigned *tocopy)
  * uninteresting and/or not to be changed.
  */
 
+/*
+ * Atomically modify the effective capabilities returning the original
+ * value. No permission check is performed here - it is assumed that the
+ * caller is permitted to set the desired effective capabilities.
+ */
+kernel_cap_t cap_set_effective(const kernel_cap_t pE_new)
+{
+        kernel_cap_t pE_old;
+
+        spin_lock(&task_capability_lock);
+
+        pE_old = current->cap_effective;
+        current->cap_effective = pE_new;
+
+        spin_unlock(&task_capability_lock);
+
+        return pE_old;
+}
+
+EXPORT_SYMBOL(cap_set_effective);
+
 /**
  * sys_capget - get the capabilities of a given process.
  * @header: pointer to struct that contains capability version and
diff --git a/kernel/cpu.c b/kernel/cpu.c
index c77bc3a1c722..cfb1d43ab801 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -15,6 +15,28 @@
 #include <linux/stop_machine.h>
 #include <linux/mutex.h>
 
+/*
+ * Represents all cpu's present in the system
+ * In systems capable of hotplug, this map could dynamically grow
+ * as new cpu's are detected in the system via any platform specific
+ * method, such as ACPI for e.g.
+ */
+cpumask_t cpu_present_map __read_mostly;
+EXPORT_SYMBOL(cpu_present_map);
+
+#ifndef CONFIG_SMP
+
+/*
+ * Represents all cpu's that are currently online.
+ */
+cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL;
+EXPORT_SYMBOL(cpu_online_map);
+
+cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL;
+EXPORT_SYMBOL(cpu_possible_map);
+
+#else /* CONFIG_SMP */
+
 /* Serializes the updates to cpu_online_map, cpu_present_map */
 static DEFINE_MUTEX(cpu_add_remove_lock);
 
@@ -277,6 +299,7 @@ int __ref cpu_down(unsigned int cpu)
         cpu_maps_update_done();
         return err;
 }
+EXPORT_SYMBOL(cpu_down);
 #endif /*CONFIG_HOTPLUG_CPU*/
 
 /* Requires cpu_add_remove_lock to be held */
@@ -403,3 +426,5 @@ out:
         cpu_maps_update_done();
 }
 #endif /* CONFIG_PM_SLEEP_SMP */
+
+#endif /* CONFIG_SMP */
diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index 039baa4cd90c..459d601947a8 100644
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -1037,8 +1037,8 @@ int current_cpuset_is_being_rebound(void)
 
 static int update_relax_domain_level(struct cpuset *cs, s64 val)
 {
-        if ((int)val < 0)
-                val = -1;
+        if (val < -1 || val >= SD_LV_MAX)
+                return -EINVAL;
 
         if (val != cs->relax_domain_level) {
                 cs->relax_domain_level = val;
@@ -1194,6 +1194,15 @@ static int cpuset_can_attach(struct cgroup_subsys *ss,
 
         if (cpus_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed))
                 return -ENOSPC;
+        if (tsk->flags & PF_THREAD_BOUND) {
+                cpumask_t mask;
+
+                mutex_lock(&callback_mutex);
+                mask = cs->cpus_allowed;
+                mutex_unlock(&callback_mutex);
+                if (!cpus_equal(tsk->cpus_allowed, mask))
+                        return -EINVAL;
+        }
 
         return security_task_setscheduler(tsk, 0, NULL);
 }
@@ -1207,11 +1216,14 @@ static void cpuset_attach(struct cgroup_subsys *ss,
         struct mm_struct *mm;
         struct cpuset *cs = cgroup_cs(cont);
         struct cpuset *oldcs = cgroup_cs(oldcont);
+        int err;
 
         mutex_lock(&callback_mutex);
         guarantee_online_cpus(cs, &cpus);
-        set_cpus_allowed_ptr(tsk, &cpus);
+        err = set_cpus_allowed_ptr(tsk, &cpus);
         mutex_unlock(&callback_mutex);
+        if (err)
+                return;
 
         from = oldcs->mems_allowed;
         to = cs->mems_allowed;
@@ -1882,7 +1894,7 @@ static void scan_for_empty_cpusets(const struct cpuset *root)
  * in order to minimize text size.
  */
 
-static void common_cpu_mem_hotplug_unplug(void)
+static void common_cpu_mem_hotplug_unplug(int rebuild_sd)
 {
         cgroup_lock();
 
@@ -1890,6 +1902,13 @@ static void common_cpu_mem_hotplug_unplug(void)
         top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY];
         scan_for_empty_cpusets(&top_cpuset);
 
+        /*
+         * Scheduler destroys domains on hotplug events.
+         * Rebuild them based on the current settings.
+         */
+        if (rebuild_sd)
+                rebuild_sched_domains();
+
         cgroup_unlock();
 }
 
@@ -1906,11 +1925,22 @@ static void common_cpu_mem_hotplug_unplug(void)
 static int cpuset_handle_cpuhp(struct notifier_block *unused_nb,
                                 unsigned long phase, void *unused_cpu)
 {
-        if (phase == CPU_DYING || phase == CPU_DYING_FROZEN)
+        switch (phase) {
+        case CPU_UP_CANCELED:
+        case CPU_UP_CANCELED_FROZEN:
+        case CPU_DOWN_FAILED:
+        case CPU_DOWN_FAILED_FROZEN:
+        case CPU_ONLINE:
+        case CPU_ONLINE_FROZEN:
+        case CPU_DEAD:
+        case CPU_DEAD_FROZEN:
+                common_cpu_mem_hotplug_unplug(1);
+                break;
+        default:
                 return NOTIFY_DONE;
+        }
 
-        common_cpu_mem_hotplug_unplug();
-        return 0;
+        return NOTIFY_OK;
 }
 
 #ifdef CONFIG_MEMORY_HOTPLUG
@@ -1923,7 +1953,7 @@ static int cpuset_handle_cpuhp(struct notifier_block *unused_nb,
 
 void cpuset_track_online_nodes(void)
 {
-        common_cpu_mem_hotplug_unplug();
+        common_cpu_mem_hotplug_unplug(0);
 }
 #endif
 
diff --git a/kernel/exit.c b/kernel/exit.c
index 8f6185e69b69..ceb258782835 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -13,6 +13,7 @@
 #include <linux/personality.h>
 #include <linux/tty.h>
 #include <linux/mnt_namespace.h>
+#include <linux/iocontext.h>
 #include <linux/key.h>
 #include <linux/security.h>
 #include <linux/cpu.h>
diff --git a/kernel/fork.c b/kernel/fork.c
index 19908b26cf80..4bd2f516401f 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -23,6 +23,7 @@
 #include <linux/sem.h>
 #include <linux/file.h>
 #include <linux/fdtable.h>
+#include <linux/iocontext.h>
 #include <linux/key.h>
 #include <linux/binfmts.h>
 #include <linux/mman.h>
@@ -909,7 +910,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
 
         rt_mutex_init_task(p);
 
-#ifdef CONFIG_TRACE_IRQFLAGS
+#ifdef CONFIG_PROVE_LOCKING
         DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
         DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
 #endif
diff --git a/kernel/futex.c b/kernel/futex.c
index 449def8074fe..7d1136e97c14 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -1096,21 +1096,64 @@ static void unqueue_me_pi(struct futex_q *q)
  * private futexes.
  */
 static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,
-                                struct task_struct *newowner)
+                                struct task_struct *newowner,
+                                struct rw_semaphore *fshared)
 {
         u32 newtid = task_pid_vnr(newowner) | FUTEX_WAITERS;
         struct futex_pi_state *pi_state = q->pi_state;
+        struct task_struct *oldowner = pi_state->owner;
         u32 uval, curval, newval;
-        int ret;
+        int ret, attempt = 0;
 
         /* Owner died? */
+        if (!pi_state->owner)
+                newtid |= FUTEX_OWNER_DIED;
+
+        /*
+         * We are here either because we stole the rtmutex from the
+         * pending owner or we are the pending owner which failed to
+         * get the rtmutex. We have to replace the pending owner TID
+         * in the user space variable. This must be atomic as we have
+         * to preserve the owner died bit here.
+         *
+         * Note: We write the user space value _before_ changing the
+         * pi_state because we can fault here. Imagine swapped out
+         * pages or a fork, which was running right before we acquired
+         * mmap_sem, that marked all the anonymous memory readonly for
+         * cow.
+         *
+         * Modifying pi_state _before_ the user space value would
+         * leave the pi_state in an inconsistent state when we fault
+         * here, because we need to drop the hash bucket lock to
+         * handle the fault. This might be observed in the PID check
+         * in lookup_pi_state.
+         */
+retry:
+        if (get_futex_value_locked(&uval, uaddr))
+                goto handle_fault;
+
+        while (1) {
+                newval = (uval & FUTEX_OWNER_DIED) | newtid;
+
+                curval = cmpxchg_futex_value_locked(uaddr, uval, newval);
+
+                if (curval == -EFAULT)
+                        goto handle_fault;
+                if (curval == uval)
+                        break;
+                uval = curval;
+        }
+
+        /*
+         * We fixed up user space. Now we need to fix the pi_state
+         * itself.
+         */
         if (pi_state->owner != NULL) {
                 spin_lock_irq(&pi_state->owner->pi_lock);
                 WARN_ON(list_empty(&pi_state->list));
                 list_del_init(&pi_state->list);
                 spin_unlock_irq(&pi_state->owner->pi_lock);
-        } else
-                newtid |= FUTEX_OWNER_DIED;
+        }
 
         pi_state->owner = newowner;
 
@@ -1118,26 +1161,35 @@ static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,
         WARN_ON(!list_empty(&pi_state->list));
         list_add(&pi_state->list, &newowner->pi_state_list);
         spin_unlock_irq(&newowner->pi_lock);
+        return 0;
 
         /*
-         * We own it, so we have to replace the pending owner
-         * TID. This must be atomic as we have preserve the
-         * owner died bit here.
+         * To handle the page fault we need to drop the hash bucket
+         * lock here. That gives the other task (either the pending
+         * owner itself or the task which stole the rtmutex) the
+         * chance to try the fixup of the pi_state. So once we are
+         * back from handling the fault we need to check the pi_state
+         * after reacquiring the hash bucket lock and before trying to
+         * do another fixup. When the fixup has been done already we
+         * simply return.
          */
-        ret = get_futex_value_locked(&uval, uaddr);
+handle_fault:
+        spin_unlock(q->lock_ptr);
 
-        while (!ret) {
-                newval = (uval & FUTEX_OWNER_DIED) | newtid;
+        ret = futex_handle_fault((unsigned long)uaddr, fshared, attempt++);
 
-                curval = cmpxchg_futex_value_locked(uaddr, uval, newval);
+        spin_lock(q->lock_ptr);
 
-                if (curval == -EFAULT)
-                        ret = -EFAULT;
-                if (curval == uval)
-                        break;
-                uval = curval;
-        }
-        return ret;
+        /*
+         * Check if someone else fixed it for us:
+         */
+        if (pi_state->owner != oldowner)
+                return 0;
+
+        if (ret)
+                return ret;
+
+        goto retry;
 }
 
 /*
@@ -1507,7 +1559,7 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
                  * that case:
                  */
                 if (q.pi_state->owner != curr)
-                        ret = fixup_pi_state_owner(uaddr, &q, curr);
+                        ret = fixup_pi_state_owner(uaddr, &q, curr, fshared);
         } else {
                 /*
                  * Catch the rare case, where the lock was released
@@ -1539,7 +1591,8 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
                                 int res;
 
                                 owner = rt_mutex_owner(&q.pi_state->pi_mutex);
-                                res = fixup_pi_state_owner(uaddr, &q, owner);
+                                res = fixup_pi_state_owner(uaddr, &q, owner,
+                                                           fshared);
 
                                 /* propagate -EFAULT, if the fixup failed */
                                 if (res)
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index 421be5fe5cc7..b8e4dce80a74 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -300,11 +300,10 @@ EXPORT_SYMBOL_GPL(ktime_sub_ns);
  */
 u64 ktime_divns(const ktime_t kt, s64 div)
 {
-        u64 dclc, inc, dns;
+        u64 dclc;
         int sft = 0;
 
-        dclc = dns = ktime_to_ns(kt);
-        inc = div;
+        dclc = ktime_to_ns(kt);
         /* Make sure the divisor is less than 2^32: */
         while (div >> 32) {
                 sft++;
@@ -623,7 +622,7 @@ static void retrigger_next_event(void *arg)
 void clock_was_set(void)
 {
         /* Retrigger the CPU local events everywhere */
-        on_each_cpu(retrigger_next_event, NULL, 0, 1);
+        on_each_cpu(retrigger_next_event, NULL, 1);
 }
 
 /*
@@ -632,8 +631,6 @@ void clock_was_set(void)
  */
 void hres_timers_resume(void)
 {
-        WARN_ON_ONCE(num_online_cpus() > 1);
-
         /* Retrigger the CPU local events: */
         retrigger_next_event(NULL);
 }
@@ -1003,10 +1000,18 @@ hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
          */
         raise = timer->state == HRTIMER_STATE_PENDING;
 
+        /*
+         * We use preempt_disable to prevent this task from migrating after
+         * setting up the softirq and raising it. Otherwise, if me migrate
+         * we will raise the softirq on the wrong CPU.
+         */
+        preempt_disable();
+
         unlock_hrtimer_base(timer, &flags);
 
         if (raise)
                 hrtimer_raise_softirq();
+        preempt_enable();
 
         return ret;
 }
@@ -1078,7 +1083,7 @@ ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
 }
 EXPORT_SYMBOL_GPL(hrtimer_get_remaining);
 
-#if defined(CONFIG_NO_IDLE_HZ) || defined(CONFIG_NO_HZ)
+#ifdef CONFIG_NO_HZ
 /**
  * hrtimer_get_next_event - get the time until next expiry event
  *
@@ -1669,7 +1674,7 @@ void __init hrtimers_init(void)
                           (void *)(long)smp_processor_id());
         register_cpu_notifier(&hrtimers_nb);
 #ifdef CONFIG_HIGH_RES_TIMERS
-        open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq, NULL);
+        open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq);
 #endif
 }
 
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 46d6611a33bb..77a51be36010 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -17,6 +17,8 @@
 
 #ifdef CONFIG_SMP
 
+cpumask_t irq_default_affinity = CPU_MASK_ALL;
+
 /**
  *      synchronize_irq - wait for pending IRQ handlers (on other CPUs)
  *      @irq: interrupt number to wait for
@@ -95,6 +97,27 @@ int irq_set_affinity(unsigned int irq, cpumask_t cpumask)
         return 0;
 }
 
+#ifndef CONFIG_AUTO_IRQ_AFFINITY
+/*
+ * Generic version of the affinity autoselector.
+ */
+int irq_select_affinity(unsigned int irq)
+{
+        cpumask_t mask;
+
+        if (!irq_can_set_affinity(irq))
+                return 0;
+
+        cpus_and(mask, cpu_online_map, irq_default_affinity);
+
+        irq_desc[irq].affinity = mask;
+        irq_desc[irq].chip->set_affinity(irq, mask);
+
+        set_balance_irq_affinity(irq, mask);
+        return 0;
+}
+#endif
+
 #endif
 
 /**
@@ -354,7 +377,7 @@ int setup_irq(unsigned int irq, struct irqaction *new)
 
                 /* Setup the type (level, edge polarity) if configured: */
                 if (new->flags & IRQF_TRIGGER_MASK) {
-                        if (desc->chip && desc->chip->set_type)
+                        if (desc->chip->set_type)
                                 desc->chip->set_type(irq,
                                                 new->flags & IRQF_TRIGGER_MASK);
                         else
@@ -364,8 +387,7 @@ int setup_irq(unsigned int irq, struct irqaction *new)
                                  */
                                 printk(KERN_WARNING "No IRQF_TRIGGER set_type "
                                        "function for IRQ %d (%s)\n", irq,
-                                       desc->chip ? desc->chip->name :
-                                       "unknown");
+                                       desc->chip->name);
                 } else
                         compat_irq_chip_set_default_handler(desc);
 
@@ -382,6 +404,9 @@ int setup_irq(unsigned int irq, struct irqaction *new)
                 } else
                         /* Undo nested disables: */
                         desc->depth = 1;
+
+                /* Set default affinity mask once everything is setup */
+                irq_select_affinity(irq);
         }
         /* Reset broken irq detection when installing new handler */
         desc->irq_count = 0;
@@ -571,8 +596,6 @@ int request_irq(unsigned int irq, irq_handler_t handler,
         action->next = NULL;
         action->dev_id = dev_id;
 
-        select_smp_affinity(irq);
-
 #ifdef CONFIG_DEBUG_SHIRQ
         if (irqflags & IRQF_SHARED) {
                 /*
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c
index c2f2ccb0549a..6c6d35d68ee9 100644
--- a/kernel/irq/proc.c
+++ b/kernel/irq/proc.c
@@ -44,7 +44,7 @@ static int irq_affinity_write_proc(struct file *file, const char __user *buffer,
                                    unsigned long count, void *data)
 {
         unsigned int irq = (int)(long)data, full_count = count, err;
-        cpumask_t new_value, tmp;
+        cpumask_t new_value;
 
         if (!irq_desc[irq].chip->set_affinity || no_irq_affinity ||
             irq_balancing_disabled(irq))
@@ -62,17 +62,51 @@ static int irq_affinity_write_proc(struct file *file, const char __user *buffer,
          * way to make the system unusable accidentally :-) At least
          * one online CPU still has to be targeted.
          */
-        cpus_and(tmp, new_value, cpu_online_map);
-        if (cpus_empty(tmp))
+        if (!cpus_intersects(new_value, cpu_online_map))
                 /* Special case for empty set - allow the architecture
                    code to set default SMP affinity. */
-                return select_smp_affinity(irq) ? -EINVAL : full_count;
+                return irq_select_affinity(irq) ? -EINVAL : full_count;
 
         irq_set_affinity(irq, new_value);
 
         return full_count;
 }
 
+static int default_affinity_read(char *page, char **start, off_t off,
+                                  int count, int *eof, void *data)
+{
+        int len = cpumask_scnprintf(page, count, irq_default_affinity);
+        if (count - len < 2)
+                return -EINVAL;
+        len += sprintf(page + len, "\n");
+        return len;
+}
+
+static int default_affinity_write(struct file *file, const char __user *buffer,
+                                   unsigned long count, void *data)
+{
+        unsigned int full_count = count, err;
+        cpumask_t new_value;
+
+        err = cpumask_parse_user(buffer, count, new_value);
+        if (err)
+                return err;
+
+        if (!is_affinity_mask_valid(new_value))
+                return -EINVAL;
+
+        /*
+         * Do not allow disabling IRQs completely - it's a too easy
+         * way to make the system unusable accidentally :-) At least
+         * one online CPU still has to be targeted.
+         */
+        if (!cpus_intersects(new_value, cpu_online_map))
+                return -EINVAL;
+
+        irq_default_affinity = new_value;
+
+        return full_count;
+}
 #endif
 
 static int irq_spurious_read(char *page, char **start, off_t off,
@@ -171,6 +205,21 @@ void unregister_handler_proc(unsigned int irq, struct irqaction *action)
                 remove_proc_entry(action->dir->name, irq_desc[irq].dir);
 }
 
+void register_default_affinity_proc(void)
+{
+#ifdef CONFIG_SMP
+        struct proc_dir_entry *entry;
+
+        /* create /proc/irq/default_smp_affinity */
+        entry = create_proc_entry("default_smp_affinity", 0600, root_irq_dir);
+        if (entry) {
+                entry->data = NULL;
+                entry->read_proc  = default_affinity_read;
+                entry->write_proc = default_affinity_write;
+        }
+#endif
+}
+
 void init_irq_proc(void)
 {
         int i;
@@ -180,6 +229,8 @@ void init_irq_proc(void)
         if (!root_irq_dir)
                 return;
 
+        register_default_affinity_proc();
+
         /*
          * Create entries for all existing IRQs.
          */
diff --git a/kernel/kgdb.c b/kernel/kgdb.c
index 79e3c90113c2..3ec23c3ec97f 100644
--- a/kernel/kgdb.c
+++ b/kernel/kgdb.c
@@ -1499,7 +1499,8 @@ int kgdb_nmicallback(int cpu, void *regs)
         return 1;
 }
 
-void kgdb_console_write(struct console *co, const char *s, unsigned count)
+static void kgdb_console_write(struct console *co, const char *s,
+   unsigned count)
 {
         unsigned long flags;
 
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 1e0250cb9486..1485ca8d0e00 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -79,7 +79,7 @@ static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
  *
  * For such cases, we now have a blacklist
  */
-struct kprobe_blackpoint kprobe_blacklist[] = {
+static struct kprobe_blackpoint kprobe_blacklist[] = {
         {"preempt_schedule",},
         {NULL}    /* Terminator */
 };
@@ -699,8 +699,9 @@ static int __register_kprobes(struct kprobe **kps, int num,
                 return -EINVAL;
         for (i = 0; i < num; i++) {
                 ret = __register_kprobe(kps[i], called_from);
-                if (ret < 0 && i > 0) {
-                        unregister_kprobes(kps, i);
+                if (ret < 0) {
+                        if (i > 0)
+                                unregister_kprobes(kps, i);
                         break;
                 }
         }
@@ -776,8 +777,9 @@ static int __register_jprobes(struct jprobe **jps, int num,
                         jp->kp.break_handler = longjmp_break_handler;
                         ret = __register_kprobe(&jp->kp, called_from);
                 }
-                if (ret < 0 && i > 0) {
-                        unregister_jprobes(jps, i);
+                if (ret < 0) {
+                        if (i > 0)
+                                unregister_jprobes(jps, i);
                         break;
                 }
         }
@@ -920,8 +922,9 @@ static int __register_kretprobes(struct kretprobe **rps, int num,
                 return -EINVAL;
         for (i = 0; i < num; i++) {
                 ret = __register_kretprobe(rps[i], called_from);
-                if (ret < 0 && i > 0) {
-                        unregister_kretprobes(rps, i);
+                if (ret < 0) {
+                        if (i > 0)
+                                unregister_kretprobes(rps, i);
                         break;
                 }
         }
diff --git a/kernel/kthread.c b/kernel/kthread.c
index bd1b9ea024e1..ac3fb7326641 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -180,6 +180,7 @@ void kthread_bind(struct task_struct *k, unsigned int cpu)
         set_task_cpu(k, cpu);
         k->cpus_allowed = cpumask_of_cpu(cpu);
         k->rt.nr_cpus_allowed = 1;
+        k->flags |= PF_THREAD_BOUND;
 }
 EXPORT_SYMBOL(kthread_bind);
 
@@ -234,7 +235,7 @@ int kthreadd(void *unused)
         set_user_nice(tsk, KTHREAD_NICE_LEVEL);
         set_cpus_allowed(tsk, CPU_MASK_ALL);
 
-        current->flags |= PF_NOFREEZE;
+        current->flags |= PF_NOFREEZE | PF_FREEZER_NOSIG;
 
         for (;;) {
                 set_current_state(TASK_INTERRUPTIBLE);
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index 81a4e4a3f087..d38a64362973 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -39,6 +39,7 @@
 #include <linux/irqflags.h>
 #include <linux/utsname.h>
 #include <linux/hash.h>
+#include <linux/ftrace.h>
 
 #include <asm/sections.h>
 
@@ -81,6 +82,8 @@ static int graph_lock(void)
                 __raw_spin_unlock(&lockdep_lock);
                 return 0;
         }
+        /* prevent any recursions within lockdep from causing deadlocks */
+        current->lockdep_recursion++;
         return 1;
 }
 
@@ -89,6 +92,7 @@ static inline int graph_unlock(void)
         if (debug_locks && !__raw_spin_is_locked(&lockdep_lock))
                 return DEBUG_LOCKS_WARN_ON(1);
 
+        current->lockdep_recursion--;
         __raw_spin_unlock(&lockdep_lock);
         return 0;
 }
@@ -982,7 +986,7 @@ check_noncircular(struct lock_class *source, unsigned int depth)
         return 1;
 }
 
-#ifdef CONFIG_TRACE_IRQFLAGS
+#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING)
 /*
  * Forwards and backwards subgraph searching, for the purposes of
  * proving that two subgraphs can be connected by a new dependency
@@ -1458,7 +1462,14 @@ out_bug:
 }
 
 unsigned long nr_lock_chains;
-static struct lock_chain lock_chains[MAX_LOCKDEP_CHAINS];
+struct lock_chain lock_chains[MAX_LOCKDEP_CHAINS];
+int nr_chain_hlocks;
+static u16 chain_hlocks[MAX_LOCKDEP_CHAIN_HLOCKS];
+
+struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i)
+{
+        return lock_classes + chain_hlocks[chain->base + i];
+}
 
 /*
  * Look up a dependency chain. If the key is not present yet then
@@ -1466,10 +1477,15 @@ static struct lock_chain lock_chains[MAX_LOCKDEP_CHAINS];
  * validated. If the key is already hashed, return 0.
  * (On return with 1 graph_lock is held.)
  */
-static inline int lookup_chain_cache(u64 chain_key, struct lock_class *class)
+static inline int lookup_chain_cache(struct task_struct *curr,
+                                     struct held_lock *hlock,
+                                     u64 chain_key)
 {
+        struct lock_class *class = hlock->class;
         struct list_head *hash_head = chainhashentry(chain_key);
         struct lock_chain *chain;
+        struct held_lock *hlock_curr, *hlock_next;
+        int i, j, n, cn;
 
         if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
                 return 0;
@@ -1517,6 +1533,32 @@ cache_hit:
         }
         chain = lock_chains + nr_lock_chains++;
         chain->chain_key = chain_key;
+        chain->irq_context = hlock->irq_context;
+        /* Find the first held_lock of current chain */
+        hlock_next = hlock;
+        for (i = curr->lockdep_depth - 1; i >= 0; i--) {
+                hlock_curr = curr->held_locks + i;
+                if (hlock_curr->irq_context != hlock_next->irq_context)
+                        break;
+                hlock_next = hlock;
+        }
+        i++;
+        chain->depth = curr->lockdep_depth + 1 - i;
+        cn = nr_chain_hlocks;
+        while (cn + chain->depth <= MAX_LOCKDEP_CHAIN_HLOCKS) {
+                n = cmpxchg(&nr_chain_hlocks, cn, cn + chain->depth);
+                if (n == cn)
+                        break;
+                cn = n;
+        }
+        if (likely(cn + chain->depth <= MAX_LOCKDEP_CHAIN_HLOCKS)) {
+                chain->base = cn;
+                for (j = 0; j < chain->depth - 1; j++, i++) {
+                        int lock_id = curr->held_locks[i].class - lock_classes;
+                        chain_hlocks[chain->base + j] = lock_id;
+                }
+                chain_hlocks[chain->base + j] = class - lock_classes;
+        }
         list_add_tail_rcu(&chain->entry, hash_head);
         debug_atomic_inc(&chain_lookup_misses);
         inc_chains();
@@ -1538,7 +1580,7 @@ static int validate_chain(struct task_struct *curr, struct lockdep_map *lock,
          * graph_lock for us)
          */
         if (!hlock->trylock && (hlock->check == 2) &&
-                        lookup_chain_cache(chain_key, hlock->class)) {
+            lookup_chain_cache(curr, hlock, chain_key)) {
                 /*
                  * Check whether last held lock:
                  *
@@ -1680,7 +1722,7 @@ valid_state(struct task_struct *curr, struct held_lock *this,
 static int mark_lock(struct task_struct *curr, struct held_lock *this,
                      enum lock_usage_bit new_bit);
 
-#ifdef CONFIG_TRACE_IRQFLAGS
+#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING)
 
 /*
  * print irq inversion bug:
@@ -2013,11 +2055,13 @@ void early_boot_irqs_on(void)
 /*
  * Hardirqs will be enabled:
  */
-void trace_hardirqs_on(void)
+void trace_hardirqs_on_caller(unsigned long a0)
 {
         struct task_struct *curr = current;
         unsigned long ip;
 
+        time_hardirqs_on(CALLER_ADDR0, a0);
+
         if (unlikely(!debug_locks || current->lockdep_recursion))
                 return;
 
@@ -2055,16 +2099,23 @@ void trace_hardirqs_on(void)
         curr->hardirq_enable_event = ++curr->irq_events;
         debug_atomic_inc(&hardirqs_on_events);
 }
+EXPORT_SYMBOL(trace_hardirqs_on_caller);
 
+void trace_hardirqs_on(void)
+{
+        trace_hardirqs_on_caller(CALLER_ADDR0);
+}
 EXPORT_SYMBOL(trace_hardirqs_on);
 
 /*
  * Hardirqs were disabled:
  */
-void trace_hardirqs_off(void)
+void trace_hardirqs_off_caller(unsigned long a0)
 {
         struct task_struct *curr = current;
 
+        time_hardirqs_off(CALLER_ADDR0, a0);
+
         if (unlikely(!debug_locks || current->lockdep_recursion))
                 return;
 
@@ -2082,7 +2133,12 @@ void trace_hardirqs_off(void)
         } else
                 debug_atomic_inc(&redundant_hardirqs_off);
 }
+EXPORT_SYMBOL(trace_hardirqs_off_caller);
 
+void trace_hardirqs_off(void)
+{
+        trace_hardirqs_off_caller(CALLER_ADDR0);
+}
 EXPORT_SYMBOL(trace_hardirqs_off);
 
 /*
@@ -2246,7 +2302,7 @@ static inline int separate_irq_context(struct task_struct *curr,
  * Mark a lock with a usage bit, and validate the state transition:
  */
 static int mark_lock(struct task_struct *curr, struct held_lock *this,
-                     enum lock_usage_bit new_bit)
+                             enum lock_usage_bit new_bit)
 {
         unsigned int new_mask = 1 << new_bit, ret = 1;
 
@@ -2650,7 +2706,8 @@ __lock_release(struct lockdep_map *lock, int nested, unsigned long ip)
  */
 static void check_flags(unsigned long flags)
 {
-#if defined(CONFIG_DEBUG_LOCKDEP) && defined(CONFIG_TRACE_IRQFLAGS)
+#if defined(CONFIG_PROVE_LOCKING) && defined(CONFIG_DEBUG_LOCKDEP) && \
+    defined(CONFIG_TRACE_IRQFLAGS)
         if (!debug_locks)
                 return;
 
@@ -2686,7 +2743,7 @@ static void check_flags(unsigned long flags)
  * and also avoid lockdep recursion:
  */
 void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
-                  int trylock, int read, int check, unsigned long ip)
+                          int trylock, int read, int check, unsigned long ip)
 {
         unsigned long flags;
 
@@ -2708,7 +2765,8 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 
 EXPORT_SYMBOL_GPL(lock_acquire);
 
-void lock_release(struct lockdep_map *lock, int nested, unsigned long ip)
+void lock_release(struct lockdep_map *lock, int nested,
+                          unsigned long ip)
 {
         unsigned long flags;
 
diff --git a/kernel/lockdep_internals.h b/kernel/lockdep_internals.h
index 8ce09bc4613d..c3600a091a28 100644
--- a/kernel/lockdep_internals.h
+++ b/kernel/lockdep_internals.h
@@ -23,6 +23,8 @@
 #define MAX_LOCKDEP_CHAINS_BITS 14
 #define MAX_LOCKDEP_CHAINS      (1UL << MAX_LOCKDEP_CHAINS_BITS)
 
+#define MAX_LOCKDEP_CHAIN_HLOCKS (MAX_LOCKDEP_CHAINS*5)
+
 /*
  * Stack-trace: tightly packed array of stack backtrace
  * addresses. Protected by the hash_lock.
@@ -30,15 +32,19 @@
 #define MAX_STACK_TRACE_ENTRIES 262144UL
 
 extern struct list_head all_lock_classes;
+extern struct lock_chain lock_chains[];
 
 extern void
 get_usage_chars(struct lock_class *class, char *c1, char *c2, char *c3, char *c4);
 
 extern const char * __get_key_name(struct lockdep_subclass_key *key, char *str);
 
+struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i);
+
 extern unsigned long nr_lock_classes;
 extern unsigned long nr_list_entries;
 extern unsigned long nr_lock_chains;
+extern int nr_chain_hlocks;
 extern unsigned long nr_stack_trace_entries;
 
 extern unsigned int nr_hardirq_chains;
diff --git a/kernel/lockdep_proc.c b/kernel/lockdep_proc.c
index dc5d29648d85..9b0e940e2545 100644
--- a/kernel/lockdep_proc.c
+++ b/kernel/lockdep_proc.c
@@ -139,7 +139,7 @@ static int l_show(struct seq_file *m, void *v)
 
         list_for_each_entry(entry, &class->locks_after, entry) {
                 if (entry->distance == 1) {
-                        seq_printf(m, " -> [%p] ", entry->class);
+                        seq_printf(m, " -> [%p] ", entry->class->key);
                         print_name(m, entry->class);
                         seq_puts(m, "\n");
                 }
@@ -178,6 +178,95 @@ static const struct file_operations proc_lockdep_operations = {
         .release        = seq_release,
 };
 
+#ifdef CONFIG_PROVE_LOCKING
+static void *lc_next(struct seq_file *m, void *v, loff_t *pos)
+{
+        struct lock_chain *chain;
+
+        (*pos)++;
+
+        if (v == SEQ_START_TOKEN)
+                chain = m->private;
+        else {
+                chain = v;
+
+                if (*pos < nr_lock_chains)
+                        chain = lock_chains + *pos;
+                else
+                        chain = NULL;
+        }
+
+        return chain;
+}
+
+static void *lc_start(struct seq_file *m, loff_t *pos)
+{
+        if (*pos == 0)
+                return SEQ_START_TOKEN;
+
+        if (*pos < nr_lock_chains)
+                return lock_chains + *pos;
+
+        return NULL;
+}
+
+static void lc_stop(struct seq_file *m, void *v)
+{
+}
+
+static int lc_show(struct seq_file *m, void *v)
+{
+        struct lock_chain *chain = v;
+        struct lock_class *class;
+        int i;
+
+        if (v == SEQ_START_TOKEN) {
+                seq_printf(m, "all lock chains:\n");
+                return 0;
+        }
+
+        seq_printf(m, "irq_context: %d\n", chain->irq_context);
+
+        for (i = 0; i < chain->depth; i++) {
+                class = lock_chain_get_class(chain, i);
+                seq_printf(m, "[%p] ", class->key);
+                print_name(m, class);
+                seq_puts(m, "\n");
+        }
+        seq_puts(m, "\n");
+
+        return 0;
+}
+
+static const struct seq_operations lockdep_chains_ops = {
+        .start  = lc_start,
+        .next   = lc_next,
+        .stop   = lc_stop,
+        .show   = lc_show,
+};
+
+static int lockdep_chains_open(struct inode *inode, struct file *file)
+{
+        int res = seq_open(file, &lockdep_chains_ops);
+        if (!res) {
+                struct seq_file *m = file->private_data;
+
+                if (nr_lock_chains)
+                        m->private = lock_chains;
+                else
+                        m->private = NULL;
+        }
+        return res;
+}
+
+static const struct file_operations proc_lockdep_chains_operations = {
+        .open           = lockdep_chains_open,
+        .read           = seq_read,
+        .llseek         = seq_lseek,
+        .release        = seq_release,
+};
+#endif /* CONFIG_PROVE_LOCKING */
+
 static void lockdep_stats_debug_show(struct seq_file *m)
 {
 #ifdef CONFIG_DEBUG_LOCKDEP
@@ -294,6 +383,8 @@ static int lockdep_stats_show(struct seq_file *m, void *v)
 #ifdef CONFIG_PROVE_LOCKING
         seq_printf(m, " dependency chains:             %11lu [max: %lu]\n",
                         nr_lock_chains, MAX_LOCKDEP_CHAINS);
+        seq_printf(m, " dependency chain hlocks:       %11d [max: %lu]\n",
+                        nr_chain_hlocks, MAX_LOCKDEP_CHAIN_HLOCKS);
 #endif
 
 #ifdef CONFIG_TRACE_IRQFLAGS
@@ -661,6 +752,10 @@ static const struct file_operations proc_lock_stat_operations = {
 static int __init lockdep_proc_init(void)
 {
         proc_create("lockdep", S_IRUSR, NULL, &proc_lockdep_operations);
+#ifdef CONFIG_PROVE_LOCKING
+        proc_create("lockdep_chains", S_IRUSR, NULL,
+                    &proc_lockdep_chains_operations);
+#endif
         proc_create("lockdep_stats", S_IRUSR, NULL,
                     &proc_lockdep_stats_operations);
 
diff --git a/kernel/marker.c b/kernel/marker.c
index b5a9fe1d50d5..1abfb923b761 100644
--- a/kernel/marker.c
+++ b/kernel/marker.c
@@ -55,8 +55,8 @@ static DEFINE_MUTEX(markers_mutex);
 struct marker_entry {
         struct hlist_node hlist;
         char *format;
-        void (*call)(const struct marker *mdata,        /* Probe wrapper */
-                void *call_private, const char *fmt, ...);
+                        /* Probe wrapper */
+        void (*call)(const struct marker *mdata, void *call_private, ...);
         struct marker_probe_closure single;
         struct marker_probe_closure *multi;
         int refcount;   /* Number of times armed. 0 if disarmed. */
@@ -91,15 +91,13 @@ EXPORT_SYMBOL_GPL(__mark_empty_function);
  * marker_probe_cb Callback that prepares the variable argument list for probes.
  * @mdata: pointer of type struct marker
  * @call_private: caller site private data
- * @fmt: format string
  * @...:  Variable argument list.
  *
  * Since we do not use "typical" pointer based RCU in the 1 argument case, we
  * need to put a full smp_rmb() in this branch. This is why we do not use
  * rcu_dereference() for the pointer read.
  */
-void marker_probe_cb(const struct marker *mdata, void *call_private,
-        const char *fmt, ...)
+void marker_probe_cb(const struct marker *mdata, void *call_private, ...)
 {
         va_list args;
         char ptype;
@@ -120,8 +118,9 @@ void marker_probe_cb(const struct marker *mdata, void *call_private,
                 /* Must read the ptr before private data. They are not data
                  * dependant, so we put an explicit smp_rmb() here. */
                 smp_rmb();
-                va_start(args, fmt);
-                func(mdata->single.probe_private, call_private, fmt, &args);
+                va_start(args, call_private);
+                func(mdata->single.probe_private, call_private, mdata->format,
+                        &args);
                 va_end(args);
         } else {
                 struct marker_probe_closure *multi;
@@ -136,9 +135,9 @@ void marker_probe_cb(const struct marker *mdata, void *call_private,
                 smp_read_barrier_depends();
                 multi = mdata->multi;
                 for (i = 0; multi[i].func; i++) {
-                        va_start(args, fmt);
-                        multi[i].func(multi[i].probe_private, call_private, fmt,
-                                &args);
+                        va_start(args, call_private);
+                        multi[i].func(multi[i].probe_private, call_private,
+                                mdata->format, &args);
                         va_end(args);
                 }
         }
@@ -150,13 +149,11 @@ EXPORT_SYMBOL_GPL(marker_probe_cb);
  * marker_probe_cb Callback that does not prepare the variable argument list.
  * @mdata: pointer of type struct marker
  * @call_private: caller site private data
- * @fmt: format string
  * @...:  Variable argument list.
  *
  * Should be connected to markers "MARK_NOARGS".
  */
-void marker_probe_cb_noarg(const struct marker *mdata,
-        void *call_private, const char *fmt, ...)
+void marker_probe_cb_noarg(const struct marker *mdata, void *call_private, ...)
 {
         va_list args;   /* not initialized */
         char ptype;
@@ -172,7 +169,8 @@ void marker_probe_cb_noarg(const struct marker *mdata,
                 /* Must read the ptr before private data. They are not data
                  * dependant, so we put an explicit smp_rmb() here. */
                 smp_rmb();
-                func(mdata->single.probe_private, call_private, fmt, &args);
+                func(mdata->single.probe_private, call_private, mdata->format,
+                        &args);
         } else {
                 struct marker_probe_closure *multi;
                 int i;
@@ -186,8 +184,8 @@ void marker_probe_cb_noarg(const struct marker *mdata,
                 smp_read_barrier_depends();
                 multi = mdata->multi;
                 for (i = 0; multi[i].func; i++)
-                        multi[i].func(multi[i].probe_private, call_private, fmt,
-                                &args);
+                        multi[i].func(multi[i].probe_private, call_private,
+                                mdata->format, &args);
         }
         preempt_enable();
 }
diff --git a/kernel/mutex-debug.c b/kernel/mutex-debug.c
index 3aaa06c561de..1d94160eb532 100644
--- a/kernel/mutex-debug.c
+++ b/kernel/mutex-debug.c
@@ -79,8 +79,8 @@ void debug_mutex_unlock(struct mutex *lock)
         if (unlikely(!debug_locks))
                 return;
 
-        DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info());
         DEBUG_LOCKS_WARN_ON(lock->magic != lock);
+        DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info());
         DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next);
         DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info());
 }
diff --git a/kernel/mutex.c b/kernel/mutex.c
index d046a345d365..bcdc9ac8ef60 100644
--- a/kernel/mutex.c
+++ b/kernel/mutex.c
@@ -165,10 +165,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                  * got a signal? (This code gets eliminated in the
                  * TASK_UNINTERRUPTIBLE case.)
                  */
-                if (unlikely((state == TASK_INTERRUPTIBLE &&
-                                        signal_pending(task)) ||
-                              (state == TASK_KILLABLE &&
-                                        fatal_signal_pending(task)))) {
+                if (unlikely(signal_pending_state(state, task))) {
                         mutex_remove_waiter(lock, &waiter,
                                             task_thread_info(task));
                         mutex_release(&lock->dep_map, 1, ip);
diff --git a/kernel/pid.c b/kernel/pid.c
index 20d59fa2d493..30bd5d4b2ac7 100644
--- a/kernel/pid.c
+++ b/kernel/pid.c
@@ -30,6 +30,7 @@
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/init.h>
+#include <linux/rculist.h>
 #include <linux/bootmem.h>
 #include <linux/hash.h>
 #include <linux/pid_namespace.h>
diff --git a/kernel/pm_qos_params.c b/kernel/pm_qos_params.c
index 0afe32be4c85..8cb757026386 100644
--- a/kernel/pm_qos_params.c
+++ b/kernel/pm_qos_params.c
@@ -29,6 +29,7 @@
 
 #include <linux/pm_qos_params.h>
 #include <linux/sched.h>
+#include <linux/smp_lock.h>
 #include <linux/spinlock.h>
 #include <linux/slab.h>
 #include <linux/time.h>
@@ -358,15 +359,19 @@ static int pm_qos_power_open(struct inode *inode, struct file *filp)
         int ret;
         long pm_qos_class;
 
+        lock_kernel();
         pm_qos_class = find_pm_qos_object_by_minor(iminor(inode));
         if (pm_qos_class >= 0) {
                 filp->private_data = (void *)pm_qos_class;
                 sprintf(name, "process_%d", current->pid);
                 ret = pm_qos_add_requirement(pm_qos_class, name,
                                         PM_QOS_DEFAULT_VALUE);
-                if (ret >= 0)
+                if (ret >= 0) {
+                        unlock_kernel();
                         return 0;
+                }
         }
+        unlock_kernel();
 
         return -EPERM;
 }
diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c
index f1525ad06cb3..c42a03aef36f 100644
--- a/kernel/posix-cpu-timers.c
+++ b/kernel/posix-cpu-timers.c
@@ -1037,6 +1037,9 @@ static void check_thread_timers(struct task_struct *tsk,
                                 sig->rlim[RLIMIT_RTTIME].rlim_cur +=
                                                                 USEC_PER_SEC;
                         }
+                        printk(KERN_INFO
+                                "RT Watchdog Timeout: %s[%d]\n",
+                                tsk->comm, task_pid_nr(tsk));
                         __group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk);
                 }
         }
diff --git a/kernel/power/process.c b/kernel/power/process.c
index f1d0b345c9ba..5fb87652f214 100644
--- a/kernel/power/process.c
+++ b/kernel/power/process.c
@@ -19,9 +19,6 @@
  */
 #define TIMEOUT (20 * HZ)
 
-#define FREEZER_KERNEL_THREADS 0
-#define FREEZER_USER_SPACE 1
-
 static inline int freezeable(struct task_struct * p)
 {
         if ((p == current) ||
@@ -84,63 +81,53 @@ static void fake_signal_wake_up(struct task_struct *p)
         spin_unlock_irqrestore(&p->sighand->siglock, flags);
 }
 
-static int has_mm(struct task_struct *p)
+static inline bool should_send_signal(struct task_struct *p)
 {
-        return (p->mm && !(p->flags & PF_BORROWED_MM));
+        return !(p->flags & PF_FREEZER_NOSIG);
 }
 
 /**
  *      freeze_task - send a freeze request to given task
  *      @p: task to send the request to
- *      @with_mm_only: if set, the request will only be sent if the task has its
- *              own mm
- *      Return value: 0, if @with_mm_only is set and the task has no mm of its
- *              own or the task is frozen, 1, otherwise
+ *      @sig_only: if set, the request will only be sent if the task has the
+ *              PF_FREEZER_NOSIG flag unset
+ *      Return value: 'false', if @sig_only is set and the task has
+ *              PF_FREEZER_NOSIG set or the task is frozen, 'true', otherwise
  *
- *      The freeze request is sent by seting the tasks's TIF_FREEZE flag and
+ *      The freeze request is sent by setting the tasks's TIF_FREEZE flag and
  *      either sending a fake signal to it or waking it up, depending on whether
- *      or not it has its own mm (ie. it is a user land task).  If @with_mm_only
- *      is set and the task has no mm of its own (ie. it is a kernel thread),
- *      its TIF_FREEZE flag should not be set.
- *
- *      The task_lock() is necessary to prevent races with exit_mm() or
- *      use_mm()/unuse_mm() from occuring.
+ *      or not it has PF_FREEZER_NOSIG set.  If @sig_only is set and the task
+ *      has PF_FREEZER_NOSIG set (ie. it is a typical kernel thread), its
+ *      TIF_FREEZE flag will not be set.
  */
-static int freeze_task(struct task_struct *p, int with_mm_only)
+static bool freeze_task(struct task_struct *p, bool sig_only)
 {
-        int ret = 1;
+        /*
+         * We first check if the task is freezing and next if it has already
+         * been frozen to avoid the race with frozen_process() which first marks
+         * the task as frozen and next clears its TIF_FREEZE.
+         */
+        if (!freezing(p)) {
+                rmb();
+                if (frozen(p))
+                        return false;
 
-        task_lock(p);
-        if (freezing(p)) {
-                if (has_mm(p)) {
-                        if (!signal_pending(p))
-                                fake_signal_wake_up(p);
-                } else {
-                        if (with_mm_only)
-                                ret = 0;
-                        else
-                                wake_up_state(p, TASK_INTERRUPTIBLE);
-                }
+                if (!sig_only || should_send_signal(p))
+                        set_freeze_flag(p);
+                else
+                        return false;
+        }
+
+        if (should_send_signal(p)) {
+                if (!signal_pending(p))
+                        fake_signal_wake_up(p);
+        } else if (sig_only) {
+                return false;
         } else {
-                rmb();
-                if (frozen(p)) {
-                        ret = 0;
-                } else {
-                        if (has_mm(p)) {
-                                set_freeze_flag(p);
-                                fake_signal_wake_up(p);
-                        } else {
-                                if (with_mm_only) {
-                                        ret = 0;
-                                } else {
-                                        set_freeze_flag(p);
-                                        wake_up_state(p, TASK_INTERRUPTIBLE);
-                                }
-                        }
-                }
+                wake_up_state(p, TASK_INTERRUPTIBLE);
         }
-        task_unlock(p);
-        return ret;
+
+        return true;
 }
 
 static void cancel_freezing(struct task_struct *p)
@@ -156,7 +143,7 @@ static void cancel_freezing(struct task_struct *p)
         }
 }
 
-static int try_to_freeze_tasks(int freeze_user_space)
+static int try_to_freeze_tasks(bool sig_only)
 {
         struct task_struct *g, *p;
         unsigned long end_time;
@@ -175,7 +162,7 @@ static int try_to_freeze_tasks(int freeze_user_space)
                         if (frozen(p) || !freezeable(p))
                                 continue;
 
-                        if (!freeze_task(p, freeze_user_space))
+                        if (!freeze_task(p, sig_only))
                                 continue;
 
                         /*
@@ -235,13 +222,13 @@ int freeze_processes(void)
         int error;
 
         printk("Freezing user space processes ... ");
-        error = try_to_freeze_tasks(FREEZER_USER_SPACE);
+        error = try_to_freeze_tasks(true);
         if (error)
                 goto Exit;
         printk("done.\n");
 
         printk("Freezing remaining freezable tasks ... ");
-        error = try_to_freeze_tasks(FREEZER_KERNEL_THREADS);
+        error = try_to_freeze_tasks(false);
         if (error)
                 goto Exit;
         printk("done.");
@@ -251,7 +238,7 @@ int freeze_processes(void)
         return error;
 }
 
-static void thaw_tasks(int thaw_user_space)
+static void thaw_tasks(bool nosig_only)
 {
         struct task_struct *g, *p;
 
@@ -260,7 +247,7 @@ static void thaw_tasks(int thaw_user_space)
                 if (!freezeable(p))
                         continue;
 
-                if (!p->mm == thaw_user_space)
+                if (nosig_only && should_send_signal(p))
                         continue;
 
                 thaw_process(p);
@@ -271,8 +258,8 @@ static void thaw_tasks(int thaw_user_space)
 void thaw_processes(void)
 {
         printk("Restarting tasks ... ");
-        thaw_tasks(FREEZER_KERNEL_THREADS);
-        thaw_tasks(FREEZER_USER_SPACE);
+        thaw_tasks(true);
+        thaw_tasks(false);
         schedule();
         printk("done.\n");
 }
diff --git a/kernel/power/user.c b/kernel/power/user.c
index f5512cb3aa86..a6332a313262 100644
--- a/kernel/power/user.c
+++ b/kernel/power/user.c
@@ -23,6 +23,7 @@
 #include <linux/console.h>
 #include <linux/cpu.h>
 #include <linux/freezer.h>
+#include <linux/smp_lock.h>
 
 #include <asm/uaccess.h>
 
@@ -69,16 +70,22 @@ static int snapshot_open(struct inode *inode, struct file *filp)
         struct snapshot_data *data;
         int error;
 
-        if (!atomic_add_unless(&snapshot_device_available, -1, 0))
-                return -EBUSY;
+        mutex_lock(&pm_mutex);
+
+        if (!atomic_add_unless(&snapshot_device_available, -1, 0)) {
+                error = -EBUSY;
+                goto Unlock;
+        }
 
         if ((filp->f_flags & O_ACCMODE) == O_RDWR) {
                 atomic_inc(&snapshot_device_available);
-                return -ENOSYS;
+                error = -ENOSYS;
+                goto Unlock;
         }
         if(create_basic_memory_bitmaps()) {
                 atomic_inc(&snapshot_device_available);
-                return -ENOMEM;
+                error = -ENOMEM;
+                goto Unlock;
         }
         nonseekable_open(inode, filp);
         data = &snapshot_state;
@@ -98,33 +105,36 @@ static int snapshot_open(struct inode *inode, struct file *filp)
                 if (error)
                         pm_notifier_call_chain(PM_POST_HIBERNATION);
         }
-        if (error) {
+        if (error)
                 atomic_inc(&snapshot_device_available);
-                return error;
-        }
         data->frozen = 0;
         data->ready = 0;
         data->platform_support = 0;
 
-        return 0;
+ Unlock:
+        mutex_unlock(&pm_mutex);
+
+        return error;
 }
 
 static int snapshot_release(struct inode *inode, struct file *filp)
 {
         struct snapshot_data *data;
 
+        mutex_lock(&pm_mutex);
+
         swsusp_free();
         free_basic_memory_bitmaps();
         data = filp->private_data;
         free_all_swap_pages(data->swap);
-        if (data->frozen) {
-                mutex_lock(&pm_mutex);
+        if (data->frozen)
                 thaw_processes();
-                mutex_unlock(&pm_mutex);
-        }
         pm_notifier_call_chain(data->mode == O_WRONLY ?
                         PM_POST_HIBERNATION : PM_POST_RESTORE);
         atomic_inc(&snapshot_device_available);
+
+        mutex_unlock(&pm_mutex);
+
         return 0;
 }
 
@@ -134,9 +144,13 @@ static ssize_t snapshot_read(struct file *filp, char __user *buf,
         struct snapshot_data *data;
         ssize_t res;
 
+        mutex_lock(&pm_mutex);
+
         data = filp->private_data;
-        if (!data->ready)
-                return -ENODATA;
+        if (!data->ready) {
+                res = -ENODATA;
+                goto Unlock;
+        }
         res = snapshot_read_next(&data->handle, count);
         if (res > 0) {
                 if (copy_to_user(buf, data_of(data->handle), res))
@@ -144,6 +158,10 @@ static ssize_t snapshot_read(struct file *filp, char __user *buf,
                 else
                         *offp = data->handle.offset;
         }
+
+ Unlock:
+        mutex_unlock(&pm_mutex);
+
         return res;
 }
 
@@ -153,6 +171,8 @@ static ssize_t snapshot_write(struct file *filp, const char __user *buf,
         struct snapshot_data *data;
         ssize_t res;
 
+        mutex_lock(&pm_mutex);
+
         data = filp->private_data;
         res = snapshot_write_next(&data->handle, count);
         if (res > 0) {
@@ -161,11 +181,14 @@ static ssize_t snapshot_write(struct file *filp, const char __user *buf,
                 else
                         *offp = data->handle.offset;
         }
+
+        mutex_unlock(&pm_mutex);
+
         return res;
 }
 
-static int snapshot_ioctl(struct inode *inode, struct file *filp,
-                          unsigned int cmd, unsigned long arg)
+static long snapshot_ioctl(struct file *filp, unsigned int cmd,
+                                                        unsigned long arg)
 {
         int error = 0;
         struct snapshot_data *data;
@@ -179,6 +202,9 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp,
         if (!capable(CAP_SYS_ADMIN))
                 return -EPERM;
 
+        if (!mutex_trylock(&pm_mutex))
+                return -EBUSY;
+
         data = filp->private_data;
 
         switch (cmd) {
@@ -186,7 +212,6 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp,
         case SNAPSHOT_FREEZE:
                 if (data->frozen)
                         break;
-                mutex_lock(&pm_mutex);
                 printk("Syncing filesystems ... ");
                 sys_sync();
                 printk("done.\n");
@@ -194,7 +219,6 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp,
                 error = freeze_processes();
                 if (error)
                         thaw_processes();
-                mutex_unlock(&pm_mutex);
                 if (!error)
                         data->frozen = 1;
                 break;
@@ -202,9 +226,7 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp,
         case SNAPSHOT_UNFREEZE:
                 if (!data->frozen || data->ready)
                         break;
-                mutex_lock(&pm_mutex);
                 thaw_processes();
-                mutex_unlock(&pm_mutex);
                 data->frozen = 0;
                 break;
 
@@ -307,16 +329,11 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp,
                         error = -EPERM;
                         break;
                 }
-                if (!mutex_trylock(&pm_mutex)) {
-                        error = -EBUSY;
-                        break;
-                }
                 /*
                  * Tasks are frozen and the notifiers have been called with
                  * PM_HIBERNATION_PREPARE
                  */
                 error = suspend_devices_and_enter(PM_SUSPEND_MEM);
-                mutex_unlock(&pm_mutex);
                 break;
 
         case SNAPSHOT_PLATFORM_SUPPORT:
@@ -390,6 +407,8 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp,
 
         }
 
+        mutex_unlock(&pm_mutex);
+
         return error;
 }
 
@@ -399,7 +418,7 @@ static const struct file_operations snapshot_fops = {
         .read = snapshot_read,
         .write = snapshot_write,
         .llseek = no_llseek,
-        .ioctl = snapshot_ioctl,
+        .unlocked_ioctl = snapshot_ioctl,
 };
 
 static struct miscdevice snapshot_device = {
diff --git a/kernel/printk.c b/kernel/printk.c
index 8fb01c32aa3b..07ad9e7f7a66 100644
--- a/kernel/printk.c
+++ b/kernel/printk.c
@@ -38,7 +38,7 @@
 /*
  * Architectures can override it:
  */
-void __attribute__((weak)) early_printk(const char *fmt, ...)
+void asmlinkage __attribute__((weak)) early_printk(const char *fmt, ...)
 {
 }
 
@@ -75,6 +75,8 @@ EXPORT_SYMBOL(oops_in_progress);
 static DECLARE_MUTEX(console_sem);
 static DECLARE_MUTEX(secondary_console_sem);
 struct console *console_drivers;
+EXPORT_SYMBOL_GPL(console_drivers);
+
 /*
  * This is used for debugging the mess that is the VT code by
  * keeping track if we have the console semaphore held. It's
@@ -121,6 +123,8 @@ struct console_cmdline
 static struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES];
 static int selected_console = -1;
 static int preferred_console = -1;
+int console_set_on_cmdline;
+EXPORT_SYMBOL(console_set_on_cmdline);
 
 /* Flag: console code may call schedule() */
 static int console_may_schedule;
@@ -231,7 +235,7 @@ static inline void boot_delay_msec(void)
 /*
  * Return the number of unread characters in the log buffer.
  */
-int log_buf_get_len(void)
+static int log_buf_get_len(void)
 {
         return logged_chars;
 }
@@ -268,19 +272,6 @@ int log_buf_copy(char *dest, int idx, int len)
 }
 
 /*
- * Extract a single character from the log buffer.
- */
-int log_buf_read(int idx)
-{
-        char ret;
-
-        if (log_buf_copy(&ret, idx, 1) == 1)
-                return ret;
-        else
-                return -1;
-}
-
-/*
  * Commands to do_syslog:
  *
  *      0 -- Close the log.  Currently a NOP.
@@ -665,18 +656,17 @@ static int acquire_console_semaphore_for_printk(unsigned int cpu)
         spin_unlock(&logbuf_lock);
         return retval;
 }
-
-const char printk_recursion_bug_msg [] =
-                        KERN_CRIT "BUG: recent printk recursion!\n";
-static int printk_recursion_bug;
+static const char recursion_bug_msg [] =
+                KERN_CRIT "BUG: recent printk recursion!\n";
+static int recursion_bug;
+        static int new_text_line = 1;
+static char printk_buf[1024];
 
 asmlinkage int vprintk(const char *fmt, va_list args)
 {
-        static int log_level_unknown = 1;
-        static char printk_buf[1024];
-
-        unsigned long flags;
         int printed_len = 0;
+        int current_log_level = default_message_loglevel;
+        unsigned long flags;
         int this_cpu;
         char *p;
 
@@ -699,7 +689,7 @@ asmlinkage int vprintk(const char *fmt, va_list args)
                  * it can be printed at the next appropriate moment:
                  */
                 if (!oops_in_progress) {
-                        printk_recursion_bug = 1;
+                        recursion_bug = 1;
                         goto out_restore_irqs;
                 }
                 zap_locks();
@@ -709,70 +699,62 @@ asmlinkage int vprintk(const char *fmt, va_list args)
         spin_lock(&logbuf_lock);
         printk_cpu = this_cpu;
 
-        if (printk_recursion_bug) {
-                printk_recursion_bug = 0;
-                strcpy(printk_buf, printk_recursion_bug_msg);
-                printed_len = sizeof(printk_recursion_bug_msg);
+        if (recursion_bug) {
+                recursion_bug = 0;
+                strcpy(printk_buf, recursion_bug_msg);
+                printed_len = sizeof(recursion_bug_msg);
         }
         /* Emit the output into the temporary buffer */
         printed_len += vscnprintf(printk_buf + printed_len,
                                   sizeof(printk_buf) - printed_len, fmt, args);
 
+
         /*
          * Copy the output into log_buf.  If the caller didn't provide
          * appropriate log level tags, we insert them here
          */
         for (p = printk_buf; *p; p++) {
-                if (log_level_unknown) {
-                        /* log_level_unknown signals the start of a new line */
+                if (new_text_line) {
+                        /* If a token, set current_log_level and skip over */
+                        if (p[0] == '<' && p[1] >= '0' && p[1] <= '7' &&
+                            p[2] == '>') {
+                                current_log_level = p[1] - '0';
+                                p += 3;
+                                printed_len -= 3;
+                        }
+
+                        /* Always output the token */
+                        emit_log_char('<');
+                        emit_log_char(current_log_level + '0');
+                        emit_log_char('>');
+                        printed_len += 3;
+                        new_text_line = 0;
+
                         if (printk_time) {
-                                int loglev_char;
+                                /* Follow the token with the time */
                                 char tbuf[50], *tp;
                                 unsigned tlen;
                                 unsigned long long t;
                                 unsigned long nanosec_rem;
 
-                                /*
-                                 * force the log level token to be
-                                 * before the time output.
-                                 */
-                                if (p[0] == '<' && p[1] >='0' &&
-                                   p[1] <= '7' && p[2] == '>') {
-                                        loglev_char = p[1];
-                                        p += 3;
-                                        printed_len -= 3;
-                                } else {
-                                        loglev_char = default_message_loglevel
-                                                + '0';
-                                }
                                 t = cpu_clock(printk_cpu);
                                 nanosec_rem = do_div(t, 1000000000);
-                                tlen = sprintf(tbuf,
-                                                "<%c>[%5lu.%06lu] ",
-                                                loglev_char,
-                                                (unsigned long)t,
-                                                nanosec_rem/1000);
+                                tlen = sprintf(tbuf, "[%5lu.%06lu] ",
+                                                (unsigned long) t,
+                                                nanosec_rem / 1000);
 
                                 for (tp = tbuf; tp < tbuf + tlen; tp++)
                                         emit_log_char(*tp);
                                 printed_len += tlen;
-                        } else {
-                                if (p[0] != '<' || p[1] < '0' ||
-                                   p[1] > '7' || p[2] != '>') {
-                                        emit_log_char('<');
-                                        emit_log_char(default_message_loglevel
-                                                + '0');
-                                        emit_log_char('>');
-                                        printed_len += 3;
-                                }
                         }
-                        log_level_unknown = 0;
+
                         if (!*p)
                                 break;
                 }
+
                 emit_log_char(*p);
                 if (*p == '\n')
-                        log_level_unknown = 1;
+                        new_text_line = 1;
         }
 
         /*
@@ -890,6 +872,7 @@ static int __init console_setup(char *str)
         *s = 0;
 
         __add_preferred_console(buf, idx, options, brl_options);
+        console_set_on_cmdline = 1;
         return 1;
 }
 __setup("console=", console_setup);
@@ -1041,7 +1024,9 @@ void release_console_sem(void)
                 _log_end = log_end;
                 con_start = log_end;            /* Flush */
                 spin_unlock(&logbuf_lock);
+                stop_critical_timings();        /* don't trace print latency */
                 call_console_drivers(_con_start, _log_end);
+                start_critical_timings();
                 local_irq_restore(flags);
         }
         console_locked = 0;
@@ -1172,8 +1157,11 @@ void register_console(struct console *console)
                         console->index = 0;
                 if (console->setup == NULL ||
                     console->setup(console, NULL) == 0) {
-                        console->flags |= CON_ENABLED | CON_CONSDEV;
-                        preferred_console = 0;
+                        console->flags |= CON_ENABLED;
+                        if (console->device) {
+                                console->flags |= CON_CONSDEV;
+                                preferred_console = 0;
+                        }
                 }
         }
 
diff --git a/kernel/profile.c b/kernel/profile.c
index ae7ead82cbc9..58926411eb2a 100644
--- a/kernel/profile.c
+++ b/kernel/profile.c
@@ -252,7 +252,7 @@ static void profile_flip_buffers(void)
         mutex_lock(&profile_flip_mutex);
         j = per_cpu(cpu_profile_flip, get_cpu());
         put_cpu();
-        on_each_cpu(__profile_flip_buffers, NULL, 0, 1);
+        on_each_cpu(__profile_flip_buffers, NULL, 1);
         for_each_online_cpu(cpu) {
                 struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[j];
                 for (i = 0; i < NR_PROFILE_HIT; ++i) {
@@ -275,7 +275,7 @@ static void profile_discard_flip_buffers(void)
         mutex_lock(&profile_flip_mutex);
         i = per_cpu(cpu_profile_flip, get_cpu());
         put_cpu();
-        on_each_cpu(__profile_flip_buffers, NULL, 0, 1);
+        on_each_cpu(__profile_flip_buffers, NULL, 1);
         for_each_online_cpu(cpu) {
                 struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[i];
                 memset(hits, 0, NR_PROFILE_HIT*sizeof(struct profile_hit));
@@ -558,7 +558,7 @@ static int __init create_hash_tables(void)
 out_cleanup:
         prof_on = 0;
         smp_mb();
-        on_each_cpu(profile_nop, NULL, 0, 1);
+        on_each_cpu(profile_nop, NULL, 1);
         for_each_online_cpu(cpu) {
                 struct page *page;
 
diff --git a/kernel/ptrace.c b/kernel/ptrace.c
index 6c19e94fd0a5..e337390fce01 100644
--- a/kernel/ptrace.c
+++ b/kernel/ptrace.c
@@ -121,7 +121,7 @@ int ptrace_check_attach(struct task_struct *child, int kill)
         return ret;
 }
 
-int __ptrace_may_attach(struct task_struct *task)
+int __ptrace_may_access(struct task_struct *task, unsigned int mode)
 {
         /* May we inspect the given task?
          * This check is used both for attaching with ptrace
@@ -148,16 +148,16 @@ int __ptrace_may_attach(struct task_struct *task)
         if (!dumpable && !capable(CAP_SYS_PTRACE))
                 return -EPERM;
 
-        return security_ptrace(current, task);
+        return security_ptrace(current, task, mode);
 }
 
-int ptrace_may_attach(struct task_struct *task)
+bool ptrace_may_access(struct task_struct *task, unsigned int mode)
 {
         int err;
         task_lock(task);
-        err = __ptrace_may_attach(task);
+        err = __ptrace_may_access(task, mode);
         task_unlock(task);
-        return !err;
+        return (!err ? true : false);
 }
 
 int ptrace_attach(struct task_struct *task)
@@ -195,7 +195,7 @@ repeat:
         /* the same process cannot be attached many times */
         if (task->ptrace & PT_PTRACED)
                 goto bad;
-        retval = __ptrace_may_attach(task);
+        retval = __ptrace_may_access(task, PTRACE_MODE_ATTACH);
         if (retval)
                 goto bad;
 
@@ -494,7 +494,8 @@ int ptrace_traceme(void)
          */
         task_lock(current);
         if (!(current->ptrace & PT_PTRACED)) {
-                ret = security_ptrace(current->parent, current);
+                ret = security_ptrace(current->parent, current,
+                                      PTRACE_MODE_ATTACH);
                 /*
                  * Set the ptrace bit in the process ptrace flags.
                  */
diff --git a/kernel/rcuclassic.c b/kernel/rcuclassic.c
index f4ffbd0f306f..16eeeaa9d618 100644
--- a/kernel/rcuclassic.c
+++ b/kernel/rcuclassic.c
@@ -89,8 +89,22 @@ static void force_quiescent_state(struct rcu_data *rdp,
                 /*
                  * Don't send IPI to itself. With irqs disabled,
                  * rdp->cpu is the current cpu.
+                 *
+                 * cpu_online_map is updated by the _cpu_down()
+                 * using stop_machine_run(). Since we're in irqs disabled
+                 * section, stop_machine_run() is not exectuting, hence
+                 * the cpu_online_map is stable.
+                 *
+                 * However,  a cpu might have been offlined _just_ before
+                 * we disabled irqs while entering here.
+                 * And rcu subsystem might not yet have handled the CPU_DEAD
+                 * notification, leading to the offlined cpu's bit
+                 * being set in the rcp->cpumask.
+                 *
+                 * Hence cpumask = (rcp->cpumask & cpu_online_map) to prevent
+                 * sending smp_reschedule() to an offlined CPU.
                  */
-                cpumask = rcp->cpumask;
+                cpus_and(cpumask, rcp->cpumask, cpu_online_map);
                 cpu_clear(rdp->cpu, cpumask);
                 for_each_cpu_mask(cpu, cpumask)
                         smp_send_reschedule(cpu);
@@ -373,6 +387,10 @@ static void __rcu_offline_cpu(struct rcu_data *this_rdp,
         rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail);
         rcu_move_batch(this_rdp, rdp->curlist, rdp->curtail);
         rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail);
+
+        local_irq_disable();
+        this_rdp->qlen += rdp->qlen;
+        local_irq_enable();
 }
 
 static void rcu_offline_cpu(int cpu)
@@ -502,10 +520,38 @@ void rcu_check_callbacks(int cpu, int user)
         if (user ||
             (idle_cpu(cpu) && !in_softirq() &&
                                 hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
+
+                /*
+                 * Get here if this CPU took its interrupt from user
+                 * mode or from the idle loop, and if this is not a
+                 * nested interrupt.  In this case, the CPU is in
+                 * a quiescent state, so count it.
+                 *
+                 * Also do a memory barrier.  This is needed to handle
+                 * the case where writes from a preempt-disable section
+                 * of code get reordered into schedule() by this CPU's
+                 * write buffer.  The memory barrier makes sure that
+                 * the rcu_qsctr_inc() and rcu_bh_qsctr_inc() are see
+                 * by other CPUs to happen after any such write.
+                 */
+
+                smp_mb();  /* See above block comment. */
                 rcu_qsctr_inc(cpu);
                 rcu_bh_qsctr_inc(cpu);
-        } else if (!in_softirq())
+
+        } else if (!in_softirq()) {
+
+                /*
+                 * Get here if this CPU did not take its interrupt from
+                 * softirq, in other words, if it is not interrupting
+                 * a rcu_bh read-side critical section.  This is an _bh
+                 * critical section, so count it.  The memory barrier
+                 * is needed for the same reason as is the above one.
+                 */
+
+                smp_mb();  /* See above block comment. */
                 rcu_bh_qsctr_inc(cpu);
+        }
         raise_rcu_softirq();
 }
 
@@ -529,7 +575,7 @@ static void __cpuinit rcu_online_cpu(int cpu)
 
         rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp);
         rcu_init_percpu_data(cpu, &rcu_bh_ctrlblk, bh_rdp);
-        open_softirq(RCU_SOFTIRQ, rcu_process_callbacks, NULL);
+        open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
 }
 
 static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index c09605f8d16c..f14f372cf6f5 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -39,16 +39,16 @@
 #include <linux/sched.h>
 #include <asm/atomic.h>
 #include <linux/bitops.h>
-#include <linux/completion.h>
 #include <linux/percpu.h>
 #include <linux/notifier.h>
 #include <linux/cpu.h>
 #include <linux/mutex.h>
 #include <linux/module.h>
 
-struct rcu_synchronize {
-        struct rcu_head head;
-        struct completion completion;
+enum rcu_barrier {
+        RCU_BARRIER_STD,
+        RCU_BARRIER_BH,
+        RCU_BARRIER_SCHED,
 };
 
 static DEFINE_PER_CPU(struct rcu_head, rcu_barrier_head) = {NULL};
@@ -60,7 +60,7 @@ static struct completion rcu_barrier_completion;
  * Awaken the corresponding synchronize_rcu() instance now that a
  * grace period has elapsed.
  */
-static void wakeme_after_rcu(struct rcu_head  *head)
+void wakeme_after_rcu(struct rcu_head  *head)
 {
         struct rcu_synchronize *rcu;
 
@@ -77,17 +77,7 @@ static void wakeme_after_rcu(struct rcu_head  *head)
  * sections are delimited by rcu_read_lock() and rcu_read_unlock(),
  * and may be nested.
  */
-void synchronize_rcu(void)
-{
-        struct rcu_synchronize rcu;
-
-        init_completion(&rcu.completion);
-        /* Will wake me after RCU finished */
-        call_rcu(&rcu.head, wakeme_after_rcu);
-
-        /* Wait for it */
-        wait_for_completion(&rcu.completion);
-}
+synchronize_rcu_xxx(synchronize_rcu, call_rcu)
 EXPORT_SYMBOL_GPL(synchronize_rcu);
 
 static void rcu_barrier_callback(struct rcu_head *notused)
@@ -99,19 +89,30 @@ static void rcu_barrier_callback(struct rcu_head *notused)
 /*
  * Called with preemption disabled, and from cross-cpu IRQ context.
  */
-static void rcu_barrier_func(void *notused)
+static void rcu_barrier_func(void *type)
 {
         int cpu = smp_processor_id();
         struct rcu_head *head = &per_cpu(rcu_barrier_head, cpu);
 
         atomic_inc(&rcu_barrier_cpu_count);
-        call_rcu(head, rcu_barrier_callback);
+        switch ((enum rcu_barrier)type) {
+        case RCU_BARRIER_STD:
+                call_rcu(head, rcu_barrier_callback);
+                break;
+        case RCU_BARRIER_BH:
+                call_rcu_bh(head, rcu_barrier_callback);
+                break;
+        case RCU_BARRIER_SCHED:
+                call_rcu_sched(head, rcu_barrier_callback);
+                break;
+        }
 }
 
-/**
- * rcu_barrier - Wait until all the in-flight RCUs are complete.
+/*
+ * Orchestrate the specified type of RCU barrier, waiting for all
+ * RCU callbacks of the specified type to complete.
  */
-void rcu_barrier(void)
+static void _rcu_barrier(enum rcu_barrier type)
 {
         BUG_ON(in_interrupt());
         /* Take cpucontrol mutex to protect against CPU hotplug */
@@ -127,13 +128,39 @@ void rcu_barrier(void)
          * until all the callbacks are queued.
          */
         rcu_read_lock();
-        on_each_cpu(rcu_barrier_func, NULL, 0, 1);
+        on_each_cpu(rcu_barrier_func, (void *)type, 1);
         rcu_read_unlock();
         wait_for_completion(&rcu_barrier_completion);
         mutex_unlock(&rcu_barrier_mutex);
 }
+
+/**
+ * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
+ */
+void rcu_barrier(void)
+{
+        _rcu_barrier(RCU_BARRIER_STD);
+}
 EXPORT_SYMBOL_GPL(rcu_barrier);
 
+/**
+ * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete.
+ */
+void rcu_barrier_bh(void)
+{
+        _rcu_barrier(RCU_BARRIER_BH);
+}
+EXPORT_SYMBOL_GPL(rcu_barrier_bh);
+
+/**
+ * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks.
+ */
+void rcu_barrier_sched(void)
+{
+        _rcu_barrier(RCU_BARRIER_SCHED);
+}
+EXPORT_SYMBOL_GPL(rcu_barrier_sched);
+
 void __init rcu_init(void)
 {
         __rcu_init();
diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c
index e1cdf196a515..6f62b77d93c4 100644
--- a/kernel/rcupreempt.c
+++ b/kernel/rcupreempt.c
@@ -46,11 +46,11 @@
 #include <asm/atomic.h>
 #include <linux/bitops.h>
 #include <linux/module.h>
+#include <linux/kthread.h>
 #include <linux/completion.h>
 #include <linux/moduleparam.h>
 #include <linux/percpu.h>
 #include <linux/notifier.h>
-#include <linux/rcupdate.h>
 #include <linux/cpu.h>
 #include <linux/random.h>
 #include <linux/delay.h>
@@ -82,14 +82,18 @@ struct rcu_data {
         spinlock_t      lock;           /* Protect rcu_data fields. */
         long            completed;      /* Number of last completed batch. */
         int             waitlistcount;
-        struct tasklet_struct rcu_tasklet;
         struct rcu_head *nextlist;
         struct rcu_head **nexttail;
         struct rcu_head *waitlist[GP_STAGES];
         struct rcu_head **waittail[GP_STAGES];
-        struct rcu_head *donelist;
+        struct rcu_head *donelist;      /* from waitlist & waitschedlist */
         struct rcu_head **donetail;
         long rcu_flipctr[2];
+        struct rcu_head *nextschedlist;
+        struct rcu_head **nextschedtail;
+        struct rcu_head *waitschedlist;
+        struct rcu_head **waitschedtail;
+        int rcu_sched_sleeping;
 #ifdef CONFIG_RCU_TRACE
         struct rcupreempt_trace trace;
 #endif /* #ifdef CONFIG_RCU_TRACE */
@@ -131,11 +135,24 @@ enum rcu_try_flip_states {
         rcu_try_flip_waitmb_state,
 };
 
+/*
+ * States for rcu_ctrlblk.rcu_sched_sleep.
+ */
+
+enum rcu_sched_sleep_states {
+        rcu_sched_not_sleeping, /* Not sleeping, callbacks need GP.  */
+        rcu_sched_sleep_prep,   /* Thinking of sleeping, rechecking. */
+        rcu_sched_sleeping,     /* Sleeping, awaken if GP needed. */
+};
+
 struct rcu_ctrlblk {
         spinlock_t      fliplock;       /* Protect state-machine transitions. */
         long            completed;      /* Number of last completed batch. */
         enum rcu_try_flip_states rcu_try_flip_state; /* The current state of
                                                         the rcu state machine */
+        spinlock_t      schedlock;      /* Protect rcu_sched sleep state. */
+        enum rcu_sched_sleep_states sched_sleep; /* rcu_sched state. */
+        wait_queue_head_t sched_wq;     /* Place for rcu_sched to sleep. */
 };
 
 static DEFINE_PER_CPU(struct rcu_data, rcu_data);
@@ -143,8 +160,12 @@ static struct rcu_ctrlblk rcu_ctrlblk = {
         .fliplock = __SPIN_LOCK_UNLOCKED(rcu_ctrlblk.fliplock),
         .completed = 0,
         .rcu_try_flip_state = rcu_try_flip_idle_state,
+        .schedlock = __SPIN_LOCK_UNLOCKED(rcu_ctrlblk.schedlock),
+        .sched_sleep = rcu_sched_not_sleeping,
+        .sched_wq = __WAIT_QUEUE_HEAD_INITIALIZER(rcu_ctrlblk.sched_wq),
 };
 
+static struct task_struct *rcu_sched_grace_period_task;
 
 #ifdef CONFIG_RCU_TRACE
 static char *rcu_try_flip_state_names[] =
@@ -207,6 +228,8 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(enum rcu_mb_flag_values, rcu_mb_flag)
  */
 #define RCU_TRACE_RDP(f, rdp) RCU_TRACE(f, &((rdp)->trace));
 
+#define RCU_SCHED_BATCH_TIME (HZ / 50)
+
 /*
  * Return the number of RCU batches processed thus far.  Useful
  * for debug and statistics.
@@ -217,8 +240,6 @@ long rcu_batches_completed(void)
 }
 EXPORT_SYMBOL_GPL(rcu_batches_completed);
 
-EXPORT_SYMBOL_GPL(rcu_batches_completed_bh);
-
 void __rcu_read_lock(void)
 {
         int idx;
@@ -413,32 +434,34 @@ static void __rcu_advance_callbacks(struct rcu_data *rdp)
         }
 }
 
-#ifdef CONFIG_NO_HZ
+DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_dyntick_sched, rcu_dyntick_sched) = {
+        .dynticks = 1,
+};
 
-DEFINE_PER_CPU(long, dynticks_progress_counter) = 1;
-static DEFINE_PER_CPU(long, rcu_dyntick_snapshot);
+#ifdef CONFIG_NO_HZ
 static DEFINE_PER_CPU(int, rcu_update_flag);
 
 /**
  * rcu_irq_enter - Called from Hard irq handlers and NMI/SMI.
  *
  * If the CPU was idle with dynamic ticks active, this updates the
- * dynticks_progress_counter to let the RCU handling know that the
+ * rcu_dyntick_sched.dynticks to let the RCU handling know that the
  * CPU is active.
  */
 void rcu_irq_enter(void)
 {
         int cpu = smp_processor_id();
+        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
 
         if (per_cpu(rcu_update_flag, cpu))
                 per_cpu(rcu_update_flag, cpu)++;
 
         /*
          * Only update if we are coming from a stopped ticks mode
-         * (dynticks_progress_counter is even).
+         * (rcu_dyntick_sched.dynticks is even).
          */
         if (!in_interrupt() &&
-            (per_cpu(dynticks_progress_counter, cpu) & 0x1) == 0) {
+            (rdssp->dynticks & 0x1) == 0) {
                 /*
                  * The following might seem like we could have a race
                  * with NMI/SMIs. But this really isn't a problem.
@@ -461,12 +484,12 @@ void rcu_irq_enter(void)
                  * RCU read-side critical sections on this CPU would
                  * have already completed.
                  */
-                per_cpu(dynticks_progress_counter, cpu)++;
+                rdssp->dynticks++;
                 /*
                  * The following memory barrier ensures that any
                  * rcu_read_lock() primitives in the irq handler
                  * are seen by other CPUs to follow the above
-                 * increment to dynticks_progress_counter. This is
+                 * increment to rcu_dyntick_sched.dynticks. This is
                  * required in order for other CPUs to correctly
                  * determine when it is safe to advance the RCU
                  * grace-period state machine.
@@ -474,7 +497,7 @@ void rcu_irq_enter(void)
                 smp_mb(); /* see above block comment. */
                 /*
                  * Since we can't determine the dynamic tick mode from
-                 * the dynticks_progress_counter after this routine,
+                 * the rcu_dyntick_sched.dynticks after this routine,
                  * we use a second flag to acknowledge that we came
                  * from an idle state with ticks stopped.
                  */
@@ -482,7 +505,7 @@ void rcu_irq_enter(void)
                 /*
                  * If we take an NMI/SMI now, they will also increment
                  * the rcu_update_flag, and will not update the
-                 * dynticks_progress_counter on exit. That is for
+                 * rcu_dyntick_sched.dynticks on exit. That is for
                  * this IRQ to do.
                  */
         }
@@ -492,12 +515,13 @@ void rcu_irq_enter(void)
  * rcu_irq_exit - Called from exiting Hard irq context.
  *
  * If the CPU was idle with dynamic ticks active, update the
- * dynticks_progress_counter to put let the RCU handling be
+ * rcu_dyntick_sched.dynticks to put let the RCU handling be
  * aware that the CPU is going back to idle with no ticks.
  */
 void rcu_irq_exit(void)
 {
         int cpu = smp_processor_id();
+        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
 
         /*
          * rcu_update_flag is set if we interrupted the CPU
@@ -505,7 +529,7 @@ void rcu_irq_exit(void)
          * Once this occurs, we keep track of interrupt nesting
          * because a NMI/SMI could also come in, and we still
          * only want the IRQ that started the increment of the
-         * dynticks_progress_counter to be the one that modifies
+         * rcu_dyntick_sched.dynticks to be the one that modifies
          * it on exit.
          */
         if (per_cpu(rcu_update_flag, cpu)) {
@@ -517,28 +541,29 @@ void rcu_irq_exit(void)
 
                 /*
                  * If an NMI/SMI happens now we are still
-                 * protected by the dynticks_progress_counter being odd.
+                 * protected by the rcu_dyntick_sched.dynticks being odd.
                  */
 
                 /*
                  * The following memory barrier ensures that any
                  * rcu_read_unlock() primitives in the irq handler
                  * are seen by other CPUs to preceed the following
-                 * increment to dynticks_progress_counter. This
+                 * increment to rcu_dyntick_sched.dynticks. This
                  * is required in order for other CPUs to determine
                  * when it is safe to advance the RCU grace-period
                  * state machine.
                  */
                 smp_mb(); /* see above block comment. */
-                per_cpu(dynticks_progress_counter, cpu)++;
-                WARN_ON(per_cpu(dynticks_progress_counter, cpu) & 0x1);
+                rdssp->dynticks++;
+                WARN_ON(rdssp->dynticks & 0x1);
         }
 }
 
 static void dyntick_save_progress_counter(int cpu)
 {
-        per_cpu(rcu_dyntick_snapshot, cpu) =
-                per_cpu(dynticks_progress_counter, cpu);
+        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
+
+        rdssp->dynticks_snap = rdssp->dynticks;
 }
 
 static inline int
@@ -546,9 +571,10 @@ rcu_try_flip_waitack_needed(int cpu)
 {
         long curr;
         long snap;
+        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
 
-        curr = per_cpu(dynticks_progress_counter, cpu);
-        snap = per_cpu(rcu_dyntick_snapshot, cpu);
+        curr = rdssp->dynticks;
+        snap = rdssp->dynticks_snap;
         smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
 
         /*
@@ -569,7 +595,7 @@ rcu_try_flip_waitack_needed(int cpu)
          * that this CPU already acknowledged the counter.
          */
 
-        if ((curr - snap) > 2 || (snap & 0x1) == 0)
+        if ((curr - snap) > 2 || (curr & 0x1) == 0)
                 return 0;
 
         /* We need this CPU to explicitly acknowledge the counter flip. */
@@ -582,9 +608,10 @@ rcu_try_flip_waitmb_needed(int cpu)
 {
         long curr;
         long snap;
+        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
 
-        curr = per_cpu(dynticks_progress_counter, cpu);
-        snap = per_cpu(rcu_dyntick_snapshot, cpu);
+        curr = rdssp->dynticks;
+        snap = rdssp->dynticks_snap;
         smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
 
         /*
@@ -611,14 +638,86 @@ rcu_try_flip_waitmb_needed(int cpu)
         return 1;
 }
 
+static void dyntick_save_progress_counter_sched(int cpu)
+{
+        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
+
+        rdssp->sched_dynticks_snap = rdssp->dynticks;
+}
+
+static int rcu_qsctr_inc_needed_dyntick(int cpu)
+{
+        long curr;
+        long snap;
+        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
+
+        curr = rdssp->dynticks;
+        snap = rdssp->sched_dynticks_snap;
+        smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
+
+        /*
+         * If the CPU remained in dynticks mode for the entire time
+         * and didn't take any interrupts, NMIs, SMIs, or whatever,
+         * then it cannot be in the middle of an rcu_read_lock(), so
+         * the next rcu_read_lock() it executes must use the new value
+         * of the counter.  Therefore, this CPU has been in a quiescent
+         * state the entire time, and we don't need to wait for it.
+         */
+
+        if ((curr == snap) && ((curr & 0x1) == 0))
+                return 0;
+
+        /*
+         * If the CPU passed through or entered a dynticks idle phase with
+         * no active irq handlers, then, as above, this CPU has already
+         * passed through a quiescent state.
+         */
+
+        if ((curr - snap) > 2 || (snap & 0x1) == 0)
+                return 0;
+
+        /* We need this CPU to go through a quiescent state. */
+
+        return 1;
+}
+
 #else /* !CONFIG_NO_HZ */
 
-# define dyntick_save_progress_counter(cpu)     do { } while (0)
-# define rcu_try_flip_waitack_needed(cpu)       (1)
-# define rcu_try_flip_waitmb_needed(cpu)        (1)
+# define dyntick_save_progress_counter(cpu)             do { } while (0)
+# define rcu_try_flip_waitack_needed(cpu)               (1)
+# define rcu_try_flip_waitmb_needed(cpu)                (1)
+
+# define dyntick_save_progress_counter_sched(cpu)       do { } while (0)
+# define rcu_qsctr_inc_needed_dyntick(cpu)              (1)
 
 #endif /* CONFIG_NO_HZ */
 
+static void save_qsctr_sched(int cpu)
+{
+        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
+
+        rdssp->sched_qs_snap = rdssp->sched_qs;
+}
+
+static inline int rcu_qsctr_inc_needed(int cpu)
+{
+        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
+
+        /*
+         * If there has been a quiescent state, no more need to wait
+         * on this CPU.
+         */
+
+        if (rdssp->sched_qs != rdssp->sched_qs_snap) {
+                smp_mb(); /* force ordering with cpu entering schedule(). */
+                return 0;
+        }
+
+        /* We need this CPU to go through a quiescent state. */
+
+        return 1;
+}
+
 /*
  * Get here when RCU is idle.  Decide whether we need to
  * move out of idle state, and return non-zero if so.
@@ -821,6 +920,26 @@ void rcu_check_callbacks(int cpu, int user)
         unsigned long flags;
         struct rcu_data *rdp = RCU_DATA_CPU(cpu);
 
+        /*
+         * If this CPU took its interrupt from user mode or from the
+         * idle loop, and this is not a nested interrupt, then
+         * this CPU has to have exited all prior preept-disable
+         * sections of code.  So increment the counter to note this.
+         *
+         * The memory barrier is needed to handle the case where
+         * writes from a preempt-disable section of code get reordered
+         * into schedule() by this CPU's write buffer.  So the memory
+         * barrier makes sure that the rcu_qsctr_inc() is seen by other
+         * CPUs to happen after any such write.
+         */
+
+        if (user ||
+            (idle_cpu(cpu) && !in_softirq() &&
+             hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
+                smp_mb();       /* Guard against aggressive schedule(). */
+                rcu_qsctr_inc(cpu);
+        }
+
         rcu_check_mb(cpu);
         if (rcu_ctrlblk.completed == rdp->completed)
                 rcu_try_flip();
@@ -871,6 +990,8 @@ void rcu_offline_cpu(int cpu)
         struct rcu_head *list = NULL;
         unsigned long flags;
         struct rcu_data *rdp = RCU_DATA_CPU(cpu);
+        struct rcu_head *schedlist = NULL;
+        struct rcu_head **schedtail = &schedlist;
         struct rcu_head **tail = &list;
 
         /*
@@ -884,6 +1005,11 @@ void rcu_offline_cpu(int cpu)
                 rcu_offline_cpu_enqueue(rdp->waitlist[i], rdp->waittail[i],
                                                 list, tail);
         rcu_offline_cpu_enqueue(rdp->nextlist, rdp->nexttail, list, tail);
+        rcu_offline_cpu_enqueue(rdp->waitschedlist, rdp->waitschedtail,
+                                schedlist, schedtail);
+        rcu_offline_cpu_enqueue(rdp->nextschedlist, rdp->nextschedtail,
+                                schedlist, schedtail);
+        rdp->rcu_sched_sleeping = 0;
         spin_unlock_irqrestore(&rdp->lock, flags);
         rdp->waitlistcount = 0;
 
@@ -918,36 +1044,50 @@ void rcu_offline_cpu(int cpu)
          * fix.
          */
 
-        local_irq_save(flags);
+        local_irq_save(flags);  /* disable preempt till we know what lock. */
         rdp = RCU_DATA_ME();
         spin_lock(&rdp->lock);
         *rdp->nexttail = list;
         if (list)
                 rdp->nexttail = tail;
+        *rdp->nextschedtail = schedlist;
+        if (schedlist)
+                rdp->nextschedtail = schedtail;
         spin_unlock_irqrestore(&rdp->lock, flags);
 }
 
-void __devinit rcu_online_cpu(int cpu)
+#else /* #ifdef CONFIG_HOTPLUG_CPU */
+
+void rcu_offline_cpu(int cpu)
+{
+}
+
+#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
+
+void __cpuinit rcu_online_cpu(int cpu)
 {
         unsigned long flags;
+        struct rcu_data *rdp;
 
         spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags);
         cpu_set(cpu, rcu_cpu_online_map);
         spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
-}
-
-#else /* #ifdef CONFIG_HOTPLUG_CPU */
 
-void rcu_offline_cpu(int cpu)
-{
-}
+        /*
+         * The rcu_sched grace-period processing might have bypassed
+         * this CPU, given that it was not in the rcu_cpu_online_map
+         * when the grace-period scan started.  This means that the
+         * grace-period task might sleep.  So make sure that if this
+         * should happen, the first callback posted to this CPU will
+         * wake up the grace-period task if need be.
+         */
 
-void __devinit rcu_online_cpu(int cpu)
-{
+        rdp = RCU_DATA_CPU(cpu);
+        spin_lock_irqsave(&rdp->lock, flags);
+        rdp->rcu_sched_sleeping = 1;
+        spin_unlock_irqrestore(&rdp->lock, flags);
 }
 
-#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
-
 static void rcu_process_callbacks(struct softirq_action *unused)
 {
         unsigned long flags;
@@ -988,31 +1128,196 @@ void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
         *rdp->nexttail = head;
         rdp->nexttail = &head->next;
         RCU_TRACE_RDP(rcupreempt_trace_next_add, rdp);
-        spin_unlock(&rdp->lock);
-        local_irq_restore(flags);
+        spin_unlock_irqrestore(&rdp->lock, flags);
 }
 EXPORT_SYMBOL_GPL(call_rcu);
 
+void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
+{
+        unsigned long flags;
+        struct rcu_data *rdp;
+        int wake_gp = 0;
+
+        head->func = func;
+        head->next = NULL;
+        local_irq_save(flags);
+        rdp = RCU_DATA_ME();
+        spin_lock(&rdp->lock);
+        *rdp->nextschedtail = head;
+        rdp->nextschedtail = &head->next;
+        if (rdp->rcu_sched_sleeping) {
+
+                /* Grace-period processing might be sleeping... */
+
+                rdp->rcu_sched_sleeping = 0;
+                wake_gp = 1;
+        }
+        spin_unlock_irqrestore(&rdp->lock, flags);
+        if (wake_gp) {
+
+                /* Wake up grace-period processing, unless someone beat us. */
+
+                spin_lock_irqsave(&rcu_ctrlblk.schedlock, flags);
+                if (rcu_ctrlblk.sched_sleep != rcu_sched_sleeping)
+                        wake_gp = 0;
+                rcu_ctrlblk.sched_sleep = rcu_sched_not_sleeping;
+                spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
+                if (wake_gp)
+                        wake_up_interruptible(&rcu_ctrlblk.sched_wq);
+        }
+}
+EXPORT_SYMBOL_GPL(call_rcu_sched);
+
 /*
  * Wait until all currently running preempt_disable() code segments
  * (including hardware-irq-disable segments) complete.  Note that
  * in -rt this does -not- necessarily result in all currently executing
  * interrupt -handlers- having completed.
  */
-void __synchronize_sched(void)
+synchronize_rcu_xxx(__synchronize_sched, call_rcu_sched)
+EXPORT_SYMBOL_GPL(__synchronize_sched);
+
+/*
+ * kthread function that manages call_rcu_sched grace periods.
+ */
+static int rcu_sched_grace_period(void *arg)
 {
-        cpumask_t oldmask;
+        int couldsleep;         /* might sleep after current pass. */
+        int couldsleepnext = 0; /* might sleep after next pass. */
         int cpu;
+        unsigned long flags;
+        struct rcu_data *rdp;
+        int ret;
 
-        if (sched_getaffinity(0, &oldmask) < 0)
-                oldmask = cpu_possible_map;
-        for_each_online_cpu(cpu) {
-                sched_setaffinity(0, &cpumask_of_cpu(cpu));
-                schedule();
-        }
-        sched_setaffinity(0, &oldmask);
+        /*
+         * Each pass through the following loop handles one
+         * rcu_sched grace period cycle.
+         */
+        do {
+                /* Save each CPU's current state. */
+
+                for_each_online_cpu(cpu) {
+                        dyntick_save_progress_counter_sched(cpu);
+                        save_qsctr_sched(cpu);
+                }
+
+                /*
+                 * Sleep for about an RCU grace-period's worth to
+                 * allow better batching and to consume less CPU.
+                 */
+                schedule_timeout_interruptible(RCU_SCHED_BATCH_TIME);
+
+                /*
+                 * If there was nothing to do last time, prepare to
+                 * sleep at the end of the current grace period cycle.
+                 */
+                couldsleep = couldsleepnext;
+                couldsleepnext = 1;
+                if (couldsleep) {
+                        spin_lock_irqsave(&rcu_ctrlblk.schedlock, flags);
+                        rcu_ctrlblk.sched_sleep = rcu_sched_sleep_prep;
+                        spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
+                }
+
+                /*
+                 * Wait on each CPU in turn to have either visited
+                 * a quiescent state or been in dynticks-idle mode.
+                 */
+                for_each_online_cpu(cpu) {
+                        while (rcu_qsctr_inc_needed(cpu) &&
+                               rcu_qsctr_inc_needed_dyntick(cpu)) {
+                                /* resched_cpu(cpu); @@@ */
+                                schedule_timeout_interruptible(1);
+                        }
+                }
+
+                /* Advance callbacks for each CPU.  */
+
+                for_each_online_cpu(cpu) {
+
+                        rdp = RCU_DATA_CPU(cpu);
+                        spin_lock_irqsave(&rdp->lock, flags);
+
+                        /*
+                         * We are running on this CPU irq-disabled, so no
+                         * CPU can go offline until we re-enable irqs.
+                         * The current CPU might have already gone
+                         * offline (between the for_each_offline_cpu and
+                         * the spin_lock_irqsave), but in that case all its
+                         * callback lists will be empty, so no harm done.
+                         *
+                         * Advance the callbacks!  We share normal RCU's
+                         * donelist, since callbacks are invoked the
+                         * same way in either case.
+                         */
+                        if (rdp->waitschedlist != NULL) {
+                                *rdp->donetail = rdp->waitschedlist;
+                                rdp->donetail = rdp->waitschedtail;
+
+                                /*
+                                 * Next rcu_check_callbacks() will
+                                 * do the required raise_softirq().
+                                 */
+                        }
+                        if (rdp->nextschedlist != NULL) {
+                                rdp->waitschedlist = rdp->nextschedlist;
+                                rdp->waitschedtail = rdp->nextschedtail;
+                                couldsleep = 0;
+                                couldsleepnext = 0;
+                        } else {
+                                rdp->waitschedlist = NULL;
+                                rdp->waitschedtail = &rdp->waitschedlist;
+                        }
+                        rdp->nextschedlist = NULL;
+                        rdp->nextschedtail = &rdp->nextschedlist;
+
+                        /* Mark sleep intention. */
+
+                        rdp->rcu_sched_sleeping = couldsleep;
+
+                        spin_unlock_irqrestore(&rdp->lock, flags);
+                }
+
+                /* If we saw callbacks on the last scan, go deal with them. */
+
+                if (!couldsleep)
+                        continue;
+
+                /* Attempt to block... */
+
+                spin_lock_irqsave(&rcu_ctrlblk.schedlock, flags);
+                if (rcu_ctrlblk.sched_sleep != rcu_sched_sleep_prep) {
+
+                        /*
+                         * Someone posted a callback after we scanned.
+                         * Go take care of it.
+                         */
+                        spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
+                        couldsleepnext = 0;
+                        continue;
+                }
+
+                /* Block until the next person posts a callback. */
+
+                rcu_ctrlblk.sched_sleep = rcu_sched_sleeping;
+                spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
+                ret = 0;
+                __wait_event_interruptible(rcu_ctrlblk.sched_wq,
+                        rcu_ctrlblk.sched_sleep != rcu_sched_sleeping,
+                        ret);
+
+                /*
+                 * Signals would prevent us from sleeping, and we cannot
+                 * do much with them in any case.  So flush them.
+                 */
+                if (ret)
+                        flush_signals(current);
+                couldsleepnext = 0;
+
+        } while (!kthread_should_stop());
+
+        return (0);
 }
-EXPORT_SYMBOL_GPL(__synchronize_sched);
 
 /*
  * Check to see if any future RCU-related work will need to be done
@@ -1029,7 +1334,9 @@ int rcu_needs_cpu(int cpu)
 
         return (rdp->donelist != NULL ||
                 !!rdp->waitlistcount ||
-                rdp->nextlist != NULL);
+                rdp->nextlist != NULL ||
+                rdp->nextschedlist != NULL ||
+                rdp->waitschedlist != NULL);
 }
 
 int rcu_pending(int cpu)
@@ -1040,7 +1347,9 @@ int rcu_pending(int cpu)
 
         if (rdp->donelist != NULL ||
             !!rdp->waitlistcount ||
-            rdp->nextlist != NULL)
+            rdp->nextlist != NULL ||
+            rdp->nextschedlist != NULL ||
+            rdp->waitschedlist != NULL)
                 return 1;
 
         /* The RCU core needs an acknowledgement from this CPU. */
@@ -1107,6 +1416,11 @@ void __init __rcu_init(void)
                 rdp->donetail = &rdp->donelist;
                 rdp->rcu_flipctr[0] = 0;
                 rdp->rcu_flipctr[1] = 0;
+                rdp->nextschedlist = NULL;
+                rdp->nextschedtail = &rdp->nextschedlist;
+                rdp->waitschedlist = NULL;
+                rdp->waitschedtail = &rdp->waitschedlist;
+                rdp->rcu_sched_sleeping = 0;
         }
         register_cpu_notifier(&rcu_nb);
 
@@ -1125,15 +1439,19 @@ void __init __rcu_init(void)
         for_each_online_cpu(cpu)
                 rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long) cpu);
 
-        open_softirq(RCU_SOFTIRQ, rcu_process_callbacks, NULL);
+        open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
 }
 
 /*
- * Deprecated, use synchronize_rcu() or synchronize_sched() instead.
+ * Late-boot-time RCU initialization that must wait until after scheduler
+ * has been initialized.
  */
-void synchronize_kernel(void)
+void __init rcu_init_sched(void)
 {
-        synchronize_rcu();
+        rcu_sched_grace_period_task = kthread_run(rcu_sched_grace_period,
+                                                  NULL,
+                                                  "rcu_sched_grace_period");
+        WARN_ON(IS_ERR(rcu_sched_grace_period_task));
 }
 
 #ifdef CONFIG_RCU_TRACE
diff --git a/kernel/rcupreempt_trace.c b/kernel/rcupreempt_trace.c
index 49ac4947af24..5edf82c34bbc 100644
--- a/kernel/rcupreempt_trace.c
+++ b/kernel/rcupreempt_trace.c
@@ -38,7 +38,6 @@
 #include <linux/moduleparam.h>
 #include <linux/percpu.h>
 #include <linux/notifier.h>
-#include <linux/rcupdate.h>
 #include <linux/cpu.h>
 #include <linux/mutex.h>
 #include <linux/rcupreempt_trace.h>
diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c
index 33acc424667e..90b5b123f7a1 100644
--- a/kernel/rcutorture.c
+++ b/kernel/rcutorture.c
@@ -57,7 +57,9 @@ static int stat_interval;	/* Interval between stats, in seconds. */
                                 /*  Defaults to "only at end of test". */
 static int verbose;             /* Print more debug info. */
 static int test_no_idle_hz;     /* Test RCU's support for tickless idle CPUs. */
-static int shuffle_interval = 5; /* Interval between shuffles (in sec)*/
+static int shuffle_interval = 3; /* Interval between shuffles (in sec)*/
+static int stutter = 5;         /* Start/stop testing interval (in sec) */
+static int irqreader = 1;       /* RCU readers from irq (timers). */
 static char *torture_type = "rcu"; /* What RCU implementation to torture. */
 
 module_param(nreaders, int, 0444);
@@ -72,6 +74,10 @@ module_param(test_no_idle_hz, bool, 0444);
 MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
 module_param(shuffle_interval, int, 0444);
 MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
+module_param(stutter, int, 0444);
+MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test");
+module_param(irqreader, int, 0444);
+MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers");
 module_param(torture_type, charp, 0444);
 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");
 
@@ -91,6 +97,7 @@ static struct task_struct **fakewriter_tasks;
 static struct task_struct **reader_tasks;
 static struct task_struct *stats_task;
 static struct task_struct *shuffler_task;
+static struct task_struct *stutter_task;
 
 #define RCU_TORTURE_PIPE_LEN 10
 
@@ -117,8 +124,18 @@ static atomic_t n_rcu_torture_alloc_fail;
 static atomic_t n_rcu_torture_free;
 static atomic_t n_rcu_torture_mberror;
 static atomic_t n_rcu_torture_error;
+static long n_rcu_torture_timers = 0;
 static struct list_head rcu_torture_removed;
 
+static int stutter_pause_test = 0;
+
+#if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE)
+#define RCUTORTURE_RUNNABLE_INIT 1
+#else
+#define RCUTORTURE_RUNNABLE_INIT 0
+#endif
+int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;
+
 /*
  * Allocate an element from the rcu_tortures pool.
  */
@@ -179,6 +196,16 @@ rcu_random(struct rcu_random_state *rrsp)
         return swahw32(rrsp->rrs_state);
 }
 
+static void
+rcu_stutter_wait(void)
+{
+        while (stutter_pause_test || !rcutorture_runnable)
+                if (rcutorture_runnable)
+                        schedule_timeout_interruptible(1);
+                else
+                        schedule_timeout_interruptible(round_jiffies_relative(HZ));
+}
+
 /*
  * Operations vector for selecting different types of tests.
  */
@@ -192,7 +219,9 @@ struct rcu_torture_ops {
         int (*completed)(void);
         void (*deferredfree)(struct rcu_torture *p);
         void (*sync)(void);
+        void (*cb_barrier)(void);
         int (*stats)(char *page);
+        int irqcapable;
         char *name;
 };
 static struct rcu_torture_ops *cur_ops = NULL;
@@ -265,7 +294,9 @@ static struct rcu_torture_ops rcu_ops = {
         .completed = rcu_torture_completed,
         .deferredfree = rcu_torture_deferred_free,
         .sync = synchronize_rcu,
+        .cb_barrier = rcu_barrier,
         .stats = NULL,
+        .irqcapable = 1,
         .name = "rcu"
 };
 
@@ -304,7 +335,9 @@ static struct rcu_torture_ops rcu_sync_ops = {
         .completed = rcu_torture_completed,
         .deferredfree = rcu_sync_torture_deferred_free,
         .sync = synchronize_rcu,
+        .cb_barrier = NULL,
         .stats = NULL,
+        .irqcapable = 1,
         .name = "rcu_sync"
 };
 
@@ -364,7 +397,9 @@ static struct rcu_torture_ops rcu_bh_ops = {
         .completed = rcu_bh_torture_completed,
         .deferredfree = rcu_bh_torture_deferred_free,
         .sync = rcu_bh_torture_synchronize,
+        .cb_barrier = rcu_barrier_bh,
         .stats = NULL,
+        .irqcapable = 1,
         .name = "rcu_bh"
 };
 
@@ -377,7 +412,9 @@ static struct rcu_torture_ops rcu_bh_sync_ops = {
         .completed = rcu_bh_torture_completed,
         .deferredfree = rcu_sync_torture_deferred_free,
         .sync = rcu_bh_torture_synchronize,
+        .cb_barrier = NULL,
         .stats = NULL,
+        .irqcapable = 1,
         .name = "rcu_bh_sync"
 };
 
@@ -458,6 +495,7 @@ static struct rcu_torture_ops srcu_ops = {
         .completed = srcu_torture_completed,
         .deferredfree = rcu_sync_torture_deferred_free,
         .sync = srcu_torture_synchronize,
+        .cb_barrier = NULL,
         .stats = srcu_torture_stats,
         .name = "srcu"
 };
@@ -482,6 +520,11 @@ static int sched_torture_completed(void)
         return 0;
 }
 
+static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
+{
+        call_rcu_sched(&p->rtort_rcu, rcu_torture_cb);
+}
+
 static void sched_torture_synchronize(void)
 {
         synchronize_sched();
@@ -494,12 +537,28 @@ static struct rcu_torture_ops sched_ops = {
         .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
         .readunlock = sched_torture_read_unlock,
         .completed = sched_torture_completed,
-        .deferredfree = rcu_sync_torture_deferred_free,
+        .deferredfree = rcu_sched_torture_deferred_free,
         .sync = sched_torture_synchronize,
+        .cb_barrier = rcu_barrier_sched,
         .stats = NULL,
+        .irqcapable = 1,
         .name = "sched"
 };
 
+static struct rcu_torture_ops sched_ops_sync = {
+        .init = rcu_sync_torture_init,
+        .cleanup = NULL,
+        .readlock = sched_torture_read_lock,
+        .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
+        .readunlock = sched_torture_read_unlock,
+        .completed = sched_torture_completed,
+        .deferredfree = rcu_sync_torture_deferred_free,
+        .sync = sched_torture_synchronize,
+        .cb_barrier = NULL,
+        .stats = NULL,
+        .name = "sched_sync"
+};
+
 /*
  * RCU torture writer kthread.  Repeatedly substitutes a new structure
  * for that pointed to by rcu_torture_current, freeing the old structure
@@ -537,6 +596,7 @@ rcu_torture_writer(void *arg)
                 }
                 rcu_torture_current_version++;
                 oldbatch = cur_ops->completed();
+                rcu_stutter_wait();
         } while (!kthread_should_stop() && !fullstop);
         VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
         while (!kthread_should_stop())
@@ -560,6 +620,7 @@ rcu_torture_fakewriter(void *arg)
                 schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10);
                 udelay(rcu_random(&rand) & 0x3ff);
                 cur_ops->sync();
+                rcu_stutter_wait();
         } while (!kthread_should_stop() && !fullstop);
 
         VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping");
@@ -569,6 +630,52 @@ rcu_torture_fakewriter(void *arg)
 }
 
 /*
+ * RCU torture reader from timer handler.  Dereferences rcu_torture_current,
+ * incrementing the corresponding element of the pipeline array.  The
+ * counter in the element should never be greater than 1, otherwise, the
+ * RCU implementation is broken.
+ */
+static void rcu_torture_timer(unsigned long unused)
+{
+        int idx;
+        int completed;
+        static DEFINE_RCU_RANDOM(rand);
+        static DEFINE_SPINLOCK(rand_lock);
+        struct rcu_torture *p;
+        int pipe_count;
+
+        idx = cur_ops->readlock();
+        completed = cur_ops->completed();
+        p = rcu_dereference(rcu_torture_current);
+        if (p == NULL) {
+                /* Leave because rcu_torture_writer is not yet underway */
+                cur_ops->readunlock(idx);
+                return;
+        }
+        if (p->rtort_mbtest == 0)
+                atomic_inc(&n_rcu_torture_mberror);
+        spin_lock(&rand_lock);
+        cur_ops->readdelay(&rand);
+        n_rcu_torture_timers++;
+        spin_unlock(&rand_lock);
+        preempt_disable();
+        pipe_count = p->rtort_pipe_count;
+        if (pipe_count > RCU_TORTURE_PIPE_LEN) {
+                /* Should not happen, but... */
+                pipe_count = RCU_TORTURE_PIPE_LEN;
+        }
+        ++__get_cpu_var(rcu_torture_count)[pipe_count];
+        completed = cur_ops->completed() - completed;
+        if (completed > RCU_TORTURE_PIPE_LEN) {
+                /* Should not happen, but... */
+                completed = RCU_TORTURE_PIPE_LEN;
+        }
+        ++__get_cpu_var(rcu_torture_batch)[completed];
+        preempt_enable();
+        cur_ops->readunlock(idx);
+}
+
+/*
  * RCU torture reader kthread.  Repeatedly dereferences rcu_torture_current,
  * incrementing the corresponding element of the pipeline array.  The
  * counter in the element should never be greater than 1, otherwise, the
@@ -582,11 +689,18 @@ rcu_torture_reader(void *arg)
         DEFINE_RCU_RANDOM(rand);
         struct rcu_torture *p;
         int pipe_count;
+        struct timer_list t;
 
         VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
         set_user_nice(current, 19);
+        if (irqreader && cur_ops->irqcapable)
+                setup_timer_on_stack(&t, rcu_torture_timer, 0);
 
         do {
+                if (irqreader && cur_ops->irqcapable) {
+                        if (!timer_pending(&t))
+                                mod_timer(&t, 1);
+                }
                 idx = cur_ops->readlock();
                 completed = cur_ops->completed();
                 p = rcu_dereference(rcu_torture_current);
@@ -615,8 +729,11 @@ rcu_torture_reader(void *arg)
                 preempt_enable();
                 cur_ops->readunlock(idx);
                 schedule();
+                rcu_stutter_wait();
         } while (!kthread_should_stop() && !fullstop);
         VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
+        if (irqreader && cur_ops->irqcapable)
+                del_timer_sync(&t);
         while (!kthread_should_stop())
                 schedule_timeout_uninterruptible(1);
         return 0;
@@ -647,20 +764,22 @@ rcu_torture_printk(char *page)
         cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG);
         cnt += sprintf(&page[cnt],
                        "rtc: %p ver: %ld tfle: %d rta: %d rtaf: %d rtf: %d "
-                       "rtmbe: %d",
+                       "rtmbe: %d nt: %ld",
                        rcu_torture_current,
                        rcu_torture_current_version,
                        list_empty(&rcu_torture_freelist),
                        atomic_read(&n_rcu_torture_alloc),
                        atomic_read(&n_rcu_torture_alloc_fail),
                        atomic_read(&n_rcu_torture_free),
-                       atomic_read(&n_rcu_torture_mberror));
+                       atomic_read(&n_rcu_torture_mberror),
+                       n_rcu_torture_timers);
         if (atomic_read(&n_rcu_torture_mberror) != 0)
                 cnt += sprintf(&page[cnt], " !!!");
         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
         if (i > 1) {
                 cnt += sprintf(&page[cnt], "!!! ");
                 atomic_inc(&n_rcu_torture_error);
+                WARN_ON_ONCE(1);
         }
         cnt += sprintf(&page[cnt], "Reader Pipe: ");
         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
@@ -785,15 +904,34 @@ rcu_torture_shuffle(void *arg)
         return 0;
 }
 
+/* Cause the rcutorture test to "stutter", starting and stopping all
+ * threads periodically.
+ */
+static int
+rcu_torture_stutter(void *arg)
+{
+        VERBOSE_PRINTK_STRING("rcu_torture_stutter task started");
+        do {
+                schedule_timeout_interruptible(stutter * HZ);
+                stutter_pause_test = 1;
+                if (!kthread_should_stop())
+                        schedule_timeout_interruptible(stutter * HZ);
+                stutter_pause_test = 0;
+        } while (!kthread_should_stop());
+        VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping");
+        return 0;
+}
+
 static inline void
 rcu_torture_print_module_parms(char *tag)
 {
         printk(KERN_ALERT "%s" TORTURE_FLAG
                 "--- %s: nreaders=%d nfakewriters=%d "
                 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
-                "shuffle_interval = %d\n",
+                "shuffle_interval=%d stutter=%d irqreader=%d\n",
                 torture_type, tag, nrealreaders, nfakewriters,
-                stat_interval, verbose, test_no_idle_hz, shuffle_interval);
+                stat_interval, verbose, test_no_idle_hz, shuffle_interval,
+                stutter, irqreader);
 }
 
 static void
@@ -802,6 +940,11 @@ rcu_torture_cleanup(void)
         int i;
 
         fullstop = 1;
+        if (stutter_task) {
+                VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
+                kthread_stop(stutter_task);
+        }
+        stutter_task = NULL;
         if (shuffler_task) {
                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task");
                 kthread_stop(shuffler_task);
@@ -848,7 +991,9 @@ rcu_torture_cleanup(void)
         stats_task = NULL;
 
         /* Wait for all RCU callbacks to fire.  */
-        rcu_barrier();
+
+        if (cur_ops->cb_barrier != NULL)
+                cur_ops->cb_barrier();
 
         rcu_torture_stats_print();  /* -After- the stats thread is stopped! */
 
@@ -868,7 +1013,7 @@ rcu_torture_init(void)
         int firsterr = 0;
         static struct rcu_torture_ops *torture_ops[] =
                 { &rcu_ops, &rcu_sync_ops, &rcu_bh_ops, &rcu_bh_sync_ops,
-                  &srcu_ops, &sched_ops, };
+                  &srcu_ops, &sched_ops, &sched_ops_sync, };
 
         /* Process args and tell the world that the torturer is on the job. */
         for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
@@ -988,6 +1133,19 @@ rcu_torture_init(void)
                         goto unwind;
                 }
         }
+        if (stutter < 0)
+                stutter = 0;
+        if (stutter) {
+                /* Create the stutter thread */
+                stutter_task = kthread_run(rcu_torture_stutter, NULL,
+                                          "rcu_torture_stutter");
+                if (IS_ERR(stutter_task)) {
+                        firsterr = PTR_ERR(stutter_task);
+                        VERBOSE_PRINTK_ERRSTRING("Failed to create stutter");
+                        stutter_task = NULL;
+                        goto unwind;
+                }
+        }
         return 0;
 
 unwind:
diff --git a/kernel/sched.c b/kernel/sched.c
index bfb8ad8ed171..99e6d850ecab 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -70,10 +70,13 @@
 #include <linux/bootmem.h>
 #include <linux/debugfs.h>
 #include <linux/ctype.h>
+#include <linux/ftrace.h>
 
 #include <asm/tlb.h>
 #include <asm/irq_regs.h>
 
+#include "sched_cpupri.h"
+
 /*
  * Convert user-nice values [ -20 ... 0 ... 19 ]
  * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
@@ -289,15 +292,15 @@ struct task_group root_task_group;
 static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
 /* Default task group's cfs_rq on each cpu */
 static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
-#endif
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #ifdef CONFIG_RT_GROUP_SCHED
 static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
 static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
-#endif
-#else
+#endif /* CONFIG_RT_GROUP_SCHED */
+#else /* !CONFIG_FAIR_GROUP_SCHED */
 #define root_task_group init_task_group
-#endif
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 
 /* task_group_lock serializes add/remove of task groups and also changes to
  * a task group's cpu shares.
@@ -307,17 +310,20 @@ static DEFINE_SPINLOCK(task_group_lock);
 #ifdef CONFIG_FAIR_GROUP_SCHED
 #ifdef CONFIG_USER_SCHED
 # define INIT_TASK_GROUP_LOAD   (2*NICE_0_LOAD)
-#else
+#else /* !CONFIG_USER_SCHED */
 # define INIT_TASK_GROUP_LOAD   NICE_0_LOAD
-#endif
+#endif /* CONFIG_USER_SCHED */
 
 /*
- * A weight of 0, 1 or ULONG_MAX can cause arithmetics problems.
+ * A weight of 0 or 1 can cause arithmetics problems.
+ * A weight of a cfs_rq is the sum of weights of which entities
+ * are queued on this cfs_rq, so a weight of a entity should not be
+ * too large, so as the shares value of a task group.
  * (The default weight is 1024 - so there's no practical
  *  limitation from this.)
  */
 #define MIN_SHARES      2
-#define MAX_SHARES      (ULONG_MAX - 1)
+#define MAX_SHARES      (1UL << 18)
 
 static int init_task_group_load = INIT_TASK_GROUP_LOAD;
 #endif
@@ -360,6 +366,10 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
 #else
 
 static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
+static inline struct task_group *task_group(struct task_struct *p)
+{
+        return NULL;
+}
 
 #endif  /* CONFIG_GROUP_SCHED */
 
@@ -370,6 +380,7 @@ struct cfs_rq {
 
         u64 exec_clock;
         u64 min_vruntime;
+        u64 pair_start;
 
         struct rb_root tasks_timeline;
         struct rb_node *rb_leftmost;
@@ -398,6 +409,31 @@ struct cfs_rq {
          */
         struct list_head leaf_cfs_rq_list;
         struct task_group *tg;  /* group that "owns" this runqueue */
+
+#ifdef CONFIG_SMP
+        /*
+         * the part of load.weight contributed by tasks
+         */
+        unsigned long task_weight;
+
+        /*
+         *   h_load = weight * f(tg)
+         *
+         * Where f(tg) is the recursive weight fraction assigned to
+         * this group.
+         */
+        unsigned long h_load;
+
+        /*
+         * this cpu's part of tg->shares
+         */
+        unsigned long shares;
+
+        /*
+         * load.weight at the time we set shares
+         */
+        unsigned long rq_weight;
+#endif
 #endif
 };
 
@@ -449,6 +485,9 @@ struct root_domain {
          */
         cpumask_t rto_mask;
         atomic_t rto_count;
+#ifdef CONFIG_SMP
+        struct cpupri cpupri;
+#endif
 };
 
 /*
@@ -523,6 +562,9 @@ struct rq {
         int push_cpu;
         /* cpu of this runqueue: */
         int cpu;
+        int online;
+
+        unsigned long avg_load_per_task;
 
         struct task_struct *migration_thread;
         struct list_head migration_queue;
@@ -604,6 +646,24 @@ static inline void update_rq_clock(struct rq *rq)
 # define const_debug static const
 #endif
 
+/**
+ * runqueue_is_locked
+ *
+ * Returns true if the current cpu runqueue is locked.
+ * This interface allows printk to be called with the runqueue lock
+ * held and know whether or not it is OK to wake up the klogd.
+ */
+int runqueue_is_locked(void)
+{
+        int cpu = get_cpu();
+        struct rq *rq = cpu_rq(cpu);
+        int ret;
+
+        ret = spin_is_locked(&rq->lock);
+        put_cpu();
+        return ret;
+}
+
 /*
  * Debugging: various feature bits
  */
@@ -746,6 +806,12 @@ late_initcall(sched_init_debug);
 const_debug unsigned int sysctl_sched_nr_migrate = 32;
 
 /*
+ * ratelimit for updating the group shares.
+ * default: 0.5ms
+ */
+const_debug unsigned int sysctl_sched_shares_ratelimit = 500000;
+
+/*
  * period over which we measure -rt task cpu usage in us.
  * default: 1s
  */
@@ -772,82 +838,6 @@ static inline u64 global_rt_runtime(void)
         return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
 }
 
-unsigned long long time_sync_thresh = 100000;
-
-static DEFINE_PER_CPU(unsigned long long, time_offset);
-static DEFINE_PER_CPU(unsigned long long, prev_cpu_time);
-
-/*
- * Global lock which we take every now and then to synchronize
- * the CPUs time. This method is not warp-safe, but it's good
- * enough to synchronize slowly diverging time sources and thus
- * it's good enough for tracing:
- */
-static DEFINE_SPINLOCK(time_sync_lock);
-static unsigned long long prev_global_time;
-
-static unsigned long long __sync_cpu_clock(unsigned long long time, int cpu)
-{
-        /*
-         * We want this inlined, to not get tracer function calls
-         * in this critical section:
-         */
-        spin_acquire(&time_sync_lock.dep_map, 0, 0, _THIS_IP_);
-        __raw_spin_lock(&time_sync_lock.raw_lock);
-
-        if (time < prev_global_time) {
-                per_cpu(time_offset, cpu) += prev_global_time - time;
-                time = prev_global_time;
-        } else {
-                prev_global_time = time;
-        }
-
-        __raw_spin_unlock(&time_sync_lock.raw_lock);
-        spin_release(&time_sync_lock.dep_map, 1, _THIS_IP_);
-
-        return time;
-}
-
-static unsigned long long __cpu_clock(int cpu)
-{
-        unsigned long long now;
-
-        /*
-         * Only call sched_clock() if the scheduler has already been
-         * initialized (some code might call cpu_clock() very early):
-         */
-        if (unlikely(!scheduler_running))
-                return 0;
-
-        now = sched_clock_cpu(cpu);
-
-        return now;
-}
-
-/*
- * For kernel-internal use: high-speed (but slightly incorrect) per-cpu
- * clock constructed from sched_clock():
- */
-unsigned long long cpu_clock(int cpu)
-{
-        unsigned long long prev_cpu_time, time, delta_time;
-        unsigned long flags;
-
-        local_irq_save(flags);
-        prev_cpu_time = per_cpu(prev_cpu_time, cpu);
-        time = __cpu_clock(cpu) + per_cpu(time_offset, cpu);
-        delta_time = time-prev_cpu_time;
-
-        if (unlikely(delta_time > time_sync_thresh)) {
-                time = __sync_cpu_clock(time, cpu);
-                per_cpu(prev_cpu_time, cpu) = time;
-        }
-        local_irq_restore(flags);
-
-        return time;
-}
-EXPORT_SYMBOL_GPL(cpu_clock);
-
 #ifndef prepare_arch_switch
 # define prepare_arch_switch(next)      do { } while (0)
 #endif
@@ -1124,6 +1114,7 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer)
         return HRTIMER_NORESTART;
 }
 
+#ifdef CONFIG_SMP
 static void hotplug_hrtick_disable(int cpu)
 {
         struct rq *rq = cpu_rq(cpu);
@@ -1179,6 +1170,7 @@ static void init_hrtick(void)
 {
         hotcpu_notifier(hotplug_hrtick, 0);
 }
+#endif /* CONFIG_SMP */
 
 static void init_rq_hrtick(struct rq *rq)
 {
@@ -1308,15 +1300,15 @@ void wake_up_idle_cpu(int cpu)
         if (!tsk_is_polling(rq->idle))
                 smp_send_reschedule(cpu);
 }
-#endif
+#endif /* CONFIG_NO_HZ */
 
-#else
+#else /* !CONFIG_SMP */
 static void __resched_task(struct task_struct *p, int tif_bit)
 {
         assert_spin_locked(&task_rq(p)->lock);
         set_tsk_thread_flag(p, tif_bit);
 }
-#endif
+#endif /* CONFIG_SMP */
 
 #if BITS_PER_LONG == 32
 # define WMULT_CONST    (~0UL)
@@ -1331,14 +1323,22 @@ static void __resched_task(struct task_struct *p, int tif_bit)
  */
 #define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
 
+/*
+ * delta *= weight / lw
+ */
 static unsigned long
 calc_delta_mine(unsigned long delta_exec, unsigned long weight,
                 struct load_weight *lw)
 {
         u64 tmp;
 
-        if (!lw->inv_weight)
-                lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)/(lw->weight+1);
+        if (!lw->inv_weight) {
+                if (BITS_PER_LONG > 32 && unlikely(lw->weight >= WMULT_CONST))
+                        lw->inv_weight = 1;
+                else
+                        lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)
+                                / (lw->weight+1);
+        }
 
         tmp = (u64)delta_exec * weight;
         /*
@@ -1353,12 +1353,6 @@ calc_delta_mine(unsigned long delta_exec, unsigned long weight,
         return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
 }
 
-static inline unsigned long
-calc_delta_fair(unsigned long delta_exec, struct load_weight *lw)
-{
-        return calc_delta_mine(delta_exec, NICE_0_LOAD, lw);
-}
-
 static inline void update_load_add(struct load_weight *lw, unsigned long inc)
 {
         lw->weight += inc;
@@ -1469,17 +1463,211 @@ static inline void dec_cpu_load(struct rq *rq, unsigned long load)
 #ifdef CONFIG_SMP
 static unsigned long source_load(int cpu, int type);
 static unsigned long target_load(int cpu, int type);
-static unsigned long cpu_avg_load_per_task(int cpu);
 static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
-#else /* CONFIG_SMP */
+
+static unsigned long cpu_avg_load_per_task(int cpu)
+{
+        struct rq *rq = cpu_rq(cpu);
+
+        if (rq->nr_running)
+                rq->avg_load_per_task = rq->load.weight / rq->nr_running;
+
+        return rq->avg_load_per_task;
+}
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
+
+typedef void (*tg_visitor)(struct task_group *, int, struct sched_domain *);
+
+/*
+ * Iterate the full tree, calling @down when first entering a node and @up when
+ * leaving it for the final time.
+ */
+static void
+walk_tg_tree(tg_visitor down, tg_visitor up, int cpu, struct sched_domain *sd)
 {
+        struct task_group *parent, *child;
+
+        rcu_read_lock();
+        parent = &root_task_group;
+down:
+        (*down)(parent, cpu, sd);
+        list_for_each_entry_rcu(child, &parent->children, siblings) {
+                parent = child;
+                goto down;
+
+up:
+                continue;
+        }
+        (*up)(parent, cpu, sd);
+
+        child = parent;
+        parent = parent->parent;
+        if (parent)
+                goto up;
+        rcu_read_unlock();
 }
+
+static void __set_se_shares(struct sched_entity *se, unsigned long shares);
+
+/*
+ * Calculate and set the cpu's group shares.
+ */
+static void
+__update_group_shares_cpu(struct task_group *tg, int cpu,
+                          unsigned long sd_shares, unsigned long sd_rq_weight)
+{
+        int boost = 0;
+        unsigned long shares;
+        unsigned long rq_weight;
+
+        if (!tg->se[cpu])
+                return;
+
+        rq_weight = tg->cfs_rq[cpu]->load.weight;
+
+        /*
+         * If there are currently no tasks on the cpu pretend there is one of
+         * average load so that when a new task gets to run here it will not
+         * get delayed by group starvation.
+         */
+        if (!rq_weight) {
+                boost = 1;
+                rq_weight = NICE_0_LOAD;
+        }
+
+        if (unlikely(rq_weight > sd_rq_weight))
+                rq_weight = sd_rq_weight;
+
+        /*
+         *           \Sum shares * rq_weight
+         * shares =  -----------------------
+         *               \Sum rq_weight
+         *
+         */
+        shares = (sd_shares * rq_weight) / (sd_rq_weight + 1);
+
+        /*
+         * record the actual number of shares, not the boosted amount.
+         */
+        tg->cfs_rq[cpu]->shares = boost ? 0 : shares;
+        tg->cfs_rq[cpu]->rq_weight = rq_weight;
+
+        if (shares < MIN_SHARES)
+                shares = MIN_SHARES;
+        else if (shares > MAX_SHARES)
+                shares = MAX_SHARES;
+
+        __set_se_shares(tg->se[cpu], shares);
+}
+
+/*
+ * Re-compute the task group their per cpu shares over the given domain.
+ * This needs to be done in a bottom-up fashion because the rq weight of a
+ * parent group depends on the shares of its child groups.
+ */
+static void
+tg_shares_up(struct task_group *tg, int cpu, struct sched_domain *sd)
+{
+        unsigned long rq_weight = 0;
+        unsigned long shares = 0;
+        int i;
+
+        for_each_cpu_mask(i, sd->span) {
+                rq_weight += tg->cfs_rq[i]->load.weight;
+                shares += tg->cfs_rq[i]->shares;
+        }
+
+        if ((!shares && rq_weight) || shares > tg->shares)
+                shares = tg->shares;
+
+        if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
+                shares = tg->shares;
+
+        if (!rq_weight)
+                rq_weight = cpus_weight(sd->span) * NICE_0_LOAD;
+
+        for_each_cpu_mask(i, sd->span) {
+                struct rq *rq = cpu_rq(i);
+                unsigned long flags;
+
+                spin_lock_irqsave(&rq->lock, flags);
+                __update_group_shares_cpu(tg, i, shares, rq_weight);
+                spin_unlock_irqrestore(&rq->lock, flags);
+        }
+}
+
+/*
+ * Compute the cpu's hierarchical load factor for each task group.
+ * This needs to be done in a top-down fashion because the load of a child
+ * group is a fraction of its parents load.
+ */
+static void
+tg_load_down(struct task_group *tg, int cpu, struct sched_domain *sd)
+{
+        unsigned long load;
+
+        if (!tg->parent) {
+                load = cpu_rq(cpu)->load.weight;
+        } else {
+                load = tg->parent->cfs_rq[cpu]->h_load;
+                load *= tg->cfs_rq[cpu]->shares;
+                load /= tg->parent->cfs_rq[cpu]->load.weight + 1;
+        }
+
+        tg->cfs_rq[cpu]->h_load = load;
+}
+
+static void
+tg_nop(struct task_group *tg, int cpu, struct sched_domain *sd)
+{
+}
+
+static void update_shares(struct sched_domain *sd)
+{
+        u64 now = cpu_clock(raw_smp_processor_id());
+        s64 elapsed = now - sd->last_update;
+
+        if (elapsed >= (s64)(u64)sysctl_sched_shares_ratelimit) {
+                sd->last_update = now;
+                walk_tg_tree(tg_nop, tg_shares_up, 0, sd);
+        }
+}
+
+static void update_shares_locked(struct rq *rq, struct sched_domain *sd)
+{
+        spin_unlock(&rq->lock);
+        update_shares(sd);
+        spin_lock(&rq->lock);
+}
+
+static void update_h_load(int cpu)
+{
+        walk_tg_tree(tg_load_down, tg_nop, cpu, NULL);
+}
+
+#else
+
+static inline void update_shares(struct sched_domain *sd)
+{
+}
+
+static inline void update_shares_locked(struct rq *rq, struct sched_domain *sd)
+{
+}
+
 #endif
 
-#endif /* CONFIG_SMP */
+#endif
+
+#ifdef CONFIG_FAIR_GROUP_SCHED
+static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
+{
+#ifdef CONFIG_SMP
+        cfs_rq->shares = shares;
+#endif
+}
+#endif
 
 #include "sched_stats.h"
 #include "sched_idletask.c"
@@ -1490,27 +1678,17 @@ static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
 #endif
 
 #define sched_class_highest (&rt_sched_class)
+#define for_each_class(class) \
+   for (class = sched_class_highest; class; class = class->next)
 
-static inline void inc_load(struct rq *rq, const struct task_struct *p)
-{
-        update_load_add(&rq->load, p->se.load.weight);
-}
-
-static inline void dec_load(struct rq *rq, const struct task_struct *p)
-{
-        update_load_sub(&rq->load, p->se.load.weight);
-}
-
-static void inc_nr_running(struct task_struct *p, struct rq *rq)
+static void inc_nr_running(struct rq *rq)
 {
         rq->nr_running++;
-        inc_load(rq, p);
 }
 
-static void dec_nr_running(struct task_struct *p, struct rq *rq)
+static void dec_nr_running(struct rq *rq)
 {
         rq->nr_running--;
-        dec_load(rq, p);
 }
 
 static void set_load_weight(struct task_struct *p)
@@ -1534,6 +1712,12 @@ static void set_load_weight(struct task_struct *p)
         p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
 }
 
+static void update_avg(u64 *avg, u64 sample)
+{
+        s64 diff = sample - *avg;
+        *avg += diff >> 3;
+}
+
 static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup)
 {
         sched_info_queued(p);
@@ -1543,6 +1727,13 @@ static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup)
 
 static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep)
 {
+        if (sleep && p->se.last_wakeup) {
+                update_avg(&p->se.avg_overlap,
+                           p->se.sum_exec_runtime - p->se.last_wakeup);
+                p->se.last_wakeup = 0;
+        }
+
+        sched_info_dequeued(p);
         p->sched_class->dequeue_task(rq, p, sleep);
         p->se.on_rq = 0;
 }
@@ -1602,7 +1793,7 @@ static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)
                 rq->nr_uninterruptible--;
 
         enqueue_task(rq, p, wakeup);
-        inc_nr_running(p, rq);
+        inc_nr_running(rq);
 }
 
 /*
@@ -1614,7 +1805,7 @@ static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep)
                 rq->nr_uninterruptible++;
 
         dequeue_task(rq, p, sleep);
-        dec_nr_running(p, rq);
+        dec_nr_running(rq);
 }
 
 /**
@@ -1626,12 +1817,6 @@ inline int task_curr(const struct task_struct *p)
         return cpu_curr(task_cpu(p)) == p;
 }
 
-/* Used instead of source_load when we know the type == 0 */
-unsigned long weighted_cpuload(const int cpu)
-{
-        return cpu_rq(cpu)->load.weight;
-}
-
 static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
 {
         set_task_rq(p, cpu);
@@ -1660,6 +1845,12 @@ static inline void check_class_changed(struct rq *rq, struct task_struct *p,
 
 #ifdef CONFIG_SMP
 
+/* Used instead of source_load when we know the type == 0 */
+static unsigned long weighted_cpuload(const int cpu)
+{
+        return cpu_rq(cpu)->load.weight;
+}
+
 /*
  * Is this task likely cache-hot:
  */
@@ -1870,7 +2061,7 @@ static unsigned long source_load(int cpu, int type)
         struct rq *rq = cpu_rq(cpu);
         unsigned long total = weighted_cpuload(cpu);
 
-        if (type == 0)
+        if (type == 0 || !sched_feat(LB_BIAS))
                 return total;
 
         return min(rq->cpu_load[type-1], total);
@@ -1885,25 +2076,13 @@ static unsigned long target_load(int cpu, int type)
         struct rq *rq = cpu_rq(cpu);
         unsigned long total = weighted_cpuload(cpu);
 
-        if (type == 0)
+        if (type == 0 || !sched_feat(LB_BIAS))
                 return total;
 
         return max(rq->cpu_load[type-1], total);
 }
 
 /*
- * Return the average load per task on the cpu's run queue
- */
-static unsigned long cpu_avg_load_per_task(int cpu)
-{
-        struct rq *rq = cpu_rq(cpu);
-        unsigned long total = weighted_cpuload(cpu);
-        unsigned long n = rq->nr_running;
-
-        return n ? total / n : SCHED_LOAD_SCALE;
-}
-
-/*
  * find_idlest_group finds and returns the least busy CPU group within the
  * domain.
  */
@@ -2009,6 +2188,9 @@ static int sched_balance_self(int cpu, int flag)
                         sd = tmp;
         }
 
+        if (sd)
+                update_shares(sd);
+
         while (sd) {
                 cpumask_t span, tmpmask;
                 struct sched_group *group;
@@ -2075,6 +2257,22 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
         if (!sched_feat(SYNC_WAKEUPS))
                 sync = 0;
 
+#ifdef CONFIG_SMP
+        if (sched_feat(LB_WAKEUP_UPDATE)) {
+                struct sched_domain *sd;
+
+                this_cpu = raw_smp_processor_id();
+                cpu = task_cpu(p);
+
+                for_each_domain(this_cpu, sd) {
+                        if (cpu_isset(cpu, sd->span)) {
+                                update_shares(sd);
+                                break;
+                        }
+                }
+        }
+#endif
+
         smp_wmb();
         rq = task_rq_lock(p, &flags);
         old_state = p->state;
@@ -2121,7 +2319,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
                         }
                 }
         }
-#endif
+#endif /* CONFIG_SCHEDSTATS */
 
 out_activate:
 #endif /* CONFIG_SMP */
@@ -2139,6 +2337,9 @@ out_activate:
         success = 1;
 
 out_running:
+        trace_mark(kernel_sched_wakeup,
+                "pid %d state %ld ## rq %p task %p rq->curr %p",
+                p->pid, p->state, rq, p, rq->curr);
         check_preempt_curr(rq, p);
 
         p->state = TASK_RUNNING;
@@ -2147,6 +2348,8 @@ out_running:
                 p->sched_class->task_wake_up(rq, p);
 #endif
 out:
+        current->se.last_wakeup = current->se.sum_exec_runtime;
+
         task_rq_unlock(rq, &flags);
 
         return success;
@@ -2267,8 +2470,11 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
                  * management (if any):
                  */
                 p->sched_class->task_new(rq, p);
-                inc_nr_running(p, rq);
+                inc_nr_running(rq);
         }
+        trace_mark(kernel_sched_wakeup_new,
+                "pid %d state %ld ## rq %p task %p rq->curr %p",
+                p->pid, p->state, rq, p, rq->curr);
         check_preempt_curr(rq, p);
 #ifdef CONFIG_SMP
         if (p->sched_class->task_wake_up)
@@ -2321,7 +2527,7 @@ fire_sched_out_preempt_notifiers(struct task_struct *curr,
                 notifier->ops->sched_out(notifier, next);
 }
 
-#else
+#else /* !CONFIG_PREEMPT_NOTIFIERS */
 
 static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
 {
@@ -2333,7 +2539,7 @@ fire_sched_out_preempt_notifiers(struct task_struct *curr,
 {
 }
 
-#endif
+#endif /* CONFIG_PREEMPT_NOTIFIERS */
 
 /**
  * prepare_task_switch - prepare to switch tasks
@@ -2441,6 +2647,11 @@ context_switch(struct rq *rq, struct task_struct *prev,
         struct mm_struct *mm, *oldmm;
 
         prepare_task_switch(rq, prev, next);
+        trace_mark(kernel_sched_schedule,
+                "prev_pid %d next_pid %d prev_state %ld "
+                "## rq %p prev %p next %p",
+                prev->pid, next->pid, prev->state,
+                rq, prev, next);
         mm = next->mm;
         oldmm = prev->active_mm;
         /*
@@ -2775,7 +2986,7 @@ balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
               enum cpu_idle_type idle, int *all_pinned,
               int *this_best_prio, struct rq_iterator *iterator)
 {
-        int loops = 0, pulled = 0, pinned = 0, skip_for_load;
+        int loops = 0, pulled = 0, pinned = 0;
         struct task_struct *p;
         long rem_load_move = max_load_move;
 
@@ -2791,14 +3002,8 @@ balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
 next:
         if (!p || loops++ > sysctl_sched_nr_migrate)
                 goto out;
-        /*
-         * To help distribute high priority tasks across CPUs we don't
-         * skip a task if it will be the highest priority task (i.e. smallest
-         * prio value) on its new queue regardless of its load weight
-         */
-        skip_for_load = (p->se.load.weight >> 1) > rem_load_move +
-                                                         SCHED_LOAD_SCALE_FUZZ;
-        if ((skip_for_load && p->prio >= *this_best_prio) ||
+
+        if ((p->se.load.weight >> 1) > rem_load_move ||
             !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
                 p = iterator->next(iterator->arg);
                 goto next;
@@ -2853,6 +3058,10 @@ static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
                                 max_load_move - total_load_moved,
                                 sd, idle, all_pinned, &this_best_prio);
                 class = class->next;
+
+                if (idle == CPU_NEWLY_IDLE && this_rq->nr_running)
+                        break;
+
         } while (class && max_load_move > total_load_moved);
 
         return total_load_moved > 0;
@@ -2929,6 +3138,7 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
         max_load = this_load = total_load = total_pwr = 0;
         busiest_load_per_task = busiest_nr_running = 0;
         this_load_per_task = this_nr_running = 0;
+
         if (idle == CPU_NOT_IDLE)
                 load_idx = sd->busy_idx;
         else if (idle == CPU_NEWLY_IDLE)
@@ -2943,6 +3153,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
                 int __group_imb = 0;
                 unsigned int balance_cpu = -1, first_idle_cpu = 0;
                 unsigned long sum_nr_running, sum_weighted_load;
+                unsigned long sum_avg_load_per_task;
+                unsigned long avg_load_per_task;
 
                 local_group = cpu_isset(this_cpu, group->cpumask);
 
@@ -2951,6 +3163,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
 
                 /* Tally up the load of all CPUs in the group */
                 sum_weighted_load = sum_nr_running = avg_load = 0;
+                sum_avg_load_per_task = avg_load_per_task = 0;
+
                 max_cpu_load = 0;
                 min_cpu_load = ~0UL;
 
@@ -2984,6 +3198,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
                         avg_load += load;
                         sum_nr_running += rq->nr_running;
                         sum_weighted_load += weighted_cpuload(i);
+
+                        sum_avg_load_per_task += cpu_avg_load_per_task(i);
                 }
 
                 /*
@@ -3005,7 +3221,20 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
                 avg_load = sg_div_cpu_power(group,
                                 avg_load * SCHED_LOAD_SCALE);
 
-                if ((max_cpu_load - min_cpu_load) > SCHED_LOAD_SCALE)
+
+                /*
+                 * Consider the group unbalanced when the imbalance is larger
+                 * than the average weight of two tasks.
+                 *
+                 * APZ: with cgroup the avg task weight can vary wildly and
+                 *      might not be a suitable number - should we keep a
+                 *      normalized nr_running number somewhere that negates
+                 *      the hierarchy?
+                 */
+                avg_load_per_task = sg_div_cpu_power(group,
+                                sum_avg_load_per_task * SCHED_LOAD_SCALE);
+
+                if ((max_cpu_load - min_cpu_load) > 2*avg_load_per_task)
                         __group_imb = 1;
 
                 group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
@@ -3146,9 +3375,9 @@ small_imbalance:
                         if (busiest_load_per_task > this_load_per_task)
                                 imbn = 1;
                 } else
-                        this_load_per_task = SCHED_LOAD_SCALE;
+                        this_load_per_task = cpu_avg_load_per_task(this_cpu);
 
-                if (max_load - this_load + SCHED_LOAD_SCALE_FUZZ >=
+                if (max_load - this_load + 2*busiest_load_per_task >=
                                         busiest_load_per_task * imbn) {
                         *imbalance = busiest_load_per_task;
                         return busiest;
@@ -3274,6 +3503,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
         schedstat_inc(sd, lb_count[idle]);
 
 redo:
+        update_shares(sd);
         group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
                                    cpus, balance);
 
@@ -3376,8 +3606,9 @@ redo:
 
         if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
             !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
-                return -1;
-        return ld_moved;
+                ld_moved = -1;
+
+        goto out;
 
 out_balanced:
         schedstat_inc(sd, lb_balanced[idle]);
@@ -3392,8 +3623,13 @@ out_one_pinned:
 
         if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
             !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
-                return -1;
-        return 0;
+                ld_moved = -1;
+        else
+                ld_moved = 0;
+out:
+        if (ld_moved)
+                update_shares(sd);
+        return ld_moved;
 }
 
 /*
@@ -3428,6 +3664,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
 
         schedstat_inc(sd, lb_count[CPU_NEWLY_IDLE]);
 redo:
+        update_shares_locked(this_rq, sd);
         group = find_busiest_group(sd, this_cpu, &imbalance, CPU_NEWLY_IDLE,
                                    &sd_idle, cpus, NULL);
         if (!group) {
@@ -3471,6 +3708,7 @@ redo:
         } else
                 sd->nr_balance_failed = 0;
 
+        update_shares_locked(this_rq, sd);
         return ld_moved;
 
 out_balanced:
@@ -3662,6 +3900,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
         /* Earliest time when we have to do rebalance again */
         unsigned long next_balance = jiffies + 60*HZ;
         int update_next_balance = 0;
+        int need_serialize;
         cpumask_t tmp;
 
         for_each_domain(cpu, sd) {
@@ -3679,8 +3918,9 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
                 if (interval > HZ*NR_CPUS/10)
                         interval = HZ*NR_CPUS/10;
 
+                need_serialize = sd->flags & SD_SERIALIZE;
 
-                if (sd->flags & SD_SERIALIZE) {
+                if (need_serialize) {
                         if (!spin_trylock(&balancing))
                                 goto out;
                 }
@@ -3696,7 +3936,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
                         }
                         sd->last_balance = jiffies;
                 }
-                if (sd->flags & SD_SERIALIZE)
+                if (need_serialize)
                         spin_unlock(&balancing);
 out:
                 if (time_after(next_balance, sd->last_balance + interval)) {
@@ -4011,26 +4251,44 @@ void scheduler_tick(void)
 #endif
 }
 
-#if defined(CONFIG_PREEMPT) && defined(CONFIG_DEBUG_PREEMPT)
+#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
+                                defined(CONFIG_PREEMPT_TRACER))
+
+static inline unsigned long get_parent_ip(unsigned long addr)
+{
+        if (in_lock_functions(addr)) {
+                addr = CALLER_ADDR2;
+                if (in_lock_functions(addr))
+                        addr = CALLER_ADDR3;
+        }
+        return addr;
+}
 
 void __kprobes add_preempt_count(int val)
 {
+#ifdef CONFIG_DEBUG_PREEMPT
         /*
          * Underflow?
          */
         if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
                 return;
+#endif
         preempt_count() += val;
+#ifdef CONFIG_DEBUG_PREEMPT
         /*
          * Spinlock count overflowing soon?
          */
         DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
                                 PREEMPT_MASK - 10);
+#endif
+        if (preempt_count() == val)
+                trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 }
 EXPORT_SYMBOL(add_preempt_count);
 
 void __kprobes sub_preempt_count(int val)
 {
+#ifdef CONFIG_DEBUG_PREEMPT
         /*
          * Underflow?
          */
@@ -4042,7 +4300,10 @@ void __kprobes sub_preempt_count(int val)
         if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
                         !(preempt_count() & PREEMPT_MASK)))
                 return;
+#endif
 
+        if (preempt_count() == val)
+                trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
         preempt_count() -= val;
 }
 EXPORT_SYMBOL(sub_preempt_count);
@@ -4060,6 +4321,7 @@ static noinline void __schedule_bug(struct task_struct *prev)
                 prev->comm, prev->pid, preempt_count());
 
         debug_show_held_locks(prev);
+        print_modules();
         if (irqs_disabled())
                 print_irqtrace_events(prev);
 
@@ -4133,7 +4395,7 @@ asmlinkage void __sched schedule(void)
         struct task_struct *prev, *next;
         unsigned long *switch_count;
         struct rq *rq;
-        int cpu;
+        int cpu, hrtick = sched_feat(HRTICK);
 
 need_resched:
         preempt_disable();
@@ -4148,7 +4410,8 @@ need_resched_nonpreemptible:
 
         schedule_debug(prev);
 
-        hrtick_clear(rq);
+        if (hrtick)
+                hrtick_clear(rq);
 
         /*
          * Do the rq-clock update outside the rq lock:
@@ -4159,12 +4422,10 @@ need_resched_nonpreemptible:
         clear_tsk_need_resched(prev);
 
         if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
-                if (unlikely((prev->state & TASK_INTERRUPTIBLE) &&
-                                signal_pending(prev))) {
+                if (unlikely(signal_pending_state(prev->state, prev)))
                         prev->state = TASK_RUNNING;
-                } else {
+                else
                         deactivate_task(rq, prev, 1);
-                }
                 switch_count = &prev->nvcsw;
         }
 
@@ -4196,7 +4457,8 @@ need_resched_nonpreemptible:
         } else
                 spin_unlock_irq(&rq->lock);
 
-        hrtick_set(rq);
+        if (hrtick)
+                hrtick_set(rq);
 
         if (unlikely(reacquire_kernel_lock(current) < 0))
                 goto need_resched_nonpreemptible;
@@ -4390,22 +4652,20 @@ do_wait_for_common(struct completion *x, long timeout, int state)
                              signal_pending(current)) ||
                             (state == TASK_KILLABLE &&
                              fatal_signal_pending(current))) {
-                                __remove_wait_queue(&x->wait, &wait);
-                                return -ERESTARTSYS;
+                                timeout = -ERESTARTSYS;
+                                break;
                         }
                         __set_current_state(state);
                         spin_unlock_irq(&x->wait.lock);
                         timeout = schedule_timeout(timeout);
                         spin_lock_irq(&x->wait.lock);
-                        if (!timeout) {
-                                __remove_wait_queue(&x->wait, &wait);
-                                return timeout;
-                        }
-                } while (!x->done);
+                } while (!x->done && timeout);
                 __remove_wait_queue(&x->wait, &wait);
+                if (!x->done)
+                        return timeout;
         }
         x->done--;
-        return timeout;
+        return timeout ?: 1;
 }
 
 static long __sched
@@ -4580,10 +4840,8 @@ void set_user_nice(struct task_struct *p, long nice)
                 goto out_unlock;
         }
         on_rq = p->se.on_rq;
-        if (on_rq) {
+        if (on_rq)
                 dequeue_task(rq, p, 0);
-                dec_load(rq, p);
-        }
 
         p->static_prio = NICE_TO_PRIO(nice);
         set_load_weight(p);
@@ -4593,7 +4851,6 @@ void set_user_nice(struct task_struct *p, long nice)
 
         if (on_rq) {
                 enqueue_task(rq, p, 0);
-                inc_load(rq, p);
                 /*
                  * If the task increased its priority or is running and
                  * lowered its priority, then reschedule its CPU:
@@ -4738,16 +4995,8 @@ __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
         set_load_weight(p);
 }
 
-/**
- * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
- * @p: the task in question.
- * @policy: new policy.
- * @param: structure containing the new RT priority.
- *
- * NOTE that the task may be already dead.
- */
-int sched_setscheduler(struct task_struct *p, int policy,
-                       struct sched_param *param)
+static int __sched_setscheduler(struct task_struct *p, int policy,
+                                struct sched_param *param, bool user)
 {
         int retval, oldprio, oldpolicy = -1, on_rq, running;
         unsigned long flags;
@@ -4779,7 +5028,7 @@ recheck:
         /*
          * Allow unprivileged RT tasks to decrease priority:
          */
-        if (!capable(CAP_SYS_NICE)) {
+        if (user && !capable(CAP_SYS_NICE)) {
                 if (rt_policy(policy)) {
                         unsigned long rlim_rtprio;
 
@@ -4815,7 +5064,8 @@ recheck:
          * Do not allow realtime tasks into groups that have no runtime
          * assigned.
          */
-        if (rt_policy(policy) && task_group(p)->rt_bandwidth.rt_runtime == 0)
+        if (user
+            && rt_policy(policy) && task_group(p)->rt_bandwidth.rt_runtime == 0)
                 return -EPERM;
 #endif
 
@@ -4864,8 +5114,39 @@ recheck:
 
         return 0;
 }
+
+/**
+ * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
+ * @p: the task in question.
+ * @policy: new policy.
+ * @param: structure containing the new RT priority.
+ *
+ * NOTE that the task may be already dead.
+ */
+int sched_setscheduler(struct task_struct *p, int policy,
+                       struct sched_param *param)
+{
+        return __sched_setscheduler(p, policy, param, true);
+}
 EXPORT_SYMBOL_GPL(sched_setscheduler);
 
+/**
+ * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
+ * @p: the task in question.
+ * @policy: new policy.
+ * @param: structure containing the new RT priority.
+ *
+ * Just like sched_setscheduler, only don't bother checking if the
+ * current context has permission.  For example, this is needed in
+ * stop_machine(): we create temporary high priority worker threads,
+ * but our caller might not have that capability.
+ */
+int sched_setscheduler_nocheck(struct task_struct *p, int policy,
+                               struct sched_param *param)
+{
+        return __sched_setscheduler(p, policy, param, false);
+}
+
 static int
 do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
 {
@@ -5064,24 +5345,6 @@ asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
         return sched_setaffinity(pid, &new_mask);
 }
 
-/*
- * Represents all cpu's present in the system
- * In systems capable of hotplug, this map could dynamically grow
- * as new cpu's are detected in the system via any platform specific
- * method, such as ACPI for e.g.
- */
-
-cpumask_t cpu_present_map __read_mostly;
-EXPORT_SYMBOL(cpu_present_map);
-
-#ifndef CONFIG_SMP
-cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL;
-EXPORT_SYMBOL(cpu_online_map);
-
-cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL;
-EXPORT_SYMBOL(cpu_possible_map);
-#endif
-
 long sched_getaffinity(pid_t pid, cpumask_t *mask)
 {
         struct task_struct *p;
@@ -5378,7 +5641,7 @@ out_unlock:
         return retval;
 }
 
-static const char stat_nam[] = "RSDTtZX";
+static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
 
 void sched_show_task(struct task_struct *p)
 {
@@ -5565,6 +5828,12 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
                 goto out;
         }
 
+        if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
+                     !cpus_equal(p->cpus_allowed, *new_mask))) {
+                ret = -EINVAL;
+                goto out;
+        }
+
         if (p->sched_class->set_cpus_allowed)
                 p->sched_class->set_cpus_allowed(p, new_mask);
         else {
@@ -5616,10 +5885,10 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
         double_rq_lock(rq_src, rq_dest);
         /* Already moved. */
         if (task_cpu(p) != src_cpu)
-                goto out;
+                goto done;
         /* Affinity changed (again). */
         if (!cpu_isset(dest_cpu, p->cpus_allowed))
-                goto out;
+                goto fail;
 
         on_rq = p->se.on_rq;
         if (on_rq)
@@ -5630,8 +5899,9 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
                 activate_task(rq_dest, p, 0);
                 check_preempt_curr(rq_dest, p);
         }
+done:
         ret = 1;
-out:
+fail:
         double_rq_unlock(rq_src, rq_dest);
         return ret;
 }
@@ -5881,6 +6151,7 @@ static void migrate_dead_tasks(unsigned int dead_cpu)
                 next = pick_next_task(rq, rq->curr);
                 if (!next)
                         break;
+                next->sched_class->put_prev_task(rq, next);
                 migrate_dead(dead_cpu, next);
 
         }
@@ -6052,6 +6323,36 @@ static void unregister_sched_domain_sysctl(void)
 }
 #endif
 
+static void set_rq_online(struct rq *rq)
+{
+        if (!rq->online) {
+                const struct sched_class *class;
+
+                cpu_set(rq->cpu, rq->rd->online);
+                rq->online = 1;
+
+                for_each_class(class) {
+                        if (class->rq_online)
+                                class->rq_online(rq);
+                }
+        }
+}
+
+static void set_rq_offline(struct rq *rq)
+{
+        if (rq->online) {
+                const struct sched_class *class;
+
+                for_each_class(class) {
+                        if (class->rq_offline)
+                                class->rq_offline(rq);
+                }
+
+                cpu_clear(rq->cpu, rq->rd->online);
+                rq->online = 0;
+        }
+}
+
 /*
  * migration_call - callback that gets triggered when a CPU is added.
  * Here we can start up the necessary migration thread for the new CPU.
@@ -6089,7 +6390,8 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 spin_lock_irqsave(&rq->lock, flags);
                 if (rq->rd) {
                         BUG_ON(!cpu_isset(cpu, rq->rd->span));
-                        cpu_set(cpu, rq->rd->online);
+
+                        set_rq_online(rq);
                 }
                 spin_unlock_irqrestore(&rq->lock, flags);
                 break;
@@ -6150,7 +6452,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 spin_lock_irqsave(&rq->lock, flags);
                 if (rq->rd) {
                         BUG_ON(!cpu_isset(cpu, rq->rd->span));
-                        cpu_clear(cpu, rq->rd->online);
+                        set_rq_offline(rq);
                 }
                 spin_unlock_irqrestore(&rq->lock, flags);
                 break;
@@ -6184,6 +6486,28 @@ void __init migration_init(void)
 
 #ifdef CONFIG_SCHED_DEBUG
 
+static inline const char *sd_level_to_string(enum sched_domain_level lvl)
+{
+        switch (lvl) {
+        case SD_LV_NONE:
+                        return "NONE";
+        case SD_LV_SIBLING:
+                        return "SIBLING";
+        case SD_LV_MC:
+                        return "MC";
+        case SD_LV_CPU:
+                        return "CPU";
+        case SD_LV_NODE:
+                        return "NODE";
+        case SD_LV_ALLNODES:
+                        return "ALLNODES";
+        case SD_LV_MAX:
+                        return "MAX";
+
+        }
+        return "MAX";
+}
+
 static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
                                   cpumask_t *groupmask)
 {
@@ -6203,7 +6527,8 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
                 return -1;
         }
 
-        printk(KERN_CONT "span %s\n", str);
+        printk(KERN_CONT "span %s level %s\n",
+                str, sd_level_to_string(sd->level));
 
         if (!cpu_isset(cpu, sd->span)) {
                 printk(KERN_ERR "ERROR: domain->span does not contain "
@@ -6287,9 +6612,9 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
         }
         kfree(groupmask);
 }
-#else
+#else /* !CONFIG_SCHED_DEBUG */
 # define sched_domain_debug(sd, cpu) do { } while (0)
-#endif
+#endif /* CONFIG_SCHED_DEBUG */
 
 static int sd_degenerate(struct sched_domain *sd)
 {
@@ -6349,20 +6674,16 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
 static void rq_attach_root(struct rq *rq, struct root_domain *rd)
 {
         unsigned long flags;
-        const struct sched_class *class;
 
         spin_lock_irqsave(&rq->lock, flags);
 
         if (rq->rd) {
                 struct root_domain *old_rd = rq->rd;
 
-                for (class = sched_class_highest; class; class = class->next) {
-                        if (class->leave_domain)
-                                class->leave_domain(rq);
-                }
+                if (cpu_isset(rq->cpu, old_rd->online))
+                        set_rq_offline(rq);
 
                 cpu_clear(rq->cpu, old_rd->span);
-                cpu_clear(rq->cpu, old_rd->online);
 
                 if (atomic_dec_and_test(&old_rd->refcount))
                         kfree(old_rd);
@@ -6373,12 +6694,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
 
         cpu_set(rq->cpu, rd->span);
         if (cpu_isset(rq->cpu, cpu_online_map))
-                cpu_set(rq->cpu, rd->online);
-
-        for (class = sched_class_highest; class; class = class->next) {
-                if (class->join_domain)
-                        class->join_domain(rq);
-        }
+                set_rq_online(rq);
 
         spin_unlock_irqrestore(&rq->lock, flags);
 }
@@ -6389,6 +6705,8 @@ static void init_rootdomain(struct root_domain *rd)
 
         cpus_clear(rd->span);
         cpus_clear(rd->online);
+
+        cpupri_init(&rd->cpupri);
 }
 
 static void init_defrootdomain(void)
@@ -6531,9 +6849,9 @@ static int find_next_best_node(int node, nodemask_t *used_nodes)
 
         min_val = INT_MAX;
 
-        for (i = 0; i < MAX_NUMNODES; i++) {
+        for (i = 0; i < nr_node_ids; i++) {
                 /* Start at @node */
-                n = (node + i) % MAX_NUMNODES;
+                n = (node + i) % nr_node_ids;
 
                 if (!nr_cpus_node(n))
                         continue;
@@ -6583,7 +6901,7 @@ static void sched_domain_node_span(int node, cpumask_t *span)
                 cpus_or(*span, *span, *nodemask);
         }
 }
-#endif
+#endif /* CONFIG_NUMA */
 
 int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
 
@@ -6602,7 +6920,7 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
                 *sg = &per_cpu(sched_group_cpus, cpu);
         return cpu;
 }
-#endif
+#endif /* CONFIG_SCHED_SMT */
 
 /*
  * multi-core sched-domains:
@@ -6610,7 +6928,7 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
 #ifdef CONFIG_SCHED_MC
 static DEFINE_PER_CPU(struct sched_domain, core_domains);
 static DEFINE_PER_CPU(struct sched_group, sched_group_core);
-#endif
+#endif /* CONFIG_SCHED_MC */
 
 #if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
 static int
@@ -6712,7 +7030,7 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
                 sg = sg->next;
         } while (sg != group_head);
 }
-#endif
+#endif /* CONFIG_NUMA */
 
 #ifdef CONFIG_NUMA
 /* Free memory allocated for various sched_group structures */
@@ -6727,7 +7045,7 @@ static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
                 if (!sched_group_nodes)
                         continue;
 
-                for (i = 0; i < MAX_NUMNODES; i++) {
+                for (i = 0; i < nr_node_ids; i++) {
                         struct sched_group *oldsg, *sg = sched_group_nodes[i];
 
                         *nodemask = node_to_cpumask(i);
@@ -6749,11 +7067,11 @@ next_sg:
                 sched_group_nodes_bycpu[cpu] = NULL;
         }
 }
-#else
+#else /* !CONFIG_NUMA */
 static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
 {
 }
-#endif
+#endif /* CONFIG_NUMA */
 
 /*
  * Initialize sched groups cpu_power.
@@ -6871,7 +7189,12 @@ static int default_relax_domain_level = -1;
 
 static int __init setup_relax_domain_level(char *str)
 {
-        default_relax_domain_level = simple_strtoul(str, NULL, 0);
+        unsigned long val;
+
+        val = simple_strtoul(str, NULL, 0);
+        if (val < SD_LV_MAX)
+                default_relax_domain_level = val;
+
         return 1;
 }
 __setup("relax_domain_level=", setup_relax_domain_level);
@@ -6915,7 +7238,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
         /*
          * Allocate the per-node list of sched groups
          */
-        sched_group_nodes = kcalloc(MAX_NUMNODES, sizeof(struct sched_group *),
+        sched_group_nodes = kcalloc(nr_node_ids, sizeof(struct sched_group *),
                                     GFP_KERNEL);
         if (!sched_group_nodes) {
                 printk(KERN_WARNING "Can not alloc sched group node list\n");
@@ -7054,7 +7377,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
 #endif
 
         /* Set up physical groups */
-        for (i = 0; i < MAX_NUMNODES; i++) {
+        for (i = 0; i < nr_node_ids; i++) {
                 SCHED_CPUMASK_VAR(nodemask, allmasks);
                 SCHED_CPUMASK_VAR(send_covered, allmasks);
 
@@ -7078,7 +7401,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
                                         send_covered, tmpmask);
         }
 
-        for (i = 0; i < MAX_NUMNODES; i++) {
+        for (i = 0; i < nr_node_ids; i++) {
                 /* Set up node groups */
                 struct sched_group *sg, *prev;
                 SCHED_CPUMASK_VAR(nodemask, allmasks);
@@ -7117,9 +7440,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
                 cpus_or(*covered, *covered, *nodemask);
                 prev = sg;
 
-                for (j = 0; j < MAX_NUMNODES; j++) {
+                for (j = 0; j < nr_node_ids; j++) {
                         SCHED_CPUMASK_VAR(notcovered, allmasks);
-                        int n = (i + j) % MAX_NUMNODES;
+                        int n = (i + j) % nr_node_ids;
                         node_to_cpumask_ptr(pnodemask, n);
 
                         cpus_complement(*notcovered, *covered);
@@ -7172,7 +7495,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
         }
 
 #ifdef CONFIG_NUMA
-        for (i = 0; i < MAX_NUMNODES; i++)
+        for (i = 0; i < nr_node_ids; i++)
                 init_numa_sched_groups_power(sched_group_nodes[i]);
 
         if (sd_allnodes) {
@@ -7230,6 +7553,18 @@ void __attribute__((weak)) arch_update_cpu_topology(void)
 }
 
 /*
+ * Free current domain masks.
+ * Called after all cpus are attached to NULL domain.
+ */
+static void free_sched_domains(void)
+{
+        ndoms_cur = 0;
+        if (doms_cur != &fallback_doms)
+                kfree(doms_cur);
+        doms_cur = &fallback_doms;
+}
+
+/*
  * Set up scheduler domains and groups. Callers must hold the hotplug lock.
  * For now this just excludes isolated cpus, but could be used to
  * exclude other special cases in the future.
@@ -7376,6 +7711,7 @@ int arch_reinit_sched_domains(void)
         get_online_cpus();
         mutex_lock(&sched_domains_mutex);
         detach_destroy_domains(&cpu_online_map);
+        free_sched_domains();
         err = arch_init_sched_domains(&cpu_online_map);
         mutex_unlock(&sched_domains_mutex);
         put_online_cpus();
@@ -7444,7 +7780,7 @@ int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
 #endif
         return err;
 }
-#endif
+#endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
 
 /*
  * Force a reinitialization of the sched domains hierarchy. The domains
@@ -7455,20 +7791,28 @@ int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
 static int update_sched_domains(struct notifier_block *nfb,
                                 unsigned long action, void *hcpu)
 {
+        int cpu = (int)(long)hcpu;
+
         switch (action) {
-        case CPU_UP_PREPARE:
-        case CPU_UP_PREPARE_FROZEN:
         case CPU_DOWN_PREPARE:
         case CPU_DOWN_PREPARE_FROZEN:
+                disable_runtime(cpu_rq(cpu));
+                /* fall-through */
+        case CPU_UP_PREPARE:
+        case CPU_UP_PREPARE_FROZEN:
                 detach_destroy_domains(&cpu_online_map);
+                free_sched_domains();
                 return NOTIFY_OK;
 
-        case CPU_UP_CANCELED:
-        case CPU_UP_CANCELED_FROZEN:
+
         case CPU_DOWN_FAILED:
         case CPU_DOWN_FAILED_FROZEN:
         case CPU_ONLINE:
         case CPU_ONLINE_FROZEN:
+                enable_runtime(cpu_rq(cpu));
+                /* fall-through */
+        case CPU_UP_CANCELED:
+        case CPU_UP_CANCELED_FROZEN:
         case CPU_DEAD:
         case CPU_DEAD_FROZEN:
                 /*
@@ -7479,8 +7823,16 @@ static int update_sched_domains(struct notifier_block *nfb,
                 return NOTIFY_DONE;
         }
 
+#ifndef CONFIG_CPUSETS
+        /*
+         * Create default domain partitioning if cpusets are disabled.
+         * Otherwise we let cpusets rebuild the domains based on the
+         * current setup.
+         */
+
         /* The hotplug lock is already held by cpu_up/cpu_down */
         arch_init_sched_domains(&cpu_online_map);
+#endif
 
         return NOTIFY_OK;
 }
@@ -7620,7 +7972,6 @@ static void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
         else
                 rt_se->rt_rq = parent->my_q;
 
-        rt_se->rt_rq = &rq->rt;
         rt_se->my_q = rt_rq;
         rt_se->parent = parent;
         INIT_LIST_HEAD(&rt_se->run_list);
@@ -7661,8 +8012,8 @@ void __init sched_init(void)
 
                 root_task_group.cfs_rq = (struct cfs_rq **)ptr;
                 ptr += nr_cpu_ids * sizeof(void **);
-#endif
-#endif
+#endif /* CONFIG_USER_SCHED */
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 #ifdef CONFIG_RT_GROUP_SCHED
                 init_task_group.rt_se = (struct sched_rt_entity **)ptr;
                 ptr += nr_cpu_ids * sizeof(void **);
@@ -7676,8 +8027,8 @@ void __init sched_init(void)
 
                 root_task_group.rt_rq = (struct rt_rq **)ptr;
                 ptr += nr_cpu_ids * sizeof(void **);
-#endif
-#endif
+#endif /* CONFIG_USER_SCHED */
+#endif /* CONFIG_RT_GROUP_SCHED */
         }
 
 #ifdef CONFIG_SMP
@@ -7693,8 +8044,8 @@ void __init sched_init(void)
 #ifdef CONFIG_USER_SCHED
         init_rt_bandwidth(&root_task_group.rt_bandwidth,
                         global_rt_period(), RUNTIME_INF);
-#endif
-#endif
+#endif /* CONFIG_USER_SCHED */
+#endif /* CONFIG_RT_GROUP_SCHED */
 
 #ifdef CONFIG_GROUP_SCHED
         list_add(&init_task_group.list, &task_groups);
@@ -7704,8 +8055,8 @@ void __init sched_init(void)
         INIT_LIST_HEAD(&root_task_group.children);
         init_task_group.parent = &root_task_group;
         list_add(&init_task_group.siblings, &root_task_group.children);
-#endif
-#endif
+#endif /* CONFIG_USER_SCHED */
+#endif /* CONFIG_GROUP_SCHED */
 
         for_each_possible_cpu(i) {
                 struct rq *rq;
@@ -7785,6 +8136,7 @@ void __init sched_init(void)
                 rq->next_balance = jiffies;
                 rq->push_cpu = 0;
                 rq->cpu = i;
+                rq->online = 0;
                 rq->migration_thread = NULL;
                 INIT_LIST_HEAD(&rq->migration_queue);
                 rq_attach_root(rq, &def_root_domain);
@@ -7800,7 +8152,7 @@ void __init sched_init(void)
 #endif
 
 #ifdef CONFIG_SMP
-        open_softirq(SCHED_SOFTIRQ, run_rebalance_domains, NULL);
+        open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
 #endif
 
 #ifdef CONFIG_RT_MUTEXES
@@ -8024,7 +8376,7 @@ static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
 {
         list_del_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list);
 }
-#else
+#else /* !CONFG_FAIR_GROUP_SCHED */
 static inline void free_fair_sched_group(struct task_group *tg)
 {
 }
@@ -8042,7 +8394,7 @@ static inline void register_fair_sched_group(struct task_group *tg, int cpu)
 static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
 {
 }
-#endif
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #ifdef CONFIG_RT_GROUP_SCHED
 static void free_rt_sched_group(struct task_group *tg)
@@ -8113,7 +8465,7 @@ static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
 {
         list_del_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list);
 }
-#else
+#else /* !CONFIG_RT_GROUP_SCHED */
 static inline void free_rt_sched_group(struct task_group *tg)
 {
 }
@@ -8131,7 +8483,7 @@ static inline void register_rt_sched_group(struct task_group *tg, int cpu)
 static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
 {
 }
-#endif
+#endif /* CONFIG_RT_GROUP_SCHED */
 
 #ifdef CONFIG_GROUP_SCHED
 static void free_sched_group(struct task_group *tg)
@@ -8242,17 +8594,14 @@ void sched_move_task(struct task_struct *tsk)
 
         task_rq_unlock(rq, &flags);
 }
-#endif
+#endif /* CONFIG_GROUP_SCHED */
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-static void set_se_shares(struct sched_entity *se, unsigned long shares)
+static void __set_se_shares(struct sched_entity *se, unsigned long shares)
 {
         struct cfs_rq *cfs_rq = se->cfs_rq;
-        struct rq *rq = cfs_rq->rq;
         int on_rq;
 
-        spin_lock_irq(&rq->lock);
-
         on_rq = se->on_rq;
         if (on_rq)
                 dequeue_entity(cfs_rq, se, 0);
@@ -8262,8 +8611,17 @@ static void set_se_shares(struct sched_entity *se, unsigned long shares)
 
         if (on_rq)
                 enqueue_entity(cfs_rq, se, 0);
+}
 
-        spin_unlock_irq(&rq->lock);
+static void set_se_shares(struct sched_entity *se, unsigned long shares)
+{
+        struct cfs_rq *cfs_rq = se->cfs_rq;
+        struct rq *rq = cfs_rq->rq;
+        unsigned long flags;
+
+        spin_lock_irqsave(&rq->lock, flags);
+        __set_se_shares(se, shares);
+        spin_unlock_irqrestore(&rq->lock, flags);
 }
 
 static DEFINE_MUTEX(shares_mutex);
@@ -8302,8 +8660,13 @@ int sched_group_set_shares(struct task_group *tg, unsigned long shares)
          * w/o tripping rebalance_share or load_balance_fair.
          */
         tg->shares = shares;
-        for_each_possible_cpu(i)
+        for_each_possible_cpu(i) {
+                /*
+                 * force a rebalance
+                 */
+                cfs_rq_set_shares(tg->cfs_rq[i], 0);
                 set_se_shares(tg->se[i], shares);
+        }
 
         /*
          * Enable load balance activity on this group, by inserting it back on
@@ -8366,7 +8729,7 @@ static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
         }
         rcu_read_unlock();
 
-        return total + to_ratio(period, runtime) <
+        return total + to_ratio(period, runtime) <=
                 to_ratio(ktime_to_ns(parent->rt_bandwidth.rt_period),
                                 parent->rt_bandwidth.rt_runtime);
 }
@@ -8469,6 +8832,9 @@ int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
         rt_period = (u64)rt_period_us * NSEC_PER_USEC;
         rt_runtime = tg->rt_bandwidth.rt_runtime;
 
+        if (rt_period == 0)
+                return -EINVAL;
+
         return tg_set_bandwidth(tg, rt_period, rt_runtime);
 }
 
@@ -8483,16 +8849,21 @@ long sched_group_rt_period(struct task_group *tg)
 
 static int sched_rt_global_constraints(void)
 {
+        struct task_group *tg = &root_task_group;
+        u64 rt_runtime, rt_period;
         int ret = 0;
 
+        rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
+        rt_runtime = tg->rt_bandwidth.rt_runtime;
+
         mutex_lock(&rt_constraints_mutex);
-        if (!__rt_schedulable(NULL, 1, 0))
+        if (!__rt_schedulable(tg, rt_period, rt_runtime))
                 ret = -EINVAL;
         mutex_unlock(&rt_constraints_mutex);
 
         return ret;
 }
-#else
+#else /* !CONFIG_RT_GROUP_SCHED */
 static int sched_rt_global_constraints(void)
 {
         unsigned long flags;
@@ -8510,7 +8881,7 @@ static int sched_rt_global_constraints(void)
 
         return 0;
 }
-#endif
+#endif /* CONFIG_RT_GROUP_SCHED */
 
 int sched_rt_handler(struct ctl_table *table, int write,
                 struct file *filp, void __user *buffer, size_t *lenp,
@@ -8618,7 +8989,7 @@ static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
 
         return (u64) tg->shares;
 }
-#endif
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #ifdef CONFIG_RT_GROUP_SCHED
 static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
@@ -8642,7 +9013,7 @@ static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
 {
         return sched_group_rt_period(cgroup_tg(cgrp));
 }
-#endif
+#endif /* CONFIG_RT_GROUP_SCHED */
 
 static struct cftype cpu_files[] = {
 #ifdef CONFIG_FAIR_GROUP_SCHED
diff --git a/kernel/sched_clock.c b/kernel/sched_clock.c
index ce05271219ab..22ed55d1167f 100644
--- a/kernel/sched_clock.c
+++ b/kernel/sched_clock.c
@@ -3,6 +3,9 @@
  *
  *  Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  *
+ *  Updates and enhancements:
+ *    Copyright (C) 2008 Red Hat, Inc. Steven Rostedt <srostedt@redhat.com>
+ *
  * Based on code by:
  *   Ingo Molnar <mingo@redhat.com>
  *   Guillaume Chazarain <guichaz@gmail.com>
@@ -32,6 +35,11 @@
 
 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
 
+#define MULTI_SHIFT 15
+/* Max is double, Min is 1/2 */
+#define MAX_MULTI (2LL << MULTI_SHIFT)
+#define MIN_MULTI (1LL << (MULTI_SHIFT-1))
+
 struct sched_clock_data {
         /*
          * Raw spinlock - this is a special case: this might be called
@@ -40,11 +48,15 @@ struct sched_clock_data {
          */
         raw_spinlock_t          lock;
 
-        unsigned long           prev_jiffies;
+        unsigned long           tick_jiffies;
         u64                     prev_raw;
         u64                     tick_raw;
         u64                     tick_gtod;
         u64                     clock;
+        s64                     multi;
+#ifdef CONFIG_NO_HZ
+        int                     check_max;
+#endif
 };
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
@@ -71,41 +83,91 @@ void sched_clock_init(void)
                 struct sched_clock_data *scd = cpu_sdc(cpu);
 
                 scd->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
-                scd->prev_jiffies = now_jiffies;
+                scd->tick_jiffies = now_jiffies;
                 scd->prev_raw = 0;
                 scd->tick_raw = 0;
                 scd->tick_gtod = ktime_now;
                 scd->clock = ktime_now;
+                scd->multi = 1 << MULTI_SHIFT;
+#ifdef CONFIG_NO_HZ
+                scd->check_max = 1;
+#endif
         }
 
         sched_clock_running = 1;
 }
 
+#ifdef CONFIG_NO_HZ
+/*
+ * The dynamic ticks makes the delta jiffies inaccurate. This
+ * prevents us from checking the maximum time update.
+ * Disable the maximum check during stopped ticks.
+ */
+void sched_clock_tick_stop(int cpu)
+{
+        struct sched_clock_data *scd = cpu_sdc(cpu);
+
+        scd->check_max = 0;
+}
+
+void sched_clock_tick_start(int cpu)
+{
+        struct sched_clock_data *scd = cpu_sdc(cpu);
+
+        scd->check_max = 1;
+}
+
+static int check_max(struct sched_clock_data *scd)
+{
+        return scd->check_max;
+}
+#else
+static int check_max(struct sched_clock_data *scd)
+{
+        return 1;
+}
+#endif /* CONFIG_NO_HZ */
+
 /*
  * update the percpu scd from the raw @now value
  *
  *  - filter out backward motion
  *  - use jiffies to generate a min,max window to clip the raw values
  */
-static void __update_sched_clock(struct sched_clock_data *scd, u64 now)
+static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *time)
 {
         unsigned long now_jiffies = jiffies;
-        long delta_jiffies = now_jiffies - scd->prev_jiffies;
+        long delta_jiffies = now_jiffies - scd->tick_jiffies;
         u64 clock = scd->clock;
         u64 min_clock, max_clock;
         s64 delta = now - scd->prev_raw;
 
         WARN_ON_ONCE(!irqs_disabled());
-        min_clock = scd->tick_gtod + delta_jiffies * TICK_NSEC;
+
+        /*
+         * At schedule tick the clock can be just under the gtod. We don't
+         * want to push it too prematurely.
+         */
+        min_clock = scd->tick_gtod + (delta_jiffies * TICK_NSEC);
+        if (min_clock > TICK_NSEC)
+                min_clock -= TICK_NSEC / 2;
 
         if (unlikely(delta < 0)) {
                 clock++;
                 goto out;
         }
 
-        max_clock = min_clock + TICK_NSEC;
+        /*
+         * The clock must stay within a jiffie of the gtod.
+         * But since we may be at the start of a jiffy or the end of one
+         * we add another jiffy buffer.
+         */
+        max_clock = scd->tick_gtod + (2 + delta_jiffies) * TICK_NSEC;
+
+        delta *= scd->multi;
+        delta >>= MULTI_SHIFT;
 
-        if (unlikely(clock + delta > max_clock)) {
+        if (unlikely(clock + delta > max_clock) && check_max(scd)) {
                 if (clock < max_clock)
                         clock = max_clock;
                 else
@@ -118,9 +180,12 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now)
         if (unlikely(clock < min_clock))
                 clock = min_clock;
 
-        scd->prev_raw = now;
-        scd->prev_jiffies = now_jiffies;
-        scd->clock = clock;
+        if (time)
+                *time = clock;
+        else {
+                scd->prev_raw = now;
+                scd->clock = clock;
+        }
 }
 
 static void lock_double_clock(struct sched_clock_data *data1,
@@ -160,25 +225,30 @@ u64 sched_clock_cpu(int cpu)
                 now -= my_scd->tick_raw;
                 now += scd->tick_raw;
 
-                now -= my_scd->tick_gtod;
-                now += scd->tick_gtod;
+                now += my_scd->tick_gtod;
+                now -= scd->tick_gtod;
 
                 __raw_spin_unlock(&my_scd->lock);
+
+                __update_sched_clock(scd, now, &clock);
+
+                __raw_spin_unlock(&scd->lock);
+
         } else {
                 __raw_spin_lock(&scd->lock);
+                __update_sched_clock(scd, now, NULL);
+                clock = scd->clock;
+                __raw_spin_unlock(&scd->lock);
         }
 
-        __update_sched_clock(scd, now);
-        clock = scd->clock;
-
-        __raw_spin_unlock(&scd->lock);
-
         return clock;
 }
 
 void sched_clock_tick(void)
 {
         struct sched_clock_data *scd = this_scd();
+        unsigned long now_jiffies = jiffies;
+        s64 mult, delta_gtod, delta_raw;
         u64 now, now_gtod;
 
         if (unlikely(!sched_clock_running))
@@ -186,18 +256,33 @@ void sched_clock_tick(void)
 
         WARN_ON_ONCE(!irqs_disabled());
 
-        now = sched_clock();
         now_gtod = ktime_to_ns(ktime_get());
+        now = sched_clock();
 
         __raw_spin_lock(&scd->lock);
-        __update_sched_clock(scd, now);
+        __update_sched_clock(scd, now, NULL);
         /*
          * update tick_gtod after __update_sched_clock() because that will
          * already observe 1 new jiffy; adding a new tick_gtod to that would
          * increase the clock 2 jiffies.
          */
+        delta_gtod = now_gtod - scd->tick_gtod;
+        delta_raw = now - scd->tick_raw;
+
+        if ((long)delta_raw > 0) {
+                mult = delta_gtod << MULTI_SHIFT;
+                do_div(mult, delta_raw);
+                scd->multi = mult;
+                if (scd->multi > MAX_MULTI)
+                        scd->multi = MAX_MULTI;
+                else if (scd->multi < MIN_MULTI)
+                        scd->multi = MIN_MULTI;
+        } else
+                scd->multi = 1 << MULTI_SHIFT;
+
         scd->tick_raw = now;
         scd->tick_gtod = now_gtod;
+        scd->tick_jiffies = now_jiffies;
         __raw_spin_unlock(&scd->lock);
 }
 
@@ -227,6 +312,7 @@ void sched_clock_idle_wakeup_event(u64 delta_ns)
         __raw_spin_lock(&scd->lock);
         scd->prev_raw = now;
         scd->clock += delta_ns;
+        scd->multi = 1 << MULTI_SHIFT;
         __raw_spin_unlock(&scd->lock);
 
         touch_softlockup_watchdog();
@@ -244,3 +330,16 @@ unsigned long long __attribute__((weak)) sched_clock(void)
 {
         return (unsigned long long)jiffies * (NSEC_PER_SEC / HZ);
 }
+
+unsigned long long cpu_clock(int cpu)
+{
+        unsigned long long clock;
+        unsigned long flags;
+
+        local_irq_save(flags);
+        clock = sched_clock_cpu(cpu);
+        local_irq_restore(flags);
+
+        return clock;
+}
+EXPORT_SYMBOL_GPL(cpu_clock);
diff --git a/kernel/sched_cpupri.c b/kernel/sched_cpupri.c
new file mode 100644
index 000000000000..52154fefab7e
--- /dev/null
+++ b/kernel/sched_cpupri.c
@@ -0,0 +1,174 @@
+/*
+ *  kernel/sched_cpupri.c
+ *
+ *  CPU priority management
+ *
+ *  Copyright (C) 2007-2008 Novell
+ *
+ *  Author: Gregory Haskins <ghaskins@novell.com>
+ *
+ *  This code tracks the priority of each CPU so that global migration
+ *  decisions are easy to calculate.  Each CPU can be in a state as follows:
+ *
+ *                 (INVALID), IDLE, NORMAL, RT1, ... RT99
+ *
+ *  going from the lowest priority to the highest.  CPUs in the INVALID state
+ *  are not eligible for routing.  The system maintains this state with
+ *  a 2 dimensional bitmap (the first for priority class, the second for cpus
+ *  in that class).  Therefore a typical application without affinity
+ *  restrictions can find a suitable CPU with O(1) complexity (e.g. two bit
+ *  searches).  For tasks with affinity restrictions, the algorithm has a
+ *  worst case complexity of O(min(102, nr_domcpus)), though the scenario that
+ *  yields the worst case search is fairly contrived.
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; version 2
+ *  of the License.
+ */
+
+#include "sched_cpupri.h"
+
+/* Convert between a 140 based task->prio, and our 102 based cpupri */
+static int convert_prio(int prio)
+{
+        int cpupri;
+
+        if (prio == CPUPRI_INVALID)
+                cpupri = CPUPRI_INVALID;
+        else if (prio == MAX_PRIO)
+                cpupri = CPUPRI_IDLE;
+        else if (prio >= MAX_RT_PRIO)
+                cpupri = CPUPRI_NORMAL;
+        else
+                cpupri = MAX_RT_PRIO - prio + 1;
+
+        return cpupri;
+}
+
+#define for_each_cpupri_active(array, idx)                    \
+  for (idx = find_first_bit(array, CPUPRI_NR_PRIORITIES);     \
+       idx < CPUPRI_NR_PRIORITIES;                            \
+       idx = find_next_bit(array, CPUPRI_NR_PRIORITIES, idx+1))
+
+/**
+ * cpupri_find - find the best (lowest-pri) CPU in the system
+ * @cp: The cpupri context
+ * @p: The task
+ * @lowest_mask: A mask to fill in with selected CPUs
+ *
+ * Note: This function returns the recommended CPUs as calculated during the
+ * current invokation.  By the time the call returns, the CPUs may have in
+ * fact changed priorities any number of times.  While not ideal, it is not
+ * an issue of correctness since the normal rebalancer logic will correct
+ * any discrepancies created by racing against the uncertainty of the current
+ * priority configuration.
+ *
+ * Returns: (int)bool - CPUs were found
+ */
+int cpupri_find(struct cpupri *cp, struct task_struct *p,
+                cpumask_t *lowest_mask)
+{
+        int                  idx      = 0;
+        int                  task_pri = convert_prio(p->prio);
+
+        for_each_cpupri_active(cp->pri_active, idx) {
+                struct cpupri_vec *vec  = &cp->pri_to_cpu[idx];
+                cpumask_t mask;
+
+                if (idx >= task_pri)
+                        break;
+
+                cpus_and(mask, p->cpus_allowed, vec->mask);
+
+                if (cpus_empty(mask))
+                        continue;
+
+                *lowest_mask = mask;
+                return 1;
+        }
+
+        return 0;
+}
+
+/**
+ * cpupri_set - update the cpu priority setting
+ * @cp: The cpupri context
+ * @cpu: The target cpu
+ * @pri: The priority (INVALID-RT99) to assign to this CPU
+ *
+ * Note: Assumes cpu_rq(cpu)->lock is locked
+ *
+ * Returns: (void)
+ */
+void cpupri_set(struct cpupri *cp, int cpu, int newpri)
+{
+        int                 *currpri = &cp->cpu_to_pri[cpu];
+        int                  oldpri  = *currpri;
+        unsigned long        flags;
+
+        newpri = convert_prio(newpri);
+
+        BUG_ON(newpri >= CPUPRI_NR_PRIORITIES);
+
+        if (newpri == oldpri)
+                return;
+
+        /*
+         * If the cpu was currently mapped to a different value, we
+         * first need to unmap the old value
+         */
+        if (likely(oldpri != CPUPRI_INVALID)) {
+                struct cpupri_vec *vec  = &cp->pri_to_cpu[oldpri];
+
+                spin_lock_irqsave(&vec->lock, flags);
+
+                vec->count--;
+                if (!vec->count)
+                        clear_bit(oldpri, cp->pri_active);
+                cpu_clear(cpu, vec->mask);
+
+                spin_unlock_irqrestore(&vec->lock, flags);
+        }
+
+        if (likely(newpri != CPUPRI_INVALID)) {
+                struct cpupri_vec *vec = &cp->pri_to_cpu[newpri];
+
+                spin_lock_irqsave(&vec->lock, flags);
+
+                cpu_set(cpu, vec->mask);
+                vec->count++;
+                if (vec->count == 1)
+                        set_bit(newpri, cp->pri_active);
+
+                spin_unlock_irqrestore(&vec->lock, flags);
+        }
+
+        *currpri = newpri;
+}
+
+/**
+ * cpupri_init - initialize the cpupri structure
+ * @cp: The cpupri context
+ *
+ * Returns: (void)
+ */
+void cpupri_init(struct cpupri *cp)
+{
+        int i;
+
+        memset(cp, 0, sizeof(*cp));
+
+        for (i = 0; i < CPUPRI_NR_PRIORITIES; i++) {
+                struct cpupri_vec *vec = &cp->pri_to_cpu[i];
+
+                spin_lock_init(&vec->lock);
+                vec->count = 0;
+                cpus_clear(vec->mask);
+        }
+
+        for_each_possible_cpu(i)
+                cp->cpu_to_pri[i] = CPUPRI_INVALID;
+}
+
+
diff --git a/kernel/sched_cpupri.h b/kernel/sched_cpupri.h
new file mode 100644
index 000000000000..f25811b0f931
--- /dev/null
+++ b/kernel/sched_cpupri.h
@@ -0,0 +1,36 @@
+#ifndef _LINUX_CPUPRI_H
+#define _LINUX_CPUPRI_H
+
+#include <linux/sched.h>
+
+#define CPUPRI_NR_PRIORITIES    (MAX_RT_PRIO + 2)
+#define CPUPRI_NR_PRI_WORDS     BITS_TO_LONGS(CPUPRI_NR_PRIORITIES)
+
+#define CPUPRI_INVALID -1
+#define CPUPRI_IDLE     0
+#define CPUPRI_NORMAL   1
+/* values 2-101 are RT priorities 0-99 */
+
+struct cpupri_vec {
+        spinlock_t lock;
+        int        count;
+        cpumask_t  mask;
+};
+
+struct cpupri {
+        struct cpupri_vec pri_to_cpu[CPUPRI_NR_PRIORITIES];
+        long              pri_active[CPUPRI_NR_PRI_WORDS];
+        int               cpu_to_pri[NR_CPUS];
+};
+
+#ifdef CONFIG_SMP
+int  cpupri_find(struct cpupri *cp,
+                 struct task_struct *p, cpumask_t *lowest_mask);
+void cpupri_set(struct cpupri *cp, int cpu, int pri);
+void cpupri_init(struct cpupri *cp);
+#else
+#define cpupri_set(cp, cpu, pri) do { } while (0)
+#define cpupri_init() do { } while (0)
+#endif
+
+#endif /* _LINUX_CPUPRI_H */
diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c
index 8bb713040ac9..bbe6b31c3c56 100644
--- a/kernel/sched_debug.c
+++ b/kernel/sched_debug.c
@@ -119,9 +119,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
         struct sched_entity *last;
         unsigned long flags;
 
-#if !defined(CONFIG_CGROUP_SCHED) || !defined(CONFIG_USER_SCHED)
-        SEQ_printf(m, "\ncfs_rq[%d]:\n", cpu);
-#else
+#if defined(CONFIG_CGROUP_SCHED) && defined(CONFIG_FAIR_GROUP_SCHED)
         char path[128] = "";
         struct cgroup *cgroup = NULL;
         struct task_group *tg = cfs_rq->tg;
@@ -133,6 +131,8 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
                 cgroup_path(cgroup, path, sizeof(path));
 
         SEQ_printf(m, "\ncfs_rq[%d]:%s\n", cpu, path);
+#else
+        SEQ_printf(m, "\ncfs_rq[%d]:\n", cpu);
 #endif
 
         SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "exec_clock",
@@ -162,11 +162,64 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
         SEQ_printf(m, "  .%-30s: %ld\n", "nr_running", cfs_rq->nr_running);
         SEQ_printf(m, "  .%-30s: %ld\n", "load", cfs_rq->load.weight);
 #ifdef CONFIG_SCHEDSTATS
-        SEQ_printf(m, "  .%-30s: %d\n", "bkl_count",
-                        rq->bkl_count);
+#define P(n) SEQ_printf(m, "  .%-30s: %d\n", #n, rq->n);
+
+        P(yld_exp_empty);
+        P(yld_act_empty);
+        P(yld_both_empty);
+        P(yld_count);
+
+        P(sched_switch);
+        P(sched_count);
+        P(sched_goidle);
+
+        P(ttwu_count);
+        P(ttwu_local);
+
+        P(bkl_count);
+
+#undef P
 #endif
         SEQ_printf(m, "  .%-30s: %ld\n", "nr_spread_over",
                         cfs_rq->nr_spread_over);
+#ifdef CONFIG_FAIR_GROUP_SCHED
+#ifdef CONFIG_SMP
+        SEQ_printf(m, "  .%-30s: %lu\n", "shares", cfs_rq->shares);
+#endif
+#endif
+}
+
+void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq)
+{
+#if defined(CONFIG_CGROUP_SCHED) && defined(CONFIG_RT_GROUP_SCHED)
+        char path[128] = "";
+        struct cgroup *cgroup = NULL;
+        struct task_group *tg = rt_rq->tg;
+
+        if (tg)
+                cgroup = tg->css.cgroup;
+
+        if (cgroup)
+                cgroup_path(cgroup, path, sizeof(path));
+
+        SEQ_printf(m, "\nrt_rq[%d]:%s\n", cpu, path);
+#else
+        SEQ_printf(m, "\nrt_rq[%d]:\n", cpu);
+#endif
+
+
+#define P(x) \
+        SEQ_printf(m, "  .%-30s: %Ld\n", #x, (long long)(rt_rq->x))
+#define PN(x) \
+        SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rt_rq->x))
+
+        P(rt_nr_running);
+        P(rt_throttled);
+        PN(rt_time);
+        PN(rt_runtime);
+
+#undef PN
+#undef P
 }
 
 static void print_cpu(struct seq_file *m, int cpu)
@@ -208,6 +261,7 @@ static void print_cpu(struct seq_file *m, int cpu)
 #undef PN
 
         print_cfs_stats(m, cpu);
+        print_rt_stats(m, cpu);
 
         print_rq(m, rq, cpu);
 }
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 08ae848b71d4..f2aa987027d6 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -63,13 +63,13 @@ unsigned int __read_mostly sysctl_sched_compat_yield;
 
 /*
  * SCHED_OTHER wake-up granularity.
- * (default: 10 msec * (1 + ilog(ncpus)), units: nanoseconds)
+ * (default: 5 msec * (1 + ilog(ncpus)), units: nanoseconds)
  *
  * This option delays the preemption effects of decoupled workloads
  * and reduces their over-scheduling. Synchronous workloads will still
  * have immediate wakeup/sleep latencies.
  */
-unsigned int sysctl_sched_wakeup_granularity = 10000000UL;
+unsigned int sysctl_sched_wakeup_granularity = 5000000UL;
 
 const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
 
@@ -334,6 +334,34 @@ int sched_nr_latency_handler(struct ctl_table *table, int write,
 #endif
 
 /*
+ * delta *= w / rw
+ */
+static inline unsigned long
+calc_delta_weight(unsigned long delta, struct sched_entity *se)
+{
+        for_each_sched_entity(se) {
+                delta = calc_delta_mine(delta,
+                                se->load.weight, &cfs_rq_of(se)->load);
+        }
+
+        return delta;
+}
+
+/*
+ * delta *= rw / w
+ */
+static inline unsigned long
+calc_delta_fair(unsigned long delta, struct sched_entity *se)
+{
+        for_each_sched_entity(se) {
+                delta = calc_delta_mine(delta,
+                                cfs_rq_of(se)->load.weight, &se->load);
+        }
+
+        return delta;
+}
+
+/*
  * The idea is to set a period in which each task runs once.
  *
  * When there are too many tasks (sysctl_sched_nr_latency) we have to stretch
@@ -362,47 +390,80 @@ static u64 __sched_period(unsigned long nr_running)
  */
 static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-        u64 slice = __sched_period(cfs_rq->nr_running);
-
-        for_each_sched_entity(se) {
-                cfs_rq = cfs_rq_of(se);
-
-                slice *= se->load.weight;
-                do_div(slice, cfs_rq->load.weight);
-        }
-
-
-        return slice;
+        return calc_delta_weight(__sched_period(cfs_rq->nr_running), se);
 }
 
 /*
  * We calculate the vruntime slice of a to be inserted task
  *
- * vs = s/w = p/rw
+ * vs = s*rw/w = p
  */
 static u64 sched_vslice_add(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         unsigned long nr_running = cfs_rq->nr_running;
-        unsigned long weight;
-        u64 vslice;
 
         if (!se->on_rq)
                 nr_running++;
 
-        vslice = __sched_period(nr_running);
+        return __sched_period(nr_running);
+}
+
+/*
+ * The goal of calc_delta_asym() is to be asymmetrically around NICE_0_LOAD, in
+ * that it favours >=0 over <0.
+ *
+ *   -20         |
+ *               |
+ *     0 --------+-------
+ *             .'
+ *    19     .'
+ *
+ */
+static unsigned long
+calc_delta_asym(unsigned long delta, struct sched_entity *se)
+{
+        struct load_weight lw = {
+                .weight = NICE_0_LOAD,
+                .inv_weight = 1UL << (WMULT_SHIFT-NICE_0_SHIFT)
+        };
 
         for_each_sched_entity(se) {
-                cfs_rq = cfs_rq_of(se);
+                struct load_weight *se_lw = &se->load;
+                unsigned long rw = cfs_rq_of(se)->load.weight;
+
+#ifdef CONFIG_FAIR_SCHED_GROUP
+                struct cfs_rq *cfs_rq = se->my_q;
+                struct task_group *tg = NULL
+
+                if (cfs_rq)
+                        tg = cfs_rq->tg;
+
+                if (tg && tg->shares < NICE_0_LOAD) {
+                        /*
+                         * scale shares to what it would have been had
+                         * tg->weight been NICE_0_LOAD:
+                         *
+                         *   weight = 1024 * shares / tg->weight
+                         */
+                        lw.weight *= se->load.weight;
+                        lw.weight /= tg->shares;
+
+                        lw.inv_weight = 0;
+
+                        se_lw = &lw;
+                        rw += lw.weight - se->load.weight;
+                } else
+#endif
 
-                weight = cfs_rq->load.weight;
-                if (!se->on_rq)
-                        weight += se->load.weight;
+                if (se->load.weight < NICE_0_LOAD) {
+                        se_lw = &lw;
+                        rw += NICE_0_LOAD - se->load.weight;
+                }
 
-                vslice *= NICE_0_LOAD;
-                do_div(vslice, weight);
+                delta = calc_delta_mine(delta, rw, se_lw);
         }
 
-        return vslice;
+        return delta;
 }
 
 /*
@@ -419,11 +480,7 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr,
 
         curr->sum_exec_runtime += delta_exec;
         schedstat_add(cfs_rq, exec_clock, delta_exec);
-        delta_exec_weighted = delta_exec;
-        if (unlikely(curr->load.weight != NICE_0_LOAD)) {
-                delta_exec_weighted = calc_delta_fair(delta_exec_weighted,
-                                                        &curr->load);
-        }
+        delta_exec_weighted = calc_delta_fair(delta_exec, curr);
         curr->vruntime += delta_exec_weighted;
 }
 
@@ -510,10 +567,27 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
  * Scheduling class queueing methods:
  */
 
+#if defined CONFIG_SMP && defined CONFIG_FAIR_GROUP_SCHED
+static void
+add_cfs_task_weight(struct cfs_rq *cfs_rq, unsigned long weight)
+{
+        cfs_rq->task_weight += weight;
+}
+#else
+static inline void
+add_cfs_task_weight(struct cfs_rq *cfs_rq, unsigned long weight)
+{
+}
+#endif
+
 static void
 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         update_load_add(&cfs_rq->load, se->load.weight);
+        if (!parent_entity(se))
+                inc_cpu_load(rq_of(cfs_rq), se->load.weight);
+        if (entity_is_task(se))
+                add_cfs_task_weight(cfs_rq, se->load.weight);
         cfs_rq->nr_running++;
         se->on_rq = 1;
         list_add(&se->group_node, &cfs_rq->tasks);
@@ -523,6 +597,10 @@ static void
 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         update_load_sub(&cfs_rq->load, se->load.weight);
+        if (!parent_entity(se))
+                dec_cpu_load(rq_of(cfs_rq), se->load.weight);
+        if (entity_is_task(se))
+                add_cfs_task_weight(cfs_rq, -se->load.weight);
         cfs_rq->nr_running--;
         se->on_rq = 0;
         list_del_init(&se->group_node);
@@ -609,8 +687,17 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
 
         if (!initial) {
                 /* sleeps upto a single latency don't count. */
-                if (sched_feat(NEW_FAIR_SLEEPERS))
-                        vruntime -= sysctl_sched_latency;
+                if (sched_feat(NEW_FAIR_SLEEPERS)) {
+                        unsigned long thresh = sysctl_sched_latency;
+
+                        /*
+                         * convert the sleeper threshold into virtual time
+                         */
+                        if (sched_feat(NORMALIZED_SLEEPER))
+                                thresh = calc_delta_fair(thresh, se);
+
+                        vruntime -= thresh;
+                }
 
                 /* ensure we never gain time by being placed backwards. */
                 vruntime = max_vruntime(se->vruntime, vruntime);
@@ -639,21 +726,6 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int wakeup)
                 __enqueue_entity(cfs_rq, se);
 }
 
-static void update_avg(u64 *avg, u64 sample)
-{
-        s64 diff = sample - *avg;
-        *avg += diff >> 3;
-}
-
-static void update_avg_stats(struct cfs_rq *cfs_rq, struct sched_entity *se)
-{
-        if (!se->last_wakeup)
-                return;
-
-        update_avg(&se->avg_overlap, se->sum_exec_runtime - se->last_wakeup);
-        se->last_wakeup = 0;
-}
-
 static void
 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
 {
@@ -664,7 +736,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
 
         update_stats_dequeue(cfs_rq, se);
         if (sleep) {
-                update_avg_stats(cfs_rq, se);
 #ifdef CONFIG_SCHEDSTATS
                 if (entity_is_task(se)) {
                         struct task_struct *tsk = task_of(se);
@@ -726,17 +797,16 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
         se->prev_sum_exec_runtime = se->sum_exec_runtime;
 }
 
-static int
-wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se);
-
 static struct sched_entity *
 pick_next(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-        if (!cfs_rq->next)
-                return se;
+        struct rq *rq = rq_of(cfs_rq);
+        u64 pair_slice = rq->clock - cfs_rq->pair_start;
 
-        if (wakeup_preempt_entity(cfs_rq->next, se) != 0)
+        if (!cfs_rq->next || pair_slice > sched_slice(cfs_rq, cfs_rq->next)) {
+                cfs_rq->pair_start = rq->clock;
                 return se;
+        }
 
         return cfs_rq->next;
 }
@@ -835,7 +905,7 @@ static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
                 hrtick_start(rq, delta, requeue);
         }
 }
-#else
+#else /* !CONFIG_SCHED_HRTICK */
 static inline void
 hrtick_start_fair(struct rq *rq, struct task_struct *p)
 {
@@ -976,7 +1046,7 @@ static int wake_idle(int cpu, struct task_struct *p)
         }
         return cpu;
 }
-#else
+#else /* !ARCH_HAS_SCHED_WAKE_IDLE*/
 static inline int wake_idle(int cpu, struct task_struct *p)
 {
         return cpu;
@@ -987,6 +1057,89 @@ static inline int wake_idle(int cpu, struct task_struct *p)
 
 static const struct sched_class fair_sched_class;
 
+#ifdef CONFIG_FAIR_GROUP_SCHED
+/*
+ * effective_load() calculates the load change as seen from the root_task_group
+ *
+ * Adding load to a group doesn't make a group heavier, but can cause movement
+ * of group shares between cpus. Assuming the shares were perfectly aligned one
+ * can calculate the shift in shares.
+ *
+ * The problem is that perfectly aligning the shares is rather expensive, hence
+ * we try to avoid doing that too often - see update_shares(), which ratelimits
+ * this change.
+ *
+ * We compensate this by not only taking the current delta into account, but
+ * also considering the delta between when the shares were last adjusted and
+ * now.
+ *
+ * We still saw a performance dip, some tracing learned us that between
+ * cgroup:/ and cgroup:/foo balancing the number of affine wakeups increased
+ * significantly. Therefore try to bias the error in direction of failing
+ * the affine wakeup.
+ *
+ */
+static long effective_load(struct task_group *tg, int cpu,
+                long wl, long wg)
+{
+        struct sched_entity *se = tg->se[cpu];
+        long more_w;
+
+        if (!tg->parent)
+                return wl;
+
+        /*
+         * By not taking the decrease of shares on the other cpu into
+         * account our error leans towards reducing the affine wakeups.
+         */
+        if (!wl && sched_feat(ASYM_EFF_LOAD))
+                return wl;
+
+        /*
+         * Instead of using this increment, also add the difference
+         * between when the shares were last updated and now.
+         */
+        more_w = se->my_q->load.weight - se->my_q->rq_weight;
+        wl += more_w;
+        wg += more_w;
+
+        for_each_sched_entity(se) {
+#define D(n) (likely(n) ? (n) : 1)
+
+                long S, rw, s, a, b;
+
+                S = se->my_q->tg->shares;
+                s = se->my_q->shares;
+                rw = se->my_q->rq_weight;
+
+                a = S*(rw + wl);
+                b = S*rw + s*wg;
+
+                wl = s*(a-b)/D(b);
+                /*
+                 * Assume the group is already running and will
+                 * thus already be accounted for in the weight.
+                 *
+                 * That is, moving shares between CPUs, does not
+                 * alter the group weight.
+                 */
+                wg = 0;
+#undef D
+        }
+
+        return wl;
+}
+
+#else
+
+static inline unsigned long effective_load(struct task_group *tg, int cpu,
+                unsigned long wl, unsigned long wg)
+{
+        return wl;
+}
+
+#endif
+
 static int
 wake_affine(struct rq *rq, struct sched_domain *this_sd, struct rq *this_rq,
             struct task_struct *p, int prev_cpu, int this_cpu, int sync,
@@ -994,8 +1147,10 @@ wake_affine(struct rq *rq, struct sched_domain *this_sd, struct rq *this_rq,
             unsigned int imbalance)
 {
         struct task_struct *curr = this_rq->curr;
+        struct task_group *tg;
         unsigned long tl = this_load;
         unsigned long tl_per_task;
+        unsigned long weight;
         int balanced;
 
         if (!(this_sd->flags & SD_WAKE_AFFINE) || !sched_feat(AFFINE_WAKEUPS))
@@ -1006,19 +1161,28 @@ wake_affine(struct rq *rq, struct sched_domain *this_sd, struct rq *this_rq,
          * effect of the currently running task from the load
          * of the current CPU:
          */
-        if (sync)
-                tl -= current->se.load.weight;
+        if (sync) {
+                tg = task_group(current);
+                weight = current->se.load.weight;
+
+                tl += effective_load(tg, this_cpu, -weight, -weight);
+                load += effective_load(tg, prev_cpu, 0, -weight);
+        }
 
-        balanced = 100*(tl + p->se.load.weight) <= imbalance*load;
+        tg = task_group(p);
+        weight = p->se.load.weight;
+
+        balanced = 100*(tl + effective_load(tg, this_cpu, weight, weight)) <=
+                imbalance*(load + effective_load(tg, prev_cpu, 0, weight));
 
         /*
          * If the currently running task will sleep within
          * a reasonable amount of time then attract this newly
          * woken task:
          */
-        if (sync && balanced && curr->sched_class == &fair_sched_class) {
+        if (sync && balanced) {
                 if (curr->se.avg_overlap < sysctl_sched_migration_cost &&
-                                p->se.avg_overlap < sysctl_sched_migration_cost)
+                    p->se.avg_overlap < sysctl_sched_migration_cost)
                         return 1;
         }
 
@@ -1111,11 +1275,13 @@ static unsigned long wakeup_gran(struct sched_entity *se)
         unsigned long gran = sysctl_sched_wakeup_granularity;
 
         /*
-         * More easily preempt - nice tasks, while not making
-         * it harder for + nice tasks.
+         * More easily preempt - nice tasks, while not making it harder for
+         * + nice tasks.
          */
-        if (unlikely(se->load.weight > NICE_0_LOAD))
-                gran = calc_delta_fair(gran, &se->load);
+        if (sched_feat(ASYM_GRAN))
+                gran = calc_delta_asym(sysctl_sched_wakeup_granularity, se);
+        else
+                gran = calc_delta_fair(sysctl_sched_wakeup_granularity, se);
 
         return gran;
 }
@@ -1177,7 +1343,6 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
                 return;
         }
 
-        se->last_wakeup = se->sum_exec_runtime;
         if (unlikely(se == pse))
                 return;
 
@@ -1275,23 +1440,18 @@ __load_balance_iterator(struct cfs_rq *cfs_rq, struct list_head *next)
         struct task_struct *p = NULL;
         struct sched_entity *se;
 
-        if (next == &cfs_rq->tasks)
-                return NULL;
-
-        /* Skip over entities that are not tasks */
-        do {
+        while (next != &cfs_rq->tasks) {
                 se = list_entry(next, struct sched_entity, group_node);
                 next = next->next;
-        } while (next != &cfs_rq->tasks && !entity_is_task(se));
 
-        if (next == &cfs_rq->tasks)
-                return NULL;
+                /* Skip over entities that are not tasks */
+                if (entity_is_task(se)) {
+                        p = task_of(se);
+                        break;
+                }
+        }
 
         cfs_rq->balance_iterator = next;
-
-        if (entity_is_task(se))
-                p = task_of(se);
-
         return p;
 }
 
@@ -1309,75 +1469,82 @@ static struct task_struct *load_balance_next_fair(void *arg)
         return __load_balance_iterator(cfs_rq, cfs_rq->balance_iterator);
 }
 
-#ifdef CONFIG_FAIR_GROUP_SCHED
-static int cfs_rq_best_prio(struct cfs_rq *cfs_rq)
+static unsigned long
+__load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
+                unsigned long max_load_move, struct sched_domain *sd,
+                enum cpu_idle_type idle, int *all_pinned, int *this_best_prio,
+                struct cfs_rq *cfs_rq)
 {
-        struct sched_entity *curr;
-        struct task_struct *p;
-
-        if (!cfs_rq->nr_running || !first_fair(cfs_rq))
-                return MAX_PRIO;
-
-        curr = cfs_rq->curr;
-        if (!curr)
-                curr = __pick_next_entity(cfs_rq);
+        struct rq_iterator cfs_rq_iterator;
 
-        p = task_of(curr);
+        cfs_rq_iterator.start = load_balance_start_fair;
+        cfs_rq_iterator.next = load_balance_next_fair;
+        cfs_rq_iterator.arg = cfs_rq;
 
-        return p->prio;
+        return balance_tasks(this_rq, this_cpu, busiest,
+                        max_load_move, sd, idle, all_pinned,
+                        this_best_prio, &cfs_rq_iterator);
 }
-#endif
 
+#ifdef CONFIG_FAIR_GROUP_SCHED
 static unsigned long
 load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
                   unsigned long max_load_move,
                   struct sched_domain *sd, enum cpu_idle_type idle,
                   int *all_pinned, int *this_best_prio)
 {
-        struct cfs_rq *busy_cfs_rq;
         long rem_load_move = max_load_move;
-        struct rq_iterator cfs_rq_iterator;
-
-        cfs_rq_iterator.start = load_balance_start_fair;
-        cfs_rq_iterator.next = load_balance_next_fair;
+        int busiest_cpu = cpu_of(busiest);
+        struct task_group *tg;
 
-        for_each_leaf_cfs_rq(busiest, busy_cfs_rq) {
-#ifdef CONFIG_FAIR_GROUP_SCHED
-                struct cfs_rq *this_cfs_rq;
-                long imbalance;
-                unsigned long maxload;
+        rcu_read_lock();
+        update_h_load(busiest_cpu);
 
-                this_cfs_rq = cpu_cfs_rq(busy_cfs_rq, this_cpu);
+        list_for_each_entry(tg, &task_groups, list) {
+                struct cfs_rq *busiest_cfs_rq = tg->cfs_rq[busiest_cpu];
+                unsigned long busiest_h_load = busiest_cfs_rq->h_load;
+                unsigned long busiest_weight = busiest_cfs_rq->load.weight;
+                u64 rem_load, moved_load;
 
-                imbalance = busy_cfs_rq->load.weight - this_cfs_rq->load.weight;
-                /* Don't pull if this_cfs_rq has more load than busy_cfs_rq */
-                if (imbalance <= 0)
+                /*
+                 * empty group
+                 */
+                if (!busiest_cfs_rq->task_weight)
                         continue;
 
-                /* Don't pull more than imbalance/2 */
-                imbalance /= 2;
-                maxload = min(rem_load_move, imbalance);
+                rem_load = (u64)rem_load_move * busiest_weight;
+                rem_load = div_u64(rem_load, busiest_h_load + 1);
 
-                *this_best_prio = cfs_rq_best_prio(this_cfs_rq);
-#else
-# define maxload rem_load_move
-#endif
-                /*
-                 * pass busy_cfs_rq argument into
-                 * load_balance_[start|next]_fair iterators
-                 */
-                cfs_rq_iterator.arg = busy_cfs_rq;
-                rem_load_move -= balance_tasks(this_rq, this_cpu, busiest,
-                                               maxload, sd, idle, all_pinned,
-                                               this_best_prio,
-                                               &cfs_rq_iterator);
+                moved_load = __load_balance_fair(this_rq, this_cpu, busiest,
+                                rem_load, sd, idle, all_pinned, this_best_prio,
+                                tg->cfs_rq[busiest_cpu]);
+
+                if (!moved_load)
+                        continue;
+
+                moved_load *= busiest_h_load;
+                moved_load = div_u64(moved_load, busiest_weight + 1);
 
-                if (rem_load_move <= 0)
+                rem_load_move -= moved_load;
+                if (rem_load_move < 0)
                         break;
         }
+        rcu_read_unlock();
 
         return max_load_move - rem_load_move;
 }
+#else
+static unsigned long
+load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
+                  unsigned long max_load_move,
+                  struct sched_domain *sd, enum cpu_idle_type idle,
+                  int *all_pinned, int *this_best_prio)
+{
+        return __load_balance_fair(this_rq, this_cpu, busiest,
+                        max_load_move, sd, idle, all_pinned,
+                        this_best_prio, &busiest->cfs);
+}
+#endif
 
 static int
 move_one_task_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
@@ -1402,7 +1569,7 @@ move_one_task_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
 
         return 0;
 }
-#endif
+#endif /* CONFIG_SMP */
 
 /*
  * scheduler tick hitting a task of our scheduling class:
diff --git a/kernel/sched_features.h b/kernel/sched_features.h
index 1c7283cb9581..862b06bd560a 100644
--- a/kernel/sched_features.h
+++ b/kernel/sched_features.h
@@ -1,4 +1,5 @@
 SCHED_FEAT(NEW_FAIR_SLEEPERS, 1)
+SCHED_FEAT(NORMALIZED_SLEEPER, 1)
 SCHED_FEAT(WAKEUP_PREEMPT, 1)
 SCHED_FEAT(START_DEBIT, 1)
 SCHED_FEAT(AFFINE_WAKEUPS, 1)
@@ -6,5 +7,7 @@ SCHED_FEAT(CACHE_HOT_BUDDY, 1)
 SCHED_FEAT(SYNC_WAKEUPS, 1)
 SCHED_FEAT(HRTICK, 1)
 SCHED_FEAT(DOUBLE_TICK, 0)
-SCHED_FEAT(NORMALIZED_SLEEPER, 1)
-SCHED_FEAT(DEADLINE, 1)
+SCHED_FEAT(ASYM_GRAN, 1)
+SCHED_FEAT(LB_BIAS, 0)
+SCHED_FEAT(LB_WAKEUP_UPDATE, 1)
+SCHED_FEAT(ASYM_EFF_LOAD, 1)
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index 3432d573205d..47ceac9e8552 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -12,6 +12,9 @@ static inline int rt_overloaded(struct rq *rq)
 
 static inline void rt_set_overload(struct rq *rq)
 {
+        if (!rq->online)
+                return;
+
         cpu_set(rq->cpu, rq->rd->rto_mask);
         /*
          * Make sure the mask is visible before we set
@@ -26,6 +29,9 @@ static inline void rt_set_overload(struct rq *rq)
 
 static inline void rt_clear_overload(struct rq *rq)
 {
+        if (!rq->online)
+                return;
+
         /* the order here really doesn't matter */
         atomic_dec(&rq->rd->rto_count);
         cpu_clear(rq->cpu, rq->rd->rto_mask);
@@ -155,7 +161,7 @@ static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
         return &rt_rq->tg->rt_bandwidth;
 }
 
-#else
+#else /* !CONFIG_RT_GROUP_SCHED */
 
 static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
 {
@@ -220,48 +226,10 @@ static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
         return &def_rt_bandwidth;
 }
 
-#endif
-
-static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
-{
-        int i, idle = 1;
-        cpumask_t span;
-
-        if (rt_b->rt_runtime == RUNTIME_INF)
-                return 1;
-
-        span = sched_rt_period_mask();
-        for_each_cpu_mask(i, span) {
-                int enqueue = 0;
-                struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
-                struct rq *rq = rq_of_rt_rq(rt_rq);
-
-                spin_lock(&rq->lock);
-                if (rt_rq->rt_time) {
-                        u64 runtime;
-
-                        spin_lock(&rt_rq->rt_runtime_lock);
-                        runtime = rt_rq->rt_runtime;
-                        rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);
-                        if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {
-                                rt_rq->rt_throttled = 0;
-                                enqueue = 1;
-                        }
-                        if (rt_rq->rt_time || rt_rq->rt_nr_running)
-                                idle = 0;
-                        spin_unlock(&rt_rq->rt_runtime_lock);
-                }
-
-                if (enqueue)
-                        sched_rt_rq_enqueue(rt_rq);
-                spin_unlock(&rq->lock);
-        }
-
-        return idle;
-}
+#endif /* CONFIG_RT_GROUP_SCHED */
 
 #ifdef CONFIG_SMP
-static int balance_runtime(struct rt_rq *rt_rq)
+static int do_balance_runtime(struct rt_rq *rt_rq)
 {
         struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
         struct root_domain *rd = cpu_rq(smp_processor_id())->rd;
@@ -280,6 +248,9 @@ static int balance_runtime(struct rt_rq *rt_rq)
                         continue;
 
                 spin_lock(&iter->rt_runtime_lock);
+                if (iter->rt_runtime == RUNTIME_INF)
+                        goto next;
+
                 diff = iter->rt_runtime - iter->rt_time;
                 if (diff > 0) {
                         do_div(diff, weight);
@@ -293,13 +264,163 @@ static int balance_runtime(struct rt_rq *rt_rq)
                                 break;
                         }
                 }
+next:
                 spin_unlock(&iter->rt_runtime_lock);
         }
         spin_unlock(&rt_b->rt_runtime_lock);
 
         return more;
 }
-#endif
+
+static void __disable_runtime(struct rq *rq)
+{
+        struct root_domain *rd = rq->rd;
+        struct rt_rq *rt_rq;
+
+        if (unlikely(!scheduler_running))
+                return;
+
+        for_each_leaf_rt_rq(rt_rq, rq) {
+                struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
+                s64 want;
+                int i;
+
+                spin_lock(&rt_b->rt_runtime_lock);
+                spin_lock(&rt_rq->rt_runtime_lock);
+                if (rt_rq->rt_runtime == RUNTIME_INF ||
+                                rt_rq->rt_runtime == rt_b->rt_runtime)
+                        goto balanced;
+                spin_unlock(&rt_rq->rt_runtime_lock);
+
+                want = rt_b->rt_runtime - rt_rq->rt_runtime;
+
+                for_each_cpu_mask(i, rd->span) {
+                        struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
+                        s64 diff;
+
+                        if (iter == rt_rq)
+                                continue;
+
+                        spin_lock(&iter->rt_runtime_lock);
+                        if (want > 0) {
+                                diff = min_t(s64, iter->rt_runtime, want);
+                                iter->rt_runtime -= diff;
+                                want -= diff;
+                        } else {
+                                iter->rt_runtime -= want;
+                                want -= want;
+                        }
+                        spin_unlock(&iter->rt_runtime_lock);
+
+                        if (!want)
+                                break;
+                }
+
+                spin_lock(&rt_rq->rt_runtime_lock);
+                BUG_ON(want);
+balanced:
+                rt_rq->rt_runtime = RUNTIME_INF;
+                spin_unlock(&rt_rq->rt_runtime_lock);
+                spin_unlock(&rt_b->rt_runtime_lock);
+        }
+}
+
+static void disable_runtime(struct rq *rq)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&rq->lock, flags);
+        __disable_runtime(rq);
+        spin_unlock_irqrestore(&rq->lock, flags);
+}
+
+static void __enable_runtime(struct rq *rq)
+{
+        struct rt_rq *rt_rq;
+
+        if (unlikely(!scheduler_running))
+                return;
+
+        for_each_leaf_rt_rq(rt_rq, rq) {
+                struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
+
+                spin_lock(&rt_b->rt_runtime_lock);
+                spin_lock(&rt_rq->rt_runtime_lock);
+                rt_rq->rt_runtime = rt_b->rt_runtime;
+                rt_rq->rt_time = 0;
+                spin_unlock(&rt_rq->rt_runtime_lock);
+                spin_unlock(&rt_b->rt_runtime_lock);
+        }
+}
+
+static void enable_runtime(struct rq *rq)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&rq->lock, flags);
+        __enable_runtime(rq);
+        spin_unlock_irqrestore(&rq->lock, flags);
+}
+
+static int balance_runtime(struct rt_rq *rt_rq)
+{
+        int more = 0;
+
+        if (rt_rq->rt_time > rt_rq->rt_runtime) {
+                spin_unlock(&rt_rq->rt_runtime_lock);
+                more = do_balance_runtime(rt_rq);
+                spin_lock(&rt_rq->rt_runtime_lock);
+        }
+
+        return more;
+}
+#else /* !CONFIG_SMP */
+static inline int balance_runtime(struct rt_rq *rt_rq)
+{
+        return 0;
+}
+#endif /* CONFIG_SMP */
+
+static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
+{
+        int i, idle = 1;
+        cpumask_t span;
+
+        if (rt_b->rt_runtime == RUNTIME_INF)
+                return 1;
+
+        span = sched_rt_period_mask();
+        for_each_cpu_mask(i, span) {
+                int enqueue = 0;
+                struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
+                struct rq *rq = rq_of_rt_rq(rt_rq);
+
+                spin_lock(&rq->lock);
+                if (rt_rq->rt_time) {
+                        u64 runtime;
+
+                        spin_lock(&rt_rq->rt_runtime_lock);
+                        if (rt_rq->rt_throttled)
+                                balance_runtime(rt_rq);
+                        runtime = rt_rq->rt_runtime;
+                        rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);
+                        if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {
+                                rt_rq->rt_throttled = 0;
+                                enqueue = 1;
+                        }
+                        if (rt_rq->rt_time || rt_rq->rt_nr_running)
+                                idle = 0;
+                        spin_unlock(&rt_rq->rt_runtime_lock);
+                } else if (rt_rq->rt_nr_running)
+                        idle = 0;
+
+                if (enqueue)
+                        sched_rt_rq_enqueue(rt_rq);
+                spin_unlock(&rq->lock);
+        }
+
+        return idle;
+}
 
 static inline int rt_se_prio(struct sched_rt_entity *rt_se)
 {
@@ -326,18 +447,10 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
         if (sched_rt_runtime(rt_rq) >= sched_rt_period(rt_rq))
                 return 0;
 
-#ifdef CONFIG_SMP
-        if (rt_rq->rt_time > runtime) {
-                int more;
-
-                spin_unlock(&rt_rq->rt_runtime_lock);
-                more = balance_runtime(rt_rq);
-                spin_lock(&rt_rq->rt_runtime_lock);
-
-                if (more)
-                        runtime = sched_rt_runtime(rt_rq);
-        }
-#endif
+        balance_runtime(rt_rq);
+        runtime = sched_rt_runtime(rt_rq);
+        if (runtime == RUNTIME_INF)
+                return 0;
 
         if (rt_rq->rt_time > runtime) {
                 rt_rq->rt_throttled = 1;
@@ -391,12 +504,21 @@ void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
         WARN_ON(!rt_prio(rt_se_prio(rt_se)));
         rt_rq->rt_nr_running++;
 #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
-        if (rt_se_prio(rt_se) < rt_rq->highest_prio)
+        if (rt_se_prio(rt_se) < rt_rq->highest_prio) {
+                struct rq *rq = rq_of_rt_rq(rt_rq);
+
                 rt_rq->highest_prio = rt_se_prio(rt_se);
+#ifdef CONFIG_SMP
+                if (rq->online)
+                        cpupri_set(&rq->rd->cpupri, rq->cpu,
+                                   rt_se_prio(rt_se));
+#endif
+        }
 #endif
 #ifdef CONFIG_SMP
         if (rt_se->nr_cpus_allowed > 1) {
                 struct rq *rq = rq_of_rt_rq(rt_rq);
+
                 rq->rt.rt_nr_migratory++;
         }
 
@@ -416,6 +538,10 @@ void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
 static inline
 void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
 {
+#ifdef CONFIG_SMP
+        int highest_prio = rt_rq->highest_prio;
+#endif
+
         WARN_ON(!rt_prio(rt_se_prio(rt_se)));
         WARN_ON(!rt_rq->rt_nr_running);
         rt_rq->rt_nr_running--;
@@ -439,6 +565,14 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
                 rq->rt.rt_nr_migratory--;
         }
 
+        if (rt_rq->highest_prio != highest_prio) {
+                struct rq *rq = rq_of_rt_rq(rt_rq);
+
+                if (rq->online)
+                        cpupri_set(&rq->rd->cpupri, rq->cpu,
+                                   rt_rq->highest_prio);
+        }
+
         update_rt_migration(rq_of_rt_rq(rt_rq));
 #endif /* CONFIG_SMP */
 #ifdef CONFIG_RT_GROUP_SCHED
@@ -449,22 +583,33 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
 #endif
 }
 
-static void enqueue_rt_entity(struct sched_rt_entity *rt_se)
+static void __enqueue_rt_entity(struct sched_rt_entity *rt_se)
 {
         struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
         struct rt_prio_array *array = &rt_rq->active;
         struct rt_rq *group_rq = group_rt_rq(rt_se);
+        struct list_head *queue = array->queue + rt_se_prio(rt_se);
 
-        if (group_rq && rt_rq_throttled(group_rq))
+        /*
+         * Don't enqueue the group if its throttled, or when empty.
+         * The latter is a consequence of the former when a child group
+         * get throttled and the current group doesn't have any other
+         * active members.
+         */
+        if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running))
                 return;
 
-        list_add_tail(&rt_se->run_list, array->queue + rt_se_prio(rt_se));
+        if (rt_se->nr_cpus_allowed == 1)
+                list_add(&rt_se->run_list, queue);
+        else
+                list_add_tail(&rt_se->run_list, queue);
+
         __set_bit(rt_se_prio(rt_se), array->bitmap);
 
         inc_rt_tasks(rt_se, rt_rq);
 }
 
-static void dequeue_rt_entity(struct sched_rt_entity *rt_se)
+static void __dequeue_rt_entity(struct sched_rt_entity *rt_se)
 {
         struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
         struct rt_prio_array *array = &rt_rq->active;
@@ -480,11 +625,10 @@ static void dequeue_rt_entity(struct sched_rt_entity *rt_se)
  * Because the prio of an upper entry depends on the lower
  * entries, we must remove entries top - down.
  */
-static void dequeue_rt_stack(struct task_struct *p)
+static void dequeue_rt_stack(struct sched_rt_entity *rt_se)
 {
-        struct sched_rt_entity *rt_se, *back = NULL;
+        struct sched_rt_entity *back = NULL;
 
-        rt_se = &p->rt;
         for_each_sched_rt_entity(rt_se) {
                 rt_se->back = back;
                 back = rt_se;
@@ -492,7 +636,26 @@ static void dequeue_rt_stack(struct task_struct *p)
 
         for (rt_se = back; rt_se; rt_se = rt_se->back) {
                 if (on_rt_rq(rt_se))
-                        dequeue_rt_entity(rt_se);
+                        __dequeue_rt_entity(rt_se);
+        }
+}
+
+static void enqueue_rt_entity(struct sched_rt_entity *rt_se)
+{
+        dequeue_rt_stack(rt_se);
+        for_each_sched_rt_entity(rt_se)
+                __enqueue_rt_entity(rt_se);
+}
+
+static void dequeue_rt_entity(struct sched_rt_entity *rt_se)
+{
+        dequeue_rt_stack(rt_se);
+
+        for_each_sched_rt_entity(rt_se) {
+                struct rt_rq *rt_rq = group_rt_rq(rt_se);
+
+                if (rt_rq && rt_rq->rt_nr_running)
+                        __enqueue_rt_entity(rt_se);
         }
 }
 
@@ -506,32 +669,19 @@ static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup)
         if (wakeup)
                 rt_se->timeout = 0;
 
-        dequeue_rt_stack(p);
+        enqueue_rt_entity(rt_se);
 
-        /*
-         * enqueue everybody, bottom - up.
-         */
-        for_each_sched_rt_entity(rt_se)
-                enqueue_rt_entity(rt_se);
+        inc_cpu_load(rq, p->se.load.weight);
 }
 
 static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep)
 {
         struct sched_rt_entity *rt_se = &p->rt;
-        struct rt_rq *rt_rq;
 
         update_curr_rt(rq);
+        dequeue_rt_entity(rt_se);
 
-        dequeue_rt_stack(p);
-
-        /*
-         * re-enqueue all non-empty rt_rq entities.
-         */
-        for_each_sched_rt_entity(rt_se) {
-                rt_rq = group_rt_rq(rt_se);
-                if (rt_rq && rt_rq->rt_nr_running)
-                        enqueue_rt_entity(rt_se);
-        }
+        dec_cpu_load(rq, p->se.load.weight);
 }
 
 /*
@@ -543,7 +693,11 @@ void requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)
 {
         struct rt_prio_array *array = &rt_rq->active;
 
-        list_move_tail(&rt_se->run_list, array->queue + rt_se_prio(rt_se));
+        if (on_rt_rq(rt_se)) {
+                list_del_init(&rt_se->run_list);
+                list_add_tail(&rt_se->run_list,
+                              array->queue + rt_se_prio(rt_se));
+        }
 }
 
 static void requeue_task_rt(struct rq *rq, struct task_struct *p)
@@ -606,8 +760,37 @@ static int select_task_rq_rt(struct task_struct *p, int sync)
  */
 static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p)
 {
-        if (p->prio < rq->curr->prio)
+        if (p->prio < rq->curr->prio) {
                 resched_task(rq->curr);
+                return;
+        }
+
+#ifdef CONFIG_SMP
+        /*
+         * If:
+         *
+         * - the newly woken task is of equal priority to the current task
+         * - the newly woken task is non-migratable while current is migratable
+         * - current will be preempted on the next reschedule
+         *
+         * we should check to see if current can readily move to a different
+         * cpu.  If so, we will reschedule to allow the push logic to try
+         * to move current somewhere else, making room for our non-migratable
+         * task.
+         */
+        if((p->prio == rq->curr->prio)
+           && p->rt.nr_cpus_allowed == 1
+           && rq->curr->rt.nr_cpus_allowed != 1) {
+                cpumask_t mask;
+
+                if (cpupri_find(&rq->rd->cpupri, rq->curr, &mask))
+                        /*
+                         * There appears to be other cpus that can accept
+                         * current, so lets reschedule to try and push it away
+                         */
+                        resched_task(rq->curr);
+        }
+#endif
 }
 
 static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq,
@@ -710,73 +893,6 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu)
 
 static DEFINE_PER_CPU(cpumask_t, local_cpu_mask);
 
-static int find_lowest_cpus(struct task_struct *task, cpumask_t *lowest_mask)
-{
-        int       lowest_prio = -1;
-        int       lowest_cpu  = -1;
-        int       count       = 0;
-        int       cpu;
-
-        cpus_and(*lowest_mask, task_rq(task)->rd->online, task->cpus_allowed);
-
-        /*
-         * Scan each rq for the lowest prio.
-         */
-        for_each_cpu_mask(cpu, *lowest_mask) {
-                struct rq *rq = cpu_rq(cpu);
-
-                /* We look for lowest RT prio or non-rt CPU */
-                if (rq->rt.highest_prio >= MAX_RT_PRIO) {
-                        /*
-                         * if we already found a low RT queue
-                         * and now we found this non-rt queue
-                         * clear the mask and set our bit.
-                         * Otherwise just return the queue as is
-                         * and the count==1 will cause the algorithm
-                         * to use the first bit found.
-                         */
-                        if (lowest_cpu != -1) {
-                                cpus_clear(*lowest_mask);
-                                cpu_set(rq->cpu, *lowest_mask);
-                        }
-                        return 1;
-                }
-
-                /* no locking for now */
-                if ((rq->rt.highest_prio > task->prio)
-                    && (rq->rt.highest_prio >= lowest_prio)) {
-                        if (rq->rt.highest_prio > lowest_prio) {
-                                /* new low - clear old data */
-                                lowest_prio = rq->rt.highest_prio;
-                                lowest_cpu = cpu;
-                                count = 0;
-                        }
-                        count++;
-                } else
-                        cpu_clear(cpu, *lowest_mask);
-        }
-
-        /*
-         * Clear out all the set bits that represent
-         * runqueues that were of higher prio than
-         * the lowest_prio.
-         */
-        if (lowest_cpu > 0) {
-                /*
-                 * Perhaps we could add another cpumask op to
-                 * zero out bits. Like cpu_zero_bits(cpumask, nrbits);
-                 * Then that could be optimized to use memset and such.
-                 */
-                for_each_cpu_mask(cpu, *lowest_mask) {
-                        if (cpu >= lowest_cpu)
-                                break;
-                        cpu_clear(cpu, *lowest_mask);
-                }
-        }
-
-        return count;
-}
-
 static inline int pick_optimal_cpu(int this_cpu, cpumask_t *mask)
 {
         int first;
@@ -798,17 +914,12 @@ static int find_lowest_rq(struct task_struct *task)
         cpumask_t *lowest_mask = &__get_cpu_var(local_cpu_mask);
         int this_cpu = smp_processor_id();
         int cpu      = task_cpu(task);
-        int count    = find_lowest_cpus(task, lowest_mask);
 
-        if (!count)
-                return -1; /* No targets found */
+        if (task->rt.nr_cpus_allowed == 1)
+                return -1; /* No other targets possible */
 
-        /*
-         * There is no sense in performing an optimal search if only one
-         * target is found.
-         */
-        if (count == 1)
-                return first_cpu(*lowest_mask);
+        if (!cpupri_find(&task_rq(task)->rd->cpupri, task, lowest_mask))
+                return -1; /* No targets found */
 
         /*
          * At this point we have built a mask of cpus representing the
@@ -1153,17 +1264,25 @@ static void set_cpus_allowed_rt(struct task_struct *p,
 }
 
 /* Assumes rq->lock is held */
-static void join_domain_rt(struct rq *rq)
+static void rq_online_rt(struct rq *rq)
 {
         if (rq->rt.overloaded)
                 rt_set_overload(rq);
+
+        __enable_runtime(rq);
+
+        cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio);
 }
 
 /* Assumes rq->lock is held */
-static void leave_domain_rt(struct rq *rq)
+static void rq_offline_rt(struct rq *rq)
 {
         if (rq->rt.overloaded)
                 rt_clear_overload(rq);
+
+        __disable_runtime(rq);
+
+        cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID);
 }
 
 /*
@@ -1326,8 +1445,8 @@ static const struct sched_class rt_sched_class = {
         .load_balance           = load_balance_rt,
         .move_one_task          = move_one_task_rt,
         .set_cpus_allowed       = set_cpus_allowed_rt,
-        .join_domain            = join_domain_rt,
-        .leave_domain           = leave_domain_rt,
+        .rq_online              = rq_online_rt,
+        .rq_offline             = rq_offline_rt,
         .pre_schedule           = pre_schedule_rt,
         .post_schedule          = post_schedule_rt,
         .task_wake_up           = task_wake_up_rt,
@@ -1340,3 +1459,17 @@ static const struct sched_class rt_sched_class = {
         .prio_changed           = prio_changed_rt,
         .switched_to            = switched_to_rt,
 };
+
+#ifdef CONFIG_SCHED_DEBUG
+extern void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq);
+
+static void print_rt_stats(struct seq_file *m, int cpu)
+{
+        struct rt_rq *rt_rq;
+
+        rcu_read_lock();
+        for_each_leaf_rt_rq(rt_rq, cpu_rq(cpu))
+                print_rt_rq(m, cpu, rt_rq);
+        rcu_read_unlock();
+}
+#endif /* CONFIG_SCHED_DEBUG */
diff --git a/kernel/sched_stats.h b/kernel/sched_stats.h
index a38878e0e49d..8385d43987e2 100644
--- a/kernel/sched_stats.h
+++ b/kernel/sched_stats.h
@@ -118,6 +118,13 @@ rq_sched_info_depart(struct rq *rq, unsigned long long delta)
         if (rq)
                 rq->rq_sched_info.cpu_time += delta;
 }
+
+static inline void
+rq_sched_info_dequeued(struct rq *rq, unsigned long long delta)
+{
+        if (rq)
+                rq->rq_sched_info.run_delay += delta;
+}
 # define schedstat_inc(rq, field)       do { (rq)->field++; } while (0)
 # define schedstat_add(rq, field, amt)  do { (rq)->field += (amt); } while (0)
 # define schedstat_set(var, val)        do { var = (val); } while (0)
@@ -126,6 +133,9 @@ static inline void
 rq_sched_info_arrive(struct rq *rq, unsigned long long delta)
 {}
 static inline void
+rq_sched_info_dequeued(struct rq *rq, unsigned long long delta)
+{}
+static inline void
 rq_sched_info_depart(struct rq *rq, unsigned long long delta)
 {}
 # define schedstat_inc(rq, field)       do { } while (0)
@@ -134,6 +144,11 @@ rq_sched_info_depart(struct rq *rq, unsigned long long delta)
 #endif
 
 #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
+static inline void sched_info_reset_dequeued(struct task_struct *t)
+{
+        t->sched_info.last_queued = 0;
+}
+
 /*
  * Called when a process is dequeued from the active array and given
  * the cpu.  We should note that with the exception of interactive
@@ -143,15 +158,22 @@ rq_sched_info_depart(struct rq *rq, unsigned long long delta)
  * active queue, thus delaying tasks in the expired queue from running;
  * see scheduler_tick()).
  *
- * This function is only called from sched_info_arrive(), rather than
- * dequeue_task(). Even though a task may be queued and dequeued multiple
- * times as it is shuffled about, we're really interested in knowing how
- * long it was from the *first* time it was queued to the time that it
- * finally hit a cpu.
+ * Though we are interested in knowing how long it was from the *first* time a
+ * task was queued to the time that it finally hit a cpu, we call this routine
+ * from dequeue_task() to account for possible rq->clock skew across cpus. The
+ * delta taken on each cpu would annul the skew.
  */
 static inline void sched_info_dequeued(struct task_struct *t)
 {
-        t->sched_info.last_queued = 0;
+        unsigned long long now = task_rq(t)->clock, delta = 0;
+
+        if (unlikely(sched_info_on()))
+                if (t->sched_info.last_queued)
+                        delta = now - t->sched_info.last_queued;
+        sched_info_reset_dequeued(t);
+        t->sched_info.run_delay += delta;
+
+        rq_sched_info_dequeued(task_rq(t), delta);
 }
 
 /*
@@ -165,7 +187,7 @@ static void sched_info_arrive(struct task_struct *t)
 
         if (t->sched_info.last_queued)
                 delta = now - t->sched_info.last_queued;
-        sched_info_dequeued(t);
+        sched_info_reset_dequeued(t);
         t->sched_info.run_delay += delta;
         t->sched_info.last_arrival = now;
         t->sched_info.pcount++;
@@ -198,6 +220,9 @@ static inline void sched_info_queued(struct task_struct *t)
 /*
  * Called when a process ceases being the active-running process, either
  * voluntarily or involuntarily.  Now we can calculate how long we ran.
+ * Also, if the process is still in the TASK_RUNNING state, call
+ * sched_info_queued() to mark that it has now again started waiting on
+ * the runqueue.
  */
 static inline void sched_info_depart(struct task_struct *t)
 {
@@ -206,6 +231,9 @@ static inline void sched_info_depart(struct task_struct *t)
 
         t->sched_info.cpu_time += delta;
         rq_sched_info_depart(task_rq(t), delta);
+
+        if (t->state == TASK_RUNNING)
+                sched_info_queued(t);
 }
 
 /*
@@ -236,7 +264,9 @@ sched_info_switch(struct task_struct *prev, struct task_struct *next)
                 __sched_info_switch(prev, next);
 }
 #else
-#define sched_info_queued(t)            do { } while (0)
-#define sched_info_switch(t, next)      do { } while (0)
+#define sched_info_queued(t)                    do { } while (0)
+#define sched_info_reset_dequeued(t)    do { } while (0)
+#define sched_info_dequeued(t)                  do { } while (0)
+#define sched_info_switch(t, next)              do { } while (0)
 #endif /* CONFIG_SCHEDSTATS || CONFIG_TASK_DELAY_ACCT */
 
diff --git a/kernel/semaphore.c b/kernel/semaphore.c
index 5c2942e768cd..aaaeae8244e7 100644
--- a/kernel/semaphore.c
+++ b/kernel/semaphore.c
@@ -31,6 +31,7 @@
 #include <linux/sched.h>
 #include <linux/semaphore.h>
 #include <linux/spinlock.h>
+#include <linux/ftrace.h>
 
 static noinline void __down(struct semaphore *sem);
 static noinline int __down_interruptible(struct semaphore *sem);
diff --git a/kernel/smp.c b/kernel/smp.c
new file mode 100644
index 000000000000..462c785ca1ee
--- /dev/null
+++ b/kernel/smp.c
@@ -0,0 +1,383 @@
+/*
+ * Generic helpers for smp ipi calls
+ *
+ * (C) Jens Axboe <jens.axboe@oracle.com> 2008
+ *
+ */
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/percpu.h>
+#include <linux/rcupdate.h>
+#include <linux/rculist.h>
+#include <linux/smp.h>
+
+static DEFINE_PER_CPU(struct call_single_queue, call_single_queue);
+static LIST_HEAD(call_function_queue);
+__cacheline_aligned_in_smp DEFINE_SPINLOCK(call_function_lock);
+
+enum {
+        CSD_FLAG_WAIT           = 0x01,
+        CSD_FLAG_ALLOC          = 0x02,
+};
+
+struct call_function_data {
+        struct call_single_data csd;
+        spinlock_t lock;
+        unsigned int refs;
+        cpumask_t cpumask;
+        struct rcu_head rcu_head;
+};
+
+struct call_single_queue {
+        struct list_head list;
+        spinlock_t lock;
+};
+
+void __cpuinit init_call_single_data(void)
+{
+        int i;
+
+        for_each_possible_cpu(i) {
+                struct call_single_queue *q = &per_cpu(call_single_queue, i);
+
+                spin_lock_init(&q->lock);
+                INIT_LIST_HEAD(&q->list);
+        }
+}
+
+static void csd_flag_wait(struct call_single_data *data)
+{
+        /* Wait for response */
+        do {
+                /*
+                 * We need to see the flags store in the IPI handler
+                 */
+                smp_mb();
+                if (!(data->flags & CSD_FLAG_WAIT))
+                        break;
+                cpu_relax();
+        } while (1);
+}
+
+/*
+ * Insert a previously allocated call_single_data element for execution
+ * on the given CPU. data must already have ->func, ->info, and ->flags set.
+ */
+static void generic_exec_single(int cpu, struct call_single_data *data)
+{
+        struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
+        int wait = data->flags & CSD_FLAG_WAIT, ipi;
+        unsigned long flags;
+
+        spin_lock_irqsave(&dst->lock, flags);
+        ipi = list_empty(&dst->list);
+        list_add_tail(&data->list, &dst->list);
+        spin_unlock_irqrestore(&dst->lock, flags);
+
+        if (ipi)
+                arch_send_call_function_single_ipi(cpu);
+
+        if (wait)
+                csd_flag_wait(data);
+}
+
+static void rcu_free_call_data(struct rcu_head *head)
+{
+        struct call_function_data *data;
+
+        data = container_of(head, struct call_function_data, rcu_head);
+
+        kfree(data);
+}
+
+/*
+ * Invoked by arch to handle an IPI for call function. Must be called with
+ * interrupts disabled.
+ */
+void generic_smp_call_function_interrupt(void)
+{
+        struct call_function_data *data;
+        int cpu = get_cpu();
+
+        /*
+         * It's ok to use list_for_each_rcu() here even though we may delete
+         * 'pos', since list_del_rcu() doesn't clear ->next
+         */
+        rcu_read_lock();
+        list_for_each_entry_rcu(data, &call_function_queue, csd.list) {
+                int refs;
+
+                if (!cpu_isset(cpu, data->cpumask))
+                        continue;
+
+                data->csd.func(data->csd.info);
+
+                spin_lock(&data->lock);
+                cpu_clear(cpu, data->cpumask);
+                WARN_ON(data->refs == 0);
+                data->refs--;
+                refs = data->refs;
+                spin_unlock(&data->lock);
+
+                if (refs)
+                        continue;
+
+                spin_lock(&call_function_lock);
+                list_del_rcu(&data->csd.list);
+                spin_unlock(&call_function_lock);
+
+                if (data->csd.flags & CSD_FLAG_WAIT) {
+                        /*
+                         * serialize stores to data with the flag clear
+                         * and wakeup
+                         */
+                        smp_wmb();
+                        data->csd.flags &= ~CSD_FLAG_WAIT;
+                } else
+                        call_rcu(&data->rcu_head, rcu_free_call_data);
+        }
+        rcu_read_unlock();
+
+        put_cpu();
+}
+
+/*
+ * Invoked by arch to handle an IPI for call function single. Must be called
+ * from the arch with interrupts disabled.
+ */
+void generic_smp_call_function_single_interrupt(void)
+{
+        struct call_single_queue *q = &__get_cpu_var(call_single_queue);
+        LIST_HEAD(list);
+
+        /*
+         * Need to see other stores to list head for checking whether
+         * list is empty without holding q->lock
+         */
+        smp_mb();
+        while (!list_empty(&q->list)) {
+                unsigned int data_flags;
+
+                spin_lock(&q->lock);
+                list_replace_init(&q->list, &list);
+                spin_unlock(&q->lock);
+
+                while (!list_empty(&list)) {
+                        struct call_single_data *data;
+
+                        data = list_entry(list.next, struct call_single_data,
+                                                list);
+                        list_del(&data->list);
+
+                        /*
+                         * 'data' can be invalid after this call if
+                         * flags == 0 (when called through
+                         * generic_exec_single(), so save them away before
+                         * making the call.
+                         */
+                        data_flags = data->flags;
+
+                        data->func(data->info);
+
+                        if (data_flags & CSD_FLAG_WAIT) {
+                                smp_wmb();
+                                data->flags &= ~CSD_FLAG_WAIT;
+                        } else if (data_flags & CSD_FLAG_ALLOC)
+                                kfree(data);
+                }
+                /*
+                 * See comment on outer loop
+                 */
+                smp_mb();
+        }
+}
+
+/*
+ * smp_call_function_single - Run a function on a specific CPU
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @wait: If true, wait until function has completed on other CPUs.
+ *
+ * Returns 0 on success, else a negative status code. Note that @wait
+ * will be implicitly turned on in case of allocation failures, since
+ * we fall back to on-stack allocation.
+ */
+int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
+                             int wait)
+{
+        struct call_single_data d;
+        unsigned long flags;
+        /* prevent preemption and reschedule on another processor */
+        int me = get_cpu();
+
+        /* Can deadlock when called with interrupts disabled */
+        WARN_ON(irqs_disabled());
+
+        if (cpu == me) {
+                local_irq_save(flags);
+                func(info);
+                local_irq_restore(flags);
+        } else {
+                struct call_single_data *data = NULL;
+
+                if (!wait) {
+                        data = kmalloc(sizeof(*data), GFP_ATOMIC);
+                        if (data)
+                                data->flags = CSD_FLAG_ALLOC;
+                }
+                if (!data) {
+                        data = &d;
+                        data->flags = CSD_FLAG_WAIT;
+                }
+
+                data->func = func;
+                data->info = info;
+                generic_exec_single(cpu, data);
+        }
+
+        put_cpu();
+        return 0;
+}
+EXPORT_SYMBOL(smp_call_function_single);
+
+/**
+ * __smp_call_function_single(): Run a function on another CPU
+ * @cpu: The CPU to run on.
+ * @data: Pre-allocated and setup data structure
+ *
+ * Like smp_call_function_single(), but allow caller to pass in a pre-allocated
+ * data structure. Useful for embedding @data inside other structures, for
+ * instance.
+ *
+ */
+void __smp_call_function_single(int cpu, struct call_single_data *data)
+{
+        /* Can deadlock when called with interrupts disabled */
+        WARN_ON((data->flags & CSD_FLAG_WAIT) && irqs_disabled());
+
+        generic_exec_single(cpu, data);
+}
+
+/**
+ * smp_call_function_mask(): Run a function on a set of other CPUs.
+ * @mask: The set of cpus to run on.
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @wait: If true, wait (atomically) until function has completed on other CPUs.
+ *
+ * Returns 0 on success, else a negative status code.
+ *
+ * If @wait is true, then returns once @func has returned. Note that @wait
+ * will be implicitly turned on in case of allocation failures, since
+ * we fall back to on-stack allocation.
+ *
+ * You must not call this function with disabled interrupts or from a
+ * hardware interrupt handler or from a bottom half handler. Preemption
+ * must be disabled when calling this function.
+ */
+int smp_call_function_mask(cpumask_t mask, void (*func)(void *), void *info,
+                           int wait)
+{
+        struct call_function_data d;
+        struct call_function_data *data = NULL;
+        cpumask_t allbutself;
+        unsigned long flags;
+        int cpu, num_cpus;
+
+        /* Can deadlock when called with interrupts disabled */
+        WARN_ON(irqs_disabled());
+
+        cpu = smp_processor_id();
+        allbutself = cpu_online_map;
+        cpu_clear(cpu, allbutself);
+        cpus_and(mask, mask, allbutself);
+        num_cpus = cpus_weight(mask);
+
+        /*
+         * If zero CPUs, return. If just a single CPU, turn this request
+         * into a targetted single call instead since it's faster.
+         */
+        if (!num_cpus)
+                return 0;
+        else if (num_cpus == 1) {
+                cpu = first_cpu(mask);
+                return smp_call_function_single(cpu, func, info, wait);
+        }
+
+        if (!wait) {
+                data = kmalloc(sizeof(*data), GFP_ATOMIC);
+                if (data)
+                        data->csd.flags = CSD_FLAG_ALLOC;
+        }
+        if (!data) {
+                data = &d;
+                data->csd.flags = CSD_FLAG_WAIT;
+                wait = 1;
+        }
+
+        spin_lock_init(&data->lock);
+        data->csd.func = func;
+        data->csd.info = info;
+        data->refs = num_cpus;
+        data->cpumask = mask;
+
+        spin_lock_irqsave(&call_function_lock, flags);
+        list_add_tail_rcu(&data->csd.list, &call_function_queue);
+        spin_unlock_irqrestore(&call_function_lock, flags);
+
+        /* Send a message to all CPUs in the map */
+        arch_send_call_function_ipi(mask);
+
+        /* optionally wait for the CPUs to complete */
+        if (wait)
+                csd_flag_wait(&data->csd);
+
+        return 0;
+}
+EXPORT_SYMBOL(smp_call_function_mask);
+
+/**
+ * smp_call_function(): Run a function on all other CPUs.
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @wait: If true, wait (atomically) until function has completed on other CPUs.
+ *
+ * Returns 0 on success, else a negative status code.
+ *
+ * If @wait is true, then returns once @func has returned; otherwise
+ * it returns just before the target cpu calls @func. In case of allocation
+ * failure, @wait will be implicitly turned on.
+ *
+ * You must not call this function with disabled interrupts or from a
+ * hardware interrupt handler or from a bottom half handler.
+ */
+int smp_call_function(void (*func)(void *), void *info, int wait)
+{
+        int ret;
+
+        preempt_disable();
+        ret = smp_call_function_mask(cpu_online_map, func, info, wait);
+        preempt_enable();
+        return ret;
+}
+EXPORT_SYMBOL(smp_call_function);
+
+void ipi_call_lock(void)
+{
+        spin_lock(&call_function_lock);
+}
+
+void ipi_call_unlock(void)
+{
+        spin_unlock(&call_function_lock);
+}
+
+void ipi_call_lock_irq(void)
+{
+        spin_lock_irq(&call_function_lock);
+}
+
+void ipi_call_unlock_irq(void)
+{
+        spin_unlock_irq(&call_function_lock);
+}
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 36e061740047..81e2fe0f983a 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -131,23 +131,17 @@ void _local_bh_enable(void)
 
 EXPORT_SYMBOL(_local_bh_enable);
 
-void local_bh_enable(void)
+static inline void _local_bh_enable_ip(unsigned long ip)
 {
+        WARN_ON_ONCE(in_irq() || irqs_disabled());
 #ifdef CONFIG_TRACE_IRQFLAGS
-        unsigned long flags;
-
-        WARN_ON_ONCE(in_irq());
-#endif
-        WARN_ON_ONCE(irqs_disabled());
-
-#ifdef CONFIG_TRACE_IRQFLAGS
-        local_irq_save(flags);
+        local_irq_disable();
 #endif
         /*
          * Are softirqs going to be turned on now:
          */
         if (softirq_count() == SOFTIRQ_OFFSET)
-                trace_softirqs_on((unsigned long)__builtin_return_address(0));
+                trace_softirqs_on(ip);
         /*
          * Keep preemption disabled until we are done with
          * softirq processing:
@@ -159,40 +153,20 @@ void local_bh_enable(void)
 
         dec_preempt_count();
 #ifdef CONFIG_TRACE_IRQFLAGS
-        local_irq_restore(flags);
+        local_irq_enable();
 #endif
         preempt_check_resched();
 }
+
+void local_bh_enable(void)
+{
+        _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
+}
 EXPORT_SYMBOL(local_bh_enable);
 
 void local_bh_enable_ip(unsigned long ip)
 {
-#ifdef CONFIG_TRACE_IRQFLAGS
-        unsigned long flags;
-
-        WARN_ON_ONCE(in_irq());
-
-        local_irq_save(flags);
-#endif
-        /*
-         * Are softirqs going to be turned on now:
-         */
-        if (softirq_count() == SOFTIRQ_OFFSET)
-                trace_softirqs_on(ip);
-        /*
-         * Keep preemption disabled until we are done with
-         * softirq processing:
-         */
-        sub_preempt_count(SOFTIRQ_OFFSET - 1);
-
-        if (unlikely(!in_interrupt() && local_softirq_pending()))
-                do_softirq();
-
-        dec_preempt_count();
-#ifdef CONFIG_TRACE_IRQFLAGS
-        local_irq_restore(flags);
-#endif
-        preempt_check_resched();
+        _local_bh_enable_ip(ip);
 }
 EXPORT_SYMBOL(local_bh_enable_ip);
 
@@ -347,9 +321,8 @@ void raise_softirq(unsigned int nr)
         local_irq_restore(flags);
 }
 
-void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
+void open_softirq(int nr, void (*action)(struct softirq_action *))
 {
-        softirq_vec[nr].data = data;
         softirq_vec[nr].action = action;
 }
 
@@ -360,10 +333,8 @@ struct tasklet_head
         struct tasklet_struct **tail;
 };
 
-/* Some compilers disobey section attribute on statics when not
-   initialized -- RR */
-static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
-static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
+static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
+static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 
 void __tasklet_schedule(struct tasklet_struct *t)
 {
@@ -503,8 +474,8 @@ void __init softirq_init(void)
                         &per_cpu(tasklet_hi_vec, cpu).head;
         }
 
-        open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
-        open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
+        open_softirq(TASKLET_SOFTIRQ, tasklet_action);
+        open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 }
 
 static int ksoftirqd(void * __bind_cpu)
@@ -645,7 +616,7 @@ static int __cpuinit cpu_callback(struct notifier_block *nfb,
 
                 p = per_cpu(ksoftirqd, hotcpu);
                 per_cpu(ksoftirqd, hotcpu) = NULL;
-                sched_setscheduler(p, SCHED_FIFO, &param);
+                sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
                 kthread_stop(p);
                 takeover_tasklets(hotcpu);
                 break;
@@ -674,12 +645,12 @@ __init int spawn_ksoftirqd(void)
 /*
  * Call a function on all processors
  */
-int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
+int on_each_cpu(void (*func) (void *info), void *info, int wait)
 {
         int ret = 0;
 
         preempt_disable();
-        ret = smp_call_function(func, info, retry, wait);
+        ret = smp_call_function(func, info, wait);
         local_irq_disable();
         func(info);
         local_irq_enable();
diff --git a/kernel/softlockup.c b/kernel/softlockup.c
index 01b6522fd92b..a272d78185eb 100644
--- a/kernel/softlockup.c
+++ b/kernel/softlockup.c
@@ -49,12 +49,17 @@ static unsigned long get_timestamp(int this_cpu)
         return cpu_clock(this_cpu) >> 30LL;  /* 2^30 ~= 10^9 */
 }
 
-void touch_softlockup_watchdog(void)
+static void __touch_softlockup_watchdog(void)
 {
         int this_cpu = raw_smp_processor_id();
 
         __raw_get_cpu_var(touch_timestamp) = get_timestamp(this_cpu);
 }
+
+void touch_softlockup_watchdog(void)
+{
+        __raw_get_cpu_var(touch_timestamp) = 0;
+}
 EXPORT_SYMBOL(touch_softlockup_watchdog);
 
 void touch_all_softlockup_watchdogs(void)
@@ -80,7 +85,7 @@ void softlockup_tick(void)
         unsigned long now;
 
         if (touch_timestamp == 0) {
-                touch_softlockup_watchdog();
+                __touch_softlockup_watchdog();
                 return;
         }
 
@@ -95,7 +100,7 @@ void softlockup_tick(void)
 
         /* do not print during early bootup: */
         if (unlikely(system_state != SYSTEM_RUNNING)) {
-                touch_softlockup_watchdog();
+                __touch_softlockup_watchdog();
                 return;
         }
 
@@ -115,6 +120,7 @@ void softlockup_tick(void)
         printk(KERN_ERR "BUG: soft lockup - CPU#%d stuck for %lus! [%s:%d]\n",
                         this_cpu, now - touch_timestamp,
                         current->comm, task_pid_nr(current));
+        print_modules();
         if (regs)
                 show_regs(regs);
         else
@@ -214,7 +220,7 @@ static int watchdog(void *__bind_cpu)
         sched_setscheduler(current, SCHED_FIFO, &param);
 
         /* initialize timestamp */
-        touch_softlockup_watchdog();
+        __touch_softlockup_watchdog();
 
         set_current_state(TASK_INTERRUPTIBLE);
         /*
@@ -223,7 +229,7 @@ static int watchdog(void *__bind_cpu)
          * debug-printout triggers in softlockup_tick().
          */
         while (!kthread_should_stop()) {
-                touch_softlockup_watchdog();
+                __touch_softlockup_watchdog();
                 schedule();
 
                 if (kthread_should_stop())
diff --git a/kernel/spinlock.c b/kernel/spinlock.c
index ae28c8245123..a1fb54c93cdd 100644
--- a/kernel/spinlock.c
+++ b/kernel/spinlock.c
@@ -436,7 +436,7 @@ int __lockfunc _spin_trylock_bh(spinlock_t *lock)
 }
 EXPORT_SYMBOL(_spin_trylock_bh);
 
-int in_lock_functions(unsigned long addr)
+notrace int in_lock_functions(unsigned long addr)
 {
         /* Linker adds these: start and end of __lockfunc functions */
         extern char __lock_text_start[], __lock_text_end[];
diff --git a/kernel/stacktrace.c b/kernel/stacktrace.c
index b71816e47a30..94b527ef1d1e 100644
--- a/kernel/stacktrace.c
+++ b/kernel/stacktrace.c
@@ -6,19 +6,21 @@
  *  Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  */
 #include <linux/sched.h>
+#include <linux/module.h>
 #include <linux/kallsyms.h>
 #include <linux/stacktrace.h>
 
 void print_stack_trace(struct stack_trace *trace, int spaces)
 {
-        int i, j;
+        int i;
 
-        for (i = 0; i < trace->nr_entries; i++) {
-                unsigned long ip = trace->entries[i];
+        if (WARN_ON(!trace->entries))
+                return;
 
-                for (j = 0; j < spaces + 1; j++)
-                        printk(" ");
-                print_ip_sym(ip);
+        for (i = 0; i < trace->nr_entries; i++) {
+                printk("%*c", 1 + spaces, ' ');
+                print_ip_sym(trace->entries[i]);
         }
 }
+EXPORT_SYMBOL_GPL(print_stack_trace);
 
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index b7350bbfb076..ba9b2054ecbd 100644
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -187,7 +187,7 @@ struct task_struct *__stop_machine_run(int (*fn)(void *), void *data,
                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 
                 /* One high-prio thread per cpu.  We'll do this one. */
-                sched_setscheduler(p, SCHED_FIFO, &param);
+                sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
                 kthread_bind(p, cpu);
                 wake_up_process(p);
                 wait_for_completion(&smdata.done);
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 29116652dca8..6b16e16428d8 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -46,6 +46,7 @@
 #include <linux/nfs_fs.h>
 #include <linux/acpi.h>
 #include <linux/reboot.h>
+#include <linux/ftrace.h>
 
 #include <asm/uaccess.h>
 #include <asm/processor.h>
@@ -82,6 +83,9 @@ extern int maps_protect;
 extern int sysctl_stat_interval;
 extern int latencytop_enabled;
 extern int sysctl_nr_open_min, sysctl_nr_open_max;
+#ifdef CONFIG_RCU_TORTURE_TEST
+extern int rcutorture_runnable;
+#endif /* #ifdef CONFIG_RCU_TORTURE_TEST */
 
 /* Constants used for minimum and  maximum */
 #if defined(CONFIG_DETECT_SOFTLOCKUP) || defined(CONFIG_HIGHMEM)
@@ -132,8 +136,6 @@ extern int sysctl_userprocess_debug;
 extern int spin_retry;
 #endif
 
-extern int sysctl_hz_timer;
-
 #ifdef CONFIG_BSD_PROCESS_ACCT
 extern int acct_parm[];
 #endif
@@ -266,6 +268,14 @@ static struct ctl_table kern_table[] = {
         },
         {
                 .ctl_name       = CTL_UNNUMBERED,
+                .procname       = "sched_shares_ratelimit",
+                .data           = &sysctl_sched_shares_ratelimit,
+                .maxlen         = sizeof(unsigned int),
+                .mode           = 0644,
+                .proc_handler   = &proc_dointvec,
+        },
+        {
+                .ctl_name       = CTL_UNNUMBERED,
                 .procname       = "sched_child_runs_first",
                 .data           = &sysctl_sched_child_runs_first,
                 .maxlen         = sizeof(unsigned int),
@@ -455,6 +465,16 @@ static struct ctl_table kern_table[] = {
                 .mode           = 0644,
                 .proc_handler   = &proc_dointvec,
         },
+#ifdef CONFIG_FTRACE
+        {
+                .ctl_name       = CTL_UNNUMBERED,
+                .procname       = "ftrace_enabled",
+                .data           = &ftrace_enabled,
+                .maxlen         = sizeof(int),
+                .mode           = 0644,
+                .proc_handler   = &ftrace_enable_sysctl,
+        },
+#endif
 #ifdef CONFIG_KMOD
         {
                 .ctl_name       = KERN_MODPROBE,
@@ -563,16 +583,6 @@ static struct ctl_table kern_table[] = {
                 .proc_handler   = &proc_dointvec,
         },
 #endif
-#ifdef CONFIG_NO_IDLE_HZ
-        {
-                .ctl_name       = KERN_HZ_TIMER,
-                .procname       = "hz_timer",
-                .data           = &sysctl_hz_timer,
-                .maxlen         = sizeof(int),
-                .mode           = 0644,
-                .proc_handler   = &proc_dointvec,
-        },
-#endif
         {
                 .ctl_name       = KERN_S390_USER_DEBUG_LOGGING,
                 .procname       = "userprocess_debug",
@@ -813,6 +823,16 @@ static struct ctl_table kern_table[] = {
                 .child          = key_sysctls,
         },
 #endif
+#ifdef CONFIG_RCU_TORTURE_TEST
+        {
+                .ctl_name       = CTL_UNNUMBERED,
+                .procname       = "rcutorture_runnable",
+                .data           = &rcutorture_runnable,
+                .maxlen         = sizeof(int),
+                .mode           = 0644,
+                .proc_handler   = &proc_dointvec,
+        },
+#endif
 /*
  * NOTE: do not add new entries to this table unless you have read
  * Documentation/sysctl/ctl_unnumbered.txt
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index 57a1f02e5ec0..f48d0f09d32f 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -30,6 +30,7 @@
 struct tick_device tick_broadcast_device;
 static cpumask_t tick_broadcast_mask;
 static DEFINE_SPINLOCK(tick_broadcast_lock);
+static int tick_broadcast_force;
 
 #ifdef CONFIG_TICK_ONESHOT
 static void tick_broadcast_clear_oneshot(int cpu);
@@ -232,10 +233,11 @@ static void tick_do_broadcast_on_off(void *why)
                                                      CLOCK_EVT_MODE_SHUTDOWN);
                 }
                 if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE)
-                        dev->features |= CLOCK_EVT_FEAT_DUMMY;
+                        tick_broadcast_force = 1;
                 break;
         case CLOCK_EVT_NOTIFY_BROADCAST_OFF:
-                if (cpu_isset(cpu, tick_broadcast_mask)) {
+                if (!tick_broadcast_force &&
+                    cpu_isset(cpu, tick_broadcast_mask)) {
                         cpu_clear(cpu, tick_broadcast_mask);
                         if (td->mode == TICKDEV_MODE_PERIODIC)
                                 tick_setup_periodic(dev, 0);
@@ -266,7 +268,7 @@ void tick_broadcast_on_off(unsigned long reason, int *oncpu)
                        "offline CPU #%d\n", *oncpu);
         else
                 smp_call_function_single(*oncpu, tick_do_broadcast_on_off,
-                                         &reason, 1, 1);
+                                         &reason, 1);
 }
 
 /*
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index b854a895591e..beef7ccdf842 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -48,6 +48,13 @@ static void tick_do_update_jiffies64(ktime_t now)
         unsigned long ticks = 0;
         ktime_t delta;
 
+        /*
+         * Do a quick check without holding xtime_lock:
+         */
+        delta = ktime_sub(now, last_jiffies_update);
+        if (delta.tv64 < tick_period.tv64)
+                return;
+
         /* Reevalute with xtime_lock held */
         write_seqlock(&xtime_lock);
 
@@ -228,6 +235,7 @@ void tick_nohz_stop_sched_tick(void)
                                local_softirq_pending());
                         ratelimit++;
                 }
+                goto end;
         }
 
         ts->idle_calls++;
@@ -276,6 +284,7 @@ void tick_nohz_stop_sched_tick(void)
                         ts->tick_stopped = 1;
                         ts->idle_jiffies = last_jiffies;
                         rcu_enter_nohz();
+                        sched_clock_tick_stop(cpu);
                 }
 
                 /*
@@ -375,6 +384,7 @@ void tick_nohz_restart_sched_tick(void)
         select_nohz_load_balancer(0);
         now = ktime_get();
         tick_do_update_jiffies64(now);
+        sched_clock_tick_start(cpu);
         cpu_clear(cpu, nohz_cpu_mask);
 
         /*
diff --git a/kernel/timer.c b/kernel/timer.c
index ceacc6626572..03bc7f1f1593 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -812,7 +812,7 @@ static inline void __run_timers(struct tvec_base *base)
         spin_unlock_irq(&base->lock);
 }
 
-#if defined(CONFIG_NO_IDLE_HZ) || defined(CONFIG_NO_HZ)
+#ifdef CONFIG_NO_HZ
 /*
  * Find out when the next timer event is due to happen. This
  * is used on S/390 to stop all activity when a cpus is idle.
@@ -947,14 +947,6 @@ unsigned long get_next_timer_interrupt(unsigned long now)
 
         return cmp_next_hrtimer_event(now, expires);
 }
-
-#ifdef CONFIG_NO_IDLE_HZ
-unsigned long next_timer_interrupt(void)
-{
-        return get_next_timer_interrupt(jiffies);
-}
-#endif
-
 #endif
 
 #ifndef CONFIG_VIRT_CPU_ACCOUNTING
@@ -1502,7 +1494,7 @@ void __init init_timers(void)
 
         BUG_ON(err == NOTIFY_BAD);
         register_cpu_notifier(&timers_nb);
-        open_softirq(TIMER_SOFTIRQ, run_timer_softirq, NULL);
+        open_softirq(TIMER_SOFTIRQ, run_timer_softirq);
 }
 
 /**
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
new file mode 100644
index 000000000000..263e9e6bbd60
--- /dev/null
+++ b/kernel/trace/Kconfig
@@ -0,0 +1,135 @@
+#
+# Architectures that offer an FTRACE implementation should select HAVE_FTRACE:
+#
+config HAVE_FTRACE
+        bool
+
+config HAVE_DYNAMIC_FTRACE
+        bool
+
+config TRACER_MAX_TRACE
+        bool
+
+config TRACING
+        bool
+        select DEBUG_FS
+        select STACKTRACE
+
+config FTRACE
+        bool "Kernel Function Tracer"
+        depends on HAVE_FTRACE
+        select FRAME_POINTER
+        select TRACING
+        select CONTEXT_SWITCH_TRACER
+        help
+          Enable the kernel to trace every kernel function. This is done
+          by using a compiler feature to insert a small, 5-byte No-Operation
+          instruction to the beginning of every kernel function, which NOP
+          sequence is then dynamically patched into a tracer call when
+          tracing is enabled by the administrator. If it's runtime disabled
+          (the bootup default), then the overhead of the instructions is very
+          small and not measurable even in micro-benchmarks.
+
+config IRQSOFF_TRACER
+        bool "Interrupts-off Latency Tracer"
+        default n
+        depends on TRACE_IRQFLAGS_SUPPORT
+        depends on GENERIC_TIME
+        depends on HAVE_FTRACE
+        select TRACE_IRQFLAGS
+        select TRACING
+        select TRACER_MAX_TRACE
+        help
+          This option measures the time spent in irqs-off critical
+          sections, with microsecond accuracy.
+
+          The default measurement method is a maximum search, which is
+          disabled by default and can be runtime (re-)started
+          via:
+
+              echo 0 > /debugfs/tracing/tracing_max_latency
+
+          (Note that kernel size and overhead increases with this option
+          enabled. This option and the preempt-off timing option can be
+          used together or separately.)
+
+config PREEMPT_TRACER
+        bool "Preemption-off Latency Tracer"
+        default n
+        depends on GENERIC_TIME
+        depends on PREEMPT
+        depends on HAVE_FTRACE
+        select TRACING
+        select TRACER_MAX_TRACE
+        help
+          This option measures the time spent in preemption off critical
+          sections, with microsecond accuracy.
+
+          The default measurement method is a maximum search, which is
+          disabled by default and can be runtime (re-)started
+          via:
+
+              echo 0 > /debugfs/tracing/tracing_max_latency
+
+          (Note that kernel size and overhead increases with this option
+          enabled. This option and the irqs-off timing option can be
+          used together or separately.)
+
+config SYSPROF_TRACER
+        bool "Sysprof Tracer"
+        depends on X86
+        select TRACING
+        help
+          This tracer provides the trace needed by the 'Sysprof' userspace
+          tool.
+
+config SCHED_TRACER
+        bool "Scheduling Latency Tracer"
+        depends on HAVE_FTRACE
+        select TRACING
+        select CONTEXT_SWITCH_TRACER
+        select TRACER_MAX_TRACE
+        help
+          This tracer tracks the latency of the highest priority task
+          to be scheduled in, starting from the point it has woken up.
+
+config CONTEXT_SWITCH_TRACER
+        bool "Trace process context switches"
+        depends on HAVE_FTRACE
+        select TRACING
+        select MARKERS
+        help
+          This tracer gets called from the context switch and records
+          all switching of tasks.
+
+config DYNAMIC_FTRACE
+        bool "enable/disable ftrace tracepoints dynamically"
+        depends on FTRACE
+        depends on HAVE_DYNAMIC_FTRACE
+        default y
+        help
+         This option will modify all the calls to ftrace dynamically
+         (will patch them out of the binary image and replaces them
+         with a No-Op instruction) as they are called. A table is
+         created to dynamically enable them again.
+
+         This way a CONFIG_FTRACE kernel is slightly larger, but otherwise
+         has native performance as long as no tracing is active.
+
+         The changes to the code are done by a kernel thread that
+         wakes up once a second and checks to see if any ftrace calls
+         were made. If so, it runs stop_machine (stops all CPUS)
+         and modifies the code to jump over the call to ftrace.
+
+config FTRACE_SELFTEST
+        bool
+
+config FTRACE_STARTUP_TEST
+        bool "Perform a startup test on ftrace"
+        depends on TRACING
+        select FTRACE_SELFTEST
+        help
+          This option performs a series of startup tests on ftrace. On bootup
+          a series of tests are made to verify that the tracer is
+          functioning properly. It will do tests on all the configured
+          tracers of ftrace.
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile
new file mode 100644
index 000000000000..71d17de17288
--- /dev/null
+++ b/kernel/trace/Makefile
@@ -0,0 +1,24 @@
+
+# Do not instrument the tracer itself:
+
+ifdef CONFIG_FTRACE
+ORIG_CFLAGS := $(KBUILD_CFLAGS)
+KBUILD_CFLAGS = $(subst -pg,,$(ORIG_CFLAGS))
+
+# selftest needs instrumentation
+CFLAGS_trace_selftest_dynamic.o = -pg
+obj-y += trace_selftest_dynamic.o
+endif
+
+obj-$(CONFIG_FTRACE) += libftrace.o
+
+obj-$(CONFIG_TRACING) += trace.o
+obj-$(CONFIG_CONTEXT_SWITCH_TRACER) += trace_sched_switch.o
+obj-$(CONFIG_SYSPROF_TRACER) += trace_sysprof.o
+obj-$(CONFIG_FTRACE) += trace_functions.o
+obj-$(CONFIG_IRQSOFF_TRACER) += trace_irqsoff.o
+obj-$(CONFIG_PREEMPT_TRACER) += trace_irqsoff.o
+obj-$(CONFIG_SCHED_TRACER) += trace_sched_wakeup.o
+obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o
+
+libftrace-y := ftrace.o
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
new file mode 100644
index 000000000000..4231a3dc224a
--- /dev/null
+++ b/kernel/trace/ftrace.c
@@ -0,0 +1,1727 @@
+/*
+ * Infrastructure for profiling code inserted by 'gcc -pg'.
+ *
+ * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
+ * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
+ *
+ * Originally ported from the -rt patch by:
+ *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
+ *
+ * Based on code in the latency_tracer, that is:
+ *
+ *  Copyright (C) 2004-2006 Ingo Molnar
+ *  Copyright (C) 2004 William Lee Irwin III
+ */
+
+#include <linux/stop_machine.h>
+#include <linux/clocksource.h>
+#include <linux/kallsyms.h>
+#include <linux/seq_file.h>
+#include <linux/debugfs.h>
+#include <linux/hardirq.h>
+#include <linux/kthread.h>
+#include <linux/uaccess.h>
+#include <linux/kprobes.h>
+#include <linux/ftrace.h>
+#include <linux/sysctl.h>
+#include <linux/ctype.h>
+#include <linux/hash.h>
+#include <linux/list.h>
+
+#include <asm/ftrace.h>
+
+#include "trace.h"
+
+/* ftrace_enabled is a method to turn ftrace on or off */
+int ftrace_enabled __read_mostly;
+static int last_ftrace_enabled;
+
+/*
+ * ftrace_disabled is set when an anomaly is discovered.
+ * ftrace_disabled is much stronger than ftrace_enabled.
+ */
+static int ftrace_disabled __read_mostly;
+
+static DEFINE_SPINLOCK(ftrace_lock);
+static DEFINE_MUTEX(ftrace_sysctl_lock);
+
+static struct ftrace_ops ftrace_list_end __read_mostly =
+{
+        .func = ftrace_stub,
+};
+
+static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
+ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
+
+static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
+{
+        struct ftrace_ops *op = ftrace_list;
+
+        /* in case someone actually ports this to alpha! */
+        read_barrier_depends();
+
+        while (op != &ftrace_list_end) {
+                /* silly alpha */
+                read_barrier_depends();
+                op->func(ip, parent_ip);
+                op = op->next;
+        };
+}
+
+/**
+ * clear_ftrace_function - reset the ftrace function
+ *
+ * This NULLs the ftrace function and in essence stops
+ * tracing.  There may be lag
+ */
+void clear_ftrace_function(void)
+{
+        ftrace_trace_function = ftrace_stub;
+}
+
+static int __register_ftrace_function(struct ftrace_ops *ops)
+{
+        /* Should never be called by interrupts */
+        spin_lock(&ftrace_lock);
+
+        ops->next = ftrace_list;
+        /*
+         * We are entering ops into the ftrace_list but another
+         * CPU might be walking that list. We need to make sure
+         * the ops->next pointer is valid before another CPU sees
+         * the ops pointer included into the ftrace_list.
+         */
+        smp_wmb();
+        ftrace_list = ops;
+
+        if (ftrace_enabled) {
+                /*
+                 * For one func, simply call it directly.
+                 * For more than one func, call the chain.
+                 */
+                if (ops->next == &ftrace_list_end)
+                        ftrace_trace_function = ops->func;
+                else
+                        ftrace_trace_function = ftrace_list_func;
+        }
+
+        spin_unlock(&ftrace_lock);
+
+        return 0;
+}
+
+static int __unregister_ftrace_function(struct ftrace_ops *ops)
+{
+        struct ftrace_ops **p;
+        int ret = 0;
+
+        spin_lock(&ftrace_lock);
+
+        /*
+         * If we are removing the last function, then simply point
+         * to the ftrace_stub.
+         */
+        if (ftrace_list == ops && ops->next == &ftrace_list_end) {
+                ftrace_trace_function = ftrace_stub;
+                ftrace_list = &ftrace_list_end;
+                goto out;
+        }
+
+        for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
+                if (*p == ops)
+                        break;
+
+        if (*p != ops) {
+                ret = -1;
+                goto out;
+        }
+
+        *p = (*p)->next;
+
+        if (ftrace_enabled) {
+                /* If we only have one func left, then call that directly */
+                if (ftrace_list == &ftrace_list_end ||
+                    ftrace_list->next == &ftrace_list_end)
+                        ftrace_trace_function = ftrace_list->func;
+        }
+
+ out:
+        spin_unlock(&ftrace_lock);
+
+        return ret;
+}
+
+#ifdef CONFIG_DYNAMIC_FTRACE
+
+static struct task_struct *ftraced_task;
+
+enum {
+        FTRACE_ENABLE_CALLS             = (1 << 0),
+        FTRACE_DISABLE_CALLS            = (1 << 1),
+        FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
+        FTRACE_ENABLE_MCOUNT            = (1 << 3),
+        FTRACE_DISABLE_MCOUNT           = (1 << 4),
+};
+
+static int ftrace_filtered;
+static int tracing_on;
+static int frozen_record_count;
+
+static struct hlist_head ftrace_hash[FTRACE_HASHSIZE];
+
+static DEFINE_PER_CPU(int, ftrace_shutdown_disable_cpu);
+
+static DEFINE_SPINLOCK(ftrace_shutdown_lock);
+static DEFINE_MUTEX(ftraced_lock);
+static DEFINE_MUTEX(ftrace_regex_lock);
+
+struct ftrace_page {
+        struct ftrace_page      *next;
+        unsigned long           index;
+        struct dyn_ftrace       records[];
+};
+
+#define ENTRIES_PER_PAGE \
+  ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
+
+/* estimate from running different kernels */
+#define NR_TO_INIT              10000
+
+static struct ftrace_page       *ftrace_pages_start;
+static struct ftrace_page       *ftrace_pages;
+
+static int ftraced_trigger;
+static int ftraced_suspend;
+static int ftraced_stop;
+
+static int ftrace_record_suspend;
+
+static struct dyn_ftrace *ftrace_free_records;
+
+
+#ifdef CONFIG_KPROBES
+static inline void freeze_record(struct dyn_ftrace *rec)
+{
+        if (!(rec->flags & FTRACE_FL_FROZEN)) {
+                rec->flags |= FTRACE_FL_FROZEN;
+                frozen_record_count++;
+        }
+}
+
+static inline void unfreeze_record(struct dyn_ftrace *rec)
+{
+        if (rec->flags & FTRACE_FL_FROZEN) {
+                rec->flags &= ~FTRACE_FL_FROZEN;
+                frozen_record_count--;
+        }
+}
+
+static inline int record_frozen(struct dyn_ftrace *rec)
+{
+        return rec->flags & FTRACE_FL_FROZEN;
+}
+#else
+# define freeze_record(rec)                     ({ 0; })
+# define unfreeze_record(rec)                   ({ 0; })
+# define record_frozen(rec)                     ({ 0; })
+#endif /* CONFIG_KPROBES */
+
+int skip_trace(unsigned long ip)
+{
+        unsigned long fl;
+        struct dyn_ftrace *rec;
+        struct hlist_node *t;
+        struct hlist_head *head;
+
+        if (frozen_record_count == 0)
+                return 0;
+
+        head = &ftrace_hash[hash_long(ip, FTRACE_HASHBITS)];
+        hlist_for_each_entry_rcu(rec, t, head, node) {
+                if (rec->ip == ip) {
+                        if (record_frozen(rec)) {
+                                if (rec->flags & FTRACE_FL_FAILED)
+                                        return 1;
+
+                                if (!(rec->flags & FTRACE_FL_CONVERTED))
+                                        return 1;
+
+                                if (!tracing_on || !ftrace_enabled)
+                                        return 1;
+
+                                if (ftrace_filtered) {
+                                        fl = rec->flags & (FTRACE_FL_FILTER |
+                                                           FTRACE_FL_NOTRACE);
+                                        if (!fl || (fl & FTRACE_FL_NOTRACE))
+                                                return 1;
+                                }
+                        }
+                        break;
+                }
+        }
+
+        return 0;
+}
+
+static inline int
+ftrace_ip_in_hash(unsigned long ip, unsigned long key)
+{
+        struct dyn_ftrace *p;
+        struct hlist_node *t;
+        int found = 0;
+
+        hlist_for_each_entry_rcu(p, t, &ftrace_hash[key], node) {
+                if (p->ip == ip) {
+                        found = 1;
+                        break;
+                }
+        }
+
+        return found;
+}
+
+static inline void
+ftrace_add_hash(struct dyn_ftrace *node, unsigned long key)
+{
+        hlist_add_head_rcu(&node->node, &ftrace_hash[key]);
+}
+
+/* called from kstop_machine */
+static inline void ftrace_del_hash(struct dyn_ftrace *node)
+{
+        hlist_del(&node->node);
+}
+
+static void ftrace_free_rec(struct dyn_ftrace *rec)
+{
+        /* no locking, only called from kstop_machine */
+
+        rec->ip = (unsigned long)ftrace_free_records;
+        ftrace_free_records = rec;
+        rec->flags |= FTRACE_FL_FREE;
+}
+
+static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
+{
+        struct dyn_ftrace *rec;
+
+        /* First check for freed records */
+        if (ftrace_free_records) {
+                rec = ftrace_free_records;
+
+                if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
+                        WARN_ON_ONCE(1);
+                        ftrace_free_records = NULL;
+                        ftrace_disabled = 1;
+                        ftrace_enabled = 0;
+                        return NULL;
+                }
+
+                ftrace_free_records = (void *)rec->ip;
+                memset(rec, 0, sizeof(*rec));
+                return rec;
+        }
+
+        if (ftrace_pages->index == ENTRIES_PER_PAGE) {
+                if (!ftrace_pages->next)
+                        return NULL;
+                ftrace_pages = ftrace_pages->next;
+        }
+
+        return &ftrace_pages->records[ftrace_pages->index++];
+}
+
+static void
+ftrace_record_ip(unsigned long ip)
+{
+        struct dyn_ftrace *node;
+        unsigned long flags;
+        unsigned long key;
+        int resched;
+        int atomic;
+        int cpu;
+
+        if (!ftrace_enabled || ftrace_disabled)
+                return;
+
+        resched = need_resched();
+        preempt_disable_notrace();
+
+        /*
+         * We simply need to protect against recursion.
+         * Use the the raw version of smp_processor_id and not
+         * __get_cpu_var which can call debug hooks that can
+         * cause a recursive crash here.
+         */
+        cpu = raw_smp_processor_id();
+        per_cpu(ftrace_shutdown_disable_cpu, cpu)++;
+        if (per_cpu(ftrace_shutdown_disable_cpu, cpu) != 1)
+                goto out;
+
+        if (unlikely(ftrace_record_suspend))
+                goto out;
+
+        key = hash_long(ip, FTRACE_HASHBITS);
+
+        WARN_ON_ONCE(key >= FTRACE_HASHSIZE);
+
+        if (ftrace_ip_in_hash(ip, key))
+                goto out;
+
+        atomic = irqs_disabled();
+
+        spin_lock_irqsave(&ftrace_shutdown_lock, flags);
+
+        /* This ip may have hit the hash before the lock */
+        if (ftrace_ip_in_hash(ip, key))
+                goto out_unlock;
+
+        node = ftrace_alloc_dyn_node(ip);
+        if (!node)
+                goto out_unlock;
+
+        node->ip = ip;
+
+        ftrace_add_hash(node, key);
+
+        ftraced_trigger = 1;
+
+ out_unlock:
+        spin_unlock_irqrestore(&ftrace_shutdown_lock, flags);
+ out:
+        per_cpu(ftrace_shutdown_disable_cpu, cpu)--;
+
+        /* prevent recursion with scheduler */
+        if (resched)
+                preempt_enable_no_resched_notrace();
+        else
+                preempt_enable_notrace();
+}
+
+#define FTRACE_ADDR ((long)(ftrace_caller))
+
+static int
+__ftrace_replace_code(struct dyn_ftrace *rec,
+                      unsigned char *old, unsigned char *new, int enable)
+{
+        unsigned long ip, fl;
+
+        ip = rec->ip;
+
+        if (ftrace_filtered && enable) {
+                /*
+                 * If filtering is on:
+                 *
+                 * If this record is set to be filtered and
+                 * is enabled then do nothing.
+                 *
+                 * If this record is set to be filtered and
+                 * it is not enabled, enable it.
+                 *
+                 * If this record is not set to be filtered
+                 * and it is not enabled do nothing.
+                 *
+                 * If this record is set not to trace then
+                 * do nothing.
+                 *
+                 * If this record is set not to trace and
+                 * it is enabled then disable it.
+                 *
+                 * If this record is not set to be filtered and
+                 * it is enabled, disable it.
+                 */
+
+                fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE |
+                                   FTRACE_FL_ENABLED);
+
+                if ((fl ==  (FTRACE_FL_FILTER | FTRACE_FL_ENABLED)) ||
+                    (fl ==  (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE)) ||
+                    !fl || (fl == FTRACE_FL_NOTRACE))
+                        return 0;
+
+                /*
+                 * If it is enabled disable it,
+                 * otherwise enable it!
+                 */
+                if (fl & FTRACE_FL_ENABLED) {
+                        /* swap new and old */
+                        new = old;
+                        old = ftrace_call_replace(ip, FTRACE_ADDR);
+                        rec->flags &= ~FTRACE_FL_ENABLED;
+                } else {
+                        new = ftrace_call_replace(ip, FTRACE_ADDR);
+                        rec->flags |= FTRACE_FL_ENABLED;
+                }
+        } else {
+
+                if (enable) {
+                        /*
+                         * If this record is set not to trace and is
+                         * not enabled, do nothing.
+                         */
+                        fl = rec->flags & (FTRACE_FL_NOTRACE | FTRACE_FL_ENABLED);
+                        if (fl == FTRACE_FL_NOTRACE)
+                                return 0;
+
+                        new = ftrace_call_replace(ip, FTRACE_ADDR);
+                } else
+                        old = ftrace_call_replace(ip, FTRACE_ADDR);
+
+                if (enable) {
+                        if (rec->flags & FTRACE_FL_ENABLED)
+                                return 0;
+                        rec->flags |= FTRACE_FL_ENABLED;
+                } else {
+                        if (!(rec->flags & FTRACE_FL_ENABLED))
+                                return 0;
+                        rec->flags &= ~FTRACE_FL_ENABLED;
+                }
+        }
+
+        return ftrace_modify_code(ip, old, new);
+}
+
+static void ftrace_replace_code(int enable)
+{
+        int i, failed;
+        unsigned char *new = NULL, *old = NULL;
+        struct dyn_ftrace *rec;
+        struct ftrace_page *pg;
+
+        if (enable)
+                old = ftrace_nop_replace();
+        else
+                new = ftrace_nop_replace();
+
+        for (pg = ftrace_pages_start; pg; pg = pg->next) {
+                for (i = 0; i < pg->index; i++) {
+                        rec = &pg->records[i];
+
+                        /* don't modify code that has already faulted */
+                        if (rec->flags & FTRACE_FL_FAILED)
+                                continue;
+
+                        /* ignore updates to this record's mcount site */
+                        if (get_kprobe((void *)rec->ip)) {
+                                freeze_record(rec);
+                                continue;
+                        } else {
+                                unfreeze_record(rec);
+                        }
+
+                        failed = __ftrace_replace_code(rec, old, new, enable);
+                        if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
+                                rec->flags |= FTRACE_FL_FAILED;
+                                if ((system_state == SYSTEM_BOOTING) ||
+                                    !core_kernel_text(rec->ip)) {
+                                        ftrace_del_hash(rec);
+                                        ftrace_free_rec(rec);
+                                }
+                        }
+                }
+        }
+}
+
+static void ftrace_shutdown_replenish(void)
+{
+        if (ftrace_pages->next)
+                return;
+
+        /* allocate another page */
+        ftrace_pages->next = (void *)get_zeroed_page(GFP_KERNEL);
+}
+
+static int
+ftrace_code_disable(struct dyn_ftrace *rec)
+{
+        unsigned long ip;
+        unsigned char *nop, *call;
+        int failed;
+
+        ip = rec->ip;
+
+        nop = ftrace_nop_replace();
+        call = ftrace_call_replace(ip, MCOUNT_ADDR);
+
+        failed = ftrace_modify_code(ip, call, nop);
+        if (failed) {
+                rec->flags |= FTRACE_FL_FAILED;
+                return 0;
+        }
+        return 1;
+}
+
+static int __ftrace_update_code(void *ignore);
+
+static int __ftrace_modify_code(void *data)
+{
+        unsigned long addr;
+        int *command = data;
+
+        if (*command & FTRACE_ENABLE_CALLS) {
+                /*
+                 * Update any recorded ips now that we have the
+                 * machine stopped
+                 */
+                __ftrace_update_code(NULL);
+                ftrace_replace_code(1);
+                tracing_on = 1;
+        } else if (*command & FTRACE_DISABLE_CALLS) {
+                ftrace_replace_code(0);
+                tracing_on = 0;
+        }
+
+        if (*command & FTRACE_UPDATE_TRACE_FUNC)
+                ftrace_update_ftrace_func(ftrace_trace_function);
+
+        if (*command & FTRACE_ENABLE_MCOUNT) {
+                addr = (unsigned long)ftrace_record_ip;
+                ftrace_mcount_set(&addr);
+        } else if (*command & FTRACE_DISABLE_MCOUNT) {
+                addr = (unsigned long)ftrace_stub;
+                ftrace_mcount_set(&addr);
+        }
+
+        return 0;
+}
+
+static void ftrace_run_update_code(int command)
+{
+        stop_machine_run(__ftrace_modify_code, &command, NR_CPUS);
+}
+
+void ftrace_disable_daemon(void)
+{
+        /* Stop the daemon from calling kstop_machine */
+        mutex_lock(&ftraced_lock);
+        ftraced_stop = 1;
+        mutex_unlock(&ftraced_lock);
+
+        ftrace_force_update();
+}
+
+void ftrace_enable_daemon(void)
+{
+        mutex_lock(&ftraced_lock);
+        ftraced_stop = 0;
+        mutex_unlock(&ftraced_lock);
+
+        ftrace_force_update();
+}
+
+static ftrace_func_t saved_ftrace_func;
+
+static void ftrace_startup(void)
+{
+        int command = 0;
+
+        if (unlikely(ftrace_disabled))
+                return;
+
+        mutex_lock(&ftraced_lock);
+        ftraced_suspend++;
+        if (ftraced_suspend == 1)
+                command |= FTRACE_ENABLE_CALLS;
+
+        if (saved_ftrace_func != ftrace_trace_function) {
+                saved_ftrace_func = ftrace_trace_function;
+                command |= FTRACE_UPDATE_TRACE_FUNC;
+        }
+
+        if (!command || !ftrace_enabled)
+                goto out;
+
+        ftrace_run_update_code(command);
+ out:
+        mutex_unlock(&ftraced_lock);
+}
+
+static void ftrace_shutdown(void)
+{
+        int command = 0;
+
+        if (unlikely(ftrace_disabled))
+                return;
+
+        mutex_lock(&ftraced_lock);
+        ftraced_suspend--;
+        if (!ftraced_suspend)
+                command |= FTRACE_DISABLE_CALLS;
+
+        if (saved_ftrace_func != ftrace_trace_function) {
+                saved_ftrace_func = ftrace_trace_function;
+                command |= FTRACE_UPDATE_TRACE_FUNC;
+        }
+
+        if (!command || !ftrace_enabled)
+                goto out;
+
+        ftrace_run_update_code(command);
+ out:
+        mutex_unlock(&ftraced_lock);
+}
+
+static void ftrace_startup_sysctl(void)
+{
+        int command = FTRACE_ENABLE_MCOUNT;
+
+        if (unlikely(ftrace_disabled))
+                return;
+
+        mutex_lock(&ftraced_lock);
+        /* Force update next time */
+        saved_ftrace_func = NULL;
+        /* ftraced_suspend is true if we want ftrace running */
+        if (ftraced_suspend)
+                command |= FTRACE_ENABLE_CALLS;
+
+        ftrace_run_update_code(command);
+        mutex_unlock(&ftraced_lock);
+}
+
+static void ftrace_shutdown_sysctl(void)
+{
+        int command = FTRACE_DISABLE_MCOUNT;
+
+        if (unlikely(ftrace_disabled))
+                return;
+
+        mutex_lock(&ftraced_lock);
+        /* ftraced_suspend is true if ftrace is running */
+        if (ftraced_suspend)
+                command |= FTRACE_DISABLE_CALLS;
+
+        ftrace_run_update_code(command);
+        mutex_unlock(&ftraced_lock);
+}
+
+static cycle_t          ftrace_update_time;
+static unsigned long    ftrace_update_cnt;
+unsigned long           ftrace_update_tot_cnt;
+
+static int __ftrace_update_code(void *ignore)
+{
+        int i, save_ftrace_enabled;
+        cycle_t start, stop;
+        struct dyn_ftrace *p;
+        struct hlist_node *t, *n;
+        struct hlist_head *head, temp_list;
+
+        /* Don't be recording funcs now */
+        ftrace_record_suspend++;
+        save_ftrace_enabled = ftrace_enabled;
+        ftrace_enabled = 0;
+
+        start = ftrace_now(raw_smp_processor_id());
+        ftrace_update_cnt = 0;
+
+        /* No locks needed, the machine is stopped! */
+        for (i = 0; i < FTRACE_HASHSIZE; i++) {
+                INIT_HLIST_HEAD(&temp_list);
+                head = &ftrace_hash[i];
+
+                /* all CPUS are stopped, we are safe to modify code */
+                hlist_for_each_entry_safe(p, t, n, head, node) {
+                        /* Skip over failed records which have not been
+                         * freed. */
+                        if (p->flags & FTRACE_FL_FAILED)
+                                continue;
+
+                        /* Unconverted records are always at the head of the
+                         * hash bucket. Once we encounter a converted record,
+                         * simply skip over to the next bucket. Saves ftraced
+                         * some processor cycles (ftrace does its bid for
+                         * global warming :-p ). */
+                        if (p->flags & (FTRACE_FL_CONVERTED))
+                                break;
+
+                        /* Ignore updates to this record's mcount site.
+                         * Reintroduce this record at the head of this
+                         * bucket to attempt to "convert" it again if
+                         * the kprobe on it is unregistered before the
+                         * next run. */
+                        if (get_kprobe((void *)p->ip)) {
+                                ftrace_del_hash(p);
+                                INIT_HLIST_NODE(&p->node);
+                                hlist_add_head(&p->node, &temp_list);
+                                freeze_record(p);
+                                continue;
+                        } else {
+                                unfreeze_record(p);
+                        }
+
+                        /* convert record (i.e, patch mcount-call with NOP) */
+                        if (ftrace_code_disable(p)) {
+                                p->flags |= FTRACE_FL_CONVERTED;
+                                ftrace_update_cnt++;
+                        } else {
+                                if ((system_state == SYSTEM_BOOTING) ||
+                                    !core_kernel_text(p->ip)) {
+                                        ftrace_del_hash(p);
+                                        ftrace_free_rec(p);
+                                }
+                        }
+                }
+
+                hlist_for_each_entry_safe(p, t, n, &temp_list, node) {
+                        hlist_del(&p->node);
+                        INIT_HLIST_NODE(&p->node);
+                        hlist_add_head(&p->node, head);
+                }
+        }
+
+        stop = ftrace_now(raw_smp_processor_id());
+        ftrace_update_time = stop - start;
+        ftrace_update_tot_cnt += ftrace_update_cnt;
+        ftraced_trigger = 0;
+
+        ftrace_enabled = save_ftrace_enabled;
+        ftrace_record_suspend--;
+
+        return 0;
+}
+
+static int ftrace_update_code(void)
+{
+        if (unlikely(ftrace_disabled) ||
+            !ftrace_enabled || !ftraced_trigger)
+                return 0;
+
+        stop_machine_run(__ftrace_update_code, NULL, NR_CPUS);
+
+        return 1;
+}
+
+static int ftraced(void *ignore)
+{
+        unsigned long usecs;
+
+        while (!kthread_should_stop()) {
+
+                set_current_state(TASK_INTERRUPTIBLE);
+
+                /* check once a second */
+                schedule_timeout(HZ);
+
+                if (unlikely(ftrace_disabled))
+                        continue;
+
+                mutex_lock(&ftrace_sysctl_lock);
+                mutex_lock(&ftraced_lock);
+                if (!ftraced_suspend && !ftraced_stop &&
+                    ftrace_update_code()) {
+                        usecs = nsecs_to_usecs(ftrace_update_time);
+                        if (ftrace_update_tot_cnt > 100000) {
+                                ftrace_update_tot_cnt = 0;
+                                pr_info("hm, dftrace overflow: %lu change%s"
+                                        " (%lu total) in %lu usec%s\n",
+                                        ftrace_update_cnt,
+                                        ftrace_update_cnt != 1 ? "s" : "",
+                                        ftrace_update_tot_cnt,
+                                        usecs, usecs != 1 ? "s" : "");
+                                ftrace_disabled = 1;
+                                WARN_ON_ONCE(1);
+                        }
+                }
+                mutex_unlock(&ftraced_lock);
+                mutex_unlock(&ftrace_sysctl_lock);
+
+                ftrace_shutdown_replenish();
+        }
+        __set_current_state(TASK_RUNNING);
+        return 0;
+}
+
+static int __init ftrace_dyn_table_alloc(void)
+{
+        struct ftrace_page *pg;
+        int cnt;
+        int i;
+
+        /* allocate a few pages */
+        ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
+        if (!ftrace_pages_start)
+                return -1;
+
+        /*
+         * Allocate a few more pages.
+         *
+         * TODO: have some parser search vmlinux before
+         *   final linking to find all calls to ftrace.
+         *   Then we can:
+         *    a) know how many pages to allocate.
+         *     and/or
+         *    b) set up the table then.
+         *
+         *  The dynamic code is still necessary for
+         *  modules.
+         */
+
+        pg = ftrace_pages = ftrace_pages_start;
+
+        cnt = NR_TO_INIT / ENTRIES_PER_PAGE;
+
+        for (i = 0; i < cnt; i++) {
+                pg->next = (void *)get_zeroed_page(GFP_KERNEL);
+
+                /* If we fail, we'll try later anyway */
+                if (!pg->next)
+                        break;
+
+                pg = pg->next;
+        }
+
+        return 0;
+}
+
+enum {
+        FTRACE_ITER_FILTER      = (1 << 0),
+        FTRACE_ITER_CONT        = (1 << 1),
+        FTRACE_ITER_NOTRACE     = (1 << 2),
+        FTRACE_ITER_FAILURES    = (1 << 3),
+};
+
+#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
+
+struct ftrace_iterator {
+        loff_t                  pos;
+        struct ftrace_page      *pg;
+        unsigned                idx;
+        unsigned                flags;
+        unsigned char           buffer[FTRACE_BUFF_MAX+1];
+        unsigned                buffer_idx;
+        unsigned                filtered;
+};
+
+static void *
+t_next(struct seq_file *m, void *v, loff_t *pos)
+{
+        struct ftrace_iterator *iter = m->private;
+        struct dyn_ftrace *rec = NULL;
+
+        (*pos)++;
+
+ retry:
+        if (iter->idx >= iter->pg->index) {
+                if (iter->pg->next) {
+                        iter->pg = iter->pg->next;
+                        iter->idx = 0;
+                        goto retry;
+                }
+        } else {
+                rec = &iter->pg->records[iter->idx++];
+                if ((!(iter->flags & FTRACE_ITER_FAILURES) &&
+                     (rec->flags & FTRACE_FL_FAILED)) ||
+
+                    ((iter->flags & FTRACE_ITER_FAILURES) &&
+                     (!(rec->flags & FTRACE_FL_FAILED) ||
+                      (rec->flags & FTRACE_FL_FREE))) ||
+
+                    ((iter->flags & FTRACE_ITER_FILTER) &&
+                     !(rec->flags & FTRACE_FL_FILTER)) ||
+
+                    ((iter->flags & FTRACE_ITER_NOTRACE) &&
+                     !(rec->flags & FTRACE_FL_NOTRACE))) {
+                        rec = NULL;
+                        goto retry;
+                }
+        }
+
+        iter->pos = *pos;
+
+        return rec;
+}
+
+static void *t_start(struct seq_file *m, loff_t *pos)
+{
+        struct ftrace_iterator *iter = m->private;
+        void *p = NULL;
+        loff_t l = -1;
+
+        if (*pos != iter->pos) {
+                for (p = t_next(m, p, &l); p && l < *pos; p = t_next(m, p, &l))
+                        ;
+        } else {
+                l = *pos;
+                p = t_next(m, p, &l);
+        }
+
+        return p;
+}
+
+static void t_stop(struct seq_file *m, void *p)
+{
+}
+
+static int t_show(struct seq_file *m, void *v)
+{
+        struct dyn_ftrace *rec = v;
+        char str[KSYM_SYMBOL_LEN];
+
+        if (!rec)
+                return 0;
+
+        kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
+
+        seq_printf(m, "%s\n", str);
+
+        return 0;
+}
+
+static struct seq_operations show_ftrace_seq_ops = {
+        .start = t_start,
+        .next = t_next,
+        .stop = t_stop,
+        .show = t_show,
+};
+
+static int
+ftrace_avail_open(struct inode *inode, struct file *file)
+{
+        struct ftrace_iterator *iter;
+        int ret;
+
+        if (unlikely(ftrace_disabled))
+                return -ENODEV;
+
+        iter = kzalloc(sizeof(*iter), GFP_KERNEL);
+        if (!iter)
+                return -ENOMEM;
+
+        iter->pg = ftrace_pages_start;
+        iter->pos = -1;
+
+        ret = seq_open(file, &show_ftrace_seq_ops);
+        if (!ret) {
+                struct seq_file *m = file->private_data;
+
+                m->private = iter;
+        } else {
+                kfree(iter);
+        }
+
+        return ret;
+}
+
+int ftrace_avail_release(struct inode *inode, struct file *file)
+{
+        struct seq_file *m = (struct seq_file *)file->private_data;
+        struct ftrace_iterator *iter = m->private;
+
+        seq_release(inode, file);
+        kfree(iter);
+
+        return 0;
+}
+
+static int
+ftrace_failures_open(struct inode *inode, struct file *file)
+{
+        int ret;
+        struct seq_file *m;
+        struct ftrace_iterator *iter;
+
+        ret = ftrace_avail_open(inode, file);
+        if (!ret) {
+                m = (struct seq_file *)file->private_data;
+                iter = (struct ftrace_iterator *)m->private;
+                iter->flags = FTRACE_ITER_FAILURES;
+        }
+
+        return ret;
+}
+
+
+static void ftrace_filter_reset(int enable)
+{
+        struct ftrace_page *pg;
+        struct dyn_ftrace *rec;
+        unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
+        unsigned i;
+
+        /* keep kstop machine from running */
+        preempt_disable();
+        if (enable)
+                ftrace_filtered = 0;
+        pg = ftrace_pages_start;
+        while (pg) {
+                for (i = 0; i < pg->index; i++) {
+                        rec = &pg->records[i];
+                        if (rec->flags & FTRACE_FL_FAILED)
+                                continue;
+                        rec->flags &= ~type;
+                }
+                pg = pg->next;
+        }
+        preempt_enable();
+}
+
+static int
+ftrace_regex_open(struct inode *inode, struct file *file, int enable)
+{
+        struct ftrace_iterator *iter;
+        int ret = 0;
+
+        if (unlikely(ftrace_disabled))
+                return -ENODEV;
+
+        iter = kzalloc(sizeof(*iter), GFP_KERNEL);
+        if (!iter)
+                return -ENOMEM;
+
+        mutex_lock(&ftrace_regex_lock);
+        if ((file->f_mode & FMODE_WRITE) &&
+            !(file->f_flags & O_APPEND))
+                ftrace_filter_reset(enable);
+
+        if (file->f_mode & FMODE_READ) {
+                iter->pg = ftrace_pages_start;
+                iter->pos = -1;
+                iter->flags = enable ? FTRACE_ITER_FILTER :
+                        FTRACE_ITER_NOTRACE;
+
+                ret = seq_open(file, &show_ftrace_seq_ops);
+                if (!ret) {
+                        struct seq_file *m = file->private_data;
+                        m->private = iter;
+                } else
+                        kfree(iter);
+        } else
+                file->private_data = iter;
+        mutex_unlock(&ftrace_regex_lock);
+
+        return ret;
+}
+
+static int
+ftrace_filter_open(struct inode *inode, struct file *file)
+{
+        return ftrace_regex_open(inode, file, 1);
+}
+
+static int
+ftrace_notrace_open(struct inode *inode, struct file *file)
+{
+        return ftrace_regex_open(inode, file, 0);
+}
+
+static ssize_t
+ftrace_regex_read(struct file *file, char __user *ubuf,
+                       size_t cnt, loff_t *ppos)
+{
+        if (file->f_mode & FMODE_READ)
+                return seq_read(file, ubuf, cnt, ppos);
+        else
+                return -EPERM;
+}
+
+static loff_t
+ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
+{
+        loff_t ret;
+
+        if (file->f_mode & FMODE_READ)
+                ret = seq_lseek(file, offset, origin);
+        else
+                file->f_pos = ret = 1;
+
+        return ret;
+}
+
+enum {
+        MATCH_FULL,
+        MATCH_FRONT_ONLY,
+        MATCH_MIDDLE_ONLY,
+        MATCH_END_ONLY,
+};
+
+static void
+ftrace_match(unsigned char *buff, int len, int enable)
+{
+        char str[KSYM_SYMBOL_LEN];
+        char *search = NULL;
+        struct ftrace_page *pg;
+        struct dyn_ftrace *rec;
+        int type = MATCH_FULL;
+        unsigned long flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
+        unsigned i, match = 0, search_len = 0;
+
+        for (i = 0; i < len; i++) {
+                if (buff[i] == '*') {
+                        if (!i) {
+                                search = buff + i + 1;
+                                type = MATCH_END_ONLY;
+                                search_len = len - (i + 1);
+                        } else {
+                                if (type == MATCH_END_ONLY) {
+                                        type = MATCH_MIDDLE_ONLY;
+                                } else {
+                                        match = i;
+                                        type = MATCH_FRONT_ONLY;
+                                }
+                                buff[i] = 0;
+                                break;
+                        }
+                }
+        }
+
+        /* keep kstop machine from running */
+        preempt_disable();
+        if (enable)
+                ftrace_filtered = 1;
+        pg = ftrace_pages_start;
+        while (pg) {
+                for (i = 0; i < pg->index; i++) {
+                        int matched = 0;
+                        char *ptr;
+
+                        rec = &pg->records[i];
+                        if (rec->flags & FTRACE_FL_FAILED)
+                                continue;
+                        kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
+                        switch (type) {
+                        case MATCH_FULL:
+                                if (strcmp(str, buff) == 0)
+                                        matched = 1;
+                                break;
+                        case MATCH_FRONT_ONLY:
+                                if (memcmp(str, buff, match) == 0)
+                                        matched = 1;
+                                break;
+                        case MATCH_MIDDLE_ONLY:
+                                if (strstr(str, search))
+                                        matched = 1;
+                                break;
+                        case MATCH_END_ONLY:
+                                ptr = strstr(str, search);
+                                if (ptr && (ptr[search_len] == 0))
+                                        matched = 1;
+                                break;
+                        }
+                        if (matched)
+                                rec->flags |= flag;
+                }
+                pg = pg->next;
+        }
+        preempt_enable();
+}
+
+static ssize_t
+ftrace_regex_write(struct file *file, const char __user *ubuf,
+                   size_t cnt, loff_t *ppos, int enable)
+{
+        struct ftrace_iterator *iter;
+        char ch;
+        size_t read = 0;
+        ssize_t ret;
+
+        if (!cnt || cnt < 0)
+                return 0;
+
+        mutex_lock(&ftrace_regex_lock);
+
+        if (file->f_mode & FMODE_READ) {
+                struct seq_file *m = file->private_data;
+                iter = m->private;
+        } else
+                iter = file->private_data;
+
+        if (!*ppos) {
+                iter->flags &= ~FTRACE_ITER_CONT;
+                iter->buffer_idx = 0;
+        }
+
+        ret = get_user(ch, ubuf++);
+        if (ret)
+                goto out;
+        read++;
+        cnt--;
+
+        if (!(iter->flags & ~FTRACE_ITER_CONT)) {
+                /* skip white space */
+                while (cnt && isspace(ch)) {
+                        ret = get_user(ch, ubuf++);
+                        if (ret)
+                                goto out;
+                        read++;
+                        cnt--;
+                }
+
+                if (isspace(ch)) {
+                        file->f_pos += read;
+                        ret = read;
+                        goto out;
+                }
+
+                iter->buffer_idx = 0;
+        }
+
+        while (cnt && !isspace(ch)) {
+                if (iter->buffer_idx < FTRACE_BUFF_MAX)
+                        iter->buffer[iter->buffer_idx++] = ch;
+                else {
+                        ret = -EINVAL;
+                        goto out;
+                }
+                ret = get_user(ch, ubuf++);
+                if (ret)
+                        goto out;
+                read++;
+                cnt--;
+        }
+
+        if (isspace(ch)) {
+                iter->filtered++;
+                iter->buffer[iter->buffer_idx] = 0;
+                ftrace_match(iter->buffer, iter->buffer_idx, enable);
+                iter->buffer_idx = 0;
+        } else
+                iter->flags |= FTRACE_ITER_CONT;
+
+
+        file->f_pos += read;
+
+        ret = read;
+ out:
+        mutex_unlock(&ftrace_regex_lock);
+
+        return ret;
+}
+
+static ssize_t
+ftrace_filter_write(struct file *file, const char __user *ubuf,
+                    size_t cnt, loff_t *ppos)
+{
+        return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
+}
+
+static ssize_t
+ftrace_notrace_write(struct file *file, const char __user *ubuf,
+                     size_t cnt, loff_t *ppos)
+{
+        return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
+}
+
+static void
+ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
+{
+        if (unlikely(ftrace_disabled))
+                return;
+
+        mutex_lock(&ftrace_regex_lock);
+        if (reset)
+                ftrace_filter_reset(enable);
+        if (buf)
+                ftrace_match(buf, len, enable);
+        mutex_unlock(&ftrace_regex_lock);
+}
+
+/**
+ * ftrace_set_filter - set a function to filter on in ftrace
+ * @buf - the string that holds the function filter text.
+ * @len - the length of the string.
+ * @reset - non zero to reset all filters before applying this filter.
+ *
+ * Filters denote which functions should be enabled when tracing is enabled.
+ * If @buf is NULL and reset is set, all functions will be enabled for tracing.
+ */
+void ftrace_set_filter(unsigned char *buf, int len, int reset)
+{
+        ftrace_set_regex(buf, len, reset, 1);
+}
+
+/**
+ * ftrace_set_notrace - set a function to not trace in ftrace
+ * @buf - the string that holds the function notrace text.
+ * @len - the length of the string.
+ * @reset - non zero to reset all filters before applying this filter.
+ *
+ * Notrace Filters denote which functions should not be enabled when tracing
+ * is enabled. If @buf is NULL and reset is set, all functions will be enabled
+ * for tracing.
+ */
+void ftrace_set_notrace(unsigned char *buf, int len, int reset)
+{
+        ftrace_set_regex(buf, len, reset, 0);
+}
+
+static int
+ftrace_regex_release(struct inode *inode, struct file *file, int enable)
+{
+        struct seq_file *m = (struct seq_file *)file->private_data;
+        struct ftrace_iterator *iter;
+
+        mutex_lock(&ftrace_regex_lock);
+        if (file->f_mode & FMODE_READ) {
+                iter = m->private;
+
+                seq_release(inode, file);
+        } else
+                iter = file->private_data;
+
+        if (iter->buffer_idx) {
+                iter->filtered++;
+                iter->buffer[iter->buffer_idx] = 0;
+                ftrace_match(iter->buffer, iter->buffer_idx, enable);
+        }
+
+        mutex_lock(&ftrace_sysctl_lock);
+        mutex_lock(&ftraced_lock);
+        if (iter->filtered && ftraced_suspend && ftrace_enabled)
+                ftrace_run_update_code(FTRACE_ENABLE_CALLS);
+        mutex_unlock(&ftraced_lock);
+        mutex_unlock(&ftrace_sysctl_lock);
+
+        kfree(iter);
+        mutex_unlock(&ftrace_regex_lock);
+        return 0;
+}
+
+static int
+ftrace_filter_release(struct inode *inode, struct file *file)
+{
+        return ftrace_regex_release(inode, file, 1);
+}
+
+static int
+ftrace_notrace_release(struct inode *inode, struct file *file)
+{
+        return ftrace_regex_release(inode, file, 0);
+}
+
+static ssize_t
+ftraced_read(struct file *filp, char __user *ubuf,
+                     size_t cnt, loff_t *ppos)
+{
+        /* don't worry about races */
+        char *buf = ftraced_stop ? "disabled\n" : "enabled\n";
+        int r = strlen(buf);
+
+        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+static ssize_t
+ftraced_write(struct file *filp, const char __user *ubuf,
+                      size_t cnt, loff_t *ppos)
+{
+        char buf[64];
+        long val;
+        int ret;
+
+        if (cnt >= sizeof(buf))
+                return -EINVAL;
+
+        if (copy_from_user(&buf, ubuf, cnt))
+                return -EFAULT;
+
+        if (strncmp(buf, "enable", 6) == 0)
+                val = 1;
+        else if (strncmp(buf, "disable", 7) == 0)
+                val = 0;
+        else {
+                buf[cnt] = 0;
+
+                ret = strict_strtoul(buf, 10, &val);
+                if (ret < 0)
+                        return ret;
+
+                val = !!val;
+        }
+
+        if (val)
+                ftrace_enable_daemon();
+        else
+                ftrace_disable_daemon();
+
+        filp->f_pos += cnt;
+
+        return cnt;
+}
+
+static struct file_operations ftrace_avail_fops = {
+        .open = ftrace_avail_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = ftrace_avail_release,
+};
+
+static struct file_operations ftrace_failures_fops = {
+        .open = ftrace_failures_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = ftrace_avail_release,
+};
+
+static struct file_operations ftrace_filter_fops = {
+        .open = ftrace_filter_open,
+        .read = ftrace_regex_read,
+        .write = ftrace_filter_write,
+        .llseek = ftrace_regex_lseek,
+        .release = ftrace_filter_release,
+};
+
+static struct file_operations ftrace_notrace_fops = {
+        .open = ftrace_notrace_open,
+        .read = ftrace_regex_read,
+        .write = ftrace_notrace_write,
+        .llseek = ftrace_regex_lseek,
+        .release = ftrace_notrace_release,
+};
+
+static struct file_operations ftraced_fops = {
+        .open = tracing_open_generic,
+        .read = ftraced_read,
+        .write = ftraced_write,
+};
+
+/**
+ * ftrace_force_update - force an update to all recording ftrace functions
+ */
+int ftrace_force_update(void)
+{
+        int ret = 0;
+
+        if (unlikely(ftrace_disabled))
+                return -ENODEV;
+
+        mutex_lock(&ftrace_sysctl_lock);
+        mutex_lock(&ftraced_lock);
+
+        /*
+         * If ftraced_trigger is not set, then there is nothing
+         * to update.
+         */
+        if (ftraced_trigger && !ftrace_update_code())
+                ret = -EBUSY;
+
+        mutex_unlock(&ftraced_lock);
+        mutex_unlock(&ftrace_sysctl_lock);
+
+        return ret;
+}
+
+static void ftrace_force_shutdown(void)
+{
+        struct task_struct *task;
+        int command = FTRACE_DISABLE_CALLS | FTRACE_UPDATE_TRACE_FUNC;
+
+        mutex_lock(&ftraced_lock);
+        task = ftraced_task;
+        ftraced_task = NULL;
+        ftraced_suspend = -1;
+        ftrace_run_update_code(command);
+        mutex_unlock(&ftraced_lock);
+
+        if (task)
+                kthread_stop(task);
+}
+
+static __init int ftrace_init_debugfs(void)
+{
+        struct dentry *d_tracer;
+        struct dentry *entry;
+
+        d_tracer = tracing_init_dentry();
+
+        entry = debugfs_create_file("available_filter_functions", 0444,
+                                    d_tracer, NULL, &ftrace_avail_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs "
+                           "'available_filter_functions' entry\n");
+
+        entry = debugfs_create_file("failures", 0444,
+                                    d_tracer, NULL, &ftrace_failures_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs 'failures' entry\n");
+
+        entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
+                                    NULL, &ftrace_filter_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs "
+                           "'set_ftrace_filter' entry\n");
+
+        entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
+                                    NULL, &ftrace_notrace_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs "
+                           "'set_ftrace_notrace' entry\n");
+
+        entry = debugfs_create_file("ftraced_enabled", 0644, d_tracer,
+                                    NULL, &ftraced_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs "
+                           "'ftraced_enabled' entry\n");
+        return 0;
+}
+
+fs_initcall(ftrace_init_debugfs);
+
+static int __init ftrace_dynamic_init(void)
+{
+        struct task_struct *p;
+        unsigned long addr;
+        int ret;
+
+        addr = (unsigned long)ftrace_record_ip;
+
+        stop_machine_run(ftrace_dyn_arch_init, &addr, NR_CPUS);
+
+        /* ftrace_dyn_arch_init places the return code in addr */
+        if (addr) {
+                ret = (int)addr;
+                goto failed;
+        }
+
+        ret = ftrace_dyn_table_alloc();
+        if (ret)
+                goto failed;
+
+        p = kthread_run(ftraced, NULL, "ftraced");
+        if (IS_ERR(p)) {
+                ret = -1;
+                goto failed;
+        }
+
+        last_ftrace_enabled = ftrace_enabled = 1;
+        ftraced_task = p;
+
+        return 0;
+
+ failed:
+        ftrace_disabled = 1;
+        return ret;
+}
+
+core_initcall(ftrace_dynamic_init);
+#else
+# define ftrace_startup()               do { } while (0)
+# define ftrace_shutdown()              do { } while (0)
+# define ftrace_startup_sysctl()        do { } while (0)
+# define ftrace_shutdown_sysctl()       do { } while (0)
+# define ftrace_force_shutdown()        do { } while (0)
+#endif /* CONFIG_DYNAMIC_FTRACE */
+
+/**
+ * ftrace_kill_atomic - kill ftrace from critical sections
+ *
+ * This function should be used by panic code. It stops ftrace
+ * but in a not so nice way. If you need to simply kill ftrace
+ * from a non-atomic section, use ftrace_kill.
+ */
+void ftrace_kill_atomic(void)
+{
+        ftrace_disabled = 1;
+        ftrace_enabled = 0;
+#ifdef CONFIG_DYNAMIC_FTRACE
+        ftraced_suspend = -1;
+#endif
+        clear_ftrace_function();
+}
+
+/**
+ * ftrace_kill - totally shutdown ftrace
+ *
+ * This is a safety measure. If something was detected that seems
+ * wrong, calling this function will keep ftrace from doing
+ * any more modifications, and updates.
+ * used when something went wrong.
+ */
+void ftrace_kill(void)
+{
+        mutex_lock(&ftrace_sysctl_lock);
+        ftrace_disabled = 1;
+        ftrace_enabled = 0;
+
+        clear_ftrace_function();
+        mutex_unlock(&ftrace_sysctl_lock);
+
+        /* Try to totally disable ftrace */
+        ftrace_force_shutdown();
+}
+
+/**
+ * register_ftrace_function - register a function for profiling
+ * @ops - ops structure that holds the function for profiling.
+ *
+ * Register a function to be called by all functions in the
+ * kernel.
+ *
+ * Note: @ops->func and all the functions it calls must be labeled
+ *       with "notrace", otherwise it will go into a
+ *       recursive loop.
+ */
+int register_ftrace_function(struct ftrace_ops *ops)
+{
+        int ret;
+
+        if (unlikely(ftrace_disabled))
+                return -1;
+
+        mutex_lock(&ftrace_sysctl_lock);
+        ret = __register_ftrace_function(ops);
+        ftrace_startup();
+        mutex_unlock(&ftrace_sysctl_lock);
+
+        return ret;
+}
+
+/**
+ * unregister_ftrace_function - unresgister a function for profiling.
+ * @ops - ops structure that holds the function to unregister
+ *
+ * Unregister a function that was added to be called by ftrace profiling.
+ */
+int unregister_ftrace_function(struct ftrace_ops *ops)
+{
+        int ret;
+
+        mutex_lock(&ftrace_sysctl_lock);
+        ret = __unregister_ftrace_function(ops);
+        ftrace_shutdown();
+        mutex_unlock(&ftrace_sysctl_lock);
+
+        return ret;
+}
+
+int
+ftrace_enable_sysctl(struct ctl_table *table, int write,
+                     struct file *file, void __user *buffer, size_t *lenp,
+                     loff_t *ppos)
+{
+        int ret;
+
+        if (unlikely(ftrace_disabled))
+                return -ENODEV;
+
+        mutex_lock(&ftrace_sysctl_lock);
+
+        ret  = proc_dointvec(table, write, file, buffer, lenp, ppos);
+
+        if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
+                goto out;
+
+        last_ftrace_enabled = ftrace_enabled;
+
+        if (ftrace_enabled) {
+
+                ftrace_startup_sysctl();
+
+                /* we are starting ftrace again */
+                if (ftrace_list != &ftrace_list_end) {
+                        if (ftrace_list->next == &ftrace_list_end)
+                                ftrace_trace_function = ftrace_list->func;
+                        else
+                                ftrace_trace_function = ftrace_list_func;
+                }
+
+        } else {
+                /* stopping ftrace calls (just send to ftrace_stub) */
+                ftrace_trace_function = ftrace_stub;
+
+                ftrace_shutdown_sysctl();
+        }
+
+ out:
+        mutex_unlock(&ftrace_sysctl_lock);
+        return ret;
+}
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
new file mode 100644
index 000000000000..868e121c8e38
--- /dev/null
+++ b/kernel/trace/trace.c
@@ -0,0 +1,3161 @@
+/*
+ * ring buffer based function tracer
+ *
+ * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
+ * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
+ *
+ * Originally taken from the RT patch by:
+ *    Arnaldo Carvalho de Melo <acme@redhat.com>
+ *
+ * Based on code from the latency_tracer, that is:
+ *  Copyright (C) 2004-2006 Ingo Molnar
+ *  Copyright (C) 2004 William Lee Irwin III
+ */
+#include <linux/utsrelease.h>
+#include <linux/kallsyms.h>
+#include <linux/seq_file.h>
+#include <linux/debugfs.h>
+#include <linux/pagemap.h>
+#include <linux/hardirq.h>
+#include <linux/linkage.h>
+#include <linux/uaccess.h>
+#include <linux/ftrace.h>
+#include <linux/module.h>
+#include <linux/percpu.h>
+#include <linux/ctype.h>
+#include <linux/init.h>
+#include <linux/poll.h>
+#include <linux/gfp.h>
+#include <linux/fs.h>
+#include <linux/kprobes.h>
+#include <linux/writeback.h>
+
+#include <linux/stacktrace.h>
+
+#include "trace.h"
+
+unsigned long __read_mostly     tracing_max_latency = (cycle_t)ULONG_MAX;
+unsigned long __read_mostly     tracing_thresh;
+
+static unsigned long __read_mostly      tracing_nr_buffers;
+static cpumask_t __read_mostly          tracing_buffer_mask;
+
+#define for_each_tracing_cpu(cpu)       \
+        for_each_cpu_mask(cpu, tracing_buffer_mask)
+
+static int trace_alloc_page(void);
+static int trace_free_page(void);
+
+static int tracing_disabled = 1;
+
+static unsigned long tracing_pages_allocated;
+
+long
+ns2usecs(cycle_t nsec)
+{
+        nsec += 500;
+        do_div(nsec, 1000);
+        return nsec;
+}
+
+cycle_t ftrace_now(int cpu)
+{
+        return cpu_clock(cpu);
+}
+
+/*
+ * The global_trace is the descriptor that holds the tracing
+ * buffers for the live tracing. For each CPU, it contains
+ * a link list of pages that will store trace entries. The
+ * page descriptor of the pages in the memory is used to hold
+ * the link list by linking the lru item in the page descriptor
+ * to each of the pages in the buffer per CPU.
+ *
+ * For each active CPU there is a data field that holds the
+ * pages for the buffer for that CPU. Each CPU has the same number
+ * of pages allocated for its buffer.
+ */
+static struct trace_array       global_trace;
+
+static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
+
+/*
+ * The max_tr is used to snapshot the global_trace when a maximum
+ * latency is reached. Some tracers will use this to store a maximum
+ * trace while it continues examining live traces.
+ *
+ * The buffers for the max_tr are set up the same as the global_trace.
+ * When a snapshot is taken, the link list of the max_tr is swapped
+ * with the link list of the global_trace and the buffers are reset for
+ * the global_trace so the tracing can continue.
+ */
+static struct trace_array       max_tr;
+
+static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
+
+/* tracer_enabled is used to toggle activation of a tracer */
+static int                      tracer_enabled = 1;
+
+/* function tracing enabled */
+int                             ftrace_function_enabled;
+
+/*
+ * trace_nr_entries is the number of entries that is allocated
+ * for a buffer. Note, the number of entries is always rounded
+ * to ENTRIES_PER_PAGE.
+ */
+static unsigned long            trace_nr_entries = 65536UL;
+
+/* trace_types holds a link list of available tracers. */
+static struct tracer            *trace_types __read_mostly;
+
+/* current_trace points to the tracer that is currently active */
+static struct tracer            *current_trace __read_mostly;
+
+/*
+ * max_tracer_type_len is used to simplify the allocating of
+ * buffers to read userspace tracer names. We keep track of
+ * the longest tracer name registered.
+ */
+static int                      max_tracer_type_len;
+
+/*
+ * trace_types_lock is used to protect the trace_types list.
+ * This lock is also used to keep user access serialized.
+ * Accesses from userspace will grab this lock while userspace
+ * activities happen inside the kernel.
+ */
+static DEFINE_MUTEX(trace_types_lock);
+
+/* trace_wait is a waitqueue for tasks blocked on trace_poll */
+static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
+
+/* trace_flags holds iter_ctrl options */
+unsigned long trace_flags = TRACE_ITER_PRINT_PARENT;
+
+static notrace void no_trace_init(struct trace_array *tr)
+{
+        int cpu;
+
+        ftrace_function_enabled = 0;
+        if(tr->ctrl)
+                for_each_online_cpu(cpu)
+                        tracing_reset(tr->data[cpu]);
+        tracer_enabled = 0;
+}
+
+/* dummy trace to disable tracing */
+static struct tracer no_tracer __read_mostly = {
+        .name           = "none",
+        .init           = no_trace_init
+};
+
+
+/**
+ * trace_wake_up - wake up tasks waiting for trace input
+ *
+ * Simply wakes up any task that is blocked on the trace_wait
+ * queue. These is used with trace_poll for tasks polling the trace.
+ */
+void trace_wake_up(void)
+{
+        /*
+         * The runqueue_is_locked() can fail, but this is the best we
+         * have for now:
+         */
+        if (!(trace_flags & TRACE_ITER_BLOCK) && !runqueue_is_locked())
+                wake_up(&trace_wait);
+}
+
+#define ENTRIES_PER_PAGE (PAGE_SIZE / sizeof(struct trace_entry))
+
+static int __init set_nr_entries(char *str)
+{
+        unsigned long nr_entries;
+        int ret;
+
+        if (!str)
+                return 0;
+        ret = strict_strtoul(str, 0, &nr_entries);
+        /* nr_entries can not be zero */
+        if (ret < 0 || nr_entries == 0)
+                return 0;
+        trace_nr_entries = nr_entries;
+        return 1;
+}
+__setup("trace_entries=", set_nr_entries);
+
+unsigned long nsecs_to_usecs(unsigned long nsecs)
+{
+        return nsecs / 1000;
+}
+
+/*
+ * trace_flag_type is an enumeration that holds different
+ * states when a trace occurs. These are:
+ *  IRQS_OFF    - interrupts were disabled
+ *  NEED_RESCED - reschedule is requested
+ *  HARDIRQ     - inside an interrupt handler
+ *  SOFTIRQ     - inside a softirq handler
+ */
+enum trace_flag_type {
+        TRACE_FLAG_IRQS_OFF             = 0x01,
+        TRACE_FLAG_NEED_RESCHED         = 0x02,
+        TRACE_FLAG_HARDIRQ              = 0x04,
+        TRACE_FLAG_SOFTIRQ              = 0x08,
+};
+
+/*
+ * TRACE_ITER_SYM_MASK masks the options in trace_flags that
+ * control the output of kernel symbols.
+ */
+#define TRACE_ITER_SYM_MASK \
+        (TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
+
+/* These must match the bit postions in trace_iterator_flags */
+static const char *trace_options[] = {
+        "print-parent",
+        "sym-offset",
+        "sym-addr",
+        "verbose",
+        "raw",
+        "hex",
+        "bin",
+        "block",
+        "stacktrace",
+        "sched-tree",
+        NULL
+};
+
+/*
+ * ftrace_max_lock is used to protect the swapping of buffers
+ * when taking a max snapshot. The buffers themselves are
+ * protected by per_cpu spinlocks. But the action of the swap
+ * needs its own lock.
+ *
+ * This is defined as a raw_spinlock_t in order to help
+ * with performance when lockdep debugging is enabled.
+ */
+static raw_spinlock_t ftrace_max_lock =
+        (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
+
+/*
+ * Copy the new maximum trace into the separate maximum-trace
+ * structure. (this way the maximum trace is permanently saved,
+ * for later retrieval via /debugfs/tracing/latency_trace)
+ */
+static void
+__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
+{
+        struct trace_array_cpu *data = tr->data[cpu];
+
+        max_tr.cpu = cpu;
+        max_tr.time_start = data->preempt_timestamp;
+
+        data = max_tr.data[cpu];
+        data->saved_latency = tracing_max_latency;
+
+        memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
+        data->pid = tsk->pid;
+        data->uid = tsk->uid;
+        data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
+        data->policy = tsk->policy;
+        data->rt_priority = tsk->rt_priority;
+
+        /* record this tasks comm */
+        tracing_record_cmdline(current);
+}
+
+#define CHECK_COND(cond)                        \
+        if (unlikely(cond)) {                   \
+                tracing_disabled = 1;           \
+                WARN_ON(1);                     \
+                return -1;                      \
+        }
+
+/**
+ * check_pages - integrity check of trace buffers
+ *
+ * As a safty measure we check to make sure the data pages have not
+ * been corrupted.
+ */
+int check_pages(struct trace_array_cpu *data)
+{
+        struct page *page, *tmp;
+
+        CHECK_COND(data->trace_pages.next->prev != &data->trace_pages);
+        CHECK_COND(data->trace_pages.prev->next != &data->trace_pages);
+
+        list_for_each_entry_safe(page, tmp, &data->trace_pages, lru) {
+                CHECK_COND(page->lru.next->prev != &page->lru);
+                CHECK_COND(page->lru.prev->next != &page->lru);
+        }
+
+        return 0;
+}
+
+/**
+ * head_page - page address of the first page in per_cpu buffer.
+ *
+ * head_page returns the page address of the first page in
+ * a per_cpu buffer. This also preforms various consistency
+ * checks to make sure the buffer has not been corrupted.
+ */
+void *head_page(struct trace_array_cpu *data)
+{
+        struct page *page;
+
+        if (list_empty(&data->trace_pages))
+                return NULL;
+
+        page = list_entry(data->trace_pages.next, struct page, lru);
+        BUG_ON(&page->lru == &data->trace_pages);
+
+        return page_address(page);
+}
+
+/**
+ * trace_seq_printf - sequence printing of trace information
+ * @s: trace sequence descriptor
+ * @fmt: printf format string
+ *
+ * The tracer may use either sequence operations or its own
+ * copy to user routines. To simplify formating of a trace
+ * trace_seq_printf is used to store strings into a special
+ * buffer (@s). Then the output may be either used by
+ * the sequencer or pulled into another buffer.
+ */
+int
+trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
+{
+        int len = (PAGE_SIZE - 1) - s->len;
+        va_list ap;
+        int ret;
+
+        if (!len)
+                return 0;
+
+        va_start(ap, fmt);
+        ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
+        va_end(ap);
+
+        /* If we can't write it all, don't bother writing anything */
+        if (ret >= len)
+                return 0;
+
+        s->len += ret;
+
+        return len;
+}
+
+/**
+ * trace_seq_puts - trace sequence printing of simple string
+ * @s: trace sequence descriptor
+ * @str: simple string to record
+ *
+ * The tracer may use either the sequence operations or its own
+ * copy to user routines. This function records a simple string
+ * into a special buffer (@s) for later retrieval by a sequencer
+ * or other mechanism.
+ */
+static int
+trace_seq_puts(struct trace_seq *s, const char *str)
+{
+        int len = strlen(str);
+
+        if (len > ((PAGE_SIZE - 1) - s->len))
+                return 0;
+
+        memcpy(s->buffer + s->len, str, len);
+        s->len += len;
+
+        return len;
+}
+
+static int
+trace_seq_putc(struct trace_seq *s, unsigned char c)
+{
+        if (s->len >= (PAGE_SIZE - 1))
+                return 0;
+
+        s->buffer[s->len++] = c;
+
+        return 1;
+}
+
+static int
+trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
+{
+        if (len > ((PAGE_SIZE - 1) - s->len))
+                return 0;
+
+        memcpy(s->buffer + s->len, mem, len);
+        s->len += len;
+
+        return len;
+}
+
+#define HEX_CHARS 17
+static const char hex2asc[] = "0123456789abcdef";
+
+static int
+trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
+{
+        unsigned char hex[HEX_CHARS];
+        unsigned char *data = mem;
+        unsigned char byte;
+        int i, j;
+
+        BUG_ON(len >= HEX_CHARS);
+
+#ifdef __BIG_ENDIAN
+        for (i = 0, j = 0; i < len; i++) {
+#else
+        for (i = len-1, j = 0; i >= 0; i--) {
+#endif
+                byte = data[i];
+
+                hex[j++] = hex2asc[byte & 0x0f];
+                hex[j++] = hex2asc[byte >> 4];
+        }
+        hex[j++] = ' ';
+
+        return trace_seq_putmem(s, hex, j);
+}
+
+static void
+trace_seq_reset(struct trace_seq *s)
+{
+        s->len = 0;
+        s->readpos = 0;
+}
+
+ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
+{
+        int len;
+        int ret;
+
+        if (s->len <= s->readpos)
+                return -EBUSY;
+
+        len = s->len - s->readpos;
+        if (cnt > len)
+                cnt = len;
+        ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
+        if (ret)
+                return -EFAULT;
+
+        s->readpos += len;
+        return cnt;
+}
+
+static void
+trace_print_seq(struct seq_file *m, struct trace_seq *s)
+{
+        int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len;
+
+        s->buffer[len] = 0;
+        seq_puts(m, s->buffer);
+
+        trace_seq_reset(s);
+}
+
+/*
+ * flip the trace buffers between two trace descriptors.
+ * This usually is the buffers between the global_trace and
+ * the max_tr to record a snapshot of a current trace.
+ *
+ * The ftrace_max_lock must be held.
+ */
+static void
+flip_trace(struct trace_array_cpu *tr1, struct trace_array_cpu *tr2)
+{
+        struct list_head flip_pages;
+
+        INIT_LIST_HEAD(&flip_pages);
+
+        memcpy(&tr1->trace_head_idx, &tr2->trace_head_idx,
+                sizeof(struct trace_array_cpu) -
+                offsetof(struct trace_array_cpu, trace_head_idx));
+
+        check_pages(tr1);
+        check_pages(tr2);
+        list_splice_init(&tr1->trace_pages, &flip_pages);
+        list_splice_init(&tr2->trace_pages, &tr1->trace_pages);
+        list_splice_init(&flip_pages, &tr2->trace_pages);
+        BUG_ON(!list_empty(&flip_pages));
+        check_pages(tr1);
+        check_pages(tr2);
+}
+
+/**
+ * update_max_tr - snapshot all trace buffers from global_trace to max_tr
+ * @tr: tracer
+ * @tsk: the task with the latency
+ * @cpu: The cpu that initiated the trace.
+ *
+ * Flip the buffers between the @tr and the max_tr and record information
+ * about which task was the cause of this latency.
+ */
+void
+update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
+{
+        struct trace_array_cpu *data;
+        int i;
+
+        WARN_ON_ONCE(!irqs_disabled());
+        __raw_spin_lock(&ftrace_max_lock);
+        /* clear out all the previous traces */
+        for_each_tracing_cpu(i) {
+                data = tr->data[i];
+                flip_trace(max_tr.data[i], data);
+                tracing_reset(data);
+        }
+
+        __update_max_tr(tr, tsk, cpu);
+        __raw_spin_unlock(&ftrace_max_lock);
+}
+
+/**
+ * update_max_tr_single - only copy one trace over, and reset the rest
+ * @tr - tracer
+ * @tsk - task with the latency
+ * @cpu - the cpu of the buffer to copy.
+ *
+ * Flip the trace of a single CPU buffer between the @tr and the max_tr.
+ */
+void
+update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
+{
+        struct trace_array_cpu *data = tr->data[cpu];
+        int i;
+
+        WARN_ON_ONCE(!irqs_disabled());
+        __raw_spin_lock(&ftrace_max_lock);
+        for_each_tracing_cpu(i)
+                tracing_reset(max_tr.data[i]);
+
+        flip_trace(max_tr.data[cpu], data);
+        tracing_reset(data);
+
+        __update_max_tr(tr, tsk, cpu);
+        __raw_spin_unlock(&ftrace_max_lock);
+}
+
+/**
+ * register_tracer - register a tracer with the ftrace system.
+ * @type - the plugin for the tracer
+ *
+ * Register a new plugin tracer.
+ */
+int register_tracer(struct tracer *type)
+{
+        struct tracer *t;
+        int len;
+        int ret = 0;
+
+        if (!type->name) {
+                pr_info("Tracer must have a name\n");
+                return -1;
+        }
+
+        mutex_lock(&trace_types_lock);
+        for (t = trace_types; t; t = t->next) {
+                if (strcmp(type->name, t->name) == 0) {
+                        /* already found */
+                        pr_info("Trace %s already registered\n",
+                                type->name);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+#ifdef CONFIG_FTRACE_STARTUP_TEST
+        if (type->selftest) {
+                struct tracer *saved_tracer = current_trace;
+                struct trace_array_cpu *data;
+                struct trace_array *tr = &global_trace;
+                int saved_ctrl = tr->ctrl;
+                int i;
+                /*
+                 * Run a selftest on this tracer.
+                 * Here we reset the trace buffer, and set the current
+                 * tracer to be this tracer. The tracer can then run some
+                 * internal tracing to verify that everything is in order.
+                 * If we fail, we do not register this tracer.
+                 */
+                for_each_tracing_cpu(i) {
+                        data = tr->data[i];
+                        if (!head_page(data))
+                                continue;
+                        tracing_reset(data);
+                }
+                current_trace = type;
+                tr->ctrl = 0;
+                /* the test is responsible for initializing and enabling */
+                pr_info("Testing tracer %s: ", type->name);
+                ret = type->selftest(type, tr);
+                /* the test is responsible for resetting too */
+                current_trace = saved_tracer;
+                tr->ctrl = saved_ctrl;
+                if (ret) {
+                        printk(KERN_CONT "FAILED!\n");
+                        goto out;
+                }
+                /* Only reset on passing, to avoid touching corrupted buffers */
+                for_each_tracing_cpu(i) {
+                        data = tr->data[i];
+                        if (!head_page(data))
+                                continue;
+                        tracing_reset(data);
+                }
+                printk(KERN_CONT "PASSED\n");
+        }
+#endif
+
+        type->next = trace_types;
+        trace_types = type;
+        len = strlen(type->name);
+        if (len > max_tracer_type_len)
+                max_tracer_type_len = len;
+
+ out:
+        mutex_unlock(&trace_types_lock);
+
+        return ret;
+}
+
+void unregister_tracer(struct tracer *type)
+{
+        struct tracer **t;
+        int len;
+
+        mutex_lock(&trace_types_lock);
+        for (t = &trace_types; *t; t = &(*t)->next) {
+                if (*t == type)
+                        goto found;
+        }
+        pr_info("Trace %s not registered\n", type->name);
+        goto out;
+
+ found:
+        *t = (*t)->next;
+        if (strlen(type->name) != max_tracer_type_len)
+                goto out;
+
+        max_tracer_type_len = 0;
+        for (t = &trace_types; *t; t = &(*t)->next) {
+                len = strlen((*t)->name);
+                if (len > max_tracer_type_len)
+                        max_tracer_type_len = len;
+        }
+ out:
+        mutex_unlock(&trace_types_lock);
+}
+
+void tracing_reset(struct trace_array_cpu *data)
+{
+        data->trace_idx = 0;
+        data->overrun = 0;
+        data->trace_head = data->trace_tail = head_page(data);
+        data->trace_head_idx = 0;
+        data->trace_tail_idx = 0;
+}
+
+#define SAVED_CMDLINES 128
+static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
+static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
+static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
+static int cmdline_idx;
+static DEFINE_SPINLOCK(trace_cmdline_lock);
+
+/* temporary disable recording */
+atomic_t trace_record_cmdline_disabled __read_mostly;
+
+static void trace_init_cmdlines(void)
+{
+        memset(&map_pid_to_cmdline, -1, sizeof(map_pid_to_cmdline));
+        memset(&map_cmdline_to_pid, -1, sizeof(map_cmdline_to_pid));
+        cmdline_idx = 0;
+}
+
+void trace_stop_cmdline_recording(void);
+
+static void trace_save_cmdline(struct task_struct *tsk)
+{
+        unsigned map;
+        unsigned idx;
+
+        if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
+                return;
+
+        /*
+         * It's not the end of the world if we don't get
+         * the lock, but we also don't want to spin
+         * nor do we want to disable interrupts,
+         * so if we miss here, then better luck next time.
+         */
+        if (!spin_trylock(&trace_cmdline_lock))
+                return;
+
+        idx = map_pid_to_cmdline[tsk->pid];
+        if (idx >= SAVED_CMDLINES) {
+                idx = (cmdline_idx + 1) % SAVED_CMDLINES;
+
+                map = map_cmdline_to_pid[idx];
+                if (map <= PID_MAX_DEFAULT)
+                        map_pid_to_cmdline[map] = (unsigned)-1;
+
+                map_pid_to_cmdline[tsk->pid] = idx;
+
+                cmdline_idx = idx;
+        }
+
+        memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
+
+        spin_unlock(&trace_cmdline_lock);
+}
+
+static char *trace_find_cmdline(int pid)
+{
+        char *cmdline = "<...>";
+        unsigned map;
+
+        if (!pid)
+                return "<idle>";
+
+        if (pid > PID_MAX_DEFAULT)
+                goto out;
+
+        map = map_pid_to_cmdline[pid];
+        if (map >= SAVED_CMDLINES)
+                goto out;
+
+        cmdline = saved_cmdlines[map];
+
+ out:
+        return cmdline;
+}
+
+void tracing_record_cmdline(struct task_struct *tsk)
+{
+        if (atomic_read(&trace_record_cmdline_disabled))
+                return;
+
+        trace_save_cmdline(tsk);
+}
+
+static inline struct list_head *
+trace_next_list(struct trace_array_cpu *data, struct list_head *next)
+{
+        /*
+         * Roundrobin - but skip the head (which is not a real page):
+         */
+        next = next->next;
+        if (unlikely(next == &data->trace_pages))
+                next = next->next;
+        BUG_ON(next == &data->trace_pages);
+
+        return next;
+}
+
+static inline void *
+trace_next_page(struct trace_array_cpu *data, void *addr)
+{
+        struct list_head *next;
+        struct page *page;
+
+        page = virt_to_page(addr);
+
+        next = trace_next_list(data, &page->lru);
+        page = list_entry(next, struct page, lru);
+
+        return page_address(page);
+}
+
+static inline struct trace_entry *
+tracing_get_trace_entry(struct trace_array *tr, struct trace_array_cpu *data)
+{
+        unsigned long idx, idx_next;
+        struct trace_entry *entry;
+
+        data->trace_idx++;
+        idx = data->trace_head_idx;
+        idx_next = idx + 1;
+
+        BUG_ON(idx * TRACE_ENTRY_SIZE >= PAGE_SIZE);
+
+        entry = data->trace_head + idx * TRACE_ENTRY_SIZE;
+
+        if (unlikely(idx_next >= ENTRIES_PER_PAGE)) {
+                data->trace_head = trace_next_page(data, data->trace_head);
+                idx_next = 0;
+        }
+
+        if (data->trace_head == data->trace_tail &&
+            idx_next == data->trace_tail_idx) {
+                /* overrun */
+                data->overrun++;
+                data->trace_tail_idx++;
+                if (data->trace_tail_idx >= ENTRIES_PER_PAGE) {
+                        data->trace_tail =
+                                trace_next_page(data, data->trace_tail);
+                        data->trace_tail_idx = 0;
+                }
+        }
+
+        data->trace_head_idx = idx_next;
+
+        return entry;
+}
+
+static inline void
+tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags)
+{
+        struct task_struct *tsk = current;
+        unsigned long pc;
+
+        pc = preempt_count();
+
+        entry->preempt_count    = pc & 0xff;
+        entry->pid              = (tsk) ? tsk->pid : 0;
+        entry->t                = ftrace_now(raw_smp_processor_id());
+        entry->flags = (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
+                ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
+                ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
+                (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
+}
+
+void
+trace_function(struct trace_array *tr, struct trace_array_cpu *data,
+               unsigned long ip, unsigned long parent_ip, unsigned long flags)
+{
+        struct trace_entry *entry;
+        unsigned long irq_flags;
+
+        raw_local_irq_save(irq_flags);
+        __raw_spin_lock(&data->lock);
+        entry                   = tracing_get_trace_entry(tr, data);
+        tracing_generic_entry_update(entry, flags);
+        entry->type             = TRACE_FN;
+        entry->fn.ip            = ip;
+        entry->fn.parent_ip     = parent_ip;
+        __raw_spin_unlock(&data->lock);
+        raw_local_irq_restore(irq_flags);
+}
+
+void
+ftrace(struct trace_array *tr, struct trace_array_cpu *data,
+       unsigned long ip, unsigned long parent_ip, unsigned long flags)
+{
+        if (likely(!atomic_read(&data->disabled)))
+                trace_function(tr, data, ip, parent_ip, flags);
+}
+
+#ifdef CONFIG_MMIOTRACE
+void __trace_mmiotrace_rw(struct trace_array *tr, struct trace_array_cpu *data,
+                                                struct mmiotrace_rw *rw)
+{
+        struct trace_entry *entry;
+        unsigned long irq_flags;
+
+        raw_local_irq_save(irq_flags);
+        __raw_spin_lock(&data->lock);
+
+        entry                   = tracing_get_trace_entry(tr, data);
+        tracing_generic_entry_update(entry, 0);
+        entry->type             = TRACE_MMIO_RW;
+        entry->mmiorw           = *rw;
+
+        __raw_spin_unlock(&data->lock);
+        raw_local_irq_restore(irq_flags);
+
+        trace_wake_up();
+}
+
+void __trace_mmiotrace_map(struct trace_array *tr, struct trace_array_cpu *data,
+                                                struct mmiotrace_map *map)
+{
+        struct trace_entry *entry;
+        unsigned long irq_flags;
+
+        raw_local_irq_save(irq_flags);
+        __raw_spin_lock(&data->lock);
+
+        entry                   = tracing_get_trace_entry(tr, data);
+        tracing_generic_entry_update(entry, 0);
+        entry->type             = TRACE_MMIO_MAP;
+        entry->mmiomap          = *map;
+
+        __raw_spin_unlock(&data->lock);
+        raw_local_irq_restore(irq_flags);
+
+        trace_wake_up();
+}
+#endif
+
+void __trace_stack(struct trace_array *tr,
+                   struct trace_array_cpu *data,
+                   unsigned long flags,
+                   int skip)
+{
+        struct trace_entry *entry;
+        struct stack_trace trace;
+
+        if (!(trace_flags & TRACE_ITER_STACKTRACE))
+                return;
+
+        entry                   = tracing_get_trace_entry(tr, data);
+        tracing_generic_entry_update(entry, flags);
+        entry->type             = TRACE_STACK;
+
+        memset(&entry->stack, 0, sizeof(entry->stack));
+
+        trace.nr_entries        = 0;
+        trace.max_entries       = FTRACE_STACK_ENTRIES;
+        trace.skip              = skip;
+        trace.entries           = entry->stack.caller;
+
+        save_stack_trace(&trace);
+}
+
+void
+__trace_special(void *__tr, void *__data,
+                unsigned long arg1, unsigned long arg2, unsigned long arg3)
+{
+        struct trace_array_cpu *data = __data;
+        struct trace_array *tr = __tr;
+        struct trace_entry *entry;
+        unsigned long irq_flags;
+
+        raw_local_irq_save(irq_flags);
+        __raw_spin_lock(&data->lock);
+        entry                   = tracing_get_trace_entry(tr, data);
+        tracing_generic_entry_update(entry, 0);
+        entry->type             = TRACE_SPECIAL;
+        entry->special.arg1     = arg1;
+        entry->special.arg2     = arg2;
+        entry->special.arg3     = arg3;
+        __trace_stack(tr, data, irq_flags, 4);
+        __raw_spin_unlock(&data->lock);
+        raw_local_irq_restore(irq_flags);
+
+        trace_wake_up();
+}
+
+void
+tracing_sched_switch_trace(struct trace_array *tr,
+                           struct trace_array_cpu *data,
+                           struct task_struct *prev,
+                           struct task_struct *next,
+                           unsigned long flags)
+{
+        struct trace_entry *entry;
+        unsigned long irq_flags;
+
+        raw_local_irq_save(irq_flags);
+        __raw_spin_lock(&data->lock);
+        entry                   = tracing_get_trace_entry(tr, data);
+        tracing_generic_entry_update(entry, flags);
+        entry->type             = TRACE_CTX;
+        entry->ctx.prev_pid     = prev->pid;
+        entry->ctx.prev_prio    = prev->prio;
+        entry->ctx.prev_state   = prev->state;
+        entry->ctx.next_pid     = next->pid;
+        entry->ctx.next_prio    = next->prio;
+        entry->ctx.next_state   = next->state;
+        __trace_stack(tr, data, flags, 5);
+        __raw_spin_unlock(&data->lock);
+        raw_local_irq_restore(irq_flags);
+}
+
+void
+tracing_sched_wakeup_trace(struct trace_array *tr,
+                           struct trace_array_cpu *data,
+                           struct task_struct *wakee,
+                           struct task_struct *curr,
+                           unsigned long flags)
+{
+        struct trace_entry *entry;
+        unsigned long irq_flags;
+
+        raw_local_irq_save(irq_flags);
+        __raw_spin_lock(&data->lock);
+        entry                   = tracing_get_trace_entry(tr, data);
+        tracing_generic_entry_update(entry, flags);
+        entry->type             = TRACE_WAKE;
+        entry->ctx.prev_pid     = curr->pid;
+        entry->ctx.prev_prio    = curr->prio;
+        entry->ctx.prev_state   = curr->state;
+        entry->ctx.next_pid     = wakee->pid;
+        entry->ctx.next_prio    = wakee->prio;
+        entry->ctx.next_state   = wakee->state;
+        __trace_stack(tr, data, flags, 6);
+        __raw_spin_unlock(&data->lock);
+        raw_local_irq_restore(irq_flags);
+
+        trace_wake_up();
+}
+
+void
+ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
+{
+        struct trace_array *tr = &global_trace;
+        struct trace_array_cpu *data;
+        unsigned long flags;
+        long disabled;
+        int cpu;
+
+        if (tracing_disabled || current_trace == &no_tracer || !tr->ctrl)
+                return;
+
+        local_irq_save(flags);
+        cpu = raw_smp_processor_id();
+        data = tr->data[cpu];
+        disabled = atomic_inc_return(&data->disabled);
+
+        if (likely(disabled == 1))
+                __trace_special(tr, data, arg1, arg2, arg3);
+
+        atomic_dec(&data->disabled);
+        local_irq_restore(flags);
+}
+
+#ifdef CONFIG_FTRACE
+static void
+function_trace_call(unsigned long ip, unsigned long parent_ip)
+{
+        struct trace_array *tr = &global_trace;
+        struct trace_array_cpu *data;
+        unsigned long flags;
+        long disabled;
+        int cpu;
+
+        if (unlikely(!ftrace_function_enabled))
+                return;
+
+        if (skip_trace(ip))
+                return;
+
+        local_irq_save(flags);
+        cpu = raw_smp_processor_id();
+        data = tr->data[cpu];
+        disabled = atomic_inc_return(&data->disabled);
+
+        if (likely(disabled == 1))
+                trace_function(tr, data, ip, parent_ip, flags);
+
+        atomic_dec(&data->disabled);
+        local_irq_restore(flags);
+}
+
+static struct ftrace_ops trace_ops __read_mostly =
+{
+        .func = function_trace_call,
+};
+
+void tracing_start_function_trace(void)
+{
+        ftrace_function_enabled = 0;
+        register_ftrace_function(&trace_ops);
+        if (tracer_enabled)
+                ftrace_function_enabled = 1;
+}
+
+void tracing_stop_function_trace(void)
+{
+        ftrace_function_enabled = 0;
+        unregister_ftrace_function(&trace_ops);
+}
+#endif
+
+enum trace_file_type {
+        TRACE_FILE_LAT_FMT      = 1,
+};
+
+static struct trace_entry *
+trace_entry_idx(struct trace_array *tr, struct trace_array_cpu *data,
+                struct trace_iterator *iter, int cpu)
+{
+        struct page *page;
+        struct trace_entry *array;
+
+        if (iter->next_idx[cpu] >= tr->entries ||
+            iter->next_idx[cpu] >= data->trace_idx ||
+            (data->trace_head == data->trace_tail &&
+             data->trace_head_idx == data->trace_tail_idx))
+                return NULL;
+
+        if (!iter->next_page[cpu]) {
+                /* Initialize the iterator for this cpu trace buffer */
+                WARN_ON(!data->trace_tail);
+                page = virt_to_page(data->trace_tail);
+                iter->next_page[cpu] = &page->lru;
+                iter->next_page_idx[cpu] = data->trace_tail_idx;
+        }
+
+        page = list_entry(iter->next_page[cpu], struct page, lru);
+        BUG_ON(&data->trace_pages == &page->lru);
+
+        array = page_address(page);
+
+        WARN_ON(iter->next_page_idx[cpu] >= ENTRIES_PER_PAGE);
+        return &array[iter->next_page_idx[cpu]];
+}
+
+static struct trace_entry *
+find_next_entry(struct trace_iterator *iter, int *ent_cpu)
+{
+        struct trace_array *tr = iter->tr;
+        struct trace_entry *ent, *next = NULL;
+        int next_cpu = -1;
+        int cpu;
+
+        for_each_tracing_cpu(cpu) {
+                if (!head_page(tr->data[cpu]))
+                        continue;
+                ent = trace_entry_idx(tr, tr->data[cpu], iter, cpu);
+                /*
+                 * Pick the entry with the smallest timestamp:
+                 */
+                if (ent && (!next || ent->t < next->t)) {
+                        next = ent;
+                        next_cpu = cpu;
+                }
+        }
+
+        if (ent_cpu)
+                *ent_cpu = next_cpu;
+
+        return next;
+}
+
+static void trace_iterator_increment(struct trace_iterator *iter)
+{
+        iter->idx++;
+        iter->next_idx[iter->cpu]++;
+        iter->next_page_idx[iter->cpu]++;
+
+        if (iter->next_page_idx[iter->cpu] >= ENTRIES_PER_PAGE) {
+                struct trace_array_cpu *data = iter->tr->data[iter->cpu];
+
+                iter->next_page_idx[iter->cpu] = 0;
+                iter->next_page[iter->cpu] =
+                        trace_next_list(data, iter->next_page[iter->cpu]);
+        }
+}
+
+static void trace_consume(struct trace_iterator *iter)
+{
+        struct trace_array_cpu *data = iter->tr->data[iter->cpu];
+
+        data->trace_tail_idx++;
+        if (data->trace_tail_idx >= ENTRIES_PER_PAGE) {
+                data->trace_tail = trace_next_page(data, data->trace_tail);
+                data->trace_tail_idx = 0;
+        }
+
+        /* Check if we empty it, then reset the index */
+        if (data->trace_head == data->trace_tail &&
+            data->trace_head_idx == data->trace_tail_idx)
+                data->trace_idx = 0;
+}
+
+static void *find_next_entry_inc(struct trace_iterator *iter)
+{
+        struct trace_entry *next;
+        int next_cpu = -1;
+
+        next = find_next_entry(iter, &next_cpu);
+
+        iter->prev_ent = iter->ent;
+        iter->prev_cpu = iter->cpu;
+
+        iter->ent = next;
+        iter->cpu = next_cpu;
+
+        if (next)
+                trace_iterator_increment(iter);
+
+        return next ? iter : NULL;
+}
+
+static void *s_next(struct seq_file *m, void *v, loff_t *pos)
+{
+        struct trace_iterator *iter = m->private;
+        void *last_ent = iter->ent;
+        int i = (int)*pos;
+        void *ent;
+
+        (*pos)++;
+
+        /* can't go backwards */
+        if (iter->idx > i)
+                return NULL;
+
+        if (iter->idx < 0)
+                ent = find_next_entry_inc(iter);
+        else
+                ent = iter;
+
+        while (ent && iter->idx < i)
+                ent = find_next_entry_inc(iter);
+
+        iter->pos = *pos;
+
+        if (last_ent && !ent)
+                seq_puts(m, "\n\nvim:ft=help\n");
+
+        return ent;
+}
+
+static void *s_start(struct seq_file *m, loff_t *pos)
+{
+        struct trace_iterator *iter = m->private;
+        void *p = NULL;
+        loff_t l = 0;
+        int i;
+
+        mutex_lock(&trace_types_lock);
+
+        if (!current_trace || current_trace != iter->trace) {
+                mutex_unlock(&trace_types_lock);
+                return NULL;
+        }
+
+        atomic_inc(&trace_record_cmdline_disabled);
+
+        /* let the tracer grab locks here if needed */
+        if (current_trace->start)
+                current_trace->start(iter);
+
+        if (*pos != iter->pos) {
+                iter->ent = NULL;
+                iter->cpu = 0;
+                iter->idx = -1;
+                iter->prev_ent = NULL;
+                iter->prev_cpu = -1;
+
+                for_each_tracing_cpu(i) {
+                        iter->next_idx[i] = 0;
+                        iter->next_page[i] = NULL;
+                }
+
+                for (p = iter; p && l < *pos; p = s_next(m, p, &l))
+                        ;
+
+        } else {
+                l = *pos - 1;
+                p = s_next(m, p, &l);
+        }
+
+        return p;
+}
+
+static void s_stop(struct seq_file *m, void *p)
+{
+        struct trace_iterator *iter = m->private;
+
+        atomic_dec(&trace_record_cmdline_disabled);
+
+        /* let the tracer release locks here if needed */
+        if (current_trace && current_trace == iter->trace && iter->trace->stop)
+                iter->trace->stop(iter);
+
+        mutex_unlock(&trace_types_lock);
+}
+
+#define KRETPROBE_MSG "[unknown/kretprobe'd]"
+
+#ifdef CONFIG_KRETPROBES
+static inline int kretprobed(unsigned long addr)
+{
+        return addr == (unsigned long)kretprobe_trampoline;
+}
+#else
+static inline int kretprobed(unsigned long addr)
+{
+        return 0;
+}
+#endif /* CONFIG_KRETPROBES */
+
+static int
+seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
+{
+#ifdef CONFIG_KALLSYMS
+        char str[KSYM_SYMBOL_LEN];
+
+        kallsyms_lookup(address, NULL, NULL, NULL, str);
+
+        return trace_seq_printf(s, fmt, str);
+#endif
+        return 1;
+}
+
+static int
+seq_print_sym_offset(struct trace_seq *s, const char *fmt,
+                     unsigned long address)
+{
+#ifdef CONFIG_KALLSYMS
+        char str[KSYM_SYMBOL_LEN];
+
+        sprint_symbol(str, address);
+        return trace_seq_printf(s, fmt, str);
+#endif
+        return 1;
+}
+
+#ifndef CONFIG_64BIT
+# define IP_FMT "%08lx"
+#else
+# define IP_FMT "%016lx"
+#endif
+
+static int
+seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
+{
+        int ret;
+
+        if (!ip)
+                return trace_seq_printf(s, "0");
+
+        if (sym_flags & TRACE_ITER_SYM_OFFSET)
+                ret = seq_print_sym_offset(s, "%s", ip);
+        else
+                ret = seq_print_sym_short(s, "%s", ip);
+
+        if (!ret)
+                return 0;
+
+        if (sym_flags & TRACE_ITER_SYM_ADDR)
+                ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
+        return ret;
+}
+
+static void print_lat_help_header(struct seq_file *m)
+{
+        seq_puts(m, "#                _------=> CPU#            \n");
+        seq_puts(m, "#               / _-----=> irqs-off        \n");
+        seq_puts(m, "#              | / _----=> need-resched    \n");
+        seq_puts(m, "#              || / _---=> hardirq/softirq \n");
+        seq_puts(m, "#              ||| / _--=> preempt-depth   \n");
+        seq_puts(m, "#              |||| /                      \n");
+        seq_puts(m, "#              |||||     delay             \n");
+        seq_puts(m, "#  cmd     pid ||||| time  |   caller      \n");
+        seq_puts(m, "#     \\   /    |||||   \\   |   /           \n");
+}
+
+static void print_func_help_header(struct seq_file *m)
+{
+        seq_puts(m, "#           TASK-PID   CPU#    TIMESTAMP  FUNCTION\n");
+        seq_puts(m, "#              | |      |          |         |\n");
+}
+
+
+static void
+print_trace_header(struct seq_file *m, struct trace_iterator *iter)
+{
+        unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
+        struct trace_array *tr = iter->tr;
+        struct trace_array_cpu *data = tr->data[tr->cpu];
+        struct tracer *type = current_trace;
+        unsigned long total   = 0;
+        unsigned long entries = 0;
+        int cpu;
+        const char *name = "preemption";
+
+        if (type)
+                name = type->name;
+
+        for_each_tracing_cpu(cpu) {
+                if (head_page(tr->data[cpu])) {
+                        total += tr->data[cpu]->trace_idx;
+                        if (tr->data[cpu]->trace_idx > tr->entries)
+                                entries += tr->entries;
+                        else
+                                entries += tr->data[cpu]->trace_idx;
+                }
+        }
+
+        seq_printf(m, "%s latency trace v1.1.5 on %s\n",
+                   name, UTS_RELEASE);
+        seq_puts(m, "-----------------------------------"
+                 "---------------------------------\n");
+        seq_printf(m, " latency: %lu us, #%lu/%lu, CPU#%d |"
+                   " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
+                   nsecs_to_usecs(data->saved_latency),
+                   entries,
+                   total,
+                   tr->cpu,
+#if defined(CONFIG_PREEMPT_NONE)
+                   "server",
+#elif defined(CONFIG_PREEMPT_VOLUNTARY)
+                   "desktop",
+#elif defined(CONFIG_PREEMPT)
+                   "preempt",
+#else
+                   "unknown",
+#endif
+                   /* These are reserved for later use */
+                   0, 0, 0, 0);
+#ifdef CONFIG_SMP
+        seq_printf(m, " #P:%d)\n", num_online_cpus());
+#else
+        seq_puts(m, ")\n");
+#endif
+        seq_puts(m, "    -----------------\n");
+        seq_printf(m, "    | task: %.16s-%d "
+                   "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
+                   data->comm, data->pid, data->uid, data->nice,
+                   data->policy, data->rt_priority);
+        seq_puts(m, "    -----------------\n");
+
+        if (data->critical_start) {
+                seq_puts(m, " => started at: ");
+                seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
+                trace_print_seq(m, &iter->seq);
+                seq_puts(m, "\n => ended at:   ");
+                seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
+                trace_print_seq(m, &iter->seq);
+                seq_puts(m, "\n");
+        }
+
+        seq_puts(m, "\n");
+}
+
+static void
+lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
+{
+        int hardirq, softirq;
+        char *comm;
+
+        comm = trace_find_cmdline(entry->pid);
+
+        trace_seq_printf(s, "%8.8s-%-5d ", comm, entry->pid);
+        trace_seq_printf(s, "%d", cpu);
+        trace_seq_printf(s, "%c%c",
+                        (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' : '.',
+                        ((entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'));
+
+        hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
+        softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
+        if (hardirq && softirq) {
+                trace_seq_putc(s, 'H');
+        } else {
+                if (hardirq) {
+                        trace_seq_putc(s, 'h');
+                } else {
+                        if (softirq)
+                                trace_seq_putc(s, 's');
+                        else
+                                trace_seq_putc(s, '.');
+                }
+        }
+
+        if (entry->preempt_count)
+                trace_seq_printf(s, "%x", entry->preempt_count);
+        else
+                trace_seq_puts(s, ".");
+}
+
+unsigned long preempt_mark_thresh = 100;
+
+static void
+lat_print_timestamp(struct trace_seq *s, unsigned long long abs_usecs,
+                    unsigned long rel_usecs)
+{
+        trace_seq_printf(s, " %4lldus", abs_usecs);
+        if (rel_usecs > preempt_mark_thresh)
+                trace_seq_puts(s, "!: ");
+        else if (rel_usecs > 1)
+                trace_seq_puts(s, "+: ");
+        else
+                trace_seq_puts(s, " : ");
+}
+
+static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
+
+static int
+print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
+{
+        struct trace_seq *s = &iter->seq;
+        unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
+        struct trace_entry *next_entry = find_next_entry(iter, NULL);
+        unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
+        struct trace_entry *entry = iter->ent;
+        unsigned long abs_usecs;
+        unsigned long rel_usecs;
+        char *comm;
+        int S, T;
+        int i;
+        unsigned state;
+
+        if (!next_entry)
+                next_entry = entry;
+        rel_usecs = ns2usecs(next_entry->t - entry->t);
+        abs_usecs = ns2usecs(entry->t - iter->tr->time_start);
+
+        if (verbose) {
+                comm = trace_find_cmdline(entry->pid);
+                trace_seq_printf(s, "%16s %5d %d %d %08x %08x [%08lx]"
+                                 " %ld.%03ldms (+%ld.%03ldms): ",
+                                 comm,
+                                 entry->pid, cpu, entry->flags,
+                                 entry->preempt_count, trace_idx,
+                                 ns2usecs(entry->t),
+                                 abs_usecs/1000,
+                                 abs_usecs % 1000, rel_usecs/1000,
+                                 rel_usecs % 1000);
+        } else {
+                lat_print_generic(s, entry, cpu);
+                lat_print_timestamp(s, abs_usecs, rel_usecs);
+        }
+        switch (entry->type) {
+        case TRACE_FN:
+                seq_print_ip_sym(s, entry->fn.ip, sym_flags);
+                trace_seq_puts(s, " (");
+                if (kretprobed(entry->fn.parent_ip))
+                        trace_seq_puts(s, KRETPROBE_MSG);
+                else
+                        seq_print_ip_sym(s, entry->fn.parent_ip, sym_flags);
+                trace_seq_puts(s, ")\n");
+                break;
+        case TRACE_CTX:
+        case TRACE_WAKE:
+                T = entry->ctx.next_state < sizeof(state_to_char) ?
+                        state_to_char[entry->ctx.next_state] : 'X';
+
+                state = entry->ctx.prev_state ? __ffs(entry->ctx.prev_state) + 1 : 0;
+                S = state < sizeof(state_to_char) - 1 ? state_to_char[state] : 'X';
+                comm = trace_find_cmdline(entry->ctx.next_pid);
+                trace_seq_printf(s, " %5d:%3d:%c %s %5d:%3d:%c %s\n",
+                                 entry->ctx.prev_pid,
+                                 entry->ctx.prev_prio,
+                                 S, entry->type == TRACE_CTX ? "==>" : "  +",
+                                 entry->ctx.next_pid,
+                                 entry->ctx.next_prio,
+                                 T, comm);
+                break;
+        case TRACE_SPECIAL:
+                trace_seq_printf(s, "# %ld %ld %ld\n",
+                                 entry->special.arg1,
+                                 entry->special.arg2,
+                                 entry->special.arg3);
+                break;
+        case TRACE_STACK:
+                for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
+                        if (i)
+                                trace_seq_puts(s, " <= ");
+                        seq_print_ip_sym(s, entry->stack.caller[i], sym_flags);
+                }
+                trace_seq_puts(s, "\n");
+                break;
+        default:
+                trace_seq_printf(s, "Unknown type %d\n", entry->type);
+        }
+        return 1;
+}
+
+static int print_trace_fmt(struct trace_iterator *iter)
+{
+        struct trace_seq *s = &iter->seq;
+        unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
+        struct trace_entry *entry;
+        unsigned long usec_rem;
+        unsigned long long t;
+        unsigned long secs;
+        char *comm;
+        int ret;
+        int S, T;
+        int i;
+
+        entry = iter->ent;
+
+        comm = trace_find_cmdline(iter->ent->pid);
+
+        t = ns2usecs(entry->t);
+        usec_rem = do_div(t, 1000000ULL);
+        secs = (unsigned long)t;
+
+        ret = trace_seq_printf(s, "%16s-%-5d ", comm, entry->pid);
+        if (!ret)
+                return 0;
+        ret = trace_seq_printf(s, "[%02d] ", iter->cpu);
+        if (!ret)
+                return 0;
+        ret = trace_seq_printf(s, "%5lu.%06lu: ", secs, usec_rem);
+        if (!ret)
+                return 0;
+
+        switch (entry->type) {
+        case TRACE_FN:
+                ret = seq_print_ip_sym(s, entry->fn.ip, sym_flags);
+                if (!ret)
+                        return 0;
+                if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
+                                                entry->fn.parent_ip) {
+                        ret = trace_seq_printf(s, " <-");
+                        if (!ret)
+                                return 0;
+                        if (kretprobed(entry->fn.parent_ip))
+                                ret = trace_seq_puts(s, KRETPROBE_MSG);
+                        else
+                                ret = seq_print_ip_sym(s, entry->fn.parent_ip,
+                                                       sym_flags);
+                        if (!ret)
+                                return 0;
+                }
+                ret = trace_seq_printf(s, "\n");
+                if (!ret)
+                        return 0;
+                break;
+        case TRACE_CTX:
+        case TRACE_WAKE:
+                S = entry->ctx.prev_state < sizeof(state_to_char) ?
+                        state_to_char[entry->ctx.prev_state] : 'X';
+                T = entry->ctx.next_state < sizeof(state_to_char) ?
+                        state_to_char[entry->ctx.next_state] : 'X';
+                ret = trace_seq_printf(s, " %5d:%3d:%c %s %5d:%3d:%c\n",
+                                       entry->ctx.prev_pid,
+                                       entry->ctx.prev_prio,
+                                       S,
+                                       entry->type == TRACE_CTX ? "==>" : "  +",
+                                       entry->ctx.next_pid,
+                                       entry->ctx.next_prio,
+                                       T);
+                if (!ret)
+                        return 0;
+                break;
+        case TRACE_SPECIAL:
+                ret = trace_seq_printf(s, "# %ld %ld %ld\n",
+                                 entry->special.arg1,
+                                 entry->special.arg2,
+                                 entry->special.arg3);
+                if (!ret)
+                        return 0;
+                break;
+        case TRACE_STACK:
+                for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
+                        if (i) {
+                                ret = trace_seq_puts(s, " <= ");
+                                if (!ret)
+                                        return 0;
+                        }
+                        ret = seq_print_ip_sym(s, entry->stack.caller[i],
+                                               sym_flags);
+                        if (!ret)
+                                return 0;
+                }
+                ret = trace_seq_puts(s, "\n");
+                if (!ret)
+                        return 0;
+                break;
+        }
+        return 1;
+}
+
+static int print_raw_fmt(struct trace_iterator *iter)
+{
+        struct trace_seq *s = &iter->seq;
+        struct trace_entry *entry;
+        int ret;
+        int S, T;
+
+        entry = iter->ent;
+
+        ret = trace_seq_printf(s, "%d %d %llu ",
+                entry->pid, iter->cpu, entry->t);
+        if (!ret)
+                return 0;
+
+        switch (entry->type) {
+        case TRACE_FN:
+                ret = trace_seq_printf(s, "%x %x\n",
+                                        entry->fn.ip, entry->fn.parent_ip);
+                if (!ret)
+                        return 0;
+                break;
+        case TRACE_CTX:
+        case TRACE_WAKE:
+                S = entry->ctx.prev_state < sizeof(state_to_char) ?
+                        state_to_char[entry->ctx.prev_state] : 'X';
+                T = entry->ctx.next_state < sizeof(state_to_char) ?
+                        state_to_char[entry->ctx.next_state] : 'X';
+                if (entry->type == TRACE_WAKE)
+                        S = '+';
+                ret = trace_seq_printf(s, "%d %d %c %d %d %c\n",
+                                       entry->ctx.prev_pid,
+                                       entry->ctx.prev_prio,
+                                       S,
+                                       entry->ctx.next_pid,
+                                       entry->ctx.next_prio,
+                                       T);
+                if (!ret)
+                        return 0;
+                break;
+        case TRACE_SPECIAL:
+        case TRACE_STACK:
+                ret = trace_seq_printf(s, "# %ld %ld %ld\n",
+                                 entry->special.arg1,
+                                 entry->special.arg2,
+                                 entry->special.arg3);
+                if (!ret)
+                        return 0;
+                break;
+        }
+        return 1;
+}
+
+#define SEQ_PUT_FIELD_RET(s, x)                         \
+do {                                                    \
+        if (!trace_seq_putmem(s, &(x), sizeof(x)))      \
+                return 0;                               \
+} while (0)
+
+#define SEQ_PUT_HEX_FIELD_RET(s, x)                     \
+do {                                                    \
+        if (!trace_seq_putmem_hex(s, &(x), sizeof(x)))  \
+                return 0;                               \
+} while (0)
+
+static int print_hex_fmt(struct trace_iterator *iter)
+{
+        struct trace_seq *s = &iter->seq;
+        unsigned char newline = '\n';
+        struct trace_entry *entry;
+        int S, T;
+
+        entry = iter->ent;
+
+        SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
+        SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
+        SEQ_PUT_HEX_FIELD_RET(s, entry->t);
+
+        switch (entry->type) {
+        case TRACE_FN:
+                SEQ_PUT_HEX_FIELD_RET(s, entry->fn.ip);
+                SEQ_PUT_HEX_FIELD_RET(s, entry->fn.parent_ip);
+                break;
+        case TRACE_CTX:
+        case TRACE_WAKE:
+                S = entry->ctx.prev_state < sizeof(state_to_char) ?
+                        state_to_char[entry->ctx.prev_state] : 'X';
+                T = entry->ctx.next_state < sizeof(state_to_char) ?
+                        state_to_char[entry->ctx.next_state] : 'X';
+                if (entry->type == TRACE_WAKE)
+                        S = '+';
+                SEQ_PUT_HEX_FIELD_RET(s, entry->ctx.prev_pid);
+                SEQ_PUT_HEX_FIELD_RET(s, entry->ctx.prev_prio);
+                SEQ_PUT_HEX_FIELD_RET(s, S);
+                SEQ_PUT_HEX_FIELD_RET(s, entry->ctx.next_pid);
+                SEQ_PUT_HEX_FIELD_RET(s, entry->ctx.next_prio);
+                SEQ_PUT_HEX_FIELD_RET(s, entry->fn.parent_ip);
+                SEQ_PUT_HEX_FIELD_RET(s, T);
+                break;
+        case TRACE_SPECIAL:
+        case TRACE_STACK:
+                SEQ_PUT_HEX_FIELD_RET(s, entry->special.arg1);
+                SEQ_PUT_HEX_FIELD_RET(s, entry->special.arg2);
+                SEQ_PUT_HEX_FIELD_RET(s, entry->special.arg3);
+                break;
+        }
+        SEQ_PUT_FIELD_RET(s, newline);
+
+        return 1;
+}
+
+static int print_bin_fmt(struct trace_iterator *iter)
+{
+        struct trace_seq *s = &iter->seq;
+        struct trace_entry *entry;
+
+        entry = iter->ent;
+
+        SEQ_PUT_FIELD_RET(s, entry->pid);
+        SEQ_PUT_FIELD_RET(s, entry->cpu);
+        SEQ_PUT_FIELD_RET(s, entry->t);
+
+        switch (entry->type) {
+        case TRACE_FN:
+                SEQ_PUT_FIELD_RET(s, entry->fn.ip);
+                SEQ_PUT_FIELD_RET(s, entry->fn.parent_ip);
+                break;
+        case TRACE_CTX:
+                SEQ_PUT_FIELD_RET(s, entry->ctx.prev_pid);
+                SEQ_PUT_FIELD_RET(s, entry->ctx.prev_prio);
+                SEQ_PUT_FIELD_RET(s, entry->ctx.prev_state);
+                SEQ_PUT_FIELD_RET(s, entry->ctx.next_pid);
+                SEQ_PUT_FIELD_RET(s, entry->ctx.next_prio);
+                SEQ_PUT_FIELD_RET(s, entry->ctx.next_state);
+                break;
+        case TRACE_SPECIAL:
+        case TRACE_STACK:
+                SEQ_PUT_FIELD_RET(s, entry->special.arg1);
+                SEQ_PUT_FIELD_RET(s, entry->special.arg2);
+                SEQ_PUT_FIELD_RET(s, entry->special.arg3);
+                break;
+        }
+        return 1;
+}
+
+static int trace_empty(struct trace_iterator *iter)
+{
+        struct trace_array_cpu *data;
+        int cpu;
+
+        for_each_tracing_cpu(cpu) {
+                data = iter->tr->data[cpu];
+
+                if (head_page(data) && data->trace_idx &&
+                    (data->trace_tail != data->trace_head ||
+                     data->trace_tail_idx != data->trace_head_idx))
+                        return 0;
+        }
+        return 1;
+}
+
+static int print_trace_line(struct trace_iterator *iter)
+{
+        if (iter->trace && iter->trace->print_line)
+                return iter->trace->print_line(iter);
+
+        if (trace_flags & TRACE_ITER_BIN)
+                return print_bin_fmt(iter);
+
+        if (trace_flags & TRACE_ITER_HEX)
+                return print_hex_fmt(iter);
+
+        if (trace_flags & TRACE_ITER_RAW)
+                return print_raw_fmt(iter);
+
+        if (iter->iter_flags & TRACE_FILE_LAT_FMT)
+                return print_lat_fmt(iter, iter->idx, iter->cpu);
+
+        return print_trace_fmt(iter);
+}
+
+static int s_show(struct seq_file *m, void *v)
+{
+        struct trace_iterator *iter = v;
+
+        if (iter->ent == NULL) {
+                if (iter->tr) {
+                        seq_printf(m, "# tracer: %s\n", iter->trace->name);
+                        seq_puts(m, "#\n");
+                }
+                if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
+                        /* print nothing if the buffers are empty */
+                        if (trace_empty(iter))
+                                return 0;
+                        print_trace_header(m, iter);
+                        if (!(trace_flags & TRACE_ITER_VERBOSE))
+                                print_lat_help_header(m);
+                } else {
+                        if (!(trace_flags & TRACE_ITER_VERBOSE))
+                                print_func_help_header(m);
+                }
+        } else {
+                print_trace_line(iter);
+                trace_print_seq(m, &iter->seq);
+        }
+
+        return 0;
+}
+
+static struct seq_operations tracer_seq_ops = {
+        .start          = s_start,
+        .next           = s_next,
+        .stop           = s_stop,
+        .show           = s_show,
+};
+
+static struct trace_iterator *
+__tracing_open(struct inode *inode, struct file *file, int *ret)
+{
+        struct trace_iterator *iter;
+
+        if (tracing_disabled) {
+                *ret = -ENODEV;
+                return NULL;
+        }
+
+        iter = kzalloc(sizeof(*iter), GFP_KERNEL);
+        if (!iter) {
+                *ret = -ENOMEM;
+                goto out;
+        }
+
+        mutex_lock(&trace_types_lock);
+        if (current_trace && current_trace->print_max)
+                iter->tr = &max_tr;
+        else
+                iter->tr = inode->i_private;
+        iter->trace = current_trace;
+        iter->pos = -1;
+
+        /* TODO stop tracer */
+        *ret = seq_open(file, &tracer_seq_ops);
+        if (!*ret) {
+                struct seq_file *m = file->private_data;
+                m->private = iter;
+
+                /* stop the trace while dumping */
+                if (iter->tr->ctrl) {
+                        tracer_enabled = 0;
+                        ftrace_function_enabled = 0;
+                }
+
+                if (iter->trace && iter->trace->open)
+                        iter->trace->open(iter);
+        } else {
+                kfree(iter);
+                iter = NULL;
+        }
+        mutex_unlock(&trace_types_lock);
+
+ out:
+        return iter;
+}
+
+int tracing_open_generic(struct inode *inode, struct file *filp)
+{
+        if (tracing_disabled)
+                return -ENODEV;
+
+        filp->private_data = inode->i_private;
+        return 0;
+}
+
+int tracing_release(struct inode *inode, struct file *file)
+{
+        struct seq_file *m = (struct seq_file *)file->private_data;
+        struct trace_iterator *iter = m->private;
+
+        mutex_lock(&trace_types_lock);
+        if (iter->trace && iter->trace->close)
+                iter->trace->close(iter);
+
+        /* reenable tracing if it was previously enabled */
+        if (iter->tr->ctrl) {
+                tracer_enabled = 1;
+                /*
+                 * It is safe to enable function tracing even if it
+                 * isn't used
+                 */
+                ftrace_function_enabled = 1;
+        }
+        mutex_unlock(&trace_types_lock);
+
+        seq_release(inode, file);
+        kfree(iter);
+        return 0;
+}
+
+static int tracing_open(struct inode *inode, struct file *file)
+{
+        int ret;
+
+        __tracing_open(inode, file, &ret);
+
+        return ret;
+}
+
+static int tracing_lt_open(struct inode *inode, struct file *file)
+{
+        struct trace_iterator *iter;
+        int ret;
+
+        iter = __tracing_open(inode, file, &ret);
+
+        if (!ret)
+                iter->iter_flags |= TRACE_FILE_LAT_FMT;
+
+        return ret;
+}
+
+
+static void *
+t_next(struct seq_file *m, void *v, loff_t *pos)
+{
+        struct tracer *t = m->private;
+
+        (*pos)++;
+
+        if (t)
+                t = t->next;
+
+        m->private = t;
+
+        return t;
+}
+
+static void *t_start(struct seq_file *m, loff_t *pos)
+{
+        struct tracer *t = m->private;
+        loff_t l = 0;
+
+        mutex_lock(&trace_types_lock);
+        for (; t && l < *pos; t = t_next(m, t, &l))
+                ;
+
+        return t;
+}
+
+static void t_stop(struct seq_file *m, void *p)
+{
+        mutex_unlock(&trace_types_lock);
+}
+
+static int t_show(struct seq_file *m, void *v)
+{
+        struct tracer *t = v;
+
+        if (!t)
+                return 0;
+
+        seq_printf(m, "%s", t->name);
+        if (t->next)
+                seq_putc(m, ' ');
+        else
+                seq_putc(m, '\n');
+
+        return 0;
+}
+
+static struct seq_operations show_traces_seq_ops = {
+        .start          = t_start,
+        .next           = t_next,
+        .stop           = t_stop,
+        .show           = t_show,
+};
+
+static int show_traces_open(struct inode *inode, struct file *file)
+{
+        int ret;
+
+        if (tracing_disabled)
+                return -ENODEV;
+
+        ret = seq_open(file, &show_traces_seq_ops);
+        if (!ret) {
+                struct seq_file *m = file->private_data;
+                m->private = trace_types;
+        }
+
+        return ret;
+}
+
+static struct file_operations tracing_fops = {
+        .open           = tracing_open,
+        .read           = seq_read,
+        .llseek         = seq_lseek,
+        .release        = tracing_release,
+};
+
+static struct file_operations tracing_lt_fops = {
+        .open           = tracing_lt_open,
+        .read           = seq_read,
+        .llseek         = seq_lseek,
+        .release        = tracing_release,
+};
+
+static struct file_operations show_traces_fops = {
+        .open           = show_traces_open,
+        .read           = seq_read,
+        .release        = seq_release,
+};
+
+/*
+ * Only trace on a CPU if the bitmask is set:
+ */
+static cpumask_t tracing_cpumask = CPU_MASK_ALL;
+
+/*
+ * When tracing/tracing_cpu_mask is modified then this holds
+ * the new bitmask we are about to install:
+ */
+static cpumask_t tracing_cpumask_new;
+
+/*
+ * The tracer itself will not take this lock, but still we want
+ * to provide a consistent cpumask to user-space:
+ */
+static DEFINE_MUTEX(tracing_cpumask_update_lock);
+
+/*
+ * Temporary storage for the character representation of the
+ * CPU bitmask (and one more byte for the newline):
+ */
+static char mask_str[NR_CPUS + 1];
+
+static ssize_t
+tracing_cpumask_read(struct file *filp, char __user *ubuf,
+                     size_t count, loff_t *ppos)
+{
+        int len;
+
+        mutex_lock(&tracing_cpumask_update_lock);
+
+        len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
+        if (count - len < 2) {
+                count = -EINVAL;
+                goto out_err;
+        }
+        len += sprintf(mask_str + len, "\n");
+        count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
+
+out_err:
+        mutex_unlock(&tracing_cpumask_update_lock);
+
+        return count;
+}
+
+static ssize_t
+tracing_cpumask_write(struct file *filp, const char __user *ubuf,
+                      size_t count, loff_t *ppos)
+{
+        int err, cpu;
+
+        mutex_lock(&tracing_cpumask_update_lock);
+        err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
+        if (err)
+                goto err_unlock;
+
+        raw_local_irq_disable();
+        __raw_spin_lock(&ftrace_max_lock);
+        for_each_tracing_cpu(cpu) {
+                /*
+                 * Increase/decrease the disabled counter if we are
+                 * about to flip a bit in the cpumask:
+                 */
+                if (cpu_isset(cpu, tracing_cpumask) &&
+                                !cpu_isset(cpu, tracing_cpumask_new)) {
+                        atomic_inc(&global_trace.data[cpu]->disabled);
+                }
+                if (!cpu_isset(cpu, tracing_cpumask) &&
+                                cpu_isset(cpu, tracing_cpumask_new)) {
+                        atomic_dec(&global_trace.data[cpu]->disabled);
+                }
+        }
+        __raw_spin_unlock(&ftrace_max_lock);
+        raw_local_irq_enable();
+
+        tracing_cpumask = tracing_cpumask_new;
+
+        mutex_unlock(&tracing_cpumask_update_lock);
+
+        return count;
+
+err_unlock:
+        mutex_unlock(&tracing_cpumask_update_lock);
+
+        return err;
+}
+
+static struct file_operations tracing_cpumask_fops = {
+        .open           = tracing_open_generic,
+        .read           = tracing_cpumask_read,
+        .write          = tracing_cpumask_write,
+};
+
+static ssize_t
+tracing_iter_ctrl_read(struct file *filp, char __user *ubuf,
+                       size_t cnt, loff_t *ppos)
+{
+        char *buf;
+        int r = 0;
+        int len = 0;
+        int i;
+
+        /* calulate max size */
+        for (i = 0; trace_options[i]; i++) {
+                len += strlen(trace_options[i]);
+                len += 3; /* "no" and space */
+        }
+
+        /* +2 for \n and \0 */
+        buf = kmalloc(len + 2, GFP_KERNEL);
+        if (!buf)
+                return -ENOMEM;
+
+        for (i = 0; trace_options[i]; i++) {
+                if (trace_flags & (1 << i))
+                        r += sprintf(buf + r, "%s ", trace_options[i]);
+                else
+                        r += sprintf(buf + r, "no%s ", trace_options[i]);
+        }
+
+        r += sprintf(buf + r, "\n");
+        WARN_ON(r >= len + 2);
+
+        r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+
+        kfree(buf);
+
+        return r;
+}
+
+static ssize_t
+tracing_iter_ctrl_write(struct file *filp, const char __user *ubuf,
+                        size_t cnt, loff_t *ppos)
+{
+        char buf[64];
+        char *cmp = buf;
+        int neg = 0;
+        int i;
+
+        if (cnt >= sizeof(buf))
+                return -EINVAL;
+
+        if (copy_from_user(&buf, ubuf, cnt))
+                return -EFAULT;
+
+        buf[cnt] = 0;
+
+        if (strncmp(buf, "no", 2) == 0) {
+                neg = 1;
+                cmp += 2;
+        }
+
+        for (i = 0; trace_options[i]; i++) {
+                int len = strlen(trace_options[i]);
+
+                if (strncmp(cmp, trace_options[i], len) == 0) {
+                        if (neg)
+                                trace_flags &= ~(1 << i);
+                        else
+                                trace_flags |= (1 << i);
+                        break;
+                }
+        }
+        /*
+         * If no option could be set, return an error:
+         */
+        if (!trace_options[i])
+                return -EINVAL;
+
+        filp->f_pos += cnt;
+
+        return cnt;
+}
+
+static struct file_operations tracing_iter_fops = {
+        .open           = tracing_open_generic,
+        .read           = tracing_iter_ctrl_read,
+        .write          = tracing_iter_ctrl_write,
+};
+
+static const char readme_msg[] =
+        "tracing mini-HOWTO:\n\n"
+        "# mkdir /debug\n"
+        "# mount -t debugfs nodev /debug\n\n"
+        "# cat /debug/tracing/available_tracers\n"
+        "wakeup preemptirqsoff preemptoff irqsoff ftrace sched_switch none\n\n"
+        "# cat /debug/tracing/current_tracer\n"
+        "none\n"
+        "# echo sched_switch > /debug/tracing/current_tracer\n"
+        "# cat /debug/tracing/current_tracer\n"
+        "sched_switch\n"
+        "# cat /debug/tracing/iter_ctrl\n"
+        "noprint-parent nosym-offset nosym-addr noverbose\n"
+        "# echo print-parent > /debug/tracing/iter_ctrl\n"
+        "# echo 1 > /debug/tracing/tracing_enabled\n"
+        "# cat /debug/tracing/trace > /tmp/trace.txt\n"
+        "echo 0 > /debug/tracing/tracing_enabled\n"
+;
+
+static ssize_t
+tracing_readme_read(struct file *filp, char __user *ubuf,
+                       size_t cnt, loff_t *ppos)
+{
+        return simple_read_from_buffer(ubuf, cnt, ppos,
+                                        readme_msg, strlen(readme_msg));
+}
+
+static struct file_operations tracing_readme_fops = {
+        .open           = tracing_open_generic,
+        .read           = tracing_readme_read,
+};
+
+static ssize_t
+tracing_ctrl_read(struct file *filp, char __user *ubuf,
+                  size_t cnt, loff_t *ppos)
+{
+        struct trace_array *tr = filp->private_data;
+        char buf[64];
+        int r;
+
+        r = sprintf(buf, "%ld\n", tr->ctrl);
+        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+static ssize_t
+tracing_ctrl_write(struct file *filp, const char __user *ubuf,
+                   size_t cnt, loff_t *ppos)
+{
+        struct trace_array *tr = filp->private_data;
+        char buf[64];
+        long val;
+        int ret;
+
+        if (cnt >= sizeof(buf))
+                return -EINVAL;
+
+        if (copy_from_user(&buf, ubuf, cnt))
+                return -EFAULT;
+
+        buf[cnt] = 0;
+
+        ret = strict_strtoul(buf, 10, &val);
+        if (ret < 0)
+                return ret;
+
+        val = !!val;
+
+        mutex_lock(&trace_types_lock);
+        if (tr->ctrl ^ val) {
+                if (val)
+                        tracer_enabled = 1;
+                else
+                        tracer_enabled = 0;
+
+                tr->ctrl = val;
+
+                if (current_trace && current_trace->ctrl_update)
+                        current_trace->ctrl_update(tr);
+        }
+        mutex_unlock(&trace_types_lock);
+
+        filp->f_pos += cnt;
+
+        return cnt;
+}
+
+static ssize_t
+tracing_set_trace_read(struct file *filp, char __user *ubuf,
+                       size_t cnt, loff_t *ppos)
+{
+        char buf[max_tracer_type_len+2];
+        int r;
+
+        mutex_lock(&trace_types_lock);
+        if (current_trace)
+                r = sprintf(buf, "%s\n", current_trace->name);
+        else
+                r = sprintf(buf, "\n");
+        mutex_unlock(&trace_types_lock);
+
+        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+static ssize_t
+tracing_set_trace_write(struct file *filp, const char __user *ubuf,
+                        size_t cnt, loff_t *ppos)
+{
+        struct trace_array *tr = &global_trace;
+        struct tracer *t;
+        char buf[max_tracer_type_len+1];
+        int i;
+
+        if (cnt > max_tracer_type_len)
+                cnt = max_tracer_type_len;
+
+        if (copy_from_user(&buf, ubuf, cnt))
+                return -EFAULT;
+
+        buf[cnt] = 0;
+
+        /* strip ending whitespace. */
+        for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
+                buf[i] = 0;
+
+        mutex_lock(&trace_types_lock);
+        for (t = trace_types; t; t = t->next) {
+                if (strcmp(t->name, buf) == 0)
+                        break;
+        }
+        if (!t || t == current_trace)
+                goto out;
+
+        if (current_trace && current_trace->reset)
+                current_trace->reset(tr);
+
+        current_trace = t;
+        if (t->init)
+                t->init(tr);
+
+ out:
+        mutex_unlock(&trace_types_lock);
+
+        filp->f_pos += cnt;
+
+        return cnt;
+}
+
+static ssize_t
+tracing_max_lat_read(struct file *filp, char __user *ubuf,
+                     size_t cnt, loff_t *ppos)
+{
+        unsigned long *ptr = filp->private_data;
+        char buf[64];
+        int r;
+
+        r = snprintf(buf, sizeof(buf), "%ld\n",
+                     *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
+        if (r > sizeof(buf))
+                r = sizeof(buf);
+        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+static ssize_t
+tracing_max_lat_write(struct file *filp, const char __user *ubuf,
+                      size_t cnt, loff_t *ppos)
+{
+        long *ptr = filp->private_data;
+        char buf[64];
+        long val;
+        int ret;
+
+        if (cnt >= sizeof(buf))
+                return -EINVAL;
+
+        if (copy_from_user(&buf, ubuf, cnt))
+                return -EFAULT;
+
+        buf[cnt] = 0;
+
+        ret = strict_strtoul(buf, 10, &val);
+        if (ret < 0)
+                return ret;
+
+        *ptr = val * 1000;
+
+        return cnt;
+}
+
+static atomic_t tracing_reader;
+
+static int tracing_open_pipe(struct inode *inode, struct file *filp)
+{
+        struct trace_iterator *iter;
+
+        if (tracing_disabled)
+                return -ENODEV;
+
+        /* We only allow for reader of the pipe */
+        if (atomic_inc_return(&tracing_reader) != 1) {
+                atomic_dec(&tracing_reader);
+                return -EBUSY;
+        }
+
+        /* create a buffer to store the information to pass to userspace */
+        iter = kzalloc(sizeof(*iter), GFP_KERNEL);
+        if (!iter)
+                return -ENOMEM;
+
+        mutex_lock(&trace_types_lock);
+        iter->tr = &global_trace;
+        iter->trace = current_trace;
+        filp->private_data = iter;
+
+        if (iter->trace->pipe_open)
+                iter->trace->pipe_open(iter);
+        mutex_unlock(&trace_types_lock);
+
+        return 0;
+}
+
+static int tracing_release_pipe(struct inode *inode, struct file *file)
+{
+        struct trace_iterator *iter = file->private_data;
+
+        kfree(iter);
+        atomic_dec(&tracing_reader);
+
+        return 0;
+}
+
+static unsigned int
+tracing_poll_pipe(struct file *filp, poll_table *poll_table)
+{
+        struct trace_iterator *iter = filp->private_data;
+
+        if (trace_flags & TRACE_ITER_BLOCK) {
+                /*
+                 * Always select as readable when in blocking mode
+                 */
+                return POLLIN | POLLRDNORM;
+        } else {
+                if (!trace_empty(iter))
+                        return POLLIN | POLLRDNORM;
+                poll_wait(filp, &trace_wait, poll_table);
+                if (!trace_empty(iter))
+                        return POLLIN | POLLRDNORM;
+
+                return 0;
+        }
+}
+
+/*
+ * Consumer reader.
+ */
+static ssize_t
+tracing_read_pipe(struct file *filp, char __user *ubuf,
+                  size_t cnt, loff_t *ppos)
+{
+        struct trace_iterator *iter = filp->private_data;
+        struct trace_array_cpu *data;
+        static cpumask_t mask;
+        unsigned long flags;
+#ifdef CONFIG_FTRACE
+        int ftrace_save;
+#endif
+        int cpu;
+        ssize_t sret;
+
+        /* return any leftover data */
+        sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
+        if (sret != -EBUSY)
+                return sret;
+        sret = 0;
+
+        trace_seq_reset(&iter->seq);
+
+        mutex_lock(&trace_types_lock);
+        if (iter->trace->read) {
+                sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
+                if (sret)
+                        goto out;
+        }
+
+        while (trace_empty(iter)) {
+
+                if ((filp->f_flags & O_NONBLOCK)) {
+                        sret = -EAGAIN;
+                        goto out;
+                }
+
+                /*
+                 * This is a make-shift waitqueue. The reason we don't use
+                 * an actual wait queue is because:
+                 *  1) we only ever have one waiter
+                 *  2) the tracing, traces all functions, we don't want
+                 *     the overhead of calling wake_up and friends
+                 *     (and tracing them too)
+                 *     Anyway, this is really very primitive wakeup.
+                 */
+                set_current_state(TASK_INTERRUPTIBLE);
+                iter->tr->waiter = current;
+
+                mutex_unlock(&trace_types_lock);
+
+                /* sleep for 100 msecs, and try again. */
+                schedule_timeout(HZ/10);
+
+                mutex_lock(&trace_types_lock);
+
+                iter->tr->waiter = NULL;
+
+                if (signal_pending(current)) {
+                        sret = -EINTR;
+                        goto out;
+                }
+
+                if (iter->trace != current_trace)
+                        goto out;
+
+                /*
+                 * We block until we read something and tracing is disabled.
+                 * We still block if tracing is disabled, but we have never
+                 * read anything. This allows a user to cat this file, and
+                 * then enable tracing. But after we have read something,
+                 * we give an EOF when tracing is again disabled.
+                 *
+                 * iter->pos will be 0 if we haven't read anything.
+                 */
+                if (!tracer_enabled && iter->pos)
+                        break;
+
+                continue;
+        }
+
+        /* stop when tracing is finished */
+        if (trace_empty(iter))
+                goto out;
+
+        if (cnt >= PAGE_SIZE)
+                cnt = PAGE_SIZE - 1;
+
+        /* reset all but tr, trace, and overruns */
+        memset(&iter->seq, 0,
+               sizeof(struct trace_iterator) -
+               offsetof(struct trace_iterator, seq));
+        iter->pos = -1;
+
+        /*
+         * We need to stop all tracing on all CPUS to read the
+         * the next buffer. This is a bit expensive, but is
+         * not done often. We fill all what we can read,
+         * and then release the locks again.
+         */
+
+        cpus_clear(mask);
+        local_irq_save(flags);
+#ifdef CONFIG_FTRACE
+        ftrace_save = ftrace_enabled;
+        ftrace_enabled = 0;
+#endif
+        smp_wmb();
+        for_each_tracing_cpu(cpu) {
+                data = iter->tr->data[cpu];
+
+                if (!head_page(data) || !data->trace_idx)
+                        continue;
+
+                atomic_inc(&data->disabled);
+                cpu_set(cpu, mask);
+        }
+
+        for_each_cpu_mask(cpu, mask) {
+                data = iter->tr->data[cpu];
+                __raw_spin_lock(&data->lock);
+
+                if (data->overrun > iter->last_overrun[cpu])
+                        iter->overrun[cpu] +=
+                                data->overrun - iter->last_overrun[cpu];
+                iter->last_overrun[cpu] = data->overrun;
+        }
+
+        while (find_next_entry_inc(iter) != NULL) {
+                int ret;
+                int len = iter->seq.len;
+
+                ret = print_trace_line(iter);
+                if (!ret) {
+                        /* don't print partial lines */
+                        iter->seq.len = len;
+                        break;
+                }
+
+                trace_consume(iter);
+
+                if (iter->seq.len >= cnt)
+                        break;
+        }
+
+        for_each_cpu_mask(cpu, mask) {
+                data = iter->tr->data[cpu];
+                __raw_spin_unlock(&data->lock);
+        }
+
+        for_each_cpu_mask(cpu, mask) {
+                data = iter->tr->data[cpu];
+                atomic_dec(&data->disabled);
+        }
+#ifdef CONFIG_FTRACE
+        ftrace_enabled = ftrace_save;
+#endif
+        local_irq_restore(flags);
+
+        /* Now copy what we have to the user */
+        sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
+        if (iter->seq.readpos >= iter->seq.len)
+                trace_seq_reset(&iter->seq);
+        if (sret == -EBUSY)
+                sret = 0;
+
+out:
+        mutex_unlock(&trace_types_lock);
+
+        return sret;
+}
+
+static ssize_t
+tracing_entries_read(struct file *filp, char __user *ubuf,
+                     size_t cnt, loff_t *ppos)
+{
+        struct trace_array *tr = filp->private_data;
+        char buf[64];
+        int r;
+
+        r = sprintf(buf, "%lu\n", tr->entries);
+        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+static ssize_t
+tracing_entries_write(struct file *filp, const char __user *ubuf,
+                      size_t cnt, loff_t *ppos)
+{
+        unsigned long val;
+        char buf[64];
+        int i, ret;
+
+        if (cnt >= sizeof(buf))
+                return -EINVAL;
+
+        if (copy_from_user(&buf, ubuf, cnt))
+                return -EFAULT;
+
+        buf[cnt] = 0;
+
+        ret = strict_strtoul(buf, 10, &val);
+        if (ret < 0)
+                return ret;
+
+        /* must have at least 1 entry */
+        if (!val)
+                return -EINVAL;
+
+        mutex_lock(&trace_types_lock);
+
+        if (current_trace != &no_tracer) {
+                cnt = -EBUSY;
+                pr_info("ftrace: set current_tracer to none"
+                        " before modifying buffer size\n");
+                goto out;
+        }
+
+        if (val > global_trace.entries) {
+                long pages_requested;
+                unsigned long freeable_pages;
+
+                /* make sure we have enough memory before mapping */
+                pages_requested =
+                        (val + (ENTRIES_PER_PAGE-1)) / ENTRIES_PER_PAGE;
+
+                /* account for each buffer (and max_tr) */
+                pages_requested *= tracing_nr_buffers * 2;
+
+                /* Check for overflow */
+                if (pages_requested < 0) {
+                        cnt = -ENOMEM;
+                        goto out;
+                }
+
+                freeable_pages = determine_dirtyable_memory();
+
+                /* we only allow to request 1/4 of useable memory */
+                if (pages_requested >
+                    ((freeable_pages + tracing_pages_allocated) / 4)) {
+                        cnt = -ENOMEM;
+                        goto out;
+                }
+
+                while (global_trace.entries < val) {
+                        if (trace_alloc_page()) {
+                                cnt = -ENOMEM;
+                                goto out;
+                        }
+                        /* double check that we don't go over the known pages */
+                        if (tracing_pages_allocated > pages_requested)
+                                break;
+                }
+
+        } else {
+                /* include the number of entries in val (inc of page entries) */
+                while (global_trace.entries > val + (ENTRIES_PER_PAGE - 1))
+                        trace_free_page();
+        }
+
+        /* check integrity */
+        for_each_tracing_cpu(i)
+                check_pages(global_trace.data[i]);
+
+        filp->f_pos += cnt;
+
+        /* If check pages failed, return ENOMEM */
+        if (tracing_disabled)
+                cnt = -ENOMEM;
+ out:
+        max_tr.entries = global_trace.entries;
+        mutex_unlock(&trace_types_lock);
+
+        return cnt;
+}
+
+static struct file_operations tracing_max_lat_fops = {
+        .open           = tracing_open_generic,
+        .read           = tracing_max_lat_read,
+        .write          = tracing_max_lat_write,
+};
+
+static struct file_operations tracing_ctrl_fops = {
+        .open           = tracing_open_generic,
+        .read           = tracing_ctrl_read,
+        .write          = tracing_ctrl_write,
+};
+
+static struct file_operations set_tracer_fops = {
+        .open           = tracing_open_generic,
+        .read           = tracing_set_trace_read,
+        .write          = tracing_set_trace_write,
+};
+
+static struct file_operations tracing_pipe_fops = {
+        .open           = tracing_open_pipe,
+        .poll           = tracing_poll_pipe,
+        .read           = tracing_read_pipe,
+        .release        = tracing_release_pipe,
+};
+
+static struct file_operations tracing_entries_fops = {
+        .open           = tracing_open_generic,
+        .read           = tracing_entries_read,
+        .write          = tracing_entries_write,
+};
+
+#ifdef CONFIG_DYNAMIC_FTRACE
+
+static ssize_t
+tracing_read_long(struct file *filp, char __user *ubuf,
+                  size_t cnt, loff_t *ppos)
+{
+        unsigned long *p = filp->private_data;
+        char buf[64];
+        int r;
+
+        r = sprintf(buf, "%ld\n", *p);
+
+        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+static struct file_operations tracing_read_long_fops = {
+        .open           = tracing_open_generic,
+        .read           = tracing_read_long,
+};
+#endif
+
+static struct dentry *d_tracer;
+
+struct dentry *tracing_init_dentry(void)
+{
+        static int once;
+
+        if (d_tracer)
+                return d_tracer;
+
+        d_tracer = debugfs_create_dir("tracing", NULL);
+
+        if (!d_tracer && !once) {
+                once = 1;
+                pr_warning("Could not create debugfs directory 'tracing'\n");
+                return NULL;
+        }
+
+        return d_tracer;
+}
+
+#ifdef CONFIG_FTRACE_SELFTEST
+/* Let selftest have access to static functions in this file */
+#include "trace_selftest.c"
+#endif
+
+static __init void tracer_init_debugfs(void)
+{
+        struct dentry *d_tracer;
+        struct dentry *entry;
+
+        d_tracer = tracing_init_dentry();
+
+        entry = debugfs_create_file("tracing_enabled", 0644, d_tracer,
+                                    &global_trace, &tracing_ctrl_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs 'tracing_enabled' entry\n");
+
+        entry = debugfs_create_file("iter_ctrl", 0644, d_tracer,
+                                    NULL, &tracing_iter_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs 'iter_ctrl' entry\n");
+
+        entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
+                                    NULL, &tracing_cpumask_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs 'tracing_cpumask' entry\n");
+
+        entry = debugfs_create_file("latency_trace", 0444, d_tracer,
+                                    &global_trace, &tracing_lt_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs 'latency_trace' entry\n");
+
+        entry = debugfs_create_file("trace", 0444, d_tracer,
+                                    &global_trace, &tracing_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs 'trace' entry\n");
+
+        entry = debugfs_create_file("available_tracers", 0444, d_tracer,
+                                    &global_trace, &show_traces_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs 'trace' entry\n");
+
+        entry = debugfs_create_file("current_tracer", 0444, d_tracer,
+                                    &global_trace, &set_tracer_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs 'trace' entry\n");
+
+        entry = debugfs_create_file("tracing_max_latency", 0644, d_tracer,
+                                    &tracing_max_latency,
+                                    &tracing_max_lat_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs "
+                           "'tracing_max_latency' entry\n");
+
+        entry = debugfs_create_file("tracing_thresh", 0644, d_tracer,
+                                    &tracing_thresh, &tracing_max_lat_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs "
+                           "'tracing_threash' entry\n");
+        entry = debugfs_create_file("README", 0644, d_tracer,
+                                    NULL, &tracing_readme_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs 'README' entry\n");
+
+        entry = debugfs_create_file("trace_pipe", 0644, d_tracer,
+                                    NULL, &tracing_pipe_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs "
+                           "'tracing_threash' entry\n");
+
+        entry = debugfs_create_file("trace_entries", 0644, d_tracer,
+                                    &global_trace, &tracing_entries_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs "
+                           "'tracing_threash' entry\n");
+
+#ifdef CONFIG_DYNAMIC_FTRACE
+        entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
+                                    &ftrace_update_tot_cnt,
+                                    &tracing_read_long_fops);
+        if (!entry)
+                pr_warning("Could not create debugfs "
+                           "'dyn_ftrace_total_info' entry\n");
+#endif
+#ifdef CONFIG_SYSPROF_TRACER
+        init_tracer_sysprof_debugfs(d_tracer);
+#endif
+}
+
+static int trace_alloc_page(void)
+{
+        struct trace_array_cpu *data;
+        struct page *page, *tmp;
+        LIST_HEAD(pages);
+        void *array;
+        unsigned pages_allocated = 0;
+        int i;
+
+        /* first allocate a page for each CPU */
+        for_each_tracing_cpu(i) {
+                array = (void *)__get_free_page(GFP_KERNEL);
+                if (array == NULL) {
+                        printk(KERN_ERR "tracer: failed to allocate page"
+                               "for trace buffer!\n");
+                        goto free_pages;
+                }
+
+                pages_allocated++;
+                page = virt_to_page(array);
+                list_add(&page->lru, &pages);
+
+/* Only allocate if we are actually using the max trace */
+#ifdef CONFIG_TRACER_MAX_TRACE
+                array = (void *)__get_free_page(GFP_KERNEL);
+                if (array == NULL) {
+                        printk(KERN_ERR "tracer: failed to allocate page"
+                               "for trace buffer!\n");
+                        goto free_pages;
+                }
+                pages_allocated++;
+                page = virt_to_page(array);
+                list_add(&page->lru, &pages);
+#endif
+        }
+
+        /* Now that we successfully allocate a page per CPU, add them */
+        for_each_tracing_cpu(i) {
+                data = global_trace.data[i];
+                page = list_entry(pages.next, struct page, lru);
+                list_del_init(&page->lru);
+                list_add_tail(&page->lru, &data->trace_pages);
+                ClearPageLRU(page);
+
+#ifdef CONFIG_TRACER_MAX_TRACE
+                data = max_tr.data[i];
+                page = list_entry(pages.next, struct page, lru);
+                list_del_init(&page->lru);
+                list_add_tail(&page->lru, &data->trace_pages);
+                SetPageLRU(page);
+#endif
+        }
+        tracing_pages_allocated += pages_allocated;
+        global_trace.entries += ENTRIES_PER_PAGE;
+
+        return 0;
+
+ free_pages:
+        list_for_each_entry_safe(page, tmp, &pages, lru) {
+                list_del_init(&page->lru);
+                __free_page(page);
+        }
+        return -ENOMEM;
+}
+
+static int trace_free_page(void)
+{
+        struct trace_array_cpu *data;
+        struct page *page;
+        struct list_head *p;
+        int i;
+        int ret = 0;
+
+        /* free one page from each buffer */
+        for_each_tracing_cpu(i) {
+                data = global_trace.data[i];
+                p = data->trace_pages.next;
+                if (p == &data->trace_pages) {
+                        /* should never happen */
+                        WARN_ON(1);
+                        tracing_disabled = 1;
+                        ret = -1;
+                        break;
+                }
+                page = list_entry(p, struct page, lru);
+                ClearPageLRU(page);
+                list_del(&page->lru);
+                tracing_pages_allocated--;
+                tracing_pages_allocated--;
+                __free_page(page);
+
+                tracing_reset(data);
+
+#ifdef CONFIG_TRACER_MAX_TRACE
+                data = max_tr.data[i];
+                p = data->trace_pages.next;
+                if (p == &data->trace_pages) {
+                        /* should never happen */
+                        WARN_ON(1);
+                        tracing_disabled = 1;
+                        ret = -1;
+                        break;
+                }
+                page = list_entry(p, struct page, lru);
+                ClearPageLRU(page);
+                list_del(&page->lru);
+                __free_page(page);
+
+                tracing_reset(data);
+#endif
+        }
+        global_trace.entries -= ENTRIES_PER_PAGE;
+
+        return ret;
+}
+
+__init static int tracer_alloc_buffers(void)
+{
+        struct trace_array_cpu *data;
+        void *array;
+        struct page *page;
+        int pages = 0;
+        int ret = -ENOMEM;
+        int i;
+
+        /* TODO: make the number of buffers hot pluggable with CPUS */
+        tracing_nr_buffers = num_possible_cpus();
+        tracing_buffer_mask = cpu_possible_map;
+
+        /* Allocate the first page for all buffers */
+        for_each_tracing_cpu(i) {
+                data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
+                max_tr.data[i] = &per_cpu(max_data, i);
+
+                array = (void *)__get_free_page(GFP_KERNEL);
+                if (array == NULL) {
+                        printk(KERN_ERR "tracer: failed to allocate page"
+                               "for trace buffer!\n");
+                        goto free_buffers;
+                }
+
+                /* set the array to the list */
+                INIT_LIST_HEAD(&data->trace_pages);
+                page = virt_to_page(array);
+                list_add(&page->lru, &data->trace_pages);
+                /* use the LRU flag to differentiate the two buffers */
+                ClearPageLRU(page);
+
+                data->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
+                max_tr.data[i]->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
+
+/* Only allocate if we are actually using the max trace */
+#ifdef CONFIG_TRACER_MAX_TRACE
+                array = (void *)__get_free_page(GFP_KERNEL);
+                if (array == NULL) {
+                        printk(KERN_ERR "tracer: failed to allocate page"
+                               "for trace buffer!\n");
+                        goto free_buffers;
+                }
+
+                INIT_LIST_HEAD(&max_tr.data[i]->trace_pages);
+                page = virt_to_page(array);
+                list_add(&page->lru, &max_tr.data[i]->trace_pages);
+                SetPageLRU(page);
+#endif
+        }
+
+        /*
+         * Since we allocate by orders of pages, we may be able to
+         * round up a bit.
+         */
+        global_trace.entries = ENTRIES_PER_PAGE;
+        pages++;
+
+        while (global_trace.entries < trace_nr_entries) {
+                if (trace_alloc_page())
+                        break;
+                pages++;
+        }
+        max_tr.entries = global_trace.entries;
+
+        pr_info("tracer: %d pages allocated for %ld entries of %ld bytes\n",
+                pages, trace_nr_entries, (long)TRACE_ENTRY_SIZE);
+        pr_info("   actual entries %ld\n", global_trace.entries);
+
+        tracer_init_debugfs();
+
+        trace_init_cmdlines();
+
+        register_tracer(&no_tracer);
+        current_trace = &no_tracer;
+
+        /* All seems OK, enable tracing */
+        global_trace.ctrl = tracer_enabled;
+        tracing_disabled = 0;
+
+        return 0;
+
+ free_buffers:
+        for (i-- ; i >= 0; i--) {
+                struct page *page, *tmp;
+                struct trace_array_cpu *data = global_trace.data[i];
+
+                if (data) {
+                        list_for_each_entry_safe(page, tmp,
+                                                 &data->trace_pages, lru) {
+                                list_del_init(&page->lru);
+                                __free_page(page);
+                        }
+                }
+
+#ifdef CONFIG_TRACER_MAX_TRACE
+                data = max_tr.data[i];
+                if (data) {
+                        list_for_each_entry_safe(page, tmp,
+                                                 &data->trace_pages, lru) {
+                                list_del_init(&page->lru);
+                                __free_page(page);
+                        }
+                }
+#endif
+        }
+        return ret;
+}
+fs_initcall(tracer_alloc_buffers);
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
new file mode 100644
index 000000000000..f69f86788c2b
--- /dev/null
+++ b/kernel/trace/trace.h
@@ -0,0 +1,339 @@
+#ifndef _LINUX_KERNEL_TRACE_H
+#define _LINUX_KERNEL_TRACE_H
+
+#include <linux/fs.h>
+#include <asm/atomic.h>
+#include <linux/sched.h>
+#include <linux/clocksource.h>
+#include <linux/mmiotrace.h>
+
+enum trace_type {
+        __TRACE_FIRST_TYPE = 0,
+
+        TRACE_FN,
+        TRACE_CTX,
+        TRACE_WAKE,
+        TRACE_STACK,
+        TRACE_SPECIAL,
+        TRACE_MMIO_RW,
+        TRACE_MMIO_MAP,
+
+        __TRACE_LAST_TYPE
+};
+
+/*
+ * Function trace entry - function address and parent function addres:
+ */
+struct ftrace_entry {
+        unsigned long           ip;
+        unsigned long           parent_ip;
+};
+
+/*
+ * Context switch trace entry - which task (and prio) we switched from/to:
+ */
+struct ctx_switch_entry {
+        unsigned int            prev_pid;
+        unsigned char           prev_prio;
+        unsigned char           prev_state;
+        unsigned int            next_pid;
+        unsigned char           next_prio;
+        unsigned char           next_state;
+};
+
+/*
+ * Special (free-form) trace entry:
+ */
+struct special_entry {
+        unsigned long           arg1;
+        unsigned long           arg2;
+        unsigned long           arg3;
+};
+
+/*
+ * Stack-trace entry:
+ */
+
+#define FTRACE_STACK_ENTRIES    8
+
+struct stack_entry {
+        unsigned long           caller[FTRACE_STACK_ENTRIES];
+};
+
+/*
+ * The trace entry - the most basic unit of tracing. This is what
+ * is printed in the end as a single line in the trace output, such as:
+ *
+ *     bash-15816 [01]   235.197585: idle_cpu <- irq_enter
+ */
+struct trace_entry {
+        char                    type;
+        char                    cpu;
+        char                    flags;
+        char                    preempt_count;
+        int                     pid;
+        cycle_t                 t;
+        union {
+                struct ftrace_entry             fn;
+                struct ctx_switch_entry         ctx;
+                struct special_entry            special;
+                struct stack_entry              stack;
+                struct mmiotrace_rw             mmiorw;
+                struct mmiotrace_map            mmiomap;
+        };
+};
+
+#define TRACE_ENTRY_SIZE        sizeof(struct trace_entry)
+
+/*
+ * The CPU trace array - it consists of thousands of trace entries
+ * plus some other descriptor data: (for example which task started
+ * the trace, etc.)
+ */
+struct trace_array_cpu {
+        struct list_head        trace_pages;
+        atomic_t                disabled;
+        raw_spinlock_t          lock;
+        struct lock_class_key   lock_key;
+
+        /* these fields get copied into max-trace: */
+        unsigned                trace_head_idx;
+        unsigned                trace_tail_idx;
+        void                    *trace_head; /* producer */
+        void                    *trace_tail; /* consumer */
+        unsigned long           trace_idx;
+        unsigned long           overrun;
+        unsigned long           saved_latency;
+        unsigned long           critical_start;
+        unsigned long           critical_end;
+        unsigned long           critical_sequence;
+        unsigned long           nice;
+        unsigned long           policy;
+        unsigned long           rt_priority;
+        cycle_t                 preempt_timestamp;
+        pid_t                   pid;
+        uid_t                   uid;
+        char                    comm[TASK_COMM_LEN];
+};
+
+struct trace_iterator;
+
+/*
+ * The trace array - an array of per-CPU trace arrays. This is the
+ * highest level data structure that individual tracers deal with.
+ * They have on/off state as well:
+ */
+struct trace_array {
+        unsigned long           entries;
+        long                    ctrl;
+        int                     cpu;
+        cycle_t                 time_start;
+        struct task_struct      *waiter;
+        struct trace_array_cpu  *data[NR_CPUS];
+};
+
+/*
+ * A specific tracer, represented by methods that operate on a trace array:
+ */
+struct tracer {
+        const char              *name;
+        void                    (*init)(struct trace_array *tr);
+        void                    (*reset)(struct trace_array *tr);
+        void                    (*open)(struct trace_iterator *iter);
+        void                    (*pipe_open)(struct trace_iterator *iter);
+        void                    (*close)(struct trace_iterator *iter);
+        void                    (*start)(struct trace_iterator *iter);
+        void                    (*stop)(struct trace_iterator *iter);
+        ssize_t                 (*read)(struct trace_iterator *iter,
+                                        struct file *filp, char __user *ubuf,
+                                        size_t cnt, loff_t *ppos);
+        void                    (*ctrl_update)(struct trace_array *tr);
+#ifdef CONFIG_FTRACE_STARTUP_TEST
+        int                     (*selftest)(struct tracer *trace,
+                                            struct trace_array *tr);
+#endif
+        int                     (*print_line)(struct trace_iterator *iter);
+        struct tracer           *next;
+        int                     print_max;
+};
+
+struct trace_seq {
+        unsigned char           buffer[PAGE_SIZE];
+        unsigned int            len;
+        unsigned int            readpos;
+};
+
+/*
+ * Trace iterator - used by printout routines who present trace
+ * results to users and which routines might sleep, etc:
+ */
+struct trace_iterator {
+        struct trace_array      *tr;
+        struct tracer           *trace;
+        void                    *private;
+        long                    last_overrun[NR_CPUS];
+        long                    overrun[NR_CPUS];
+
+        /* The below is zeroed out in pipe_read */
+        struct trace_seq        seq;
+        struct trace_entry      *ent;
+        int                     cpu;
+
+        struct trace_entry      *prev_ent;
+        int                     prev_cpu;
+
+        unsigned long           iter_flags;
+        loff_t                  pos;
+        unsigned long           next_idx[NR_CPUS];
+        struct list_head        *next_page[NR_CPUS];
+        unsigned                next_page_idx[NR_CPUS];
+        long                    idx;
+};
+
+void tracing_reset(struct trace_array_cpu *data);
+int tracing_open_generic(struct inode *inode, struct file *filp);
+struct dentry *tracing_init_dentry(void);
+void init_tracer_sysprof_debugfs(struct dentry *d_tracer);
+
+void ftrace(struct trace_array *tr,
+                            struct trace_array_cpu *data,
+                            unsigned long ip,
+                            unsigned long parent_ip,
+                            unsigned long flags);
+void tracing_sched_switch_trace(struct trace_array *tr,
+                                struct trace_array_cpu *data,
+                                struct task_struct *prev,
+                                struct task_struct *next,
+                                unsigned long flags);
+void tracing_record_cmdline(struct task_struct *tsk);
+
+void tracing_sched_wakeup_trace(struct trace_array *tr,
+                                struct trace_array_cpu *data,
+                                struct task_struct *wakee,
+                                struct task_struct *cur,
+                                unsigned long flags);
+void trace_special(struct trace_array *tr,
+                   struct trace_array_cpu *data,
+                   unsigned long arg1,
+                   unsigned long arg2,
+                   unsigned long arg3);
+void trace_function(struct trace_array *tr,
+                    struct trace_array_cpu *data,
+                    unsigned long ip,
+                    unsigned long parent_ip,
+                    unsigned long flags);
+
+void tracing_start_cmdline_record(void);
+void tracing_stop_cmdline_record(void);
+int register_tracer(struct tracer *type);
+void unregister_tracer(struct tracer *type);
+
+extern unsigned long nsecs_to_usecs(unsigned long nsecs);
+
+extern unsigned long tracing_max_latency;
+extern unsigned long tracing_thresh;
+
+void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu);
+void update_max_tr_single(struct trace_array *tr,
+                          struct task_struct *tsk, int cpu);
+
+extern cycle_t ftrace_now(int cpu);
+
+#ifdef CONFIG_FTRACE
+void tracing_start_function_trace(void);
+void tracing_stop_function_trace(void);
+#else
+# define tracing_start_function_trace()         do { } while (0)
+# define tracing_stop_function_trace()          do { } while (0)
+#endif
+
+#ifdef CONFIG_CONTEXT_SWITCH_TRACER
+typedef void
+(*tracer_switch_func_t)(void *private,
+                        void *__rq,
+                        struct task_struct *prev,
+                        struct task_struct *next);
+
+struct tracer_switch_ops {
+        tracer_switch_func_t            func;
+        void                            *private;
+        struct tracer_switch_ops        *next;
+};
+
+#endif /* CONFIG_CONTEXT_SWITCH_TRACER */
+
+#ifdef CONFIG_DYNAMIC_FTRACE
+extern unsigned long ftrace_update_tot_cnt;
+#define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func
+extern int DYN_FTRACE_TEST_NAME(void);
+#endif
+
+#ifdef CONFIG_MMIOTRACE
+extern void __trace_mmiotrace_rw(struct trace_array *tr,
+                                struct trace_array_cpu *data,
+                                struct mmiotrace_rw *rw);
+extern void __trace_mmiotrace_map(struct trace_array *tr,
+                                struct trace_array_cpu *data,
+                                struct mmiotrace_map *map);
+#endif
+
+#ifdef CONFIG_FTRACE_STARTUP_TEST
+#ifdef CONFIG_FTRACE
+extern int trace_selftest_startup_function(struct tracer *trace,
+                                           struct trace_array *tr);
+#endif
+#ifdef CONFIG_IRQSOFF_TRACER
+extern int trace_selftest_startup_irqsoff(struct tracer *trace,
+                                          struct trace_array *tr);
+#endif
+#ifdef CONFIG_PREEMPT_TRACER
+extern int trace_selftest_startup_preemptoff(struct tracer *trace,
+                                             struct trace_array *tr);
+#endif
+#if defined(CONFIG_IRQSOFF_TRACER) && defined(CONFIG_PREEMPT_TRACER)
+extern int trace_selftest_startup_preemptirqsoff(struct tracer *trace,
+                                                 struct trace_array *tr);
+#endif
+#ifdef CONFIG_SCHED_TRACER
+extern int trace_selftest_startup_wakeup(struct tracer *trace,
+                                         struct trace_array *tr);
+#endif
+#ifdef CONFIG_CONTEXT_SWITCH_TRACER
+extern int trace_selftest_startup_sched_switch(struct tracer *trace,
+                                               struct trace_array *tr);
+#endif
+#ifdef CONFIG_SYSPROF_TRACER
+extern int trace_selftest_startup_sysprof(struct tracer *trace,
+                                               struct trace_array *tr);
+#endif
+#endif /* CONFIG_FTRACE_STARTUP_TEST */
+
+extern void *head_page(struct trace_array_cpu *data);
+extern int trace_seq_printf(struct trace_seq *s, const char *fmt, ...);
+extern ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf,
+                                 size_t cnt);
+extern long ns2usecs(cycle_t nsec);
+
+extern unsigned long trace_flags;
+
+/*
+ * trace_iterator_flags is an enumeration that defines bit
+ * positions into trace_flags that controls the output.
+ *
+ * NOTE: These bits must match the trace_options array in
+ *       trace.c.
+ */
+enum trace_iterator_flags {
+        TRACE_ITER_PRINT_PARENT         = 0x01,
+        TRACE_ITER_SYM_OFFSET           = 0x02,
+        TRACE_ITER_SYM_ADDR             = 0x04,
+        TRACE_ITER_VERBOSE              = 0x08,
+        TRACE_ITER_RAW                  = 0x10,
+        TRACE_ITER_HEX                  = 0x20,
+        TRACE_ITER_BIN                  = 0x40,
+        TRACE_ITER_BLOCK                = 0x80,
+        TRACE_ITER_STACKTRACE           = 0x100,
+        TRACE_ITER_SCHED_TREE           = 0x200,
+};
+
+#endif /* _LINUX_KERNEL_TRACE_H */
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
new file mode 100644
index 000000000000..312144897970
--- /dev/null
+++ b/kernel/trace/trace_functions.c
@@ -0,0 +1,81 @@
+/*
+ * ring buffer based function tracer
+ *
+ * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
+ * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
+ *
+ * Based on code from the latency_tracer, that is:
+ *
+ *  Copyright (C) 2004-2006 Ingo Molnar
+ *  Copyright (C) 2004 William Lee Irwin III
+ */
+#include <linux/debugfs.h>
+#include <linux/uaccess.h>
+#include <linux/ftrace.h>
+#include <linux/fs.h>
+
+#include "trace.h"
+
+static void function_reset(struct trace_array *tr)
+{
+        int cpu;
+
+        tr->time_start = ftrace_now(tr->cpu);
+
+        for_each_online_cpu(cpu)
+                tracing_reset(tr->data[cpu]);
+}
+
+static void start_function_trace(struct trace_array *tr)
+{
+        tr->cpu = get_cpu();
+        function_reset(tr);
+        put_cpu();
+
+        tracing_start_cmdline_record();
+        tracing_start_function_trace();
+}
+
+static void stop_function_trace(struct trace_array *tr)
+{
+        tracing_stop_function_trace();
+        tracing_stop_cmdline_record();
+}
+
+static void function_trace_init(struct trace_array *tr)
+{
+        if (tr->ctrl)
+                start_function_trace(tr);
+}
+
+static void function_trace_reset(struct trace_array *tr)
+{
+        if (tr->ctrl)
+                stop_function_trace(tr);
+}
+
+static void function_trace_ctrl_update(struct trace_array *tr)
+{
+        if (tr->ctrl)
+                start_function_trace(tr);
+        else
+                stop_function_trace(tr);
+}
+
+static struct tracer function_trace __read_mostly =
+{
+        .name        = "ftrace",
+        .init        = function_trace_init,
+        .reset       = function_trace_reset,
+        .ctrl_update = function_trace_ctrl_update,
+#ifdef CONFIG_FTRACE_SELFTEST
+        .selftest    = trace_selftest_startup_function,
+#endif
+};
+
+static __init int init_function_trace(void)
+{
+        return register_tracer(&function_trace);
+}
+
+device_initcall(init_function_trace);
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
new file mode 100644
index 000000000000..421d6fe3650e
--- /dev/null
+++ b/kernel/trace/trace_irqsoff.c
@@ -0,0 +1,486 @@
+/*
+ * trace irqs off criticall timings
+ *
+ * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
+ * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
+ *
+ * From code in the latency_tracer, that is:
+ *
+ *  Copyright (C) 2004-2006 Ingo Molnar
+ *  Copyright (C) 2004 William Lee Irwin III
+ */
+#include <linux/kallsyms.h>
+#include <linux/debugfs.h>
+#include <linux/uaccess.h>
+#include <linux/module.h>
+#include <linux/ftrace.h>
+#include <linux/fs.h>
+
+#include "trace.h"
+
+static struct trace_array               *irqsoff_trace __read_mostly;
+static int                              tracer_enabled __read_mostly;
+
+static DEFINE_PER_CPU(int, tracing_cpu);
+
+static DEFINE_SPINLOCK(max_trace_lock);
+
+enum {
+        TRACER_IRQS_OFF         = (1 << 1),
+        TRACER_PREEMPT_OFF      = (1 << 2),
+};
+
+static int trace_type __read_mostly;
+
+#ifdef CONFIG_PREEMPT_TRACER
+static inline int
+preempt_trace(void)
+{
+        return ((trace_type & TRACER_PREEMPT_OFF) && preempt_count());
+}
+#else
+# define preempt_trace() (0)
+#endif
+
+#ifdef CONFIG_IRQSOFF_TRACER
+static inline int
+irq_trace(void)
+{
+        return ((trace_type & TRACER_IRQS_OFF) &&
+                irqs_disabled());
+}
+#else
+# define irq_trace() (0)
+#endif
+
+/*
+ * Sequence count - we record it when starting a measurement and
+ * skip the latency if the sequence has changed - some other section
+ * did a maximum and could disturb our measurement with serial console
+ * printouts, etc. Truly coinciding maximum latencies should be rare
+ * and what happens together happens separately as well, so this doesnt
+ * decrease the validity of the maximum found:
+ */
+static __cacheline_aligned_in_smp       unsigned long max_sequence;
+
+#ifdef CONFIG_FTRACE
+/*
+ * irqsoff uses its own tracer function to keep the overhead down:
+ */
+static void
+irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip)
+{
+        struct trace_array *tr = irqsoff_trace;
+        struct trace_array_cpu *data;
+        unsigned long flags;
+        long disabled;
+        int cpu;
+
+        /*
+         * Does not matter if we preempt. We test the flags
+         * afterward, to see if irqs are disabled or not.
+         * If we preempt and get a false positive, the flags
+         * test will fail.
+         */
+        cpu = raw_smp_processor_id();
+        if (likely(!per_cpu(tracing_cpu, cpu)))
+                return;
+
+        local_save_flags(flags);
+        /* slight chance to get a false positive on tracing_cpu */
+        if (!irqs_disabled_flags(flags))
+                return;
+
+        data = tr->data[cpu];
+        disabled = atomic_inc_return(&data->disabled);
+
+        if (likely(disabled == 1))
+                trace_function(tr, data, ip, parent_ip, flags);
+
+        atomic_dec(&data->disabled);
+}
+
+static struct ftrace_ops trace_ops __read_mostly =
+{
+        .func = irqsoff_tracer_call,
+};
+#endif /* CONFIG_FTRACE */
+
+/*
+ * Should this new latency be reported/recorded?
+ */
+static int report_latency(cycle_t delta)
+{
+        if (tracing_thresh) {
+                if (delta < tracing_thresh)
+                        return 0;
+        } else {
+                if (delta <= tracing_max_latency)
+                        return 0;
+        }
+        return 1;
+}
+
+static void
+check_critical_timing(struct trace_array *tr,
+                      struct trace_array_cpu *data,
+                      unsigned long parent_ip,
+                      int cpu)
+{
+        unsigned long latency, t0, t1;
+        cycle_t T0, T1, delta;
+        unsigned long flags;
+
+        /*
+         * usecs conversion is slow so we try to delay the conversion
+         * as long as possible:
+         */
+        T0 = data->preempt_timestamp;
+        T1 = ftrace_now(cpu);
+        delta = T1-T0;
+
+        local_save_flags(flags);
+
+        if (!report_latency(delta))
+                goto out;
+
+        spin_lock_irqsave(&max_trace_lock, flags);
+
+        /* check if we are still the max latency */
+        if (!report_latency(delta))
+                goto out_unlock;
+
+        trace_function(tr, data, CALLER_ADDR0, parent_ip, flags);
+
+        latency = nsecs_to_usecs(delta);
+
+        if (data->critical_sequence != max_sequence)
+                goto out_unlock;
+
+        tracing_max_latency = delta;
+        t0 = nsecs_to_usecs(T0);
+        t1 = nsecs_to_usecs(T1);
+
+        data->critical_end = parent_ip;
+
+        update_max_tr_single(tr, current, cpu);
+
+        max_sequence++;
+
+out_unlock:
+        spin_unlock_irqrestore(&max_trace_lock, flags);
+
+out:
+        data->critical_sequence = max_sequence;
+        data->preempt_timestamp = ftrace_now(cpu);
+        tracing_reset(data);
+        trace_function(tr, data, CALLER_ADDR0, parent_ip, flags);
+}
+
+static inline void
+start_critical_timing(unsigned long ip, unsigned long parent_ip)
+{
+        int cpu;
+        struct trace_array *tr = irqsoff_trace;
+        struct trace_array_cpu *data;
+        unsigned long flags;
+
+        if (likely(!tracer_enabled))
+                return;
+
+        cpu = raw_smp_processor_id();
+
+        if (per_cpu(tracing_cpu, cpu))
+                return;
+
+        data = tr->data[cpu];
+
+        if (unlikely(!data) || atomic_read(&data->disabled))
+                return;
+
+        atomic_inc(&data->disabled);
+
+        data->critical_sequence = max_sequence;
+        data->preempt_timestamp = ftrace_now(cpu);
+        data->critical_start = parent_ip ? : ip;
+        tracing_reset(data);
+
+        local_save_flags(flags);
+
+        trace_function(tr, data, ip, parent_ip, flags);
+
+        per_cpu(tracing_cpu, cpu) = 1;
+
+        atomic_dec(&data->disabled);
+}
+
+static inline void
+stop_critical_timing(unsigned long ip, unsigned long parent_ip)
+{
+        int cpu;
+        struct trace_array *tr = irqsoff_trace;
+        struct trace_array_cpu *data;
+        unsigned long flags;
+
+        cpu = raw_smp_processor_id();
+        /* Always clear the tracing cpu on stopping the trace */
+        if (unlikely(per_cpu(tracing_cpu, cpu)))
+                per_cpu(tracing_cpu, cpu) = 0;
+        else
+                return;
+
+        if (!tracer_enabled)
+                return;
+
+        data = tr->data[cpu];
+
+        if (unlikely(!data) || unlikely(!head_page(data)) ||
+            !data->critical_start || atomic_read(&data->disabled))
+                return;
+
+        atomic_inc(&data->disabled);
+
+        local_save_flags(flags);
+        trace_function(tr, data, ip, parent_ip, flags);
+        check_critical_timing(tr, data, parent_ip ? : ip, cpu);
+        data->critical_start = 0;
+        atomic_dec(&data->disabled);
+}
+
+/* start and stop critical timings used to for stoppage (in idle) */
+void start_critical_timings(void)
+{
+        if (preempt_trace() || irq_trace())
+                start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
+}
+
+void stop_critical_timings(void)
+{
+        if (preempt_trace() || irq_trace())
+                stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
+}
+
+#ifdef CONFIG_IRQSOFF_TRACER
+#ifdef CONFIG_PROVE_LOCKING
+void time_hardirqs_on(unsigned long a0, unsigned long a1)
+{
+        if (!preempt_trace() && irq_trace())
+                stop_critical_timing(a0, a1);
+}
+
+void time_hardirqs_off(unsigned long a0, unsigned long a1)
+{
+        if (!preempt_trace() && irq_trace())
+                start_critical_timing(a0, a1);
+}
+
+#else /* !CONFIG_PROVE_LOCKING */
+
+/*
+ * Stubs:
+ */
+
+void early_boot_irqs_off(void)
+{
+}
+
+void early_boot_irqs_on(void)
+{
+}
+
+void trace_softirqs_on(unsigned long ip)
+{
+}
+
+void trace_softirqs_off(unsigned long ip)
+{
+}
+
+inline void print_irqtrace_events(struct task_struct *curr)
+{
+}
+
+/*
+ * We are only interested in hardirq on/off events:
+ */
+void trace_hardirqs_on(void)
+{
+        if (!preempt_trace() && irq_trace())
+                stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
+}
+EXPORT_SYMBOL(trace_hardirqs_on);
+
+void trace_hardirqs_off(void)
+{
+        if (!preempt_trace() && irq_trace())
+                start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
+}
+EXPORT_SYMBOL(trace_hardirqs_off);
+
+void trace_hardirqs_on_caller(unsigned long caller_addr)
+{
+        if (!preempt_trace() && irq_trace())
+                stop_critical_timing(CALLER_ADDR0, caller_addr);
+}
+EXPORT_SYMBOL(trace_hardirqs_on_caller);
+
+void trace_hardirqs_off_caller(unsigned long caller_addr)
+{
+        if (!preempt_trace() && irq_trace())
+                start_critical_timing(CALLER_ADDR0, caller_addr);
+}
+EXPORT_SYMBOL(trace_hardirqs_off_caller);
+
+#endif /* CONFIG_PROVE_LOCKING */
+#endif /*  CONFIG_IRQSOFF_TRACER */
+
+#ifdef CONFIG_PREEMPT_TRACER
+void trace_preempt_on(unsigned long a0, unsigned long a1)
+{
+        stop_critical_timing(a0, a1);
+}
+
+void trace_preempt_off(unsigned long a0, unsigned long a1)
+{
+        start_critical_timing(a0, a1);
+}
+#endif /* CONFIG_PREEMPT_TRACER */
+
+static void start_irqsoff_tracer(struct trace_array *tr)
+{
+        register_ftrace_function(&trace_ops);
+        tracer_enabled = 1;
+}
+
+static void stop_irqsoff_tracer(struct trace_array *tr)
+{
+        tracer_enabled = 0;
+        unregister_ftrace_function(&trace_ops);
+}
+
+static void __irqsoff_tracer_init(struct trace_array *tr)
+{
+        irqsoff_trace = tr;
+        /* make sure that the tracer is visible */
+        smp_wmb();
+
+        if (tr->ctrl)
+                start_irqsoff_tracer(tr);
+}
+
+static void irqsoff_tracer_reset(struct trace_array *tr)
+{
+        if (tr->ctrl)
+                stop_irqsoff_tracer(tr);
+}
+
+static void irqsoff_tracer_ctrl_update(struct trace_array *tr)
+{
+        if (tr->ctrl)
+                start_irqsoff_tracer(tr);
+        else
+                stop_irqsoff_tracer(tr);
+}
+
+static void irqsoff_tracer_open(struct trace_iterator *iter)
+{
+        /* stop the trace while dumping */
+        if (iter->tr->ctrl)
+                stop_irqsoff_tracer(iter->tr);
+}
+
+static void irqsoff_tracer_close(struct trace_iterator *iter)
+{
+        if (iter->tr->ctrl)
+                start_irqsoff_tracer(iter->tr);
+}
+
+#ifdef CONFIG_IRQSOFF_TRACER
+static void irqsoff_tracer_init(struct trace_array *tr)
+{
+        trace_type = TRACER_IRQS_OFF;
+
+        __irqsoff_tracer_init(tr);
+}
+static struct tracer irqsoff_tracer __read_mostly =
+{
+        .name           = "irqsoff",
+        .init           = irqsoff_tracer_init,
+        .reset          = irqsoff_tracer_reset,
+        .open           = irqsoff_tracer_open,
+        .close          = irqsoff_tracer_close,
+        .ctrl_update    = irqsoff_tracer_ctrl_update,
+        .print_max      = 1,
+#ifdef CONFIG_FTRACE_SELFTEST
+        .selftest    = trace_selftest_startup_irqsoff,
+#endif
+};
+# define register_irqsoff(trace) register_tracer(&trace)
+#else
+# define register_irqsoff(trace) do { } while (0)
+#endif
+
+#ifdef CONFIG_PREEMPT_TRACER
+static void preemptoff_tracer_init(struct trace_array *tr)
+{
+        trace_type = TRACER_PREEMPT_OFF;
+
+        __irqsoff_tracer_init(tr);
+}
+
+static struct tracer preemptoff_tracer __read_mostly =
+{
+        .name           = "preemptoff",
+        .init           = preemptoff_tracer_init,
+        .reset          = irqsoff_tracer_reset,
+        .open           = irqsoff_tracer_open,
+        .close          = irqsoff_tracer_close,
+        .ctrl_update    = irqsoff_tracer_ctrl_update,
+        .print_max      = 1,
+#ifdef CONFIG_FTRACE_SELFTEST
+        .selftest    = trace_selftest_startup_preemptoff,
+#endif
+};
+# define register_preemptoff(trace) register_tracer(&trace)
+#else
+# define register_preemptoff(trace) do { } while (0)
+#endif
+
+#if defined(CONFIG_IRQSOFF_TRACER) && \
+        defined(CONFIG_PREEMPT_TRACER)
+
+static void preemptirqsoff_tracer_init(struct trace_array *tr)
+{
+        trace_type = TRACER_IRQS_OFF | TRACER_PREEMPT_OFF;
+
+        __irqsoff_tracer_init(tr);
+}
+
+static struct tracer preemptirqsoff_tracer __read_mostly =
+{
+        .name           = "preemptirqsoff",
+        .init           = preemptirqsoff_tracer_init,
+        .reset          = irqsoff_tracer_reset,
+        .open           = irqsoff_tracer_open,
+        .close          = irqsoff_tracer_close,
+        .ctrl_update    = irqsoff_tracer_ctrl_update,
+        .print_max      = 1,
+#ifdef CONFIG_FTRACE_SELFTEST
+        .selftest    = trace_selftest_startup_preemptirqsoff,
+#endif
+};
+
+# define register_preemptirqsoff(trace) register_tracer(&trace)
+#else
+# define register_preemptirqsoff(trace) do { } while (0)
+#endif
+
+__init static int init_irqsoff_tracer(void)
+{
+        register_irqsoff(irqsoff_tracer);
+        register_preemptoff(preemptoff_tracer);
+        register_preemptirqsoff(preemptirqsoff_tracer);
+
+        return 0;
+}
+device_initcall(init_irqsoff_tracer);
diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c
new file mode 100644
index 000000000000..b13dc19dcbb4
--- /dev/null
+++ b/kernel/trace/trace_mmiotrace.c
@@ -0,0 +1,295 @@
+/*
+ * Memory mapped I/O tracing
+ *
+ * Copyright (C) 2008 Pekka Paalanen <pq@iki.fi>
+ */
+
+#define DEBUG 1
+
+#include <linux/kernel.h>
+#include <linux/mmiotrace.h>
+#include <linux/pci.h>
+
+#include "trace.h"
+
+struct header_iter {
+        struct pci_dev *dev;
+};
+
+static struct trace_array *mmio_trace_array;
+static bool overrun_detected;
+
+static void mmio_reset_data(struct trace_array *tr)
+{
+        int cpu;
+
+        overrun_detected = false;
+        tr->time_start = ftrace_now(tr->cpu);
+
+        for_each_online_cpu(cpu)
+                tracing_reset(tr->data[cpu]);
+}
+
+static void mmio_trace_init(struct trace_array *tr)
+{
+        pr_debug("in %s\n", __func__);
+        mmio_trace_array = tr;
+        if (tr->ctrl) {
+                mmio_reset_data(tr);
+                enable_mmiotrace();
+        }
+}
+
+static void mmio_trace_reset(struct trace_array *tr)
+{
+        pr_debug("in %s\n", __func__);
+        if (tr->ctrl)
+                disable_mmiotrace();
+        mmio_reset_data(tr);
+        mmio_trace_array = NULL;
+}
+
+static void mmio_trace_ctrl_update(struct trace_array *tr)
+{
+        pr_debug("in %s\n", __func__);
+        if (tr->ctrl) {
+                mmio_reset_data(tr);
+                enable_mmiotrace();
+        } else {
+                disable_mmiotrace();
+        }
+}
+
+static int mmio_print_pcidev(struct trace_seq *s, const struct pci_dev *dev)
+{
+        int ret = 0;
+        int i;
+        resource_size_t start, end;
+        const struct pci_driver *drv = pci_dev_driver(dev);
+
+        /* XXX: incomplete checks for trace_seq_printf() return value */
+        ret += trace_seq_printf(s, "PCIDEV %02x%02x %04x%04x %x",
+                                dev->bus->number, dev->devfn,
+                                dev->vendor, dev->device, dev->irq);
+        /*
+         * XXX: is pci_resource_to_user() appropriate, since we are
+         * supposed to interpret the __ioremap() phys_addr argument based on
+         * these printed values?
+         */
+        for (i = 0; i < 7; i++) {
+                pci_resource_to_user(dev, i, &dev->resource[i], &start, &end);
+                ret += trace_seq_printf(s, " %llx",
+                        (unsigned long long)(start |
+                        (dev->resource[i].flags & PCI_REGION_FLAG_MASK)));
+        }
+        for (i = 0; i < 7; i++) {
+                pci_resource_to_user(dev, i, &dev->resource[i], &start, &end);
+                ret += trace_seq_printf(s, " %llx",
+                        dev->resource[i].start < dev->resource[i].end ?
+                        (unsigned long long)(end - start) + 1 : 0);
+        }
+        if (drv)
+                ret += trace_seq_printf(s, " %s\n", drv->name);
+        else
+                ret += trace_seq_printf(s, " \n");
+        return ret;
+}
+
+static void destroy_header_iter(struct header_iter *hiter)
+{
+        if (!hiter)
+                return;
+        pci_dev_put(hiter->dev);
+        kfree(hiter);
+}
+
+static void mmio_pipe_open(struct trace_iterator *iter)
+{
+        struct header_iter *hiter;
+        struct trace_seq *s = &iter->seq;
+
+        trace_seq_printf(s, "VERSION 20070824\n");
+
+        hiter = kzalloc(sizeof(*hiter), GFP_KERNEL);
+        if (!hiter)
+                return;
+
+        hiter->dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, NULL);
+        iter->private = hiter;
+}
+
+/* XXX: This is not called when the pipe is closed! */
+static void mmio_close(struct trace_iterator *iter)
+{
+        struct header_iter *hiter = iter->private;
+        destroy_header_iter(hiter);
+        iter->private = NULL;
+}
+
+static unsigned long count_overruns(struct trace_iterator *iter)
+{
+        int cpu;
+        unsigned long cnt = 0;
+        for_each_online_cpu(cpu) {
+                cnt += iter->overrun[cpu];
+                iter->overrun[cpu] = 0;
+        }
+        return cnt;
+}
+
+static ssize_t mmio_read(struct trace_iterator *iter, struct file *filp,
+                                char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+        ssize_t ret;
+        struct header_iter *hiter = iter->private;
+        struct trace_seq *s = &iter->seq;
+        unsigned long n;
+
+        n = count_overruns(iter);
+        if (n) {
+                /* XXX: This is later than where events were lost. */
+                trace_seq_printf(s, "MARK 0.000000 Lost %lu events.\n", n);
+                if (!overrun_detected)
+                        pr_warning("mmiotrace has lost events.\n");
+                overrun_detected = true;
+                goto print_out;
+        }
+
+        if (!hiter)
+                return 0;
+
+        mmio_print_pcidev(s, hiter->dev);
+        hiter->dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, hiter->dev);
+
+        if (!hiter->dev) {
+                destroy_header_iter(hiter);
+                iter->private = NULL;
+        }
+
+print_out:
+        ret = trace_seq_to_user(s, ubuf, cnt);
+        return (ret == -EBUSY) ? 0 : ret;
+}
+
+static int mmio_print_rw(struct trace_iterator *iter)
+{
+        struct trace_entry *entry = iter->ent;
+        struct mmiotrace_rw *rw = &entry->mmiorw;
+        struct trace_seq *s     = &iter->seq;
+        unsigned long long t    = ns2usecs(entry->t);
+        unsigned long usec_rem  = do_div(t, 1000000ULL);
+        unsigned secs           = (unsigned long)t;
+        int ret = 1;
+
+        switch (entry->mmiorw.opcode) {
+        case MMIO_READ:
+                ret = trace_seq_printf(s,
+                        "R %d %lu.%06lu %d 0x%llx 0x%lx 0x%lx %d\n",
+                        rw->width, secs, usec_rem, rw->map_id,
+                        (unsigned long long)rw->phys,
+                        rw->value, rw->pc, 0);
+                break;
+        case MMIO_WRITE:
+                ret = trace_seq_printf(s,
+                        "W %d %lu.%06lu %d 0x%llx 0x%lx 0x%lx %d\n",
+                        rw->width, secs, usec_rem, rw->map_id,
+                        (unsigned long long)rw->phys,
+                        rw->value, rw->pc, 0);
+                break;
+        case MMIO_UNKNOWN_OP:
+                ret = trace_seq_printf(s,
+                        "UNKNOWN %lu.%06lu %d 0x%llx %02x,%02x,%02x 0x%lx %d\n",
+                        secs, usec_rem, rw->map_id,
+                        (unsigned long long)rw->phys,
+                        (rw->value >> 16) & 0xff, (rw->value >> 8) & 0xff,
+                        (rw->value >> 0) & 0xff, rw->pc, 0);
+                break;
+        default:
+                ret = trace_seq_printf(s, "rw what?\n");
+                break;
+        }
+        if (ret)
+                return 1;
+        return 0;
+}
+
+static int mmio_print_map(struct trace_iterator *iter)
+{
+        struct trace_entry *entry = iter->ent;
+        struct mmiotrace_map *m = &entry->mmiomap;
+        struct trace_seq *s     = &iter->seq;
+        unsigned long long t    = ns2usecs(entry->t);
+        unsigned long usec_rem  = do_div(t, 1000000ULL);
+        unsigned secs           = (unsigned long)t;
+        int ret = 1;
+
+        switch (entry->mmiorw.opcode) {
+        case MMIO_PROBE:
+                ret = trace_seq_printf(s,
+                        "MAP %lu.%06lu %d 0x%llx 0x%lx 0x%lx 0x%lx %d\n",
+                        secs, usec_rem, m->map_id,
+                        (unsigned long long)m->phys, m->virt, m->len,
+                        0UL, 0);
+                break;
+        case MMIO_UNPROBE:
+                ret = trace_seq_printf(s,
+                        "UNMAP %lu.%06lu %d 0x%lx %d\n",
+                        secs, usec_rem, m->map_id, 0UL, 0);
+                break;
+        default:
+                ret = trace_seq_printf(s, "map what?\n");
+                break;
+        }
+        if (ret)
+                return 1;
+        return 0;
+}
+
+/* return 0 to abort printing without consuming current entry in pipe mode */
+static int mmio_print_line(struct trace_iterator *iter)
+{
+        switch (iter->ent->type) {
+        case TRACE_MMIO_RW:
+                return mmio_print_rw(iter);
+        case TRACE_MMIO_MAP:
+                return mmio_print_map(iter);
+        default:
+                return 1; /* ignore unknown entries */
+        }
+}
+
+static struct tracer mmio_tracer __read_mostly =
+{
+        .name           = "mmiotrace",
+        .init           = mmio_trace_init,
+        .reset          = mmio_trace_reset,
+        .pipe_open      = mmio_pipe_open,
+        .close          = mmio_close,
+        .read           = mmio_read,
+        .ctrl_update    = mmio_trace_ctrl_update,
+        .print_line     = mmio_print_line,
+};
+
+__init static int init_mmio_trace(void)
+{
+        return register_tracer(&mmio_tracer);
+}
+device_initcall(init_mmio_trace);
+
+void mmio_trace_rw(struct mmiotrace_rw *rw)
+{
+        struct trace_array *tr = mmio_trace_array;
+        struct trace_array_cpu *data = tr->data[smp_processor_id()];
+        __trace_mmiotrace_rw(tr, data, rw);
+}
+
+void mmio_trace_mapping(struct mmiotrace_map *map)
+{
+        struct trace_array *tr = mmio_trace_array;
+        struct trace_array_cpu *data;
+
+        preempt_disable();
+        data = tr->data[smp_processor_id()];
+        __trace_mmiotrace_map(tr, data, map);
+        preempt_enable();
+}
diff --git a/kernel/trace/trace_sched_switch.c b/kernel/trace/trace_sched_switch.c
new file mode 100644
index 000000000000..cb817a209aa0
--- /dev/null
+++ b/kernel/trace/trace_sched_switch.c
@@ -0,0 +1,286 @@
+/*
+ * trace context switch
+ *
+ * Copyright (C) 2007 Steven Rostedt <srostedt@redhat.com>
+ *
+ */
+#include <linux/module.h>
+#include <linux/fs.h>
+#include <linux/debugfs.h>
+#include <linux/kallsyms.h>
+#include <linux/uaccess.h>
+#include <linux/marker.h>
+#include <linux/ftrace.h>
+
+#include "trace.h"
+
+static struct trace_array       *ctx_trace;
+static int __read_mostly        tracer_enabled;
+static atomic_t                 sched_ref;
+
+static void
+sched_switch_func(void *private, void *__rq, struct task_struct *prev,
+                        struct task_struct *next)
+{
+        struct trace_array **ptr = private;
+        struct trace_array *tr = *ptr;
+        struct trace_array_cpu *data;
+        unsigned long flags;
+        long disabled;
+        int cpu;
+
+        tracing_record_cmdline(prev);
+        tracing_record_cmdline(next);
+
+        if (!tracer_enabled)
+                return;
+
+        local_irq_save(flags);
+        cpu = raw_smp_processor_id();
+        data = tr->data[cpu];
+        disabled = atomic_inc_return(&data->disabled);
+
+        if (likely(disabled == 1))
+                tracing_sched_switch_trace(tr, data, prev, next, flags);
+
+        atomic_dec(&data->disabled);
+        local_irq_restore(flags);
+}
+
+static notrace void
+sched_switch_callback(void *probe_data, void *call_data,
+                      const char *format, va_list *args)
+{
+        struct task_struct *prev;
+        struct task_struct *next;
+        struct rq *__rq;
+
+        if (!atomic_read(&sched_ref))
+                return;
+
+        /* skip prev_pid %d next_pid %d prev_state %ld */
+        (void)va_arg(*args, int);
+        (void)va_arg(*args, int);
+        (void)va_arg(*args, long);
+        __rq = va_arg(*args, typeof(__rq));
+        prev = va_arg(*args, typeof(prev));
+        next = va_arg(*args, typeof(next));
+
+        /*
+         * If tracer_switch_func only points to the local
+         * switch func, it still needs the ptr passed to it.
+         */
+        sched_switch_func(probe_data, __rq, prev, next);
+}
+
+static void
+wakeup_func(void *private, void *__rq, struct task_struct *wakee, struct
+                        task_struct *curr)
+{
+        struct trace_array **ptr = private;
+        struct trace_array *tr = *ptr;
+        struct trace_array_cpu *data;
+        unsigned long flags;
+        long disabled;
+        int cpu;
+
+        if (!tracer_enabled)
+                return;
+
+        tracing_record_cmdline(curr);
+
+        local_irq_save(flags);
+        cpu = raw_smp_processor_id();
+        data = tr->data[cpu];
+        disabled = atomic_inc_return(&data->disabled);
+
+        if (likely(disabled == 1))
+                tracing_sched_wakeup_trace(tr, data, wakee, curr, flags);
+
+        atomic_dec(&data->disabled);
+        local_irq_restore(flags);
+}
+
+static notrace void
+wake_up_callback(void *probe_data, void *call_data,
+                 const char *format, va_list *args)
+{
+        struct task_struct *curr;
+        struct task_struct *task;
+        struct rq *__rq;
+
+        if (likely(!tracer_enabled))
+                return;
+
+        /* Skip pid %d state %ld */
+        (void)va_arg(*args, int);
+        (void)va_arg(*args, long);
+        /* now get the meat: "rq %p task %p rq->curr %p" */
+        __rq = va_arg(*args, typeof(__rq));
+        task = va_arg(*args, typeof(task));
+        curr = va_arg(*args, typeof(curr));
+
+        tracing_record_cmdline(task);
+        tracing_record_cmdline(curr);
+
+        wakeup_func(probe_data, __rq, task, curr);
+}
+
+static void sched_switch_reset(struct trace_array *tr)
+{
+        int cpu;
+
+        tr->time_start = ftrace_now(tr->cpu);
+
+        for_each_online_cpu(cpu)
+                tracing_reset(tr->data[cpu]);
+}
+
+static int tracing_sched_register(void)
+{
+        int ret;
+
+        ret = marker_probe_register("kernel_sched_wakeup",
+                        "pid %d state %ld ## rq %p task %p rq->curr %p",
+                        wake_up_callback,
+                        &ctx_trace);
+        if (ret) {
+                pr_info("wakeup trace: Couldn't add marker"
+                        " probe to kernel_sched_wakeup\n");
+                return ret;
+        }
+
+        ret = marker_probe_register("kernel_sched_wakeup_new",
+                        "pid %d state %ld ## rq %p task %p rq->curr %p",
+                        wake_up_callback,
+                        &ctx_trace);
+        if (ret) {
+                pr_info("wakeup trace: Couldn't add marker"
+                        " probe to kernel_sched_wakeup_new\n");
+                goto fail_deprobe;
+        }
+
+        ret = marker_probe_register("kernel_sched_schedule",
+                "prev_pid %d next_pid %d prev_state %ld "
+                "## rq %p prev %p next %p",
+                sched_switch_callback,
+                &ctx_trace);
+        if (ret) {
+                pr_info("sched trace: Couldn't add marker"
+                        " probe to kernel_sched_schedule\n");
+                goto fail_deprobe_wake_new;
+        }
+
+        return ret;
+fail_deprobe_wake_new:
+        marker_probe_unregister("kernel_sched_wakeup_new",
+                                wake_up_callback,
+                                &ctx_trace);
+fail_deprobe:
+        marker_probe_unregister("kernel_sched_wakeup",
+                                wake_up_callback,
+                                &ctx_trace);
+        return ret;
+}
+
+static void tracing_sched_unregister(void)
+{
+        marker_probe_unregister("kernel_sched_schedule",
+                                sched_switch_callback,
+                                &ctx_trace);
+        marker_probe_unregister("kernel_sched_wakeup_new",
+                                wake_up_callback,
+                                &ctx_trace);
+        marker_probe_unregister("kernel_sched_wakeup",
+                                wake_up_callback,
+                                &ctx_trace);
+}
+
+static void tracing_start_sched_switch(void)
+{
+        long ref;
+
+        ref = atomic_inc_return(&sched_ref);
+        if (ref == 1)
+                tracing_sched_register();
+}
+
+static void tracing_stop_sched_switch(void)
+{
+        long ref;
+
+        ref = atomic_dec_and_test(&sched_ref);
+        if (ref)
+                tracing_sched_unregister();
+}
+
+void tracing_start_cmdline_record(void)
+{
+        tracing_start_sched_switch();
+}
+
+void tracing_stop_cmdline_record(void)
+{
+        tracing_stop_sched_switch();
+}
+
+static void start_sched_trace(struct trace_array *tr)
+{
+        sched_switch_reset(tr);
+        tracing_start_cmdline_record();
+        tracer_enabled = 1;
+}
+
+static void stop_sched_trace(struct trace_array *tr)
+{
+        tracer_enabled = 0;
+        tracing_stop_cmdline_record();
+}
+
+static void sched_switch_trace_init(struct trace_array *tr)
+{
+        ctx_trace = tr;
+
+        if (tr->ctrl)
+                start_sched_trace(tr);
+}
+
+static void sched_switch_trace_reset(struct trace_array *tr)
+{
+        if (tr->ctrl)
+                stop_sched_trace(tr);
+}
+
+static void sched_switch_trace_ctrl_update(struct trace_array *tr)
+{
+        /* When starting a new trace, reset the buffers */
+        if (tr->ctrl)
+                start_sched_trace(tr);
+        else
+                stop_sched_trace(tr);
+}
+
+static struct tracer sched_switch_trace __read_mostly =
+{
+        .name           = "sched_switch",
+        .init           = sched_switch_trace_init,
+        .reset          = sched_switch_trace_reset,
+        .ctrl_update    = sched_switch_trace_ctrl_update,
+#ifdef CONFIG_FTRACE_SELFTEST
+        .selftest    = trace_selftest_startup_sched_switch,
+#endif
+};
+
+__init static int init_sched_switch_trace(void)
+{
+        int ret = 0;
+
+        if (atomic_read(&sched_ref))
+                ret = tracing_sched_register();
+        if (ret) {
+                pr_info("error registering scheduler trace\n");
+                return ret;
+        }
+        return register_tracer(&sched_switch_trace);
+}
+device_initcall(init_sched_switch_trace);
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
new file mode 100644
index 000000000000..3c8d61df4474
--- /dev/null
+++ b/kernel/trace/trace_sched_wakeup.c
@@ -0,0 +1,448 @@
+/*
+ * trace task wakeup timings
+ *
+ * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
+ * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
+ *
+ * Based on code from the latency_tracer, that is:
+ *
+ *  Copyright (C) 2004-2006 Ingo Molnar
+ *  Copyright (C) 2004 William Lee Irwin III
+ */
+#include <linux/module.h>
+#include <linux/fs.h>
+#include <linux/debugfs.h>
+#include <linux/kallsyms.h>
+#include <linux/uaccess.h>
+#include <linux/ftrace.h>
+#include <linux/marker.h>
+
+#include "trace.h"
+
+static struct trace_array       *wakeup_trace;
+static int __read_mostly        tracer_enabled;
+
+static struct task_struct       *wakeup_task;
+static int                      wakeup_cpu;
+static unsigned                 wakeup_prio = -1;
+
+static DEFINE_SPINLOCK(wakeup_lock);
+
+static void __wakeup_reset(struct trace_array *tr);
+
+#ifdef CONFIG_FTRACE
+/*
+ * irqsoff uses its own tracer function to keep the overhead down:
+ */
+static void
+wakeup_tracer_call(unsigned long ip, unsigned long parent_ip)
+{
+        struct trace_array *tr = wakeup_trace;
+        struct trace_array_cpu *data;
+        unsigned long flags;
+        long disabled;
+        int resched;
+        int cpu;
+
+        if (likely(!wakeup_task))
+                return;
+
+        resched = need_resched();
+        preempt_disable_notrace();
+
+        cpu = raw_smp_processor_id();
+        data = tr->data[cpu];
+        disabled = atomic_inc_return(&data->disabled);
+        if (unlikely(disabled != 1))
+                goto out;
+
+        spin_lock_irqsave(&wakeup_lock, flags);
+
+        if (unlikely(!wakeup_task))
+                goto unlock;
+
+        /*
+         * The task can't disappear because it needs to
+         * wake up first, and we have the wakeup_lock.
+         */
+        if (task_cpu(wakeup_task) != cpu)
+                goto unlock;
+
+        trace_function(tr, data, ip, parent_ip, flags);
+
+ unlock:
+        spin_unlock_irqrestore(&wakeup_lock, flags);
+
+ out:
+        atomic_dec(&data->disabled);
+
+        /*
+         * To prevent recursion from the scheduler, if the
+         * resched flag was set before we entered, then
+         * don't reschedule.
+         */
+        if (resched)
+                preempt_enable_no_resched_notrace();
+        else
+                preempt_enable_notrace();
+}
+
+static struct ftrace_ops trace_ops __read_mostly =
+{
+        .func = wakeup_tracer_call,
+};
+#endif /* CONFIG_FTRACE */
+
+/*
+ * Should this new latency be reported/recorded?
+ */
+static int report_latency(cycle_t delta)
+{
+        if (tracing_thresh) {
+                if (delta < tracing_thresh)
+                        return 0;
+        } else {
+                if (delta <= tracing_max_latency)
+                        return 0;
+        }
+        return 1;
+}
+
+static void notrace
+wakeup_sched_switch(void *private, void *rq, struct task_struct *prev,
+        struct task_struct *next)
+{
+        unsigned long latency = 0, t0 = 0, t1 = 0;
+        struct trace_array **ptr = private;
+        struct trace_array *tr = *ptr;
+        struct trace_array_cpu *data;
+        cycle_t T0, T1, delta;
+        unsigned long flags;
+        long disabled;
+        int cpu;
+
+        if (unlikely(!tracer_enabled))
+                return;
+
+        /*
+         * When we start a new trace, we set wakeup_task to NULL
+         * and then set tracer_enabled = 1. We want to make sure
+         * that another CPU does not see the tracer_enabled = 1
+         * and the wakeup_task with an older task, that might
+         * actually be the same as next.
+         */
+        smp_rmb();
+
+        if (next != wakeup_task)
+                return;
+
+        /* The task we are waiting for is waking up */
+        data = tr->data[wakeup_cpu];
+
+        /* disable local data, not wakeup_cpu data */
+        cpu = raw_smp_processor_id();
+        disabled = atomic_inc_return(&tr->data[cpu]->disabled);
+        if (likely(disabled != 1))
+                goto out;
+
+        spin_lock_irqsave(&wakeup_lock, flags);
+
+        /* We could race with grabbing wakeup_lock */
+        if (unlikely(!tracer_enabled || next != wakeup_task))
+                goto out_unlock;
+
+        trace_function(tr, data, CALLER_ADDR1, CALLER_ADDR2, flags);
+
+        /*
+         * usecs conversion is slow so we try to delay the conversion
+         * as long as possible:
+         */
+        T0 = data->preempt_timestamp;
+        T1 = ftrace_now(cpu);
+        delta = T1-T0;
+
+        if (!report_latency(delta))
+                goto out_unlock;
+
+        latency = nsecs_to_usecs(delta);
+
+        tracing_max_latency = delta;
+        t0 = nsecs_to_usecs(T0);
+        t1 = nsecs_to_usecs(T1);
+
+        update_max_tr(tr, wakeup_task, wakeup_cpu);
+
+out_unlock:
+        __wakeup_reset(tr);
+        spin_unlock_irqrestore(&wakeup_lock, flags);
+out:
+        atomic_dec(&tr->data[cpu]->disabled);
+}
+
+static notrace void
+sched_switch_callback(void *probe_data, void *call_data,
+                      const char *format, va_list *args)
+{
+        struct task_struct *prev;
+        struct task_struct *next;
+        struct rq *__rq;
+
+        /* skip prev_pid %d next_pid %d prev_state %ld */
+        (void)va_arg(*args, int);
+        (void)va_arg(*args, int);
+        (void)va_arg(*args, long);
+        __rq = va_arg(*args, typeof(__rq));
+        prev = va_arg(*args, typeof(prev));
+        next = va_arg(*args, typeof(next));
+
+        tracing_record_cmdline(prev);
+
+        /*
+         * If tracer_switch_func only points to the local
+         * switch func, it still needs the ptr passed to it.
+         */
+        wakeup_sched_switch(probe_data, __rq, prev, next);
+}
+
+static void __wakeup_reset(struct trace_array *tr)
+{
+        struct trace_array_cpu *data;
+        int cpu;
+
+        assert_spin_locked(&wakeup_lock);
+
+        for_each_possible_cpu(cpu) {
+                data = tr->data[cpu];
+                tracing_reset(data);
+        }
+
+        wakeup_cpu = -1;
+        wakeup_prio = -1;
+
+        if (wakeup_task)
+                put_task_struct(wakeup_task);
+
+        wakeup_task = NULL;
+}
+
+static void wakeup_reset(struct trace_array *tr)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&wakeup_lock, flags);
+        __wakeup_reset(tr);
+        spin_unlock_irqrestore(&wakeup_lock, flags);
+}
+
+static void
+wakeup_check_start(struct trace_array *tr, struct task_struct *p,
+                   struct task_struct *curr)
+{
+        int cpu = smp_processor_id();
+        unsigned long flags;
+        long disabled;
+
+        if (likely(!rt_task(p)) ||
+                        p->prio >= wakeup_prio ||
+                        p->prio >= curr->prio)
+                return;
+
+        disabled = atomic_inc_return(&tr->data[cpu]->disabled);
+        if (unlikely(disabled != 1))
+                goto out;
+
+        /* interrupts should be off from try_to_wake_up */
+        spin_lock(&wakeup_lock);
+
+        /* check for races. */
+        if (!tracer_enabled || p->prio >= wakeup_prio)
+                goto out_locked;
+
+        /* reset the trace */
+        __wakeup_reset(tr);
+
+        wakeup_cpu = task_cpu(p);
+        wakeup_prio = p->prio;
+
+        wakeup_task = p;
+        get_task_struct(wakeup_task);
+
+        local_save_flags(flags);
+
+        tr->data[wakeup_cpu]->preempt_timestamp = ftrace_now(cpu);
+        trace_function(tr, tr->data[wakeup_cpu],
+                       CALLER_ADDR1, CALLER_ADDR2, flags);
+
+out_locked:
+        spin_unlock(&wakeup_lock);
+out:
+        atomic_dec(&tr->data[cpu]->disabled);
+}
+
+static notrace void
+wake_up_callback(void *probe_data, void *call_data,
+                 const char *format, va_list *args)
+{
+        struct trace_array **ptr = probe_data;
+        struct trace_array *tr = *ptr;
+        struct task_struct *curr;
+        struct task_struct *task;
+        struct rq *__rq;
+
+        if (likely(!tracer_enabled))
+                return;
+
+        /* Skip pid %d state %ld */
+        (void)va_arg(*args, int);
+        (void)va_arg(*args, long);
+        /* now get the meat: "rq %p task %p rq->curr %p" */
+        __rq = va_arg(*args, typeof(__rq));
+        task = va_arg(*args, typeof(task));
+        curr = va_arg(*args, typeof(curr));
+
+        tracing_record_cmdline(task);
+        tracing_record_cmdline(curr);
+
+        wakeup_check_start(tr, task, curr);
+}
+
+static void start_wakeup_tracer(struct trace_array *tr)
+{
+        int ret;
+
+        ret = marker_probe_register("kernel_sched_wakeup",
+                        "pid %d state %ld ## rq %p task %p rq->curr %p",
+                        wake_up_callback,
+                        &wakeup_trace);
+        if (ret) {
+                pr_info("wakeup trace: Couldn't add marker"
+                        " probe to kernel_sched_wakeup\n");
+                return;
+        }
+
+        ret = marker_probe_register("kernel_sched_wakeup_new",
+                        "pid %d state %ld ## rq %p task %p rq->curr %p",
+                        wake_up_callback,
+                        &wakeup_trace);
+        if (ret) {
+                pr_info("wakeup trace: Couldn't add marker"
+                        " probe to kernel_sched_wakeup_new\n");
+                goto fail_deprobe;
+        }
+
+        ret = marker_probe_register("kernel_sched_schedule",
+                "prev_pid %d next_pid %d prev_state %ld "
+                "## rq %p prev %p next %p",
+                sched_switch_callback,
+                &wakeup_trace);
+        if (ret) {
+                pr_info("sched trace: Couldn't add marker"
+                        " probe to kernel_sched_schedule\n");
+                goto fail_deprobe_wake_new;
+        }
+
+        wakeup_reset(tr);
+
+        /*
+         * Don't let the tracer_enabled = 1 show up before
+         * the wakeup_task is reset. This may be overkill since
+         * wakeup_reset does a spin_unlock after setting the
+         * wakeup_task to NULL, but I want to be safe.
+         * This is a slow path anyway.
+         */
+        smp_wmb();
+
+        register_ftrace_function(&trace_ops);
+
+        tracer_enabled = 1;
+
+        return;
+fail_deprobe_wake_new:
+        marker_probe_unregister("kernel_sched_wakeup_new",
+                                wake_up_callback,
+                                &wakeup_trace);
+fail_deprobe:
+        marker_probe_unregister("kernel_sched_wakeup",
+                                wake_up_callback,
+                                &wakeup_trace);
+}
+
+static void stop_wakeup_tracer(struct trace_array *tr)
+{
+        tracer_enabled = 0;
+        unregister_ftrace_function(&trace_ops);
+        marker_probe_unregister("kernel_sched_schedule",
+                                sched_switch_callback,
+                                &wakeup_trace);
+        marker_probe_unregister("kernel_sched_wakeup_new",
+                                wake_up_callback,
+                                &wakeup_trace);
+        marker_probe_unregister("kernel_sched_wakeup",
+                                wake_up_callback,
+                                &wakeup_trace);
+}
+
+static void wakeup_tracer_init(struct trace_array *tr)
+{
+        wakeup_trace = tr;
+
+        if (tr->ctrl)
+                start_wakeup_tracer(tr);
+}
+
+static void wakeup_tracer_reset(struct trace_array *tr)
+{
+        if (tr->ctrl) {
+                stop_wakeup_tracer(tr);
+                /* make sure we put back any tasks we are tracing */
+                wakeup_reset(tr);
+        }
+}
+
+static void wakeup_tracer_ctrl_update(struct trace_array *tr)
+{
+        if (tr->ctrl)
+                start_wakeup_tracer(tr);
+        else
+                stop_wakeup_tracer(tr);
+}
+
+static void wakeup_tracer_open(struct trace_iterator *iter)
+{
+        /* stop the trace while dumping */
+        if (iter->tr->ctrl)
+                stop_wakeup_tracer(iter->tr);
+}
+
+static void wakeup_tracer_close(struct trace_iterator *iter)
+{
+        /* forget about any processes we were recording */
+        if (iter->tr->ctrl)
+                start_wakeup_tracer(iter->tr);
+}
+
+static struct tracer wakeup_tracer __read_mostly =
+{
+        .name           = "wakeup",
+        .init           = wakeup_tracer_init,
+        .reset          = wakeup_tracer_reset,
+        .open           = wakeup_tracer_open,
+        .close          = wakeup_tracer_close,
+        .ctrl_update    = wakeup_tracer_ctrl_update,
+        .print_max      = 1,
+#ifdef CONFIG_FTRACE_SELFTEST
+        .selftest    = trace_selftest_startup_wakeup,
+#endif
+};
+
+__init static int init_wakeup_tracer(void)
+{
+        int ret;
+
+        ret = register_tracer(&wakeup_tracer);
+        if (ret)
+                return ret;
+
+        return 0;
+}
+device_initcall(init_wakeup_tracer);
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c
new file mode 100644
index 000000000000..0911b7e073bf
--- /dev/null
+++ b/kernel/trace/trace_selftest.c
@@ -0,0 +1,563 @@
+/* Include in trace.c */
+
+#include <linux/kthread.h>
+#include <linux/delay.h>
+
+static inline int trace_valid_entry(struct trace_entry *entry)
+{
+        switch (entry->type) {
+        case TRACE_FN:
+        case TRACE_CTX:
+        case TRACE_WAKE:
+        case TRACE_STACK:
+        case TRACE_SPECIAL:
+                return 1;
+        }
+        return 0;
+}
+
+static int
+trace_test_buffer_cpu(struct trace_array *tr, struct trace_array_cpu *data)
+{
+        struct trace_entry *entries;
+        struct page *page;
+        int idx = 0;
+        int i;
+
+        BUG_ON(list_empty(&data->trace_pages));
+        page = list_entry(data->trace_pages.next, struct page, lru);
+        entries = page_address(page);
+
+        check_pages(data);
+        if (head_page(data) != entries)
+                goto failed;
+
+        /*
+         * The starting trace buffer always has valid elements,
+         * if any element exists.
+         */
+        entries = head_page(data);
+
+        for (i = 0; i < tr->entries; i++) {
+
+                if (i < data->trace_idx && !trace_valid_entry(&entries[idx])) {
+                        printk(KERN_CONT ".. invalid entry %d ",
+                                entries[idx].type);
+                        goto failed;
+                }
+
+                idx++;
+                if (idx >= ENTRIES_PER_PAGE) {
+                        page = virt_to_page(entries);
+                        if (page->lru.next == &data->trace_pages) {
+                                if (i != tr->entries - 1) {
+                                        printk(KERN_CONT ".. entries buffer mismatch");
+                                        goto failed;
+                                }
+                        } else {
+                                page = list_entry(page->lru.next, struct page, lru);
+                                entries = page_address(page);
+                        }
+                        idx = 0;
+                }
+        }
+
+        page = virt_to_page(entries);
+        if (page->lru.next != &data->trace_pages) {
+                printk(KERN_CONT ".. too many entries");
+                goto failed;
+        }
+
+        return 0;
+
+ failed:
+        /* disable tracing */
+        tracing_disabled = 1;
+        printk(KERN_CONT ".. corrupted trace buffer .. ");
+        return -1;
+}
+
+/*
+ * Test the trace buffer to see if all the elements
+ * are still sane.
+ */
+static int trace_test_buffer(struct trace_array *tr, unsigned long *count)
+{
+        unsigned long flags, cnt = 0;
+        int cpu, ret = 0;
+
+        /* Don't allow flipping of max traces now */
+        raw_local_irq_save(flags);
+        __raw_spin_lock(&ftrace_max_lock);
+        for_each_possible_cpu(cpu) {
+                if (!head_page(tr->data[cpu]))
+                        continue;
+
+                cnt += tr->data[cpu]->trace_idx;
+
+                ret = trace_test_buffer_cpu(tr, tr->data[cpu]);
+                if (ret)
+                        break;
+        }
+        __raw_spin_unlock(&ftrace_max_lock);
+        raw_local_irq_restore(flags);
+
+        if (count)
+                *count = cnt;
+
+        return ret;
+}
+
+#ifdef CONFIG_FTRACE
+
+#ifdef CONFIG_DYNAMIC_FTRACE
+
+#define __STR(x) #x
+#define STR(x) __STR(x)
+
+/* Test dynamic code modification and ftrace filters */
+int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
+                                           struct trace_array *tr,
+                                           int (*func)(void))
+{
+        unsigned long count;
+        int ret;
+        int save_ftrace_enabled = ftrace_enabled;
+        int save_tracer_enabled = tracer_enabled;
+        char *func_name;
+
+        /* The ftrace test PASSED */
+        printk(KERN_CONT "PASSED\n");
+        pr_info("Testing dynamic ftrace: ");
+
+        /* enable tracing, and record the filter function */
+        ftrace_enabled = 1;
+        tracer_enabled = 1;
+
+        /* passed in by parameter to fool gcc from optimizing */
+        func();
+
+        /* update the records */
+        ret = ftrace_force_update();
+        if (ret) {
+                printk(KERN_CONT ".. ftraced failed .. ");
+                return ret;
+        }
+
+        /*
+         * Some archs *cough*PowerPC*cough* add charachters to the
+         * start of the function names. We simply put a '*' to
+         * accomodate them.
+         */
+        func_name = "*" STR(DYN_FTRACE_TEST_NAME);
+
+        /* filter only on our function */
+        ftrace_set_filter(func_name, strlen(func_name), 1);
+
+        /* enable tracing */
+        tr->ctrl = 1;
+        trace->init(tr);
+        /* Sleep for a 1/10 of a second */
+        msleep(100);
+
+        /* we should have nothing in the buffer */
+        ret = trace_test_buffer(tr, &count);
+        if (ret)
+                goto out;
+
+        if (count) {
+                ret = -1;
+                printk(KERN_CONT ".. filter did not filter .. ");
+                goto out;
+        }
+
+        /* call our function again */
+        func();
+
+        /* sleep again */
+        msleep(100);
+
+        /* stop the tracing. */
+        tr->ctrl = 0;
+        trace->ctrl_update(tr);
+        ftrace_enabled = 0;
+
+        /* check the trace buffer */
+        ret = trace_test_buffer(tr, &count);
+        trace->reset(tr);
+
+        /* we should only have one item */
+        if (!ret && count != 1) {
+                printk(KERN_CONT ".. filter failed count=%ld ..", count);
+                ret = -1;
+                goto out;
+        }
+ out:
+        ftrace_enabled = save_ftrace_enabled;
+        tracer_enabled = save_tracer_enabled;
+
+        /* Enable tracing on all functions again */
+        ftrace_set_filter(NULL, 0, 1);
+
+        return ret;
+}
+#else
+# define trace_selftest_startup_dynamic_tracing(trace, tr, func) ({ 0; })
+#endif /* CONFIG_DYNAMIC_FTRACE */
+/*
+ * Simple verification test of ftrace function tracer.
+ * Enable ftrace, sleep 1/10 second, and then read the trace
+ * buffer to see if all is in order.
+ */
+int
+trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr)
+{
+        unsigned long count;
+        int ret;
+        int save_ftrace_enabled = ftrace_enabled;
+        int save_tracer_enabled = tracer_enabled;
+
+        /* make sure msleep has been recorded */
+        msleep(1);
+
+        /* force the recorded functions to be traced */
+        ret = ftrace_force_update();
+        if (ret) {
+                printk(KERN_CONT ".. ftraced failed .. ");
+                return ret;
+        }
+
+        /* start the tracing */
+        ftrace_enabled = 1;
+        tracer_enabled = 1;
+
+        tr->ctrl = 1;
+        trace->init(tr);
+        /* Sleep for a 1/10 of a second */
+        msleep(100);
+        /* stop the tracing. */
+        tr->ctrl = 0;
+        trace->ctrl_update(tr);
+        ftrace_enabled = 0;
+
+        /* check the trace buffer */
+        ret = trace_test_buffer(tr, &count);
+        trace->reset(tr);
+
+        if (!ret && !count) {
+                printk(KERN_CONT ".. no entries found ..");
+                ret = -1;
+                goto out;
+        }
+
+        ret = trace_selftest_startup_dynamic_tracing(trace, tr,
+                                                     DYN_FTRACE_TEST_NAME);
+
+ out:
+        ftrace_enabled = save_ftrace_enabled;
+        tracer_enabled = save_tracer_enabled;
+
+        /* kill ftrace totally if we failed */
+        if (ret)
+                ftrace_kill();
+
+        return ret;
+}
+#endif /* CONFIG_FTRACE */
+
+#ifdef CONFIG_IRQSOFF_TRACER
+int
+trace_selftest_startup_irqsoff(struct tracer *trace, struct trace_array *tr)
+{
+        unsigned long save_max = tracing_max_latency;
+        unsigned long count;
+        int ret;
+
+        /* start the tracing */
+        tr->ctrl = 1;
+        trace->init(tr);
+        /* reset the max latency */
+        tracing_max_latency = 0;
+        /* disable interrupts for a bit */
+        local_irq_disable();
+        udelay(100);
+        local_irq_enable();
+        /* stop the tracing. */
+        tr->ctrl = 0;
+        trace->ctrl_update(tr);
+        /* check both trace buffers */
+        ret = trace_test_buffer(tr, NULL);
+        if (!ret)
+                ret = trace_test_buffer(&max_tr, &count);
+        trace->reset(tr);
+
+        if (!ret && !count) {
+                printk(KERN_CONT ".. no entries found ..");
+                ret = -1;
+        }
+
+        tracing_max_latency = save_max;
+
+        return ret;
+}
+#endif /* CONFIG_IRQSOFF_TRACER */
+
+#ifdef CONFIG_PREEMPT_TRACER
+int
+trace_selftest_startup_preemptoff(struct tracer *trace, struct trace_array *tr)
+{
+        unsigned long save_max = tracing_max_latency;
+        unsigned long count;
+        int ret;
+
+        /* start the tracing */
+        tr->ctrl = 1;
+        trace->init(tr);
+        /* reset the max latency */
+        tracing_max_latency = 0;
+        /* disable preemption for a bit */
+        preempt_disable();
+        udelay(100);
+        preempt_enable();
+        /* stop the tracing. */
+        tr->ctrl = 0;
+        trace->ctrl_update(tr);
+        /* check both trace buffers */
+        ret = trace_test_buffer(tr, NULL);
+        if (!ret)
+                ret = trace_test_buffer(&max_tr, &count);
+        trace->reset(tr);
+
+        if (!ret && !count) {
+                printk(KERN_CONT ".. no entries found ..");
+                ret = -1;
+        }
+
+        tracing_max_latency = save_max;
+
+        return ret;
+}
+#endif /* CONFIG_PREEMPT_TRACER */
+
+#if defined(CONFIG_IRQSOFF_TRACER) && defined(CONFIG_PREEMPT_TRACER)
+int
+trace_selftest_startup_preemptirqsoff(struct tracer *trace, struct trace_array *tr)
+{
+        unsigned long save_max = tracing_max_latency;
+        unsigned long count;
+        int ret;
+
+        /* start the tracing */
+        tr->ctrl = 1;
+        trace->init(tr);
+
+        /* reset the max latency */
+        tracing_max_latency = 0;
+
+        /* disable preemption and interrupts for a bit */
+        preempt_disable();
+        local_irq_disable();
+        udelay(100);
+        preempt_enable();
+        /* reverse the order of preempt vs irqs */
+        local_irq_enable();
+
+        /* stop the tracing. */
+        tr->ctrl = 0;
+        trace->ctrl_update(tr);
+        /* check both trace buffers */
+        ret = trace_test_buffer(tr, NULL);
+        if (ret)
+                goto out;
+
+        ret = trace_test_buffer(&max_tr, &count);
+        if (ret)
+                goto out;
+
+        if (!ret && !count) {
+                printk(KERN_CONT ".. no entries found ..");
+                ret = -1;
+                goto out;
+        }
+
+        /* do the test by disabling interrupts first this time */
+        tracing_max_latency = 0;
+        tr->ctrl = 1;
+        trace->ctrl_update(tr);
+        preempt_disable();
+        local_irq_disable();
+        udelay(100);
+        preempt_enable();
+        /* reverse the order of preempt vs irqs */
+        local_irq_enable();
+
+        /* stop the tracing. */
+        tr->ctrl = 0;
+        trace->ctrl_update(tr);
+        /* check both trace buffers */
+        ret = trace_test_buffer(tr, NULL);
+        if (ret)
+                goto out;
+
+        ret = trace_test_buffer(&max_tr, &count);
+
+        if (!ret && !count) {
+                printk(KERN_CONT ".. no entries found ..");
+                ret = -1;
+                goto out;
+        }
+
+ out:
+        trace->reset(tr);
+        tracing_max_latency = save_max;
+
+        return ret;
+}
+#endif /* CONFIG_IRQSOFF_TRACER && CONFIG_PREEMPT_TRACER */
+
+#ifdef CONFIG_SCHED_TRACER
+static int trace_wakeup_test_thread(void *data)
+{
+        /* Make this a RT thread, doesn't need to be too high */
+        struct sched_param param = { .sched_priority = 5 };
+        struct completion *x = data;
+
+        sched_setscheduler(current, SCHED_FIFO, &param);
+
+        /* Make it know we have a new prio */
+        complete(x);
+
+        /* now go to sleep and let the test wake us up */
+        set_current_state(TASK_INTERRUPTIBLE);
+        schedule();
+
+        /* we are awake, now wait to disappear */
+        while (!kthread_should_stop()) {
+                /*
+                 * This is an RT task, do short sleeps to let
+                 * others run.
+                 */
+                msleep(100);
+        }
+
+        return 0;
+}
+
+int
+trace_selftest_startup_wakeup(struct tracer *trace, struct trace_array *tr)
+{
+        unsigned long save_max = tracing_max_latency;
+        struct task_struct *p;
+        struct completion isrt;
+        unsigned long count;
+        int ret;
+
+        init_completion(&isrt);
+
+        /* create a high prio thread */
+        p = kthread_run(trace_wakeup_test_thread, &isrt, "ftrace-test");
+        if (IS_ERR(p)) {
+                printk(KERN_CONT "Failed to create ftrace wakeup test thread ");
+                return -1;
+        }
+
+        /* make sure the thread is running at an RT prio */
+        wait_for_completion(&isrt);
+
+        /* start the tracing */
+        tr->ctrl = 1;
+        trace->init(tr);
+        /* reset the max latency */
+        tracing_max_latency = 0;
+
+        /* sleep to let the RT thread sleep too */
+        msleep(100);
+
+        /*
+         * Yes this is slightly racy. It is possible that for some
+         * strange reason that the RT thread we created, did not
+         * call schedule for 100ms after doing the completion,
+         * and we do a wakeup on a task that already is awake.
+         * But that is extremely unlikely, and the worst thing that
+         * happens in such a case, is that we disable tracing.
+         * Honestly, if this race does happen something is horrible
+         * wrong with the system.
+         */
+
+        wake_up_process(p);
+
+        /* stop the tracing. */
+        tr->ctrl = 0;
+        trace->ctrl_update(tr);
+        /* check both trace buffers */
+        ret = trace_test_buffer(tr, NULL);
+        if (!ret)
+                ret = trace_test_buffer(&max_tr, &count);
+
+
+        trace->reset(tr);
+
+        tracing_max_latency = save_max;
+
+        /* kill the thread */
+        kthread_stop(p);
+
+        if (!ret && !count) {
+                printk(KERN_CONT ".. no entries found ..");
+                ret = -1;
+        }
+
+        return ret;
+}
+#endif /* CONFIG_SCHED_TRACER */
+
+#ifdef CONFIG_CONTEXT_SWITCH_TRACER
+int
+trace_selftest_startup_sched_switch(struct tracer *trace, struct trace_array *tr)
+{
+        unsigned long count;
+        int ret;
+
+        /* start the tracing */
+        tr->ctrl = 1;
+        trace->init(tr);
+        /* Sleep for a 1/10 of a second */
+        msleep(100);
+        /* stop the tracing. */
+        tr->ctrl = 0;
+        trace->ctrl_update(tr);
+        /* check the trace buffer */
+        ret = trace_test_buffer(tr, &count);
+        trace->reset(tr);
+
+        if (!ret && !count) {
+                printk(KERN_CONT ".. no entries found ..");
+                ret = -1;
+        }
+
+        return ret;
+}
+#endif /* CONFIG_CONTEXT_SWITCH_TRACER */
+
+#ifdef CONFIG_SYSPROF_TRACER
+int
+trace_selftest_startup_sysprof(struct tracer *trace, struct trace_array *tr)
+{
+        unsigned long count;
+        int ret;
+
+        /* start the tracing */
+        tr->ctrl = 1;
+        trace->init(tr);
+        /* Sleep for a 1/10 of a second */
+        msleep(100);
+        /* stop the tracing. */
+        tr->ctrl = 0;
+        trace->ctrl_update(tr);
+        /* check the trace buffer */
+        ret = trace_test_buffer(tr, &count);
+        trace->reset(tr);
+
+        return ret;
+}
+#endif /* CONFIG_SYSPROF_TRACER */
diff --git a/kernel/trace/trace_selftest_dynamic.c b/kernel/trace/trace_selftest_dynamic.c
new file mode 100644
index 000000000000..54dd77cce5bf
--- /dev/null
+++ b/kernel/trace/trace_selftest_dynamic.c
@@ -0,0 +1,7 @@
+#include "trace.h"
+
+int DYN_FTRACE_TEST_NAME(void)
+{
+        /* used to call mcount */
+        return 0;
+}
diff --git a/kernel/trace/trace_sysprof.c b/kernel/trace/trace_sysprof.c
new file mode 100644
index 000000000000..2301e1e7c606
--- /dev/null
+++ b/kernel/trace/trace_sysprof.c
@@ -0,0 +1,363 @@
+/*
+ * trace stack traces
+ *
+ * Copyright (C) 2004-2008, Soeren Sandmann
+ * Copyright (C) 2007 Steven Rostedt <srostedt@redhat.com>
+ * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
+ */
+#include <linux/kallsyms.h>
+#include <linux/debugfs.h>
+#include <linux/hrtimer.h>
+#include <linux/uaccess.h>
+#include <linux/ftrace.h>
+#include <linux/module.h>
+#include <linux/irq.h>
+#include <linux/fs.h>
+
+#include <asm/stacktrace.h>
+
+#include "trace.h"
+
+static struct trace_array       *sysprof_trace;
+static int __read_mostly        tracer_enabled;
+
+/*
+ * 1 msec sample interval by default:
+ */
+static unsigned long sample_period = 1000000;
+static const unsigned int sample_max_depth = 512;
+
+static DEFINE_MUTEX(sample_timer_lock);
+/*
+ * Per CPU hrtimers that do the profiling:
+ */
+static DEFINE_PER_CPU(struct hrtimer, stack_trace_hrtimer);
+
+struct stack_frame {
+        const void __user       *next_fp;
+        unsigned long           return_address;
+};
+
+static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
+{
+        int ret;
+
+        if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
+                return 0;
+
+        ret = 1;
+        pagefault_disable();
+        if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
+                ret = 0;
+        pagefault_enable();
+
+        return ret;
+}
+
+struct backtrace_info {
+        struct trace_array_cpu  *data;
+        struct trace_array      *tr;
+        int                     pos;
+};
+
+static void
+backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
+{
+        /* Ignore warnings */
+}
+
+static void backtrace_warning(void *data, char *msg)
+{
+        /* Ignore warnings */
+}
+
+static int backtrace_stack(void *data, char *name)
+{
+        /* Don't bother with IRQ stacks for now */
+        return -1;
+}
+
+static void backtrace_address(void *data, unsigned long addr, int reliable)
+{
+        struct backtrace_info *info = data;
+
+        if (info->pos < sample_max_depth && reliable) {
+                __trace_special(info->tr, info->data, 1, addr, 0);
+
+                info->pos++;
+        }
+}
+
+const static struct stacktrace_ops backtrace_ops = {
+        .warning                = backtrace_warning,
+        .warning_symbol         = backtrace_warning_symbol,
+        .stack                  = backtrace_stack,
+        .address                = backtrace_address,
+};
+
+static int
+trace_kernel(struct pt_regs *regs, struct trace_array *tr,
+             struct trace_array_cpu *data)
+{
+        struct backtrace_info info;
+        unsigned long bp;
+        char *stack;
+
+        info.tr = tr;
+        info.data = data;
+        info.pos = 1;
+
+        __trace_special(info.tr, info.data, 1, regs->ip, 0);
+
+        stack = ((char *)regs + sizeof(struct pt_regs));
+#ifdef CONFIG_FRAME_POINTER
+        bp = regs->bp;
+#else
+        bp = 0;
+#endif
+
+        dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, &info);
+
+        return info.pos;
+}
+
+static void timer_notify(struct pt_regs *regs, int cpu)
+{
+        struct trace_array_cpu *data;
+        struct stack_frame frame;
+        struct trace_array *tr;
+        const void __user *fp;
+        int is_user;
+        int i;
+
+        if (!regs)
+                return;
+
+        tr = sysprof_trace;
+        data = tr->data[cpu];
+        is_user = user_mode(regs);
+
+        if (!current || current->pid == 0)
+                return;
+
+        if (is_user && current->state != TASK_RUNNING)
+                return;
+
+        __trace_special(tr, data, 0, 0, current->pid);
+
+        if (!is_user)
+                i = trace_kernel(regs, tr, data);
+        else
+                i = 0;
+
+        /*
+         * Trace user stack if we are not a kernel thread
+         */
+        if (current->mm && i < sample_max_depth) {
+                regs = (struct pt_regs *)current->thread.sp0 - 1;
+
+                fp = (void __user *)regs->bp;
+
+                __trace_special(tr, data, 2, regs->ip, 0);
+
+                while (i < sample_max_depth) {
+                        frame.next_fp = 0;
+                        frame.return_address = 0;
+                        if (!copy_stack_frame(fp, &frame))
+                                break;
+                        if ((unsigned long)fp < regs->sp)
+                                break;
+
+                        __trace_special(tr, data, 2, frame.return_address,
+                                        (unsigned long)fp);
+                        fp = frame.next_fp;
+
+                        i++;
+                }
+
+        }
+
+        /*
+         * Special trace entry if we overflow the max depth:
+         */
+        if (i == sample_max_depth)
+                __trace_special(tr, data, -1, -1, -1);
+
+        __trace_special(tr, data, 3, current->pid, i);
+}
+
+static enum hrtimer_restart stack_trace_timer_fn(struct hrtimer *hrtimer)
+{
+        /* trace here */
+        timer_notify(get_irq_regs(), smp_processor_id());
+
+        hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
+
+        return HRTIMER_RESTART;
+}
+
+static void start_stack_timer(int cpu)
+{
+        struct hrtimer *hrtimer = &per_cpu(stack_trace_hrtimer, cpu);
+
+        hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+        hrtimer->function = stack_trace_timer_fn;
+        hrtimer->cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;
+
+        hrtimer_start(hrtimer, ns_to_ktime(sample_period), HRTIMER_MODE_REL);
+}
+
+static void start_stack_timers(void)
+{
+        cpumask_t saved_mask = current->cpus_allowed;
+        int cpu;
+
+        for_each_online_cpu(cpu) {
+                set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu));
+                start_stack_timer(cpu);
+        }
+        set_cpus_allowed_ptr(current, &saved_mask);
+}
+
+static void stop_stack_timer(int cpu)
+{
+        struct hrtimer *hrtimer = &per_cpu(stack_trace_hrtimer, cpu);
+
+        hrtimer_cancel(hrtimer);
+}
+
+static void stop_stack_timers(void)
+{
+        int cpu;
+
+        for_each_online_cpu(cpu)
+                stop_stack_timer(cpu);
+}
+
+static void stack_reset(struct trace_array *tr)
+{
+        int cpu;
+
+        tr->time_start = ftrace_now(tr->cpu);
+
+        for_each_online_cpu(cpu)
+                tracing_reset(tr->data[cpu]);
+}
+
+static void start_stack_trace(struct trace_array *tr)
+{
+        mutex_lock(&sample_timer_lock);
+        stack_reset(tr);
+        start_stack_timers();
+        tracer_enabled = 1;
+        mutex_unlock(&sample_timer_lock);
+}
+
+static void stop_stack_trace(struct trace_array *tr)
+{
+        mutex_lock(&sample_timer_lock);
+        stop_stack_timers();
+        tracer_enabled = 0;
+        mutex_unlock(&sample_timer_lock);
+}
+
+static void stack_trace_init(struct trace_array *tr)
+{
+        sysprof_trace = tr;
+
+        if (tr->ctrl)
+                start_stack_trace(tr);
+}
+
+static void stack_trace_reset(struct trace_array *tr)
+{
+        if (tr->ctrl)
+                stop_stack_trace(tr);
+}
+
+static void stack_trace_ctrl_update(struct trace_array *tr)
+{
+        /* When starting a new trace, reset the buffers */
+        if (tr->ctrl)
+                start_stack_trace(tr);
+        else
+                stop_stack_trace(tr);
+}
+
+static struct tracer stack_trace __read_mostly =
+{
+        .name           = "sysprof",
+        .init           = stack_trace_init,
+        .reset          = stack_trace_reset,
+        .ctrl_update    = stack_trace_ctrl_update,
+#ifdef CONFIG_FTRACE_SELFTEST
+        .selftest    = trace_selftest_startup_sysprof,
+#endif
+};
+
+__init static int init_stack_trace(void)
+{
+        return register_tracer(&stack_trace);
+}
+device_initcall(init_stack_trace);
+
+#define MAX_LONG_DIGITS 22
+
+static ssize_t
+sysprof_sample_read(struct file *filp, char __user *ubuf,
+                    size_t cnt, loff_t *ppos)
+{
+        char buf[MAX_LONG_DIGITS];
+        int r;
+
+        r = sprintf(buf, "%ld\n", nsecs_to_usecs(sample_period));
+
+        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+static ssize_t
+sysprof_sample_write(struct file *filp, const char __user *ubuf,
+                     size_t cnt, loff_t *ppos)
+{
+        char buf[MAX_LONG_DIGITS];
+        unsigned long val;
+
+        if (cnt > MAX_LONG_DIGITS-1)
+                cnt = MAX_LONG_DIGITS-1;
+
+        if (copy_from_user(&buf, ubuf, cnt))
+                return -EFAULT;
+
+        buf[cnt] = 0;
+
+        val = simple_strtoul(buf, NULL, 10);
+        /*
+         * Enforce a minimum sample period of 100 usecs:
+         */
+        if (val < 100)
+                val = 100;
+
+        mutex_lock(&sample_timer_lock);
+        stop_stack_timers();
+        sample_period = val * 1000;
+        start_stack_timers();
+        mutex_unlock(&sample_timer_lock);
+
+        return cnt;
+}
+
+static struct file_operations sysprof_sample_fops = {
+        .read           = sysprof_sample_read,
+        .write          = sysprof_sample_write,
+};
+
+void init_tracer_sysprof_debugfs(struct dentry *d_tracer)
+{
+        struct dentry *entry;
+
+        entry = debugfs_create_file("sysprof_sample_period", 0644,
+                        d_tracer, NULL, &sysprof_sample_fops);
+        if (entry)
+                return;
+        pr_warning("Could not create debugfs 'dyn_ftrace_total_info' entry\n");
+}
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 29fc39f1029c..ce7799540c91 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -13,7 +13,7 @@
  *   Kai Petzke <wpp@marie.physik.tu-berlin.de>
  *   Theodore Ts'o <tytso@mit.edu>
  *
- * Made to use alloc_percpu by Christoph Lameter <clameter@sgi.com>.
+ * Made to use alloc_percpu by Christoph Lameter.
  */
 
 #include <linux/module.h>