56 files changed, 3134 insertions, 1272 deletions

diff --git a/kernel/Makefile b/kernel/Makefile
index 1ce47553fb02..a4d1aa8da9bc 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -6,9 +6,9 @@ obj-y     = fork.o exec_domain.o panic.o \
             cpu.o exit.o itimer.o time.o softirq.o resource.o \
             sysctl.o sysctl_binary.o capability.o ptrace.o timer.o user.o \
             signal.o sys.o kmod.o workqueue.o pid.o task_work.o \
-            rcupdate.o extable.o params.o posix-timers.o \
-            kthread.o wait.o sys_ni.o posix-cpu-timers.o mutex.o \
-            hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
+            extable.o params.o posix-timers.o \
+            kthread.o sys_ni.o posix-cpu-timers.o mutex.o \
+            hrtimer.o rwsem.o nsproxy.o semaphore.o \
             notifier.o ksysfs.o cred.o reboot.o \
             async.o range.o groups.o lglock.o smpboot.o
 
@@ -27,6 +27,7 @@ obj-y += power/
 obj-y += printk/
 obj-y += cpu/
 obj-y += irq/
+obj-y += rcu/
 
 obj-$(CONFIG_CHECKPOINT_RESTORE) += kcmp.o
 obj-$(CONFIG_FREEZER) += freezer.o
@@ -81,12 +82,6 @@ obj-$(CONFIG_KGDB) += debug/
 obj-$(CONFIG_DETECT_HUNG_TASK) += hung_task.o
 obj-$(CONFIG_LOCKUP_DETECTOR) += watchdog.o
 obj-$(CONFIG_SECCOMP) += seccomp.o
-obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
-obj-$(CONFIG_TREE_RCU) += rcutree.o
-obj-$(CONFIG_TREE_PREEMPT_RCU) += rcutree.o
-obj-$(CONFIG_TREE_RCU_TRACE) += rcutree_trace.o
-obj-$(CONFIG_TINY_RCU) += rcutiny.o
-obj-$(CONFIG_TINY_PREEMPT_RCU) += rcutiny.o
 obj-$(CONFIG_RELAY) += relay.o
 obj-$(CONFIG_SYSCTL) += utsname_sysctl.o
 obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
diff --git a/kernel/bounds.c b/kernel/bounds.c
index 0c9b862292b2..e8ca97b5c386 100644
--- a/kernel/bounds.c
+++ b/kernel/bounds.c
@@ -10,6 +10,7 @@
 #include <linux/mmzone.h>
 #include <linux/kbuild.h>
 #include <linux/page_cgroup.h>
+#include <linux/log2.h>
 
 void foo(void)
 {
@@ -17,5 +18,8 @@ void foo(void)
         DEFINE(NR_PAGEFLAGS, __NR_PAGEFLAGS);
         DEFINE(MAX_NR_ZONES, __MAX_NR_ZONES);
         DEFINE(NR_PCG_FLAGS, __NR_PCG_FLAGS);
+#ifdef CONFIG_SMP
+        DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS));
+#endif
         /* End of constants */
 }
diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c
index 859c8dfd78a1..e5f3917aa05b 100644
--- a/kernel/context_tracking.c
+++ b/kernel/context_tracking.c
@@ -120,7 +120,7 @@ void context_tracking_user_enter(void)
  * instead of preempt_schedule() to exit user context if needed before
  * calling the scheduler.
  */
-void __sched notrace preempt_schedule_context(void)
+asmlinkage void __sched notrace preempt_schedule_context(void)
 {
         enum ctx_state prev_ctx;
 
diff --git a/kernel/cpu.c b/kernel/cpu.c
index d7f07a2da5a6..63aa50d7ce1e 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -308,6 +308,23 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
         }
         smpboot_park_threads(cpu);
 
+        /*
+         * By now we've cleared cpu_active_mask, wait for all preempt-disabled
+         * and RCU users of this state to go away such that all new such users
+         * will observe it.
+         *
+         * For CONFIG_PREEMPT we have preemptible RCU and its sync_rcu() might
+         * not imply sync_sched(), so explicitly call both.
+         */
+#ifdef CONFIG_PREEMPT
+        synchronize_sched();
+#endif
+        synchronize_rcu();
+
+        /*
+         * So now all preempt/rcu users must observe !cpu_active().
+         */
+
         err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu));
         if (err) {
                 /* CPU didn't die: tell everyone.  Can't complain. */
diff --git a/kernel/cpu/idle.c b/kernel/cpu/idle.c
index e695c0a0bcb5..988573a9a387 100644
--- a/kernel/cpu/idle.c
+++ b/kernel/cpu/idle.c
@@ -44,7 +44,7 @@ static inline int cpu_idle_poll(void)
         rcu_idle_enter();
         trace_cpu_idle_rcuidle(0, smp_processor_id());
         local_irq_enable();
-        while (!need_resched())
+        while (!tif_need_resched())
                 cpu_relax();
         trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
         rcu_idle_exit();
@@ -92,8 +92,7 @@ static void cpu_idle_loop(void)
                         if (cpu_idle_force_poll || tick_check_broadcast_expired()) {
                                 cpu_idle_poll();
                         } else {
-                                current_clr_polling();
-                                if (!need_resched()) {
+                                if (!current_clr_polling_and_test()) {
                                         stop_critical_timings();
                                         rcu_idle_enter();
                                         arch_cpu_idle();
@@ -103,9 +102,16 @@ static void cpu_idle_loop(void)
                                 } else {
                                         local_irq_enable();
                                 }
-                                current_set_polling();
+                                __current_set_polling();
                         }
                         arch_cpu_idle_exit();
+                        /*
+                         * We need to test and propagate the TIF_NEED_RESCHED
+                         * bit here because we might not have send the
+                         * reschedule IPI to idle tasks.
+                         */
+                        if (tif_need_resched())
+                                set_preempt_need_resched();
                 }
                 tick_nohz_idle_exit();
                 schedule_preempt_disabled();
@@ -129,7 +135,7 @@ void cpu_startup_entry(enum cpuhp_state state)
          */
         boot_init_stack_canary();
 #endif
-        current_set_polling();
+        __current_set_polling();
         arch_cpu_idle_prepare();
         cpu_idle_loop();
 }
diff --git a/kernel/events/core.c b/kernel/events/core.c
index d49a9d29334c..8c875ef6e120 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -175,8 +175,8 @@ int sysctl_perf_event_sample_rate __read_mostly	= DEFAULT_MAX_SAMPLE_RATE;
 static int max_samples_per_tick __read_mostly   = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
 static int perf_sample_period_ns __read_mostly  = DEFAULT_SAMPLE_PERIOD_NS;
 
-static atomic_t perf_sample_allowed_ns __read_mostly =
-        ATOMIC_INIT( DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100);
+static int perf_sample_allowed_ns __read_mostly =
+        DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100;
 
 void update_perf_cpu_limits(void)
 {
@@ -184,7 +184,7 @@ void update_perf_cpu_limits(void)
 
         tmp *= sysctl_perf_cpu_time_max_percent;
         do_div(tmp, 100);
-        atomic_set(&perf_sample_allowed_ns, tmp);
+        ACCESS_ONCE(perf_sample_allowed_ns) = tmp;
 }
 
 static int perf_rotate_context(struct perf_cpu_context *cpuctx);
@@ -193,7 +193,7 @@ int perf_proc_update_handler(struct ctl_table *table, int write,
                 void __user *buffer, size_t *lenp,
                 loff_t *ppos)
 {
-        int ret = proc_dointvec(table, write, buffer, lenp, ppos);
+        int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
 
         if (ret || !write)
                 return ret;
@@ -228,14 +228,15 @@ int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
  * we detect that events are taking too long.
  */
 #define NR_ACCUMULATED_SAMPLES 128
-DEFINE_PER_CPU(u64, running_sample_length);
+static DEFINE_PER_CPU(u64, running_sample_length);
 
 void perf_sample_event_took(u64 sample_len_ns)
 {
         u64 avg_local_sample_len;
         u64 local_samples_len;
+        u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns);
 
-        if (atomic_read(&perf_sample_allowed_ns) == 0)
+        if (allowed_ns == 0)
                 return;
 
         /* decay the counter by 1 average sample */
@@ -251,7 +252,7 @@ void perf_sample_event_took(u64 sample_len_ns)
          */
         avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES;
 
-        if (avg_local_sample_len <= atomic_read(&perf_sample_allowed_ns))
+        if (avg_local_sample_len <= allowed_ns)
                 return;
 
         if (max_samples_per_tick <= 1)
@@ -262,10 +263,9 @@ void perf_sample_event_took(u64 sample_len_ns)
         perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
 
         printk_ratelimited(KERN_WARNING
-                        "perf samples too long (%lld > %d), lowering "
+                        "perf samples too long (%lld > %lld), lowering "
                         "kernel.perf_event_max_sample_rate to %d\n",
-                        avg_local_sample_len,
-                        atomic_read(&perf_sample_allowed_ns),
+                        avg_local_sample_len, allowed_ns,
                         sysctl_perf_event_sample_rate);
 
         update_perf_cpu_limits();
@@ -899,6 +899,7 @@ static void unclone_ctx(struct perf_event_context *ctx)
                 put_ctx(ctx->parent_ctx);
                 ctx->parent_ctx = NULL;
         }
+        ctx->generation++;
 }
 
 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
@@ -1136,6 +1137,8 @@ list_add_event(struct perf_event *event, struct perf_event_context *ctx)
         ctx->nr_events++;
         if (event->attr.inherit_stat)
                 ctx->nr_stat++;
+
+        ctx->generation++;
 }
 
 /*
@@ -1201,6 +1204,9 @@ static void perf_event__header_size(struct perf_event *event)
         if (sample_type & PERF_SAMPLE_DATA_SRC)
                 size += sizeof(data->data_src.val);
 
+        if (sample_type & PERF_SAMPLE_TRANSACTION)
+                size += sizeof(data->txn);
+
         event->header_size = size;
 }
 
@@ -1310,6 +1316,8 @@ list_del_event(struct perf_event *event, struct perf_event_context *ctx)
          */
         if (event->state > PERF_EVENT_STATE_OFF)
                 event->state = PERF_EVENT_STATE_OFF;
+
+        ctx->generation++;
 }
 
 static void perf_group_detach(struct perf_event *event)
@@ -2146,22 +2154,38 @@ static void ctx_sched_out(struct perf_event_context *ctx,
 }
 
 /*
- * Test whether two contexts are equivalent, i.e. whether they
- * have both been cloned from the same version of the same context
- * and they both have the same number of enabled events.
- * If the number of enabled events is the same, then the set
- * of enabled events should be the same, because these are both
- * inherited contexts, therefore we can't access individual events
- * in them directly with an fd; we can only enable/disable all
- * events via prctl, or enable/disable all events in a family
- * via ioctl, which will have the same effect on both contexts.
+ * Test whether two contexts are equivalent, i.e. whether they have both been
+ * cloned from the same version of the same context.
+ *
+ * Equivalence is measured using a generation number in the context that is
+ * incremented on each modification to it; see unclone_ctx(), list_add_event()
+ * and list_del_event().
  */
 static int context_equiv(struct perf_event_context *ctx1,
                          struct perf_event_context *ctx2)
 {
-        return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
-                && ctx1->parent_gen == ctx2->parent_gen
-                && !ctx1->pin_count && !ctx2->pin_count;
+        /* Pinning disables the swap optimization */
+        if (ctx1->pin_count || ctx2->pin_count)
+                return 0;
+
+        /* If ctx1 is the parent of ctx2 */
+        if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)
+                return 1;
+
+        /* If ctx2 is the parent of ctx1 */
+        if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)
+                return 1;
+
+        /*
+         * If ctx1 and ctx2 have the same parent; we flatten the parent
+         * hierarchy, see perf_event_init_context().
+         */
+        if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&
+                        ctx1->parent_gen == ctx2->parent_gen)
+                return 1;
+
+        /* Unmatched */
+        return 0;
 }
 
 static void __perf_event_sync_stat(struct perf_event *event,
@@ -2244,7 +2268,7 @@ static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
 {
         struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
         struct perf_event_context *next_ctx;
-        struct perf_event_context *parent;
+        struct perf_event_context *parent, *next_parent;
         struct perf_cpu_context *cpuctx;
         int do_switch = 1;
 
@@ -2256,10 +2280,18 @@ static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
                 return;
 
         rcu_read_lock();
-        parent = rcu_dereference(ctx->parent_ctx);
         next_ctx = next->perf_event_ctxp[ctxn];
-        if (parent && next_ctx &&
-            rcu_dereference(next_ctx->parent_ctx) == parent) {
+        if (!next_ctx)
+                goto unlock;
+
+        parent = rcu_dereference(ctx->parent_ctx);
+        next_parent = rcu_dereference(next_ctx->parent_ctx);
+
+        /* If neither context have a parent context; they cannot be clones. */
+        if (!parent && !next_parent)
+                goto unlock;
+
+        if (next_parent == ctx || next_ctx == parent || next_parent == parent) {
                 /*
                  * Looks like the two contexts are clones, so we might be
                  * able to optimize the context switch.  We lock both
@@ -2287,6 +2319,7 @@ static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
                 raw_spin_unlock(&next_ctx->lock);
                 raw_spin_unlock(&ctx->lock);
         }
+unlock:
         rcu_read_unlock();
 
         if (do_switch) {
@@ -4572,6 +4605,9 @@ void perf_output_sample(struct perf_output_handle *handle,
         if (sample_type & PERF_SAMPLE_DATA_SRC)
                 perf_output_put(handle, data->data_src.val);
 
+        if (sample_type & PERF_SAMPLE_TRANSACTION)
+                perf_output_put(handle, data->txn);
+
         if (!event->attr.watermark) {
                 int wakeup_events = event->attr.wakeup_events;
 
@@ -5100,27 +5136,26 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
         unsigned int size;
         char tmp[16];
         char *buf = NULL;
-        const char *name;
-
-        memset(tmp, 0, sizeof(tmp));
+        char *name;
 
         if (file) {
                 struct inode *inode;
                 dev_t dev;
+
+                buf = kmalloc(PATH_MAX, GFP_KERNEL);
+                if (!buf) {
+                        name = "//enomem";
+                        goto cpy_name;
+                }
                 /*
-                 * d_path works from the end of the rb backwards, so we
+                 * d_path() works from the end of the rb backwards, so we
                  * need to add enough zero bytes after the string to handle
                  * the 64bit alignment we do later.
                  */
-                buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
-                if (!buf) {
-                        name = strncpy(tmp, "//enomem", sizeof(tmp));
-                        goto got_name;
-                }
-                name = d_path(&file->f_path, buf, PATH_MAX);
+                name = d_path(&file->f_path, buf, PATH_MAX - sizeof(u64));
                 if (IS_ERR(name)) {
-                        name = strncpy(tmp, "//toolong", sizeof(tmp));
-                        goto got_name;
+                        name = "//toolong";
+                        goto cpy_name;
                 }
                 inode = file_inode(vma->vm_file);
                 dev = inode->i_sb->s_dev;
@@ -5128,34 +5163,39 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
                 gen = inode->i_generation;
                 maj = MAJOR(dev);
                 min = MINOR(dev);
-
+                goto got_name;
         } else {
-                if (arch_vma_name(mmap_event->vma)) {
-                        name = strncpy(tmp, arch_vma_name(mmap_event->vma),
-                                       sizeof(tmp) - 1);
-                        tmp[sizeof(tmp) - 1] = '\0';
-                        goto got_name;
-                }
+                name = (char *)arch_vma_name(vma);
+                if (name)
+                        goto cpy_name;
 
-                if (!vma->vm_mm) {
-                        name = strncpy(tmp, "[vdso]", sizeof(tmp));
-                        goto got_name;
-                } else if (vma->vm_start <= vma->vm_mm->start_brk &&
+                if (vma->vm_start <= vma->vm_mm->start_brk &&
                                 vma->vm_end >= vma->vm_mm->brk) {
-                        name = strncpy(tmp, "[heap]", sizeof(tmp));
-                        goto got_name;
-                } else if (vma->vm_start <= vma->vm_mm->start_stack &&
+                        name = "[heap]";
+                        goto cpy_name;
+                }
+                if (vma->vm_start <= vma->vm_mm->start_stack &&
                                 vma->vm_end >= vma->vm_mm->start_stack) {
-                        name = strncpy(tmp, "[stack]", sizeof(tmp));
-                        goto got_name;
+                        name = "[stack]";
+                        goto cpy_name;
                 }
 
-                name = strncpy(tmp, "//anon", sizeof(tmp));
-                goto got_name;
+                name = "//anon";
+                goto cpy_name;
         }
 
+cpy_name:
+        strlcpy(tmp, name, sizeof(tmp));
+        name = tmp;
 got_name:
-        size = ALIGN(strlen(name)+1, sizeof(u64));
+        /*
+         * Since our buffer works in 8 byte units we need to align our string
+         * size to a multiple of 8. However, we must guarantee the tail end is
+         * zero'd out to avoid leaking random bits to userspace.
+         */
+        size = strlen(name)+1;
+        while (!IS_ALIGNED(size, sizeof(u64)))
+                name[size++] = '\0';
 
         mmap_event->file_name = name;
         mmap_event->file_size = size;
@@ -6292,6 +6332,7 @@ type_show(struct device *dev, struct device_attribute *attr, char *page)
 
         return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
 }
+static DEVICE_ATTR_RO(type);
 
 static ssize_t
 perf_event_mux_interval_ms_show(struct device *dev,
@@ -6336,17 +6377,19 @@ perf_event_mux_interval_ms_store(struct device *dev,
 
         return count;
 }
+static DEVICE_ATTR_RW(perf_event_mux_interval_ms);
 
-static struct device_attribute pmu_dev_attrs[] = {
-        __ATTR_RO(type),
-        __ATTR_RW(perf_event_mux_interval_ms),
-        __ATTR_NULL,
+static struct attribute *pmu_dev_attrs[] = {
+        &dev_attr_type.attr,
+        &dev_attr_perf_event_mux_interval_ms.attr,
+        NULL,
 };
+ATTRIBUTE_GROUPS(pmu_dev);
 
 static int pmu_bus_running;
 static struct bus_type pmu_bus = {
         .name           = "event_source",
-        .dev_attrs      = pmu_dev_attrs,
+        .dev_groups     = pmu_dev_groups,
 };
 
 static void pmu_dev_release(struct device *dev)
@@ -6767,6 +6810,10 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr,
         if (ret)
                 return -EFAULT;
 
+        /* disabled for now */
+        if (attr->mmap2)
+                return -EINVAL;
+
         if (attr->__reserved_1)
                 return -EINVAL;
 
@@ -7122,7 +7169,6 @@ SYSCALL_DEFINE5(perf_event_open,
         }
 
         perf_install_in_context(ctx, event, event->cpu);
-        ++ctx->generation;
         perf_unpin_context(ctx);
         mutex_unlock(&ctx->mutex);
 
@@ -7205,7 +7251,6 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
         WARN_ON_ONCE(ctx->parent_ctx);
         mutex_lock(&ctx->mutex);
         perf_install_in_context(ctx, event, cpu);
-        ++ctx->generation;
         perf_unpin_context(ctx);
         mutex_unlock(&ctx->mutex);
 
diff --git a/kernel/events/internal.h b/kernel/events/internal.h
index ca6599723be5..569b218782ad 100644
--- a/kernel/events/internal.h
+++ b/kernel/events/internal.h
@@ -82,16 +82,16 @@ static inline unsigned long perf_data_size(struct ring_buffer *rb)
 }
 
 #define DEFINE_OUTPUT_COPY(func_name, memcpy_func)                      \
-static inline unsigned int                                              \
+static inline unsigned long                                             \
 func_name(struct perf_output_handle *handle,                            \
-          const void *buf, unsigned int len)                            \
+          const void *buf, unsigned long len)                           \
 {                                                                       \
         unsigned long size, written;                                    \
                                                                         \
         do {                                                            \
-                size = min_t(unsigned long, handle->size, len);         \
-                                                                        \
+                size    = min(handle->size, len);                       \
                 written = memcpy_func(handle->addr, buf, size);         \
+                written = size - written;                               \
                                                                         \
                 len -= written;                                         \
                 handle->addr += written;                                \
@@ -110,20 +110,37 @@ func_name(struct perf_output_handle *handle,				\
         return len;                                                     \
 }
 
-static inline int memcpy_common(void *dst, const void *src, size_t n)
+static inline unsigned long
+memcpy_common(void *dst, const void *src, unsigned long n)
 {
         memcpy(dst, src, n);
-        return n;
+        return 0;
 }
 
 DEFINE_OUTPUT_COPY(__output_copy, memcpy_common)
 
-#define MEMCPY_SKIP(dst, src, n) (n)
+static inline unsigned long
+memcpy_skip(void *dst, const void *src, unsigned long n)
+{
+        return 0;
+}
 
-DEFINE_OUTPUT_COPY(__output_skip, MEMCPY_SKIP)
+DEFINE_OUTPUT_COPY(__output_skip, memcpy_skip)
 
 #ifndef arch_perf_out_copy_user
-#define arch_perf_out_copy_user __copy_from_user_inatomic
+#define arch_perf_out_copy_user arch_perf_out_copy_user
+
+static inline unsigned long
+arch_perf_out_copy_user(void *dst, const void *src, unsigned long n)
+{
+        unsigned long ret;
+
+        pagefault_disable();
+        ret = __copy_from_user_inatomic(dst, src, n);
+        pagefault_enable();
+
+        return ret;
+}
 #endif
 
 DEFINE_OUTPUT_COPY(__output_copy_user, arch_perf_out_copy_user)
diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c
index cd55144270b5..e8b168af135b 100644
--- a/kernel/events/ring_buffer.c
+++ b/kernel/events/ring_buffer.c
@@ -12,40 +12,10 @@
 #include <linux/perf_event.h>
 #include <linux/vmalloc.h>
 #include <linux/slab.h>
+#include <linux/circ_buf.h>
 
 #include "internal.h"
 
-static bool perf_output_space(struct ring_buffer *rb, unsigned long tail,
-                              unsigned long offset, unsigned long head)
-{
-        unsigned long sz = perf_data_size(rb);
-        unsigned long mask = sz - 1;
-
-        /*
-         * check if user-writable
-         * overwrite : over-write its own tail
-         * !overwrite: buffer possibly drops events.
-         */
-        if (rb->overwrite)
-                return true;
-
-        /*
-         * verify that payload is not bigger than buffer
-         * otherwise masking logic may fail to detect
-         * the "not enough space" condition
-         */
-        if ((head - offset) > sz)
-                return false;
-
-        offset = (offset - tail) & mask;
-        head   = (head   - tail) & mask;
-
-        if ((int)(head - offset) < 0)
-                return false;
-
-        return true;
-}
-
 static void perf_output_wakeup(struct perf_output_handle *handle)
 {
         atomic_set(&handle->rb->poll, POLL_IN);
@@ -87,15 +57,36 @@ again:
                 goto out;
 
         /*
-         * Publish the known good head. Rely on the full barrier implied
-         * by atomic_dec_and_test() order the rb->head read and this
-         * write.
+         * Since the mmap() consumer (userspace) can run on a different CPU:
+         *
+         *   kernel                             user
+         *
+         *   READ ->data_tail                   READ ->data_head
+         *   smp_mb()   (A)                     smp_rmb()       (C)
+         *   WRITE $data                        READ $data
+         *   smp_wmb()  (B)                     smp_mb()        (D)
+         *   STORE ->data_head                  WRITE ->data_tail
+         *
+         * Where A pairs with D, and B pairs with C.
+         *
+         * I don't think A needs to be a full barrier because we won't in fact
+         * write data until we see the store from userspace. So we simply don't
+         * issue the data WRITE until we observe it. Be conservative for now.
+         *
+         * OTOH, D needs to be a full barrier since it separates the data READ
+         * from the tail WRITE.
+         *
+         * For B a WMB is sufficient since it separates two WRITEs, and for C
+         * an RMB is sufficient since it separates two READs.
+         *
+         * See perf_output_begin().
          */
+        smp_wmb();
         rb->user_page->data_head = head;
 
         /*
-         * Now check if we missed an update, rely on the (compiler)
-         * barrier in atomic_dec_and_test() to re-read rb->head.
+         * Now check if we missed an update -- rely on previous implied
+         * compiler barriers to force a re-read.
          */
         if (unlikely(head != local_read(&rb->head))) {
                 local_inc(&rb->nest);
@@ -114,8 +105,7 @@ int perf_output_begin(struct perf_output_handle *handle,
 {
         struct ring_buffer *rb;
         unsigned long tail, offset, head;
-        int have_lost;
-        struct perf_sample_data sample_data;
+        int have_lost, page_shift;
         struct {
                 struct perf_event_header header;
                 u64                      id;
@@ -130,55 +120,63 @@ int perf_output_begin(struct perf_output_handle *handle,
                 event = event->parent;
 
         rb = rcu_dereference(event->rb);
-        if (!rb)
+        if (unlikely(!rb))
                 goto out;
 
-        handle->rb      = rb;
-        handle->event   = event;
-
-        if (!rb->nr_pages)
+        if (unlikely(!rb->nr_pages))
                 goto out;
 
+        handle->rb    = rb;
+        handle->event = event;
+
         have_lost = local_read(&rb->lost);
-        if (have_lost) {
-                lost_event.header.size = sizeof(lost_event);
-                perf_event_header__init_id(&lost_event.header, &sample_data,
-                                           event);
-                size += lost_event.header.size;
+        if (unlikely(have_lost)) {
+                size += sizeof(lost_event);
+                if (event->attr.sample_id_all)
+                        size += event->id_header_size;
         }
 
         perf_output_get_handle(handle);
 
         do {
-                /*
-                 * Userspace could choose to issue a mb() before updating the
-                 * tail pointer. So that all reads will be completed before the
-                 * write is issued.
-                 */
                 tail = ACCESS_ONCE(rb->user_page->data_tail);
-                smp_rmb();
                 offset = head = local_read(&rb->head);
-                head += size;
-                if (unlikely(!perf_output_space(rb, tail, offset, head)))
+                if (!rb->overwrite &&
+                    unlikely(CIRC_SPACE(head, tail, perf_data_size(rb)) < size))
                         goto fail;
+                head += size;
         } while (local_cmpxchg(&rb->head, offset, head) != offset);
 
-        if (head - local_read(&rb->wakeup) > rb->watermark)
+        /*
+         * Separate the userpage->tail read from the data stores below.
+         * Matches the MB userspace SHOULD issue after reading the data
+         * and before storing the new tail position.
+         *
+         * See perf_output_put_handle().
+         */
+        smp_mb();
+
+        if (unlikely(head - local_read(&rb->wakeup) > rb->watermark))
                 local_add(rb->watermark, &rb->wakeup);
 
-        handle->page = offset >> (PAGE_SHIFT + page_order(rb));
-        handle->page &= rb->nr_pages - 1;
-        handle->size = offset & ((PAGE_SIZE << page_order(rb)) - 1);
-        handle->addr = rb->data_pages[handle->page];
-        handle->addr += handle->size;
-        handle->size = (PAGE_SIZE << page_order(rb)) - handle->size;
+        page_shift = PAGE_SHIFT + page_order(rb);
 
-        if (have_lost) {
+        handle->page = (offset >> page_shift) & (rb->nr_pages - 1);
+        offset &= (1UL << page_shift) - 1;
+        handle->addr = rb->data_pages[handle->page] + offset;
+        handle->size = (1UL << page_shift) - offset;
+
+        if (unlikely(have_lost)) {
+                struct perf_sample_data sample_data;
+
+                lost_event.header.size = sizeof(lost_event);
                 lost_event.header.type = PERF_RECORD_LOST;
                 lost_event.header.misc = 0;
                 lost_event.id          = event->id;
                 lost_event.lost        = local_xchg(&rb->lost, 0);
 
+                perf_event_header__init_id(&lost_event.header,
+                                           &sample_data, event);
                 perf_output_put(handle, lost_event);
                 perf_event__output_id_sample(event, handle, &sample_data);
         }
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index ad8e1bdca70e..24b7d6ca871b 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -35,6 +35,7 @@
 #include <linux/kdebug.h>       /* notifier mechanism */
 #include "../../mm/internal.h"  /* munlock_vma_page */
 #include <linux/percpu-rwsem.h>
+#include <linux/task_work.h>
 
 #include <linux/uprobes.h>
 
@@ -244,12 +245,12 @@ static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t
  * the architecture. If an arch has variable length instruction and the
  * breakpoint instruction is not of the smallest length instruction
  * supported by that architecture then we need to modify is_trap_at_addr and
- * write_opcode accordingly. This would never be a problem for archs that
- * have fixed length instructions.
+ * uprobe_write_opcode accordingly. This would never be a problem for archs
+ * that have fixed length instructions.
  */
 
 /*
- * write_opcode - write the opcode at a given virtual address.
+ * uprobe_write_opcode - write the opcode at a given virtual address.
  * @mm: the probed process address space.
  * @vaddr: the virtual address to store the opcode.
  * @opcode: opcode to be written at @vaddr.
@@ -260,7 +261,7 @@ static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t
  * For mm @mm, write the opcode at @vaddr.
  * Return 0 (success) or a negative errno.
  */
-static int write_opcode(struct mm_struct *mm, unsigned long vaddr,
+int uprobe_write_opcode(struct mm_struct *mm, unsigned long vaddr,
                         uprobe_opcode_t opcode)
 {
         struct page *old_page, *new_page;
@@ -314,7 +315,7 @@ put_old:
  */
 int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
 {
-        return write_opcode(mm, vaddr, UPROBE_SWBP_INSN);
+        return uprobe_write_opcode(mm, vaddr, UPROBE_SWBP_INSN);
 }
 
 /**
@@ -329,7 +330,7 @@ int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned
 int __weak
 set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
 {
-        return write_opcode(mm, vaddr, *(uprobe_opcode_t *)auprobe->insn);
+        return uprobe_write_opcode(mm, vaddr, *(uprobe_opcode_t *)auprobe->insn);
 }
 
 static int match_uprobe(struct uprobe *l, struct uprobe *r)
@@ -503,9 +504,8 @@ static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)
         return ret;
 }
 
-static int
-__copy_insn(struct address_space *mapping, struct file *filp, char *insn,
-                        unsigned long nbytes, loff_t offset)
+static int __copy_insn(struct address_space *mapping, struct file *filp,
+                        void *insn, int nbytes, loff_t offset)
 {
         struct page *page;
 
@@ -527,28 +527,28 @@ __copy_insn(struct address_space *mapping, struct file *filp, char *insn,
 
 static int copy_insn(struct uprobe *uprobe, struct file *filp)
 {
-        struct address_space *mapping;
-        unsigned long nbytes;
-        int bytes;
-
-        nbytes = PAGE_SIZE - (uprobe->offset & ~PAGE_MASK);
-        mapping = uprobe->inode->i_mapping;
+        struct address_space *mapping = uprobe->inode->i_mapping;
+        loff_t offs = uprobe->offset;
+        void *insn = uprobe->arch.insn;
+        int size = MAX_UINSN_BYTES;
+        int len, err = -EIO;
 
-        /* Instruction at end of binary; copy only available bytes */
-        if (uprobe->offset + MAX_UINSN_BYTES > uprobe->inode->i_size)
-                bytes = uprobe->inode->i_size - uprobe->offset;
-        else
-                bytes = MAX_UINSN_BYTES;
+        /* Copy only available bytes, -EIO if nothing was read */
+        do {
+                if (offs >= i_size_read(uprobe->inode))
+                        break;
 
-        /* Instruction at the page-boundary; copy bytes in second page */
-        if (nbytes < bytes) {
-                int err = __copy_insn(mapping, filp, uprobe->arch.insn + nbytes,
-                                bytes - nbytes, uprobe->offset + nbytes);
+                len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK));
+                err = __copy_insn(mapping, filp, insn, len, offs);
                 if (err)
-                        return err;
-                bytes = nbytes;
-        }
-        return __copy_insn(mapping, filp, uprobe->arch.insn, bytes, uprobe->offset);
+                        break;
+
+                insn += len;
+                offs += len;
+                size -= len;
+        } while (size);
+
+        return err;
 }
 
 static int prepare_uprobe(struct uprobe *uprobe, struct file *file,
@@ -576,7 +576,7 @@ static int prepare_uprobe(struct uprobe *uprobe, struct file *file,
         if (ret)
                 goto out;
 
-        /* write_opcode() assumes we don't cross page boundary */
+        /* uprobe_write_opcode() assumes we don't cross page boundary */
         BUG_ON((uprobe->offset & ~PAGE_MASK) +
                         UPROBE_SWBP_INSN_SIZE > PAGE_SIZE);
 
@@ -1096,21 +1096,22 @@ void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned lon
 }
 
 /* Slot allocation for XOL */
-static int xol_add_vma(struct xol_area *area)
+static int xol_add_vma(struct mm_struct *mm, struct xol_area *area)
 {
-        struct mm_struct *mm = current->mm;
         int ret = -EALREADY;
 
         down_write(&mm->mmap_sem);
         if (mm->uprobes_state.xol_area)
                 goto fail;
 
-        ret = -ENOMEM;
-        /* Try to map as high as possible, this is only a hint. */
-        area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0);
-        if (area->vaddr & ~PAGE_MASK) {
-                ret = area->vaddr;
-                goto fail;
+        if (!area->vaddr) {
+                /* Try to map as high as possible, this is only a hint. */
+                area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE,
+                                                PAGE_SIZE, 0, 0);
+                if (area->vaddr & ~PAGE_MASK) {
+                        ret = area->vaddr;
+                        goto fail;
+                }
         }
 
         ret = install_special_mapping(mm, area->vaddr, PAGE_SIZE,
@@ -1120,30 +1121,19 @@ static int xol_add_vma(struct xol_area *area)
 
         smp_wmb();      /* pairs with get_xol_area() */
         mm->uprobes_state.xol_area = area;
-        ret = 0;
  fail:
         up_write(&mm->mmap_sem);
 
         return ret;
 }
 
-/*
- * get_xol_area - Allocate process's xol_area if necessary.
- * This area will be used for storing instructions for execution out of line.
- *
- * Returns the allocated area or NULL.
- */
-static struct xol_area *get_xol_area(void)
+static struct xol_area *__create_xol_area(unsigned long vaddr)
 {
         struct mm_struct *mm = current->mm;
-        struct xol_area *area;
         uprobe_opcode_t insn = UPROBE_SWBP_INSN;
+        struct xol_area *area;
 
-        area = mm->uprobes_state.xol_area;
-        if (area)
-                goto ret;
-
-        area = kzalloc(sizeof(*area), GFP_KERNEL);
+        area = kmalloc(sizeof(*area), GFP_KERNEL);
         if (unlikely(!area))
                 goto out;
 
@@ -1155,13 +1145,14 @@ static struct xol_area *get_xol_area(void)
         if (!area->page)
                 goto free_bitmap;
 
-        /* allocate first slot of task's xol_area for the return probes */
+        area->vaddr = vaddr;
+        init_waitqueue_head(&area->wq);
+        /* Reserve the 1st slot for get_trampoline_vaddr() */
         set_bit(0, area->bitmap);
-        copy_to_page(area->page, 0, &insn, UPROBE_SWBP_INSN_SIZE);
         atomic_set(&area->slot_count, 1);
-        init_waitqueue_head(&area->wq);
+        copy_to_page(area->page, 0, &insn, UPROBE_SWBP_INSN_SIZE);
 
-        if (!xol_add_vma(area))
+        if (!xol_add_vma(mm, area))
                 return area;
 
         __free_page(area->page);
@@ -1170,9 +1161,25 @@ static struct xol_area *get_xol_area(void)
  free_area:
         kfree(area);
  out:
+        return NULL;
+}
+
+/*
+ * get_xol_area - Allocate process's xol_area if necessary.
+ * This area will be used for storing instructions for execution out of line.
+ *
+ * Returns the allocated area or NULL.
+ */
+static struct xol_area *get_xol_area(void)
+{
+        struct mm_struct *mm = current->mm;
+        struct xol_area *area;
+
+        if (!mm->uprobes_state.xol_area)
+                __create_xol_area(0);
+
         area = mm->uprobes_state.xol_area;
- ret:
-        smp_read_barrier_depends();     /* pairs with wmb in xol_add_vma() */
+        smp_read_barrier_depends();     /* pairs with wmb in xol_add_vma() */
         return area;
 }
 
@@ -1256,7 +1263,8 @@ static unsigned long xol_get_insn_slot(struct uprobe *uprobe)
                 return 0;
 
         /* Initialize the slot */
-        copy_to_page(area->page, xol_vaddr, uprobe->arch.insn, MAX_UINSN_BYTES);
+        copy_to_page(area->page, xol_vaddr,
+                        uprobe->arch.ixol, sizeof(uprobe->arch.ixol));
         /*
          * We probably need flush_icache_user_range() but it needs vma.
          * This should work on supported architectures too.
@@ -1345,14 +1353,6 @@ void uprobe_free_utask(struct task_struct *t)
 }
 
 /*
- * Called in context of a new clone/fork from copy_process.
- */
-void uprobe_copy_process(struct task_struct *t)
-{
-        t->utask = NULL;
-}
-
-/*
  * Allocate a uprobe_task object for the task if if necessary.
  * Called when the thread hits a breakpoint.
  *
@@ -1367,6 +1367,90 @@ static struct uprobe_task *get_utask(void)
         return current->utask;
 }
 
+static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask)
+{
+        struct uprobe_task *n_utask;
+        struct return_instance **p, *o, *n;
+
+        n_utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
+        if (!n_utask)
+                return -ENOMEM;
+        t->utask = n_utask;
+
+        p = &n_utask->return_instances;
+        for (o = o_utask->return_instances; o; o = o->next) {
+                n = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
+                if (!n)
+                        return -ENOMEM;
+
+                *n = *o;
+                atomic_inc(&n->uprobe->ref);
+                n->next = NULL;
+
+                *p = n;
+                p = &n->next;
+                n_utask->depth++;
+        }
+
+        return 0;
+}
+
+static void uprobe_warn(struct task_struct *t, const char *msg)
+{
+        pr_warn("uprobe: %s:%d failed to %s\n",
+                        current->comm, current->pid, msg);
+}
+
+static void dup_xol_work(struct callback_head *work)
+{
+        kfree(work);
+
+        if (current->flags & PF_EXITING)
+                return;
+
+        if (!__create_xol_area(current->utask->vaddr))
+                uprobe_warn(current, "dup xol area");
+}
+
+/*
+ * Called in context of a new clone/fork from copy_process.
+ */
+void uprobe_copy_process(struct task_struct *t, unsigned long flags)
+{
+        struct uprobe_task *utask = current->utask;
+        struct mm_struct *mm = current->mm;
+        struct callback_head *work;
+        struct xol_area *area;
+
+        t->utask = NULL;
+
+        if (!utask || !utask->return_instances)
+                return;
+
+        if (mm == t->mm && !(flags & CLONE_VFORK))
+                return;
+
+        if (dup_utask(t, utask))
+                return uprobe_warn(t, "dup ret instances");
+
+        /* The task can fork() after dup_xol_work() fails */
+        area = mm->uprobes_state.xol_area;
+        if (!area)
+                return uprobe_warn(t, "dup xol area");
+
+        if (mm == t->mm)
+                return;
+
+        /* TODO: move it into the union in uprobe_task */
+        work = kmalloc(sizeof(*work), GFP_KERNEL);
+        if (!work)
+                return uprobe_warn(t, "dup xol area");
+
+        t->utask->vaddr = area->vaddr;
+        init_task_work(work, dup_xol_work);
+        task_work_add(t, work, true);
+}
+
 /*
  * Current area->vaddr notion assume the trampoline address is always
  * equal area->vaddr.
@@ -1857,9 +1941,4 @@ static int __init init_uprobes(void)
 
         return register_die_notifier(&uprobe_exception_nb);
 }
-module_init(init_uprobes);
-
-static void __exit exit_uprobes(void)
-{
-}
-module_exit(exit_uprobes);
+__initcall(init_uprobes);
diff --git a/kernel/fork.c b/kernel/fork.c
index 086fe73ad6bd..f6d11fc67f72 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -817,9 +817,6 @@ struct mm_struct *dup_mm(struct task_struct *tsk)
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
         mm->pmd_huge_pte = NULL;
 #endif
-#ifdef CONFIG_NUMA_BALANCING
-        mm->first_nid = NUMA_PTE_SCAN_INIT;
-#endif
         if (!mm_init(mm, tsk))
                 goto fail_nomem;
 
@@ -1313,7 +1310,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
 #endif
 
         /* Perform scheduler related setup. Assign this task to a CPU. */
-        sched_fork(p);
+        sched_fork(clone_flags, p);
 
         retval = perf_event_init_task(p);
         if (retval)
@@ -1373,7 +1370,6 @@ static struct task_struct *copy_process(unsigned long clone_flags,
         INIT_LIST_HEAD(&p->pi_state_list);
         p->pi_state_cache = NULL;
 #endif
-        uprobe_copy_process(p);
         /*
          * sigaltstack should be cleared when sharing the same VM
          */
@@ -1490,6 +1486,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
         perf_event_fork(p);
 
         trace_task_newtask(p, clone_flags);
+        uprobe_copy_process(p, clone_flags);
 
         return p;
 
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 514bcfd855a8..3e59f951d42f 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -956,7 +956,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
                         goto out_mput;
                 }
 
-                sched_setscheduler(t, SCHED_FIFO, &param);
+                sched_setscheduler_nocheck(t, SCHED_FIFO, &param);
 
                 /*
                  * We keep the reference to the task struct even if
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index e16c45b9ee77..4e8e14c34e42 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -4224,7 +4224,7 @@ void lockdep_rcu_suspicious(const char *file, const int line, const char *s)
         printk("\n%srcu_scheduler_active = %d, debug_locks = %d\n",
                !rcu_lockdep_current_cpu_online()
                         ? "RCU used illegally from offline CPU!\n"
-                        : rcu_is_cpu_idle()
+                        : !rcu_is_watching()
                                 ? "RCU used illegally from idle CPU!\n"
                                 : "",
                rcu_scheduler_active, debug_locks);
@@ -4247,7 +4247,7 @@ void lockdep_rcu_suspicious(const char *file, const int line, const char *s)
          * So complain bitterly if someone does call rcu_read_lock(),
          * rcu_read_lock_bh() and so on from extended quiescent states.
          */
-        if (rcu_is_cpu_idle())
+        if (!rcu_is_watching())
                 printk("RCU used illegally from extended quiescent state!\n");
 
         lockdep_print_held_locks(curr);
diff --git a/kernel/mutex.c b/kernel/mutex.c
index 6d647aedffea..d24105b1b794 100644
--- a/kernel/mutex.c
+++ b/kernel/mutex.c
@@ -410,7 +410,7 @@ ww_mutex_set_context_fastpath(struct ww_mutex *lock,
 static __always_inline int __sched
 __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                     struct lockdep_map *nest_lock, unsigned long ip,
-                    struct ww_acquire_ctx *ww_ctx)
+                    struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
 {
         struct task_struct *task = current;
         struct mutex_waiter waiter;
@@ -450,7 +450,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                 struct task_struct *owner;
                 struct mspin_node  node;
 
-                if (!__builtin_constant_p(ww_ctx == NULL) && ww_ctx->acquired > 0) {
+                if (use_ww_ctx && ww_ctx->acquired > 0) {
                         struct ww_mutex *ww;
 
                         ww = container_of(lock, struct ww_mutex, base);
@@ -480,7 +480,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                 if ((atomic_read(&lock->count) == 1) &&
                     (atomic_cmpxchg(&lock->count, 1, 0) == 1)) {
                         lock_acquired(&lock->dep_map, ip);
-                        if (!__builtin_constant_p(ww_ctx == NULL)) {
+                        if (use_ww_ctx) {
                                 struct ww_mutex *ww;
                                 ww = container_of(lock, struct ww_mutex, base);
 
@@ -551,7 +551,7 @@ slowpath:
                         goto err;
                 }
 
-                if (!__builtin_constant_p(ww_ctx == NULL) && ww_ctx->acquired > 0) {
+                if (use_ww_ctx && ww_ctx->acquired > 0) {
                         ret = __mutex_lock_check_stamp(lock, ww_ctx);
                         if (ret)
                                 goto err;
@@ -575,7 +575,7 @@ skip_wait:
         lock_acquired(&lock->dep_map, ip);
         mutex_set_owner(lock);
 
-        if (!__builtin_constant_p(ww_ctx == NULL)) {
+        if (use_ww_ctx) {
                 struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
                 struct mutex_waiter *cur;
 
@@ -615,7 +615,7 @@ mutex_lock_nested(struct mutex *lock, unsigned int subclass)
 {
         might_sleep();
         __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
-                            subclass, NULL, _RET_IP_, NULL);
+                            subclass, NULL, _RET_IP_, NULL, 0);
 }
 
 EXPORT_SYMBOL_GPL(mutex_lock_nested);
@@ -625,7 +625,7 @@ _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
 {
         might_sleep();
         __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
-                            0, nest, _RET_IP_, NULL);
+                            0, nest, _RET_IP_, NULL, 0);
 }
 
 EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock);
@@ -635,7 +635,7 @@ mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass)
 {
         might_sleep();
         return __mutex_lock_common(lock, TASK_KILLABLE,
-                                   subclass, NULL, _RET_IP_, NULL);
+                                   subclass, NULL, _RET_IP_, NULL, 0);
 }
 EXPORT_SYMBOL_GPL(mutex_lock_killable_nested);
 
@@ -644,7 +644,7 @@ mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)
 {
         might_sleep();
         return __mutex_lock_common(lock, TASK_INTERRUPTIBLE,
-                                   subclass, NULL, _RET_IP_, NULL);
+                                   subclass, NULL, _RET_IP_, NULL, 0);
 }
 
 EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
@@ -682,7 +682,7 @@ __ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
 
         might_sleep();
         ret =  __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE,
-                                   0, &ctx->dep_map, _RET_IP_, ctx);
+                                   0, &ctx->dep_map, _RET_IP_, ctx, 1);
         if (!ret && ctx->acquired > 1)
                 return ww_mutex_deadlock_injection(lock, ctx);
 
@@ -697,7 +697,7 @@ __ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
 
         might_sleep();
         ret = __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE,
-                                  0, &ctx->dep_map, _RET_IP_, ctx);
+                                  0, &ctx->dep_map, _RET_IP_, ctx, 1);
 
         if (!ret && ctx->acquired > 1)
                 return ww_mutex_deadlock_injection(lock, ctx);
@@ -809,28 +809,28 @@ __mutex_lock_slowpath(atomic_t *lock_count)
         struct mutex *lock = container_of(lock_count, struct mutex, count);
 
         __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0,
-                            NULL, _RET_IP_, NULL);
+                            NULL, _RET_IP_, NULL, 0);
 }
 
 static noinline int __sched
 __mutex_lock_killable_slowpath(struct mutex *lock)
 {
         return __mutex_lock_common(lock, TASK_KILLABLE, 0,
-                                   NULL, _RET_IP_, NULL);
+                                   NULL, _RET_IP_, NULL, 0);
 }
 
 static noinline int __sched
 __mutex_lock_interruptible_slowpath(struct mutex *lock)
 {
         return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0,
-                                   NULL, _RET_IP_, NULL);
+                                   NULL, _RET_IP_, NULL, 0);
 }
 
 static noinline int __sched
 __ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
 {
         return __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE, 0,
-                                   NULL, _RET_IP_, ctx);
+                                   NULL, _RET_IP_, ctx, 1);
 }
 
 static noinline int __sched
@@ -838,7 +838,7 @@ __ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock,
                                             struct ww_acquire_ctx *ctx)
 {
         return __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE, 0,
-                                   NULL, _RET_IP_, ctx);
+                                   NULL, _RET_IP_, ctx, 1);
 }
 
 #endif
diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c
index c9c759d5a15c..0121dab83f43 100644
--- a/kernel/power/hibernate.c
+++ b/kernel/power/hibernate.c
@@ -846,7 +846,7 @@ static int software_resume(void)
         goto Finish;
 }
 
-late_initcall(software_resume);
+late_initcall_sync(software_resume);
 
 
 static const char * const hibernation_modes[] = {
diff --git a/kernel/rcu/Makefile b/kernel/rcu/Makefile
new file mode 100644
index 000000000000..01e9ec37a3e3
--- /dev/null
+++ b/kernel/rcu/Makefile
@@ -0,0 +1,6 @@
+obj-y += update.o srcu.o
+obj-$(CONFIG_RCU_TORTURE_TEST) += torture.o
+obj-$(CONFIG_TREE_RCU) += tree.o
+obj-$(CONFIG_TREE_PREEMPT_RCU) += tree.o
+obj-$(CONFIG_TREE_RCU_TRACE) += tree_trace.o
+obj-$(CONFIG_TINY_RCU) += tiny.o
diff --git a/kernel/rcu.h b/kernel/rcu/rcu.h
index 77131966c4ad..7859a0a3951e 100644
--- a/kernel/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -122,4 +122,11 @@ int rcu_jiffies_till_stall_check(void);
 
 #endif /* #ifdef CONFIG_RCU_STALL_COMMON */
 
+/*
+ * Strings used in tracepoints need to be exported via the
+ * tracing system such that tools like perf and trace-cmd can
+ * translate the string address pointers to actual text.
+ */
+#define TPS(x)  tracepoint_string(x)
+
 #endif /* __LINUX_RCU_H */
diff --git a/kernel/srcu.c b/kernel/rcu/srcu.c
index 01d5ccb8bfe3..01d5ccb8bfe3 100644
--- a/kernel/srcu.c
+++ b/kernel/rcu/srcu.c
diff --git a/kernel/rcutiny.c b/kernel/rcu/tiny.c
index 9ed6075dc562..0c9a934cfec1 100644
--- a/kernel/rcutiny.c
+++ b/kernel/rcu/tiny.c
@@ -35,6 +35,7 @@
 #include <linux/time.h>
 #include <linux/cpu.h>
 #include <linux/prefetch.h>
+#include <linux/ftrace_event.h>
 
 #ifdef CONFIG_RCU_TRACE
 #include <trace/events/rcu.h>
@@ -42,7 +43,7 @@
 
 #include "rcu.h"
 
-/* Forward declarations for rcutiny_plugin.h. */
+/* Forward declarations for tiny_plugin.h. */
 struct rcu_ctrlblk;
 static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp);
 static void rcu_process_callbacks(struct softirq_action *unused);
@@ -52,22 +53,23 @@ static void __call_rcu(struct rcu_head *head,
 
 static long long rcu_dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
 
-#include "rcutiny_plugin.h"
+#include "tiny_plugin.h"
 
 /* Common code for rcu_idle_enter() and rcu_irq_exit(), see kernel/rcutree.c. */
 static void rcu_idle_enter_common(long long newval)
 {
         if (newval) {
-                RCU_TRACE(trace_rcu_dyntick("--=",
+                RCU_TRACE(trace_rcu_dyntick(TPS("--="),
                                             rcu_dynticks_nesting, newval));
                 rcu_dynticks_nesting = newval;
                 return;
         }
-        RCU_TRACE(trace_rcu_dyntick("Start", rcu_dynticks_nesting, newval));
+        RCU_TRACE(trace_rcu_dyntick(TPS("Start"),
+                                    rcu_dynticks_nesting, newval));
         if (!is_idle_task(current)) {
-                struct task_struct *idle = idle_task(smp_processor_id());
+                struct task_struct *idle __maybe_unused = idle_task(smp_processor_id());
 
-                RCU_TRACE(trace_rcu_dyntick("Error on entry: not idle task",
+                RCU_TRACE(trace_rcu_dyntick(TPS("Entry error: not idle task"),
                                             rcu_dynticks_nesting, newval));
                 ftrace_dump(DUMP_ALL);
                 WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
@@ -120,15 +122,15 @@ EXPORT_SYMBOL_GPL(rcu_irq_exit);
 static void rcu_idle_exit_common(long long oldval)
 {
         if (oldval) {
-                RCU_TRACE(trace_rcu_dyntick("++=",
+                RCU_TRACE(trace_rcu_dyntick(TPS("++="),
                                             oldval, rcu_dynticks_nesting));
                 return;
         }
-        RCU_TRACE(trace_rcu_dyntick("End", oldval, rcu_dynticks_nesting));
+        RCU_TRACE(trace_rcu_dyntick(TPS("End"), oldval, rcu_dynticks_nesting));
         if (!is_idle_task(current)) {
-                struct task_struct *idle = idle_task(smp_processor_id());
+                struct task_struct *idle __maybe_unused = idle_task(smp_processor_id());
 
-                RCU_TRACE(trace_rcu_dyntick("Error on exit: not idle task",
+                RCU_TRACE(trace_rcu_dyntick(TPS("Exit error: not idle task"),
                           oldval, rcu_dynticks_nesting));
                 ftrace_dump(DUMP_ALL);
                 WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
@@ -174,18 +176,18 @@ void rcu_irq_enter(void)
 }
 EXPORT_SYMBOL_GPL(rcu_irq_enter);
 
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
+#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE)
 
 /*
  * Test whether RCU thinks that the current CPU is idle.
  */
-int rcu_is_cpu_idle(void)
+bool __rcu_is_watching(void)
 {
-        return !rcu_dynticks_nesting;
+        return rcu_dynticks_nesting;
 }
-EXPORT_SYMBOL(rcu_is_cpu_idle);
+EXPORT_SYMBOL(__rcu_is_watching);
 
-#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
+#endif /* defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) */
 
 /*
  * Test whether the current CPU was interrupted from idle.  Nested
@@ -273,7 +275,7 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
         if (&rcp->rcucblist == rcp->donetail) {
                 RCU_TRACE(trace_rcu_batch_start(rcp->name, 0, 0, -1));
                 RCU_TRACE(trace_rcu_batch_end(rcp->name, 0,
-                                              ACCESS_ONCE(rcp->rcucblist),
+                                              !!ACCESS_ONCE(rcp->rcucblist),
                                               need_resched(),
                                               is_idle_task(current),
                                               false));
@@ -304,7 +306,8 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
                 RCU_TRACE(cb_count++);
         }
         RCU_TRACE(rcu_trace_sub_qlen(rcp, cb_count));
-        RCU_TRACE(trace_rcu_batch_end(rcp->name, cb_count, 0, need_resched(),
+        RCU_TRACE(trace_rcu_batch_end(rcp->name,
+                                      cb_count, 0, need_resched(),
                                       is_idle_task(current),
                                       false));
 }
diff --git a/kernel/rcutiny_plugin.h b/kernel/rcu/tiny_plugin.h
index 280d06cae352..280d06cae352 100644
--- a/kernel/rcutiny_plugin.h
+++ b/kernel/rcu/tiny_plugin.h
diff --git a/kernel/rcutorture.c b/kernel/rcu/torture.c
index be63101c6175..3929cd451511 100644
--- a/kernel/rcutorture.c
+++ b/kernel/rcu/torture.c
@@ -52,6 +52,12 @@
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@freedesktop.org>");
 
+MODULE_ALIAS("rcutorture");
+#ifdef MODULE_PARAM_PREFIX
+#undef MODULE_PARAM_PREFIX
+#endif
+#define MODULE_PARAM_PREFIX "rcutorture."
+
 static int fqs_duration;
 module_param(fqs_duration, int, 0444);
 MODULE_PARM_DESC(fqs_duration, "Duration of fqs bursts (us), 0 to disable");
diff --git a/kernel/rcutree.c b/kernel/rcu/tree.c
index 32618b3fe4e6..4c06ddfea7cd 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcu/tree.c
@@ -41,6 +41,7 @@
 #include <linux/export.h>
 #include <linux/completion.h>
 #include <linux/moduleparam.h>
+#include <linux/module.h>
 #include <linux/percpu.h>
 #include <linux/notifier.h>
 #include <linux/cpu.h>
@@ -56,17 +57,16 @@
 #include <linux/ftrace_event.h>
 #include <linux/suspend.h>
 
-#include "rcutree.h"
+#include "tree.h"
 #include <trace/events/rcu.h>
 
 #include "rcu.h"
 
-/*
- * Strings used in tracepoints need to be exported via the
- * tracing system such that tools like perf and trace-cmd can
- * translate the string address pointers to actual text.
- */
-#define TPS(x)  tracepoint_string(x)
+MODULE_ALIAS("rcutree");
+#ifdef MODULE_PARAM_PREFIX
+#undef MODULE_PARAM_PREFIX
+#endif
+#define MODULE_PARAM_PREFIX "rcutree."
 
 /* Data structures. */
 
@@ -222,7 +222,7 @@ void rcu_note_context_switch(int cpu)
 }
 EXPORT_SYMBOL_GPL(rcu_note_context_switch);
 
-DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
+static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
         .dynticks_nesting = DYNTICK_TASK_EXIT_IDLE,
         .dynticks = ATOMIC_INIT(1),
 #ifdef CONFIG_NO_HZ_FULL_SYSIDLE
@@ -371,7 +371,8 @@ static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval,
 {
         trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
         if (!user && !is_idle_task(current)) {
-                struct task_struct *idle = idle_task(smp_processor_id());
+                struct task_struct *idle __maybe_unused =
+                        idle_task(smp_processor_id());
 
                 trace_rcu_dyntick(TPS("Error on entry: not idle task"), oldval, 0);
                 ftrace_dump(DUMP_ORIG);
@@ -407,7 +408,7 @@ static void rcu_eqs_enter(bool user)
         long long oldval;
         struct rcu_dynticks *rdtp;
 
-        rdtp = &__get_cpu_var(rcu_dynticks);
+        rdtp = this_cpu_ptr(&rcu_dynticks);
         oldval = rdtp->dynticks_nesting;
         WARN_ON_ONCE((oldval & DYNTICK_TASK_NEST_MASK) == 0);
         if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE)
@@ -435,7 +436,7 @@ void rcu_idle_enter(void)
 
         local_irq_save(flags);
         rcu_eqs_enter(false);
-        rcu_sysidle_enter(&__get_cpu_var(rcu_dynticks), 0);
+        rcu_sysidle_enter(this_cpu_ptr(&rcu_dynticks), 0);
         local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(rcu_idle_enter);
@@ -478,7 +479,7 @@ void rcu_irq_exit(void)
         struct rcu_dynticks *rdtp;
 
         local_irq_save(flags);
-        rdtp = &__get_cpu_var(rcu_dynticks);
+        rdtp = this_cpu_ptr(&rcu_dynticks);
         oldval = rdtp->dynticks_nesting;
         rdtp->dynticks_nesting--;
         WARN_ON_ONCE(rdtp->dynticks_nesting < 0);
@@ -508,7 +509,8 @@ static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval,
         rcu_cleanup_after_idle(smp_processor_id());
         trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
         if (!user && !is_idle_task(current)) {
-                struct task_struct *idle = idle_task(smp_processor_id());
+                struct task_struct *idle __maybe_unused =
+                        idle_task(smp_processor_id());
 
                 trace_rcu_dyntick(TPS("Error on exit: not idle task"),
                                   oldval, rdtp->dynticks_nesting);
@@ -528,7 +530,7 @@ static void rcu_eqs_exit(bool user)
         struct rcu_dynticks *rdtp;
         long long oldval;
 
-        rdtp = &__get_cpu_var(rcu_dynticks);
+        rdtp = this_cpu_ptr(&rcu_dynticks);
         oldval = rdtp->dynticks_nesting;
         WARN_ON_ONCE(oldval < 0);
         if (oldval & DYNTICK_TASK_NEST_MASK)
@@ -555,7 +557,7 @@ void rcu_idle_exit(void)
 
         local_irq_save(flags);
         rcu_eqs_exit(false);
-        rcu_sysidle_exit(&__get_cpu_var(rcu_dynticks), 0);
+        rcu_sysidle_exit(this_cpu_ptr(&rcu_dynticks), 0);
         local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(rcu_idle_exit);
@@ -599,7 +601,7 @@ void rcu_irq_enter(void)
         long long oldval;
 
         local_irq_save(flags);
-        rdtp = &__get_cpu_var(rcu_dynticks);
+        rdtp = this_cpu_ptr(&rcu_dynticks);
         oldval = rdtp->dynticks_nesting;
         rdtp->dynticks_nesting++;
         WARN_ON_ONCE(rdtp->dynticks_nesting == 0);
@@ -620,7 +622,7 @@ void rcu_irq_enter(void)
  */
 void rcu_nmi_enter(void)
 {
-        struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
+        struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
 
         if (rdtp->dynticks_nmi_nesting == 0 &&
             (atomic_read(&rdtp->dynticks) & 0x1))
@@ -642,7 +644,7 @@ void rcu_nmi_enter(void)
  */
 void rcu_nmi_exit(void)
 {
-        struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
+        struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
 
         if (rdtp->dynticks_nmi_nesting == 0 ||
             --rdtp->dynticks_nmi_nesting != 0)
@@ -655,21 +657,34 @@ void rcu_nmi_exit(void)
 }
 
 /**
- * rcu_is_cpu_idle - see if RCU thinks that the current CPU is idle
+ * __rcu_is_watching - are RCU read-side critical sections safe?
+ *
+ * Return true if RCU is watching the running CPU, which means that
+ * this CPU can safely enter RCU read-side critical sections.  Unlike
+ * rcu_is_watching(), the caller of __rcu_is_watching() must have at
+ * least disabled preemption.
+ */
+bool __rcu_is_watching(void)
+{
+        return atomic_read(this_cpu_ptr(&rcu_dynticks.dynticks)) & 0x1;
+}
+
+/**
+ * rcu_is_watching - see if RCU thinks that the current CPU is idle
  *
  * If the current CPU is in its idle loop and is neither in an interrupt
  * or NMI handler, return true.
  */
-int rcu_is_cpu_idle(void)
+bool rcu_is_watching(void)
 {
         int ret;
 
         preempt_disable();
-        ret = (atomic_read(&__get_cpu_var(rcu_dynticks).dynticks) & 0x1) == 0;
+        ret = __rcu_is_watching();
         preempt_enable();
         return ret;
 }
-EXPORT_SYMBOL(rcu_is_cpu_idle);
+EXPORT_SYMBOL_GPL(rcu_is_watching);
 
 #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
 
@@ -703,7 +718,7 @@ bool rcu_lockdep_current_cpu_online(void)
         if (in_nmi())
                 return 1;
         preempt_disable();
-        rdp = &__get_cpu_var(rcu_sched_data);
+        rdp = this_cpu_ptr(&rcu_sched_data);
         rnp = rdp->mynode;
         ret = (rdp->grpmask & rnp->qsmaskinit) ||
               !rcu_scheduler_fully_active;
@@ -723,7 +738,7 @@ EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online);
  */
 static int rcu_is_cpu_rrupt_from_idle(void)
 {
-        return __get_cpu_var(rcu_dynticks).dynticks_nesting <= 1;
+        return __this_cpu_read(rcu_dynticks.dynticks_nesting) <= 1;
 }
 
 /*
@@ -802,8 +817,11 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 
 static void record_gp_stall_check_time(struct rcu_state *rsp)
 {
-        rsp->gp_start = jiffies;
-        rsp->jiffies_stall = jiffies + rcu_jiffies_till_stall_check();
+        unsigned long j = ACCESS_ONCE(jiffies);
+
+        rsp->gp_start = j;
+        smp_wmb(); /* Record start time before stall time. */
+        rsp->jiffies_stall = j + rcu_jiffies_till_stall_check();
 }
 
 /*
@@ -898,6 +916,12 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
         force_quiescent_state(rsp);  /* Kick them all. */
 }
 
+/*
+ * This function really isn't for public consumption, but RCU is special in
+ * that context switches can allow the state machine to make progress.
+ */
+extern void resched_cpu(int cpu);
+
 static void print_cpu_stall(struct rcu_state *rsp)
 {
         int cpu;
@@ -927,22 +951,60 @@ static void print_cpu_stall(struct rcu_state *rsp)
                                      3 * rcu_jiffies_till_stall_check() + 3;
         raw_spin_unlock_irqrestore(&rnp->lock, flags);
 
-        set_need_resched();  /* kick ourselves to get things going. */
+        /*
+         * Attempt to revive the RCU machinery by forcing a context switch.
+         *
+         * A context switch would normally allow the RCU state machine to make
+         * progress and it could be we're stuck in kernel space without context
+         * switches for an entirely unreasonable amount of time.
+         */
+        resched_cpu(smp_processor_id());
 }
 
 static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
 {
+        unsigned long completed;
+        unsigned long gpnum;
+        unsigned long gps;
         unsigned long j;
         unsigned long js;
         struct rcu_node *rnp;
 
-        if (rcu_cpu_stall_suppress)
+        if (rcu_cpu_stall_suppress || !rcu_gp_in_progress(rsp))
                 return;
         j = ACCESS_ONCE(jiffies);
+
+        /*
+         * Lots of memory barriers to reject false positives.
+         *
+         * The idea is to pick up rsp->gpnum, then rsp->jiffies_stall,
+         * then rsp->gp_start, and finally rsp->completed.  These values
+         * are updated in the opposite order with memory barriers (or
+         * equivalent) during grace-period initialization and cleanup.
+         * Now, a false positive can occur if we get an new value of
+         * rsp->gp_start and a old value of rsp->jiffies_stall.  But given
+         * the memory barriers, the only way that this can happen is if one
+         * grace period ends and another starts between these two fetches.
+         * Detect this by comparing rsp->completed with the previous fetch
+         * from rsp->gpnum.
+         *
+         * Given this check, comparisons of jiffies, rsp->jiffies_stall,
+         * and rsp->gp_start suffice to forestall false positives.
+         */
+        gpnum = ACCESS_ONCE(rsp->gpnum);
+        smp_rmb(); /* Pick up ->gpnum first... */
         js = ACCESS_ONCE(rsp->jiffies_stall);
+        smp_rmb(); /* ...then ->jiffies_stall before the rest... */
+        gps = ACCESS_ONCE(rsp->gp_start);
+        smp_rmb(); /* ...and finally ->gp_start before ->completed. */
+        completed = ACCESS_ONCE(rsp->completed);
+        if (ULONG_CMP_GE(completed, gpnum) ||
+            ULONG_CMP_LT(j, js) ||
+            ULONG_CMP_GE(gps, js))
+                return; /* No stall or GP completed since entering function. */
         rnp = rdp->mynode;
         if (rcu_gp_in_progress(rsp) &&
-            (ACCESS_ONCE(rnp->qsmask) & rdp->grpmask) && ULONG_CMP_GE(j, js)) {
+            (ACCESS_ONCE(rnp->qsmask) & rdp->grpmask)) {
 
                 /* We haven't checked in, so go dump stack. */
                 print_cpu_stall(rsp);
@@ -1297,7 +1359,7 @@ static void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp)
 }
 
 /*
- * Initialize a new grace period.
+ * Initialize a new grace period.  Return 0 if no grace period required.
  */
 static int rcu_gp_init(struct rcu_state *rsp)
 {
@@ -1306,18 +1368,27 @@ static int rcu_gp_init(struct rcu_state *rsp)
 
         rcu_bind_gp_kthread();
         raw_spin_lock_irq(&rnp->lock);
+        if (rsp->gp_flags == 0) {
+                /* Spurious wakeup, tell caller to go back to sleep.  */
+                raw_spin_unlock_irq(&rnp->lock);
+                return 0;
+        }
         rsp->gp_flags = 0; /* Clear all flags: New grace period. */
 
-        if (rcu_gp_in_progress(rsp)) {
-                /* Grace period already in progress, don't start another.  */
+        if (WARN_ON_ONCE(rcu_gp_in_progress(rsp))) {
+                /*
+                 * Grace period already in progress, don't start another.
+                 * Not supposed to be able to happen.
+                 */
                 raw_spin_unlock_irq(&rnp->lock);
                 return 0;
         }
 
         /* Advance to a new grace period and initialize state. */
+        record_gp_stall_check_time(rsp);
+        smp_wmb(); /* Record GP times before starting GP. */
         rsp->gpnum++;
         trace_rcu_grace_period(rsp->name, rsp->gpnum, TPS("start"));
-        record_gp_stall_check_time(rsp);
         raw_spin_unlock_irq(&rnp->lock);
 
         /* Exclude any concurrent CPU-hotplug operations. */
@@ -1366,7 +1437,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
 /*
  * Do one round of quiescent-state forcing.
  */
-int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in)
+static int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in)
 {
         int fqs_state = fqs_state_in;
         bool isidle = false;
@@ -1451,8 +1522,12 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
         rsp->fqs_state = RCU_GP_IDLE;
         rdp = this_cpu_ptr(rsp->rda);
         rcu_advance_cbs(rsp, rnp, rdp);  /* Reduce false positives below. */
-        if (cpu_needs_another_gp(rsp, rdp))
-                rsp->gp_flags = 1;
+        if (cpu_needs_another_gp(rsp, rdp)) {
+                rsp->gp_flags = RCU_GP_FLAG_INIT;
+                trace_rcu_grace_period(rsp->name,
+                                       ACCESS_ONCE(rsp->gpnum),
+                                       TPS("newreq"));
+        }
         raw_spin_unlock_irq(&rnp->lock);
 }
 
@@ -1462,6 +1537,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
 static int __noreturn rcu_gp_kthread(void *arg)
 {
         int fqs_state;
+        int gf;
         unsigned long j;
         int ret;
         struct rcu_state *rsp = arg;
@@ -1471,14 +1547,19 @@ static int __noreturn rcu_gp_kthread(void *arg)
 
                 /* Handle grace-period start. */
                 for (;;) {
+                        trace_rcu_grace_period(rsp->name,
+                                               ACCESS_ONCE(rsp->gpnum),
+                                               TPS("reqwait"));
                         wait_event_interruptible(rsp->gp_wq,
-                                                 rsp->gp_flags &
+                                                 ACCESS_ONCE(rsp->gp_flags) &
                                                  RCU_GP_FLAG_INIT);
-                        if ((rsp->gp_flags & RCU_GP_FLAG_INIT) &&
-                            rcu_gp_init(rsp))
+                        if (rcu_gp_init(rsp))
                                 break;
                         cond_resched();
                         flush_signals(current);
+                        trace_rcu_grace_period(rsp->name,
+                                               ACCESS_ONCE(rsp->gpnum),
+                                               TPS("reqwaitsig"));
                 }
 
                 /* Handle quiescent-state forcing. */
@@ -1488,10 +1569,16 @@ static int __noreturn rcu_gp_kthread(void *arg)
                         j = HZ;
                         jiffies_till_first_fqs = HZ;
                 }
+                ret = 0;
                 for (;;) {
-                        rsp->jiffies_force_qs = jiffies + j;
+                        if (!ret)
+                                rsp->jiffies_force_qs = jiffies + j;
+                        trace_rcu_grace_period(rsp->name,
+                                               ACCESS_ONCE(rsp->gpnum),
+                                               TPS("fqswait"));
                         ret = wait_event_interruptible_timeout(rsp->gp_wq,
-                                        (rsp->gp_flags & RCU_GP_FLAG_FQS) ||
+                                        ((gf = ACCESS_ONCE(rsp->gp_flags)) &
+                                         RCU_GP_FLAG_FQS) ||
                                         (!ACCESS_ONCE(rnp->qsmask) &&
                                          !rcu_preempt_blocked_readers_cgp(rnp)),
                                         j);
@@ -1500,13 +1587,23 @@ static int __noreturn rcu_gp_kthread(void *arg)
                             !rcu_preempt_blocked_readers_cgp(rnp))
                                 break;
                         /* If time for quiescent-state forcing, do it. */
-                        if (ret == 0 || (rsp->gp_flags & RCU_GP_FLAG_FQS)) {
+                        if (ULONG_CMP_GE(jiffies, rsp->jiffies_force_qs) ||
+                            (gf & RCU_GP_FLAG_FQS)) {
+                                trace_rcu_grace_period(rsp->name,
+                                                       ACCESS_ONCE(rsp->gpnum),
+                                                       TPS("fqsstart"));
                                 fqs_state = rcu_gp_fqs(rsp, fqs_state);
+                                trace_rcu_grace_period(rsp->name,
+                                                       ACCESS_ONCE(rsp->gpnum),
+                                                       TPS("fqsend"));
                                 cond_resched();
                         } else {
                                 /* Deal with stray signal. */
                                 cond_resched();
                                 flush_signals(current);
+                                trace_rcu_grace_period(rsp->name,
+                                                       ACCESS_ONCE(rsp->gpnum),
+                                                       TPS("fqswaitsig"));
                         }
                         j = jiffies_till_next_fqs;
                         if (j > HZ) {
@@ -1554,6 +1651,8 @@ rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp,
                 return;
         }
         rsp->gp_flags = RCU_GP_FLAG_INIT;
+        trace_rcu_grace_period(rsp->name, ACCESS_ONCE(rsp->gpnum),
+                               TPS("newreq"));
 
         /*
          * We can't do wakeups while holding the rnp->lock, as that
@@ -2255,7 +2354,7 @@ static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp,
          * If called from an extended quiescent state, invoke the RCU
          * core in order to force a re-evaluation of RCU's idleness.
          */
-        if (rcu_is_cpu_idle() && cpu_online(smp_processor_id()))
+        if (!rcu_is_watching() && cpu_online(smp_processor_id()))
                 invoke_rcu_core();
 
         /* If interrupts were disabled or CPU offline, don't invoke RCU core. */
@@ -2725,10 +2824,13 @@ static int rcu_cpu_has_callbacks(int cpu, bool *all_lazy)
 
         for_each_rcu_flavor(rsp) {
                 rdp = per_cpu_ptr(rsp->rda, cpu);
-                if (rdp->qlen != rdp->qlen_lazy)
+                if (!rdp->nxtlist)
+                        continue;
+                hc = true;
+                if (rdp->qlen != rdp->qlen_lazy || !all_lazy) {
                         al = false;
-                if (rdp->nxtlist)
-                        hc = true;
+                        break;
+                }
         }
         if (all_lazy)
                 *all_lazy = al;
@@ -3216,7 +3318,7 @@ static void __init rcu_init_one(struct rcu_state *rsp,
 
 /*
  * Compute the rcu_node tree geometry from kernel parameters.  This cannot
- * replace the definitions in rcutree.h because those are needed to size
+ * replace the definitions in tree.h because those are needed to size
  * the ->node array in the rcu_state structure.
  */
 static void __init rcu_init_geometry(void)
@@ -3295,8 +3397,8 @@ void __init rcu_init(void)
 
         rcu_bootup_announce();
         rcu_init_geometry();
-        rcu_init_one(&rcu_sched_state, &rcu_sched_data);
         rcu_init_one(&rcu_bh_state, &rcu_bh_data);
+        rcu_init_one(&rcu_sched_state, &rcu_sched_data);
         __rcu_init_preempt();
         open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
 
@@ -3311,4 +3413,4 @@ void __init rcu_init(void)
                 rcu_cpu_notify(NULL, CPU_UP_PREPARE, (void *)(long)cpu);
 }
 
-#include "rcutree_plugin.h"
+#include "tree_plugin.h"
diff --git a/kernel/rcutree.h b/kernel/rcu/tree.h
index 5f97eab602cd..52be957c9fe2 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcu/tree.h
@@ -104,6 +104,8 @@ struct rcu_dynticks {
                                     /* idle-period nonlazy_posted snapshot. */
         unsigned long last_accelerate;
                                     /* Last jiffy CBs were accelerated. */
+        unsigned long last_advance_all;
+                                    /* Last jiffy CBs were all advanced. */
         int tick_nohz_enabled_snap; /* Previously seen value from sysfs. */
 #endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */
 };
diff --git a/kernel/rcutree_plugin.h b/kernel/rcu/tree_plugin.h
index 130c97b027f2..3822ac0c4b27 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -28,7 +28,7 @@
 #include <linux/gfp.h>
 #include <linux/oom.h>
 #include <linux/smpboot.h>
-#include "time/tick-internal.h"
+#include "../time/tick-internal.h"
 
 #define RCU_KTHREAD_PRIO 1
 
@@ -96,10 +96,15 @@ static void __init rcu_bootup_announce_oddness(void)
 #endif /* #ifdef CONFIG_RCU_NOCB_CPU_ZERO */
 #ifdef CONFIG_RCU_NOCB_CPU_ALL
         pr_info("\tOffload RCU callbacks from all CPUs\n");
-        cpumask_setall(rcu_nocb_mask);
+        cpumask_copy(rcu_nocb_mask, cpu_possible_mask);
 #endif /* #ifdef CONFIG_RCU_NOCB_CPU_ALL */
 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_NONE */
         if (have_rcu_nocb_mask) {
+                if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
+                        pr_info("\tNote: kernel parameter 'rcu_nocbs=' contains nonexistent CPUs.\n");
+                        cpumask_and(rcu_nocb_mask, cpu_possible_mask,
+                                    rcu_nocb_mask);
+                }
                 cpulist_scnprintf(nocb_buf, sizeof(nocb_buf), rcu_nocb_mask);
                 pr_info("\tOffload RCU callbacks from CPUs: %s.\n", nocb_buf);
                 if (rcu_nocb_poll)
@@ -660,7 +665,7 @@ static void rcu_preempt_check_callbacks(int cpu)
 
 static void rcu_preempt_do_callbacks(void)
 {
-        rcu_do_batch(&rcu_preempt_state, &__get_cpu_var(rcu_preempt_data));
+        rcu_do_batch(&rcu_preempt_state, this_cpu_ptr(&rcu_preempt_data));
 }
 
 #endif /* #ifdef CONFIG_RCU_BOOST */
@@ -1128,7 +1133,7 @@ void exit_rcu(void)
 
 #ifdef CONFIG_RCU_BOOST
 
-#include "rtmutex_common.h"
+#include "../rtmutex_common.h"
 
 #ifdef CONFIG_RCU_TRACE
 
@@ -1332,7 +1337,7 @@ static void invoke_rcu_callbacks_kthread(void)
  */
 static bool rcu_is_callbacks_kthread(void)
 {
-        return __get_cpu_var(rcu_cpu_kthread_task) == current;
+        return __this_cpu_read(rcu_cpu_kthread_task) == current;
 }
 
 #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
@@ -1382,8 +1387,8 @@ static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
 
 static void rcu_kthread_do_work(void)
 {
-        rcu_do_batch(&rcu_sched_state, &__get_cpu_var(rcu_sched_data));
-        rcu_do_batch(&rcu_bh_state, &__get_cpu_var(rcu_bh_data));
+        rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data));
+        rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data));
         rcu_preempt_do_callbacks();
 }
 
@@ -1402,7 +1407,7 @@ static void rcu_cpu_kthread_park(unsigned int cpu)
 
 static int rcu_cpu_kthread_should_run(unsigned int cpu)
 {
-        return __get_cpu_var(rcu_cpu_has_work);
+        return __this_cpu_read(rcu_cpu_has_work);
 }
 
 /*
@@ -1412,8 +1417,8 @@ static int rcu_cpu_kthread_should_run(unsigned int cpu)
  */
 static void rcu_cpu_kthread(unsigned int cpu)
 {
-        unsigned int *statusp = &__get_cpu_var(rcu_cpu_kthread_status);
-        char work, *workp = &__get_cpu_var(rcu_cpu_has_work);
+        unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
+        char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
         int spincnt;
 
         for (spincnt = 0; spincnt < 10; spincnt++) {
@@ -1630,17 +1635,23 @@ module_param(rcu_idle_lazy_gp_delay, int, 0644);
 extern int tick_nohz_enabled;
 
 /*
- * Try to advance callbacks for all flavors of RCU on the current CPU.
- * Afterwards, if there are any callbacks ready for immediate invocation,
- * return true.
+ * Try to advance callbacks for all flavors of RCU on the current CPU, but
+ * only if it has been awhile since the last time we did so.  Afterwards,
+ * if there are any callbacks ready for immediate invocation, return true.
  */
 static bool rcu_try_advance_all_cbs(void)
 {
         bool cbs_ready = false;
         struct rcu_data *rdp;
+        struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
         struct rcu_node *rnp;
         struct rcu_state *rsp;
 
+        /* Exit early if we advanced recently. */
+        if (jiffies == rdtp->last_advance_all)
+                return 0;
+        rdtp->last_advance_all = jiffies;
+
         for_each_rcu_flavor(rsp) {
                 rdp = this_cpu_ptr(rsp->rda);
                 rnp = rdp->mynode;
@@ -1739,6 +1750,8 @@ static void rcu_prepare_for_idle(int cpu)
          */
         if (rdtp->all_lazy &&
             rdtp->nonlazy_posted != rdtp->nonlazy_posted_snap) {
+                rdtp->all_lazy = false;
+                rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
                 invoke_rcu_core();
                 return;
         }
@@ -1768,17 +1781,11 @@ static void rcu_prepare_for_idle(int cpu)
  */
 static void rcu_cleanup_after_idle(int cpu)
 {
-        struct rcu_data *rdp;
-        struct rcu_state *rsp;
 
         if (rcu_is_nocb_cpu(cpu))
                 return;
-        rcu_try_advance_all_cbs();
-        for_each_rcu_flavor(rsp) {
-                rdp = per_cpu_ptr(rsp->rda, cpu);
-                if (cpu_has_callbacks_ready_to_invoke(rdp))
-                        invoke_rcu_core();
-        }
+        if (rcu_try_advance_all_cbs())
+                invoke_rcu_core();
 }
 
 /*
@@ -2108,15 +2115,22 @@ static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
 
         /* If we are not being polled and there is a kthread, awaken it ... */
         t = ACCESS_ONCE(rdp->nocb_kthread);
-        if (rcu_nocb_poll | !t)
+        if (rcu_nocb_poll || !t) {
+                trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
+                                    TPS("WakeNotPoll"));
                 return;
+        }
         len = atomic_long_read(&rdp->nocb_q_count);
         if (old_rhpp == &rdp->nocb_head) {
                 wake_up(&rdp->nocb_wq); /* ... only if queue was empty ... */
                 rdp->qlen_last_fqs_check = 0;
+                trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeEmpty"));
         } else if (len > rdp->qlen_last_fqs_check + qhimark) {
                 wake_up_process(t); /* ... or if many callbacks queued. */
                 rdp->qlen_last_fqs_check = LONG_MAX / 2;
+                trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeOvf"));
+        } else {
+                trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeNot"));
         }
         return;
 }
@@ -2140,10 +2154,12 @@ static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
         if (__is_kfree_rcu_offset((unsigned long)rhp->func))
                 trace_rcu_kfree_callback(rdp->rsp->name, rhp,
                                          (unsigned long)rhp->func,
-                                         rdp->qlen_lazy, rdp->qlen);
+                                         -atomic_long_read(&rdp->nocb_q_count_lazy),
+                                         -atomic_long_read(&rdp->nocb_q_count));
         else
                 trace_rcu_callback(rdp->rsp->name, rhp,
-                                   rdp->qlen_lazy, rdp->qlen);
+                                   -atomic_long_read(&rdp->nocb_q_count_lazy),
+                                   -atomic_long_read(&rdp->nocb_q_count));
         return 1;
 }
 
@@ -2221,6 +2237,7 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp)
 static int rcu_nocb_kthread(void *arg)
 {
         int c, cl;
+        bool firsttime = 1;
         struct rcu_head *list;
         struct rcu_head *next;
         struct rcu_head **tail;
@@ -2229,14 +2246,27 @@ static int rcu_nocb_kthread(void *arg)
         /* Each pass through this loop invokes one batch of callbacks */
         for (;;) {
                 /* If not polling, wait for next batch of callbacks. */
-                if (!rcu_nocb_poll)
+                if (!rcu_nocb_poll) {
+                        trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
+                                            TPS("Sleep"));
                         wait_event_interruptible(rdp->nocb_wq, rdp->nocb_head);
+                } else if (firsttime) {
+                        firsttime = 0;
+                        trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
+                                            TPS("Poll"));
+                }
                 list = ACCESS_ONCE(rdp->nocb_head);
                 if (!list) {
+                        if (!rcu_nocb_poll)
+                                trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
+                                                    TPS("WokeEmpty"));
                         schedule_timeout_interruptible(1);
                         flush_signals(current);
                         continue;
                 }
+                firsttime = 1;
+                trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
+                                    TPS("WokeNonEmpty"));
 
                 /*
                  * Extract queued callbacks, update counts, and wait
@@ -2257,7 +2287,11 @@ static int rcu_nocb_kthread(void *arg)
                         next = list->next;
                         /* Wait for enqueuing to complete, if needed. */
                         while (next == NULL && &list->next != tail) {
+                                trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
+                                                    TPS("WaitQueue"));
                                 schedule_timeout_interruptible(1);
+                                trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
+                                                    TPS("WokeQueue"));
                                 next = list->next;
                         }
                         debug_rcu_head_unqueue(list);
diff --git a/kernel/rcutree_trace.c b/kernel/rcu/tree_trace.c
index cf6c17412932..3596797b7e46 100644
--- a/kernel/rcutree_trace.c
+++ b/kernel/rcu/tree_trace.c
@@ -44,7 +44,7 @@
 #include <linux/seq_file.h>
 
 #define RCU_TREE_NONCORE
-#include "rcutree.h"
+#include "tree.h"
 
 static int r_open(struct inode *inode, struct file *file,
                                         const struct seq_operations *op)
diff --git a/kernel/rcupdate.c b/kernel/rcu/update.c
index b02a339836b4..6cb3dff89e2b 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcu/update.c
@@ -53,6 +53,12 @@
 
 #include "rcu.h"
 
+MODULE_ALIAS("rcupdate");
+#ifdef MODULE_PARAM_PREFIX
+#undef MODULE_PARAM_PREFIX
+#endif
+#define MODULE_PARAM_PREFIX "rcupdate."
+
 module_param(rcu_expedited, int, 0);
 
 #ifdef CONFIG_PREEMPT_RCU
@@ -148,7 +154,7 @@ int rcu_read_lock_bh_held(void)
 {
         if (!debug_lockdep_rcu_enabled())
                 return 1;
-        if (rcu_is_cpu_idle())
+        if (!rcu_is_watching())
                 return 0;
         if (!rcu_lockdep_current_cpu_online())
                 return 0;
@@ -298,7 +304,7 @@ EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read);
 #endif
 
 int rcu_cpu_stall_suppress __read_mostly; /* 1 = suppress stall warnings. */
-int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
+static int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
 
 module_param(rcu_cpu_stall_suppress, int, 0644);
 module_param(rcu_cpu_stall_timeout, int, 0644);
diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile
index 54adcf35f495..7b621409cf15 100644
--- a/kernel/sched/Makefile
+++ b/kernel/sched/Makefile
@@ -12,6 +12,7 @@ CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer
 endif
 
 obj-y += core.o proc.o clock.o cputime.o idle_task.o fair.o rt.o stop_task.o
+obj-y += wait.o completion.o
 obj-$(CONFIG_SMP) += cpupri.o
 obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o
 obj-$(CONFIG_SCHEDSTATS) += stats.o
diff --git a/kernel/sched/completion.c b/kernel/sched/completion.c
new file mode 100644
index 000000000000..a63f4dc27909
--- /dev/null
+++ b/kernel/sched/completion.c
@@ -0,0 +1,299 @@
+/*
+ * Generic wait-for-completion handler;
+ *
+ * It differs from semaphores in that their default case is the opposite,
+ * wait_for_completion default blocks whereas semaphore default non-block. The
+ * interface also makes it easy to 'complete' multiple waiting threads,
+ * something which isn't entirely natural for semaphores.
+ *
+ * But more importantly, the primitive documents the usage. Semaphores would
+ * typically be used for exclusion which gives rise to priority inversion.
+ * Waiting for completion is a typically sync point, but not an exclusion point.
+ */
+
+#include <linux/sched.h>
+#include <linux/completion.h>
+
+/**
+ * complete: - signals a single thread waiting on this completion
+ * @x:  holds the state of this particular completion
+ *
+ * This will wake up a single thread waiting on this completion. Threads will be
+ * awakened in the same order in which they were queued.
+ *
+ * See also complete_all(), wait_for_completion() and related routines.
+ *
+ * It may be assumed that this function implies a write memory barrier before
+ * changing the task state if and only if any tasks are woken up.
+ */
+void complete(struct completion *x)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&x->wait.lock, flags);
+        x->done++;
+        __wake_up_locked(&x->wait, TASK_NORMAL, 1);
+        spin_unlock_irqrestore(&x->wait.lock, flags);
+}
+EXPORT_SYMBOL(complete);
+
+/**
+ * complete_all: - signals all threads waiting on this completion
+ * @x:  holds the state of this particular completion
+ *
+ * This will wake up all threads waiting on this particular completion event.
+ *
+ * It may be assumed that this function implies a write memory barrier before
+ * changing the task state if and only if any tasks are woken up.
+ */
+void complete_all(struct completion *x)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&x->wait.lock, flags);
+        x->done += UINT_MAX/2;
+        __wake_up_locked(&x->wait, TASK_NORMAL, 0);
+        spin_unlock_irqrestore(&x->wait.lock, flags);
+}
+EXPORT_SYMBOL(complete_all);
+
+static inline long __sched
+do_wait_for_common(struct completion *x,
+                   long (*action)(long), long timeout, int state)
+{
+        if (!x->done) {
+                DECLARE_WAITQUEUE(wait, current);
+
+                __add_wait_queue_tail_exclusive(&x->wait, &wait);
+                do {
+                        if (signal_pending_state(state, current)) {
+                                timeout = -ERESTARTSYS;
+                                break;
+                        }
+                        __set_current_state(state);
+                        spin_unlock_irq(&x->wait.lock);
+                        timeout = action(timeout);
+                        spin_lock_irq(&x->wait.lock);
+                } while (!x->done && timeout);
+                __remove_wait_queue(&x->wait, &wait);
+                if (!x->done)
+                        return timeout;
+        }
+        x->done--;
+        return timeout ?: 1;
+}
+
+static inline long __sched
+__wait_for_common(struct completion *x,
+                  long (*action)(long), long timeout, int state)
+{
+        might_sleep();
+
+        spin_lock_irq(&x->wait.lock);
+        timeout = do_wait_for_common(x, action, timeout, state);
+        spin_unlock_irq(&x->wait.lock);
+        return timeout;
+}
+
+static long __sched
+wait_for_common(struct completion *x, long timeout, int state)
+{
+        return __wait_for_common(x, schedule_timeout, timeout, state);
+}
+
+static long __sched
+wait_for_common_io(struct completion *x, long timeout, int state)
+{
+        return __wait_for_common(x, io_schedule_timeout, timeout, state);
+}
+
+/**
+ * wait_for_completion: - waits for completion of a task
+ * @x:  holds the state of this particular completion
+ *
+ * This waits to be signaled for completion of a specific task. It is NOT
+ * interruptible and there is no timeout.
+ *
+ * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
+ * and interrupt capability. Also see complete().
+ */
+void __sched wait_for_completion(struct completion *x)
+{
+        wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
+}
+EXPORT_SYMBOL(wait_for_completion);
+
+/**
+ * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
+ * @x:  holds the state of this particular completion
+ * @timeout:  timeout value in jiffies
+ *
+ * This waits for either a completion of a specific task to be signaled or for a
+ * specified timeout to expire. The timeout is in jiffies. It is not
+ * interruptible.
+ *
+ * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
+ * till timeout) if completed.
+ */
+unsigned long __sched
+wait_for_completion_timeout(struct completion *x, unsigned long timeout)
+{
+        return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
+}
+EXPORT_SYMBOL(wait_for_completion_timeout);
+
+/**
+ * wait_for_completion_io: - waits for completion of a task
+ * @x:  holds the state of this particular completion
+ *
+ * This waits to be signaled for completion of a specific task. It is NOT
+ * interruptible and there is no timeout. The caller is accounted as waiting
+ * for IO.
+ */
+void __sched wait_for_completion_io(struct completion *x)
+{
+        wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
+}
+EXPORT_SYMBOL(wait_for_completion_io);
+
+/**
+ * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
+ * @x:  holds the state of this particular completion
+ * @timeout:  timeout value in jiffies
+ *
+ * This waits for either a completion of a specific task to be signaled or for a
+ * specified timeout to expire. The timeout is in jiffies. It is not
+ * interruptible. The caller is accounted as waiting for IO.
+ *
+ * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
+ * till timeout) if completed.
+ */
+unsigned long __sched
+wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
+{
+        return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
+}
+EXPORT_SYMBOL(wait_for_completion_io_timeout);
+
+/**
+ * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
+ * @x:  holds the state of this particular completion
+ *
+ * This waits for completion of a specific task to be signaled. It is
+ * interruptible.
+ *
+ * Return: -ERESTARTSYS if interrupted, 0 if completed.
+ */
+int __sched wait_for_completion_interruptible(struct completion *x)
+{
+        long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
+        if (t == -ERESTARTSYS)
+                return t;
+        return 0;
+}
+EXPORT_SYMBOL(wait_for_completion_interruptible);
+
+/**
+ * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
+ * @x:  holds the state of this particular completion
+ * @timeout:  timeout value in jiffies
+ *
+ * This waits for either a completion of a specific task to be signaled or for a
+ * specified timeout to expire. It is interruptible. The timeout is in jiffies.
+ *
+ * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
+ * or number of jiffies left till timeout) if completed.
+ */
+long __sched
+wait_for_completion_interruptible_timeout(struct completion *x,
+                                          unsigned long timeout)
+{
+        return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
+}
+EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
+
+/**
+ * wait_for_completion_killable: - waits for completion of a task (killable)
+ * @x:  holds the state of this particular completion
+ *
+ * This waits to be signaled for completion of a specific task. It can be
+ * interrupted by a kill signal.
+ *
+ * Return: -ERESTARTSYS if interrupted, 0 if completed.
+ */
+int __sched wait_for_completion_killable(struct completion *x)
+{
+        long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
+        if (t == -ERESTARTSYS)
+                return t;
+        return 0;
+}
+EXPORT_SYMBOL(wait_for_completion_killable);
+
+/**
+ * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
+ * @x:  holds the state of this particular completion
+ * @timeout:  timeout value in jiffies
+ *
+ * This waits for either a completion of a specific task to be
+ * signaled or for a specified timeout to expire. It can be
+ * interrupted by a kill signal. The timeout is in jiffies.
+ *
+ * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
+ * or number of jiffies left till timeout) if completed.
+ */
+long __sched
+wait_for_completion_killable_timeout(struct completion *x,
+                                     unsigned long timeout)
+{
+        return wait_for_common(x, timeout, TASK_KILLABLE);
+}
+EXPORT_SYMBOL(wait_for_completion_killable_timeout);
+
+/**
+ *      try_wait_for_completion - try to decrement a completion without blocking
+ *      @x:     completion structure
+ *
+ *      Return: 0 if a decrement cannot be done without blocking
+ *               1 if a decrement succeeded.
+ *
+ *      If a completion is being used as a counting completion,
+ *      attempt to decrement the counter without blocking. This
+ *      enables us to avoid waiting if the resource the completion
+ *      is protecting is not available.
+ */
+bool try_wait_for_completion(struct completion *x)
+{
+        unsigned long flags;
+        int ret = 1;
+
+        spin_lock_irqsave(&x->wait.lock, flags);
+        if (!x->done)
+                ret = 0;
+        else
+                x->done--;
+        spin_unlock_irqrestore(&x->wait.lock, flags);
+        return ret;
+}
+EXPORT_SYMBOL(try_wait_for_completion);
+
+/**
+ *      completion_done - Test to see if a completion has any waiters
+ *      @x:     completion structure
+ *
+ *      Return: 0 if there are waiters (wait_for_completion() in progress)
+ *               1 if there are no waiters.
+ *
+ */
+bool completion_done(struct completion *x)
+{
+        unsigned long flags;
+        int ret = 1;
+
+        spin_lock_irqsave(&x->wait.lock, flags);
+        if (!x->done)
+                ret = 0;
+        spin_unlock_irqrestore(&x->wait.lock, flags);
+        return ret;
+}
+EXPORT_SYMBOL(completion_done);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 5ac63c9a995a..1deccd78be98 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -513,12 +513,11 @@ static inline void init_hrtick(void)
  * might also involve a cross-CPU call to trigger the scheduler on
  * the target CPU.
  */
-#ifdef CONFIG_SMP
 void resched_task(struct task_struct *p)
 {
         int cpu;
 
-        assert_raw_spin_locked(&task_rq(p)->lock);
+        lockdep_assert_held(&task_rq(p)->lock);
 
         if (test_tsk_need_resched(p))
                 return;
@@ -526,8 +525,10 @@ void resched_task(struct task_struct *p)
         set_tsk_need_resched(p);
 
         cpu = task_cpu(p);
-        if (cpu == smp_processor_id())
+        if (cpu == smp_processor_id()) {
+                set_preempt_need_resched();
                 return;
+        }
 
         /* NEED_RESCHED must be visible before we test polling */
         smp_mb();
@@ -546,6 +547,7 @@ void resched_cpu(int cpu)
         raw_spin_unlock_irqrestore(&rq->lock, flags);
 }
 
+#ifdef CONFIG_SMP
 #ifdef CONFIG_NO_HZ_COMMON
 /*
  * In the semi idle case, use the nearest busy cpu for migrating timers
@@ -693,12 +695,6 @@ void sched_avg_update(struct rq *rq)
         }
 }
 
-#else /* !CONFIG_SMP */
-void resched_task(struct task_struct *p)
-{
-        assert_raw_spin_locked(&task_rq(p)->lock);
-        set_tsk_need_resched(p);
-}
 #endif /* CONFIG_SMP */
 
 #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
@@ -767,14 +763,14 @@ static void set_load_weight(struct task_struct *p)
 static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
 {
         update_rq_clock(rq);
-        sched_info_queued(p);
+        sched_info_queued(rq, p);
         p->sched_class->enqueue_task(rq, p, flags);
 }
 
 static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
 {
         update_rq_clock(rq);
-        sched_info_dequeued(p);
+        sched_info_dequeued(rq, p);
         p->sched_class->dequeue_task(rq, p, flags);
 }
 
@@ -987,7 +983,7 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
          * ttwu() will sort out the placement.
          */
         WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
-                        !(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE));
+                        !(task_preempt_count(p) & PREEMPT_ACTIVE));
 
 #ifdef CONFIG_LOCKDEP
         /*
@@ -1017,6 +1013,107 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
         __set_task_cpu(p, new_cpu);
 }
 
+static void __migrate_swap_task(struct task_struct *p, int cpu)
+{
+        if (p->on_rq) {
+                struct rq *src_rq, *dst_rq;
+
+                src_rq = task_rq(p);
+                dst_rq = cpu_rq(cpu);
+
+                deactivate_task(src_rq, p, 0);
+                set_task_cpu(p, cpu);
+                activate_task(dst_rq, p, 0);
+                check_preempt_curr(dst_rq, p, 0);
+        } else {
+                /*
+                 * Task isn't running anymore; make it appear like we migrated
+                 * it before it went to sleep. This means on wakeup we make the
+                 * previous cpu our targer instead of where it really is.
+                 */
+                p->wake_cpu = cpu;
+        }
+}
+
+struct migration_swap_arg {
+        struct task_struct *src_task, *dst_task;
+        int src_cpu, dst_cpu;
+};
+
+static int migrate_swap_stop(void *data)
+{
+        struct migration_swap_arg *arg = data;
+        struct rq *src_rq, *dst_rq;
+        int ret = -EAGAIN;
+
+        src_rq = cpu_rq(arg->src_cpu);
+        dst_rq = cpu_rq(arg->dst_cpu);
+
+        double_raw_lock(&arg->src_task->pi_lock,
+                        &arg->dst_task->pi_lock);
+        double_rq_lock(src_rq, dst_rq);
+        if (task_cpu(arg->dst_task) != arg->dst_cpu)
+                goto unlock;
+
+        if (task_cpu(arg->src_task) != arg->src_cpu)
+                goto unlock;
+
+        if (!cpumask_test_cpu(arg->dst_cpu, tsk_cpus_allowed(arg->src_task)))
+                goto unlock;
+
+        if (!cpumask_test_cpu(arg->src_cpu, tsk_cpus_allowed(arg->dst_task)))
+                goto unlock;
+
+        __migrate_swap_task(arg->src_task, arg->dst_cpu);
+        __migrate_swap_task(arg->dst_task, arg->src_cpu);
+
+        ret = 0;
+
+unlock:
+        double_rq_unlock(src_rq, dst_rq);
+        raw_spin_unlock(&arg->dst_task->pi_lock);
+        raw_spin_unlock(&arg->src_task->pi_lock);
+
+        return ret;
+}
+
+/*
+ * Cross migrate two tasks
+ */
+int migrate_swap(struct task_struct *cur, struct task_struct *p)
+{
+        struct migration_swap_arg arg;
+        int ret = -EINVAL;
+
+        arg = (struct migration_swap_arg){
+                .src_task = cur,
+                .src_cpu = task_cpu(cur),
+                .dst_task = p,
+                .dst_cpu = task_cpu(p),
+        };
+
+        if (arg.src_cpu == arg.dst_cpu)
+                goto out;
+
+        /*
+         * These three tests are all lockless; this is OK since all of them
+         * will be re-checked with proper locks held further down the line.
+         */
+        if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
+                goto out;
+
+        if (!cpumask_test_cpu(arg.dst_cpu, tsk_cpus_allowed(arg.src_task)))
+                goto out;
+
+        if (!cpumask_test_cpu(arg.src_cpu, tsk_cpus_allowed(arg.dst_task)))
+                goto out;
+
+        ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg);
+
+out:
+        return ret;
+}
+
 struct migration_arg {
         struct task_struct *task;
         int dest_cpu;
@@ -1236,9 +1333,9 @@ out:
  * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable.
  */
 static inline
-int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags)
+int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
 {
-        int cpu = p->sched_class->select_task_rq(p, sd_flags, wake_flags);
+        cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
 
         /*
          * In order not to call set_task_cpu() on a blocking task we need
@@ -1330,12 +1427,13 @@ ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
 
         if (rq->idle_stamp) {
                 u64 delta = rq_clock(rq) - rq->idle_stamp;
-                u64 max = 2*sysctl_sched_migration_cost;
+                u64 max = 2*rq->max_idle_balance_cost;
+
+                update_avg(&rq->avg_idle, delta);
 
-                if (delta > max)
+                if (rq->avg_idle > max)
                         rq->avg_idle = max;
-                else
-                        update_avg(&rq->avg_idle, delta);
+
                 rq->idle_stamp = 0;
         }
 #endif
@@ -1396,6 +1494,14 @@ static void sched_ttwu_pending(void)
 
 void scheduler_ipi(void)
 {
+        /*
+         * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
+         * TIF_NEED_RESCHED remotely (for the first time) will also send
+         * this IPI.
+         */
+        if (tif_need_resched())
+                set_preempt_need_resched();
+
         if (llist_empty(&this_rq()->wake_list)
                         && !tick_nohz_full_cpu(smp_processor_id())
                         && !got_nohz_idle_kick())
@@ -1513,7 +1619,7 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
         if (p->sched_class->task_waking)
                 p->sched_class->task_waking(p);
 
-        cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags);
+        cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);
         if (task_cpu(p) != cpu) {
                 wake_flags |= WF_MIGRATED;
                 set_task_cpu(p, cpu);
@@ -1595,7 +1701,7 @@ int wake_up_state(struct task_struct *p, unsigned int state)
  *
  * __sched_fork() is basic setup used by init_idle() too:
  */
-static void __sched_fork(struct task_struct *p)
+static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
 {
         p->on_rq                        = 0;
 
@@ -1619,16 +1725,24 @@ static void __sched_fork(struct task_struct *p)
 
 #ifdef CONFIG_NUMA_BALANCING
         if (p->mm && atomic_read(&p->mm->mm_users) == 1) {
-                p->mm->numa_next_scan = jiffies;
-                p->mm->numa_next_reset = jiffies;
+                p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
                 p->mm->numa_scan_seq = 0;
         }
 
+        if (clone_flags & CLONE_VM)
+                p->numa_preferred_nid = current->numa_preferred_nid;
+        else
+                p->numa_preferred_nid = -1;
+
         p->node_stamp = 0ULL;
         p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0;
-        p->numa_migrate_seq = p->mm ? p->mm->numa_scan_seq - 1 : 0;
         p->numa_scan_period = sysctl_numa_balancing_scan_delay;
         p->numa_work.next = &p->numa_work;
+        p->numa_faults = NULL;
+        p->numa_faults_buffer = NULL;
+
+        INIT_LIST_HEAD(&p->numa_entry);
+        p->numa_group = NULL;
 #endif /* CONFIG_NUMA_BALANCING */
 }
 
@@ -1654,12 +1768,12 @@ void set_numabalancing_state(bool enabled)
 /*
  * fork()/clone()-time setup:
  */
-void sched_fork(struct task_struct *p)
+void sched_fork(unsigned long clone_flags, struct task_struct *p)
 {
         unsigned long flags;
         int cpu = get_cpu();
 
-        __sched_fork(p);
+        __sched_fork(clone_flags, p);
         /*
          * We mark the process as running here. This guarantees that
          * nobody will actually run it, and a signal or other external
@@ -1717,10 +1831,7 @@ void sched_fork(struct task_struct *p)
 #if defined(CONFIG_SMP)
         p->on_cpu = 0;
 #endif
-#ifdef CONFIG_PREEMPT_COUNT
-        /* Want to start with kernel preemption disabled. */
-        task_thread_info(p)->preempt_count = 1;
-#endif
+        init_task_preempt_count(p);
 #ifdef CONFIG_SMP
         plist_node_init(&p->pushable_tasks, MAX_PRIO);
 #endif
@@ -1747,7 +1858,7 @@ void wake_up_new_task(struct task_struct *p)
          *  - cpus_allowed can change in the fork path
          *  - any previously selected cpu might disappear through hotplug
          */
-        set_task_cpu(p, select_task_rq(p, SD_BALANCE_FORK, 0));
+        set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0));
 #endif
 
         /* Initialize new task's runnable average */
@@ -1838,7 +1949,7 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev,
                     struct task_struct *next)
 {
         trace_sched_switch(prev, next);
-        sched_info_switch(prev, next);
+        sched_info_switch(rq, prev, next);
         perf_event_task_sched_out(prev, next);
         fire_sched_out_preempt_notifiers(prev, next);
         prepare_lock_switch(rq, next);
@@ -1890,6 +2001,8 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
         if (mm)
                 mmdrop(mm);
         if (unlikely(prev_state == TASK_DEAD)) {
+                task_numa_free(prev);
+
                 /*
                  * Remove function-return probe instances associated with this
                  * task and put them back on the free list.
@@ -2073,7 +2186,7 @@ void sched_exec(void)
         int dest_cpu;
 
         raw_spin_lock_irqsave(&p->pi_lock, flags);
-        dest_cpu = p->sched_class->select_task_rq(p, SD_BALANCE_EXEC, 0);
+        dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0);
         if (dest_cpu == smp_processor_id())
                 goto unlock;
 
@@ -2215,7 +2328,7 @@ notrace unsigned long get_parent_ip(unsigned long addr)
 #if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
                                 defined(CONFIG_PREEMPT_TRACER))
 
-void __kprobes add_preempt_count(int val)
+void __kprobes preempt_count_add(int val)
 {
 #ifdef CONFIG_DEBUG_PREEMPT
         /*
@@ -2224,7 +2337,7 @@ void __kprobes add_preempt_count(int val)
         if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
                 return;
 #endif
-        preempt_count() += val;
+        __preempt_count_add(val);
 #ifdef CONFIG_DEBUG_PREEMPT
         /*
          * Spinlock count overflowing soon?
@@ -2235,9 +2348,9 @@ void __kprobes add_preempt_count(int val)
         if (preempt_count() == val)
                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 }
-EXPORT_SYMBOL(add_preempt_count);
+EXPORT_SYMBOL(preempt_count_add);
 
-void __kprobes sub_preempt_count(int val)
+void __kprobes preempt_count_sub(int val)
 {
 #ifdef CONFIG_DEBUG_PREEMPT
         /*
@@ -2255,9 +2368,9 @@ void __kprobes sub_preempt_count(int val)
 
         if (preempt_count() == val)
                 trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
-        preempt_count() -= val;
+        __preempt_count_sub(val);
 }
-EXPORT_SYMBOL(sub_preempt_count);
+EXPORT_SYMBOL(preempt_count_sub);
 
 #endif
 
@@ -2430,6 +2543,7 @@ need_resched:
         put_prev_task(rq, prev);
         next = pick_next_task(rq);
         clear_tsk_need_resched(prev);
+        clear_preempt_need_resched();
         rq->skip_clock_update = 0;
 
         if (likely(prev != next)) {
@@ -2520,9 +2634,9 @@ asmlinkage void __sched notrace preempt_schedule(void)
                 return;
 
         do {
-                add_preempt_count_notrace(PREEMPT_ACTIVE);
+                __preempt_count_add(PREEMPT_ACTIVE);
                 __schedule();
-                sub_preempt_count_notrace(PREEMPT_ACTIVE);
+                __preempt_count_sub(PREEMPT_ACTIVE);
 
                 /*
                  * Check again in case we missed a preemption opportunity
@@ -2541,20 +2655,19 @@ EXPORT_SYMBOL(preempt_schedule);
  */
 asmlinkage void __sched preempt_schedule_irq(void)
 {
-        struct thread_info *ti = current_thread_info();
         enum ctx_state prev_state;
 
         /* Catch callers which need to be fixed */
-        BUG_ON(ti->preempt_count || !irqs_disabled());
+        BUG_ON(preempt_count() || !irqs_disabled());
 
         prev_state = exception_enter();
 
         do {
-                add_preempt_count(PREEMPT_ACTIVE);
+                __preempt_count_add(PREEMPT_ACTIVE);
                 local_irq_enable();
                 __schedule();
                 local_irq_disable();
-                sub_preempt_count(PREEMPT_ACTIVE);
+                __preempt_count_sub(PREEMPT_ACTIVE);
 
                 /*
                  * Check again in case we missed a preemption opportunity
@@ -2575,393 +2688,6 @@ int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
 }
 EXPORT_SYMBOL(default_wake_function);
 
-/*
- * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
- * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
- * number) then we wake all the non-exclusive tasks and one exclusive task.
- *
- * There are circumstances in which we can try to wake a task which has already
- * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
- * zero in this (rare) case, and we handle it by continuing to scan the queue.
- */
-static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
-                        int nr_exclusive, int wake_flags, void *key)
-{
-        wait_queue_t *curr, *next;
-
-        list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
-                unsigned flags = curr->flags;
-
-                if (curr->func(curr, mode, wake_flags, key) &&
-                                (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
-                        break;
-        }
-}
-
-/**
- * __wake_up - wake up threads blocked on a waitqueue.
- * @q: the waitqueue
- * @mode: which threads
- * @nr_exclusive: how many wake-one or wake-many threads to wake up
- * @key: is directly passed to the wakeup function
- *
- * It may be assumed that this function implies a write memory barrier before
- * changing the task state if and only if any tasks are woken up.
- */
-void __wake_up(wait_queue_head_t *q, unsigned int mode,
-                        int nr_exclusive, void *key)
-{
-        unsigned long flags;
-
-        spin_lock_irqsave(&q->lock, flags);
-        __wake_up_common(q, mode, nr_exclusive, 0, key);
-        spin_unlock_irqrestore(&q->lock, flags);
-}
-EXPORT_SYMBOL(__wake_up);
-
-/*
- * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
- */
-void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr)
-{
-        __wake_up_common(q, mode, nr, 0, NULL);
-}
-EXPORT_SYMBOL_GPL(__wake_up_locked);
-
-void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key)
-{
-        __wake_up_common(q, mode, 1, 0, key);
-}
-EXPORT_SYMBOL_GPL(__wake_up_locked_key);
-
-/**
- * __wake_up_sync_key - wake up threads blocked on a waitqueue.
- * @q: the waitqueue
- * @mode: which threads
- * @nr_exclusive: how many wake-one or wake-many threads to wake up
- * @key: opaque value to be passed to wakeup targets
- *
- * The sync wakeup differs that the waker knows that it will schedule
- * away soon, so while the target thread will be woken up, it will not
- * be migrated to another CPU - ie. the two threads are 'synchronized'
- * with each other. This can prevent needless bouncing between CPUs.
- *
- * On UP it can prevent extra preemption.
- *
- * It may be assumed that this function implies a write memory barrier before
- * changing the task state if and only if any tasks are woken up.
- */
-void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode,
-                        int nr_exclusive, void *key)
-{
-        unsigned long flags;
-        int wake_flags = WF_SYNC;
-
-        if (unlikely(!q))
-                return;
-
-        if (unlikely(nr_exclusive != 1))
-                wake_flags = 0;
-
-        spin_lock_irqsave(&q->lock, flags);
-        __wake_up_common(q, mode, nr_exclusive, wake_flags, key);
-        spin_unlock_irqrestore(&q->lock, flags);
-}
-EXPORT_SYMBOL_GPL(__wake_up_sync_key);
-
-/*
- * __wake_up_sync - see __wake_up_sync_key()
- */
-void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
-{
-        __wake_up_sync_key(q, mode, nr_exclusive, NULL);
-}
-EXPORT_SYMBOL_GPL(__wake_up_sync);      /* For internal use only */
-
-/**
- * complete: - signals a single thread waiting on this completion
- * @x:  holds the state of this particular completion
- *
- * This will wake up a single thread waiting on this completion. Threads will be
- * awakened in the same order in which they were queued.
- *
- * See also complete_all(), wait_for_completion() and related routines.
- *
- * It may be assumed that this function implies a write memory barrier before
- * changing the task state if and only if any tasks are woken up.
- */
-void complete(struct completion *x)
-{
-        unsigned long flags;
-
-        spin_lock_irqsave(&x->wait.lock, flags);
-        x->done++;
-        __wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL);
-        spin_unlock_irqrestore(&x->wait.lock, flags);
-}
-EXPORT_SYMBOL(complete);
-
-/**
- * complete_all: - signals all threads waiting on this completion
- * @x:  holds the state of this particular completion
- *
- * This will wake up all threads waiting on this particular completion event.
- *
- * It may be assumed that this function implies a write memory barrier before
- * changing the task state if and only if any tasks are woken up.
- */
-void complete_all(struct completion *x)
-{
-        unsigned long flags;
-
-        spin_lock_irqsave(&x->wait.lock, flags);
-        x->done += UINT_MAX/2;
-        __wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL);
-        spin_unlock_irqrestore(&x->wait.lock, flags);
-}
-EXPORT_SYMBOL(complete_all);
-
-static inline long __sched
-do_wait_for_common(struct completion *x,
-                   long (*action)(long), long timeout, int state)
-{
-        if (!x->done) {
-                DECLARE_WAITQUEUE(wait, current);
-
-                __add_wait_queue_tail_exclusive(&x->wait, &wait);
-                do {
-                        if (signal_pending_state(state, current)) {
-                                timeout = -ERESTARTSYS;
-                                break;
-                        }
-                        __set_current_state(state);
-                        spin_unlock_irq(&x->wait.lock);
-                        timeout = action(timeout);
-                        spin_lock_irq(&x->wait.lock);
-                } while (!x->done && timeout);
-                __remove_wait_queue(&x->wait, &wait);
-                if (!x->done)
-                        return timeout;
-        }
-        x->done--;
-        return timeout ?: 1;
-}
-
-static inline long __sched
-__wait_for_common(struct completion *x,
-                  long (*action)(long), long timeout, int state)
-{
-        might_sleep();
-
-        spin_lock_irq(&x->wait.lock);
-        timeout = do_wait_for_common(x, action, timeout, state);
-        spin_unlock_irq(&x->wait.lock);
-        return timeout;
-}
-
-static long __sched
-wait_for_common(struct completion *x, long timeout, int state)
-{
-        return __wait_for_common(x, schedule_timeout, timeout, state);
-}
-
-static long __sched
-wait_for_common_io(struct completion *x, long timeout, int state)
-{
-        return __wait_for_common(x, io_schedule_timeout, timeout, state);
-}
-
-/**
- * wait_for_completion: - waits for completion of a task
- * @x:  holds the state of this particular completion
- *
- * This waits to be signaled for completion of a specific task. It is NOT
- * interruptible and there is no timeout.
- *
- * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
- * and interrupt capability. Also see complete().
- */
-void __sched wait_for_completion(struct completion *x)
-{
-        wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
-}
-EXPORT_SYMBOL(wait_for_completion);
-
-/**
- * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
- * @x:  holds the state of this particular completion
- * @timeout:  timeout value in jiffies
- *
- * This waits for either a completion of a specific task to be signaled or for a
- * specified timeout to expire. The timeout is in jiffies. It is not
- * interruptible.
- *
- * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
- * till timeout) if completed.
- */
-unsigned long __sched
-wait_for_completion_timeout(struct completion *x, unsigned long timeout)
-{
-        return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
-}
-EXPORT_SYMBOL(wait_for_completion_timeout);
-
-/**
- * wait_for_completion_io: - waits for completion of a task
- * @x:  holds the state of this particular completion
- *
- * This waits to be signaled for completion of a specific task. It is NOT
- * interruptible and there is no timeout. The caller is accounted as waiting
- * for IO.
- */
-void __sched wait_for_completion_io(struct completion *x)
-{
-        wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
-}
-EXPORT_SYMBOL(wait_for_completion_io);
-
-/**
- * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
- * @x:  holds the state of this particular completion
- * @timeout:  timeout value in jiffies
- *
- * This waits for either a completion of a specific task to be signaled or for a
- * specified timeout to expire. The timeout is in jiffies. It is not
- * interruptible. The caller is accounted as waiting for IO.
- *
- * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
- * till timeout) if completed.
- */
-unsigned long __sched
-wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
-{
-        return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
-}
-EXPORT_SYMBOL(wait_for_completion_io_timeout);
-
-/**
- * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
- * @x:  holds the state of this particular completion
- *
- * This waits for completion of a specific task to be signaled. It is
- * interruptible.
- *
- * Return: -ERESTARTSYS if interrupted, 0 if completed.
- */
-int __sched wait_for_completion_interruptible(struct completion *x)
-{
-        long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
-        if (t == -ERESTARTSYS)
-                return t;
-        return 0;
-}
-EXPORT_SYMBOL(wait_for_completion_interruptible);
-
-/**
- * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
- * @x:  holds the state of this particular completion
- * @timeout:  timeout value in jiffies
- *
- * This waits for either a completion of a specific task to be signaled or for a
- * specified timeout to expire. It is interruptible. The timeout is in jiffies.
- *
- * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
- * or number of jiffies left till timeout) if completed.
- */
-long __sched
-wait_for_completion_interruptible_timeout(struct completion *x,
-                                          unsigned long timeout)
-{
-        return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
-}
-EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
-
-/**
- * wait_for_completion_killable: - waits for completion of a task (killable)
- * @x:  holds the state of this particular completion
- *
- * This waits to be signaled for completion of a specific task. It can be
- * interrupted by a kill signal.
- *
- * Return: -ERESTARTSYS if interrupted, 0 if completed.
- */
-int __sched wait_for_completion_killable(struct completion *x)
-{
-        long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
-        if (t == -ERESTARTSYS)
-                return t;
-        return 0;
-}
-EXPORT_SYMBOL(wait_for_completion_killable);
-
-/**
- * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
- * @x:  holds the state of this particular completion
- * @timeout:  timeout value in jiffies
- *
- * This waits for either a completion of a specific task to be
- * signaled or for a specified timeout to expire. It can be
- * interrupted by a kill signal. The timeout is in jiffies.
- *
- * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
- * or number of jiffies left till timeout) if completed.
- */
-long __sched
-wait_for_completion_killable_timeout(struct completion *x,
-                                     unsigned long timeout)
-{
-        return wait_for_common(x, timeout, TASK_KILLABLE);
-}
-EXPORT_SYMBOL(wait_for_completion_killable_timeout);
-
-/**
- *      try_wait_for_completion - try to decrement a completion without blocking
- *      @x:     completion structure
- *
- *      Return: 0 if a decrement cannot be done without blocking
- *               1 if a decrement succeeded.
- *
- *      If a completion is being used as a counting completion,
- *      attempt to decrement the counter without blocking. This
- *      enables us to avoid waiting if the resource the completion
- *      is protecting is not available.
- */
-bool try_wait_for_completion(struct completion *x)
-{
-        unsigned long flags;
-        int ret = 1;
-
-        spin_lock_irqsave(&x->wait.lock, flags);
-        if (!x->done)
-                ret = 0;
-        else
-                x->done--;
-        spin_unlock_irqrestore(&x->wait.lock, flags);
-        return ret;
-}
-EXPORT_SYMBOL(try_wait_for_completion);
-
-/**
- *      completion_done - Test to see if a completion has any waiters
- *      @x:     completion structure
- *
- *      Return: 0 if there are waiters (wait_for_completion() in progress)
- *               1 if there are no waiters.
- *
- */
-bool completion_done(struct completion *x)
-{
-        unsigned long flags;
-        int ret = 1;
-
-        spin_lock_irqsave(&x->wait.lock, flags);
-        if (!x->done)
-                ret = 0;
-        spin_unlock_irqrestore(&x->wait.lock, flags);
-        return ret;
-}
-EXPORT_SYMBOL(completion_done);
-
 static long __sched
 sleep_on_common(wait_queue_head_t *q, int state, long timeout)
 {
@@ -3598,13 +3324,11 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
         struct task_struct *p;
         int retval;
 
-        get_online_cpus();
         rcu_read_lock();
 
         p = find_process_by_pid(pid);
         if (!p) {
                 rcu_read_unlock();
-                put_online_cpus();
                 return -ESRCH;
         }
 
@@ -3661,7 +3385,6 @@ out_free_cpus_allowed:
         free_cpumask_var(cpus_allowed);
 out_put_task:
         put_task_struct(p);
-        put_online_cpus();
         return retval;
 }
 
@@ -3706,7 +3429,6 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask)
         unsigned long flags;
         int retval;
 
-        get_online_cpus();
         rcu_read_lock();
 
         retval = -ESRCH;
@@ -3719,12 +3441,11 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask)
                 goto out_unlock;
 
         raw_spin_lock_irqsave(&p->pi_lock, flags);
-        cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
+        cpumask_and(mask, &p->cpus_allowed, cpu_active_mask);
         raw_spin_unlock_irqrestore(&p->pi_lock, flags);
 
 out_unlock:
         rcu_read_unlock();
-        put_online_cpus();
 
         return retval;
 }
@@ -3794,16 +3515,11 @@ SYSCALL_DEFINE0(sched_yield)
         return 0;
 }
 
-static inline int should_resched(void)
-{
-        return need_resched() && !(preempt_count() & PREEMPT_ACTIVE);
-}
-
 static void __cond_resched(void)
 {
-        add_preempt_count(PREEMPT_ACTIVE);
+        __preempt_count_add(PREEMPT_ACTIVE);
         __schedule();
-        sub_preempt_count(PREEMPT_ACTIVE);
+        __preempt_count_sub(PREEMPT_ACTIVE);
 }
 
 int __sched _cond_resched(void)
@@ -4186,7 +3902,7 @@ void init_idle(struct task_struct *idle, int cpu)
 
         raw_spin_lock_irqsave(&rq->lock, flags);
 
-        __sched_fork(idle);
+        __sched_fork(0, idle);
         idle->state = TASK_RUNNING;
         idle->se.exec_start = sched_clock();
 
@@ -4212,7 +3928,7 @@ void init_idle(struct task_struct *idle, int cpu)
         raw_spin_unlock_irqrestore(&rq->lock, flags);
 
         /* Set the preempt count _outside_ the spinlocks! */
-        task_thread_info(idle)->preempt_count = 0;
+        init_idle_preempt_count(idle, cpu);
 
         /*
          * The idle tasks have their own, simple scheduling class:
@@ -4346,6 +4062,53 @@ fail:
         return ret;
 }
 
+#ifdef CONFIG_NUMA_BALANCING
+/* Migrate current task p to target_cpu */
+int migrate_task_to(struct task_struct *p, int target_cpu)
+{
+        struct migration_arg arg = { p, target_cpu };
+        int curr_cpu = task_cpu(p);
+
+        if (curr_cpu == target_cpu)
+                return 0;
+
+        if (!cpumask_test_cpu(target_cpu, tsk_cpus_allowed(p)))
+                return -EINVAL;
+
+        /* TODO: This is not properly updating schedstats */
+
+        return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg);
+}
+
+/*
+ * Requeue a task on a given node and accurately track the number of NUMA
+ * tasks on the runqueues
+ */
+void sched_setnuma(struct task_struct *p, int nid)
+{
+        struct rq *rq;
+        unsigned long flags;
+        bool on_rq, running;
+
+        rq = task_rq_lock(p, &flags);
+        on_rq = p->on_rq;
+        running = task_current(rq, p);
+
+        if (on_rq)
+                dequeue_task(rq, p, 0);
+        if (running)
+                p->sched_class->put_prev_task(rq, p);
+
+        p->numa_preferred_nid = nid;
+
+        if (running)
+                p->sched_class->set_curr_task(rq);
+        if (on_rq)
+                enqueue_task(rq, p, 0);
+        task_rq_unlock(rq, p, &flags);
+}
+#endif
+
 /*
  * migration_cpu_stop - this will be executed by a highprio stopper thread
  * and performs thread migration by bumping thread off CPU then
@@ -5119,6 +4882,9 @@ static void destroy_sched_domains(struct sched_domain *sd, int cpu)
 DEFINE_PER_CPU(struct sched_domain *, sd_llc);
 DEFINE_PER_CPU(int, sd_llc_size);
 DEFINE_PER_CPU(int, sd_llc_id);
+DEFINE_PER_CPU(struct sched_domain *, sd_numa);
+DEFINE_PER_CPU(struct sched_domain *, sd_busy);
+DEFINE_PER_CPU(struct sched_domain *, sd_asym);
 
 static void update_top_cache_domain(int cpu)
 {
@@ -5130,11 +4896,18 @@ static void update_top_cache_domain(int cpu)
         if (sd) {
                 id = cpumask_first(sched_domain_span(sd));
                 size = cpumask_weight(sched_domain_span(sd));
+                rcu_assign_pointer(per_cpu(sd_busy, cpu), sd->parent);
         }
 
         rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
         per_cpu(sd_llc_size, cpu) = size;
         per_cpu(sd_llc_id, cpu) = id;
+
+        sd = lowest_flag_domain(cpu, SD_NUMA);
+        rcu_assign_pointer(per_cpu(sd_numa, cpu), sd);
+
+        sd = highest_flag_domain(cpu, SD_ASYM_PACKING);
+        rcu_assign_pointer(per_cpu(sd_asym, cpu), sd);
 }
 
 /*
@@ -5654,6 +5427,7 @@ sd_numa_init(struct sched_domain_topology_level *tl, int cpu)
                                         | 0*SD_SHARE_PKG_RESOURCES
                                         | 1*SD_SERIALIZE
                                         | 0*SD_PREFER_SIBLING
+                                        | 1*SD_NUMA
                                         | sd_local_flags(level)
                                         ,
                 .last_balance           = jiffies,
@@ -6335,14 +6109,17 @@ void __init sched_init_smp(void)
 
         sched_init_numa();
 
-        get_online_cpus();
+        /*
+         * There's no userspace yet to cause hotplug operations; hence all the
+         * cpu masks are stable and all blatant races in the below code cannot
+         * happen.
+         */
         mutex_lock(&sched_domains_mutex);
         init_sched_domains(cpu_active_mask);
         cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
         if (cpumask_empty(non_isolated_cpus))
                 cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
         mutex_unlock(&sched_domains_mutex);
-        put_online_cpus();
 
         hotcpu_notifier(sched_domains_numa_masks_update, CPU_PRI_SCHED_ACTIVE);
         hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE);
@@ -6505,6 +6282,7 @@ void __init sched_init(void)
                 rq->online = 0;
                 rq->idle_stamp = 0;
                 rq->avg_idle = 2*sysctl_sched_migration_cost;
+                rq->max_idle_balance_cost = sysctl_sched_migration_cost;
 
                 INIT_LIST_HEAD(&rq->cfs_tasks);
 
@@ -7277,7 +7055,12 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 
         runtime_enabled = quota != RUNTIME_INF;
         runtime_was_enabled = cfs_b->quota != RUNTIME_INF;
-        account_cfs_bandwidth_used(runtime_enabled, runtime_was_enabled);
+        /*
+         * If we need to toggle cfs_bandwidth_used, off->on must occur
+         * before making related changes, and on->off must occur afterwards
+         */
+        if (runtime_enabled && !runtime_was_enabled)
+                cfs_bandwidth_usage_inc();
         raw_spin_lock_irq(&cfs_b->lock);
         cfs_b->period = ns_to_ktime(period);
         cfs_b->quota = quota;
@@ -7303,6 +7086,8 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
                         unthrottle_cfs_rq(cfs_rq);
                 raw_spin_unlock_irq(&rq->lock);
         }
+        if (runtime_was_enabled && !runtime_enabled)
+                cfs_bandwidth_usage_dec();
 out_unlock:
         mutex_unlock(&cfs_constraints_mutex);
 
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 196559994f7c..5c34d1817e8f 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -15,6 +15,7 @@
 #include <linux/seq_file.h>
 #include <linux/kallsyms.h>
 #include <linux/utsname.h>
+#include <linux/mempolicy.h>
 
 #include "sched.h"
 
@@ -137,6 +138,9 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p)
         SEQ_printf(m, "%15Ld %15Ld %15Ld.%06ld %15Ld.%06ld %15Ld.%06ld",
                 0LL, 0LL, 0LL, 0L, 0LL, 0L, 0LL, 0L);
 #endif
+#ifdef CONFIG_NUMA_BALANCING
+        SEQ_printf(m, " %d", cpu_to_node(task_cpu(p)));
+#endif
 #ifdef CONFIG_CGROUP_SCHED
         SEQ_printf(m, " %s", task_group_path(task_group(p)));
 #endif
@@ -159,7 +163,7 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu)
         read_lock_irqsave(&tasklist_lock, flags);
 
         do_each_thread(g, p) {
-                if (!p->on_rq || task_cpu(p) != rq_cpu)
+                if (task_cpu(p) != rq_cpu)
                         continue;
 
                 print_task(m, rq, p);
@@ -225,6 +229,14 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
                         atomic_read(&cfs_rq->tg->runnable_avg));
 #endif
 #endif
+#ifdef CONFIG_CFS_BANDWIDTH
+        SEQ_printf(m, "  .%-30s: %d\n", "tg->cfs_bandwidth.timer_active",
+                        cfs_rq->tg->cfs_bandwidth.timer_active);
+        SEQ_printf(m, "  .%-30s: %d\n", "throttled",
+                        cfs_rq->throttled);
+        SEQ_printf(m, "  .%-30s: %d\n", "throttle_count",
+                        cfs_rq->throttle_count);
+#endif
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
         print_cfs_group_stats(m, cpu, cfs_rq->tg);
@@ -345,7 +357,7 @@ static void sched_debug_header(struct seq_file *m)
         cpu_clk = local_clock();
         local_irq_restore(flags);
 
-        SEQ_printf(m, "Sched Debug Version: v0.10, %s %.*s\n",
+        SEQ_printf(m, "Sched Debug Version: v0.11, %s %.*s\n",
                 init_utsname()->release,
                 (int)strcspn(init_utsname()->version, " "),
                 init_utsname()->version);
@@ -488,6 +500,56 @@ static int __init init_sched_debug_procfs(void)
 
 __initcall(init_sched_debug_procfs);
 
+#define __P(F) \
+        SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)F)
+#define P(F) \
+        SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)p->F)
+#define __PN(F) \
+        SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F))
+#define PN(F) \
+        SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F))
+
+
+static void sched_show_numa(struct task_struct *p, struct seq_file *m)
+{
+#ifdef CONFIG_NUMA_BALANCING
+        struct mempolicy *pol;
+        int node, i;
+
+        if (p->mm)
+                P(mm->numa_scan_seq);
+
+        task_lock(p);
+        pol = p->mempolicy;
+        if (pol && !(pol->flags & MPOL_F_MORON))
+                pol = NULL;
+        mpol_get(pol);
+        task_unlock(p);
+
+        SEQ_printf(m, "numa_migrations, %ld\n", xchg(&p->numa_pages_migrated, 0));
+
+        for_each_online_node(node) {
+                for (i = 0; i < 2; i++) {
+                        unsigned long nr_faults = -1;
+                        int cpu_current, home_node;
+
+                        if (p->numa_faults)
+                                nr_faults = p->numa_faults[2*node + i];
+
+                        cpu_current = !i ? (task_node(p) == node) :
+                                (pol && node_isset(node, pol->v.nodes));
+
+                        home_node = (p->numa_preferred_nid == node);
+
+                        SEQ_printf(m, "numa_faults, %d, %d, %d, %d, %ld\n",
+                                i, node, cpu_current, home_node, nr_faults);
+                }
+        }
+
+        mpol_put(pol);
+#endif
+}
+
 void proc_sched_show_task(struct task_struct *p, struct seq_file *m)
 {
         unsigned long nr_switches;
@@ -591,6 +653,8 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m)
                 SEQ_printf(m, "%-45s:%21Ld\n",
                            "clock-delta", (long long)(t1-t0));
         }
+
+        sched_show_numa(p, m);
 }
 
 void proc_sched_set_task(struct task_struct *p)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 7c70201fbc61..df77c605c7a6 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -681,6 +681,8 @@ static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se)
 }
 
 #ifdef CONFIG_SMP
+static unsigned long task_h_load(struct task_struct *p);
+
 static inline void __update_task_entity_contrib(struct sched_entity *se);
 
 /* Give new task start runnable values to heavy its load in infant time */
@@ -818,11 +820,12 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
 
 #ifdef CONFIG_NUMA_BALANCING
 /*
- * numa task sample period in ms
+ * Approximate time to scan a full NUMA task in ms. The task scan period is
+ * calculated based on the tasks virtual memory size and
+ * numa_balancing_scan_size.
  */
-unsigned int sysctl_numa_balancing_scan_period_min = 100;
-unsigned int sysctl_numa_balancing_scan_period_max = 100*50;
-unsigned int sysctl_numa_balancing_scan_period_reset = 100*600;
+unsigned int sysctl_numa_balancing_scan_period_min = 1000;
+unsigned int sysctl_numa_balancing_scan_period_max = 60000;
 
 /* Portion of address space to scan in MB */
 unsigned int sysctl_numa_balancing_scan_size = 256;
@@ -830,41 +833,810 @@ unsigned int sysctl_numa_balancing_scan_size = 256;
 /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */
 unsigned int sysctl_numa_balancing_scan_delay = 1000;
 
-static void task_numa_placement(struct task_struct *p)
+/*
+ * After skipping a page migration on a shared page, skip N more numa page
+ * migrations unconditionally. This reduces the number of NUMA migrations
+ * in shared memory workloads, and has the effect of pulling tasks towards
+ * where their memory lives, over pulling the memory towards the task.
+ */
+unsigned int sysctl_numa_balancing_migrate_deferred = 16;
+
+static unsigned int task_nr_scan_windows(struct task_struct *p)
+{
+        unsigned long rss = 0;
+        unsigned long nr_scan_pages;
+
+        /*
+         * Calculations based on RSS as non-present and empty pages are skipped
+         * by the PTE scanner and NUMA hinting faults should be trapped based
+         * on resident pages
+         */
+        nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT);
+        rss = get_mm_rss(p->mm);
+        if (!rss)
+                rss = nr_scan_pages;
+
+        rss = round_up(rss, nr_scan_pages);
+        return rss / nr_scan_pages;
+}
+
+/* For sanitys sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */
+#define MAX_SCAN_WINDOW 2560
+
+static unsigned int task_scan_min(struct task_struct *p)
+{
+        unsigned int scan, floor;
+        unsigned int windows = 1;
+
+        if (sysctl_numa_balancing_scan_size < MAX_SCAN_WINDOW)
+                windows = MAX_SCAN_WINDOW / sysctl_numa_balancing_scan_size;
+        floor = 1000 / windows;
+
+        scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p);
+        return max_t(unsigned int, floor, scan);
+}
+
+static unsigned int task_scan_max(struct task_struct *p)
+{
+        unsigned int smin = task_scan_min(p);
+        unsigned int smax;
+
+        /* Watch for min being lower than max due to floor calculations */
+        smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p);
+        return max(smin, smax);
+}
+
+/*
+ * Once a preferred node is selected the scheduler balancer will prefer moving
+ * a task to that node for sysctl_numa_balancing_settle_count number of PTE
+ * scans. This will give the process the chance to accumulate more faults on
+ * the preferred node but still allow the scheduler to move the task again if
+ * the nodes CPUs are overloaded.
+ */
+unsigned int sysctl_numa_balancing_settle_count __read_mostly = 4;
+
+static void account_numa_enqueue(struct rq *rq, struct task_struct *p)
+{
+        rq->nr_numa_running += (p->numa_preferred_nid != -1);
+        rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p));
+}
+
+static void account_numa_dequeue(struct rq *rq, struct task_struct *p)
+{
+        rq->nr_numa_running -= (p->numa_preferred_nid != -1);
+        rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p));
+}
+
+struct numa_group {
+        atomic_t refcount;
+
+        spinlock_t lock; /* nr_tasks, tasks */
+        int nr_tasks;
+        pid_t gid;
+        struct list_head task_list;
+
+        struct rcu_head rcu;
+        unsigned long total_faults;
+        unsigned long faults[0];
+};
+
+pid_t task_numa_group_id(struct task_struct *p)
+{
+        return p->numa_group ? p->numa_group->gid : 0;
+}
+
+static inline int task_faults_idx(int nid, int priv)
+{
+        return 2 * nid + priv;
+}
+
+static inline unsigned long task_faults(struct task_struct *p, int nid)
+{
+        if (!p->numa_faults)
+                return 0;
+
+        return p->numa_faults[task_faults_idx(nid, 0)] +
+                p->numa_faults[task_faults_idx(nid, 1)];
+}
+
+static inline unsigned long group_faults(struct task_struct *p, int nid)
+{
+        if (!p->numa_group)
+                return 0;
+
+        return p->numa_group->faults[2*nid] + p->numa_group->faults[2*nid+1];
+}
+
+/*
+ * These return the fraction of accesses done by a particular task, or
+ * task group, on a particular numa node.  The group weight is given a
+ * larger multiplier, in order to group tasks together that are almost
+ * evenly spread out between numa nodes.
+ */
+static inline unsigned long task_weight(struct task_struct *p, int nid)
+{
+        unsigned long total_faults;
+
+        if (!p->numa_faults)
+                return 0;
+
+        total_faults = p->total_numa_faults;
+
+        if (!total_faults)
+                return 0;
+
+        return 1000 * task_faults(p, nid) / total_faults;
+}
+
+static inline unsigned long group_weight(struct task_struct *p, int nid)
 {
-        int seq;
+        if (!p->numa_group || !p->numa_group->total_faults)
+                return 0;
 
-        if (!p->mm)     /* for example, ksmd faulting in a user's mm */
+        return 1000 * group_faults(p, nid) / p->numa_group->total_faults;
+}
+
+static unsigned long weighted_cpuload(const int cpu);
+static unsigned long source_load(int cpu, int type);
+static unsigned long target_load(int cpu, int type);
+static unsigned long power_of(int cpu);
+static long effective_load(struct task_group *tg, int cpu, long wl, long wg);
+
+/* Cached statistics for all CPUs within a node */
+struct numa_stats {
+        unsigned long nr_running;
+        unsigned long load;
+
+        /* Total compute capacity of CPUs on a node */
+        unsigned long power;
+
+        /* Approximate capacity in terms of runnable tasks on a node */
+        unsigned long capacity;
+        int has_capacity;
+};
+
+/*
+ * XXX borrowed from update_sg_lb_stats
+ */
+static void update_numa_stats(struct numa_stats *ns, int nid)
+{
+        int cpu;
+
+        memset(ns, 0, sizeof(*ns));
+        for_each_cpu(cpu, cpumask_of_node(nid)) {
+                struct rq *rq = cpu_rq(cpu);
+
+                ns->nr_running += rq->nr_running;
+                ns->load += weighted_cpuload(cpu);
+                ns->power += power_of(cpu);
+        }
+
+        ns->load = (ns->load * SCHED_POWER_SCALE) / ns->power;
+        ns->capacity = DIV_ROUND_CLOSEST(ns->power, SCHED_POWER_SCALE);
+        ns->has_capacity = (ns->nr_running < ns->capacity);
+}
+
+struct task_numa_env {
+        struct task_struct *p;
+
+        int src_cpu, src_nid;
+        int dst_cpu, dst_nid;
+
+        struct numa_stats src_stats, dst_stats;
+
+        int imbalance_pct, idx;
+
+        struct task_struct *best_task;
+        long best_imp;
+        int best_cpu;
+};
+
+static void task_numa_assign(struct task_numa_env *env,
+                             struct task_struct *p, long imp)
+{
+        if (env->best_task)
+                put_task_struct(env->best_task);
+        if (p)
+                get_task_struct(p);
+
+        env->best_task = p;
+        env->best_imp = imp;
+        env->best_cpu = env->dst_cpu;
+}
+
+/*
+ * This checks if the overall compute and NUMA accesses of the system would
+ * be improved if the source tasks was migrated to the target dst_cpu taking
+ * into account that it might be best if task running on the dst_cpu should
+ * be exchanged with the source task
+ */
+static void task_numa_compare(struct task_numa_env *env,
+                              long taskimp, long groupimp)
+{
+        struct rq *src_rq = cpu_rq(env->src_cpu);
+        struct rq *dst_rq = cpu_rq(env->dst_cpu);
+        struct task_struct *cur;
+        long dst_load, src_load;
+        long load;
+        long imp = (groupimp > 0) ? groupimp : taskimp;
+
+        rcu_read_lock();
+        cur = ACCESS_ONCE(dst_rq->curr);
+        if (cur->pid == 0) /* idle */
+                cur = NULL;
+
+        /*
+         * "imp" is the fault differential for the source task between the
+         * source and destination node. Calculate the total differential for
+         * the source task and potential destination task. The more negative
+         * the value is, the more rmeote accesses that would be expected to
+         * be incurred if the tasks were swapped.
+         */
+        if (cur) {
+                /* Skip this swap candidate if cannot move to the source cpu */
+                if (!cpumask_test_cpu(env->src_cpu, tsk_cpus_allowed(cur)))
+                        goto unlock;
+
+                /*
+                 * If dst and source tasks are in the same NUMA group, or not
+                 * in any group then look only at task weights.
+                 */
+                if (cur->numa_group == env->p->numa_group) {
+                        imp = taskimp + task_weight(cur, env->src_nid) -
+                              task_weight(cur, env->dst_nid);
+                        /*
+                         * Add some hysteresis to prevent swapping the
+                         * tasks within a group over tiny differences.
+                         */
+                        if (cur->numa_group)
+                                imp -= imp/16;
+                } else {
+                        /*
+                         * Compare the group weights. If a task is all by
+                         * itself (not part of a group), use the task weight
+                         * instead.
+                         */
+                        if (env->p->numa_group)
+                                imp = groupimp;
+                        else
+                                imp = taskimp;
+
+                        if (cur->numa_group)
+                                imp += group_weight(cur, env->src_nid) -
+                                       group_weight(cur, env->dst_nid);
+                        else
+                                imp += task_weight(cur, env->src_nid) -
+                                       task_weight(cur, env->dst_nid);
+                }
+        }
+
+        if (imp < env->best_imp)
+                goto unlock;
+
+        if (!cur) {
+                /* Is there capacity at our destination? */
+                if (env->src_stats.has_capacity &&
+                    !env->dst_stats.has_capacity)
+                        goto unlock;
+
+                goto balance;
+        }
+
+        /* Balance doesn't matter much if we're running a task per cpu */
+        if (src_rq->nr_running == 1 && dst_rq->nr_running == 1)
+                goto assign;
+
+        /*
+         * In the overloaded case, try and keep the load balanced.
+         */
+balance:
+        dst_load = env->dst_stats.load;
+        src_load = env->src_stats.load;
+
+        /* XXX missing power terms */
+        load = task_h_load(env->p);
+        dst_load += load;
+        src_load -= load;
+
+        if (cur) {
+                load = task_h_load(cur);
+                dst_load -= load;
+                src_load += load;
+        }
+
+        /* make src_load the smaller */
+        if (dst_load < src_load)
+                swap(dst_load, src_load);
+
+        if (src_load * env->imbalance_pct < dst_load * 100)
+                goto unlock;
+
+assign:
+        task_numa_assign(env, cur, imp);
+unlock:
+        rcu_read_unlock();
+}
+
+static void task_numa_find_cpu(struct task_numa_env *env,
+                                long taskimp, long groupimp)
+{
+        int cpu;
+
+        for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) {
+                /* Skip this CPU if the source task cannot migrate */
+                if (!cpumask_test_cpu(cpu, tsk_cpus_allowed(env->p)))
+                        continue;
+
+                env->dst_cpu = cpu;
+                task_numa_compare(env, taskimp, groupimp);
+        }
+}
+
+static int task_numa_migrate(struct task_struct *p)
+{
+        struct task_numa_env env = {
+                .p = p,
+
+                .src_cpu = task_cpu(p),
+                .src_nid = task_node(p),
+
+                .imbalance_pct = 112,
+
+                .best_task = NULL,
+                .best_imp = 0,
+                .best_cpu = -1
+        };
+        struct sched_domain *sd;
+        unsigned long taskweight, groupweight;
+        int nid, ret;
+        long taskimp, groupimp;
+
+        /*
+         * Pick the lowest SD_NUMA domain, as that would have the smallest
+         * imbalance and would be the first to start moving tasks about.
+         *
+         * And we want to avoid any moving of tasks about, as that would create
+         * random movement of tasks -- counter the numa conditions we're trying
+         * to satisfy here.
+         */
+        rcu_read_lock();
+        sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu));
+        env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2;
+        rcu_read_unlock();
+
+        taskweight = task_weight(p, env.src_nid);
+        groupweight = group_weight(p, env.src_nid);
+        update_numa_stats(&env.src_stats, env.src_nid);
+        env.dst_nid = p->numa_preferred_nid;
+        taskimp = task_weight(p, env.dst_nid) - taskweight;
+        groupimp = group_weight(p, env.dst_nid) - groupweight;
+        update_numa_stats(&env.dst_stats, env.dst_nid);
+
+        /* If the preferred nid has capacity, try to use it. */
+        if (env.dst_stats.has_capacity)
+                task_numa_find_cpu(&env, taskimp, groupimp);
+
+        /* No space available on the preferred nid. Look elsewhere. */
+        if (env.best_cpu == -1) {
+                for_each_online_node(nid) {
+                        if (nid == env.src_nid || nid == p->numa_preferred_nid)
+                                continue;
+
+                        /* Only consider nodes where both task and groups benefit */
+                        taskimp = task_weight(p, nid) - taskweight;
+                        groupimp = group_weight(p, nid) - groupweight;
+                        if (taskimp < 0 && groupimp < 0)
+                                continue;
+
+                        env.dst_nid = nid;
+                        update_numa_stats(&env.dst_stats, env.dst_nid);
+                        task_numa_find_cpu(&env, taskimp, groupimp);
+                }
+        }
+
+        /* No better CPU than the current one was found. */
+        if (env.best_cpu == -1)
+                return -EAGAIN;
+
+        sched_setnuma(p, env.dst_nid);
+
+        /*
+         * Reset the scan period if the task is being rescheduled on an
+         * alternative node to recheck if the tasks is now properly placed.
+         */
+        p->numa_scan_period = task_scan_min(p);
+
+        if (env.best_task == NULL) {
+                int ret = migrate_task_to(p, env.best_cpu);
+                return ret;
+        }
+
+        ret = migrate_swap(p, env.best_task);
+        put_task_struct(env.best_task);
+        return ret;
+}
+
+/* Attempt to migrate a task to a CPU on the preferred node. */
+static void numa_migrate_preferred(struct task_struct *p)
+{
+        /* This task has no NUMA fault statistics yet */
+        if (unlikely(p->numa_preferred_nid == -1 || !p->numa_faults))
+                return;
+
+        /* Periodically retry migrating the task to the preferred node */
+        p->numa_migrate_retry = jiffies + HZ;
+
+        /* Success if task is already running on preferred CPU */
+        if (cpu_to_node(task_cpu(p)) == p->numa_preferred_nid)
                 return;
+
+        /* Otherwise, try migrate to a CPU on the preferred node */
+        task_numa_migrate(p);
+}
+
+/*
+ * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS
+ * increments. The more local the fault statistics are, the higher the scan
+ * period will be for the next scan window. If local/remote ratio is below
+ * NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) the
+ * scan period will decrease
+ */
+#define NUMA_PERIOD_SLOTS 10
+#define NUMA_PERIOD_THRESHOLD 3
+
+/*
+ * Increase the scan period (slow down scanning) if the majority of
+ * our memory is already on our local node, or if the majority of
+ * the page accesses are shared with other processes.
+ * Otherwise, decrease the scan period.
+ */
+static void update_task_scan_period(struct task_struct *p,
+                        unsigned long shared, unsigned long private)
+{
+        unsigned int period_slot;
+        int ratio;
+        int diff;
+
+        unsigned long remote = p->numa_faults_locality[0];
+        unsigned long local = p->numa_faults_locality[1];
+
+        /*
+         * If there were no record hinting faults then either the task is
+         * completely idle or all activity is areas that are not of interest
+         * to automatic numa balancing. Scan slower
+         */
+        if (local + shared == 0) {
+                p->numa_scan_period = min(p->numa_scan_period_max,
+                        p->numa_scan_period << 1);
+
+                p->mm->numa_next_scan = jiffies +
+                        msecs_to_jiffies(p->numa_scan_period);
+
+                return;
+        }
+
+        /*
+         * Prepare to scale scan period relative to the current period.
+         *       == NUMA_PERIOD_THRESHOLD scan period stays the same
+         *       <  NUMA_PERIOD_THRESHOLD scan period decreases (scan faster)
+         *       >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower)
+         */
+        period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS);
+        ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote);
+        if (ratio >= NUMA_PERIOD_THRESHOLD) {
+                int slot = ratio - NUMA_PERIOD_THRESHOLD;
+                if (!slot)
+                        slot = 1;
+                diff = slot * period_slot;
+        } else {
+                diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot;
+
+                /*
+                 * Scale scan rate increases based on sharing. There is an
+                 * inverse relationship between the degree of sharing and
+                 * the adjustment made to the scanning period. Broadly
+                 * speaking the intent is that there is little point
+                 * scanning faster if shared accesses dominate as it may
+                 * simply bounce migrations uselessly
+                 */
+                period_slot = DIV_ROUND_UP(diff, NUMA_PERIOD_SLOTS);
+                ratio = DIV_ROUND_UP(private * NUMA_PERIOD_SLOTS, (private + shared));
+                diff = (diff * ratio) / NUMA_PERIOD_SLOTS;
+        }
+
+        p->numa_scan_period = clamp(p->numa_scan_period + diff,
+                        task_scan_min(p), task_scan_max(p));
+        memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
+}
+
+static void task_numa_placement(struct task_struct *p)
+{
+        int seq, nid, max_nid = -1, max_group_nid = -1;
+        unsigned long max_faults = 0, max_group_faults = 0;
+        unsigned long fault_types[2] = { 0, 0 };
+        spinlock_t *group_lock = NULL;
+
         seq = ACCESS_ONCE(p->mm->numa_scan_seq);
         if (p->numa_scan_seq == seq)
                 return;
         p->numa_scan_seq = seq;
+        p->numa_scan_period_max = task_scan_max(p);
+
+        /* If the task is part of a group prevent parallel updates to group stats */
+        if (p->numa_group) {
+                group_lock = &p->numa_group->lock;
+                spin_lock(group_lock);
+        }
+
+        /* Find the node with the highest number of faults */
+        for_each_online_node(nid) {
+                unsigned long faults = 0, group_faults = 0;
+                int priv, i;
+
+                for (priv = 0; priv < 2; priv++) {
+                        long diff;
+
+                        i = task_faults_idx(nid, priv);
+                        diff = -p->numa_faults[i];
+
+                        /* Decay existing window, copy faults since last scan */
+                        p->numa_faults[i] >>= 1;
+                        p->numa_faults[i] += p->numa_faults_buffer[i];
+                        fault_types[priv] += p->numa_faults_buffer[i];
+                        p->numa_faults_buffer[i] = 0;
+
+                        faults += p->numa_faults[i];
+                        diff += p->numa_faults[i];
+                        p->total_numa_faults += diff;
+                        if (p->numa_group) {
+                                /* safe because we can only change our own group */
+                                p->numa_group->faults[i] += diff;
+                                p->numa_group->total_faults += diff;
+                                group_faults += p->numa_group->faults[i];
+                        }
+                }
+
+                if (faults > max_faults) {
+                        max_faults = faults;
+                        max_nid = nid;
+                }
+
+                if (group_faults > max_group_faults) {
+                        max_group_faults = group_faults;
+                        max_group_nid = nid;
+                }
+        }
+
+        update_task_scan_period(p, fault_types[0], fault_types[1]);
+
+        if (p->numa_group) {
+                /*
+                 * If the preferred task and group nids are different,
+                 * iterate over the nodes again to find the best place.
+                 */
+                if (max_nid != max_group_nid) {
+                        unsigned long weight, max_weight = 0;
+
+                        for_each_online_node(nid) {
+                                weight = task_weight(p, nid) + group_weight(p, nid);
+                                if (weight > max_weight) {
+                                        max_weight = weight;
+                                        max_nid = nid;
+                                }
+                        }
+                }
+
+                spin_unlock(group_lock);
+        }
+
+        /* Preferred node as the node with the most faults */
+        if (max_faults && max_nid != p->numa_preferred_nid) {
+                /* Update the preferred nid and migrate task if possible */
+                sched_setnuma(p, max_nid);
+                numa_migrate_preferred(p);
+        }
+}
+
+static inline int get_numa_group(struct numa_group *grp)
+{
+        return atomic_inc_not_zero(&grp->refcount);
+}
+
+static inline void put_numa_group(struct numa_group *grp)
+{
+        if (atomic_dec_and_test(&grp->refcount))
+                kfree_rcu(grp, rcu);
+}
+
+static void task_numa_group(struct task_struct *p, int cpupid, int flags,
+                        int *priv)
+{
+        struct numa_group *grp, *my_grp;
+        struct task_struct *tsk;
+        bool join = false;
+        int cpu = cpupid_to_cpu(cpupid);
+        int i;
+
+        if (unlikely(!p->numa_group)) {
+                unsigned int size = sizeof(struct numa_group) +
+                                    2*nr_node_ids*sizeof(unsigned long);
+
+                grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
+                if (!grp)
+                        return;
+
+                atomic_set(&grp->refcount, 1);
+                spin_lock_init(&grp->lock);
+                INIT_LIST_HEAD(&grp->task_list);
+                grp->gid = p->pid;
+
+                for (i = 0; i < 2*nr_node_ids; i++)
+                        grp->faults[i] = p->numa_faults[i];
+
+                grp->total_faults = p->total_numa_faults;
+
+                list_add(&p->numa_entry, &grp->task_list);
+                grp->nr_tasks++;
+                rcu_assign_pointer(p->numa_group, grp);
+        }
+
+        rcu_read_lock();
+        tsk = ACCESS_ONCE(cpu_rq(cpu)->curr);
+
+        if (!cpupid_match_pid(tsk, cpupid))
+                goto no_join;
+
+        grp = rcu_dereference(tsk->numa_group);
+        if (!grp)
+                goto no_join;
+
+        my_grp = p->numa_group;
+        if (grp == my_grp)
+                goto no_join;
+
+        /*
+         * Only join the other group if its bigger; if we're the bigger group,
+         * the other task will join us.
+         */
+        if (my_grp->nr_tasks > grp->nr_tasks)
+                goto no_join;
+
+        /*
+         * Tie-break on the grp address.
+         */
+        if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp)
+                goto no_join;
+
+        /* Always join threads in the same process. */
+        if (tsk->mm == current->mm)
+                join = true;
+
+        /* Simple filter to avoid false positives due to PID collisions */
+        if (flags & TNF_SHARED)
+                join = true;
+
+        /* Update priv based on whether false sharing was detected */
+        *priv = !join;
+
+        if (join && !get_numa_group(grp))
+                goto no_join;
 
-        /* FIXME: Scheduling placement policy hints go here */
+        rcu_read_unlock();
+
+        if (!join)
+                return;
+
+        double_lock(&my_grp->lock, &grp->lock);
+
+        for (i = 0; i < 2*nr_node_ids; i++) {
+                my_grp->faults[i] -= p->numa_faults[i];
+                grp->faults[i] += p->numa_faults[i];
+        }
+        my_grp->total_faults -= p->total_numa_faults;
+        grp->total_faults += p->total_numa_faults;
+
+        list_move(&p->numa_entry, &grp->task_list);
+        my_grp->nr_tasks--;
+        grp->nr_tasks++;
+
+        spin_unlock(&my_grp->lock);
+        spin_unlock(&grp->lock);
+
+        rcu_assign_pointer(p->numa_group, grp);
+
+        put_numa_group(my_grp);
+        return;
+
+no_join:
+        rcu_read_unlock();
+        return;
+}
+
+void task_numa_free(struct task_struct *p)
+{
+        struct numa_group *grp = p->numa_group;
+        int i;
+        void *numa_faults = p->numa_faults;
+
+        if (grp) {
+                spin_lock(&grp->lock);
+                for (i = 0; i < 2*nr_node_ids; i++)
+                        grp->faults[i] -= p->numa_faults[i];
+                grp->total_faults -= p->total_numa_faults;
+
+                list_del(&p->numa_entry);
+                grp->nr_tasks--;
+                spin_unlock(&grp->lock);
+                rcu_assign_pointer(p->numa_group, NULL);
+                put_numa_group(grp);
+        }
+
+        p->numa_faults = NULL;
+        p->numa_faults_buffer = NULL;
+        kfree(numa_faults);
 }
 
 /*
  * Got a PROT_NONE fault for a page on @node.
  */
-void task_numa_fault(int node, int pages, bool migrated)
+void task_numa_fault(int last_cpupid, int node, int pages, int flags)
 {
         struct task_struct *p = current;
+        bool migrated = flags & TNF_MIGRATED;
+        int priv;
 
         if (!numabalancing_enabled)
                 return;
 
-        /* FIXME: Allocate task-specific structure for placement policy here */
+        /* for example, ksmd faulting in a user's mm */
+        if (!p->mm)
+                return;
+
+        /* Do not worry about placement if exiting */
+        if (p->state == TASK_DEAD)
+                return;
+
+        /* Allocate buffer to track faults on a per-node basis */
+        if (unlikely(!p->numa_faults)) {
+                int size = sizeof(*p->numa_faults) * 2 * nr_node_ids;
+
+                /* numa_faults and numa_faults_buffer share the allocation */
+                p->numa_faults = kzalloc(size * 2, GFP_KERNEL|__GFP_NOWARN);
+                if (!p->numa_faults)
+                        return;
+
+                BUG_ON(p->numa_faults_buffer);
+                p->numa_faults_buffer = p->numa_faults + (2 * nr_node_ids);
+                p->total_numa_faults = 0;
+                memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
+        }
 
         /*
-         * If pages are properly placed (did not migrate) then scan slower.
-         * This is reset periodically in case of phase changes
+         * First accesses are treated as private, otherwise consider accesses
+         * to be private if the accessing pid has not changed
          */
-        if (!migrated)
-                p->numa_scan_period = min(sysctl_numa_balancing_scan_period_max,
-                        p->numa_scan_period + jiffies_to_msecs(10));
+        if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) {
+                priv = 1;
+        } else {
+                priv = cpupid_match_pid(p, last_cpupid);
+                if (!priv && !(flags & TNF_NO_GROUP))
+                        task_numa_group(p, last_cpupid, flags, &priv);
+        }
 
         task_numa_placement(p);
+
+        /*
+         * Retry task to preferred node migration periodically, in case it
+         * case it previously failed, or the scheduler moved us.
+         */
+        if (time_after(jiffies, p->numa_migrate_retry))
+                numa_migrate_preferred(p);
+
+        if (migrated)
+                p->numa_pages_migrated += pages;
+
+        p->numa_faults_buffer[task_faults_idx(node, priv)] += pages;
+        p->numa_faults_locality[!!(flags & TNF_FAULT_LOCAL)] += pages;
 }
 
 static void reset_ptenuma_scan(struct task_struct *p)
@@ -884,6 +1656,7 @@ void task_numa_work(struct callback_head *work)
         struct mm_struct *mm = p->mm;
         struct vm_area_struct *vma;
         unsigned long start, end;
+        unsigned long nr_pte_updates = 0;
         long pages;
 
         WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work));
@@ -900,35 +1673,9 @@ void task_numa_work(struct callback_head *work)
         if (p->flags & PF_EXITING)
                 return;
 
-        /*
-         * We do not care about task placement until a task runs on a node
-         * other than the first one used by the address space. This is
-         * largely because migrations are driven by what CPU the task
-         * is running on. If it's never scheduled on another node, it'll
-         * not migrate so why bother trapping the fault.
-         */
-        if (mm->first_nid == NUMA_PTE_SCAN_INIT)
-                mm->first_nid = numa_node_id();
-        if (mm->first_nid != NUMA_PTE_SCAN_ACTIVE) {
-                /* Are we running on a new node yet? */
-                if (numa_node_id() == mm->first_nid &&
-                    !sched_feat_numa(NUMA_FORCE))
-                        return;
-
-                mm->first_nid = NUMA_PTE_SCAN_ACTIVE;
-        }
-
-        /*
-         * Reset the scan period if enough time has gone by. Objective is that
-         * scanning will be reduced if pages are properly placed. As tasks
-         * can enter different phases this needs to be re-examined. Lacking
-         * proper tracking of reference behaviour, this blunt hammer is used.
-         */
-        migrate = mm->numa_next_reset;
-        if (time_after(now, migrate)) {
-                p->numa_scan_period = sysctl_numa_balancing_scan_period_min;
-                next_scan = now + msecs_to_jiffies(sysctl_numa_balancing_scan_period_reset);
-                xchg(&mm->numa_next_reset, next_scan);
+        if (!mm->numa_next_scan) {
+                mm->numa_next_scan = now +
+                        msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
         }
 
         /*
@@ -938,20 +1685,20 @@ void task_numa_work(struct callback_head *work)
         if (time_before(now, migrate))
                 return;
 
-        if (p->numa_scan_period == 0)
-                p->numa_scan_period = sysctl_numa_balancing_scan_period_min;
+        if (p->numa_scan_period == 0) {
+                p->numa_scan_period_max = task_scan_max(p);
+                p->numa_scan_period = task_scan_min(p);
+        }
 
         next_scan = now + msecs_to_jiffies(p->numa_scan_period);
         if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate)
                 return;
 
         /*
-         * Do not set pte_numa if the current running node is rate-limited.
-         * This loses statistics on the fault but if we are unwilling to
-         * migrate to this node, it is less likely we can do useful work
+         * Delay this task enough that another task of this mm will likely win
+         * the next time around.
          */
-        if (migrate_ratelimited(numa_node_id()))
-                return;
+        p->node_stamp += 2 * TICK_NSEC;
 
         start = mm->numa_scan_offset;
         pages = sysctl_numa_balancing_scan_size;
@@ -967,18 +1714,32 @@ void task_numa_work(struct callback_head *work)
                 vma = mm->mmap;
         }
         for (; vma; vma = vma->vm_next) {
-                if (!vma_migratable(vma))
+                if (!vma_migratable(vma) || !vma_policy_mof(p, vma))
                         continue;
 
-                /* Skip small VMAs. They are not likely to be of relevance */
-                if (vma->vm_end - vma->vm_start < HPAGE_SIZE)
+                /*
+                 * Shared library pages mapped by multiple processes are not
+                 * migrated as it is expected they are cache replicated. Avoid
+                 * hinting faults in read-only file-backed mappings or the vdso
+                 * as migrating the pages will be of marginal benefit.
+                 */
+                if (!vma->vm_mm ||
+                    (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ)))
                         continue;
 
                 do {
                         start = max(start, vma->vm_start);
                         end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE);
                         end = min(end, vma->vm_end);
-                        pages -= change_prot_numa(vma, start, end);
+                        nr_pte_updates += change_prot_numa(vma, start, end);
+
+                        /*
+                         * Scan sysctl_numa_balancing_scan_size but ensure that
+                         * at least one PTE is updated so that unused virtual
+                         * address space is quickly skipped.
+                         */
+                        if (nr_pte_updates)
+                                pages -= (end - start) >> PAGE_SHIFT;
 
                         start = end;
                         if (pages <= 0)
@@ -988,10 +1749,10 @@ void task_numa_work(struct callback_head *work)
 
 out:
         /*
-         * It is possible to reach the end of the VMA list but the last few VMAs are
-         * not guaranteed to the vma_migratable. If they are not, we would find the
-         * !migratable VMA on the next scan but not reset the scanner to the start
-         * so check it now.
+         * It is possible to reach the end of the VMA list but the last few
+         * VMAs are not guaranteed to the vma_migratable. If they are not, we
+         * would find the !migratable VMA on the next scan but not reset the
+         * scanner to the start so check it now.
          */
         if (vma)
                 mm->numa_scan_offset = start;
@@ -1025,8 +1786,8 @@ void task_tick_numa(struct rq *rq, struct task_struct *curr)
 
         if (now - curr->node_stamp > period) {
                 if (!curr->node_stamp)
-                        curr->numa_scan_period = sysctl_numa_balancing_scan_period_min;
-                curr->node_stamp = now;
+                        curr->numa_scan_period = task_scan_min(curr);
+                curr->node_stamp += period;
 
                 if (!time_before(jiffies, curr->mm->numa_next_scan)) {
                         init_task_work(work, task_numa_work); /* TODO: move this into sched_fork() */
@@ -1038,6 +1799,14 @@ void task_tick_numa(struct rq *rq, struct task_struct *curr)
 static void task_tick_numa(struct rq *rq, struct task_struct *curr)
 {
 }
+
+static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p)
+{
+}
+
+static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p)
+{
+}
 #endif /* CONFIG_NUMA_BALANCING */
 
 static void
@@ -1047,8 +1816,12 @@ account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
         if (!parent_entity(se))
                 update_load_add(&rq_of(cfs_rq)->load, se->load.weight);
 #ifdef CONFIG_SMP
-        if (entity_is_task(se))
-                list_add(&se->group_node, &rq_of(cfs_rq)->cfs_tasks);
+        if (entity_is_task(se)) {
+                struct rq *rq = rq_of(cfs_rq);
+
+                account_numa_enqueue(rq, task_of(se));
+                list_add(&se->group_node, &rq->cfs_tasks);
+        }
 #endif
         cfs_rq->nr_running++;
 }
@@ -1059,8 +1832,10 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
         update_load_sub(&cfs_rq->load, se->load.weight);
         if (!parent_entity(se))
                 update_load_sub(&rq_of(cfs_rq)->load, se->load.weight);
-        if (entity_is_task(se))
+        if (entity_is_task(se)) {
+                account_numa_dequeue(rq_of(cfs_rq), task_of(se));
                 list_del_init(&se->group_node);
+        }
         cfs_rq->nr_running--;
 }
 
@@ -2070,13 +2845,14 @@ static inline bool cfs_bandwidth_used(void)
         return static_key_false(&__cfs_bandwidth_used);
 }
 
-void account_cfs_bandwidth_used(int enabled, int was_enabled)
+void cfs_bandwidth_usage_inc(void)
+{
+        static_key_slow_inc(&__cfs_bandwidth_used);
+}
+
+void cfs_bandwidth_usage_dec(void)
 {
-        /* only need to count groups transitioning between enabled/!enabled */
-        if (enabled && !was_enabled)
-                static_key_slow_inc(&__cfs_bandwidth_used);
-        else if (!enabled && was_enabled)
-                static_key_slow_dec(&__cfs_bandwidth_used);
+        static_key_slow_dec(&__cfs_bandwidth_used);
 }
 #else /* HAVE_JUMP_LABEL */
 static bool cfs_bandwidth_used(void)
@@ -2084,7 +2860,8 @@ static bool cfs_bandwidth_used(void)
         return true;
 }
 
-void account_cfs_bandwidth_used(int enabled, int was_enabled) {}
+void cfs_bandwidth_usage_inc(void) {}
+void cfs_bandwidth_usage_dec(void) {}
 #endif /* HAVE_JUMP_LABEL */
 
 /*
@@ -2335,6 +3112,8 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
         cfs_rq->throttled_clock = rq_clock(rq);
         raw_spin_lock(&cfs_b->lock);
         list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);
+        if (!cfs_b->timer_active)
+                __start_cfs_bandwidth(cfs_b);
         raw_spin_unlock(&cfs_b->lock);
 }
 
@@ -2448,6 +3227,13 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
         if (idle)
                 goto out_unlock;
 
+        /*
+         * if we have relooped after returning idle once, we need to update our
+         * status as actually running, so that other cpus doing
+         * __start_cfs_bandwidth will stop trying to cancel us.
+         */
+        cfs_b->timer_active = 1;
+
         __refill_cfs_bandwidth_runtime(cfs_b);
 
         if (!throttled) {
@@ -2508,7 +3294,13 @@ static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC;
 /* how long we wait to gather additional slack before distributing */
 static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC;
 
-/* are we near the end of the current quota period? */
+/*
+ * Are we near the end of the current quota period?
+ *
+ * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the
+ * hrtimer base being cleared by __hrtimer_start_range_ns. In the case of
+ * migrate_hrtimers, base is never cleared, so we are fine.
+ */
 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)
 {
         struct hrtimer *refresh_timer = &cfs_b->period_timer;
@@ -2584,10 +3376,12 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
         u64 expires;
 
         /* confirm we're still not at a refresh boundary */
-        if (runtime_refresh_within(cfs_b, min_bandwidth_expiration))
+        raw_spin_lock(&cfs_b->lock);
+        if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {
+                raw_spin_unlock(&cfs_b->lock);
                 return;
+        }
 
-        raw_spin_lock(&cfs_b->lock);
         if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) {
                 runtime = cfs_b->runtime;
                 cfs_b->runtime = 0;
@@ -2708,11 +3502,11 @@ void __start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
          * (timer_active==0 becomes visible before the hrtimer call-back
          * terminates).  In either case we ensure that it's re-programmed
          */
-        while (unlikely(hrtimer_active(&cfs_b->period_timer))) {
+        while (unlikely(hrtimer_active(&cfs_b->period_timer)) &&
+               hrtimer_try_to_cancel(&cfs_b->period_timer) < 0) {
+                /* bounce the lock to allow do_sched_cfs_period_timer to run */
                 raw_spin_unlock(&cfs_b->lock);
-                /* ensure cfs_b->lock is available while we wait */
-                hrtimer_cancel(&cfs_b->period_timer);
-
+                cpu_relax();
                 raw_spin_lock(&cfs_b->lock);
                 /* if someone else restarted the timer then we're done */
                 if (cfs_b->timer_active)
@@ -3113,7 +3907,7 @@ static long effective_load(struct task_group *tg, int cpu, long wl, long wg)
 {
         struct sched_entity *se = tg->se[cpu];
 
-        if (!tg->parent)        /* the trivial, non-cgroup case */
+        if (!tg->parent || !wl) /* the trivial, non-cgroup case */
                 return wl;
 
         for_each_sched_entity(se) {
@@ -3166,8 +3960,7 @@ static long effective_load(struct task_group *tg, int cpu, long wl, long wg)
 }
 #else
 
-static inline unsigned long effective_load(struct task_group *tg, int cpu,
-                unsigned long wl, unsigned long wg)
+static long effective_load(struct task_group *tg, int cpu, long wl, long wg)
 {
         return wl;
 }
@@ -3420,11 +4213,10 @@ done:
  * preempt must be disabled.
  */
 static int
-select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags)
+select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags)
 {
         struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL;
         int cpu = smp_processor_id();
-        int prev_cpu = task_cpu(p);
         int new_cpu = cpu;
         int want_affine = 0;
         int sync = wake_flags & WF_SYNC;
@@ -3904,9 +4696,12 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preemp
 
 static unsigned long __read_mostly max_load_balance_interval = HZ/10;
 
+enum fbq_type { regular, remote, all };
+
 #define LBF_ALL_PINNED  0x01
 #define LBF_NEED_BREAK  0x02
-#define LBF_SOME_PINNED 0x04
+#define LBF_DST_PINNED  0x04
+#define LBF_SOME_PINNED 0x08
 
 struct lb_env {
         struct sched_domain     *sd;
@@ -3929,6 +4724,8 @@ struct lb_env {
         unsigned int            loop;
         unsigned int            loop_break;
         unsigned int            loop_max;
+
+        enum fbq_type           fbq_type;
 };
 
 /*
@@ -3975,6 +4772,78 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
         return delta < (s64)sysctl_sched_migration_cost;
 }
 
+#ifdef CONFIG_NUMA_BALANCING
+/* Returns true if the destination node has incurred more faults */
+static bool migrate_improves_locality(struct task_struct *p, struct lb_env *env)
+{
+        int src_nid, dst_nid;
+
+        if (!sched_feat(NUMA_FAVOUR_HIGHER) || !p->numa_faults ||
+            !(env->sd->flags & SD_NUMA)) {
+                return false;
+        }
+
+        src_nid = cpu_to_node(env->src_cpu);
+        dst_nid = cpu_to_node(env->dst_cpu);
+
+        if (src_nid == dst_nid)
+                return false;
+
+        /* Always encourage migration to the preferred node. */
+        if (dst_nid == p->numa_preferred_nid)
+                return true;
+
+        /* If both task and group weight improve, this move is a winner. */
+        if (task_weight(p, dst_nid) > task_weight(p, src_nid) &&
+            group_weight(p, dst_nid) > group_weight(p, src_nid))
+                return true;
+
+        return false;
+}
+
+
+static bool migrate_degrades_locality(struct task_struct *p, struct lb_env *env)
+{
+        int src_nid, dst_nid;
+
+        if (!sched_feat(NUMA) || !sched_feat(NUMA_RESIST_LOWER))
+                return false;
+
+        if (!p->numa_faults || !(env->sd->flags & SD_NUMA))
+                return false;
+
+        src_nid = cpu_to_node(env->src_cpu);
+        dst_nid = cpu_to_node(env->dst_cpu);
+
+        if (src_nid == dst_nid)
+                return false;
+
+        /* Migrating away from the preferred node is always bad. */
+        if (src_nid == p->numa_preferred_nid)
+                return true;
+
+        /* If either task or group weight get worse, don't do it. */
+        if (task_weight(p, dst_nid) < task_weight(p, src_nid) ||
+            group_weight(p, dst_nid) < group_weight(p, src_nid))
+                return true;
+
+        return false;
+}
+
+#else
+static inline bool migrate_improves_locality(struct task_struct *p,
+                                             struct lb_env *env)
+{
+        return false;
+}
+
+static inline bool migrate_degrades_locality(struct task_struct *p,
+                                             struct lb_env *env)
+{
+        return false;
+}
+#endif
+
 /*
  * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
  */
@@ -3997,6 +4866,8 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
 
                 schedstat_inc(p, se.statistics.nr_failed_migrations_affine);
 
+                env->flags |= LBF_SOME_PINNED;
+
                 /*
                  * Remember if this task can be migrated to any other cpu in
                  * our sched_group. We may want to revisit it if we couldn't
@@ -4005,13 +4876,13 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
                  * Also avoid computing new_dst_cpu if we have already computed
                  * one in current iteration.
                  */
-                if (!env->dst_grpmask || (env->flags & LBF_SOME_PINNED))
+                if (!env->dst_grpmask || (env->flags & LBF_DST_PINNED))
                         return 0;
 
                 /* Prevent to re-select dst_cpu via env's cpus */
                 for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) {
                         if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) {
-                                env->flags |= LBF_SOME_PINNED;
+                                env->flags |= LBF_DST_PINNED;
                                 env->new_dst_cpu = cpu;
                                 break;
                         }
@@ -4030,11 +4901,24 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
 
         /*
          * Aggressive migration if:
-         * 1) task is cache cold, or
-         * 2) too many balance attempts have failed.
+         * 1) destination numa is preferred
+         * 2) task is cache cold, or
+         * 3) too many balance attempts have failed.
          */
-
         tsk_cache_hot = task_hot(p, rq_clock_task(env->src_rq), env->sd);
+        if (!tsk_cache_hot)
+                tsk_cache_hot = migrate_degrades_locality(p, env);
+
+        if (migrate_improves_locality(p, env)) {
+#ifdef CONFIG_SCHEDSTATS
+                if (tsk_cache_hot) {
+                        schedstat_inc(env->sd, lb_hot_gained[env->idle]);
+                        schedstat_inc(p, se.statistics.nr_forced_migrations);
+                }
+#endif
+                return 1;
+        }
+
         if (!tsk_cache_hot ||
                 env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
 
@@ -4077,8 +4961,6 @@ static int move_one_task(struct lb_env *env)
         return 0;
 }
 
-static unsigned long task_h_load(struct task_struct *p);
-
 static const unsigned int sched_nr_migrate_break = 32;
 
 /*
@@ -4291,6 +5173,10 @@ struct sg_lb_stats {
         unsigned int group_weight;
         int group_imb; /* Is there an imbalance in the group ? */
         int group_has_capacity; /* Is there extra capacity in the group? */
+#ifdef CONFIG_NUMA_BALANCING
+        unsigned int nr_numa_running;
+        unsigned int nr_preferred_running;
+#endif
 };
 
 /*
@@ -4330,7 +5216,7 @@ static inline void init_sd_lb_stats(struct sd_lb_stats *sds)
 /**
  * get_sd_load_idx - Obtain the load index for a given sched domain.
  * @sd: The sched_domain whose load_idx is to be obtained.
- * @idle: The Idle status of the CPU for whose sd load_icx is obtained.
+ * @idle: The idle status of the CPU for whose sd load_idx is obtained.
  *
  * Return: The load index.
  */
@@ -4447,7 +5333,7 @@ void update_group_power(struct sched_domain *sd, int cpu)
 {
         struct sched_domain *child = sd->child;
         struct sched_group *group, *sdg = sd->groups;
-        unsigned long power;
+        unsigned long power, power_orig;
         unsigned long interval;
 
         interval = msecs_to_jiffies(sd->balance_interval);
@@ -4459,7 +5345,7 @@ void update_group_power(struct sched_domain *sd, int cpu)
                 return;
         }
 
-        power = 0;
+        power_orig = power = 0;
 
         if (child->flags & SD_OVERLAP) {
                 /*
@@ -4467,8 +5353,12 @@ void update_group_power(struct sched_domain *sd, int cpu)
                  * span the current group.
                  */
 
-                for_each_cpu(cpu, sched_group_cpus(sdg))
-                        power += power_of(cpu);
+                for_each_cpu(cpu, sched_group_cpus(sdg)) {
+                        struct sched_group *sg = cpu_rq(cpu)->sd->groups;
+
+                        power_orig += sg->sgp->power_orig;
+                        power += sg->sgp->power;
+                }
         } else  {
                 /*
                  * !SD_OVERLAP domains can assume that child groups
@@ -4477,12 +5367,14 @@ void update_group_power(struct sched_domain *sd, int cpu)
 
                 group = child->groups;
                 do {
+                        power_orig += group->sgp->power_orig;
                         power += group->sgp->power;
                         group = group->next;
                 } while (group != child->groups);
         }
 
-        sdg->sgp->power_orig = sdg->sgp->power = power;
+        sdg->sgp->power_orig = power_orig;
+        sdg->sgp->power = power;
 }
 
 /*
@@ -4526,13 +5418,12 @@ fix_small_capacity(struct sched_domain *sd, struct sched_group *group)
  * cpu 3 and leave one of the cpus in the second group unused.
  *
  * The current solution to this issue is detecting the skew in the first group
- * by noticing it has a cpu that is overloaded while the remaining cpus are
- * idle -- or rather, there's a distinct imbalance in the cpus; see
- * sg_imbalanced().
+ * by noticing the lower domain failed to reach balance and had difficulty
+ * moving tasks due to affinity constraints.
  *
  * When this is so detected; this group becomes a candidate for busiest; see
- * update_sd_pick_busiest(). And calculcate_imbalance() and
- * find_busiest_group() avoid some of the usual balance conditional to allow it
+ * update_sd_pick_busiest(). And calculate_imbalance() and
+ * find_busiest_group() avoid some of the usual balance conditions to allow it
  * to create an effective group imbalance.
  *
  * This is a somewhat tricky proposition since the next run might not find the
@@ -4540,49 +5431,36 @@ fix_small_capacity(struct sched_domain *sd, struct sched_group *group)
  * subtle and fragile situation.
  */
 
-struct sg_imb_stats {
-        unsigned long max_nr_running, min_nr_running;
-        unsigned long max_cpu_load, min_cpu_load;
-};
-
-static inline void init_sg_imb_stats(struct sg_imb_stats *sgi)
+static inline int sg_imbalanced(struct sched_group *group)
 {
-        sgi->max_cpu_load = sgi->max_nr_running = 0UL;
-        sgi->min_cpu_load = sgi->min_nr_running = ~0UL;
+        return group->sgp->imbalance;
 }
 
-static inline void
-update_sg_imb_stats(struct sg_imb_stats *sgi,
-                    unsigned long load, unsigned long nr_running)
+/*
+ * Compute the group capacity.
+ *
+ * Avoid the issue where N*frac(smt_power) >= 1 creates 'phantom' cores by
+ * first dividing out the smt factor and computing the actual number of cores
+ * and limit power unit capacity with that.
+ */
+static inline int sg_capacity(struct lb_env *env, struct sched_group *group)
 {
-        if (load > sgi->max_cpu_load)
-                sgi->max_cpu_load = load;
-        if (sgi->min_cpu_load > load)
-                sgi->min_cpu_load = load;
+        unsigned int capacity, smt, cpus;
+        unsigned int power, power_orig;
 
-        if (nr_running > sgi->max_nr_running)
-                sgi->max_nr_running = nr_running;
-        if (sgi->min_nr_running > nr_running)
-                sgi->min_nr_running = nr_running;
-}
+        power = group->sgp->power;
+        power_orig = group->sgp->power_orig;
+        cpus = group->group_weight;
 
-static inline int
-sg_imbalanced(struct sg_lb_stats *sgs, struct sg_imb_stats *sgi)
-{
-        /*
-         * Consider the group unbalanced when the imbalance is larger
-         * than the average weight of a task.
-         *
-         * APZ: with cgroup the avg task weight can vary wildly and
-         *      might not be a suitable number - should we keep a
-         *      normalized nr_running number somewhere that negates
-         *      the hierarchy?
-         */
-        if ((sgi->max_cpu_load - sgi->min_cpu_load) >= sgs->load_per_task &&
-            (sgi->max_nr_running - sgi->min_nr_running) > 1)
-                return 1;
+        /* smt := ceil(cpus / power), assumes: 1 < smt_power < 2 */
+        smt = DIV_ROUND_UP(SCHED_POWER_SCALE * cpus, power_orig);
+        capacity = cpus / smt; /* cores */
 
-        return 0;
+        capacity = min_t(unsigned, capacity, DIV_ROUND_CLOSEST(power, SCHED_POWER_SCALE));
+        if (!capacity)
+                capacity = fix_small_capacity(env->sd, group);
+
+        return capacity;
 }
 
 /**
@@ -4597,12 +5475,11 @@ static inline void update_sg_lb_stats(struct lb_env *env,
                         struct sched_group *group, int load_idx,
                         int local_group, struct sg_lb_stats *sgs)
 {
-        struct sg_imb_stats sgi;
         unsigned long nr_running;
         unsigned long load;
         int i;
 
-        init_sg_imb_stats(&sgi);
+        memset(sgs, 0, sizeof(*sgs));
 
         for_each_cpu_and(i, sched_group_cpus(group), env->cpus) {
                 struct rq *rq = cpu_rq(i);
@@ -4610,24 +5487,22 @@ static inline void update_sg_lb_stats(struct lb_env *env,
                 nr_running = rq->nr_running;
 
                 /* Bias balancing toward cpus of our domain */
-                if (local_group) {
+                if (local_group)
                         load = target_load(i, load_idx);
-                } else {
+                else
                         load = source_load(i, load_idx);
-                        update_sg_imb_stats(&sgi, load, nr_running);
-                }
 
                 sgs->group_load += load;
                 sgs->sum_nr_running += nr_running;
+#ifdef CONFIG_NUMA_BALANCING
+                sgs->nr_numa_running += rq->nr_numa_running;
+                sgs->nr_preferred_running += rq->nr_preferred_running;
+#endif
                 sgs->sum_weighted_load += weighted_cpuload(i);
                 if (idle_cpu(i))
                         sgs->idle_cpus++;
         }
 
-        if (local_group && (env->idle != CPU_NEWLY_IDLE ||
-                        time_after_eq(jiffies, group->sgp->next_update)))
-                update_group_power(env->sd, env->dst_cpu);
-
         /* Adjust by relative CPU power of the group */
         sgs->group_power = group->sgp->power;
         sgs->avg_load = (sgs->group_load*SCHED_POWER_SCALE) / sgs->group_power;
@@ -4635,16 +5510,11 @@ static inline void update_sg_lb_stats(struct lb_env *env,
         if (sgs->sum_nr_running)
                 sgs->load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running;
 
-        sgs->group_imb = sg_imbalanced(sgs, &sgi);
-
-        sgs->group_capacity =
-                DIV_ROUND_CLOSEST(sgs->group_power, SCHED_POWER_SCALE);
-
-        if (!sgs->group_capacity)
-                sgs->group_capacity = fix_small_capacity(env->sd, group);
-
         sgs->group_weight = group->group_weight;
 
+        sgs->group_imb = sg_imbalanced(group);
+        sgs->group_capacity = sg_capacity(env, group);
+
         if (sgs->group_capacity > sgs->sum_nr_running)
                 sgs->group_has_capacity = 1;
 }
@@ -4693,14 +5563,42 @@ static bool update_sd_pick_busiest(struct lb_env *env,
         return false;
 }
 
+#ifdef CONFIG_NUMA_BALANCING
+static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
+{
+        if (sgs->sum_nr_running > sgs->nr_numa_running)
+                return regular;
+        if (sgs->sum_nr_running > sgs->nr_preferred_running)
+                return remote;
+        return all;
+}
+
+static inline enum fbq_type fbq_classify_rq(struct rq *rq)
+{
+        if (rq->nr_running > rq->nr_numa_running)
+                return regular;
+        if (rq->nr_running > rq->nr_preferred_running)
+                return remote;
+        return all;
+}
+#else
+static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
+{
+        return all;
+}
+
+static inline enum fbq_type fbq_classify_rq(struct rq *rq)
+{
+        return regular;
+}
+#endif /* CONFIG_NUMA_BALANCING */
+
 /**
  * update_sd_lb_stats - Update sched_domain's statistics for load balancing.
  * @env: The load balancing environment.
- * @balance: Should we balance.
  * @sds: variable to hold the statistics for this sched_domain.
  */
-static inline void update_sd_lb_stats(struct lb_env *env,
-                                        struct sd_lb_stats *sds)
+static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds)
 {
         struct sched_domain *child = env->sd->child;
         struct sched_group *sg = env->sd->groups;
@@ -4720,11 +5618,17 @@ static inline void update_sd_lb_stats(struct lb_env *env,
                 if (local_group) {
                         sds->local = sg;
                         sgs = &sds->local_stat;
+
+                        if (env->idle != CPU_NEWLY_IDLE ||
+                            time_after_eq(jiffies, sg->sgp->next_update))
+                                update_group_power(env->sd, env->dst_cpu);
                 }
 
-                memset(sgs, 0, sizeof(*sgs));
                 update_sg_lb_stats(env, sg, load_idx, local_group, sgs);
 
+                if (local_group)
+                        goto next_group;
+
                 /*
                  * In case the child domain prefers tasks go to siblings
                  * first, lower the sg capacity to one so that we'll try
@@ -4735,21 +5639,25 @@ static inline void update_sd_lb_stats(struct lb_env *env,
                  * heaviest group when it is already under-utilized (possible
                  * with a large weight task outweighs the tasks on the system).
                  */
-                if (prefer_sibling && !local_group &&
-                                sds->local && sds->local_stat.group_has_capacity)
+                if (prefer_sibling && sds->local &&
+                    sds->local_stat.group_has_capacity)
                         sgs->group_capacity = min(sgs->group_capacity, 1U);
 
-                /* Now, start updating sd_lb_stats */
-                sds->total_load += sgs->group_load;
-                sds->total_pwr += sgs->group_power;
-
-                if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) {
+                if (update_sd_pick_busiest(env, sds, sg, sgs)) {
                         sds->busiest = sg;
                         sds->busiest_stat = *sgs;
                 }
 
+next_group:
+                /* Now, start updating sd_lb_stats */
+                sds->total_load += sgs->group_load;
+                sds->total_pwr += sgs->group_power;
+
                 sg = sg->next;
         } while (sg != env->sd->groups);
+
+        if (env->sd->flags & SD_NUMA)
+                env->fbq_type = fbq_classify_group(&sds->busiest_stat);
 }
 
 /**
@@ -5053,15 +5961,39 @@ static struct rq *find_busiest_queue(struct lb_env *env,
         int i;
 
         for_each_cpu_and(i, sched_group_cpus(group), env->cpus) {
-                unsigned long power = power_of(i);
-                unsigned long capacity = DIV_ROUND_CLOSEST(power,
-                                                           SCHED_POWER_SCALE);
-                unsigned long wl;
+                unsigned long power, capacity, wl;
+                enum fbq_type rt;
+
+                rq = cpu_rq(i);
+                rt = fbq_classify_rq(rq);
+
+                /*
+                 * We classify groups/runqueues into three groups:
+                 *  - regular: there are !numa tasks
+                 *  - remote:  there are numa tasks that run on the 'wrong' node
+                 *  - all:     there is no distinction
+                 *
+                 * In order to avoid migrating ideally placed numa tasks,
+                 * ignore those when there's better options.
+                 *
+                 * If we ignore the actual busiest queue to migrate another
+                 * task, the next balance pass can still reduce the busiest
+                 * queue by moving tasks around inside the node.
+                 *
+                 * If we cannot move enough load due to this classification
+                 * the next pass will adjust the group classification and
+                 * allow migration of more tasks.
+                 *
+                 * Both cases only affect the total convergence complexity.
+                 */
+                if (rt > env->fbq_type)
+                        continue;
 
+                power = power_of(i);
+                capacity = DIV_ROUND_CLOSEST(power, SCHED_POWER_SCALE);
                 if (!capacity)
                         capacity = fix_small_capacity(env->sd, group);
 
-                rq = cpu_rq(i);
                 wl = weighted_cpuload(i);
 
                 /*
@@ -5164,6 +6096,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
                         int *continue_balancing)
 {
         int ld_moved, cur_ld_moved, active_balance = 0;
+        struct sched_domain *sd_parent = sd->parent;
         struct sched_group *group;
         struct rq *busiest;
         unsigned long flags;
@@ -5177,6 +6110,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
                 .idle           = idle,
                 .loop_break     = sched_nr_migrate_break,
                 .cpus           = cpus,
+                .fbq_type       = all,
         };
 
         /*
@@ -5268,17 +6202,17 @@ more_balance:
                  * moreover subsequent load balance cycles should correct the
                  * excess load moved.
                  */
-                if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) {
+                if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) {
+
+                        /* Prevent to re-select dst_cpu via env's cpus */
+                        cpumask_clear_cpu(env.dst_cpu, env.cpus);
 
                         env.dst_rq       = cpu_rq(env.new_dst_cpu);
                         env.dst_cpu      = env.new_dst_cpu;
-                        env.flags       &= ~LBF_SOME_PINNED;
+                        env.flags       &= ~LBF_DST_PINNED;
                         env.loop         = 0;
                         env.loop_break   = sched_nr_migrate_break;
 
-                        /* Prevent to re-select dst_cpu via env's cpus */
-                        cpumask_clear_cpu(env.dst_cpu, env.cpus);
-
                         /*
                          * Go back to "more_balance" rather than "redo" since we
                          * need to continue with same src_cpu.
@@ -5286,6 +6220,18 @@ more_balance:
                         goto more_balance;
                 }
 
+                /*
+                 * We failed to reach balance because of affinity.
+                 */
+                if (sd_parent) {
+                        int *group_imbalance = &sd_parent->groups->sgp->imbalance;
+
+                        if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) {
+                                *group_imbalance = 1;
+                        } else if (*group_imbalance)
+                                *group_imbalance = 0;
+                }
+
                 /* All tasks on this runqueue were pinned by CPU affinity */
                 if (unlikely(env.flags & LBF_ALL_PINNED)) {
                         cpumask_clear_cpu(cpu_of(busiest), cpus);
@@ -5393,6 +6339,7 @@ void idle_balance(int this_cpu, struct rq *this_rq)
         struct sched_domain *sd;
         int pulled_task = 0;
         unsigned long next_balance = jiffies + HZ;
+        u64 curr_cost = 0;
 
         this_rq->idle_stamp = rq_clock(this_rq);
 
@@ -5409,15 +6356,27 @@ void idle_balance(int this_cpu, struct rq *this_rq)
         for_each_domain(this_cpu, sd) {
                 unsigned long interval;
                 int continue_balancing = 1;
+                u64 t0, domain_cost;
 
                 if (!(sd->flags & SD_LOAD_BALANCE))
                         continue;
 
+                if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost)
+                        break;
+
                 if (sd->flags & SD_BALANCE_NEWIDLE) {
+                        t0 = sched_clock_cpu(this_cpu);
+
                         /* If we've pulled tasks over stop searching: */
                         pulled_task = load_balance(this_cpu, this_rq,
                                                    sd, CPU_NEWLY_IDLE,
                                                    &continue_balancing);
+
+                        domain_cost = sched_clock_cpu(this_cpu) - t0;
+                        if (domain_cost > sd->max_newidle_lb_cost)
+                                sd->max_newidle_lb_cost = domain_cost;
+
+                        curr_cost += domain_cost;
                 }
 
                 interval = msecs_to_jiffies(sd->balance_interval);
@@ -5439,6 +6398,9 @@ void idle_balance(int this_cpu, struct rq *this_rq)
                  */
                 this_rq->next_balance = next_balance;
         }
+
+        if (curr_cost > this_rq->max_idle_balance_cost)
+                this_rq->max_idle_balance_cost = curr_cost;
 }
 
 /*
@@ -5572,16 +6534,16 @@ static inline void nohz_balance_exit_idle(int cpu)
 static inline void set_cpu_sd_state_busy(void)
 {
         struct sched_domain *sd;
+        int cpu = smp_processor_id();
 
         rcu_read_lock();
-        sd = rcu_dereference_check_sched_domain(this_rq()->sd);
+        sd = rcu_dereference(per_cpu(sd_busy, cpu));
 
         if (!sd || !sd->nohz_idle)
                 goto unlock;
         sd->nohz_idle = 0;
 
-        for (; sd; sd = sd->parent)
-                atomic_inc(&sd->groups->sgp->nr_busy_cpus);
+        atomic_inc(&sd->groups->sgp->nr_busy_cpus);
 unlock:
         rcu_read_unlock();
 }
@@ -5589,16 +6551,16 @@ unlock:
 void set_cpu_sd_state_idle(void)
 {
         struct sched_domain *sd;
+        int cpu = smp_processor_id();
 
         rcu_read_lock();
-        sd = rcu_dereference_check_sched_domain(this_rq()->sd);
+        sd = rcu_dereference(per_cpu(sd_busy, cpu));
 
         if (!sd || sd->nohz_idle)
                 goto unlock;
         sd->nohz_idle = 1;
 
-        for (; sd; sd = sd->parent)
-                atomic_dec(&sd->groups->sgp->nr_busy_cpus);
+        atomic_dec(&sd->groups->sgp->nr_busy_cpus);
 unlock:
         rcu_read_unlock();
 }
@@ -5662,15 +6624,39 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
         /* Earliest time when we have to do rebalance again */
         unsigned long next_balance = jiffies + 60*HZ;
         int update_next_balance = 0;
-        int need_serialize;
+        int need_serialize, need_decay = 0;
+        u64 max_cost = 0;
 
         update_blocked_averages(cpu);
 
         rcu_read_lock();
         for_each_domain(cpu, sd) {
+                /*
+                 * Decay the newidle max times here because this is a regular
+                 * visit to all the domains. Decay ~1% per second.
+                 */
+                if (time_after(jiffies, sd->next_decay_max_lb_cost)) {
+                        sd->max_newidle_lb_cost =
+                                (sd->max_newidle_lb_cost * 253) / 256;
+                        sd->next_decay_max_lb_cost = jiffies + HZ;
+                        need_decay = 1;
+                }
+                max_cost += sd->max_newidle_lb_cost;
+
                 if (!(sd->flags & SD_LOAD_BALANCE))
                         continue;
 
+                /*
+                 * Stop the load balance at this level. There is another
+                 * CPU in our sched group which is doing load balancing more
+                 * actively.
+                 */
+                if (!continue_balancing) {
+                        if (need_decay)
+                                continue;
+                        break;
+                }
+
                 interval = sd->balance_interval;
                 if (idle != CPU_IDLE)
                         interval *= sd->busy_factor;
@@ -5689,7 +6675,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
                 if (time_after_eq(jiffies, sd->last_balance + interval)) {
                         if (load_balance(cpu, rq, sd, idle, &continue_balancing)) {
                                 /*
-                                 * The LBF_SOME_PINNED logic could have changed
+                                 * The LBF_DST_PINNED logic could have changed
                                  * env->dst_cpu, so we can't know our idle
                                  * state even if we migrated tasks. Update it.
                                  */
@@ -5704,14 +6690,14 @@ out:
                         next_balance = sd->last_balance + interval;
                         update_next_balance = 1;
                 }
-
+        }
+        if (need_decay) {
                 /*
-                 * Stop the load balance at this level. There is another
-                 * CPU in our sched group which is doing load balancing more
-                 * actively.
+                 * Ensure the rq-wide value also decays but keep it at a
+                 * reasonable floor to avoid funnies with rq->avg_idle.
                  */
-                if (!continue_balancing)
-                        break;
+                rq->max_idle_balance_cost =
+                        max((u64)sysctl_sched_migration_cost, max_cost);
         }
         rcu_read_unlock();
 
@@ -5781,6 +6767,8 @@ static inline int nohz_kick_needed(struct rq *rq, int cpu)
 {
         unsigned long now = jiffies;
         struct sched_domain *sd;
+        struct sched_group_power *sgp;
+        int nr_busy;
 
         if (unlikely(idle_cpu(cpu)))
                 return 0;
@@ -5806,22 +6794,22 @@ static inline int nohz_kick_needed(struct rq *rq, int cpu)
                 goto need_kick;
 
         rcu_read_lock();
-        for_each_domain(cpu, sd) {
-                struct sched_group *sg = sd->groups;
-                struct sched_group_power *sgp = sg->sgp;
-                int nr_busy = atomic_read(&sgp->nr_busy_cpus);
+        sd = rcu_dereference(per_cpu(sd_busy, cpu));
 
-                if (sd->flags & SD_SHARE_PKG_RESOURCES && nr_busy > 1)
-                        goto need_kick_unlock;
+        if (sd) {
+                sgp = sd->groups->sgp;
+                nr_busy = atomic_read(&sgp->nr_busy_cpus);
 
-                if (sd->flags & SD_ASYM_PACKING && nr_busy != sg->group_weight
-                    && (cpumask_first_and(nohz.idle_cpus_mask,
-                                          sched_domain_span(sd)) < cpu))
+                if (nr_busy > 1)
                         goto need_kick_unlock;
-
-                if (!(sd->flags & (SD_SHARE_PKG_RESOURCES | SD_ASYM_PACKING)))
-                        break;
         }
+
+        sd = rcu_dereference(per_cpu(sd_asym, cpu));
+
+        if (sd && (cpumask_first_and(nohz.idle_cpus_mask,
+                                  sched_domain_span(sd)) < cpu))
+                goto need_kick_unlock;
+
         rcu_read_unlock();
         return 0;
 
@@ -6214,7 +7202,8 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
                 se->cfs_rq = parent->my_q;
 
         se->my_q = cfs_rq;
-        update_load_set(&se->load, 0);
+        /* guarantee group entities always have weight */
+        update_load_set(&se->load, NICE_0_LOAD);
         se->parent = parent;
 }
 
diff --git a/kernel/sched/features.h b/kernel/sched/features.h
index 99399f8e4799..5716929a2e3a 100644
--- a/kernel/sched/features.h
+++ b/kernel/sched/features.h
@@ -63,10 +63,23 @@ SCHED_FEAT(LB_MIN, false)
 /*
  * Apply the automatic NUMA scheduling policy. Enabled automatically
  * at runtime if running on a NUMA machine. Can be controlled via
- * numa_balancing=. Allow PTE scanning to be forced on UMA machines
- * for debugging the core machinery.
+ * numa_balancing=
  */
 #ifdef CONFIG_NUMA_BALANCING
 SCHED_FEAT(NUMA,        false)
-SCHED_FEAT(NUMA_FORCE,  false)
+
+/*
+ * NUMA_FAVOUR_HIGHER will favor moving tasks towards nodes where a
+ * higher number of hinting faults are recorded during active load
+ * balancing.
+ */
+SCHED_FEAT(NUMA_FAVOUR_HIGHER, true)
+
+/*
+ * NUMA_RESIST_LOWER will resist moving tasks towards nodes where a
+ * lower number of hinting faults have been recorded. As this has
+ * the potential to prevent a task ever migrating to a new node
+ * due to CPU overload it is disabled by default.
+ */
+SCHED_FEAT(NUMA_RESIST_LOWER, false)
 #endif
diff --git a/kernel/sched/idle_task.c b/kernel/sched/idle_task.c
index d8da01008d39..516c3d9ceea1 100644
--- a/kernel/sched/idle_task.c
+++ b/kernel/sched/idle_task.c
@@ -9,7 +9,7 @@
 
 #ifdef CONFIG_SMP
 static int
-select_task_rq_idle(struct task_struct *p, int sd_flag, int flags)
+select_task_rq_idle(struct task_struct *p, int cpu, int sd_flag, int flags)
 {
         return task_cpu(p); /* IDLE tasks as never migrated */
 }
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 01970c8e64df..7d57275fc396 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -246,8 +246,10 @@ static inline void rt_set_overload(struct rq *rq)
          * if we should look at the mask. It would be a shame
          * if we looked at the mask, but the mask was not
          * updated yet.
+         *
+         * Matched by the barrier in pull_rt_task().
          */
-        wmb();
+        smp_wmb();
         atomic_inc(&rq->rd->rto_count);
 }
 
@@ -1169,13 +1171,10 @@ static void yield_task_rt(struct rq *rq)
 static int find_lowest_rq(struct task_struct *task);
 
 static int
-select_task_rq_rt(struct task_struct *p, int sd_flag, int flags)
+select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags)
 {
         struct task_struct *curr;
         struct rq *rq;
-        int cpu;
-
-        cpu = task_cpu(p);
 
         if (p->nr_cpus_allowed == 1)
                 goto out;
@@ -1213,8 +1212,7 @@ select_task_rq_rt(struct task_struct *p, int sd_flag, int flags)
          */
         if (curr && unlikely(rt_task(curr)) &&
             (curr->nr_cpus_allowed < 2 ||
-             curr->prio <= p->prio) &&
-            (p->nr_cpus_allowed > 1)) {
+             curr->prio <= p->prio)) {
                 int target = find_lowest_rq(p);
 
                 if (target != -1)
@@ -1630,6 +1628,12 @@ static int pull_rt_task(struct rq *this_rq)
         if (likely(!rt_overloaded(this_rq)))
                 return 0;
 
+        /*
+         * Match the barrier from rt_set_overloaded; this guarantees that if we
+         * see overloaded we must also see the rto_mask bit.
+         */
+        smp_rmb();
+
         for_each_cpu(cpu, this_rq->rd->rto_mask) {
                 if (this_cpu == cpu)
                         continue;
@@ -1931,8 +1935,8 @@ static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued)
         p->rt.time_slice = sched_rr_timeslice;
 
         /*
-         * Requeue to the end of queue if we (and all of our ancestors) are the
-         * only element on the queue
+         * Requeue to the end of queue if we (and all of our ancestors) are not
+         * the only element on the queue
          */
         for_each_sched_rt_entity(rt_se) {
                 if (rt_se->run_list.prev != rt_se->run_list.next) {
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index b3c5653e1dca..88c85b21d633 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -6,6 +6,7 @@
 #include <linux/spinlock.h>
 #include <linux/stop_machine.h>
 #include <linux/tick.h>
+#include <linux/slab.h>
 
 #include "cpupri.h"
 #include "cpuacct.h"
@@ -408,6 +409,10 @@ struct rq {
          * remote CPUs use both these fields when doing load calculation.
          */
         unsigned int nr_running;
+#ifdef CONFIG_NUMA_BALANCING
+        unsigned int nr_numa_running;
+        unsigned int nr_preferred_running;
+#endif
         #define CPU_LOAD_IDX_MAX 5
         unsigned long cpu_load[CPU_LOAD_IDX_MAX];
         unsigned long last_load_update_tick;
@@ -476,6 +481,9 @@ struct rq {
         u64 age_stamp;
         u64 idle_stamp;
         u64 avg_idle;
+
+        /* This is used to determine avg_idle's max value */
+        u64 max_idle_balance_cost;
 #endif
 
 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
@@ -552,6 +560,12 @@ static inline u64 rq_clock_task(struct rq *rq)
         return rq->clock_task;
 }
 
+#ifdef CONFIG_NUMA_BALANCING
+extern void sched_setnuma(struct task_struct *p, int node);
+extern int migrate_task_to(struct task_struct *p, int cpu);
+extern int migrate_swap(struct task_struct *, struct task_struct *);
+#endif /* CONFIG_NUMA_BALANCING */
+
 #ifdef CONFIG_SMP
 
 #define rcu_dereference_check_sched_domain(p) \
@@ -593,9 +607,24 @@ static inline struct sched_domain *highest_flag_domain(int cpu, int flag)
         return hsd;
 }
 
+static inline struct sched_domain *lowest_flag_domain(int cpu, int flag)
+{
+        struct sched_domain *sd;
+
+        for_each_domain(cpu, sd) {
+                if (sd->flags & flag)
+                        break;
+        }
+
+        return sd;
+}
+
 DECLARE_PER_CPU(struct sched_domain *, sd_llc);
 DECLARE_PER_CPU(int, sd_llc_size);
 DECLARE_PER_CPU(int, sd_llc_id);
+DECLARE_PER_CPU(struct sched_domain *, sd_numa);
+DECLARE_PER_CPU(struct sched_domain *, sd_busy);
+DECLARE_PER_CPU(struct sched_domain *, sd_asym);
 
 struct sched_group_power {
         atomic_t ref;
@@ -605,6 +634,7 @@ struct sched_group_power {
          */
         unsigned int power, power_orig;
         unsigned long next_update;
+        int imbalance; /* XXX unrelated to power but shared group state */
         /*
          * Number of busy cpus in this group.
          */
@@ -719,6 +749,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
          */
         smp_wmb();
         task_thread_info(p)->cpu = cpu;
+        p->wake_cpu = cpu;
 #endif
 }
 
@@ -974,7 +1005,7 @@ struct sched_class {
         void (*put_prev_task) (struct rq *rq, struct task_struct *p);
 
 #ifdef CONFIG_SMP
-        int  (*select_task_rq)(struct task_struct *p, int sd_flag, int flags);
+        int  (*select_task_rq)(struct task_struct *p, int task_cpu, int sd_flag, int flags);
         void (*migrate_task_rq)(struct task_struct *p, int next_cpu);
 
         void (*pre_schedule) (struct rq *this_rq, struct task_struct *task);
@@ -1220,6 +1251,24 @@ static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
         lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
 }
 
+static inline void double_lock(spinlock_t *l1, spinlock_t *l2)
+{
+        if (l1 > l2)
+                swap(l1, l2);
+
+        spin_lock(l1);
+        spin_lock_nested(l2, SINGLE_DEPTH_NESTING);
+}
+
+static inline void double_raw_lock(raw_spinlock_t *l1, raw_spinlock_t *l2)
+{
+        if (l1 > l2)
+                swap(l1, l2);
+
+        raw_spin_lock(l1);
+        raw_spin_lock_nested(l2, SINGLE_DEPTH_NESTING);
+}
+
 /*
  * double_rq_lock - safely lock two runqueues
  *
@@ -1305,7 +1354,8 @@ extern void print_rt_stats(struct seq_file *m, int cpu);
 extern void init_cfs_rq(struct cfs_rq *cfs_rq);
 extern void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq);
 
-extern void account_cfs_bandwidth_used(int enabled, int was_enabled);
+extern void cfs_bandwidth_usage_inc(void);
+extern void cfs_bandwidth_usage_dec(void);
 
 #ifdef CONFIG_NO_HZ_COMMON
 enum rq_nohz_flag_bits {
diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h
index c7edee71bce8..4ab704339656 100644
--- a/kernel/sched/stats.h
+++ b/kernel/sched/stats.h
@@ -59,9 +59,9 @@ static inline void sched_info_reset_dequeued(struct task_struct *t)
  * from dequeue_task() to account for possible rq->clock skew across cpus. The
  * delta taken on each cpu would annul the skew.
  */
-static inline void sched_info_dequeued(struct task_struct *t)
+static inline void sched_info_dequeued(struct rq *rq, struct task_struct *t)
 {
-        unsigned long long now = rq_clock(task_rq(t)), delta = 0;
+        unsigned long long now = rq_clock(rq), delta = 0;
 
         if (unlikely(sched_info_on()))
                 if (t->sched_info.last_queued)
@@ -69,7 +69,7 @@ static inline void sched_info_dequeued(struct task_struct *t)
         sched_info_reset_dequeued(t);
         t->sched_info.run_delay += delta;
 
-        rq_sched_info_dequeued(task_rq(t), delta);
+        rq_sched_info_dequeued(rq, delta);
 }
 
 /*
@@ -77,9 +77,9 @@ static inline void sched_info_dequeued(struct task_struct *t)
  * long it was waiting to run.  We also note when it began so that we
  * can keep stats on how long its timeslice is.
  */
-static void sched_info_arrive(struct task_struct *t)
+static void sched_info_arrive(struct rq *rq, struct task_struct *t)
 {
-        unsigned long long now = rq_clock(task_rq(t)), delta = 0;
+        unsigned long long now = rq_clock(rq), delta = 0;
 
         if (t->sched_info.last_queued)
                 delta = now - t->sched_info.last_queued;
@@ -88,7 +88,7 @@ static void sched_info_arrive(struct task_struct *t)
         t->sched_info.last_arrival = now;
         t->sched_info.pcount++;
 
-        rq_sched_info_arrive(task_rq(t), delta);
+        rq_sched_info_arrive(rq, delta);
 }
 
 /*
@@ -96,11 +96,11 @@ static void sched_info_arrive(struct task_struct *t)
  * the timestamp if it is already not set.  It's assumed that
  * sched_info_dequeued() will clear that stamp when appropriate.
  */
-static inline void sched_info_queued(struct task_struct *t)
+static inline void sched_info_queued(struct rq *rq, struct task_struct *t)
 {
         if (unlikely(sched_info_on()))
                 if (!t->sched_info.last_queued)
-                        t->sched_info.last_queued = rq_clock(task_rq(t));
+                        t->sched_info.last_queued = rq_clock(rq);
 }
 
 /*
@@ -111,15 +111,15 @@ static inline void sched_info_queued(struct task_struct *t)
  * sched_info_queued() to mark that it has now again started waiting on
  * the runqueue.
  */
-static inline void sched_info_depart(struct task_struct *t)
+static inline void sched_info_depart(struct rq *rq, struct task_struct *t)
 {
-        unsigned long long delta = rq_clock(task_rq(t)) -
+        unsigned long long delta = rq_clock(rq) -
                                         t->sched_info.last_arrival;
 
-        rq_sched_info_depart(task_rq(t), delta);
+        rq_sched_info_depart(rq, delta);
 
         if (t->state == TASK_RUNNING)
-                sched_info_queued(t);
+                sched_info_queued(rq, t);
 }
 
 /*
@@ -128,32 +128,34 @@ static inline void sched_info_depart(struct task_struct *t)
  * the idle task.)  We are only called when prev != next.
  */
 static inline void
-__sched_info_switch(struct task_struct *prev, struct task_struct *next)
+__sched_info_switch(struct rq *rq,
+                    struct task_struct *prev, struct task_struct *next)
 {
-        struct rq *rq = task_rq(prev);
-
         /*
          * prev now departs the cpu.  It's not interesting to record
          * stats about how efficient we were at scheduling the idle
          * process, however.
          */
         if (prev != rq->idle)
-                sched_info_depart(prev);
+                sched_info_depart(rq, prev);
 
         if (next != rq->idle)
-                sched_info_arrive(next);
+                sched_info_arrive(rq, next);
 }
 static inline void
-sched_info_switch(struct task_struct *prev, struct task_struct *next)
+sched_info_switch(struct rq *rq,
+                  struct task_struct *prev, struct task_struct *next)
 {
         if (unlikely(sched_info_on()))
-                __sched_info_switch(prev, next);
+                __sched_info_switch(rq, prev, next);
 }
 #else
-#define sched_info_queued(t)                    do { } while (0)
+#define sched_info_queued(rq, t)                do { } while (0)
 #define sched_info_reset_dequeued(t)    do { } while (0)
-#define sched_info_dequeued(t)                  do { } while (0)
-#define sched_info_switch(t, next)              do { } while (0)
+#define sched_info_dequeued(rq, t)              do { } while (0)
+#define sched_info_depart(rq, t)                do { } while (0)
+#define sched_info_arrive(rq, next)             do { } while (0)
+#define sched_info_switch(rq, t, next)          do { } while (0)
 #endif /* CONFIG_SCHEDSTATS || CONFIG_TASK_DELAY_ACCT */
 
 /*
diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c
index e08fbeeb54b9..47197de8abd9 100644
--- a/kernel/sched/stop_task.c
+++ b/kernel/sched/stop_task.c
@@ -11,7 +11,7 @@
 
 #ifdef CONFIG_SMP
 static int
-select_task_rq_stop(struct task_struct *p, int sd_flag, int flags)
+select_task_rq_stop(struct task_struct *p, int cpu, int sd_flag, int flags)
 {
         return task_cpu(p); /* stop tasks as never migrate */
 }
diff --git a/kernel/wait.c b/kernel/sched/wait.c
index d550920e040c..7d50f794e248 100644
--- a/kernel/wait.c
+++ b/kernel/sched/wait.c
@@ -53,6 +53,109 @@ EXPORT_SYMBOL(remove_wait_queue);
 
 
 /*
+ * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
+ * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
+ * number) then we wake all the non-exclusive tasks and one exclusive task.
+ *
+ * There are circumstances in which we can try to wake a task which has already
+ * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
+ * zero in this (rare) case, and we handle it by continuing to scan the queue.
+ */
+static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
+                        int nr_exclusive, int wake_flags, void *key)
+{
+        wait_queue_t *curr, *next;
+
+        list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
+                unsigned flags = curr->flags;
+
+                if (curr->func(curr, mode, wake_flags, key) &&
+                                (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
+                        break;
+        }
+}
+
+/**
+ * __wake_up - wake up threads blocked on a waitqueue.
+ * @q: the waitqueue
+ * @mode: which threads
+ * @nr_exclusive: how many wake-one or wake-many threads to wake up
+ * @key: is directly passed to the wakeup function
+ *
+ * It may be assumed that this function implies a write memory barrier before
+ * changing the task state if and only if any tasks are woken up.
+ */
+void __wake_up(wait_queue_head_t *q, unsigned int mode,
+                        int nr_exclusive, void *key)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&q->lock, flags);
+        __wake_up_common(q, mode, nr_exclusive, 0, key);
+        spin_unlock_irqrestore(&q->lock, flags);
+}
+EXPORT_SYMBOL(__wake_up);
+
+/*
+ * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
+ */
+void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr)
+{
+        __wake_up_common(q, mode, nr, 0, NULL);
+}
+EXPORT_SYMBOL_GPL(__wake_up_locked);
+
+void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key)
+{
+        __wake_up_common(q, mode, 1, 0, key);
+}
+EXPORT_SYMBOL_GPL(__wake_up_locked_key);
+
+/**
+ * __wake_up_sync_key - wake up threads blocked on a waitqueue.
+ * @q: the waitqueue
+ * @mode: which threads
+ * @nr_exclusive: how many wake-one or wake-many threads to wake up
+ * @key: opaque value to be passed to wakeup targets
+ *
+ * The sync wakeup differs that the waker knows that it will schedule
+ * away soon, so while the target thread will be woken up, it will not
+ * be migrated to another CPU - ie. the two threads are 'synchronized'
+ * with each other. This can prevent needless bouncing between CPUs.
+ *
+ * On UP it can prevent extra preemption.
+ *
+ * It may be assumed that this function implies a write memory barrier before
+ * changing the task state if and only if any tasks are woken up.
+ */
+void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode,
+                        int nr_exclusive, void *key)
+{
+        unsigned long flags;
+        int wake_flags = 1; /* XXX WF_SYNC */
+
+        if (unlikely(!q))
+                return;
+
+        if (unlikely(nr_exclusive != 1))
+                wake_flags = 0;
+
+        spin_lock_irqsave(&q->lock, flags);
+        __wake_up_common(q, mode, nr_exclusive, wake_flags, key);
+        spin_unlock_irqrestore(&q->lock, flags);
+}
+EXPORT_SYMBOL_GPL(__wake_up_sync_key);
+
+/*
+ * __wake_up_sync - see __wake_up_sync_key()
+ */
+void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
+{
+        __wake_up_sync_key(q, mode, nr_exclusive, NULL);
+}
+EXPORT_SYMBOL_GPL(__wake_up_sync);      /* For internal use only */
+
+/*
  * Note: we use "set_current_state()" _after_ the wait-queue add,
  * because we need a memory barrier there on SMP, so that any
  * wake-function that tests for the wait-queue being active
@@ -92,6 +195,30 @@ prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state)
 }
 EXPORT_SYMBOL(prepare_to_wait_exclusive);
 
+long prepare_to_wait_event(wait_queue_head_t *q, wait_queue_t *wait, int state)
+{
+        unsigned long flags;
+
+        if (signal_pending_state(state, current))
+                return -ERESTARTSYS;
+
+        wait->private = current;
+        wait->func = autoremove_wake_function;
+
+        spin_lock_irqsave(&q->lock, flags);
+        if (list_empty(&wait->task_list)) {
+                if (wait->flags & WQ_FLAG_EXCLUSIVE)
+                        __add_wait_queue_tail(q, wait);
+                else
+                        __add_wait_queue(q, wait);
+        }
+        set_current_state(state);
+        spin_unlock_irqrestore(&q->lock, flags);
+
+        return 0;
+}
+EXPORT_SYMBOL(prepare_to_wait_event);
+
 /**
  * finish_wait - clean up after waiting in a queue
  * @q: waitqueue waited on
diff --git a/kernel/smp.c b/kernel/smp.c
index 0564571dcdf7..f5768b0c816a 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -524,6 +524,11 @@ void __init setup_nr_cpu_ids(void)
         nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1;
 }
 
+void __weak smp_announce(void)
+{
+        printk(KERN_INFO "Brought up %d CPUs\n", num_online_cpus());
+}
+
 /* Called by boot processor to activate the rest. */
 void __init smp_init(void)
 {
@@ -540,7 +545,7 @@ void __init smp_init(void)
         }
 
         /* Any cleanup work */
-        printk(KERN_INFO "Brought up %ld CPUs\n", (long)num_online_cpus());
+        smp_announce();
         smp_cpus_done(setup_max_cpus);
 }
 
diff --git a/kernel/softirq.c b/kernel/softirq.c
index d7d498d8cc4f..b24988353458 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -29,7 +29,6 @@
 #define CREATE_TRACE_POINTS
 #include <trace/events/irq.h>
 
-#include <asm/irq.h>
 /*
    - No shared variables, all the data are CPU local.
    - If a softirq needs serialization, let it serialize itself
@@ -100,13 +99,13 @@ static void __local_bh_disable(unsigned long ip, unsigned int cnt)
 
         raw_local_irq_save(flags);
         /*
-         * The preempt tracer hooks into add_preempt_count and will break
+         * The preempt tracer hooks into preempt_count_add and will break
          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
          * is set and before current->softirq_enabled is cleared.
          * We must manually increment preempt_count here and manually
          * call the trace_preempt_off later.
          */
-        preempt_count() += cnt;
+        __preempt_count_add(cnt);
         /*
          * Were softirqs turned off above:
          */
@@ -120,7 +119,7 @@ static void __local_bh_disable(unsigned long ip, unsigned int cnt)
 #else /* !CONFIG_TRACE_IRQFLAGS */
 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
 {
-        add_preempt_count(cnt);
+        preempt_count_add(cnt);
         barrier();
 }
 #endif /* CONFIG_TRACE_IRQFLAGS */
@@ -134,12 +133,11 @@ EXPORT_SYMBOL(local_bh_disable);
 
 static void __local_bh_enable(unsigned int cnt)
 {
-        WARN_ON_ONCE(in_irq());
         WARN_ON_ONCE(!irqs_disabled());
 
         if (softirq_count() == cnt)
                 trace_softirqs_on(_RET_IP_);
-        sub_preempt_count(cnt);
+        preempt_count_sub(cnt);
 }
 
 /*
@@ -149,6 +147,7 @@ static void __local_bh_enable(unsigned int cnt)
  */
 void _local_bh_enable(void)
 {
+        WARN_ON_ONCE(in_irq());
         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 }
 
@@ -169,12 +168,17 @@ static inline void _local_bh_enable_ip(unsigned long ip)
          * Keep preemption disabled until we are done with
          * softirq processing:
          */
-        sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
+        preempt_count_sub(SOFTIRQ_DISABLE_OFFSET - 1);
 
-        if (unlikely(!in_interrupt() && local_softirq_pending()))
+        if (unlikely(!in_interrupt() && local_softirq_pending())) {
+                /*
+                 * Run softirq if any pending. And do it in its own stack
+                 * as we may be calling this deep in a task call stack already.
+                 */
                 do_softirq();
+        }
 
-        dec_preempt_count();
+        preempt_count_dec();
 #ifdef CONFIG_TRACE_IRQFLAGS
         local_irq_enable();
 #endif
@@ -256,7 +260,7 @@ restart:
                                        " exited with %08x?\n", vec_nr,
                                        softirq_to_name[vec_nr], h->action,
                                        prev_count, preempt_count());
-                                preempt_count() = prev_count;
+                                preempt_count_set(prev_count);
                         }
 
                         rcu_bh_qs(cpu);
@@ -280,10 +284,11 @@ restart:
 
         account_irq_exit_time(current);
         __local_bh_enable(SOFTIRQ_OFFSET);
+        WARN_ON_ONCE(in_interrupt());
         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 }
 
-#ifndef __ARCH_HAS_DO_SOFTIRQ
+
 
 asmlinkage void do_softirq(void)
 {
@@ -298,13 +303,11 @@ asmlinkage void do_softirq(void)
         pending = local_softirq_pending();
 
         if (pending)
-                __do_softirq();
+                do_softirq_own_stack();
 
         local_irq_restore(flags);
 }
 
-#endif
-
 /*
  * Enter an interrupt context.
  */
@@ -329,15 +332,21 @@ void irq_enter(void)
 static inline void invoke_softirq(void)
 {
         if (!force_irqthreads) {
+#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
                 /*
                  * We can safely execute softirq on the current stack if
                  * it is the irq stack, because it should be near empty
-                 * at this stage. But we have no way to know if the arch
-                 * calls irq_exit() on the irq stack. So call softirq
-                 * in its own stack to prevent from any overrun on top
-                 * of a potentially deep task stack.
+                 * at this stage.
                  */
-                do_softirq();
+                __do_softirq();
+#else
+                /*
+                 * Otherwise, irq_exit() is called on the task stack that can
+                 * be potentially deep already. So call softirq in its own stack
+                 * to prevent from any overrun.
+                 */
+                do_softirq_own_stack();
+#endif
         } else {
                 wakeup_softirqd();
         }
@@ -369,7 +378,7 @@ void irq_exit(void)
 
         account_irq_exit_time(current);
         trace_hardirq_exit();
-        sub_preempt_count(HARDIRQ_OFFSET);
+        preempt_count_sub(HARDIRQ_OFFSET);
         if (!in_interrupt() && local_softirq_pending())
                 invoke_softirq();
 
@@ -771,6 +780,10 @@ static void run_ksoftirqd(unsigned int cpu)
 {
         local_irq_disable();
         if (local_softirq_pending()) {
+                /*
+                 * We can safely run softirq on inline stack, as we are not deep
+                 * in the task stack here.
+                 */
                 __do_softirq();
                 rcu_note_context_switch(cpu);
                 local_irq_enable();
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index c09f2955ae30..84571e09c907 100644
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -20,6 +20,7 @@
 #include <linux/kallsyms.h>
 #include <linux/smpboot.h>
 #include <linux/atomic.h>
+#include <linux/lglock.h>
 
 /*
  * Structure to determine completion condition and record errors.  May
@@ -43,6 +44,14 @@ static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
 static DEFINE_PER_CPU(struct task_struct *, cpu_stopper_task);
 static bool stop_machine_initialized = false;
 
+/*
+ * Avoids a race between stop_two_cpus and global stop_cpus, where
+ * the stoppers could get queued up in reverse order, leading to
+ * system deadlock. Using an lglock means stop_two_cpus remains
+ * relatively cheap.
+ */
+DEFINE_STATIC_LGLOCK(stop_cpus_lock);
+
 static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
 {
         memset(done, 0, sizeof(*done));
@@ -115,6 +124,184 @@ int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
         return done.executed ? done.ret : -ENOENT;
 }
 
+/* This controls the threads on each CPU. */
+enum multi_stop_state {
+        /* Dummy starting state for thread. */
+        MULTI_STOP_NONE,
+        /* Awaiting everyone to be scheduled. */
+        MULTI_STOP_PREPARE,
+        /* Disable interrupts. */
+        MULTI_STOP_DISABLE_IRQ,
+        /* Run the function */
+        MULTI_STOP_RUN,
+        /* Exit */
+        MULTI_STOP_EXIT,
+};
+
+struct multi_stop_data {
+        int                     (*fn)(void *);
+        void                    *data;
+        /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
+        unsigned int            num_threads;
+        const struct cpumask    *active_cpus;
+
+        enum multi_stop_state   state;
+        atomic_t                thread_ack;
+};
+
+static void set_state(struct multi_stop_data *msdata,
+                      enum multi_stop_state newstate)
+{
+        /* Reset ack counter. */
+        atomic_set(&msdata->thread_ack, msdata->num_threads);
+        smp_wmb();
+        msdata->state = newstate;
+}
+
+/* Last one to ack a state moves to the next state. */
+static void ack_state(struct multi_stop_data *msdata)
+{
+        if (atomic_dec_and_test(&msdata->thread_ack))
+                set_state(msdata, msdata->state + 1);
+}
+
+/* This is the cpu_stop function which stops the CPU. */
+static int multi_cpu_stop(void *data)
+{
+        struct multi_stop_data *msdata = data;
+        enum multi_stop_state curstate = MULTI_STOP_NONE;
+        int cpu = smp_processor_id(), err = 0;
+        unsigned long flags;
+        bool is_active;
+
+        /*
+         * When called from stop_machine_from_inactive_cpu(), irq might
+         * already be disabled.  Save the state and restore it on exit.
+         */
+        local_save_flags(flags);
+
+        if (!msdata->active_cpus)
+                is_active = cpu == cpumask_first(cpu_online_mask);
+        else
+                is_active = cpumask_test_cpu(cpu, msdata->active_cpus);
+
+        /* Simple state machine */
+        do {
+                /* Chill out and ensure we re-read multi_stop_state. */
+                cpu_relax();
+                if (msdata->state != curstate) {
+                        curstate = msdata->state;
+                        switch (curstate) {
+                        case MULTI_STOP_DISABLE_IRQ:
+                                local_irq_disable();
+                                hard_irq_disable();
+                                break;
+                        case MULTI_STOP_RUN:
+                                if (is_active)
+                                        err = msdata->fn(msdata->data);
+                                break;
+                        default:
+                                break;
+                        }
+                        ack_state(msdata);
+                }
+        } while (curstate != MULTI_STOP_EXIT);
+
+        local_irq_restore(flags);
+        return err;
+}
+
+struct irq_cpu_stop_queue_work_info {
+        int cpu1;
+        int cpu2;
+        struct cpu_stop_work *work1;
+        struct cpu_stop_work *work2;
+};
+
+/*
+ * This function is always run with irqs and preemption disabled.
+ * This guarantees that both work1 and work2 get queued, before
+ * our local migrate thread gets the chance to preempt us.
+ */
+static void irq_cpu_stop_queue_work(void *arg)
+{
+        struct irq_cpu_stop_queue_work_info *info = arg;
+        cpu_stop_queue_work(info->cpu1, info->work1);
+        cpu_stop_queue_work(info->cpu2, info->work2);
+}
+
+/**
+ * stop_two_cpus - stops two cpus
+ * @cpu1: the cpu to stop
+ * @cpu2: the other cpu to stop
+ * @fn: function to execute
+ * @arg: argument to @fn
+ *
+ * Stops both the current and specified CPU and runs @fn on one of them.
+ *
+ * returns when both are completed.
+ */
+int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *arg)
+{
+        struct cpu_stop_done done;
+        struct cpu_stop_work work1, work2;
+        struct irq_cpu_stop_queue_work_info call_args;
+        struct multi_stop_data msdata;
+
+        preempt_disable();
+        msdata = (struct multi_stop_data){
+                .fn = fn,
+                .data = arg,
+                .num_threads = 2,
+                .active_cpus = cpumask_of(cpu1),
+        };
+
+        work1 = work2 = (struct cpu_stop_work){
+                .fn = multi_cpu_stop,
+                .arg = &msdata,
+                .done = &done
+        };
+
+        call_args = (struct irq_cpu_stop_queue_work_info){
+                .cpu1 = cpu1,
+                .cpu2 = cpu2,
+                .work1 = &work1,
+                .work2 = &work2,
+        };
+
+        cpu_stop_init_done(&done, 2);
+        set_state(&msdata, MULTI_STOP_PREPARE);
+
+        /*
+         * If we observe both CPUs active we know _cpu_down() cannot yet have
+         * queued its stop_machine works and therefore ours will get executed
+         * first. Or its not either one of our CPUs that's getting unplugged,
+         * in which case we don't care.
+         *
+         * This relies on the stopper workqueues to be FIFO.
+         */
+        if (!cpu_active(cpu1) || !cpu_active(cpu2)) {
+                preempt_enable();
+                return -ENOENT;
+        }
+
+        lg_local_lock(&stop_cpus_lock);
+        /*
+         * Queuing needs to be done by the lowest numbered CPU, to ensure
+         * that works are always queued in the same order on every CPU.
+         * This prevents deadlocks.
+         */
+        smp_call_function_single(min(cpu1, cpu2),
+                                 &irq_cpu_stop_queue_work,
+                                 &call_args, 0);
+        lg_local_unlock(&stop_cpus_lock);
+        preempt_enable();
+
+        wait_for_completion(&done.completion);
+
+        return done.executed ? done.ret : -ENOENT;
+}
+
 /**
  * stop_one_cpu_nowait - stop a cpu but don't wait for completion
  * @cpu: cpu to stop
@@ -159,10 +346,10 @@ static void queue_stop_cpus_work(const struct cpumask *cpumask,
          * preempted by a stopper which might wait for other stoppers
          * to enter @fn which can lead to deadlock.
          */
-        preempt_disable();
+        lg_global_lock(&stop_cpus_lock);
         for_each_cpu(cpu, cpumask)
                 cpu_stop_queue_work(cpu, &per_cpu(stop_cpus_work, cpu));
-        preempt_enable();
+        lg_global_unlock(&stop_cpus_lock);
 }
 
 static int __stop_cpus(const struct cpumask *cpumask,
@@ -359,98 +546,14 @@ early_initcall(cpu_stop_init);
 
 #ifdef CONFIG_STOP_MACHINE
 
-/* This controls the threads on each CPU. */
-enum stopmachine_state {
-        /* Dummy starting state for thread. */
-        STOPMACHINE_NONE,
-        /* Awaiting everyone to be scheduled. */
-        STOPMACHINE_PREPARE,
-        /* Disable interrupts. */
-        STOPMACHINE_DISABLE_IRQ,
-        /* Run the function */
-        STOPMACHINE_RUN,
-        /* Exit */
-        STOPMACHINE_EXIT,
-};
-
-struct stop_machine_data {
-        int                     (*fn)(void *);
-        void                    *data;
-        /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
-        unsigned int            num_threads;
-        const struct cpumask    *active_cpus;
-
-        enum stopmachine_state  state;
-        atomic_t                thread_ack;
-};
-
-static void set_state(struct stop_machine_data *smdata,
-                      enum stopmachine_state newstate)
-{
-        /* Reset ack counter. */
-        atomic_set(&smdata->thread_ack, smdata->num_threads);
-        smp_wmb();
-        smdata->state = newstate;
-}
-
-/* Last one to ack a state moves to the next state. */
-static void ack_state(struct stop_machine_data *smdata)
-{
-        if (atomic_dec_and_test(&smdata->thread_ack))
-                set_state(smdata, smdata->state + 1);
-}
-
-/* This is the cpu_stop function which stops the CPU. */
-static int stop_machine_cpu_stop(void *data)
-{
-        struct stop_machine_data *smdata = data;
-        enum stopmachine_state curstate = STOPMACHINE_NONE;
-        int cpu = smp_processor_id(), err = 0;
-        unsigned long flags;
-        bool is_active;
-
-        /*
-         * When called from stop_machine_from_inactive_cpu(), irq might
-         * already be disabled.  Save the state and restore it on exit.
-         */
-        local_save_flags(flags);
-
-        if (!smdata->active_cpus)
-                is_active = cpu == cpumask_first(cpu_online_mask);
-        else
-                is_active = cpumask_test_cpu(cpu, smdata->active_cpus);
-
-        /* Simple state machine */
-        do {
-                /* Chill out and ensure we re-read stopmachine_state. */
-                cpu_relax();
-                if (smdata->state != curstate) {
-                        curstate = smdata->state;
-                        switch (curstate) {
-                        case STOPMACHINE_DISABLE_IRQ:
-                                local_irq_disable();
-                                hard_irq_disable();
-                                break;
-                        case STOPMACHINE_RUN:
-                                if (is_active)
-                                        err = smdata->fn(smdata->data);
-                                break;
-                        default:
-                                break;
-                        }
-                        ack_state(smdata);
-                }
-        } while (curstate != STOPMACHINE_EXIT);
-
-        local_irq_restore(flags);
-        return err;
-}
-
 int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
 {
-        struct stop_machine_data smdata = { .fn = fn, .data = data,
-                                            .num_threads = num_online_cpus(),
-                                            .active_cpus = cpus };
+        struct multi_stop_data msdata = {
+                .fn = fn,
+                .data = data,
+                .num_threads = num_online_cpus(),
+                .active_cpus = cpus,
+        };
 
         if (!stop_machine_initialized) {
                 /*
@@ -461,7 +564,7 @@ int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
                 unsigned long flags;
                 int ret;
 
-                WARN_ON_ONCE(smdata.num_threads != 1);
+                WARN_ON_ONCE(msdata.num_threads != 1);
 
                 local_irq_save(flags);
                 hard_irq_disable();
@@ -472,8 +575,8 @@ int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
         }
 
         /* Set the initial state and stop all online cpus. */
-        set_state(&smdata, STOPMACHINE_PREPARE);
-        return stop_cpus(cpu_online_mask, stop_machine_cpu_stop, &smdata);
+        set_state(&msdata, MULTI_STOP_PREPARE);
+        return stop_cpus(cpu_online_mask, multi_cpu_stop, &msdata);
 }
 
 int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
@@ -513,25 +616,25 @@ EXPORT_SYMBOL_GPL(stop_machine);
 int stop_machine_from_inactive_cpu(int (*fn)(void *), void *data,
                                   const struct cpumask *cpus)
 {
-        struct stop_machine_data smdata = { .fn = fn, .data = data,
+        struct multi_stop_data msdata = { .fn = fn, .data = data,
                                             .active_cpus = cpus };
         struct cpu_stop_done done;
         int ret;
 
         /* Local CPU must be inactive and CPU hotplug in progress. */
         BUG_ON(cpu_active(raw_smp_processor_id()));
-        smdata.num_threads = num_active_cpus() + 1;     /* +1 for local */
+        msdata.num_threads = num_active_cpus() + 1;     /* +1 for local */
 
         /* No proper task established and can't sleep - busy wait for lock. */
         while (!mutex_trylock(&stop_cpus_mutex))
                 cpu_relax();
 
         /* Schedule work on other CPUs and execute directly for local CPU */
-        set_state(&smdata, STOPMACHINE_PREPARE);
+        set_state(&msdata, MULTI_STOP_PREPARE);
         cpu_stop_init_done(&done, num_active_cpus());
-        queue_stop_cpus_work(cpu_active_mask, stop_machine_cpu_stop, &smdata,
+        queue_stop_cpus_work(cpu_active_mask, multi_cpu_stop, &msdata,
                              &done);
-        ret = stop_machine_cpu_stop(&smdata);
+        ret = multi_cpu_stop(&msdata);
 
         /* Busy wait for completion. */
         while (!completion_done(&done.completion))
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index b2f06f3c6a3f..36547dddcdb8 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -190,7 +190,7 @@ static int proc_dostring_coredump(struct ctl_table *table, int write,
 
 #ifdef CONFIG_MAGIC_SYSRQ
 /* Note: sysrq code uses it's own private copy */
-static int __sysrq_enabled = SYSRQ_DEFAULT_ENABLE;
+static int __sysrq_enabled = CONFIG_MAGIC_SYSRQ_DEFAULT_ENABLE;
 
 static int sysrq_sysctl_handler(ctl_table *table, int write,
                                 void __user *buffer, size_t *lenp,
@@ -371,13 +371,6 @@ static struct ctl_table kern_table[] = {
                 .proc_handler   = proc_dointvec,
         },
         {
-                .procname       = "numa_balancing_scan_period_reset",
-                .data           = &sysctl_numa_balancing_scan_period_reset,
-                .maxlen         = sizeof(unsigned int),
-                .mode           = 0644,
-                .proc_handler   = proc_dointvec,
-        },
-        {
                 .procname       = "numa_balancing_scan_period_max_ms",
                 .data           = &sysctl_numa_balancing_scan_period_max,
                 .maxlen         = sizeof(unsigned int),
@@ -391,6 +384,20 @@ static struct ctl_table kern_table[] = {
                 .mode           = 0644,
                 .proc_handler   = proc_dointvec,
         },
+        {
+                .procname       = "numa_balancing_settle_count",
+                .data           = &sysctl_numa_balancing_settle_count,
+                .maxlen         = sizeof(unsigned int),
+                .mode           = 0644,
+                .proc_handler   = proc_dointvec,
+        },
+        {
+                .procname       = "numa_balancing_migrate_deferred",
+                .data           = &sysctl_numa_balancing_migrate_deferred,
+                .maxlen         = sizeof(unsigned int),
+                .mode           = 0644,
+                .proc_handler   = proc_dointvec,
+        },
 #endif /* CONFIG_NUMA_BALANCING */
 #endif /* CONFIG_SCHED_DEBUG */
         {
@@ -1049,6 +1056,7 @@ static struct ctl_table kern_table[] = {
                 .maxlen         = sizeof(sysctl_perf_event_sample_rate),
                 .mode           = 0644,
                 .proc_handler   = perf_proc_update_handler,
+                .extra1         = &one,
         },
         {
                 .procname       = "perf_cpu_time_max_percent",
diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
index 2b62fe86f9ec..3ce6e8c5f3fc 100644
--- a/kernel/time/Kconfig
+++ b/kernel/time/Kconfig
@@ -100,7 +100,7 @@ config NO_HZ_FULL
         # RCU_USER_QS dependency
         depends on HAVE_CONTEXT_TRACKING
         # VIRT_CPU_ACCOUNTING_GEN dependency
-        depends on 64BIT
+        depends on HAVE_VIRT_CPU_ACCOUNTING_GEN
         select NO_HZ_COMMON
         select RCU_USER_QS
         select RCU_NOCB_CPU
diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c
index eec50fcef9e4..88c9c65a430d 100644
--- a/kernel/time/alarmtimer.c
+++ b/kernel/time/alarmtimer.c
@@ -490,7 +490,7 @@ static int alarm_clock_getres(const clockid_t which_clock, struct timespec *tp)
         clockid_t baseid = alarm_bases[clock2alarm(which_clock)].base_clockid;
 
         if (!alarmtimer_get_rtcdev())
-                return -ENOTSUPP;
+                return -EINVAL;
 
         return hrtimer_get_res(baseid, tp);
 }
@@ -507,7 +507,7 @@ static int alarm_clock_get(clockid_t which_clock, struct timespec *tp)
         struct alarm_base *base = &alarm_bases[clock2alarm(which_clock)];
 
         if (!alarmtimer_get_rtcdev())
-                return -ENOTSUPP;
+                return -EINVAL;
 
         *tp = ktime_to_timespec(base->gettime());
         return 0;
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
index 38959c866789..086ad6043bcb 100644
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -33,29 +33,64 @@ struct ce_unbind {
         int res;
 };
 
-/**
- * clockevents_delta2ns - Convert a latch value (device ticks) to nanoseconds
- * @latch:      value to convert
- * @evt:        pointer to clock event device descriptor
- *
- * Math helper, returns latch value converted to nanoseconds (bound checked)
- */
-u64 clockevent_delta2ns(unsigned long latch, struct clock_event_device *evt)
+static u64 cev_delta2ns(unsigned long latch, struct clock_event_device *evt,
+                        bool ismax)
 {
         u64 clc = (u64) latch << evt->shift;
+        u64 rnd;
 
         if (unlikely(!evt->mult)) {
                 evt->mult = 1;
                 WARN_ON(1);
         }
+        rnd = (u64) evt->mult - 1;
+
+        /*
+         * Upper bound sanity check. If the backwards conversion is
+         * not equal latch, we know that the above shift overflowed.
+         */
+        if ((clc >> evt->shift) != (u64)latch)
+                clc = ~0ULL;
+
+        /*
+         * Scaled math oddities:
+         *
+         * For mult <= (1 << shift) we can safely add mult - 1 to
+         * prevent integer rounding loss. So the backwards conversion
+         * from nsec to device ticks will be correct.
+         *
+         * For mult > (1 << shift), i.e. device frequency is > 1GHz we
+         * need to be careful. Adding mult - 1 will result in a value
+         * which when converted back to device ticks can be larger
+         * than latch by up to (mult - 1) >> shift. For the min_delta
+         * calculation we still want to apply this in order to stay
+         * above the minimum device ticks limit. For the upper limit
+         * we would end up with a latch value larger than the upper
+         * limit of the device, so we omit the add to stay below the
+         * device upper boundary.
+         *
+         * Also omit the add if it would overflow the u64 boundary.
+         */
+        if ((~0ULL - clc > rnd) &&
+            (!ismax || evt->mult <= (1U << evt->shift)))
+                clc += rnd;
 
         do_div(clc, evt->mult);
-        if (clc < 1000)
-                clc = 1000;
-        if (clc > KTIME_MAX)
-                clc = KTIME_MAX;
 
-        return clc;
+        /* Deltas less than 1usec are pointless noise */
+        return clc > 1000 ? clc : 1000;
+}
+
+/**
+ * clockevents_delta2ns - Convert a latch value (device ticks) to nanoseconds
+ * @latch:      value to convert
+ * @evt:        pointer to clock event device descriptor
+ *
+ * Math helper, returns latch value converted to nanoseconds (bound checked)
+ */
+u64 clockevent_delta2ns(unsigned long latch, struct clock_event_device *evt)
+{
+        return cev_delta2ns(latch, evt, false);
 }
 EXPORT_SYMBOL_GPL(clockevent_delta2ns);
 
@@ -380,8 +415,8 @@ void clockevents_config(struct clock_event_device *dev, u32 freq)
                 sec = 600;
 
         clockevents_calc_mult_shift(dev, freq, sec);
-        dev->min_delta_ns = clockevent_delta2ns(dev->min_delta_ticks, dev);
-        dev->max_delta_ns = clockevent_delta2ns(dev->max_delta_ticks, dev);
+        dev->min_delta_ns = cev_delta2ns(dev->min_delta_ticks, dev, false);
+        dev->max_delta_ns = cev_delta2ns(dev->max_delta_ticks, dev, true);
 }
 
 /**
@@ -584,7 +619,7 @@ static ssize_t sysfs_unbind_tick_dev(struct device *dev,
                                      const char *buf, size_t count)
 {
         char name[CS_NAME_LEN];
-        size_t ret = sysfs_get_uname(buf, name, count);
+        ssize_t ret = sysfs_get_uname(buf, name, count);
         struct clock_event_device *ce;
 
         if (ret < 0)
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index 50a8736757f3..ba3e502c955a 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -479,6 +479,7 @@ static inline void clocksource_dequeue_watchdog(struct clocksource *cs) { }
 static inline void clocksource_resume_watchdog(void) { }
 static inline int __clocksource_watchdog_kthread(void) { return 0; }
 static bool clocksource_is_watchdog(struct clocksource *cs) { return false; }
+void clocksource_mark_unstable(struct clocksource *cs) { }
 
 #endif /* CONFIG_CLOCKSOURCE_WATCHDOG */
 
@@ -537,40 +538,55 @@ static u32 clocksource_max_adjustment(struct clocksource *cs)
 }
 
 /**
- * clocksource_max_deferment - Returns max time the clocksource can be deferred
- * @cs:         Pointer to clocksource
- *
+ * clocks_calc_max_nsecs - Returns maximum nanoseconds that can be converted
+ * @mult:       cycle to nanosecond multiplier
+ * @shift:      cycle to nanosecond divisor (power of two)
+ * @maxadj:     maximum adjustment value to mult (~11%)
+ * @mask:       bitmask for two's complement subtraction of non 64 bit counters
  */
-static u64 clocksource_max_deferment(struct clocksource *cs)
+u64 clocks_calc_max_nsecs(u32 mult, u32 shift, u32 maxadj, u64 mask)
 {
         u64 max_nsecs, max_cycles;
 
         /*
          * Calculate the maximum number of cycles that we can pass to the
          * cyc2ns function without overflowing a 64-bit signed result. The
-         * maximum number of cycles is equal to ULLONG_MAX/(cs->mult+cs->maxadj)
+         * maximum number of cycles is equal to ULLONG_MAX/(mult+maxadj)
          * which is equivalent to the below.
-         * max_cycles < (2^63)/(cs->mult + cs->maxadj)
-         * max_cycles < 2^(log2((2^63)/(cs->mult + cs->maxadj)))
-         * max_cycles < 2^(log2(2^63) - log2(cs->mult + cs->maxadj))
-         * max_cycles < 2^(63 - log2(cs->mult + cs->maxadj))
-         * max_cycles < 1 << (63 - log2(cs->mult + cs->maxadj))
+         * max_cycles < (2^63)/(mult + maxadj)
+         * max_cycles < 2^(log2((2^63)/(mult + maxadj)))
+         * max_cycles < 2^(log2(2^63) - log2(mult + maxadj))
+         * max_cycles < 2^(63 - log2(mult + maxadj))
+         * max_cycles < 1 << (63 - log2(mult + maxadj))
          * Please note that we add 1 to the result of the log2 to account for
          * any rounding errors, ensure the above inequality is satisfied and
          * no overflow will occur.
          */
-        max_cycles = 1ULL << (63 - (ilog2(cs->mult + cs->maxadj) + 1));
+        max_cycles = 1ULL << (63 - (ilog2(mult + maxadj) + 1));
 
         /*
          * The actual maximum number of cycles we can defer the clocksource is
-         * determined by the minimum of max_cycles and cs->mask.
+         * determined by the minimum of max_cycles and mask.
          * Note: Here we subtract the maxadj to make sure we don't sleep for
          * too long if there's a large negative adjustment.
          */
-        max_cycles = min_t(u64, max_cycles, (u64) cs->mask);
-        max_nsecs = clocksource_cyc2ns(max_cycles, cs->mult - cs->maxadj,
-                                        cs->shift);
+        max_cycles = min(max_cycles, mask);
+        max_nsecs = clocksource_cyc2ns(max_cycles, mult - maxadj, shift);
+
+        return max_nsecs;
+}
+
+/**
+ * clocksource_max_deferment - Returns max time the clocksource can be deferred
+ * @cs:         Pointer to clocksource
+ *
+ */
+static u64 clocksource_max_deferment(struct clocksource *cs)
+{
+        u64 max_nsecs;
 
+        max_nsecs = clocks_calc_max_nsecs(cs->mult, cs->shift, cs->maxadj,
+                                          cs->mask);
         /*
          * To ensure that the clocksource does not wrap whilst we are idle,
          * limit the time the clocksource can be deferred by 12.5%. Please
@@ -893,7 +909,7 @@ sysfs_show_current_clocksources(struct device *dev,
         return count;
 }
 
-size_t sysfs_get_uname(const char *buf, char *dst, size_t cnt)
+ssize_t sysfs_get_uname(const char *buf, char *dst, size_t cnt)
 {
         size_t ret = cnt;
 
@@ -924,7 +940,7 @@ static ssize_t sysfs_override_clocksource(struct device *dev,
                                           struct device_attribute *attr,
                                           const char *buf, size_t count)
 {
-        size_t ret;
+        ssize_t ret;
 
         mutex_lock(&clocksource_mutex);
 
@@ -952,7 +968,7 @@ static ssize_t sysfs_unbind_clocksource(struct device *dev,
 {
         struct clocksource *cs;
         char name[CS_NAME_LEN];
-        size_t ret;
+        ssize_t ret;
 
         ret = sysfs_get_uname(buf, name, count);
         if (ret < 0)
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c
index bb2215174f05..af8d1d4f3d55 100644
--- a/kernel/time/ntp.c
+++ b/kernel/time/ntp.c
@@ -475,6 +475,7 @@ static void sync_cmos_clock(struct work_struct *work)
          * called as close as possible to 500 ms before the new second starts.
          * This code is run on a timer.  If the clock is set, that timer
          * may not expire at the correct time.  Thus, we adjust...
+         * We want the clock to be within a couple of ticks from the target.
          */
         if (!ntp_synced()) {
                 /*
@@ -485,7 +486,7 @@ static void sync_cmos_clock(struct work_struct *work)
         }
 
         getnstimeofday(&now);
-        if (abs(now.tv_nsec - (NSEC_PER_SEC / 2)) <= tick_nsec / 2) {
+        if (abs(now.tv_nsec - (NSEC_PER_SEC / 2)) <= tick_nsec * 5) {
                 struct timespec adjust = now;
 
                 fail = -ENODEV;
diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c
index 0b479a6a22bb..68b799375981 100644
--- a/kernel/time/sched_clock.c
+++ b/kernel/time/sched_clock.c
@@ -8,25 +8,28 @@
 #include <linux/clocksource.h>
 #include <linux/init.h>
 #include <linux/jiffies.h>
+#include <linux/ktime.h>
 #include <linux/kernel.h>
 #include <linux/moduleparam.h>
 #include <linux/sched.h>
 #include <linux/syscore_ops.h>
-#include <linux/timer.h>
+#include <linux/hrtimer.h>
 #include <linux/sched_clock.h>
+#include <linux/seqlock.h>
+#include <linux/bitops.h>
 
 struct clock_data {
+        ktime_t wrap_kt;
         u64 epoch_ns;
-        u32 epoch_cyc;
-        u32 epoch_cyc_copy;
+        u64 epoch_cyc;
+        seqcount_t seq;
         unsigned long rate;
         u32 mult;
         u32 shift;
         bool suspended;
 };
 
-static void sched_clock_poll(unsigned long wrap_ticks);
-static DEFINE_TIMER(sched_clock_timer, sched_clock_poll, 0, 0);
+static struct hrtimer sched_clock_timer;
 static int irqtime = -1;
 
 core_param(irqtime, irqtime, int, 0400);
@@ -35,42 +38,46 @@ static struct clock_data cd = {
         .mult   = NSEC_PER_SEC / HZ,
 };
 
-static u32 __read_mostly sched_clock_mask = 0xffffffff;
+static u64 __read_mostly sched_clock_mask;
 
-static u32 notrace jiffy_sched_clock_read(void)
+static u64 notrace jiffy_sched_clock_read(void)
 {
-        return (u32)(jiffies - INITIAL_JIFFIES);
+        /*
+         * We don't need to use get_jiffies_64 on 32-bit arches here
+         * because we register with BITS_PER_LONG
+         */
+        return (u64)(jiffies - INITIAL_JIFFIES);
 }
 
-static u32 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read;
+static u32 __read_mostly (*read_sched_clock_32)(void);
+
+static u64 notrace read_sched_clock_32_wrapper(void)
+{
+        return read_sched_clock_32();
+}
+
+static u64 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read;
 
 static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift)
 {
         return (cyc * mult) >> shift;
 }
 
-static unsigned long long notrace sched_clock_32(void)
+unsigned long long notrace sched_clock(void)
 {
         u64 epoch_ns;
-        u32 epoch_cyc;
-        u32 cyc;
+        u64 epoch_cyc;
+        u64 cyc;
+        unsigned long seq;
 
         if (cd.suspended)
                 return cd.epoch_ns;
 
-        /*
-         * Load the epoch_cyc and epoch_ns atomically.  We do this by
-         * ensuring that we always write epoch_cyc, epoch_ns and
-         * epoch_cyc_copy in strict order, and read them in strict order.
-         * If epoch_cyc and epoch_cyc_copy are not equal, then we're in
-         * the middle of an update, and we should repeat the load.
-         */
         do {
+                seq = read_seqcount_begin(&cd.seq);
                 epoch_cyc = cd.epoch_cyc;
-                smp_rmb();
                 epoch_ns = cd.epoch_ns;
-                smp_rmb();
-        } while (epoch_cyc != cd.epoch_cyc_copy);
+        } while (read_seqcount_retry(&cd.seq, seq));
 
         cyc = read_sched_clock();
         cyc = (cyc - epoch_cyc) & sched_clock_mask;
@@ -83,49 +90,46 @@ static unsigned long long notrace sched_clock_32(void)
 static void notrace update_sched_clock(void)
 {
         unsigned long flags;
-        u32 cyc;
+        u64 cyc;
         u64 ns;
 
         cyc = read_sched_clock();
         ns = cd.epoch_ns +
                 cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask,
                           cd.mult, cd.shift);
-        /*
-         * Write epoch_cyc and epoch_ns in a way that the update is
-         * detectable in cyc_to_fixed_sched_clock().
-         */
+
         raw_local_irq_save(flags);
-        cd.epoch_cyc_copy = cyc;
-        smp_wmb();
+        write_seqcount_begin(&cd.seq);
         cd.epoch_ns = ns;
-        smp_wmb();
         cd.epoch_cyc = cyc;
+        write_seqcount_end(&cd.seq);
         raw_local_irq_restore(flags);
 }
 
-static void sched_clock_poll(unsigned long wrap_ticks)
+static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt)
 {
-        mod_timer(&sched_clock_timer, round_jiffies(jiffies + wrap_ticks));
         update_sched_clock();
+        hrtimer_forward_now(hrt, cd.wrap_kt);
+        return HRTIMER_RESTART;
 }
 
-void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate)
+void __init sched_clock_register(u64 (*read)(void), int bits,
+                                 unsigned long rate)
 {
-        unsigned long r, w;
+        unsigned long r;
         u64 res, wrap;
         char r_unit;
 
         if (cd.rate > rate)
                 return;
 
-        BUG_ON(bits > 32);
         WARN_ON(!irqs_disabled());
         read_sched_clock = read;
-        sched_clock_mask = (1ULL << bits) - 1;
+        sched_clock_mask = CLOCKSOURCE_MASK(bits);
         cd.rate = rate;
 
         /* calculate the mult/shift to convert counter ticks to ns. */
-        clocks_calc_mult_shift(&cd.mult, &cd.shift, rate, NSEC_PER_SEC, 0);
+        clocks_calc_mult_shift(&cd.mult, &cd.shift, rate, NSEC_PER_SEC, 3600);
 
         r = rate;
         if (r >= 4000000) {
@@ -138,20 +142,14 @@ void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate)
                 r_unit = ' ';
 
         /* calculate how many ns until we wrap */
-        wrap = cyc_to_ns((1ULL << bits) - 1, cd.mult, cd.shift);
-        do_div(wrap, NSEC_PER_MSEC);
-        w = wrap;
+        wrap = clocks_calc_max_nsecs(cd.mult, cd.shift, 0, sched_clock_mask);
+        cd.wrap_kt = ns_to_ktime(wrap - (wrap >> 3));
 
         /* calculate the ns resolution of this counter */
         res = cyc_to_ns(1ULL, cd.mult, cd.shift);
-        pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lums\n",
-                bits, r, r_unit, res, w);
+        pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n",
+                bits, r, r_unit, res, wrap);
 
-        /*
-         * Start the timer to keep sched_clock() properly updated and
-         * sets the initial epoch.
-         */
-        sched_clock_timer.data = msecs_to_jiffies(w - (w / 10));
         update_sched_clock();
 
         /*
@@ -166,11 +164,10 @@ void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate)
         pr_debug("Registered %pF as sched_clock source\n", read);
 }
 
-unsigned long long __read_mostly (*sched_clock_func)(void) = sched_clock_32;
-
-unsigned long long notrace sched_clock(void)
+void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate)
 {
-        return sched_clock_func();
+        read_sched_clock_32 = read;
+        sched_clock_register(read_sched_clock_32_wrapper, bits, rate);
 }
 
 void __init sched_clock_postinit(void)
@@ -180,14 +177,22 @@ void __init sched_clock_postinit(void)
          * make it the final one one.
          */
         if (read_sched_clock == jiffy_sched_clock_read)
-                setup_sched_clock(jiffy_sched_clock_read, 32, HZ);
+                sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ);
 
-        sched_clock_poll(sched_clock_timer.data);
+        update_sched_clock();
+
+        /*
+         * Start the timer to keep sched_clock() properly updated and
+         * sets the initial epoch.
+         */
+        hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+        sched_clock_timer.function = sched_clock_poll;
+        hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL);
 }
 
 static int sched_clock_suspend(void)
 {
-        sched_clock_poll(sched_clock_timer.data);
+        sched_clock_poll(&sched_clock_timer);
         cd.suspended = true;
         return 0;
 }
@@ -195,7 +200,6 @@ static int sched_clock_suspend(void)
 static void sched_clock_resume(void)
 {
         cd.epoch_cyc = read_sched_clock();
-        cd.epoch_cyc_copy = cd.epoch_cyc;
         cd.suspended = false;
 }
 
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index 218bcb565fed..9532690daaa9 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -70,6 +70,7 @@ static bool tick_check_broadcast_device(struct clock_event_device *curdev,
                                         struct clock_event_device *newdev)
 {
         if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||
+            (newdev->features & CLOCK_EVT_FEAT_PERCPU) ||
             (newdev->features & CLOCK_EVT_FEAT_C3STOP))
                 return false;
 
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h
index bc906cad709b..18e71f7fbc2a 100644
--- a/kernel/time/tick-internal.h
+++ b/kernel/time/tick-internal.h
@@ -31,7 +31,7 @@ extern void tick_install_replacement(struct clock_event_device *dev);
 
 extern void clockevents_shutdown(struct clock_event_device *dev);
 
-extern size_t sysfs_get_uname(const char *buf, char *dst, size_t cnt);
+extern ssize_t sysfs_get_uname(const char *buf, char *dst, size_t cnt);
 
 /*
  * NO_HZ / high resolution timer shared code
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 947ba25a95a0..3abf53418b67 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -1613,9 +1613,10 @@ void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim,
  * ktime_get_update_offsets - hrtimer helper
  * @offs_real:  pointer to storage for monotonic -> realtime offset
  * @offs_boot:  pointer to storage for monotonic -> boottime offset
+ * @offs_tai:   pointer to storage for monotonic -> clock tai offset
  *
  * Returns current monotonic time and updates the offsets
- * Called from hrtimer_interupt() or retrigger_next_event()
+ * Called from hrtimer_interrupt() or retrigger_next_event()
  */
 ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot,
                                                         ktime_t *offs_tai)
diff --git a/kernel/time/timer_stats.c b/kernel/time/timer_stats.c
index 0b537f27b559..1fb08f21302e 100644
--- a/kernel/time/timer_stats.c
+++ b/kernel/time/timer_stats.c
@@ -298,15 +298,15 @@ static int tstats_show(struct seq_file *m, void *v)
         period = ktime_to_timespec(time);
         ms = period.tv_nsec / 1000000;
 
-        seq_puts(m, "Timer Stats Version: v0.2\n");
+        seq_puts(m, "Timer Stats Version: v0.3\n");
         seq_printf(m, "Sample period: %ld.%03ld s\n", period.tv_sec, ms);
         if (atomic_read(&overflow_count))
-                seq_printf(m, "Overflow: %d entries\n",
-                        atomic_read(&overflow_count));
+                seq_printf(m, "Overflow: %d entries\n", atomic_read(&overflow_count));
+        seq_printf(m, "Collection: %s\n", timer_stats_active ? "active" : "inactive");
 
         for (i = 0; i < nr_entries; i++) {
                 entry = entries + i;
-                if (entry->timer_flag & TIMER_STATS_FLAG_DEFERRABLE) {
+                if (entry->timer_flag & TIMER_STATS_FLAG_DEFERRABLE) {
                         seq_printf(m, "%4luD, %5d %-16s ",
                                 entry->count, entry->pid, entry->comm);
                 } else {
diff --git a/kernel/timer.c b/kernel/timer.c
index 4296d13db3d1..6582b82fa966 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -1092,7 +1092,7 @@ static int cascade(struct tvec_base *base, struct tvec *tv, int index)
 static void call_timer_fn(struct timer_list *timer, void (*fn)(unsigned long),
                           unsigned long data)
 {
-        int preempt_count = preempt_count();
+        int count = preempt_count();
 
 #ifdef CONFIG_LOCKDEP
         /*
@@ -1119,16 +1119,16 @@ static void call_timer_fn(struct timer_list *timer, void (*fn)(unsigned long),
 
         lock_map_release(&lockdep_map);
 
-        if (preempt_count != preempt_count()) {
+        if (count != preempt_count()) {
                 WARN_ONCE(1, "timer: %pF preempt leak: %08x -> %08x\n",
-                          fn, preempt_count, preempt_count());
+                          fn, count, preempt_count());
                 /*
                  * Restore the preempt count. That gives us a decent
                  * chance to survive and extract information. If the
                  * callback kept a lock held, bad luck, but not worse
                  * than the BUG() we had.
                  */
-                preempt_count() = preempt_count;
+                preempt_count_set(count);
         }
 }
 
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 7974ba20557d..d9fea7dfd5d3 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -1509,7 +1509,8 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
 #endif
                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
-                (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
+                (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
+                (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
 }
 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
 
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 10c86fb7a2b4..73d08aa25b55 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -124,6 +124,7 @@ enum trace_flag_type {
         TRACE_FLAG_NEED_RESCHED         = 0x04,
         TRACE_FLAG_HARDIRQ              = 0x08,
         TRACE_FLAG_SOFTIRQ              = 0x10,
+        TRACE_FLAG_PREEMPT_RESCHED      = 0x20,
 };
 
 #define TRACE_BUF_SIZE          1024
diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c
index 80c36bcf66e8..78e27e3b52ac 100644
--- a/kernel/trace/trace_event_perf.c
+++ b/kernel/trace/trace_event_perf.c
@@ -26,7 +26,7 @@ static int perf_trace_event_perm(struct ftrace_event_call *tp_event,
 {
         /* The ftrace function trace is allowed only for root. */
         if (ftrace_event_is_function(tp_event) &&
-            perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
+            perf_paranoid_tracepoint_raw() && !capable(CAP_SYS_ADMIN))
                 return -EPERM;
 
         /* No tracing, just counting, so no obvious leak */
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c
index 34e7cbac0c9c..ed32284fbe32 100644
--- a/kernel/trace/trace_output.c
+++ b/kernel/trace/trace_output.c
@@ -618,8 +618,23 @@ int trace_print_lat_fmt(struct trace_seq *s, struct trace_entry *entry)
                 (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
                 (entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ? 'X' :
                 '.';
-        need_resched =
-                (entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.';
+
+        switch (entry->flags & (TRACE_FLAG_NEED_RESCHED |
+                                TRACE_FLAG_PREEMPT_RESCHED)) {
+        case TRACE_FLAG_NEED_RESCHED | TRACE_FLAG_PREEMPT_RESCHED:
+                need_resched = 'N';
+                break;
+        case TRACE_FLAG_NEED_RESCHED:
+                need_resched = 'n';
+                break;
+        case TRACE_FLAG_PREEMPT_RESCHED:
+                need_resched = 'p';
+                break;
+        default:
+                need_resched = '.';
+                break;
+        }
+
         hardsoft_irq =
                 (hardirq && softirq) ? 'H' :
                 hardirq ? 'h' :