Merge branch 'sched-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler updates from Ingo Molnar:
 "The main changes in this cycle are:

   - Make schedstats a runtime tunable (disabled by default) and
     optimize it via static keys.

     As most distributions enable CONFIG_SCHEDSTATS=y due to its
     instrumentation value, this is a nice performance enhancement.
     (Mel Gorman)

   - Implement 'simple waitqueues' (swait): these are just pure
     waitqueues without any of the more complex features of full-blown
     waitqueues (callbacks, wake flags, wake keys, etc.).  Simple
     waitqueues have less memory overhead and are faster.

     Use simple waitqueues in the RCU code (in 4 different places) and
     for handling KVM vCPU wakeups.

     (Peter Zijlstra, Daniel Wagner, Thomas Gleixner, Paul Gortmaker,
     Marcelo Tosatti)

   - sched/numa enhancements (Rik van Riel)

   - NOHZ performance enhancements (Rik van Riel)

   - Various sched/deadline enhancements (Steven Rostedt)

   - Various fixes (Peter Zijlstra)

   - ... and a number of other fixes, cleanups and smaller enhancements"

* 'sched-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (29 commits)
  sched/cputime: Fix steal_account_process_tick() to always return jiffies
  sched/deadline: Remove dl_new from struct sched_dl_entity
  Revert "kbuild: Add option to turn incompatible pointer check into error"
  sched/deadline: Remove superfluous call to switched_to_dl()
  sched/debug: Fix preempt_disable_ip recording for preempt_disable()
  sched, time: Switch VIRT_CPU_ACCOUNTING_GEN to jiffy granularity
  time, acct: Drop irq save & restore from __acct_update_integrals()
  acct, time: Change indentation in __acct_update_integrals()
  sched, time: Remove non-power-of-two divides from __acct_update_integrals()
  sched/rt: Kick RT bandwidth timer immediately on start up
  sched/debug: Add deadline scheduler bandwidth ratio to /proc/sched_debug
  sched/debug: Move sched_domain_sysctl to debug.c
  sched/debug: Move the /sys/kernel/debug/sched_features file setup into debug.c
  sched/rt: Fix PI handling vs. sched_setscheduler()
  sched/core: Remove duplicated sched_group_set_shares() prototype
  sched/fair: Consolidate nohz CPU load update code
  sched/fair: Avoid using decay_load_missed() with a negative value
  sched/deadline: Always calculate end of period on sched_yield()
  sched/cgroup: Fix cgroup entity load tracking tear-down
  rcu: Use simple wait queues where possible in rcutree
  ...

7 files changed, 202 insertions, 10 deletions

diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index c2b340e23f62..6d9df3f7e334 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -713,6 +713,18 @@ static inline void __ftrace_enabled_restore(int enabled)
 #define CALLER_ADDR5 ((unsigned long)ftrace_return_address(5))
 #define CALLER_ADDR6 ((unsigned long)ftrace_return_address(6))
 
+static inline unsigned long get_lock_parent_ip(void)
+{
+        unsigned long addr = CALLER_ADDR0;
+
+        if (!in_lock_functions(addr))
+                return addr;
+        addr = CALLER_ADDR1;
+        if (!in_lock_functions(addr))
+                return addr;
+        return CALLER_ADDR2;
+}
+
 #ifdef CONFIG_IRQSOFF_TRACER
   extern void time_hardirqs_on(unsigned long a0, unsigned long a1);
   extern void time_hardirqs_off(unsigned long a0, unsigned long a1);
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 861f690aa791..5276fe0916fc 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -25,6 +25,7 @@
 #include <linux/irqflags.h>
 #include <linux/context_tracking.h>
 #include <linux/irqbypass.h>
+#include <linux/swait.h>
 #include <asm/signal.h>
 
 #include <linux/kvm.h>
@@ -218,7 +219,7 @@ struct kvm_vcpu {
         int fpu_active;
         int guest_fpu_loaded, guest_xcr0_loaded;
         unsigned char fpu_counter;
-        wait_queue_head_t wq;
+        struct swait_queue_head wq;
         struct pid *pid;
         int sigset_active;
         sigset_t sigset;
@@ -782,7 +783,7 @@ static inline bool kvm_arch_has_assigned_device(struct kvm *kvm)
 }
 #endif
 
-static inline wait_queue_head_t *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu)
+static inline struct swait_queue_head *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu)
 {
 #ifdef __KVM_HAVE_ARCH_WQP
         return vcpu->arch.wqp;
diff --git a/include/linux/latencytop.h b/include/linux/latencytop.h
index e23121f9d82a..59ccab297ae0 100644
--- a/include/linux/latencytop.h
+++ b/include/linux/latencytop.h
@@ -37,6 +37,9 @@ account_scheduler_latency(struct task_struct *task, int usecs, int inter)
 
 void clear_all_latency_tracing(struct task_struct *p);
 
+extern int sysctl_latencytop(struct ctl_table *table, int write,
+                        void __user *buffer, size_t *lenp, loff_t *ppos);
+
 #else
 
 static inline void
diff --git a/include/linux/sched.h b/include/linux/sched.h
index a10494a94cc3..838a89a78332 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -182,8 +182,6 @@ extern void update_cpu_load_nohz(int active);
 static inline void update_cpu_load_nohz(int active) { }
 #endif
 
-extern unsigned long get_parent_ip(unsigned long addr);
-
 extern void dump_cpu_task(int cpu);
 
 struct seq_file;
@@ -920,6 +918,10 @@ static inline int sched_info_on(void)
 #endif
 }
 
+#ifdef CONFIG_SCHEDSTATS
+void force_schedstat_enabled(void);
+#endif
+
 enum cpu_idle_type {
         CPU_IDLE,
         CPU_NOT_IDLE,
@@ -1289,6 +1291,8 @@ struct sched_rt_entity {
         unsigned long timeout;
         unsigned long watchdog_stamp;
         unsigned int time_slice;
+        unsigned short on_rq;
+        unsigned short on_list;
 
         struct sched_rt_entity *back;
 #ifdef CONFIG_RT_GROUP_SCHED
@@ -1329,10 +1333,6 @@ struct sched_dl_entity {
          * task has to wait for a replenishment to be performed at the
          * next firing of dl_timer.
          *
-         * @dl_new tells if a new instance arrived. If so we must
-         * start executing it with full runtime and reset its absolute
-         * deadline;
-         *
          * @dl_boosted tells if we are boosted due to DI. If so we are
          * outside bandwidth enforcement mechanism (but only until we
          * exit the critical section);
@@ -1340,7 +1340,7 @@ struct sched_dl_entity {
          * @dl_yielded tells if task gave up the cpu before consuming
          * all its available runtime during the last job.
          */
-        int dl_throttled, dl_new, dl_boosted, dl_yielded;
+        int dl_throttled, dl_boosted, dl_yielded;
 
         /*
          * Bandwidth enforcement timer. Each -deadline task has its
diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h
index c9e4731cf10b..4f080ab4f2cd 100644
--- a/include/linux/sched/sysctl.h
+++ b/include/linux/sched/sysctl.h
@@ -95,4 +95,8 @@ extern int sysctl_numa_balancing(struct ctl_table *table, int write,
                                  void __user *buffer, size_t *lenp,
                                  loff_t *ppos);
 
+extern int sysctl_schedstats(struct ctl_table *table, int write,
+                                 void __user *buffer, size_t *lenp,
+                                 loff_t *ppos);
+
 #endif /* _SCHED_SYSCTL_H */
diff --git a/include/linux/swait.h b/include/linux/swait.h
new file mode 100644
index 000000000000..c1f9c62a8a50
--- /dev/null
+++ b/include/linux/swait.h
@@ -0,0 +1,172 @@
+#ifndef _LINUX_SWAIT_H
+#define _LINUX_SWAIT_H
+
+#include <linux/list.h>
+#include <linux/stddef.h>
+#include <linux/spinlock.h>
+#include <asm/current.h>
+
+/*
+ * Simple wait queues
+ *
+ * While these are very similar to the other/complex wait queues (wait.h) the
+ * most important difference is that the simple waitqueue allows for
+ * deterministic behaviour -- IOW it has strictly bounded IRQ and lock hold
+ * times.
+ *
+ * In order to make this so, we had to drop a fair number of features of the
+ * other waitqueue code; notably:
+ *
+ *  - mixing INTERRUPTIBLE and UNINTERRUPTIBLE sleeps on the same waitqueue;
+ *    all wakeups are TASK_NORMAL in order to avoid O(n) lookups for the right
+ *    sleeper state.
+ *
+ *  - the exclusive mode; because this requires preserving the list order
+ *    and this is hard.
+ *
+ *  - custom wake functions; because you cannot give any guarantees about
+ *    random code.
+ *
+ * As a side effect of this; the data structures are slimmer.
+ *
+ * One would recommend using this wait queue where possible.
+ */
+
+struct task_struct;
+
+struct swait_queue_head {
+        raw_spinlock_t          lock;
+        struct list_head        task_list;
+};
+
+struct swait_queue {
+        struct task_struct      *task;
+        struct list_head        task_list;
+};
+
+#define __SWAITQUEUE_INITIALIZER(name) {                                \
+        .task           = current,                                      \
+        .task_list      = LIST_HEAD_INIT((name).task_list),             \
+}
+
+#define DECLARE_SWAITQUEUE(name)                                        \
+        struct swait_queue name = __SWAITQUEUE_INITIALIZER(name)
+
+#define __SWAIT_QUEUE_HEAD_INITIALIZER(name) {                          \
+        .lock           = __RAW_SPIN_LOCK_UNLOCKED(name.lock),          \
+        .task_list      = LIST_HEAD_INIT((name).task_list),             \
+}
+
+#define DECLARE_SWAIT_QUEUE_HEAD(name)                                  \
+        struct swait_queue_head name = __SWAIT_QUEUE_HEAD_INITIALIZER(name)
+
+extern void __init_swait_queue_head(struct swait_queue_head *q, const char *name,
+                                    struct lock_class_key *key);
+
+#define init_swait_queue_head(q)                                \
+        do {                                                    \
+                static struct lock_class_key __key;             \
+                __init_swait_queue_head((q), #q, &__key);       \
+        } while (0)
+
+#ifdef CONFIG_LOCKDEP
+# define __SWAIT_QUEUE_HEAD_INIT_ONSTACK(name)                  \
+        ({ init_swait_queue_head(&name); name; })
+# define DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(name)                 \
+        struct swait_queue_head name = __SWAIT_QUEUE_HEAD_INIT_ONSTACK(name)
+#else
+# define DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(name)                 \
+        DECLARE_SWAIT_QUEUE_HEAD(name)
+#endif
+
+static inline int swait_active(struct swait_queue_head *q)
+{
+        return !list_empty(&q->task_list);
+}
+
+extern void swake_up(struct swait_queue_head *q);
+extern void swake_up_all(struct swait_queue_head *q);
+extern void swake_up_locked(struct swait_queue_head *q);
+
+extern void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait);
+extern void prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait, int state);
+extern long prepare_to_swait_event(struct swait_queue_head *q, struct swait_queue *wait, int state);
+
+extern void __finish_swait(struct swait_queue_head *q, struct swait_queue *wait);
+extern void finish_swait(struct swait_queue_head *q, struct swait_queue *wait);
+
+/* as per ___wait_event() but for swait, therefore "exclusive == 0" */
+#define ___swait_event(wq, condition, state, ret, cmd)                  \
+({                                                                      \
+        struct swait_queue __wait;                                      \
+        long __ret = ret;                                               \
+                                                                        \
+        INIT_LIST_HEAD(&__wait.task_list);                              \
+        for (;;) {                                                      \
+                long __int = prepare_to_swait_event(&wq, &__wait, state);\
+                                                                        \
+                if (condition)                                          \
+                        break;                                          \
+                                                                        \
+                if (___wait_is_interruptible(state) && __int) {         \
+                        __ret = __int;                                  \
+                        break;                                          \
+                }                                                       \
+                                                                        \
+                cmd;                                                    \
+        }                                                               \
+        finish_swait(&wq, &__wait);                                     \
+        __ret;                                                          \
+})
+
+#define __swait_event(wq, condition)                                    \
+        (void)___swait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0,    \
+                            schedule())
+
+#define swait_event(wq, condition)                                      \
+do {                                                                    \
+        if (condition)                                                  \
+                break;                                                  \
+        __swait_event(wq, condition);                                   \
+} while (0)
+
+#define __swait_event_timeout(wq, condition, timeout)                   \
+        ___swait_event(wq, ___wait_cond_timeout(condition),             \
+                      TASK_UNINTERRUPTIBLE, timeout,                    \
+                      __ret = schedule_timeout(__ret))
+
+#define swait_event_timeout(wq, condition, timeout)                     \
+({                                                                      \
+        long __ret = timeout;                                           \
+        if (!___wait_cond_timeout(condition))                           \
+                __ret = __swait_event_timeout(wq, condition, timeout);  \
+        __ret;                                                          \
+})
+
+#define __swait_event_interruptible(wq, condition)                      \
+        ___swait_event(wq, condition, TASK_INTERRUPTIBLE, 0,            \
+                      schedule())
+
+#define swait_event_interruptible(wq, condition)                        \
+({                                                                      \
+        int __ret = 0;                                                  \
+        if (!(condition))                                               \
+                __ret = __swait_event_interruptible(wq, condition);     \
+        __ret;                                                          \
+})
+
+#define __swait_event_interruptible_timeout(wq, condition, timeout)     \
+        ___swait_event(wq, ___wait_cond_timeout(condition),             \
+                      TASK_INTERRUPTIBLE, timeout,                      \
+                      __ret = schedule_timeout(__ret))
+
+#define swait_event_interruptible_timeout(wq, condition, timeout)       \
+({                                                                      \
+        long __ret = timeout;                                           \
+        if (!___wait_cond_timeout(condition))                           \
+                __ret = __swait_event_interruptible_timeout(wq,         \
+                                                condition, timeout);    \
+        __ret;                                                          \
+})
+
+#endif /* _LINUX_SWAIT_H */
diff --git a/include/linux/wait.h b/include/linux/wait.h
index ae71a769b89e..27d7a0ab5da3 100644
--- a/include/linux/wait.h
+++ b/include/linux/wait.h
@@ -338,7 +338,7 @@ do {									\
                             schedule(); try_to_freeze())
 
 /**
- * wait_event - sleep (or freeze) until a condition gets true
+ * wait_event_freezable - sleep (or freeze) until a condition gets true
  * @wq: the waitqueue to wait on
  * @condition: a C expression for the event to wait for
  *