1 files changed, 105 insertions, 40 deletions

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 26a2e6122734..6633e83e608a 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -25,7 +25,7 @@ struct sched_param {
 #include <linux/errno.h>
 #include <linux/nodemask.h>
 #include <linux/mm_types.h>
-#include <linux/preempt_mask.h>
+#include <linux/preempt.h>
 
 #include <asm/page.h>
 #include <asm/ptrace.h>
@@ -132,6 +132,7 @@ struct fs_struct;
 struct perf_event_context;
 struct blk_plug;
 struct filename;
+struct nameidata;
 
 #define VMACACHE_BITS 2
 #define VMACACHE_SIZE (1U << VMACACHE_BITS)
@@ -173,7 +174,12 @@ extern unsigned long nr_iowait_cpu(int cpu);
 extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load);
 
 extern void calc_global_load(unsigned long ticks);
+
+#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
 extern void update_cpu_load_nohz(void);
+#else
+static inline void update_cpu_load_nohz(void) { }
+#endif
 
 extern unsigned long get_parent_ip(unsigned long addr);
 
@@ -213,9 +219,10 @@ print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq);
 #define TASK_WAKEKILL           128
 #define TASK_WAKING             256
 #define TASK_PARKED             512
-#define TASK_STATE_MAX          1024
+#define TASK_NOLOAD             1024
+#define TASK_STATE_MAX          2048
 
-#define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWP"
+#define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWPN"
 
 extern char ___assert_task_state[1 - 2*!!(
                 sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1)];
@@ -225,6 +232,8 @@ extern char ___assert_task_state[1 - 2*!!(
 #define TASK_STOPPED            (TASK_WAKEKILL | __TASK_STOPPED)
 #define TASK_TRACED             (TASK_WAKEKILL | __TASK_TRACED)
 
+#define TASK_IDLE               (TASK_UNINTERRUPTIBLE | TASK_NOLOAD)
+
 /* Convenience macros for the sake of wake_up */
 #define TASK_NORMAL             (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
 #define TASK_ALL                (TASK_NORMAL | __TASK_STOPPED | __TASK_TRACED)
@@ -240,7 +249,8 @@ extern char ___assert_task_state[1 - 2*!!(
                         ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
 #define task_contributes_to_load(task)  \
                                 ((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
-                                 (task->flags & PF_FROZEN) == 0)
+                                 (task->flags & PF_FROZEN) == 0 && \
+                                 (task->state & TASK_NOLOAD) == 0)
 
 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
 
@@ -252,7 +262,7 @@ extern char ___assert_task_state[1 - 2*!!(
 #define set_task_state(tsk, state_value)                        \
         do {                                                    \
                 (tsk)->task_state_change = _THIS_IP_;           \
-                set_mb((tsk)->state, (state_value));            \
+                smp_store_mb((tsk)->state, (state_value));              \
         } while (0)
 
 /*
@@ -274,7 +284,7 @@ extern char ___assert_task_state[1 - 2*!!(
 #define set_current_state(state_value)                          \
         do {                                                    \
                 current->task_state_change = _THIS_IP_;         \
-                set_mb(current->state, (state_value));          \
+                smp_store_mb(current->state, (state_value));            \
         } while (0)
 
 #else
@@ -282,7 +292,7 @@ extern char ___assert_task_state[1 - 2*!!(
 #define __set_task_state(tsk, state_value)              \
         do { (tsk)->state = (state_value); } while (0)
 #define set_task_state(tsk, state_value)                \
-        set_mb((tsk)->state, (state_value))
+        smp_store_mb((tsk)->state, (state_value))
 
 /*
  * set_current_state() includes a barrier so that the write of current->state
@@ -298,7 +308,7 @@ extern char ___assert_task_state[1 - 2*!!(
 #define __set_current_state(state_value)                \
         do { current->state = (state_value); } while (0)
 #define set_current_state(state_value)                  \
-        set_mb(current->state, (state_value))
+        smp_store_mb(current->state, (state_value))
 
 #endif
 
@@ -335,14 +345,10 @@ extern int runqueue_is_locked(int cpu);
 #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
 extern void nohz_balance_enter_idle(int cpu);
 extern void set_cpu_sd_state_idle(void);
-extern int get_nohz_timer_target(int pinned);
+extern int get_nohz_timer_target(void);
 #else
 static inline void nohz_balance_enter_idle(int cpu) { }
 static inline void set_cpu_sd_state_idle(void) { }
-static inline int get_nohz_timer_target(int pinned)
-{
-        return smp_processor_id();
-}
 #endif
 
 /*
@@ -567,6 +573,23 @@ struct task_cputime {
                 .sum_exec_runtime = 0,                          \
         }
 
+/*
+ * This is the atomic variant of task_cputime, which can be used for
+ * storing and updating task_cputime statistics without locking.
+ */
+struct task_cputime_atomic {
+        atomic64_t utime;
+        atomic64_t stime;
+        atomic64_t sum_exec_runtime;
+};
+
+#define INIT_CPUTIME_ATOMIC \
+        (struct task_cputime_atomic) {                          \
+                .utime = ATOMIC64_INIT(0),                      \
+                .stime = ATOMIC64_INIT(0),                      \
+                .sum_exec_runtime = ATOMIC64_INIT(0),           \
+        }
+
 #ifdef CONFIG_PREEMPT_COUNT
 #define PREEMPT_DISABLED        (1 + PREEMPT_ENABLED)
 #else
@@ -584,18 +607,16 @@ struct task_cputime {
 
 /**
  * struct thread_group_cputimer - thread group interval timer counts
- * @cputime:            thread group interval timers.
+ * @cputime_atomic:     atomic thread group interval timers.
  * @running:            non-zero when there are timers running and
  *                      @cputime receives updates.
- * @lock:               lock for fields in this struct.
  *
  * This structure contains the version of task_cputime, above, that is
  * used for thread group CPU timer calculations.
  */
 struct thread_group_cputimer {
-        struct task_cputime cputime;
+        struct task_cputime_atomic cputime_atomic;
         int running;
-        raw_spinlock_t lock;
 };
 
 #include <linux/rwsem.h>
@@ -900,6 +921,50 @@ enum cpu_idle_type {
 #define SCHED_CAPACITY_SCALE    (1L << SCHED_CAPACITY_SHIFT)
 
 /*
+ * Wake-queues are lists of tasks with a pending wakeup, whose
+ * callers have already marked the task as woken internally,
+ * and can thus carry on. A common use case is being able to
+ * do the wakeups once the corresponding user lock as been
+ * released.
+ *
+ * We hold reference to each task in the list across the wakeup,
+ * thus guaranteeing that the memory is still valid by the time
+ * the actual wakeups are performed in wake_up_q().
+ *
+ * One per task suffices, because there's never a need for a task to be
+ * in two wake queues simultaneously; it is forbidden to abandon a task
+ * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is
+ * already in a wake queue, the wakeup will happen soon and the second
+ * waker can just skip it.
+ *
+ * The WAKE_Q macro declares and initializes the list head.
+ * wake_up_q() does NOT reinitialize the list; it's expected to be
+ * called near the end of a function, where the fact that the queue is
+ * not used again will be easy to see by inspection.
+ *
+ * Note that this can cause spurious wakeups. schedule() callers
+ * must ensure the call is done inside a loop, confirming that the
+ * wakeup condition has in fact occurred.
+ */
+struct wake_q_node {
+        struct wake_q_node *next;
+};
+
+struct wake_q_head {
+        struct wake_q_node *first;
+        struct wake_q_node **lastp;
+};
+
+#define WAKE_Q_TAIL ((struct wake_q_node *) 0x01)
+
+#define WAKE_Q(name)                                    \
+        struct wake_q_head name = { WAKE_Q_TAIL, &name.first }
+
+extern void wake_q_add(struct wake_q_head *head,
+                       struct task_struct *task);
+extern void wake_up_q(struct wake_q_head *head);
+
+/*
  * sched-domains (multiprocessor balancing) declarations:
  */
 #ifdef CONFIG_SMP
@@ -1334,8 +1399,6 @@ struct task_struct {
         int rcu_read_lock_nesting;
         union rcu_special rcu_read_unlock_special;
         struct list_head rcu_node_entry;
-#endif /* #ifdef CONFIG_PREEMPT_RCU */
-#ifdef CONFIG_PREEMPT_RCU
         struct rcu_node *rcu_blocked_node;
 #endif /* #ifdef CONFIG_PREEMPT_RCU */
 #ifdef CONFIG_TASKS_RCU
@@ -1356,9 +1419,6 @@ struct task_struct {
 #endif
 
         struct mm_struct *mm, *active_mm;
-#ifdef CONFIG_COMPAT_BRK
-        unsigned brk_randomized:1;
-#endif
         /* per-thread vma caching */
         u32 vmacache_seqnum;
         struct vm_area_struct *vmacache[VMACACHE_SIZE];
@@ -1369,7 +1429,7 @@ struct task_struct {
         int exit_state;
         int exit_code, exit_signal;
         int pdeath_signal;  /*  The signal sent when the parent dies  */
-        unsigned int jobctl;    /* JOBCTL_*, siglock protected */
+        unsigned long jobctl;   /* JOBCTL_*, siglock protected */
 
         /* Used for emulating ABI behavior of previous Linux versions */
         unsigned int personality;
@@ -1381,10 +1441,14 @@ struct task_struct {
         /* Revert to default priority/policy when forking */
         unsigned sched_reset_on_fork:1;
         unsigned sched_contributes_to_load:1;
+        unsigned sched_migrated:1;
 
 #ifdef CONFIG_MEMCG_KMEM
         unsigned memcg_kmem_skip_account:1;
 #endif
+#ifdef CONFIG_COMPAT_BRK
+        unsigned brk_randomized:1;
+#endif
 
         unsigned long atomic_flags; /* Flags needing atomic access. */
 
@@ -1461,7 +1525,7 @@ struct task_struct {
                                        it with task_lock())
                                      - initialized normally by setup_new_exec */
 /* file system info */
-        int link_count, total_link_count;
+        struct nameidata *nameidata;
 #ifdef CONFIG_SYSVIPC
 /* ipc stuff */
         struct sysv_sem sysvsem;
@@ -1511,6 +1575,8 @@ struct task_struct {
         /* Protection of the PI data structures: */
         raw_spinlock_t pi_lock;
 
+        struct wake_q_node wake_q;
+
 #ifdef CONFIG_RT_MUTEXES
         /* PI waiters blocked on a rt_mutex held by this task */
         struct rb_root pi_waiters;
@@ -1724,6 +1790,7 @@ struct task_struct {
 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
         unsigned long   task_state_change;
 #endif
+        int pagefault_disabled;
 };
 
 /* Future-safe accessor for struct task_struct's cpus_allowed. */
@@ -2077,22 +2144,22 @@ TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
 #define JOBCTL_TRAPPING_BIT     21      /* switching to TRACED */
 #define JOBCTL_LISTENING_BIT    22      /* ptracer is listening for events */
 
-#define JOBCTL_STOP_DEQUEUED    (1 << JOBCTL_STOP_DEQUEUED_BIT)
-#define JOBCTL_STOP_PENDING     (1 << JOBCTL_STOP_PENDING_BIT)
-#define JOBCTL_STOP_CONSUME     (1 << JOBCTL_STOP_CONSUME_BIT)
-#define JOBCTL_TRAP_STOP        (1 << JOBCTL_TRAP_STOP_BIT)
-#define JOBCTL_TRAP_NOTIFY      (1 << JOBCTL_TRAP_NOTIFY_BIT)
-#define JOBCTL_TRAPPING         (1 << JOBCTL_TRAPPING_BIT)
-#define JOBCTL_LISTENING        (1 << JOBCTL_LISTENING_BIT)
+#define JOBCTL_STOP_DEQUEUED    (1UL << JOBCTL_STOP_DEQUEUED_BIT)
+#define JOBCTL_STOP_PENDING     (1UL << JOBCTL_STOP_PENDING_BIT)
+#define JOBCTL_STOP_CONSUME     (1UL << JOBCTL_STOP_CONSUME_BIT)
+#define JOBCTL_TRAP_STOP        (1UL << JOBCTL_TRAP_STOP_BIT)
+#define JOBCTL_TRAP_NOTIFY      (1UL << JOBCTL_TRAP_NOTIFY_BIT)
+#define JOBCTL_TRAPPING         (1UL << JOBCTL_TRAPPING_BIT)
+#define JOBCTL_LISTENING        (1UL << JOBCTL_LISTENING_BIT)
 
 #define JOBCTL_TRAP_MASK        (JOBCTL_TRAP_STOP | JOBCTL_TRAP_NOTIFY)
 #define JOBCTL_PENDING_MASK     (JOBCTL_STOP_PENDING | JOBCTL_TRAP_MASK)
 
 extern bool task_set_jobctl_pending(struct task_struct *task,
-                                    unsigned int mask);
+                                    unsigned long mask);
 extern void task_clear_jobctl_trapping(struct task_struct *task);
 extern void task_clear_jobctl_pending(struct task_struct *task,
-                                      unsigned int mask);
+                                      unsigned long mask);
 
 static inline void rcu_copy_process(struct task_struct *p)
 {
@@ -2532,6 +2599,9 @@ static inline unsigned long wait_task_inactive(struct task_struct *p,
 }
 #endif
 
+#define tasklist_empty() \
+        list_empty(&init_task.tasks)
+
 #define next_task(p) \
         list_entry_rcu((p)->tasks.next, struct task_struct, tasks)
 
@@ -2962,11 +3032,6 @@ static __always_inline bool need_resched(void)
 void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times);
 void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times);
 
-static inline void thread_group_cputime_init(struct signal_struct *sig)
-{
-        raw_spin_lock_init(&sig->cputimer.lock);
-}
-
 /*
  * Reevaluate whether the task has signals pending delivery.
  * Wake the task if so.
@@ -3080,13 +3145,13 @@ static inline void mm_update_next_owner(struct mm_struct *mm)
 static inline unsigned long task_rlimit(const struct task_struct *tsk,
                 unsigned int limit)
 {
-        return ACCESS_ONCE(tsk->signal->rlim[limit].rlim_cur);
+        return READ_ONCE(tsk->signal->rlim[limit].rlim_cur);
 }
 
 static inline unsigned long task_rlimit_max(const struct task_struct *tsk,
                 unsigned int limit)
 {
-        return ACCESS_ONCE(tsk->signal->rlim[limit].rlim_max);
+        return READ_ONCE(tsk->signal->rlim[limit].rlim_max);
 }
 
 static inline unsigned long rlimit(unsigned int limit)