1 files changed, 101 insertions, 45 deletions

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 0f1ea4a66957..8af3d249170e 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -38,6 +38,8 @@
 #define SCHED_BATCH             3
 /* SCHED_ISO: reserved but not implemented yet */
 #define SCHED_IDLE              5
+/* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */
+#define SCHED_RESET_ON_FORK     0x40000000
 
 #ifdef __KERNEL__
 
@@ -188,6 +190,7 @@ extern unsigned long long time_sync_thresh;
 /* in tsk->state again */
 #define TASK_DEAD               64
 #define TASK_WAKEKILL           128
+#define TASK_WAKING             256
 
 /* Convenience macros for the sake of set_task_state */
 #define TASK_KILLABLE           (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
@@ -796,18 +799,19 @@ enum cpu_idle_type {
 #define SCHED_LOAD_SCALE_FUZZ   SCHED_LOAD_SCALE
 
 #ifdef CONFIG_SMP
-#define SD_LOAD_BALANCE         1       /* Do load balancing on this domain. */
-#define SD_BALANCE_NEWIDLE      2       /* Balance when about to become idle */
-#define SD_BALANCE_EXEC         4       /* Balance on exec */
-#define SD_BALANCE_FORK         8       /* Balance on fork, clone */
-#define SD_WAKE_IDLE            16      /* Wake to idle CPU on task wakeup */
-#define SD_WAKE_AFFINE          32      /* Wake task to waking CPU */
-#define SD_WAKE_BALANCE         64      /* Perform balancing at task wakeup */
-#define SD_SHARE_CPUPOWER       128     /* Domain members share cpu power */
-#define SD_POWERSAVINGS_BALANCE 256     /* Balance for power savings */
-#define SD_SHARE_PKG_RESOURCES  512     /* Domain members share cpu pkg resources */
-#define SD_SERIALIZE            1024    /* Only a single load balancing instance */
-#define SD_WAKE_IDLE_FAR        2048    /* Gain latency sacrificing cache hit */
+#define SD_LOAD_BALANCE         0x0001  /* Do load balancing on this domain. */
+#define SD_BALANCE_NEWIDLE      0x0002  /* Balance when about to become idle */
+#define SD_BALANCE_EXEC         0x0004  /* Balance on exec */
+#define SD_BALANCE_FORK         0x0008  /* Balance on fork, clone */
+#define SD_BALANCE_WAKE         0x0010  /* Balance on wakeup */
+#define SD_WAKE_AFFINE          0x0020  /* Wake task to waking CPU */
+#define SD_PREFER_LOCAL         0x0040  /* Prefer to keep tasks local to this domain */
+#define SD_SHARE_CPUPOWER       0x0080  /* Domain members share cpu power */
+#define SD_POWERSAVINGS_BALANCE 0x0100  /* Balance for power savings */
+#define SD_SHARE_PKG_RESOURCES  0x0200  /* Domain members share cpu pkg resources */
+#define SD_SERIALIZE            0x0400  /* Only a single load balancing instance */
+
+#define SD_PREFER_SIBLING       0x1000  /* Prefer to place tasks in a sibling domain */
 
 enum powersavings_balance_level {
         POWERSAVINGS_BALANCE_NONE = 0,  /* No power saving load balance */
@@ -827,7 +831,7 @@ static inline int sd_balance_for_mc_power(void)
         if (sched_smt_power_savings)
                 return SD_POWERSAVINGS_BALANCE;
 
-        return 0;
+        return SD_PREFER_SIBLING;
 }
 
 static inline int sd_balance_for_package_power(void)
@@ -835,7 +839,7 @@ static inline int sd_balance_for_package_power(void)
         if (sched_mc_power_savings | sched_smt_power_savings)
                 return SD_POWERSAVINGS_BALANCE;
 
-        return 0;
+        return SD_PREFER_SIBLING;
 }
 
 /*
@@ -857,15 +861,9 @@ struct sched_group {
 
         /*
          * CPU power of this group, SCHED_LOAD_SCALE being max power for a
-         * single CPU. This is read only (except for setup, hotplug CPU).
-         * Note : Never change cpu_power without recompute its reciprocal
-         */
-        unsigned int __cpu_power;
-        /*
-         * reciprocal value of cpu_power to avoid expensive divides
-         * (see include/linux/reciprocal_div.h)
+         * single CPU.
          */
-        u32 reciprocal_cpu_power;
+        unsigned int cpu_power;
 
         /*
          * The CPUs this group covers.
@@ -918,6 +916,7 @@ struct sched_domain {
         unsigned int newidle_idx;
         unsigned int wake_idx;
         unsigned int forkexec_idx;
+        unsigned int smt_gain;
         int flags;                      /* See SD_* */
         enum sched_domain_level level;
 
@@ -993,6 +992,9 @@ static inline int test_sd_parent(struct sched_domain *sd, int flag)
         return 0;
 }
 
+unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu);
+unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu);
+
 #else /* CONFIG_SMP */
 
 struct sched_domain_attr;
@@ -1004,6 +1006,7 @@ partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
 }
 #endif  /* !CONFIG_SMP */
 
+
 struct io_context;                      /* See blkdev.h */
 
 
@@ -1021,6 +1024,12 @@ struct uts_namespace;
 struct rq;
 struct sched_domain;
 
+/*
+ * wake flags
+ */
+#define WF_SYNC         0x01            /* waker goes to sleep after wakup */
+#define WF_FORK         0x02            /* child wakeup after fork */
+
 struct sched_class {
         const struct sched_class *next;
 
@@ -1028,13 +1037,13 @@ struct sched_class {
         void (*dequeue_task) (struct rq *rq, struct task_struct *p, int sleep);
         void (*yield_task) (struct rq *rq);
 
-        void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int sync);
+        void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int flags);
 
         struct task_struct * (*pick_next_task) (struct rq *rq);
         void (*put_prev_task) (struct rq *rq, struct task_struct *p);
 
 #ifdef CONFIG_SMP
-        int  (*select_task_rq)(struct task_struct *p, int sync);
+        int  (*select_task_rq)(struct task_struct *p, int sd_flag, int flags);
 
         unsigned long (*load_balance) (struct rq *this_rq, int this_cpu,
                         struct rq *busiest, unsigned long max_load_move,
@@ -1045,7 +1054,6 @@ struct sched_class {
                               struct rq *busiest, struct sched_domain *sd,
                               enum cpu_idle_type idle);
         void (*pre_schedule) (struct rq *this_rq, struct task_struct *task);
-        int (*needs_post_schedule) (struct rq *this_rq);
         void (*post_schedule) (struct rq *this_rq);
         void (*task_wake_up) (struct rq *this_rq, struct task_struct *task);
 
@@ -1105,11 +1113,15 @@ struct sched_entity {
         u64                     start_runtime;
         u64                     avg_wakeup;
 
+        u64                     avg_running;
+
 #ifdef CONFIG_SCHEDSTATS
         u64                     wait_start;
         u64                     wait_max;
         u64                     wait_count;
         u64                     wait_sum;
+        u64                     iowait_count;
+        u64                     iowait_sum;
 
         u64                     sleep_start;
         u64                     sleep_max;
@@ -1163,6 +1175,8 @@ struct sched_rt_entity {
 #endif
 };
 
+struct rcu_node;
+
 struct task_struct {
         volatile long state;    /* -1 unrunnable, 0 runnable, >0 stopped */
         void *stack;
@@ -1206,10 +1220,12 @@ struct task_struct {
         unsigned int policy;
         cpumask_t cpus_allowed;
 
-#ifdef CONFIG_PREEMPT_RCU
+#ifdef CONFIG_TREE_PREEMPT_RCU
         int rcu_read_lock_nesting;
-        int rcu_flipctr_idx;
-#endif /* #ifdef CONFIG_PREEMPT_RCU */
+        char rcu_read_unlock_special;
+        struct rcu_node *rcu_blocked_node;
+        struct list_head rcu_node_entry;
+#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
 
 #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
         struct sched_info sched_info;
@@ -1230,11 +1246,19 @@ struct task_struct {
         unsigned did_exec:1;
         unsigned in_execve:1;   /* Tell the LSMs that the process is doing an
                                  * execve */
+        unsigned in_iowait:1;
+
+
+        /* Revert to default priority/policy when forking */
+        unsigned sched_reset_on_fork:1;
+
         pid_t pid;
         pid_t tgid;
 
+#ifdef CONFIG_CC_STACKPROTECTOR
         /* Canary value for the -fstack-protector gcc feature */
         unsigned long stack_canary;
+#endif
 
         /* 
          * pointers to (original) parent process, youngest child, younger sibling,
@@ -1292,6 +1316,7 @@ struct task_struct {
         struct mutex cred_guard_mutex;  /* guard against foreign influences on
                                          * credential calculations
                                          * (notably. ptrace) */
+        struct cred *replacement_session_keyring; /* for KEYCTL_SESSION_TO_PARENT */
 
         char comm[TASK_COMM_LEN]; /* executable name excluding path
                                      - access with [gs]et_task_comm (which lock
@@ -1724,6 +1749,28 @@ extern cputime_t task_gtime(struct task_struct *p);
 #define tsk_used_math(p) ((p)->flags & PF_USED_MATH)
 #define used_math() tsk_used_math(current)
 
+#ifdef CONFIG_TREE_PREEMPT_RCU
+
+#define RCU_READ_UNLOCK_BLOCKED (1 << 0) /* blocked while in RCU read-side. */
+#define RCU_READ_UNLOCK_NEED_QS (1 << 1) /* RCU core needs CPU response. */
+#define RCU_READ_UNLOCK_GOT_QS  (1 << 2) /* CPU has responded to RCU core. */
+
+static inline void rcu_copy_process(struct task_struct *p)
+{
+        p->rcu_read_lock_nesting = 0;
+        p->rcu_read_unlock_special = 0;
+        p->rcu_blocked_node = NULL;
+        INIT_LIST_HEAD(&p->rcu_node_entry);
+}
+
+#else
+
+static inline void rcu_copy_process(struct task_struct *p)
+{
+}
+
+#endif
+
 #ifdef CONFIG_SMP
 extern int set_cpus_allowed_ptr(struct task_struct *p,
                                 const struct cpumask *new_mask);
@@ -1813,11 +1860,12 @@ extern unsigned int sysctl_sched_min_granularity;
 extern unsigned int sysctl_sched_wakeup_granularity;
 extern unsigned int sysctl_sched_shares_ratelimit;
 extern unsigned int sysctl_sched_shares_thresh;
-#ifdef CONFIG_SCHED_DEBUG
 extern unsigned int sysctl_sched_child_runs_first;
+#ifdef CONFIG_SCHED_DEBUG
 extern unsigned int sysctl_sched_features;
 extern unsigned int sysctl_sched_migration_cost;
 extern unsigned int sysctl_sched_nr_migrate;
+extern unsigned int sysctl_sched_time_avg;
 extern unsigned int sysctl_timer_migration;
 
 int sched_nr_latency_handler(struct ctl_table *table, int write,
@@ -2077,7 +2125,7 @@ static inline unsigned long wait_task_inactive(struct task_struct *p,
 #define for_each_process(p) \
         for (p = &init_task ; (p = next_task(p)) != &init_task ; )
 
-extern bool is_single_threaded(struct task_struct *);
+extern bool current_is_single_threaded(void);
 
 /*
  * Careful: do_each_thread/while_each_thread is a double loop so
@@ -2281,23 +2329,31 @@ static inline int need_resched(void)
  * cond_resched_softirq() will enable bhs before scheduling.
  */
 extern int _cond_resched(void);
-#ifdef CONFIG_PREEMPT_BKL
-static inline int cond_resched(void)
-{
-        return 0;
-}
+
+#define cond_resched() ({                       \
+        __might_sleep(__FILE__, __LINE__, 0);   \
+        _cond_resched();                        \
+})
+
+extern int __cond_resched_lock(spinlock_t *lock);
+
+#ifdef CONFIG_PREEMPT
+#define PREEMPT_LOCK_OFFSET     PREEMPT_OFFSET
 #else
-static inline int cond_resched(void)
-{
-        return _cond_resched();
-}
+#define PREEMPT_LOCK_OFFSET     0
 #endif
-extern int cond_resched_lock(spinlock_t * lock);
-extern int cond_resched_softirq(void);
-static inline int cond_resched_bkl(void)
-{
-        return _cond_resched();
-}
+
+#define cond_resched_lock(lock) ({                              \
+        __might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \
+        __cond_resched_lock(lock);                              \
+})
+
+extern int __cond_resched_softirq(void);
+
+#define cond_resched_softirq() ({                               \
+        __might_sleep(__FILE__, __LINE__, SOFTIRQ_OFFSET);      \
+        __cond_resched_softirq();                               \
+})
 
 /*
  * Does a critical section need to be broken due to another