Merge branch 'linus' into perf/core, to pick up fixes

Signed-off-by: Ingo Molnar <mingo@kernel.org>

21 files changed, 201 insertions, 86 deletions

diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index 3ea87fb19a94..afe5bab376c9 100644
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -13,11 +13,12 @@
 #include <linux/bpf.h>
 #include <linux/jhash.h>
 #include <linux/filter.h>
+#include <linux/rculist_nulls.h>
 #include "percpu_freelist.h"
 #include "bpf_lru_list.h"
 
 struct bucket {
-        struct hlist_head head;
+        struct hlist_nulls_head head;
         raw_spinlock_t lock;
 };
 
@@ -44,9 +45,14 @@ enum extra_elem_state {
 /* each htab element is struct htab_elem + key + value */
 struct htab_elem {
         union {
-                struct hlist_node hash_node;
-                struct bpf_htab *htab;
-                struct pcpu_freelist_node fnode;
+                struct hlist_nulls_node hash_node;
+                struct {
+                        void *padding;
+                        union {
+                                struct bpf_htab *htab;
+                                struct pcpu_freelist_node fnode;
+                        };
+                };
         };
         union {
                 struct rcu_head rcu;
@@ -162,7 +168,8 @@ skip_percpu_elems:
                                  offsetof(struct htab_elem, lru_node),
                                  htab->elem_size, htab->map.max_entries);
         else
-                pcpu_freelist_populate(&htab->freelist, htab->elems,
+                pcpu_freelist_populate(&htab->freelist,
+                                       htab->elems + offsetof(struct htab_elem, fnode),
                                        htab->elem_size, htab->map.max_entries);
 
         return 0;
@@ -217,6 +224,11 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
         int err, i;
         u64 cost;
 
+        BUILD_BUG_ON(offsetof(struct htab_elem, htab) !=
+                     offsetof(struct htab_elem, hash_node.pprev));
+        BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) !=
+                     offsetof(struct htab_elem, hash_node.pprev));
+
         if (lru && !capable(CAP_SYS_ADMIN))
                 /* LRU implementation is much complicated than other
                  * maps.  Hence, limit to CAP_SYS_ADMIN for now.
@@ -326,7 +338,7 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
                 goto free_htab;
 
         for (i = 0; i < htab->n_buckets; i++) {
-                INIT_HLIST_HEAD(&htab->buckets[i].head);
+                INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i);
                 raw_spin_lock_init(&htab->buckets[i].lock);
         }
 
@@ -366,20 +378,44 @@ static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash)
         return &htab->buckets[hash & (htab->n_buckets - 1)];
 }
 
-static inline struct hlist_head *select_bucket(struct bpf_htab *htab, u32 hash)
+static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32 hash)
 {
         return &__select_bucket(htab, hash)->head;
 }
 
-static struct htab_elem *lookup_elem_raw(struct hlist_head *head, u32 hash,
+/* this lookup function can only be called with bucket lock taken */
+static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash,
                                          void *key, u32 key_size)
 {
+        struct hlist_nulls_node *n;
+        struct htab_elem *l;
+
+        hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
+                if (l->hash == hash && !memcmp(&l->key, key, key_size))
+                        return l;
+
+        return NULL;
+}
+
+/* can be called without bucket lock. it will repeat the loop in
+ * the unlikely event when elements moved from one bucket into another
+ * while link list is being walked
+ */
+static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,
+                                               u32 hash, void *key,
+                                               u32 key_size, u32 n_buckets)
+{
+        struct hlist_nulls_node *n;
         struct htab_elem *l;
 
-        hlist_for_each_entry_rcu(l, head, hash_node)
+again:
+        hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
                 if (l->hash == hash && !memcmp(&l->key, key, key_size))
                         return l;
 
+        if (unlikely(get_nulls_value(n) != (hash & (n_buckets - 1))))
+                goto again;
+
         return NULL;
 }
 
@@ -387,7 +423,7 @@ static struct htab_elem *lookup_elem_raw(struct hlist_head *head, u32 hash,
 static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         struct htab_elem *l;
         u32 hash, key_size;
 
@@ -400,7 +436,7 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)
 
         head = select_bucket(htab, hash);
 
-        l = lookup_elem_raw(head, hash, key, key_size);
+        l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);
 
         return l;
 }
@@ -433,8 +469,9 @@ static void *htab_lru_map_lookup_elem(struct bpf_map *map, void *key)
 static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node)
 {
         struct bpf_htab *htab = (struct bpf_htab *)arg;
-        struct htab_elem *l, *tgt_l;
-        struct hlist_head *head;
+        struct htab_elem *l = NULL, *tgt_l;
+        struct hlist_nulls_head *head;
+        struct hlist_nulls_node *n;
         unsigned long flags;
         struct bucket *b;
 
@@ -444,9 +481,9 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node)
 
         raw_spin_lock_irqsave(&b->lock, flags);
 
-        hlist_for_each_entry_rcu(l, head, hash_node)
+        hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
                 if (l == tgt_l) {
-                        hlist_del_rcu(&l->hash_node);
+                        hlist_nulls_del_rcu(&l->hash_node);
                         break;
                 }
 
@@ -459,7 +496,7 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node)
 static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         struct htab_elem *l, *next_l;
         u32 hash, key_size;
         int i;
@@ -473,7 +510,7 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
         head = select_bucket(htab, hash);
 
         /* lookup the key */
-        l = lookup_elem_raw(head, hash, key, key_size);
+        l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);
 
         if (!l) {
                 i = 0;
@@ -481,7 +518,7 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
         }
 
         /* key was found, get next key in the same bucket */
-        next_l = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(&l->hash_node)),
+        next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_next_rcu(&l->hash_node)),
                                   struct htab_elem, hash_node);
 
         if (next_l) {
@@ -500,7 +537,7 @@ find_first_elem:
                 head = select_bucket(htab, i);
 
                 /* pick first element in the bucket */
-                next_l = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),
+                next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_first_rcu(head)),
                                           struct htab_elem, hash_node);
                 if (next_l) {
                         /* if it's not empty, just return it */
@@ -582,9 +619,13 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
         int err = 0;
 
         if (prealloc) {
-                l_new = (struct htab_elem *)pcpu_freelist_pop(&htab->freelist);
-                if (!l_new)
+                struct pcpu_freelist_node *l;
+
+                l = pcpu_freelist_pop(&htab->freelist);
+                if (!l)
                         err = -E2BIG;
+                else
+                        l_new = container_of(l, struct htab_elem, fnode);
         } else {
                 if (atomic_inc_return(&htab->count) > htab->map.max_entries) {
                         atomic_dec(&htab->count);
@@ -661,7 +702,7 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value,
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
         struct htab_elem *l_new = NULL, *l_old;
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         unsigned long flags;
         struct bucket *b;
         u32 key_size, hash;
@@ -700,9 +741,9 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value,
         /* add new element to the head of the list, so that
          * concurrent search will find it before old elem
          */
-        hlist_add_head_rcu(&l_new->hash_node, head);
+        hlist_nulls_add_head_rcu(&l_new->hash_node, head);
         if (l_old) {
-                hlist_del_rcu(&l_old->hash_node);
+                hlist_nulls_del_rcu(&l_old->hash_node);
                 free_htab_elem(htab, l_old);
         }
         ret = 0;
@@ -716,7 +757,7 @@ static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value,
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
         struct htab_elem *l_new, *l_old = NULL;
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         unsigned long flags;
         struct bucket *b;
         u32 key_size, hash;
@@ -757,10 +798,10 @@ static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value,
         /* add new element to the head of the list, so that
          * concurrent search will find it before old elem
          */
-        hlist_add_head_rcu(&l_new->hash_node, head);
+        hlist_nulls_add_head_rcu(&l_new->hash_node, head);
         if (l_old) {
                 bpf_lru_node_set_ref(&l_new->lru_node);
-                hlist_del_rcu(&l_old->hash_node);
+                hlist_nulls_del_rcu(&l_old->hash_node);
         }
         ret = 0;
 
@@ -781,7 +822,7 @@ static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key,
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
         struct htab_elem *l_new = NULL, *l_old;
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         unsigned long flags;
         struct bucket *b;
         u32 key_size, hash;
@@ -820,7 +861,7 @@ static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key,
                         ret = PTR_ERR(l_new);
                         goto err;
                 }
-                hlist_add_head_rcu(&l_new->hash_node, head);
+                hlist_nulls_add_head_rcu(&l_new->hash_node, head);
         }
         ret = 0;
 err:
@@ -834,7 +875,7 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
         struct htab_elem *l_new = NULL, *l_old;
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         unsigned long flags;
         struct bucket *b;
         u32 key_size, hash;
@@ -882,7 +923,7 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
         } else {
                 pcpu_copy_value(htab, htab_elem_get_ptr(l_new, key_size),
                                 value, onallcpus);
-                hlist_add_head_rcu(&l_new->hash_node, head);
+                hlist_nulls_add_head_rcu(&l_new->hash_node, head);
                 l_new = NULL;
         }
         ret = 0;
@@ -910,7 +951,7 @@ static int htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
 static int htab_map_delete_elem(struct bpf_map *map, void *key)
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         struct bucket *b;
         struct htab_elem *l;
         unsigned long flags;
@@ -930,7 +971,7 @@ static int htab_map_delete_elem(struct bpf_map *map, void *key)
         l = lookup_elem_raw(head, hash, key, key_size);
 
         if (l) {
-                hlist_del_rcu(&l->hash_node);
+                hlist_nulls_del_rcu(&l->hash_node);
                 free_htab_elem(htab, l);
                 ret = 0;
         }
@@ -942,7 +983,7 @@ static int htab_map_delete_elem(struct bpf_map *map, void *key)
 static int htab_lru_map_delete_elem(struct bpf_map *map, void *key)
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         struct bucket *b;
         struct htab_elem *l;
         unsigned long flags;
@@ -962,7 +1003,7 @@ static int htab_lru_map_delete_elem(struct bpf_map *map, void *key)
         l = lookup_elem_raw(head, hash, key, key_size);
 
         if (l) {
-                hlist_del_rcu(&l->hash_node);
+                hlist_nulls_del_rcu(&l->hash_node);
                 ret = 0;
         }
 
@@ -977,12 +1018,12 @@ static void delete_all_elements(struct bpf_htab *htab)
         int i;
 
         for (i = 0; i < htab->n_buckets; i++) {
-                struct hlist_head *head = select_bucket(htab, i);
-                struct hlist_node *n;
+                struct hlist_nulls_head *head = select_bucket(htab, i);
+                struct hlist_nulls_node *n;
                 struct htab_elem *l;
 
-                hlist_for_each_entry_safe(l, n, head, hash_node) {
-                        hlist_del_rcu(&l->hash_node);
+                hlist_nulls_for_each_entry_safe(l, n, head, hash_node) {
+                        hlist_nulls_del_rcu(&l->hash_node);
                         if (l->state != HTAB_EXTRA_ELEM_USED)
                                 htab_elem_free(htab, l);
                 }
diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c
index 8bfe0afaee10..b37bd9ab7f57 100644
--- a/kernel/bpf/lpm_trie.c
+++ b/kernel/bpf/lpm_trie.c
@@ -500,9 +500,15 @@ unlock:
         raw_spin_unlock(&trie->lock);
 }
 
+static int trie_get_next_key(struct bpf_map *map, void *key, void *next_key)
+{
+        return -ENOTSUPP;
+}
+
 static const struct bpf_map_ops trie_ops = {
         .map_alloc = trie_alloc,
         .map_free = trie_free,
+        .map_get_next_key = trie_get_next_key,
         .map_lookup_elem = trie_lookup_elem,
         .map_update_elem = trie_update_elem,
         .map_delete_elem = trie_delete_elem,
diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c
index 56eba9caa632..1dc22f6b49f5 100644
--- a/kernel/cgroup/cgroup-v1.c
+++ b/kernel/cgroup/cgroup-v1.c
@@ -1329,7 +1329,7 @@ static int cgroup_css_links_read(struct seq_file *seq, void *v)
                 struct task_struct *task;
                 int count = 0;
 
-                seq_printf(seq, "css_set %p\n", cset);
+                seq_printf(seq, "css_set %pK\n", cset);
 
                 list_for_each_entry(task, &cset->tasks, cg_list) {
                         if (count++ > MAX_TASKS_SHOWN_PER_CSS)
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 0125589c7428..48851327a15e 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -2669,7 +2669,7 @@ static bool css_visible(struct cgroup_subsys_state *css)
  *
  * Returns 0 on success, -errno on failure.  On failure, csses which have
  * been processed already aren't cleaned up.  The caller is responsible for
- * cleaning up with cgroup_apply_control_disble().
+ * cleaning up with cgroup_apply_control_disable().
  */
 static int cgroup_apply_control_enable(struct cgroup *cgrp)
 {
diff --git a/kernel/cgroup/pids.c b/kernel/cgroup/pids.c
index e756dae49300..2237201d66d5 100644
--- a/kernel/cgroup/pids.c
+++ b/kernel/cgroup/pids.c
@@ -229,7 +229,7 @@ static int pids_can_fork(struct task_struct *task)
                 /* Only log the first time events_limit is incremented. */
                 if (atomic64_inc_return(&pids->events_limit) == 1) {
                         pr_info("cgroup: fork rejected by pids controller in ");
-                        pr_cont_cgroup_path(task_cgroup(current, pids_cgrp_id));
+                        pr_cont_cgroup_path(css->cgroup);
                         pr_cont("\n");
                 }
                 cgroup_file_notify(&pids->events_file);
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 16c877a121c8..2d7990d4e988 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -1001,7 +1001,7 @@ list_update_cgroup_event(struct perf_event *event,
  */
 #define PERF_CPU_HRTIMER (1000 / HZ)
 /*
- * function must be called with interrupts disbled
+ * function must be called with interrupts disabled
  */
 static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr)
 {
diff --git a/kernel/exit.c b/kernel/exit.c
index e126ebf2400c..516acdb0e0ec 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -554,7 +554,6 @@ static void exit_mm(void)
         enter_lazy_tlb(mm, current);
         task_unlock(current);
         mm_update_next_owner(mm);
-        userfaultfd_exit(mm);
         mmput(mm);
         if (test_thread_flag(TIF_MEMDIE))
                 exit_oom_victim();
diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c
index b56a558e406d..b118735fea9d 100644
--- a/kernel/kexec_file.c
+++ b/kernel/kexec_file.c
@@ -614,13 +614,13 @@ static int kexec_calculate_store_digests(struct kimage *image)
                 ret = crypto_shash_final(desc, digest);
                 if (ret)
                         goto out_free_digest;
-                ret = kexec_purgatory_get_set_symbol(image, "sha_regions",
-                                                sha_regions, sha_region_sz, 0);
+                ret = kexec_purgatory_get_set_symbol(image, "purgatory_sha_regions",
+                                                     sha_regions, sha_region_sz, 0);
                 if (ret)
                         goto out_free_digest;
 
-                ret = kexec_purgatory_get_set_symbol(image, "sha256_digest",
-                                                digest, SHA256_DIGEST_SIZE, 0);
+                ret = kexec_purgatory_get_set_symbol(image, "purgatory_sha256_digest",
+                                                     digest, SHA256_DIGEST_SIZE, 0);
                 if (ret)
                         goto out_free_digest;
         }
diff --git a/kernel/kexec_internal.h b/kernel/kexec_internal.h
index 4cef7e4706b0..799a8a452187 100644
--- a/kernel/kexec_internal.h
+++ b/kernel/kexec_internal.h
@@ -15,11 +15,7 @@ int kimage_is_destination_range(struct kimage *image,
 extern struct mutex kexec_mutex;
 
 #ifdef CONFIG_KEXEC_FILE
-struct kexec_sha_region {
-        unsigned long start;
-        unsigned long len;
-};
-
+#include <linux/purgatory.h>
 void kimage_file_post_load_cleanup(struct kimage *image);
 #else /* CONFIG_KEXEC_FILE */
 static inline void kimage_file_post_load_cleanup(struct kimage *image) { }
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index 12e38c213b70..a95e5d1f4a9c 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -3262,10 +3262,17 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
         if (depth) {
                 hlock = curr->held_locks + depth - 1;
                 if (hlock->class_idx == class_idx && nest_lock) {
-                        if (hlock->references)
+                        if (hlock->references) {
+                                /*
+                                 * Check: unsigned int references:12, overflow.
+                                 */
+                                if (DEBUG_LOCKS_WARN_ON(hlock->references == (1 << 12)-1))
+                                        return 0;
+
                                 hlock->references++;
-                        else
+                        } else {
                                 hlock->references = 2;
+                        }
 
                         return 1;
                 }
diff --git a/kernel/locking/test-ww_mutex.c b/kernel/locking/test-ww_mutex.c
index da6c9a34f62f..6b7abb334ca6 100644
--- a/kernel/locking/test-ww_mutex.c
+++ b/kernel/locking/test-ww_mutex.c
@@ -50,7 +50,7 @@ static void test_mutex_work(struct work_struct *work)
 
         if (mtx->flags & TEST_MTX_TRY) {
                 while (!ww_mutex_trylock(&mtx->mutex))
-                        cpu_relax();
+                        cond_resched();
         } else {
                 ww_mutex_lock(&mtx->mutex, NULL);
         }
@@ -88,7 +88,7 @@ static int __test_mutex(unsigned int flags)
                                 ret = -EINVAL;
                                 break;
                         }
-                        cpu_relax();
+                        cond_resched();
                 } while (time_before(jiffies, timeout));
         } else {
                 ret = wait_for_completion_timeout(&mtx.done, TIMEOUT);
@@ -627,7 +627,7 @@ static int __init test_ww_mutex_init(void)
         if (ret)
                 return ret;
 
-        ret = stress(4096, hweight32(STRESS_ALL)*ncpus, 1<<12, STRESS_ALL);
+        ret = stress(4095, hweight32(STRESS_ALL)*ncpus, 1<<12, STRESS_ALL);
         if (ret)
                 return ret;
 
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 956383844116..3b31fc05a0f1 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3287,10 +3287,15 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
         struct task_struct *p;
 
         /*
-         * Optimization: we know that if all tasks are in
-         * the fair class we can call that function directly:
+         * Optimization: we know that if all tasks are in the fair class we can
+         * call that function directly, but only if the @prev task wasn't of a
+         * higher scheduling class, because otherwise those loose the
+         * opportunity to pull in more work from other CPUs.
          */
-        if (likely(rq->nr_running == rq->cfs.h_nr_running)) {
+        if (likely((prev->sched_class == &idle_sched_class ||
+                    prev->sched_class == &fair_sched_class) &&
+                   rq->nr_running == rq->cfs.h_nr_running)) {
+
                 p = fair_sched_class.pick_next_task(rq, prev, rf);
                 if (unlikely(p == RETRY_TASK))
                         goto again;
diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index 8f8de3d4d6b7..cd7cd489f739 100644
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -36,6 +36,7 @@ struct sugov_policy {
         u64 last_freq_update_time;
         s64 freq_update_delay_ns;
         unsigned int next_freq;
+        unsigned int cached_raw_freq;
 
         /* The next fields are only needed if fast switch cannot be used. */
         struct irq_work irq_work;
@@ -52,7 +53,6 @@ struct sugov_cpu {
         struct update_util_data update_util;
         struct sugov_policy *sg_policy;
 
-        unsigned int cached_raw_freq;
         unsigned long iowait_boost;
         unsigned long iowait_boost_max;
         u64 last_update;
@@ -116,7 +116,7 @@ static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time,
 
 /**
  * get_next_freq - Compute a new frequency for a given cpufreq policy.
- * @sg_cpu: schedutil cpu object to compute the new frequency for.
+ * @sg_policy: schedutil policy object to compute the new frequency for.
  * @util: Current CPU utilization.
  * @max: CPU capacity.
  *
@@ -136,19 +136,18 @@ static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time,
  * next_freq (as calculated above) is returned, subject to policy min/max and
  * cpufreq driver limitations.
  */
-static unsigned int get_next_freq(struct sugov_cpu *sg_cpu, unsigned long util,
-                                  unsigned long max)
+static unsigned int get_next_freq(struct sugov_policy *sg_policy,
+                                  unsigned long util, unsigned long max)
 {
-        struct sugov_policy *sg_policy = sg_cpu->sg_policy;
         struct cpufreq_policy *policy = sg_policy->policy;
         unsigned int freq = arch_scale_freq_invariant() ?
                                 policy->cpuinfo.max_freq : policy->cur;
 
         freq = (freq + (freq >> 2)) * util / max;
 
-        if (freq == sg_cpu->cached_raw_freq && sg_policy->next_freq != UINT_MAX)
+        if (freq == sg_policy->cached_raw_freq && sg_policy->next_freq != UINT_MAX)
                 return sg_policy->next_freq;
-        sg_cpu->cached_raw_freq = freq;
+        sg_policy->cached_raw_freq = freq;
         return cpufreq_driver_resolve_freq(policy, freq);
 }
 
@@ -213,7 +212,7 @@ static void sugov_update_single(struct update_util_data *hook, u64 time,
         } else {
                 sugov_get_util(&util, &max);
                 sugov_iowait_boost(sg_cpu, &util, &max);
-                next_f = get_next_freq(sg_cpu, util, max);
+                next_f = get_next_freq(sg_policy, util, max);
         }
         sugov_update_commit(sg_policy, time, next_f);
 }
@@ -267,7 +266,7 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu,
                 sugov_iowait_boost(j_sg_cpu, &util, &max);
         }
 
-        return get_next_freq(sg_cpu, util, max);
+        return get_next_freq(sg_policy, util, max);
 }
 
 static void sugov_update_shared(struct update_util_data *hook, u64 time,
@@ -580,6 +579,7 @@ static int sugov_start(struct cpufreq_policy *policy)
         sg_policy->next_freq = UINT_MAX;
         sg_policy->work_in_progress = false;
         sg_policy->need_freq_update = false;
+        sg_policy->cached_raw_freq = 0;
 
         for_each_cpu(cpu, policy->cpus) {
                 struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);
@@ -590,7 +590,6 @@ static int sugov_start(struct cpufreq_policy *policy)
                         sg_cpu->max = 0;
                         sg_cpu->flags = SCHED_CPUFREQ_RT;
                         sg_cpu->last_update = 0;
-                        sg_cpu->cached_raw_freq = 0;
                         sg_cpu->iowait_boost = 0;
                         sg_cpu->iowait_boost_max = policy->cpuinfo.max_freq;
                         cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util,
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 3e88b35ac157..dea138964b91 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5799,7 +5799,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t
          * Due to large variance we need a large fuzz factor; hackbench in
          * particularly is sensitive here.
          */
-        if ((avg_idle / 512) < avg_cost)
+        if (sched_feat(SIS_AVG_CPU) && (avg_idle / 512) < avg_cost)
                 return -1;
 
         time = local_clock();
diff --git a/kernel/sched/features.h b/kernel/sched/features.h
index 69631fa46c2f..1b3c8189b286 100644
--- a/kernel/sched/features.h
+++ b/kernel/sched/features.h
@@ -51,6 +51,11 @@ SCHED_FEAT(NONTASK_CAPACITY, true)
  */
 SCHED_FEAT(TTWU_QUEUE, true)
 
+/*
+ * When doing wakeups, attempt to limit superfluous scans of the LLC domain.
+ */
+SCHED_FEAT(SIS_AVG_CPU, false)
+
 #ifdef HAVE_RT_PUSH_IPI
 /*
  * In order to avoid a thundering herd attack of CPUs that are
diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c
index 4d2ea6f25568..b8c84c6dee64 100644
--- a/kernel/sched/wait.c
+++ b/kernel/sched/wait.c
@@ -242,6 +242,45 @@ long prepare_to_wait_event(wait_queue_head_t *q, wait_queue_t *wait, int state)
 }
 EXPORT_SYMBOL(prepare_to_wait_event);
 
+/*
+ * Note! These two wait functions are entered with the
+ * wait-queue lock held (and interrupts off in the _irq
+ * case), so there is no race with testing the wakeup
+ * condition in the caller before they add the wait
+ * entry to the wake queue.
+ */
+int do_wait_intr(wait_queue_head_t *wq, wait_queue_t *wait)
+{
+        if (likely(list_empty(&wait->task_list)))
+                __add_wait_queue_tail(wq, wait);
+
+        set_current_state(TASK_INTERRUPTIBLE);
+        if (signal_pending(current))
+                return -ERESTARTSYS;
+
+        spin_unlock(&wq->lock);
+        schedule();
+        spin_lock(&wq->lock);
+        return 0;
+}
+EXPORT_SYMBOL(do_wait_intr);
+
+int do_wait_intr_irq(wait_queue_head_t *wq, wait_queue_t *wait)
+{
+        if (likely(list_empty(&wait->task_list)))
+                __add_wait_queue_tail(wq, wait);
+
+        set_current_state(TASK_INTERRUPTIBLE);
+        if (signal_pending(current))
+                return -ERESTARTSYS;
+
+        spin_unlock_irq(&wq->lock);
+        schedule();
+        spin_lock_irq(&wq->lock);
+        return 0;
+}
+EXPORT_SYMBOL(do_wait_intr_irq);
+
 /**
  * finish_wait - clean up after waiting in a queue
  * @q: waitqueue waited on
diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c
index 7906b3f0c41a..497719127bf9 100644
--- a/kernel/time/jiffies.c
+++ b/kernel/time/jiffies.c
@@ -125,7 +125,7 @@ int register_refined_jiffies(long cycles_per_second)
         shift_hz += cycles_per_tick/2;
         do_div(shift_hz, cycles_per_tick);
         /* Calculate nsec_per_tick using shift_hz */
-        nsec_per_tick = (u64)TICK_NSEC << 8;
+        nsec_per_tick = (u64)NSEC_PER_SEC << 8;
         nsec_per_tick += (u32)shift_hz/2;
         do_div(nsec_per_tick, (u32)shift_hz);
 
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 0d1597c9ee30..b9691ee8f6c1 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -4416,16 +4416,24 @@ static int __init set_graph_notrace_function(char *str)
 }
 __setup("ftrace_graph_notrace=", set_graph_notrace_function);
 
+static int __init set_graph_max_depth_function(char *str)
+{
+        if (!str)
+                return 0;
+        fgraph_max_depth = simple_strtoul(str, NULL, 0);
+        return 1;
+}
+__setup("ftrace_graph_max_depth=", set_graph_max_depth_function);
+
 static void __init set_ftrace_early_graph(char *buf, int enable)
 {
         int ret;
         char *func;
         struct ftrace_hash *hash;
 
-        if (enable)
-                hash = ftrace_graph_hash;
-        else
-                hash = ftrace_graph_notrace_hash;
+        hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
+        if (WARN_ON(!hash))
+                return;
 
         while (buf) {
                 func = strsep(&buf, ",");
@@ -4435,6 +4443,11 @@ static void __init set_ftrace_early_graph(char *buf, int enable)
                         printk(KERN_DEBUG "ftrace: function %s not "
                                           "traceable\n", func);
         }
+
+        if (enable)
+                ftrace_graph_hash = hash;
+        else
+                ftrace_graph_notrace_hash = hash;
 }
 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 
@@ -5488,7 +5501,7 @@ static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
  * Normally the mcount trampoline will call the ops->func, but there
  * are times that it should not. For example, if the ops does not
  * have its own recursion protection, then it should call the
- * ftrace_ops_recurs_func() instead.
+ * ftrace_ops_assist_func() instead.
  *
  * Returns the function that the trampoline should call for @ops.
  */
diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c
index 1d68b5b7ad41..5fb1f2c87e6b 100644
--- a/kernel/trace/trace_stack.c
+++ b/kernel/trace/trace_stack.c
@@ -65,7 +65,7 @@ void stack_trace_print(void)
 }
 
 /*
- * When arch-specific code overides this function, the following
+ * When arch-specific code overrides this function, the following
  * data should be filled up, assuming stack_trace_max_lock is held to
  * prevent concurrent updates.
  *     stack_trace_index[]
diff --git a/kernel/ucount.c b/kernel/ucount.c
index 62630a40ab3a..b4eeee03934f 100644
--- a/kernel/ucount.c
+++ b/kernel/ucount.c
@@ -144,7 +144,7 @@ static struct ucounts *get_ucounts(struct user_namespace *ns, kuid_t uid)
 
                 new->ns = ns;
                 new->uid = uid;
-                atomic_set(&new->count, 0);
+                new->count = 0;
 
                 spin_lock_irq(&ucounts_lock);
                 ucounts = find_ucounts(ns, uid, hashent);
@@ -155,8 +155,10 @@ static struct ucounts *get_ucounts(struct user_namespace *ns, kuid_t uid)
                         ucounts = new;
                 }
         }
-        if (!atomic_add_unless(&ucounts->count, 1, INT_MAX))
+        if (ucounts->count == INT_MAX)
                 ucounts = NULL;
+        else
+                ucounts->count += 1;
         spin_unlock_irq(&ucounts_lock);
         return ucounts;
 }
@@ -165,13 +167,15 @@ static void put_ucounts(struct ucounts *ucounts)
 {
         unsigned long flags;
 
-        if (atomic_dec_and_test(&ucounts->count)) {
-                spin_lock_irqsave(&ucounts_lock, flags);
+        spin_lock_irqsave(&ucounts_lock, flags);
+        ucounts->count -= 1;
+        if (!ucounts->count)
                 hlist_del_init(&ucounts->node);
-                spin_unlock_irqrestore(&ucounts_lock, flags);
+        else
+                ucounts = NULL;
+        spin_unlock_irqrestore(&ucounts_lock, flags);
 
-                kfree(ucounts);
-        }
+        kfree(ucounts);
 }
 
 static inline bool atomic_inc_below(atomic_t *v, int u)
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 072cbc9b175d..c0168b7da1ea 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -1507,6 +1507,7 @@ static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
         struct timer_list *timer = &dwork->timer;
         struct work_struct *work = &dwork->work;
 
+        WARN_ON_ONCE(!wq);
         WARN_ON_ONCE(timer->function != delayed_work_timer_fn ||
                      timer->data != (unsigned long)dwork);
         WARN_ON_ONCE(timer_pending(timer));