Merge tag 'v3.13-rc4' into core/locking

Merge Linux 3.13-rc4, to refresh this rather old tree with the latest fixes.

Signed-off-by: Ingo Molnar <mingo@kernel.org>

10 files changed, 482 insertions, 474 deletions

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index bccd5a628ea6..33a5dc492810 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1481,8 +1481,18 @@ int move_huge_pmd(struct vm_area_struct *vma, struct vm_area_struct *new_vma,
                 pmd = pmdp_get_and_clear(mm, old_addr, old_pmd);
                 VM_BUG_ON(!pmd_none(*new_pmd));
                 set_pmd_at(mm, new_addr, new_pmd, pmd_mksoft_dirty(pmd));
-                if (new_ptl != old_ptl)
+                if (new_ptl != old_ptl) {
+                        pgtable_t pgtable;
+
+                        /*
+                         * Move preallocated PTE page table if new_pmd is on
+                         * different PMD page table.
+                         */
+                        pgtable = pgtable_trans_huge_withdraw(mm, old_pmd);
+                        pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
+
                         spin_unlock(new_ptl);
+                }
                 spin_unlock(old_ptl);
         }
 out:
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 7d57af21f49e..dee6cf4e6d34 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -476,40 +476,6 @@ static int vma_has_reserves(struct vm_area_struct *vma, long chg)
         return 0;
 }
 
-static void copy_gigantic_page(struct page *dst, struct page *src)
-{
-        int i;
-        struct hstate *h = page_hstate(src);
-        struct page *dst_base = dst;
-        struct page *src_base = src;
-
-        for (i = 0; i < pages_per_huge_page(h); ) {
-                cond_resched();
-                copy_highpage(dst, src);
-
-                i++;
-                dst = mem_map_next(dst, dst_base, i);
-                src = mem_map_next(src, src_base, i);
-        }
-}
-
-void copy_huge_page(struct page *dst, struct page *src)
-{
-        int i;
-        struct hstate *h = page_hstate(src);
-
-        if (unlikely(pages_per_huge_page(h) > MAX_ORDER_NR_PAGES)) {
-                copy_gigantic_page(dst, src);
-                return;
-        }
-
-        might_sleep();
-        for (i = 0; i < pages_per_huge_page(h); i++) {
-                cond_resched();
-                copy_highpage(dst + i, src + i);
-        }
-}
-
 static void enqueue_huge_page(struct hstate *h, struct page *page)
 {
         int nid = page_to_nid(page);
@@ -736,6 +702,23 @@ int PageHuge(struct page *page)
 }
 EXPORT_SYMBOL_GPL(PageHuge);
 
+/*
+ * PageHeadHuge() only returns true for hugetlbfs head page, but not for
+ * normal or transparent huge pages.
+ */
+int PageHeadHuge(struct page *page_head)
+{
+        compound_page_dtor *dtor;
+
+        if (!PageHead(page_head))
+                return 0;
+
+        dtor = get_compound_page_dtor(page_head);
+
+        return dtor == free_huge_page;
+}
+EXPORT_SYMBOL_GPL(PageHeadHuge);
+
 pgoff_t __basepage_index(struct page *page)
 {
         struct page *page_head = compound_head(page);
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index f1a0ae6e11b8..bf5e89457149 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2694,7 +2694,10 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
                 goto bypass;
 
         if (unlikely(task_in_memcg_oom(current)))
-                goto bypass;
+                goto nomem;
+
+        if (gfp_mask & __GFP_NOFAIL)
+                oom = false;
 
         /*
          * We always charge the cgroup the mm_struct belongs to.
@@ -6352,6 +6355,42 @@ static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
 static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
 {
         struct mem_cgroup *memcg = mem_cgroup_from_css(css);
+        /*
+         * XXX: css_offline() would be where we should reparent all
+         * memory to prepare the cgroup for destruction.  However,
+         * memcg does not do css_tryget() and res_counter charging
+         * under the same RCU lock region, which means that charging
+         * could race with offlining.  Offlining only happens to
+         * cgroups with no tasks in them but charges can show up
+         * without any tasks from the swapin path when the target
+         * memcg is looked up from the swapout record and not from the
+         * current task as it usually is.  A race like this can leak
+         * charges and put pages with stale cgroup pointers into
+         * circulation:
+         *
+         * #0                        #1
+         *                           lookup_swap_cgroup_id()
+         *                           rcu_read_lock()
+         *                           mem_cgroup_lookup()
+         *                           css_tryget()
+         *                           rcu_read_unlock()
+         * disable css_tryget()
+         * call_rcu()
+         *   offline_css()
+         *     reparent_charges()
+         *                           res_counter_charge()
+         *                           css_put()
+         *                             css_free()
+         *                           pc->mem_cgroup = dead memcg
+         *                           add page to lru
+         *
+         * The bulk of the charges are still moved in offline_css() to
+         * avoid pinning a lot of pages in case a long-term reference
+         * like a swapout record is deferring the css_free() to long
+         * after offlining.  But this makes sure we catch any charges
+         * made after offlining:
+         */
+        mem_cgroup_reparent_charges(memcg);
 
         memcg_destroy_kmem(memcg);
         __mem_cgroup_free(memcg);
diff --git a/mm/memory.c b/mm/memory.c
index 0409e8f43fa0..5d9025f3b3e1 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -4272,13 +4272,6 @@ void copy_user_huge_page(struct page *dst, struct page *src,
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
 
 #if USE_SPLIT_PTE_PTLOCKS && BLOATED_SPINLOCKS
-static struct kmem_cache *page_ptl_cachep;
-void __init ptlock_cache_init(void)
-{
-        page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
-                        SLAB_PANIC, NULL);
-}
-
 bool ptlock_alloc(struct page *page)
 {
         spinlock_t *ptl;
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index c4403cdf3433..eca4a3129129 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -2950,7 +2950,7 @@ void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
                 return;
         }
 
-        p += snprintf(p, maxlen, policy_modes[mode]);
+        p += snprintf(p, maxlen, "%s", policy_modes[mode]);
 
         if (flags & MPOL_MODE_FLAGS) {
                 p += snprintf(p, buffer + maxlen - p, "=");
diff --git a/mm/migrate.c b/mm/migrate.c
index 316e720a2023..bb940045fe85 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -442,6 +442,54 @@ int migrate_huge_page_move_mapping(struct address_space *mapping,
 }
 
 /*
+ * Gigantic pages are so large that we do not guarantee that page++ pointer
+ * arithmetic will work across the entire page.  We need something more
+ * specialized.
+ */
+static void __copy_gigantic_page(struct page *dst, struct page *src,
+                                int nr_pages)
+{
+        int i;
+        struct page *dst_base = dst;
+        struct page *src_base = src;
+
+        for (i = 0; i < nr_pages; ) {
+                cond_resched();
+                copy_highpage(dst, src);
+
+                i++;
+                dst = mem_map_next(dst, dst_base, i);
+                src = mem_map_next(src, src_base, i);
+        }
+}
+
+static void copy_huge_page(struct page *dst, struct page *src)
+{
+        int i;
+        int nr_pages;
+
+        if (PageHuge(src)) {
+                /* hugetlbfs page */
+                struct hstate *h = page_hstate(src);
+                nr_pages = pages_per_huge_page(h);
+
+                if (unlikely(nr_pages > MAX_ORDER_NR_PAGES)) {
+                        __copy_gigantic_page(dst, src, nr_pages);
+                        return;
+                }
+        } else {
+                /* thp page */
+                BUG_ON(!PageTransHuge(src));
+                nr_pages = hpage_nr_pages(src);
+        }
+
+        for (i = 0; i < nr_pages; i++) {
+                cond_resched();
+                copy_highpage(dst + i, src + i);
+        }
+}
+
+/*
  * Copy the page to its new location
  */
 void migrate_page_copy(struct page *newpage, struct page *page)
diff --git a/mm/shmem.c b/mm/shmem.c
index 8297623fcaed..902a14842b74 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -2918,13 +2918,8 @@ static struct dentry_operations anon_ops = {
         .d_dname = simple_dname
 };
 
-/**
- * shmem_file_setup - get an unlinked file living in tmpfs
- * @name: name for dentry (to be seen in /proc/<pid>/maps
- * @size: size to be set for the file
- * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
- */
-struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
+static struct file *__shmem_file_setup(const char *name, loff_t size,
+                                       unsigned long flags, unsigned int i_flags)
 {
         struct file *res;
         struct inode *inode;
@@ -2957,6 +2952,7 @@ struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags
         if (!inode)
                 goto put_dentry;
 
+        inode->i_flags |= i_flags;
         d_instantiate(path.dentry, inode);
         inode->i_size = size;
         clear_nlink(inode);     /* It is unlinked */
@@ -2977,6 +2973,32 @@ put_memory:
         shmem_unacct_size(flags, size);
         return res;
 }
+
+/**
+ * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
+ *      kernel internal.  There will be NO LSM permission checks against the
+ *      underlying inode.  So users of this interface must do LSM checks at a
+ *      higher layer.  The one user is the big_key implementation.  LSM checks
+ *      are provided at the key level rather than the inode level.
+ * @name: name for dentry (to be seen in /proc/<pid>/maps
+ * @size: size to be set for the file
+ * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
+ */
+struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
+{
+        return __shmem_file_setup(name, size, flags, S_PRIVATE);
+}
+
+/**
+ * shmem_file_setup - get an unlinked file living in tmpfs
+ * @name: name for dentry (to be seen in /proc/<pid>/maps
+ * @size: size to be set for the file
+ * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
+ */
+struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
+{
+        return __shmem_file_setup(name, size, flags, 0);
+}
 EXPORT_SYMBOL_GPL(shmem_file_setup);
 
 /**
diff --git a/mm/slab.c b/mm/slab.c
index 0c8967bb2018..eb043bf05f4c 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -164,72 +164,6 @@
 static bool pfmemalloc_active __read_mostly;
 
 /*
- * kmem_bufctl_t:
- *
- * Bufctl's are used for linking objs within a slab
- * linked offsets.
- *
- * This implementation relies on "struct page" for locating the cache &
- * slab an object belongs to.
- * This allows the bufctl structure to be small (one int), but limits
- * the number of objects a slab (not a cache) can contain when off-slab
- * bufctls are used. The limit is the size of the largest general cache
- * that does not use off-slab slabs.
- * For 32bit archs with 4 kB pages, is this 56.
- * This is not serious, as it is only for large objects, when it is unwise
- * to have too many per slab.
- * Note: This limit can be raised by introducing a general cache whose size
- * is less than 512 (PAGE_SIZE<<3), but greater than 256.
- */
-
-typedef unsigned int kmem_bufctl_t;
-#define BUFCTL_END      (((kmem_bufctl_t)(~0U))-0)
-#define BUFCTL_FREE     (((kmem_bufctl_t)(~0U))-1)
-#define BUFCTL_ACTIVE   (((kmem_bufctl_t)(~0U))-2)
-#define SLAB_LIMIT      (((kmem_bufctl_t)(~0U))-3)
-
-/*
- * struct slab_rcu
- *
- * slab_destroy on a SLAB_DESTROY_BY_RCU cache uses this structure to
- * arrange for kmem_freepages to be called via RCU.  This is useful if
- * we need to approach a kernel structure obliquely, from its address
- * obtained without the usual locking.  We can lock the structure to
- * stabilize it and check it's still at the given address, only if we
- * can be sure that the memory has not been meanwhile reused for some
- * other kind of object (which our subsystem's lock might corrupt).
- *
- * rcu_read_lock before reading the address, then rcu_read_unlock after
- * taking the spinlock within the structure expected at that address.
- */
-struct slab_rcu {
-        struct rcu_head head;
-        struct kmem_cache *cachep;
-        void *addr;
-};
-
-/*
- * struct slab
- *
- * Manages the objs in a slab. Placed either at the beginning of mem allocated
- * for a slab, or allocated from an general cache.
- * Slabs are chained into three list: fully used, partial, fully free slabs.
- */
-struct slab {
-        union {
-                struct {
-                        struct list_head list;
-                        unsigned long colouroff;
-                        void *s_mem;            /* including colour offset */
-                        unsigned int inuse;     /* num of objs active in slab */
-                        kmem_bufctl_t free;
-                        unsigned short nodeid;
-                };
-                struct slab_rcu __slab_cover_slab_rcu;
-        };
-};
-
-/*
  * struct array_cache
  *
  * Purpose:
@@ -456,18 +390,10 @@ static inline struct kmem_cache *virt_to_cache(const void *obj)
         return page->slab_cache;
 }
 
-static inline struct slab *virt_to_slab(const void *obj)
-{
-        struct page *page = virt_to_head_page(obj);
-
-        VM_BUG_ON(!PageSlab(page));
-        return page->slab_page;
-}
-
-static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab,
+static inline void *index_to_obj(struct kmem_cache *cache, struct page *page,
                                  unsigned int idx)
 {
-        return slab->s_mem + cache->size * idx;
+        return page->s_mem + cache->size * idx;
 }
 
 /*
@@ -477,9 +403,9 @@ static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab,
  *   reciprocal_divide(offset, cache->reciprocal_buffer_size)
  */
 static inline unsigned int obj_to_index(const struct kmem_cache *cache,
-                                        const struct slab *slab, void *obj)
+                                        const struct page *page, void *obj)
 {
-        u32 offset = (obj - slab->s_mem);
+        u32 offset = (obj - page->s_mem);
         return reciprocal_divide(offset, cache->reciprocal_buffer_size);
 }
 
@@ -641,7 +567,7 @@ static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
 
 static size_t slab_mgmt_size(size_t nr_objs, size_t align)
 {
-        return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align);
+        return ALIGN(nr_objs * sizeof(unsigned int), align);
 }
 
 /*
@@ -660,8 +586,7 @@ static void cache_estimate(unsigned long gfporder, size_t buffer_size,
          * on it. For the latter case, the memory allocated for a
          * slab is used for:
          *
-         * - The struct slab
-         * - One kmem_bufctl_t for each object
+         * - One unsigned int for each object
          * - Padding to respect alignment of @align
          * - @buffer_size bytes for each object
          *
@@ -674,8 +599,6 @@ static void cache_estimate(unsigned long gfporder, size_t buffer_size,
                 mgmt_size = 0;
                 nr_objs = slab_size / buffer_size;
 
-                if (nr_objs > SLAB_LIMIT)
-                        nr_objs = SLAB_LIMIT;
         } else {
                 /*
                  * Ignore padding for the initial guess. The padding
@@ -685,8 +608,7 @@ static void cache_estimate(unsigned long gfporder, size_t buffer_size,
                  * into the memory allocation when taking the padding
                  * into account.
                  */
-                nr_objs = (slab_size - sizeof(struct slab)) /
-                          (buffer_size + sizeof(kmem_bufctl_t));
+                nr_objs = (slab_size) / (buffer_size + sizeof(unsigned int));
 
                 /*
                  * This calculated number will be either the right
@@ -696,9 +618,6 @@ static void cache_estimate(unsigned long gfporder, size_t buffer_size,
                        > slab_size)
                         nr_objs--;
 
-                if (nr_objs > SLAB_LIMIT)
-                        nr_objs = SLAB_LIMIT;
-
                 mgmt_size = slab_mgmt_size(nr_objs, align);
         }
         *num = nr_objs;
@@ -829,10 +748,8 @@ static struct array_cache *alloc_arraycache(int node, int entries,
         return nc;
 }
 
-static inline bool is_slab_pfmemalloc(struct slab *slabp)
+static inline bool is_slab_pfmemalloc(struct page *page)
 {
-        struct page *page = virt_to_page(slabp->s_mem);
-
         return PageSlabPfmemalloc(page);
 }
 
@@ -841,23 +758,23 @@ static void recheck_pfmemalloc_active(struct kmem_cache *cachep,
                                                 struct array_cache *ac)
 {
         struct kmem_cache_node *n = cachep->node[numa_mem_id()];
-        struct slab *slabp;
+        struct page *page;
         unsigned long flags;
 
         if (!pfmemalloc_active)
                 return;
 
         spin_lock_irqsave(&n->list_lock, flags);
-        list_for_each_entry(slabp, &n->slabs_full, list)
-                if (is_slab_pfmemalloc(slabp))
+        list_for_each_entry(page, &n->slabs_full, lru)
+                if (is_slab_pfmemalloc(page))
                         goto out;
 
-        list_for_each_entry(slabp, &n->slabs_partial, list)
-                if (is_slab_pfmemalloc(slabp))
+        list_for_each_entry(page, &n->slabs_partial, lru)
+                if (is_slab_pfmemalloc(page))
                         goto out;
 
-        list_for_each_entry(slabp, &n->slabs_free, list)
-                if (is_slab_pfmemalloc(slabp))
+        list_for_each_entry(page, &n->slabs_free, lru)
+                if (is_slab_pfmemalloc(page))
                         goto out;
 
         pfmemalloc_active = false;
@@ -897,8 +814,8 @@ static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac,
                  */
                 n = cachep->node[numa_mem_id()];
                 if (!list_empty(&n->slabs_free) && force_refill) {
-                        struct slab *slabp = virt_to_slab(objp);
-                        ClearPageSlabPfmemalloc(virt_to_head_page(slabp->s_mem));
+                        struct page *page = virt_to_head_page(objp);
+                        ClearPageSlabPfmemalloc(page);
                         clear_obj_pfmemalloc(&objp);
                         recheck_pfmemalloc_active(cachep, ac);
                         return objp;
@@ -1099,8 +1016,7 @@ static void drain_alien_cache(struct kmem_cache *cachep,
 
 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
 {
-        struct slab *slabp = virt_to_slab(objp);
-        int nodeid = slabp->nodeid;
+        int nodeid = page_to_nid(virt_to_page(objp));
         struct kmem_cache_node *n;
         struct array_cache *alien = NULL;
         int node;
@@ -1111,7 +1027,7 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
          * Make sure we are not freeing a object from another node to the array
          * cache on this cpu.
          */
-        if (likely(slabp->nodeid == node))
+        if (likely(nodeid == node))
                 return 0;
 
         n = cachep->node[node];
@@ -1512,6 +1428,8 @@ void __init kmem_cache_init(void)
 {
         int i;
 
+        BUILD_BUG_ON(sizeof(((struct page *)NULL)->lru) <
+                                        sizeof(struct rcu_head));
         kmem_cache = &kmem_cache_boot;
         setup_node_pointer(kmem_cache);
 
@@ -1687,7 +1605,7 @@ static noinline void
 slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
 {
         struct kmem_cache_node *n;
-        struct slab *slabp;
+        struct page *page;
         unsigned long flags;
         int node;
 
@@ -1706,15 +1624,15 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
                         continue;
 
                 spin_lock_irqsave(&n->list_lock, flags);
-                list_for_each_entry(slabp, &n->slabs_full, list) {
+                list_for_each_entry(page, &n->slabs_full, lru) {
                         active_objs += cachep->num;
                         active_slabs++;
                 }
-                list_for_each_entry(slabp, &n->slabs_partial, list) {
-                        active_objs += slabp->inuse;
+                list_for_each_entry(page, &n->slabs_partial, lru) {
+                        active_objs += page->active;
                         active_slabs++;
                 }
-                list_for_each_entry(slabp, &n->slabs_free, list)
+                list_for_each_entry(page, &n->slabs_free, lru)
                         num_slabs++;
 
                 free_objects += n->free_objects;
@@ -1736,19 +1654,11 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
  * did not request dmaable memory, we might get it, but that
  * would be relatively rare and ignorable.
  */
-static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
+static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
+                                                                int nodeid)
 {
         struct page *page;
         int nr_pages;
-        int i;
-
-#ifndef CONFIG_MMU
-        /*
-         * Nommu uses slab's for process anonymous memory allocations, and thus
-         * requires __GFP_COMP to properly refcount higher order allocations
-         */
-        flags |= __GFP_COMP;
-#endif
 
         flags |= cachep->allocflags;
         if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
@@ -1772,12 +1682,9 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
         else
                 add_zone_page_state(page_zone(page),
                         NR_SLAB_UNRECLAIMABLE, nr_pages);
-        for (i = 0; i < nr_pages; i++) {
-                __SetPageSlab(page + i);
-
-                if (page->pfmemalloc)
-                        SetPageSlabPfmemalloc(page + i);
-        }
+        __SetPageSlab(page);
+        if (page->pfmemalloc)
+                SetPageSlabPfmemalloc(page);
         memcg_bind_pages(cachep, cachep->gfporder);
 
         if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
@@ -1789,17 +1696,15 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
                         kmemcheck_mark_unallocated_pages(page, nr_pages);
         }
 
-        return page_address(page);
+        return page;
 }
 
 /*
  * Interface to system's page release.
  */
-static void kmem_freepages(struct kmem_cache *cachep, void *addr)
+static void kmem_freepages(struct kmem_cache *cachep, struct page *page)
 {
-        unsigned long i = (1 << cachep->gfporder);
-        struct page *page = virt_to_page(addr);
-        const unsigned long nr_freed = i;
+        const unsigned long nr_freed = (1 << cachep->gfporder);
 
         kmemcheck_free_shadow(page, cachep->gfporder);
 
@@ -1809,27 +1714,28 @@ static void kmem_freepages(struct kmem_cache *cachep, void *addr)
         else
                 sub_zone_page_state(page_zone(page),
                                 NR_SLAB_UNRECLAIMABLE, nr_freed);
-        while (i--) {
-                BUG_ON(!PageSlab(page));
-                __ClearPageSlabPfmemalloc(page);
-                __ClearPageSlab(page);
-                page++;
-        }
+
+        BUG_ON(!PageSlab(page));
+        __ClearPageSlabPfmemalloc(page);
+        __ClearPageSlab(page);
+        page_mapcount_reset(page);
+        page->mapping = NULL;
 
         memcg_release_pages(cachep, cachep->gfporder);
         if (current->reclaim_state)
                 current->reclaim_state->reclaimed_slab += nr_freed;
-        free_memcg_kmem_pages((unsigned long)addr, cachep->gfporder);
+        __free_memcg_kmem_pages(page, cachep->gfporder);
 }
 
 static void kmem_rcu_free(struct rcu_head *head)
 {
-        struct slab_rcu *slab_rcu = (struct slab_rcu *)head;
-        struct kmem_cache *cachep = slab_rcu->cachep;
+        struct kmem_cache *cachep;
+        struct page *page;
 
-        kmem_freepages(cachep, slab_rcu->addr);
-        if (OFF_SLAB(cachep))
-                kmem_cache_free(cachep->slabp_cache, slab_rcu);
+        page = container_of(head, struct page, rcu_head);
+        cachep = page->slab_cache;
+
+        kmem_freepages(cachep, page);
 }
 
 #if DEBUG
@@ -1978,19 +1884,19 @@ static void check_poison_obj(struct kmem_cache *cachep, void *objp)
                 /* Print some data about the neighboring objects, if they
                  * exist:
                  */
-                struct slab *slabp = virt_to_slab(objp);
+                struct page *page = virt_to_head_page(objp);
                 unsigned int objnr;
 
-                objnr = obj_to_index(cachep, slabp, objp);
+                objnr = obj_to_index(cachep, page, objp);
                 if (objnr) {
-                        objp = index_to_obj(cachep, slabp, objnr - 1);
+                        objp = index_to_obj(cachep, page, objnr - 1);
                         realobj = (char *)objp + obj_offset(cachep);
                         printk(KERN_ERR "Prev obj: start=%p, len=%d\n",
                                realobj, size);
                         print_objinfo(cachep, objp, 2);
                 }
                 if (objnr + 1 < cachep->num) {
-                        objp = index_to_obj(cachep, slabp, objnr + 1);
+                        objp = index_to_obj(cachep, page, objnr + 1);
                         realobj = (char *)objp + obj_offset(cachep);
                         printk(KERN_ERR "Next obj: start=%p, len=%d\n",
                                realobj, size);
@@ -2001,11 +1907,12 @@ static void check_poison_obj(struct kmem_cache *cachep, void *objp)
 #endif
 
 #if DEBUG
-static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slabp)
+static void slab_destroy_debugcheck(struct kmem_cache *cachep,
+                                                struct page *page)
 {
         int i;
         for (i = 0; i < cachep->num; i++) {
-                void *objp = index_to_obj(cachep, slabp, i);
+                void *objp = index_to_obj(cachep, page, i);
 
                 if (cachep->flags & SLAB_POISON) {
 #ifdef CONFIG_DEBUG_PAGEALLOC
@@ -2030,7 +1937,8 @@ static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slab
         }
 }
 #else
-static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slabp)
+static void slab_destroy_debugcheck(struct kmem_cache *cachep,
+                                                struct page *page)
 {
 }
 #endif
@@ -2044,23 +1952,34 @@ static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slab
  * Before calling the slab must have been unlinked from the cache.  The
  * cache-lock is not held/needed.
  */
-static void slab_destroy(struct kmem_cache *cachep, struct slab *slabp)
+static void slab_destroy(struct kmem_cache *cachep, struct page *page)
 {
-        void *addr = slabp->s_mem - slabp->colouroff;
+        void *freelist;
 
-        slab_destroy_debugcheck(cachep, slabp);
+        freelist = page->freelist;
+        slab_destroy_debugcheck(cachep, page);
         if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) {
-                struct slab_rcu *slab_rcu;
+                struct rcu_head *head;
+
+                /*
+                 * RCU free overloads the RCU head over the LRU.
+                 * slab_page has been overloeaded over the LRU,
+                 * however it is not used from now on so that
+                 * we can use it safely.
+                 */
+                head = (void *)&page->rcu_head;
+                call_rcu(head, kmem_rcu_free);
 
-                slab_rcu = (struct slab_rcu *)slabp;
-                slab_rcu->cachep = cachep;
-                slab_rcu->addr = addr;
-                call_rcu(&slab_rcu->head, kmem_rcu_free);
         } else {
-                kmem_freepages(cachep, addr);
-                if (OFF_SLAB(cachep))
-                        kmem_cache_free(cachep->slabp_cache, slabp);
+                kmem_freepages(cachep, page);
         }
+
+        /*
+         * From now on, we don't use freelist
+         * although actual page can be freed in rcu context
+         */
+        if (OFF_SLAB(cachep))
+                kmem_cache_free(cachep->freelist_cache, freelist);
 }
 
 /**
@@ -2097,8 +2016,8 @@ static size_t calculate_slab_order(struct kmem_cache *cachep,
                          * use off-slab slabs. Needed to avoid a possible
                          * looping condition in cache_grow().
                          */
-                        offslab_limit = size - sizeof(struct slab);
-                        offslab_limit /= sizeof(kmem_bufctl_t);
+                        offslab_limit = size;
+                        offslab_limit /= sizeof(unsigned int);
 
                         if (num > offslab_limit)
                                 break;
@@ -2220,7 +2139,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
 int
 __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
 {
-        size_t left_over, slab_size, ralign;
+        size_t left_over, freelist_size, ralign;
         gfp_t gfp;
         int err;
         size_t size = cachep->size;
@@ -2339,22 +2258,21 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
         if (!cachep->num)
                 return -E2BIG;
 
-        slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t)
-                          + sizeof(struct slab), cachep->align);
+        freelist_size =
+                ALIGN(cachep->num * sizeof(unsigned int), cachep->align);
 
         /*
          * If the slab has been placed off-slab, and we have enough space then
          * move it on-slab. This is at the expense of any extra colouring.
          */
-        if (flags & CFLGS_OFF_SLAB && left_over >= slab_size) {
+        if (flags & CFLGS_OFF_SLAB && left_over >= freelist_size) {
                 flags &= ~CFLGS_OFF_SLAB;
-                left_over -= slab_size;
+                left_over -= freelist_size;
         }
 
         if (flags & CFLGS_OFF_SLAB) {
                 /* really off slab. No need for manual alignment */
-                slab_size =
-                    cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab);
+                freelist_size = cachep->num * sizeof(unsigned int);
 
 #ifdef CONFIG_PAGE_POISONING
                 /* If we're going to use the generic kernel_map_pages()
@@ -2371,16 +2289,16 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
         if (cachep->colour_off < cachep->align)
                 cachep->colour_off = cachep->align;
         cachep->colour = left_over / cachep->colour_off;
-        cachep->slab_size = slab_size;
+        cachep->freelist_size = freelist_size;
         cachep->flags = flags;
-        cachep->allocflags = 0;
+        cachep->allocflags = __GFP_COMP;
         if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA))
                 cachep->allocflags |= GFP_DMA;
         cachep->size = size;
         cachep->reciprocal_buffer_size = reciprocal_value(size);
 
         if (flags & CFLGS_OFF_SLAB) {
-                cachep->slabp_cache = kmalloc_slab(slab_size, 0u);
+                cachep->freelist_cache = kmalloc_slab(freelist_size, 0u);
                 /*
                  * This is a possibility for one of the malloc_sizes caches.
                  * But since we go off slab only for object size greater than
@@ -2388,7 +2306,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
                  * this should not happen at all.
                  * But leave a BUG_ON for some lucky dude.
                  */
-                BUG_ON(ZERO_OR_NULL_PTR(cachep->slabp_cache));
+                BUG_ON(ZERO_OR_NULL_PTR(cachep->freelist_cache));
         }
 
         err = setup_cpu_cache(cachep, gfp);
@@ -2494,7 +2412,7 @@ static int drain_freelist(struct kmem_cache *cache,
 {
         struct list_head *p;
         int nr_freed;
-        struct slab *slabp;
+        struct page *page;
 
         nr_freed = 0;
         while (nr_freed < tofree && !list_empty(&n->slabs_free)) {
@@ -2506,18 +2424,18 @@ static int drain_freelist(struct kmem_cache *cache,
                         goto out;
                 }
 
-                slabp = list_entry(p, struct slab, list);
+                page = list_entry(p, struct page, lru);
 #if DEBUG
-                BUG_ON(slabp->inuse);
+                BUG_ON(page->active);
 #endif
-                list_del(&slabp->list);
+                list_del(&page->lru);
                 /*
                  * Safe to drop the lock. The slab is no longer linked
                  * to the cache.
                  */
                 n->free_objects -= cache->num;
                 spin_unlock_irq(&n->list_lock);
-                slab_destroy(cache, slabp);
+                slab_destroy(cache, page);
                 nr_freed++;
         }
 out:
@@ -2600,52 +2518,42 @@ int __kmem_cache_shutdown(struct kmem_cache *cachep)
  * descriptors in kmem_cache_create, we search through the malloc_sizes array.
  * If we are creating a malloc_sizes cache here it would not be visible to
  * kmem_find_general_cachep till the initialization is complete.
- * Hence we cannot have slabp_cache same as the original cache.
+ * Hence we cannot have freelist_cache same as the original cache.
  */
-static struct slab *alloc_slabmgmt(struct kmem_cache *cachep, void *objp,
-                                   int colour_off, gfp_t local_flags,
-                                   int nodeid)
+static void *alloc_slabmgmt(struct kmem_cache *cachep,
+                                   struct page *page, int colour_off,
+                                   gfp_t local_flags, int nodeid)
 {
-        struct slab *slabp;
+        void *freelist;
+        void *addr = page_address(page);
 
         if (OFF_SLAB(cachep)) {
                 /* Slab management obj is off-slab. */
-                slabp = kmem_cache_alloc_node(cachep->slabp_cache,
+                freelist = kmem_cache_alloc_node(cachep->freelist_cache,
                                               local_flags, nodeid);
-                /*
-                 * If the first object in the slab is leaked (it's allocated
-                 * but no one has a reference to it), we want to make sure
-                 * kmemleak does not treat the ->s_mem pointer as a reference
-                 * to the object. Otherwise we will not report the leak.
-                 */
-                kmemleak_scan_area(&slabp->list, sizeof(struct list_head),
-                                   local_flags);
-                if (!slabp)
+                if (!freelist)
                         return NULL;
         } else {
-                slabp = objp + colour_off;
-                colour_off += cachep->slab_size;
+                freelist = addr + colour_off;
+                colour_off += cachep->freelist_size;
         }
-        slabp->inuse = 0;
-        slabp->colouroff = colour_off;
-        slabp->s_mem = objp + colour_off;
-        slabp->nodeid = nodeid;
-        slabp->free = 0;
-        return slabp;
+        page->active = 0;
+        page->s_mem = addr + colour_off;
+        return freelist;
 }
 
-static inline kmem_bufctl_t *slab_bufctl(struct slab *slabp)
+static inline unsigned int *slab_freelist(struct page *page)
 {
-        return (kmem_bufctl_t *) (slabp + 1);
+        return (unsigned int *)(page->freelist);
 }
 
 static void cache_init_objs(struct kmem_cache *cachep,
-                            struct slab *slabp)
+                            struct page *page)
 {
         int i;
 
         for (i = 0; i < cachep->num; i++) {
-                void *objp = index_to_obj(cachep, slabp, i);
+                void *objp = index_to_obj(cachep, page, i);
 #if DEBUG
                 /* need to poison the objs? */
                 if (cachep->flags & SLAB_POISON)
@@ -2681,9 +2589,8 @@ static void cache_init_objs(struct kmem_cache *cachep,
                 if (cachep->ctor)
                         cachep->ctor(objp);
 #endif
-                slab_bufctl(slabp)[i] = i + 1;
+                slab_freelist(page)[i] = i;
         }
-        slab_bufctl(slabp)[i - 1] = BUFCTL_END;
 }
 
 static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags)
@@ -2696,41 +2603,41 @@ static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags)
         }
 }
 
-static void *slab_get_obj(struct kmem_cache *cachep, struct slab *slabp,
+static void *slab_get_obj(struct kmem_cache *cachep, struct page *page,
                                 int nodeid)
 {
-        void *objp = index_to_obj(cachep, slabp, slabp->free);
-        kmem_bufctl_t next;
+        void *objp;
 
-        slabp->inuse++;
-        next = slab_bufctl(slabp)[slabp->free];
+        objp = index_to_obj(cachep, page, slab_freelist(page)[page->active]);
+        page->active++;
 #if DEBUG
-        slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE;
-        WARN_ON(slabp->nodeid != nodeid);
+        WARN_ON(page_to_nid(virt_to_page(objp)) != nodeid);
 #endif
-        slabp->free = next;
 
         return objp;
 }
 
-static void slab_put_obj(struct kmem_cache *cachep, struct slab *slabp,
+static void slab_put_obj(struct kmem_cache *cachep, struct page *page,
                                 void *objp, int nodeid)
 {
-        unsigned int objnr = obj_to_index(cachep, slabp, objp);
-
+        unsigned int objnr = obj_to_index(cachep, page, objp);
 #if DEBUG
+        unsigned int i;
+
         /* Verify that the slab belongs to the intended node */
-        WARN_ON(slabp->nodeid != nodeid);
+        WARN_ON(page_to_nid(virt_to_page(objp)) != nodeid);
 
-        if (slab_bufctl(slabp)[objnr] + 1 <= SLAB_LIMIT + 1) {
-                printk(KERN_ERR "slab: double free detected in cache "
-                                "'%s', objp %p\n", cachep->name, objp);
-                BUG();
+        /* Verify double free bug */
+        for (i = page->active; i < cachep->num; i++) {
+                if (slab_freelist(page)[i] == objnr) {
+                        printk(KERN_ERR "slab: double free detected in cache "
+                                        "'%s', objp %p\n", cachep->name, objp);
+                        BUG();
+                }
         }
 #endif
-        slab_bufctl(slabp)[objnr] = slabp->free;
-        slabp->free = objnr;
-        slabp->inuse--;
+        page->active--;
+        slab_freelist(page)[page->active] = objnr;
 }
 
 /*
@@ -2738,23 +2645,11 @@ static void slab_put_obj(struct kmem_cache *cachep, struct slab *slabp,
  * for the slab allocator to be able to lookup the cache and slab of a
  * virtual address for kfree, ksize, and slab debugging.
  */
-static void slab_map_pages(struct kmem_cache *cache, struct slab *slab,
-                           void *addr)
+static void slab_map_pages(struct kmem_cache *cache, struct page *page,
+                           void *freelist)
 {
-        int nr_pages;
-        struct page *page;
-
-        page = virt_to_page(addr);
-
-        nr_pages = 1;
-        if (likely(!PageCompound(page)))
-                nr_pages <<= cache->gfporder;
-
-        do {
-                page->slab_cache = cache;
-                page->slab_page = slab;
-                page++;
-        } while (--nr_pages);
+        page->slab_cache = cache;
+        page->freelist = freelist;
 }
 
 /*
@@ -2762,9 +2657,9 @@ static void slab_map_pages(struct kmem_cache *cache, struct slab *slab,
  * kmem_cache_alloc() when there are no active objs left in a cache.
  */
 static int cache_grow(struct kmem_cache *cachep,
-                gfp_t flags, int nodeid, void *objp)
+                gfp_t flags, int nodeid, struct page *page)
 {
-        struct slab *slabp;
+        void *freelist;
         size_t offset;
         gfp_t local_flags;
         struct kmem_cache_node *n;
@@ -2805,20 +2700,20 @@ static int cache_grow(struct kmem_cache *cachep,
          * Get mem for the objs.  Attempt to allocate a physical page from
          * 'nodeid'.
          */
-        if (!objp)
-                objp = kmem_getpages(cachep, local_flags, nodeid);
-        if (!objp)
+        if (!page)
+                page = kmem_getpages(cachep, local_flags, nodeid);
+        if (!page)
                 goto failed;
 
         /* Get slab management. */
-        slabp = alloc_slabmgmt(cachep, objp, offset,
+        freelist = alloc_slabmgmt(cachep, page, offset,
                         local_flags & ~GFP_CONSTRAINT_MASK, nodeid);
-        if (!slabp)
+        if (!freelist)
                 goto opps1;
 
-        slab_map_pages(cachep, slabp, objp);
+        slab_map_pages(cachep, page, freelist);
 
-        cache_init_objs(cachep, slabp);
+        cache_init_objs(cachep, page);
 
         if (local_flags & __GFP_WAIT)
                 local_irq_disable();
@@ -2826,13 +2721,13 @@ static int cache_grow(struct kmem_cache *cachep,
         spin_lock(&n->list_lock);
 
         /* Make slab active. */
-        list_add_tail(&slabp->list, &(n->slabs_free));
+        list_add_tail(&page->lru, &(n->slabs_free));
         STATS_INC_GROWN(cachep);
         n->free_objects += cachep->num;
         spin_unlock(&n->list_lock);
         return 1;
 opps1:
-        kmem_freepages(cachep, objp);
+        kmem_freepages(cachep, page);
 failed:
         if (local_flags & __GFP_WAIT)
                 local_irq_disable();
@@ -2880,9 +2775,8 @@ static inline void verify_redzone_free(struct kmem_cache *cache, void *obj)
 static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
                                    unsigned long caller)
 {
-        struct page *page;
         unsigned int objnr;
-        struct slab *slabp;
+        struct page *page;
 
         BUG_ON(virt_to_cache(objp) != cachep);
 
@@ -2890,8 +2784,6 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
         kfree_debugcheck(objp);
         page = virt_to_head_page(objp);
 
-        slabp = page->slab_page;
-
         if (cachep->flags & SLAB_RED_ZONE) {
                 verify_redzone_free(cachep, objp);
                 *dbg_redzone1(cachep, objp) = RED_INACTIVE;
@@ -2900,14 +2792,11 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
         if (cachep->flags & SLAB_STORE_USER)
                 *dbg_userword(cachep, objp) = (void *)caller;
 
-        objnr = obj_to_index(cachep, slabp, objp);
+        objnr = obj_to_index(cachep, page, objp);
 
         BUG_ON(objnr >= cachep->num);
-        BUG_ON(objp != index_to_obj(cachep, slabp, objnr));
+        BUG_ON(objp != index_to_obj(cachep, page, objnr));
 
-#ifdef CONFIG_DEBUG_SLAB_LEAK
-        slab_bufctl(slabp)[objnr] = BUFCTL_FREE;
-#endif
         if (cachep->flags & SLAB_POISON) {
 #ifdef CONFIG_DEBUG_PAGEALLOC
                 if ((cachep->size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) {
@@ -2924,33 +2813,9 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
         return objp;
 }
 
-static void check_slabp(struct kmem_cache *cachep, struct slab *slabp)
-{
-        kmem_bufctl_t i;
-        int entries = 0;
-
-        /* Check slab's freelist to see if this obj is there. */
-        for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) {
-                entries++;
-                if (entries > cachep->num || i >= cachep->num)
-                        goto bad;
-        }
-        if (entries != cachep->num - slabp->inuse) {
-bad:
-                printk(KERN_ERR "slab: Internal list corruption detected in "
-                        "cache '%s'(%d), slabp %p(%d). Tainted(%s). Hexdump:\n",
-                        cachep->name, cachep->num, slabp, slabp->inuse,
-                        print_tainted());
-                print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, slabp,
-                        sizeof(*slabp) + cachep->num * sizeof(kmem_bufctl_t),
-                        1);
-                BUG();
-        }
-}
 #else
 #define kfree_debugcheck(x) do { } while(0)
 #define cache_free_debugcheck(x,objp,z) (objp)
-#define check_slabp(x,y) do { } while(0)
 #endif
 
 static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags,
@@ -2989,7 +2854,7 @@ retry:
 
         while (batchcount > 0) {
                 struct list_head *entry;
-                struct slab *slabp;
+                struct page *page;
                 /* Get slab alloc is to come from. */
                 entry = n->slabs_partial.next;
                 if (entry == &n->slabs_partial) {
@@ -2999,8 +2864,7 @@ retry:
                                 goto must_grow;
                 }
 
-                slabp = list_entry(entry, struct slab, list);
-                check_slabp(cachep, slabp);
+                page = list_entry(entry, struct page, lru);
                 check_spinlock_acquired(cachep);
 
                 /*
@@ -3008,24 +2872,23 @@ retry:
                  * there must be at least one object available for
                  * allocation.
                  */
-                BUG_ON(slabp->inuse >= cachep->num);
+                BUG_ON(page->active >= cachep->num);
 
-                while (slabp->inuse < cachep->num && batchcount--) {
+                while (page->active < cachep->num && batchcount--) {
                         STATS_INC_ALLOCED(cachep);
                         STATS_INC_ACTIVE(cachep);
                         STATS_SET_HIGH(cachep);
 
-                        ac_put_obj(cachep, ac, slab_get_obj(cachep, slabp,
+                        ac_put_obj(cachep, ac, slab_get_obj(cachep, page,
                                                                         node));
                 }
-                check_slabp(cachep, slabp);
 
                 /* move slabp to correct slabp list: */
-                list_del(&slabp->list);
-                if (slabp->free == BUFCTL_END)
-                        list_add(&slabp->list, &n->slabs_full);
+                list_del(&page->lru);
+                if (page->active == cachep->num)
+                        list_add(&page->list, &n->slabs_full);
                 else
-                        list_add(&slabp->list, &n->slabs_partial);
+                        list_add(&page->list, &n->slabs_partial);
         }
 
 must_grow:
@@ -3097,16 +2960,6 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
                 *dbg_redzone1(cachep, objp) = RED_ACTIVE;
                 *dbg_redzone2(cachep, objp) = RED_ACTIVE;
         }
-#ifdef CONFIG_DEBUG_SLAB_LEAK
-        {
-                struct slab *slabp;
-                unsigned objnr;
-
-                slabp = virt_to_head_page(objp)->slab_page;
-                objnr = (unsigned)(objp - slabp->s_mem) / cachep->size;
-                slab_bufctl(slabp)[objnr] = BUFCTL_ACTIVE;
-        }
-#endif
         objp += obj_offset(cachep);
         if (cachep->ctor && cachep->flags & SLAB_POISON)
                 cachep->ctor(objp);
@@ -3248,18 +3101,20 @@ retry:
                  * We may trigger various forms of reclaim on the allowed
                  * set and go into memory reserves if necessary.
                  */
+                struct page *page;
+
                 if (local_flags & __GFP_WAIT)
                         local_irq_enable();
                 kmem_flagcheck(cache, flags);
-                obj = kmem_getpages(cache, local_flags, numa_mem_id());
+                page = kmem_getpages(cache, local_flags, numa_mem_id());
                 if (local_flags & __GFP_WAIT)
                         local_irq_disable();
-                if (obj) {
+                if (page) {
                         /*
                          * Insert into the appropriate per node queues
                          */
-                        nid = page_to_nid(virt_to_page(obj));
-                        if (cache_grow(cache, flags, nid, obj)) {
+                        nid = page_to_nid(page);
+                        if (cache_grow(cache, flags, nid, page)) {
                                 obj = ____cache_alloc_node(cache,
                                         flags | GFP_THISNODE, nid);
                                 if (!obj)
@@ -3288,7 +3143,7 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
                                 int nodeid)
 {
         struct list_head *entry;
-        struct slab *slabp;
+        struct page *page;
         struct kmem_cache_node *n;
         void *obj;
         int x;
@@ -3308,26 +3163,24 @@ retry:
                         goto must_grow;
         }
 
-        slabp = list_entry(entry, struct slab, list);
+        page = list_entry(entry, struct page, lru);
         check_spinlock_acquired_node(cachep, nodeid);
-        check_slabp(cachep, slabp);
 
         STATS_INC_NODEALLOCS(cachep);
         STATS_INC_ACTIVE(cachep);
         STATS_SET_HIGH(cachep);
 
-        BUG_ON(slabp->inuse == cachep->num);
+        BUG_ON(page->active == cachep->num);
 
-        obj = slab_get_obj(cachep, slabp, nodeid);
-        check_slabp(cachep, slabp);
+        obj = slab_get_obj(cachep, page, nodeid);
         n->free_objects--;
         /* move slabp to correct slabp list: */
-        list_del(&slabp->list);
+        list_del(&page->lru);
 
-        if (slabp->free == BUFCTL_END)
-                list_add(&slabp->list, &n->slabs_full);
+        if (page->active == cachep->num)
+                list_add(&page->lru, &n->slabs_full);
         else
-                list_add(&slabp->list, &n->slabs_partial);
+                list_add(&page->lru, &n->slabs_partial);
 
         spin_unlock(&n->list_lock);
         goto done;
@@ -3477,23 +3330,21 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects,
 
         for (i = 0; i < nr_objects; i++) {
                 void *objp;
-                struct slab *slabp;
+                struct page *page;
 
                 clear_obj_pfmemalloc(&objpp[i]);
                 objp = objpp[i];
 
-                slabp = virt_to_slab(objp);
+                page = virt_to_head_page(objp);
                 n = cachep->node[node];
-                list_del(&slabp->list);
+                list_del(&page->lru);
                 check_spinlock_acquired_node(cachep, node);
-                check_slabp(cachep, slabp);
-                slab_put_obj(cachep, slabp, objp, node);
+                slab_put_obj(cachep, page, objp, node);
                 STATS_DEC_ACTIVE(cachep);
                 n->free_objects++;
-                check_slabp(cachep, slabp);
 
                 /* fixup slab chains */
-                if (slabp->inuse == 0) {
+                if (page->active == 0) {
                         if (n->free_objects > n->free_limit) {
                                 n->free_objects -= cachep->num;
                                 /* No need to drop any previously held
@@ -3502,16 +3353,16 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects,
                                  * a different cache, refer to comments before
                                  * alloc_slabmgmt.
                                  */
-                                slab_destroy(cachep, slabp);
+                                slab_destroy(cachep, page);
                         } else {
-                                list_add(&slabp->list, &n->slabs_free);
+                                list_add(&page->lru, &n->slabs_free);
                         }
                 } else {
                         /* Unconditionally move a slab to the end of the
                          * partial list on free - maximum time for the
                          * other objects to be freed, too.
                          */
-                        list_add_tail(&slabp->list, &n->slabs_partial);
+                        list_add_tail(&page->lru, &n->slabs_partial);
                 }
         }
 }
@@ -3551,10 +3402,10 @@ free_done:
 
                 p = n->slabs_free.next;
                 while (p != &(n->slabs_free)) {
-                        struct slab *slabp;
+                        struct page *page;
 
-                        slabp = list_entry(p, struct slab, list);
-                        BUG_ON(slabp->inuse);
+                        page = list_entry(p, struct page, lru);
+                        BUG_ON(page->active);
 
                         i++;
                         p = p->next;
@@ -4158,7 +4009,7 @@ out:
 #ifdef CONFIG_SLABINFO
 void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo)
 {
-        struct slab *slabp;
+        struct page *page;
         unsigned long active_objs;
         unsigned long num_objs;
         unsigned long active_slabs = 0;
@@ -4178,23 +4029,23 @@ void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo)
                 check_irq_on();
                 spin_lock_irq(&n->list_lock);
 
-                list_for_each_entry(slabp, &n->slabs_full, list) {
-                        if (slabp->inuse != cachep->num && !error)
+                list_for_each_entry(page, &n->slabs_full, lru) {
+                        if (page->active != cachep->num && !error)
                                 error = "slabs_full accounting error";
                         active_objs += cachep->num;
                         active_slabs++;
                 }
-                list_for_each_entry(slabp, &n->slabs_partial, list) {
-                        if (slabp->inuse == cachep->num && !error)
-                                error = "slabs_partial inuse accounting error";
-                        if (!slabp->inuse && !error)
-                                error = "slabs_partial/inuse accounting error";
-                        active_objs += slabp->inuse;
+                list_for_each_entry(page, &n->slabs_partial, lru) {
+                        if (page->active == cachep->num && !error)
+                                error = "slabs_partial accounting error";
+                        if (!page->active && !error)
+                                error = "slabs_partial accounting error";
+                        active_objs += page->active;
                         active_slabs++;
                 }
-                list_for_each_entry(slabp, &n->slabs_free, list) {
-                        if (slabp->inuse && !error)
-                                error = "slabs_free/inuse accounting error";
+                list_for_each_entry(page, &n->slabs_free, lru) {
+                        if (page->active && !error)
+                                error = "slabs_free accounting error";
                         num_slabs++;
                 }
                 free_objects += n->free_objects;
@@ -4346,15 +4197,27 @@ static inline int add_caller(unsigned long *n, unsigned long v)
         return 1;
 }
 
-static void handle_slab(unsigned long *n, struct kmem_cache *c, struct slab *s)
+static void handle_slab(unsigned long *n, struct kmem_cache *c,
+                                                struct page *page)
 {
         void *p;
-        int i;
+        int i, j;
+
         if (n[0] == n[1])
                 return;
-        for (i = 0, p = s->s_mem; i < c->num; i++, p += c->size) {
-                if (slab_bufctl(s)[i] != BUFCTL_ACTIVE)
+        for (i = 0, p = page->s_mem; i < c->num; i++, p += c->size) {
+                bool active = true;
+
+                for (j = page->active; j < c->num; j++) {
+                        /* Skip freed item */
+                        if (slab_freelist(page)[j] == i) {
+                                active = false;
+                                break;
+                        }
+                }
+                if (!active)
                         continue;
+
                 if (!add_caller(n, (unsigned long)*dbg_userword(c, p)))
                         return;
         }
@@ -4379,7 +4242,7 @@ static void show_symbol(struct seq_file *m, unsigned long address)
 static int leaks_show(struct seq_file *m, void *p)
 {
         struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
-        struct slab *slabp;
+        struct page *page;
         struct kmem_cache_node *n;
         const char *name;
         unsigned long *x = m->private;
@@ -4403,10 +4266,10 @@ static int leaks_show(struct seq_file *m, void *p)
                 check_irq_on();
                 spin_lock_irq(&n->list_lock);
 
-                list_for_each_entry(slabp, &n->slabs_full, list)
-                        handle_slab(x, cachep, slabp);
-                list_for_each_entry(slabp, &n->slabs_partial, list)
-                        handle_slab(x, cachep, slabp);
+                list_for_each_entry(page, &n->slabs_full, lru)
+                        handle_slab(x, cachep, page);
+                list_for_each_entry(page, &n->slabs_partial, lru)
+                        handle_slab(x, cachep, page);
                 spin_unlock_irq(&n->list_lock);
         }
         name = cachep->name;
diff --git a/mm/slub.c b/mm/slub.c
index 7e8bd8d828bc..545a170ebf9f 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -155,7 +155,7 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
 /*
  * Maximum number of desirable partial slabs.
  * The existence of more partial slabs makes kmem_cache_shrink
- * sort the partial list by the number of objects in the.
+ * sort the partial list by the number of objects in use.
  */
 #define MAX_PARTIAL 10
 
@@ -933,6 +933,16 @@ static void trace(struct kmem_cache *s, struct page *page, void *object,
  * Hooks for other subsystems that check memory allocations. In a typical
  * production configuration these hooks all should produce no code at all.
  */
+static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
+{
+        kmemleak_alloc(ptr, size, 1, flags);
+}
+
+static inline void kfree_hook(const void *x)
+{
+        kmemleak_free(x);
+}
+
 static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
 {
         flags &= gfp_allowed_mask;
@@ -1217,8 +1227,8 @@ static unsigned long kmem_cache_flags(unsigned long object_size,
         /*
          * Enable debugging if selected on the kernel commandline.
          */
-        if (slub_debug && (!slub_debug_slabs ||
-                !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs))))
+        if (slub_debug && (!slub_debug_slabs || (name &&
+                !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs)))))
                 flags |= slub_debug;
 
         return flags;
@@ -1260,13 +1270,30 @@ static inline void inc_slabs_node(struct kmem_cache *s, int node,
 static inline void dec_slabs_node(struct kmem_cache *s, int node,
                                                         int objects) {}
 
+static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
+{
+        kmemleak_alloc(ptr, size, 1, flags);
+}
+
+static inline void kfree_hook(const void *x)
+{
+        kmemleak_free(x);
+}
+
 static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
                                                         { return 0; }
 
 static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
-                void *object) {}
+                void *object)
+{
+        kmemleak_alloc_recursive(object, s->object_size, 1, s->flags,
+                flags & gfp_allowed_mask);
+}
 
-static inline void slab_free_hook(struct kmem_cache *s, void *x) {}
+static inline void slab_free_hook(struct kmem_cache *s, void *x)
+{
+        kmemleak_free_recursive(x, s->flags);
+}
 
 #endif /* CONFIG_SLUB_DEBUG */
 
@@ -2829,8 +2856,8 @@ static struct kmem_cache *kmem_cache_node;
  * slab on the node for this slabcache. There are no concurrent accesses
  * possible.
  *
- * Note that this function only works on the kmalloc_node_cache
- * when allocating for the kmalloc_node_cache. This is used for bootstrapping
+ * Note that this function only works on the kmem_cache_node
+ * when allocating for the kmem_cache_node. This is used for bootstrapping
  * memory on a fresh node that has no slab structures yet.
  */
 static void early_kmem_cache_node_alloc(int node)
@@ -3272,7 +3299,7 @@ static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
         if (page)
                 ptr = page_address(page);
 
-        kmemleak_alloc(ptr, size, 1, flags);
+        kmalloc_large_node_hook(ptr, size, flags);
         return ptr;
 }
 
@@ -3336,7 +3363,7 @@ void kfree(const void *x)
         page = virt_to_head_page(x);
         if (unlikely(!PageSlab(page))) {
                 BUG_ON(!PageCompound(page));
-                kmemleak_free(x);
+                kfree_hook(x);
                 __free_memcg_kmem_pages(page, compound_order(page));
                 return;
         }
diff --git a/mm/swap.c b/mm/swap.c
index 7a9f80d451f5..84b26aaabd03 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -82,19 +82,6 @@ static void __put_compound_page(struct page *page)
 
 static void put_compound_page(struct page *page)
 {
-        /*
-         * hugetlbfs pages cannot be split from under us.  If this is a
-         * hugetlbfs page, check refcount on head page and release the page if
-         * the refcount becomes zero.
-         */
-        if (PageHuge(page)) {
-                page = compound_head(page);
-                if (put_page_testzero(page))
-                        __put_compound_page(page);
-
-                return;
-        }
-
         if (unlikely(PageTail(page))) {
                 /* __split_huge_page_refcount can run under us */
                 struct page *page_head = compound_trans_head(page);
@@ -111,14 +98,31 @@ static void put_compound_page(struct page *page)
                          * still hot on arches that do not support
                          * this_cpu_cmpxchg_double().
                          */
-                        if (PageSlab(page_head)) {
-                                if (PageTail(page)) {
+                        if (PageSlab(page_head) || PageHeadHuge(page_head)) {
+                                if (likely(PageTail(page))) {
+                                        /*
+                                         * __split_huge_page_refcount
+                                         * cannot race here.
+                                         */
+                                        VM_BUG_ON(!PageHead(page_head));
+                                        atomic_dec(&page->_mapcount);
                                         if (put_page_testzero(page_head))
                                                 VM_BUG_ON(1);
-
-                                        atomic_dec(&page->_mapcount);
-                                        goto skip_lock_tail;
+                                        if (put_page_testzero(page_head))
+                                                __put_compound_page(page_head);
+                                        return;
                                 } else
+                                        /*
+                                         * __split_huge_page_refcount
+                                         * run before us, "page" was a
+                                         * THP tail. The split
+                                         * page_head has been freed
+                                         * and reallocated as slab or
+                                         * hugetlbfs page of smaller
+                                         * order (only possible if
+                                         * reallocated as slab on
+                                         * x86).
+                                         */
                                         goto skip_lock;
                         }
                         /*
@@ -132,8 +136,27 @@ static void put_compound_page(struct page *page)
                                 /* __split_huge_page_refcount run before us */
                                 compound_unlock_irqrestore(page_head, flags);
 skip_lock:
-                                if (put_page_testzero(page_head))
-                                        __put_single_page(page_head);
+                                if (put_page_testzero(page_head)) {
+                                        /*
+                                         * The head page may have been
+                                         * freed and reallocated as a
+                                         * compound page of smaller
+                                         * order and then freed again.
+                                         * All we know is that it
+                                         * cannot have become: a THP
+                                         * page, a compound page of
+                                         * higher order, a tail page.
+                                         * That is because we still
+                                         * hold the refcount of the
+                                         * split THP tail and
+                                         * page_head was the THP head
+                                         * before the split.
+                                         */
+                                        if (PageHead(page_head))
+                                                __put_compound_page(page_head);
+                                        else
+                                                __put_single_page(page_head);
+                                }
 out_put_single:
                                 if (put_page_testzero(page))
                                         __put_single_page(page);
@@ -155,7 +178,6 @@ out_put_single:
                         VM_BUG_ON(atomic_read(&page->_count) != 0);
                         compound_unlock_irqrestore(page_head, flags);
 
-skip_lock_tail:
                         if (put_page_testzero(page_head)) {
                                 if (PageHead(page_head))
                                         __put_compound_page(page_head);
@@ -198,51 +220,52 @@ bool __get_page_tail(struct page *page)
          * proper PT lock that already serializes against
          * split_huge_page().
          */
+        unsigned long flags;
         bool got = false;
-        struct page *page_head;
-
-        /*
-         * If this is a hugetlbfs page it cannot be split under us.  Simply
-         * increment refcount for the head page.
-         */
-        if (PageHuge(page)) {
-                page_head = compound_head(page);
-                atomic_inc(&page_head->_count);
-                got = true;
-        } else {
-                unsigned long flags;
+        struct page *page_head = compound_trans_head(page);
 
-                page_head = compound_trans_head(page);
-                if (likely(page != page_head &&
-                                        get_page_unless_zero(page_head))) {
-
-                        /* Ref to put_compound_page() comment. */
-                        if (PageSlab(page_head)) {
-                                if (likely(PageTail(page))) {
-                                        __get_page_tail_foll(page, false);
-                                        return true;
-                                } else {
-                                        put_page(page_head);
-                                        return false;
-                                }
-                        }
-
-                        /*
-                         * page_head wasn't a dangling pointer but it
-                         * may not be a head page anymore by the time
-                         * we obtain the lock. That is ok as long as it
-                         * can't be freed from under us.
-                         */
-                        flags = compound_lock_irqsave(page_head);
-                        /* here __split_huge_page_refcount won't run anymore */
+        if (likely(page != page_head && get_page_unless_zero(page_head))) {
+                /* Ref to put_compound_page() comment. */
+                if (PageSlab(page_head) || PageHeadHuge(page_head)) {
                         if (likely(PageTail(page))) {
+                                /*
+                                 * This is a hugetlbfs page or a slab
+                                 * page. __split_huge_page_refcount
+                                 * cannot race here.
+                                 */
+                                VM_BUG_ON(!PageHead(page_head));
                                 __get_page_tail_foll(page, false);
-                                got = true;
-                        }
-                        compound_unlock_irqrestore(page_head, flags);
-                        if (unlikely(!got))
+                                return true;
+                        } else {
+                                /*
+                                 * __split_huge_page_refcount run
+                                 * before us, "page" was a THP
+                                 * tail. The split page_head has been
+                                 * freed and reallocated as slab or
+                                 * hugetlbfs page of smaller order
+                                 * (only possible if reallocated as
+                                 * slab on x86).
+                                 */
                                 put_page(page_head);
+                                return false;
+                        }
+                }
+
+                /*
+                 * page_head wasn't a dangling pointer but it
+                 * may not be a head page anymore by the time
+                 * we obtain the lock. That is ok as long as it
+                 * can't be freed from under us.
+                 */
+                flags = compound_lock_irqsave(page_head);
+                /* here __split_huge_page_refcount won't run anymore */
+                if (likely(PageTail(page))) {
+                        __get_page_tail_foll(page, false);
+                        got = true;
                 }
+                compound_unlock_irqrestore(page_head, flags);
+                if (unlikely(!got))
+                        put_page(page_head);
         }
         return got;
 }