64 files changed, 12516 insertions, 3905 deletions

diff --git a/mm/Kconfig b/mm/Kconfig
index b53427ad30a3..44cf6f0a3a6d 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -67,7 +67,7 @@ config DISCONTIGMEM
 
 config SPARSEMEM
         def_bool y
-        depends on SPARSEMEM_MANUAL
+        depends on (!SELECT_MEMORY_MODEL && ARCH_SPARSEMEM_ENABLE) || SPARSEMEM_MANUAL
 
 config FLATMEM
         def_bool y
@@ -128,11 +128,8 @@ config SPARSEMEM_VMEMMAP
 config MEMORY_HOTPLUG
         bool "Allow for memory hot-add"
         depends on SPARSEMEM || X86_64_ACPI_NUMA
-        depends on HOTPLUG && !HIBERNATION && ARCH_ENABLE_MEMORY_HOTPLUG
-        depends on (IA64 || X86 || PPC64 || SUPERH || S390)
-
-comment "Memory hotplug is currently incompatible with Software Suspend"
-        depends on SPARSEMEM && HOTPLUG && HIBERNATION
+        depends on HOTPLUG && ARCH_ENABLE_MEMORY_HOTPLUG
+        depends on (IA64 || X86 || PPC_BOOK3S_64 || SUPERH || S390)
 
 config MEMORY_HOTPLUG_SPARSE
         def_bool y
@@ -153,7 +150,7 @@ config MEMORY_HOTREMOVE
 #
 config PAGEFLAGS_EXTENDED
         def_bool y
-        depends on 64BIT || SPARSEMEM_VMEMMAP || !NUMA || !SPARSEMEM
+        depends on 64BIT || SPARSEMEM_VMEMMAP || !SPARSEMEM
 
 # Heavily threaded applications may benefit from splitting the mm-wide
 # page_table_lock, so that faults on different parts of the user address
@@ -203,23 +200,90 @@ config VIRT_TO_BUS
         def_bool y
         depends on !ARCH_NO_VIRT_TO_BUS
 
-config UNEVICTABLE_LRU
-        bool "Add LRU list to track non-evictable pages"
-        default y
-        help
-          Keeps unevictable pages off of the active and inactive pageout
-          lists, so kswapd will not waste CPU time or have its balancing
-          algorithms thrown off by scanning these pages.  Selecting this
-          will use one page flag and increase the code size a little,
-          say Y unless you know what you are doing.
-
 config HAVE_MLOCK
         bool
         default y if MMU=y
 
 config HAVE_MLOCKED_PAGE_BIT
         bool
-        default y if HAVE_MLOCK=y && UNEVICTABLE_LRU=y
+        default y if HAVE_MLOCK=y
 
 config MMU_NOTIFIER
         bool
+
+config KSM
+        bool "Enable KSM for page merging"
+        depends on MMU
+        help
+          Enable Kernel Samepage Merging: KSM periodically scans those areas
+          of an application's address space that an app has advised may be
+          mergeable.  When it finds pages of identical content, it replaces
+          the many instances by a single resident page with that content, so
+          saving memory until one or another app needs to modify the content.
+          Recommended for use with KVM, or with other duplicative applications.
+          See Documentation/vm/ksm.txt for more information: KSM is inactive
+          until a program has madvised that an area is MADV_MERGEABLE, and
+          root has set /sys/kernel/mm/ksm/run to 1 (if CONFIG_SYSFS is set).
+
+config DEFAULT_MMAP_MIN_ADDR
+        int "Low address space to protect from user allocation"
+        default 4096
+        help
+          This is the portion of low virtual memory which should be protected
+          from userspace allocation.  Keeping a user from writing to low pages
+          can help reduce the impact of kernel NULL pointer bugs.
+
+          For most ia64, ppc64 and x86 users with lots of address space
+          a value of 65536 is reasonable and should cause no problems.
+          On arm and other archs it should not be higher than 32768.
+          Programs which use vm86 functionality or have some need to map
+          this low address space will need CAP_SYS_RAWIO or disable this
+          protection by setting the value to 0.
+
+          This value can be changed after boot using the
+          /proc/sys/vm/mmap_min_addr tunable.
+
+config ARCH_SUPPORTS_MEMORY_FAILURE
+        bool
+
+config MEMORY_FAILURE
+        depends on MMU
+        depends on ARCH_SUPPORTS_MEMORY_FAILURE
+        bool "Enable recovery from hardware memory errors"
+        help
+          Enables code to recover from some memory failures on systems
+          with MCA recovery. This allows a system to continue running
+          even when some of its memory has uncorrected errors. This requires
+          special hardware support and typically ECC memory.
+
+config HWPOISON_INJECT
+        tristate "Poison pages injector"
+        depends on MEMORY_FAILURE && DEBUG_KERNEL
+
+config NOMMU_INITIAL_TRIM_EXCESS
+        int "Turn on mmap() excess space trimming before booting"
+        depends on !MMU
+        default 1
+        help
+          The NOMMU mmap() frequently needs to allocate large contiguous chunks
+          of memory on which to store mappings, but it can only ask the system
+          allocator for chunks in 2^N*PAGE_SIZE amounts - which is frequently
+          more than it requires.  To deal with this, mmap() is able to trim off
+          the excess and return it to the allocator.
+
+          If trimming is enabled, the excess is trimmed off and returned to the
+          system allocator, which can cause extra fragmentation, particularly
+          if there are a lot of transient processes.
+
+          If trimming is disabled, the excess is kept, but not used, which for
+          long-term mappings means that the space is wasted.
+
+          Trimming can be dynamically controlled through a sysctl option
+          (/proc/sys/vm/nr_trim_pages) which specifies the minimum number of
+          excess pages there must be before trimming should occur, or zero if
+          no trimming is to occur.
+
+          This option specifies the initial value of this option.  The default
+          of 1 says that all excess pages should be trimmed.
+
+          See Documentation/nommu-mmap.txt for more information.
diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug
index c8d62d49a44e..af7cfb43d2f0 100644
--- a/mm/Kconfig.debug
+++ b/mm/Kconfig.debug
@@ -1,3 +1,13 @@
+config DEBUG_PAGEALLOC
+        bool "Debug page memory allocations"
+        depends on DEBUG_KERNEL && ARCH_SUPPORTS_DEBUG_PAGEALLOC
+        depends on !HIBERNATION || !PPC && !SPARC
+        depends on !KMEMCHECK
+        ---help---
+          Unmap pages from the kernel linear mapping after free_pages().
+          This results in a large slowdown, but helps to find certain types
+          of memory corruption.
+
 config WANT_PAGE_DEBUG_FLAGS
         bool
 
@@ -7,11 +17,11 @@ config PAGE_POISONING
         depends on !HIBERNATION
         select DEBUG_PAGEALLOC
         select WANT_PAGE_DEBUG_FLAGS
-        help
+        ---help---
            Fill the pages with poison patterns after free_pages() and verify
            the patterns before alloc_pages(). This results in a large slowdown,
-           but helps to find certain types of memory corruptions.
+           but helps to find certain types of memory corruption.
 
-           This option cannot enalbe with hibernation. Otherwise, it will get
-           wrong messages for memory corruption because the free pages are not
-           saved to the suspend image.
+           This option cannot be enabled in combination with hibernation as
+           that would result in incorrect warnings of memory corruption after
+           a resume because free pages are not saved to the suspend image.
diff --git a/mm/Makefile b/mm/Makefile
index ec73c68b6015..ebf849042ed3 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -5,15 +5,16 @@
 mmu-y                   := nommu.o
 mmu-$(CONFIG_MMU)       := fremap.o highmem.o madvise.o memory.o mincore.o \
                            mlock.o mmap.o mprotect.o mremap.o msync.o rmap.o \
-                           vmalloc.o
+                           vmalloc.o pagewalk.o
 
 obj-y                   := bootmem.o filemap.o mempool.o oom_kill.o fadvise.o \
-                           maccess.o page_alloc.o page-writeback.o pdflush.o \
+                           maccess.o page_alloc.o page-writeback.o \
                            readahead.o swap.o truncate.o vmscan.o shmem.o \
                            prio_tree.o util.o mmzone.o vmstat.o backing-dev.o \
-                           page_isolation.o mm_init.o $(mmu-y)
+                           page_isolation.o mm_init.o mmu_context.o \
+                           $(mmu-y)
+obj-y += init-mm.o
 
-obj-$(CONFIG_PROC_PAGE_MONITOR) += pagewalk.o
 obj-$(CONFIG_BOUNCE)    += bounce.o
 obj-$(CONFIG_SWAP)      += page_io.o swap_state.o swapfile.o thrash.o
 obj-$(CONFIG_HAS_DMA)   += dmapool.o
@@ -24,17 +25,23 @@ obj-$(CONFIG_SPARSEMEM_VMEMMAP) += sparse-vmemmap.o
 obj-$(CONFIG_TMPFS_POSIX_ACL) += shmem_acl.o
 obj-$(CONFIG_SLOB) += slob.o
 obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o
+obj-$(CONFIG_KSM) += ksm.o
 obj-$(CONFIG_PAGE_POISONING) += debug-pagealloc.o
 obj-$(CONFIG_SLAB) += slab.o
 obj-$(CONFIG_SLUB) += slub.o
+obj-$(CONFIG_KMEMCHECK) += kmemcheck.o
 obj-$(CONFIG_FAILSLAB) += failslab.o
 obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o
 obj-$(CONFIG_FS_XIP) += filemap_xip.o
 obj-$(CONFIG_MIGRATION) += migrate.o
-ifdef CONFIG_HAVE_DYNAMIC_PER_CPU_AREA
+ifndef CONFIG_HAVE_LEGACY_PER_CPU_AREA
 obj-$(CONFIG_SMP) += percpu.o
 else
 obj-$(CONFIG_SMP) += allocpercpu.o
 endif
 obj-$(CONFIG_QUICKLIST) += quicklist.o
 obj-$(CONFIG_CGROUP_MEM_RES_CTLR) += memcontrol.o page_cgroup.o
+obj-$(CONFIG_MEMORY_FAILURE) += memory-failure.o
+obj-$(CONFIG_HWPOISON_INJECT) += hwpoison-inject.o
+obj-$(CONFIG_DEBUG_KMEMLEAK) += kmemleak.o
+obj-$(CONFIG_DEBUG_KMEMLEAK_TEST) += kmemleak-test.o
diff --git a/mm/allocpercpu.c b/mm/allocpercpu.c
index 139d5b7b6621..df34ceae0c67 100644
--- a/mm/allocpercpu.c
+++ b/mm/allocpercpu.c
@@ -5,6 +5,8 @@
  */
 #include <linux/mm.h>
 #include <linux/module.h>
+#include <linux/bootmem.h>
+#include <asm/sections.h>
 
 #ifndef cache_line_size
 #define cache_line_size()       L1_CACHE_BYTES
@@ -31,7 +33,7 @@ static void percpu_depopulate(void *__pdata, int cpu)
  * @__pdata: per-cpu data to depopulate
  * @mask: depopulate per-cpu data for cpu's selected through mask bits
  */
-static void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask)
+static void __percpu_depopulate_mask(void *__pdata, const cpumask_t *mask)
 {
         int cpu;
         for_each_cpu_mask_nr(cpu, *mask)
@@ -147,3 +149,29 @@ void free_percpu(void *__pdata)
         kfree(__percpu_disguise(__pdata));
 }
 EXPORT_SYMBOL_GPL(free_percpu);
+
+/*
+ * Generic percpu area setup.
+ */
+#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
+unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
+
+EXPORT_SYMBOL(__per_cpu_offset);
+
+void __init setup_per_cpu_areas(void)
+{
+        unsigned long size, i;
+        char *ptr;
+        unsigned long nr_possible_cpus = num_possible_cpus();
+
+        /* Copy section for each CPU (we discard the original) */
+        size = ALIGN(PERCPU_ENOUGH_ROOM, PAGE_SIZE);
+        ptr = alloc_bootmem_pages(size * nr_possible_cpus);
+
+        for_each_possible_cpu(i) {
+                __per_cpu_offset[i] = ptr - __per_cpu_start;
+                memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
+                ptr += size;
+        }
+}
+#endif /* CONFIG_HAVE_SETUP_PER_CPU_AREA */
diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index be68c956a660..0e8ca0347707 100644
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -1,8 +1,11 @@
 
 #include <linux/wait.h>
 #include <linux/backing-dev.h>
+#include <linux/kthread.h>
+#include <linux/freezer.h>
 #include <linux/fs.h>
 #include <linux/pagemap.h>
+#include <linux/mm.h>
 #include <linux/sched.h>
 #include <linux/module.h>
 #include <linux/writeback.h>
@@ -14,6 +17,7 @@ void default_unplug_io_fn(struct backing_dev_info *bdi, struct page *page)
 EXPORT_SYMBOL(default_unplug_io_fn);
 
 struct backing_dev_info default_backing_dev_info = {
+        .name           = "default",
         .ra_pages       = VM_MAX_READAHEAD * 1024 / PAGE_CACHE_SIZE,
         .state          = 0,
         .capabilities   = BDI_CAP_MAP_COPY,
@@ -23,6 +27,24 @@ EXPORT_SYMBOL_GPL(default_backing_dev_info);
 
 static struct class *bdi_class;
 
+/*
+ * bdi_lock protects updates to bdi_list and bdi_pending_list, as well as
+ * reader side protection for bdi_pending_list. bdi_list has RCU reader side
+ * locking.
+ */
+DEFINE_SPINLOCK(bdi_lock);
+LIST_HEAD(bdi_list);
+LIST_HEAD(bdi_pending_list);
+
+static struct task_struct *sync_supers_tsk;
+static struct timer_list sync_supers_timer;
+
+static int bdi_sync_supers(void *);
+static void sync_supers_timer_fn(unsigned long);
+static void arm_supers_timer(void);
+
+static void bdi_add_default_flusher_task(struct backing_dev_info *bdi);
+
 #ifdef CONFIG_DEBUG_FS
 #include <linux/debugfs.h>
 #include <linux/seq_file.h>
@@ -37,9 +59,29 @@ static void bdi_debug_init(void)
 static int bdi_debug_stats_show(struct seq_file *m, void *v)
 {
         struct backing_dev_info *bdi = m->private;
+        struct bdi_writeback *wb;
         unsigned long background_thresh;
         unsigned long dirty_thresh;
         unsigned long bdi_thresh;
+        unsigned long nr_dirty, nr_io, nr_more_io, nr_wb;
+        struct inode *inode;
+
+        /*
+         * inode lock is enough here, the bdi->wb_list is protected by
+         * RCU on the reader side
+         */
+        nr_wb = nr_dirty = nr_io = nr_more_io = 0;
+        spin_lock(&inode_lock);
+        list_for_each_entry(wb, &bdi->wb_list, list) {
+                nr_wb++;
+                list_for_each_entry(inode, &wb->b_dirty, i_list)
+                        nr_dirty++;
+                list_for_each_entry(inode, &wb->b_io, i_list)
+                        nr_io++;
+                list_for_each_entry(inode, &wb->b_more_io, i_list)
+                        nr_more_io++;
+        }
+        spin_unlock(&inode_lock);
 
         get_dirty_limits(&background_thresh, &dirty_thresh, &bdi_thresh, bdi);
 
@@ -49,12 +91,22 @@ static int bdi_debug_stats_show(struct seq_file *m, void *v)
                    "BdiReclaimable:   %8lu kB\n"
                    "BdiDirtyThresh:   %8lu kB\n"
                    "DirtyThresh:      %8lu kB\n"
-                   "BackgroundThresh: %8lu kB\n",
+                   "BackgroundThresh: %8lu kB\n"
+                   "WritebackThreads: %8lu\n"
+                   "b_dirty:          %8lu\n"
+                   "b_io:             %8lu\n"
+                   "b_more_io:        %8lu\n"
+                   "bdi_list:         %8u\n"
+                   "state:            %8lx\n"
+                   "wb_mask:          %8lx\n"
+                   "wb_list:          %8u\n"
+                   "wb_cnt:           %8u\n",
                    (unsigned long) K(bdi_stat(bdi, BDI_WRITEBACK)),
                    (unsigned long) K(bdi_stat(bdi, BDI_RECLAIMABLE)),
-                   K(bdi_thresh),
-                   K(dirty_thresh),
-                   K(background_thresh));
+                   K(bdi_thresh), K(dirty_thresh),
+                   K(background_thresh), nr_wb, nr_dirty, nr_io, nr_more_io,
+                   !list_empty(&bdi->bdi_list), bdi->state, bdi->wb_mask,
+                   !list_empty(&bdi->wb_list), bdi->wb_cnt);
 #undef K
 
         return 0;
@@ -185,6 +237,13 @@ static int __init default_bdi_init(void)
 {
         int err;
 
+        sync_supers_tsk = kthread_run(bdi_sync_supers, NULL, "sync_supers");
+        BUG_ON(IS_ERR(sync_supers_tsk));
+
+        init_timer(&sync_supers_timer);
+        setup_timer(&sync_supers_timer, sync_supers_timer_fn, 0);
+        arm_supers_timer();
+
         err = bdi_init(&default_backing_dev_info);
         if (!err)
                 bdi_register(&default_backing_dev_info, NULL, "default");
@@ -193,6 +252,279 @@ static int __init default_bdi_init(void)
 }
 subsys_initcall(default_bdi_init);
 
+static void bdi_wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi)
+{
+        memset(wb, 0, sizeof(*wb));
+
+        wb->bdi = bdi;
+        wb->last_old_flush = jiffies;
+        INIT_LIST_HEAD(&wb->b_dirty);
+        INIT_LIST_HEAD(&wb->b_io);
+        INIT_LIST_HEAD(&wb->b_more_io);
+}
+
+static void bdi_task_init(struct backing_dev_info *bdi,
+                          struct bdi_writeback *wb)
+{
+        struct task_struct *tsk = current;
+
+        spin_lock(&bdi->wb_lock);
+        list_add_tail_rcu(&wb->list, &bdi->wb_list);
+        spin_unlock(&bdi->wb_lock);
+
+        tsk->flags |= PF_FLUSHER | PF_SWAPWRITE;
+        set_freezable();
+
+        /*
+         * Our parent may run at a different priority, just set us to normal
+         */
+        set_user_nice(tsk, 0);
+}
+
+static int bdi_start_fn(void *ptr)
+{
+        struct bdi_writeback *wb = ptr;
+        struct backing_dev_info *bdi = wb->bdi;
+        int ret;
+
+        /*
+         * Add us to the active bdi_list
+         */
+        spin_lock_bh(&bdi_lock);
+        list_add_rcu(&bdi->bdi_list, &bdi_list);
+        spin_unlock_bh(&bdi_lock);
+
+        bdi_task_init(bdi, wb);
+
+        /*
+         * Clear pending bit and wakeup anybody waiting to tear us down
+         */
+        clear_bit(BDI_pending, &bdi->state);
+        smp_mb__after_clear_bit();
+        wake_up_bit(&bdi->state, BDI_pending);
+
+        ret = bdi_writeback_task(wb);
+
+        /*
+         * Remove us from the list
+         */
+        spin_lock(&bdi->wb_lock);
+        list_del_rcu(&wb->list);
+        spin_unlock(&bdi->wb_lock);
+
+        /*
+         * Flush any work that raced with us exiting. No new work
+         * will be added, since this bdi isn't discoverable anymore.
+         */
+        if (!list_empty(&bdi->work_list))
+                wb_do_writeback(wb, 1);
+
+        wb->task = NULL;
+        return ret;
+}
+
+int bdi_has_dirty_io(struct backing_dev_info *bdi)
+{
+        return wb_has_dirty_io(&bdi->wb);
+}
+
+static void bdi_flush_io(struct backing_dev_info *bdi)
+{
+        struct writeback_control wbc = {
+                .bdi                    = bdi,
+                .sync_mode              = WB_SYNC_NONE,
+                .older_than_this        = NULL,
+                .range_cyclic           = 1,
+                .nr_to_write            = 1024,
+        };
+
+        writeback_inodes_wbc(&wbc);
+}
+
+/*
+ * kupdated() used to do this. We cannot do it from the bdi_forker_task()
+ * or we risk deadlocking on ->s_umount. The longer term solution would be
+ * to implement sync_supers_bdi() or similar and simply do it from the
+ * bdi writeback tasks individually.
+ */
+static int bdi_sync_supers(void *unused)
+{
+        set_user_nice(current, 0);
+
+        while (!kthread_should_stop()) {
+                set_current_state(TASK_INTERRUPTIBLE);
+                schedule();
+
+                /*
+                 * Do this periodically, like kupdated() did before.
+                 */
+                sync_supers();
+        }
+
+        return 0;
+}
+
+static void arm_supers_timer(void)
+{
+        unsigned long next;
+
+        next = msecs_to_jiffies(dirty_writeback_interval * 10) + jiffies;
+        mod_timer(&sync_supers_timer, round_jiffies_up(next));
+}
+
+static void sync_supers_timer_fn(unsigned long unused)
+{
+        wake_up_process(sync_supers_tsk);
+        arm_supers_timer();
+}
+
+static int bdi_forker_task(void *ptr)
+{
+        struct bdi_writeback *me = ptr;
+
+        bdi_task_init(me->bdi, me);
+
+        for (;;) {
+                struct backing_dev_info *bdi, *tmp;
+                struct bdi_writeback *wb;
+
+                /*
+                 * Temporary measure, we want to make sure we don't see
+                 * dirty data on the default backing_dev_info
+                 */
+                if (wb_has_dirty_io(me) || !list_empty(&me->bdi->work_list))
+                        wb_do_writeback(me, 0);
+
+                spin_lock_bh(&bdi_lock);
+
+                /*
+                 * Check if any existing bdi's have dirty data without
+                 * a thread registered. If so, set that up.
+                 */
+                list_for_each_entry_safe(bdi, tmp, &bdi_list, bdi_list) {
+                        if (bdi->wb.task)
+                                continue;
+                        if (list_empty(&bdi->work_list) &&
+                            !bdi_has_dirty_io(bdi))
+                                continue;
+
+                        bdi_add_default_flusher_task(bdi);
+                }
+
+                set_current_state(TASK_INTERRUPTIBLE);
+
+                if (list_empty(&bdi_pending_list)) {
+                        unsigned long wait;
+
+                        spin_unlock_bh(&bdi_lock);
+                        wait = msecs_to_jiffies(dirty_writeback_interval * 10);
+                        schedule_timeout(wait);
+                        try_to_freeze();
+                        continue;
+                }
+
+                __set_current_state(TASK_RUNNING);
+
+                /*
+                 * This is our real job - check for pending entries in
+                 * bdi_pending_list, and create the tasks that got added
+                 */
+                bdi = list_entry(bdi_pending_list.next, struct backing_dev_info,
+                                 bdi_list);
+                list_del_init(&bdi->bdi_list);
+                spin_unlock_bh(&bdi_lock);
+
+                wb = &bdi->wb;
+                wb->task = kthread_run(bdi_start_fn, wb, "flush-%s",
+                                        dev_name(bdi->dev));
+                /*
+                 * If task creation fails, then readd the bdi to
+                 * the pending list and force writeout of the bdi
+                 * from this forker thread. That will free some memory
+                 * and we can try again.
+                 */
+                if (IS_ERR(wb->task)) {
+                        wb->task = NULL;
+
+                        /*
+                         * Add this 'bdi' to the back, so we get
+                         * a chance to flush other bdi's to free
+                         * memory.
+                         */
+                        spin_lock_bh(&bdi_lock);
+                        list_add_tail(&bdi->bdi_list, &bdi_pending_list);
+                        spin_unlock_bh(&bdi_lock);
+
+                        bdi_flush_io(bdi);
+                }
+        }
+
+        return 0;
+}
+
+static void bdi_add_to_pending(struct rcu_head *head)
+{
+        struct backing_dev_info *bdi;
+
+        bdi = container_of(head, struct backing_dev_info, rcu_head);
+        INIT_LIST_HEAD(&bdi->bdi_list);
+
+        spin_lock(&bdi_lock);
+        list_add_tail(&bdi->bdi_list, &bdi_pending_list);
+        spin_unlock(&bdi_lock);
+
+        /*
+         * We are now on the pending list, wake up bdi_forker_task()
+         * to finish the job and add us back to the active bdi_list
+         */
+        wake_up_process(default_backing_dev_info.wb.task);
+}
+
+/*
+ * Add the default flusher task that gets created for any bdi
+ * that has dirty data pending writeout
+ */
+void static bdi_add_default_flusher_task(struct backing_dev_info *bdi)
+{
+        if (!bdi_cap_writeback_dirty(bdi))
+                return;
+
+        if (WARN_ON(!test_bit(BDI_registered, &bdi->state))) {
+                printk(KERN_ERR "bdi %p/%s is not registered!\n",
+                                                        bdi, bdi->name);
+                return;
+        }
+
+        /*
+         * Check with the helper whether to proceed adding a task. Will only
+         * abort if we two or more simultanous calls to
+         * bdi_add_default_flusher_task() occured, further additions will block
+         * waiting for previous additions to finish.
+         */
+        if (!test_and_set_bit(BDI_pending, &bdi->state)) {
+                list_del_rcu(&bdi->bdi_list);
+
+                /*
+                 * We must wait for the current RCU period to end before
+                 * moving to the pending list. So schedule that operation
+                 * from an RCU callback.
+                 */
+                call_rcu(&bdi->rcu_head, bdi_add_to_pending);
+        }
+}
+
+/*
+ * Remove bdi from bdi_list, and ensure that it is no longer visible
+ */
+static void bdi_remove_from_list(struct backing_dev_info *bdi)
+{
+        spin_lock_bh(&bdi_lock);
+        list_del_rcu(&bdi->bdi_list);
+        spin_unlock_bh(&bdi_lock);
+
+        synchronize_rcu();
+}
+
 int bdi_register(struct backing_dev_info *bdi, struct device *parent,
                 const char *fmt, ...)
 {
@@ -211,9 +543,33 @@ int bdi_register(struct backing_dev_info *bdi, struct device *parent,
                 goto exit;
         }
 
+        spin_lock_bh(&bdi_lock);
+        list_add_tail_rcu(&bdi->bdi_list, &bdi_list);
+        spin_unlock_bh(&bdi_lock);
+
         bdi->dev = dev;
-        bdi_debug_register(bdi, dev_name(dev));
 
+        /*
+         * Just start the forker thread for our default backing_dev_info,
+         * and add other bdi's to the list. They will get a thread created
+         * on-demand when they need it.
+         */
+        if (bdi_cap_flush_forker(bdi)) {
+                struct bdi_writeback *wb = &bdi->wb;
+
+                wb->task = kthread_run(bdi_forker_task, wb, "bdi-%s",
+                                                dev_name(dev));
+                if (IS_ERR(wb->task)) {
+                        wb->task = NULL;
+                        ret = -ENOMEM;
+
+                        bdi_remove_from_list(bdi);
+                        goto exit;
+                }
+        }
+
+        bdi_debug_register(bdi, dev_name(dev));
+        set_bit(BDI_registered, &bdi->state);
 exit:
         return ret;
 }
@@ -225,9 +581,61 @@ int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev)
 }
 EXPORT_SYMBOL(bdi_register_dev);
 
+/*
+ * Remove bdi from the global list and shutdown any threads we have running
+ */
+static void bdi_wb_shutdown(struct backing_dev_info *bdi)
+{
+        struct bdi_writeback *wb;
+
+        if (!bdi_cap_writeback_dirty(bdi))
+                return;
+
+        /*
+         * If setup is pending, wait for that to complete first
+         */
+        wait_on_bit(&bdi->state, BDI_pending, bdi_sched_wait,
+                        TASK_UNINTERRUPTIBLE);
+
+        /*
+         * Make sure nobody finds us on the bdi_list anymore
+         */
+        bdi_remove_from_list(bdi);
+
+        /*
+         * Finally, kill the kernel threads. We don't need to be RCU
+         * safe anymore, since the bdi is gone from visibility. Force
+         * unfreeze of the thread before calling kthread_stop(), otherwise
+         * it would never exet if it is currently stuck in the refrigerator.
+         */
+        list_for_each_entry(wb, &bdi->wb_list, list) {
+                thaw_process(wb->task);
+                kthread_stop(wb->task);
+        }
+}
+
+/*
+ * This bdi is going away now, make sure that no super_blocks point to it
+ */
+static void bdi_prune_sb(struct backing_dev_info *bdi)
+{
+        struct super_block *sb;
+
+        spin_lock(&sb_lock);
+        list_for_each_entry(sb, &super_blocks, s_list) {
+                if (sb->s_bdi == bdi)
+                        sb->s_bdi = NULL;
+        }
+        spin_unlock(&sb_lock);
+}
+
 void bdi_unregister(struct backing_dev_info *bdi)
 {
         if (bdi->dev) {
+                bdi_prune_sb(bdi);
+
+                if (!bdi_cap_flush_forker(bdi))
+                        bdi_wb_shutdown(bdi);
                 bdi_debug_unregister(bdi);
                 device_unregister(bdi->dev);
                 bdi->dev = NULL;
@@ -237,14 +645,26 @@ EXPORT_SYMBOL(bdi_unregister);
 
 int bdi_init(struct backing_dev_info *bdi)
 {
-        int i;
-        int err;
+        int i, err;
 
         bdi->dev = NULL;
 
         bdi->min_ratio = 0;
         bdi->max_ratio = 100;
         bdi->max_prop_frac = PROP_FRAC_BASE;
+        spin_lock_init(&bdi->wb_lock);
+        INIT_RCU_HEAD(&bdi->rcu_head);
+        INIT_LIST_HEAD(&bdi->bdi_list);
+        INIT_LIST_HEAD(&bdi->wb_list);
+        INIT_LIST_HEAD(&bdi->work_list);
+
+        bdi_wb_init(&bdi->wb, bdi);
+
+        /*
+         * Just one thread support for now, hard code mask and count
+         */
+        bdi->wb_mask = 1;
+        bdi->wb_cnt = 1;
 
         for (i = 0; i < NR_BDI_STAT_ITEMS; i++) {
                 err = percpu_counter_init(&bdi->bdi_stat[i], 0);
@@ -269,6 +689,20 @@ void bdi_destroy(struct backing_dev_info *bdi)
 {
         int i;
 
+        /*
+         * Splice our entries to the default_backing_dev_info, if this
+         * bdi disappears
+         */
+        if (bdi_has_dirty_io(bdi)) {
+                struct bdi_writeback *dst = &default_backing_dev_info.wb;
+
+                spin_lock(&inode_lock);
+                list_splice(&bdi->wb.b_dirty, &dst->b_dirty);
+                list_splice(&bdi->wb.b_io, &dst->b_io);
+                list_splice(&bdi->wb.b_more_io, &dst->b_more_io);
+                spin_unlock(&inode_lock);
+        }
+
         bdi_unregister(bdi);
 
         for (i = 0; i < NR_BDI_STAT_ITEMS; i++)
@@ -283,13 +717,12 @@ static wait_queue_head_t congestion_wqh[2] = {
                 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1])
         };
 
-
-void clear_bdi_congested(struct backing_dev_info *bdi, int rw)
+void clear_bdi_congested(struct backing_dev_info *bdi, int sync)
 {
         enum bdi_state bit;
-        wait_queue_head_t *wqh = &congestion_wqh[rw];
+        wait_queue_head_t *wqh = &congestion_wqh[sync];
 
-        bit = (rw == WRITE) ? BDI_write_congested : BDI_read_congested;
+        bit = sync ? BDI_sync_congested : BDI_async_congested;
         clear_bit(bit, &bdi->state);
         smp_mb__after_clear_bit();
         if (waitqueue_active(wqh))
@@ -297,29 +730,29 @@ void clear_bdi_congested(struct backing_dev_info *bdi, int rw)
 }
 EXPORT_SYMBOL(clear_bdi_congested);
 
-void set_bdi_congested(struct backing_dev_info *bdi, int rw)
+void set_bdi_congested(struct backing_dev_info *bdi, int sync)
 {
         enum bdi_state bit;
 
-        bit = (rw == WRITE) ? BDI_write_congested : BDI_read_congested;
+        bit = sync ? BDI_sync_congested : BDI_async_congested;
         set_bit(bit, &bdi->state);
 }
 EXPORT_SYMBOL(set_bdi_congested);
 
 /**
  * congestion_wait - wait for a backing_dev to become uncongested
- * @rw: READ or WRITE
+ * @sync: SYNC or ASYNC IO
  * @timeout: timeout in jiffies
  *
  * Waits for up to @timeout jiffies for a backing_dev (any backing_dev) to exit
  * write congestion.  If no backing_devs are congested then just wait for the
  * next write to be completed.
  */
-long congestion_wait(int rw, long timeout)
+long congestion_wait(int sync, long timeout)
 {
         long ret;
         DEFINE_WAIT(wait);
-        wait_queue_head_t *wqh = &congestion_wqh[rw];
+        wait_queue_head_t *wqh = &congestion_wqh[sync];
 
         prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
         ret = io_schedule_timeout(timeout);
diff --git a/mm/bootmem.c b/mm/bootmem.c
index daf92713f7de..d1dc23cc7f10 100644
--- a/mm/bootmem.c
+++ b/mm/bootmem.c
@@ -12,6 +12,7 @@
 #include <linux/pfn.h>
 #include <linux/bootmem.h>
 #include <linux/module.h>
+#include <linux/kmemleak.h>
 
 #include <asm/bug.h>
 #include <asm/io.h>
@@ -142,6 +143,30 @@ unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
         return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
 }
 
+/*
+ * free_bootmem_late - free bootmem pages directly to page allocator
+ * @addr: starting address of the range
+ * @size: size of the range in bytes
+ *
+ * This is only useful when the bootmem allocator has already been torn
+ * down, but we are still initializing the system.  Pages are given directly
+ * to the page allocator, no bootmem metadata is updated because it is gone.
+ */
+void __init free_bootmem_late(unsigned long addr, unsigned long size)
+{
+        unsigned long cursor, end;
+
+        kmemleak_free_part(__va(addr), size);
+
+        cursor = PFN_UP(addr);
+        end = PFN_DOWN(addr + size);
+
+        for (; cursor < end; cursor++) {
+                __free_pages_bootmem(pfn_to_page(cursor), 0);
+                totalram_pages++;
+        }
+}
+
 static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
 {
         int aligned;
@@ -335,6 +360,8 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
 {
         unsigned long start, end;
 
+        kmemleak_free_part(__va(physaddr), size);
+
         start = PFN_UP(physaddr);
         end = PFN_DOWN(physaddr + size);
 
@@ -354,6 +381,8 @@ void __init free_bootmem(unsigned long addr, unsigned long size)
 {
         unsigned long start, end;
 
+        kmemleak_free_part(__va(addr), size);
+
         start = PFN_UP(addr);
         end = PFN_DOWN(addr + size);
 
@@ -516,6 +545,11 @@ find_block:
                 region = phys_to_virt(PFN_PHYS(bdata->node_min_pfn) +
                                 start_off);
                 memset(region, 0, size);
+                /*
+                 * The min_count is set to 0 so that bootmem allocated blocks
+                 * are never reported as leaks.
+                 */
+                kmemleak_alloc(region, size, 0, 0);
                 return region;
         }
 
@@ -532,12 +566,19 @@ static void * __init alloc_arch_preferred_bootmem(bootmem_data_t *bdata,
                                         unsigned long size, unsigned long align,
                                         unsigned long goal, unsigned long limit)
 {
-#ifdef CONFIG_HAVE_ARCH_BOOTMEM
-        bootmem_data_t *p_bdata;
+        if (WARN_ON_ONCE(slab_is_available()))
+                return kzalloc(size, GFP_NOWAIT);
 
-        p_bdata = bootmem_arch_preferred_node(bdata, size, align, goal, limit);
-        if (p_bdata)
-                return alloc_bootmem_core(p_bdata, size, align, goal, limit);
+#ifdef CONFIG_HAVE_ARCH_BOOTMEM
+        {
+                bootmem_data_t *p_bdata;
+
+                p_bdata = bootmem_arch_preferred_node(bdata, size, align,
+                                                        goal, limit);
+                if (p_bdata)
+                        return alloc_bootmem_core(p_bdata, size, align,
+                                                        goal, limit);
+        }
 #endif
         return NULL;
 }
@@ -662,6 +703,9 @@ static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
 void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
                                    unsigned long align, unsigned long goal)
 {
+        if (WARN_ON_ONCE(slab_is_available()))
+                return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+
         return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
 }
 
@@ -693,6 +737,9 @@ void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
 {
         void *ptr;
 
+        if (WARN_ON_ONCE(slab_is_available()))
+                return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+
         ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0);
         if (ptr)
                 return ptr;
@@ -745,6 +792,9 @@ void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
 void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
                                        unsigned long align, unsigned long goal)
 {
+        if (WARN_ON_ONCE(slab_is_available()))
+                return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+
         return ___alloc_bootmem_node(pgdat->bdata, size, align,
                                 goal, ARCH_LOW_ADDRESS_LIMIT);
 }
diff --git a/mm/bounce.c b/mm/bounce.c
index e590272fe7a8..a2b76a588e34 100644
--- a/mm/bounce.c
+++ b/mm/bounce.c
@@ -13,17 +13,15 @@
 #include <linux/init.h>
 #include <linux/hash.h>
 #include <linux/highmem.h>
-#include <linux/blktrace_api.h>
-#include <trace/block.h>
 #include <asm/tlbflush.h>
 
+#include <trace/events/block.h>
+
 #define POOL_SIZE       64
 #define ISA_POOL_SIZE   16
 
 static mempool_t *page_pool, *isa_page_pool;
 
-DEFINE_TRACE(block_bio_bounce);
-
 #ifdef CONFIG_HIGHMEM
 static __init int init_emergency_pool(void)
 {
@@ -192,7 +190,7 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
                 /*
                  * is destination page below bounce pfn?
                  */
-                if (page_to_pfn(page) <= q->bounce_pfn)
+                if (page_to_pfn(page) <= queue_bounce_pfn(q))
                         continue;
 
                 /*
@@ -284,7 +282,7 @@ void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
          * don't waste time iterating over bio segments
          */
         if (!(q->bounce_gfp & GFP_DMA)) {
-                if (q->bounce_pfn >= blk_max_pfn)
+                if (queue_bounce_pfn(q) >= blk_max_pfn)
                         return;
                 pool = page_pool;
         } else {
diff --git a/mm/dmapool.c b/mm/dmapool.c
index b1f0885dda22..3df063706f53 100644
--- a/mm/dmapool.c
+++ b/mm/dmapool.c
@@ -86,10 +86,12 @@ show_pools(struct device *dev, struct device_attribute *attr, char *buf)
                 unsigned pages = 0;
                 unsigned blocks = 0;
 
+                spin_lock_irq(&pool->lock);
                 list_for_each_entry(page, &pool->page_list, page_list) {
                         pages++;
                         blocks += page->in_use;
                 }
+                spin_unlock_irq(&pool->lock);
 
                 /* per-pool info, no real statistics yet */
                 temp = scnprintf(next, size, "%-16s %4u %4Zu %4Zu %2u\n",
diff --git a/mm/fadvise.c b/mm/fadvise.c
index 54a0f8040afa..e43359214f6f 100644
--- a/mm/fadvise.c
+++ b/mm/fadvise.c
@@ -101,7 +101,7 @@ SYSCALL_DEFINE(fadvise64_64)(int fd, loff_t offset, loff_t len, int advice)
                 
                 ret = force_page_cache_readahead(mapping, file,
                                 start_index,
-                                max_sane_readahead(nrpages));
+                                nrpages);
                 if (ret > 0)
                         ret = 0;
                 break;
diff --git a/mm/failslab.c b/mm/failslab.c
index 7c6ea6493f80..9339de5f0a91 100644
--- a/mm/failslab.c
+++ b/mm/failslab.c
@@ -1,4 +1,5 @@
 #include <linux/fault-inject.h>
+#include <linux/gfp.h>
 
 static struct {
         struct fault_attr attr;
diff --git a/mm/filemap.c b/mm/filemap.c
index 126d3973b3d1..ef169f37156d 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -39,11 +39,10 @@
 /*
  * FIXME: remove all knowledge of the buffer layer from the core VM
  */
-#include <linux/buffer_head.h> /* for generic_osync_inode */
+#include <linux/buffer_head.h> /* for try_to_free_buffers */
 
 #include <asm/mman.h>
 
-
 /*
  * Shared mappings implemented 30.11.1994. It's not fully working yet,
  * though.
@@ -59,7 +58,7 @@
 /*
  * Lock ordering:
  *
- *  ->i_mmap_lock               (vmtruncate)
+ *  ->i_mmap_lock               (truncate_pagecache)
  *    ->private_lock            (__free_pte->__set_page_dirty_buffers)
  *      ->swap_lock             (exclusive_swap_page, others)
  *        ->mapping->tree_lock
@@ -105,6 +104,10 @@
  *
  *  ->task->proc_lock
  *    ->dcache_lock             (proc_pid_lookup)
+ *
+ *  (code doesn't rely on that order, so you could switch it around)
+ *  ->tasklist_lock             (memory_failure, collect_procs_ao)
+ *    ->i_mmap_lock
  */
 
 /*
@@ -120,8 +123,9 @@ void __remove_from_page_cache(struct page *page)
         page->mapping = NULL;
         mapping->nrpages--;
         __dec_zone_page_state(page, NR_FILE_PAGES);
+        if (PageSwapBacked(page))
+                __dec_zone_page_state(page, NR_SHMEM);
         BUG_ON(page_mapped(page));
-        mem_cgroup_uncharge_cache_page(page);
 
         /*
          * Some filesystems seem to re-dirty the page even after
@@ -145,6 +149,7 @@ void remove_from_page_cache(struct page *page)
         spin_lock_irq(&mapping->tree_lock);
         __remove_from_page_cache(page);
         spin_unlock_irq(&mapping->tree_lock);
+        mem_cgroup_uncharge_cache_page(page);
 }
 
 static int sync_page(void *word)
@@ -307,68 +312,24 @@ int wait_on_page_writeback_range(struct address_space *mapping,
 }
 
 /**
- * sync_page_range - write and wait on all pages in the passed range
- * @inode:      target inode
- * @mapping:    target address_space
- * @pos:        beginning offset in pages to write
- * @count:      number of bytes to write
- *
- * Write and wait upon all the pages in the passed range.  This is a "data
- * integrity" operation.  It waits upon in-flight writeout before starting and
- * waiting upon new writeout.  If there was an IO error, return it.
+ * filemap_fdatawait_range - wait for all under-writeback pages to complete in a given range
+ * @mapping: address space structure to wait for
+ * @start:      offset in bytes where the range starts
+ * @end:        offset in bytes where the range ends (inclusive)
  *
- * We need to re-take i_mutex during the generic_osync_inode list walk because
- * it is otherwise livelockable.
- */
-int sync_page_range(struct inode *inode, struct address_space *mapping,
-                        loff_t pos, loff_t count)
-{
-        pgoff_t start = pos >> PAGE_CACHE_SHIFT;
-        pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT;
-        int ret;
-
-        if (!mapping_cap_writeback_dirty(mapping) || !count)
-                return 0;
-        ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1);
-        if (ret == 0) {
-                mutex_lock(&inode->i_mutex);
-                ret = generic_osync_inode(inode, mapping, OSYNC_METADATA);
-                mutex_unlock(&inode->i_mutex);
-        }
-        if (ret == 0)
-                ret = wait_on_page_writeback_range(mapping, start, end);
-        return ret;
-}
-EXPORT_SYMBOL(sync_page_range);
-
-/**
- * sync_page_range_nolock - write & wait on all pages in the passed range without locking
- * @inode:      target inode
- * @mapping:    target address_space
- * @pos:        beginning offset in pages to write
- * @count:      number of bytes to write
+ * Walk the list of under-writeback pages of the given address space
+ * in the given range and wait for all of them.
  *
- * Note: Holding i_mutex across sync_page_range_nolock() is not a good idea
- * as it forces O_SYNC writers to different parts of the same file
- * to be serialised right until io completion.
+ * This is just a simple wrapper so that callers don't have to convert offsets
+ * to page indexes themselves
  */
-int sync_page_range_nolock(struct inode *inode, struct address_space *mapping,
-                           loff_t pos, loff_t count)
+int filemap_fdatawait_range(struct address_space *mapping, loff_t start,
+                            loff_t end)
 {
-        pgoff_t start = pos >> PAGE_CACHE_SHIFT;
-        pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT;
-        int ret;
-
-        if (!mapping_cap_writeback_dirty(mapping) || !count)
-                return 0;
-        ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1);
-        if (ret == 0)
-                ret = generic_osync_inode(inode, mapping, OSYNC_METADATA);
-        if (ret == 0)
-                ret = wait_on_page_writeback_range(mapping, start, end);
-        return ret;
+        return wait_on_page_writeback_range(mapping, start >> PAGE_CACHE_SHIFT,
+                                            end >> PAGE_CACHE_SHIFT);
 }
-EXPORT_SYMBOL(sync_page_range_nolock);
+EXPORT_SYMBOL(filemap_fdatawait_range);
 
 /**
  * filemap_fdatawait - wait for all under-writeback pages to complete
@@ -441,6 +402,7 @@ int filemap_write_and_wait_range(struct address_space *mapping,
         }
         return err;
 }
+EXPORT_SYMBOL(filemap_write_and_wait_range);
 
 /**
  * add_to_page_cache_locked - add a locked page to the pagecache
@@ -475,13 +437,15 @@ int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
                 if (likely(!error)) {
                         mapping->nrpages++;
                         __inc_zone_page_state(page, NR_FILE_PAGES);
+                        if (PageSwapBacked(page))
+                                __inc_zone_page_state(page, NR_SHMEM);
+                        spin_unlock_irq(&mapping->tree_lock);
                 } else {
                         page->mapping = NULL;
+                        spin_unlock_irq(&mapping->tree_lock);
                         mem_cgroup_uncharge_cache_page(page);
                         page_cache_release(page);
                 }
-
-                spin_unlock_irq(&mapping->tree_lock);
                 radix_tree_preload_end();
         } else
                 mem_cgroup_uncharge_cache_page(page);
@@ -513,13 +477,14 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
         }
         return ret;
 }
+EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
 
 #ifdef CONFIG_NUMA
 struct page *__page_cache_alloc(gfp_t gfp)
 {
         if (cpuset_do_page_mem_spread()) {
                 int n = cpuset_mem_spread_node();
-                return alloc_pages_node(n, gfp, 0);
+                return alloc_pages_exact_node(n, gfp, 0);
         }
         return alloc_pages(gfp, 0);
 }
@@ -565,6 +530,24 @@ void wait_on_page_bit(struct page *page, int bit_nr)
 EXPORT_SYMBOL(wait_on_page_bit);
 
 /**
+ * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
+ * @page: Page defining the wait queue of interest
+ * @waiter: Waiter to add to the queue
+ *
+ * Add an arbitrary @waiter to the wait queue for the nominated @page.
+ */
+void add_page_wait_queue(struct page *page, wait_queue_t *waiter)
+{
+        wait_queue_head_t *q = page_waitqueue(page);
+        unsigned long flags;
+
+        spin_lock_irqsave(&q->lock, flags);
+        __add_wait_queue(q, waiter);
+        spin_unlock_irqrestore(&q->lock, flags);
+}
+EXPORT_SYMBOL_GPL(add_page_wait_queue);
+
+/**
  * unlock_page - unlock a locked page
  * @page: the page
  *
@@ -627,6 +610,7 @@ int __lock_page_killable(struct page *page)
         return __wait_on_bit_lock(page_waitqueue(page), &wait,
                                         sync_page_killable, TASK_KILLABLE);
 }
+EXPORT_SYMBOL_GPL(__lock_page_killable);
 
 /**
  * __lock_page_nosync - get a lock on the page, without calling sync_page()
@@ -983,9 +967,6 @@ EXPORT_SYMBOL(grab_cache_page_nowait);
 static void shrink_readahead_size_eio(struct file *filp,
                                         struct file_ra_state *ra)
 {
-        if (!ra->ra_pages)
-                return;
-
         ra->ra_pages /= 4;
 }
 
@@ -1369,8 +1350,7 @@ do_readahead(struct address_space *mapping, struct file *filp,
         if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage)
                 return -EINVAL;
 
-        force_page_cache_readahead(mapping, filp, index,
-                                        max_sane_readahead(nr));
+        force_page_cache_readahead(mapping, filp, index, nr);
         return 0;
 }
 
@@ -1436,6 +1416,73 @@ static int page_cache_read(struct file *file, pgoff_t offset)
 
 #define MMAP_LOTSAMISS  (100)
 
+/*
+ * Synchronous readahead happens when we don't even find
+ * a page in the page cache at all.
+ */
+static void do_sync_mmap_readahead(struct vm_area_struct *vma,
+                                   struct file_ra_state *ra,
+                                   struct file *file,
+                                   pgoff_t offset)
+{
+        unsigned long ra_pages;
+        struct address_space *mapping = file->f_mapping;
+
+        /* If we don't want any read-ahead, don't bother */
+        if (VM_RandomReadHint(vma))
+                return;
+
+        if (VM_SequentialReadHint(vma) ||
+                        offset - 1 == (ra->prev_pos >> PAGE_CACHE_SHIFT)) {
+                page_cache_sync_readahead(mapping, ra, file, offset,
+                                          ra->ra_pages);
+                return;
+        }
+
+        if (ra->mmap_miss < INT_MAX)
+                ra->mmap_miss++;
+
+        /*
+         * Do we miss much more than hit in this file? If so,
+         * stop bothering with read-ahead. It will only hurt.
+         */
+        if (ra->mmap_miss > MMAP_LOTSAMISS)
+                return;
+
+        /*
+         * mmap read-around
+         */
+        ra_pages = max_sane_readahead(ra->ra_pages);
+        if (ra_pages) {
+                ra->start = max_t(long, 0, offset - ra_pages/2);
+                ra->size = ra_pages;
+                ra->async_size = 0;
+                ra_submit(ra, mapping, file);
+        }
+}
+
+/*
+ * Asynchronous readahead happens when we find the page and PG_readahead,
+ * so we want to possibly extend the readahead further..
+ */
+static void do_async_mmap_readahead(struct vm_area_struct *vma,
+                                    struct file_ra_state *ra,
+                                    struct file *file,
+                                    struct page *page,
+                                    pgoff_t offset)
+{
+        struct address_space *mapping = file->f_mapping;
+
+        /* If we don't want any read-ahead, don't bother */
+        if (VM_RandomReadHint(vma))
+                return;
+        if (ra->mmap_miss > 0)
+                ra->mmap_miss--;
+        if (PageReadahead(page))
+                page_cache_async_readahead(mapping, ra, file,
+                                           page, offset, ra->ra_pages);
+}
+
 /**
  * filemap_fault - read in file data for page fault handling
  * @vma:        vma in which the fault was taken
@@ -1455,78 +1502,44 @@ int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
         struct address_space *mapping = file->f_mapping;
         struct file_ra_state *ra = &file->f_ra;
         struct inode *inode = mapping->host;
+        pgoff_t offset = vmf->pgoff;
         struct page *page;
         pgoff_t size;
-        int did_readaround = 0;
         int ret = 0;
 
         size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
-        if (vmf->pgoff >= size)
+        if (offset >= size)
                 return VM_FAULT_SIGBUS;
 
-        /* If we don't want any read-ahead, don't bother */
-        if (VM_RandomReadHint(vma))
-                goto no_cached_page;
-
         /*
          * Do we have something in the page cache already?
          */
-retry_find:
-        page = find_lock_page(mapping, vmf->pgoff);
-        /*
-         * For sequential accesses, we use the generic readahead logic.
-         */
-        if (VM_SequentialReadHint(vma)) {
-                if (!page) {
-                        page_cache_sync_readahead(mapping, ra, file,
-                                                           vmf->pgoff, 1);
-                        page = find_lock_page(mapping, vmf->pgoff);
-                        if (!page)
-                                goto no_cached_page;
-                }
-                if (PageReadahead(page)) {
-                        page_cache_async_readahead(mapping, ra, file, page,
-                                                           vmf->pgoff, 1);
-                }
-        }
-
-        if (!page) {
-                unsigned long ra_pages;
-
-                ra->mmap_miss++;
-
+        page = find_get_page(mapping, offset);
+        if (likely(page)) {
                 /*
-                 * Do we miss much more than hit in this file? If so,
-                 * stop bothering with read-ahead. It will only hurt.
+                 * We found the page, so try async readahead before
+                 * waiting for the lock.
                  */
-                if (ra->mmap_miss > MMAP_LOTSAMISS)
-                        goto no_cached_page;
+                do_async_mmap_readahead(vma, ra, file, page, offset);
+                lock_page(page);
 
-                /*
-                 * To keep the pgmajfault counter straight, we need to
-                 * check did_readaround, as this is an inner loop.
-                 */
-                if (!did_readaround) {
-                        ret = VM_FAULT_MAJOR;
-                        count_vm_event(PGMAJFAULT);
-                }
-                did_readaround = 1;
-                ra_pages = max_sane_readahead(file->f_ra.ra_pages);
-                if (ra_pages) {
-                        pgoff_t start = 0;
-
-                        if (vmf->pgoff > ra_pages / 2)
-                                start = vmf->pgoff - ra_pages / 2;
-                        do_page_cache_readahead(mapping, file, start, ra_pages);
+                /* Did it get truncated? */
+                if (unlikely(page->mapping != mapping)) {
+                        unlock_page(page);
+                        put_page(page);
+                        goto no_cached_page;
                 }
-                page = find_lock_page(mapping, vmf->pgoff);
+        } else {
+                /* No page in the page cache at all */
+                do_sync_mmap_readahead(vma, ra, file, offset);
+                count_vm_event(PGMAJFAULT);
+                ret = VM_FAULT_MAJOR;
+retry_find:
+                page = find_lock_page(mapping, offset);
                 if (!page)
                         goto no_cached_page;
         }
 
-        if (!did_readaround)
-                ra->mmap_miss--;
-
         /*
          * We have a locked page in the page cache, now we need to check
          * that it's up-to-date. If not, it is going to be due to an error.
@@ -1534,18 +1547,18 @@ retry_find:
         if (unlikely(!PageUptodate(page)))
                 goto page_not_uptodate;
 
-        /* Must recheck i_size under page lock */
+        /*
+         * Found the page and have a reference on it.
+         * We must recheck i_size under page lock.
+         */
         size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
-        if (unlikely(vmf->pgoff >= size)) {
+        if (unlikely(offset >= size)) {
                 unlock_page(page);
                 page_cache_release(page);
                 return VM_FAULT_SIGBUS;
         }
 
-        /*
-         * Found the page and have a reference on it.
-         */
-        ra->prev_pos = (loff_t)page->index << PAGE_CACHE_SHIFT;
+        ra->prev_pos = (loff_t)offset << PAGE_CACHE_SHIFT;
         vmf->page = page;
         return ret | VM_FAULT_LOCKED;
 
@@ -1554,7 +1567,7 @@ no_cached_page:
          * We're only likely to ever get here if MADV_RANDOM is in
          * effect.
          */
-        error = page_cache_read(file, vmf->pgoff);
+        error = page_cache_read(file, offset);
 
         /*
          * The page we want has now been added to the page cache.
@@ -1574,12 +1587,6 @@ no_cached_page:
         return VM_FAULT_SIGBUS;
 
 page_not_uptodate:
-        /* IO error path */
-        if (!did_readaround) {
-                ret = VM_FAULT_MAJOR;
-                count_vm_event(PGMAJFAULT);
-        }
-
         /*
          * Umm, take care of errors if the page isn't up-to-date.
          * Try to re-read it _once_. We do this synchronously,
@@ -1604,7 +1611,7 @@ page_not_uptodate:
 }
 EXPORT_SYMBOL(filemap_fault);
 
-struct vm_operations_struct generic_file_vm_ops = {
+const struct vm_operations_struct generic_file_vm_ops = {
         .fault          = filemap_fault,
 };
 
@@ -2123,20 +2130,7 @@ generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
                 }
                 *ppos = end;
         }
-
-        /*
-         * Sync the fs metadata but not the minor inode changes and
-         * of course not the data as we did direct DMA for the IO.
-         * i_mutex is held, which protects generic_osync_inode() from
-         * livelocking.  AIO O_DIRECT ops attempt to sync metadata here.
-         */
 out:
-        if ((written >= 0 || written == -EIOCBQUEUED) &&
-            ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
-                int err = generic_osync_inode(inode, mapping, OSYNC_METADATA);
-                if (err < 0)
-                        written = err;
-        }
         return written;
 }
 EXPORT_SYMBOL(generic_file_direct_write);
@@ -2228,6 +2222,7 @@ again:
                 pagefault_enable();
                 flush_dcache_page(page);
 
+                mark_page_accessed(page);
                 status = a_ops->write_end(file, mapping, pos, bytes, copied,
                                                 page, fsdata);
                 if (unlikely(status < 0))
@@ -2267,8 +2262,6 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
 {
         struct file *file = iocb->ki_filp;
         struct address_space *mapping = file->f_mapping;
-        const struct address_space_operations *a_ops = mapping->a_ops;
-        struct inode *inode = mapping->host;
         ssize_t status;
         struct iov_iter i;
 
@@ -2278,16 +2271,6 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
         if (likely(status >= 0)) {
                 written += status;
                 *ppos = pos + status;
-
-                /*
-                 * For now, when the user asks for O_SYNC, we'll actually give
-                 * O_DSYNC
-                 */
-                if (unlikely((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
-                        if (!a_ops->writepage || !is_sync_kiocb(iocb))
-                                status = generic_osync_inode(inode, mapping,
-                                                OSYNC_METADATA|OSYNC_DATA);
-                }
         }
         
         /*
@@ -2303,9 +2286,27 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
 }
 EXPORT_SYMBOL(generic_file_buffered_write);
 
-static ssize_t
-__generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
-                                unsigned long nr_segs, loff_t *ppos)
+/**
+ * __generic_file_aio_write - write data to a file
+ * @iocb:       IO state structure (file, offset, etc.)
+ * @iov:        vector with data to write
+ * @nr_segs:    number of segments in the vector
+ * @ppos:       position where to write
+ *
+ * This function does all the work needed for actually writing data to a
+ * file. It does all basic checks, removes SUID from the file, updates
+ * modification times and calls proper subroutines depending on whether we
+ * do direct IO or a standard buffered write.
+ *
+ * It expects i_mutex to be grabbed unless we work on a block device or similar
+ * object which does not need locking at all.
+ *
+ * This function does *not* take care of syncing data in case of O_SYNC write.
+ * A caller has to handle it. This is mainly due to the fact that we want to
+ * avoid syncing under i_mutex.
+ */
+ssize_t __generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
+                                 unsigned long nr_segs, loff_t *ppos)
 {
         struct file *file = iocb->ki_filp;
         struct address_space * mapping = file->f_mapping;
@@ -2402,51 +2403,37 @@ out:
         current->backing_dev_info = NULL;
         return written ? written : err;
 }
+EXPORT_SYMBOL(__generic_file_aio_write);
 
-ssize_t generic_file_aio_write_nolock(struct kiocb *iocb,
-                const struct iovec *iov, unsigned long nr_segs, loff_t pos)
-{
-        struct file *file = iocb->ki_filp;
-        struct address_space *mapping = file->f_mapping;
-        struct inode *inode = mapping->host;
-        ssize_t ret;
-
-        BUG_ON(iocb->ki_pos != pos);
-
-        ret = __generic_file_aio_write_nolock(iocb, iov, nr_segs,
-                        &iocb->ki_pos);
-
-        if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
-                ssize_t err;
-
-                err = sync_page_range_nolock(inode, mapping, pos, ret);
-                if (err < 0)
-                        ret = err;
-        }
-        return ret;
-}
-EXPORT_SYMBOL(generic_file_aio_write_nolock);
-
+/**
+ * generic_file_aio_write - write data to a file
+ * @iocb:       IO state structure
+ * @iov:        vector with data to write
+ * @nr_segs:    number of segments in the vector
+ * @pos:        position in file where to write
+ *
+ * This is a wrapper around __generic_file_aio_write() to be used by most
+ * filesystems. It takes care of syncing the file in case of O_SYNC file
+ * and acquires i_mutex as needed.
+ */
 ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
                 unsigned long nr_segs, loff_t pos)
 {
         struct file *file = iocb->ki_filp;
-        struct address_space *mapping = file->f_mapping;
-        struct inode *inode = mapping->host;
+        struct inode *inode = file->f_mapping->host;
         ssize_t ret;
 
         BUG_ON(iocb->ki_pos != pos);
 
         mutex_lock(&inode->i_mutex);
-        ret = __generic_file_aio_write_nolock(iocb, iov, nr_segs,
-                        &iocb->ki_pos);
+        ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos);
         mutex_unlock(&inode->i_mutex);
 
-        if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
+        if (ret > 0 || ret == -EIOCBQUEUED) {
                 ssize_t err;
 
-                err = sync_page_range(inode, mapping, pos, ret);
-                if (err < 0)
+                err = generic_write_sync(file, pos, ret);
+                if (err < 0 && ret > 0)
                         ret = err;
         }
         return ret;
@@ -2463,6 +2450,9 @@ EXPORT_SYMBOL(generic_file_aio_write);
  * (presumably at page->private).  If the release was successful, return `1'.
  * Otherwise return zero.
  *
+ * This may also be called if PG_fscache is set on a page, indicating that the
+ * page is known to the local caching routines.
+ *
  * The @gfp_mask argument specifies whether I/O may be performed to release
  * this page (__GFP_IO), and whether the call may block (__GFP_WAIT & __GFP_FS).
  *
diff --git a/mm/filemap_xip.c b/mm/filemap_xip.c
index 0c04615651b7..1888b2d71bb8 100644
--- a/mm/filemap_xip.c
+++ b/mm/filemap_xip.c
@@ -89,8 +89,8 @@ do_xip_mapping_read(struct address_space *mapping,
                         }
                 }
                 nr = nr - offset;
-                if (nr > len)
-                        nr = len;
+                if (nr > len - copied)
+                        nr = len - copied;
 
                 error = mapping->a_ops->get_xip_mem(mapping, index, 0,
                                                         &xip_mem, &xip_pfn);
@@ -296,7 +296,7 @@ out:
         }
 }
 
-static struct vm_operations_struct xip_file_vm_ops = {
+static const struct vm_operations_struct xip_file_vm_ops = {
         .fault  = xip_file_fault,
 };
 
diff --git a/mm/highmem.c b/mm/highmem.c
index 68eb1d9b63fa..9c1e627f282e 100644
--- a/mm/highmem.c
+++ b/mm/highmem.c
@@ -26,7 +26,6 @@
 #include <linux/init.h>
 #include <linux/hash.h>
 #include <linux/highmem.h>
-#include <linux/blktrace_api.h>
 #include <asm/tlbflush.h>
 
 /*
@@ -427,16 +426,21 @@ void __init page_address_init(void)
 
 void debug_kmap_atomic(enum km_type type)
 {
-        static unsigned warn_count = 10;
+        static int warn_count = 10;
 
-        if (unlikely(warn_count == 0))
+        if (unlikely(warn_count < 0))
                 return;
 
         if (unlikely(in_interrupt())) {
-                if (in_irq()) {
+                if (in_nmi()) {
+                        if (type != KM_NMI && type != KM_NMI_PTE) {
+                                WARN_ON(1);
+                                warn_count--;
+                        }
+                } else if (in_irq()) {
                         if (type != KM_IRQ0 && type != KM_IRQ1 &&
                             type != KM_BIO_SRC_IRQ && type != KM_BIO_DST_IRQ &&
-                            type != KM_BOUNCE_READ) {
+                            type != KM_BOUNCE_READ && type != KM_IRQ_PTE) {
                                 WARN_ON(1);
                                 warn_count--;
                         }
@@ -453,7 +457,9 @@ void debug_kmap_atomic(enum km_type type)
         }
 
         if (type == KM_IRQ0 || type == KM_IRQ1 || type == KM_BOUNCE_READ ||
-                        type == KM_BIO_SRC_IRQ || type == KM_BIO_DST_IRQ) {
+                        type == KM_BIO_SRC_IRQ || type == KM_BIO_DST_IRQ ||
+                        type == KM_IRQ_PTE || type == KM_NMI ||
+                        type == KM_NMI_PTE ) {
                 if (!irqs_disabled()) {
                         WARN_ON(1);
                         warn_count--;
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 28c655ba9353..5d7601b02874 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -234,6 +234,7 @@ unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
 
         return 1UL << (hstate->order + PAGE_SHIFT);
 }
+EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
 
 /*
  * Return the page size being used by the MMU to back a VMA. In the majority
@@ -316,7 +317,7 @@ static void resv_map_release(struct kref *ref)
 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
 {
         VM_BUG_ON(!is_vm_hugetlb_page(vma));
-        if (!(vma->vm_flags & VM_SHARED))
+        if (!(vma->vm_flags & VM_MAYSHARE))
                 return (struct resv_map *)(get_vma_private_data(vma) &
                                                         ~HPAGE_RESV_MASK);
         return NULL;
@@ -325,7 +326,7 @@ static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
 {
         VM_BUG_ON(!is_vm_hugetlb_page(vma));
-        VM_BUG_ON(vma->vm_flags & VM_SHARED);
+        VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
 
         set_vma_private_data(vma, (get_vma_private_data(vma) &
                                 HPAGE_RESV_MASK) | (unsigned long)map);
@@ -334,7 +335,7 @@ static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
 {
         VM_BUG_ON(!is_vm_hugetlb_page(vma));
-        VM_BUG_ON(vma->vm_flags & VM_SHARED);
+        VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
 
         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
 }
@@ -353,7 +354,7 @@ static void decrement_hugepage_resv_vma(struct hstate *h,
         if (vma->vm_flags & VM_NORESERVE)
                 return;
 
-        if (vma->vm_flags & VM_SHARED) {
+        if (vma->vm_flags & VM_MAYSHARE) {
                 /* Shared mappings always use reserves */
                 h->resv_huge_pages--;
         } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
@@ -369,14 +370,14 @@ static void decrement_hugepage_resv_vma(struct hstate *h,
 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
 {
         VM_BUG_ON(!is_vm_hugetlb_page(vma));
-        if (!(vma->vm_flags & VM_SHARED))
+        if (!(vma->vm_flags & VM_MAYSHARE))
                 vma->vm_private_data = (void *)0;
 }
 
 /* Returns true if the VMA has associated reserve pages */
 static int vma_has_reserves(struct vm_area_struct *vma)
 {
-        if (vma->vm_flags & VM_SHARED)
+        if (vma->vm_flags & VM_MAYSHARE)
                 return 1;
         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
                 return 1;
@@ -455,24 +456,6 @@ static void enqueue_huge_page(struct hstate *h, struct page *page)
         h->free_huge_pages_node[nid]++;
 }
 
-static struct page *dequeue_huge_page(struct hstate *h)
-{
-        int nid;
-        struct page *page = NULL;
-
-        for (nid = 0; nid < MAX_NUMNODES; ++nid) {
-                if (!list_empty(&h->hugepage_freelists[nid])) {
-                        page = list_entry(h->hugepage_freelists[nid].next,
-                                          struct page, lru);
-                        list_del(&page->lru);
-                        h->free_huge_pages--;
-                        h->free_huge_pages_node[nid]--;
-                        break;
-                }
-        }
-        return page;
-}
-
 static struct page *dequeue_huge_page_vma(struct hstate *h,
                                 struct vm_area_struct *vma,
                                 unsigned long address, int avoid_reserve)
@@ -578,41 +561,6 @@ static void free_huge_page(struct page *page)
                 hugetlb_put_quota(mapping, 1);
 }
 
-/*
- * Increment or decrement surplus_huge_pages.  Keep node-specific counters
- * balanced by operating on them in a round-robin fashion.
- * Returns 1 if an adjustment was made.
- */
-static int adjust_pool_surplus(struct hstate *h, int delta)
-{
-        static int prev_nid;
-        int nid = prev_nid;
-        int ret = 0;
-
-        VM_BUG_ON(delta != -1 && delta != 1);
-        do {
-                nid = next_node(nid, node_online_map);
-                if (nid == MAX_NUMNODES)
-                        nid = first_node(node_online_map);
-
-                /* To shrink on this node, there must be a surplus page */
-                if (delta < 0 && !h->surplus_huge_pages_node[nid])
-                        continue;
-                /* Surplus cannot exceed the total number of pages */
-                if (delta > 0 && h->surplus_huge_pages_node[nid] >=
-                                                h->nr_huge_pages_node[nid])
-                        continue;
-
-                h->surplus_huge_pages += delta;
-                h->surplus_huge_pages_node[nid] += delta;
-                ret = 1;
-                break;
-        } while (nid != prev_nid);
-
-        prev_nid = nid;
-        return ret;
-}
-
 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
 {
         set_compound_page_dtor(page, free_huge_page);
@@ -623,6 +571,34 @@ static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
         put_page(page); /* free it into the hugepage allocator */
 }
 
+static void prep_compound_gigantic_page(struct page *page, unsigned long order)
+{
+        int i;
+        int nr_pages = 1 << order;
+        struct page *p = page + 1;
+
+        /* we rely on prep_new_huge_page to set the destructor */
+        set_compound_order(page, order);
+        __SetPageHead(page);
+        for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
+                __SetPageTail(p);
+                p->first_page = page;
+        }
+}
+
+int PageHuge(struct page *page)
+{
+        compound_page_dtor *dtor;
+
+        if (!PageCompound(page))
+                return 0;
+
+        page = compound_head(page);
+        dtor = get_compound_page_dtor(page);
+
+        return dtor == free_huge_page;
+}
+
 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
 {
         struct page *page;
@@ -630,7 +606,7 @@ static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
         if (h->order >= MAX_ORDER)
                 return NULL;
 
-        page = alloc_pages_node(nid,
+        page = alloc_pages_exact_node(nid,
                 htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
                                                 __GFP_REPEAT|__GFP_NOWARN,
                 huge_page_order(h));
@@ -647,22 +623,22 @@ static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
 
 /*
  * Use a helper variable to find the next node and then
- * copy it back to hugetlb_next_nid afterwards:
+ * copy it back to next_nid_to_alloc afterwards:
  * otherwise there's a window in which a racer might
- * pass invalid nid MAX_NUMNODES to alloc_pages_node.
+ * pass invalid nid MAX_NUMNODES to alloc_pages_exact_node.
  * But we don't need to use a spin_lock here: it really
  * doesn't matter if occasionally a racer chooses the
  * same nid as we do.  Move nid forward in the mask even
  * if we just successfully allocated a hugepage so that
  * the next caller gets hugepages on the next node.
  */
-static int hstate_next_node(struct hstate *h)
+static int hstate_next_node_to_alloc(struct hstate *h)
 {
         int next_nid;
-        next_nid = next_node(h->hugetlb_next_nid, node_online_map);
+        next_nid = next_node(h->next_nid_to_alloc, node_online_map);
         if (next_nid == MAX_NUMNODES)
                 next_nid = first_node(node_online_map);
-        h->hugetlb_next_nid = next_nid;
+        h->next_nid_to_alloc = next_nid;
         return next_nid;
 }
 
@@ -673,14 +649,15 @@ static int alloc_fresh_huge_page(struct hstate *h)
         int next_nid;
         int ret = 0;
 
-        start_nid = h->hugetlb_next_nid;
+        start_nid = h->next_nid_to_alloc;
+        next_nid = start_nid;
 
         do {
-                page = alloc_fresh_huge_page_node(h, h->hugetlb_next_nid);
+                page = alloc_fresh_huge_page_node(h, next_nid);
                 if (page)
                         ret = 1;
-                next_nid = hstate_next_node(h);
-        } while (!page && h->hugetlb_next_nid != start_nid);
+                next_nid = hstate_next_node_to_alloc(h);
+        } while (!page && next_nid != start_nid);
 
         if (ret)
                 count_vm_event(HTLB_BUDDY_PGALLOC);
@@ -690,6 +667,61 @@ static int alloc_fresh_huge_page(struct hstate *h)
         return ret;
 }
 
+/*
+ * helper for free_pool_huge_page() - find next node
+ * from which to free a huge page
+ */
+static int hstate_next_node_to_free(struct hstate *h)
+{
+        int next_nid;
+        next_nid = next_node(h->next_nid_to_free, node_online_map);
+        if (next_nid == MAX_NUMNODES)
+                next_nid = first_node(node_online_map);
+        h->next_nid_to_free = next_nid;
+        return next_nid;
+}
+
+/*
+ * Free huge page from pool from next node to free.
+ * Attempt to keep persistent huge pages more or less
+ * balanced over allowed nodes.
+ * Called with hugetlb_lock locked.
+ */
+static int free_pool_huge_page(struct hstate *h, bool acct_surplus)
+{
+        int start_nid;
+        int next_nid;
+        int ret = 0;
+
+        start_nid = h->next_nid_to_free;
+        next_nid = start_nid;
+
+        do {
+                /*
+                 * If we're returning unused surplus pages, only examine
+                 * nodes with surplus pages.
+                 */
+                if ((!acct_surplus || h->surplus_huge_pages_node[next_nid]) &&
+                    !list_empty(&h->hugepage_freelists[next_nid])) {
+                        struct page *page =
+                                list_entry(h->hugepage_freelists[next_nid].next,
+                                          struct page, lru);
+                        list_del(&page->lru);
+                        h->free_huge_pages--;
+                        h->free_huge_pages_node[next_nid]--;
+                        if (acct_surplus) {
+                                h->surplus_huge_pages--;
+                                h->surplus_huge_pages_node[next_nid]--;
+                        }
+                        update_and_free_page(h, page);
+                        ret = 1;
+                }
+                next_nid = hstate_next_node_to_free(h);
+        } while (!ret && next_nid != start_nid);
+
+        return ret;
+}
+
 static struct page *alloc_buddy_huge_page(struct hstate *h,
                         struct vm_area_struct *vma, unsigned long address)
 {
@@ -861,22 +893,13 @@ free:
  * When releasing a hugetlb pool reservation, any surplus pages that were
  * allocated to satisfy the reservation must be explicitly freed if they were
  * never used.
+ * Called with hugetlb_lock held.
  */
 static void return_unused_surplus_pages(struct hstate *h,
                                         unsigned long unused_resv_pages)
 {
-        static int nid = -1;
-        struct page *page;
         unsigned long nr_pages;
 
-        /*
-         * We want to release as many surplus pages as possible, spread
-         * evenly across all nodes. Iterate across all nodes until we
-         * can no longer free unreserved surplus pages. This occurs when
-         * the nodes with surplus pages have no free pages.
-         */
-        unsigned long remaining_iterations = num_online_nodes();
-
         /* Uncommit the reservation */
         h->resv_huge_pages -= unused_resv_pages;
 
@@ -886,26 +909,17 @@ static void return_unused_surplus_pages(struct hstate *h,
 
         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
 
-        while (remaining_iterations-- && nr_pages) {
-                nid = next_node(nid, node_online_map);
-                if (nid == MAX_NUMNODES)
-                        nid = first_node(node_online_map);
-
-                if (!h->surplus_huge_pages_node[nid])
-                        continue;
-
-                if (!list_empty(&h->hugepage_freelists[nid])) {
-                        page = list_entry(h->hugepage_freelists[nid].next,
-                                          struct page, lru);
-                        list_del(&page->lru);
-                        update_and_free_page(h, page);
-                        h->free_huge_pages--;
-                        h->free_huge_pages_node[nid]--;
-                        h->surplus_huge_pages--;
-                        h->surplus_huge_pages_node[nid]--;
-                        nr_pages--;
-                        remaining_iterations = num_online_nodes();
-                }
+        /*
+         * We want to release as many surplus pages as possible, spread
+         * evenly across all nodes. Iterate across all nodes until we
+         * can no longer free unreserved surplus pages. This occurs when
+         * the nodes with surplus pages have no free pages.
+         * free_pool_huge_page() will balance the the frees across the
+         * on-line nodes for us and will handle the hstate accounting.
+         */
+        while (nr_pages--) {
+                if (!free_pool_huge_page(h, 1))
+                        break;
         }
 }
 
@@ -924,7 +938,7 @@ static long vma_needs_reservation(struct hstate *h,
         struct address_space *mapping = vma->vm_file->f_mapping;
         struct inode *inode = mapping->host;
 
-        if (vma->vm_flags & VM_SHARED) {
+        if (vma->vm_flags & VM_MAYSHARE) {
                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
                 return region_chg(&inode->i_mapping->private_list,
                                                         idx, idx + 1);
@@ -949,7 +963,7 @@ static void vma_commit_reservation(struct hstate *h,
         struct address_space *mapping = vma->vm_file->f_mapping;
         struct inode *inode = mapping->host;
 
-        if (vma->vm_flags & VM_SHARED) {
+        if (vma->vm_flags & VM_MAYSHARE) {
                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
                 region_add(&inode->i_mapping->private_list, idx, idx + 1);
 
@@ -1014,9 +1028,10 @@ int __weak alloc_bootmem_huge_page(struct hstate *h)
                 void *addr;
 
                 addr = __alloc_bootmem_node_nopanic(
-                                NODE_DATA(h->hugetlb_next_nid),
+                                NODE_DATA(h->next_nid_to_alloc),
                                 huge_page_size(h), huge_page_size(h), 0);
 
+                hstate_next_node_to_alloc(h);
                 if (addr) {
                         /*
                          * Use the beginning of the huge page to store the
@@ -1026,7 +1041,6 @@ int __weak alloc_bootmem_huge_page(struct hstate *h)
                         m = addr;
                         goto found;
                 }
-                hstate_next_node(h);
                 nr_nodes--;
         }
         return 0;
@@ -1140,6 +1154,53 @@ static inline void try_to_free_low(struct hstate *h, unsigned long count)
 }
 #endif
 
+/*
+ * Increment or decrement surplus_huge_pages.  Keep node-specific counters
+ * balanced by operating on them in a round-robin fashion.
+ * Returns 1 if an adjustment was made.
+ */
+static int adjust_pool_surplus(struct hstate *h, int delta)
+{
+        int start_nid, next_nid;
+        int ret = 0;
+
+        VM_BUG_ON(delta != -1 && delta != 1);
+
+        if (delta < 0)
+                start_nid = h->next_nid_to_alloc;
+        else
+                start_nid = h->next_nid_to_free;
+        next_nid = start_nid;
+
+        do {
+                int nid = next_nid;
+                if (delta < 0)  {
+                        next_nid = hstate_next_node_to_alloc(h);
+                        /*
+                         * To shrink on this node, there must be a surplus page
+                         */
+                        if (!h->surplus_huge_pages_node[nid])
+                                continue;
+                }
+                if (delta > 0) {
+                        next_nid = hstate_next_node_to_free(h);
+                        /*
+                         * Surplus cannot exceed the total number of pages
+                         */
+                        if (h->surplus_huge_pages_node[nid] >=
+                                                h->nr_huge_pages_node[nid])
+                                continue;
+                }
+
+                h->surplus_huge_pages += delta;
+                h->surplus_huge_pages_node[nid] += delta;
+                ret = 1;
+                break;
+        } while (next_nid != start_nid);
+
+        return ret;
+}
+
 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count)
 {
@@ -1198,10 +1259,8 @@ static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count)
         min_count = max(count, min_count);
         try_to_free_low(h, min_count);
         while (min_count < persistent_huge_pages(h)) {
-                struct page *page = dequeue_huge_page(h);
-                if (!page)
+                if (!free_pool_huge_page(h, 0))
                         break;
-                update_and_free_page(h, page);
         }
         while (count < persistent_huge_pages(h)) {
                 if (!adjust_pool_surplus(h, 1))
@@ -1413,7 +1472,8 @@ void __init hugetlb_add_hstate(unsigned order)
         h->free_huge_pages = 0;
         for (i = 0; i < MAX_NUMNODES; ++i)
                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
-        h->hugetlb_next_nid = first_node(node_online_map);
+        h->next_nid_to_alloc = first_node(node_online_map);
+        h->next_nid_to_free = first_node(node_online_map);
         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
                                         huge_page_size(h)/1024);
 
@@ -1477,7 +1537,7 @@ static unsigned int cpuset_mems_nr(unsigned int *array)
 
 #ifdef CONFIG_SYSCTL
 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
-                           struct file *file, void __user *buffer,
+                           void __user *buffer,
                            size_t *length, loff_t *ppos)
 {
         struct hstate *h = &default_hstate;
@@ -1488,7 +1548,7 @@ int hugetlb_sysctl_handler(struct ctl_table *table, int write,
 
         table->data = &tmp;
         table->maxlen = sizeof(unsigned long);
-        proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
+        proc_doulongvec_minmax(table, write, buffer, length, ppos);
 
         if (write)
                 h->max_huge_pages = set_max_huge_pages(h, tmp);
@@ -1497,10 +1557,10 @@ int hugetlb_sysctl_handler(struct ctl_table *table, int write,
 }
 
 int hugetlb_treat_movable_handler(struct ctl_table *table, int write,
-                        struct file *file, void __user *buffer,
+                        void __user *buffer,
                         size_t *length, loff_t *ppos)
 {
-        proc_dointvec(table, write, file, buffer, length, ppos);
+        proc_dointvec(table, write, buffer, length, ppos);
         if (hugepages_treat_as_movable)
                 htlb_alloc_mask = GFP_HIGHUSER_MOVABLE;
         else
@@ -1509,7 +1569,7 @@ int hugetlb_treat_movable_handler(struct ctl_table *table, int write,
 }
 
 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
-                        struct file *file, void __user *buffer,
+                        void __user *buffer,
                         size_t *length, loff_t *ppos)
 {
         struct hstate *h = &default_hstate;
@@ -1520,7 +1580,7 @@ int hugetlb_overcommit_handler(struct ctl_table *table, int write,
 
         table->data = &tmp;
         table->maxlen = sizeof(unsigned long);
-        proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
+        proc_doulongvec_minmax(table, write, buffer, length, ppos);
 
         if (write) {
                 spin_lock(&hugetlb_lock);
@@ -1661,7 +1721,7 @@ static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
         return 0;
 }
 
-struct vm_operations_struct hugetlb_vm_ops = {
+const struct vm_operations_struct hugetlb_vm_ops = {
         .fault = hugetlb_vm_op_fault,
         .open = hugetlb_vm_op_open,
         .close = hugetlb_vm_op_close,
@@ -1893,7 +1953,7 @@ retry_avoidcopy:
          * at the time of fork() could consume its reserves on COW instead
          * of the full address range.
          */
-        if (!(vma->vm_flags & VM_SHARED) &&
+        if (!(vma->vm_flags & VM_MAYSHARE) &&
                         is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
                         old_page != pagecache_page)
                 outside_reserve = 1;
@@ -1956,8 +2016,28 @@ static struct page *hugetlbfs_pagecache_page(struct hstate *h,
         return find_lock_page(mapping, idx);
 }
 
+/*
+ * Return whether there is a pagecache page to back given address within VMA.
+ * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
+ */
+static bool hugetlbfs_pagecache_present(struct hstate *h,
+                        struct vm_area_struct *vma, unsigned long address)
+{
+        struct address_space *mapping;
+        pgoff_t idx;
+        struct page *page;
+
+        mapping = vma->vm_file->f_mapping;
+        idx = vma_hugecache_offset(h, vma, address);
+
+        page = find_get_page(mapping, idx);
+        if (page)
+                put_page(page);
+        return page != NULL;
+}
+
 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
-                        unsigned long address, pte_t *ptep, int write_access)
+                        unsigned long address, pte_t *ptep, unsigned int flags)
 {
         struct hstate *h = hstate_vma(vma);
         int ret = VM_FAULT_SIGBUS;
@@ -2000,7 +2080,7 @@ retry:
                 clear_huge_page(page, address, huge_page_size(h));
                 __SetPageUptodate(page);
 
-                if (vma->vm_flags & VM_SHARED) {
+                if (vma->vm_flags & VM_MAYSHARE) {
                         int err;
                         struct inode *inode = mapping->host;
 
@@ -2025,7 +2105,7 @@ retry:
          * any allocations necessary to record that reservation occur outside
          * the spinlock.
          */
-        if (write_access && !(vma->vm_flags & VM_SHARED))
+        if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
                 if (vma_needs_reservation(h, vma, address) < 0) {
                         ret = VM_FAULT_OOM;
                         goto backout_unlocked;
@@ -2044,7 +2124,7 @@ retry:
                                 && (vma->vm_flags & VM_SHARED)));
         set_huge_pte_at(mm, address, ptep, new_pte);
 
-        if (write_access && !(vma->vm_flags & VM_SHARED)) {
+        if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
                 /* Optimization, do the COW without a second fault */
                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page);
         }
@@ -2063,7 +2143,7 @@ backout_unlocked:
 }
 
 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
-                        unsigned long address, int write_access)
+                        unsigned long address, unsigned int flags)
 {
         pte_t *ptep;
         pte_t entry;
@@ -2084,7 +2164,7 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
         mutex_lock(&hugetlb_instantiation_mutex);
         entry = huge_ptep_get(ptep);
         if (huge_pte_none(entry)) {
-                ret = hugetlb_no_page(mm, vma, address, ptep, write_access);
+                ret = hugetlb_no_page(mm, vma, address, ptep, flags);
                 goto out_mutex;
         }
 
@@ -2098,13 +2178,13 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
          * page now as it is used to determine if a reservation has been
          * consumed.
          */
-        if (write_access && !pte_write(entry)) {
+        if ((flags & FAULT_FLAG_WRITE) && !pte_write(entry)) {
                 if (vma_needs_reservation(h, vma, address) < 0) {
                         ret = VM_FAULT_OOM;
                         goto out_mutex;
                 }
 
-                if (!(vma->vm_flags & VM_SHARED))
+                if (!(vma->vm_flags & VM_MAYSHARE))
                         pagecache_page = hugetlbfs_pagecache_page(h,
                                                                 vma, address);
         }
@@ -2115,7 +2195,7 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
                 goto out_page_table_lock;
 
 
-        if (write_access) {
+        if (flags & FAULT_FLAG_WRITE) {
                 if (!pte_write(entry)) {
                         ret = hugetlb_cow(mm, vma, address, ptep, entry,
                                                         pagecache_page);
@@ -2124,7 +2204,8 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
                 entry = pte_mkdirty(entry);
         }
         entry = pte_mkyoung(entry);
-        if (huge_ptep_set_access_flags(vma, address, ptep, entry, write_access))
+        if (huge_ptep_set_access_flags(vma, address, ptep, entry,
+                                                flags & FAULT_FLAG_WRITE))
                 update_mmu_cache(vma, address, entry);
 
 out_page_table_lock:
@@ -2150,54 +2231,55 @@ follow_huge_pud(struct mm_struct *mm, unsigned long address,
         return NULL;
 }
 
-static int huge_zeropage_ok(pte_t *ptep, int write, int shared)
-{
-        if (!ptep || write || shared)
-                return 0;
-        else
-                return huge_pte_none(huge_ptep_get(ptep));
-}
-
 int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
                         struct page **pages, struct vm_area_struct **vmas,
                         unsigned long *position, int *length, int i,
-                        int write)
+                        unsigned int flags)
 {
         unsigned long pfn_offset;
         unsigned long vaddr = *position;
         int remainder = *length;
         struct hstate *h = hstate_vma(vma);
-        int zeropage_ok = 0;
-        int shared = vma->vm_flags & VM_SHARED;
 
         spin_lock(&mm->page_table_lock);
         while (vaddr < vma->vm_end && remainder) {
                 pte_t *pte;
+                int absent;
                 struct page *page;
 
                 /*
                  * Some archs (sparc64, sh*) have multiple pte_ts to
-                 * each hugepage.  We have to make * sure we get the
+                 * each hugepage.  We have to make sure we get the
                  * first, for the page indexing below to work.
                  */
                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
-                if (huge_zeropage_ok(pte, write, shared))
-                        zeropage_ok = 1;
+                absent = !pte || huge_pte_none(huge_ptep_get(pte));
+
+                /*
+                 * When coredumping, it suits get_dump_page if we just return
+                 * an error where there's an empty slot with no huge pagecache
+                 * to back it.  This way, we avoid allocating a hugepage, and
+                 * the sparse dumpfile avoids allocating disk blocks, but its
+                 * huge holes still show up with zeroes where they need to be.
+                 */
+                if (absent && (flags & FOLL_DUMP) &&
+                    !hugetlbfs_pagecache_present(h, vma, vaddr)) {
+                        remainder = 0;
+                        break;
+                }
 
-                if (!pte ||
-                    (huge_pte_none(huge_ptep_get(pte)) && !zeropage_ok) ||
-                    (write && !pte_write(huge_ptep_get(pte)))) {
+                if (absent ||
+                    ((flags & FOLL_WRITE) && !pte_write(huge_ptep_get(pte)))) {
                         int ret;
 
                         spin_unlock(&mm->page_table_lock);
-                        ret = hugetlb_fault(mm, vma, vaddr, write);
+                        ret = hugetlb_fault(mm, vma, vaddr,
+                                (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
                         spin_lock(&mm->page_table_lock);
                         if (!(ret & VM_FAULT_ERROR))
                                 continue;
 
                         remainder = 0;
-                        if (!i)
-                                i = -EFAULT;
                         break;
                 }
 
@@ -2205,10 +2287,7 @@ int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 page = pte_page(huge_ptep_get(pte));
 same_page:
                 if (pages) {
-                        if (zeropage_ok)
-                                pages[i] = ZERO_PAGE(0);
-                        else
-                                pages[i] = mem_map_offset(page, pfn_offset);
+                        pages[i] = mem_map_offset(page, pfn_offset);
                         get_page(pages[i]);
                 }
 
@@ -2232,7 +2311,7 @@ same_page:
         *length = remainder;
         *position = vaddr;
 
-        return i;
+        return i ? i : -EFAULT;
 }
 
 void hugetlb_change_protection(struct vm_area_struct *vma,
@@ -2289,7 +2368,7 @@ int hugetlb_reserve_pages(struct inode *inode,
          * to reserve the full area even if read-only as mprotect() may be
          * called to make the mapping read-write. Assume !vma is a shm mapping
          */
-        if (!vma || vma->vm_flags & VM_SHARED)
+        if (!vma || vma->vm_flags & VM_MAYSHARE)
                 chg = region_chg(&inode->i_mapping->private_list, from, to);
         else {
                 struct resv_map *resv_map = resv_map_alloc();
@@ -2330,7 +2409,7 @@ int hugetlb_reserve_pages(struct inode *inode,
          * consumed reservations are stored in the map. Hence, nothing
          * else has to be done for private mappings here
          */
-        if (!vma || vma->vm_flags & VM_SHARED)
+        if (!vma || vma->vm_flags & VM_MAYSHARE)
                 region_add(&inode->i_mapping->private_list, from, to);
         return 0;
 }
@@ -2341,7 +2420,7 @@ void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
         long chg = region_truncate(&inode->i_mapping->private_list, offset);
 
         spin_lock(&inode->i_lock);
-        inode->i_blocks -= blocks_per_huge_page(h);
+        inode->i_blocks -= (blocks_per_huge_page(h) * freed);
         spin_unlock(&inode->i_lock);
 
         hugetlb_put_quota(inode->i_mapping, (chg - freed));
diff --git a/mm/hwpoison-inject.c b/mm/hwpoison-inject.c
new file mode 100644
index 000000000000..e1d85137f086
--- /dev/null
+++ b/mm/hwpoison-inject.c
@@ -0,0 +1,41 @@
+/* Inject a hwpoison memory failure on a arbitary pfn */
+#include <linux/module.h>
+#include <linux/debugfs.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+
+static struct dentry *hwpoison_dir, *corrupt_pfn;
+
+static int hwpoison_inject(void *data, u64 val)
+{
+        if (!capable(CAP_SYS_ADMIN))
+                return -EPERM;
+        printk(KERN_INFO "Injecting memory failure at pfn %Lx\n", val);
+        return __memory_failure(val, 18, 0);
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(hwpoison_fops, NULL, hwpoison_inject, "%lli\n");
+
+static void pfn_inject_exit(void)
+{
+        if (hwpoison_dir)
+                debugfs_remove_recursive(hwpoison_dir);
+}
+
+static int pfn_inject_init(void)
+{
+        hwpoison_dir = debugfs_create_dir("hwpoison", NULL);
+        if (hwpoison_dir == NULL)
+                return -ENOMEM;
+        corrupt_pfn = debugfs_create_file("corrupt-pfn", 0600, hwpoison_dir,
+                                          NULL, &hwpoison_fops);
+        if (corrupt_pfn == NULL) {
+                pfn_inject_exit();
+                return -ENOMEM;
+        }
+        return 0;
+}
+
+module_init(pfn_inject_init);
+module_exit(pfn_inject_exit);
+MODULE_LICENSE("GPL");
diff --git a/mm/init-mm.c b/mm/init-mm.c
new file mode 100644
index 000000000000..57aba0da9668
--- /dev/null
+++ b/mm/init-mm.c
@@ -0,0 +1,20 @@
+#include <linux/mm_types.h>
+#include <linux/rbtree.h>
+#include <linux/rwsem.h>
+#include <linux/spinlock.h>
+#include <linux/list.h>
+#include <linux/cpumask.h>
+
+#include <asm/atomic.h>
+#include <asm/pgtable.h>
+
+struct mm_struct init_mm = {
+        .mm_rb          = RB_ROOT,
+        .pgd            = swapper_pg_dir,
+        .mm_users       = ATOMIC_INIT(2),
+        .mm_count       = ATOMIC_INIT(1),
+        .mmap_sem       = __RWSEM_INITIALIZER(init_mm.mmap_sem),
+        .page_table_lock =  __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock),
+        .mmlist         = LIST_HEAD_INIT(init_mm.mmlist),
+        .cpu_vm_mask    = CPU_MASK_ALL,
+};
diff --git a/mm/internal.h b/mm/internal.h
index 987bb03fbdd8..22ec8d2b0fb8 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -16,9 +16,6 @@
 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
                 unsigned long floor, unsigned long ceiling);
 
-extern void prep_compound_page(struct page *page, unsigned long order);
-extern void prep_compound_gigantic_page(struct page *page, unsigned long order);
-
 static inline void set_page_count(struct page *page, int v)
 {
         atomic_set(&page->_count, v);
@@ -40,6 +37,8 @@ static inline void __put_page(struct page *page)
         atomic_dec(&page->_count);
 }
 
+extern unsigned long highest_memmap_pfn;
+
 /*
  * in mm/vmscan.c:
  */
@@ -49,8 +48,9 @@ extern void putback_lru_page(struct page *page);
 /*
  * in mm/page_alloc.c
  */
-extern unsigned long highest_memmap_pfn;
 extern void __free_pages_bootmem(struct page *page, unsigned int order);
+extern void prep_compound_page(struct page *page, unsigned long order);
+
 
 /*
  * function for dealing with page's order in buddy system.
@@ -74,7 +74,6 @@ static inline void munlock_vma_pages_all(struct vm_area_struct *vma)
 }
 #endif
 
-#ifdef CONFIG_UNEVICTABLE_LRU
 /*
  * unevictable_migrate_page() called only from migrate_page_copy() to
  * migrate unevictable flag to new page.
@@ -86,11 +85,6 @@ static inline void unevictable_migrate_page(struct page *new, struct page *old)
         if (TestClearPageUnevictable(old))
                 SetPageUnevictable(new);
 }
-#else
-static inline void unevictable_migrate_page(struct page *new, struct page *old)
-{
-}
-#endif
 
 #ifdef CONFIG_HAVE_MLOCKED_PAGE_BIT
 /*
@@ -150,23 +144,6 @@ static inline void mlock_migrate_page(struct page *newpage, struct page *page)
         }
 }
 
-/*
- * free_page_mlock() -- clean up attempts to free and mlocked() page.
- * Page should not be on lru, so no need to fix that up.
- * free_pages_check() will verify...
- */
-static inline void free_page_mlock(struct page *page)
-{
-        if (unlikely(TestClearPageMlocked(page))) {
-                unsigned long flags;
-
-                local_irq_save(flags);
-                __dec_zone_page_state(page, NR_MLOCK);
-                __count_vm_event(UNEVICTABLE_MLOCKFREED);
-                local_irq_restore(flags);
-        }
-}
-
 #else /* CONFIG_HAVE_MLOCKED_PAGE_BIT */
 static inline int is_mlocked_vma(struct vm_area_struct *v, struct page *p)
 {
@@ -175,7 +152,6 @@ static inline int is_mlocked_vma(struct vm_area_struct *v, struct page *p)
 static inline void clear_page_mlock(struct page *page) { }
 static inline void mlock_vma_page(struct page *page) { }
 static inline void mlock_migrate_page(struct page *new, struct page *old) { }
-static inline void free_page_mlock(struct page *page) { }
 
 #endif /* CONFIG_HAVE_MLOCKED_PAGE_BIT */
 
@@ -275,13 +251,12 @@ static inline void mminit_validate_memmodel_limits(unsigned long *start_pfn,
 }
 #endif /* CONFIG_SPARSEMEM */
 
-#define GUP_FLAGS_WRITE                  0x1
-#define GUP_FLAGS_FORCE                  0x2
-#define GUP_FLAGS_IGNORE_VMA_PERMISSIONS 0x4
-#define GUP_FLAGS_IGNORE_SIGKILL         0x8
-
 int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
-                     unsigned long start, int len, int flags,
+                     unsigned long start, int len, unsigned int foll_flags,
                      struct page **pages, struct vm_area_struct **vmas);
 
+#define ZONE_RECLAIM_NOSCAN     -2
+#define ZONE_RECLAIM_FULL       -1
+#define ZONE_RECLAIM_SOME       0
+#define ZONE_RECLAIM_SUCCESS    1
 #endif
diff --git a/mm/kmemcheck.c b/mm/kmemcheck.c
new file mode 100644
index 000000000000..fd814fd61319
--- /dev/null
+++ b/mm/kmemcheck.c
@@ -0,0 +1,122 @@
+#include <linux/gfp.h>
+#include <linux/mm_types.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/kmemcheck.h>
+
+void kmemcheck_alloc_shadow(struct page *page, int order, gfp_t flags, int node)
+{
+        struct page *shadow;
+        int pages;
+        int i;
+
+        pages = 1 << order;
+
+        /*
+         * With kmemcheck enabled, we need to allocate a memory area for the
+         * shadow bits as well.
+         */
+        shadow = alloc_pages_node(node, flags | __GFP_NOTRACK, order);
+        if (!shadow) {
+                if (printk_ratelimit())
+                        printk(KERN_ERR "kmemcheck: failed to allocate "
+                                "shadow bitmap\n");
+                return;
+        }
+
+        for(i = 0; i < pages; ++i)
+                page[i].shadow = page_address(&shadow[i]);
+
+        /*
+         * Mark it as non-present for the MMU so that our accesses to
+         * this memory will trigger a page fault and let us analyze
+         * the memory accesses.
+         */
+        kmemcheck_hide_pages(page, pages);
+}
+
+void kmemcheck_free_shadow(struct page *page, int order)
+{
+        struct page *shadow;
+        int pages;
+        int i;
+
+        if (!kmemcheck_page_is_tracked(page))
+                return;
+
+        pages = 1 << order;
+
+        kmemcheck_show_pages(page, pages);
+
+        shadow = virt_to_page(page[0].shadow);
+
+        for(i = 0; i < pages; ++i)
+                page[i].shadow = NULL;
+
+        __free_pages(shadow, order);
+}
+
+void kmemcheck_slab_alloc(struct kmem_cache *s, gfp_t gfpflags, void *object,
+                          size_t size)
+{
+        /*
+         * Has already been memset(), which initializes the shadow for us
+         * as well.
+         */
+        if (gfpflags & __GFP_ZERO)
+                return;
+
+        /* No need to initialize the shadow of a non-tracked slab. */
+        if (s->flags & SLAB_NOTRACK)
+                return;
+
+        if (!kmemcheck_enabled || gfpflags & __GFP_NOTRACK) {
+                /*
+                 * Allow notracked objects to be allocated from
+                 * tracked caches. Note however that these objects
+                 * will still get page faults on access, they just
+                 * won't ever be flagged as uninitialized. If page
+                 * faults are not acceptable, the slab cache itself
+                 * should be marked NOTRACK.
+                 */
+                kmemcheck_mark_initialized(object, size);
+        } else if (!s->ctor) {
+                /*
+                 * New objects should be marked uninitialized before
+                 * they're returned to the called.
+                 */
+                kmemcheck_mark_uninitialized(object, size);
+        }
+}
+
+void kmemcheck_slab_free(struct kmem_cache *s, void *object, size_t size)
+{
+        /* TODO: RCU freeing is unsupported for now; hide false positives. */
+        if (!s->ctor && !(s->flags & SLAB_DESTROY_BY_RCU))
+                kmemcheck_mark_freed(object, size);
+}
+
+void kmemcheck_pagealloc_alloc(struct page *page, unsigned int order,
+                               gfp_t gfpflags)
+{
+        int pages;
+
+        if (gfpflags & (__GFP_HIGHMEM | __GFP_NOTRACK))
+                return;
+
+        pages = 1 << order;
+
+        /*
+         * NOTE: We choose to track GFP_ZERO pages too; in fact, they
+         * can become uninitialized by copying uninitialized memory
+         * into them.
+         */
+
+        /* XXX: Can use zone->node for node? */
+        kmemcheck_alloc_shadow(page, order, gfpflags, -1);
+
+        if (gfpflags & __GFP_ZERO)
+                kmemcheck_mark_initialized_pages(page, pages);
+        else
+                kmemcheck_mark_uninitialized_pages(page, pages);
+}
diff --git a/mm/kmemleak-test.c b/mm/kmemleak-test.c
new file mode 100644
index 000000000000..177a5169bbde
--- /dev/null
+++ b/mm/kmemleak-test.c
@@ -0,0 +1,111 @@
+/*
+ * mm/kmemleak-test.c
+ *
+ * Copyright (C) 2008 ARM Limited
+ * Written by Catalin Marinas <catalin.marinas@arm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/list.h>
+#include <linux/percpu.h>
+#include <linux/fdtable.h>
+
+#include <linux/kmemleak.h>
+
+struct test_node {
+        long header[25];
+        struct list_head list;
+        long footer[25];
+};
+
+static LIST_HEAD(test_list);
+static DEFINE_PER_CPU(void *, kmemleak_test_pointer);
+
+/*
+ * Some very simple testing. This function needs to be extended for
+ * proper testing.
+ */
+static int __init kmemleak_test_init(void)
+{
+        struct test_node *elem;
+        int i;
+
+        printk(KERN_INFO "Kmemleak testing\n");
+
+        /* make some orphan objects */
+        pr_info("kmemleak: kmalloc(32) = %p\n", kmalloc(32, GFP_KERNEL));
+        pr_info("kmemleak: kmalloc(32) = %p\n", kmalloc(32, GFP_KERNEL));
+        pr_info("kmemleak: kmalloc(1024) = %p\n", kmalloc(1024, GFP_KERNEL));
+        pr_info("kmemleak: kmalloc(1024) = %p\n", kmalloc(1024, GFP_KERNEL));
+        pr_info("kmemleak: kmalloc(2048) = %p\n", kmalloc(2048, GFP_KERNEL));
+        pr_info("kmemleak: kmalloc(2048) = %p\n", kmalloc(2048, GFP_KERNEL));
+        pr_info("kmemleak: kmalloc(4096) = %p\n", kmalloc(4096, GFP_KERNEL));
+        pr_info("kmemleak: kmalloc(4096) = %p\n", kmalloc(4096, GFP_KERNEL));
+#ifndef CONFIG_MODULES
+        pr_info("kmemleak: kmem_cache_alloc(files_cachep) = %p\n",
+                kmem_cache_alloc(files_cachep, GFP_KERNEL));
+        pr_info("kmemleak: kmem_cache_alloc(files_cachep) = %p\n",
+                kmem_cache_alloc(files_cachep, GFP_KERNEL));
+#endif
+        pr_info("kmemleak: vmalloc(64) = %p\n", vmalloc(64));
+        pr_info("kmemleak: vmalloc(64) = %p\n", vmalloc(64));
+        pr_info("kmemleak: vmalloc(64) = %p\n", vmalloc(64));
+        pr_info("kmemleak: vmalloc(64) = %p\n", vmalloc(64));
+        pr_info("kmemleak: vmalloc(64) = %p\n", vmalloc(64));
+
+        /*
+         * Add elements to a list. They should only appear as orphan
+         * after the module is removed.
+         */
+        for (i = 0; i < 10; i++) {
+                elem = kmalloc(sizeof(*elem), GFP_KERNEL);
+                pr_info("kmemleak: kmalloc(sizeof(*elem)) = %p\n", elem);
+                if (!elem)
+                        return -ENOMEM;
+                memset(elem, 0, sizeof(*elem));
+                INIT_LIST_HEAD(&elem->list);
+
+                list_add_tail(&elem->list, &test_list);
+        }
+
+        for_each_possible_cpu(i) {
+                per_cpu(kmemleak_test_pointer, i) = kmalloc(129, GFP_KERNEL);
+                pr_info("kmemleak: kmalloc(129) = %p\n",
+                        per_cpu(kmemleak_test_pointer, i));
+        }
+
+        return 0;
+}
+module_init(kmemleak_test_init);
+
+static void __exit kmemleak_test_exit(void)
+{
+        struct test_node *elem, *tmp;
+
+        /*
+         * Remove the list elements without actually freeing the
+         * memory.
+         */
+        list_for_each_entry_safe(elem, tmp, &test_list, list)
+                list_del(&elem->list);
+}
+module_exit(kmemleak_test_exit);
+
+MODULE_LICENSE("GPL");
diff --git a/mm/kmemleak.c b/mm/kmemleak.c
new file mode 100644
index 000000000000..8bf765c4f58d
--- /dev/null
+++ b/mm/kmemleak.c
@@ -0,0 +1,1689 @@
+/*
+ * mm/kmemleak.c
+ *
+ * Copyright (C) 2008 ARM Limited
+ * Written by Catalin Marinas <catalin.marinas@arm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *
+ *
+ * For more information on the algorithm and kmemleak usage, please see
+ * Documentation/kmemleak.txt.
+ *
+ * Notes on locking
+ * ----------------
+ *
+ * The following locks and mutexes are used by kmemleak:
+ *
+ * - kmemleak_lock (rwlock): protects the object_list modifications and
+ *   accesses to the object_tree_root. The object_list is the main list
+ *   holding the metadata (struct kmemleak_object) for the allocated memory
+ *   blocks. The object_tree_root is a priority search tree used to look-up
+ *   metadata based on a pointer to the corresponding memory block.  The
+ *   kmemleak_object structures are added to the object_list and
+ *   object_tree_root in the create_object() function called from the
+ *   kmemleak_alloc() callback and removed in delete_object() called from the
+ *   kmemleak_free() callback
+ * - kmemleak_object.lock (spinlock): protects a kmemleak_object. Accesses to
+ *   the metadata (e.g. count) are protected by this lock. Note that some
+ *   members of this structure may be protected by other means (atomic or
+ *   kmemleak_lock). This lock is also held when scanning the corresponding
+ *   memory block to avoid the kernel freeing it via the kmemleak_free()
+ *   callback. This is less heavyweight than holding a global lock like
+ *   kmemleak_lock during scanning
+ * - scan_mutex (mutex): ensures that only one thread may scan the memory for
+ *   unreferenced objects at a time. The gray_list contains the objects which
+ *   are already referenced or marked as false positives and need to be
+ *   scanned. This list is only modified during a scanning episode when the
+ *   scan_mutex is held. At the end of a scan, the gray_list is always empty.
+ *   Note that the kmemleak_object.use_count is incremented when an object is
+ *   added to the gray_list and therefore cannot be freed. This mutex also
+ *   prevents multiple users of the "kmemleak" debugfs file together with
+ *   modifications to the memory scanning parameters including the scan_thread
+ *   pointer
+ *
+ * The kmemleak_object structures have a use_count incremented or decremented
+ * using the get_object()/put_object() functions. When the use_count becomes
+ * 0, this count can no longer be incremented and put_object() schedules the
+ * kmemleak_object freeing via an RCU callback. All calls to the get_object()
+ * function must be protected by rcu_read_lock() to avoid accessing a freed
+ * structure.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/sched.h>
+#include <linux/jiffies.h>
+#include <linux/delay.h>
+#include <linux/module.h>
+#include <linux/kthread.h>
+#include <linux/prio_tree.h>
+#include <linux/gfp.h>
+#include <linux/fs.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/cpumask.h>
+#include <linux/spinlock.h>
+#include <linux/mutex.h>
+#include <linux/rcupdate.h>
+#include <linux/stacktrace.h>
+#include <linux/cache.h>
+#include <linux/percpu.h>
+#include <linux/hardirq.h>
+#include <linux/mmzone.h>
+#include <linux/slab.h>
+#include <linux/thread_info.h>
+#include <linux/err.h>
+#include <linux/uaccess.h>
+#include <linux/string.h>
+#include <linux/nodemask.h>
+#include <linux/mm.h>
+#include <linux/workqueue.h>
+
+#include <asm/sections.h>
+#include <asm/processor.h>
+#include <asm/atomic.h>
+
+#include <linux/kmemcheck.h>
+#include <linux/kmemleak.h>
+
+/*
+ * Kmemleak configuration and common defines.
+ */
+#define MAX_TRACE               16      /* stack trace length */
+#define MSECS_MIN_AGE           5000    /* minimum object age for reporting */
+#define SECS_FIRST_SCAN         60      /* delay before the first scan */
+#define SECS_SCAN_WAIT          600     /* subsequent auto scanning delay */
+#define GRAY_LIST_PASSES        25      /* maximum number of gray list scans */
+#define MAX_SCAN_SIZE           4096    /* maximum size of a scanned block */
+
+#define BYTES_PER_POINTER       sizeof(void *)
+
+/* GFP bitmask for kmemleak internal allocations */
+#define GFP_KMEMLEAK_MASK       (GFP_KERNEL | GFP_ATOMIC)
+
+/* scanning area inside a memory block */
+struct kmemleak_scan_area {
+        struct hlist_node node;
+        unsigned long offset;
+        size_t length;
+};
+
+#define KMEMLEAK_GREY   0
+#define KMEMLEAK_BLACK  -1
+
+/*
+ * Structure holding the metadata for each allocated memory block.
+ * Modifications to such objects should be made while holding the
+ * object->lock. Insertions or deletions from object_list, gray_list or
+ * tree_node are already protected by the corresponding locks or mutex (see
+ * the notes on locking above). These objects are reference-counted
+ * (use_count) and freed using the RCU mechanism.
+ */
+struct kmemleak_object {
+        spinlock_t lock;
+        unsigned long flags;            /* object status flags */
+        struct list_head object_list;
+        struct list_head gray_list;
+        struct prio_tree_node tree_node;
+        struct rcu_head rcu;            /* object_list lockless traversal */
+        /* object usage count; object freed when use_count == 0 */
+        atomic_t use_count;
+        unsigned long pointer;
+        size_t size;
+        /* minimum number of a pointers found before it is considered leak */
+        int min_count;
+        /* the total number of pointers found pointing to this object */
+        int count;
+        /* memory ranges to be scanned inside an object (empty for all) */
+        struct hlist_head area_list;
+        unsigned long trace[MAX_TRACE];
+        unsigned int trace_len;
+        unsigned long jiffies;          /* creation timestamp */
+        pid_t pid;                      /* pid of the current task */
+        char comm[TASK_COMM_LEN];       /* executable name */
+};
+
+/* flag representing the memory block allocation status */
+#define OBJECT_ALLOCATED        (1 << 0)
+/* flag set after the first reporting of an unreference object */
+#define OBJECT_REPORTED         (1 << 1)
+/* flag set to not scan the object */
+#define OBJECT_NO_SCAN          (1 << 2)
+/* flag set on newly allocated objects */
+#define OBJECT_NEW              (1 << 3)
+
+/* number of bytes to print per line; must be 16 or 32 */
+#define HEX_ROW_SIZE            16
+/* number of bytes to print at a time (1, 2, 4, 8) */
+#define HEX_GROUP_SIZE          1
+/* include ASCII after the hex output */
+#define HEX_ASCII               1
+/* max number of lines to be printed */
+#define HEX_MAX_LINES           2
+
+/* the list of all allocated objects */
+static LIST_HEAD(object_list);
+/* the list of gray-colored objects (see color_gray comment below) */
+static LIST_HEAD(gray_list);
+/* prio search tree for object boundaries */
+static struct prio_tree_root object_tree_root;
+/* rw_lock protecting the access to object_list and prio_tree_root */
+static DEFINE_RWLOCK(kmemleak_lock);
+
+/* allocation caches for kmemleak internal data */
+static struct kmem_cache *object_cache;
+static struct kmem_cache *scan_area_cache;
+
+/* set if tracing memory operations is enabled */
+static atomic_t kmemleak_enabled = ATOMIC_INIT(0);
+/* set in the late_initcall if there were no errors */
+static atomic_t kmemleak_initialized = ATOMIC_INIT(0);
+/* enables or disables early logging of the memory operations */
+static atomic_t kmemleak_early_log = ATOMIC_INIT(1);
+/* set if a fata kmemleak error has occurred */
+static atomic_t kmemleak_error = ATOMIC_INIT(0);
+
+/* minimum and maximum address that may be valid pointers */
+static unsigned long min_addr = ULONG_MAX;
+static unsigned long max_addr;
+
+static struct task_struct *scan_thread;
+/* used to avoid reporting of recently allocated objects */
+static unsigned long jiffies_min_age;
+static unsigned long jiffies_last_scan;
+/* delay between automatic memory scannings */
+static signed long jiffies_scan_wait;
+/* enables or disables the task stacks scanning */
+static int kmemleak_stack_scan = 1;
+/* protects the memory scanning, parameters and debug/kmemleak file access */
+static DEFINE_MUTEX(scan_mutex);
+
+/*
+ * Early object allocation/freeing logging. Kmemleak is initialized after the
+ * kernel allocator. However, both the kernel allocator and kmemleak may
+ * allocate memory blocks which need to be tracked. Kmemleak defines an
+ * arbitrary buffer to hold the allocation/freeing information before it is
+ * fully initialized.
+ */
+
+/* kmemleak operation type for early logging */
+enum {
+        KMEMLEAK_ALLOC,
+        KMEMLEAK_FREE,
+        KMEMLEAK_FREE_PART,
+        KMEMLEAK_NOT_LEAK,
+        KMEMLEAK_IGNORE,
+        KMEMLEAK_SCAN_AREA,
+        KMEMLEAK_NO_SCAN
+};
+
+/*
+ * Structure holding the information passed to kmemleak callbacks during the
+ * early logging.
+ */
+struct early_log {
+        int op_type;                    /* kmemleak operation type */
+        const void *ptr;                /* allocated/freed memory block */
+        size_t size;                    /* memory block size */
+        int min_count;                  /* minimum reference count */
+        unsigned long offset;           /* scan area offset */
+        size_t length;                  /* scan area length */
+        unsigned long trace[MAX_TRACE]; /* stack trace */
+        unsigned int trace_len;         /* stack trace length */
+};
+
+/* early logging buffer and current position */
+static struct early_log
+        early_log[CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE] __initdata;
+static int crt_early_log __initdata;
+
+static void kmemleak_disable(void);
+
+/*
+ * Print a warning and dump the stack trace.
+ */
+#define kmemleak_warn(x...)     do {    \
+        pr_warning(x);                  \
+        dump_stack();                   \
+} while (0)
+
+/*
+ * Macro invoked when a serious kmemleak condition occured and cannot be
+ * recovered from. Kmemleak will be disabled and further allocation/freeing
+ * tracing no longer available.
+ */
+#define kmemleak_stop(x...)     do {    \
+        kmemleak_warn(x);               \
+        kmemleak_disable();             \
+} while (0)
+
+/*
+ * Printing of the objects hex dump to the seq file. The number of lines to be
+ * printed is limited to HEX_MAX_LINES to prevent seq file spamming. The
+ * actual number of printed bytes depends on HEX_ROW_SIZE. It must be called
+ * with the object->lock held.
+ */
+static void hex_dump_object(struct seq_file *seq,
+                            struct kmemleak_object *object)
+{
+        const u8 *ptr = (const u8 *)object->pointer;
+        int i, len, remaining;
+        unsigned char linebuf[HEX_ROW_SIZE * 5];
+
+        /* limit the number of lines to HEX_MAX_LINES */
+        remaining = len =
+                min(object->size, (size_t)(HEX_MAX_LINES * HEX_ROW_SIZE));
+
+        seq_printf(seq, "  hex dump (first %d bytes):\n", len);
+        for (i = 0; i < len; i += HEX_ROW_SIZE) {
+                int linelen = min(remaining, HEX_ROW_SIZE);
+
+                remaining -= HEX_ROW_SIZE;
+                hex_dump_to_buffer(ptr + i, linelen, HEX_ROW_SIZE,
+                                   HEX_GROUP_SIZE, linebuf, sizeof(linebuf),
+                                   HEX_ASCII);
+                seq_printf(seq, "    %s\n", linebuf);
+        }
+}
+
+/*
+ * Object colors, encoded with count and min_count:
+ * - white - orphan object, not enough references to it (count < min_count)
+ * - gray  - not orphan, not marked as false positive (min_count == 0) or
+ *              sufficient references to it (count >= min_count)
+ * - black - ignore, it doesn't contain references (e.g. text section)
+ *              (min_count == -1). No function defined for this color.
+ * Newly created objects don't have any color assigned (object->count == -1)
+ * before the next memory scan when they become white.
+ */
+static bool color_white(const struct kmemleak_object *object)
+{
+        return object->count != KMEMLEAK_BLACK &&
+                object->count < object->min_count;
+}
+
+static bool color_gray(const struct kmemleak_object *object)
+{
+        return object->min_count != KMEMLEAK_BLACK &&
+                object->count >= object->min_count;
+}
+
+static bool color_black(const struct kmemleak_object *object)
+{
+        return object->min_count == KMEMLEAK_BLACK;
+}
+
+/*
+ * Objects are considered unreferenced only if their color is white, they have
+ * not be deleted and have a minimum age to avoid false positives caused by
+ * pointers temporarily stored in CPU registers.
+ */
+static bool unreferenced_object(struct kmemleak_object *object)
+{
+        return (object->flags & OBJECT_ALLOCATED) && color_white(object) &&
+                time_before_eq(object->jiffies + jiffies_min_age,
+                               jiffies_last_scan);
+}
+
+/*
+ * Printing of the unreferenced objects information to the seq file. The
+ * print_unreferenced function must be called with the object->lock held.
+ */
+static void print_unreferenced(struct seq_file *seq,
+                               struct kmemleak_object *object)
+{
+        int i;
+
+        seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",
+                   object->pointer, object->size);
+        seq_printf(seq, "  comm \"%s\", pid %d, jiffies %lu\n",
+                   object->comm, object->pid, object->jiffies);
+        hex_dump_object(seq, object);
+        seq_printf(seq, "  backtrace:\n");
+
+        for (i = 0; i < object->trace_len; i++) {
+                void *ptr = (void *)object->trace[i];
+                seq_printf(seq, "    [<%p>] %pS\n", ptr, ptr);
+        }
+}
+
+/*
+ * Print the kmemleak_object information. This function is used mainly for
+ * debugging special cases when kmemleak operations. It must be called with
+ * the object->lock held.
+ */
+static void dump_object_info(struct kmemleak_object *object)
+{
+        struct stack_trace trace;
+
+        trace.nr_entries = object->trace_len;
+        trace.entries = object->trace;
+
+        pr_notice("Object 0x%08lx (size %zu):\n",
+                  object->tree_node.start, object->size);
+        pr_notice("  comm \"%s\", pid %d, jiffies %lu\n",
+                  object->comm, object->pid, object->jiffies);
+        pr_notice("  min_count = %d\n", object->min_count);
+        pr_notice("  count = %d\n", object->count);
+        pr_notice("  flags = 0x%lx\n", object->flags);
+        pr_notice("  backtrace:\n");
+        print_stack_trace(&trace, 4);
+}
+
+/*
+ * Look-up a memory block metadata (kmemleak_object) in the priority search
+ * tree based on a pointer value. If alias is 0, only values pointing to the
+ * beginning of the memory block are allowed. The kmemleak_lock must be held
+ * when calling this function.
+ */
+static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
+{
+        struct prio_tree_node *node;
+        struct prio_tree_iter iter;
+        struct kmemleak_object *object;
+
+        prio_tree_iter_init(&iter, &object_tree_root, ptr, ptr);
+        node = prio_tree_next(&iter);
+        if (node) {
+                object = prio_tree_entry(node, struct kmemleak_object,
+                                         tree_node);
+                if (!alias && object->pointer != ptr) {
+                        kmemleak_warn("Found object by alias");
+                        object = NULL;
+                }
+        } else
+                object = NULL;
+
+        return object;
+}
+
+/*
+ * Increment the object use_count. Return 1 if successful or 0 otherwise. Note
+ * that once an object's use_count reached 0, the RCU freeing was already
+ * registered and the object should no longer be used. This function must be
+ * called under the protection of rcu_read_lock().
+ */
+static int get_object(struct kmemleak_object *object)
+{
+        return atomic_inc_not_zero(&object->use_count);
+}
+
+/*
+ * RCU callback to free a kmemleak_object.
+ */
+static void free_object_rcu(struct rcu_head *rcu)
+{
+        struct hlist_node *elem, *tmp;
+        struct kmemleak_scan_area *area;
+        struct kmemleak_object *object =
+                container_of(rcu, struct kmemleak_object, rcu);
+
+        /*
+         * Once use_count is 0 (guaranteed by put_object), there is no other
+         * code accessing this object, hence no need for locking.
+         */
+        hlist_for_each_entry_safe(area, elem, tmp, &object->area_list, node) {
+                hlist_del(elem);
+                kmem_cache_free(scan_area_cache, area);
+        }
+        kmem_cache_free(object_cache, object);
+}
+
+/*
+ * Decrement the object use_count. Once the count is 0, free the object using
+ * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
+ * delete_object() path, the delayed RCU freeing ensures that there is no
+ * recursive call to the kernel allocator. Lock-less RCU object_list traversal
+ * is also possible.
+ */
+static void put_object(struct kmemleak_object *object)
+{
+        if (!atomic_dec_and_test(&object->use_count))
+                return;
+
+        /* should only get here after delete_object was called */
+        WARN_ON(object->flags & OBJECT_ALLOCATED);
+
+        call_rcu(&object->rcu, free_object_rcu);
+}
+
+/*
+ * Look up an object in the prio search tree and increase its use_count.
+ */
+static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
+{
+        unsigned long flags;
+        struct kmemleak_object *object = NULL;
+
+        rcu_read_lock();
+        read_lock_irqsave(&kmemleak_lock, flags);
+        if (ptr >= min_addr && ptr < max_addr)
+                object = lookup_object(ptr, alias);
+        read_unlock_irqrestore(&kmemleak_lock, flags);
+
+        /* check whether the object is still available */
+        if (object && !get_object(object))
+                object = NULL;
+        rcu_read_unlock();
+
+        return object;
+}
+
+/*
+ * Save stack trace to the given array of MAX_TRACE size.
+ */
+static int __save_stack_trace(unsigned long *trace)
+{
+        struct stack_trace stack_trace;
+
+        stack_trace.max_entries = MAX_TRACE;
+        stack_trace.nr_entries = 0;
+        stack_trace.entries = trace;
+        stack_trace.skip = 2;
+        save_stack_trace(&stack_trace);
+
+        return stack_trace.nr_entries;
+}
+
+/*
+ * Create the metadata (struct kmemleak_object) corresponding to an allocated
+ * memory block and add it to the object_list and object_tree_root.
+ */
+static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
+                                             int min_count, gfp_t gfp)
+{
+        unsigned long flags;
+        struct kmemleak_object *object;
+        struct prio_tree_node *node;
+
+        object = kmem_cache_alloc(object_cache, gfp & GFP_KMEMLEAK_MASK);
+        if (!object) {
+                kmemleak_stop("Cannot allocate a kmemleak_object structure\n");
+                return NULL;
+        }
+
+        INIT_LIST_HEAD(&object->object_list);
+        INIT_LIST_HEAD(&object->gray_list);
+        INIT_HLIST_HEAD(&object->area_list);
+        spin_lock_init(&object->lock);
+        atomic_set(&object->use_count, 1);
+        object->flags = OBJECT_ALLOCATED | OBJECT_NEW;
+        object->pointer = ptr;
+        object->size = size;
+        object->min_count = min_count;
+        object->count = -1;                     /* no color initially */
+        object->jiffies = jiffies;
+
+        /* task information */
+        if (in_irq()) {
+                object->pid = 0;
+                strncpy(object->comm, "hardirq", sizeof(object->comm));
+        } else if (in_softirq()) {
+                object->pid = 0;
+                strncpy(object->comm, "softirq", sizeof(object->comm));
+        } else {
+                object->pid = current->pid;
+                /*
+                 * There is a small chance of a race with set_task_comm(),
+                 * however using get_task_comm() here may cause locking
+                 * dependency issues with current->alloc_lock. In the worst
+                 * case, the command line is not correct.
+                 */
+                strncpy(object->comm, current->comm, sizeof(object->comm));
+        }
+
+        /* kernel backtrace */
+        object->trace_len = __save_stack_trace(object->trace);
+
+        INIT_PRIO_TREE_NODE(&object->tree_node);
+        object->tree_node.start = ptr;
+        object->tree_node.last = ptr + size - 1;
+
+        write_lock_irqsave(&kmemleak_lock, flags);
+
+        min_addr = min(min_addr, ptr);
+        max_addr = max(max_addr, ptr + size);
+        node = prio_tree_insert(&object_tree_root, &object->tree_node);
+        /*
+         * The code calling the kernel does not yet have the pointer to the
+         * memory block to be able to free it.  However, we still hold the
+         * kmemleak_lock here in case parts of the kernel started freeing
+         * random memory blocks.
+         */
+        if (node != &object->tree_node) {
+                kmemleak_stop("Cannot insert 0x%lx into the object search tree "
+                              "(already existing)\n", ptr);
+                object = lookup_object(ptr, 1);
+                spin_lock(&object->lock);
+                dump_object_info(object);
+                spin_unlock(&object->lock);
+
+                goto out;
+        }
+        list_add_tail_rcu(&object->object_list, &object_list);
+out:
+        write_unlock_irqrestore(&kmemleak_lock, flags);
+        return object;
+}
+
+/*
+ * Remove the metadata (struct kmemleak_object) for a memory block from the
+ * object_list and object_tree_root and decrement its use_count.
+ */
+static void __delete_object(struct kmemleak_object *object)
+{
+        unsigned long flags;
+
+        write_lock_irqsave(&kmemleak_lock, flags);
+        prio_tree_remove(&object_tree_root, &object->tree_node);
+        list_del_rcu(&object->object_list);
+        write_unlock_irqrestore(&kmemleak_lock, flags);
+
+        WARN_ON(!(object->flags & OBJECT_ALLOCATED));
+        WARN_ON(atomic_read(&object->use_count) < 2);
+
+        /*
+         * Locking here also ensures that the corresponding memory block
+         * cannot be freed when it is being scanned.
+         */
+        spin_lock_irqsave(&object->lock, flags);
+        object->flags &= ~OBJECT_ALLOCATED;
+        spin_unlock_irqrestore(&object->lock, flags);
+        put_object(object);
+}
+
+/*
+ * Look up the metadata (struct kmemleak_object) corresponding to ptr and
+ * delete it.
+ */
+static void delete_object_full(unsigned long ptr)
+{
+        struct kmemleak_object *object;
+
+        object = find_and_get_object(ptr, 0);
+        if (!object) {
+#ifdef DEBUG
+                kmemleak_warn("Freeing unknown object at 0x%08lx\n",
+                              ptr);
+#endif
+                return;
+        }
+        __delete_object(object);
+        put_object(object);
+}
+
+/*
+ * Look up the metadata (struct kmemleak_object) corresponding to ptr and
+ * delete it. If the memory block is partially freed, the function may create
+ * additional metadata for the remaining parts of the block.
+ */
+static void delete_object_part(unsigned long ptr, size_t size)
+{
+        struct kmemleak_object *object;
+        unsigned long start, end;
+
+        object = find_and_get_object(ptr, 1);
+        if (!object) {
+#ifdef DEBUG
+                kmemleak_warn("Partially freeing unknown object at 0x%08lx "
+                              "(size %zu)\n", ptr, size);
+#endif
+                return;
+        }
+        __delete_object(object);
+
+        /*
+         * Create one or two objects that may result from the memory block
+         * split. Note that partial freeing is only done by free_bootmem() and
+         * this happens before kmemleak_init() is called. The path below is
+         * only executed during early log recording in kmemleak_init(), so
+         * GFP_KERNEL is enough.
+         */
+        start = object->pointer;
+        end = object->pointer + object->size;
+        if (ptr > start)
+                create_object(start, ptr - start, object->min_count,
+                              GFP_KERNEL);
+        if (ptr + size < end)
+                create_object(ptr + size, end - ptr - size, object->min_count,
+                              GFP_KERNEL);
+
+        put_object(object);
+}
+
+static void __paint_it(struct kmemleak_object *object, int color)
+{
+        object->min_count = color;
+        if (color == KMEMLEAK_BLACK)
+                object->flags |= OBJECT_NO_SCAN;
+}
+
+static void paint_it(struct kmemleak_object *object, int color)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&object->lock, flags);
+        __paint_it(object, color);
+        spin_unlock_irqrestore(&object->lock, flags);
+}
+
+static void paint_ptr(unsigned long ptr, int color)
+{
+        struct kmemleak_object *object;
+
+        object = find_and_get_object(ptr, 0);
+        if (!object) {
+                kmemleak_warn("Trying to color unknown object "
+                              "at 0x%08lx as %s\n", ptr,
+                              (color == KMEMLEAK_GREY) ? "Grey" :
+                              (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
+                return;
+        }
+        paint_it(object, color);
+        put_object(object);
+}
+
+/*
+ * Make a object permanently as gray-colored so that it can no longer be
+ * reported as a leak. This is used in general to mark a false positive.
+ */
+static void make_gray_object(unsigned long ptr)
+{
+        paint_ptr(ptr, KMEMLEAK_GREY);
+}
+
+/*
+ * Mark the object as black-colored so that it is ignored from scans and
+ * reporting.
+ */
+static void make_black_object(unsigned long ptr)
+{
+        paint_ptr(ptr, KMEMLEAK_BLACK);
+}
+
+/*
+ * Add a scanning area to the object. If at least one such area is added,
+ * kmemleak will only scan these ranges rather than the whole memory block.
+ */
+static void add_scan_area(unsigned long ptr, unsigned long offset,
+                          size_t length, gfp_t gfp)
+{
+        unsigned long flags;
+        struct kmemleak_object *object;
+        struct kmemleak_scan_area *area;
+
+        object = find_and_get_object(ptr, 0);
+        if (!object) {
+                kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
+                              ptr);
+                return;
+        }
+
+        area = kmem_cache_alloc(scan_area_cache, gfp & GFP_KMEMLEAK_MASK);
+        if (!area) {
+                kmemleak_warn("Cannot allocate a scan area\n");
+                goto out;
+        }
+
+        spin_lock_irqsave(&object->lock, flags);
+        if (offset + length > object->size) {
+                kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
+                dump_object_info(object);
+                kmem_cache_free(scan_area_cache, area);
+                goto out_unlock;
+        }
+
+        INIT_HLIST_NODE(&area->node);
+        area->offset = offset;
+        area->length = length;
+
+        hlist_add_head(&area->node, &object->area_list);
+out_unlock:
+        spin_unlock_irqrestore(&object->lock, flags);
+out:
+        put_object(object);
+}
+
+/*
+ * Set the OBJECT_NO_SCAN flag for the object corresponding to the give
+ * pointer. Such object will not be scanned by kmemleak but references to it
+ * are searched.
+ */
+static void object_no_scan(unsigned long ptr)
+{
+        unsigned long flags;
+        struct kmemleak_object *object;
+
+        object = find_and_get_object(ptr, 0);
+        if (!object) {
+                kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
+                return;
+        }
+
+        spin_lock_irqsave(&object->lock, flags);
+        object->flags |= OBJECT_NO_SCAN;
+        spin_unlock_irqrestore(&object->lock, flags);
+        put_object(object);
+}
+
+/*
+ * Log an early kmemleak_* call to the early_log buffer. These calls will be
+ * processed later once kmemleak is fully initialized.
+ */
+static void __init log_early(int op_type, const void *ptr, size_t size,
+                             int min_count, unsigned long offset, size_t length)
+{
+        unsigned long flags;
+        struct early_log *log;
+
+        if (crt_early_log >= ARRAY_SIZE(early_log)) {
+                pr_warning("Early log buffer exceeded, "
+                           "please increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n");
+                kmemleak_disable();
+                return;
+        }
+
+        /*
+         * There is no need for locking since the kernel is still in UP mode
+         * at this stage. Disabling the IRQs is enough.
+         */
+        local_irq_save(flags);
+        log = &early_log[crt_early_log];
+        log->op_type = op_type;
+        log->ptr = ptr;
+        log->size = size;
+        log->min_count = min_count;
+        log->offset = offset;
+        log->length = length;
+        if (op_type == KMEMLEAK_ALLOC)
+                log->trace_len = __save_stack_trace(log->trace);
+        crt_early_log++;
+        local_irq_restore(flags);
+}
+
+/*
+ * Log an early allocated block and populate the stack trace.
+ */
+static void early_alloc(struct early_log *log)
+{
+        struct kmemleak_object *object;
+        unsigned long flags;
+        int i;
+
+        if (!atomic_read(&kmemleak_enabled) || !log->ptr || IS_ERR(log->ptr))
+                return;
+
+        /*
+         * RCU locking needed to ensure object is not freed via put_object().
+         */
+        rcu_read_lock();
+        object = create_object((unsigned long)log->ptr, log->size,
+                               log->min_count, GFP_ATOMIC);
+        if (!object)
+                goto out;
+        spin_lock_irqsave(&object->lock, flags);
+        for (i = 0; i < log->trace_len; i++)
+                object->trace[i] = log->trace[i];
+        object->trace_len = log->trace_len;
+        spin_unlock_irqrestore(&object->lock, flags);
+out:
+        rcu_read_unlock();
+}
+
+/*
+ * Memory allocation function callback. This function is called from the
+ * kernel allocators when a new block is allocated (kmem_cache_alloc, kmalloc,
+ * vmalloc etc.).
+ */
+void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
+                          gfp_t gfp)
+{
+        pr_debug("%s(0x%p, %zu, %d)\n", __func__, ptr, size, min_count);
+
+        if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
+                create_object((unsigned long)ptr, size, min_count, gfp);
+        else if (atomic_read(&kmemleak_early_log))
+                log_early(KMEMLEAK_ALLOC, ptr, size, min_count, 0, 0);
+}
+EXPORT_SYMBOL_GPL(kmemleak_alloc);
+
+/*
+ * Memory freeing function callback. This function is called from the kernel
+ * allocators when a block is freed (kmem_cache_free, kfree, vfree etc.).
+ */
+void __ref kmemleak_free(const void *ptr)
+{
+        pr_debug("%s(0x%p)\n", __func__, ptr);
+
+        if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
+                delete_object_full((unsigned long)ptr);
+        else if (atomic_read(&kmemleak_early_log))
+                log_early(KMEMLEAK_FREE, ptr, 0, 0, 0, 0);
+}
+EXPORT_SYMBOL_GPL(kmemleak_free);
+
+/*
+ * Partial memory freeing function callback. This function is usually called
+ * from bootmem allocator when (part of) a memory block is freed.
+ */
+void __ref kmemleak_free_part(const void *ptr, size_t size)
+{
+        pr_debug("%s(0x%p)\n", __func__, ptr);
+
+        if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
+                delete_object_part((unsigned long)ptr, size);
+        else if (atomic_read(&kmemleak_early_log))
+                log_early(KMEMLEAK_FREE_PART, ptr, size, 0, 0, 0);
+}
+EXPORT_SYMBOL_GPL(kmemleak_free_part);
+
+/*
+ * Mark an already allocated memory block as a false positive. This will cause
+ * the block to no longer be reported as leak and always be scanned.
+ */
+void __ref kmemleak_not_leak(const void *ptr)
+{
+        pr_debug("%s(0x%p)\n", __func__, ptr);
+
+        if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
+                make_gray_object((unsigned long)ptr);
+        else if (atomic_read(&kmemleak_early_log))
+                log_early(KMEMLEAK_NOT_LEAK, ptr, 0, 0, 0, 0);
+}
+EXPORT_SYMBOL(kmemleak_not_leak);
+
+/*
+ * Ignore a memory block. This is usually done when it is known that the
+ * corresponding block is not a leak and does not contain any references to
+ * other allocated memory blocks.
+ */
+void __ref kmemleak_ignore(const void *ptr)
+{
+        pr_debug("%s(0x%p)\n", __func__, ptr);
+
+        if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
+                make_black_object((unsigned long)ptr);
+        else if (atomic_read(&kmemleak_early_log))
+                log_early(KMEMLEAK_IGNORE, ptr, 0, 0, 0, 0);
+}
+EXPORT_SYMBOL(kmemleak_ignore);
+
+/*
+ * Limit the range to be scanned in an allocated memory block.
+ */
+void __ref kmemleak_scan_area(const void *ptr, unsigned long offset,
+                              size_t length, gfp_t gfp)
+{
+        pr_debug("%s(0x%p)\n", __func__, ptr);
+
+        if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
+                add_scan_area((unsigned long)ptr, offset, length, gfp);
+        else if (atomic_read(&kmemleak_early_log))
+                log_early(KMEMLEAK_SCAN_AREA, ptr, 0, 0, offset, length);
+}
+EXPORT_SYMBOL(kmemleak_scan_area);
+
+/*
+ * Inform kmemleak not to scan the given memory block.
+ */
+void __ref kmemleak_no_scan(const void *ptr)
+{
+        pr_debug("%s(0x%p)\n", __func__, ptr);
+
+        if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
+                object_no_scan((unsigned long)ptr);
+        else if (atomic_read(&kmemleak_early_log))
+                log_early(KMEMLEAK_NO_SCAN, ptr, 0, 0, 0, 0);
+}
+EXPORT_SYMBOL(kmemleak_no_scan);
+
+/*
+ * Memory scanning is a long process and it needs to be interruptable. This
+ * function checks whether such interrupt condition occured.
+ */
+static int scan_should_stop(void)
+{
+        if (!atomic_read(&kmemleak_enabled))
+                return 1;
+
+        /*
+         * This function may be called from either process or kthread context,
+         * hence the need to check for both stop conditions.
+         */
+        if (current->mm)
+                return signal_pending(current);
+        else
+                return kthread_should_stop();
+
+        return 0;
+}
+
+/*
+ * Scan a memory block (exclusive range) for valid pointers and add those
+ * found to the gray list.
+ */
+static void scan_block(void *_start, void *_end,
+                       struct kmemleak_object *scanned, int allow_resched)
+{
+        unsigned long *ptr;
+        unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
+        unsigned long *end = _end - (BYTES_PER_POINTER - 1);
+
+        for (ptr = start; ptr < end; ptr++) {
+                struct kmemleak_object *object;
+                unsigned long flags;
+                unsigned long pointer;
+
+                if (allow_resched)
+                        cond_resched();
+                if (scan_should_stop())
+                        break;
+
+                /* don't scan uninitialized memory */
+                if (!kmemcheck_is_obj_initialized((unsigned long)ptr,
+                                                  BYTES_PER_POINTER))
+                        continue;
+
+                pointer = *ptr;
+
+                object = find_and_get_object(pointer, 1);
+                if (!object)
+                        continue;
+                if (object == scanned) {
+                        /* self referenced, ignore */
+                        put_object(object);
+                        continue;
+                }
+
+                /*
+                 * Avoid the lockdep recursive warning on object->lock being
+                 * previously acquired in scan_object(). These locks are
+                 * enclosed by scan_mutex.
+                 */
+                spin_lock_irqsave_nested(&object->lock, flags,
+                                         SINGLE_DEPTH_NESTING);
+                if (!color_white(object)) {
+                        /* non-orphan, ignored or new */
+                        spin_unlock_irqrestore(&object->lock, flags);
+                        put_object(object);
+                        continue;
+                }
+
+                /*
+                 * Increase the object's reference count (number of pointers
+                 * to the memory block). If this count reaches the required
+                 * minimum, the object's color will become gray and it will be
+                 * added to the gray_list.
+                 */
+                object->count++;
+                if (color_gray(object))
+                        list_add_tail(&object->gray_list, &gray_list);
+                else
+                        put_object(object);
+                spin_unlock_irqrestore(&object->lock, flags);
+        }
+}
+
+/*
+ * Scan a memory block corresponding to a kmemleak_object. A condition is
+ * that object->use_count >= 1.
+ */
+static void scan_object(struct kmemleak_object *object)
+{
+        struct kmemleak_scan_area *area;
+        struct hlist_node *elem;
+        unsigned long flags;
+
+        /*
+         * Once the object->lock is aquired, the corresponding memory block
+         * cannot be freed (the same lock is aquired in delete_object).
+         */
+        spin_lock_irqsave(&object->lock, flags);
+        if (object->flags & OBJECT_NO_SCAN)
+                goto out;
+        if (!(object->flags & OBJECT_ALLOCATED))
+                /* already freed object */
+                goto out;
+        if (hlist_empty(&object->area_list)) {
+                void *start = (void *)object->pointer;
+                void *end = (void *)(object->pointer + object->size);
+
+                while (start < end && (object->flags & OBJECT_ALLOCATED) &&
+                       !(object->flags & OBJECT_NO_SCAN)) {
+                        scan_block(start, min(start + MAX_SCAN_SIZE, end),
+                                   object, 0);
+                        start += MAX_SCAN_SIZE;
+
+                        spin_unlock_irqrestore(&object->lock, flags);
+                        cond_resched();
+                        spin_lock_irqsave(&object->lock, flags);
+                }
+        } else
+                hlist_for_each_entry(area, elem, &object->area_list, node)
+                        scan_block((void *)(object->pointer + area->offset),
+                                   (void *)(object->pointer + area->offset
+                                            + area->length), object, 0);
+out:
+        spin_unlock_irqrestore(&object->lock, flags);
+}
+
+/*
+ * Scan data sections and all the referenced memory blocks allocated via the
+ * kernel's standard allocators. This function must be called with the
+ * scan_mutex held.
+ */
+static void kmemleak_scan(void)
+{
+        unsigned long flags;
+        struct kmemleak_object *object, *tmp;
+        int i;
+        int new_leaks = 0;
+        int gray_list_pass = 0;
+
+        jiffies_last_scan = jiffies;
+
+        /* prepare the kmemleak_object's */
+        rcu_read_lock();
+        list_for_each_entry_rcu(object, &object_list, object_list) {
+                spin_lock_irqsave(&object->lock, flags);
+#ifdef DEBUG
+                /*
+                 * With a few exceptions there should be a maximum of
+                 * 1 reference to any object at this point.
+                 */
+                if (atomic_read(&object->use_count) > 1) {
+                        pr_debug("object->use_count = %d\n",
+                                 atomic_read(&object->use_count));
+                        dump_object_info(object);
+                }
+#endif
+                /* reset the reference count (whiten the object) */
+                object->count = 0;
+                object->flags &= ~OBJECT_NEW;
+                if (color_gray(object) && get_object(object))
+                        list_add_tail(&object->gray_list, &gray_list);
+
+                spin_unlock_irqrestore(&object->lock, flags);
+        }
+        rcu_read_unlock();
+
+        /* data/bss scanning */
+        scan_block(_sdata, _edata, NULL, 1);
+        scan_block(__bss_start, __bss_stop, NULL, 1);
+
+#ifdef CONFIG_SMP
+        /* per-cpu sections scanning */
+        for_each_possible_cpu(i)
+                scan_block(__per_cpu_start + per_cpu_offset(i),
+                           __per_cpu_end + per_cpu_offset(i), NULL, 1);
+#endif
+
+        /*
+         * Struct page scanning for each node. The code below is not yet safe
+         * with MEMORY_HOTPLUG.
+         */
+        for_each_online_node(i) {
+                pg_data_t *pgdat = NODE_DATA(i);
+                unsigned long start_pfn = pgdat->node_start_pfn;
+                unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages;
+                unsigned long pfn;
+
+                for (pfn = start_pfn; pfn < end_pfn; pfn++) {
+                        struct page *page;
+
+                        if (!pfn_valid(pfn))
+                                continue;
+                        page = pfn_to_page(pfn);
+                        /* only scan if page is in use */
+                        if (page_count(page) == 0)
+                                continue;
+                        scan_block(page, page + 1, NULL, 1);
+                }
+        }
+
+        /*
+         * Scanning the task stacks (may introduce false negatives).
+         */
+        if (kmemleak_stack_scan) {
+                struct task_struct *p, *g;
+
+                read_lock(&tasklist_lock);
+                do_each_thread(g, p) {
+                        scan_block(task_stack_page(p), task_stack_page(p) +
+                                   THREAD_SIZE, NULL, 0);
+                } while_each_thread(g, p);
+                read_unlock(&tasklist_lock);
+        }
+
+        /*
+         * Scan the objects already referenced from the sections scanned
+         * above. More objects will be referenced and, if there are no memory
+         * leaks, all the objects will be scanned. The list traversal is safe
+         * for both tail additions and removals from inside the loop. The
+         * kmemleak objects cannot be freed from outside the loop because their
+         * use_count was increased.
+         */
+repeat:
+        object = list_entry(gray_list.next, typeof(*object), gray_list);
+        while (&object->gray_list != &gray_list) {
+                cond_resched();
+
+                /* may add new objects to the list */
+                if (!scan_should_stop())
+                        scan_object(object);
+
+                tmp = list_entry(object->gray_list.next, typeof(*object),
+                                 gray_list);
+
+                /* remove the object from the list and release it */
+                list_del(&object->gray_list);
+                put_object(object);
+
+                object = tmp;
+        }
+
+        if (scan_should_stop() || ++gray_list_pass >= GRAY_LIST_PASSES)
+                goto scan_end;
+
+        /*
+         * Check for new objects allocated during this scanning and add them
+         * to the gray list.
+         */
+        rcu_read_lock();
+        list_for_each_entry_rcu(object, &object_list, object_list) {
+                spin_lock_irqsave(&object->lock, flags);
+                if ((object->flags & OBJECT_NEW) && !color_black(object) &&
+                    get_object(object)) {
+                        object->flags &= ~OBJECT_NEW;
+                        list_add_tail(&object->gray_list, &gray_list);
+                }
+                spin_unlock_irqrestore(&object->lock, flags);
+        }
+        rcu_read_unlock();
+
+        if (!list_empty(&gray_list))
+                goto repeat;
+
+scan_end:
+        WARN_ON(!list_empty(&gray_list));
+
+        /*
+         * If scanning was stopped or new objects were being allocated at a
+         * higher rate than gray list scanning, do not report any new
+         * unreferenced objects.
+         */
+        if (scan_should_stop() || gray_list_pass >= GRAY_LIST_PASSES)
+                return;
+
+        /*
+         * Scanning result reporting.
+         */
+        rcu_read_lock();
+        list_for_each_entry_rcu(object, &object_list, object_list) {
+                spin_lock_irqsave(&object->lock, flags);
+                if (unreferenced_object(object) &&
+                    !(object->flags & OBJECT_REPORTED)) {
+                        object->flags |= OBJECT_REPORTED;
+                        new_leaks++;
+                }
+                spin_unlock_irqrestore(&object->lock, flags);
+        }
+        rcu_read_unlock();
+
+        if (new_leaks)
+                pr_info("%d new suspected memory leaks (see "
+                        "/sys/kernel/debug/kmemleak)\n", new_leaks);
+
+}
+
+/*
+ * Thread function performing automatic memory scanning. Unreferenced objects
+ * at the end of a memory scan are reported but only the first time.
+ */
+static int kmemleak_scan_thread(void *arg)
+{
+        static int first_run = 1;
+
+        pr_info("Automatic memory scanning thread started\n");
+        set_user_nice(current, 10);
+
+        /*
+         * Wait before the first scan to allow the system to fully initialize.
+         */
+        if (first_run) {
+                first_run = 0;
+                ssleep(SECS_FIRST_SCAN);
+        }
+
+        while (!kthread_should_stop()) {
+                signed long timeout = jiffies_scan_wait;
+
+                mutex_lock(&scan_mutex);
+                kmemleak_scan();
+                mutex_unlock(&scan_mutex);
+
+                /* wait before the next scan */
+                while (timeout && !kthread_should_stop())
+                        timeout = schedule_timeout_interruptible(timeout);
+        }
+
+        pr_info("Automatic memory scanning thread ended\n");
+
+        return 0;
+}
+
+/*
+ * Start the automatic memory scanning thread. This function must be called
+ * with the scan_mutex held.
+ */
+static void start_scan_thread(void)
+{
+        if (scan_thread)
+                return;
+        scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
+        if (IS_ERR(scan_thread)) {
+                pr_warning("Failed to create the scan thread\n");
+                scan_thread = NULL;
+        }
+}
+
+/*
+ * Stop the automatic memory scanning thread. This function must be called
+ * with the scan_mutex held.
+ */
+static void stop_scan_thread(void)
+{
+        if (scan_thread) {
+                kthread_stop(scan_thread);
+                scan_thread = NULL;
+        }
+}
+
+/*
+ * Iterate over the object_list and return the first valid object at or after
+ * the required position with its use_count incremented. The function triggers
+ * a memory scanning when the pos argument points to the first position.
+ */
+static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)
+{
+        struct kmemleak_object *object;
+        loff_t n = *pos;
+        int err;
+
+        err = mutex_lock_interruptible(&scan_mutex);
+        if (err < 0)
+                return ERR_PTR(err);
+
+        rcu_read_lock();
+        list_for_each_entry_rcu(object, &object_list, object_list) {
+                if (n-- > 0)
+                        continue;
+                if (get_object(object))
+                        goto out;
+        }
+        object = NULL;
+out:
+        return object;
+}
+
+/*
+ * Return the next object in the object_list. The function decrements the
+ * use_count of the previous object and increases that of the next one.
+ */
+static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+        struct kmemleak_object *prev_obj = v;
+        struct kmemleak_object *next_obj = NULL;
+        struct list_head *n = &prev_obj->object_list;
+
+        ++(*pos);
+
+        list_for_each_continue_rcu(n, &object_list) {
+                next_obj = list_entry(n, struct kmemleak_object, object_list);
+                if (get_object(next_obj))
+                        break;
+        }
+
+        put_object(prev_obj);
+        return next_obj;
+}
+
+/*
+ * Decrement the use_count of the last object required, if any.
+ */
+static void kmemleak_seq_stop(struct seq_file *seq, void *v)
+{
+        if (!IS_ERR(v)) {
+                /*
+                 * kmemleak_seq_start may return ERR_PTR if the scan_mutex
+                 * waiting was interrupted, so only release it if !IS_ERR.
+                 */
+                rcu_read_unlock();
+                mutex_unlock(&scan_mutex);
+                if (v)
+                        put_object(v);
+        }
+}
+
+/*
+ * Print the information for an unreferenced object to the seq file.
+ */
+static int kmemleak_seq_show(struct seq_file *seq, void *v)
+{
+        struct kmemleak_object *object = v;
+        unsigned long flags;
+
+        spin_lock_irqsave(&object->lock, flags);
+        if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
+                print_unreferenced(seq, object);
+        spin_unlock_irqrestore(&object->lock, flags);
+        return 0;
+}
+
+static const struct seq_operations kmemleak_seq_ops = {
+        .start = kmemleak_seq_start,
+        .next  = kmemleak_seq_next,
+        .stop  = kmemleak_seq_stop,
+        .show  = kmemleak_seq_show,
+};
+
+static int kmemleak_open(struct inode *inode, struct file *file)
+{
+        if (!atomic_read(&kmemleak_enabled))
+                return -EBUSY;
+
+        return seq_open(file, &kmemleak_seq_ops);
+}
+
+static int kmemleak_release(struct inode *inode, struct file *file)
+{
+        return seq_release(inode, file);
+}
+
+static int dump_str_object_info(const char *str)
+{
+        unsigned long flags;
+        struct kmemleak_object *object;
+        unsigned long addr;
+
+        addr= simple_strtoul(str, NULL, 0);
+        object = find_and_get_object(addr, 0);
+        if (!object) {
+                pr_info("Unknown object at 0x%08lx\n", addr);
+                return -EINVAL;
+        }
+
+        spin_lock_irqsave(&object->lock, flags);
+        dump_object_info(object);
+        spin_unlock_irqrestore(&object->lock, flags);
+
+        put_object(object);
+        return 0;
+}
+
+/*
+ * We use grey instead of black to ensure we can do future scans on the same
+ * objects. If we did not do future scans these black objects could
+ * potentially contain references to newly allocated objects in the future and
+ * we'd end up with false positives.
+ */
+static void kmemleak_clear(void)
+{
+        struct kmemleak_object *object;
+        unsigned long flags;
+
+        rcu_read_lock();
+        list_for_each_entry_rcu(object, &object_list, object_list) {
+                spin_lock_irqsave(&object->lock, flags);
+                if ((object->flags & OBJECT_REPORTED) &&
+                    unreferenced_object(object))
+                        __paint_it(object, KMEMLEAK_GREY);
+                spin_unlock_irqrestore(&object->lock, flags);
+        }
+        rcu_read_unlock();
+}
+
+/*
+ * File write operation to configure kmemleak at run-time. The following
+ * commands can be written to the /sys/kernel/debug/kmemleak file:
+ *   off        - disable kmemleak (irreversible)
+ *   stack=on   - enable the task stacks scanning
+ *   stack=off  - disable the tasks stacks scanning
+ *   scan=on    - start the automatic memory scanning thread
+ *   scan=off   - stop the automatic memory scanning thread
+ *   scan=...   - set the automatic memory scanning period in seconds (0 to
+ *                disable it)
+ *   scan       - trigger a memory scan
+ *   clear      - mark all current reported unreferenced kmemleak objects as
+ *                grey to ignore printing them
+ *   dump=...   - dump information about the object found at the given address
+ */
+static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,
+                              size_t size, loff_t *ppos)
+{
+        char buf[64];
+        int buf_size;
+        int ret;
+
+        buf_size = min(size, (sizeof(buf) - 1));
+        if (strncpy_from_user(buf, user_buf, buf_size) < 0)
+                return -EFAULT;
+        buf[buf_size] = 0;
+
+        ret = mutex_lock_interruptible(&scan_mutex);
+        if (ret < 0)
+                return ret;
+
+        if (strncmp(buf, "off", 3) == 0)
+                kmemleak_disable();
+        else if (strncmp(buf, "stack=on", 8) == 0)
+                kmemleak_stack_scan = 1;
+        else if (strncmp(buf, "stack=off", 9) == 0)
+                kmemleak_stack_scan = 0;
+        else if (strncmp(buf, "scan=on", 7) == 0)
+                start_scan_thread();
+        else if (strncmp(buf, "scan=off", 8) == 0)
+                stop_scan_thread();
+        else if (strncmp(buf, "scan=", 5) == 0) {
+                unsigned long secs;
+
+                ret = strict_strtoul(buf + 5, 0, &secs);
+                if (ret < 0)
+                        goto out;
+                stop_scan_thread();
+                if (secs) {
+                        jiffies_scan_wait = msecs_to_jiffies(secs * 1000);
+                        start_scan_thread();
+                }
+        } else if (strncmp(buf, "scan", 4) == 0)
+                kmemleak_scan();
+        else if (strncmp(buf, "clear", 5) == 0)
+                kmemleak_clear();
+        else if (strncmp(buf, "dump=", 5) == 0)
+                ret = dump_str_object_info(buf + 5);
+        else
+                ret = -EINVAL;
+
+out:
+        mutex_unlock(&scan_mutex);
+        if (ret < 0)
+                return ret;
+
+        /* ignore the rest of the buffer, only one command at a time */
+        *ppos += size;
+        return size;
+}
+
+static const struct file_operations kmemleak_fops = {
+        .owner          = THIS_MODULE,
+        .open           = kmemleak_open,
+        .read           = seq_read,
+        .write          = kmemleak_write,
+        .llseek         = seq_lseek,
+        .release        = kmemleak_release,
+};
+
+/*
+ * Perform the freeing of the kmemleak internal objects after waiting for any
+ * current memory scan to complete.
+ */
+static void kmemleak_do_cleanup(struct work_struct *work)
+{
+        struct kmemleak_object *object;
+
+        mutex_lock(&scan_mutex);
+        stop_scan_thread();
+
+        rcu_read_lock();
+        list_for_each_entry_rcu(object, &object_list, object_list)
+                delete_object_full(object->pointer);
+        rcu_read_unlock();
+        mutex_unlock(&scan_mutex);
+}
+
+static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
+
+/*
+ * Disable kmemleak. No memory allocation/freeing will be traced once this
+ * function is called. Disabling kmemleak is an irreversible operation.
+ */
+static void kmemleak_disable(void)
+{
+        /* atomically check whether it was already invoked */
+        if (atomic_cmpxchg(&kmemleak_error, 0, 1))
+                return;
+
+        /* stop any memory operation tracing */
+        atomic_set(&kmemleak_early_log, 0);
+        atomic_set(&kmemleak_enabled, 0);
+
+        /* check whether it is too early for a kernel thread */
+        if (atomic_read(&kmemleak_initialized))
+                schedule_work(&cleanup_work);
+
+        pr_info("Kernel memory leak detector disabled\n");
+}
+
+/*
+ * Allow boot-time kmemleak disabling (enabled by default).
+ */
+static int kmemleak_boot_config(char *str)
+{
+        if (!str)
+                return -EINVAL;
+        if (strcmp(str, "off") == 0)
+                kmemleak_disable();
+        else if (strcmp(str, "on") != 0)
+                return -EINVAL;
+        return 0;
+}
+early_param("kmemleak", kmemleak_boot_config);
+
+/*
+ * Kmemleak initialization.
+ */
+void __init kmemleak_init(void)
+{
+        int i;
+        unsigned long flags;
+
+        jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
+        jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
+
+        object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
+        scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
+        INIT_PRIO_TREE_ROOT(&object_tree_root);
+
+        /* the kernel is still in UP mode, so disabling the IRQs is enough */
+        local_irq_save(flags);
+        if (!atomic_read(&kmemleak_error)) {
+                atomic_set(&kmemleak_enabled, 1);
+                atomic_set(&kmemleak_early_log, 0);
+        }
+        local_irq_restore(flags);
+
+        /*
+         * This is the point where tracking allocations is safe. Automatic
+         * scanning is started during the late initcall. Add the early logged
+         * callbacks to the kmemleak infrastructure.
+         */
+        for (i = 0; i < crt_early_log; i++) {
+                struct early_log *log = &early_log[i];
+
+                switch (log->op_type) {
+                case KMEMLEAK_ALLOC:
+                        early_alloc(log);
+                        break;
+                case KMEMLEAK_FREE:
+                        kmemleak_free(log->ptr);
+                        break;
+                case KMEMLEAK_FREE_PART:
+                        kmemleak_free_part(log->ptr, log->size);
+                        break;
+                case KMEMLEAK_NOT_LEAK:
+                        kmemleak_not_leak(log->ptr);
+                        break;
+                case KMEMLEAK_IGNORE:
+                        kmemleak_ignore(log->ptr);
+                        break;
+                case KMEMLEAK_SCAN_AREA:
+                        kmemleak_scan_area(log->ptr, log->offset, log->length,
+                                           GFP_KERNEL);
+                        break;
+                case KMEMLEAK_NO_SCAN:
+                        kmemleak_no_scan(log->ptr);
+                        break;
+                default:
+                        WARN_ON(1);
+                }
+        }
+}
+
+/*
+ * Late initialization function.
+ */
+static int __init kmemleak_late_init(void)
+{
+        struct dentry *dentry;
+
+        atomic_set(&kmemleak_initialized, 1);
+
+        if (atomic_read(&kmemleak_error)) {
+                /*
+                 * Some error occured and kmemleak was disabled. There is a
+                 * small chance that kmemleak_disable() was called immediately
+                 * after setting kmemleak_initialized and we may end up with
+                 * two clean-up threads but serialized by scan_mutex.
+                 */
+                schedule_work(&cleanup_work);
+                return -ENOMEM;
+        }
+
+        dentry = debugfs_create_file("kmemleak", S_IRUGO, NULL, NULL,
+                                     &kmemleak_fops);
+        if (!dentry)
+                pr_warning("Failed to create the debugfs kmemleak file\n");
+        mutex_lock(&scan_mutex);
+        start_scan_thread();
+        mutex_unlock(&scan_mutex);
+
+        pr_info("Kernel memory leak detector initialized\n");
+
+        return 0;
+}
+late_initcall(kmemleak_late_init);
diff --git a/mm/ksm.c b/mm/ksm.c
new file mode 100644
index 000000000000..5575f8628fef
--- /dev/null
+++ b/mm/ksm.c
@@ -0,0 +1,1710 @@
+/*
+ * Memory merging support.
+ *
+ * This code enables dynamic sharing of identical pages found in different
+ * memory areas, even if they are not shared by fork()
+ *
+ * Copyright (C) 2008-2009 Red Hat, Inc.
+ * Authors:
+ *      Izik Eidus
+ *      Andrea Arcangeli
+ *      Chris Wright
+ *      Hugh Dickins
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.
+ */
+
+#include <linux/errno.h>
+#include <linux/mm.h>
+#include <linux/fs.h>
+#include <linux/mman.h>
+#include <linux/sched.h>
+#include <linux/rwsem.h>
+#include <linux/pagemap.h>
+#include <linux/rmap.h>
+#include <linux/spinlock.h>
+#include <linux/jhash.h>
+#include <linux/delay.h>
+#include <linux/kthread.h>
+#include <linux/wait.h>
+#include <linux/slab.h>
+#include <linux/rbtree.h>
+#include <linux/mmu_notifier.h>
+#include <linux/swap.h>
+#include <linux/ksm.h>
+
+#include <asm/tlbflush.h>
+
+/*
+ * A few notes about the KSM scanning process,
+ * to make it easier to understand the data structures below:
+ *
+ * In order to reduce excessive scanning, KSM sorts the memory pages by their
+ * contents into a data structure that holds pointers to the pages' locations.
+ *
+ * Since the contents of the pages may change at any moment, KSM cannot just
+ * insert the pages into a normal sorted tree and expect it to find anything.
+ * Therefore KSM uses two data structures - the stable and the unstable tree.
+ *
+ * The stable tree holds pointers to all the merged pages (ksm pages), sorted
+ * by their contents.  Because each such page is write-protected, searching on
+ * this tree is fully assured to be working (except when pages are unmapped),
+ * and therefore this tree is called the stable tree.
+ *
+ * In addition to the stable tree, KSM uses a second data structure called the
+ * unstable tree: this tree holds pointers to pages which have been found to
+ * be "unchanged for a period of time".  The unstable tree sorts these pages
+ * by their contents, but since they are not write-protected, KSM cannot rely
+ * upon the unstable tree to work correctly - the unstable tree is liable to
+ * be corrupted as its contents are modified, and so it is called unstable.
+ *
+ * KSM solves this problem by several techniques:
+ *
+ * 1) The unstable tree is flushed every time KSM completes scanning all
+ *    memory areas, and then the tree is rebuilt again from the beginning.
+ * 2) KSM will only insert into the unstable tree, pages whose hash value
+ *    has not changed since the previous scan of all memory areas.
+ * 3) The unstable tree is a RedBlack Tree - so its balancing is based on the
+ *    colors of the nodes and not on their contents, assuring that even when
+ *    the tree gets "corrupted" it won't get out of balance, so scanning time
+ *    remains the same (also, searching and inserting nodes in an rbtree uses
+ *    the same algorithm, so we have no overhead when we flush and rebuild).
+ * 4) KSM never flushes the stable tree, which means that even if it were to
+ *    take 10 attempts to find a page in the unstable tree, once it is found,
+ *    it is secured in the stable tree.  (When we scan a new page, we first
+ *    compare it against the stable tree, and then against the unstable tree.)
+ */
+
+/**
+ * struct mm_slot - ksm information per mm that is being scanned
+ * @link: link to the mm_slots hash list
+ * @mm_list: link into the mm_slots list, rooted in ksm_mm_head
+ * @rmap_list: head for this mm_slot's list of rmap_items
+ * @mm: the mm that this information is valid for
+ */
+struct mm_slot {
+        struct hlist_node link;
+        struct list_head mm_list;
+        struct list_head rmap_list;
+        struct mm_struct *mm;
+};
+
+/**
+ * struct ksm_scan - cursor for scanning
+ * @mm_slot: the current mm_slot we are scanning
+ * @address: the next address inside that to be scanned
+ * @rmap_item: the current rmap that we are scanning inside the rmap_list
+ * @seqnr: count of completed full scans (needed when removing unstable node)
+ *
+ * There is only the one ksm_scan instance of this cursor structure.
+ */
+struct ksm_scan {
+        struct mm_slot *mm_slot;
+        unsigned long address;
+        struct rmap_item *rmap_item;
+        unsigned long seqnr;
+};
+
+/**
+ * struct rmap_item - reverse mapping item for virtual addresses
+ * @link: link into mm_slot's rmap_list (rmap_list is per mm)
+ * @mm: the memory structure this rmap_item is pointing into
+ * @address: the virtual address this rmap_item tracks (+ flags in low bits)
+ * @oldchecksum: previous checksum of the page at that virtual address
+ * @node: rb_node of this rmap_item in either unstable or stable tree
+ * @next: next rmap_item hanging off the same node of the stable tree
+ * @prev: previous rmap_item hanging off the same node of the stable tree
+ */
+struct rmap_item {
+        struct list_head link;
+        struct mm_struct *mm;
+        unsigned long address;          /* + low bits used for flags below */
+        union {
+                unsigned int oldchecksum;               /* when unstable */
+                struct rmap_item *next;                 /* when stable */
+        };
+        union {
+                struct rb_node node;                    /* when tree node */
+                struct rmap_item *prev;                 /* in stable list */
+        };
+};
+
+#define SEQNR_MASK      0x0ff   /* low bits of unstable tree seqnr */
+#define NODE_FLAG       0x100   /* is a node of unstable or stable tree */
+#define STABLE_FLAG     0x200   /* is a node or list item of stable tree */
+
+/* The stable and unstable tree heads */
+static struct rb_root root_stable_tree = RB_ROOT;
+static struct rb_root root_unstable_tree = RB_ROOT;
+
+#define MM_SLOTS_HASH_HEADS 1024
+static struct hlist_head *mm_slots_hash;
+
+static struct mm_slot ksm_mm_head = {
+        .mm_list = LIST_HEAD_INIT(ksm_mm_head.mm_list),
+};
+static struct ksm_scan ksm_scan = {
+        .mm_slot = &ksm_mm_head,
+};
+
+static struct kmem_cache *rmap_item_cache;
+static struct kmem_cache *mm_slot_cache;
+
+/* The number of nodes in the stable tree */
+static unsigned long ksm_pages_shared;
+
+/* The number of page slots additionally sharing those nodes */
+static unsigned long ksm_pages_sharing;
+
+/* The number of nodes in the unstable tree */
+static unsigned long ksm_pages_unshared;
+
+/* The number of rmap_items in use: to calculate pages_volatile */
+static unsigned long ksm_rmap_items;
+
+/* Limit on the number of unswappable pages used */
+static unsigned long ksm_max_kernel_pages;
+
+/* Number of pages ksmd should scan in one batch */
+static unsigned int ksm_thread_pages_to_scan = 100;
+
+/* Milliseconds ksmd should sleep between batches */
+static unsigned int ksm_thread_sleep_millisecs = 20;
+
+#define KSM_RUN_STOP    0
+#define KSM_RUN_MERGE   1
+#define KSM_RUN_UNMERGE 2
+static unsigned int ksm_run = KSM_RUN_STOP;
+
+static DECLARE_WAIT_QUEUE_HEAD(ksm_thread_wait);
+static DEFINE_MUTEX(ksm_thread_mutex);
+static DEFINE_SPINLOCK(ksm_mmlist_lock);
+
+#define KSM_KMEM_CACHE(__struct, __flags) kmem_cache_create("ksm_"#__struct,\
+                sizeof(struct __struct), __alignof__(struct __struct),\
+                (__flags), NULL)
+
+static int __init ksm_slab_init(void)
+{
+        rmap_item_cache = KSM_KMEM_CACHE(rmap_item, 0);
+        if (!rmap_item_cache)
+                goto out;
+
+        mm_slot_cache = KSM_KMEM_CACHE(mm_slot, 0);
+        if (!mm_slot_cache)
+                goto out_free;
+
+        return 0;
+
+out_free:
+        kmem_cache_destroy(rmap_item_cache);
+out:
+        return -ENOMEM;
+}
+
+static void __init ksm_slab_free(void)
+{
+        kmem_cache_destroy(mm_slot_cache);
+        kmem_cache_destroy(rmap_item_cache);
+        mm_slot_cache = NULL;
+}
+
+static inline struct rmap_item *alloc_rmap_item(void)
+{
+        struct rmap_item *rmap_item;
+
+        rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL);
+        if (rmap_item)
+                ksm_rmap_items++;
+        return rmap_item;
+}
+
+static inline void free_rmap_item(struct rmap_item *rmap_item)
+{
+        ksm_rmap_items--;
+        rmap_item->mm = NULL;   /* debug safety */
+        kmem_cache_free(rmap_item_cache, rmap_item);
+}
+
+static inline struct mm_slot *alloc_mm_slot(void)
+{
+        if (!mm_slot_cache)     /* initialization failed */
+                return NULL;
+        return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
+}
+
+static inline void free_mm_slot(struct mm_slot *mm_slot)
+{
+        kmem_cache_free(mm_slot_cache, mm_slot);
+}
+
+static int __init mm_slots_hash_init(void)
+{
+        mm_slots_hash = kzalloc(MM_SLOTS_HASH_HEADS * sizeof(struct hlist_head),
+                                GFP_KERNEL);
+        if (!mm_slots_hash)
+                return -ENOMEM;
+        return 0;
+}
+
+static void __init mm_slots_hash_free(void)
+{
+        kfree(mm_slots_hash);
+}
+
+static struct mm_slot *get_mm_slot(struct mm_struct *mm)
+{
+        struct mm_slot *mm_slot;
+        struct hlist_head *bucket;
+        struct hlist_node *node;
+
+        bucket = &mm_slots_hash[((unsigned long)mm / sizeof(struct mm_struct))
+                                % MM_SLOTS_HASH_HEADS];
+        hlist_for_each_entry(mm_slot, node, bucket, link) {
+                if (mm == mm_slot->mm)
+                        return mm_slot;
+        }
+        return NULL;
+}
+
+static void insert_to_mm_slots_hash(struct mm_struct *mm,
+                                    struct mm_slot *mm_slot)
+{
+        struct hlist_head *bucket;
+
+        bucket = &mm_slots_hash[((unsigned long)mm / sizeof(struct mm_struct))
+                                % MM_SLOTS_HASH_HEADS];
+        mm_slot->mm = mm;
+        INIT_LIST_HEAD(&mm_slot->rmap_list);
+        hlist_add_head(&mm_slot->link, bucket);
+}
+
+static inline int in_stable_tree(struct rmap_item *rmap_item)
+{
+        return rmap_item->address & STABLE_FLAG;
+}
+
+/*
+ * ksmd, and unmerge_and_remove_all_rmap_items(), must not touch an mm's
+ * page tables after it has passed through ksm_exit() - which, if necessary,
+ * takes mmap_sem briefly to serialize against them.  ksm_exit() does not set
+ * a special flag: they can just back out as soon as mm_users goes to zero.
+ * ksm_test_exit() is used throughout to make this test for exit: in some
+ * places for correctness, in some places just to avoid unnecessary work.
+ */
+static inline bool ksm_test_exit(struct mm_struct *mm)
+{
+        return atomic_read(&mm->mm_users) == 0;
+}
+
+/*
+ * We use break_ksm to break COW on a ksm page: it's a stripped down
+ *
+ *      if (get_user_pages(current, mm, addr, 1, 1, 1, &page, NULL) == 1)
+ *              put_page(page);
+ *
+ * but taking great care only to touch a ksm page, in a VM_MERGEABLE vma,
+ * in case the application has unmapped and remapped mm,addr meanwhile.
+ * Could a ksm page appear anywhere else?  Actually yes, in a VM_PFNMAP
+ * mmap of /dev/mem or /dev/kmem, where we would not want to touch it.
+ */
+static int break_ksm(struct vm_area_struct *vma, unsigned long addr)
+{
+        struct page *page;
+        int ret = 0;
+
+        do {
+                cond_resched();
+                page = follow_page(vma, addr, FOLL_GET);
+                if (!page)
+                        break;
+                if (PageKsm(page))
+                        ret = handle_mm_fault(vma->vm_mm, vma, addr,
+                                                        FAULT_FLAG_WRITE);
+                else
+                        ret = VM_FAULT_WRITE;
+                put_page(page);
+        } while (!(ret & (VM_FAULT_WRITE | VM_FAULT_SIGBUS | VM_FAULT_OOM)));
+        /*
+         * We must loop because handle_mm_fault() may back out if there's
+         * any difficulty e.g. if pte accessed bit gets updated concurrently.
+         *
+         * VM_FAULT_WRITE is what we have been hoping for: it indicates that
+         * COW has been broken, even if the vma does not permit VM_WRITE;
+         * but note that a concurrent fault might break PageKsm for us.
+         *
+         * VM_FAULT_SIGBUS could occur if we race with truncation of the
+         * backing file, which also invalidates anonymous pages: that's
+         * okay, that truncation will have unmapped the PageKsm for us.
+         *
+         * VM_FAULT_OOM: at the time of writing (late July 2009), setting
+         * aside mem_cgroup limits, VM_FAULT_OOM would only be set if the
+         * current task has TIF_MEMDIE set, and will be OOM killed on return
+         * to user; and ksmd, having no mm, would never be chosen for that.
+         *
+         * But if the mm is in a limited mem_cgroup, then the fault may fail
+         * with VM_FAULT_OOM even if the current task is not TIF_MEMDIE; and
+         * even ksmd can fail in this way - though it's usually breaking ksm
+         * just to undo a merge it made a moment before, so unlikely to oom.
+         *
+         * That's a pity: we might therefore have more kernel pages allocated
+         * than we're counting as nodes in the stable tree; but ksm_do_scan
+         * will retry to break_cow on each pass, so should recover the page
+         * in due course.  The important thing is to not let VM_MERGEABLE
+         * be cleared while any such pages might remain in the area.
+         */
+        return (ret & VM_FAULT_OOM) ? -ENOMEM : 0;
+}
+
+static void break_cow(struct mm_struct *mm, unsigned long addr)
+{
+        struct vm_area_struct *vma;
+
+        down_read(&mm->mmap_sem);
+        if (ksm_test_exit(mm))
+                goto out;
+        vma = find_vma(mm, addr);
+        if (!vma || vma->vm_start > addr)
+                goto out;
+        if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
+                goto out;
+        break_ksm(vma, addr);
+out:
+        up_read(&mm->mmap_sem);
+}
+
+static struct page *get_mergeable_page(struct rmap_item *rmap_item)
+{
+        struct mm_struct *mm = rmap_item->mm;
+        unsigned long addr = rmap_item->address;
+        struct vm_area_struct *vma;
+        struct page *page;
+
+        down_read(&mm->mmap_sem);
+        if (ksm_test_exit(mm))
+                goto out;
+        vma = find_vma(mm, addr);
+        if (!vma || vma->vm_start > addr)
+                goto out;
+        if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
+                goto out;
+
+        page = follow_page(vma, addr, FOLL_GET);
+        if (!page)
+                goto out;
+        if (PageAnon(page)) {
+                flush_anon_page(vma, page, addr);
+                flush_dcache_page(page);
+        } else {
+                put_page(page);
+out:            page = NULL;
+        }
+        up_read(&mm->mmap_sem);
+        return page;
+}
+
+/*
+ * get_ksm_page: checks if the page at the virtual address in rmap_item
+ * is still PageKsm, in which case we can trust the content of the page,
+ * and it returns the gotten page; but NULL if the page has been zapped.
+ */
+static struct page *get_ksm_page(struct rmap_item *rmap_item)
+{
+        struct page *page;
+
+        page = get_mergeable_page(rmap_item);
+        if (page && !PageKsm(page)) {
+                put_page(page);
+                page = NULL;
+        }
+        return page;
+}
+
+/*
+ * Removing rmap_item from stable or unstable tree.
+ * This function will clean the information from the stable/unstable tree.
+ */
+static void remove_rmap_item_from_tree(struct rmap_item *rmap_item)
+{
+        if (in_stable_tree(rmap_item)) {
+                struct rmap_item *next_item = rmap_item->next;
+
+                if (rmap_item->address & NODE_FLAG) {
+                        if (next_item) {
+                                rb_replace_node(&rmap_item->node,
+                                                &next_item->node,
+                                                &root_stable_tree);
+                                next_item->address |= NODE_FLAG;
+                                ksm_pages_sharing--;
+                        } else {
+                                rb_erase(&rmap_item->node, &root_stable_tree);
+                                ksm_pages_shared--;
+                        }
+                } else {
+                        struct rmap_item *prev_item = rmap_item->prev;
+
+                        BUG_ON(prev_item->next != rmap_item);
+                        prev_item->next = next_item;
+                        if (next_item) {
+                                BUG_ON(next_item->prev != rmap_item);
+                                next_item->prev = rmap_item->prev;
+                        }
+                        ksm_pages_sharing--;
+                }
+
+                rmap_item->next = NULL;
+
+        } else if (rmap_item->address & NODE_FLAG) {
+                unsigned char age;
+                /*
+                 * Usually ksmd can and must skip the rb_erase, because
+                 * root_unstable_tree was already reset to RB_ROOT.
+                 * But be careful when an mm is exiting: do the rb_erase
+                 * if this rmap_item was inserted by this scan, rather
+                 * than left over from before.
+                 */
+                age = (unsigned char)(ksm_scan.seqnr - rmap_item->address);
+                BUG_ON(age > 1);
+                if (!age)
+                        rb_erase(&rmap_item->node, &root_unstable_tree);
+                ksm_pages_unshared--;
+        }
+
+        rmap_item->address &= PAGE_MASK;
+
+        cond_resched();         /* we're called from many long loops */
+}
+
+static void remove_trailing_rmap_items(struct mm_slot *mm_slot,
+                                       struct list_head *cur)
+{
+        struct rmap_item *rmap_item;
+
+        while (cur != &mm_slot->rmap_list) {
+                rmap_item = list_entry(cur, struct rmap_item, link);
+                cur = cur->next;
+                remove_rmap_item_from_tree(rmap_item);
+                list_del(&rmap_item->link);
+                free_rmap_item(rmap_item);
+        }
+}
+
+/*
+ * Though it's very tempting to unmerge in_stable_tree(rmap_item)s rather
+ * than check every pte of a given vma, the locking doesn't quite work for
+ * that - an rmap_item is assigned to the stable tree after inserting ksm
+ * page and upping mmap_sem.  Nor does it fit with the way we skip dup'ing
+ * rmap_items from parent to child at fork time (so as not to waste time
+ * if exit comes before the next scan reaches it).
+ *
+ * Similarly, although we'd like to remove rmap_items (so updating counts
+ * and freeing memory) when unmerging an area, it's easier to leave that
+ * to the next pass of ksmd - consider, for example, how ksmd might be
+ * in cmp_and_merge_page on one of the rmap_items we would be removing.
+ */
+static int unmerge_ksm_pages(struct vm_area_struct *vma,
+                             unsigned long start, unsigned long end)
+{
+        unsigned long addr;
+        int err = 0;
+
+        for (addr = start; addr < end && !err; addr += PAGE_SIZE) {
+                if (ksm_test_exit(vma->vm_mm))
+                        break;
+                if (signal_pending(current))
+                        err = -ERESTARTSYS;
+                else
+                        err = break_ksm(vma, addr);
+        }
+        return err;
+}
+
+#ifdef CONFIG_SYSFS
+/*
+ * Only called through the sysfs control interface:
+ */
+static int unmerge_and_remove_all_rmap_items(void)
+{
+        struct mm_slot *mm_slot;
+        struct mm_struct *mm;
+        struct vm_area_struct *vma;
+        int err = 0;
+
+        spin_lock(&ksm_mmlist_lock);
+        ksm_scan.mm_slot = list_entry(ksm_mm_head.mm_list.next,
+                                                struct mm_slot, mm_list);
+        spin_unlock(&ksm_mmlist_lock);
+
+        for (mm_slot = ksm_scan.mm_slot;
+                        mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) {
+                mm = mm_slot->mm;
+                down_read(&mm->mmap_sem);
+                for (vma = mm->mmap; vma; vma = vma->vm_next) {
+                        if (ksm_test_exit(mm))
+                                break;
+                        if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
+                                continue;
+                        err = unmerge_ksm_pages(vma,
+                                                vma->vm_start, vma->vm_end);
+                        if (err)
+                                goto error;
+                }
+
+                remove_trailing_rmap_items(mm_slot, mm_slot->rmap_list.next);
+
+                spin_lock(&ksm_mmlist_lock);
+                ksm_scan.mm_slot = list_entry(mm_slot->mm_list.next,
+                                                struct mm_slot, mm_list);
+                if (ksm_test_exit(mm)) {
+                        hlist_del(&mm_slot->link);
+                        list_del(&mm_slot->mm_list);
+                        spin_unlock(&ksm_mmlist_lock);
+
+                        free_mm_slot(mm_slot);
+                        clear_bit(MMF_VM_MERGEABLE, &mm->flags);
+                        up_read(&mm->mmap_sem);
+                        mmdrop(mm);
+                } else {
+                        spin_unlock(&ksm_mmlist_lock);
+                        up_read(&mm->mmap_sem);
+                }
+        }
+
+        ksm_scan.seqnr = 0;
+        return 0;
+
+error:
+        up_read(&mm->mmap_sem);
+        spin_lock(&ksm_mmlist_lock);
+        ksm_scan.mm_slot = &ksm_mm_head;
+        spin_unlock(&ksm_mmlist_lock);
+        return err;
+}
+#endif /* CONFIG_SYSFS */
+
+static u32 calc_checksum(struct page *page)
+{
+        u32 checksum;
+        void *addr = kmap_atomic(page, KM_USER0);
+        checksum = jhash2(addr, PAGE_SIZE / 4, 17);
+        kunmap_atomic(addr, KM_USER0);
+        return checksum;
+}
+
+static int memcmp_pages(struct page *page1, struct page *page2)
+{
+        char *addr1, *addr2;
+        int ret;
+
+        addr1 = kmap_atomic(page1, KM_USER0);
+        addr2 = kmap_atomic(page2, KM_USER1);
+        ret = memcmp(addr1, addr2, PAGE_SIZE);
+        kunmap_atomic(addr2, KM_USER1);
+        kunmap_atomic(addr1, KM_USER0);
+        return ret;
+}
+
+static inline int pages_identical(struct page *page1, struct page *page2)
+{
+        return !memcmp_pages(page1, page2);
+}
+
+static int write_protect_page(struct vm_area_struct *vma, struct page *page,
+                              pte_t *orig_pte)
+{
+        struct mm_struct *mm = vma->vm_mm;
+        unsigned long addr;
+        pte_t *ptep;
+        spinlock_t *ptl;
+        int swapped;
+        int err = -EFAULT;
+
+        addr = page_address_in_vma(page, vma);
+        if (addr == -EFAULT)
+                goto out;
+
+        ptep = page_check_address(page, mm, addr, &ptl, 0);
+        if (!ptep)
+                goto out;
+
+        if (pte_write(*ptep)) {
+                pte_t entry;
+
+                swapped = PageSwapCache(page);
+                flush_cache_page(vma, addr, page_to_pfn(page));
+                /*
+                 * Ok this is tricky, when get_user_pages_fast() run it doesnt
+                 * take any lock, therefore the check that we are going to make
+                 * with the pagecount against the mapcount is racey and
+                 * O_DIRECT can happen right after the check.
+                 * So we clear the pte and flush the tlb before the check
+                 * this assure us that no O_DIRECT can happen after the check
+                 * or in the middle of the check.
+                 */
+                entry = ptep_clear_flush(vma, addr, ptep);
+                /*
+                 * Check that no O_DIRECT or similar I/O is in progress on the
+                 * page
+                 */
+                if ((page_mapcount(page) + 2 + swapped) != page_count(page)) {
+                        set_pte_at_notify(mm, addr, ptep, entry);
+                        goto out_unlock;
+                }
+                entry = pte_wrprotect(entry);
+                set_pte_at_notify(mm, addr, ptep, entry);
+        }
+        *orig_pte = *ptep;
+        err = 0;
+
+out_unlock:
+        pte_unmap_unlock(ptep, ptl);
+out:
+        return err;
+}
+
+/**
+ * replace_page - replace page in vma by new ksm page
+ * @vma:      vma that holds the pte pointing to oldpage
+ * @oldpage:  the page we are replacing by newpage
+ * @newpage:  the ksm page we replace oldpage by
+ * @orig_pte: the original value of the pte
+ *
+ * Returns 0 on success, -EFAULT on failure.
+ */
+static int replace_page(struct vm_area_struct *vma, struct page *oldpage,
+                        struct page *newpage, pte_t orig_pte)
+{
+        struct mm_struct *mm = vma->vm_mm;
+        pgd_t *pgd;
+        pud_t *pud;
+        pmd_t *pmd;
+        pte_t *ptep;
+        spinlock_t *ptl;
+        unsigned long addr;
+        pgprot_t prot;
+        int err = -EFAULT;
+
+        prot = vm_get_page_prot(vma->vm_flags & ~VM_WRITE);
+
+        addr = page_address_in_vma(oldpage, vma);
+        if (addr == -EFAULT)
+                goto out;
+
+        pgd = pgd_offset(mm, addr);
+        if (!pgd_present(*pgd))
+                goto out;
+
+        pud = pud_offset(pgd, addr);
+        if (!pud_present(*pud))
+                goto out;
+
+        pmd = pmd_offset(pud, addr);
+        if (!pmd_present(*pmd))
+                goto out;
+
+        ptep = pte_offset_map_lock(mm, pmd, addr, &ptl);
+        if (!pte_same(*ptep, orig_pte)) {
+                pte_unmap_unlock(ptep, ptl);
+                goto out;
+        }
+
+        get_page(newpage);
+        page_add_ksm_rmap(newpage);
+
+        flush_cache_page(vma, addr, pte_pfn(*ptep));
+        ptep_clear_flush(vma, addr, ptep);
+        set_pte_at_notify(mm, addr, ptep, mk_pte(newpage, prot));
+
+        page_remove_rmap(oldpage);
+        put_page(oldpage);
+
+        pte_unmap_unlock(ptep, ptl);
+        err = 0;
+out:
+        return err;
+}
+
+/*
+ * try_to_merge_one_page - take two pages and merge them into one
+ * @vma: the vma that hold the pte pointing into oldpage
+ * @oldpage: the page that we want to replace with newpage
+ * @newpage: the page that we want to map instead of oldpage
+ *
+ * Note:
+ * oldpage should be a PageAnon page, while newpage should be a PageKsm page,
+ * or a newly allocated kernel page which page_add_ksm_rmap will make PageKsm.
+ *
+ * This function returns 0 if the pages were merged, -EFAULT otherwise.
+ */
+static int try_to_merge_one_page(struct vm_area_struct *vma,
+                                 struct page *oldpage,
+                                 struct page *newpage)
+{
+        pte_t orig_pte = __pte(0);
+        int err = -EFAULT;
+
+        if (!(vma->vm_flags & VM_MERGEABLE))
+                goto out;
+
+        if (!PageAnon(oldpage))
+                goto out;
+
+        get_page(newpage);
+        get_page(oldpage);
+
+        /*
+         * We need the page lock to read a stable PageSwapCache in
+         * write_protect_page().  We use trylock_page() instead of
+         * lock_page() because we don't want to wait here - we
+         * prefer to continue scanning and merging different pages,
+         * then come back to this page when it is unlocked.
+         */
+        if (!trylock_page(oldpage))
+                goto out_putpage;
+        /*
+         * If this anonymous page is mapped only here, its pte may need
+         * to be write-protected.  If it's mapped elsewhere, all of its
+         * ptes are necessarily already write-protected.  But in either
+         * case, we need to lock and check page_count is not raised.
+         */
+        if (write_protect_page(vma, oldpage, &orig_pte)) {
+                unlock_page(oldpage);
+                goto out_putpage;
+        }
+        unlock_page(oldpage);
+
+        if (pages_identical(oldpage, newpage))
+                err = replace_page(vma, oldpage, newpage, orig_pte);
+
+out_putpage:
+        put_page(oldpage);
+        put_page(newpage);
+out:
+        return err;
+}
+
+/*
+ * try_to_merge_with_ksm_page - like try_to_merge_two_pages,
+ * but no new kernel page is allocated: kpage must already be a ksm page.
+ */
+static int try_to_merge_with_ksm_page(struct mm_struct *mm1,
+                                      unsigned long addr1,
+                                      struct page *page1,
+                                      struct page *kpage)
+{
+        struct vm_area_struct *vma;
+        int err = -EFAULT;
+
+        down_read(&mm1->mmap_sem);
+        if (ksm_test_exit(mm1))
+                goto out;
+
+        vma = find_vma(mm1, addr1);
+        if (!vma || vma->vm_start > addr1)
+                goto out;
+
+        err = try_to_merge_one_page(vma, page1, kpage);
+out:
+        up_read(&mm1->mmap_sem);
+        return err;
+}
+
+/*
+ * try_to_merge_two_pages - take two identical pages and prepare them
+ * to be merged into one page.
+ *
+ * This function returns 0 if we successfully mapped two identical pages
+ * into one page, -EFAULT otherwise.
+ *
+ * Note that this function allocates a new kernel page: if one of the pages
+ * is already a ksm page, try_to_merge_with_ksm_page should be used.
+ */
+static int try_to_merge_two_pages(struct mm_struct *mm1, unsigned long addr1,
+                                  struct page *page1, struct mm_struct *mm2,
+                                  unsigned long addr2, struct page *page2)
+{
+        struct vm_area_struct *vma;
+        struct page *kpage;
+        int err = -EFAULT;
+
+        /*
+         * The number of nodes in the stable tree
+         * is the number of kernel pages that we hold.
+         */
+        if (ksm_max_kernel_pages &&
+            ksm_max_kernel_pages <= ksm_pages_shared)
+                return err;
+
+        kpage = alloc_page(GFP_HIGHUSER);
+        if (!kpage)
+                return err;
+
+        down_read(&mm1->mmap_sem);
+        if (ksm_test_exit(mm1)) {
+                up_read(&mm1->mmap_sem);
+                goto out;
+        }
+        vma = find_vma(mm1, addr1);
+        if (!vma || vma->vm_start > addr1) {
+                up_read(&mm1->mmap_sem);
+                goto out;
+        }
+
+        copy_user_highpage(kpage, page1, addr1, vma);
+        err = try_to_merge_one_page(vma, page1, kpage);
+        up_read(&mm1->mmap_sem);
+
+        if (!err) {
+                err = try_to_merge_with_ksm_page(mm2, addr2, page2, kpage);
+                /*
+                 * If that fails, we have a ksm page with only one pte
+                 * pointing to it: so break it.
+                 */
+                if (err)
+                        break_cow(mm1, addr1);
+        }
+out:
+        put_page(kpage);
+        return err;
+}
+
+/*
+ * stable_tree_search - search page inside the stable tree
+ * @page: the page that we are searching identical pages to.
+ * @page2: pointer into identical page that we are holding inside the stable
+ *         tree that we have found.
+ * @rmap_item: the reverse mapping item
+ *
+ * This function checks if there is a page inside the stable tree
+ * with identical content to the page that we are scanning right now.
+ *
+ * This function return rmap_item pointer to the identical item if found,
+ * NULL otherwise.
+ */
+static struct rmap_item *stable_tree_search(struct page *page,
+                                            struct page **page2,
+                                            struct rmap_item *rmap_item)
+{
+        struct rb_node *node = root_stable_tree.rb_node;
+
+        while (node) {
+                struct rmap_item *tree_rmap_item, *next_rmap_item;
+                int ret;
+
+                tree_rmap_item = rb_entry(node, struct rmap_item, node);
+                while (tree_rmap_item) {
+                        BUG_ON(!in_stable_tree(tree_rmap_item));
+                        cond_resched();
+                        page2[0] = get_ksm_page(tree_rmap_item);
+                        if (page2[0])
+                                break;
+                        next_rmap_item = tree_rmap_item->next;
+                        remove_rmap_item_from_tree(tree_rmap_item);
+                        tree_rmap_item = next_rmap_item;
+                }
+                if (!tree_rmap_item)
+                        return NULL;
+
+                ret = memcmp_pages(page, page2[0]);
+
+                if (ret < 0) {
+                        put_page(page2[0]);
+                        node = node->rb_left;
+                } else if (ret > 0) {
+                        put_page(page2[0]);
+                        node = node->rb_right;
+                } else {
+                        return tree_rmap_item;
+                }
+        }
+
+        return NULL;
+}
+
+/*
+ * stable_tree_insert - insert rmap_item pointing to new ksm page
+ * into the stable tree.
+ *
+ * @page: the page that we are searching identical page to inside the stable
+ *        tree.
+ * @rmap_item: pointer to the reverse mapping item.
+ *
+ * This function returns rmap_item if success, NULL otherwise.
+ */
+static struct rmap_item *stable_tree_insert(struct page *page,
+                                            struct rmap_item *rmap_item)
+{
+        struct rb_node **new = &root_stable_tree.rb_node;
+        struct rb_node *parent = NULL;
+
+        while (*new) {
+                struct rmap_item *tree_rmap_item, *next_rmap_item;
+                struct page *tree_page;
+                int ret;
+
+                tree_rmap_item = rb_entry(*new, struct rmap_item, node);
+                while (tree_rmap_item) {
+                        BUG_ON(!in_stable_tree(tree_rmap_item));
+                        cond_resched();
+                        tree_page = get_ksm_page(tree_rmap_item);
+                        if (tree_page)
+                                break;
+                        next_rmap_item = tree_rmap_item->next;
+                        remove_rmap_item_from_tree(tree_rmap_item);
+                        tree_rmap_item = next_rmap_item;
+                }
+                if (!tree_rmap_item)
+                        return NULL;
+
+                ret = memcmp_pages(page, tree_page);
+                put_page(tree_page);
+
+                parent = *new;
+                if (ret < 0)
+                        new = &parent->rb_left;
+                else if (ret > 0)
+                        new = &parent->rb_right;
+                else {
+                        /*
+                         * It is not a bug that stable_tree_search() didn't
+                         * find this node: because at that time our page was
+                         * not yet write-protected, so may have changed since.
+                         */
+                        return NULL;
+                }
+        }
+
+        rmap_item->address |= NODE_FLAG | STABLE_FLAG;
+        rmap_item->next = NULL;
+        rb_link_node(&rmap_item->node, parent, new);
+        rb_insert_color(&rmap_item->node, &root_stable_tree);
+
+        ksm_pages_shared++;
+        return rmap_item;
+}
+
+/*
+ * unstable_tree_search_insert - search and insert items into the unstable tree.
+ *
+ * @page: the page that we are going to search for identical page or to insert
+ *        into the unstable tree
+ * @page2: pointer into identical page that was found inside the unstable tree
+ * @rmap_item: the reverse mapping item of page
+ *
+ * This function searches for a page in the unstable tree identical to the
+ * page currently being scanned; and if no identical page is found in the
+ * tree, we insert rmap_item as a new object into the unstable tree.
+ *
+ * This function returns pointer to rmap_item found to be identical
+ * to the currently scanned page, NULL otherwise.
+ *
+ * This function does both searching and inserting, because they share
+ * the same walking algorithm in an rbtree.
+ */
+static struct rmap_item *unstable_tree_search_insert(struct page *page,
+                                                struct page **page2,
+                                                struct rmap_item *rmap_item)
+{
+        struct rb_node **new = &root_unstable_tree.rb_node;
+        struct rb_node *parent = NULL;
+
+        while (*new) {
+                struct rmap_item *tree_rmap_item;
+                int ret;
+
+                cond_resched();
+                tree_rmap_item = rb_entry(*new, struct rmap_item, node);
+                page2[0] = get_mergeable_page(tree_rmap_item);
+                if (!page2[0])
+                        return NULL;
+
+                /*
+                 * Don't substitute an unswappable ksm page
+                 * just for one good swappable forked page.
+                 */
+                if (page == page2[0]) {
+                        put_page(page2[0]);
+                        return NULL;
+                }
+
+                ret = memcmp_pages(page, page2[0]);
+
+                parent = *new;
+                if (ret < 0) {
+                        put_page(page2[0]);
+                        new = &parent->rb_left;
+                } else if (ret > 0) {
+                        put_page(page2[0]);
+                        new = &parent->rb_right;
+                } else {
+                        return tree_rmap_item;
+                }
+        }
+
+        rmap_item->address |= NODE_FLAG;
+        rmap_item->address |= (ksm_scan.seqnr & SEQNR_MASK);
+        rb_link_node(&rmap_item->node, parent, new);
+        rb_insert_color(&rmap_item->node, &root_unstable_tree);
+
+        ksm_pages_unshared++;
+        return NULL;
+}
+
+/*
+ * stable_tree_append - add another rmap_item to the linked list of
+ * rmap_items hanging off a given node of the stable tree, all sharing
+ * the same ksm page.
+ */
+static void stable_tree_append(struct rmap_item *rmap_item,
+                               struct rmap_item *tree_rmap_item)
+{
+        rmap_item->next = tree_rmap_item->next;
+        rmap_item->prev = tree_rmap_item;
+
+        if (tree_rmap_item->next)
+                tree_rmap_item->next->prev = rmap_item;
+
+        tree_rmap_item->next = rmap_item;
+        rmap_item->address |= STABLE_FLAG;
+
+        ksm_pages_sharing++;
+}
+
+/*
+ * cmp_and_merge_page - first see if page can be merged into the stable tree;
+ * if not, compare checksum to previous and if it's the same, see if page can
+ * be inserted into the unstable tree, or merged with a page already there and
+ * both transferred to the stable tree.
+ *
+ * @page: the page that we are searching identical page to.
+ * @rmap_item: the reverse mapping into the virtual address of this page
+ */
+static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item)
+{
+        struct page *page2[1];
+        struct rmap_item *tree_rmap_item;
+        unsigned int checksum;
+        int err;
+
+        if (in_stable_tree(rmap_item))
+                remove_rmap_item_from_tree(rmap_item);
+
+        /* We first start with searching the page inside the stable tree */
+        tree_rmap_item = stable_tree_search(page, page2, rmap_item);
+        if (tree_rmap_item) {
+                if (page == page2[0])                   /* forked */
+                        err = 0;
+                else
+                        err = try_to_merge_with_ksm_page(rmap_item->mm,
+                                                         rmap_item->address,
+                                                         page, page2[0]);
+                put_page(page2[0]);
+
+                if (!err) {
+                        /*
+                         * The page was successfully merged:
+                         * add its rmap_item to the stable tree.
+                         */
+                        stable_tree_append(rmap_item, tree_rmap_item);
+                }
+                return;
+        }
+
+        /*
+         * A ksm page might have got here by fork, but its other
+         * references have already been removed from the stable tree.
+         * Or it might be left over from a break_ksm which failed
+         * when the mem_cgroup had reached its limit: try again now.
+         */
+        if (PageKsm(page))
+                break_cow(rmap_item->mm, rmap_item->address);
+
+        /*
+         * In case the hash value of the page was changed from the last time we
+         * have calculated it, this page to be changed frequely, therefore we
+         * don't want to insert it to the unstable tree, and we don't want to
+         * waste our time to search if there is something identical to it there.
+         */
+        checksum = calc_checksum(page);
+        if (rmap_item->oldchecksum != checksum) {
+                rmap_item->oldchecksum = checksum;
+                return;
+        }
+
+        tree_rmap_item = unstable_tree_search_insert(page, page2, rmap_item);
+        if (tree_rmap_item) {
+                err = try_to_merge_two_pages(rmap_item->mm,
+                                             rmap_item->address, page,
+                                             tree_rmap_item->mm,
+                                             tree_rmap_item->address, page2[0]);
+                /*
+                 * As soon as we merge this page, we want to remove the
+                 * rmap_item of the page we have merged with from the unstable
+                 * tree, and insert it instead as new node in the stable tree.
+                 */
+                if (!err) {
+                        rb_erase(&tree_rmap_item->node, &root_unstable_tree);
+                        tree_rmap_item->address &= ~NODE_FLAG;
+                        ksm_pages_unshared--;
+
+                        /*
+                         * If we fail to insert the page into the stable tree,
+                         * we will have 2 virtual addresses that are pointing
+                         * to a ksm page left outside the stable tree,
+                         * in which case we need to break_cow on both.
+                         */
+                        if (stable_tree_insert(page2[0], tree_rmap_item))
+                                stable_tree_append(rmap_item, tree_rmap_item);
+                        else {
+                                break_cow(tree_rmap_item->mm,
+                                                tree_rmap_item->address);
+                                break_cow(rmap_item->mm, rmap_item->address);
+                        }
+                }
+
+                put_page(page2[0]);
+        }
+}
+
+static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot,
+                                            struct list_head *cur,
+                                            unsigned long addr)
+{
+        struct rmap_item *rmap_item;
+
+        while (cur != &mm_slot->rmap_list) {
+                rmap_item = list_entry(cur, struct rmap_item, link);
+                if ((rmap_item->address & PAGE_MASK) == addr) {
+                        if (!in_stable_tree(rmap_item))
+                                remove_rmap_item_from_tree(rmap_item);
+                        return rmap_item;
+                }
+                if (rmap_item->address > addr)
+                        break;
+                cur = cur->next;
+                remove_rmap_item_from_tree(rmap_item);
+                list_del(&rmap_item->link);
+                free_rmap_item(rmap_item);
+        }
+
+        rmap_item = alloc_rmap_item();
+        if (rmap_item) {
+                /* It has already been zeroed */
+                rmap_item->mm = mm_slot->mm;
+                rmap_item->address = addr;
+                list_add_tail(&rmap_item->link, cur);
+        }
+        return rmap_item;
+}
+
+static struct rmap_item *scan_get_next_rmap_item(struct page **page)
+{
+        struct mm_struct *mm;
+        struct mm_slot *slot;
+        struct vm_area_struct *vma;
+        struct rmap_item *rmap_item;
+
+        if (list_empty(&ksm_mm_head.mm_list))
+                return NULL;
+
+        slot = ksm_scan.mm_slot;
+        if (slot == &ksm_mm_head) {
+                root_unstable_tree = RB_ROOT;
+
+                spin_lock(&ksm_mmlist_lock);
+                slot = list_entry(slot->mm_list.next, struct mm_slot, mm_list);
+                ksm_scan.mm_slot = slot;
+                spin_unlock(&ksm_mmlist_lock);
+next_mm:
+                ksm_scan.address = 0;
+                ksm_scan.rmap_item = list_entry(&slot->rmap_list,
+                                                struct rmap_item, link);
+        }
+
+        mm = slot->mm;
+        down_read(&mm->mmap_sem);
+        if (ksm_test_exit(mm))
+                vma = NULL;
+        else
+                vma = find_vma(mm, ksm_scan.address);
+
+        for (; vma; vma = vma->vm_next) {
+                if (!(vma->vm_flags & VM_MERGEABLE))
+                        continue;
+                if (ksm_scan.address < vma->vm_start)
+                        ksm_scan.address = vma->vm_start;
+                if (!vma->anon_vma)
+                        ksm_scan.address = vma->vm_end;
+
+                while (ksm_scan.address < vma->vm_end) {
+                        if (ksm_test_exit(mm))
+                                break;
+                        *page = follow_page(vma, ksm_scan.address, FOLL_GET);
+                        if (*page && PageAnon(*page)) {
+                                flush_anon_page(vma, *page, ksm_scan.address);
+                                flush_dcache_page(*page);
+                                rmap_item = get_next_rmap_item(slot,
+                                        ksm_scan.rmap_item->link.next,
+                                        ksm_scan.address);
+                                if (rmap_item) {
+                                        ksm_scan.rmap_item = rmap_item;
+                                        ksm_scan.address += PAGE_SIZE;
+                                } else
+                                        put_page(*page);
+                                up_read(&mm->mmap_sem);
+                                return rmap_item;
+                        }
+                        if (*page)
+                                put_page(*page);
+                        ksm_scan.address += PAGE_SIZE;
+                        cond_resched();
+                }
+        }
+
+        if (ksm_test_exit(mm)) {
+                ksm_scan.address = 0;
+                ksm_scan.rmap_item = list_entry(&slot->rmap_list,
+                                                struct rmap_item, link);
+        }
+        /*
+         * Nuke all the rmap_items that are above this current rmap:
+         * because there were no VM_MERGEABLE vmas with such addresses.
+         */
+        remove_trailing_rmap_items(slot, ksm_scan.rmap_item->link.next);
+
+        spin_lock(&ksm_mmlist_lock);
+        ksm_scan.mm_slot = list_entry(slot->mm_list.next,
+                                                struct mm_slot, mm_list);
+        if (ksm_scan.address == 0) {
+                /*
+                 * We've completed a full scan of all vmas, holding mmap_sem
+                 * throughout, and found no VM_MERGEABLE: so do the same as
+                 * __ksm_exit does to remove this mm from all our lists now.
+                 * This applies either when cleaning up after __ksm_exit
+                 * (but beware: we can reach here even before __ksm_exit),
+                 * or when all VM_MERGEABLE areas have been unmapped (and
+                 * mmap_sem then protects against race with MADV_MERGEABLE).
+                 */
+                hlist_del(&slot->link);
+                list_del(&slot->mm_list);
+                spin_unlock(&ksm_mmlist_lock);
+
+                free_mm_slot(slot);
+                clear_bit(MMF_VM_MERGEABLE, &mm->flags);
+                up_read(&mm->mmap_sem);
+                mmdrop(mm);
+        } else {
+                spin_unlock(&ksm_mmlist_lock);
+                up_read(&mm->mmap_sem);
+        }
+
+        /* Repeat until we've completed scanning the whole list */
+        slot = ksm_scan.mm_slot;
+        if (slot != &ksm_mm_head)
+                goto next_mm;
+
+        ksm_scan.seqnr++;
+        return NULL;
+}
+
+/**
+ * ksm_do_scan  - the ksm scanner main worker function.
+ * @scan_npages - number of pages we want to scan before we return.
+ */
+static void ksm_do_scan(unsigned int scan_npages)
+{
+        struct rmap_item *rmap_item;
+        struct page *page;
+
+        while (scan_npages--) {
+                cond_resched();
+                rmap_item = scan_get_next_rmap_item(&page);
+                if (!rmap_item)
+                        return;
+                if (!PageKsm(page) || !in_stable_tree(rmap_item))
+                        cmp_and_merge_page(page, rmap_item);
+                else if (page_mapcount(page) == 1) {
+                        /*
+                         * Replace now-unshared ksm page by ordinary page.
+                         */
+                        break_cow(rmap_item->mm, rmap_item->address);
+                        remove_rmap_item_from_tree(rmap_item);
+                        rmap_item->oldchecksum = calc_checksum(page);
+                }
+                put_page(page);
+        }
+}
+
+static int ksmd_should_run(void)
+{
+        return (ksm_run & KSM_RUN_MERGE) && !list_empty(&ksm_mm_head.mm_list);
+}
+
+static int ksm_scan_thread(void *nothing)
+{
+        set_user_nice(current, 5);
+
+        while (!kthread_should_stop()) {
+                mutex_lock(&ksm_thread_mutex);
+                if (ksmd_should_run())
+                        ksm_do_scan(ksm_thread_pages_to_scan);
+                mutex_unlock(&ksm_thread_mutex);
+
+                if (ksmd_should_run()) {
+                        schedule_timeout_interruptible(
+                                msecs_to_jiffies(ksm_thread_sleep_millisecs));
+                } else {
+                        wait_event_interruptible(ksm_thread_wait,
+                                ksmd_should_run() || kthread_should_stop());
+                }
+        }
+        return 0;
+}
+
+int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
+                unsigned long end, int advice, unsigned long *vm_flags)
+{
+        struct mm_struct *mm = vma->vm_mm;
+        int err;
+
+        switch (advice) {
+        case MADV_MERGEABLE:
+                /*
+                 * Be somewhat over-protective for now!
+                 */
+                if (*vm_flags & (VM_MERGEABLE | VM_SHARED  | VM_MAYSHARE   |
+                                 VM_PFNMAP    | VM_IO      | VM_DONTEXPAND |
+                                 VM_RESERVED  | VM_HUGETLB | VM_INSERTPAGE |
+                                 VM_MIXEDMAP  | VM_SAO))
+                        return 0;               /* just ignore the advice */
+
+                if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) {
+                        err = __ksm_enter(mm);
+                        if (err)
+                                return err;
+                }
+
+                *vm_flags |= VM_MERGEABLE;
+                break;
+
+        case MADV_UNMERGEABLE:
+                if (!(*vm_flags & VM_MERGEABLE))
+                        return 0;               /* just ignore the advice */
+
+                if (vma->anon_vma) {
+                        err = unmerge_ksm_pages(vma, start, end);
+                        if (err)
+                                return err;
+                }
+
+                *vm_flags &= ~VM_MERGEABLE;
+                break;
+        }
+
+        return 0;
+}
+
+int __ksm_enter(struct mm_struct *mm)
+{
+        struct mm_slot *mm_slot;
+        int needs_wakeup;
+
+        mm_slot = alloc_mm_slot();
+        if (!mm_slot)
+                return -ENOMEM;
+
+        /* Check ksm_run too?  Would need tighter locking */
+        needs_wakeup = list_empty(&ksm_mm_head.mm_list);
+
+        spin_lock(&ksm_mmlist_lock);
+        insert_to_mm_slots_hash(mm, mm_slot);
+        /*
+         * Insert just behind the scanning cursor, to let the area settle
+         * down a little; when fork is followed by immediate exec, we don't
+         * want ksmd to waste time setting up and tearing down an rmap_list.
+         */
+        list_add_tail(&mm_slot->mm_list, &ksm_scan.mm_slot->mm_list);
+        spin_unlock(&ksm_mmlist_lock);
+
+        set_bit(MMF_VM_MERGEABLE, &mm->flags);
+        atomic_inc(&mm->mm_count);
+
+        if (needs_wakeup)
+                wake_up_interruptible(&ksm_thread_wait);
+
+        return 0;
+}
+
+void __ksm_exit(struct mm_struct *mm)
+{
+        struct mm_slot *mm_slot;
+        int easy_to_free = 0;
+
+        /*
+         * This process is exiting: if it's straightforward (as is the
+         * case when ksmd was never running), free mm_slot immediately.
+         * But if it's at the cursor or has rmap_items linked to it, use
+         * mmap_sem to synchronize with any break_cows before pagetables
+         * are freed, and leave the mm_slot on the list for ksmd to free.
+         * Beware: ksm may already have noticed it exiting and freed the slot.
+         */
+
+        spin_lock(&ksm_mmlist_lock);
+        mm_slot = get_mm_slot(mm);
+        if (mm_slot && ksm_scan.mm_slot != mm_slot) {
+                if (list_empty(&mm_slot->rmap_list)) {
+                        hlist_del(&mm_slot->link);
+                        list_del(&mm_slot->mm_list);
+                        easy_to_free = 1;
+                } else {
+                        list_move(&mm_slot->mm_list,
+                                  &ksm_scan.mm_slot->mm_list);
+                }
+        }
+        spin_unlock(&ksm_mmlist_lock);
+
+        if (easy_to_free) {
+                free_mm_slot(mm_slot);
+                clear_bit(MMF_VM_MERGEABLE, &mm->flags);
+                mmdrop(mm);
+        } else if (mm_slot) {
+                down_write(&mm->mmap_sem);
+                up_write(&mm->mmap_sem);
+        }
+}
+
+#ifdef CONFIG_SYSFS
+/*
+ * This all compiles without CONFIG_SYSFS, but is a waste of space.
+ */
+
+#define KSM_ATTR_RO(_name) \
+        static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
+#define KSM_ATTR(_name) \
+        static struct kobj_attribute _name##_attr = \
+                __ATTR(_name, 0644, _name##_show, _name##_store)
+
+static ssize_t sleep_millisecs_show(struct kobject *kobj,
+                                    struct kobj_attribute *attr, char *buf)
+{
+        return sprintf(buf, "%u\n", ksm_thread_sleep_millisecs);
+}
+
+static ssize_t sleep_millisecs_store(struct kobject *kobj,
+                                     struct kobj_attribute *attr,
+                                     const char *buf, size_t count)
+{
+        unsigned long msecs;
+        int err;
+
+        err = strict_strtoul(buf, 10, &msecs);
+        if (err || msecs > UINT_MAX)
+                return -EINVAL;
+
+        ksm_thread_sleep_millisecs = msecs;
+
+        return count;
+}
+KSM_ATTR(sleep_millisecs);
+
+static ssize_t pages_to_scan_show(struct kobject *kobj,
+                                  struct kobj_attribute *attr, char *buf)
+{
+        return sprintf(buf, "%u\n", ksm_thread_pages_to_scan);
+}
+
+static ssize_t pages_to_scan_store(struct kobject *kobj,
+                                   struct kobj_attribute *attr,
+                                   const char *buf, size_t count)
+{
+        int err;
+        unsigned long nr_pages;
+
+        err = strict_strtoul(buf, 10, &nr_pages);
+        if (err || nr_pages > UINT_MAX)
+                return -EINVAL;
+
+        ksm_thread_pages_to_scan = nr_pages;
+
+        return count;
+}
+KSM_ATTR(pages_to_scan);
+
+static ssize_t run_show(struct kobject *kobj, struct kobj_attribute *attr,
+                        char *buf)
+{
+        return sprintf(buf, "%u\n", ksm_run);
+}
+
+static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr,
+                         const char *buf, size_t count)
+{
+        int err;
+        unsigned long flags;
+
+        err = strict_strtoul(buf, 10, &flags);
+        if (err || flags > UINT_MAX)
+                return -EINVAL;
+        if (flags > KSM_RUN_UNMERGE)
+                return -EINVAL;
+
+        /*
+         * KSM_RUN_MERGE sets ksmd running, and 0 stops it running.
+         * KSM_RUN_UNMERGE stops it running and unmerges all rmap_items,
+         * breaking COW to free the unswappable pages_shared (but leaves
+         * mm_slots on the list for when ksmd may be set running again).
+         */
+
+        mutex_lock(&ksm_thread_mutex);
+        if (ksm_run != flags) {
+                ksm_run = flags;
+                if (flags & KSM_RUN_UNMERGE) {
+                        current->flags |= PF_OOM_ORIGIN;
+                        err = unmerge_and_remove_all_rmap_items();
+                        current->flags &= ~PF_OOM_ORIGIN;
+                        if (err) {
+                                ksm_run = KSM_RUN_STOP;
+                                count = err;
+                        }
+                }
+        }
+        mutex_unlock(&ksm_thread_mutex);
+
+        if (flags & KSM_RUN_MERGE)
+                wake_up_interruptible(&ksm_thread_wait);
+
+        return count;
+}
+KSM_ATTR(run);
+
+static ssize_t max_kernel_pages_store(struct kobject *kobj,
+                                      struct kobj_attribute *attr,
+                                      const char *buf, size_t count)
+{
+        int err;
+        unsigned long nr_pages;
+
+        err = strict_strtoul(buf, 10, &nr_pages);
+        if (err)
+                return -EINVAL;
+
+        ksm_max_kernel_pages = nr_pages;
+
+        return count;
+}
+
+static ssize_t max_kernel_pages_show(struct kobject *kobj,
+                                     struct kobj_attribute *attr, char *buf)
+{
+        return sprintf(buf, "%lu\n", ksm_max_kernel_pages);
+}
+KSM_ATTR(max_kernel_pages);
+
+static ssize_t pages_shared_show(struct kobject *kobj,
+                                 struct kobj_attribute *attr, char *buf)
+{
+        return sprintf(buf, "%lu\n", ksm_pages_shared);
+}
+KSM_ATTR_RO(pages_shared);
+
+static ssize_t pages_sharing_show(struct kobject *kobj,
+                                  struct kobj_attribute *attr, char *buf)
+{
+        return sprintf(buf, "%lu\n", ksm_pages_sharing);
+}
+KSM_ATTR_RO(pages_sharing);
+
+static ssize_t pages_unshared_show(struct kobject *kobj,
+                                   struct kobj_attribute *attr, char *buf)
+{
+        return sprintf(buf, "%lu\n", ksm_pages_unshared);
+}
+KSM_ATTR_RO(pages_unshared);
+
+static ssize_t pages_volatile_show(struct kobject *kobj,
+                                   struct kobj_attribute *attr, char *buf)
+{
+        long ksm_pages_volatile;
+
+        ksm_pages_volatile = ksm_rmap_items - ksm_pages_shared
+                                - ksm_pages_sharing - ksm_pages_unshared;
+        /*
+         * It was not worth any locking to calculate that statistic,
+         * but it might therefore sometimes be negative: conceal that.
+         */
+        if (ksm_pages_volatile < 0)
+                ksm_pages_volatile = 0;
+        return sprintf(buf, "%ld\n", ksm_pages_volatile);
+}
+KSM_ATTR_RO(pages_volatile);
+
+static ssize_t full_scans_show(struct kobject *kobj,
+                               struct kobj_attribute *attr, char *buf)
+{
+        return sprintf(buf, "%lu\n", ksm_scan.seqnr);
+}
+KSM_ATTR_RO(full_scans);
+
+static struct attribute *ksm_attrs[] = {
+        &sleep_millisecs_attr.attr,
+        &pages_to_scan_attr.attr,
+        &run_attr.attr,
+        &max_kernel_pages_attr.attr,
+        &pages_shared_attr.attr,
+        &pages_sharing_attr.attr,
+        &pages_unshared_attr.attr,
+        &pages_volatile_attr.attr,
+        &full_scans_attr.attr,
+        NULL,
+};
+
+static struct attribute_group ksm_attr_group = {
+        .attrs = ksm_attrs,
+        .name = "ksm",
+};
+#endif /* CONFIG_SYSFS */
+
+static int __init ksm_init(void)
+{
+        struct task_struct *ksm_thread;
+        int err;
+
+        ksm_max_kernel_pages = totalram_pages / 4;
+
+        err = ksm_slab_init();
+        if (err)
+                goto out;
+
+        err = mm_slots_hash_init();
+        if (err)
+                goto out_free1;
+
+        ksm_thread = kthread_run(ksm_scan_thread, NULL, "ksmd");
+        if (IS_ERR(ksm_thread)) {
+                printk(KERN_ERR "ksm: creating kthread failed\n");
+                err = PTR_ERR(ksm_thread);
+                goto out_free2;
+        }
+
+#ifdef CONFIG_SYSFS
+        err = sysfs_create_group(mm_kobj, &ksm_attr_group);
+        if (err) {
+                printk(KERN_ERR "ksm: register sysfs failed\n");
+                kthread_stop(ksm_thread);
+                goto out_free2;
+        }
+#else
+        ksm_run = KSM_RUN_MERGE;        /* no way for user to start it */
+
+#endif /* CONFIG_SYSFS */
+
+        return 0;
+
+out_free2:
+        mm_slots_hash_free();
+out_free1:
+        ksm_slab_free();
+out:
+        return err;
+}
+module_init(ksm_init)
diff --git a/mm/maccess.c b/mm/maccess.c
index ac40796cfb15..9073695ff25f 100644
--- a/mm/maccess.c
+++ b/mm/maccess.c
@@ -39,7 +39,7 @@ EXPORT_SYMBOL_GPL(probe_kernel_read);
  * Safely write to address @dst from the buffer at @src.  If a kernel fault
  * happens, handle that and return -EFAULT.
  */
-long probe_kernel_write(void *dst, void *src, size_t size)
+long notrace __weak probe_kernel_write(void *dst, void *src, size_t size)
 {
         long ret;
         mm_segment_t old_fs = get_fs();
diff --git a/mm/madvise.c b/mm/madvise.c
index b9ce574827c8..35b1479b7c9d 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -11,6 +11,7 @@
 #include <linux/mempolicy.h>
 #include <linux/hugetlb.h>
 #include <linux/sched.h>
+#include <linux/ksm.h>
 
 /*
  * Any behaviour which results in changes to the vma->vm_flags needs to
@@ -41,7 +42,7 @@ static long madvise_behavior(struct vm_area_struct * vma,
         struct mm_struct * mm = vma->vm_mm;
         int error = 0;
         pgoff_t pgoff;
-        int new_flags = vma->vm_flags;
+        unsigned long new_flags = vma->vm_flags;
 
         switch (behavior) {
         case MADV_NORMAL:
@@ -57,8 +58,18 @@ static long madvise_behavior(struct vm_area_struct * vma,
                 new_flags |= VM_DONTCOPY;
                 break;
         case MADV_DOFORK:
+                if (vma->vm_flags & VM_IO) {
+                        error = -EINVAL;
+                        goto out;
+                }
                 new_flags &= ~VM_DONTCOPY;
                 break;
+        case MADV_MERGEABLE:
+        case MADV_UNMERGEABLE:
+                error = ksm_madvise(vma, start, end, behavior, &new_flags);
+                if (error)
+                        goto out;
+                break;
         }
 
         if (new_flags == vma->vm_flags) {
@@ -123,8 +134,7 @@ static long madvise_willneed(struct vm_area_struct * vma,
                 end = vma->vm_end;
         end = ((end - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
 
-        force_page_cache_readahead(file->f_mapping,
-                        file, start, max_sane_readahead(end - start));
+        force_page_cache_readahead(file->f_mapping, file, start, end - start);
         return 0;
 }
 
@@ -208,41 +218,69 @@ static long madvise_remove(struct vm_area_struct *vma,
         return error;
 }
 
+#ifdef CONFIG_MEMORY_FAILURE
+/*
+ * Error injection support for memory error handling.
+ */
+static int madvise_hwpoison(unsigned long start, unsigned long end)
+{
+        int ret = 0;
+
+        if (!capable(CAP_SYS_ADMIN))
+                return -EPERM;
+        for (; start < end; start += PAGE_SIZE) {
+                struct page *p;
+                int ret = get_user_pages(current, current->mm, start, 1,
+                                                0, 0, &p, NULL);
+                if (ret != 1)
+                        return ret;
+                printk(KERN_INFO "Injecting memory failure for page %lx at %lx\n",
+                       page_to_pfn(p), start);
+                /* Ignore return value for now */
+                __memory_failure(page_to_pfn(p), 0, 1);
+                put_page(p);
+        }
+        return ret;
+}
+#endif
+
 static long
 madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
                 unsigned long start, unsigned long end, int behavior)
 {
-        long error;
+        switch (behavior) {
+        case MADV_REMOVE:
+                return madvise_remove(vma, prev, start, end);
+        case MADV_WILLNEED:
+                return madvise_willneed(vma, prev, start, end);
+        case MADV_DONTNEED:
+                return madvise_dontneed(vma, prev, start, end);
+        default:
+                return madvise_behavior(vma, prev, start, end, behavior);
+        }
+}
 
+static int
+madvise_behavior_valid(int behavior)
+{
         switch (behavior) {
         case MADV_DOFORK:
-                if (vma->vm_flags & VM_IO) {
-                        error = -EINVAL;
-                        break;
-                }
         case MADV_DONTFORK:
         case MADV_NORMAL:
         case MADV_SEQUENTIAL:
         case MADV_RANDOM:
-                error = madvise_behavior(vma, prev, start, end, behavior);
-                break;
         case MADV_REMOVE:
-                error = madvise_remove(vma, prev, start, end);
-                break;
-
         case MADV_WILLNEED:
-                error = madvise_willneed(vma, prev, start, end);
-                break;
-
         case MADV_DONTNEED:
-                error = madvise_dontneed(vma, prev, start, end);
-                break;
+#ifdef CONFIG_KSM
+        case MADV_MERGEABLE:
+        case MADV_UNMERGEABLE:
+#endif
+                return 1;
 
         default:
-                error = -EINVAL;
-                break;
+                return 0;
         }
-        return error;
 }
 
 /*
@@ -269,6 +307,12 @@ madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
  *              so the kernel can free resources associated with it.
  *  MADV_REMOVE - the application wants to free up the given range of
  *              pages and associated backing store.
+ *  MADV_DONTFORK - omit this area from child's address space when forking:
+ *              typically, to avoid COWing pages pinned by get_user_pages().
+ *  MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
+ *  MADV_MERGEABLE - the application recommends that KSM try to merge pages in
+ *              this area with pages of identical content from other such areas.
+ *  MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
  *
  * return values:
  *  zero    - success
@@ -290,6 +334,13 @@ SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
         int write;
         size_t len;
 
+#ifdef CONFIG_MEMORY_FAILURE
+        if (behavior == MADV_HWPOISON)
+                return madvise_hwpoison(start, start+len_in);
+#endif
+        if (!madvise_behavior_valid(behavior))
+                return error;
+
         write = madvise_need_mmap_write(behavior);
         if (write)
                 down_write(&current->mm->mmap_sem);
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 8e4be9cb2a6a..f99f5991d6bb 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -27,7 +27,9 @@
 #include <linux/backing-dev.h>
 #include <linux/bit_spinlock.h>
 #include <linux/rcupdate.h>
+#include <linux/limits.h>
 #include <linux/mutex.h>
+#include <linux/rbtree.h>
 #include <linux/slab.h>
 #include <linux/swap.h>
 #include <linux/spinlock.h>
@@ -42,9 +44,10 @@
 
 struct cgroup_subsys mem_cgroup_subsys __read_mostly;
 #define MEM_CGROUP_RECLAIM_RETRIES      5
+struct mem_cgroup *root_mem_cgroup __read_mostly;
 
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
-/* Turned on only when memory cgroup is enabled && really_do_swap_account = 0 */
+/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
 int do_swap_account __read_mostly;
 static int really_do_swap_account __initdata = 1; /* for remember boot option*/
 #else
@@ -52,6 +55,7 @@ static int really_do_swap_account __initdata = 1; /* for remember boot option*/
 #endif
 
 static DEFINE_MUTEX(memcg_tasklist);    /* can be hold under cgroup_mutex */
+#define SOFTLIMIT_EVENTS_THRESH (1000)
 
 /*
  * Statistics for memory cgroup.
@@ -61,9 +65,12 @@ enum mem_cgroup_stat_index {
          * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
          */
         MEM_CGROUP_STAT_CACHE,     /* # of pages charged as cache */
-        MEM_CGROUP_STAT_RSS,       /* # of pages charged as rss */
+        MEM_CGROUP_STAT_RSS,       /* # of pages charged as anon rss */
+        MEM_CGROUP_STAT_MAPPED_FILE,  /* # of pages charged as file rss */
         MEM_CGROUP_STAT_PGPGIN_COUNT,   /* # of pages paged in */
         MEM_CGROUP_STAT_PGPGOUT_COUNT,  /* # of pages paged out */
+        MEM_CGROUP_STAT_EVENTS, /* sum of pagein + pageout for internal use */
+        MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
 
         MEM_CGROUP_STAT_NSTATS,
 };
@@ -76,6 +83,20 @@ struct mem_cgroup_stat {
         struct mem_cgroup_stat_cpu cpustat[0];
 };
 
+static inline void
+__mem_cgroup_stat_reset_safe(struct mem_cgroup_stat_cpu *stat,
+                                enum mem_cgroup_stat_index idx)
+{
+        stat->count[idx] = 0;
+}
+
+static inline s64
+__mem_cgroup_stat_read_local(struct mem_cgroup_stat_cpu *stat,
+                                enum mem_cgroup_stat_index idx)
+{
+        return stat->count[idx];
+}
+
 /*
  * For accounting under irq disable, no need for increment preempt count.
  */
@@ -95,6 +116,15 @@ static s64 mem_cgroup_read_stat(struct mem_cgroup_stat *stat,
         return ret;
 }
 
+static s64 mem_cgroup_local_usage(struct mem_cgroup_stat *stat)
+{
+        s64 ret;
+
+        ret = mem_cgroup_read_stat(stat, MEM_CGROUP_STAT_CACHE);
+        ret += mem_cgroup_read_stat(stat, MEM_CGROUP_STAT_RSS);
+        return ret;
+}
+
 /*
  * per-zone information in memory controller.
  */
@@ -106,6 +136,12 @@ struct mem_cgroup_per_zone {
         unsigned long           count[NR_LRU_LISTS];
 
         struct zone_reclaim_stat reclaim_stat;
+        struct rb_node          tree_node;      /* RB tree node */
+        unsigned long long      usage_in_excess;/* Set to the value by which */
+                                                /* the soft limit is exceeded*/
+        bool                    on_tree;
+        struct mem_cgroup       *mem;           /* Back pointer, we cannot */
+                                                /* use container_of        */
 };
 /* Macro for accessing counter */
 #define MEM_CGROUP_ZSTAT(mz, idx)       ((mz)->count[(idx)])
@@ -119,6 +155,26 @@ struct mem_cgroup_lru_info {
 };
 
 /*
+ * Cgroups above their limits are maintained in a RB-Tree, independent of
+ * their hierarchy representation
+ */
+
+struct mem_cgroup_tree_per_zone {
+        struct rb_root rb_root;
+        spinlock_t lock;
+};
+
+struct mem_cgroup_tree_per_node {
+        struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
+};
+
+struct mem_cgroup_tree {
+        struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
+};
+
+static struct mem_cgroup_tree soft_limit_tree __read_mostly;
+
+/*
  * The memory controller data structure. The memory controller controls both
  * page cache and RSS per cgroup. We would eventually like to provide
  * statistics based on the statistics developed by Rik Van Riel for clock-pro,
@@ -154,9 +210,9 @@ struct mem_cgroup {
 
         /*
          * While reclaiming in a hiearchy, we cache the last child we
-         * reclaimed from. Protected by hierarchy_mutex
+         * reclaimed from.
          */
-        struct mem_cgroup *last_scanned_child;
+        int last_scanned_child;
         /*
          * Should the accounting and control be hierarchical, per subtree?
          */
@@ -166,18 +222,29 @@ struct mem_cgroup {
 
         unsigned int    swappiness;
 
+        /* set when res.limit == memsw.limit */
+        bool            memsw_is_minimum;
+
         /*
          * statistics. This must be placed at the end of memcg.
          */
         struct mem_cgroup_stat stat;
 };
 
+/*
+ * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
+ * limit reclaim to prevent infinite loops, if they ever occur.
+ */
+#define MEM_CGROUP_MAX_RECLAIM_LOOPS            (100)
+#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS (2)
+
 enum charge_type {
         MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
         MEM_CGROUP_CHARGE_TYPE_MAPPED,
         MEM_CGROUP_CHARGE_TYPE_SHMEM,   /* used by page migration of shmem */
         MEM_CGROUP_CHARGE_TYPE_FORCE,   /* used by force_empty */
         MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
+        MEM_CGROUP_CHARGE_TYPE_DROP,    /* a page was unused swap cache */
         NR_CHARGE_TYPE,
 };
 
@@ -185,13 +252,8 @@ enum charge_type {
 #define PCGF_CACHE      (1UL << PCG_CACHE)
 #define PCGF_USED       (1UL << PCG_USED)
 #define PCGF_LOCK       (1UL << PCG_LOCK)
-static const unsigned long
-pcg_default_flags[NR_CHARGE_TYPE] = {
-        PCGF_CACHE | PCGF_USED | PCGF_LOCK, /* File Cache */
-        PCGF_USED | PCGF_LOCK, /* Anon */
-        PCGF_CACHE | PCGF_USED | PCGF_LOCK, /* Shmem */
-        0, /* FORCE */
-};
+/* Not used, but added here for completeness */
+#define PCGF_ACCT       (1UL << PCG_ACCT)
 
 /* for encoding cft->private value on file */
 #define _MEM                    (0)
@@ -200,15 +262,237 @@ pcg_default_flags[NR_CHARGE_TYPE] = {
 #define MEMFILE_TYPE(val)       (((val) >> 16) & 0xffff)
 #define MEMFILE_ATTR(val)       ((val) & 0xffff)
 
+/*
+ * Reclaim flags for mem_cgroup_hierarchical_reclaim
+ */
+#define MEM_CGROUP_RECLAIM_NOSWAP_BIT   0x0
+#define MEM_CGROUP_RECLAIM_NOSWAP       (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
+#define MEM_CGROUP_RECLAIM_SHRINK_BIT   0x1
+#define MEM_CGROUP_RECLAIM_SHRINK       (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
+#define MEM_CGROUP_RECLAIM_SOFT_BIT     0x2
+#define MEM_CGROUP_RECLAIM_SOFT         (1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
+
 static void mem_cgroup_get(struct mem_cgroup *mem);
 static void mem_cgroup_put(struct mem_cgroup *mem);
 static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
 
+static struct mem_cgroup_per_zone *
+mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
+{
+        return &mem->info.nodeinfo[nid]->zoneinfo[zid];
+}
+
+static struct mem_cgroup_per_zone *
+page_cgroup_zoneinfo(struct page_cgroup *pc)
+{
+        struct mem_cgroup *mem = pc->mem_cgroup;
+        int nid = page_cgroup_nid(pc);
+        int zid = page_cgroup_zid(pc);
+
+        if (!mem)
+                return NULL;
+
+        return mem_cgroup_zoneinfo(mem, nid, zid);
+}
+
+static struct mem_cgroup_tree_per_zone *
+soft_limit_tree_node_zone(int nid, int zid)
+{
+        return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
+}
+
+static struct mem_cgroup_tree_per_zone *
+soft_limit_tree_from_page(struct page *page)
+{
+        int nid = page_to_nid(page);
+        int zid = page_zonenum(page);
+
+        return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
+}
+
+static void
+__mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
+                                struct mem_cgroup_per_zone *mz,
+                                struct mem_cgroup_tree_per_zone *mctz,
+                                unsigned long long new_usage_in_excess)
+{
+        struct rb_node **p = &mctz->rb_root.rb_node;
+        struct rb_node *parent = NULL;
+        struct mem_cgroup_per_zone *mz_node;
+
+        if (mz->on_tree)
+                return;
+
+        mz->usage_in_excess = new_usage_in_excess;
+        if (!mz->usage_in_excess)
+                return;
+        while (*p) {
+                parent = *p;
+                mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
+                                        tree_node);
+                if (mz->usage_in_excess < mz_node->usage_in_excess)
+                        p = &(*p)->rb_left;
+                /*
+                 * We can't avoid mem cgroups that are over their soft
+                 * limit by the same amount
+                 */
+                else if (mz->usage_in_excess >= mz_node->usage_in_excess)
+                        p = &(*p)->rb_right;
+        }
+        rb_link_node(&mz->tree_node, parent, p);
+        rb_insert_color(&mz->tree_node, &mctz->rb_root);
+        mz->on_tree = true;
+}
+
+static void
+__mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
+                                struct mem_cgroup_per_zone *mz,
+                                struct mem_cgroup_tree_per_zone *mctz)
+{
+        if (!mz->on_tree)
+                return;
+        rb_erase(&mz->tree_node, &mctz->rb_root);
+        mz->on_tree = false;
+}
+
+static void
+mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
+                                struct mem_cgroup_per_zone *mz,
+                                struct mem_cgroup_tree_per_zone *mctz)
+{
+        spin_lock(&mctz->lock);
+        __mem_cgroup_remove_exceeded(mem, mz, mctz);
+        spin_unlock(&mctz->lock);
+}
+
+static bool mem_cgroup_soft_limit_check(struct mem_cgroup *mem)
+{
+        bool ret = false;
+        int cpu;
+        s64 val;
+        struct mem_cgroup_stat_cpu *cpustat;
+
+        cpu = get_cpu();
+        cpustat = &mem->stat.cpustat[cpu];
+        val = __mem_cgroup_stat_read_local(cpustat, MEM_CGROUP_STAT_EVENTS);
+        if (unlikely(val > SOFTLIMIT_EVENTS_THRESH)) {
+                __mem_cgroup_stat_reset_safe(cpustat, MEM_CGROUP_STAT_EVENTS);
+                ret = true;
+        }
+        put_cpu();
+        return ret;
+}
+
+static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
+{
+        unsigned long long excess;
+        struct mem_cgroup_per_zone *mz;
+        struct mem_cgroup_tree_per_zone *mctz;
+        int nid = page_to_nid(page);
+        int zid = page_zonenum(page);
+        mctz = soft_limit_tree_from_page(page);
+
+        /*
+         * Necessary to update all ancestors when hierarchy is used.
+         * because their event counter is not touched.
+         */
+        for (; mem; mem = parent_mem_cgroup(mem)) {
+                mz = mem_cgroup_zoneinfo(mem, nid, zid);
+                excess = res_counter_soft_limit_excess(&mem->res);
+                /*
+                 * We have to update the tree if mz is on RB-tree or
+                 * mem is over its softlimit.
+                 */
+                if (excess || mz->on_tree) {
+                        spin_lock(&mctz->lock);
+                        /* if on-tree, remove it */
+                        if (mz->on_tree)
+                                __mem_cgroup_remove_exceeded(mem, mz, mctz);
+                        /*
+                         * Insert again. mz->usage_in_excess will be updated.
+                         * If excess is 0, no tree ops.
+                         */
+                        __mem_cgroup_insert_exceeded(mem, mz, mctz, excess);
+                        spin_unlock(&mctz->lock);
+                }
+        }
+}
+
+static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem)
+{
+        int node, zone;
+        struct mem_cgroup_per_zone *mz;
+        struct mem_cgroup_tree_per_zone *mctz;
+
+        for_each_node_state(node, N_POSSIBLE) {
+                for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+                        mz = mem_cgroup_zoneinfo(mem, node, zone);
+                        mctz = soft_limit_tree_node_zone(node, zone);
+                        mem_cgroup_remove_exceeded(mem, mz, mctz);
+                }
+        }
+}
+
+static inline unsigned long mem_cgroup_get_excess(struct mem_cgroup *mem)
+{
+        return res_counter_soft_limit_excess(&mem->res) >> PAGE_SHIFT;
+}
+
+static struct mem_cgroup_per_zone *
+__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
+{
+        struct rb_node *rightmost = NULL;
+        struct mem_cgroup_per_zone *mz;
+
+retry:
+        mz = NULL;
+        rightmost = rb_last(&mctz->rb_root);
+        if (!rightmost)
+                goto done;              /* Nothing to reclaim from */
+
+        mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
+        /*
+         * Remove the node now but someone else can add it back,
+         * we will to add it back at the end of reclaim to its correct
+         * position in the tree.
+         */
+        __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
+        if (!res_counter_soft_limit_excess(&mz->mem->res) ||
+                !css_tryget(&mz->mem->css))
+                goto retry;
+done:
+        return mz;
+}
+
+static struct mem_cgroup_per_zone *
+mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
+{
+        struct mem_cgroup_per_zone *mz;
+
+        spin_lock(&mctz->lock);
+        mz = __mem_cgroup_largest_soft_limit_node(mctz);
+        spin_unlock(&mctz->lock);
+        return mz;
+}
+
+static void mem_cgroup_swap_statistics(struct mem_cgroup *mem,
+                                         bool charge)
+{
+        int val = (charge) ? 1 : -1;
+        struct mem_cgroup_stat *stat = &mem->stat;
+        struct mem_cgroup_stat_cpu *cpustat;
+        int cpu = get_cpu();
+
+        cpustat = &stat->cpustat[cpu];
+        __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_SWAPOUT, val);
+        put_cpu();
+}
+
 static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
                                          struct page_cgroup *pc,
                                          bool charge)
 {
-        int val = (charge)? 1 : -1;
+        int val = (charge) ? 1 : -1;
         struct mem_cgroup_stat *stat = &mem->stat;
         struct mem_cgroup_stat_cpu *cpustat;
         int cpu = get_cpu();
@@ -225,29 +509,11 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
         else
                 __mem_cgroup_stat_add_safe(cpustat,
                                 MEM_CGROUP_STAT_PGPGOUT_COUNT, 1);
+        __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_EVENTS, 1);
         put_cpu();
 }
 
-static struct mem_cgroup_per_zone *
-mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
-{
-        return &mem->info.nodeinfo[nid]->zoneinfo[zid];
-}
-
-static struct mem_cgroup_per_zone *
-page_cgroup_zoneinfo(struct page_cgroup *pc)
-{
-        struct mem_cgroup *mem = pc->mem_cgroup;
-        int nid = page_cgroup_nid(pc);
-        int zid = page_cgroup_zid(pc);
-
-        if (!mem)
-                return NULL;
-
-        return mem_cgroup_zoneinfo(mem, nid, zid);
-}
-
-static unsigned long mem_cgroup_get_all_zonestat(struct mem_cgroup *mem,
+static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
                                         enum lru_list idx)
 {
         int nid, zid;
@@ -286,6 +552,9 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
 static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
 {
         struct mem_cgroup *mem = NULL;
+
+        if (!mm)
+                return NULL;
         /*
          * Because we have no locks, mm->owner's may be being moved to other
          * cgroup. We use css_tryget() here even if this looks
@@ -301,11 +570,44 @@ static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
         return mem;
 }
 
-static bool mem_cgroup_is_obsolete(struct mem_cgroup *mem)
+/*
+ * Call callback function against all cgroup under hierarchy tree.
+ */
+static int mem_cgroup_walk_tree(struct mem_cgroup *root, void *data,
+                          int (*func)(struct mem_cgroup *, void *))
 {
-        if (!mem)
-                return true;
-        return css_is_removed(&mem->css);
+        int found, ret, nextid;
+        struct cgroup_subsys_state *css;
+        struct mem_cgroup *mem;
+
+        if (!root->use_hierarchy)
+                return (*func)(root, data);
+
+        nextid = 1;
+        do {
+                ret = 0;
+                mem = NULL;
+
+                rcu_read_lock();
+                css = css_get_next(&mem_cgroup_subsys, nextid, &root->css,
+                                   &found);
+                if (css && css_tryget(css))
+                        mem = container_of(css, struct mem_cgroup, css);
+                rcu_read_unlock();
+
+                if (mem) {
+                        ret = (*func)(mem, data);
+                        css_put(&mem->css);
+                }
+                nextid = found + 1;
+        } while (!ret && css);
+
+        return ret;
+}
+
+static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
+{
+        return (mem == root_mem_cgroup);
 }
 
 /*
@@ -325,22 +627,24 @@ static bool mem_cgroup_is_obsolete(struct mem_cgroup *mem)
 void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
 {
         struct page_cgroup *pc;
-        struct mem_cgroup *mem;
         struct mem_cgroup_per_zone *mz;
 
         if (mem_cgroup_disabled())
                 return;
         pc = lookup_page_cgroup(page);
         /* can happen while we handle swapcache. */
-        if (list_empty(&pc->lru) || !pc->mem_cgroup)
+        if (!TestClearPageCgroupAcctLRU(pc))
                 return;
+        VM_BUG_ON(!pc->mem_cgroup);
         /*
          * We don't check PCG_USED bit. It's cleared when the "page" is finally
          * removed from global LRU.
          */
         mz = page_cgroup_zoneinfo(pc);
-        mem = pc->mem_cgroup;
         MEM_CGROUP_ZSTAT(mz, lru) -= 1;
+        if (mem_cgroup_is_root(pc->mem_cgroup))
+                return;
+        VM_BUG_ON(list_empty(&pc->lru));
         list_del_init(&pc->lru);
         return;
 }
@@ -364,8 +668,8 @@ void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
          * For making pc->mem_cgroup visible, insert smp_rmb() here.
          */
         smp_rmb();
-        /* unused page is not rotated. */
-        if (!PageCgroupUsed(pc))
+        /* unused or root page is not rotated. */
+        if (!PageCgroupUsed(pc) || mem_cgroup_is_root(pc->mem_cgroup))
                 return;
         mz = page_cgroup_zoneinfo(pc);
         list_move(&pc->lru, &mz->lists[lru]);
@@ -379,6 +683,7 @@ void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
         if (mem_cgroup_disabled())
                 return;
         pc = lookup_page_cgroup(page);
+        VM_BUG_ON(PageCgroupAcctLRU(pc));
         /*
          * Used bit is set without atomic ops but after smp_wmb().
          * For making pc->mem_cgroup visible, insert smp_rmb() here.
@@ -389,6 +694,9 @@ void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
 
         mz = page_cgroup_zoneinfo(pc);
         MEM_CGROUP_ZSTAT(mz, lru) += 1;
+        SetPageCgroupAcctLRU(pc);
+        if (mem_cgroup_is_root(pc->mem_cgroup))
+                return;
         list_add(&pc->lru, &mz->lists[lru]);
 }
 
@@ -423,7 +731,7 @@ static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page)
 
         spin_lock_irqsave(&zone->lru_lock, flags);
         /* link when the page is linked to LRU but page_cgroup isn't */
-        if (PageLRU(page) && list_empty(&pc->lru))
+        if (PageLRU(page) && !PageCgroupAcctLRU(pc))
                 mem_cgroup_add_lru_list(page, page_lru(page));
         spin_unlock_irqrestore(&zone->lru_lock, flags);
 }
@@ -441,31 +749,24 @@ void mem_cgroup_move_lists(struct page *page,
 int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
 {
         int ret;
+        struct mem_cgroup *curr = NULL;
 
         task_lock(task);
-        ret = task->mm && mm_match_cgroup(task->mm, mem);
+        rcu_read_lock();
+        curr = try_get_mem_cgroup_from_mm(task->mm);
+        rcu_read_unlock();
         task_unlock(task);
+        if (!curr)
+                return 0;
+        if (curr->use_hierarchy)
+                ret = css_is_ancestor(&curr->css, &mem->css);
+        else
+                ret = (curr == mem);
+        css_put(&curr->css);
         return ret;
 }
 
 /*
- * Calculate mapped_ratio under memory controller. This will be used in
- * vmscan.c for deteremining we have to reclaim mapped pages.
- */
-int mem_cgroup_calc_mapped_ratio(struct mem_cgroup *mem)
-{
-        long total, rss;
-
-        /*
-         * usage is recorded in bytes. But, here, we assume the number of
-         * physical pages can be represented by "long" on any arch.
-         */
-        total = (long) (mem->res.usage >> PAGE_SHIFT) + 1L;
-        rss = (long)mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_RSS);
-        return (int)((rss * 100L) / total);
-}
-
-/*
  * prev_priority control...this will be used in memory reclaim path.
  */
 int mem_cgroup_get_reclaim_priority(struct mem_cgroup *mem)
@@ -501,8 +802,8 @@ static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_
         unsigned long gb;
         unsigned long inactive_ratio;
 
-        inactive = mem_cgroup_get_all_zonestat(memcg, LRU_INACTIVE_ANON);
-        active = mem_cgroup_get_all_zonestat(memcg, LRU_ACTIVE_ANON);
+        inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON);
+        active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON);
 
         gb = (inactive + active) >> (30 - PAGE_SHIFT);
         if (gb)
@@ -536,6 +837,17 @@ int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg)
         return 0;
 }
 
+int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
+{
+        unsigned long active;
+        unsigned long inactive;
+
+        inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE);
+        active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE);
+
+        return (active > inactive);
+}
+
 unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg,
                                        struct zone *zone,
                                        enum lru_list lru)
@@ -598,7 +910,8 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
         int nid = z->zone_pgdat->node_id;
         int zid = zone_idx(z);
         struct mem_cgroup_per_zone *mz;
-        int lru = LRU_FILE * !!file + !!active;
+        int lru = LRU_FILE * file + active;
+        int ret;
 
         BUG_ON(!mem_cont);
         mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
@@ -616,9 +929,19 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
                         continue;
 
                 scan++;
-                if (__isolate_lru_page(page, mode, file) == 0) {
+                ret = __isolate_lru_page(page, mode, file);
+                switch (ret) {
+                case 0:
                         list_move(&page->lru, dst);
+                        mem_cgroup_del_lru(page);
                         nr_taken++;
+                        break;
+                case -EBUSY:
+                        /* we don't affect global LRU but rotate in our LRU */
+                        mem_cgroup_rotate_lru_list(page, page_lru(page));
+                        break;
+                default:
+                        break;
                 }
         }
 
@@ -629,172 +952,243 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
 #define mem_cgroup_from_res_counter(counter, member)    \
         container_of(counter, struct mem_cgroup, member)
 
-/*
- * This routine finds the DFS walk successor. This routine should be
- * called with hierarchy_mutex held
- */
-static struct mem_cgroup *
-__mem_cgroup_get_next_node(struct mem_cgroup *curr, struct mem_cgroup *root_mem)
+static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem)
 {
-        struct cgroup *cgroup, *curr_cgroup, *root_cgroup;
-
-        curr_cgroup = curr->css.cgroup;
-        root_cgroup = root_mem->css.cgroup;
+        if (do_swap_account) {
+                if (res_counter_check_under_limit(&mem->res) &&
+                        res_counter_check_under_limit(&mem->memsw))
+                        return true;
+        } else
+                if (res_counter_check_under_limit(&mem->res))
+                        return true;
+        return false;
+}
 
-        if (!list_empty(&curr_cgroup->children)) {
-                /*
-                 * Walk down to children
-                 */
-                cgroup = list_entry(curr_cgroup->children.next,
-                                                struct cgroup, sibling);
-                curr = mem_cgroup_from_cont(cgroup);
-                goto done;
-        }
+static unsigned int get_swappiness(struct mem_cgroup *memcg)
+{
+        struct cgroup *cgrp = memcg->css.cgroup;
+        unsigned int swappiness;
 
-visit_parent:
-        if (curr_cgroup == root_cgroup) {
-                /* caller handles NULL case */
-                curr = NULL;
-                goto done;
-        }
+        /* root ? */
+        if (cgrp->parent == NULL)
+                return vm_swappiness;
 
-        /*
-         * Goto next sibling
-         */
-        if (curr_cgroup->sibling.next != &curr_cgroup->parent->children) {
-                cgroup = list_entry(curr_cgroup->sibling.next, struct cgroup,
-                                                sibling);
-                curr = mem_cgroup_from_cont(cgroup);
-                goto done;
-        }
+        spin_lock(&memcg->reclaim_param_lock);
+        swappiness = memcg->swappiness;
+        spin_unlock(&memcg->reclaim_param_lock);
 
-        /*
-         * Go up to next parent and next parent's sibling if need be
-         */
-        curr_cgroup = curr_cgroup->parent;
-        goto visit_parent;
+        return swappiness;
+}
 
-done:
-        return curr;
+static int mem_cgroup_count_children_cb(struct mem_cgroup *mem, void *data)
+{
+        int *val = data;
+        (*val)++;
+        return 0;
 }
 
-/*
- * Visit the first child (need not be the first child as per the ordering
- * of the cgroup list, since we track last_scanned_child) of @mem and use
- * that to reclaim free pages from.
+/**
+ * mem_cgroup_print_mem_info: Called from OOM with tasklist_lock held in read mode.
+ * @memcg: The memory cgroup that went over limit
+ * @p: Task that is going to be killed
+ *
+ * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
+ * enabled
  */
-static struct mem_cgroup *
-mem_cgroup_get_next_node(struct mem_cgroup *root_mem)
+void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
 {
-        struct cgroup *cgroup;
-        struct mem_cgroup *orig, *next;
-        bool obsolete;
-
+        struct cgroup *task_cgrp;
+        struct cgroup *mem_cgrp;
         /*
-         * Scan all children under the mem_cgroup mem
+         * Need a buffer in BSS, can't rely on allocations. The code relies
+         * on the assumption that OOM is serialized for memory controller.
+         * If this assumption is broken, revisit this code.
          */
-        mutex_lock(&mem_cgroup_subsys.hierarchy_mutex);
+        static char memcg_name[PATH_MAX];
+        int ret;
+
+        if (!memcg)
+                return;
 
-        orig = root_mem->last_scanned_child;
-        obsolete = mem_cgroup_is_obsolete(orig);
 
-        if (list_empty(&root_mem->css.cgroup->children)) {
+        rcu_read_lock();
+
+        mem_cgrp = memcg->css.cgroup;
+        task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
+
+        ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
+        if (ret < 0) {
                 /*
-                 * root_mem might have children before and last_scanned_child
-                 * may point to one of them. We put it later.
+                 * Unfortunately, we are unable to convert to a useful name
+                 * But we'll still print out the usage information
                  */
-                if (orig)
-                        VM_BUG_ON(!obsolete);
-                next = NULL;
+                rcu_read_unlock();
                 goto done;
         }
+        rcu_read_unlock();
 
-        if (!orig || obsolete) {
-                cgroup = list_first_entry(&root_mem->css.cgroup->children,
-                                struct cgroup, sibling);
-                next = mem_cgroup_from_cont(cgroup);
-        } else
-                next = __mem_cgroup_get_next_node(orig, root_mem);
+        printk(KERN_INFO "Task in %s killed", memcg_name);
+
+        rcu_read_lock();
+        ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
+        if (ret < 0) {
+                rcu_read_unlock();
+                goto done;
+        }
+        rcu_read_unlock();
 
+        /*
+         * Continues from above, so we don't need an KERN_ level
+         */
+        printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
 done:
-        if (next)
-                mem_cgroup_get(next);
-        root_mem->last_scanned_child = next;
-        if (orig)
-                mem_cgroup_put(orig);
-        mutex_unlock(&mem_cgroup_subsys.hierarchy_mutex);
-        return (next) ? next : root_mem;
+
+        printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
+                res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
+                res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
+                res_counter_read_u64(&memcg->res, RES_FAILCNT));
+        printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
+                "failcnt %llu\n",
+                res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
+                res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
+                res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
 }
 
-static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem)
+/*
+ * This function returns the number of memcg under hierarchy tree. Returns
+ * 1(self count) if no children.
+ */
+static int mem_cgroup_count_children(struct mem_cgroup *mem)
 {
-        if (do_swap_account) {
-                if (res_counter_check_under_limit(&mem->res) &&
-                        res_counter_check_under_limit(&mem->memsw))
-                        return true;
-        } else
-                if (res_counter_check_under_limit(&mem->res))
-                        return true;
-        return false;
+        int num = 0;
+        mem_cgroup_walk_tree(mem, &num, mem_cgroup_count_children_cb);
+        return num;
 }
 
-static unsigned int get_swappiness(struct mem_cgroup *memcg)
+/*
+ * Visit the first child (need not be the first child as per the ordering
+ * of the cgroup list, since we track last_scanned_child) of @mem and use
+ * that to reclaim free pages from.
+ */
+static struct mem_cgroup *
+mem_cgroup_select_victim(struct mem_cgroup *root_mem)
 {
-        struct cgroup *cgrp = memcg->css.cgroup;
-        unsigned int swappiness;
+        struct mem_cgroup *ret = NULL;
+        struct cgroup_subsys_state *css;
+        int nextid, found;
 
-        /* root ? */
-        if (cgrp->parent == NULL)
-                return vm_swappiness;
+        if (!root_mem->use_hierarchy) {
+                css_get(&root_mem->css);
+                ret = root_mem;
+        }
 
-        spin_lock(&memcg->reclaim_param_lock);
-        swappiness = memcg->swappiness;
-        spin_unlock(&memcg->reclaim_param_lock);
+        while (!ret) {
+                rcu_read_lock();
+                nextid = root_mem->last_scanned_child + 1;
+                css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css,
+                                   &found);
+                if (css && css_tryget(css))
+                        ret = container_of(css, struct mem_cgroup, css);
+
+                rcu_read_unlock();
+                /* Updates scanning parameter */
+                spin_lock(&root_mem->reclaim_param_lock);
+                if (!css) {
+                        /* this means start scan from ID:1 */
+                        root_mem->last_scanned_child = 0;
+                } else
+                        root_mem->last_scanned_child = found;
+                spin_unlock(&root_mem->reclaim_param_lock);
+        }
 
-        return swappiness;
+        return ret;
 }
 
 /*
- * Dance down the hierarchy if needed to reclaim memory. We remember the
- * last child we reclaimed from, so that we don't end up penalizing
- * one child extensively based on its position in the children list.
+ * Scan the hierarchy if needed to reclaim memory. We remember the last child
+ * we reclaimed from, so that we don't end up penalizing one child extensively
+ * based on its position in the children list.
  *
  * root_mem is the original ancestor that we've been reclaim from.
+ *
+ * We give up and return to the caller when we visit root_mem twice.
+ * (other groups can be removed while we're walking....)
+ *
+ * If shrink==true, for avoiding to free too much, this returns immedieately.
  */
 static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
-                                                gfp_t gfp_mask, bool noswap)
-{
-        struct mem_cgroup *next_mem;
-        int ret = 0;
-
-        /*
-         * Reclaim unconditionally and don't check for return value.
-         * We need to reclaim in the current group and down the tree.
-         * One might think about checking for children before reclaiming,
-         * but there might be left over accounting, even after children
-         * have left.
-         */
-        ret += try_to_free_mem_cgroup_pages(root_mem, gfp_mask, noswap,
-                                           get_swappiness(root_mem));
-        if (mem_cgroup_check_under_limit(root_mem))
-                return 1;       /* indicate reclaim has succeeded */
-        if (!root_mem->use_hierarchy)
-                return ret;
+                                                struct zone *zone,
+                                                gfp_t gfp_mask,
+                                                unsigned long reclaim_options)
+{
+        struct mem_cgroup *victim;
+        int ret, total = 0;
+        int loop = 0;
+        bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP;
+        bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK;
+        bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT;
+        unsigned long excess = mem_cgroup_get_excess(root_mem);
+
+        /* If memsw_is_minimum==1, swap-out is of-no-use. */
+        if (root_mem->memsw_is_minimum)
+                noswap = true;
 
-        next_mem = mem_cgroup_get_next_node(root_mem);
-
-        while (next_mem != root_mem) {
-                if (mem_cgroup_is_obsolete(next_mem)) {
-                        next_mem = mem_cgroup_get_next_node(root_mem);
+        while (1) {
+                victim = mem_cgroup_select_victim(root_mem);
+                if (victim == root_mem) {
+                        loop++;
+                        if (loop >= 2) {
+                                /*
+                                 * If we have not been able to reclaim
+                                 * anything, it might because there are
+                                 * no reclaimable pages under this hierarchy
+                                 */
+                                if (!check_soft || !total) {
+                                        css_put(&victim->css);
+                                        break;
+                                }
+                                /*
+                                 * We want to do more targetted reclaim.
+                                 * excess >> 2 is not to excessive so as to
+                                 * reclaim too much, nor too less that we keep
+                                 * coming back to reclaim from this cgroup
+                                 */
+                                if (total >= (excess >> 2) ||
+                                        (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) {
+                                        css_put(&victim->css);
+                                        break;
+                                }
+                        }
+                }
+                if (!mem_cgroup_local_usage(&victim->stat)) {
+                        /* this cgroup's local usage == 0 */
+                        css_put(&victim->css);
                         continue;
                 }
-                ret += try_to_free_mem_cgroup_pages(next_mem, gfp_mask, noswap,
-                                                   get_swappiness(next_mem));
-                if (mem_cgroup_check_under_limit(root_mem))
-                        return 1;       /* indicate reclaim has succeeded */
-                next_mem = mem_cgroup_get_next_node(root_mem);
+                /* we use swappiness of local cgroup */
+                if (check_soft)
+                        ret = mem_cgroup_shrink_node_zone(victim, gfp_mask,
+                                noswap, get_swappiness(victim), zone,
+                                zone->zone_pgdat->node_id);
+                else
+                        ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
+                                                noswap, get_swappiness(victim));
+                css_put(&victim->css);
+                /*
+                 * At shrinking usage, we can't check we should stop here or
+                 * reclaim more. It's depends on callers. last_scanned_child
+                 * will work enough for keeping fairness under tree.
+                 */
+                if (shrink)
+                        return ret;
+                total += ret;
+                if (check_soft) {
+                        if (res_counter_check_under_soft_limit(&root_mem->res))
+                                return total;
+                } else if (mem_cgroup_check_under_limit(root_mem))
+                        return 1 + total;
         }
-        return ret;
+        return total;
 }
 
 bool mem_cgroup_oom_called(struct task_struct *task)
@@ -813,13 +1207,64 @@ bool mem_cgroup_oom_called(struct task_struct *task)
         rcu_read_unlock();
         return ret;
 }
+
+static int record_last_oom_cb(struct mem_cgroup *mem, void *data)
+{
+        mem->last_oom_jiffies = jiffies;
+        return 0;
+}
+
+static void record_last_oom(struct mem_cgroup *mem)
+{
+        mem_cgroup_walk_tree(mem, NULL, record_last_oom_cb);
+}
+
+/*
+ * Currently used to update mapped file statistics, but the routine can be
+ * generalized to update other statistics as well.
+ */
+void mem_cgroup_update_mapped_file_stat(struct page *page, int val)
+{
+        struct mem_cgroup *mem;
+        struct mem_cgroup_stat *stat;
+        struct mem_cgroup_stat_cpu *cpustat;
+        int cpu;
+        struct page_cgroup *pc;
+
+        if (!page_is_file_cache(page))
+                return;
+
+        pc = lookup_page_cgroup(page);
+        if (unlikely(!pc))
+                return;
+
+        lock_page_cgroup(pc);
+        mem = pc->mem_cgroup;
+        if (!mem)
+                goto done;
+
+        if (!PageCgroupUsed(pc))
+                goto done;
+
+        /*
+         * Preemption is already disabled, we don't need get_cpu()
+         */
+        cpu = smp_processor_id();
+        stat = &mem->stat;
+        cpustat = &stat->cpustat[cpu];
+
+        __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_MAPPED_FILE, val);
+done:
+        unlock_page_cgroup(pc);
+}
+
 /*
  * Unlike exported interface, "oom" parameter is added. if oom==true,
  * oom-killer can be invoked.
  */
 static int __mem_cgroup_try_charge(struct mm_struct *mm,
                         gfp_t gfp_mask, struct mem_cgroup **memcg,
-                        bool oom)
+                        bool oom, struct page *page)
 {
         struct mem_cgroup *mem, *mem_over_limit;
         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
@@ -847,12 +1292,14 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
         if (unlikely(!mem))
                 return 0;
 
-        VM_BUG_ON(mem_cgroup_is_obsolete(mem));
+        VM_BUG_ON(css_is_removed(&mem->css));
 
         while (1) {
-                int ret;
-                bool noswap = false;
+                int ret = 0;
+                unsigned long flags = 0;
 
+                if (mem_cgroup_is_root(mem))
+                        goto done;
                 ret = res_counter_charge(&mem->res, PAGE_SIZE, &fail_res);
                 if (likely(!ret)) {
                         if (!do_swap_account)
@@ -863,7 +1310,7 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
                                 break;
                         /* mem+swap counter fails */
                         res_counter_uncharge(&mem->res, PAGE_SIZE);
-                        noswap = true;
+                        flags |= MEM_CGROUP_RECLAIM_NOSWAP;
                         mem_over_limit = mem_cgroup_from_res_counter(fail_res,
                                                                         memsw);
                 } else
@@ -874,8 +1321,8 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
                 if (!(gfp_mask & __GFP_WAIT))
                         goto nomem;
 
-                ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, gfp_mask,
-                                                        noswap);
+                ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
+                                                gfp_mask, flags);
                 if (ret)
                         continue;
 
@@ -895,31 +1342,71 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
                                 mutex_lock(&memcg_tasklist);
                                 mem_cgroup_out_of_memory(mem_over_limit, gfp_mask);
                                 mutex_unlock(&memcg_tasklist);
-                                mem_over_limit->last_oom_jiffies = jiffies;
+                                record_last_oom(mem_over_limit);
                         }
                         goto nomem;
                 }
         }
+        /*
+         * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
+         * if they exceeds softlimit.
+         */
+        if (mem_cgroup_soft_limit_check(mem))
+                mem_cgroup_update_tree(mem, page);
+done:
         return 0;
 nomem:
         css_put(&mem->css);
         return -ENOMEM;
 }
 
+/*
+ * A helper function to get mem_cgroup from ID. must be called under
+ * rcu_read_lock(). The caller must check css_is_removed() or some if
+ * it's concern. (dropping refcnt from swap can be called against removed
+ * memcg.)
+ */
+static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
+{
+        struct cgroup_subsys_state *css;
+
+        /* ID 0 is unused ID */
+        if (!id)
+                return NULL;
+        css = css_lookup(&mem_cgroup_subsys, id);
+        if (!css)
+                return NULL;
+        return container_of(css, struct mem_cgroup, css);
+}
+
 static struct mem_cgroup *try_get_mem_cgroup_from_swapcache(struct page *page)
 {
         struct mem_cgroup *mem;
+        struct page_cgroup *pc;
+        unsigned short id;
         swp_entry_t ent;
 
+        VM_BUG_ON(!PageLocked(page));
+
         if (!PageSwapCache(page))
                 return NULL;
 
-        ent.val = page_private(page);
-        mem = lookup_swap_cgroup(ent);
-        if (!mem)
-                return NULL;
-        if (!css_tryget(&mem->css))
-                return NULL;
+        pc = lookup_page_cgroup(page);
+        lock_page_cgroup(pc);
+        if (PageCgroupUsed(pc)) {
+                mem = pc->mem_cgroup;
+                if (mem && !css_tryget(&mem->css))
+                        mem = NULL;
+        } else {
+                ent.val = page_private(page);
+                id = lookup_swap_cgroup(ent);
+                rcu_read_lock();
+                mem = mem_cgroup_lookup(id);
+                if (mem && !css_tryget(&mem->css))
+                        mem = NULL;
+                rcu_read_unlock();
+        }
+        unlock_page_cgroup(pc);
         return mem;
 }
 
@@ -939,15 +1426,37 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
         lock_page_cgroup(pc);
         if (unlikely(PageCgroupUsed(pc))) {
                 unlock_page_cgroup(pc);
-                res_counter_uncharge(&mem->res, PAGE_SIZE);
-                if (do_swap_account)
-                        res_counter_uncharge(&mem->memsw, PAGE_SIZE);
+                if (!mem_cgroup_is_root(mem)) {
+                        res_counter_uncharge(&mem->res, PAGE_SIZE);
+                        if (do_swap_account)
+                                res_counter_uncharge(&mem->memsw, PAGE_SIZE);
+                }
                 css_put(&mem->css);
                 return;
         }
+
         pc->mem_cgroup = mem;
+        /*
+         * We access a page_cgroup asynchronously without lock_page_cgroup().
+         * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
+         * is accessed after testing USED bit. To make pc->mem_cgroup visible
+         * before USED bit, we need memory barrier here.
+         * See mem_cgroup_add_lru_list(), etc.
+         */
         smp_wmb();
-        pc->flags = pcg_default_flags[ctype];
+        switch (ctype) {
+        case MEM_CGROUP_CHARGE_TYPE_CACHE:
+        case MEM_CGROUP_CHARGE_TYPE_SHMEM:
+                SetPageCgroupCache(pc);
+                SetPageCgroupUsed(pc);
+                break;
+        case MEM_CGROUP_CHARGE_TYPE_MAPPED:
+                ClearPageCgroupCache(pc);
+                SetPageCgroupUsed(pc);
+                break;
+        default:
+                break;
+        }
 
         mem_cgroup_charge_statistics(mem, pc, true);
 
@@ -976,6 +1485,10 @@ static int mem_cgroup_move_account(struct page_cgroup *pc,
         struct mem_cgroup_per_zone *from_mz, *to_mz;
         int nid, zid;
         int ret = -EBUSY;
+        struct page *page;
+        int cpu;
+        struct mem_cgroup_stat *stat;
+        struct mem_cgroup_stat_cpu *cpustat;
 
         VM_BUG_ON(from == to);
         VM_BUG_ON(PageLRU(pc->page));
@@ -994,9 +1507,27 @@ static int mem_cgroup_move_account(struct page_cgroup *pc,
         if (pc->mem_cgroup != from)
                 goto out;
 
-        res_counter_uncharge(&from->res, PAGE_SIZE);
+        if (!mem_cgroup_is_root(from))
+                res_counter_uncharge(&from->res, PAGE_SIZE);
         mem_cgroup_charge_statistics(from, pc, false);
-        if (do_swap_account)
+
+        page = pc->page;
+        if (page_is_file_cache(page) && page_mapped(page)) {
+                cpu = smp_processor_id();
+                /* Update mapped_file data for mem_cgroup "from" */
+                stat = &from->stat;
+                cpustat = &stat->cpustat[cpu];
+                __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_MAPPED_FILE,
+                                                -1);
+
+                /* Update mapped_file data for mem_cgroup "to" */
+                stat = &to->stat;
+                cpustat = &stat->cpustat[cpu];
+                __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_MAPPED_FILE,
+                                                1);
+        }
+
+        if (do_swap_account && !mem_cgroup_is_root(from))
                 res_counter_uncharge(&from->memsw, PAGE_SIZE);
         css_put(&from->css);
 
@@ -1006,6 +1537,12 @@ static int mem_cgroup_move_account(struct page_cgroup *pc,
         ret = 0;
 out:
         unlock_page_cgroup(pc);
+        /*
+         * We charges against "to" which may not have any tasks. Then, "to"
+         * can be under rmdir(). But in current implementation, caller of
+         * this function is just force_empty() and it's garanteed that
+         * "to" is never removed. So, we don't check rmdir status here.
+         */
         return ret;
 }
 
@@ -1031,7 +1568,7 @@ static int mem_cgroup_move_parent(struct page_cgroup *pc,
         parent = mem_cgroup_from_cont(pcg);
 
 
-        ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false);
+        ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false, page);
         if (ret || !parent)
                 return ret;
 
@@ -1061,9 +1598,11 @@ uncharge:
         /* drop extra refcnt by try_charge() */
         css_put(&parent->css);
         /* uncharge if move fails */
-        res_counter_uncharge(&parent->res, PAGE_SIZE);
-        if (do_swap_account)
-                res_counter_uncharge(&parent->memsw, PAGE_SIZE);
+        if (!mem_cgroup_is_root(parent)) {
+                res_counter_uncharge(&parent->res, PAGE_SIZE);
+                if (do_swap_account)
+                        res_counter_uncharge(&parent->memsw, PAGE_SIZE);
+        }
         return ret;
 }
 
@@ -1088,7 +1627,7 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
         prefetchw(pc);
 
         mem = memcg;
-        ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true);
+        ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true, page);
         if (ret || !mem)
                 return ret;
 
@@ -1118,6 +1657,10 @@ int mem_cgroup_newpage_charge(struct page *page,
                                 MEM_CGROUP_CHARGE_TYPE_MAPPED, NULL);
 }
 
+static void
+__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
+                                        enum charge_type ctype);
+
 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
                                 gfp_t gfp_mask)
 {
@@ -1154,16 +1697,6 @@ int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
                 unlock_page_cgroup(pc);
         }
 
-        if (do_swap_account && PageSwapCache(page)) {
-                mem = try_get_mem_cgroup_from_swapcache(page);
-                if (mem)
-                        mm = NULL;
-                  else
-                        mem = NULL;
-                /* SwapCache may be still linked to LRU now. */
-                mem_cgroup_lru_del_before_commit_swapcache(page);
-        }
-
         if (unlikely(!mm && !mem))
                 mm = &init_mm;
 
@@ -1171,22 +1704,16 @@ int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
                 return mem_cgroup_charge_common(page, mm, gfp_mask,
                                 MEM_CGROUP_CHARGE_TYPE_CACHE, NULL);
 
-        ret = mem_cgroup_charge_common(page, mm, gfp_mask,
-                                MEM_CGROUP_CHARGE_TYPE_SHMEM, mem);
-        if (mem)
-                css_put(&mem->css);
-        if (PageSwapCache(page))
-                mem_cgroup_lru_add_after_commit_swapcache(page);
+        /* shmem */
+        if (PageSwapCache(page)) {
+                ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
+                if (!ret)
+                        __mem_cgroup_commit_charge_swapin(page, mem,
+                                        MEM_CGROUP_CHARGE_TYPE_SHMEM);
+        } else
+                ret = mem_cgroup_charge_common(page, mm, gfp_mask,
+                                        MEM_CGROUP_CHARGE_TYPE_SHMEM, mem);
 
-        if (do_swap_account && !ret && PageSwapCache(page)) {
-                swp_entry_t ent = {.val = page_private(page)};
-                /* avoid double counting */
-                mem = swap_cgroup_record(ent, NULL);
-                if (mem) {
-                        res_counter_uncharge(&mem->memsw, PAGE_SIZE);
-                        mem_cgroup_put(mem);
-                }
-        }
         return ret;
 }
 
@@ -1219,17 +1746,19 @@ int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
         if (!mem)
                 goto charge_cur_mm;
         *ptr = mem;
-        ret = __mem_cgroup_try_charge(NULL, mask, ptr, true);
+        ret = __mem_cgroup_try_charge(NULL, mask, ptr, true, page);
         /* drop extra refcnt from tryget */
         css_put(&mem->css);
         return ret;
 charge_cur_mm:
         if (unlikely(!mm))
                 mm = &init_mm;
-        return __mem_cgroup_try_charge(mm, mask, ptr, true);
+        return __mem_cgroup_try_charge(mm, mask, ptr, true, page);
 }
 
-void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
+static void
+__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
+                                        enum charge_type ctype)
 {
         struct page_cgroup *pc;
 
@@ -1237,9 +1766,10 @@ void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
                 return;
         if (!ptr)
                 return;
+        cgroup_exclude_rmdir(&ptr->css);
         pc = lookup_page_cgroup(page);
         mem_cgroup_lru_del_before_commit_swapcache(page);
-        __mem_cgroup_commit_charge(ptr, pc, MEM_CGROUP_CHARGE_TYPE_MAPPED);
+        __mem_cgroup_commit_charge(ptr, pc, ctype);
         mem_cgroup_lru_add_after_commit_swapcache(page);
         /*
          * Now swap is on-memory. This means this page may be
@@ -1250,16 +1780,36 @@ void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
          */
         if (do_swap_account && PageSwapCache(page)) {
                 swp_entry_t ent = {.val = page_private(page)};
+                unsigned short id;
                 struct mem_cgroup *memcg;
-                memcg = swap_cgroup_record(ent, NULL);
+
+                id = swap_cgroup_record(ent, 0);
+                rcu_read_lock();
+                memcg = mem_cgroup_lookup(id);
                 if (memcg) {
-                        res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
+                        /*
+                         * This recorded memcg can be obsolete one. So, avoid
+                         * calling css_tryget
+                         */
+                        if (!mem_cgroup_is_root(memcg))
+                                res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
+                        mem_cgroup_swap_statistics(memcg, false);
                         mem_cgroup_put(memcg);
                 }
-
+                rcu_read_unlock();
         }
-        /* add this page(page_cgroup) to the LRU we want. */
+        /*
+         * At swapin, we may charge account against cgroup which has no tasks.
+         * So, rmdir()->pre_destroy() can be called while we do this charge.
+         * In that case, we need to call pre_destroy() again. check it here.
+         */
+        cgroup_release_and_wakeup_rmdir(&ptr->css);
+}
 
+void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
+{
+        __mem_cgroup_commit_charge_swapin(page, ptr,
+                                        MEM_CGROUP_CHARGE_TYPE_MAPPED);
 }
 
 void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
@@ -1268,9 +1818,11 @@ void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
                 return;
         if (!mem)
                 return;
-        res_counter_uncharge(&mem->res, PAGE_SIZE);
-        if (do_swap_account)
-                res_counter_uncharge(&mem->memsw, PAGE_SIZE);
+        if (!mem_cgroup_is_root(mem)) {
+                res_counter_uncharge(&mem->res, PAGE_SIZE);
+                if (do_swap_account)
+                        res_counter_uncharge(&mem->memsw, PAGE_SIZE);
+        }
         css_put(&mem->css);
 }
 
@@ -1307,6 +1859,7 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
 
         switch (ctype) {
         case MEM_CGROUP_CHARGE_TYPE_MAPPED:
+        case MEM_CGROUP_CHARGE_TYPE_DROP:
                 if (page_mapped(page))
                         goto unlock_out;
                 break;
@@ -1321,11 +1874,16 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
                 break;
         }
 
-        res_counter_uncharge(&mem->res, PAGE_SIZE);
-        if (do_swap_account && (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT))
-                res_counter_uncharge(&mem->memsw, PAGE_SIZE);
-
+        if (!mem_cgroup_is_root(mem)) {
+                res_counter_uncharge(&mem->res, PAGE_SIZE);
+                if (do_swap_account &&
+                                (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT))
+                        res_counter_uncharge(&mem->memsw, PAGE_SIZE);
+        }
+        if (ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
+                mem_cgroup_swap_statistics(mem, true);
         mem_cgroup_charge_statistics(mem, pc, false);
+
         ClearPageCgroupUsed(pc);
         /*
          * pc->mem_cgroup is not cleared here. It will be accessed when it's
@@ -1337,6 +1895,8 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
         mz = page_cgroup_zoneinfo(pc);
         unlock_page_cgroup(pc);
 
+        if (mem_cgroup_soft_limit_check(mem))
+                mem_cgroup_update_tree(mem, page);
         /* at swapout, this memcg will be accessed to record to swap */
         if (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
                 css_put(&mem->css);
@@ -1365,24 +1925,31 @@ void mem_cgroup_uncharge_cache_page(struct page *page)
         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
 }
 
+#ifdef CONFIG_SWAP
 /*
- * called from __delete_from_swap_cache() and drop "page" account.
+ * called after __delete_from_swap_cache() and drop "page" account.
  * memcg information is recorded to swap_cgroup of "ent"
  */
-void mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent)
+void
+mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
 {
         struct mem_cgroup *memcg;
+        int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
+
+        if (!swapout) /* this was a swap cache but the swap is unused ! */
+                ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
+
+        memcg = __mem_cgroup_uncharge_common(page, ctype);
 
-        memcg = __mem_cgroup_uncharge_common(page,
-                                        MEM_CGROUP_CHARGE_TYPE_SWAPOUT);
         /* record memcg information */
-        if (do_swap_account && memcg) {
-                swap_cgroup_record(ent, memcg);
+        if (do_swap_account && swapout && memcg) {
+                swap_cgroup_record(ent, css_id(&memcg->css));
                 mem_cgroup_get(memcg);
         }
-        if (memcg)
+        if (swapout && memcg)
                 css_put(&memcg->css);
 }
+#endif
 
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
 /*
@@ -1392,15 +1959,25 @@ void mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent)
 void mem_cgroup_uncharge_swap(swp_entry_t ent)
 {
         struct mem_cgroup *memcg;
+        unsigned short id;
 
         if (!do_swap_account)
                 return;
 
-        memcg = swap_cgroup_record(ent, NULL);
+        id = swap_cgroup_record(ent, 0);
+        rcu_read_lock();
+        memcg = mem_cgroup_lookup(id);
         if (memcg) {
-                res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
+                /*
+                 * We uncharge this because swap is freed.
+                 * This memcg can be obsolete one. We avoid calling css_tryget
+                 */
+                if (!mem_cgroup_is_root(memcg))
+                        res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
+                mem_cgroup_swap_statistics(memcg, false);
                 mem_cgroup_put(memcg);
         }
+        rcu_read_unlock();
 }
 #endif
 
@@ -1426,7 +2003,8 @@ int mem_cgroup_prepare_migration(struct page *page, struct mem_cgroup **ptr)
         unlock_page_cgroup(pc);
 
         if (mem) {
-                ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false);
+                ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false,
+                                                page);
                 css_put(&mem->css);
         }
         *ptr = mem;
@@ -1443,7 +2021,7 @@ void mem_cgroup_end_migration(struct mem_cgroup *mem,
 
         if (!mem)
                 return;
-
+        cgroup_exclude_rmdir(&mem->css);
         /* at migration success, oldpage->mapping is NULL. */
         if (oldpage->mapping) {
                 target = oldpage;
@@ -1483,39 +2061,37 @@ void mem_cgroup_end_migration(struct mem_cgroup *mem,
          */
         if (ctype == MEM_CGROUP_CHARGE_TYPE_MAPPED)
                 mem_cgroup_uncharge_page(target);
+        /*
+         * At migration, we may charge account against cgroup which has no tasks
+         * So, rmdir()->pre_destroy() can be called while we do this charge.
+         * In that case, we need to call pre_destroy() again. check it here.
+         */
+        cgroup_release_and_wakeup_rmdir(&mem->css);
 }
 
 /*
- * A call to try to shrink memory usage under specified resource controller.
- * This is typically used for page reclaiming for shmem for reducing side
- * effect of page allocation from shmem, which is used by some mem_cgroup.
+ * A call to try to shrink memory usage on charge failure at shmem's swapin.
+ * Calling hierarchical_reclaim is not enough because we should update
+ * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM.
+ * Moreover considering hierarchy, we should reclaim from the mem_over_limit,
+ * not from the memcg which this page would be charged to.
+ * try_charge_swapin does all of these works properly.
  */
-int mem_cgroup_shrink_usage(struct page *page,
+int mem_cgroup_shmem_charge_fallback(struct page *page,
                             struct mm_struct *mm,
                             gfp_t gfp_mask)
 {
         struct mem_cgroup *mem = NULL;
-        int progress = 0;
-        int retry = MEM_CGROUP_RECLAIM_RETRIES;
+        int ret;
 
         if (mem_cgroup_disabled())
                 return 0;
-        if (page)
-                mem = try_get_mem_cgroup_from_swapcache(page);
-        if (!mem && mm)
-                mem = try_get_mem_cgroup_from_mm(mm);
-        if (unlikely(!mem))
-                return 0;
 
-        do {
-                progress = mem_cgroup_hierarchical_reclaim(mem, gfp_mask, true);
-                progress += mem_cgroup_check_under_limit(mem);
-        } while (!progress && --retry);
+        ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
+        if (!ret)
+                mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */
 
-        css_put(&mem->css);
-        if (!retry)
-                return -ENOMEM;
-        return 0;
+        return ret;
 }
 
 static DEFINE_MUTEX(set_limit_mutex);
@@ -1523,11 +2099,21 @@ static DEFINE_MUTEX(set_limit_mutex);
 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
                                 unsigned long long val)
 {
-
-        int retry_count = MEM_CGROUP_RECLAIM_RETRIES;
+        int retry_count;
         int progress;
         u64 memswlimit;
         int ret = 0;
+        int children = mem_cgroup_count_children(memcg);
+        u64 curusage, oldusage;
+
+        /*
+         * For keeping hierarchical_reclaim simple, how long we should retry
+         * is depends on callers. We set our retry-count to be function
+         * of # of children which we should visit in this loop.
+         */
+        retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
+
+        oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
 
         while (retry_count) {
                 if (signal_pending(current)) {
@@ -1547,29 +2133,42 @@ static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
                         break;
                 }
                 ret = res_counter_set_limit(&memcg->res, val);
+                if (!ret) {
+                        if (memswlimit == val)
+                                memcg->memsw_is_minimum = true;
+                        else
+                                memcg->memsw_is_minimum = false;
+                }
                 mutex_unlock(&set_limit_mutex);
 
                 if (!ret)
                         break;
 
-                progress = mem_cgroup_hierarchical_reclaim(memcg, GFP_KERNEL,
-                                                           false);
-                if (!progress)                  retry_count--;
+                progress = mem_cgroup_hierarchical_reclaim(memcg, NULL,
+                                                GFP_KERNEL,
+                                                MEM_CGROUP_RECLAIM_SHRINK);
+                curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
+                /* Usage is reduced ? */
+                if (curusage >= oldusage)
+                        retry_count--;
+                else
+                        oldusage = curusage;
         }
 
         return ret;
 }
 
-int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
-                                unsigned long long val)
+static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
+                                        unsigned long long val)
 {
-        int retry_count = MEM_CGROUP_RECLAIM_RETRIES;
+        int retry_count;
         u64 memlimit, oldusage, curusage;
-        int ret;
-
-        if (!do_swap_account)
-                return -EINVAL;
+        int children = mem_cgroup_count_children(memcg);
+        int ret = -EBUSY;
 
+        /* see mem_cgroup_resize_res_limit */
+        retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
+        oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
         while (retry_count) {
                 if (signal_pending(current)) {
                         ret = -EINTR;
@@ -1588,20 +2187,121 @@ int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
                         break;
                 }
                 ret = res_counter_set_limit(&memcg->memsw, val);
+                if (!ret) {
+                        if (memlimit == val)
+                                memcg->memsw_is_minimum = true;
+                        else
+                                memcg->memsw_is_minimum = false;
+                }
                 mutex_unlock(&set_limit_mutex);
 
                 if (!ret)
                         break;
 
-                oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
-                mem_cgroup_hierarchical_reclaim(memcg, GFP_KERNEL, true);
+                mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
+                                                MEM_CGROUP_RECLAIM_NOSWAP |
+                                                MEM_CGROUP_RECLAIM_SHRINK);
                 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
+                /* Usage is reduced ? */
                 if (curusage >= oldusage)
                         retry_count--;
+                else
+                        oldusage = curusage;
         }
         return ret;
 }
 
+unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
+                                                gfp_t gfp_mask, int nid,
+                                                int zid)
+{
+        unsigned long nr_reclaimed = 0;
+        struct mem_cgroup_per_zone *mz, *next_mz = NULL;
+        unsigned long reclaimed;
+        int loop = 0;
+        struct mem_cgroup_tree_per_zone *mctz;
+        unsigned long long excess;
+
+        if (order > 0)
+                return 0;
+
+        mctz = soft_limit_tree_node_zone(nid, zid);
+        /*
+         * This loop can run a while, specially if mem_cgroup's continuously
+         * keep exceeding their soft limit and putting the system under
+         * pressure
+         */
+        do {
+                if (next_mz)
+                        mz = next_mz;
+                else
+                        mz = mem_cgroup_largest_soft_limit_node(mctz);
+                if (!mz)
+                        break;
+
+                reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone,
+                                                gfp_mask,
+                                                MEM_CGROUP_RECLAIM_SOFT);
+                nr_reclaimed += reclaimed;
+                spin_lock(&mctz->lock);
+
+                /*
+                 * If we failed to reclaim anything from this memory cgroup
+                 * it is time to move on to the next cgroup
+                 */
+                next_mz = NULL;
+                if (!reclaimed) {
+                        do {
+                                /*
+                                 * Loop until we find yet another one.
+                                 *
+                                 * By the time we get the soft_limit lock
+                                 * again, someone might have aded the
+                                 * group back on the RB tree. Iterate to
+                                 * make sure we get a different mem.
+                                 * mem_cgroup_largest_soft_limit_node returns
+                                 * NULL if no other cgroup is present on
+                                 * the tree
+                                 */
+                                next_mz =
+                                __mem_cgroup_largest_soft_limit_node(mctz);
+                                if (next_mz == mz) {
+                                        css_put(&next_mz->mem->css);
+                                        next_mz = NULL;
+                                } else /* next_mz == NULL or other memcg */
+                                        break;
+                        } while (1);
+                }
+                __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
+                excess = res_counter_soft_limit_excess(&mz->mem->res);
+                /*
+                 * One school of thought says that we should not add
+                 * back the node to the tree if reclaim returns 0.
+                 * But our reclaim could return 0, simply because due
+                 * to priority we are exposing a smaller subset of
+                 * memory to reclaim from. Consider this as a longer
+                 * term TODO.
+                 */
+                /* If excess == 0, no tree ops */
+                __mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess);
+                spin_unlock(&mctz->lock);
+                css_put(&mz->mem->css);
+                loop++;
+                /*
+                 * Could not reclaim anything and there are no more
+                 * mem cgroups to try or we seem to be looping without
+                 * reclaiming anything.
+                 */
+                if (!nr_reclaimed &&
+                        (next_mz == NULL ||
+                        loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
+                        break;
+        } while (!nr_reclaimed);
+        if (next_mz)
+                css_put(&next_mz->mem->css);
+        return nr_reclaimed;
+}
+
 /*
  * This routine traverse page_cgroup in given list and drop them all.
  * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
@@ -1730,7 +2430,7 @@ try_to_free:
                 if (!progress) {
                         nr_retries--;
                         /* maybe some writeback is necessary */
-                        congestion_wait(WRITE, HZ/10);
+                        congestion_wait(BLK_RW_ASYNC, HZ/10);
                 }
 
         }
@@ -1786,20 +2486,63 @@ static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
         return retval;
 }
 
+struct mem_cgroup_idx_data {
+        s64 val;
+        enum mem_cgroup_stat_index idx;
+};
+
+static int
+mem_cgroup_get_idx_stat(struct mem_cgroup *mem, void *data)
+{
+        struct mem_cgroup_idx_data *d = data;
+        d->val += mem_cgroup_read_stat(&mem->stat, d->idx);
+        return 0;
+}
+
+static void
+mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem,
+                                enum mem_cgroup_stat_index idx, s64 *val)
+{
+        struct mem_cgroup_idx_data d;
+        d.idx = idx;
+        d.val = 0;
+        mem_cgroup_walk_tree(mem, &d, mem_cgroup_get_idx_stat);
+        *val = d.val;
+}
+
 static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
 {
         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
-        u64 val = 0;
+        u64 idx_val, val;
         int type, name;
 
         type = MEMFILE_TYPE(cft->private);
         name = MEMFILE_ATTR(cft->private);
         switch (type) {
         case _MEM:
-                val = res_counter_read_u64(&mem->res, name);
+                if (name == RES_USAGE && mem_cgroup_is_root(mem)) {
+                        mem_cgroup_get_recursive_idx_stat(mem,
+                                MEM_CGROUP_STAT_CACHE, &idx_val);
+                        val = idx_val;
+                        mem_cgroup_get_recursive_idx_stat(mem,
+                                MEM_CGROUP_STAT_RSS, &idx_val);
+                        val += idx_val;
+                        val <<= PAGE_SHIFT;
+                } else
+                        val = res_counter_read_u64(&mem->res, name);
                 break;
         case _MEMSWAP:
-                if (do_swap_account)
+                if (name == RES_USAGE && mem_cgroup_is_root(mem)) {
+                        mem_cgroup_get_recursive_idx_stat(mem,
+                                MEM_CGROUP_STAT_CACHE, &idx_val);
+                        val = idx_val;
+                        mem_cgroup_get_recursive_idx_stat(mem,
+                                MEM_CGROUP_STAT_RSS, &idx_val);
+                        val += idx_val;
+                        mem_cgroup_get_recursive_idx_stat(mem,
+                                MEM_CGROUP_STAT_SWAPOUT, &idx_val);
+                        val <<= PAGE_SHIFT;
+                } else
                         val = res_counter_read_u64(&mem->memsw, name);
                 break;
         default:
@@ -1824,6 +2567,10 @@ static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
         name = MEMFILE_ATTR(cft->private);
         switch (name) {
         case RES_LIMIT:
+                if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
+                        ret = -EINVAL;
+                        break;
+                }
                 /* This function does all necessary parse...reuse it */
                 ret = res_counter_memparse_write_strategy(buffer, &val);
                 if (ret)
@@ -1833,6 +2580,20 @@ static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
                 else
                         ret = mem_cgroup_resize_memsw_limit(memcg, val);
                 break;
+        case RES_SOFT_LIMIT:
+                ret = res_counter_memparse_write_strategy(buffer, &val);
+                if (ret)
+                        break;
+                /*
+                 * For memsw, soft limits are hard to implement in terms
+                 * of semantics, for now, we support soft limits for
+                 * control without swap
+                 */
+                if (type == _MEM)
+                        ret = res_counter_set_soft_limit(&memcg->res, val);
+                else
+                        ret = -EINVAL;
+                break;
         default:
                 ret = -EINVAL; /* should be BUG() ? */
                 break;
@@ -1890,57 +2651,107 @@ static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
                         res_counter_reset_failcnt(&mem->memsw);
                 break;
         }
+
         return 0;
 }
 
-static const struct mem_cgroup_stat_desc {
-        const char *msg;
-        u64 unit;
-} mem_cgroup_stat_desc[] = {
-        [MEM_CGROUP_STAT_CACHE] = { "cache", PAGE_SIZE, },
-        [MEM_CGROUP_STAT_RSS] = { "rss", PAGE_SIZE, },
-        [MEM_CGROUP_STAT_PGPGIN_COUNT] = {"pgpgin", 1, },
-        [MEM_CGROUP_STAT_PGPGOUT_COUNT] = {"pgpgout", 1, },
+
+/* For read statistics */
+enum {
+        MCS_CACHE,
+        MCS_RSS,
+        MCS_MAPPED_FILE,
+        MCS_PGPGIN,
+        MCS_PGPGOUT,
+        MCS_SWAP,
+        MCS_INACTIVE_ANON,
+        MCS_ACTIVE_ANON,
+        MCS_INACTIVE_FILE,
+        MCS_ACTIVE_FILE,
+        MCS_UNEVICTABLE,
+        NR_MCS_STAT,
+};
+
+struct mcs_total_stat {
+        s64 stat[NR_MCS_STAT];
+};
+
+struct {
+        char *local_name;
+        char *total_name;
+} memcg_stat_strings[NR_MCS_STAT] = {
+        {"cache", "total_cache"},
+        {"rss", "total_rss"},
+        {"mapped_file", "total_mapped_file"},
+        {"pgpgin", "total_pgpgin"},
+        {"pgpgout", "total_pgpgout"},
+        {"swap", "total_swap"},
+        {"inactive_anon", "total_inactive_anon"},
+        {"active_anon", "total_active_anon"},
+        {"inactive_file", "total_inactive_file"},
+        {"active_file", "total_active_file"},
+        {"unevictable", "total_unevictable"}
 };
 
+
+static int mem_cgroup_get_local_stat(struct mem_cgroup *mem, void *data)
+{
+        struct mcs_total_stat *s = data;
+        s64 val;
+
+        /* per cpu stat */
+        val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_CACHE);
+        s->stat[MCS_CACHE] += val * PAGE_SIZE;
+        val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_RSS);
+        s->stat[MCS_RSS] += val * PAGE_SIZE;
+        val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_MAPPED_FILE);
+        s->stat[MCS_MAPPED_FILE] += val * PAGE_SIZE;
+        val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_PGPGIN_COUNT);
+        s->stat[MCS_PGPGIN] += val;
+        val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_PGPGOUT_COUNT);
+        s->stat[MCS_PGPGOUT] += val;
+        if (do_swap_account) {
+                val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_SWAPOUT);
+                s->stat[MCS_SWAP] += val * PAGE_SIZE;
+        }
+
+        /* per zone stat */
+        val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON);
+        s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
+        val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON);
+        s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
+        val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE);
+        s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
+        val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE);
+        s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
+        val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
+        s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
+        return 0;
+}
+
+static void
+mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
+{
+        mem_cgroup_walk_tree(mem, s, mem_cgroup_get_local_stat);
+}
+
 static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
                                  struct cgroup_map_cb *cb)
 {
         struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
-        struct mem_cgroup_stat *stat = &mem_cont->stat;
+        struct mcs_total_stat mystat;
         int i;
 
-        for (i = 0; i < ARRAY_SIZE(stat->cpustat[0].count); i++) {
-                s64 val;
+        memset(&mystat, 0, sizeof(mystat));
+        mem_cgroup_get_local_stat(mem_cont, &mystat);
 
-                val = mem_cgroup_read_stat(stat, i);
-                val *= mem_cgroup_stat_desc[i].unit;
-                cb->fill(cb, mem_cgroup_stat_desc[i].msg, val);
+        for (i = 0; i < NR_MCS_STAT; i++) {
+                if (i == MCS_SWAP && !do_swap_account)
+                        continue;
+                cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]);
         }
-        /* showing # of active pages */
-        {
-                unsigned long active_anon, inactive_anon;
-                unsigned long active_file, inactive_file;
-                unsigned long unevictable;
-
-                inactive_anon = mem_cgroup_get_all_zonestat(mem_cont,
-                                                LRU_INACTIVE_ANON);
-                active_anon = mem_cgroup_get_all_zonestat(mem_cont,
-                                                LRU_ACTIVE_ANON);
-                inactive_file = mem_cgroup_get_all_zonestat(mem_cont,
-                                                LRU_INACTIVE_FILE);
-                active_file = mem_cgroup_get_all_zonestat(mem_cont,
-                                                LRU_ACTIVE_FILE);
-                unevictable = mem_cgroup_get_all_zonestat(mem_cont,
-                                                        LRU_UNEVICTABLE);
-
-                cb->fill(cb, "active_anon", (active_anon) * PAGE_SIZE);
-                cb->fill(cb, "inactive_anon", (inactive_anon) * PAGE_SIZE);
-                cb->fill(cb, "active_file", (active_file) * PAGE_SIZE);
-                cb->fill(cb, "inactive_file", (inactive_file) * PAGE_SIZE);
-                cb->fill(cb, "unevictable", unevictable * PAGE_SIZE);
 
-        }
+        /* Hierarchical information */
         {
                 unsigned long long limit, memsw_limit;
                 memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit);
@@ -1949,6 +2760,14 @@ static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
                         cb->fill(cb, "hierarchical_memsw_limit", memsw_limit);
         }
 
+        memset(&mystat, 0, sizeof(mystat));
+        mem_cgroup_get_total_stat(mem_cont, &mystat);
+        for (i = 0; i < NR_MCS_STAT; i++) {
+                if (i == MCS_SWAP && !do_swap_account)
+                        continue;
+                cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]);
+        }
+
 #ifdef CONFIG_DEBUG_VM
         cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL));
 
@@ -2040,6 +2859,12 @@ static struct cftype mem_cgroup_files[] = {
                 .read_u64 = mem_cgroup_read,
         },
         {
+                .name = "soft_limit_in_bytes",
+                .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
+                .write_string = mem_cgroup_write,
+                .read_u64 = mem_cgroup_read,
+        },
+        {
                 .name = "failcnt",
                 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
                 .trigger = mem_cgroup_reset,
@@ -2133,6 +2958,9 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
                 mz = &pn->zoneinfo[zone];
                 for_each_lru(l)
                         INIT_LIST_HEAD(&mz->lists[l]);
+                mz->usage_in_excess = 0;
+                mz->on_tree = false;
+                mz->mem = mem;
         }
         return 0;
 }
@@ -2178,6 +3006,9 @@ static void __mem_cgroup_free(struct mem_cgroup *mem)
 {
         int node;
 
+        mem_cgroup_remove_from_trees(mem);
+        free_css_id(&mem_cgroup_subsys, &mem->css);
+
         for_each_node_state(node, N_POSSIBLE)
                 free_mem_cgroup_per_zone_info(mem, node);
 
@@ -2224,23 +3055,54 @@ static void __init enable_swap_cgroup(void)
 }
 #endif
 
+static int mem_cgroup_soft_limit_tree_init(void)
+{
+        struct mem_cgroup_tree_per_node *rtpn;
+        struct mem_cgroup_tree_per_zone *rtpz;
+        int tmp, node, zone;
+
+        for_each_node_state(node, N_POSSIBLE) {
+                tmp = node;
+                if (!node_state(node, N_NORMAL_MEMORY))
+                        tmp = -1;
+                rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
+                if (!rtpn)
+                        return 1;
+
+                soft_limit_tree.rb_tree_per_node[node] = rtpn;
+
+                for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+                        rtpz = &rtpn->rb_tree_per_zone[zone];
+                        rtpz->rb_root = RB_ROOT;
+                        spin_lock_init(&rtpz->lock);
+                }
+        }
+        return 0;
+}
+
 static struct cgroup_subsys_state * __ref
 mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
 {
         struct mem_cgroup *mem, *parent;
+        long error = -ENOMEM;
         int node;
 
         mem = mem_cgroup_alloc();
         if (!mem)
-                return ERR_PTR(-ENOMEM);
+                return ERR_PTR(error);
 
         for_each_node_state(node, N_POSSIBLE)
                 if (alloc_mem_cgroup_per_zone_info(mem, node))
                         goto free_out;
+
         /* root ? */
         if (cont->parent == NULL) {
                 enable_swap_cgroup();
                 parent = NULL;
+                root_mem_cgroup = mem;
+                if (mem_cgroup_soft_limit_tree_init())
+                        goto free_out;
+
         } else {
                 parent = mem_cgroup_from_cont(cont->parent);
                 mem->use_hierarchy = parent->use_hierarchy;
@@ -2260,7 +3122,7 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
                 res_counter_init(&mem->res, NULL);
                 res_counter_init(&mem->memsw, NULL);
         }
-        mem->last_scanned_child = NULL;
+        mem->last_scanned_child = 0;
         spin_lock_init(&mem->reclaim_param_lock);
 
         if (parent)
@@ -2269,26 +3131,23 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
         return &mem->css;
 free_out:
         __mem_cgroup_free(mem);
-        return ERR_PTR(-ENOMEM);
+        root_mem_cgroup = NULL;
+        return ERR_PTR(error);
 }
 
-static void mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
+static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
                                         struct cgroup *cont)
 {
         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
-        mem_cgroup_force_empty(mem, false);
+
+        return mem_cgroup_force_empty(mem, false);
 }
 
 static void mem_cgroup_destroy(struct cgroup_subsys *ss,
                                 struct cgroup *cont)
 {
         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
-        struct mem_cgroup *last_scanned_child = mem->last_scanned_child;
 
-        if (last_scanned_child) {
-                VM_BUG_ON(!mem_cgroup_is_obsolete(last_scanned_child));
-                mem_cgroup_put(last_scanned_child);
-        }
         mem_cgroup_put(mem);
 }
 
@@ -2308,7 +3167,8 @@ static int mem_cgroup_populate(struct cgroup_subsys *ss,
 static void mem_cgroup_move_task(struct cgroup_subsys *ss,
                                 struct cgroup *cont,
                                 struct cgroup *old_cont,
-                                struct task_struct *p)
+                                struct task_struct *p,
+                                bool threadgroup)
 {
         mutex_lock(&memcg_tasklist);
         /*
@@ -2327,6 +3187,7 @@ struct cgroup_subsys mem_cgroup_subsys = {
         .populate = mem_cgroup_populate,
         .attach = mem_cgroup_move_task,
         .early_init = 0,
+        .use_id = 1,
 };
 
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
new file mode 100644
index 000000000000..dacc64183874
--- /dev/null
+++ b/mm/memory-failure.c
@@ -0,0 +1,835 @@
+/*
+ * Copyright (C) 2008, 2009 Intel Corporation
+ * Authors: Andi Kleen, Fengguang Wu
+ *
+ * This software may be redistributed and/or modified under the terms of
+ * the GNU General Public License ("GPL") version 2 only as published by the
+ * Free Software Foundation.
+ *
+ * High level machine check handler. Handles pages reported by the
+ * hardware as being corrupted usually due to a 2bit ECC memory or cache
+ * failure.
+ *
+ * Handles page cache pages in various states.  The tricky part
+ * here is that we can access any page asynchronous to other VM
+ * users, because memory failures could happen anytime and anywhere,
+ * possibly violating some of their assumptions. This is why this code
+ * has to be extremely careful. Generally it tries to use normal locking
+ * rules, as in get the standard locks, even if that means the
+ * error handling takes potentially a long time.
+ *
+ * The operation to map back from RMAP chains to processes has to walk
+ * the complete process list and has non linear complexity with the number
+ * mappings. In short it can be quite slow. But since memory corruptions
+ * are rare we hope to get away with this.
+ */
+
+/*
+ * Notebook:
+ * - hugetlb needs more code
+ * - kcore/oldmem/vmcore/mem/kmem check for hwpoison pages
+ * - pass bad pages to kdump next kernel
+ */
+#define DEBUG 1         /* remove me in 2.6.34 */
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/page-flags.h>
+#include <linux/sched.h>
+#include <linux/ksm.h>
+#include <linux/rmap.h>
+#include <linux/pagemap.h>
+#include <linux/swap.h>
+#include <linux/backing-dev.h>
+#include "internal.h"
+
+int sysctl_memory_failure_early_kill __read_mostly = 0;
+
+int sysctl_memory_failure_recovery __read_mostly = 1;
+
+atomic_long_t mce_bad_pages __read_mostly = ATOMIC_LONG_INIT(0);
+
+/*
+ * Send all the processes who have the page mapped an ``action optional''
+ * signal.
+ */
+static int kill_proc_ao(struct task_struct *t, unsigned long addr, int trapno,
+                        unsigned long pfn)
+{
+        struct siginfo si;
+        int ret;
+
+        printk(KERN_ERR
+                "MCE %#lx: Killing %s:%d early due to hardware memory corruption\n",
+                pfn, t->comm, t->pid);
+        si.si_signo = SIGBUS;
+        si.si_errno = 0;
+        si.si_code = BUS_MCEERR_AO;
+        si.si_addr = (void *)addr;
+#ifdef __ARCH_SI_TRAPNO
+        si.si_trapno = trapno;
+#endif
+        si.si_addr_lsb = PAGE_SHIFT;
+        /*
+         * Don't use force here, it's convenient if the signal
+         * can be temporarily blocked.
+         * This could cause a loop when the user sets SIGBUS
+         * to SIG_IGN, but hopefully noone will do that?
+         */
+        ret = send_sig_info(SIGBUS, &si, t);  /* synchronous? */
+        if (ret < 0)
+                printk(KERN_INFO "MCE: Error sending signal to %s:%d: %d\n",
+                       t->comm, t->pid, ret);
+        return ret;
+}
+
+/*
+ * Kill all processes that have a poisoned page mapped and then isolate
+ * the page.
+ *
+ * General strategy:
+ * Find all processes having the page mapped and kill them.
+ * But we keep a page reference around so that the page is not
+ * actually freed yet.
+ * Then stash the page away
+ *
+ * There's no convenient way to get back to mapped processes
+ * from the VMAs. So do a brute-force search over all
+ * running processes.
+ *
+ * Remember that machine checks are not common (or rather
+ * if they are common you have other problems), so this shouldn't
+ * be a performance issue.
+ *
+ * Also there are some races possible while we get from the
+ * error detection to actually handle it.
+ */
+
+struct to_kill {
+        struct list_head nd;
+        struct task_struct *tsk;
+        unsigned long addr;
+        unsigned addr_valid:1;
+};
+
+/*
+ * Failure handling: if we can't find or can't kill a process there's
+ * not much we can do.  We just print a message and ignore otherwise.
+ */
+
+/*
+ * Schedule a process for later kill.
+ * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM.
+ * TBD would GFP_NOIO be enough?
+ */
+static void add_to_kill(struct task_struct *tsk, struct page *p,
+                       struct vm_area_struct *vma,
+                       struct list_head *to_kill,
+                       struct to_kill **tkc)
+{
+        struct to_kill *tk;
+
+        if (*tkc) {
+                tk = *tkc;
+                *tkc = NULL;
+        } else {
+                tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
+                if (!tk) {
+                        printk(KERN_ERR
+                "MCE: Out of memory while machine check handling\n");
+                        return;
+                }
+        }
+        tk->addr = page_address_in_vma(p, vma);
+        tk->addr_valid = 1;
+
+        /*
+         * In theory we don't have to kill when the page was
+         * munmaped. But it could be also a mremap. Since that's
+         * likely very rare kill anyways just out of paranoia, but use
+         * a SIGKILL because the error is not contained anymore.
+         */
+        if (tk->addr == -EFAULT) {
+                pr_debug("MCE: Unable to find user space address %lx in %s\n",
+                        page_to_pfn(p), tsk->comm);
+                tk->addr_valid = 0;
+        }
+        get_task_struct(tsk);
+        tk->tsk = tsk;
+        list_add_tail(&tk->nd, to_kill);
+}
+
+/*
+ * Kill the processes that have been collected earlier.
+ *
+ * Only do anything when DOIT is set, otherwise just free the list
+ * (this is used for clean pages which do not need killing)
+ * Also when FAIL is set do a force kill because something went
+ * wrong earlier.
+ */
+static void kill_procs_ao(struct list_head *to_kill, int doit, int trapno,
+                          int fail, unsigned long pfn)
+{
+        struct to_kill *tk, *next;
+
+        list_for_each_entry_safe (tk, next, to_kill, nd) {
+                if (doit) {
+                        /*
+                         * In case something went wrong with munmaping
+                         * make sure the process doesn't catch the
+                         * signal and then access the memory. Just kill it.
+                         * the signal handlers
+                         */
+                        if (fail || tk->addr_valid == 0) {
+                                printk(KERN_ERR
+                "MCE %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n",
+                                        pfn, tk->tsk->comm, tk->tsk->pid);
+                                force_sig(SIGKILL, tk->tsk);
+                        }
+
+                        /*
+                         * In theory the process could have mapped
+                         * something else on the address in-between. We could
+                         * check for that, but we need to tell the
+                         * process anyways.
+                         */
+                        else if (kill_proc_ao(tk->tsk, tk->addr, trapno,
+                                              pfn) < 0)
+                                printk(KERN_ERR
+                "MCE %#lx: Cannot send advisory machine check signal to %s:%d\n",
+                                        pfn, tk->tsk->comm, tk->tsk->pid);
+                }
+                put_task_struct(tk->tsk);
+                kfree(tk);
+        }
+}
+
+static int task_early_kill(struct task_struct *tsk)
+{
+        if (!tsk->mm)
+                return 0;
+        if (tsk->flags & PF_MCE_PROCESS)
+                return !!(tsk->flags & PF_MCE_EARLY);
+        return sysctl_memory_failure_early_kill;
+}
+
+/*
+ * Collect processes when the error hit an anonymous page.
+ */
+static void collect_procs_anon(struct page *page, struct list_head *to_kill,
+                              struct to_kill **tkc)
+{
+        struct vm_area_struct *vma;
+        struct task_struct *tsk;
+        struct anon_vma *av;
+
+        read_lock(&tasklist_lock);
+        av = page_lock_anon_vma(page);
+        if (av == NULL) /* Not actually mapped anymore */
+                goto out;
+        for_each_process (tsk) {
+                if (!task_early_kill(tsk))
+                        continue;
+                list_for_each_entry (vma, &av->head, anon_vma_node) {
+                        if (!page_mapped_in_vma(page, vma))
+                                continue;
+                        if (vma->vm_mm == tsk->mm)
+                                add_to_kill(tsk, page, vma, to_kill, tkc);
+                }
+        }
+        page_unlock_anon_vma(av);
+out:
+        read_unlock(&tasklist_lock);
+}
+
+/*
+ * Collect processes when the error hit a file mapped page.
+ */
+static void collect_procs_file(struct page *page, struct list_head *to_kill,
+                              struct to_kill **tkc)
+{
+        struct vm_area_struct *vma;
+        struct task_struct *tsk;
+        struct prio_tree_iter iter;
+        struct address_space *mapping = page->mapping;
+
+        /*
+         * A note on the locking order between the two locks.
+         * We don't rely on this particular order.
+         * If you have some other code that needs a different order
+         * feel free to switch them around. Or add a reverse link
+         * from mm_struct to task_struct, then this could be all
+         * done without taking tasklist_lock and looping over all tasks.
+         */
+
+        read_lock(&tasklist_lock);
+        spin_lock(&mapping->i_mmap_lock);
+        for_each_process(tsk) {
+                pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
+
+                if (!task_early_kill(tsk))
+                        continue;
+
+                vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff,
+                                      pgoff) {
+                        /*
+                         * Send early kill signal to tasks where a vma covers
+                         * the page but the corrupted page is not necessarily
+                         * mapped it in its pte.
+                         * Assume applications who requested early kill want
+                         * to be informed of all such data corruptions.
+                         */
+                        if (vma->vm_mm == tsk->mm)
+                                add_to_kill(tsk, page, vma, to_kill, tkc);
+                }
+        }
+        spin_unlock(&mapping->i_mmap_lock);
+        read_unlock(&tasklist_lock);
+}
+
+/*
+ * Collect the processes who have the corrupted page mapped to kill.
+ * This is done in two steps for locking reasons.
+ * First preallocate one tokill structure outside the spin locks,
+ * so that we can kill at least one process reasonably reliable.
+ */
+static void collect_procs(struct page *page, struct list_head *tokill)
+{
+        struct to_kill *tk;
+
+        if (!page->mapping)
+                return;
+
+        tk = kmalloc(sizeof(struct to_kill), GFP_NOIO);
+        if (!tk)
+                return;
+        if (PageAnon(page))
+                collect_procs_anon(page, tokill, &tk);
+        else
+                collect_procs_file(page, tokill, &tk);
+        kfree(tk);
+}
+
+/*
+ * Error handlers for various types of pages.
+ */
+
+enum outcome {
+        FAILED,         /* Error handling failed */
+        DELAYED,        /* Will be handled later */
+        IGNORED,        /* Error safely ignored */
+        RECOVERED,      /* Successfully recovered */
+};
+
+static const char *action_name[] = {
+        [FAILED] = "Failed",
+        [DELAYED] = "Delayed",
+        [IGNORED] = "Ignored",
+        [RECOVERED] = "Recovered",
+};
+
+/*
+ * Error hit kernel page.
+ * Do nothing, try to be lucky and not touch this instead. For a few cases we
+ * could be more sophisticated.
+ */
+static int me_kernel(struct page *p, unsigned long pfn)
+{
+        return DELAYED;
+}
+
+/*
+ * Already poisoned page.
+ */
+static int me_ignore(struct page *p, unsigned long pfn)
+{
+        return IGNORED;
+}
+
+/*
+ * Page in unknown state. Do nothing.
+ */
+static int me_unknown(struct page *p, unsigned long pfn)
+{
+        printk(KERN_ERR "MCE %#lx: Unknown page state\n", pfn);
+        return FAILED;
+}
+
+/*
+ * Free memory
+ */
+static int me_free(struct page *p, unsigned long pfn)
+{
+        return DELAYED;
+}
+
+/*
+ * Clean (or cleaned) page cache page.
+ */
+static int me_pagecache_clean(struct page *p, unsigned long pfn)
+{
+        int err;
+        int ret = FAILED;
+        struct address_space *mapping;
+
+        /*
+         * For anonymous pages we're done the only reference left
+         * should be the one m_f() holds.
+         */
+        if (PageAnon(p))
+                return RECOVERED;
+
+        /*
+         * Now truncate the page in the page cache. This is really
+         * more like a "temporary hole punch"
+         * Don't do this for block devices when someone else
+         * has a reference, because it could be file system metadata
+         * and that's not safe to truncate.
+         */
+        mapping = page_mapping(p);
+        if (!mapping) {
+                /*
+                 * Page has been teared down in the meanwhile
+                 */
+                return FAILED;
+        }
+
+        /*
+         * Truncation is a bit tricky. Enable it per file system for now.
+         *
+         * Open: to take i_mutex or not for this? Right now we don't.
+         */
+        if (mapping->a_ops->error_remove_page) {
+                err = mapping->a_ops->error_remove_page(mapping, p);
+                if (err != 0) {
+                        printk(KERN_INFO "MCE %#lx: Failed to punch page: %d\n",
+                                        pfn, err);
+                } else if (page_has_private(p) &&
+                                !try_to_release_page(p, GFP_NOIO)) {
+                        pr_debug("MCE %#lx: failed to release buffers\n", pfn);
+                } else {
+                        ret = RECOVERED;
+                }
+        } else {
+                /*
+                 * If the file system doesn't support it just invalidate
+                 * This fails on dirty or anything with private pages
+                 */
+                if (invalidate_inode_page(p))
+                        ret = RECOVERED;
+                else
+                        printk(KERN_INFO "MCE %#lx: Failed to invalidate\n",
+                                pfn);
+        }
+        return ret;
+}
+
+/*
+ * Dirty cache page page
+ * Issues: when the error hit a hole page the error is not properly
+ * propagated.
+ */
+static int me_pagecache_dirty(struct page *p, unsigned long pfn)
+{
+        struct address_space *mapping = page_mapping(p);
+
+        SetPageError(p);
+        /* TBD: print more information about the file. */
+        if (mapping) {
+                /*
+                 * IO error will be reported by write(), fsync(), etc.
+                 * who check the mapping.
+                 * This way the application knows that something went
+                 * wrong with its dirty file data.
+                 *
+                 * There's one open issue:
+                 *
+                 * The EIO will be only reported on the next IO
+                 * operation and then cleared through the IO map.
+                 * Normally Linux has two mechanisms to pass IO error
+                 * first through the AS_EIO flag in the address space
+                 * and then through the PageError flag in the page.
+                 * Since we drop pages on memory failure handling the
+                 * only mechanism open to use is through AS_AIO.
+                 *
+                 * This has the disadvantage that it gets cleared on
+                 * the first operation that returns an error, while
+                 * the PageError bit is more sticky and only cleared
+                 * when the page is reread or dropped.  If an
+                 * application assumes it will always get error on
+                 * fsync, but does other operations on the fd before
+                 * and the page is dropped inbetween then the error
+                 * will not be properly reported.
+                 *
+                 * This can already happen even without hwpoisoned
+                 * pages: first on metadata IO errors (which only
+                 * report through AS_EIO) or when the page is dropped
+                 * at the wrong time.
+                 *
+                 * So right now we assume that the application DTRT on
+                 * the first EIO, but we're not worse than other parts
+                 * of the kernel.
+                 */
+                mapping_set_error(mapping, EIO);
+        }
+
+        return me_pagecache_clean(p, pfn);
+}
+
+/*
+ * Clean and dirty swap cache.
+ *
+ * Dirty swap cache page is tricky to handle. The page could live both in page
+ * cache and swap cache(ie. page is freshly swapped in). So it could be
+ * referenced concurrently by 2 types of PTEs:
+ * normal PTEs and swap PTEs. We try to handle them consistently by calling
+ * try_to_unmap(TTU_IGNORE_HWPOISON) to convert the normal PTEs to swap PTEs,
+ * and then
+ *      - clear dirty bit to prevent IO
+ *      - remove from LRU
+ *      - but keep in the swap cache, so that when we return to it on
+ *        a later page fault, we know the application is accessing
+ *        corrupted data and shall be killed (we installed simple
+ *        interception code in do_swap_page to catch it).
+ *
+ * Clean swap cache pages can be directly isolated. A later page fault will
+ * bring in the known good data from disk.
+ */
+static int me_swapcache_dirty(struct page *p, unsigned long pfn)
+{
+        ClearPageDirty(p);
+        /* Trigger EIO in shmem: */
+        ClearPageUptodate(p);
+
+        return DELAYED;
+}
+
+static int me_swapcache_clean(struct page *p, unsigned long pfn)
+{
+        delete_from_swap_cache(p);
+
+        return RECOVERED;
+}
+
+/*
+ * Huge pages. Needs work.
+ * Issues:
+ * No rmap support so we cannot find the original mapper. In theory could walk
+ * all MMs and look for the mappings, but that would be non atomic and racy.
+ * Need rmap for hugepages for this. Alternatively we could employ a heuristic,
+ * like just walking the current process and hoping it has it mapped (that
+ * should be usually true for the common "shared database cache" case)
+ * Should handle free huge pages and dequeue them too, but this needs to
+ * handle huge page accounting correctly.
+ */
+static int me_huge_page(struct page *p, unsigned long pfn)
+{
+        return FAILED;
+}
+
+/*
+ * Various page states we can handle.
+ *
+ * A page state is defined by its current page->flags bits.
+ * The table matches them in order and calls the right handler.
+ *
+ * This is quite tricky because we can access page at any time
+ * in its live cycle, so all accesses have to be extremly careful.
+ *
+ * This is not complete. More states could be added.
+ * For any missing state don't attempt recovery.
+ */
+
+#define dirty           (1UL << PG_dirty)
+#define sc              (1UL << PG_swapcache)
+#define unevict         (1UL << PG_unevictable)
+#define mlock           (1UL << PG_mlocked)
+#define writeback       (1UL << PG_writeback)
+#define lru             (1UL << PG_lru)
+#define swapbacked      (1UL << PG_swapbacked)
+#define head            (1UL << PG_head)
+#define tail            (1UL << PG_tail)
+#define compound        (1UL << PG_compound)
+#define slab            (1UL << PG_slab)
+#define buddy           (1UL << PG_buddy)
+#define reserved        (1UL << PG_reserved)
+
+static struct page_state {
+        unsigned long mask;
+        unsigned long res;
+        char *msg;
+        int (*action)(struct page *p, unsigned long pfn);
+} error_states[] = {
+        { reserved,     reserved,       "reserved kernel",      me_ignore },
+        { buddy,        buddy,          "free kernel",  me_free },
+
+        /*
+         * Could in theory check if slab page is free or if we can drop
+         * currently unused objects without touching them. But just
+         * treat it as standard kernel for now.
+         */
+        { slab,         slab,           "kernel slab",  me_kernel },
+
+#ifdef CONFIG_PAGEFLAGS_EXTENDED
+        { head,         head,           "huge",         me_huge_page },
+        { tail,         tail,           "huge",         me_huge_page },
+#else
+        { compound,     compound,       "huge",         me_huge_page },
+#endif
+
+        { sc|dirty,     sc|dirty,       "swapcache",    me_swapcache_dirty },
+        { sc|dirty,     sc,             "swapcache",    me_swapcache_clean },
+
+        { unevict|dirty, unevict|dirty, "unevictable LRU", me_pagecache_dirty},
+        { unevict,      unevict,        "unevictable LRU", me_pagecache_clean},
+
+#ifdef CONFIG_HAVE_MLOCKED_PAGE_BIT
+        { mlock|dirty,  mlock|dirty,    "mlocked LRU",  me_pagecache_dirty },
+        { mlock,        mlock,          "mlocked LRU",  me_pagecache_clean },
+#endif
+
+        { lru|dirty,    lru|dirty,      "LRU",          me_pagecache_dirty },
+        { lru|dirty,    lru,            "clean LRU",    me_pagecache_clean },
+        { swapbacked,   swapbacked,     "anonymous",    me_pagecache_clean },
+
+        /*
+         * Catchall entry: must be at end.
+         */
+        { 0,            0,              "unknown page state",   me_unknown },
+};
+
+static void action_result(unsigned long pfn, char *msg, int result)
+{
+        struct page *page = NULL;
+        if (pfn_valid(pfn))
+                page = pfn_to_page(pfn);
+
+        printk(KERN_ERR "MCE %#lx: %s%s page recovery: %s\n",
+                pfn,
+                page && PageDirty(page) ? "dirty " : "",
+                msg, action_name[result]);
+}
+
+static int page_action(struct page_state *ps, struct page *p,
+                        unsigned long pfn, int ref)
+{
+        int result;
+        int count;
+
+        result = ps->action(p, pfn);
+        action_result(pfn, ps->msg, result);
+
+        count = page_count(p) - 1 - ref;
+        if (count != 0)
+                printk(KERN_ERR
+                       "MCE %#lx: %s page still referenced by %d users\n",
+                       pfn, ps->msg, count);
+
+        /* Could do more checks here if page looks ok */
+        /*
+         * Could adjust zone counters here to correct for the missing page.
+         */
+
+        return result == RECOVERED ? 0 : -EBUSY;
+}
+
+#define N_UNMAP_TRIES 5
+
+/*
+ * Do all that is necessary to remove user space mappings. Unmap
+ * the pages and send SIGBUS to the processes if the data was dirty.
+ */
+static void hwpoison_user_mappings(struct page *p, unsigned long pfn,
+                                  int trapno)
+{
+        enum ttu_flags ttu = TTU_UNMAP | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS;
+        struct address_space *mapping;
+        LIST_HEAD(tokill);
+        int ret;
+        int i;
+        int kill = 1;
+
+        if (PageReserved(p) || PageCompound(p) || PageSlab(p) || PageKsm(p))
+                return;
+
+        /*
+         * This check implies we don't kill processes if their pages
+         * are in the swap cache early. Those are always late kills.
+         */
+        if (!page_mapped(p))
+                return;
+
+        if (PageSwapCache(p)) {
+                printk(KERN_ERR
+                       "MCE %#lx: keeping poisoned page in swap cache\n", pfn);
+                ttu |= TTU_IGNORE_HWPOISON;
+        }
+
+        /*
+         * Propagate the dirty bit from PTEs to struct page first, because we
+         * need this to decide if we should kill or just drop the page.
+         */
+        mapping = page_mapping(p);
+        if (!PageDirty(p) && mapping && mapping_cap_writeback_dirty(mapping)) {
+                if (page_mkclean(p)) {
+                        SetPageDirty(p);
+                } else {
+                        kill = 0;
+                        ttu |= TTU_IGNORE_HWPOISON;
+                        printk(KERN_INFO
+        "MCE %#lx: corrupted page was clean: dropped without side effects\n",
+                                pfn);
+                }
+        }
+
+        /*
+         * First collect all the processes that have the page
+         * mapped in dirty form.  This has to be done before try_to_unmap,
+         * because ttu takes the rmap data structures down.
+         *
+         * Error handling: We ignore errors here because
+         * there's nothing that can be done.
+         */
+        if (kill)
+                collect_procs(p, &tokill);
+
+        /*
+         * try_to_unmap can fail temporarily due to races.
+         * Try a few times (RED-PEN better strategy?)
+         */
+        for (i = 0; i < N_UNMAP_TRIES; i++) {
+                ret = try_to_unmap(p, ttu);
+                if (ret == SWAP_SUCCESS)
+                        break;
+                pr_debug("MCE %#lx: try_to_unmap retry needed %d\n", pfn,  ret);
+        }
+
+        if (ret != SWAP_SUCCESS)
+                printk(KERN_ERR "MCE %#lx: failed to unmap page (mapcount=%d)\n",
+                                pfn, page_mapcount(p));
+
+        /*
+         * Now that the dirty bit has been propagated to the
+         * struct page and all unmaps done we can decide if
+         * killing is needed or not.  Only kill when the page
+         * was dirty, otherwise the tokill list is merely
+         * freed.  When there was a problem unmapping earlier
+         * use a more force-full uncatchable kill to prevent
+         * any accesses to the poisoned memory.
+         */
+        kill_procs_ao(&tokill, !!PageDirty(p), trapno,
+                      ret != SWAP_SUCCESS, pfn);
+}
+
+int __memory_failure(unsigned long pfn, int trapno, int ref)
+{
+        unsigned long lru_flag;
+        struct page_state *ps;
+        struct page *p;
+        int res;
+
+        if (!sysctl_memory_failure_recovery)
+                panic("Memory failure from trap %d on page %lx", trapno, pfn);
+
+        if (!pfn_valid(pfn)) {
+                action_result(pfn, "memory outside kernel control", IGNORED);
+                return -EIO;
+        }
+
+        p = pfn_to_page(pfn);
+        if (TestSetPageHWPoison(p)) {
+                action_result(pfn, "already hardware poisoned", IGNORED);
+                return 0;
+        }
+
+        atomic_long_add(1, &mce_bad_pages);
+
+        /*
+         * We need/can do nothing about count=0 pages.
+         * 1) it's a free page, and therefore in safe hand:
+         *    prep_new_page() will be the gate keeper.
+         * 2) it's part of a non-compound high order page.
+         *    Implies some kernel user: cannot stop them from
+         *    R/W the page; let's pray that the page has been
+         *    used and will be freed some time later.
+         * In fact it's dangerous to directly bump up page count from 0,
+         * that may make page_freeze_refs()/page_unfreeze_refs() mismatch.
+         */
+        if (!get_page_unless_zero(compound_head(p))) {
+                action_result(pfn, "free or high order kernel", IGNORED);
+                return PageBuddy(compound_head(p)) ? 0 : -EBUSY;
+        }
+
+        /*
+         * We ignore non-LRU pages for good reasons.
+         * - PG_locked is only well defined for LRU pages and a few others
+         * - to avoid races with __set_page_locked()
+         * - to avoid races with __SetPageSlab*() (and more non-atomic ops)
+         * The check (unnecessarily) ignores LRU pages being isolated and
+         * walked by the page reclaim code, however that's not a big loss.
+         */
+        if (!PageLRU(p))
+                lru_add_drain_all();
+        lru_flag = p->flags & lru;
+        if (isolate_lru_page(p)) {
+                action_result(pfn, "non LRU", IGNORED);
+                put_page(p);
+                return -EBUSY;
+        }
+        page_cache_release(p);
+
+        /*
+         * Lock the page and wait for writeback to finish.
+         * It's very difficult to mess with pages currently under IO
+         * and in many cases impossible, so we just avoid it here.
+         */
+        lock_page_nosync(p);
+        wait_on_page_writeback(p);
+
+        /*
+         * Now take care of user space mappings.
+         */
+        hwpoison_user_mappings(p, pfn, trapno);
+
+        /*
+         * Torn down by someone else?
+         */
+        if ((lru_flag & lru) && !PageSwapCache(p) && p->mapping == NULL) {
+                action_result(pfn, "already truncated LRU", IGNORED);
+                res = 0;
+                goto out;
+        }
+
+        res = -EBUSY;
+        for (ps = error_states;; ps++) {
+                if (((p->flags | lru_flag)& ps->mask) == ps->res) {
+                        res = page_action(ps, p, pfn, ref);
+                        break;
+                }
+        }
+out:
+        unlock_page(p);
+        return res;
+}
+EXPORT_SYMBOL_GPL(__memory_failure);
+
+/**
+ * memory_failure - Handle memory failure of a page.
+ * @pfn: Page Number of the corrupted page
+ * @trapno: Trap number reported in the signal to user space.
+ *
+ * This function is called by the low level machine check code
+ * of an architecture when it detects hardware memory corruption
+ * of a page. It tries its best to recover, which includes
+ * dropping pages, killing processes etc.
+ *
+ * The function is primarily of use for corruptions that
+ * happen outside the current execution context (e.g. when
+ * detected by a background scrubber)
+ *
+ * Must run in process context (e.g. a work queue) with interrupts
+ * enabled and no spinlocks hold.
+ */
+void memory_failure(unsigned long pfn, int trapno)
+{
+        __memory_failure(pfn, trapno, 0);
+}
diff --git a/mm/memory.c b/mm/memory.c
index cf6873e91c6a..6ab19dd4a199 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -45,6 +45,7 @@
 #include <linux/swap.h>
 #include <linux/highmem.h>
 #include <linux/pagemap.h>
+#include <linux/ksm.h>
 #include <linux/rmap.h>
 #include <linux/module.h>
 #include <linux/delayacct.h>
@@ -56,6 +57,7 @@
 #include <linux/swapops.h>
 #include <linux/elf.h>
 
+#include <asm/io.h>
 #include <asm/pgalloc.h>
 #include <asm/uaccess.h>
 #include <asm/tlb.h>
@@ -106,6 +108,18 @@ static int __init disable_randmaps(char *s)
 }
 __setup("norandmaps", disable_randmaps);
 
+unsigned long zero_pfn __read_mostly;
+unsigned long highest_memmap_pfn __read_mostly;
+
+/*
+ * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
+ */
+static int __init init_zero_pfn(void)
+{
+        zero_pfn = page_to_pfn(ZERO_PAGE(0));
+        return 0;
+}
+core_initcall(init_zero_pfn);
 
 /*
  * If a p?d_bad entry is found while walking page tables, report
@@ -135,11 +149,12 @@ void pmd_clear_bad(pmd_t *pmd)
  * Note: this doesn't free the actual pages themselves. That
  * has been handled earlier when unmapping all the memory regions.
  */
-static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd)
+static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
+                           unsigned long addr)
 {
         pgtable_t token = pmd_pgtable(*pmd);
         pmd_clear(pmd);
-        pte_free_tlb(tlb, token);
+        pte_free_tlb(tlb, token, addr);
         tlb->mm->nr_ptes--;
 }
 
@@ -157,7 +172,7 @@ static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
                 next = pmd_addr_end(addr, end);
                 if (pmd_none_or_clear_bad(pmd))
                         continue;
-                free_pte_range(tlb, pmd);
+                free_pte_range(tlb, pmd, addr);
         } while (pmd++, addr = next, addr != end);
 
         start &= PUD_MASK;
@@ -173,7 +188,7 @@ static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
 
         pmd = pmd_offset(pud, start);
         pud_clear(pud);
-        pmd_free_tlb(tlb, pmd);
+        pmd_free_tlb(tlb, pmd, start);
 }
 
 static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
@@ -206,7 +221,7 @@ static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
 
         pud = pud_offset(pgd, start);
         pgd_clear(pgd);
-        pud_free_tlb(tlb, pud);
+        pud_free_tlb(tlb, pud, start);
 }
 
 /*
@@ -282,7 +297,8 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
                 unsigned long addr = vma->vm_start;
 
                 /*
-                 * Hide vma from rmap and vmtruncate before freeing pgtables
+                 * Hide vma from rmap and truncate_pagecache before freeing
+                 * pgtables
                  */
                 anon_vma_unlink(vma);
                 unlink_file_vma(vma);
@@ -441,6 +457,20 @@ static inline int is_cow_mapping(unsigned int flags)
         return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
 }
 
+#ifndef is_zero_pfn
+static inline int is_zero_pfn(unsigned long pfn)
+{
+        return pfn == zero_pfn;
+}
+#endif
+
+#ifndef my_zero_pfn
+static inline unsigned long my_zero_pfn(unsigned long addr)
+{
+        return zero_pfn;
+}
+#endif
+
 /*
  * vm_normal_page -- This function gets the "struct page" associated with a pte.
  *
@@ -496,7 +526,9 @@ struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
         if (HAVE_PTE_SPECIAL) {
                 if (likely(!pte_special(pte)))
                         goto check_pfn;
-                if (!(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)))
+                if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
+                        return NULL;
+                if (!is_zero_pfn(pfn))
                         print_bad_pte(vma, addr, pte, NULL);
                 return NULL;
         }
@@ -518,6 +550,8 @@ struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
                 }
         }
 
+        if (is_zero_pfn(pfn))
+                return NULL;
 check_pfn:
         if (unlikely(pfn > highest_memmap_pfn)) {
                 print_bad_pte(vma, addr, pte, NULL);
@@ -595,8 +629,8 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
         page = vm_normal_page(vma, addr, pte);
         if (page) {
                 get_page(page);
-                page_dup_rmap(page, vma, addr);
-                rss[!!PageAnon(page)]++;
+                page_dup_rmap(page);
+                rss[PageAnon(page)]++;
         }
 
 out_set_pte:
@@ -607,6 +641,7 @@ static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
                 pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma,
                 unsigned long addr, unsigned long end)
 {
+        pte_t *orig_src_pte, *orig_dst_pte;
         pte_t *src_pte, *dst_pte;
         spinlock_t *src_ptl, *dst_ptl;
         int progress = 0;
@@ -620,6 +655,8 @@ again:
         src_pte = pte_offset_map_nested(src_pmd, addr);
         src_ptl = pte_lockptr(src_mm, src_pmd);
         spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
+        orig_src_pte = src_pte;
+        orig_dst_pte = dst_pte;
         arch_enter_lazy_mmu_mode();
 
         do {
@@ -643,9 +680,9 @@ again:
 
         arch_leave_lazy_mmu_mode();
         spin_unlock(src_ptl);
-        pte_unmap_nested(src_pte - 1);
+        pte_unmap_nested(orig_src_pte);
         add_mm_rss(dst_mm, rss[0], rss[1]);
-        pte_unmap_unlock(dst_pte - 1, dst_ptl);
+        pte_unmap_unlock(orig_dst_pte, dst_ptl);
         cond_resched();
         if (addr != end)
                 goto again;
@@ -1141,9 +1178,14 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
                 goto no_page;
         if ((flags & FOLL_WRITE) && !pte_write(pte))
                 goto unlock;
+
         page = vm_normal_page(vma, address, pte);
-        if (unlikely(!page))
-                goto bad_page;
+        if (unlikely(!page)) {
+                if ((flags & FOLL_DUMP) ||
+                    !is_zero_pfn(pte_pfn(pte)))
+                        goto bad_page;
+                page = pte_page(pte);
+        }
 
         if (flags & FOLL_GET)
                 get_page(page);
@@ -1171,65 +1213,46 @@ no_page:
         pte_unmap_unlock(ptep, ptl);
         if (!pte_none(pte))
                 return page;
-        /* Fall through to ZERO_PAGE handling */
+
 no_page_table:
         /*
          * When core dumping an enormous anonymous area that nobody
-         * has touched so far, we don't want to allocate page tables.
+         * has touched so far, we don't want to allocate unnecessary pages or
+         * page tables.  Return error instead of NULL to skip handle_mm_fault,
+         * then get_dump_page() will return NULL to leave a hole in the dump.
+         * But we can only make this optimization where a hole would surely
+         * be zero-filled if handle_mm_fault() actually did handle it.
          */
-        if (flags & FOLL_ANON) {
-                page = ZERO_PAGE(0);
-                if (flags & FOLL_GET)
-                        get_page(page);
-                BUG_ON(flags & FOLL_WRITE);
-        }
+        if ((flags & FOLL_DUMP) &&
+            (!vma->vm_ops || !vma->vm_ops->fault))
+                return ERR_PTR(-EFAULT);
         return page;
 }
 
-/* Can we do the FOLL_ANON optimization? */
-static inline int use_zero_page(struct vm_area_struct *vma)
-{
-        /*
-         * We don't want to optimize FOLL_ANON for make_pages_present()
-         * when it tries to page in a VM_LOCKED region. As to VM_SHARED,
-         * we want to get the page from the page tables to make sure
-         * that we serialize and update with any other user of that
-         * mapping.
-         */
-        if (vma->vm_flags & (VM_LOCKED | VM_SHARED))
-                return 0;
-        /*
-         * And if we have a fault routine, it's not an anonymous region.
-         */
-        return !vma->vm_ops || !vma->vm_ops->fault;
-}
-
-
-
 int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
-                     unsigned long start, int len, int flags,
-                struct page **pages, struct vm_area_struct **vmas)
+                     unsigned long start, int nr_pages, unsigned int gup_flags,
+                     struct page **pages, struct vm_area_struct **vmas)
 {
         int i;
-        unsigned int vm_flags = 0;
-        int write = !!(flags & GUP_FLAGS_WRITE);
-        int force = !!(flags & GUP_FLAGS_FORCE);
-        int ignore = !!(flags & GUP_FLAGS_IGNORE_VMA_PERMISSIONS);
-        int ignore_sigkill = !!(flags & GUP_FLAGS_IGNORE_SIGKILL);
+        unsigned long vm_flags;
 
-        if (len <= 0)
+        if (nr_pages <= 0)
                 return 0;
+
+        VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
+
         /* 
          * Require read or write permissions.
-         * If 'force' is set, we only require the "MAY" flags.
+         * If FOLL_FORCE is set, we only require the "MAY" flags.
          */
-        vm_flags  = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
-        vm_flags &= force ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
+        vm_flags  = (gup_flags & FOLL_WRITE) ?
+                        (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
+        vm_flags &= (gup_flags & FOLL_FORCE) ?
+                        (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
         i = 0;
 
         do {
                 struct vm_area_struct *vma;
-                unsigned int foll_flags;
 
                 vma = find_extend_vma(mm, start);
                 if (!vma && in_gate_area(tsk, start)) {
@@ -1241,7 +1264,7 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                         pte_t *pte;
 
                         /* user gate pages are read-only */
-                        if (!ignore && write)
+                        if (gup_flags & FOLL_WRITE)
                                 return i ? : -EFAULT;
                         if (pg > TASK_SIZE)
                                 pgd = pgd_offset_k(pg);
@@ -1269,53 +1292,45 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                                 vmas[i] = gate_vma;
                         i++;
                         start += PAGE_SIZE;
-                        len--;
+                        nr_pages--;
                         continue;
                 }
 
                 if (!vma ||
                     (vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
-                    (!ignore && !(vm_flags & vma->vm_flags)))
+                    !(vm_flags & vma->vm_flags))
                         return i ? : -EFAULT;
 
                 if (is_vm_hugetlb_page(vma)) {
                         i = follow_hugetlb_page(mm, vma, pages, vmas,
-                                                &start, &len, i, write);
+                                        &start, &nr_pages, i, gup_flags);
                         continue;
                 }
 
-                foll_flags = FOLL_TOUCH;
-                if (pages)
-                        foll_flags |= FOLL_GET;
-                if (!write && use_zero_page(vma))
-                        foll_flags |= FOLL_ANON;
-
                 do {
                         struct page *page;
+                        unsigned int foll_flags = gup_flags;
 
                         /*
                          * If we have a pending SIGKILL, don't keep faulting
-                         * pages and potentially allocating memory, unless
-                         * current is handling munlock--e.g., on exit. In
-                         * that case, we are not allocating memory.  Rather,
-                         * we're only unlocking already resident/mapped pages.
+                         * pages and potentially allocating memory.
                          */
-                        if (unlikely(!ignore_sigkill &&
-                                        fatal_signal_pending(current)))
+                        if (unlikely(fatal_signal_pending(current)))
                                 return i ? i : -ERESTARTSYS;
 
-                        if (write)
-                                foll_flags |= FOLL_WRITE;
-
                         cond_resched();
                         while (!(page = follow_page(vma, start, foll_flags))) {
                                 int ret;
+
                                 ret = handle_mm_fault(mm, vma, start,
-                                                foll_flags & FOLL_WRITE);
+                                        (foll_flags & FOLL_WRITE) ?
+                                        FAULT_FLAG_WRITE : 0);
+
                                 if (ret & VM_FAULT_ERROR) {
                                         if (ret & VM_FAULT_OOM)
                                                 return i ? i : -ENOMEM;
-                                        else if (ret & VM_FAULT_SIGBUS)
+                                        if (ret &
+                                            (VM_FAULT_HWPOISON|VM_FAULT_SIGBUS))
                                                 return i ? i : -EFAULT;
                                         BUG();
                                 }
@@ -1354,30 +1369,107 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                                 vmas[i] = vma;
                         i++;
                         start += PAGE_SIZE;
-                        len--;
-                } while (len && start < vma->vm_end);
-        } while (len);
+                        nr_pages--;
+                } while (nr_pages && start < vma->vm_end);
+        } while (nr_pages);
         return i;
 }
 
+/**
+ * get_user_pages() - pin user pages in memory
+ * @tsk:        task_struct of target task
+ * @mm:         mm_struct of target mm
+ * @start:      starting user address
+ * @nr_pages:   number of pages from start to pin
+ * @write:      whether pages will be written to by the caller
+ * @force:      whether to force write access even if user mapping is
+ *              readonly. This will result in the page being COWed even
+ *              in MAP_SHARED mappings. You do not want this.
+ * @pages:      array that receives pointers to the pages pinned.
+ *              Should be at least nr_pages long. Or NULL, if caller
+ *              only intends to ensure the pages are faulted in.
+ * @vmas:       array of pointers to vmas corresponding to each page.
+ *              Or NULL if the caller does not require them.
+ *
+ * Returns number of pages pinned. This may be fewer than the number
+ * requested. If nr_pages is 0 or negative, returns 0. If no pages
+ * were pinned, returns -errno. Each page returned must be released
+ * with a put_page() call when it is finished with. vmas will only
+ * remain valid while mmap_sem is held.
+ *
+ * Must be called with mmap_sem held for read or write.
+ *
+ * get_user_pages walks a process's page tables and takes a reference to
+ * each struct page that each user address corresponds to at a given
+ * instant. That is, it takes the page that would be accessed if a user
+ * thread accesses the given user virtual address at that instant.
+ *
+ * This does not guarantee that the page exists in the user mappings when
+ * get_user_pages returns, and there may even be a completely different
+ * page there in some cases (eg. if mmapped pagecache has been invalidated
+ * and subsequently re faulted). However it does guarantee that the page
+ * won't be freed completely. And mostly callers simply care that the page
+ * contains data that was valid *at some point in time*. Typically, an IO
+ * or similar operation cannot guarantee anything stronger anyway because
+ * locks can't be held over the syscall boundary.
+ *
+ * If write=0, the page must not be written to. If the page is written to,
+ * set_page_dirty (or set_page_dirty_lock, as appropriate) must be called
+ * after the page is finished with, and before put_page is called.
+ *
+ * get_user_pages is typically used for fewer-copy IO operations, to get a
+ * handle on the memory by some means other than accesses via the user virtual
+ * addresses. The pages may be submitted for DMA to devices or accessed via
+ * their kernel linear mapping (via the kmap APIs). Care should be taken to
+ * use the correct cache flushing APIs.
+ *
+ * See also get_user_pages_fast, for performance critical applications.
+ */
 int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
-                unsigned long start, int len, int write, int force,
+                unsigned long start, int nr_pages, int write, int force,
                 struct page **pages, struct vm_area_struct **vmas)
 {
-        int flags = 0;
+        int flags = FOLL_TOUCH;
 
+        if (pages)
+                flags |= FOLL_GET;
         if (write)
-                flags |= GUP_FLAGS_WRITE;
+                flags |= FOLL_WRITE;
         if (force)
-                flags |= GUP_FLAGS_FORCE;
+                flags |= FOLL_FORCE;
 
-        return __get_user_pages(tsk, mm,
-                                start, len, flags,
-                                pages, vmas);
+        return __get_user_pages(tsk, mm, start, nr_pages, flags, pages, vmas);
 }
-
 EXPORT_SYMBOL(get_user_pages);
 
+/**
+ * get_dump_page() - pin user page in memory while writing it to core dump
+ * @addr: user address
+ *
+ * Returns struct page pointer of user page pinned for dump,
+ * to be freed afterwards by page_cache_release() or put_page().
+ *
+ * Returns NULL on any kind of failure - a hole must then be inserted into
+ * the corefile, to preserve alignment with its headers; and also returns
+ * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found -
+ * allowing a hole to be left in the corefile to save diskspace.
+ *
+ * Called without mmap_sem, but after all other threads have been killed.
+ */
+#ifdef CONFIG_ELF_CORE
+struct page *get_dump_page(unsigned long addr)
+{
+        struct vm_area_struct *vma;
+        struct page *page;
+
+        if (__get_user_pages(current, current->mm, addr, 1,
+                        FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma) < 1)
+                return NULL;
+        flush_cache_page(vma, addr, page_to_pfn(page));
+        return page;
+}
+#endif /* CONFIG_ELF_CORE */
+
 pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
                         spinlock_t **ptl)
 {
@@ -1555,7 +1647,8 @@ int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
          * If we don't have pte special, then we have to use the pfn_valid()
          * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
          * refcount the page if pfn_valid is true (hence insert_page rather
-         * than insert_pfn).
+         * than insert_pfn).  If a zero_pfn were inserted into a VM_MIXEDMAP
+         * without pte special, it would there be refcounted as a normal page.
          */
         if (!HAVE_PTE_SPECIAL && pfn_valid(pfn)) {
                 struct page *page;
@@ -1730,10 +1823,10 @@ static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
         token = pmd_pgtable(*pmd);
 
         do {
-                err = fn(pte, token, addr, data);
+                err = fn(pte++, token, addr, data);
                 if (err)
                         break;
-        } while (pte++, addr += PAGE_SIZE, addr != end);
+        } while (addr += PAGE_SIZE, addr != end);
 
         arch_leave_lazy_mmu_mode();
 
@@ -1921,7 +2014,7 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
          * Take out anonymous pages first, anonymous shared vmas are
          * not dirty accountable.
          */
-        if (PageAnon(old_page)) {
+        if (PageAnon(old_page) && !PageKsm(old_page)) {
                 if (!trylock_page(old_page)) {
                         page_cache_get(old_page);
                         pte_unmap_unlock(page_table, ptl);
@@ -1971,6 +2064,15 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
                                 ret = tmp;
                                 goto unwritable_page;
                         }
+                        if (unlikely(!(tmp & VM_FAULT_LOCKED))) {
+                                lock_page(old_page);
+                                if (!old_page->mapping) {
+                                        ret = 0; /* retry the fault */
+                                        unlock_page(old_page);
+                                        goto unwritable_page;
+                                }
+                        } else
+                                VM_BUG_ON(!PageLocked(old_page));
 
                         /*
                          * Since we dropped the lock we need to revalidate
@@ -1980,9 +2082,11 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
                          */
                         page_table = pte_offset_map_lock(mm, pmd, address,
                                                          &ptl);
-                        page_cache_release(old_page);
-                        if (!pte_same(*page_table, orig_pte))
+                        if (!pte_same(*page_table, orig_pte)) {
+                                unlock_page(old_page);
+                                page_cache_release(old_page);
                                 goto unlock;
+                        }
 
                         page_mkwrite = 1;
                 }
@@ -2011,10 +2115,19 @@ gotten:
 
         if (unlikely(anon_vma_prepare(vma)))
                 goto oom;
-        VM_BUG_ON(old_page == ZERO_PAGE(0));
-        new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
-        if (!new_page)
-                goto oom;
+
+        if (is_zero_pfn(pte_pfn(orig_pte))) {
+                new_page = alloc_zeroed_user_highpage_movable(vma, address);
+                if (!new_page)
+                        goto oom;
+        } else {
+                new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
+                if (!new_page)
+                        goto oom;
+                cow_user_page(new_page, old_page, address, vma);
+        }
+        __SetPageUptodate(new_page);
+
         /*
          * Don't let another task, with possibly unlocked vma,
          * keep the mlocked page.
@@ -2024,8 +2137,6 @@ gotten:
                 clear_page_mlock(old_page);
                 unlock_page(old_page);
         }
-        cow_user_page(new_page, old_page, address, vma);
-        __SetPageUptodate(new_page);
 
         if (mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))
                 goto oom_free_new;
@@ -2051,9 +2162,14 @@ gotten:
                  * seen in the presence of one thread doing SMC and another
                  * thread doing COW.
                  */
-                ptep_clear_flush_notify(vma, address, page_table);
+                ptep_clear_flush(vma, address, page_table);
                 page_add_new_anon_rmap(new_page, vma, address);
-                set_pte_at(mm, address, page_table, entry);
+                /*
+                 * We call the notify macro here because, when using secondary
+                 * mmu page tables (such as kvm shadow page tables), we want the
+                 * new page to be mapped directly into the secondary page table.
+                 */
+                set_pte_at_notify(mm, address, page_table, entry);
                 update_mmu_cache(vma, address, entry);
                 if (old_page) {
                         /*
@@ -2094,9 +2210,6 @@ gotten:
 unlock:
         pte_unmap_unlock(page_table, ptl);
         if (dirty_page) {
-                if (vma->vm_file)
-                        file_update_time(vma->vm_file);
-
                 /*
                  * Yes, Virginia, this is actually required to prevent a race
                  * with clear_page_dirty_for_io() from clearing the page dirty
@@ -2105,16 +2218,41 @@ unlock:
                  *
                  * do_no_page is protected similarly.
                  */
-                wait_on_page_locked(dirty_page);
-                set_page_dirty_balance(dirty_page, page_mkwrite);
+                if (!page_mkwrite) {
+                        wait_on_page_locked(dirty_page);
+                        set_page_dirty_balance(dirty_page, page_mkwrite);
+                }
                 put_page(dirty_page);
+                if (page_mkwrite) {
+                        struct address_space *mapping = dirty_page->mapping;
+
+                        set_page_dirty(dirty_page);
+                        unlock_page(dirty_page);
+                        page_cache_release(dirty_page);
+                        if (mapping)    {
+                                /*
+                                 * Some device drivers do not set page.mapping
+                                 * but still dirty their pages
+                                 */
+                                balance_dirty_pages_ratelimited(mapping);
+                        }
+                }
+
+                /* file_update_time outside page_lock */
+                if (vma->vm_file)
+                        file_update_time(vma->vm_file);
         }
         return ret;
 oom_free_new:
         page_cache_release(new_page);
 oom:
-        if (old_page)
+        if (old_page) {
+                if (page_mkwrite) {
+                        unlock_page(old_page);
+                        page_cache_release(old_page);
+                }
                 page_cache_release(old_page);
+        }
         return VM_FAULT_OOM;
 
 unwritable_page:
@@ -2274,7 +2412,7 @@ restart:
  * @mapping: the address space containing mmaps to be unmapped.
  * @holebegin: byte in first page to unmap, relative to the start of
  * the underlying file.  This will be rounded down to a PAGE_SIZE
- * boundary.  Note that this is different from vmtruncate(), which
+ * boundary.  Note that this is different from truncate_pagecache(), which
  * must keep the partial page.  In contrast, we must get rid of
  * partial pages.
  * @holelen: size of prospective hole in bytes.  This will be rounded
@@ -2325,63 +2463,6 @@ void unmap_mapping_range(struct address_space *mapping,
 }
 EXPORT_SYMBOL(unmap_mapping_range);
 
-/**
- * vmtruncate - unmap mappings "freed" by truncate() syscall
- * @inode: inode of the file used
- * @offset: file offset to start truncating
- *
- * NOTE! We have to be ready to update the memory sharing
- * between the file and the memory map for a potential last
- * incomplete page.  Ugly, but necessary.
- */
-int vmtruncate(struct inode * inode, loff_t offset)
-{
-        if (inode->i_size < offset) {
-                unsigned long limit;
-
-                limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
-                if (limit != RLIM_INFINITY && offset > limit)
-                        goto out_sig;
-                if (offset > inode->i_sb->s_maxbytes)
-                        goto out_big;
-                i_size_write(inode, offset);
-        } else {
-                struct address_space *mapping = inode->i_mapping;
-
-                /*
-                 * truncation of in-use swapfiles is disallowed - it would
-                 * cause subsequent swapout to scribble on the now-freed
-                 * blocks.
-                 */
-                if (IS_SWAPFILE(inode))
-                        return -ETXTBSY;
-                i_size_write(inode, offset);
-
-                /*
-                 * unmap_mapping_range is called twice, first simply for
-                 * efficiency so that truncate_inode_pages does fewer
-                 * single-page unmaps.  However after this first call, and
-                 * before truncate_inode_pages finishes, it is possible for
-                 * private pages to be COWed, which remain after
-                 * truncate_inode_pages finishes, hence the second
-                 * unmap_mapping_range call must be made for correctness.
-                 */
-                unmap_mapping_range(mapping, offset + PAGE_SIZE - 1, 0, 1);
-                truncate_inode_pages(mapping, offset);
-                unmap_mapping_range(mapping, offset + PAGE_SIZE - 1, 0, 1);
-        }
-
-        if (inode->i_op->truncate)
-                inode->i_op->truncate(inode);
-        return 0;
-
-out_sig:
-        send_sig(SIGXFSZ, current, 0);
-out_big:
-        return -EFBIG;
-}
-EXPORT_SYMBOL(vmtruncate);
-
 int vmtruncate_range(struct inode *inode, loff_t offset, loff_t end)
 {
         struct address_space *mapping = inode->i_mapping;
@@ -2413,7 +2494,7 @@ int vmtruncate_range(struct inode *inode, loff_t offset, loff_t end)
  */
 static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 unsigned long address, pte_t *page_table, pmd_t *pmd,
-                int write_access, pte_t orig_pte)
+                unsigned int flags, pte_t orig_pte)
 {
         spinlock_t *ptl;
         struct page *page;
@@ -2426,14 +2507,21 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 goto out;
 
         entry = pte_to_swp_entry(orig_pte);
-        if (is_migration_entry(entry)) {
-                migration_entry_wait(mm, pmd, address);
+        if (unlikely(non_swap_entry(entry))) {
+                if (is_migration_entry(entry)) {
+                        migration_entry_wait(mm, pmd, address);
+                } else if (is_hwpoison_entry(entry)) {
+                        ret = VM_FAULT_HWPOISON;
+                } else {
+                        print_bad_pte(vma, address, orig_pte, NULL);
+                        ret = VM_FAULT_OOM;
+                }
                 goto out;
         }
         delayacct_set_flag(DELAYACCT_PF_SWAPIN);
         page = lookup_swap_cache(entry);
         if (!page) {
-                grab_swap_token(); /* Contend for token _before_ read-in */
+                grab_swap_token(mm); /* Contend for token _before_ read-in */
                 page = swapin_readahead(entry,
                                         GFP_HIGHUSER_MOVABLE, vma, address);
                 if (!page) {
@@ -2451,6 +2539,10 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 /* Had to read the page from swap area: Major fault */
                 ret = VM_FAULT_MAJOR;
                 count_vm_event(PGMAJFAULT);
+        } else if (PageHWPoison(page)) {
+                ret = VM_FAULT_HWPOISON;
+                delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
+                goto out_release;
         }
 
         lock_page(page);
@@ -2458,8 +2550,7 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
 
         if (mem_cgroup_try_charge_swapin(mm, page, GFP_KERNEL, &ptr)) {
                 ret = VM_FAULT_OOM;
-                unlock_page(page);
-                goto out;
+                goto out_page;
         }
 
         /*
@@ -2490,9 +2581,9 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
 
         inc_mm_counter(mm, anon_rss);
         pte = mk_pte(page, vma->vm_page_prot);
-        if (write_access && reuse_swap_page(page)) {
+        if ((flags & FAULT_FLAG_WRITE) && reuse_swap_page(page)) {
                 pte = maybe_mkwrite(pte_mkdirty(pte), vma);
-                write_access = 0;
+                flags &= ~FAULT_FLAG_WRITE;
         }
         flush_icache_page(vma, page);
         set_pte_at(mm, address, page_table, pte);
@@ -2505,7 +2596,7 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 try_to_free_swap(page);
         unlock_page(page);
 
-        if (write_access) {
+        if (flags & FAULT_FLAG_WRITE) {
                 ret |= do_wp_page(mm, vma, address, page_table, pmd, ptl, pte);
                 if (ret & VM_FAULT_ERROR)
                         ret &= VM_FAULT_ERROR;
@@ -2521,7 +2612,9 @@ out:
 out_nomap:
         mem_cgroup_cancel_charge_swapin(ptr);
         pte_unmap_unlock(page_table, ptl);
+out_page:
         unlock_page(page);
+out_release:
         page_cache_release(page);
         return ret;
 }
@@ -2533,12 +2626,22 @@ out_nomap:
  */
 static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 unsigned long address, pte_t *page_table, pmd_t *pmd,
-                int write_access)
+                unsigned int flags)
 {
         struct page *page;
         spinlock_t *ptl;
         pte_t entry;
 
+        if (!(flags & FAULT_FLAG_WRITE)) {
+                entry = pte_mkspecial(pfn_pte(my_zero_pfn(address),
+                                                vma->vm_page_prot));
+                ptl = pte_lockptr(mm, pmd);
+                spin_lock(ptl);
+                if (!pte_none(*page_table))
+                        goto unlock;
+                goto setpte;
+        }
+
         /* Allocate our own private page. */
         pte_unmap(page_table);
 
@@ -2553,13 +2656,16 @@ static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 goto oom_free_page;
 
         entry = mk_pte(page, vma->vm_page_prot);
-        entry = maybe_mkwrite(pte_mkdirty(entry), vma);
+        if (vma->vm_flags & VM_WRITE)
+                entry = pte_mkwrite(pte_mkdirty(entry));
 
         page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
         if (!pte_none(*page_table))
                 goto release;
+
         inc_mm_counter(mm, anon_rss);
         page_add_new_anon_rmap(page, vma, address);
+setpte:
         set_pte_at(mm, address, page_table, entry);
 
         /* No need to invalidate - it was non-present before */
@@ -2614,6 +2720,12 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
         if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
                 return ret;
 
+        if (unlikely(PageHWPoison(vmf.page))) {
+                if (ret & VM_FAULT_LOCKED)
+                        unlock_page(vmf.page);
+                return VM_FAULT_HWPOISON;
+        }
+
         /*
          * For consistency in subsequent calls, make the faulted page always
          * locked.
@@ -2664,27 +2776,22 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
                                 int tmp;
 
                                 unlock_page(page);
-                                vmf.flags |= FAULT_FLAG_MKWRITE;
+                                vmf.flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
                                 tmp = vma->vm_ops->page_mkwrite(vma, &vmf);
                                 if (unlikely(tmp &
                                           (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
                                         ret = tmp;
-                                        anon = 1; /* no anon but release vmf.page */
-                                        goto out_unlocked;
-                                }
-                                lock_page(page);
-                                /*
-                                 * XXX: this is not quite right (racy vs
-                                 * invalidate) to unlock and relock the page
-                                 * like this, however a better fix requires
-                                 * reworking page_mkwrite locking API, which
-                                 * is better done later.
-                                 */
-                                if (!page->mapping) {
-                                        ret = 0;
-                                        anon = 1; /* no anon but release vmf.page */
-                                        goto out;
+                                        goto unwritable_page;
                                 }
+                                if (unlikely(!(tmp & VM_FAULT_LOCKED))) {
+                                        lock_page(page);
+                                        if (!page->mapping) {
+                                                ret = 0; /* retry the fault */
+                                                unlock_page(page);
+                                                goto unwritable_page;
+                                        }
+                                } else
+                                        VM_BUG_ON(!PageLocked(page));
                                 page_mkwrite = 1;
                         }
                 }
@@ -2698,7 +2805,7 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
          * due to the bad i386 page protection. But it's valid
          * for other architectures too.
          *
-         * Note that if write_access is true, we either now have
+         * Note that if FAULT_FLAG_WRITE is set, we either now have
          * an exclusive copy of the page, or this is a shared mapping,
          * so we can make it writable and dirty to avoid having to
          * handle that later.
@@ -2736,28 +2843,43 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
         pte_unmap_unlock(page_table, ptl);
 
 out:
-        unlock_page(vmf.page);
-out_unlocked:
-        if (anon)
-                page_cache_release(vmf.page);
-        else if (dirty_page) {
-                if (vma->vm_file)
-                        file_update_time(vma->vm_file);
+        if (dirty_page) {
+                struct address_space *mapping = page->mapping;
 
-                set_page_dirty_balance(dirty_page, page_mkwrite);
+                if (set_page_dirty(dirty_page))
+                        page_mkwrite = 1;
+                unlock_page(dirty_page);
                 put_page(dirty_page);
+                if (page_mkwrite && mapping) {
+                        /*
+                         * Some device drivers do not set page.mapping but still
+                         * dirty their pages
+                         */
+                        balance_dirty_pages_ratelimited(mapping);
+                }
+
+                /* file_update_time outside page_lock */
+                if (vma->vm_file)
+                        file_update_time(vma->vm_file);
+        } else {
+                unlock_page(vmf.page);
+                if (anon)
+                        page_cache_release(vmf.page);
         }
 
         return ret;
+
+unwritable_page:
+        page_cache_release(page);
+        return ret;
 }
 
 static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma,
                 unsigned long address, pte_t *page_table, pmd_t *pmd,
-                int write_access, pte_t orig_pte)
+                unsigned int flags, pte_t orig_pte)
 {
         pgoff_t pgoff = (((address & PAGE_MASK)
                         - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
-        unsigned int flags = (write_access ? FAULT_FLAG_WRITE : 0);
 
         pte_unmap(page_table);
         return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte);
@@ -2774,12 +2896,12 @@ static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  */
 static int do_nonlinear_fault(struct mm_struct *mm, struct vm_area_struct *vma,
                 unsigned long address, pte_t *page_table, pmd_t *pmd,
-                int write_access, pte_t orig_pte)
+                unsigned int flags, pte_t orig_pte)
 {
-        unsigned int flags = FAULT_FLAG_NONLINEAR |
-                                (write_access ? FAULT_FLAG_WRITE : 0);
         pgoff_t pgoff;
 
+        flags |= FAULT_FLAG_NONLINEAR;
+
         if (!pte_unmap_same(mm, pmd, page_table, orig_pte))
                 return 0;
 
@@ -2810,7 +2932,7 @@ static int do_nonlinear_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  */
 static inline int handle_pte_fault(struct mm_struct *mm,
                 struct vm_area_struct *vma, unsigned long address,
-                pte_t *pte, pmd_t *pmd, int write_access)
+                pte_t *pte, pmd_t *pmd, unsigned int flags)
 {
         pte_t entry;
         spinlock_t *ptl;
@@ -2821,30 +2943,30 @@ static inline int handle_pte_fault(struct mm_struct *mm,
                         if (vma->vm_ops) {
                                 if (likely(vma->vm_ops->fault))
                                         return do_linear_fault(mm, vma, address,
-                                                pte, pmd, write_access, entry);
+                                                pte, pmd, flags, entry);
                         }
                         return do_anonymous_page(mm, vma, address,
-                                                 pte, pmd, write_access);
+                                                 pte, pmd, flags);
                 }
                 if (pte_file(entry))
                         return do_nonlinear_fault(mm, vma, address,
-                                        pte, pmd, write_access, entry);
+                                        pte, pmd, flags, entry);
                 return do_swap_page(mm, vma, address,
-                                        pte, pmd, write_access, entry);
+                                        pte, pmd, flags, entry);
         }
 
         ptl = pte_lockptr(mm, pmd);
         spin_lock(ptl);
         if (unlikely(!pte_same(*pte, entry)))
                 goto unlock;
-        if (write_access) {
+        if (flags & FAULT_FLAG_WRITE) {
                 if (!pte_write(entry))
                         return do_wp_page(mm, vma, address,
                                         pte, pmd, ptl, entry);
                 entry = pte_mkdirty(entry);
         }
         entry = pte_mkyoung(entry);
-        if (ptep_set_access_flags(vma, address, pte, entry, write_access)) {
+        if (ptep_set_access_flags(vma, address, pte, entry, flags & FAULT_FLAG_WRITE)) {
                 update_mmu_cache(vma, address, entry);
         } else {
                 /*
@@ -2853,7 +2975,7 @@ static inline int handle_pte_fault(struct mm_struct *mm,
                  * This still avoids useless tlb flushes for .text page faults
                  * with threads.
                  */
-                if (write_access)
+                if (flags & FAULT_FLAG_WRITE)
                         flush_tlb_page(vma, address);
         }
 unlock:
@@ -2865,7 +2987,7 @@ unlock:
  * By the time we get here, we already hold the mm semaphore
  */
 int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
-                unsigned long address, int write_access)
+                unsigned long address, unsigned int flags)
 {
         pgd_t *pgd;
         pud_t *pud;
@@ -2877,7 +2999,7 @@ int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
         count_vm_event(PGFAULT);
 
         if (unlikely(is_vm_hugetlb_page(vma)))
-                return hugetlb_fault(mm, vma, address, write_access);
+                return hugetlb_fault(mm, vma, address, flags);
 
         pgd = pgd_offset(mm, address);
         pud = pud_alloc(mm, pgd, address);
@@ -2890,7 +3012,7 @@ int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
         if (!pte)
                 return VM_FAULT_OOM;
 
-        return handle_pte_fault(mm, vma, address, pte, pmd, write_access);
+        return handle_pte_fault(mm, vma, address, pte, pmd, flags);
 }
 
 #ifndef __PAGETABLE_PUD_FOLDED
@@ -3009,22 +3131,13 @@ int in_gate_area_no_task(unsigned long addr)
 
 #endif  /* __HAVE_ARCH_GATE_AREA */
 
-#ifdef CONFIG_HAVE_IOREMAP_PROT
-int follow_phys(struct vm_area_struct *vma,
-                unsigned long address, unsigned int flags,
-                unsigned long *prot, resource_size_t *phys)
+static int follow_pte(struct mm_struct *mm, unsigned long address,
+                pte_t **ptepp, spinlock_t **ptlp)
 {
         pgd_t *pgd;
         pud_t *pud;
         pmd_t *pmd;
-        pte_t *ptep, pte;
-        spinlock_t *ptl;
-        resource_size_t phys_addr = 0;
-        struct mm_struct *mm = vma->vm_mm;
-        int ret = -EINVAL;
-
-        if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
-                goto out;
+        pte_t *ptep;
 
         pgd = pgd_offset(mm, address);
         if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
@@ -3042,22 +3155,71 @@ int follow_phys(struct vm_area_struct *vma,
         if (pmd_huge(*pmd))
                 goto out;
 
-        ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
+        ptep = pte_offset_map_lock(mm, pmd, address, ptlp);
         if (!ptep)
                 goto out;
+        if (!pte_present(*ptep))
+                goto unlock;
+        *ptepp = ptep;
+        return 0;
+unlock:
+        pte_unmap_unlock(ptep, *ptlp);
+out:
+        return -EINVAL;
+}
+
+/**
+ * follow_pfn - look up PFN at a user virtual address
+ * @vma: memory mapping
+ * @address: user virtual address
+ * @pfn: location to store found PFN
+ *
+ * Only IO mappings and raw PFN mappings are allowed.
+ *
+ * Returns zero and the pfn at @pfn on success, -ve otherwise.
+ */
+int follow_pfn(struct vm_area_struct *vma, unsigned long address,
+        unsigned long *pfn)
+{
+        int ret = -EINVAL;
+        spinlock_t *ptl;
+        pte_t *ptep;
+
+        if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
+                return ret;
 
+        ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
+        if (ret)
+                return ret;
+        *pfn = pte_pfn(*ptep);
+        pte_unmap_unlock(ptep, ptl);
+        return 0;
+}
+EXPORT_SYMBOL(follow_pfn);
+
+#ifdef CONFIG_HAVE_IOREMAP_PROT
+int follow_phys(struct vm_area_struct *vma,
+                unsigned long address, unsigned int flags,
+                unsigned long *prot, resource_size_t *phys)
+{
+        int ret = -EINVAL;
+        pte_t *ptep, pte;
+        spinlock_t *ptl;
+
+        if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
+                goto out;
+
+        if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
+                goto out;
         pte = *ptep;
-        if (!pte_present(pte))
-                goto unlock;
+
         if ((flags & FOLL_WRITE) && !pte_write(pte))
                 goto unlock;
-        phys_addr = pte_pfn(pte);
-        phys_addr <<= PAGE_SHIFT; /* Shift here to avoid overflow on PAE */
 
         *prot = pgprot_val(pte_pgprot(pte));
-        *phys = phys_addr;
-        ret = 0;
+        *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
 
+        ret = 0;
 unlock:
         pte_unmap_unlock(ptep, ptl);
 out:
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index c083cf5fd6df..2047465cd27c 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -26,6 +26,7 @@
 #include <linux/migrate.h>
 #include <linux/page-isolation.h>
 #include <linux/pfn.h>
+#include <linux/suspend.h>
 
 #include <asm/tlbflush.h>
 
@@ -339,8 +340,11 @@ EXPORT_SYMBOL_GPL(__remove_pages);
 
 void online_page(struct page *page)
 {
+        unsigned long pfn = page_to_pfn(page);
+
         totalram_pages++;
-        num_physpages++;
+        if (pfn >= num_physpages)
+                num_physpages = pfn + 1;
 
 #ifdef CONFIG_HIGHMEM
         if (PageHighMem(page))
@@ -410,7 +414,7 @@ int online_pages(unsigned long pfn, unsigned long nr_pages)
         if (!populated_zone(zone))
                 need_zonelists_rebuild = 1;
 
-        ret = walk_memory_resource(pfn, nr_pages, &onlined_pages,
+        ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages,
                 online_pages_range);
         if (ret) {
                 printk(KERN_DEBUG "online_pages %lx at %lx failed\n",
@@ -422,7 +426,9 @@ int online_pages(unsigned long pfn, unsigned long nr_pages)
         zone->present_pages += onlined_pages;
         zone->zone_pgdat->node_present_pages += onlined_pages;
 
-        setup_per_zone_pages_min();
+        zone_pcp_update(zone);
+        setup_per_zone_wmarks();
+        calculate_zone_inactive_ratio(zone);
         if (onlined_pages) {
                 kswapd_run(zone_to_nid(zone));
                 node_set_state(zone_to_nid(zone), N_HIGH_MEMORY);
@@ -442,7 +448,8 @@ int online_pages(unsigned long pfn, unsigned long nr_pages)
 }
 #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
 
-static pg_data_t *hotadd_new_pgdat(int nid, u64 start)
+/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
+static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start)
 {
         struct pglist_data *pgdat;
         unsigned long zones_size[MAX_NR_ZONES] = {0};
@@ -479,14 +486,18 @@ int __ref add_memory(int nid, u64 start, u64 size)
         struct resource *res;
         int ret;
 
+        lock_system_sleep();
+
         res = register_memory_resource(start, size);
+        ret = -EEXIST;
         if (!res)
-                return -EEXIST;
+                goto out;
 
         if (!node_online(nid)) {
                 pgdat = hotadd_new_pgdat(nid, start);
+                ret = -ENOMEM;
                 if (!pgdat)
-                        return -ENOMEM;
+                        goto out;
                 new_pgdat = 1;
         }
 
@@ -509,7 +520,8 @@ int __ref add_memory(int nid, u64 start, u64 size)
                 BUG_ON(ret);
         }
 
-        return ret;
+        goto out;
+
 error:
         /* rollback pgdat allocation and others */
         if (new_pgdat)
@@ -517,6 +529,8 @@ error:
         if (res)
                 release_memory_resource(res);
 
+out:
+        unlock_system_sleep();
         return ret;
 }
 EXPORT_SYMBOL_GPL(add_memory);
@@ -700,7 +714,7 @@ offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages,
 static void
 offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
 {
-        walk_memory_resource(start_pfn, end_pfn - start_pfn, NULL,
+        walk_system_ram_range(start_pfn, end_pfn - start_pfn, NULL,
                                 offline_isolated_pages_cb);
 }
 
@@ -726,7 +740,7 @@ check_pages_isolated(unsigned long start_pfn, unsigned long end_pfn)
         long offlined = 0;
         int ret;
 
-        ret = walk_memory_resource(start_pfn, end_pfn - start_pfn, &offlined,
+        ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, &offlined,
                         check_pages_isolated_cb);
         if (ret < 0)
                 offlined = (long)ret;
@@ -753,6 +767,8 @@ int offline_pages(unsigned long start_pfn,
         if (!test_pages_in_a_zone(start_pfn, end_pfn))
                 return -EINVAL;
 
+        lock_system_sleep();
+
         zone = page_zone(pfn_to_page(start_pfn));
         node = zone_to_nid(zone);
         nr_pages = end_pfn - start_pfn;
@@ -760,7 +776,7 @@ int offline_pages(unsigned long start_pfn,
         /* set above range as isolated */
         ret = start_isolate_page_range(start_pfn, end_pfn);
         if (ret)
-                return ret;
+                goto out;
 
         arg.start_pfn = start_pfn;
         arg.nr_pages = nr_pages;
@@ -830,12 +846,15 @@ repeat:
         zone->present_pages -= offlined_pages;
         zone->zone_pgdat->node_present_pages -= offlined_pages;
         totalram_pages -= offlined_pages;
-        num_physpages -= offlined_pages;
+
+        setup_per_zone_wmarks();
+        calculate_zone_inactive_ratio(zone);
 
         vm_total_pages = nr_free_pagecache_pages();
         writeback_set_ratelimit();
 
         memory_notify(MEM_OFFLINE, &arg);
+        unlock_system_sleep();
         return 0;
 
 failed_removal:
@@ -845,6 +864,8 @@ failed_removal:
         /* pushback to free area */
         undo_isolate_page_range(start_pfn, end_pfn);
 
+out:
+        unlock_system_sleep();
         return ret;
 }
 
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 3eb4a6fdc043..4545d5944243 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -182,13 +182,58 @@ static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
         return 0;
 }
 
-/* Create a new policy */
+/*
+ * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
+ * any, for the new policy.  mpol_new() has already validated the nodes
+ * parameter with respect to the policy mode and flags.  But, we need to
+ * handle an empty nodemask with MPOL_PREFERRED here.
+ *
+ * Must be called holding task's alloc_lock to protect task's mems_allowed
+ * and mempolicy.  May also be called holding the mmap_semaphore for write.
+ */
+static int mpol_set_nodemask(struct mempolicy *pol,
+                     const nodemask_t *nodes, struct nodemask_scratch *nsc)
+{
+        int ret;
+
+        /* if mode is MPOL_DEFAULT, pol is NULL. This is right. */
+        if (pol == NULL)
+                return 0;
+        /* Check N_HIGH_MEMORY */
+        nodes_and(nsc->mask1,
+                  cpuset_current_mems_allowed, node_states[N_HIGH_MEMORY]);
+
+        VM_BUG_ON(!nodes);
+        if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))
+                nodes = NULL;   /* explicit local allocation */
+        else {
+                if (pol->flags & MPOL_F_RELATIVE_NODES)
+                        mpol_relative_nodemask(&nsc->mask2, nodes,&nsc->mask1);
+                else
+                        nodes_and(nsc->mask2, *nodes, nsc->mask1);
+
+                if (mpol_store_user_nodemask(pol))
+                        pol->w.user_nodemask = *nodes;
+                else
+                        pol->w.cpuset_mems_allowed =
+                                                cpuset_current_mems_allowed;
+        }
+
+        if (nodes)
+                ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
+        else
+                ret = mpol_ops[pol->mode].create(pol, NULL);
+        return ret;
+}
+
+/*
+ * This function just creates a new policy, does some check and simple
+ * initialization. You must invoke mpol_set_nodemask() to set nodes.
+ */
 static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
                                   nodemask_t *nodes)
 {
         struct mempolicy *policy;
-        nodemask_t cpuset_context_nmask;
-        int ret;
 
         pr_debug("setting mode %d flags %d nodes[0] %lx\n",
                  mode, flags, nodes ? nodes_addr(*nodes)[0] : -1);
@@ -210,7 +255,6 @@ static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
                         if (((flags & MPOL_F_STATIC_NODES) ||
                              (flags & MPOL_F_RELATIVE_NODES)))
                                 return ERR_PTR(-EINVAL);
-                        nodes = NULL;   /* flag local alloc */
                 }
         } else if (nodes_empty(*nodes))
                 return ERR_PTR(-EINVAL);
@@ -221,30 +265,6 @@ static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
         policy->mode = mode;
         policy->flags = flags;
 
-        if (nodes) {
-                /*
-                 * cpuset related setup doesn't apply to local allocation
-                 */
-                cpuset_update_task_memory_state();
-                if (flags & MPOL_F_RELATIVE_NODES)
-                        mpol_relative_nodemask(&cpuset_context_nmask, nodes,
-                                               &cpuset_current_mems_allowed);
-                else
-                        nodes_and(cpuset_context_nmask, *nodes,
-                                  cpuset_current_mems_allowed);
-                if (mpol_store_user_nodemask(policy))
-                        policy->w.user_nodemask = *nodes;
-                else
-                        policy->w.cpuset_mems_allowed =
-                                                cpuset_mems_allowed(current);
-        }
-
-        ret = mpol_ops[mode].create(policy,
-                                nodes ? &cpuset_context_nmask : NULL);
-        if (ret < 0) {
-                kmem_cache_free(policy_cache, policy);
-                return ERR_PTR(ret);
-        }
         return policy;
 }
 
@@ -324,6 +344,8 @@ static void mpol_rebind_policy(struct mempolicy *pol,
 /*
  * Wrapper for mpol_rebind_policy() that just requires task
  * pointer, and updates task mempolicy.
+ *
+ * Called with task's alloc_lock held.
  */
 
 void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
@@ -600,13 +622,19 @@ static void mpol_set_task_struct_flag(void)
 static long do_set_mempolicy(unsigned short mode, unsigned short flags,
                              nodemask_t *nodes)
 {
-        struct mempolicy *new;
+        struct mempolicy *new, *old;
         struct mm_struct *mm = current->mm;
+        NODEMASK_SCRATCH(scratch);
+        int ret;
 
-        new = mpol_new(mode, flags, nodes);
-        if (IS_ERR(new))
-                return PTR_ERR(new);
+        if (!scratch)
+                return -ENOMEM;
 
+        new = mpol_new(mode, flags, nodes);
+        if (IS_ERR(new)) {
+                ret = PTR_ERR(new);
+                goto out;
+        }
         /*
          * prevent changing our mempolicy while show_numa_maps()
          * is using it.
@@ -615,20 +643,36 @@ static long do_set_mempolicy(unsigned short mode, unsigned short flags,
          */
         if (mm)
                 down_write(&mm->mmap_sem);
-        mpol_put(current->mempolicy);
+        task_lock(current);
+        ret = mpol_set_nodemask(new, nodes, scratch);
+        if (ret) {
+                task_unlock(current);
+                if (mm)
+                        up_write(&mm->mmap_sem);
+                mpol_put(new);
+                goto out;
+        }
+        old = current->mempolicy;
         current->mempolicy = new;
         mpol_set_task_struct_flag();
         if (new && new->mode == MPOL_INTERLEAVE &&
             nodes_weight(new->v.nodes))
                 current->il_next = first_node(new->v.nodes);
+        task_unlock(current);
         if (mm)
                 up_write(&mm->mmap_sem);
 
-        return 0;
+        mpol_put(old);
+        ret = 0;
+out:
+        NODEMASK_SCRATCH_FREE(scratch);
+        return ret;
 }
 
 /*
  * Return nodemask for policy for get_mempolicy() query
+ *
+ * Called with task's alloc_lock held
  */
 static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
 {
@@ -674,7 +718,6 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask,
         struct vm_area_struct *vma = NULL;
         struct mempolicy *pol = current->mempolicy;
 
-        cpuset_update_task_memory_state();
         if (flags &
                 ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
                 return -EINVAL;
@@ -683,7 +726,9 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask,
                 if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
                         return -EINVAL;
                 *policy = 0;    /* just so it's initialized */
+                task_lock(current);
                 *nmask  = cpuset_current_mems_allowed;
+                task_unlock(current);
                 return 0;
         }
 
@@ -738,8 +783,11 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask,
         }
 
         err = 0;
-        if (nmask)
+        if (nmask) {
+                task_lock(current);
                 get_policy_nodemask(pol, nmask);
+                task_unlock(current);
+        }
 
  out:
         mpol_cond_put(pol);
@@ -767,7 +815,7 @@ static void migrate_page_add(struct page *page, struct list_head *pagelist,
 
 static struct page *new_node_page(struct page *page, unsigned long node, int **x)
 {
-        return alloc_pages_node(node, GFP_HIGHUSER_MOVABLE, 0);
+        return alloc_pages_exact_node(node, GFP_HIGHUSER_MOVABLE, 0);
 }
 
 /*
@@ -976,9 +1024,24 @@ static long do_mbind(unsigned long start, unsigned long len,
 
                 err = migrate_prep();
                 if (err)
-                        return err;
+                        goto mpol_out;
         }
-        down_write(&mm->mmap_sem);
+        {
+                NODEMASK_SCRATCH(scratch);
+                if (scratch) {
+                        down_write(&mm->mmap_sem);
+                        task_lock(current);
+                        err = mpol_set_nodemask(new, nmask, scratch);
+                        task_unlock(current);
+                        if (err)
+                                up_write(&mm->mmap_sem);
+                } else
+                        err = -ENOMEM;
+                NODEMASK_SCRATCH_FREE(scratch);
+        }
+        if (err)
+                goto mpol_out;
+
         vma = check_range(mm, start, end, nmask,
                           flags | MPOL_MF_INVERT, &pagelist);
 
@@ -994,9 +1057,11 @@ static long do_mbind(unsigned long start, unsigned long len,
 
                 if (!err && nr_failed && (flags & MPOL_MF_STRICT))
                         err = -EIO;
-        }
+        } else
+                putback_lru_pages(&pagelist);
 
         up_write(&mm->mmap_sem);
+ mpol_out:
         mpol_put(new);
         return err;
 }
@@ -1545,8 +1610,6 @@ alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr)
         struct mempolicy *pol = get_vma_policy(current, vma, addr);
         struct zonelist *zl;
 
-        cpuset_update_task_memory_state();
-
         if (unlikely(pol->mode == MPOL_INTERLEAVE)) {
                 unsigned nid;
 
@@ -1593,8 +1656,6 @@ struct page *alloc_pages_current(gfp_t gfp, unsigned order)
 {
         struct mempolicy *pol = current->mempolicy;
 
-        if ((gfp & __GFP_WAIT) && !in_interrupt())
-                cpuset_update_task_memory_state();
         if (!pol || in_interrupt() || (gfp & __GFP_THISNODE))
                 pol = &default_policy;
 
@@ -1851,27 +1912,46 @@ restart:
  * Install non-NULL @mpol in inode's shared policy rb-tree.
  * On entry, the current task has a reference on a non-NULL @mpol.
  * This must be released on exit.
+ * This is called at get_inode() calls and we can use GFP_KERNEL.
  */
 void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
 {
+        int ret;
+
         sp->root = RB_ROOT;             /* empty tree == default mempolicy */
         spin_lock_init(&sp->lock);
 
         if (mpol) {
                 struct vm_area_struct pvma;
                 struct mempolicy *new;
+                NODEMASK_SCRATCH(scratch);
 
+                if (!scratch)
+                        return;
                 /* contextualize the tmpfs mount point mempolicy */
                 new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
-                mpol_put(mpol); /* drop our ref on sb mpol */
-                if (IS_ERR(new))
+                if (IS_ERR(new)) {
+                        mpol_put(mpol); /* drop our ref on sb mpol */
+                        NODEMASK_SCRATCH_FREE(scratch);
                         return;         /* no valid nodemask intersection */
+                }
+
+                task_lock(current);
+                ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch);
+                task_unlock(current);
+                mpol_put(mpol); /* drop our ref on sb mpol */
+                if (ret) {
+                        NODEMASK_SCRATCH_FREE(scratch);
+                        mpol_put(new);
+                        return;
+                }
 
                 /* Create pseudo-vma that contains just the policy */
                 memset(&pvma, 0, sizeof(struct vm_area_struct));
                 pvma.vm_end = TASK_SIZE;        /* policy covers entire file */
                 mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
                 mpol_put(new);                  /* drop initial ref */
+                NODEMASK_SCRATCH_FREE(scratch);
         }
 }
 
@@ -2086,8 +2166,24 @@ int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context)
         new = mpol_new(mode, mode_flags, &nodes);
         if (IS_ERR(new))
                 err = 1;
-        else if (no_context)
-                new->w.user_nodemask = nodes;   /* save for contextualization */
+        else {
+                int ret;
+                NODEMASK_SCRATCH(scratch);
+                if (scratch) {
+                        task_lock(current);
+                        ret = mpol_set_nodemask(new, &nodes, scratch);
+                        task_unlock(current);
+                } else
+                        ret = -ENOMEM;
+                NODEMASK_SCRATCH_FREE(scratch);
+                if (ret) {
+                        err = 1;
+                        mpol_put(new);
+                } else if (no_context) {
+                        /* save for contextualization */
+                        new->w.user_nodemask = nodes;
+                }
+        }
 
 out:
         /* Restore string for error message */
diff --git a/mm/mempool.c b/mm/mempool.c
index a46eb1b4bb66..1a3bc3d4d554 100644
--- a/mm/mempool.c
+++ b/mm/mempool.c
@@ -303,18 +303,11 @@ EXPORT_SYMBOL(mempool_free_slab);
  */
 void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data)
 {
-        size_t size = (size_t)(long)pool_data;
+        size_t size = (size_t)pool_data;
         return kmalloc(size, gfp_mask);
 }
 EXPORT_SYMBOL(mempool_kmalloc);
 
-void *mempool_kzalloc(gfp_t gfp_mask, void *pool_data)
-{
-        size_t size = (size_t) pool_data;
-        return kzalloc(size, gfp_mask);
-}
-EXPORT_SYMBOL(mempool_kzalloc);
-
 void mempool_kfree(void *element, void *pool_data)
 {
         kfree(element);
diff --git a/mm/migrate.c b/mm/migrate.c
index a9eff3f092f6..7dbcb22316d2 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -67,6 +67,8 @@ int putback_lru_pages(struct list_head *l)
 
         list_for_each_entry_safe(page, page2, l, lru) {
                 list_del(&page->lru);
+                dec_zone_page_state(page, NR_ISOLATED_ANON +
+                                page_is_file_cache(page));
                 putback_lru_page(page);
                 count++;
         }
@@ -147,7 +149,7 @@ out:
 static void remove_file_migration_ptes(struct page *old, struct page *new)
 {
         struct vm_area_struct *vma;
-        struct address_space *mapping = page_mapping(new);
+        struct address_space *mapping = new->mapping;
         struct prio_tree_iter iter;
         pgoff_t pgoff = new->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
 
@@ -250,7 +252,7 @@ out:
  * The number of remaining references must be:
  * 1 for anonymous pages without a mapping
  * 2 for pages with a mapping
- * 3 for pages with a mapping and PagePrivate set.
+ * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
  */
 static int migrate_page_move_mapping(struct address_space *mapping,
                 struct page *newpage, struct page *page)
@@ -270,7 +272,7 @@ static int migrate_page_move_mapping(struct address_space *mapping,
         pslot = radix_tree_lookup_slot(&mapping->page_tree,
                                         page_index(page));
 
-        expected_count = 2 + !!PagePrivate(page);
+        expected_count = 2 + page_has_private(page);
         if (page_count(page) != expected_count ||
                         (struct page *)radix_tree_deref_slot(pslot) != page) {
                 spin_unlock_irq(&mapping->tree_lock);
@@ -312,7 +314,10 @@ static int migrate_page_move_mapping(struct address_space *mapping,
          */
         __dec_zone_page_state(page, NR_FILE_PAGES);
         __inc_zone_page_state(newpage, NR_FILE_PAGES);
-
+        if (PageSwapBacked(page)) {
+                __dec_zone_page_state(page, NR_SHMEM);
+                __inc_zone_page_state(newpage, NR_SHMEM);
+        }
         spin_unlock_irq(&mapping->tree_lock);
 
         return 0;
@@ -386,7 +391,7 @@ EXPORT_SYMBOL(fail_migrate_page);
 
 /*
  * Common logic to directly migrate a single page suitable for
- * pages that do not use PagePrivate.
+ * pages that do not use PagePrivate/PagePrivate2.
  *
  * Pages are locked upon entry and exit.
  */
@@ -522,7 +527,7 @@ static int fallback_migrate_page(struct address_space *mapping,
          * Buffers may be managed in a filesystem specific way.
          * We must have no buffers or drop them.
          */
-        if (PagePrivate(page) &&
+        if (page_has_private(page) &&
             !try_to_release_page(page, GFP_KERNEL))
                 return -EAGAIN;
 
@@ -597,7 +602,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
         struct page *newpage = get_new_page(page, private, &result);
         int rcu_locked = 0;
         int charge = 0;
-        struct mem_cgroup *mem;
+        struct mem_cgroup *mem = NULL;
 
         if (!newpage)
                 return -ENOMEM;
@@ -655,7 +660,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
          * free the metadata, so the page can be freed.
          */
         if (!page->mapping) {
-                if (!PageAnon(page) && PagePrivate(page)) {
+                if (!PageAnon(page) && page_has_private(page)) {
                         /*
                          * Go direct to try_to_free_buffers() here because
                          * a) that's what try_to_release_page() would do anyway
@@ -664,13 +669,15 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
                          *    needs to be effective.
                          */
                         try_to_free_buffers(page);
+                        goto rcu_unlock;
                 }
-                goto rcu_unlock;
+                goto skip_unmap;
         }
 
         /* Establish migration ptes or remove ptes */
-        try_to_unmap(page, 1);
+        try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
 
+skip_unmap:
         if (!page_mapped(page))
                 rc = move_to_new_page(newpage, page);
 
@@ -693,6 +700,8 @@ unlock:
                  * restored.
                  */
                 list_del(&page->lru);
+                dec_zone_page_state(page, NR_ISOLATED_ANON +
+                                page_is_file_cache(page));
                 putback_lru_page(page);
         }
 
@@ -737,6 +746,13 @@ int migrate_pages(struct list_head *from,
         struct page *page2;
         int swapwrite = current->flags & PF_SWAPWRITE;
         int rc;
+        unsigned long flags;
+
+        local_irq_save(flags);
+        list_for_each_entry(page, from, lru)
+                __inc_zone_page_state(page, NR_ISOLATED_ANON +
+                                page_is_file_cache(page));
+        local_irq_restore(flags);
 
         if (!swapwrite)
                 current->flags |= PF_SWAPWRITE;
@@ -802,7 +818,7 @@ static struct page *new_page_node(struct page *p, unsigned long private,
 
         *result = &pm->status;
 
-        return alloc_pages_node(pm->node,
+        return alloc_pages_exact_node(pm->node,
                                 GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0);
 }
 
@@ -820,7 +836,6 @@ static int do_move_page_to_node_array(struct mm_struct *mm,
         struct page_to_node *pp;
         LIST_HEAD(pagelist);
 
-        migrate_prep();
         down_read(&mm->mmap_sem);
 
         /*
@@ -907,6 +922,9 @@ static int do_pages_move(struct mm_struct *mm, struct task_struct *task,
         pm = (struct page_to_node *)__get_free_page(GFP_KERNEL);
         if (!pm)
                 goto out;
+
+        migrate_prep();
+
         /*
          * Store a chunk of page_to_node array in a page,
          * but keep the last one as a marker
diff --git a/mm/mlock.c b/mm/mlock.c
index cbe9e0581b75..bd6f0e466f6c 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -31,7 +31,6 @@ int can_do_mlock(void)
 }
 EXPORT_SYMBOL(can_do_mlock);
 
-#ifdef CONFIG_UNEVICTABLE_LRU
 /*
  * Mlocked pages are marked with PageMlocked() flag for efficient testing
  * in vmscan and, possibly, the fault path; and to support semi-accurate
@@ -140,49 +139,36 @@ static void munlock_vma_page(struct page *page)
 }
 
 /**
- * __mlock_vma_pages_range() -  mlock/munlock a range of pages in the vma.
+ * __mlock_vma_pages_range() -  mlock a range of pages in the vma.
  * @vma:   target vma
  * @start: start address
  * @end:   end address
- * @mlock: 0 indicate munlock, otherwise mlock.
  *
- * If @mlock == 0, unlock an mlocked range;
- * else mlock the range of pages.  This takes care of making the pages present ,
- * too.
+ * This takes care of making the pages present too.
  *
  * return 0 on success, negative error code on error.
  *
  * vma->vm_mm->mmap_sem must be held for at least read.
  */
 static long __mlock_vma_pages_range(struct vm_area_struct *vma,
-                                   unsigned long start, unsigned long end,
-                                   int mlock)
+                                    unsigned long start, unsigned long end)
 {
         struct mm_struct *mm = vma->vm_mm;
         unsigned long addr = start;
         struct page *pages[16]; /* 16 gives a reasonable batch */
         int nr_pages = (end - start) / PAGE_SIZE;
         int ret = 0;
-        int gup_flags = 0;
+        int gup_flags;
 
         VM_BUG_ON(start & ~PAGE_MASK);
         VM_BUG_ON(end   & ~PAGE_MASK);
         VM_BUG_ON(start < vma->vm_start);
         VM_BUG_ON(end   > vma->vm_end);
-        VM_BUG_ON((!rwsem_is_locked(&mm->mmap_sem)) &&
-                  (atomic_read(&mm->mm_users) != 0));
-
-        /*
-         * mlock:   don't page populate if vma has PROT_NONE permission.
-         * munlock: always do munlock although the vma has PROT_NONE
-         *          permission, or SIGKILL is pending.
-         */
-        if (!mlock)
-                gup_flags |= GUP_FLAGS_IGNORE_VMA_PERMISSIONS |
-                             GUP_FLAGS_IGNORE_SIGKILL;
+        VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
 
+        gup_flags = FOLL_TOUCH | FOLL_GET;
         if (vma->vm_flags & VM_WRITE)
-                gup_flags |= GUP_FLAGS_WRITE;
+                gup_flags |= FOLL_WRITE;
 
         while (nr_pages > 0) {
                 int i;
@@ -202,51 +188,45 @@ static long __mlock_vma_pages_range(struct vm_area_struct *vma,
                  * This can happen for, e.g., VM_NONLINEAR regions before
                  * a page has been allocated and mapped at a given offset,
                  * or for addresses that map beyond end of a file.
-                 * We'll mlock the the pages if/when they get faulted in.
+                 * We'll mlock the pages if/when they get faulted in.
                  */
                 if (ret < 0)
                         break;
-                if (ret == 0) {
-                        /*
-                         * We know the vma is there, so the only time
-                         * we cannot get a single page should be an
-                         * error (ret < 0) case.
-                         */
-                        WARN_ON(1);
-                        break;
-                }
 
                 lru_add_drain();        /* push cached pages to LRU */
 
                 for (i = 0; i < ret; i++) {
                         struct page *page = pages[i];
 
-                        lock_page(page);
-                        /*
-                         * Because we lock page here and migration is blocked
-                         * by the elevated reference, we need only check for
-                         * page truncation (file-cache only).
-                         */
                         if (page->mapping) {
-                                if (mlock)
+                                /*
+                                 * That preliminary check is mainly to avoid
+                                 * the pointless overhead of lock_page on the
+                                 * ZERO_PAGE: which might bounce very badly if
+                                 * there is contention.  However, we're still
+                                 * dirtying its cacheline with get/put_page:
+                                 * we'll add another __get_user_pages flag to
+                                 * avoid it if that case turns out to matter.
+                                 */
+                                lock_page(page);
+                                /*
+                                 * Because we lock page here and migration is
+                                 * blocked by the elevated reference, we need
+                                 * only check for file-cache page truncation.
+                                 */
+                                if (page->mapping)
                                         mlock_vma_page(page);
-                                else
-                                        munlock_vma_page(page);
+                                unlock_page(page);
                         }
-                        unlock_page(page);
-                        put_page(page);         /* ref from get_user_pages() */
-
-                        /*
-                         * here we assume that get_user_pages() has given us
-                         * a list of virtually contiguous pages.
-                         */
-                        addr += PAGE_SIZE;      /* for next get_user_pages() */
-                        nr_pages--;
+                        put_page(page); /* ref from get_user_pages() */
                 }
+
+                addr += ret * PAGE_SIZE;
+                nr_pages -= ret;
                 ret = 0;
         }
 
-        return ret;     /* count entire vma as locked_vm */
+        return ret;     /* 0 or negative error code */
 }
 
 /*
@@ -261,27 +241,6 @@ static int __mlock_posix_error_return(long retval)
         return retval;
 }
 
-#else /* CONFIG_UNEVICTABLE_LRU */
-
-/*
- * Just make pages present if VM_LOCKED.  No-op if unlocking.
- */
-static long __mlock_vma_pages_range(struct vm_area_struct *vma,
-                                   unsigned long start, unsigned long end,
-                                   int mlock)
-{
-        if (mlock && (vma->vm_flags & VM_LOCKED))
-                return make_pages_present(start, end);
-        return 0;
-}
-
-static inline int __mlock_posix_error_return(long retval)
-{
-        return 0;
-}
-
-#endif /* CONFIG_UNEVICTABLE_LRU */
-
 /**
  * mlock_vma_pages_range() - mlock pages in specified vma range.
  * @vma - the vma containing the specfied address range
@@ -311,7 +270,7 @@ long mlock_vma_pages_range(struct vm_area_struct *vma,
                         is_vm_hugetlb_page(vma) ||
                         vma == get_gate_vma(current))) {
 
-                __mlock_vma_pages_range(vma, start, end, 1);
+                __mlock_vma_pages_range(vma, start, end);
 
                 /* Hide errors from mmap() and other callers */
                 return 0;
@@ -332,7 +291,6 @@ no_mlock:
         return nr_pages;                /* error or pages NOT mlocked */
 }
 
-
 /*
  * munlock_vma_pages_range() - munlock all pages in the vma range.'
  * @vma - vma containing range to be munlock()ed.
@@ -352,10 +310,38 @@ no_mlock:
  * free them.  This will result in freeing mlocked pages.
  */
 void munlock_vma_pages_range(struct vm_area_struct *vma,
-                           unsigned long start, unsigned long end)
+                             unsigned long start, unsigned long end)
 {
+        unsigned long addr;
+
+        lru_add_drain();
         vma->vm_flags &= ~VM_LOCKED;
-        __mlock_vma_pages_range(vma, start, end, 0);
+
+        for (addr = start; addr < end; addr += PAGE_SIZE) {
+                struct page *page;
+                /*
+                 * Although FOLL_DUMP is intended for get_dump_page(),
+                 * it just so happens that its special treatment of the
+                 * ZERO_PAGE (returning an error instead of doing get_page)
+                 * suits munlock very well (and if somehow an abnormal page
+                 * has sneaked into the range, we won't oops here: great).
+                 */
+                page = follow_page(vma, addr, FOLL_GET | FOLL_DUMP);
+                if (page && !IS_ERR(page)) {
+                        lock_page(page);
+                        /*
+                         * Like in __mlock_vma_pages_range(),
+                         * because we lock page here and migration is
+                         * blocked by the elevated reference, we need
+                         * only check for file-cache page truncation.
+                         */
+                        if (page->mapping)
+                                munlock_vma_page(page);
+                        unlock_page(page);
+                        put_page(page);
+                }
+                cond_resched();
+        }
 }
 
 /*
@@ -422,18 +408,14 @@ success:
          * It's okay if try_to_unmap_one unmaps a page just after we
          * set VM_LOCKED, __mlock_vma_pages_range will bring it back.
          */
-        vma->vm_flags = newflags;
 
         if (lock) {
-                ret = __mlock_vma_pages_range(vma, start, end, 1);
-
-                if (ret > 0) {
-                        mm->locked_vm -= ret;
-                        ret = 0;
-                } else
-                        ret = __mlock_posix_error_return(ret); /* translate if needed */
+                vma->vm_flags = newflags;
+                ret = __mlock_vma_pages_range(vma, start, end);
+                if (ret < 0)
+                        ret = __mlock_posix_error_return(ret);
         } else {
-                __mlock_vma_pages_range(vma, start, end, 0);
+                munlock_vma_pages_range(vma, start, end);
         }
 
 out:
@@ -629,52 +611,43 @@ void user_shm_unlock(size_t size, struct user_struct *user)
         free_uid(user);
 }
 
-void *alloc_locked_buffer(size_t size)
+int account_locked_memory(struct mm_struct *mm, struct rlimit *rlim,
+                          size_t size)
 {
-        unsigned long rlim, vm, pgsz;
-        void *buffer = NULL;
+        unsigned long lim, vm, pgsz;
+        int error = -ENOMEM;
 
         pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT;
 
-        down_write(&current->mm->mmap_sem);
+        down_write(&mm->mmap_sem);
 
-        rlim = current->signal->rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT;
-        vm   = current->mm->total_vm + pgsz;
-        if (rlim < vm)
+        lim = rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT;
+        vm   = mm->total_vm + pgsz;
+        if (lim < vm)
                 goto out;
 
-        rlim = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT;
-        vm   = current->mm->locked_vm + pgsz;
-        if (rlim < vm)
+        lim = rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT;
+        vm   = mm->locked_vm + pgsz;
+        if (lim < vm)
                 goto out;
 
-        buffer = kzalloc(size, GFP_KERNEL);
-        if (!buffer)
-                goto out;
-
-        current->mm->total_vm  += pgsz;
-        current->mm->locked_vm += pgsz;
+        mm->total_vm  += pgsz;
+        mm->locked_vm += pgsz;
 
+        error = 0;
  out:
-        up_write(&current->mm->mmap_sem);
-        return buffer;
+        up_write(&mm->mmap_sem);
+        return error;
 }
 
-void release_locked_buffer(void *buffer, size_t size)
+void refund_locked_memory(struct mm_struct *mm, size_t size)
 {
         unsigned long pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT;
 
-        down_write(&current->mm->mmap_sem);
-
-        current->mm->total_vm  -= pgsz;
-        current->mm->locked_vm -= pgsz;
-
-        up_write(&current->mm->mmap_sem);
-}
+        down_write(&mm->mmap_sem);
 
-void free_locked_buffer(void *buffer, size_t size)
-{
-        release_locked_buffer(buffer, size);
+        mm->total_vm  -= pgsz;
+        mm->locked_vm -= pgsz;
 
-        kfree(buffer);
+        up_write(&mm->mmap_sem);
 }
diff --git a/mm/mmap.c b/mm/mmap.c
index 1abb9185a686..292ddc3cef9c 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -20,7 +20,6 @@
 #include <linux/fs.h>
 #include <linux/personality.h>
 #include <linux/security.h>
-#include <linux/ima.h>
 #include <linux/hugetlb.h>
 #include <linux/profile.h>
 #include <linux/module.h>
@@ -28,6 +27,7 @@
 #include <linux/mempolicy.h>
 #include <linux/rmap.h>
 #include <linux/mmu_notifier.h>
+#include <linux/perf_event.h>
 
 #include <asm/uaccess.h>
 #include <asm/cacheflush.h>
@@ -85,7 +85,7 @@ EXPORT_SYMBOL(vm_get_page_prot);
 int sysctl_overcommit_memory = OVERCOMMIT_GUESS;  /* heuristic overcommit */
 int sysctl_overcommit_ratio = 50;       /* default is 50% */
 int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
-atomic_long_t vm_committed_space = ATOMIC_LONG_INIT(0);
+struct percpu_counter vm_committed_as;
 
 /*
  * Check that a process has enough memory to allocate a new virtual
@@ -179,11 +179,7 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
         if (mm)
                 allowed -= mm->total_vm / 32;
 
-        /*
-         * cast `allowed' as a signed long because vm_committed_space
-         * sometimes has a negative value
-         */
-        if (atomic_long_read(&vm_committed_space) < (long)allowed)
+        if (percpu_counter_read_positive(&vm_committed_as) < allowed)
                 return 0;
 error:
         vm_unacct_memory(pages);
@@ -573,9 +569,9 @@ again:			remove_next = 1 + (end > next->vm_end);
 
         /*
          * When changing only vma->vm_end, we don't really need
-         * anon_vma lock: but is that case worth optimizing out?
+         * anon_vma lock.
          */
-        if (vma->anon_vma)
+        if (vma->anon_vma && (insert || importer || start != vma->vm_start))
                 anon_vma = vma->anon_vma;
         if (anon_vma) {
                 spin_lock(&anon_vma->lock);
@@ -659,9 +655,6 @@ again:			remove_next = 1 + (end > next->vm_end);
         validate_mm(mm);
 }
 
-/* Flags that can be inherited from an existing mapping when merging */
-#define VM_MERGEABLE_FLAGS (VM_CAN_NONLINEAR)
-
 /*
  * If the vma has a ->close operation then the driver probably needs to release
  * per-vma resources, so we don't attempt to merge those.
@@ -669,7 +662,8 @@ again:			remove_next = 1 + (end > next->vm_end);
 static inline int is_mergeable_vma(struct vm_area_struct *vma,
                         struct file *file, unsigned long vm_flags)
 {
-        if ((vma->vm_flags ^ vm_flags) & ~VM_MERGEABLE_FLAGS)
+        /* VM_CAN_NONLINEAR may get set later by f_op->mmap() */
+        if ((vma->vm_flags ^ vm_flags) & ~VM_CAN_NONLINEAR)
                 return 0;
         if (vma->vm_file != file)
                 return 0;
@@ -908,7 +902,7 @@ void vm_stat_account(struct mm_struct *mm, unsigned long flags,
 #endif /* CONFIG_PROC_FS */
 
 /*
- * The caller must hold down_write(current->mm->mmap_sem).
+ * The caller must hold down_write(&current->mm->mmap_sem).
  */
 
 unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
@@ -954,6 +948,24 @@ unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
         if (mm->map_count > sysctl_max_map_count)
                 return -ENOMEM;
 
+        if (flags & MAP_HUGETLB) {
+                struct user_struct *user = NULL;
+                if (file)
+                        return -EINVAL;
+
+                /*
+                 * VM_NORESERVE is used because the reservations will be
+                 * taken when vm_ops->mmap() is called
+                 * A dummy user value is used because we are not locking
+                 * memory so no accounting is necessary
+                 */
+                len = ALIGN(len, huge_page_size(&default_hstate));
+                file = hugetlb_file_setup(HUGETLB_ANON_FILE, len, VM_NORESERVE,
+                                                &user, HUGETLB_ANONHUGE_INODE);
+                if (IS_ERR(file))
+                        return PTR_ERR(file);
+        }
+
         /* Obtain the address to map to. we verify (or select) it and ensure
          * that it represents a valid section of the address space.
          */
@@ -968,11 +980,9 @@ unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
         vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) |
                         mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
 
-        if (flags & MAP_LOCKED) {
+        if (flags & MAP_LOCKED)
                 if (!can_do_mlock())
                         return -EPERM;
-                vm_flags |= VM_LOCKED;
-        }
 
         /* mlock MCL_FUTURE? */
         if (vm_flags & VM_LOCKED) {
@@ -1050,9 +1060,6 @@ unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
         error = security_file_mmap(file, reqprot, prot, flags, addr, 0);
         if (error)
                 return error;
-        error = ima_file_mmap(file, prot);
-        if (error)
-                return error;
 
         return mmap_region(file, addr, len, flags, vm_flags, pgoff);
 }
@@ -1198,21 +1205,21 @@ munmap_back:
                         goto unmap_and_free_vma;
                 if (vm_flags & VM_EXECUTABLE)
                         added_exe_file_vma(mm);
+
+                /* Can addr have changed??
+                 *
+                 * Answer: Yes, several device drivers can do it in their
+                 *         f_op->mmap method. -DaveM
+                 */
+                addr = vma->vm_start;
+                pgoff = vma->vm_pgoff;
+                vm_flags = vma->vm_flags;
         } else if (vm_flags & VM_SHARED) {
                 error = shmem_zero_setup(vma);
                 if (error)
                         goto free_vma;
         }
 
-        /* Can addr have changed??
-         *
-         * Answer: Yes, several device drivers can do it in their
-         *         f_op->mmap method. -DaveM
-         */
-        addr = vma->vm_start;
-        pgoff = vma->vm_pgoff;
-        vm_flags = vma->vm_flags;
-
         if (vma_wants_writenotify(vma))
                 vma->vm_page_prot = vm_get_page_prot(vm_flags & ~VM_SHARED);
 
@@ -1223,6 +1230,8 @@ munmap_back:
         if (correct_wcount)
                 atomic_inc(&inode->i_writecount);
 out:
+        perf_event_mmap(vma);
+
         mm->total_vm += len >> PAGE_SHIFT;
         vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
         if (vm_flags & VM_LOCKED) {
@@ -1575,7 +1584,7 @@ static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, uns
          * Overcommit..  This must be the final test, as it will
          * update security statistics.
          */
-        if (security_vm_enough_memory(grow))
+        if (security_vm_enough_memory_mm(mm, grow))
                 return -ENOMEM;
 
         /* Ok, everything looks good - let it rip */
@@ -2112,6 +2121,7 @@ void exit_mmap(struct mm_struct *mm)
         /* Use -1 here to ensure all VMAs in the mm are unmapped */
         end = unmap_vmas(&tlb, vma, 0, -1, &nr_accounted, NULL);
         vm_unacct_memory(nr_accounted);
+
         free_pgtables(tlb, vma, FIRST_USER_ADDRESS, 0);
         tlb_finish_mmu(tlb, 0, end);
 
@@ -2268,7 +2278,7 @@ static void special_mapping_close(struct vm_area_struct *vma)
 {
 }
 
-static struct vm_operations_struct special_mapping_vmops = {
+static const struct vm_operations_struct special_mapping_vmops = {
         .close = special_mapping_close,
         .fault = special_mapping_fault,
 };
@@ -2309,6 +2319,8 @@ int install_special_mapping(struct mm_struct *mm,
 
         mm->total_vm += len >> PAGE_SHIFT;
 
+        perf_event_mmap(vma);
+
         return 0;
 }
 
@@ -2481,7 +2493,8 @@ void mm_drop_all_locks(struct mm_struct *mm)
  */
 void __init mmap_init(void)
 {
-        vm_area_cachep = kmem_cache_create("vm_area_struct",
-                        sizeof(struct vm_area_struct), 0,
-                        SLAB_PANIC, NULL);
+        int ret;
+
+        ret = percpu_counter_init(&vm_committed_as, 0);
+        VM_BUG_ON(ret);
 }
diff --git a/mm/mmu_context.c b/mm/mmu_context.c
new file mode 100644
index 000000000000..ded9081f4021
--- /dev/null
+++ b/mm/mmu_context.c
@@ -0,0 +1,58 @@
+/* Copyright (C) 2009 Red Hat, Inc.
+ *
+ * See ../COPYING for licensing terms.
+ */
+
+#include <linux/mm.h>
+#include <linux/mmu_context.h>
+#include <linux/sched.h>
+
+#include <asm/mmu_context.h>
+
+/*
+ * use_mm
+ *      Makes the calling kernel thread take on the specified
+ *      mm context.
+ *      Called by the retry thread execute retries within the
+ *      iocb issuer's mm context, so that copy_from/to_user
+ *      operations work seamlessly for aio.
+ *      (Note: this routine is intended to be called only
+ *      from a kernel thread context)
+ */
+void use_mm(struct mm_struct *mm)
+{
+        struct mm_struct *active_mm;
+        struct task_struct *tsk = current;
+
+        task_lock(tsk);
+        active_mm = tsk->active_mm;
+        if (active_mm != mm) {
+                atomic_inc(&mm->mm_count);
+                tsk->active_mm = mm;
+        }
+        tsk->mm = mm;
+        switch_mm(active_mm, mm, tsk);
+        task_unlock(tsk);
+
+        if (active_mm != mm)
+                mmdrop(active_mm);
+}
+
+/*
+ * unuse_mm
+ *      Reverses the effect of use_mm, i.e. releases the
+ *      specified mm context which was earlier taken on
+ *      by the calling kernel thread
+ *      (Note: this routine is intended to be called only
+ *      from a kernel thread context)
+ */
+void unuse_mm(struct mm_struct *mm)
+{
+        struct task_struct *tsk = current;
+
+        task_lock(tsk);
+        tsk->mm = NULL;
+        /* active_mm is still 'mm' */
+        enter_lazy_tlb(mm, tsk);
+        task_unlock(tsk);
+}
diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
index 5f4ef0250bee..7e33f2cb3c77 100644
--- a/mm/mmu_notifier.c
+++ b/mm/mmu_notifier.c
@@ -99,6 +99,26 @@ int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
         return young;
 }
 
+void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,
+                               pte_t pte)
+{
+        struct mmu_notifier *mn;
+        struct hlist_node *n;
+
+        rcu_read_lock();
+        hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
+                if (mn->ops->change_pte)
+                        mn->ops->change_pte(mn, mm, address, pte);
+                /*
+                 * Some drivers don't have change_pte,
+                 * so we must call invalidate_page in that case.
+                 */
+                else if (mn->ops->invalidate_page)
+                        mn->ops->invalidate_page(mn, mm, address);
+        }
+        rcu_read_unlock();
+}
+
 void __mmu_notifier_invalidate_page(struct mm_struct *mm,
                                           unsigned long address)
 {
diff --git a/mm/mmzone.c b/mm/mmzone.c
index 16ce8b955dcf..f5b7d1760213 100644
--- a/mm/mmzone.c
+++ b/mm/mmzone.c
@@ -6,6 +6,7 @@
 
 
 #include <linux/stddef.h>
+#include <linux/mm.h>
 #include <linux/mmzone.h>
 #include <linux/module.h>
 
@@ -72,3 +73,17 @@ struct zoneref *next_zones_zonelist(struct zoneref *z,
         *zone = zonelist_zone(z);
         return z;
 }
+
+#ifdef CONFIG_ARCH_HAS_HOLES_MEMORYMODEL
+int memmap_valid_within(unsigned long pfn,
+                                        struct page *page, struct zone *zone)
+{
+        if (page_to_pfn(page) != pfn)
+                return 0;
+
+        if (page_zone(page) != zone)
+                return 0;
+
+        return 1;
+}
+#endif /* CONFIG_ARCH_HAS_HOLES_MEMORYMODEL */
diff --git a/mm/mprotect.c b/mm/mprotect.c
index 258197b76fb4..8bc969d8112d 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -23,6 +23,7 @@
 #include <linux/swapops.h>
 #include <linux/mmu_notifier.h>
 #include <linux/migrate.h>
+#include <linux/perf_event.h>
 #include <asm/uaccess.h>
 #include <asm/pgtable.h>
 #include <asm/cacheflush.h>
@@ -299,6 +300,7 @@ SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len,
                 error = mprotect_fixup(vma, &prev, nstart, tmp, newflags);
                 if (error)
                         goto out;
+                perf_event_mmap(vma);
                 nstart = tmp;
 
                 if (nstart < prev->vm_end)
diff --git a/mm/mremap.c b/mm/mremap.c
index a39b7b91be46..97bff2547719 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -11,6 +11,7 @@
 #include <linux/hugetlb.h>
 #include <linux/slab.h>
 #include <linux/shm.h>
+#include <linux/ksm.h>
 #include <linux/mman.h>
 #include <linux/swap.h>
 #include <linux/capability.h>
@@ -85,8 +86,8 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
         if (vma->vm_file) {
                 /*
                  * Subtle point from Rajesh Venkatasubramanian: before
-                 * moving file-based ptes, we must lock vmtruncate out,
-                 * since it might clean the dst vma before the src vma,
+                 * moving file-based ptes, we must lock truncate_pagecache
+                 * out, since it might clean the dst vma before the src vma,
                  * and we propagate stale pages into the dst afterward.
                  */
                 mapping = vma->vm_file->f_mapping;
@@ -174,6 +175,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
         unsigned long excess = 0;
         unsigned long hiwater_vm;
         int split = 0;
+        int err;
 
         /*
          * We'd prefer to avoid failure later on in do_munmap:
@@ -182,6 +184,18 @@ static unsigned long move_vma(struct vm_area_struct *vma,
         if (mm->map_count >= sysctl_max_map_count - 3)
                 return -ENOMEM;
 
+        /*
+         * Advise KSM to break any KSM pages in the area to be moved:
+         * it would be confusing if they were to turn up at the new
+         * location, where they happen to coincide with different KSM
+         * pages recently unmapped.  But leave vma->vm_flags as it was,
+         * so KSM can come around to merge on vma and new_vma afterwards.
+         */
+        err = ksm_madvise(vma, old_addr, old_addr + old_len,
+                                                MADV_UNMERGEABLE, &vm_flags);
+        if (err)
+                return err;
+
         new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT);
         new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff);
         if (!new_vma)
diff --git a/mm/nommu.c b/mm/nommu.c
index 2fcf47d449b4..9876fa0c3ad3 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -33,6 +33,7 @@
 #include <asm/uaccess.h>
 #include <asm/tlb.h>
 #include <asm/tlbflush.h>
+#include <asm/mmu_context.h>
 #include "internal.h"
 
 static inline __attribute__((format(printf, 1, 2)))
@@ -56,20 +57,19 @@ void no_printk(const char *fmt, ...)
         no_printk(KERN_DEBUG FMT"\n", ##__VA_ARGS__)
 #endif
 
-#include "internal.h"
-
 void *high_memory;
 struct page *mem_map;
 unsigned long max_mapnr;
 unsigned long num_physpages;
-atomic_long_t vm_committed_space = ATOMIC_LONG_INIT(0);
+unsigned long highest_memmap_pfn;
+struct percpu_counter vm_committed_as;
 int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */
 int sysctl_overcommit_ratio = 50; /* default is 50% */
 int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT;
-int sysctl_nr_trim_pages = 1; /* page trimming behaviour */
+int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;
 int heap_stack_gap = 0;
 
-atomic_t mmap_pages_allocated;
+atomic_long_t mmap_pages_allocated;
 
 EXPORT_SYMBOL(mem_map);
 EXPORT_SYMBOL(num_physpages);
@@ -79,50 +79,10 @@ static struct kmem_cache *vm_region_jar;
 struct rb_root nommu_region_tree = RB_ROOT;
 DECLARE_RWSEM(nommu_region_sem);
 
-struct vm_operations_struct generic_file_vm_ops = {
+const struct vm_operations_struct generic_file_vm_ops = {
 };
 
 /*
- * Handle all mappings that got truncated by a "truncate()"
- * system call.
- *
- * NOTE! We have to be ready to update the memory sharing
- * between the file and the memory map for a potential last
- * incomplete page.  Ugly, but necessary.
- */
-int vmtruncate(struct inode *inode, loff_t offset)
-{
-        struct address_space *mapping = inode->i_mapping;
-        unsigned long limit;
-
-        if (inode->i_size < offset)
-                goto do_expand;
-        i_size_write(inode, offset);
-
-        truncate_inode_pages(mapping, offset);
-        goto out_truncate;
-
-do_expand:
-        limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
-        if (limit != RLIM_INFINITY && offset > limit)
-                goto out_sig;
-        if (offset > inode->i_sb->s_maxbytes)
-                goto out;
-        i_size_write(inode, offset);
-
-out_truncate:
-        if (inode->i_op->truncate)
-                inode->i_op->truncate(inode);
-        return 0;
-out_sig:
-        send_sig(SIGXFSZ, current, 0);
-out:
-        return -EFBIG;
-}
-
-EXPORT_SYMBOL(vmtruncate);
-
-/*
  * Return the total memory allocated for this pointer, not
  * just what the caller asked for.
  *
@@ -170,30 +130,29 @@ unsigned int kobjsize(const void *objp)
 }
 
 int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
-                     unsigned long start, int len, int flags,
-                struct page **pages, struct vm_area_struct **vmas)
+                     unsigned long start, int nr_pages, unsigned int foll_flags,
+                     struct page **pages, struct vm_area_struct **vmas)
 {
         struct vm_area_struct *vma;
         unsigned long vm_flags;
         int i;
-        int write = !!(flags & GUP_FLAGS_WRITE);
-        int force = !!(flags & GUP_FLAGS_FORCE);
-        int ignore = !!(flags & GUP_FLAGS_IGNORE_VMA_PERMISSIONS);
 
         /* calculate required read or write permissions.
-         * - if 'force' is set, we only require the "MAY" flags.
+         * If FOLL_FORCE is set, we only require the "MAY" flags.
          */
-        vm_flags  = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
-        vm_flags &= force ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
+        vm_flags  = (foll_flags & FOLL_WRITE) ?
+                        (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
+        vm_flags &= (foll_flags & FOLL_FORCE) ?
+                        (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
 
-        for (i = 0; i < len; i++) {
+        for (i = 0; i < nr_pages; i++) {
                 vma = find_vma(mm, start);
                 if (!vma)
                         goto finish_or_fault;
 
                 /* protect what we can, including chardevs */
-                if (vma->vm_flags & (VM_IO | VM_PFNMAP) ||
-                    (!ignore && !(vm_flags & vma->vm_flags)))
+                if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
+                    !(vm_flags & vma->vm_flags))
                         goto finish_or_fault;
 
                 if (pages) {
@@ -212,7 +171,6 @@ finish_or_fault:
         return i ? : -EFAULT;
 }
 
-
 /*
  * get a list of pages in an address range belonging to the specified process
  * and indicate the VMA that covers each page
@@ -221,22 +179,41 @@ finish_or_fault:
  * - don't permit access to VMAs that don't support it, such as I/O mappings
  */
 int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
-        unsigned long start, int len, int write, int force,
+        unsigned long start, int nr_pages, int write, int force,
         struct page **pages, struct vm_area_struct **vmas)
 {
         int flags = 0;
 
         if (write)
-                flags |= GUP_FLAGS_WRITE;
+                flags |= FOLL_WRITE;
         if (force)
-                flags |= GUP_FLAGS_FORCE;
+                flags |= FOLL_FORCE;
 
-        return __get_user_pages(tsk, mm,
-                                start, len, flags,
-                                pages, vmas);
+        return __get_user_pages(tsk, mm, start, nr_pages, flags, pages, vmas);
 }
 EXPORT_SYMBOL(get_user_pages);
 
+/**
+ * follow_pfn - look up PFN at a user virtual address
+ * @vma: memory mapping
+ * @address: user virtual address
+ * @pfn: location to store found PFN
+ *
+ * Only IO mappings and raw PFN mappings are allowed.
+ *
+ * Returns zero and the pfn at @pfn on success, -ve otherwise.
+ */
+int follow_pfn(struct vm_area_struct *vma, unsigned long address,
+        unsigned long *pfn)
+{
+        if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
+                return -EINVAL;
+
+        *pfn = address >> PAGE_SHIFT;
+        return 0;
+}
+EXPORT_SYMBOL(follow_pfn);
+
 DEFINE_RWLOCK(vmlist_lock);
 struct vm_struct *vmlist;
 
@@ -463,12 +440,11 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
  */
 void __init mmap_init(void)
 {
-        vm_region_jar = kmem_cache_create("vm_region_jar",
-                                          sizeof(struct vm_region), 0,
-                                          SLAB_PANIC, NULL);
-        vm_area_cachep = kmem_cache_create("vm_area_struct",
-                                           sizeof(struct vm_area_struct), 0,
-                                           SLAB_PANIC, NULL);
+        int ret;
+
+        ret = percpu_counter_init(&vm_committed_as, 0);
+        VM_BUG_ON(ret);
+        vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC);
 }
 
 /*
@@ -486,27 +462,24 @@ static noinline void validate_nommu_regions(void)
                 return;
 
         last = rb_entry(lastp, struct vm_region, vm_rb);
-        if (unlikely(last->vm_end <= last->vm_start))
-                BUG();
-        if (unlikely(last->vm_top < last->vm_end))
-                BUG();
+        BUG_ON(unlikely(last->vm_end <= last->vm_start));
+        BUG_ON(unlikely(last->vm_top < last->vm_end));
 
         while ((p = rb_next(lastp))) {
                 region = rb_entry(p, struct vm_region, vm_rb);
                 last = rb_entry(lastp, struct vm_region, vm_rb);
 
-                if (unlikely(region->vm_end <= region->vm_start))
-                        BUG();
-                if (unlikely(region->vm_top < region->vm_end))
-                        BUG();
-                if (unlikely(region->vm_start < last->vm_top))
-                        BUG();
+                BUG_ON(unlikely(region->vm_end <= region->vm_start));
+                BUG_ON(unlikely(region->vm_top < region->vm_end));
+                BUG_ON(unlikely(region->vm_start < last->vm_top));
 
                 lastp = p;
         }
 }
 #else
-#define validate_nommu_regions() do {} while(0)
+static void validate_nommu_regions(void)
+{
+}
 #endif
 
 /*
@@ -519,8 +492,6 @@ static void add_nommu_region(struct vm_region *region)
 
         validate_nommu_regions();
 
-        BUG_ON(region->vm_start & ~PAGE_MASK);
-
         parent = NULL;
         p = &nommu_region_tree.rb_node;
         while (*p) {
@@ -563,16 +534,17 @@ static void free_page_series(unsigned long from, unsigned long to)
                 struct page *page = virt_to_page(from);
 
                 kdebug("- free %lx", from);
-                atomic_dec(&mmap_pages_allocated);
+                atomic_long_dec(&mmap_pages_allocated);
                 if (page_count(page) != 1)
-                        kdebug("free page %p [%d]", page, page_count(page));
+                        kdebug("free page %p: refcount not one: %d",
+                               page, page_count(page));
                 put_page(page);
         }
 }
 
 /*
  * release a reference to a region
- * - the caller must hold the region semaphore, which this releases
+ * - the caller must hold the region semaphore for writing, which this releases
  * - the region may not have been added to the tree yet, in which case vm_top
  *   will equal vm_start
  */
@@ -613,6 +585,22 @@ static void put_nommu_region(struct vm_region *region)
 }
 
 /*
+ * update protection on a vma
+ */
+static void protect_vma(struct vm_area_struct *vma, unsigned long flags)
+{
+#ifdef CONFIG_MPU
+        struct mm_struct *mm = vma->vm_mm;
+        long start = vma->vm_start & PAGE_MASK;
+        while (start < vma->vm_end) {
+                protect_page(mm, start, flags);
+                start += PAGE_SIZE;
+        }
+        update_protections(mm);
+#endif
+}
+
+/*
  * add a VMA into a process's mm_struct in the appropriate place in the list
  * and tree and add to the address space's page tree also if not an anonymous
  * page
@@ -631,6 +619,8 @@ static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
         mm->map_count++;
         vma->vm_mm = mm;
 
+        protect_vma(vma, vma->vm_flags);
+
         /* add the VMA to the mapping */
         if (vma->vm_file) {
                 mapping = vma->vm_file->f_mapping;
@@ -693,6 +683,8 @@ static void delete_vma_from_mm(struct vm_area_struct *vma)
 
         kenter("%p", vma);
 
+        protect_vma(vma, 0);
+
         mm->map_count--;
         if (mm->mmap_cache == vma)
                 mm->mmap_cache = NULL;
@@ -834,7 +826,7 @@ static int validate_mmap_request(struct file *file,
         int ret;
 
         /* do the simple checks first */
-        if (flags & MAP_FIXED || addr) {
+        if (flags & MAP_FIXED) {
                 printk(KERN_DEBUG
                        "%d: Can't do fixed-address/overlay mmap of RAM\n",
                        current->pid);
@@ -905,6 +897,10 @@ static int validate_mmap_request(struct file *file,
                 if (!file->f_op->read)
                         capabilities &= ~BDI_CAP_MAP_COPY;
 
+                /* The file shall have been opened with read permission. */
+                if (!(file->f_mode & FMODE_READ))
+                        return -EACCES;
+
                 if (flags & MAP_SHARED) {
                         /* do checks for writing, appending and locking */
                         if ((prot & PROT_WRITE) &&
@@ -1038,7 +1034,7 @@ static int do_mmap_shared_file(struct vm_area_struct *vma)
         ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
         if (ret == 0) {
                 vma->vm_region->vm_top = vma->vm_region->vm_end;
-                return ret;
+                return 0;
         }
         if (ret != -ENOSYS)
                 return ret;
@@ -1055,7 +1051,8 @@ static int do_mmap_shared_file(struct vm_area_struct *vma)
  */
 static int do_mmap_private(struct vm_area_struct *vma,
                            struct vm_region *region,
-                           unsigned long len)
+                           unsigned long len,
+                           unsigned long capabilities)
 {
         struct page *pages;
         unsigned long total, point, n, rlen;
@@ -1066,13 +1063,13 @@ static int do_mmap_private(struct vm_area_struct *vma,
          * shared mappings on devices or memory
          * - VM_MAYSHARE will be set if it may attempt to share
          */
-        if (vma->vm_file) {
+        if (capabilities & BDI_CAP_MAP_DIRECT) {
                 ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
                 if (ret == 0) {
                         /* shouldn't return success if we're not sharing */
                         BUG_ON(!(vma->vm_flags & VM_MAYSHARE));
                         vma->vm_region->vm_top = vma->vm_region->vm_end;
-                        return ret;
+                        return 0;
                 }
                 if (ret != -ENOSYS)
                         return ret;
@@ -1096,7 +1093,7 @@ static int do_mmap_private(struct vm_area_struct *vma,
                 goto enomem;
 
         total = 1 << order;
-        atomic_add(total, &mmap_pages_allocated);
+        atomic_long_add(total, &mmap_pages_allocated);
 
         point = rlen >> PAGE_SHIFT;
 
@@ -1107,7 +1104,7 @@ static int do_mmap_private(struct vm_area_struct *vma,
                         order = ilog2(total - point);
                         n = 1 << order;
                         kdebug("shave %lu/%lu @%lu", n, total - point, total);
-                        atomic_sub(n, &mmap_pages_allocated);
+                        atomic_long_sub(n, &mmap_pages_allocated);
                         total -= n;
                         set_page_refcounted(pages + total);
                         __free_pages(pages + total, order);
@@ -1185,9 +1182,6 @@ unsigned long do_mmap_pgoff(struct file *file,
 
         kenter(",%lx,%lx,%lx,%lx,%lx", addr, len, prot, flags, pgoff);
 
-        if (!(flags & MAP_FIXED))
-                addr = round_hint_to_min(addr);
-
         /* decide whether we should attempt the mapping, and if so what sort of
          * mapping */
         ret = validate_mmap_request(file, addr, len, prot, flags, pgoff,
@@ -1197,6 +1191,9 @@ unsigned long do_mmap_pgoff(struct file *file,
                 return ret;
         }
 
+        /* we ignore the address hint */
+        addr = 0;
+
         /* we've determined that we can make the mapping, now translate what we
          * now know into VMA flags */
         vm_flags = determine_vm_flags(file, prot, flags, capabilities);
@@ -1310,7 +1307,7 @@ unsigned long do_mmap_pgoff(struct file *file,
                  * - this is the hook for quasi-memory character devices to
                  *   tell us the location of a shared mapping
                  */
-                if (file && file->f_op->get_unmapped_area) {
+                if (capabilities & BDI_CAP_MAP_DIRECT) {
                         addr = file->f_op->get_unmapped_area(file, addr, len,
                                                              pgoff, flags);
                         if (IS_ERR((void *) addr)) {
@@ -1335,14 +1332,15 @@ unsigned long do_mmap_pgoff(struct file *file,
 
         vma->vm_region = region;
 
-        /* set up the mapping */
+        /* set up the mapping
+         * - the region is filled in if BDI_CAP_MAP_DIRECT is still set
+         */
         if (file && vma->vm_flags & VM_SHARED)
                 ret = do_mmap_shared_file(vma);
         else
-                ret = do_mmap_private(vma, region, len);
+                ret = do_mmap_private(vma, region, len, capabilities);
         if (ret < 0)
-                goto error_put_region;
-
+                goto error_just_free;
         add_nommu_region(region);
 
         /* okay... we have a mapping; now we have to register it */
@@ -1361,25 +1359,14 @@ share:
         kleave(" = %lx", result);
         return result;
 
-error_put_region:
-        __put_nommu_region(region);
-        if (vma) {
-                if (vma->vm_file) {
-                        fput(vma->vm_file);
-                        if (vma->vm_flags & VM_EXECUTABLE)
-                                removed_exe_file_vma(vma->vm_mm);
-                }
-                kmem_cache_free(vm_area_cachep, vma);
-        }
-        kleave(" = %d [pr]", ret);
-        return ret;
-
 error_just_free:
         up_write(&nommu_region_sem);
 error:
-        fput(region->vm_file);
+        if (region->vm_file)
+                fput(region->vm_file);
         kmem_cache_free(vm_region_jar, region);
-        fput(vma->vm_file);
+        if (vma->vm_file)
+                fput(vma->vm_file);
         if (vma->vm_flags & VM_EXECUTABLE)
                 removed_exe_file_vma(vma->vm_mm);
         kmem_cache_free(vm_area_cachep, vma);
@@ -1536,10 +1523,15 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
         /* find the first potentially overlapping VMA */
         vma = find_vma(mm, start);
         if (!vma) {
-                printk(KERN_WARNING
-                       "munmap of memory not mmapped by process %d (%s):"
-                       " 0x%lx-0x%lx\n",
-                       current->pid, current->comm, start, start + len - 1);
+                static int limit = 0;
+                if (limit < 5) {
+                        printk(KERN_WARNING
+                               "munmap of memory not mmapped by process %d"
+                               " (%s): 0x%lx-0x%lx\n",
+                               current->pid, current->comm,
+                               start, start + len - 1);
+                        limit++;
+                }
                 return -EINVAL;
         }
 
@@ -1849,12 +1841,9 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
         if (mm)
                 allowed -= mm->total_vm / 32;
 
-        /*
-         * cast `allowed' as a signed long because vm_committed_space
-         * sometimes has a negative value
-         */
-        if (atomic_long_read(&vm_committed_space) < (long)allowed)
+        if (percpu_counter_read_positive(&vm_committed_as) < allowed)
                 return 0;
+
 error:
         vm_unacct_memory(pages);
 
diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index d3b9bac085b5..ea2147dabba6 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -34,6 +34,23 @@ int sysctl_oom_dump_tasks;
 static DEFINE_SPINLOCK(zone_scan_lock);
 /* #define DEBUG */
 
+/*
+ * Is all threads of the target process nodes overlap ours?
+ */
+static int has_intersects_mems_allowed(struct task_struct *tsk)
+{
+        struct task_struct *t;
+
+        t = tsk;
+        do {
+                if (cpuset_mems_allowed_intersects(current, t))
+                        return 1;
+                t = next_thread(t);
+        } while (t != tsk);
+
+        return 0;
+}
+
 /**
  * badness - calculate a numeric value for how bad this task has been
  * @p: task struct of which task we should calculate
@@ -58,6 +75,13 @@ unsigned long badness(struct task_struct *p, unsigned long uptime)
         unsigned long points, cpu_time, run_time;
         struct mm_struct *mm;
         struct task_struct *child;
+        int oom_adj = p->signal->oom_adj;
+        struct task_cputime task_time;
+        unsigned long utime;
+        unsigned long stime;
+
+        if (oom_adj == OOM_DISABLE)
+                return 0;
 
         task_lock(p);
         mm = p->mm;
@@ -79,7 +103,7 @@ unsigned long badness(struct task_struct *p, unsigned long uptime)
         /*
          * swapoff can easily use up all memory, so kill those first.
          */
-        if (p->flags & PF_SWAPOFF)
+        if (p->flags & PF_OOM_ORIGIN)
                 return ULONG_MAX;
 
         /*
@@ -102,8 +126,11 @@ unsigned long badness(struct task_struct *p, unsigned long uptime)
          * of seconds. There is no particular reason for this other than
          * that it turned out to work very well in practice.
          */
-        cpu_time = (cputime_to_jiffies(p->utime) + cputime_to_jiffies(p->stime))
-                >> (SHIFT_HZ + 3);
+        thread_group_cputime(p, &task_time);
+        utime = cputime_to_jiffies(task_time.utime);
+        stime = cputime_to_jiffies(task_time.stime);
+        cpu_time = (utime + stime) >> (SHIFT_HZ + 3);
+
 
         if (uptime >= p->start_time.tv_sec)
                 run_time = (uptime - p->start_time.tv_sec) >> 10;
@@ -144,19 +171,19 @@ unsigned long badness(struct task_struct *p, unsigned long uptime)
          * because p may have allocated or otherwise mapped memory on
          * this node before. However it will be less likely.
          */
-        if (!cpuset_mems_allowed_intersects(current, p))
+        if (!has_intersects_mems_allowed(p))
                 points /= 8;
 
         /*
-         * Adjust the score by oomkilladj.
+         * Adjust the score by oom_adj.
          */
-        if (p->oomkilladj) {
-                if (p->oomkilladj > 0) {
+        if (oom_adj) {
+                if (oom_adj > 0) {
                         if (!points)
                                 points = 1;
-                        points <<= p->oomkilladj;
+                        points <<= oom_adj;
                 } else
-                        points >>= -(p->oomkilladj);
+                        points >>= -(oom_adj);
         }
 
 #ifdef DEBUG
@@ -200,13 +227,13 @@ static inline enum oom_constraint constrained_alloc(struct zonelist *zonelist,
 static struct task_struct *select_bad_process(unsigned long *ppoints,
                                                 struct mem_cgroup *mem)
 {
-        struct task_struct *g, *p;
+        struct task_struct *p;
         struct task_struct *chosen = NULL;
         struct timespec uptime;
         *ppoints = 0;
 
         do_posix_clock_monotonic_gettime(&uptime);
-        do_each_thread(g, p) {
+        for_each_process(p) {
                 unsigned long points;
 
                 /*
@@ -251,7 +278,7 @@ static struct task_struct *select_bad_process(unsigned long *ppoints,
                         *ppoints = ULONG_MAX;
                 }
 
-                if (p->oomkilladj == OOM_DISABLE)
+                if (p->signal->oom_adj == OOM_DISABLE)
                         continue;
 
                 points = badness(p, uptime.tv_sec);
@@ -259,7 +286,7 @@ static struct task_struct *select_bad_process(unsigned long *ppoints,
                         chosen = p;
                         *ppoints = points;
                 }
-        } while_each_thread(g, p);
+        }
 
         return chosen;
 }
@@ -284,22 +311,28 @@ static void dump_tasks(const struct mem_cgroup *mem)
         printk(KERN_INFO "[ pid ]   uid  tgid total_vm      rss cpu oom_adj "
                "name\n");
         do_each_thread(g, p) {
-                /*
-                 * total_vm and rss sizes do not exist for tasks with a
-                 * detached mm so there's no need to report them.
-                 */
-                if (!p->mm)
-                        continue;
+                struct mm_struct *mm;
+
                 if (mem && !task_in_mem_cgroup(p, mem))
                         continue;
                 if (!thread_group_leader(p))
                         continue;
 
                 task_lock(p);
+                mm = p->mm;
+                if (!mm) {
+                        /*
+                         * total_vm and rss sizes do not exist for tasks with no
+                         * mm so there's no need to report them; they can't be
+                         * oom killed anyway.
+                         */
+                        task_unlock(p);
+                        continue;
+                }
                 printk(KERN_INFO "[%5d] %5d %5d %8lu %8lu %3d     %3d %s\n",
-                       p->pid, __task_cred(p)->uid, p->tgid,
-                       p->mm->total_vm, get_mm_rss(p->mm), (int)task_cpu(p),
-                       p->oomkilladj, p->comm);
+                       p->pid, __task_cred(p)->uid, p->tgid, mm->total_vm,
+                       get_mm_rss(mm), (int)task_cpu(p), p->signal->oom_adj,
+                       p->comm);
                 task_unlock(p);
         } while_each_thread(g, p);
 }
@@ -340,11 +373,6 @@ static void __oom_kill_task(struct task_struct *p, int verbose)
 
 static int oom_kill_task(struct task_struct *p)
 {
-        struct mm_struct *mm;
-        struct task_struct *g, *q;
-
-        mm = p->mm;
-
         /* WARNING: mm may not be dereferenced since we did not obtain its
          * value from get_task_mm(p).  This is OK since all we need to do is
          * compare mm to q->mm below.
@@ -353,30 +381,11 @@ static int oom_kill_task(struct task_struct *p)
          * change to NULL at any time since we do not hold task_lock(p).
          * However, this is of no concern to us.
          */
-
-        if (mm == NULL)
+        if (!p->mm || p->signal->oom_adj == OOM_DISABLE)
                 return 1;
 
-        /*
-         * Don't kill the process if any threads are set to OOM_DISABLE
-         */
-        do_each_thread(g, q) {
-                if (q->mm == mm && q->oomkilladj == OOM_DISABLE)
-                        return 1;
-        } while_each_thread(g, q);
-
         __oom_kill_task(p, 1);
 
-        /*
-         * kill all processes that share the ->mm (i.e. all threads),
-         * but are in a different thread group. Don't let them have access
-         * to memory reserves though, otherwise we might deplete all memory.
-         */
-        do_each_thread(g, q) {
-                if (q->mm == mm && !same_thread_group(q, p))
-                        force_sig(SIGKILL, q);
-        } while_each_thread(g, q);
-
         return 0;
 }
 
@@ -388,12 +397,14 @@ static int oom_kill_process(struct task_struct *p, gfp_t gfp_mask, int order,
 
         if (printk_ratelimit()) {
                 printk(KERN_WARNING "%s invoked oom-killer: "
-                        "gfp_mask=0x%x, order=%d, oomkilladj=%d\n",
-                        current->comm, gfp_mask, order, current->oomkilladj);
+                        "gfp_mask=0x%x, order=%d, oom_adj=%d\n",
+                        current->comm, gfp_mask, order,
+                        current->signal->oom_adj);
                 task_lock(current);
                 cpuset_print_task_mems_allowed(current);
                 task_unlock(current);
                 dump_stack();
+                mem_cgroup_print_oom_info(mem, current);
                 show_mem();
                 if (sysctl_oom_dump_tasks)
                         dump_tasks(mem);
@@ -513,34 +524,32 @@ void clear_zonelist_oom(struct zonelist *zonelist, gfp_t gfp_mask)
  */
 static void __out_of_memory(gfp_t gfp_mask, int order)
 {
-        if (sysctl_oom_kill_allocating_task) {
-                oom_kill_process(current, gfp_mask, order, 0, NULL,
-                                "Out of memory (oom_kill_allocating_task)");
-
-        } else {
-                unsigned long points;
-                struct task_struct *p;
-
-retry:
-                /*
-                 * Rambo mode: Shoot down a process and hope it solves whatever
-                 * issues we may have.
-                 */
-                p = select_bad_process(&points, NULL);
+        struct task_struct *p;
+        unsigned long points;
 
-                if (PTR_ERR(p) == -1UL)
+        if (sysctl_oom_kill_allocating_task)
+                if (!oom_kill_process(current, gfp_mask, order, 0, NULL,
+                                "Out of memory (oom_kill_allocating_task)"))
                         return;
+retry:
+        /*
+         * Rambo mode: Shoot down a process and hope it solves whatever
+         * issues we may have.
+         */
+        p = select_bad_process(&points, NULL);
 
-                /* Found nothing?!?! Either we hang forever, or we panic. */
-                if (!p) {
-                        read_unlock(&tasklist_lock);
-                        panic("Out of memory and no killable processes...\n");
-                }
+        if (PTR_ERR(p) == -1UL)
+                return;
 
-                if (oom_kill_process(p, gfp_mask, order, points, NULL,
-                                     "Out of memory"))
-                        goto retry;
+        /* Found nothing?!?! Either we hang forever, or we panic. */
+        if (!p) {
+                read_unlock(&tasklist_lock);
+                panic("Out of memory and no killable processes...\n");
         }
+
+        if (oom_kill_process(p, gfp_mask, order, points, NULL,
+                             "Out of memory"))
+                goto retry;
 }
 
 /*
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 30351f0063ac..0b19943ecf8b 100644
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -36,15 +36,6 @@
 #include <linux/pagevec.h>
 
 /*
- * The maximum number of pages to writeout in a single bdflush/kupdate
- * operation.  We do this so we don't hold I_SYNC against an inode for
- * enormous amounts of time, which would block a userspace task which has
- * been forced to throttle against that inode.  Also, the code reevaluates
- * the dirty each time it has written this many pages.
- */
-#define MAX_WRITEBACK_PAGES     1024
-
-/*
  * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited
  * will look to see if it needs to force writeback or throttling.
  */
@@ -53,18 +44,21 @@ static long ratelimit_pages = 32;
 /*
  * When balance_dirty_pages decides that the caller needs to perform some
  * non-background writeback, this is how many pages it will attempt to write.
- * It should be somewhat larger than RATELIMIT_PAGES to ensure that reasonably
+ * It should be somewhat larger than dirtied pages to ensure that reasonably
  * large amounts of I/O are submitted.
  */
-static inline long sync_writeback_pages(void)
+static inline long sync_writeback_pages(unsigned long dirtied)
 {
-        return ratelimit_pages + ratelimit_pages / 2;
+        if (dirtied < ratelimit_pages)
+                dirtied = ratelimit_pages;
+
+        return dirtied + dirtied / 2;
 }
 
 /* The following parameters are exported via /proc/sys/vm */
 
 /*
- * Start background writeback (via pdflush) at this percentage
+ * Start background writeback (via writeback threads) at this percentage
  */
 int dirty_background_ratio = 10;
 
@@ -94,12 +88,12 @@ unsigned long vm_dirty_bytes;
 /*
  * The interval between `kupdate'-style writebacks
  */
-unsigned int dirty_writeback_interval = 5 * 100; /* sentiseconds */
+unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */
 
 /*
  * The longest time for which data is allowed to remain dirty
  */
-unsigned int dirty_expire_interval = 30 * 100; /* sentiseconds */
+unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */
 
 /*
  * Flag that makes the machine dump writes/reads and block dirtyings.
@@ -117,8 +111,6 @@ EXPORT_SYMBOL(laptop_mode);
 /* End of sysctl-exported parameters */
 
 
-static void background_writeout(unsigned long _min_pages);
-
 /*
  * Scale the writeback cache size proportional to the relative writeout speeds.
  *
@@ -166,37 +158,37 @@ static void update_completion_period(void)
 }
 
 int dirty_background_ratio_handler(struct ctl_table *table, int write,
-                struct file *filp, void __user *buffer, size_t *lenp,
+                void __user *buffer, size_t *lenp,
                 loff_t *ppos)
 {
         int ret;
 
-        ret = proc_dointvec_minmax(table, write, filp, buffer, lenp, ppos);
+        ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
         if (ret == 0 && write)
                 dirty_background_bytes = 0;
         return ret;
 }
 
 int dirty_background_bytes_handler(struct ctl_table *table, int write,
-                struct file *filp, void __user *buffer, size_t *lenp,
+                void __user *buffer, size_t *lenp,
                 loff_t *ppos)
 {
         int ret;
 
-        ret = proc_doulongvec_minmax(table, write, filp, buffer, lenp, ppos);
+        ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
         if (ret == 0 && write)
                 dirty_background_ratio = 0;
         return ret;
 }
 
 int dirty_ratio_handler(struct ctl_table *table, int write,
-                struct file *filp, void __user *buffer, size_t *lenp,
+                void __user *buffer, size_t *lenp,
                 loff_t *ppos)
 {
         int old_ratio = vm_dirty_ratio;
         int ret;
 
-        ret = proc_dointvec_minmax(table, write, filp, buffer, lenp, ppos);
+        ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
         if (ret == 0 && write && vm_dirty_ratio != old_ratio) {
                 update_completion_period();
                 vm_dirty_bytes = 0;
@@ -206,13 +198,13 @@ int dirty_ratio_handler(struct ctl_table *table, int write,
 
 
 int dirty_bytes_handler(struct ctl_table *table, int write,
-                struct file *filp, void __user *buffer, size_t *lenp,
+                void __user *buffer, size_t *lenp,
                 loff_t *ppos)
 {
         unsigned long old_bytes = vm_dirty_bytes;
         int ret;
 
-        ret = proc_doulongvec_minmax(table, write, filp, buffer, lenp, ppos);
+        ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
         if (ret == 0 && write && vm_dirty_bytes != old_bytes) {
                 update_completion_period();
                 vm_dirty_ratio = 0;
@@ -265,18 +257,19 @@ static void bdi_writeout_fraction(struct backing_dev_info *bdi,
  * This avoids exceeding the total dirty_limit when the floating averages
  * fluctuate too quickly.
  */
-static void
-clip_bdi_dirty_limit(struct backing_dev_info *bdi, long dirty, long *pbdi_dirty)
+static void clip_bdi_dirty_limit(struct backing_dev_info *bdi,
+                unsigned long dirty, unsigned long *pbdi_dirty)
 {
-        long avail_dirty;
+        unsigned long avail_dirty;
 
-        avail_dirty = dirty -
-                (global_page_state(NR_FILE_DIRTY) +
+        avail_dirty = global_page_state(NR_FILE_DIRTY) +
                  global_page_state(NR_WRITEBACK) +
                  global_page_state(NR_UNSTABLE_NFS) +
-                 global_page_state(NR_WRITEBACK_TEMP));
+                 global_page_state(NR_WRITEBACK_TEMP);
 
-        if (avail_dirty < 0)
+        if (avail_dirty < dirty)
+                avail_dirty = dirty - avail_dirty;
+        else
                 avail_dirty = 0;
 
         avail_dirty += bdi_stat(bdi, BDI_RECLAIMABLE) +
@@ -299,10 +292,10 @@ static inline void task_dirties_fraction(struct task_struct *tsk,
  *
  *   dirty -= (dirty/8) * p_{t}
  */
-static void task_dirty_limit(struct task_struct *tsk, long *pdirty)
+static void task_dirty_limit(struct task_struct *tsk, unsigned long *pdirty)
 {
         long numerator, denominator;
-        long dirty = *pdirty;
+        unsigned long dirty = *pdirty;
         u64 inv = dirty >> 3;
 
         task_dirties_fraction(tsk, &numerator, &denominator);
@@ -319,15 +312,13 @@ static void task_dirty_limit(struct task_struct *tsk, long *pdirty)
 /*
  *
  */
-static DEFINE_SPINLOCK(bdi_lock);
 static unsigned int bdi_min_ratio;
 
 int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
 {
         int ret = 0;
-        unsigned long flags;
 
-        spin_lock_irqsave(&bdi_lock, flags);
+        spin_lock_bh(&bdi_lock);
         if (min_ratio > bdi->max_ratio) {
                 ret = -EINVAL;
         } else {
@@ -339,27 +330,26 @@ int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
                         ret = -EINVAL;
                 }
         }
-        spin_unlock_irqrestore(&bdi_lock, flags);
+        spin_unlock_bh(&bdi_lock);
 
         return ret;
 }
 
 int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned max_ratio)
 {
-        unsigned long flags;
         int ret = 0;
 
         if (max_ratio > 100)
                 return -EINVAL;
 
-        spin_lock_irqsave(&bdi_lock, flags);
+        spin_lock_bh(&bdi_lock);
         if (bdi->min_ratio > max_ratio) {
                 ret = -EINVAL;
         } else {
                 bdi->max_ratio = max_ratio;
                 bdi->max_prop_frac = (PROP_FRAC_BASE * max_ratio) / 100;
         }
-        spin_unlock_irqrestore(&bdi_lock, flags);
+        spin_unlock_bh(&bdi_lock);
 
         return ret;
 }
@@ -393,7 +383,8 @@ static unsigned long highmem_dirtyable_memory(unsigned long total)
                 struct zone *z =
                         &NODE_DATA(node)->node_zones[ZONE_HIGHMEM];
 
-                x += zone_page_state(z, NR_FREE_PAGES) + zone_lru_pages(z);
+                x += zone_page_state(z, NR_FREE_PAGES) +
+                     zone_reclaimable_pages(z);
         }
         /*
          * Make sure that the number of highmem pages is never larger
@@ -417,7 +408,7 @@ unsigned long determine_dirtyable_memory(void)
 {
         unsigned long x;
 
-        x = global_page_state(NR_FREE_PAGES) + global_lru_pages();
+        x = global_page_state(NR_FREE_PAGES) + global_reclaimable_pages();
 
         if (!vm_highmem_is_dirtyable)
                 x -= highmem_dirtyable_memory(x);
@@ -486,10 +477,11 @@ get_dirty_limits(unsigned long *pbackground, unsigned long *pdirty,
  * balance_dirty_pages() must be called by processes which are generating dirty
  * data.  It looks at the number of dirty pages in the machine and will force
  * the caller to perform writeback if the system is over `vm_dirty_ratio'.
- * If we're over `background_thresh' then pdflush is woken to perform some
- * writeout.
+ * If we're over `background_thresh' then the writeback threads are woken to
+ * perform some writeout.
  */
-static void balance_dirty_pages(struct address_space *mapping)
+static void balance_dirty_pages(struct address_space *mapping,
+                                unsigned long write_chunk)
 {
         long nr_reclaimable, bdi_nr_reclaimable;
         long nr_writeback, bdi_nr_writeback;
@@ -497,7 +489,7 @@ static void balance_dirty_pages(struct address_space *mapping)
         unsigned long dirty_thresh;
         unsigned long bdi_thresh;
         unsigned long pages_written = 0;
-        unsigned long write_chunk = sync_writeback_pages();
+        unsigned long pause = 1;
 
         struct backing_dev_info *bdi = mapping->backing_dev_info;
 
@@ -540,9 +532,12 @@ static void balance_dirty_pages(struct address_space *mapping)
                  * filesystems (i.e. NFS) in which data may have been
                  * written to the server's write cache, but has not yet
                  * been flushed to permanent storage.
+                 * Only move pages to writeback if this bdi is over its
+                 * threshold otherwise wait until the disk writes catch
+                 * up.
                  */
-                if (bdi_nr_reclaimable) {
-                        writeback_inodes(&wbc);
+                if (bdi_nr_reclaimable > bdi_thresh) {
+                        writeback_inodes_wbc(&wbc);
                         pages_written += write_chunk - wbc.nr_to_write;
                         get_dirty_limits(&background_thresh, &dirty_thresh,
                                        &bdi_thresh, bdi);
@@ -571,7 +566,16 @@ static void balance_dirty_pages(struct address_space *mapping)
                 if (pages_written >= write_chunk)
                         break;          /* We've done our duty */
 
-                congestion_wait(WRITE, HZ/10);
+                __set_current_state(TASK_INTERRUPTIBLE);
+                io_schedule_timeout(pause);
+
+                /*
+                 * Increase the delay for each loop, up to our previous
+                 * default of taking a 100ms nap.
+                 */
+                pause <<= 1;
+                if (pause > HZ / 10)
+                        pause = HZ / 10;
         }
 
         if (bdi_nr_reclaimable + bdi_nr_writeback < bdi_thresh &&
@@ -579,7 +583,7 @@ static void balance_dirty_pages(struct address_space *mapping)
                 bdi->dirty_exceeded = 0;
 
         if (writeback_in_progress(bdi))
-                return;         /* pdflush is already working this queue */
+                return;
 
         /*
          * In laptop mode, we wait until hitting the higher threshold before
@@ -590,10 +594,10 @@ static void balance_dirty_pages(struct address_space *mapping)
          * background_thresh, to keep the amount of dirty memory low.
          */
         if ((laptop_mode && pages_written) ||
-                        (!laptop_mode && (global_page_state(NR_FILE_DIRTY)
-                                          + global_page_state(NR_UNSTABLE_NFS)
+            (!laptop_mode && ((global_page_state(NR_FILE_DIRTY)
+                               + global_page_state(NR_UNSTABLE_NFS))
                                           > background_thresh)))
-                pdflush_operation(background_writeout, 0);
+                bdi_start_writeback(bdi, NULL, 0);
 }
 
 void set_page_dirty_balance(struct page *page, int page_mkwrite)
@@ -606,6 +610,8 @@ void set_page_dirty_balance(struct page *page, int page_mkwrite)
         }
 }
 
+static DEFINE_PER_CPU(unsigned long, bdp_ratelimits) = 0;
+
 /**
  * balance_dirty_pages_ratelimited_nr - balance dirty memory state
  * @mapping: address_space which was dirtied
@@ -623,7 +629,6 @@ void set_page_dirty_balance(struct page *page, int page_mkwrite)
 void balance_dirty_pages_ratelimited_nr(struct address_space *mapping,
                                         unsigned long nr_pages_dirtied)
 {
-        static DEFINE_PER_CPU(unsigned long, ratelimits) = 0;
         unsigned long ratelimit;
         unsigned long *p;
 
@@ -636,12 +641,13 @@ void balance_dirty_pages_ratelimited_nr(struct address_space *mapping,
          * tasks in balance_dirty_pages(). Period.
          */
         preempt_disable();
-        p =  &__get_cpu_var(ratelimits);
+        p =  &__get_cpu_var(bdp_ratelimits);
         *p += nr_pages_dirtied;
         if (unlikely(*p >= ratelimit)) {
+                ratelimit = sync_writeback_pages(*p);
                 *p = 0;
                 preempt_enable();
-                balance_dirty_pages(mapping);
+                balance_dirty_pages(mapping, ratelimit);
                 return;
         }
         preempt_enable();
@@ -665,7 +671,7 @@ void throttle_vm_writeout(gfp_t gfp_mask)
                 if (global_page_state(NR_UNSTABLE_NFS) +
                         global_page_state(NR_WRITEBACK) <= dirty_thresh)
                                 break;
-                congestion_wait(WRITE, HZ/10);
+                congestion_wait(BLK_RW_ASYNC, HZ/10);
 
                 /*
                  * The caller might hold locks which can prevent IO completion
@@ -677,153 +683,35 @@ void throttle_vm_writeout(gfp_t gfp_mask)
         }
 }
 
-/*
- * writeback at least _min_pages, and keep writing until the amount of dirty
- * memory is less than the background threshold, or until we're all clean.
- */
-static void background_writeout(unsigned long _min_pages)
-{
-        long min_pages = _min_pages;
-        struct writeback_control wbc = {
-                .bdi            = NULL,
-                .sync_mode      = WB_SYNC_NONE,
-                .older_than_this = NULL,
-                .nr_to_write    = 0,
-                .nonblocking    = 1,
-                .range_cyclic   = 1,
-        };
-
-        for ( ; ; ) {
-                unsigned long background_thresh;
-                unsigned long dirty_thresh;
-
-                get_dirty_limits(&background_thresh, &dirty_thresh, NULL, NULL);
-                if (global_page_state(NR_FILE_DIRTY) +
-                        global_page_state(NR_UNSTABLE_NFS) < background_thresh
-                                && min_pages <= 0)
-                        break;
-                wbc.more_io = 0;
-                wbc.encountered_congestion = 0;
-                wbc.nr_to_write = MAX_WRITEBACK_PAGES;
-                wbc.pages_skipped = 0;
-                writeback_inodes(&wbc);
-                min_pages -= MAX_WRITEBACK_PAGES - wbc.nr_to_write;
-                if (wbc.nr_to_write > 0 || wbc.pages_skipped > 0) {
-                        /* Wrote less than expected */
-                        if (wbc.encountered_congestion || wbc.more_io)
-                                congestion_wait(WRITE, HZ/10);
-                        else
-                                break;
-                }
-        }
-}
-
-/*
- * Start writeback of `nr_pages' pages.  If `nr_pages' is zero, write back
- * the whole world.  Returns 0 if a pdflush thread was dispatched.  Returns
- * -1 if all pdflush threads were busy.
- */
-int wakeup_pdflush(long nr_pages)
-{
-        if (nr_pages == 0)
-                nr_pages = global_page_state(NR_FILE_DIRTY) +
-                                global_page_state(NR_UNSTABLE_NFS);
-        return pdflush_operation(background_writeout, nr_pages);
-}
-
-static void wb_timer_fn(unsigned long unused);
 static void laptop_timer_fn(unsigned long unused);
 
-static DEFINE_TIMER(wb_timer, wb_timer_fn, 0, 0);
 static DEFINE_TIMER(laptop_mode_wb_timer, laptop_timer_fn, 0, 0);
 
 /*
- * Periodic writeback of "old" data.
- *
- * Define "old": the first time one of an inode's pages is dirtied, we mark the
- * dirtying-time in the inode's address_space.  So this periodic writeback code
- * just walks the superblock inode list, writing back any inodes which are
- * older than a specific point in time.
- *
- * Try to run once per dirty_writeback_interval.  But if a writeback event
- * takes longer than a dirty_writeback_interval interval, then leave a
- * one-second gap.
- *
- * older_than_this takes precedence over nr_to_write.  So we'll only write back
- * all dirty pages if they are all attached to "old" mappings.
- */
-static void wb_kupdate(unsigned long arg)
-{
-        unsigned long oldest_jif;
-        unsigned long start_jif;
-        unsigned long next_jif;
-        long nr_to_write;
-        struct writeback_control wbc = {
-                .bdi            = NULL,
-                .sync_mode      = WB_SYNC_NONE,
-                .older_than_this = &oldest_jif,
-                .nr_to_write    = 0,
-                .nonblocking    = 1,
-                .for_kupdate    = 1,
-                .range_cyclic   = 1,
-        };
-
-        sync_supers();
-
-        oldest_jif = jiffies - msecs_to_jiffies(dirty_expire_interval);
-        start_jif = jiffies;
-        next_jif = start_jif + msecs_to_jiffies(dirty_writeback_interval * 10);
-        nr_to_write = global_page_state(NR_FILE_DIRTY) +
-                        global_page_state(NR_UNSTABLE_NFS) +
-                        (inodes_stat.nr_inodes - inodes_stat.nr_unused);
-        while (nr_to_write > 0) {
-                wbc.more_io = 0;
-                wbc.encountered_congestion = 0;
-                wbc.nr_to_write = MAX_WRITEBACK_PAGES;
-                writeback_inodes(&wbc);
-                if (wbc.nr_to_write > 0) {
-                        if (wbc.encountered_congestion || wbc.more_io)
-                                congestion_wait(WRITE, HZ/10);
-                        else
-                                break;  /* All the old data is written */
-                }
-                nr_to_write -= MAX_WRITEBACK_PAGES - wbc.nr_to_write;
-        }
-        if (time_before(next_jif, jiffies + HZ))
-                next_jif = jiffies + HZ;
-        if (dirty_writeback_interval)
-                mod_timer(&wb_timer, next_jif);
-}
-
-/*
  * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs
  */
 int dirty_writeback_centisecs_handler(ctl_table *table, int write,
-        struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
+        void __user *buffer, size_t *length, loff_t *ppos)
 {
-        proc_dointvec(table, write, file, buffer, length, ppos);
-        if (dirty_writeback_interval)
-                mod_timer(&wb_timer, jiffies +
-                        msecs_to_jiffies(dirty_writeback_interval * 10));
-        else
-                del_timer(&wb_timer);
+        proc_dointvec(table, write, buffer, length, ppos);
         return 0;
 }
 
-static void wb_timer_fn(unsigned long unused)
+static void do_laptop_sync(struct work_struct *work)
 {
-        if (pdflush_operation(wb_kupdate, 0) < 0)
-                mod_timer(&wb_timer, jiffies + HZ); /* delay 1 second */
-}
-
-static void laptop_flush(unsigned long unused)
-{
-        sys_sync();
+        wakeup_flusher_threads(0);
+        kfree(work);
 }
 
 static void laptop_timer_fn(unsigned long unused)
 {
-        pdflush_operation(laptop_flush, 0);
+        struct work_struct *work;
+
+        work = kmalloc(sizeof(*work), GFP_ATOMIC);
+        if (work) {
+                INIT_WORK(work, do_laptop_sync);
+                schedule_work(work);
+        }
 }
 
 /*
@@ -906,8 +794,6 @@ void __init page_writeback_init(void)
 {
         int shift;
 
-        mod_timer(&wb_timer,
-                  jiffies + msecs_to_jiffies(dirty_writeback_interval * 10));
         writeback_set_ratelimit();
         register_cpu_notifier(&ratelimit_nb);
 
@@ -935,7 +821,6 @@ int write_cache_pages(struct address_space *mapping,
                       struct writeback_control *wbc, writepage_t writepage,
                       void *data)
 {
-        struct backing_dev_info *bdi = mapping->backing_dev_info;
         int ret = 0;
         int done = 0;
         struct pagevec pvec;
@@ -948,11 +833,6 @@ int write_cache_pages(struct address_space *mapping,
         int range_whole = 0;
         long nr_to_write = wbc->nr_to_write;
 
-        if (wbc->nonblocking && bdi_write_congested(bdi)) {
-                wbc->encountered_congestion = 1;
-                return 0;
-        }
-
         pagevec_init(&pvec, 0);
         if (wbc->range_cyclic) {
                 writeback_index = mapping->writeback_index; /* prev offset */
@@ -1071,12 +951,6 @@ continue_unlock:
                                         break;
                                 }
                         }
-
-                        if (wbc->nonblocking && bdi_write_congested(bdi)) {
-                                wbc->encountered_congestion = 1;
-                                done = 1;
-                                break;
-                        }
                 }
                 pagevec_release(&pvec);
                 cond_resched();
@@ -1141,12 +1015,10 @@ int do_writepages(struct address_space *mapping, struct writeback_control *wbc)
 
         if (wbc->nr_to_write <= 0)
                 return 0;
-        wbc->for_writepages = 1;
         if (mapping->a_ops->writepages)
                 ret = mapping->a_ops->writepages(mapping, wbc);
         else
                 ret = generic_writepages(mapping, wbc);
-        wbc->for_writepages = 0;
         return ret;
 }
 
@@ -1270,6 +1142,13 @@ int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page)
 EXPORT_SYMBOL(redirty_page_for_writepage);
 
 /*
+ * Dirty a page.
+ *
+ * For pages with a mapping this should be done under the page lock
+ * for the benefit of asynchronous memory errors who prefer a consistent
+ * dirty state. This rule can be broken in some special cases,
+ * but should be better not to.
+ *
  * If the mapping doesn't provide a set_page_dirty a_op, then
  * just fall through and assume that it wants buffer_heads.
  */
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 0284e528748d..2bc2ac63f41e 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -23,6 +23,7 @@
 #include <linux/bootmem.h>
 #include <linux/compiler.h>
 #include <linux/kernel.h>
+#include <linux/kmemcheck.h>
 #include <linux/module.h>
 #include <linux/suspend.h>
 #include <linux/pagevec.h>
@@ -46,6 +47,8 @@
 #include <linux/page-isolation.h>
 #include <linux/page_cgroup.h>
 #include <linux/debugobjects.h>
+#include <linux/kmemleak.h>
+#include <trace/events/kmem.h>
 
 #include <asm/tlbflush.h>
 #include <asm/div64.h>
@@ -69,8 +72,8 @@ EXPORT_SYMBOL(node_states);
 
 unsigned long totalram_pages __read_mostly;
 unsigned long totalreserve_pages __read_mostly;
-unsigned long highest_memmap_pfn __read_mostly;
 int percpu_pagelist_fraction;
+gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
 
 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
 int pageblock_order __read_mostly;
@@ -120,8 +123,8 @@ static char * const zone_names[MAX_NR_ZONES] = {
 
 int min_free_kbytes = 1024;
 
-unsigned long __meminitdata nr_kernel_pages;
-unsigned long __meminitdata nr_all_pages;
+static unsigned long __meminitdata nr_kernel_pages;
+static unsigned long __meminitdata nr_all_pages;
 static unsigned long __meminitdata dma_reserve;
 
 #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
@@ -149,10 +152,6 @@ static unsigned long __meminitdata dma_reserve;
   static int __meminitdata nr_nodemap_entries;
   static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
   static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
-#ifdef CONFIG_MEMORY_HOTPLUG_RESERVE
-  static unsigned long __meminitdata node_boundary_start_pfn[MAX_NUMNODES];
-  static unsigned long __meminitdata node_boundary_end_pfn[MAX_NUMNODES];
-#endif /* CONFIG_MEMORY_HOTPLUG_RESERVE */
   static unsigned long __initdata required_kernelcore;
   static unsigned long __initdata required_movablecore;
   static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
@@ -164,17 +163,25 @@ static unsigned long __meminitdata dma_reserve;
 
 #if MAX_NUMNODES > 1
 int nr_node_ids __read_mostly = MAX_NUMNODES;
+int nr_online_nodes __read_mostly = 1;
 EXPORT_SYMBOL(nr_node_ids);
+EXPORT_SYMBOL(nr_online_nodes);
 #endif
 
 int page_group_by_mobility_disabled __read_mostly;
 
 static void set_pageblock_migratetype(struct page *page, int migratetype)
 {
+
+        if (unlikely(page_group_by_mobility_disabled))
+                migratetype = MIGRATE_UNMOVABLE;
+
         set_pageblock_flags_group(page, (unsigned long)migratetype,
                                         PB_migrate, PB_migrate_end);
 }
 
+bool oom_killer_disabled __read_mostly;
+
 #ifdef CONFIG_DEBUG_VM
 static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
 {
@@ -227,6 +234,12 @@ static void bad_page(struct page *page)
         static unsigned long nr_shown;
         static unsigned long nr_unshown;
 
+        /* Don't complain about poisoned pages */
+        if (PageHWPoison(page)) {
+                __ClearPageBuddy(page);
+                return;
+        }
+
         /*
          * Allow a burst of 60 reports, then keep quiet for that minute;
          * or allow a steady drip of one report per second.
@@ -297,23 +310,6 @@ void prep_compound_page(struct page *page, unsigned long order)
         }
 }
 
-#ifdef CONFIG_HUGETLBFS
-void prep_compound_gigantic_page(struct page *page, unsigned long order)
-{
-        int i;
-        int nr_pages = 1 << order;
-        struct page *p = page + 1;
-
-        set_compound_page_dtor(page, free_compound_page);
-        set_compound_order(page, order);
-        __SetPageHead(page);
-        for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
-                __SetPageTail(p);
-                p->first_page = page;
-        }
-}
-#endif
-
 static int destroy_compound_page(struct page *page, unsigned long order)
 {
         int i;
@@ -331,7 +327,7 @@ static int destroy_compound_page(struct page *page, unsigned long order)
         for (i = 1; i < nr_pages; i++) {
                 struct page *p = page + i;
 
-                if (unlikely(!PageTail(p) | (p->first_page != page))) {
+                if (unlikely(!PageTail(p) || (p->first_page != page))) {
                         bad_page(page);
                         bad++;
                 }
@@ -420,7 +416,7 @@ static inline int page_is_buddy(struct page *page, struct page *buddy,
                 return 0;
 
         if (PageBuddy(buddy) && page_order(buddy) == order) {
-                BUG_ON(page_count(buddy) != 0);
+                VM_BUG_ON(page_count(buddy) != 0);
                 return 1;
         }
         return 0;
@@ -451,22 +447,22 @@ static inline int page_is_buddy(struct page *page, struct page *buddy,
  */
 
 static inline void __free_one_page(struct page *page,
-                struct zone *zone, unsigned int order)
+                struct zone *zone, unsigned int order,
+                int migratetype)
 {
         unsigned long page_idx;
-        int order_size = 1 << order;
-        int migratetype = get_pageblock_migratetype(page);
 
         if (unlikely(PageCompound(page)))
                 if (unlikely(destroy_compound_page(page, order)))
                         return;
 
+        VM_BUG_ON(migratetype == -1);
+
         page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
 
-        VM_BUG_ON(page_idx & (order_size - 1));
+        VM_BUG_ON(page_idx & ((1 << order) - 1));
         VM_BUG_ON(bad_range(zone, page));
 
-        __mod_zone_page_state(zone, NR_FREE_PAGES, order_size);
         while (order < MAX_ORDER-1) {
                 unsigned long combined_idx;
                 struct page *buddy;
@@ -490,12 +486,26 @@ static inline void __free_one_page(struct page *page,
         zone->free_area[order].nr_free++;
 }
 
+#ifdef CONFIG_HAVE_MLOCKED_PAGE_BIT
+/*
+ * free_page_mlock() -- clean up attempts to free and mlocked() page.
+ * Page should not be on lru, so no need to fix that up.
+ * free_pages_check() will verify...
+ */
+static inline void free_page_mlock(struct page *page)
+{
+        __dec_zone_page_state(page, NR_MLOCK);
+        __count_vm_event(UNEVICTABLE_MLOCKFREED);
+}
+#else
+static void free_page_mlock(struct page *page) { }
+#endif
+
 static inline int free_pages_check(struct page *page)
 {
-        free_page_mlock(page);
         if (unlikely(page_mapcount(page) |
                 (page->mapping != NULL)  |
-                (page_count(page) != 0)  |
+                (atomic_read(&page->_count) != 0) |
                 (page->flags & PAGE_FLAGS_CHECK_AT_FREE))) {
                 bad_page(page);
                 return 1;
@@ -506,7 +516,7 @@ static inline int free_pages_check(struct page *page)
 }
 
 /*
- * Frees a list of pages. 
+ * Frees a number of pages from the PCP lists
  * Assumes all pages on list are in same zone, and of same order.
  * count is the number of pages to free.
  *
@@ -516,30 +526,55 @@ static inline int free_pages_check(struct page *page)
  * And clear the zone's pages_scanned counter, to hold off the "all pages are
  * pinned" detection logic.
  */
-static void free_pages_bulk(struct zone *zone, int count,
-                                        struct list_head *list, int order)
+static void free_pcppages_bulk(struct zone *zone, int count,
+                                        struct per_cpu_pages *pcp)
 {
+        int migratetype = 0;
+        int batch_free = 0;
+
         spin_lock(&zone->lock);
         zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);
         zone->pages_scanned = 0;
-        while (count--) {
+
+        __mod_zone_page_state(zone, NR_FREE_PAGES, count);
+        while (count) {
                 struct page *page;
+                struct list_head *list;
 
-                VM_BUG_ON(list_empty(list));
-                page = list_entry(list->prev, struct page, lru);
-                /* have to delete it as __free_one_page list manipulates */
-                list_del(&page->lru);
-                __free_one_page(page, zone, order);
+                /*
+                 * Remove pages from lists in a round-robin fashion. A
+                 * batch_free count is maintained that is incremented when an
+                 * empty list is encountered.  This is so more pages are freed
+                 * off fuller lists instead of spinning excessively around empty
+                 * lists
+                 */
+                do {
+                        batch_free++;
+                        if (++migratetype == MIGRATE_PCPTYPES)
+                                migratetype = 0;
+                        list = &pcp->lists[migratetype];
+                } while (list_empty(list));
+
+                do {
+                        page = list_entry(list->prev, struct page, lru);
+                        /* must delete as __free_one_page list manipulates */
+                        list_del(&page->lru);
+                        __free_one_page(page, zone, 0, migratetype);
+                        trace_mm_page_pcpu_drain(page, 0, migratetype);
+                } while (--count && --batch_free && !list_empty(list));
         }
         spin_unlock(&zone->lock);
 }
 
-static void free_one_page(struct zone *zone, struct page *page, int order)
+static void free_one_page(struct zone *zone, struct page *page, int order,
+                                int migratetype)
 {
         spin_lock(&zone->lock);
         zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);
         zone->pages_scanned = 0;
-        __free_one_page(page, zone, order);
+
+        __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
+        __free_one_page(page, zone, order, migratetype);
         spin_unlock(&zone->lock);
 }
 
@@ -548,6 +583,9 @@ static void __free_pages_ok(struct page *page, unsigned int order)
         unsigned long flags;
         int i;
         int bad = 0;
+        int wasMlocked = __TestClearPageMlocked(page);
+
+        kmemcheck_free_shadow(page, order);
 
         for (i = 0 ; i < (1 << order) ; ++i)
                 bad += free_pages_check(page + i);
@@ -563,8 +601,11 @@ static void __free_pages_ok(struct page *page, unsigned int order)
         kernel_map_pages(page, 1 << order, 0);
 
         local_irq_save(flags);
+        if (unlikely(wasMlocked))
+                free_page_mlock(page);
         __count_vm_events(PGFREE, 1 << order);
-        free_one_page(page_zone(page), page, order);
+        free_one_page(page_zone(page), page, order,
+                                        get_pageblock_migratetype(page));
         local_irq_restore(flags);
 }
 
@@ -631,15 +672,27 @@ static inline void expand(struct zone *zone, struct page *page,
 /*
  * This page is about to be returned from the page allocator
  */
-static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
+static inline int check_new_page(struct page *page)
 {
         if (unlikely(page_mapcount(page) |
                 (page->mapping != NULL)  |
-                (page_count(page) != 0)  |
+                (atomic_read(&page->_count) != 0)  |
                 (page->flags & PAGE_FLAGS_CHECK_AT_PREP))) {
                 bad_page(page);
                 return 1;
         }
+        return 0;
+}
+
+static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
+{
+        int i;
+
+        for (i = 0; i < (1 << order); i++) {
+                struct page *p = page + i;
+                if (unlikely(check_new_page(p)))
+                        return 1;
+        }
 
         set_page_private(page, 0);
         set_page_refcounted(page);
@@ -660,7 +713,8 @@ static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
  * Go through the free lists for the given migratetype and remove
  * the smallest available page from the freelists
  */
-static struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
+static inline
+struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
                                                 int migratetype)
 {
         unsigned int current_order;
@@ -678,7 +732,6 @@ static struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
                 list_del(&page->lru);
                 rmv_page_order(page);
                 area->nr_free--;
-                __mod_zone_page_state(zone, NR_FREE_PAGES, - (1UL << order));
                 expand(zone, page, order, current_order, area, migratetype);
                 return page;
         }
@@ -768,9 +821,20 @@ static int move_freepages_block(struct zone *zone, struct page *page,
         return move_freepages(zone, start_page, end_page, migratetype);
 }
 
+static void change_pageblock_range(struct page *pageblock_page,
+                                        int start_order, int migratetype)
+{
+        int nr_pageblocks = 1 << (start_order - pageblock_order);
+
+        while (nr_pageblocks--) {
+                set_pageblock_migratetype(pageblock_page, migratetype);
+                pageblock_page += pageblock_nr_pages;
+        }
+}
+
 /* Remove an element from the buddy allocator from the fallback list */
-static struct page *__rmqueue_fallback(struct zone *zone, int order,
-                                                int start_migratetype)
+static inline struct page *
+__rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
 {
         struct free_area * area;
         int current_order;
@@ -802,13 +866,15 @@ static struct page *__rmqueue_fallback(struct zone *zone, int order,
                          * agressive about taking ownership of free pages
                          */
                         if (unlikely(current_order >= (pageblock_order >> 1)) ||
-                                        start_migratetype == MIGRATE_RECLAIMABLE) {
+                                        start_migratetype == MIGRATE_RECLAIMABLE ||
+                                        page_group_by_mobility_disabled) {
                                 unsigned long pages;
                                 pages = move_freepages_block(zone, page,
                                                                 start_migratetype);
 
                                 /* Claim the whole block if over half of it is free */
-                                if (pages >= (1 << (pageblock_order-1)))
+                                if (pages >= (1 << (pageblock_order-1)) ||
+                                                page_group_by_mobility_disabled)
                                         set_pageblock_migratetype(page,
                                                                 start_migratetype);
 
@@ -818,20 +884,22 @@ static struct page *__rmqueue_fallback(struct zone *zone, int order,
                         /* Remove the page from the freelists */
                         list_del(&page->lru);
                         rmv_page_order(page);
-                        __mod_zone_page_state(zone, NR_FREE_PAGES,
-                                                        -(1UL << order));
 
-                        if (current_order == pageblock_order)
-                                set_pageblock_migratetype(page,
+                        /* Take ownership for orders >= pageblock_order */
+                        if (current_order >= pageblock_order)
+                                change_pageblock_range(page, current_order,
                                                         start_migratetype);
 
                         expand(zone, page, order, current_order, area, migratetype);
+
+                        trace_mm_page_alloc_extfrag(page, order, current_order,
+                                start_migratetype, migratetype);
+
                         return page;
                 }
         }
 
-        /* Use MIGRATE_RESERVE rather than fail an allocation */
-        return __rmqueue_smallest(zone, order, MIGRATE_RESERVE);
+        return NULL;
 }
 
 /*
@@ -843,11 +911,24 @@ static struct page *__rmqueue(struct zone *zone, unsigned int order,
 {
         struct page *page;
 
+retry_reserve:
         page = __rmqueue_smallest(zone, order, migratetype);
 
-        if (unlikely(!page))
+        if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
                 page = __rmqueue_fallback(zone, order, migratetype);
 
+                /*
+                 * Use MIGRATE_RESERVE rather than fail an allocation. goto
+                 * is used because __rmqueue_smallest is an inline function
+                 * and we want just one call site
+                 */
+                if (!page) {
+                        migratetype = MIGRATE_RESERVE;
+                        goto retry_reserve;
+                }
+        }
+
+        trace_mm_page_alloc_zone_locked(page, order, migratetype);
         return page;
 }
 
@@ -858,7 +939,7 @@ static struct page *__rmqueue(struct zone *zone, unsigned int order,
  */
 static int rmqueue_bulk(struct zone *zone, unsigned int order, 
                         unsigned long count, struct list_head *list,
-                        int migratetype)
+                        int migratetype, int cold)
 {
         int i;
         
@@ -877,10 +958,14 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
                  * merge IO requests if the physical pages are ordered
                  * properly.
                  */
-                list_add(&page->lru, list);
+                if (likely(cold == 0))
+                        list_add(&page->lru, list);
+                else
+                        list_add_tail(&page->lru, list);
                 set_page_private(page, migratetype);
                 list = &page->lru;
         }
+        __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
         spin_unlock(&zone->lock);
         return i;
 }
@@ -904,7 +989,7 @@ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
                 to_drain = pcp->batch;
         else
                 to_drain = pcp->count;
-        free_pages_bulk(zone, to_drain, &pcp->list, 0);
+        free_pcppages_bulk(zone, to_drain, pcp);
         pcp->count -= to_drain;
         local_irq_restore(flags);
 }
@@ -930,7 +1015,7 @@ static void drain_pages(unsigned int cpu)
 
                 pcp = &pset->pcp;
                 local_irq_save(flags);
-                free_pages_bulk(zone, pcp->count, &pcp->list, 0);
+                free_pcppages_bulk(zone, pcp->count, pcp);
                 pcp->count = 0;
                 local_irq_restore(flags);
         }
@@ -996,6 +1081,10 @@ static void free_hot_cold_page(struct page *page, int cold)
         struct zone *zone = page_zone(page);
         struct per_cpu_pages *pcp;
         unsigned long flags;
+        int migratetype;
+        int wasMlocked = __TestClearPageMlocked(page);
+
+        kmemcheck_free_shadow(page, 0);
 
         if (PageAnon(page))
                 page->mapping = NULL;
@@ -1010,32 +1099,49 @@ static void free_hot_cold_page(struct page *page, int cold)
         kernel_map_pages(page, 1, 0);
 
         pcp = &zone_pcp(zone, get_cpu())->pcp;
+        migratetype = get_pageblock_migratetype(page);
+        set_page_private(page, migratetype);
         local_irq_save(flags);
+        if (unlikely(wasMlocked))
+                free_page_mlock(page);
         __count_vm_event(PGFREE);
+
+        /*
+         * We only track unmovable, reclaimable and movable on pcp lists.
+         * Free ISOLATE pages back to the allocator because they are being
+         * offlined but treat RESERVE as movable pages so we can get those
+         * areas back if necessary. Otherwise, we may have to free
+         * excessively into the page allocator
+         */
+        if (migratetype >= MIGRATE_PCPTYPES) {
+                if (unlikely(migratetype == MIGRATE_ISOLATE)) {
+                        free_one_page(zone, page, 0, migratetype);
+                        goto out;
+                }
+                migratetype = MIGRATE_MOVABLE;
+        }
+
         if (cold)
-                list_add_tail(&page->lru, &pcp->list);
+                list_add_tail(&page->lru, &pcp->lists[migratetype]);
         else
-                list_add(&page->lru, &pcp->list);
-        set_page_private(page, get_pageblock_migratetype(page));
+                list_add(&page->lru, &pcp->lists[migratetype]);
         pcp->count++;
         if (pcp->count >= pcp->high) {
-                free_pages_bulk(zone, pcp->batch, &pcp->list, 0);
+                free_pcppages_bulk(zone, pcp->batch, pcp);
                 pcp->count -= pcp->batch;
         }
+
+out:
         local_irq_restore(flags);
         put_cpu();
 }
 
 void free_hot_page(struct page *page)
 {
+        trace_mm_page_free_direct(page, 0);
         free_hot_cold_page(page, 0);
 }
         
-void free_cold_page(struct page *page)
-{
-        free_hot_cold_page(page, 1);
-}
-
 /*
  * split_page takes a non-compound higher-order page, and splits it into
  * n (1<<order) sub-pages: page[0..n]
@@ -1050,6 +1156,16 @@ void split_page(struct page *page, unsigned int order)
 
         VM_BUG_ON(PageCompound(page));
         VM_BUG_ON(!page_count(page));
+
+#ifdef CONFIG_KMEMCHECK
+        /*
+         * Split shadow pages too, because free(page[0]) would
+         * otherwise free the whole shadow.
+         */
+        if (kmemcheck_page_is_tracked(page))
+                split_page(virt_to_page(page[0].shadow), order);
+#endif
+
         for (i = 1; i < (1 << order); i++)
                 set_page_refcounted(page + i);
 }
@@ -1059,52 +1175,57 @@ void split_page(struct page *page, unsigned int order)
  * we cheat by calling it from here, in the order > 0 path.  Saves a branch
  * or two.
  */
-static struct page *buffered_rmqueue(struct zone *preferred_zone,
-                        struct zone *zone, int order, gfp_t gfp_flags)
+static inline
+struct page *buffered_rmqueue(struct zone *preferred_zone,
+                        struct zone *zone, int order, gfp_t gfp_flags,
+                        int migratetype)
 {
         unsigned long flags;
         struct page *page;
         int cold = !!(gfp_flags & __GFP_COLD);
         int cpu;
-        int migratetype = allocflags_to_migratetype(gfp_flags);
 
 again:
         cpu  = get_cpu();
         if (likely(order == 0)) {
                 struct per_cpu_pages *pcp;
+                struct list_head *list;
 
                 pcp = &zone_pcp(zone, cpu)->pcp;
+                list = &pcp->lists[migratetype];
                 local_irq_save(flags);
-                if (!pcp->count) {
-                        pcp->count = rmqueue_bulk(zone, 0,
-                                        pcp->batch, &pcp->list, migratetype);
-                        if (unlikely(!pcp->count))
+                if (list_empty(list)) {
+                        pcp->count += rmqueue_bulk(zone, 0,
+                                        pcp->batch, list,
+                                        migratetype, cold);
+                        if (unlikely(list_empty(list)))
                                 goto failed;
                 }
 
-                /* Find a page of the appropriate migrate type */
-                if (cold) {
-                        list_for_each_entry_reverse(page, &pcp->list, lru)
-                                if (page_private(page) == migratetype)
-                                        break;
-                } else {
-                        list_for_each_entry(page, &pcp->list, lru)
-                                if (page_private(page) == migratetype)
-                                        break;
-                }
-
-                /* Allocate more to the pcp list if necessary */
-                if (unlikely(&page->lru == &pcp->list)) {
-                        pcp->count += rmqueue_bulk(zone, 0,
-                                        pcp->batch, &pcp->list, migratetype);
-                        page = list_entry(pcp->list.next, struct page, lru);
-                }
+                if (cold)
+                        page = list_entry(list->prev, struct page, lru);
+                else
+                        page = list_entry(list->next, struct page, lru);
 
                 list_del(&page->lru);
                 pcp->count--;
         } else {
+                if (unlikely(gfp_flags & __GFP_NOFAIL)) {
+                        /*
+                         * __GFP_NOFAIL is not to be used in new code.
+                         *
+                         * All __GFP_NOFAIL callers should be fixed so that they
+                         * properly detect and handle allocation failures.
+                         *
+                         * We most definitely don't want callers attempting to
+                         * allocate greater than order-1 page units with
+                         * __GFP_NOFAIL.
+                         */
+                        WARN_ON_ONCE(order > 1);
+                }
                 spin_lock_irqsave(&zone->lock, flags);
                 page = __rmqueue(zone, order, migratetype);
+                __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order));
                 spin_unlock(&zone->lock);
                 if (!page)
                         goto failed;
@@ -1126,10 +1247,15 @@ failed:
         return NULL;
 }
 
-#define ALLOC_NO_WATERMARKS     0x01 /* don't check watermarks at all */
-#define ALLOC_WMARK_MIN         0x02 /* use pages_min watermark */
-#define ALLOC_WMARK_LOW         0x04 /* use pages_low watermark */
-#define ALLOC_WMARK_HIGH        0x08 /* use pages_high watermark */
+/* The ALLOC_WMARK bits are used as an index to zone->watermark */
+#define ALLOC_WMARK_MIN         WMARK_MIN
+#define ALLOC_WMARK_LOW         WMARK_LOW
+#define ALLOC_WMARK_HIGH        WMARK_HIGH
+#define ALLOC_NO_WATERMARKS     0x04 /* don't check watermarks at all */
+
+/* Mask to get the watermark bits */
+#define ALLOC_WMARK_MASK        (ALLOC_NO_WATERMARKS-1)
+
 #define ALLOC_HARDER            0x10 /* try to alloc harder */
 #define ALLOC_HIGH              0x20 /* __GFP_HIGH set */
 #define ALLOC_CPUSET            0x40 /* check for correct cpuset */
@@ -1387,23 +1513,18 @@ static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  */
 static struct page *
 get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
-                struct zonelist *zonelist, int high_zoneidx, int alloc_flags)
+                struct zonelist *zonelist, int high_zoneidx, int alloc_flags,
+                struct zone *preferred_zone, int migratetype)
 {
         struct zoneref *z;
         struct page *page = NULL;
         int classzone_idx;
-        struct zone *zone, *preferred_zone;
+        struct zone *zone;
         nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */
         int zlc_active = 0;             /* set if using zonelist_cache */
         int did_zlc_setup = 0;          /* just call zlc_setup() one time */
 
-        (void)first_zones_zonelist(zonelist, high_zoneidx, nodemask,
-                                                        &preferred_zone);
-        if (!preferred_zone)
-                return NULL;
-
         classzone_idx = zone_idx(preferred_zone);
-
 zonelist_scan:
         /*
          * Scan zonelist, looking for a zone with enough free.
@@ -1418,31 +1539,49 @@ zonelist_scan:
                         !cpuset_zone_allowed_softwall(zone, gfp_mask))
                                 goto try_next_zone;
 
+                BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
                 if (!(alloc_flags & ALLOC_NO_WATERMARKS)) {
                         unsigned long mark;
-                        if (alloc_flags & ALLOC_WMARK_MIN)
-                                mark = zone->pages_min;
-                        else if (alloc_flags & ALLOC_WMARK_LOW)
-                                mark = zone->pages_low;
-                        else
-                                mark = zone->pages_high;
-                        if (!zone_watermark_ok(zone, order, mark,
-                                    classzone_idx, alloc_flags)) {
-                                if (!zone_reclaim_mode ||
-                                    !zone_reclaim(zone, gfp_mask, order))
+                        int ret;
+
+                        mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
+                        if (zone_watermark_ok(zone, order, mark,
+                                    classzone_idx, alloc_flags))
+                                goto try_this_zone;
+
+                        if (zone_reclaim_mode == 0)
+                                goto this_zone_full;
+
+                        ret = zone_reclaim(zone, gfp_mask, order);
+                        switch (ret) {
+                        case ZONE_RECLAIM_NOSCAN:
+                                /* did not scan */
+                                goto try_next_zone;
+                        case ZONE_RECLAIM_FULL:
+                                /* scanned but unreclaimable */
+                                goto this_zone_full;
+                        default:
+                                /* did we reclaim enough */
+                                if (!zone_watermark_ok(zone, order, mark,
+                                                classzone_idx, alloc_flags))
                                         goto this_zone_full;
                         }
                 }
 
-                page = buffered_rmqueue(preferred_zone, zone, order, gfp_mask);
+try_this_zone:
+                page = buffered_rmqueue(preferred_zone, zone, order,
+                                                gfp_mask, migratetype);
                 if (page)
                         break;
 this_zone_full:
                 if (NUMA_BUILD)
                         zlc_mark_zone_full(zonelist, z);
 try_next_zone:
-                if (NUMA_BUILD && !did_zlc_setup) {
-                        /* we do zlc_setup after the first zone is tried */
+                if (NUMA_BUILD && !did_zlc_setup && nr_online_nodes > 1) {
+                        /*
+                         * we do zlc_setup after the first zone is tried but only
+                         * if there are multiple nodes make it worthwhile
+                         */
                         allowednodes = zlc_setup(zonelist, alloc_flags);
                         zlc_active = 1;
                         did_zlc_setup = 1;
@@ -1457,47 +1596,215 @@ try_next_zone:
         return page;
 }
 
+static inline int
+should_alloc_retry(gfp_t gfp_mask, unsigned int order,
+                                unsigned long pages_reclaimed)
+{
+        /* Do not loop if specifically requested */
+        if (gfp_mask & __GFP_NORETRY)
+                return 0;
+
+        /*
+         * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER
+         * means __GFP_NOFAIL, but that may not be true in other
+         * implementations.
+         */
+        if (order <= PAGE_ALLOC_COSTLY_ORDER)
+                return 1;
+
+        /*
+         * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is
+         * specified, then we retry until we no longer reclaim any pages
+         * (above), or we've reclaimed an order of pages at least as
+         * large as the allocation's order. In both cases, if the
+         * allocation still fails, we stop retrying.
+         */
+        if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order))
+                return 1;
+
+        /*
+         * Don't let big-order allocations loop unless the caller
+         * explicitly requests that.
+         */
+        if (gfp_mask & __GFP_NOFAIL)
+                return 1;
+
+        return 0;
+}
+
+static inline struct page *
+__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
+        struct zonelist *zonelist, enum zone_type high_zoneidx,
+        nodemask_t *nodemask, struct zone *preferred_zone,
+        int migratetype)
+{
+        struct page *page;
+
+        /* Acquire the OOM killer lock for the zones in zonelist */
+        if (!try_set_zone_oom(zonelist, gfp_mask)) {
+                schedule_timeout_uninterruptible(1);
+                return NULL;
+        }
+
+        /*
+         * Go through the zonelist yet one more time, keep very high watermark
+         * here, this is only to catch a parallel oom killing, we must fail if
+         * we're still under heavy pressure.
+         */
+        page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask,
+                order, zonelist, high_zoneidx,
+                ALLOC_WMARK_HIGH|ALLOC_CPUSET,
+                preferred_zone, migratetype);
+        if (page)
+                goto out;
+
+        /* The OOM killer will not help higher order allocs */
+        if (order > PAGE_ALLOC_COSTLY_ORDER && !(gfp_mask & __GFP_NOFAIL))
+                goto out;
+
+        /* Exhausted what can be done so it's blamo time */
+        out_of_memory(zonelist, gfp_mask, order);
+
+out:
+        clear_zonelist_oom(zonelist, gfp_mask);
+        return page;
+}
+
+/* The really slow allocator path where we enter direct reclaim */
+static inline struct page *
+__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
+        struct zonelist *zonelist, enum zone_type high_zoneidx,
+        nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
+        int migratetype, unsigned long *did_some_progress)
+{
+        struct page *page = NULL;
+        struct reclaim_state reclaim_state;
+        struct task_struct *p = current;
+
+        cond_resched();
+
+        /* We now go into synchronous reclaim */
+        cpuset_memory_pressure_bump();
+        p->flags |= PF_MEMALLOC;
+        lockdep_set_current_reclaim_state(gfp_mask);
+        reclaim_state.reclaimed_slab = 0;
+        p->reclaim_state = &reclaim_state;
+
+        *did_some_progress = try_to_free_pages(zonelist, order, gfp_mask, nodemask);
+
+        p->reclaim_state = NULL;
+        lockdep_clear_current_reclaim_state();
+        p->flags &= ~PF_MEMALLOC;
+
+        cond_resched();
+
+        if (order != 0)
+                drain_all_pages();
+
+        if (likely(*did_some_progress))
+                page = get_page_from_freelist(gfp_mask, nodemask, order,
+                                        zonelist, high_zoneidx,
+                                        alloc_flags, preferred_zone,
+                                        migratetype);
+        return page;
+}
+
 /*
- * This is the 'heart' of the zoned buddy allocator.
+ * This is called in the allocator slow-path if the allocation request is of
+ * sufficient urgency to ignore watermarks and take other desperate measures
  */
-struct page *
-__alloc_pages_internal(gfp_t gfp_mask, unsigned int order,
-                        struct zonelist *zonelist, nodemask_t *nodemask)
+static inline struct page *
+__alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order,
+        struct zonelist *zonelist, enum zone_type high_zoneidx,
+        nodemask_t *nodemask, struct zone *preferred_zone,
+        int migratetype)
+{
+        struct page *page;
+
+        do {
+                page = get_page_from_freelist(gfp_mask, nodemask, order,
+                        zonelist, high_zoneidx, ALLOC_NO_WATERMARKS,
+                        preferred_zone, migratetype);
+
+                if (!page && gfp_mask & __GFP_NOFAIL)
+                        congestion_wait(BLK_RW_ASYNC, HZ/50);
+        } while (!page && (gfp_mask & __GFP_NOFAIL));
+
+        return page;
+}
+
+static inline
+void wake_all_kswapd(unsigned int order, struct zonelist *zonelist,
+                                                enum zone_type high_zoneidx)
 {
-        const gfp_t wait = gfp_mask & __GFP_WAIT;
-        enum zone_type high_zoneidx = gfp_zone(gfp_mask);
         struct zoneref *z;
         struct zone *zone;
-        struct page *page;
-        struct reclaim_state reclaim_state;
-        struct task_struct *p = current;
-        int do_retry;
-        int alloc_flags;
-        unsigned long did_some_progress;
-        unsigned long pages_reclaimed = 0;
 
-        lockdep_trace_alloc(gfp_mask);
+        for_each_zone_zonelist(zone, z, zonelist, high_zoneidx)
+                wakeup_kswapd(zone, order);
+}
 
-        might_sleep_if(wait);
+static inline int
+gfp_to_alloc_flags(gfp_t gfp_mask)
+{
+        struct task_struct *p = current;
+        int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
+        const gfp_t wait = gfp_mask & __GFP_WAIT;
 
-        if (should_fail_alloc_page(gfp_mask, order))
-                return NULL;
+        /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
+        BUILD_BUG_ON(__GFP_HIGH != ALLOC_HIGH);
 
-restart:
-        z = zonelist->_zonerefs;  /* the list of zones suitable for gfp_mask */
+        /*
+         * The caller may dip into page reserves a bit more if the caller
+         * cannot run direct reclaim, or if the caller has realtime scheduling
+         * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
+         * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).
+         */
+        alloc_flags |= (gfp_mask & __GFP_HIGH);
 
-        if (unlikely(!z->zone)) {
+        if (!wait) {
+                alloc_flags |= ALLOC_HARDER;
                 /*
-                 * Happens if we have an empty zonelist as a result of
-                 * GFP_THISNODE being used on a memoryless node
+                 * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.
+                 * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
                  */
-                return NULL;
+                alloc_flags &= ~ALLOC_CPUSET;
+        } else if (unlikely(rt_task(p)) && !in_interrupt())
+                alloc_flags |= ALLOC_HARDER;
+
+        if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) {
+                if (!in_interrupt() &&
+                    ((p->flags & PF_MEMALLOC) ||
+                     unlikely(test_thread_flag(TIF_MEMDIE))))
+                        alloc_flags |= ALLOC_NO_WATERMARKS;
         }
 
-        page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
-                        zonelist, high_zoneidx, ALLOC_WMARK_LOW|ALLOC_CPUSET);
-        if (page)
-                goto got_pg;
+        return alloc_flags;
+}
+
+static inline struct page *
+__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
+        struct zonelist *zonelist, enum zone_type high_zoneidx,
+        nodemask_t *nodemask, struct zone *preferred_zone,
+        int migratetype)
+{
+        const gfp_t wait = gfp_mask & __GFP_WAIT;
+        struct page *page = NULL;
+        int alloc_flags;
+        unsigned long pages_reclaimed = 0;
+        unsigned long did_some_progress;
+        struct task_struct *p = current;
+
+        /*
+         * In the slowpath, we sanity check order to avoid ever trying to
+         * reclaim >= MAX_ORDER areas which will never succeed. Callers may
+         * be using allocators in order of preference for an area that is
+         * too large.
+         */
+        if (order >= MAX_ORDER) {
+                WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
+                return NULL;
+        }
 
         /*
          * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and
@@ -1510,155 +1817,88 @@ restart:
         if (NUMA_BUILD && (gfp_mask & GFP_THISNODE) == GFP_THISNODE)
                 goto nopage;
 
-        for_each_zone_zonelist(zone, z, zonelist, high_zoneidx)
-                wakeup_kswapd(zone, order);
+restart:
+        wake_all_kswapd(order, zonelist, high_zoneidx);
 
         /*
          * OK, we're below the kswapd watermark and have kicked background
          * reclaim. Now things get more complex, so set up alloc_flags according
          * to how we want to proceed.
-         *
-         * The caller may dip into page reserves a bit more if the caller
-         * cannot run direct reclaim, or if the caller has realtime scheduling
-         * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
-         * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).
          */
-        alloc_flags = ALLOC_WMARK_MIN;
-        if ((unlikely(rt_task(p)) && !in_interrupt()) || !wait)
-                alloc_flags |= ALLOC_HARDER;
-        if (gfp_mask & __GFP_HIGH)
-                alloc_flags |= ALLOC_HIGH;
-        if (wait)
-                alloc_flags |= ALLOC_CPUSET;
+        alloc_flags = gfp_to_alloc_flags(gfp_mask);
 
-        /*
-         * Go through the zonelist again. Let __GFP_HIGH and allocations
-         * coming from realtime tasks go deeper into reserves.
-         *
-         * This is the last chance, in general, before the goto nopage.
-         * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.
-         * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
-         */
+        /* This is the last chance, in general, before the goto nopage. */
         page = get_page_from_freelist(gfp_mask, nodemask, order, zonelist,
-                                                high_zoneidx, alloc_flags);
+                        high_zoneidx, alloc_flags & ~ALLOC_NO_WATERMARKS,
+                        preferred_zone, migratetype);
         if (page)
                 goto got_pg;
 
-        /* This allocation should allow future memory freeing. */
-
 rebalance:
-        if (((p->flags & PF_MEMALLOC) || unlikely(test_thread_flag(TIF_MEMDIE)))
-                        && !in_interrupt()) {
-                if (!(gfp_mask & __GFP_NOMEMALLOC)) {
-nofail_alloc:
-                        /* go through the zonelist yet again, ignoring mins */
-                        page = get_page_from_freelist(gfp_mask, nodemask, order,
-                                zonelist, high_zoneidx, ALLOC_NO_WATERMARKS);
-                        if (page)
-                                goto got_pg;
-                        if (gfp_mask & __GFP_NOFAIL) {
-                                congestion_wait(WRITE, HZ/50);
-                                goto nofail_alloc;
-                        }
-                }
-                goto nopage;
+        /* Allocate without watermarks if the context allows */
+        if (alloc_flags & ALLOC_NO_WATERMARKS) {
+                page = __alloc_pages_high_priority(gfp_mask, order,
+                                zonelist, high_zoneidx, nodemask,
+                                preferred_zone, migratetype);
+                if (page)
+                        goto got_pg;
         }
 
         /* Atomic allocations - we can't balance anything */
         if (!wait)
                 goto nopage;
 
-        cond_resched();
-
-        /* We now go into synchronous reclaim */
-        cpuset_memory_pressure_bump();
-        /*
-         * The task's cpuset might have expanded its set of allowable nodes
-         */
-        cpuset_update_task_memory_state();
-        p->flags |= PF_MEMALLOC;
-
-        lockdep_set_current_reclaim_state(gfp_mask);
-        reclaim_state.reclaimed_slab = 0;
-        p->reclaim_state = &reclaim_state;
+        /* Avoid recursion of direct reclaim */
+        if (p->flags & PF_MEMALLOC)
+                goto nopage;
 
-        did_some_progress = try_to_free_pages(zonelist, order,
-                                                gfp_mask, nodemask);
+        /* Avoid allocations with no watermarks from looping endlessly */
+        if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
+                goto nopage;
 
-        p->reclaim_state = NULL;
-        lockdep_clear_current_reclaim_state();
-        p->flags &= ~PF_MEMALLOC;
+        /* Try direct reclaim and then allocating */
+        page = __alloc_pages_direct_reclaim(gfp_mask, order,
+                                        zonelist, high_zoneidx,
+                                        nodemask,
+                                        alloc_flags, preferred_zone,
+                                        migratetype, &did_some_progress);
+        if (page)
+                goto got_pg;
 
-        cond_resched();
+        /*
+         * If we failed to make any progress reclaiming, then we are
+         * running out of options and have to consider going OOM
+         */
+        if (!did_some_progress) {
+                if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) {
+                        if (oom_killer_disabled)
+                                goto nopage;
+                        page = __alloc_pages_may_oom(gfp_mask, order,
+                                        zonelist, high_zoneidx,
+                                        nodemask, preferred_zone,
+                                        migratetype);
+                        if (page)
+                                goto got_pg;
 
-        if (order != 0)
-                drain_all_pages();
+                        /*
+                         * The OOM killer does not trigger for high-order
+                         * ~__GFP_NOFAIL allocations so if no progress is being
+                         * made, there are no other options and retrying is
+                         * unlikely to help.
+                         */
+                        if (order > PAGE_ALLOC_COSTLY_ORDER &&
+                                                !(gfp_mask & __GFP_NOFAIL))
+                                goto nopage;
 
-        if (likely(did_some_progress)) {
-                page = get_page_from_freelist(gfp_mask, nodemask, order,
-                                        zonelist, high_zoneidx, alloc_flags);
-                if (page)
-                        goto got_pg;
-        } else if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) {
-                if (!try_set_zone_oom(zonelist, gfp_mask)) {
-                        schedule_timeout_uninterruptible(1);
                         goto restart;
                 }
-
-                /*
-                 * Go through the zonelist yet one more time, keep
-                 * very high watermark here, this is only to catch
-                 * a parallel oom killing, we must fail if we're still
-                 * under heavy pressure.
-                 */
-                page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask,
-                        order, zonelist, high_zoneidx,
-                        ALLOC_WMARK_HIGH|ALLOC_CPUSET);
-                if (page) {
-                        clear_zonelist_oom(zonelist, gfp_mask);
-                        goto got_pg;
-                }
-
-                /* The OOM killer will not help higher order allocs so fail */
-                if (order > PAGE_ALLOC_COSTLY_ORDER) {
-                        clear_zonelist_oom(zonelist, gfp_mask);
-                        goto nopage;
-                }
-
-                out_of_memory(zonelist, gfp_mask, order);
-                clear_zonelist_oom(zonelist, gfp_mask);
-                goto restart;
         }
 
-        /*
-         * Don't let big-order allocations loop unless the caller explicitly
-         * requests that.  Wait for some write requests to complete then retry.
-         *
-         * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER
-         * means __GFP_NOFAIL, but that may not be true in other
-         * implementations.
-         *
-         * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is
-         * specified, then we retry until we no longer reclaim any pages
-         * (above), or we've reclaimed an order of pages at least as
-         * large as the allocation's order. In both cases, if the
-         * allocation still fails, we stop retrying.
-         */
+        /* Check if we should retry the allocation */
         pages_reclaimed += did_some_progress;
-        do_retry = 0;
-        if (!(gfp_mask & __GFP_NORETRY)) {
-                if (order <= PAGE_ALLOC_COSTLY_ORDER) {
-                        do_retry = 1;
-                } else {
-                        if (gfp_mask & __GFP_REPEAT &&
-                                pages_reclaimed < (1 << order))
-                                        do_retry = 1;
-                }
-                if (gfp_mask & __GFP_NOFAIL)
-                        do_retry = 1;
-        }
-        if (do_retry) {
-                congestion_wait(WRITE, HZ/50);
+        if (should_alloc_retry(gfp_mask, order, pages_reclaimed)) {
+                /* Wait for some write requests to complete then retry */
+                congestion_wait(BLK_RW_ASYNC, HZ/50);
                 goto rebalance;
         }
 
@@ -1670,54 +1910,102 @@ nopage:
                 dump_stack();
                 show_mem();
         }
+        return page;
 got_pg:
+        if (kmemcheck_enabled)
+                kmemcheck_pagealloc_alloc(page, order, gfp_mask);
         return page;
+
 }
-EXPORT_SYMBOL(__alloc_pages_internal);
+
+/*
+ * This is the 'heart' of the zoned buddy allocator.
+ */
+struct page *
+__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
+                        struct zonelist *zonelist, nodemask_t *nodemask)
+{
+        enum zone_type high_zoneidx = gfp_zone(gfp_mask);
+        struct zone *preferred_zone;
+        struct page *page;
+        int migratetype = allocflags_to_migratetype(gfp_mask);
+
+        gfp_mask &= gfp_allowed_mask;
+
+        lockdep_trace_alloc(gfp_mask);
+
+        might_sleep_if(gfp_mask & __GFP_WAIT);
+
+        if (should_fail_alloc_page(gfp_mask, order))
+                return NULL;
+
+        /*
+         * Check the zones suitable for the gfp_mask contain at least one
+         * valid zone. It's possible to have an empty zonelist as a result
+         * of GFP_THISNODE and a memoryless node
+         */
+        if (unlikely(!zonelist->_zonerefs->zone))
+                return NULL;
+
+        /* The preferred zone is used for statistics later */
+        first_zones_zonelist(zonelist, high_zoneidx, nodemask, &preferred_zone);
+        if (!preferred_zone)
+                return NULL;
+
+        /* First allocation attempt */
+        page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
+                        zonelist, high_zoneidx, ALLOC_WMARK_LOW|ALLOC_CPUSET,
+                        preferred_zone, migratetype);
+        if (unlikely(!page))
+                page = __alloc_pages_slowpath(gfp_mask, order,
+                                zonelist, high_zoneidx, nodemask,
+                                preferred_zone, migratetype);
+
+        trace_mm_page_alloc(page, order, gfp_mask, migratetype);
+        return page;
+}
+EXPORT_SYMBOL(__alloc_pages_nodemask);
 
 /*
  * Common helper functions.
  */
 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
 {
-        struct page * page;
+        struct page *page;
+
+        /*
+         * __get_free_pages() returns a 32-bit address, which cannot represent
+         * a highmem page
+         */
+        VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
+
         page = alloc_pages(gfp_mask, order);
         if (!page)
                 return 0;
         return (unsigned long) page_address(page);
 }
-
 EXPORT_SYMBOL(__get_free_pages);
 
 unsigned long get_zeroed_page(gfp_t gfp_mask)
 {
-        struct page * page;
-
-        /*
-         * get_zeroed_page() returns a 32-bit address, which cannot represent
-         * a highmem page
-         */
-        VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
-
-        page = alloc_pages(gfp_mask | __GFP_ZERO, 0);
-        if (page)
-                return (unsigned long) page_address(page);
-        return 0;
+        return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
 }
-
 EXPORT_SYMBOL(get_zeroed_page);
 
 void __pagevec_free(struct pagevec *pvec)
 {
         int i = pagevec_count(pvec);
 
-        while (--i >= 0)
+        while (--i >= 0) {
+                trace_mm_pagevec_free(pvec->pages[i], pvec->cold);
                 free_hot_cold_page(pvec->pages[i], pvec->cold);
+        }
 }
 
 void __free_pages(struct page *page, unsigned int order)
 {
         if (put_page_testzero(page)) {
+                trace_mm_page_free_direct(page, order);
                 if (order == 0)
                         free_hot_page(page);
                 else
@@ -1760,7 +2048,7 @@ void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
                 unsigned long alloc_end = addr + (PAGE_SIZE << order);
                 unsigned long used = addr + PAGE_ALIGN(size);
 
-                split_page(virt_to_page(addr), order);
+                split_page(virt_to_page((void *)addr), order);
                 while (used < alloc_end) {
                         free_page(used);
                         used += PAGE_SIZE;
@@ -1802,7 +2090,7 @@ static unsigned int nr_free_zone_pages(int offset)
 
         for_each_zone_zonelist(zone, z, zonelist, offset) {
                 unsigned long size = zone->present_pages;
-                unsigned long high = zone->pages_high;
+                unsigned long high = high_wmark_pages(zone);
                 if (size > high)
                         sum += size - high;
         }
@@ -1892,28 +2180,27 @@ void show_free_areas(void)
                 }
         }
 
-        printk("Active_anon:%lu active_file:%lu inactive_anon:%lu\n"
-                " inactive_file:%lu"
-//TODO:  check/adjust line lengths
-#ifdef CONFIG_UNEVICTABLE_LRU
+        printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
+                " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
                 " unevictable:%lu"
-#endif
                 " dirty:%lu writeback:%lu unstable:%lu\n"
-                " free:%lu slab:%lu mapped:%lu pagetables:%lu bounce:%lu\n",
+                " free:%lu slab_reclaimable:%lu slab_unreclaimable:%lu\n"
+                " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n",
                 global_page_state(NR_ACTIVE_ANON),
-                global_page_state(NR_ACTIVE_FILE),
                 global_page_state(NR_INACTIVE_ANON),
+                global_page_state(NR_ISOLATED_ANON),
+                global_page_state(NR_ACTIVE_FILE),
                 global_page_state(NR_INACTIVE_FILE),
-#ifdef CONFIG_UNEVICTABLE_LRU
+                global_page_state(NR_ISOLATED_FILE),
                 global_page_state(NR_UNEVICTABLE),
-#endif
                 global_page_state(NR_FILE_DIRTY),
                 global_page_state(NR_WRITEBACK),
                 global_page_state(NR_UNSTABLE_NFS),
                 global_page_state(NR_FREE_PAGES),
-                global_page_state(NR_SLAB_RECLAIMABLE) +
-                        global_page_state(NR_SLAB_UNRECLAIMABLE),
+                global_page_state(NR_SLAB_RECLAIMABLE),
+                global_page_state(NR_SLAB_UNRECLAIMABLE),
                 global_page_state(NR_FILE_MAPPED),
+                global_page_state(NR_SHMEM),
                 global_page_state(NR_PAGETABLE),
                 global_page_state(NR_BOUNCE));
 
@@ -1930,26 +2217,51 @@ void show_free_areas(void)
                         " inactive_anon:%lukB"
                         " active_file:%lukB"
                         " inactive_file:%lukB"
-#ifdef CONFIG_UNEVICTABLE_LRU
                         " unevictable:%lukB"
-#endif
+                        " isolated(anon):%lukB"
+                        " isolated(file):%lukB"
                         " present:%lukB"
+                        " mlocked:%lukB"
+                        " dirty:%lukB"
+                        " writeback:%lukB"
+                        " mapped:%lukB"
+                        " shmem:%lukB"
+                        " slab_reclaimable:%lukB"
+                        " slab_unreclaimable:%lukB"
+                        " kernel_stack:%lukB"
+                        " pagetables:%lukB"
+                        " unstable:%lukB"
+                        " bounce:%lukB"
+                        " writeback_tmp:%lukB"
                         " pages_scanned:%lu"
                         " all_unreclaimable? %s"
                         "\n",
                         zone->name,
                         K(zone_page_state(zone, NR_FREE_PAGES)),
-                        K(zone->pages_min),
-                        K(zone->pages_low),
-                        K(zone->pages_high),
+                        K(min_wmark_pages(zone)),
+                        K(low_wmark_pages(zone)),
+                        K(high_wmark_pages(zone)),
                         K(zone_page_state(zone, NR_ACTIVE_ANON)),
                         K(zone_page_state(zone, NR_INACTIVE_ANON)),
                         K(zone_page_state(zone, NR_ACTIVE_FILE)),
                         K(zone_page_state(zone, NR_INACTIVE_FILE)),
-#ifdef CONFIG_UNEVICTABLE_LRU
                         K(zone_page_state(zone, NR_UNEVICTABLE)),
-#endif
+                        K(zone_page_state(zone, NR_ISOLATED_ANON)),
+                        K(zone_page_state(zone, NR_ISOLATED_FILE)),
                         K(zone->present_pages),
+                        K(zone_page_state(zone, NR_MLOCK)),
+                        K(zone_page_state(zone, NR_FILE_DIRTY)),
+                        K(zone_page_state(zone, NR_WRITEBACK)),
+                        K(zone_page_state(zone, NR_FILE_MAPPED)),
+                        K(zone_page_state(zone, NR_SHMEM)),
+                        K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)),
+                        K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)),
+                        zone_page_state(zone, NR_KERNEL_STACK) *
+                                THREAD_SIZE / 1024,
+                        K(zone_page_state(zone, NR_PAGETABLE)),
+                        K(zone_page_state(zone, NR_UNSTABLE_NFS)),
+                        K(zone_page_state(zone, NR_BOUNCE)),
+                        K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
                         zone->pages_scanned,
                         (zone_is_all_unreclaimable(zone) ? "yes" : "no")
                         );
@@ -2078,7 +2390,7 @@ early_param("numa_zonelist_order", setup_numa_zonelist_order);
  * sysctl handler for numa_zonelist_order
  */
 int numa_zonelist_order_handler(ctl_table *table, int write,
-                struct file *file, void __user *buffer, size_t *length,
+                void __user *buffer, size_t *length,
                 loff_t *ppos)
 {
         char saved_string[NUMA_ZONELIST_ORDER_LEN];
@@ -2087,7 +2399,7 @@ int numa_zonelist_order_handler(ctl_table *table, int write,
         if (write)
                 strncpy(saved_string, (char*)table->data,
                         NUMA_ZONELIST_ORDER_LEN);
-        ret = proc_dostring(table, write, file, buffer, length, ppos);
+        ret = proc_dostring(table, write, buffer, length, ppos);
         if (ret)
                 return ret;
         if (write) {
@@ -2106,7 +2418,7 @@ int numa_zonelist_order_handler(ctl_table *table, int write,
 }
 
 
-#define MAX_NODE_LOAD (num_online_nodes())
+#define MAX_NODE_LOAD (nr_online_nodes)
 static int node_load[MAX_NUMNODES];
 
 /**
@@ -2128,7 +2440,7 @@ static int find_next_best_node(int node, nodemask_t *used_node_mask)
         int n, val;
         int min_val = INT_MAX;
         int best_node = -1;
-        node_to_cpumask_ptr(tmp, 0);
+        const struct cpumask *tmp = cpumask_of_node(0);
 
         /* Use the local node if we haven't already */
         if (!node_isset(node, *used_node_mask)) {
@@ -2149,8 +2461,8 @@ static int find_next_best_node(int node, nodemask_t *used_node_mask)
                 val += (n < node);
 
                 /* Give preference to headless and unused nodes */
-                node_to_cpumask_ptr_next(tmp, n);
-                if (!cpus_empty(*tmp))
+                tmp = cpumask_of_node(n);
+                if (!cpumask_empty(tmp))
                         val += PENALTY_FOR_NODE_WITH_CPUS;
 
                 /* Slight preference for less loaded node */
@@ -2315,11 +2627,10 @@ static void build_zonelists(pg_data_t *pgdat)
 
         /* NUMA-aware ordering of nodes */
         local_node = pgdat->node_id;
-        load = num_online_nodes();
+        load = nr_online_nodes;
         prev_node = local_node;
         nodes_clear(used_mask);
 
-        memset(node_load, 0, sizeof(node_load));
         memset(node_order, 0, sizeof(node_order));
         j = 0;
 
@@ -2428,6 +2739,9 @@ static int __build_all_zonelists(void *dummy)
 {
         int nid;
 
+#ifdef CONFIG_NUMA
+        memset(node_load, 0, sizeof(node_load));
+#endif
         for_each_online_node(nid) {
                 pg_data_t *pgdat = NODE_DATA(nid);
 
@@ -2466,7 +2780,7 @@ void build_all_zonelists(void)
 
         printk("Built %i zonelists in %s order, mobility grouping %s.  "
                 "Total pages: %ld\n",
-                        num_online_nodes(),
+                        nr_online_nodes,
                         zonelist_order_name[current_zonelist_order],
                         page_group_by_mobility_disabled ? "off" : "on",
                         vm_total_pages);
@@ -2545,8 +2859,8 @@ static inline unsigned long wait_table_bits(unsigned long size)
 
 /*
  * Mark a number of pageblocks as MIGRATE_RESERVE. The number
- * of blocks reserved is based on zone->pages_min. The memory within the
- * reserve will tend to store contiguous free pages. Setting min_free_kbytes
+ * of blocks reserved is based on min_wmark_pages(zone). The memory within
+ * the reserve will tend to store contiguous free pages. Setting min_free_kbytes
  * higher will lead to a bigger reserve which will get freed as contiguous
  * blocks as reclaim kicks in
  */
@@ -2554,14 +2868,24 @@ static void setup_zone_migrate_reserve(struct zone *zone)
 {
         unsigned long start_pfn, pfn, end_pfn;
         struct page *page;
-        unsigned long reserve, block_migratetype;
+        unsigned long block_migratetype;
+        int reserve;
 
         /* Get the start pfn, end pfn and the number of blocks to reserve */
         start_pfn = zone->zone_start_pfn;
         end_pfn = start_pfn + zone->spanned_pages;
-        reserve = roundup(zone->pages_min, pageblock_nr_pages) >>
+        reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >>
                                                         pageblock_order;
 
+        /*
+         * Reserve blocks are generally in place to help high-order atomic
+         * allocations that are short-lived. A min_free_kbytes value that
+         * would result in more than 2 reserve blocks for atomic allocations
+         * is assumed to be in place to help anti-fragmentation for the
+         * future allocation of hugepages at runtime.
+         */
+        reserve = min(2, reserve);
+
         for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
                 if (!pfn_valid(pfn))
                         continue;
@@ -2681,6 +3005,7 @@ static void __meminit zone_init_free_lists(struct zone *zone)
 
 static int zone_batchsize(struct zone *zone)
 {
+#ifdef CONFIG_MMU
         int batch;
 
         /*
@@ -2706,14 +3031,32 @@ static int zone_batchsize(struct zone *zone)
          * of pages of one half of the possible page colors
          * and the other with pages of the other colors.
          */
-        batch = (1 << (fls(batch + batch/2)-1)) - 1;
+        batch = rounddown_pow_of_two(batch + batch/2) - 1;
 
         return batch;
+
+#else
+        /* The deferral and batching of frees should be suppressed under NOMMU
+         * conditions.
+         *
+         * The problem is that NOMMU needs to be able to allocate large chunks
+         * of contiguous memory as there's no hardware page translation to
+         * assemble apparent contiguous memory from discontiguous pages.
+         *
+         * Queueing large contiguous runs of pages for batching, however,
+         * causes the pages to actually be freed in smaller chunks.  As there
+         * can be a significant delay between the individual batches being
+         * recycled, this leads to the once large chunks of space being
+         * fragmented and becoming unavailable for high-order allocations.
+         */
+        return 0;
+#endif
 }
 
 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
 {
         struct per_cpu_pages *pcp;
+        int migratetype;
 
         memset(p, 0, sizeof(*p));
 
@@ -2721,7 +3064,8 @@ static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
         pcp->count = 0;
         pcp->high = 6 * batch;
         pcp->batch = max(1UL, 1 * batch);
-        INIT_LIST_HEAD(&pcp->list);
+        for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
+                INIT_LIST_HEAD(&pcp->lists[migratetype]);
 }
 
 /*
@@ -2794,7 +3138,7 @@ bad:
                 if (dzone == zone)
                         break;
                 kfree(zone_pcp(dzone, cpu));
-                zone_pcp(dzone, cpu) = NULL;
+                zone_pcp(dzone, cpu) = &boot_pageset[cpu];
         }
         return -ENOMEM;
 }
@@ -2809,7 +3153,7 @@ static inline void free_zone_pagesets(int cpu)
                 /* Free per_cpu_pageset if it is slab allocated */
                 if (pset != &boot_pageset[cpu])
                         kfree(pset);
-                zone_pcp(zone, cpu) = NULL;
+                zone_pcp(zone, cpu) = &boot_pageset[cpu];
         }
 }
 
@@ -2899,6 +3243,32 @@ int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
         return 0;
 }
 
+static int __zone_pcp_update(void *data)
+{
+        struct zone *zone = data;
+        int cpu;
+        unsigned long batch = zone_batchsize(zone), flags;
+
+        for (cpu = 0; cpu < NR_CPUS; cpu++) {
+                struct per_cpu_pageset *pset;
+                struct per_cpu_pages *pcp;
+
+                pset = zone_pcp(zone, cpu);
+                pcp = &pset->pcp;
+
+                local_irq_save(flags);
+                free_pcppages_bulk(zone, pcp->count, pcp);
+                setup_pageset(pset, batch);
+                local_irq_restore(flags);
+        }
+        return 0;
+}
+
+void zone_pcp_update(struct zone *zone)
+{
+        stop_machine(__zone_pcp_update, zone, NULL);
+}
+
 static __meminit void zone_pcp_init(struct zone *zone)
 {
         int cpu;
@@ -3085,64 +3455,6 @@ void __init sparse_memory_present_with_active_regions(int nid)
 }
 
 /**
- * push_node_boundaries - Push node boundaries to at least the requested boundary
- * @nid: The nid of the node to push the boundary for
- * @start_pfn: The start pfn of the node
- * @end_pfn: The end pfn of the node
- *
- * In reserve-based hot-add, mem_map is allocated that is unused until hotadd
- * time. Specifically, on x86_64, SRAT will report ranges that can potentially
- * be hotplugged even though no physical memory exists. This function allows
- * an arch to push out the node boundaries so mem_map is allocated that can
- * be used later.
- */
-#ifdef CONFIG_MEMORY_HOTPLUG_RESERVE
-void __init push_node_boundaries(unsigned int nid,
-                unsigned long start_pfn, unsigned long end_pfn)
-{
-        mminit_dprintk(MMINIT_TRACE, "zoneboundary",
-                        "Entering push_node_boundaries(%u, %lu, %lu)\n",
-                        nid, start_pfn, end_pfn);
-
-        /* Initialise the boundary for this node if necessary */
-        if (node_boundary_end_pfn[nid] == 0)
-                node_boundary_start_pfn[nid] = -1UL;
-
-        /* Update the boundaries */
-        if (node_boundary_start_pfn[nid] > start_pfn)
-                node_boundary_start_pfn[nid] = start_pfn;
-        if (node_boundary_end_pfn[nid] < end_pfn)
-                node_boundary_end_pfn[nid] = end_pfn;
-}
-
-/* If necessary, push the node boundary out for reserve hotadd */
-static void __meminit account_node_boundary(unsigned int nid,
-                unsigned long *start_pfn, unsigned long *end_pfn)
-{
-        mminit_dprintk(MMINIT_TRACE, "zoneboundary",
-                        "Entering account_node_boundary(%u, %lu, %lu)\n",
-                        nid, *start_pfn, *end_pfn);
-
-        /* Return if boundary information has not been provided */
-        if (node_boundary_end_pfn[nid] == 0)
-                return;
-
-        /* Check the boundaries and update if necessary */
-        if (node_boundary_start_pfn[nid] < *start_pfn)
-                *start_pfn = node_boundary_start_pfn[nid];
-        if (node_boundary_end_pfn[nid] > *end_pfn)
-                *end_pfn = node_boundary_end_pfn[nid];
-}
-#else
-void __init push_node_boundaries(unsigned int nid,
-                unsigned long start_pfn, unsigned long end_pfn) {}
-
-static void __meminit account_node_boundary(unsigned int nid,
-                unsigned long *start_pfn, unsigned long *end_pfn) {}
-#endif
-
-
-/**
  * get_pfn_range_for_nid - Return the start and end page frames for a node
  * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
  * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
@@ -3167,9 +3479,6 @@ void __meminit get_pfn_range_for_nid(unsigned int nid,
 
         if (*start_pfn == -1UL)
                 *start_pfn = 0;
-
-        /* Push the node boundaries out if requested */
-        account_node_boundary(nid, start_pfn, end_pfn);
 }
 
 /*
@@ -3534,7 +3843,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
                 zone_pcp_init(zone);
                 for_each_lru(l) {
                         INIT_LIST_HEAD(&zone->lru[l].list);
-                        zone->lru[l].nr_scan = 0;
+                        zone->reclaim_stat.nr_saved_scan[l] = 0;
                 }
                 zone->reclaim_stat.recent_rotated[0] = 0;
                 zone->reclaim_stat.recent_rotated[1] = 0;
@@ -3775,10 +4084,6 @@ void __init remove_all_active_ranges(void)
 {
         memset(early_node_map, 0, sizeof(early_node_map));
         nr_nodemap_entries = 0;
-#ifdef CONFIG_MEMORY_HOTPLUG_RESERVE
-        memset(node_boundary_start_pfn, 0, sizeof(node_boundary_start_pfn));
-        memset(node_boundary_end_pfn, 0, sizeof(node_boundary_end_pfn));
-#endif /* CONFIG_MEMORY_HOTPLUG_RESERVE */
 }
 
 /* Compare two active node_active_regions */
@@ -3865,6 +4170,8 @@ static void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn)
         int i, nid;
         unsigned long usable_startpfn;
         unsigned long kernelcore_node, kernelcore_remaining;
+        /* save the state before borrow the nodemask */
+        nodemask_t saved_node_state = node_states[N_HIGH_MEMORY];
         unsigned long totalpages = early_calculate_totalpages();
         int usable_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
 
@@ -3892,7 +4199,7 @@ static void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn)
 
         /* If kernelcore was not specified, there is no ZONE_MOVABLE */
         if (!required_kernelcore)
-                return;
+                goto out;
 
         /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
         find_usable_zone_for_movable();
@@ -3991,6 +4298,10 @@ restart:
         for (nid = 0; nid < MAX_NUMNODES; nid++)
                 zone_movable_pfn[nid] =
                         roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
+
+out:
+        /* restore the node_state */
+        node_states[N_HIGH_MEMORY] = saved_node_state;
 }
 
 /* Any regular memory on that node ? */
@@ -4209,8 +4520,8 @@ static void calculate_totalreserve_pages(void)
                                         max = zone->lowmem_reserve[j];
                         }
 
-                        /* we treat pages_high as reserved pages. */
-                        max += zone->pages_high;
+                        /* we treat the high watermark as reserved pages. */
+                        max += high_wmark_pages(zone);
 
                         if (max > zone->present_pages)
                                 max = zone->present_pages;
@@ -4260,12 +4571,13 @@ static void setup_per_zone_lowmem_reserve(void)
 }
 
 /**
- * setup_per_zone_pages_min - called when min_free_kbytes changes.
+ * setup_per_zone_wmarks - called when min_free_kbytes changes
+ * or when memory is hot-{added|removed}
  *
- * Ensures that the pages_{min,low,high} values for each zone are set correctly
- * with respect to min_free_kbytes.
+ * Ensures that the watermark[min,low,high] values for each zone are set
+ * correctly with respect to min_free_kbytes.
  */
-void setup_per_zone_pages_min(void)
+void setup_per_zone_wmarks(void)
 {
         unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
         unsigned long lowmem_pages = 0;
@@ -4290,7 +4602,7 @@ void setup_per_zone_pages_min(void)
                          * need highmem pages, so cap pages_min to a small
                          * value here.
                          *
-                         * The (pages_high-pages_low) and (pages_low-pages_min)
+                         * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
                          * deltas controls asynch page reclaim, and so should
                          * not be capped for highmem.
                          */
@@ -4301,17 +4613,17 @@ void setup_per_zone_pages_min(void)
                                 min_pages = SWAP_CLUSTER_MAX;
                         if (min_pages > 128)
                                 min_pages = 128;
-                        zone->pages_min = min_pages;
+                        zone->watermark[WMARK_MIN] = min_pages;
                 } else {
                         /*
                          * If it's a lowmem zone, reserve a number of pages
                          * proportionate to the zone's size.
                          */
-                        zone->pages_min = tmp;
+                        zone->watermark[WMARK_MIN] = tmp;
                 }
 
-                zone->pages_low   = zone->pages_min + (tmp >> 2);
-                zone->pages_high  = zone->pages_min + (tmp >> 1);
+                zone->watermark[WMARK_LOW]  = min_wmark_pages(zone) + (tmp >> 2);
+                zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
                 setup_zone_migrate_reserve(zone);
                 spin_unlock_irqrestore(&zone->lock, flags);
         }
@@ -4320,9 +4632,7 @@ void setup_per_zone_pages_min(void)
         calculate_totalreserve_pages();
 }
 
-/**
- * setup_per_zone_inactive_ratio - called when min_free_kbytes changes.
- *
+/*
  * The inactive anon list should be small enough that the VM never has to
  * do too much work, but large enough that each inactive page has a chance
  * to be referenced again before it is swapped out.
@@ -4343,21 +4653,26 @@ void setup_per_zone_pages_min(void)
  *    1TB     101        10GB
  *   10TB     320        32GB
  */
-static void setup_per_zone_inactive_ratio(void)
+void calculate_zone_inactive_ratio(struct zone *zone)
 {
-        struct zone *zone;
+        unsigned int gb, ratio;
 
-        for_each_zone(zone) {
-                unsigned int gb, ratio;
-
-                /* Zone size in gigabytes */
-                gb = zone->present_pages >> (30 - PAGE_SHIFT);
+        /* Zone size in gigabytes */
+        gb = zone->present_pages >> (30 - PAGE_SHIFT);
+        if (gb)
                 ratio = int_sqrt(10 * gb);
-                if (!ratio)
-                        ratio = 1;
+        else
+                ratio = 1;
 
-                zone->inactive_ratio = ratio;
-        }
+        zone->inactive_ratio = ratio;
+}
+
+static void __init setup_per_zone_inactive_ratio(void)
+{
+        struct zone *zone;
+
+        for_each_zone(zone)
+                calculate_zone_inactive_ratio(zone);
 }
 
 /*
@@ -4384,7 +4699,7 @@ static void setup_per_zone_inactive_ratio(void)
  * 8192MB:      11584k
  * 16384MB:     16384k
  */
-static int __init init_per_zone_pages_min(void)
+static int __init init_per_zone_wmark_min(void)
 {
         unsigned long lowmem_kbytes;
 
@@ -4395,12 +4710,12 @@ static int __init init_per_zone_pages_min(void)
                 min_free_kbytes = 128;
         if (min_free_kbytes > 65536)
                 min_free_kbytes = 65536;
-        setup_per_zone_pages_min();
+        setup_per_zone_wmarks();
         setup_per_zone_lowmem_reserve();
         setup_per_zone_inactive_ratio();
         return 0;
 }
-module_init(init_per_zone_pages_min)
+module_init(init_per_zone_wmark_min)
 
 /*
  * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 
@@ -4408,22 +4723,22 @@ module_init(init_per_zone_pages_min)
  *      changes.
  */
 int min_free_kbytes_sysctl_handler(ctl_table *table, int write, 
-        struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
+        void __user *buffer, size_t *length, loff_t *ppos)
 {
-        proc_dointvec(table, write, file, buffer, length, ppos);
+        proc_dointvec(table, write, buffer, length, ppos);
         if (write)
-                setup_per_zone_pages_min();
+                setup_per_zone_wmarks();
         return 0;
 }
 
 #ifdef CONFIG_NUMA
 int sysctl_min_unmapped_ratio_sysctl_handler(ctl_table *table, int write,
-        struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
+        void __user *buffer, size_t *length, loff_t *ppos)
 {
         struct zone *zone;
         int rc;
 
-        rc = proc_dointvec_minmax(table, write, file, buffer, length, ppos);
+        rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
         if (rc)
                 return rc;
 
@@ -4434,12 +4749,12 @@ int sysctl_min_unmapped_ratio_sysctl_handler(ctl_table *table, int write,
 }
 
 int sysctl_min_slab_ratio_sysctl_handler(ctl_table *table, int write,
-        struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
+        void __user *buffer, size_t *length, loff_t *ppos)
 {
         struct zone *zone;
         int rc;
 
-        rc = proc_dointvec_minmax(table, write, file, buffer, length, ppos);
+        rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
         if (rc)
                 return rc;
 
@@ -4456,13 +4771,13 @@ int sysctl_min_slab_ratio_sysctl_handler(ctl_table *table, int write,
  *      whenever sysctl_lowmem_reserve_ratio changes.
  *
  * The reserve ratio obviously has absolutely no relation with the
- * pages_min watermarks. The lowmem reserve ratio can only make sense
+ * minimum watermarks. The lowmem reserve ratio can only make sense
  * if in function of the boot time zone sizes.
  */
 int lowmem_reserve_ratio_sysctl_handler(ctl_table *table, int write,
-        struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
+        void __user *buffer, size_t *length, loff_t *ppos)
 {
-        proc_dointvec_minmax(table, write, file, buffer, length, ppos);
+        proc_dointvec_minmax(table, write, buffer, length, ppos);
         setup_per_zone_lowmem_reserve();
         return 0;
 }
@@ -4474,16 +4789,16 @@ int lowmem_reserve_ratio_sysctl_handler(ctl_table *table, int write,
  */
 
 int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
-        struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
+        void __user *buffer, size_t *length, loff_t *ppos)
 {
         struct zone *zone;
         unsigned int cpu;
         int ret;
 
-        ret = proc_dointvec_minmax(table, write, file, buffer, length, ppos);
+        ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
         if (!write || (ret == -EINVAL))
                 return ret;
-        for_each_zone(zone) {
+        for_each_populated_zone(zone) {
                 for_each_online_cpu(cpu) {
                         unsigned long  high;
                         high = zone->present_pages / percpu_pagelist_fraction;
@@ -4540,7 +4855,14 @@ void *__init alloc_large_system_hash(const char *tablename,
                         numentries <<= (PAGE_SHIFT - scale);
 
                 /* Make sure we've got at least a 0-order allocation.. */
-                if (unlikely((numentries * bucketsize) < PAGE_SIZE))
+                if (unlikely(flags & HASH_SMALL)) {
+                        /* Makes no sense without HASH_EARLY */
+                        WARN_ON(!(flags & HASH_EARLY));
+                        if (!(numentries >> *_hash_shift)) {
+                                numentries = 1UL << *_hash_shift;
+                                BUG_ON(!numentries);
+                        }
+                } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
                         numentries = PAGE_SIZE / bucketsize;
         }
         numentries = roundup_pow_of_two(numentries);
@@ -4563,22 +4885,14 @@ void *__init alloc_large_system_hash(const char *tablename,
                 else if (hashdist)
                         table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
                 else {
-                        unsigned long order = get_order(size);
-                        table = (void*) __get_free_pages(GFP_ATOMIC, order);
                         /*
                          * If bucketsize is not a power-of-two, we may free
-                         * some pages at the end of hash table.
+                         * some pages at the end of hash table which
+                         * alloc_pages_exact() automatically does
                          */
-                        if (table) {
-                                unsigned long alloc_end = (unsigned long)table +
-                                                (PAGE_SIZE << order);
-                                unsigned long used = (unsigned long)table +
-                                                PAGE_ALIGN(size);
-                                split_page(virt_to_page(table), order);
-                                while (used < alloc_end) {
-                                        free_page(used);
-                                        used += PAGE_SIZE;
-                                }
+                        if (get_order(size) < MAX_ORDER) {
+                                table = alloc_pages_exact(size, GFP_ATOMIC);
+                                kmemleak_alloc(table, size, 1, GFP_ATOMIC);
                         }
                 }
         } while (!table && size > PAGE_SIZE && --log2qty);
@@ -4690,13 +5004,16 @@ int set_migratetype_isolate(struct page *page)
         struct zone *zone;
         unsigned long flags;
         int ret = -EBUSY;
+        int zone_idx;
 
         zone = page_zone(page);
+        zone_idx = zone_idx(zone);
         spin_lock_irqsave(&zone->lock, flags);
         /*
          * In future, more migrate types will be able to be isolation target.
          */
-        if (get_pageblock_migratetype(page) != MIGRATE_MOVABLE)
+        if (get_pageblock_migratetype(page) != MIGRATE_MOVABLE &&
+            zone_idx != ZONE_MOVABLE)
                 goto out;
         set_pageblock_migratetype(page, MIGRATE_ISOLATE);
         move_freepages_block(zone, page, MIGRATE_ISOLATE);
diff --git a/mm/page_cgroup.c b/mm/page_cgroup.c
index ceecfbb143fa..3d535d594826 100644
--- a/mm/page_cgroup.c
+++ b/mm/page_cgroup.c
@@ -69,7 +69,7 @@ static int __init alloc_node_page_cgroup(int nid)
         return 0;
 }
 
-void __init page_cgroup_init(void)
+void __init page_cgroup_init_flatmem(void)
 {
 
         int nid, fail;
@@ -83,12 +83,12 @@ void __init page_cgroup_init(void)
                         goto fail;
         }
         printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
-        printk(KERN_INFO "please try cgroup_disable=memory option if you"
-        " don't want\n");
+        printk(KERN_INFO "please try 'cgroup_disable=memory' option if you"
+        " don't want memory cgroups\n");
         return;
 fail:
-        printk(KERN_CRIT "allocation of page_cgroup was failed.\n");
-        printk(KERN_CRIT "please try cgroup_disable=memory boot option\n");
+        printk(KERN_CRIT "allocation of page_cgroup failed.\n");
+        printk(KERN_CRIT "please try 'cgroup_disable=memory' boot option\n");
         panic("Out of memory");
 }
 
@@ -99,6 +99,8 @@ struct page_cgroup *lookup_page_cgroup(struct page *page)
         unsigned long pfn = page_to_pfn(page);
         struct mem_section *section = __pfn_to_section(pfn);
 
+        if (!section->page_cgroup)
+                return NULL;
         return section->page_cgroup + pfn;
 }
 
@@ -113,15 +115,16 @@ static int __init_refok init_section_page_cgroup(unsigned long pfn)
         if (!section->page_cgroup) {
                 nid = page_to_nid(pfn_to_page(pfn));
                 table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION;
-                if (slab_is_available()) {
+                VM_BUG_ON(!slab_is_available());
+                if (node_state(nid, N_HIGH_MEMORY)) {
                         base = kmalloc_node(table_size,
-                                        GFP_KERNEL | __GFP_NOWARN, nid);
+                                GFP_KERNEL | __GFP_NOWARN, nid);
                         if (!base)
                                 base = vmalloc_node(table_size, nid);
                 } else {
-                        base = __alloc_bootmem_node_nopanic(NODE_DATA(nid),
-                                table_size,
-                                PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
+                        base = kmalloc(table_size, GFP_KERNEL | __GFP_NOWARN);
+                        if (!base)
+                                base = vmalloc(table_size);
                 }
         } else {
                 /*
@@ -257,14 +260,14 @@ void __init page_cgroup_init(void)
                 fail = init_section_page_cgroup(pfn);
         }
         if (fail) {
-                printk(KERN_CRIT "try cgroup_disable=memory boot option\n");
+                printk(KERN_CRIT "try 'cgroup_disable=memory' boot option\n");
                 panic("Out of memory");
         } else {
                 hotplug_memory_notifier(page_cgroup_callback, 0);
         }
         printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
-        printk(KERN_INFO "please try cgroup_disable=memory option if you don't"
-        " want\n");
+        printk(KERN_INFO "please try 'cgroup_disable=memory' option if you don't"
+        " want memory cgroups\n");
 }
 
 void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
@@ -285,12 +288,8 @@ struct swap_cgroup_ctrl {
 
 struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
 
-/*
- * This 8bytes seems big..maybe we can reduce this when we can use "id" for
- * cgroup rather than pointer.
- */
 struct swap_cgroup {
-        struct mem_cgroup       *val;
+        unsigned short          id;
 };
 #define SC_PER_PAGE     (PAGE_SIZE/sizeof(struct swap_cgroup))
 #define SC_POS_MASK     (SC_PER_PAGE - 1)
@@ -318,8 +317,6 @@ static int swap_cgroup_prepare(int type)
         struct swap_cgroup_ctrl *ctrl;
         unsigned long idx, max;
 
-        if (!do_swap_account)
-                return 0;
         ctrl = &swap_cgroup_ctrl[type];
 
         for (idx = 0; idx < ctrl->length; idx++) {
@@ -342,10 +339,10 @@ not_enough_page:
  * @ent: swap entry to be recorded into
  * @mem: mem_cgroup to be recorded
  *
- * Returns old value at success, NULL at failure.
- * (Of course, old value can be NULL.)
+ * Returns old value at success, 0 at failure.
+ * (Of course, old value can be 0.)
  */
-struct mem_cgroup *swap_cgroup_record(swp_entry_t ent, struct mem_cgroup *mem)
+unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
 {
         int type = swp_type(ent);
         unsigned long offset = swp_offset(ent);
@@ -354,18 +351,15 @@ struct mem_cgroup *swap_cgroup_record(swp_entry_t ent, struct mem_cgroup *mem)
         struct swap_cgroup_ctrl *ctrl;
         struct page *mappage;
         struct swap_cgroup *sc;
-        struct mem_cgroup *old;
-
-        if (!do_swap_account)
-                return NULL;
+        unsigned short old;
 
         ctrl = &swap_cgroup_ctrl[type];
 
         mappage = ctrl->map[idx];
         sc = page_address(mappage);
         sc += pos;
-        old = sc->val;
-        sc->val = mem;
+        old = sc->id;
+        sc->id = id;
 
         return old;
 }
@@ -374,9 +368,9 @@ struct mem_cgroup *swap_cgroup_record(swp_entry_t ent, struct mem_cgroup *mem)
  * lookup_swap_cgroup - lookup mem_cgroup tied to swap entry
  * @ent: swap entry to be looked up.
  *
- * Returns pointer to mem_cgroup at success. NULL at failure.
+ * Returns CSS ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
  */
-struct mem_cgroup *lookup_swap_cgroup(swp_entry_t ent)
+unsigned short lookup_swap_cgroup(swp_entry_t ent)
 {
         int type = swp_type(ent);
         unsigned long offset = swp_offset(ent);
@@ -385,16 +379,13 @@ struct mem_cgroup *lookup_swap_cgroup(swp_entry_t ent)
         struct swap_cgroup_ctrl *ctrl;
         struct page *mappage;
         struct swap_cgroup *sc;
-        struct mem_cgroup *ret;
-
-        if (!do_swap_account)
-                return NULL;
+        unsigned short ret;
 
         ctrl = &swap_cgroup_ctrl[type];
         mappage = ctrl->map[idx];
         sc = page_address(mappage);
         sc += pos;
-        ret = sc->val;
+        ret = sc->id;
         return ret;
 }
 
@@ -430,13 +421,6 @@ int swap_cgroup_swapon(int type, unsigned long max_pages)
         }
         mutex_unlock(&swap_cgroup_mutex);
 
-        printk(KERN_INFO
-                "swap_cgroup: uses %ld bytes of vmalloc for pointer array space"
-                " and %ld bytes to hold mem_cgroup pointers on swap\n",
-                array_size, length * PAGE_SIZE);
-        printk(KERN_INFO
-        "swap_cgroup can be disabled by noswapaccount boot option.\n");
-
         return 0;
 nomem:
         printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n");
diff --git a/mm/page_io.c b/mm/page_io.c
index 3023c475e041..c6f3e5071de3 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -120,7 +120,7 @@ out:
         return ret;
 }
 
-int swap_readpage(struct file *file, struct page *page)
+int swap_readpage(struct page *page)
 {
         struct bio *bio;
         int ret = 0;
diff --git a/mm/pdflush.c b/mm/pdflush.c
deleted file mode 100644
index 118905e3d788..000000000000
--- a/mm/pdflush.c
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * mm/pdflush.c - worker threads for writing back filesystem data
- *
- * Copyright (C) 2002, Linus Torvalds.
- *
- * 09Apr2002    Andrew Morton
- *              Initial version
- * 29Feb2004    kaos@sgi.com
- *              Move worker thread creation to kthread to avoid chewing
- *              up stack space with nested calls to kernel_thread.
- */
-
-#include <linux/sched.h>
-#include <linux/list.h>
-#include <linux/signal.h>
-#include <linux/spinlock.h>
-#include <linux/gfp.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/fs.h>           /* Needed by writeback.h          */
-#include <linux/writeback.h>    /* Prototypes pdflush_operation() */
-#include <linux/kthread.h>
-#include <linux/cpuset.h>
-#include <linux/freezer.h>
-
-
-/*
- * Minimum and maximum number of pdflush instances
- */
-#define MIN_PDFLUSH_THREADS     2
-#define MAX_PDFLUSH_THREADS     8
-
-static void start_one_pdflush_thread(void);
-
-
-/*
- * The pdflush threads are worker threads for writing back dirty data.
- * Ideally, we'd like one thread per active disk spindle.  But the disk
- * topology is very hard to divine at this level.   Instead, we take
- * care in various places to prevent more than one pdflush thread from
- * performing writeback against a single filesystem.  pdflush threads
- * have the PF_FLUSHER flag set in current->flags to aid in this.
- */
-
-/*
- * All the pdflush threads.  Protected by pdflush_lock
- */
-static LIST_HEAD(pdflush_list);
-static DEFINE_SPINLOCK(pdflush_lock);
-
-/*
- * The count of currently-running pdflush threads.  Protected
- * by pdflush_lock.
- *
- * Readable by sysctl, but not writable.  Published to userspace at
- * /proc/sys/vm/nr_pdflush_threads.
- */
-int nr_pdflush_threads = 0;
-
-/*
- * The time at which the pdflush thread pool last went empty
- */
-static unsigned long last_empty_jifs;
-
-/*
- * The pdflush thread.
- *
- * Thread pool management algorithm:
- * 
- * - The minimum and maximum number of pdflush instances are bound
- *   by MIN_PDFLUSH_THREADS and MAX_PDFLUSH_THREADS.
- * 
- * - If there have been no idle pdflush instances for 1 second, create
- *   a new one.
- * 
- * - If the least-recently-went-to-sleep pdflush thread has been asleep
- *   for more than one second, terminate a thread.
- */
-
-/*
- * A structure for passing work to a pdflush thread.  Also for passing
- * state information between pdflush threads.  Protected by pdflush_lock.
- */
-struct pdflush_work {
-        struct task_struct *who;        /* The thread */
-        void (*fn)(unsigned long);      /* A callback function */
-        unsigned long arg0;             /* An argument to the callback */
-        struct list_head list;          /* On pdflush_list, when idle */
-        unsigned long when_i_went_to_sleep;
-};
-
-static int __pdflush(struct pdflush_work *my_work)
-{
-        current->flags |= PF_FLUSHER | PF_SWAPWRITE;
-        set_freezable();
-        my_work->fn = NULL;
-        my_work->who = current;
-        INIT_LIST_HEAD(&my_work->list);
-
-        spin_lock_irq(&pdflush_lock);
-        nr_pdflush_threads++;
-        for ( ; ; ) {
-                struct pdflush_work *pdf;
-
-                set_current_state(TASK_INTERRUPTIBLE);
-                list_move(&my_work->list, &pdflush_list);
-                my_work->when_i_went_to_sleep = jiffies;
-                spin_unlock_irq(&pdflush_lock);
-                schedule();
-                try_to_freeze();
-                spin_lock_irq(&pdflush_lock);
-                if (!list_empty(&my_work->list)) {
-                        /*
-                         * Someone woke us up, but without removing our control
-                         * structure from the global list.  swsusp will do this
-                         * in try_to_freeze()->refrigerator().  Handle it.
-                         */
-                        my_work->fn = NULL;
-                        continue;
-                }
-                if (my_work->fn == NULL) {
-                        printk("pdflush: bogus wakeup\n");
-                        continue;
-                }
-                spin_unlock_irq(&pdflush_lock);
-
-                (*my_work->fn)(my_work->arg0);
-
-                /*
-                 * Thread creation: For how long have there been zero
-                 * available threads?
-                 */
-                if (time_after(jiffies, last_empty_jifs + 1 * HZ)) {
-                        /* unlocked list_empty() test is OK here */
-                        if (list_empty(&pdflush_list)) {
-                                /* unlocked test is OK here */
-                                if (nr_pdflush_threads < MAX_PDFLUSH_THREADS)
-                                        start_one_pdflush_thread();
-                        }
-                }
-
-                spin_lock_irq(&pdflush_lock);
-                my_work->fn = NULL;
-
-                /*
-                 * Thread destruction: For how long has the sleepiest
-                 * thread slept?
-                 */
-                if (list_empty(&pdflush_list))
-                        continue;
-                if (nr_pdflush_threads <= MIN_PDFLUSH_THREADS)
-                        continue;
-                pdf = list_entry(pdflush_list.prev, struct pdflush_work, list);
-                if (time_after(jiffies, pdf->when_i_went_to_sleep + 1 * HZ)) {
-                        /* Limit exit rate */
-                        pdf->when_i_went_to_sleep = jiffies;
-                        break;                                  /* exeunt */
-                }
-        }
-        nr_pdflush_threads--;
-        spin_unlock_irq(&pdflush_lock);
-        return 0;
-}
-
-/*
- * Of course, my_work wants to be just a local in __pdflush().  It is
- * separated out in this manner to hopefully prevent the compiler from
- * performing unfortunate optimisations against the auto variables.  Because
- * these are visible to other tasks and CPUs.  (No problem has actually
- * been observed.  This is just paranoia).
- */
-static int pdflush(void *dummy)
-{
-        struct pdflush_work my_work;
-        cpumask_var_t cpus_allowed;
-
-        /*
-         * Since the caller doesn't even check kthread_run() worked, let's not
-         * freak out too much if this fails.
-         */
-        if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
-                printk(KERN_WARNING "pdflush failed to allocate cpumask\n");
-                return 0;
-        }
-
-        /*
-         * pdflush can spend a lot of time doing encryption via dm-crypt.  We
-         * don't want to do that at keventd's priority.
-         */
-        set_user_nice(current, 0);
-
-        /*
-         * Some configs put our parent kthread in a limited cpuset,
-         * which kthread() overrides, forcing cpus_allowed == cpu_all_mask.
-         * Our needs are more modest - cut back to our cpusets cpus_allowed.
-         * This is needed as pdflush's are dynamically created and destroyed.
-         * The boottime pdflush's are easily placed w/o these 2 lines.
-         */
-        cpuset_cpus_allowed(current, cpus_allowed);
-        set_cpus_allowed_ptr(current, cpus_allowed);
-        free_cpumask_var(cpus_allowed);
-
-        return __pdflush(&my_work);
-}
-
-/*
- * Attempt to wake up a pdflush thread, and get it to do some work for you.
- * Returns zero if it indeed managed to find a worker thread, and passed your
- * payload to it.
- */
-int pdflush_operation(void (*fn)(unsigned long), unsigned long arg0)
-{
-        unsigned long flags;
-        int ret = 0;
-
-        BUG_ON(fn == NULL);     /* Hard to diagnose if it's deferred */
-
-        spin_lock_irqsave(&pdflush_lock, flags);
-        if (list_empty(&pdflush_list)) {
-                ret = -1;
-        } else {
-                struct pdflush_work *pdf;
-
-                pdf = list_entry(pdflush_list.next, struct pdflush_work, list);
-                list_del_init(&pdf->list);
-                if (list_empty(&pdflush_list))
-                        last_empty_jifs = jiffies;
-                pdf->fn = fn;
-                pdf->arg0 = arg0;
-                wake_up_process(pdf->who);
-        }
-        spin_unlock_irqrestore(&pdflush_lock, flags);
-
-        return ret;
-}
-
-static void start_one_pdflush_thread(void)
-{
-        kthread_run(pdflush, NULL, "pdflush");
-}
-
-static int __init pdflush_init(void)
-{
-        int i;
-
-        for (i = 0; i < MIN_PDFLUSH_THREADS; i++)
-                start_one_pdflush_thread();
-        return 0;
-}
-
-module_init(pdflush_init);
diff --git a/mm/percpu.c b/mm/percpu.c
index 1aa5d8fbca12..5adfc268b408 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -8,12 +8,13 @@
  *
  * This is percpu allocator which can handle both static and dynamic
  * areas.  Percpu areas are allocated in chunks in vmalloc area.  Each
- * chunk is consisted of num_possible_cpus() units and the first chunk
- * is used for static percpu variables in the kernel image (special
- * boot time alloc/init handling necessary as these areas need to be
- * brought up before allocation services are running).  Unit grows as
- * necessary and all units grow or shrink in unison.  When a chunk is
- * filled up, another chunk is allocated.  ie. in vmalloc area
+ * chunk is consisted of boot-time determined number of units and the
+ * first chunk is used for static percpu variables in the kernel image
+ * (special boot time alloc/init handling necessary as these areas
+ * need to be brought up before allocation services are running).
+ * Unit grows as necessary and all units grow or shrink in unison.
+ * When a chunk is filled up, another chunk is allocated.  ie. in
+ * vmalloc area
  *
  *  c0                           c1                         c2
  *  -------------------          -------------------        ------------
@@ -22,11 +23,13 @@
  *
  * Allocation is done in offset-size areas of single unit space.  Ie,
  * an area of 512 bytes at 6k in c1 occupies 512 bytes at 6k of c1:u0,
- * c1:u1, c1:u2 and c1:u3.  Percpu access can be done by configuring
- * percpu base registers UNIT_SIZE apart.
+ * c1:u1, c1:u2 and c1:u3.  On UMA, units corresponds directly to
+ * cpus.  On NUMA, the mapping can be non-linear and even sparse.
+ * Percpu access can be done by configuring percpu base registers
+ * according to cpu to unit mapping and pcpu_unit_size.
  *
- * There are usually many small percpu allocations many of them as
- * small as 4 bytes.  The allocator organizes chunks into lists
+ * There are usually many small percpu allocations many of them being
+ * as small as 4 bytes.  The allocator organizes chunks into lists
  * according to free size and tries to allocate from the fullest one.
  * Each chunk keeps the maximum contiguous area size hint which is
  * guaranteed to be eqaul to or larger than the maximum contiguous
@@ -38,12 +41,12 @@
  * region and negative allocated.  Allocation inside a chunk is done
  * by scanning this map sequentially and serving the first matching
  * entry.  This is mostly copied from the percpu_modalloc() allocator.
- * Chunks are also linked into a rb tree to ease address to chunk
- * mapping during free.
+ * Chunks can be determined from the address using the index field
+ * in the page struct. The index field contains a pointer to the chunk.
  *
  * To use this allocator, arch code should do the followings.
  *
- * - define CONFIG_HAVE_DYNAMIC_PER_CPU_AREA
+ * - drop CONFIG_HAVE_LEGACY_PER_CPU_AREA
  *
  * - define __addr_to_pcpu_ptr() and __pcpu_ptr_to_addr() to translate
  *   regular address to percpu pointer and back if they need to be
@@ -55,13 +58,14 @@
 
 #include <linux/bitmap.h>
 #include <linux/bootmem.h>
+#include <linux/err.h>
 #include <linux/list.h>
+#include <linux/log2.h>
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/percpu.h>
 #include <linux/pfn.h>
-#include <linux/rbtree.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/vmalloc.h>
@@ -88,44 +92,71 @@
 
 struct pcpu_chunk {
         struct list_head        list;           /* linked to pcpu_slot lists */
-        struct rb_node          rb_node;        /* key is chunk->vm->addr */
         int                     free_size;      /* free bytes in the chunk */
         int                     contig_hint;    /* max contiguous size hint */
-        struct vm_struct        *vm;            /* mapped vmalloc region */
+        void                    *base_addr;     /* base address of this chunk */
         int                     map_used;       /* # of map entries used */
         int                     map_alloc;      /* # of map entries allocated */
         int                     *map;           /* allocation map */
+        struct vm_struct        **vms;          /* mapped vmalloc regions */
         bool                    immutable;      /* no [de]population allowed */
-        struct page             **page;         /* points to page array */
-        struct page             *page_ar[];     /* #cpus * UNIT_PAGES */
+        unsigned long           populated[];    /* populated bitmap */
 };
 
 static int pcpu_unit_pages __read_mostly;
 static int pcpu_unit_size __read_mostly;
-static int pcpu_chunk_size __read_mostly;
+static int pcpu_nr_units __read_mostly;
+static int pcpu_atom_size __read_mostly;
 static int pcpu_nr_slots __read_mostly;
 static size_t pcpu_chunk_struct_size __read_mostly;
 
+/* cpus with the lowest and highest unit numbers */
+static unsigned int pcpu_first_unit_cpu __read_mostly;
+static unsigned int pcpu_last_unit_cpu __read_mostly;
+
 /* the address of the first chunk which starts with the kernel static area */
 void *pcpu_base_addr __read_mostly;
 EXPORT_SYMBOL_GPL(pcpu_base_addr);
 
-/* optional reserved chunk, only accessible for reserved allocations */
+static const int *pcpu_unit_map __read_mostly;          /* cpu -> unit */
+const unsigned long *pcpu_unit_offsets __read_mostly;   /* cpu -> unit offset */
+
+/* group information, used for vm allocation */
+static int pcpu_nr_groups __read_mostly;
+static const unsigned long *pcpu_group_offsets __read_mostly;
+static const size_t *pcpu_group_sizes __read_mostly;
+
+/*
+ * The first chunk which always exists.  Note that unlike other
+ * chunks, this one can be allocated and mapped in several different
+ * ways and thus often doesn't live in the vmalloc area.
+ */
+static struct pcpu_chunk *pcpu_first_chunk;
+
+/*
+ * Optional reserved chunk.  This chunk reserves part of the first
+ * chunk and serves it for reserved allocations.  The amount of
+ * reserved offset is in pcpu_reserved_chunk_limit.  When reserved
+ * area doesn't exist, the following variables contain NULL and 0
+ * respectively.
+ */
 static struct pcpu_chunk *pcpu_reserved_chunk;
-/* offset limit of the reserved chunk */
 static int pcpu_reserved_chunk_limit;
 
 /*
  * Synchronization rules.
  *
  * There are two locks - pcpu_alloc_mutex and pcpu_lock.  The former
- * protects allocation/reclaim paths, chunks and chunk->page arrays.
- * The latter is a spinlock and protects the index data structures -
- * chunk slots, rbtree, chunks and area maps in chunks.
+ * protects allocation/reclaim paths, chunks, populated bitmap and
+ * vmalloc mapping.  The latter is a spinlock and protects the index
+ * data structures - chunk slots, chunks and area maps in chunks.
  *
  * During allocation, pcpu_alloc_mutex is kept locked all the time and
  * pcpu_lock is grabbed and released as necessary.  All actual memory
- * allocations are done using GFP_KERNEL with pcpu_lock released.
+ * allocations are done using GFP_KERNEL with pcpu_lock released.  In
+ * general, percpu memory can't be allocated with irq off but
+ * irqsave/restore are still used in alloc path so that it can be used
+ * from early init path - sched_init() specifically.
  *
  * Free path accesses and alters only the index data structures, so it
  * can be safely called from atomic context.  When memory needs to be
@@ -140,7 +171,6 @@ static DEFINE_MUTEX(pcpu_alloc_mutex);	/* protects whole alloc and reclaim */
 static DEFINE_SPINLOCK(pcpu_lock);      /* protects index data structures */
 
 static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */
-static struct rb_root pcpu_addr_root = RB_ROOT; /* chunks by address */
 
 /* reclaim work to release fully free chunks, scheduled from free path */
 static void pcpu_reclaim(struct work_struct *work);
@@ -169,28 +199,65 @@ static int pcpu_chunk_slot(const struct pcpu_chunk *chunk)
 
 static int pcpu_page_idx(unsigned int cpu, int page_idx)
 {
-        return cpu * pcpu_unit_pages + page_idx;
+        return pcpu_unit_map[cpu] * pcpu_unit_pages + page_idx;
 }
 
-static struct page **pcpu_chunk_pagep(struct pcpu_chunk *chunk,
-                                      unsigned int cpu, int page_idx)
+static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk,
+                                     unsigned int cpu, int page_idx)
 {
-        return &chunk->page[pcpu_page_idx(cpu, page_idx)];
+        return (unsigned long)chunk->base_addr + pcpu_unit_offsets[cpu] +
+                (page_idx << PAGE_SHIFT);
 }
 
-static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk,
-                                     unsigned int cpu, int page_idx)
+static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,
+                                    unsigned int cpu, int page_idx)
+{
+        /* must not be used on pre-mapped chunk */
+        WARN_ON(chunk->immutable);
+
+        return vmalloc_to_page((void *)pcpu_chunk_addr(chunk, cpu, page_idx));
+}
+
+/* set the pointer to a chunk in a page struct */
+static void pcpu_set_page_chunk(struct page *page, struct pcpu_chunk *pcpu)
+{
+        page->index = (unsigned long)pcpu;
+}
+
+/* obtain pointer to a chunk from a page struct */
+static struct pcpu_chunk *pcpu_get_page_chunk(struct page *page)
+{
+        return (struct pcpu_chunk *)page->index;
+}
+
+static void pcpu_next_unpop(struct pcpu_chunk *chunk, int *rs, int *re, int end)
 {
-        return (unsigned long)chunk->vm->addr +
-                (pcpu_page_idx(cpu, page_idx) << PAGE_SHIFT);
+        *rs = find_next_zero_bit(chunk->populated, end, *rs);
+        *re = find_next_bit(chunk->populated, end, *rs + 1);
 }
 
-static bool pcpu_chunk_page_occupied(struct pcpu_chunk *chunk,
-                                     int page_idx)
+static void pcpu_next_pop(struct pcpu_chunk *chunk, int *rs, int *re, int end)
 {
-        return *pcpu_chunk_pagep(chunk, 0, page_idx) != NULL;
+        *rs = find_next_bit(chunk->populated, end, *rs);
+        *re = find_next_zero_bit(chunk->populated, end, *rs + 1);
 }
 
+/*
+ * (Un)populated page region iterators.  Iterate over (un)populated
+ * page regions betwen @start and @end in @chunk.  @rs and @re should
+ * be integer variables and will be set to start and end page index of
+ * the current region.
+ */
+#define pcpu_for_each_unpop_region(chunk, rs, re, start, end)               \
+        for ((rs) = (start), pcpu_next_unpop((chunk), &(rs), &(re), (end)); \
+             (rs) < (re);                                                   \
+             (rs) = (re) + 1, pcpu_next_unpop((chunk), &(rs), &(re), (end)))
+
+#define pcpu_for_each_pop_region(chunk, rs, re, start, end)                 \
+        for ((rs) = (start), pcpu_next_pop((chunk), &(rs), &(re), (end));   \
+             (rs) < (re);                                                   \
+             (rs) = (re) + 1, pcpu_next_pop((chunk), &(rs), &(re), (end)))
+
 /**
  * pcpu_mem_alloc - allocate memory
  * @size: bytes to allocate
@@ -257,152 +324,117 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
         }
 }
 
-static struct rb_node **pcpu_chunk_rb_search(void *addr,
-                                             struct rb_node **parentp)
-{
-        struct rb_node **p = &pcpu_addr_root.rb_node;
-        struct rb_node *parent = NULL;
-        struct pcpu_chunk *chunk;
-
-        while (*p) {
-                parent = *p;
-                chunk = rb_entry(parent, struct pcpu_chunk, rb_node);
-
-                if (addr < chunk->vm->addr)
-                        p = &(*p)->rb_left;
-                else if (addr > chunk->vm->addr)
-                        p = &(*p)->rb_right;
-                else
-                        break;
-        }
-
-        if (parentp)
-                *parentp = parent;
-        return p;
-}
-
 /**
- * pcpu_chunk_addr_search - search for chunk containing specified address
- * @addr: address to search for
- *
- * Look for chunk which might contain @addr.  More specifically, it
- * searchs for the chunk with the highest start address which isn't
- * beyond @addr.
- *
- * CONTEXT:
- * pcpu_lock.
+ * pcpu_chunk_addr_search - determine chunk containing specified address
+ * @addr: address for which the chunk needs to be determined.
  *
  * RETURNS:
  * The address of the found chunk.
  */
 static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
 {
-        struct rb_node *n, *parent;
-        struct pcpu_chunk *chunk;
+        void *first_start = pcpu_first_chunk->base_addr;
 
-        /* is it in the reserved chunk? */
-        if (pcpu_reserved_chunk) {
-                void *start = pcpu_reserved_chunk->vm->addr;
-
-                if (addr >= start && addr < start + pcpu_reserved_chunk_limit)
+        /* is it in the first chunk? */
+        if (addr >= first_start && addr < first_start + pcpu_unit_size) {
+                /* is it in the reserved area? */
+                if (addr < first_start + pcpu_reserved_chunk_limit)
                         return pcpu_reserved_chunk;
+                return pcpu_first_chunk;
         }
 
-        /* nah... search the regular ones */
-        n = *pcpu_chunk_rb_search(addr, &parent);
-        if (!n) {
-                /* no exactly matching chunk, the parent is the closest */
-                n = parent;
-                BUG_ON(!n);
-        }
-        chunk = rb_entry(n, struct pcpu_chunk, rb_node);
-
-        if (addr < chunk->vm->addr) {
-                /* the parent was the next one, look for the previous one */
-                n = rb_prev(n);
-                BUG_ON(!n);
-                chunk = rb_entry(n, struct pcpu_chunk, rb_node);
-        }
-
-        return chunk;
+        /*
+         * The address is relative to unit0 which might be unused and
+         * thus unmapped.  Offset the address to the unit space of the
+         * current processor before looking it up in the vmalloc
+         * space.  Note that any possible cpu id can be used here, so
+         * there's no need to worry about preemption or cpu hotplug.
+         */
+        addr += pcpu_unit_offsets[raw_smp_processor_id()];
+        return pcpu_get_page_chunk(vmalloc_to_page(addr));
 }
 
 /**
- * pcpu_chunk_addr_insert - insert chunk into address rb tree
- * @new: chunk to insert
+ * pcpu_need_to_extend - determine whether chunk area map needs to be extended
+ * @chunk: chunk of interest
  *
- * Insert @new into address rb tree.
+ * Determine whether area map of @chunk needs to be extended to
+ * accomodate a new allocation.
  *
  * CONTEXT:
  * pcpu_lock.
+ *
+ * RETURNS:
+ * New target map allocation length if extension is necessary, 0
+ * otherwise.
  */
-static void pcpu_chunk_addr_insert(struct pcpu_chunk *new)
+static int pcpu_need_to_extend(struct pcpu_chunk *chunk)
 {
-        struct rb_node **p, *parent;
+        int new_alloc;
 
-        p = pcpu_chunk_rb_search(new->vm->addr, &parent);
-        BUG_ON(*p);
-        rb_link_node(&new->rb_node, parent, p);
-        rb_insert_color(&new->rb_node, &pcpu_addr_root);
+        if (chunk->map_alloc >= chunk->map_used + 2)
+                return 0;
+
+        new_alloc = PCPU_DFL_MAP_ALLOC;
+        while (new_alloc < chunk->map_used + 2)
+                new_alloc *= 2;
+
+        return new_alloc;
 }
 
 /**
- * pcpu_extend_area_map - extend area map for allocation
- * @chunk: target chunk
+ * pcpu_extend_area_map - extend area map of a chunk
+ * @chunk: chunk of interest
+ * @new_alloc: new target allocation length of the area map
  *
- * Extend area map of @chunk so that it can accomodate an allocation.
- * A single allocation can split an area into three areas, so this
- * function makes sure that @chunk->map has at least two extra slots.
+ * Extend area map of @chunk to have @new_alloc entries.
  *
  * CONTEXT:
- * pcpu_alloc_mutex, pcpu_lock.  pcpu_lock is released and reacquired
- * if area map is extended.
+ * Does GFP_KERNEL allocation.  Grabs and releases pcpu_lock.
  *
  * RETURNS:
- * 0 if noop, 1 if successfully extended, -errno on failure.
+ * 0 on success, -errno on failure.
  */
-static int pcpu_extend_area_map(struct pcpu_chunk *chunk)
+static int pcpu_extend_area_map(struct pcpu_chunk *chunk, int new_alloc)
 {
-        int new_alloc;
-        int *new;
-        size_t size;
-
-        /* has enough? */
-        if (chunk->map_alloc >= chunk->map_used + 2)
-                return 0;
-
-        spin_unlock_irq(&pcpu_lock);
-
-        new_alloc = PCPU_DFL_MAP_ALLOC;
-        while (new_alloc < chunk->map_used + 2)
-                new_alloc *= 2;
+        int *old = NULL, *new = NULL;
+        size_t old_size = 0, new_size = new_alloc * sizeof(new[0]);
+        unsigned long flags;
 
-        new = pcpu_mem_alloc(new_alloc * sizeof(new[0]));
-        if (!new) {
-                spin_lock_irq(&pcpu_lock);
+        new = pcpu_mem_alloc(new_size);
+        if (!new)
                 return -ENOMEM;
-        }
 
-        /*
-         * Acquire pcpu_lock and switch to new area map.  Only free
-         * could have happened inbetween, so map_used couldn't have
-         * grown.
-         */
-        spin_lock_irq(&pcpu_lock);
-        BUG_ON(new_alloc < chunk->map_used + 2);
+        /* acquire pcpu_lock and switch to new area map */
+        spin_lock_irqsave(&pcpu_lock, flags);
 
-        size = chunk->map_alloc * sizeof(chunk->map[0]);
-        memcpy(new, chunk->map, size);
+        if (new_alloc <= chunk->map_alloc)
+                goto out_unlock;
+
+        old_size = chunk->map_alloc * sizeof(chunk->map[0]);
+        memcpy(new, chunk->map, old_size);
 
         /*
          * map_alloc < PCPU_DFL_MAP_ALLOC indicates that the chunk is
          * one of the first chunks and still using static map.
          */
         if (chunk->map_alloc >= PCPU_DFL_MAP_ALLOC)
-                pcpu_mem_free(chunk->map, size);
+                old = chunk->map;
 
         chunk->map_alloc = new_alloc;
         chunk->map = new;
+        new = NULL;
+
+out_unlock:
+        spin_unlock_irqrestore(&pcpu_lock, flags);
+
+        /*
+         * pcpu_mem_free() might end up calling vfree() which uses
+         * IRQ-unsafe lock and thus can't be called under pcpu_lock.
+         */
+        pcpu_mem_free(old, old_size);
+        pcpu_mem_free(new, new_size);
+
         return 0;
 }
 
@@ -591,126 +623,327 @@ static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme)
 }
 
 /**
- * pcpu_unmap - unmap pages out of a pcpu_chunk
+ * pcpu_get_pages_and_bitmap - get temp pages array and bitmap
  * @chunk: chunk of interest
- * @page_start: page index of the first page to unmap
- * @page_end: page index of the last page to unmap + 1
- * @flush: whether to flush cache and tlb or not
+ * @bitmapp: output parameter for bitmap
+ * @may_alloc: may allocate the array
  *
- * For each cpu, unmap pages [@page_start,@page_end) out of @chunk.
- * If @flush is true, vcache is flushed before unmapping and tlb
- * after.
+ * Returns pointer to array of pointers to struct page and bitmap,
+ * both of which can be indexed with pcpu_page_idx().  The returned
+ * array is cleared to zero and *@bitmapp is copied from
+ * @chunk->populated.  Note that there is only one array and bitmap
+ * and access exclusion is the caller's responsibility.
+ *
+ * CONTEXT:
+ * pcpu_alloc_mutex and does GFP_KERNEL allocation if @may_alloc.
+ * Otherwise, don't care.
+ *
+ * RETURNS:
+ * Pointer to temp pages array on success, NULL on failure.
  */
-static void pcpu_unmap(struct pcpu_chunk *chunk, int page_start, int page_end,
-                       bool flush)
+static struct page **pcpu_get_pages_and_bitmap(struct pcpu_chunk *chunk,
+                                               unsigned long **bitmapp,
+                                               bool may_alloc)
 {
-        unsigned int last = num_possible_cpus() - 1;
-        unsigned int cpu;
+        static struct page **pages;
+        static unsigned long *bitmap;
+        size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]);
+        size_t bitmap_size = BITS_TO_LONGS(pcpu_unit_pages) *
+                             sizeof(unsigned long);
+
+        if (!pages || !bitmap) {
+                if (may_alloc && !pages)
+                        pages = pcpu_mem_alloc(pages_size);
+                if (may_alloc && !bitmap)
+                        bitmap = pcpu_mem_alloc(bitmap_size);
+                if (!pages || !bitmap)
+                        return NULL;
+        }
 
-        /* unmap must not be done on immutable chunk */
-        WARN_ON(chunk->immutable);
+        memset(pages, 0, pages_size);
+        bitmap_copy(bitmap, chunk->populated, pcpu_unit_pages);
 
-        /*
-         * Each flushing trial can be very expensive, issue flush on
-         * the whole region at once rather than doing it for each cpu.
-         * This could be an overkill but is more scalable.
-         */
-        if (flush)
-                flush_cache_vunmap(pcpu_chunk_addr(chunk, 0, page_start),
-                                   pcpu_chunk_addr(chunk, last, page_end));
+        *bitmapp = bitmap;
+        return pages;
+}
 
-        for_each_possible_cpu(cpu)
-                unmap_kernel_range_noflush(
-                                pcpu_chunk_addr(chunk, cpu, page_start),
-                                (page_end - page_start) << PAGE_SHIFT);
-
-        /* ditto as flush_cache_vunmap() */
-        if (flush)
-                flush_tlb_kernel_range(pcpu_chunk_addr(chunk, 0, page_start),
-                                       pcpu_chunk_addr(chunk, last, page_end));
+/**
+ * pcpu_free_pages - free pages which were allocated for @chunk
+ * @chunk: chunk pages were allocated for
+ * @pages: array of pages to be freed, indexed by pcpu_page_idx()
+ * @populated: populated bitmap
+ * @page_start: page index of the first page to be freed
+ * @page_end: page index of the last page to be freed + 1
+ *
+ * Free pages [@page_start and @page_end) in @pages for all units.
+ * The pages were allocated for @chunk.
+ */
+static void pcpu_free_pages(struct pcpu_chunk *chunk,
+                            struct page **pages, unsigned long *populated,
+                            int page_start, int page_end)
+{
+        unsigned int cpu;
+        int i;
+
+        for_each_possible_cpu(cpu) {
+                for (i = page_start; i < page_end; i++) {
+                        struct page *page = pages[pcpu_page_idx(cpu, i)];
+
+                        if (page)
+                                __free_page(page);
+                }
+        }
 }
 
 /**
- * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk
- * @chunk: chunk to depopulate
- * @off: offset to the area to depopulate
- * @size: size of the area to depopulate in bytes
- * @flush: whether to flush cache and tlb or not
- *
- * For each cpu, depopulate and unmap pages [@page_start,@page_end)
- * from @chunk.  If @flush is true, vcache is flushed before unmapping
- * and tlb after.
- *
- * CONTEXT:
- * pcpu_alloc_mutex.
+ * pcpu_alloc_pages - allocates pages for @chunk
+ * @chunk: target chunk
+ * @pages: array to put the allocated pages into, indexed by pcpu_page_idx()
+ * @populated: populated bitmap
+ * @page_start: page index of the first page to be allocated
+ * @page_end: page index of the last page to be allocated + 1
+ *
+ * Allocate pages [@page_start,@page_end) into @pages for all units.
+ * The allocation is for @chunk.  Percpu core doesn't care about the
+ * content of @pages and will pass it verbatim to pcpu_map_pages().
  */
-static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size,
-                                  bool flush)
+static int pcpu_alloc_pages(struct pcpu_chunk *chunk,
+                            struct page **pages, unsigned long *populated,
+                            int page_start, int page_end)
 {
-        int page_start = PFN_DOWN(off);
-        int page_end = PFN_UP(off + size);
-        int unmap_start = -1;
-        int uninitialized_var(unmap_end);
+        const gfp_t gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_COLD;
         unsigned int cpu;
         int i;
 
-        for (i = page_start; i < page_end; i++) {
-                for_each_possible_cpu(cpu) {
-                        struct page **pagep = pcpu_chunk_pagep(chunk, cpu, i);
+        for_each_possible_cpu(cpu) {
+                for (i = page_start; i < page_end; i++) {
+                        struct page **pagep = &pages[pcpu_page_idx(cpu, i)];
+
+                        *pagep = alloc_pages_node(cpu_to_node(cpu), gfp, 0);
+                        if (!*pagep) {
+                                pcpu_free_pages(chunk, pages, populated,
+                                                page_start, page_end);
+                                return -ENOMEM;
+                        }
+                }
+        }
+        return 0;
+}
 
-                        if (!*pagep)
-                                continue;
+/**
+ * pcpu_pre_unmap_flush - flush cache prior to unmapping
+ * @chunk: chunk the regions to be flushed belongs to
+ * @page_start: page index of the first page to be flushed
+ * @page_end: page index of the last page to be flushed + 1
+ *
+ * Pages in [@page_start,@page_end) of @chunk are about to be
+ * unmapped.  Flush cache.  As each flushing trial can be very
+ * expensive, issue flush on the whole region at once rather than
+ * doing it for each cpu.  This could be an overkill but is more
+ * scalable.
+ */
+static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk,
+                                 int page_start, int page_end)
+{
+        flush_cache_vunmap(
+                pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start),
+                pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end));
+}
 
-                        __free_page(*pagep);
+static void __pcpu_unmap_pages(unsigned long addr, int nr_pages)
+{
+        unmap_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT);
+}
 
-                        /*
-                         * If it's partial depopulation, it might get
-                         * populated or depopulated again.  Mark the
-                         * page gone.
-                         */
-                        *pagep = NULL;
+/**
+ * pcpu_unmap_pages - unmap pages out of a pcpu_chunk
+ * @chunk: chunk of interest
+ * @pages: pages array which can be used to pass information to free
+ * @populated: populated bitmap
+ * @page_start: page index of the first page to unmap
+ * @page_end: page index of the last page to unmap + 1
+ *
+ * For each cpu, unmap pages [@page_start,@page_end) out of @chunk.
+ * Corresponding elements in @pages were cleared by the caller and can
+ * be used to carry information to pcpu_free_pages() which will be
+ * called after all unmaps are finished.  The caller should call
+ * proper pre/post flush functions.
+ */
+static void pcpu_unmap_pages(struct pcpu_chunk *chunk,
+                             struct page **pages, unsigned long *populated,
+                             int page_start, int page_end)
+{
+        unsigned int cpu;
+        int i;
 
-                        unmap_start = unmap_start < 0 ? i : unmap_start;
-                        unmap_end = i + 1;
+        for_each_possible_cpu(cpu) {
+                for (i = page_start; i < page_end; i++) {
+                        struct page *page;
+
+                        page = pcpu_chunk_page(chunk, cpu, i);
+                        WARN_ON(!page);
+                        pages[pcpu_page_idx(cpu, i)] = page;
                 }
+                __pcpu_unmap_pages(pcpu_chunk_addr(chunk, cpu, page_start),
+                                   page_end - page_start);
         }
 
-        if (unmap_start >= 0)
-                pcpu_unmap(chunk, unmap_start, unmap_end, flush);
+        for (i = page_start; i < page_end; i++)
+                __clear_bit(i, populated);
 }
 
 /**
- * pcpu_map - map pages into a pcpu_chunk
+ * pcpu_post_unmap_tlb_flush - flush TLB after unmapping
+ * @chunk: pcpu_chunk the regions to be flushed belong to
+ * @page_start: page index of the first page to be flushed
+ * @page_end: page index of the last page to be flushed + 1
+ *
+ * Pages [@page_start,@page_end) of @chunk have been unmapped.  Flush
+ * TLB for the regions.  This can be skipped if the area is to be
+ * returned to vmalloc as vmalloc will handle TLB flushing lazily.
+ *
+ * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
+ * for the whole region.
+ */
+static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,
+                                      int page_start, int page_end)
+{
+        flush_tlb_kernel_range(
+                pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start),
+                pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end));
+}
+
+static int __pcpu_map_pages(unsigned long addr, struct page **pages,
+                            int nr_pages)
+{
+        return map_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT,
+                                        PAGE_KERNEL, pages);
+}
+
+/**
+ * pcpu_map_pages - map pages into a pcpu_chunk
  * @chunk: chunk of interest
+ * @pages: pages array containing pages to be mapped
+ * @populated: populated bitmap
  * @page_start: page index of the first page to map
  * @page_end: page index of the last page to map + 1
  *
- * For each cpu, map pages [@page_start,@page_end) into @chunk.
- * vcache is flushed afterwards.
+ * For each cpu, map pages [@page_start,@page_end) into @chunk.  The
+ * caller is responsible for calling pcpu_post_map_flush() after all
+ * mappings are complete.
+ *
+ * This function is responsible for setting corresponding bits in
+ * @chunk->populated bitmap and whatever is necessary for reverse
+ * lookup (addr -> chunk).
  */
-static int pcpu_map(struct pcpu_chunk *chunk, int page_start, int page_end)
+static int pcpu_map_pages(struct pcpu_chunk *chunk,
+                          struct page **pages, unsigned long *populated,
+                          int page_start, int page_end)
 {
-        unsigned int last = num_possible_cpus() - 1;
-        unsigned int cpu;
-        int err;
-
-        /* map must not be done on immutable chunk */
-        WARN_ON(chunk->immutable);
+        unsigned int cpu, tcpu;
+        int i, err;
 
         for_each_possible_cpu(cpu) {
-                err = map_kernel_range_noflush(
-                                pcpu_chunk_addr(chunk, cpu, page_start),
-                                (page_end - page_start) << PAGE_SHIFT,
-                                PAGE_KERNEL,
-                                pcpu_chunk_pagep(chunk, cpu, page_start));
+                err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start),
+                                       &pages[pcpu_page_idx(cpu, page_start)],
+                                       page_end - page_start);
                 if (err < 0)
-                        return err;
+                        goto err;
+        }
+
+        /* mapping successful, link chunk and mark populated */
+        for (i = page_start; i < page_end; i++) {
+                for_each_possible_cpu(cpu)
+                        pcpu_set_page_chunk(pages[pcpu_page_idx(cpu, i)],
+                                            chunk);
+                __set_bit(i, populated);
         }
 
-        /* flush at once, please read comments in pcpu_unmap() */
-        flush_cache_vmap(pcpu_chunk_addr(chunk, 0, page_start),
-                         pcpu_chunk_addr(chunk, last, page_end));
         return 0;
+
+err:
+        for_each_possible_cpu(tcpu) {
+                if (tcpu == cpu)
+                        break;
+                __pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start),
+                                   page_end - page_start);
+        }
+        return err;
+}
+
+/**
+ * pcpu_post_map_flush - flush cache after mapping
+ * @chunk: pcpu_chunk the regions to be flushed belong to
+ * @page_start: page index of the first page to be flushed
+ * @page_end: page index of the last page to be flushed + 1
+ *
+ * Pages [@page_start,@page_end) of @chunk have been mapped.  Flush
+ * cache.
+ *
+ * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
+ * for the whole region.
+ */
+static void pcpu_post_map_flush(struct pcpu_chunk *chunk,
+                                int page_start, int page_end)
+{
+        flush_cache_vmap(
+                pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start),
+                pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end));
+}
+
+/**
+ * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk
+ * @chunk: chunk to depopulate
+ * @off: offset to the area to depopulate
+ * @size: size of the area to depopulate in bytes
+ * @flush: whether to flush cache and tlb or not
+ *
+ * For each cpu, depopulate and unmap pages [@page_start,@page_end)
+ * from @chunk.  If @flush is true, vcache is flushed before unmapping
+ * and tlb after.
+ *
+ * CONTEXT:
+ * pcpu_alloc_mutex.
+ */
+static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size)
+{
+        int page_start = PFN_DOWN(off);
+        int page_end = PFN_UP(off + size);
+        struct page **pages;
+        unsigned long *populated;
+        int rs, re;
+
+        /* quick path, check whether it's empty already */
+        pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
+                if (rs == page_start && re == page_end)
+                        return;
+                break;
+        }
+
+        /* immutable chunks can't be depopulated */
+        WARN_ON(chunk->immutable);
+
+        /*
+         * If control reaches here, there must have been at least one
+         * successful population attempt so the temp pages array must
+         * be available now.
+         */
+        pages = pcpu_get_pages_and_bitmap(chunk, &populated, false);
+        BUG_ON(!pages);
+
+        /* unmap and free */
+        pcpu_pre_unmap_flush(chunk, page_start, page_end);
+
+        pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end)
+                pcpu_unmap_pages(chunk, pages, populated, rs, re);
+
+        /* no need to flush tlb, vmalloc will handle it lazily */
+
+        pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end)
+                pcpu_free_pages(chunk, pages, populated, rs, re);
+
+        /* commit new bitmap */
+        bitmap_copy(chunk->populated, populated, pcpu_unit_pages);
 }
 
 /**
@@ -727,57 +960,68 @@ static int pcpu_map(struct pcpu_chunk *chunk, int page_start, int page_end)
  */
 static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size)
 {
-        const gfp_t alloc_mask = GFP_KERNEL | __GFP_HIGHMEM | __GFP_COLD;
         int page_start = PFN_DOWN(off);
         int page_end = PFN_UP(off + size);
-        int map_start = -1;
-        int uninitialized_var(map_end);
+        int free_end = page_start, unmap_end = page_start;
+        struct page **pages;
+        unsigned long *populated;
         unsigned int cpu;
-        int i;
+        int rs, re, rc;
 
-        for (i = page_start; i < page_end; i++) {
-                if (pcpu_chunk_page_occupied(chunk, i)) {
-                        if (map_start >= 0) {
-                                if (pcpu_map(chunk, map_start, map_end))
-                                        goto err;
-                                map_start = -1;
-                        }
-                        continue;
-                }
+        /* quick path, check whether all pages are already there */
+        pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end) {
+                if (rs == page_start && re == page_end)
+                        goto clear;
+                break;
+        }
 
-                map_start = map_start < 0 ? i : map_start;
-                map_end = i + 1;
+        /* need to allocate and map pages, this chunk can't be immutable */
+        WARN_ON(chunk->immutable);
 
-                for_each_possible_cpu(cpu) {
-                        struct page **pagep = pcpu_chunk_pagep(chunk, cpu, i);
+        pages = pcpu_get_pages_and_bitmap(chunk, &populated, true);
+        if (!pages)
+                return -ENOMEM;
 
-                        *pagep = alloc_pages_node(cpu_to_node(cpu),
-                                                  alloc_mask, 0);
-                        if (!*pagep)
-                                goto err;
-                }
+        /* alloc and map */
+        pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
+                rc = pcpu_alloc_pages(chunk, pages, populated, rs, re);
+                if (rc)
+                        goto err_free;
+                free_end = re;
         }
 
-        if (map_start >= 0 && pcpu_map(chunk, map_start, map_end))
-                goto err;
+        pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
+                rc = pcpu_map_pages(chunk, pages, populated, rs, re);
+                if (rc)
+                        goto err_unmap;
+                unmap_end = re;
+        }
+        pcpu_post_map_flush(chunk, page_start, page_end);
 
+        /* commit new bitmap */
+        bitmap_copy(chunk->populated, populated, pcpu_unit_pages);
+clear:
         for_each_possible_cpu(cpu)
-                memset(chunk->vm->addr + cpu * pcpu_unit_size + off, 0,
-                       size);
-
+                memset((void *)pcpu_chunk_addr(chunk, cpu, 0) + off, 0, size);
         return 0;
-err:
-        /* likely under heavy memory pressure, give memory back */
-        pcpu_depopulate_chunk(chunk, off, size, true);
-        return -ENOMEM;
+
+err_unmap:
+        pcpu_pre_unmap_flush(chunk, page_start, unmap_end);
+        pcpu_for_each_unpop_region(chunk, rs, re, page_start, unmap_end)
+                pcpu_unmap_pages(chunk, pages, populated, rs, re);
+        pcpu_post_unmap_tlb_flush(chunk, page_start, unmap_end);
+err_free:
+        pcpu_for_each_unpop_region(chunk, rs, re, page_start, free_end)
+                pcpu_free_pages(chunk, pages, populated, rs, re);
+        return rc;
 }
 
 static void free_pcpu_chunk(struct pcpu_chunk *chunk)
 {
         if (!chunk)
                 return;
-        if (chunk->vm)
-                free_vm_area(chunk->vm);
+        if (chunk->vms)
+                pcpu_free_vm_areas(chunk->vms, pcpu_nr_groups);
         pcpu_mem_free(chunk->map, chunk->map_alloc * sizeof(chunk->map[0]));
         kfree(chunk);
 }
@@ -793,10 +1037,11 @@ static struct pcpu_chunk *alloc_pcpu_chunk(void)
         chunk->map = pcpu_mem_alloc(PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0]));
         chunk->map_alloc = PCPU_DFL_MAP_ALLOC;
         chunk->map[chunk->map_used++] = pcpu_unit_size;
-        chunk->page = chunk->page_ar;
 
-        chunk->vm = get_vm_area(pcpu_chunk_size, GFP_KERNEL);
-        if (!chunk->vm) {
+        chunk->vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes,
+                                       pcpu_nr_groups, pcpu_atom_size,
+                                       GFP_KERNEL);
+        if (!chunk->vms) {
                 free_pcpu_chunk(chunk);
                 return NULL;
         }
@@ -804,6 +1049,7 @@ static struct pcpu_chunk *alloc_pcpu_chunk(void)
         INIT_LIST_HEAD(&chunk->list);
         chunk->free_size = pcpu_unit_size;
         chunk->contig_hint = pcpu_unit_size;
+        chunk->base_addr = chunk->vms[0]->addr - pcpu_group_offsets[0];
 
         return chunk;
 }
@@ -824,8 +1070,11 @@ static struct pcpu_chunk *alloc_pcpu_chunk(void)
  */
 static void *pcpu_alloc(size_t size, size_t align, bool reserved)
 {
+        static int warn_limit = 10;
         struct pcpu_chunk *chunk;
-        int slot, off;
+        const char *err;
+        int slot, off, new_alloc;
+        unsigned long flags;
 
         if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) {
                 WARN(true, "illegal size (%zu) or align (%zu) for "
@@ -834,17 +1083,31 @@ static void *pcpu_alloc(size_t size, size_t align, bool reserved)
         }
 
         mutex_lock(&pcpu_alloc_mutex);
-        spin_lock_irq(&pcpu_lock);
+        spin_lock_irqsave(&pcpu_lock, flags);
 
         /* serve reserved allocations from the reserved chunk if available */
         if (reserved && pcpu_reserved_chunk) {
                 chunk = pcpu_reserved_chunk;
-                if (size > chunk->contig_hint ||
-                    pcpu_extend_area_map(chunk) < 0)
+
+                if (size > chunk->contig_hint) {
+                        err = "alloc from reserved chunk failed";
                         goto fail_unlock;
+                }
+
+                while ((new_alloc = pcpu_need_to_extend(chunk))) {
+                        spin_unlock_irqrestore(&pcpu_lock, flags);
+                        if (pcpu_extend_area_map(chunk, new_alloc) < 0) {
+                                err = "failed to extend area map of reserved chunk";
+                                goto fail_unlock_mutex;
+                        }
+                        spin_lock_irqsave(&pcpu_lock, flags);
+                }
+
                 off = pcpu_alloc_area(chunk, size, align);
                 if (off >= 0)
                         goto area_found;
+
+                err = "alloc from reserved chunk failed";
                 goto fail_unlock;
         }
 
@@ -855,13 +1118,20 @@ restart:
                         if (size > chunk->contig_hint)
                                 continue;
 
-                        switch (pcpu_extend_area_map(chunk)) {
-                        case 0:
-                                break;
-                        case 1:
-                                goto restart;   /* pcpu_lock dropped, restart */
-                        default:
-                                goto fail_unlock;
+                        new_alloc = pcpu_need_to_extend(chunk);
+                        if (new_alloc) {
+                                spin_unlock_irqrestore(&pcpu_lock, flags);
+                                if (pcpu_extend_area_map(chunk,
+                                                         new_alloc) < 0) {
+                                        err = "failed to extend area map";
+                                        goto fail_unlock_mutex;
+                                }
+                                spin_lock_irqsave(&pcpu_lock, flags);
+                                /*
+                                 * pcpu_lock has been dropped, need to
+                                 * restart cpu_slot list walking.
+                                 */
+                                goto restart;
                         }
 
                         off = pcpu_alloc_area(chunk, size, align);
@@ -871,35 +1141,45 @@ restart:
         }
 
         /* hmmm... no space left, create a new chunk */
-        spin_unlock_irq(&pcpu_lock);
+        spin_unlock_irqrestore(&pcpu_lock, flags);
 
         chunk = alloc_pcpu_chunk();
-        if (!chunk)
+        if (!chunk) {
+                err = "failed to allocate new chunk";
                 goto fail_unlock_mutex;
+        }
 
-        spin_lock_irq(&pcpu_lock);
+        spin_lock_irqsave(&pcpu_lock, flags);
         pcpu_chunk_relocate(chunk, -1);
-        pcpu_chunk_addr_insert(chunk);
         goto restart;
 
 area_found:
-        spin_unlock_irq(&pcpu_lock);
+        spin_unlock_irqrestore(&pcpu_lock, flags);
 
         /* populate, map and clear the area */
         if (pcpu_populate_chunk(chunk, off, size)) {
-                spin_lock_irq(&pcpu_lock);
+                spin_lock_irqsave(&pcpu_lock, flags);
                 pcpu_free_area(chunk, off);
+                err = "failed to populate";
                 goto fail_unlock;
         }
 
         mutex_unlock(&pcpu_alloc_mutex);
 
-        return __addr_to_pcpu_ptr(chunk->vm->addr + off);
+        /* return address relative to base address */
+        return __addr_to_pcpu_ptr(chunk->base_addr + off);
 
 fail_unlock:
-        spin_unlock_irq(&pcpu_lock);
+        spin_unlock_irqrestore(&pcpu_lock, flags);
 fail_unlock_mutex:
         mutex_unlock(&pcpu_alloc_mutex);
+        if (warn_limit) {
+                pr_warning("PERCPU: allocation failed, size=%zu align=%zu, "
+                           "%s\n", size, align, err);
+                dump_stack();
+                if (!--warn_limit)
+                        pr_info("PERCPU: limit reached, disable warning\n");
+        }
         return NULL;
 }
 
@@ -968,17 +1248,17 @@ static void pcpu_reclaim(struct work_struct *work)
                 if (chunk == list_first_entry(head, struct pcpu_chunk, list))
                         continue;
 
-                rb_erase(&chunk->rb_node, &pcpu_addr_root);
                 list_move(&chunk->list, &todo);
         }
 
         spin_unlock_irq(&pcpu_lock);
-        mutex_unlock(&pcpu_alloc_mutex);
 
         list_for_each_entry_safe(chunk, next, &todo, list) {
-                pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size, false);
+                pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size);
                 free_pcpu_chunk(chunk);
         }
+
+        mutex_unlock(&pcpu_alloc_mutex);
 }
 
 /**
@@ -1003,7 +1283,7 @@ void free_percpu(void *ptr)
         spin_lock_irqsave(&pcpu_lock, flags);
 
         chunk = pcpu_chunk_addr_search(addr);
-        off = addr - chunk->vm->addr;
+        off = addr - chunk->base_addr;
 
         pcpu_free_area(chunk, off);
 
@@ -1022,30 +1302,299 @@ void free_percpu(void *ptr)
 }
 EXPORT_SYMBOL_GPL(free_percpu);
 
+static inline size_t pcpu_calc_fc_sizes(size_t static_size,
+                                        size_t reserved_size,
+                                        ssize_t *dyn_sizep)
+{
+        size_t size_sum;
+
+        size_sum = PFN_ALIGN(static_size + reserved_size +
+                             (*dyn_sizep >= 0 ? *dyn_sizep : 0));
+        if (*dyn_sizep != 0)
+                *dyn_sizep = size_sum - static_size - reserved_size;
+
+        return size_sum;
+}
+
 /**
- * pcpu_setup_first_chunk - initialize the first percpu chunk
- * @get_page_fn: callback to fetch page pointer
- * @static_size: the size of static percpu area in bytes
+ * pcpu_alloc_alloc_info - allocate percpu allocation info
+ * @nr_groups: the number of groups
+ * @nr_units: the number of units
+ *
+ * Allocate ai which is large enough for @nr_groups groups containing
+ * @nr_units units.  The returned ai's groups[0].cpu_map points to the
+ * cpu_map array which is long enough for @nr_units and filled with
+ * NR_CPUS.  It's the caller's responsibility to initialize cpu_map
+ * pointer of other groups.
+ *
+ * RETURNS:
+ * Pointer to the allocated pcpu_alloc_info on success, NULL on
+ * failure.
+ */
+struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups,
+                                                      int nr_units)
+{
+        struct pcpu_alloc_info *ai;
+        size_t base_size, ai_size;
+        void *ptr;
+        int unit;
+
+        base_size = ALIGN(sizeof(*ai) + nr_groups * sizeof(ai->groups[0]),
+                          __alignof__(ai->groups[0].cpu_map[0]));
+        ai_size = base_size + nr_units * sizeof(ai->groups[0].cpu_map[0]);
+
+        ptr = alloc_bootmem_nopanic(PFN_ALIGN(ai_size));
+        if (!ptr)
+                return NULL;
+        ai = ptr;
+        ptr += base_size;
+
+        ai->groups[0].cpu_map = ptr;
+
+        for (unit = 0; unit < nr_units; unit++)
+                ai->groups[0].cpu_map[unit] = NR_CPUS;
+
+        ai->nr_groups = nr_groups;
+        ai->__ai_size = PFN_ALIGN(ai_size);
+
+        return ai;
+}
+
+/**
+ * pcpu_free_alloc_info - free percpu allocation info
+ * @ai: pcpu_alloc_info to free
+ *
+ * Free @ai which was allocated by pcpu_alloc_alloc_info().
+ */
+void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai)
+{
+        free_bootmem(__pa(ai), ai->__ai_size);
+}
+
+/**
+ * pcpu_build_alloc_info - build alloc_info considering distances between CPUs
  * @reserved_size: the size of reserved percpu area in bytes
  * @dyn_size: free size for dynamic allocation in bytes, -1 for auto
- * @unit_size: unit size in bytes, must be multiple of PAGE_SIZE, -1 for auto
- * @base_addr: mapped address, NULL for auto
- * @populate_pte_fn: callback to allocate pagetable, NULL if unnecessary
+ * @atom_size: allocation atom size
+ * @cpu_distance_fn: callback to determine distance between cpus, optional
+ *
+ * This function determines grouping of units, their mappings to cpus
+ * and other parameters considering needed percpu size, allocation
+ * atom size and distances between CPUs.
+ *
+ * Groups are always mutliples of atom size and CPUs which are of
+ * LOCAL_DISTANCE both ways are grouped together and share space for
+ * units in the same group.  The returned configuration is guaranteed
+ * to have CPUs on different nodes on different groups and >=75% usage
+ * of allocated virtual address space.
+ *
+ * RETURNS:
+ * On success, pointer to the new allocation_info is returned.  On
+ * failure, ERR_PTR value is returned.
+ */
+struct pcpu_alloc_info * __init pcpu_build_alloc_info(
+                                size_t reserved_size, ssize_t dyn_size,
+                                size_t atom_size,
+                                pcpu_fc_cpu_distance_fn_t cpu_distance_fn)
+{
+        static int group_map[NR_CPUS] __initdata;
+        static int group_cnt[NR_CPUS] __initdata;
+        const size_t static_size = __per_cpu_end - __per_cpu_start;
+        int group_cnt_max = 0, nr_groups = 1, nr_units = 0;
+        size_t size_sum, min_unit_size, alloc_size;
+        int upa, max_upa, uninitialized_var(best_upa);  /* units_per_alloc */
+        int last_allocs, group, unit;
+        unsigned int cpu, tcpu;
+        struct pcpu_alloc_info *ai;
+        unsigned int *cpu_map;
+
+        /* this function may be called multiple times */
+        memset(group_map, 0, sizeof(group_map));
+        memset(group_cnt, 0, sizeof(group_map));
+
+        /*
+         * Determine min_unit_size, alloc_size and max_upa such that
+         * alloc_size is multiple of atom_size and is the smallest
+         * which can accomodate 4k aligned segments which are equal to
+         * or larger than min_unit_size.
+         */
+        size_sum = pcpu_calc_fc_sizes(static_size, reserved_size, &dyn_size);
+        min_unit_size = max_t(size_t, size_sum, PCPU_MIN_UNIT_SIZE);
+
+        alloc_size = roundup(min_unit_size, atom_size);
+        upa = alloc_size / min_unit_size;
+        while (alloc_size % upa || ((alloc_size / upa) & ~PAGE_MASK))
+                upa--;
+        max_upa = upa;
+
+        /* group cpus according to their proximity */
+        for_each_possible_cpu(cpu) {
+                group = 0;
+        next_group:
+                for_each_possible_cpu(tcpu) {
+                        if (cpu == tcpu)
+                                break;
+                        if (group_map[tcpu] == group && cpu_distance_fn &&
+                            (cpu_distance_fn(cpu, tcpu) > LOCAL_DISTANCE ||
+                             cpu_distance_fn(tcpu, cpu) > LOCAL_DISTANCE)) {
+                                group++;
+                                nr_groups = max(nr_groups, group + 1);
+                                goto next_group;
+                        }
+                }
+                group_map[cpu] = group;
+                group_cnt[group]++;
+                group_cnt_max = max(group_cnt_max, group_cnt[group]);
+        }
+
+        /*
+         * Expand unit size until address space usage goes over 75%
+         * and then as much as possible without using more address
+         * space.
+         */
+        last_allocs = INT_MAX;
+        for (upa = max_upa; upa; upa--) {
+                int allocs = 0, wasted = 0;
+
+                if (alloc_size % upa || ((alloc_size / upa) & ~PAGE_MASK))
+                        continue;
+
+                for (group = 0; group < nr_groups; group++) {
+                        int this_allocs = DIV_ROUND_UP(group_cnt[group], upa);
+                        allocs += this_allocs;
+                        wasted += this_allocs * upa - group_cnt[group];
+                }
+
+                /*
+                 * Don't accept if wastage is over 25%.  The
+                 * greater-than comparison ensures upa==1 always
+                 * passes the following check.
+                 */
+                if (wasted > num_possible_cpus() / 3)
+                        continue;
+
+                /* and then don't consume more memory */
+                if (allocs > last_allocs)
+                        break;
+                last_allocs = allocs;
+                best_upa = upa;
+        }
+        upa = best_upa;
+
+        /* allocate and fill alloc_info */
+        for (group = 0; group < nr_groups; group++)
+                nr_units += roundup(group_cnt[group], upa);
+
+        ai = pcpu_alloc_alloc_info(nr_groups, nr_units);
+        if (!ai)
+                return ERR_PTR(-ENOMEM);
+        cpu_map = ai->groups[0].cpu_map;
+
+        for (group = 0; group < nr_groups; group++) {
+                ai->groups[group].cpu_map = cpu_map;
+                cpu_map += roundup(group_cnt[group], upa);
+        }
+
+        ai->static_size = static_size;
+        ai->reserved_size = reserved_size;
+        ai->dyn_size = dyn_size;
+        ai->unit_size = alloc_size / upa;
+        ai->atom_size = atom_size;
+        ai->alloc_size = alloc_size;
+
+        for (group = 0, unit = 0; group_cnt[group]; group++) {
+                struct pcpu_group_info *gi = &ai->groups[group];
+
+                /*
+                 * Initialize base_offset as if all groups are located
+                 * back-to-back.  The caller should update this to
+                 * reflect actual allocation.
+                 */
+                gi->base_offset = unit * ai->unit_size;
+
+                for_each_possible_cpu(cpu)
+                        if (group_map[cpu] == group)
+                                gi->cpu_map[gi->nr_units++] = cpu;
+                gi->nr_units = roundup(gi->nr_units, upa);
+                unit += gi->nr_units;
+        }
+        BUG_ON(unit != nr_units);
+
+        return ai;
+}
+
+/**
+ * pcpu_dump_alloc_info - print out information about pcpu_alloc_info
+ * @lvl: loglevel
+ * @ai: allocation info to dump
+ *
+ * Print out information about @ai using loglevel @lvl.
+ */
+static void pcpu_dump_alloc_info(const char *lvl,
+                                 const struct pcpu_alloc_info *ai)
+{
+        int group_width = 1, cpu_width = 1, width;
+        char empty_str[] = "--------";
+        int alloc = 0, alloc_end = 0;
+        int group, v;
+        int upa, apl;   /* units per alloc, allocs per line */
+
+        v = ai->nr_groups;
+        while (v /= 10)
+                group_width++;
+
+        v = num_possible_cpus();
+        while (v /= 10)
+                cpu_width++;
+        empty_str[min_t(int, cpu_width, sizeof(empty_str) - 1)] = '\0';
+
+        upa = ai->alloc_size / ai->unit_size;
+        width = upa * (cpu_width + 1) + group_width + 3;
+        apl = rounddown_pow_of_two(max(60 / width, 1));
+
+        printk("%spcpu-alloc: s%zu r%zu d%zu u%zu alloc=%zu*%zu",
+               lvl, ai->static_size, ai->reserved_size, ai->dyn_size,
+               ai->unit_size, ai->alloc_size / ai->atom_size, ai->atom_size);
+
+        for (group = 0; group < ai->nr_groups; group++) {
+                const struct pcpu_group_info *gi = &ai->groups[group];
+                int unit = 0, unit_end = 0;
+
+                BUG_ON(gi->nr_units % upa);
+                for (alloc_end += gi->nr_units / upa;
+                     alloc < alloc_end; alloc++) {
+                        if (!(alloc % apl)) {
+                                printk("\n");
+                                printk("%spcpu-alloc: ", lvl);
+                        }
+                        printk("[%0*d] ", group_width, group);
+
+                        for (unit_end += upa; unit < unit_end; unit++)
+                                if (gi->cpu_map[unit] != NR_CPUS)
+                                        printk("%0*d ", cpu_width,
+                                               gi->cpu_map[unit]);
+                                else
+                                        printk("%s ", empty_str);
+                }
+        }
+        printk("\n");
+}
+
+/**
+ * pcpu_setup_first_chunk - initialize the first percpu chunk
+ * @ai: pcpu_alloc_info describing how to percpu area is shaped
+ * @base_addr: mapped address
  *
  * Initialize the first percpu chunk which contains the kernel static
  * perpcu area.  This function is to be called from arch percpu area
- * setup path.  The first two parameters are mandatory.  The rest are
- * optional.
- *
- * @get_page_fn() should return pointer to percpu page given cpu
- * number and page number.  It should at least return enough pages to
- * cover the static area.  The returned pages for static area should
- * have been initialized with valid data.  If @unit_size is specified,
- * it can also return pages after the static area.  NULL return
- * indicates end of pages for the cpu.  Note that @get_page_fn() must
- * return the same number of pages for all cpus.
- *
- * @reserved_size, if non-zero, specifies the amount of bytes to
+ * setup path.
+ *
+ * @ai contains all information necessary to initialize the first
+ * chunk and prime the dynamic percpu allocator.
+ *
+ * @ai->static_size is the size of static percpu area.
+ *
+ * @ai->reserved_size, if non-zero, specifies the amount of bytes to
  * reserve after the static area in the first chunk.  This reserves
  * the first chunk such that it's available only through reserved
  * percpu allocation.  This is primarily used to serve module percpu
@@ -1053,22 +1602,29 @@ EXPORT_SYMBOL_GPL(free_percpu);
  * limited offset range for symbol relocations to guarantee module
  * percpu symbols fall inside the relocatable range.
  *
- * @dyn_size, if non-negative, determines the number of bytes
- * available for dynamic allocation in the first chunk.  Specifying
- * non-negative value makes percpu leave alone the area beyond
- * @static_size + @reserved_size + @dyn_size.
+ * @ai->dyn_size determines the number of bytes available for dynamic
+ * allocation in the first chunk.  The area between @ai->static_size +
+ * @ai->reserved_size + @ai->dyn_size and @ai->unit_size is unused.
+ *
+ * @ai->unit_size specifies unit size and must be aligned to PAGE_SIZE
+ * and equal to or larger than @ai->static_size + @ai->reserved_size +
+ * @ai->dyn_size.
+ *
+ * @ai->atom_size is the allocation atom size and used as alignment
+ * for vm areas.
  *
- * @unit_size, if non-negative, specifies unit size and must be
- * aligned to PAGE_SIZE and equal to or larger than @static_size +
- * @reserved_size + if non-negative, @dyn_size.
+ * @ai->alloc_size is the allocation size and always multiple of
+ * @ai->atom_size.  This is larger than @ai->atom_size if
+ * @ai->unit_size is larger than @ai->atom_size.
  *
- * Non-null @base_addr means that the caller already allocated virtual
- * region for the first chunk and mapped it.  percpu must not mess
- * with the chunk.  Note that @base_addr with 0 @unit_size or non-NULL
- * @populate_pte_fn doesn't make any sense.
+ * @ai->nr_groups and @ai->groups describe virtual memory layout of
+ * percpu areas.  Units which should be colocated are put into the
+ * same group.  Dynamic VM areas will be allocated according to these
+ * groupings.  If @ai->nr_groups is zero, a single group containing
+ * all units is assumed.
  *
- * @populate_pte_fn is used to populate the pagetable.  NULL means the
- * caller already populated the pagetable.
+ * The caller should have mapped the first chunk at @base_addr and
+ * copied static data to each unit.
  *
  * If the first chunk ends up with both reserved and dynamic areas, it
  * is served by two chunks - one to serve the core static and reserved
@@ -1078,49 +1634,98 @@ EXPORT_SYMBOL_GPL(free_percpu);
  * and available for dynamic allocation like any other chunks.
  *
  * RETURNS:
- * The determined pcpu_unit_size which can be used to initialize
- * percpu access.
+ * 0 on success, -errno on failure.
  */
-size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
-                                     size_t static_size, size_t reserved_size,
-                                     ssize_t dyn_size, ssize_t unit_size,
-                                     void *base_addr,
-                                     pcpu_populate_pte_fn_t populate_pte_fn)
+int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
+                                  void *base_addr)
 {
-        static struct vm_struct first_vm;
+        static char cpus_buf[4096] __initdata;
         static int smap[2], dmap[2];
-        size_t size_sum = static_size + reserved_size +
-                          (dyn_size >= 0 ? dyn_size : 0);
+        size_t dyn_size = ai->dyn_size;
+        size_t size_sum = ai->static_size + ai->reserved_size + dyn_size;
         struct pcpu_chunk *schunk, *dchunk = NULL;
+        unsigned long *group_offsets;
+        size_t *group_sizes;
+        unsigned long *unit_off;
         unsigned int cpu;
-        int nr_pages;
-        int err, i;
+        int *unit_map;
+        int group, unit, i;
 
-        /* santiy checks */
+        cpumask_scnprintf(cpus_buf, sizeof(cpus_buf), cpu_possible_mask);
+
+#define PCPU_SETUP_BUG_ON(cond) do {                                    \
+        if (unlikely(cond)) {                                           \
+                pr_emerg("PERCPU: failed to initialize, %s", #cond);    \
+                pr_emerg("PERCPU: cpu_possible_mask=%s\n", cpus_buf);   \
+                pcpu_dump_alloc_info(KERN_EMERG, ai);                   \
+                BUG();                                                  \
+        }                                                               \
+} while (0)
+
+        /* sanity checks */
         BUILD_BUG_ON(ARRAY_SIZE(smap) >= PCPU_DFL_MAP_ALLOC ||
                      ARRAY_SIZE(dmap) >= PCPU_DFL_MAP_ALLOC);
-        BUG_ON(!static_size);
-        if (unit_size >= 0) {
-                BUG_ON(unit_size < size_sum);
-                BUG_ON(unit_size & ~PAGE_MASK);
-                BUG_ON(unit_size < PCPU_MIN_UNIT_SIZE);
-        } else
-                BUG_ON(base_addr);
-        BUG_ON(base_addr && populate_pte_fn);
-
-        if (unit_size >= 0)
-                pcpu_unit_pages = unit_size >> PAGE_SHIFT;
-        else
-                pcpu_unit_pages = max_t(int, PCPU_MIN_UNIT_SIZE >> PAGE_SHIFT,
-                                        PFN_UP(size_sum));
+        PCPU_SETUP_BUG_ON(ai->nr_groups <= 0);
+        PCPU_SETUP_BUG_ON(!ai->static_size);
+        PCPU_SETUP_BUG_ON(!base_addr);
+        PCPU_SETUP_BUG_ON(ai->unit_size < size_sum);
+        PCPU_SETUP_BUG_ON(ai->unit_size & ~PAGE_MASK);
+        PCPU_SETUP_BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE);
+
+        /* process group information and build config tables accordingly */
+        group_offsets = alloc_bootmem(ai->nr_groups * sizeof(group_offsets[0]));
+        group_sizes = alloc_bootmem(ai->nr_groups * sizeof(group_sizes[0]));
+        unit_map = alloc_bootmem(nr_cpu_ids * sizeof(unit_map[0]));
+        unit_off = alloc_bootmem(nr_cpu_ids * sizeof(unit_off[0]));
+
+        for (cpu = 0; cpu < nr_cpu_ids; cpu++)
+                unit_map[cpu] = UINT_MAX;
+        pcpu_first_unit_cpu = NR_CPUS;
+
+        for (group = 0, unit = 0; group < ai->nr_groups; group++, unit += i) {
+                const struct pcpu_group_info *gi = &ai->groups[group];
+
+                group_offsets[group] = gi->base_offset;
+                group_sizes[group] = gi->nr_units * ai->unit_size;
+
+                for (i = 0; i < gi->nr_units; i++) {
+                        cpu = gi->cpu_map[i];
+                        if (cpu == NR_CPUS)
+                                continue;
 
-        pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT;
-        pcpu_chunk_size = num_possible_cpus() * pcpu_unit_size;
-        pcpu_chunk_struct_size = sizeof(struct pcpu_chunk)
-                + num_possible_cpus() * pcpu_unit_pages * sizeof(struct page *);
+                        PCPU_SETUP_BUG_ON(cpu > nr_cpu_ids);
+                        PCPU_SETUP_BUG_ON(!cpu_possible(cpu));
+                        PCPU_SETUP_BUG_ON(unit_map[cpu] != UINT_MAX);
+
+                        unit_map[cpu] = unit + i;
+                        unit_off[cpu] = gi->base_offset + i * ai->unit_size;
+
+                        if (pcpu_first_unit_cpu == NR_CPUS)
+                                pcpu_first_unit_cpu = cpu;
+                }
+        }
+        pcpu_last_unit_cpu = cpu;
+        pcpu_nr_units = unit;
+
+        for_each_possible_cpu(cpu)
+                PCPU_SETUP_BUG_ON(unit_map[cpu] == UINT_MAX);
+
+        /* we're done parsing the input, undefine BUG macro and dump config */
+#undef PCPU_SETUP_BUG_ON
+        pcpu_dump_alloc_info(KERN_INFO, ai);
 
-        if (dyn_size < 0)
-                dyn_size = pcpu_unit_size - static_size - reserved_size;
+        pcpu_nr_groups = ai->nr_groups;
+        pcpu_group_offsets = group_offsets;
+        pcpu_group_sizes = group_sizes;
+        pcpu_unit_map = unit_map;
+        pcpu_unit_offsets = unit_off;
+
+        /* determine basic parameters */
+        pcpu_unit_pages = ai->unit_size >> PAGE_SHIFT;
+        pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT;
+        pcpu_atom_size = ai->atom_size;
+        pcpu_chunk_struct_size = sizeof(struct pcpu_chunk) +
+                BITS_TO_LONGS(pcpu_unit_pages) * sizeof(unsigned long);
 
         /*
          * Allocate chunk slots.  The additional last slot is for
@@ -1140,187 +1745,368 @@ size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
          */
         schunk = alloc_bootmem(pcpu_chunk_struct_size);
         INIT_LIST_HEAD(&schunk->list);
-        schunk->vm = &first_vm;
+        schunk->base_addr = base_addr;
         schunk->map = smap;
         schunk->map_alloc = ARRAY_SIZE(smap);
-        schunk->page = schunk->page_ar;
+        schunk->immutable = true;
+        bitmap_fill(schunk->populated, pcpu_unit_pages);
 
-        if (reserved_size) {
-                schunk->free_size = reserved_size;
-                pcpu_reserved_chunk = schunk;   /* not for dynamic alloc */
+        if (ai->reserved_size) {
+                schunk->free_size = ai->reserved_size;
+                pcpu_reserved_chunk = schunk;
+                pcpu_reserved_chunk_limit = ai->static_size + ai->reserved_size;
         } else {
                 schunk->free_size = dyn_size;
                 dyn_size = 0;                   /* dynamic area covered */
         }
         schunk->contig_hint = schunk->free_size;
 
-        schunk->map[schunk->map_used++] = -static_size;
+        schunk->map[schunk->map_used++] = -ai->static_size;
         if (schunk->free_size)
                 schunk->map[schunk->map_used++] = schunk->free_size;
 
-        pcpu_reserved_chunk_limit = static_size + schunk->free_size;
-
         /* init dynamic chunk if necessary */
         if (dyn_size) {
-                dchunk = alloc_bootmem(sizeof(struct pcpu_chunk));
+                dchunk = alloc_bootmem(pcpu_chunk_struct_size);
                 INIT_LIST_HEAD(&dchunk->list);
-                dchunk->vm = &first_vm;
+                dchunk->base_addr = base_addr;
                 dchunk->map = dmap;
                 dchunk->map_alloc = ARRAY_SIZE(dmap);
-                dchunk->page = schunk->page_ar; /* share page map with schunk */
+                dchunk->immutable = true;
+                bitmap_fill(dchunk->populated, pcpu_unit_pages);
 
                 dchunk->contig_hint = dchunk->free_size = dyn_size;
                 dchunk->map[dchunk->map_used++] = -pcpu_reserved_chunk_limit;
                 dchunk->map[dchunk->map_used++] = dchunk->free_size;
         }
 
-        /* allocate vm address */
-        first_vm.flags = VM_ALLOC;
-        first_vm.size = pcpu_chunk_size;
-
-        if (!base_addr)
-                vm_area_register_early(&first_vm, PAGE_SIZE);
-        else {
-                /*
-                 * Pages already mapped.  No need to remap into
-                 * vmalloc area.  In this case the first chunks can't
-                 * be mapped or unmapped by percpu and are marked
-                 * immutable.
-                 */
-                first_vm.addr = base_addr;
-                schunk->immutable = true;
-                if (dchunk)
-                        dchunk->immutable = true;
-        }
-
-        /* assign pages */
-        nr_pages = -1;
-        for_each_possible_cpu(cpu) {
-                for (i = 0; i < pcpu_unit_pages; i++) {
-                        struct page *page = get_page_fn(cpu, i);
-
-                        if (!page)
-                                break;
-                        *pcpu_chunk_pagep(schunk, cpu, i) = page;
-                }
-
-                BUG_ON(i < PFN_UP(static_size));
-
-                if (nr_pages < 0)
-                        nr_pages = i;
-                else
-                        BUG_ON(nr_pages != i);
-        }
-
-        /* map them */
-        if (populate_pte_fn) {
-                for_each_possible_cpu(cpu)
-                        for (i = 0; i < nr_pages; i++)
-                                populate_pte_fn(pcpu_chunk_addr(schunk,
-                                                                cpu, i));
-
-                err = pcpu_map(schunk, 0, nr_pages);
-                if (err)
-                        panic("failed to setup static percpu area, err=%d\n",
-                              err);
-        }
-
         /* link the first chunk in */
-        if (!dchunk) {
-                pcpu_chunk_relocate(schunk, -1);
-                pcpu_chunk_addr_insert(schunk);
-        } else {
-                pcpu_chunk_relocate(dchunk, -1);
-                pcpu_chunk_addr_insert(dchunk);
-        }
+        pcpu_first_chunk = dchunk ?: schunk;
+        pcpu_chunk_relocate(pcpu_first_chunk, -1);
 
         /* we're done */
-        pcpu_base_addr = (void *)pcpu_chunk_addr(schunk, 0, 0);
-        return pcpu_unit_size;
+        pcpu_base_addr = base_addr;
+        return 0;
 }
 
-/*
- * Embedding first chunk setup helper.
- */
-static void *pcpue_ptr __initdata;
-static size_t pcpue_size __initdata;
-static size_t pcpue_unit_size __initdata;
+const char *pcpu_fc_names[PCPU_FC_NR] __initdata = {
+        [PCPU_FC_AUTO]  = "auto",
+        [PCPU_FC_EMBED] = "embed",
+        [PCPU_FC_PAGE]  = "page",
+};
 
-static struct page * __init pcpue_get_page(unsigned int cpu, int pageno)
-{
-        size_t off = (size_t)pageno << PAGE_SHIFT;
+enum pcpu_fc pcpu_chosen_fc __initdata = PCPU_FC_AUTO;
 
-        if (off >= pcpue_size)
-                return NULL;
+static int __init percpu_alloc_setup(char *str)
+{
+        if (0)
+                /* nada */;
+#ifdef CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK
+        else if (!strcmp(str, "embed"))
+                pcpu_chosen_fc = PCPU_FC_EMBED;
+#endif
+#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
+        else if (!strcmp(str, "page"))
+                pcpu_chosen_fc = PCPU_FC_PAGE;
+#endif
+        else
+                pr_warning("PERCPU: unknown allocator %s specified\n", str);
 
-        return virt_to_page(pcpue_ptr + cpu * pcpue_unit_size + off);
+        return 0;
 }
+early_param("percpu_alloc", percpu_alloc_setup);
 
+#if defined(CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK) || \
+        !defined(CONFIG_HAVE_SETUP_PER_CPU_AREA)
 /**
  * pcpu_embed_first_chunk - embed the first percpu chunk into bootmem
- * @static_size: the size of static percpu area in bytes
  * @reserved_size: the size of reserved percpu area in bytes
  * @dyn_size: free size for dynamic allocation in bytes, -1 for auto
- * @unit_size: unit size in bytes, must be multiple of PAGE_SIZE, -1 for auto
+ * @atom_size: allocation atom size
+ * @cpu_distance_fn: callback to determine distance between cpus, optional
+ * @alloc_fn: function to allocate percpu page
+ * @free_fn: funtion to free percpu page
  *
  * This is a helper to ease setting up embedded first percpu chunk and
  * can be called where pcpu_setup_first_chunk() is expected.
  *
  * If this function is used to setup the first chunk, it is allocated
- * as a contiguous area using bootmem allocator and used as-is without
- * being mapped into vmalloc area.  This enables the first chunk to
- * piggy back on the linear physical mapping which often uses larger
- * page size.
+ * by calling @alloc_fn and used as-is without being mapped into
+ * vmalloc area.  Allocations are always whole multiples of @atom_size
+ * aligned to @atom_size.
+ *
+ * This enables the first chunk to piggy back on the linear physical
+ * mapping which often uses larger page size.  Please note that this
+ * can result in very sparse cpu->unit mapping on NUMA machines thus
+ * requiring large vmalloc address space.  Don't use this allocator if
+ * vmalloc space is not orders of magnitude larger than distances
+ * between node memory addresses (ie. 32bit NUMA machines).
  *
  * When @dyn_size is positive, dynamic area might be larger than
- * specified to fill page alignment.  Also, when @dyn_size is auto,
- * @dyn_size does not fill the whole first chunk but only what's
- * necessary for page alignment after static and reserved areas.
+ * specified to fill page alignment.  When @dyn_size is auto,
+ * @dyn_size is just big enough to fill page alignment after static
+ * and reserved areas.
  *
  * If the needed size is smaller than the minimum or specified unit
- * size, the leftover is returned to the bootmem allocator.
+ * size, the leftover is returned using @free_fn.
  *
  * RETURNS:
- * The determined pcpu_unit_size which can be used to initialize
- * percpu access on success, -errno on failure.
+ * 0 on success, -errno on failure.
  */
-ssize_t __init pcpu_embed_first_chunk(size_t static_size, size_t reserved_size,
-                                      ssize_t dyn_size, ssize_t unit_size)
+int __init pcpu_embed_first_chunk(size_t reserved_size, ssize_t dyn_size,
+                                  size_t atom_size,
+                                  pcpu_fc_cpu_distance_fn_t cpu_distance_fn,
+                                  pcpu_fc_alloc_fn_t alloc_fn,
+                                  pcpu_fc_free_fn_t free_fn)
 {
-        unsigned int cpu;
+        void *base = (void *)ULONG_MAX;
+        void **areas = NULL;
+        struct pcpu_alloc_info *ai;
+        size_t size_sum, areas_size, max_distance;
+        int group, i, rc;
+
+        ai = pcpu_build_alloc_info(reserved_size, dyn_size, atom_size,
+                                   cpu_distance_fn);
+        if (IS_ERR(ai))
+                return PTR_ERR(ai);
+
+        size_sum = ai->static_size + ai->reserved_size + ai->dyn_size;
+        areas_size = PFN_ALIGN(ai->nr_groups * sizeof(void *));
+
+        areas = alloc_bootmem_nopanic(areas_size);
+        if (!areas) {
+                rc = -ENOMEM;
+                goto out_free;
+        }
 
-        /* determine parameters and allocate */
-        pcpue_size = PFN_ALIGN(static_size + reserved_size +
-                               (dyn_size >= 0 ? dyn_size : 0));
-        if (dyn_size != 0)
-                dyn_size = pcpue_size - static_size - reserved_size;
-
-        if (unit_size >= 0) {
-                BUG_ON(unit_size < pcpue_size);
-                pcpue_unit_size = unit_size;
-        } else
-                pcpue_unit_size = max_t(size_t, pcpue_size, PCPU_MIN_UNIT_SIZE);
-
-        pcpue_ptr = __alloc_bootmem_nopanic(
-                                        num_possible_cpus() * pcpue_unit_size,
-                                        PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
-        if (!pcpue_ptr)
-                return -ENOMEM;
+        /* allocate, copy and determine base address */
+        for (group = 0; group < ai->nr_groups; group++) {
+                struct pcpu_group_info *gi = &ai->groups[group];
+                unsigned int cpu = NR_CPUS;
+                void *ptr;
+
+                for (i = 0; i < gi->nr_units && cpu == NR_CPUS; i++)
+                        cpu = gi->cpu_map[i];
+                BUG_ON(cpu == NR_CPUS);
+
+                /* allocate space for the whole group */
+                ptr = alloc_fn(cpu, gi->nr_units * ai->unit_size, atom_size);
+                if (!ptr) {
+                        rc = -ENOMEM;
+                        goto out_free_areas;
+                }
+                areas[group] = ptr;
 
-        /* return the leftover and copy */
-        for_each_possible_cpu(cpu) {
-                void *ptr = pcpue_ptr + cpu * pcpue_unit_size;
+                base = min(ptr, base);
+
+                for (i = 0; i < gi->nr_units; i++, ptr += ai->unit_size) {
+                        if (gi->cpu_map[i] == NR_CPUS) {
+                                /* unused unit, free whole */
+                                free_fn(ptr, ai->unit_size);
+                                continue;
+                        }
+                        /* copy and return the unused part */
+                        memcpy(ptr, __per_cpu_load, ai->static_size);
+                        free_fn(ptr + size_sum, ai->unit_size - size_sum);
+                }
+        }
+
+        /* base address is now known, determine group base offsets */
+        max_distance = 0;
+        for (group = 0; group < ai->nr_groups; group++) {
+                ai->groups[group].base_offset = areas[group] - base;
+                max_distance = max_t(size_t, max_distance,
+                                     ai->groups[group].base_offset);
+        }
+        max_distance += ai->unit_size;
+
+        /* warn if maximum distance is further than 75% of vmalloc space */
+        if (max_distance > (VMALLOC_END - VMALLOC_START) * 3 / 4) {
+                pr_warning("PERCPU: max_distance=0x%zx too large for vmalloc "
+                           "space 0x%lx\n",
+                           max_distance, VMALLOC_END - VMALLOC_START);
+#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
+                /* and fail if we have fallback */
+                rc = -EINVAL;
+                goto out_free;
+#endif
+        }
+
+        pr_info("PERCPU: Embedded %zu pages/cpu @%p s%zu r%zu d%zu u%zu\n",
+                PFN_DOWN(size_sum), base, ai->static_size, ai->reserved_size,
+                ai->dyn_size, ai->unit_size);
+
+        rc = pcpu_setup_first_chunk(ai, base);
+        goto out_free;
+
+out_free_areas:
+        for (group = 0; group < ai->nr_groups; group++)
+                free_fn(areas[group],
+                        ai->groups[group].nr_units * ai->unit_size);
+out_free:
+        pcpu_free_alloc_info(ai);
+        if (areas)
+                free_bootmem(__pa(areas), areas_size);
+        return rc;
+}
+#endif /* CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK ||
+          !CONFIG_HAVE_SETUP_PER_CPU_AREA */
+
+#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
+/**
+ * pcpu_page_first_chunk - map the first chunk using PAGE_SIZE pages
+ * @reserved_size: the size of reserved percpu area in bytes
+ * @alloc_fn: function to allocate percpu page, always called with PAGE_SIZE
+ * @free_fn: funtion to free percpu page, always called with PAGE_SIZE
+ * @populate_pte_fn: function to populate pte
+ *
+ * This is a helper to ease setting up page-remapped first percpu
+ * chunk and can be called where pcpu_setup_first_chunk() is expected.
+ *
+ * This is the basic allocator.  Static percpu area is allocated
+ * page-by-page into vmalloc area.
+ *
+ * RETURNS:
+ * 0 on success, -errno on failure.
+ */
+int __init pcpu_page_first_chunk(size_t reserved_size,
+                                 pcpu_fc_alloc_fn_t alloc_fn,
+                                 pcpu_fc_free_fn_t free_fn,
+                                 pcpu_fc_populate_pte_fn_t populate_pte_fn)
+{
+        static struct vm_struct vm;
+        struct pcpu_alloc_info *ai;
+        char psize_str[16];
+        int unit_pages;
+        size_t pages_size;
+        struct page **pages;
+        int unit, i, j, rc;
+
+        snprintf(psize_str, sizeof(psize_str), "%luK", PAGE_SIZE >> 10);
+
+        ai = pcpu_build_alloc_info(reserved_size, -1, PAGE_SIZE, NULL);
+        if (IS_ERR(ai))
+                return PTR_ERR(ai);
+        BUG_ON(ai->nr_groups != 1);
+        BUG_ON(ai->groups[0].nr_units != num_possible_cpus());
+
+        unit_pages = ai->unit_size >> PAGE_SHIFT;
+
+        /* unaligned allocations can't be freed, round up to page size */
+        pages_size = PFN_ALIGN(unit_pages * num_possible_cpus() *
+                               sizeof(pages[0]));
+        pages = alloc_bootmem(pages_size);
+
+        /* allocate pages */
+        j = 0;
+        for (unit = 0; unit < num_possible_cpus(); unit++)
+                for (i = 0; i < unit_pages; i++) {
+                        unsigned int cpu = ai->groups[0].cpu_map[unit];
+                        void *ptr;
+
+                        ptr = alloc_fn(cpu, PAGE_SIZE, PAGE_SIZE);
+                        if (!ptr) {
+                                pr_warning("PERCPU: failed to allocate %s page "
+                                           "for cpu%u\n", psize_str, cpu);
+                                goto enomem;
+                        }
+                        pages[j++] = virt_to_page(ptr);
+                }
+
+        /* allocate vm area, map the pages and copy static data */
+        vm.flags = VM_ALLOC;
+        vm.size = num_possible_cpus() * ai->unit_size;
+        vm_area_register_early(&vm, PAGE_SIZE);
+
+        for (unit = 0; unit < num_possible_cpus(); unit++) {
+                unsigned long unit_addr =
+                        (unsigned long)vm.addr + unit * ai->unit_size;
+
+                for (i = 0; i < unit_pages; i++)
+                        populate_pte_fn(unit_addr + (i << PAGE_SHIFT));
+
+                /* pte already populated, the following shouldn't fail */
+                rc = __pcpu_map_pages(unit_addr, &pages[unit * unit_pages],
+                                      unit_pages);
+                if (rc < 0)
+                        panic("failed to map percpu area, err=%d\n", rc);
 
-                free_bootmem(__pa(ptr + pcpue_size),
-                             pcpue_unit_size - pcpue_size);
-                memcpy(ptr, __per_cpu_load, static_size);
+                /*
+                 * FIXME: Archs with virtual cache should flush local
+                 * cache for the linear mapping here - something
+                 * equivalent to flush_cache_vmap() on the local cpu.
+                 * flush_cache_vmap() can't be used as most supporting
+                 * data structures are not set up yet.
+                 */
+
+                /* copy static data */
+                memcpy((void *)unit_addr, __per_cpu_load, ai->static_size);
         }
 
         /* we're ready, commit */
-        pr_info("PERCPU: Embedded %zu pages at %p, static data %zu bytes\n",
-                pcpue_size >> PAGE_SHIFT, pcpue_ptr, static_size);
+        pr_info("PERCPU: %d %s pages/cpu @%p s%zu r%zu d%zu\n",
+                unit_pages, psize_str, vm.addr, ai->static_size,
+                ai->reserved_size, ai->dyn_size);
+
+        rc = pcpu_setup_first_chunk(ai, vm.addr);
+        goto out_free_ar;
+
+enomem:
+        while (--j >= 0)
+                free_fn(page_address(pages[j]), PAGE_SIZE);
+        rc = -ENOMEM;
+out_free_ar:
+        free_bootmem(__pa(pages), pages_size);
+        pcpu_free_alloc_info(ai);
+        return rc;
+}
+#endif /* CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK */
+
+/*
+ * Generic percpu area setup.
+ *
+ * The embedding helper is used because its behavior closely resembles
+ * the original non-dynamic generic percpu area setup.  This is
+ * important because many archs have addressing restrictions and might
+ * fail if the percpu area is located far away from the previous
+ * location.  As an added bonus, in non-NUMA cases, embedding is
+ * generally a good idea TLB-wise because percpu area can piggy back
+ * on the physical linear memory mapping which uses large page
+ * mappings on applicable archs.
+ */
+#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
+unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
+EXPORT_SYMBOL(__per_cpu_offset);
 
-        return pcpu_setup_first_chunk(pcpue_get_page, static_size,
-                                      reserved_size, dyn_size,
-                                      pcpue_unit_size, pcpue_ptr, NULL);
+static void * __init pcpu_dfl_fc_alloc(unsigned int cpu, size_t size,
+                                       size_t align)
+{
+        return __alloc_bootmem_nopanic(size, align, __pa(MAX_DMA_ADDRESS));
+}
+
+static void __init pcpu_dfl_fc_free(void *ptr, size_t size)
+{
+        free_bootmem(__pa(ptr), size);
+}
+
+void __init setup_per_cpu_areas(void)
+{
+        unsigned long delta;
+        unsigned int cpu;
+        int rc;
+
+        /*
+         * Always reserve area for module percpu variables.  That's
+         * what the legacy allocator did.
+         */
+        rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
+                                    PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, NULL,
+                                    pcpu_dfl_fc_alloc, pcpu_dfl_fc_free);
+        if (rc < 0)
+                panic("Failed to initialized percpu areas.");
+
+        delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
+        for_each_possible_cpu(cpu)
+                __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
 }
+#endif /* CONFIG_HAVE_SETUP_PER_CPU_AREA */
diff --git a/mm/quicklist.c b/mm/quicklist.c
index 8dbb6805ef35..6633965bb27b 100644
--- a/mm/quicklist.c
+++ b/mm/quicklist.c
@@ -19,7 +19,7 @@
 #include <linux/module.h>
 #include <linux/quicklist.h>
 
-DEFINE_PER_CPU(struct quicklist, quicklist)[CONFIG_NR_QUICK];
+DEFINE_PER_CPU(struct quicklist [CONFIG_NR_QUICK], quicklist);
 
 #define FRACTION_OF_NODE_MEM    16
 
@@ -29,7 +29,6 @@ static unsigned long max_pages(unsigned long min_pages)
         int node = numa_node_id();
         struct zone *zones = NODE_DATA(node)->node_zones;
         int num_cpus_on_node;
-        node_to_cpumask_ptr(cpumask_on_node, node);
 
         node_free_pages =
 #ifdef CONFIG_ZONE_DMA
@@ -42,7 +41,7 @@ static unsigned long max_pages(unsigned long min_pages)
 
         max = node_free_pages / FRACTION_OF_NODE_MEM;
 
-        num_cpus_on_node = cpus_weight_nr(*cpumask_on_node);
+        num_cpus_on_node = cpumask_weight(cpumask_of_node(node));
         max /= num_cpus_on_node;
 
         return max(max, min_pages);
diff --git a/mm/readahead.c b/mm/readahead.c
index 9ce303d4b810..aa1aa2345235 100644
--- a/mm/readahead.c
+++ b/mm/readahead.c
@@ -31,6 +31,42 @@ EXPORT_SYMBOL_GPL(file_ra_state_init);
 
 #define list_to_page(head) (list_entry((head)->prev, struct page, lru))
 
+/*
+ * see if a page needs releasing upon read_cache_pages() failure
+ * - the caller of read_cache_pages() may have set PG_private or PG_fscache
+ *   before calling, such as the NFS fs marking pages that are cached locally
+ *   on disk, thus we need to give the fs a chance to clean up in the event of
+ *   an error
+ */
+static void read_cache_pages_invalidate_page(struct address_space *mapping,
+                                             struct page *page)
+{
+        if (page_has_private(page)) {
+                if (!trylock_page(page))
+                        BUG();
+                page->mapping = mapping;
+                do_invalidatepage(page, 0);
+                page->mapping = NULL;
+                unlock_page(page);
+        }
+        page_cache_release(page);
+}
+
+/*
+ * release a list of pages, invalidating them first if need be
+ */
+static void read_cache_pages_invalidate_pages(struct address_space *mapping,
+                                              struct list_head *pages)
+{
+        struct page *victim;
+
+        while (!list_empty(pages)) {
+                victim = list_to_page(pages);
+                list_del(&victim->lru);
+                read_cache_pages_invalidate_page(mapping, victim);
+        }
+}
+
 /**
  * read_cache_pages - populate an address space with some pages & start reads against them
  * @mapping: the address_space
@@ -52,14 +88,14 @@ int read_cache_pages(struct address_space *mapping, struct list_head *pages,
                 list_del(&page->lru);
                 if (add_to_page_cache_lru(page, mapping,
                                         page->index, GFP_KERNEL)) {
-                        page_cache_release(page);
+                        read_cache_pages_invalidate_page(mapping, page);
                         continue;
                 }
                 page_cache_release(page);
 
                 ret = filler(data, page);
                 if (unlikely(ret)) {
-                        put_pages_list(pages);
+                        read_cache_pages_invalidate_pages(mapping, pages);
                         break;
                 }
                 task_io_account_read(PAGE_CACHE_SIZE);
@@ -97,15 +133,12 @@ out:
 }
 
 /*
- * do_page_cache_readahead actually reads a chunk of disk.  It allocates all
+ * __do_page_cache_readahead() actually reads a chunk of disk.  It allocates all
  * the pages first, then submits them all for I/O. This avoids the very bad
  * behaviour which would occur if page allocations are causing VM writeback.
  * We really don't want to intermingle reads and writes like that.
  *
  * Returns the number of pages requested, or the maximum amount of I/O allowed.
- *
- * do_page_cache_readahead() returns -1 if it encountered request queue
- * congestion.
  */
 static int
 __do_page_cache_readahead(struct address_space *mapping, struct file *filp,
@@ -174,6 +207,7 @@ int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
         if (unlikely(!mapping->a_ops->readpage && !mapping->a_ops->readpages))
                 return -EINVAL;
 
+        nr_to_read = max_sane_readahead(nr_to_read);
         while (nr_to_read) {
                 int err;
 
@@ -195,22 +229,6 @@ int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
 }
 
 /*
- * This version skips the IO if the queue is read-congested, and will tell the
- * block layer to abandon the readahead if request allocation would block.
- *
- * force_page_cache_readahead() will ignore queue congestion and will block on
- * request queues.
- */
-int do_page_cache_readahead(struct address_space *mapping, struct file *filp,
-                        pgoff_t offset, unsigned long nr_to_read)
-{
-        if (bdi_read_congested(mapping->backing_dev_info))
-                return -1;
-
-        return __do_page_cache_readahead(mapping, filp, offset, nr_to_read, 0);
-}
-
-/*
  * Given a desired number of PAGE_CACHE_SIZE readahead pages, return a
  * sensible upper limit.
  */
@@ -223,7 +241,7 @@ unsigned long max_sane_readahead(unsigned long nr)
 /*
  * Submit IO for the read-ahead request in file_ra_state.
  */
-static unsigned long ra_submit(struct file_ra_state *ra,
+unsigned long ra_submit(struct file_ra_state *ra,
                        struct address_space *mapping, struct file *filp)
 {
         int actual;
@@ -312,6 +330,59 @@ static unsigned long get_next_ra_size(struct file_ra_state *ra,
  */
 
 /*
+ * Count contiguously cached pages from @offset-1 to @offset-@max,
+ * this count is a conservative estimation of
+ *      - length of the sequential read sequence, or
+ *      - thrashing threshold in memory tight systems
+ */
+static pgoff_t count_history_pages(struct address_space *mapping,
+                                   struct file_ra_state *ra,
+                                   pgoff_t offset, unsigned long max)
+{
+        pgoff_t head;
+
+        rcu_read_lock();
+        head = radix_tree_prev_hole(&mapping->page_tree, offset - 1, max);
+        rcu_read_unlock();
+
+        return offset - 1 - head;
+}
+
+/*
+ * page cache context based read-ahead
+ */
+static int try_context_readahead(struct address_space *mapping,
+                                 struct file_ra_state *ra,
+                                 pgoff_t offset,
+                                 unsigned long req_size,
+                                 unsigned long max)
+{
+        pgoff_t size;
+
+        size = count_history_pages(mapping, ra, offset, max);
+
+        /*
+         * no history pages:
+         * it could be a random read
+         */
+        if (!size)
+                return 0;
+
+        /*
+         * starts from beginning of file:
+         * it is a strong indication of long-run stream (or whole-file-read)
+         */
+        if (size >= offset)
+                size *= 2;
+
+        ra->start = offset;
+        ra->size = get_init_ra_size(size + req_size, max);
+        ra->async_size = ra->size;
+
+        return 1;
+}
+
+/*
  * A minimal readahead algorithm for trivial sequential/random reads.
  */
 static unsigned long
@@ -320,34 +391,26 @@ ondemand_readahead(struct address_space *mapping,
                    bool hit_readahead_marker, pgoff_t offset,
                    unsigned long req_size)
 {
-        int     max = ra->ra_pages;     /* max readahead pages */
-        pgoff_t prev_offset;
-        int     sequential;
+        unsigned long max = max_sane_readahead(ra->ra_pages);
+
+        /*
+         * start of file
+         */
+        if (!offset)
+                goto initial_readahead;
 
         /*
          * It's the expected callback offset, assume sequential access.
          * Ramp up sizes, and push forward the readahead window.
          */
-        if (offset && (offset == (ra->start + ra->size - ra->async_size) ||
-                        offset == (ra->start + ra->size))) {
+        if ((offset == (ra->start + ra->size - ra->async_size) ||
+             offset == (ra->start + ra->size))) {
                 ra->start += ra->size;
                 ra->size = get_next_ra_size(ra, max);
                 ra->async_size = ra->size;
                 goto readit;
         }
 
-        prev_offset = ra->prev_pos >> PAGE_CACHE_SHIFT;
-        sequential = offset - prev_offset <= 1UL || req_size > max;
-
-        /*
-         * Standalone, small read.
-         * Read as is, and do not pollute the readahead state.
-         */
-        if (!hit_readahead_marker && !sequential) {
-                return __do_page_cache_readahead(mapping, filp,
-                                                offset, req_size, 0);
-        }
-
         /*
          * Hit a marked page without valid readahead state.
          * E.g. interleaved reads.
@@ -358,7 +421,7 @@ ondemand_readahead(struct address_space *mapping,
                 pgoff_t start;
 
                 rcu_read_lock();
-                start = radix_tree_next_hole(&mapping->page_tree, offset,max+1);
+                start = radix_tree_next_hole(&mapping->page_tree, offset+1,max);
                 rcu_read_unlock();
 
                 if (!start || start - offset > max)
@@ -366,23 +429,53 @@ ondemand_readahead(struct address_space *mapping,
 
                 ra->start = start;
                 ra->size = start - offset;      /* old async_size */
+                ra->size += req_size;
                 ra->size = get_next_ra_size(ra, max);
                 ra->async_size = ra->size;
                 goto readit;
         }
 
         /*
-         * It may be one of
-         *      - first read on start of file
-         *      - sequential cache miss
-         *      - oversize random read
-         * Start readahead for it.
+         * oversize read
          */
+        if (req_size > max)
+                goto initial_readahead;
+
+        /*
+         * sequential cache miss
+         */
+        if (offset - (ra->prev_pos >> PAGE_CACHE_SHIFT) <= 1UL)
+                goto initial_readahead;
+
+        /*
+         * Query the page cache and look for the traces(cached history pages)
+         * that a sequential stream would leave behind.
+         */
+        if (try_context_readahead(mapping, ra, offset, req_size, max))
+                goto readit;
+
+        /*
+         * standalone, small random read
+         * Read as is, and do not pollute the readahead state.
+         */
+        return __do_page_cache_readahead(mapping, filp, offset, req_size, 0);
+
+initial_readahead:
         ra->start = offset;
         ra->size = get_init_ra_size(req_size, max);
         ra->async_size = ra->size > req_size ? ra->size - req_size : ra->size;
 
 readit:
+        /*
+         * Will this read hit the readahead marker made by itself?
+         * If so, trigger the readahead marker hit now, and merge
+         * the resulted next readahead window into the current one.
+         */
+        if (offset == ra->start && ra->size == ra->async_size) {
+                ra->async_size = get_next_ra_size(ra, max);
+                ra->size += ra->async_size;
+        }
+
         return ra_submit(ra, mapping, filp);
 }
 
diff --git a/mm/rmap.c b/mm/rmap.c
index 16521664010d..dd43373a483f 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -14,7 +14,7 @@
  * Original design by Rik van Riel <riel@conectiva.com.br> 2001
  * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004
  * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004
- * Contributions by Hugh Dickins <hugh@veritas.com> 2003, 2004
+ * Contributions by Hugh Dickins 2003, 2004
  */
 
 /*
@@ -36,6 +36,11 @@
  *                 mapping->tree_lock (widely used, in set_page_dirty,
  *                           in arch-dependent flush_dcache_mmap_lock,
  *                           within inode_lock in __sync_single_inode)
+ *
+ * (code doesn't rely on that order so it could be switched around)
+ * ->tasklist_lock
+ *   anon_vma->lock      (memory_failure, collect_procs_anon)
+ *     pte map lock
  */
 
 #include <linux/mm.h>
@@ -191,7 +196,7 @@ void __init anon_vma_init(void)
  * Getting a lock on a stable anon_vma from a page off the LRU is
  * tricky: page_lock_anon_vma rely on RCU to guard against the races.
  */
-static struct anon_vma *page_lock_anon_vma(struct page *page)
+struct anon_vma *page_lock_anon_vma(struct page *page)
 {
         struct anon_vma *anon_vma;
         unsigned long anon_mapping;
@@ -211,7 +216,7 @@ out:
         return NULL;
 }
 
-static void page_unlock_anon_vma(struct anon_vma *anon_vma)
+void page_unlock_anon_vma(struct anon_vma *anon_vma)
 {
         spin_unlock(&anon_vma->lock);
         rcu_read_unlock();
@@ -237,8 +242,8 @@ vma_address(struct page *page, struct vm_area_struct *vma)
 }
 
 /*
- * At what user virtual address is page expected in vma? checking that the
- * page matches the vma: currently only used on anon pages, by unuse_vma;
+ * At what user virtual address is page expected in vma?
+ * checking that the page matches the vma.
  */
 unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
 {
@@ -311,7 +316,7 @@ pte_t *page_check_address(struct page *page, struct mm_struct *mm,
  * if the page is not mapped into the page tables of this VMA.  Only
  * valid for normal file or anonymous VMAs.
  */
-static int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma)
+int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma)
 {
         unsigned long address;
         pte_t *pte;
@@ -333,7 +338,9 @@ static int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma)
  * repeatedly from either page_referenced_anon or page_referenced_file.
  */
 static int page_referenced_one(struct page *page,
-        struct vm_area_struct *vma, unsigned int *mapcount)
+                               struct vm_area_struct *vma,
+                               unsigned int *mapcount,
+                               unsigned long *vm_flags)
 {
         struct mm_struct *mm = vma->vm_mm;
         unsigned long address;
@@ -356,6 +363,7 @@ static int page_referenced_one(struct page *page,
          */
         if (vma->vm_flags & VM_LOCKED) {
                 *mapcount = 1;  /* break early from loop */
+                *vm_flags |= VM_LOCKED;
                 goto out_unmap;
         }
 
@@ -381,11 +389,14 @@ out_unmap:
         (*mapcount)--;
         pte_unmap_unlock(pte, ptl);
 out:
+        if (referenced)
+                *vm_flags |= vma->vm_flags;
         return referenced;
 }
 
 static int page_referenced_anon(struct page *page,
-                                struct mem_cgroup *mem_cont)
+                                struct mem_cgroup *mem_cont,
+                                unsigned long *vm_flags)
 {
         unsigned int mapcount;
         struct anon_vma *anon_vma;
@@ -405,7 +416,8 @@ static int page_referenced_anon(struct page *page,
                  */
                 if (mem_cont && !mm_match_cgroup(vma->vm_mm, mem_cont))
                         continue;
-                referenced += page_referenced_one(page, vma, &mapcount);
+                referenced += page_referenced_one(page, vma,
+                                                  &mapcount, vm_flags);
                 if (!mapcount)
                         break;
         }
@@ -418,6 +430,7 @@ static int page_referenced_anon(struct page *page,
  * page_referenced_file - referenced check for object-based rmap
  * @page: the page we're checking references on.
  * @mem_cont: target memory controller
+ * @vm_flags: collect encountered vma->vm_flags who actually referenced the page
  *
  * For an object-based mapped page, find all the places it is mapped and
  * check/clear the referenced flag.  This is done by following the page->mapping
@@ -427,7 +440,8 @@ static int page_referenced_anon(struct page *page,
  * This function is only called from page_referenced for object-based pages.
  */
 static int page_referenced_file(struct page *page,
-                                struct mem_cgroup *mem_cont)
+                                struct mem_cgroup *mem_cont,
+                                unsigned long *vm_flags)
 {
         unsigned int mapcount;
         struct address_space *mapping = page->mapping;
@@ -467,7 +481,8 @@ static int page_referenced_file(struct page *page,
                  */
                 if (mem_cont && !mm_match_cgroup(vma->vm_mm, mem_cont))
                         continue;
-                referenced += page_referenced_one(page, vma, &mapcount);
+                referenced += page_referenced_one(page, vma,
+                                                  &mapcount, vm_flags);
                 if (!mapcount)
                         break;
         }
@@ -481,29 +496,35 @@ static int page_referenced_file(struct page *page,
  * @page: the page to test
  * @is_locked: caller holds lock on the page
  * @mem_cont: target memory controller
+ * @vm_flags: collect encountered vma->vm_flags who actually referenced the page
  *
  * Quick test_and_clear_referenced for all mappings to a page,
  * returns the number of ptes which referenced the page.
  */
-int page_referenced(struct page *page, int is_locked,
-                        struct mem_cgroup *mem_cont)
+int page_referenced(struct page *page,
+                    int is_locked,
+                    struct mem_cgroup *mem_cont,
+                    unsigned long *vm_flags)
 {
         int referenced = 0;
 
         if (TestClearPageReferenced(page))
                 referenced++;
 
+        *vm_flags = 0;
         if (page_mapped(page) && page->mapping) {
                 if (PageAnon(page))
-                        referenced += page_referenced_anon(page, mem_cont);
+                        referenced += page_referenced_anon(page, mem_cont,
+                                                                vm_flags);
                 else if (is_locked)
-                        referenced += page_referenced_file(page, mem_cont);
+                        referenced += page_referenced_file(page, mem_cont,
+                                                                vm_flags);
                 else if (!trylock_page(page))
                         referenced++;
                 else {
                         if (page->mapping)
-                                referenced +=
-                                        page_referenced_file(page, mem_cont);
+                                referenced += page_referenced_file(page,
+                                                        mem_cont, vm_flags);
                         unlock_page(page);
                 }
         }
@@ -688,31 +709,12 @@ void page_add_new_anon_rmap(struct page *page,
  */
 void page_add_file_rmap(struct page *page)
 {
-        if (atomic_inc_and_test(&page->_mapcount))
+        if (atomic_inc_and_test(&page->_mapcount)) {
                 __inc_zone_page_state(page, NR_FILE_MAPPED);
+                mem_cgroup_update_mapped_file_stat(page, 1);
+        }
 }
 
-#ifdef CONFIG_DEBUG_VM
-/**
- * page_dup_rmap - duplicate pte mapping to a page
- * @page:       the page to add the mapping to
- * @vma:        the vm area being duplicated
- * @address:    the user virtual address mapped
- *
- * For copy_page_range only: minimal extract from page_add_file_rmap /
- * page_add_anon_rmap, avoiding unnecessary tests (already checked) so it's
- * quicker.
- *
- * The caller needs to hold the pte lock.
- */
-void page_dup_rmap(struct page *page, struct vm_area_struct *vma, unsigned long address)
-{
-        if (PageAnon(page))
-                __page_check_anon_rmap(page, vma, address);
-        atomic_inc(&page->_mapcount);
-}
-#endif
-
 /**
  * page_remove_rmap - take down pte mapping from a page
  * @page: page to remove mapping from
@@ -721,33 +723,37 @@ void page_dup_rmap(struct page *page, struct vm_area_struct *vma, unsigned long
  */
 void page_remove_rmap(struct page *page)
 {
-        if (atomic_add_negative(-1, &page->_mapcount)) {
-                /*
-                 * Now that the last pte has gone, s390 must transfer dirty
-                 * flag from storage key to struct page.  We can usually skip
-                 * this if the page is anon, so about to be freed; but perhaps
-                 * not if it's in swapcache - there might be another pte slot
-                 * containing the swap entry, but page not yet written to swap.
-                 */
-                if ((!PageAnon(page) || PageSwapCache(page)) &&
-                    page_test_dirty(page)) {
-                        page_clear_dirty(page);
-                        set_page_dirty(page);
-                }
-                if (PageAnon(page))
-                        mem_cgroup_uncharge_page(page);
-                __dec_zone_page_state(page,
-                        PageAnon(page) ? NR_ANON_PAGES : NR_FILE_MAPPED);
-                /*
-                 * It would be tidy to reset the PageAnon mapping here,
-                 * but that might overwrite a racing page_add_anon_rmap
-                 * which increments mapcount after us but sets mapping
-                 * before us: so leave the reset to free_hot_cold_page,
-                 * and remember that it's only reliable while mapped.
-                 * Leaving it set also helps swapoff to reinstate ptes
-                 * faster for those pages still in swapcache.
-                 */
+        /* page still mapped by someone else? */
+        if (!atomic_add_negative(-1, &page->_mapcount))
+                return;
+
+        /*
+         * Now that the last pte has gone, s390 must transfer dirty
+         * flag from storage key to struct page.  We can usually skip
+         * this if the page is anon, so about to be freed; but perhaps
+         * not if it's in swapcache - there might be another pte slot
+         * containing the swap entry, but page not yet written to swap.
+         */
+        if ((!PageAnon(page) || PageSwapCache(page)) && page_test_dirty(page)) {
+                page_clear_dirty(page);
+                set_page_dirty(page);
+        }
+        if (PageAnon(page)) {
+                mem_cgroup_uncharge_page(page);
+                __dec_zone_page_state(page, NR_ANON_PAGES);
+        } else {
+                __dec_zone_page_state(page, NR_FILE_MAPPED);
         }
+        mem_cgroup_update_mapped_file_stat(page, -1);
+        /*
+         * It would be tidy to reset the PageAnon mapping here,
+         * but that might overwrite a racing page_add_anon_rmap
+         * which increments mapcount after us but sets mapping
+         * before us: so leave the reset to free_hot_cold_page,
+         * and remember that it's only reliable while mapped.
+         * Leaving it set also helps swapoff to reinstate ptes
+         * faster for those pages still in swapcache.
+         */
 }
 
 /*
@@ -755,7 +761,7 @@ void page_remove_rmap(struct page *page)
  * repeatedly from either try_to_unmap_anon or try_to_unmap_file.
  */
 static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
-                                int migration)
+                                enum ttu_flags flags)
 {
         struct mm_struct *mm = vma->vm_mm;
         unsigned long address;
@@ -777,11 +783,13 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
          * If it's recently referenced (perhaps page_referenced
          * skipped over this mm) then we should reactivate it.
          */
-        if (!migration) {
+        if (!(flags & TTU_IGNORE_MLOCK)) {
                 if (vma->vm_flags & VM_LOCKED) {
                         ret = SWAP_MLOCK;
                         goto out_unmap;
                 }
+        }
+        if (!(flags & TTU_IGNORE_ACCESS)) {
                 if (ptep_clear_flush_young_notify(vma, address, pte)) {
                         ret = SWAP_FAIL;
                         goto out_unmap;
@@ -799,7 +807,14 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
         /* Update high watermark before we lower rss */
         update_hiwater_rss(mm);
 
-        if (PageAnon(page)) {
+        if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) {
+                if (PageAnon(page))
+                        dec_mm_counter(mm, anon_rss);
+                else
+                        dec_mm_counter(mm, file_rss);
+                set_pte_at(mm, address, pte,
+                                swp_entry_to_pte(make_hwpoison_entry(page)));
+        } else if (PageAnon(page)) {
                 swp_entry_t entry = { .val = page_private(page) };
 
                 if (PageSwapCache(page)) {
@@ -821,12 +836,12 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
                          * pte. do_swap_page() will wait until the migration
                          * pte is removed and then restart fault handling.
                          */
-                        BUG_ON(!migration);
+                        BUG_ON(TTU_ACTION(flags) != TTU_MIGRATION);
                         entry = make_migration_entry(page, pte_write(pteval));
                 }
                 set_pte_at(mm, address, pte, swp_entry_to_pte(entry));
                 BUG_ON(pte_file(*pte));
-        } else if (PAGE_MIGRATION && migration) {
+        } else if (PAGE_MIGRATION && (TTU_ACTION(flags) == TTU_MIGRATION)) {
                 /* Establish migration entry for a file page */
                 swp_entry_t entry;
                 entry = make_migration_entry(page, pte_write(pteval));
@@ -995,12 +1010,13 @@ static int try_to_mlock_page(struct page *page, struct vm_area_struct *vma)
  * vm_flags for that VMA.  That should be OK, because that vma shouldn't be
  * 'LOCKED.
  */
-static int try_to_unmap_anon(struct page *page, int unlock, int migration)
+static int try_to_unmap_anon(struct page *page, enum ttu_flags flags)
 {
         struct anon_vma *anon_vma;
         struct vm_area_struct *vma;
         unsigned int mlocked = 0;
         int ret = SWAP_AGAIN;
+        int unlock = TTU_ACTION(flags) == TTU_MUNLOCK;
 
         if (MLOCK_PAGES && unlikely(unlock))
                 ret = SWAP_SUCCESS;     /* default for try_to_munlock() */
@@ -1016,7 +1032,7 @@ static int try_to_unmap_anon(struct page *page, int unlock, int migration)
                                 continue;  /* must visit all unlocked vmas */
                         ret = SWAP_MLOCK;  /* saw at least one mlocked vma */
                 } else {
-                        ret = try_to_unmap_one(page, vma, migration);
+                        ret = try_to_unmap_one(page, vma, flags);
                         if (ret == SWAP_FAIL || !page_mapped(page))
                                 break;
                 }
@@ -1040,8 +1056,7 @@ static int try_to_unmap_anon(struct page *page, int unlock, int migration)
 /**
  * try_to_unmap_file - unmap/unlock file page using the object-based rmap method
  * @page: the page to unmap/unlock
- * @unlock:  request for unlock rather than unmap [unlikely]
- * @migration:  unmapping for migration - ignored if @unlock
+ * @flags: action and flags
  *
  * Find all the mappings of a page using the mapping pointer and the vma chains
  * contained in the address_space struct it points to.
@@ -1053,7 +1068,7 @@ static int try_to_unmap_anon(struct page *page, int unlock, int migration)
  * vm_flags for that VMA.  That should be OK, because that vma shouldn't be
  * 'LOCKED.
  */
-static int try_to_unmap_file(struct page *page, int unlock, int migration)
+static int try_to_unmap_file(struct page *page, enum ttu_flags flags)
 {
         struct address_space *mapping = page->mapping;
         pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
@@ -1065,6 +1080,7 @@ static int try_to_unmap_file(struct page *page, int unlock, int migration)
         unsigned long max_nl_size = 0;
         unsigned int mapcount;
         unsigned int mlocked = 0;
+        int unlock = TTU_ACTION(flags) == TTU_MUNLOCK;
 
         if (MLOCK_PAGES && unlikely(unlock))
                 ret = SWAP_SUCCESS;     /* default for try_to_munlock() */
@@ -1077,7 +1093,7 @@ static int try_to_unmap_file(struct page *page, int unlock, int migration)
                                 continue;       /* must visit all vmas */
                         ret = SWAP_MLOCK;
                 } else {
-                        ret = try_to_unmap_one(page, vma, migration);
+                        ret = try_to_unmap_one(page, vma, flags);
                         if (ret == SWAP_FAIL || !page_mapped(page))
                                 goto out;
                 }
@@ -1102,7 +1118,8 @@ static int try_to_unmap_file(struct page *page, int unlock, int migration)
                         ret = SWAP_MLOCK;       /* leave mlocked == 0 */
                         goto out;               /* no need to look further */
                 }
-                if (!MLOCK_PAGES && !migration && (vma->vm_flags & VM_LOCKED))
+                if (!MLOCK_PAGES && !(flags & TTU_IGNORE_MLOCK) &&
+                        (vma->vm_flags & VM_LOCKED))
                         continue;
                 cursor = (unsigned long) vma->vm_private_data;
                 if (cursor > max_nl_cursor)
@@ -1136,7 +1153,7 @@ static int try_to_unmap_file(struct page *page, int unlock, int migration)
         do {
                 list_for_each_entry(vma, &mapping->i_mmap_nonlinear,
                                                 shared.vm_set.list) {
-                        if (!MLOCK_PAGES && !migration &&
+                        if (!MLOCK_PAGES && !(flags & TTU_IGNORE_MLOCK) &&
                             (vma->vm_flags & VM_LOCKED))
                                 continue;
                         cursor = (unsigned long) vma->vm_private_data;
@@ -1176,7 +1193,7 @@ out:
 /**
  * try_to_unmap - try to remove all page table mappings to a page
  * @page: the page to get unmapped
- * @migration: migration flag
+ * @flags: action and flags
  *
  * Tries to remove all the page table entries which are mapping this
  * page, used in the pageout path.  Caller must hold the page lock.
@@ -1187,22 +1204,21 @@ out:
  * SWAP_FAIL    - the page is unswappable
  * SWAP_MLOCK   - page is mlocked.
  */
-int try_to_unmap(struct page *page, int migration)
+int try_to_unmap(struct page *page, enum ttu_flags flags)
 {
         int ret;
 
         BUG_ON(!PageLocked(page));
 
         if (PageAnon(page))
-                ret = try_to_unmap_anon(page, 0, migration);
+                ret = try_to_unmap_anon(page, flags);
         else
-                ret = try_to_unmap_file(page, 0, migration);
+                ret = try_to_unmap_file(page, flags);
         if (ret != SWAP_MLOCK && !page_mapped(page))
                 ret = SWAP_SUCCESS;
         return ret;
 }
 
-#ifdef CONFIG_UNEVICTABLE_LRU
 /**
  * try_to_munlock - try to munlock a page
  * @page: the page to be munlocked
@@ -1222,8 +1238,8 @@ int try_to_munlock(struct page *page)
         VM_BUG_ON(!PageLocked(page) || PageLRU(page));
 
         if (PageAnon(page))
-                return try_to_unmap_anon(page, 1, 0);
+                return try_to_unmap_anon(page, TTU_MUNLOCK);
         else
-                return try_to_unmap_file(page, 1, 0);
+                return try_to_unmap_file(page, TTU_MUNLOCK);
 }
-#endif
+
diff --git a/mm/shmem.c b/mm/shmem.c
index d94d2e9146bc..356dd99566ec 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -24,6 +24,7 @@
 #include <linux/init.h>
 #include <linux/vfs.h>
 #include <linux/mount.h>
+#include <linux/pagemap.h>
 #include <linux/file.h>
 #include <linux/mm.h>
 #include <linux/module.h>
@@ -43,13 +44,11 @@ static struct vfsmount *shm_mnt;
 #include <linux/exportfs.h>
 #include <linux/generic_acl.h>
 #include <linux/mman.h>
-#include <linux/pagemap.h>
 #include <linux/string.h>
 #include <linux/slab.h>
 #include <linux/backing-dev.h>
 #include <linux/shmem_fs.h>
 #include <linux/writeback.h>
-#include <linux/vfs.h>
 #include <linux/blkdev.h>
 #include <linux/security.h>
 #include <linux/swapops.h>
@@ -65,13 +64,28 @@ static struct vfsmount *shm_mnt;
 #include <asm/div64.h>
 #include <asm/pgtable.h>
 
+/*
+ * The maximum size of a shmem/tmpfs file is limited by the maximum size of
+ * its triple-indirect swap vector - see illustration at shmem_swp_entry().
+ *
+ * With 4kB page size, maximum file size is just over 2TB on a 32-bit kernel,
+ * but one eighth of that on a 64-bit kernel.  With 8kB page size, maximum
+ * file size is just over 4TB on a 64-bit kernel, but 16TB on a 32-bit kernel,
+ * MAX_LFS_FILESIZE being then more restrictive than swap vector layout.
+ *
+ * We use / and * instead of shifts in the definitions below, so that the swap
+ * vector can be tested with small even values (e.g. 20) for ENTRIES_PER_PAGE.
+ */
 #define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
-#define ENTRIES_PER_PAGEPAGE (ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
-#define BLOCKS_PER_PAGE  (PAGE_CACHE_SIZE/512)
+#define ENTRIES_PER_PAGEPAGE ((unsigned long long)ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
+
+#define SHMSWP_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
+#define SHMSWP_MAX_BYTES (SHMSWP_MAX_INDEX << PAGE_CACHE_SHIFT)
 
-#define SHMEM_MAX_INDEX  (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
-#define SHMEM_MAX_BYTES  ((unsigned long long)SHMEM_MAX_INDEX << PAGE_CACHE_SHIFT)
+#define SHMEM_MAX_BYTES  min_t(unsigned long long, SHMSWP_MAX_BYTES, MAX_LFS_FILESIZE)
+#define SHMEM_MAX_INDEX  ((unsigned long)((SHMEM_MAX_BYTES+1) >> PAGE_CACHE_SHIFT))
 
+#define BLOCKS_PER_PAGE  (PAGE_CACHE_SIZE/512)
 #define VM_ACCT(size)    (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
 
 /* info->flags needs VM_flags to handle pagein/truncate races efficiently */
@@ -204,7 +218,7 @@ static const struct file_operations shmem_file_operations;
 static const struct inode_operations shmem_inode_operations;
 static const struct inode_operations shmem_dir_inode_operations;
 static const struct inode_operations shmem_special_inode_operations;
-static struct vm_operations_struct shmem_vm_ops;
+static const struct vm_operations_struct shmem_vm_ops;
 
 static struct backing_dev_info shmem_backing_dev_info  __read_mostly = {
         .ra_pages       = 0,    /* No readahead */
@@ -1032,8 +1046,9 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc)
          * sync from ever calling shmem_writepage; but a stacking filesystem
          * may use the ->writepage of its underlying filesystem, in which case
          * tmpfs should write out to swap only in response to memory pressure,
-         * and not for pdflush or sync.  However, in those cases, we do still
-         * want to check if there's a redundant swappage to be discarded.
+         * and not for the writeback threads or sync.  However, in those cases,
+         * we do still want to check if there's a redundant swappage to be
+         * discarded.
          */
         if (wbc->for_reclaim)
                 swap = get_swap_page();
@@ -1082,7 +1097,11 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc)
         shmem_swp_unmap(entry);
 unlock:
         spin_unlock(&info->lock);
-        swap_free(swap);
+        /*
+         * add_to_swap_cache() doesn't return -EEXIST, so we can safely
+         * clear SWAP_HAS_CACHE flag.
+         */
+        swapcache_free(swap, NULL);
 redirty:
         set_page_dirty(page);
         if (wbc->for_reclaim)
@@ -1325,8 +1344,12 @@ repeat:
                         shmem_swp_unmap(entry);
                         spin_unlock(&info->lock);
                         if (error == -ENOMEM) {
-                                /* allow reclaim from this memory cgroup */
-                                error = mem_cgroup_shrink_usage(swappage,
+                                /*
+                                 * reclaim from proper memory cgroup and
+                                 * call memcg's OOM if needed.
+                                 */
+                                error = mem_cgroup_shmem_charge_fallback(
+                                                                swappage,
                                                                 current->mm,
                                                                 gfp);
                                 if (error) {
@@ -1539,6 +1562,7 @@ static struct inode *shmem_get_inode(struct super_block *sb, int mode,
                 spin_lock_init(&info->lock);
                 info->flags = flags & VM_NORESERVE;
                 INIT_LIST_HEAD(&info->swaplist);
+                cache_no_acl(inode);
 
                 switch (mode & S_IFMT) {
                 default:
@@ -1610,8 +1634,8 @@ shmem_write_end(struct file *file, struct address_space *mapping,
         if (pos + copied > inode->i_size)
                 i_size_write(inode, pos + copied);
 
-        unlock_page(page);
         set_page_dirty(page);
+        unlock_page(page);
         page_cache_release(page);
 
         return copied;
@@ -1948,13 +1972,13 @@ static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *s
                         iput(inode);
                         return error;
                 }
-                unlock_page(page);
                 inode->i_mapping->a_ops = &shmem_aops;
                 inode->i_op = &shmem_symlink_inode_operations;
                 kaddr = kmap_atomic(page, KM_USER0);
                 memcpy(kaddr, symname, len);
                 kunmap_atomic(kaddr, KM_USER0);
                 set_page_dirty(page);
+                unlock_page(page);
                 page_cache_release(page);
         }
         if (dir->i_mode & S_ISGID)
@@ -2278,8 +2302,7 @@ static void shmem_put_super(struct super_block *sb)
         sb->s_fs_info = NULL;
 }
 
-static int shmem_fill_super(struct super_block *sb,
-                            void *data, int silent)
+int shmem_fill_super(struct super_block *sb, void *data, int silent)
 {
         struct inode *inode;
         struct dentry *root;
@@ -2287,17 +2310,14 @@ static int shmem_fill_super(struct super_block *sb,
         int err = -ENOMEM;
 
         /* Round up to L1_CACHE_BYTES to resist false sharing */
-        sbinfo = kmalloc(max((int)sizeof(struct shmem_sb_info),
+        sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
                                 L1_CACHE_BYTES), GFP_KERNEL);
         if (!sbinfo)
                 return -ENOMEM;
 
-        sbinfo->max_blocks = 0;
-        sbinfo->max_inodes = 0;
         sbinfo->mode = S_IRWXUGO | S_ISVTX;
         sbinfo->uid = current_fsuid();
         sbinfo->gid = current_fsgid();
-        sbinfo->mpol = NULL;
         sb->s_fs_info = sbinfo;
 
 #ifdef CONFIG_TMPFS
@@ -2369,7 +2389,6 @@ static void shmem_destroy_inode(struct inode *inode)
                 /* only struct inode is valid if it's an inline symlink */
                 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
         }
-        shmem_acl_destroy_inode(inode);
         kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
 }
 
@@ -2378,10 +2397,6 @@ static void init_once(void *foo)
         struct shmem_inode_info *p = (struct shmem_inode_info *) foo;
 
         inode_init_once(&p->vfs_inode);
-#ifdef CONFIG_TMPFS_POSIX_ACL
-        p->i_acl = NULL;
-        p->i_default_acl = NULL;
-#endif
 }
 
 static int init_inodecache(void)
@@ -2406,6 +2421,7 @@ static const struct address_space_operations shmem_aops = {
         .write_end      = shmem_write_end,
 #endif
         .migratepage    = migrate_page,
+        .error_remove_page = generic_error_remove_page,
 };
 
 static const struct file_operations shmem_file_operations = {
@@ -2431,7 +2447,7 @@ static const struct inode_operations shmem_inode_operations = {
         .getxattr       = generic_getxattr,
         .listxattr      = generic_listxattr,
         .removexattr    = generic_removexattr,
-        .permission     = shmem_permission,
+        .check_acl      = shmem_check_acl,
 #endif
 
 };
@@ -2454,7 +2470,7 @@ static const struct inode_operations shmem_dir_inode_operations = {
         .getxattr       = generic_getxattr,
         .listxattr      = generic_listxattr,
         .removexattr    = generic_removexattr,
-        .permission     = shmem_permission,
+        .check_acl      = shmem_check_acl,
 #endif
 };
 
@@ -2465,7 +2481,7 @@ static const struct inode_operations shmem_special_inode_operations = {
         .getxattr       = generic_getxattr,
         .listxattr      = generic_listxattr,
         .removexattr    = generic_removexattr,
-        .permission     = shmem_permission,
+        .check_acl      = shmem_check_acl,
 #endif
 };
 
@@ -2482,7 +2498,7 @@ static const struct super_operations shmem_ops = {
         .put_super      = shmem_put_super,
 };
 
-static struct vm_operations_struct shmem_vm_ops = {
+static const struct vm_operations_struct shmem_vm_ops = {
         .fault          = shmem_fault,
 #ifdef CONFIG_NUMA
         .set_policy     = shmem_set_policy,
@@ -2504,7 +2520,7 @@ static struct file_system_type tmpfs_fs_type = {
         .kill_sb        = kill_litter_super,
 };
 
-static int __init init_tmpfs(void)
+int __init init_tmpfs(void)
 {
         int error;
 
@@ -2561,7 +2577,7 @@ static struct file_system_type tmpfs_fs_type = {
         .kill_sb        = kill_litter_super,
 };
 
-static int __init init_tmpfs(void)
+int __init init_tmpfs(void)
 {
         BUG_ON(register_filesystem(&tmpfs_fs_type) != 0);
 
@@ -2576,12 +2592,17 @@ int shmem_unuse(swp_entry_t entry, struct page *page)
         return 0;
 }
 
+int shmem_lock(struct file *file, int lock, struct user_struct *user)
+{
+        return 0;
+}
+
 #define shmem_vm_ops                            generic_file_vm_ops
 #define shmem_file_operations                   ramfs_file_operations
 #define shmem_get_inode(sb, mode, dev, flags)   ramfs_get_inode(sb, mode, dev)
 #define shmem_acct_size(flags, size)            0
 #define shmem_unacct_size(flags, size)          do {} while (0)
-#define SHMEM_MAX_BYTES                         LLONG_MAX
+#define SHMEM_MAX_BYTES                         MAX_LFS_FILESIZE
 
 #endif /* CONFIG_SHMEM */
 
@@ -2593,7 +2614,7 @@ int shmem_unuse(swp_entry_t entry, struct page *page)
  * @size: size to be set for the file
  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
  */
-struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags)
+struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
 {
         int error;
         struct file *file;
@@ -2640,6 +2661,7 @@ struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags)
         if (error)
                 goto close_file;
 #endif
+        ima_counts_get(file);
         return file;
 
 close_file:
@@ -2665,12 +2687,9 @@ int shmem_zero_setup(struct vm_area_struct *vma)
         if (IS_ERR(file))
                 return PTR_ERR(file);
 
-        ima_shm_check(file);
         if (vma->vm_file)
                 fput(vma->vm_file);
         vma->vm_file = file;
         vma->vm_ops = &shmem_vm_ops;
         return 0;
 }
-
-module_init(init_tmpfs)
diff --git a/mm/shmem_acl.c b/mm/shmem_acl.c
index 8e5aadd7dcd6..df2c87fdae50 100644
--- a/mm/shmem_acl.c
+++ b/mm/shmem_acl.c
@@ -22,11 +22,11 @@ shmem_get_acl(struct inode *inode, int type)
         spin_lock(&inode->i_lock);
         switch(type) {
                 case ACL_TYPE_ACCESS:
-                        acl = posix_acl_dup(SHMEM_I(inode)->i_acl);
+                        acl = posix_acl_dup(inode->i_acl);
                         break;
 
                 case ACL_TYPE_DEFAULT:
-                        acl = posix_acl_dup(SHMEM_I(inode)->i_default_acl);
+                        acl = posix_acl_dup(inode->i_default_acl);
                         break;
         }
         spin_unlock(&inode->i_lock);
@@ -45,13 +45,13 @@ shmem_set_acl(struct inode *inode, int type, struct posix_acl *acl)
         spin_lock(&inode->i_lock);
         switch(type) {
                 case ACL_TYPE_ACCESS:
-                        free = SHMEM_I(inode)->i_acl;
-                        SHMEM_I(inode)->i_acl = posix_acl_dup(acl);
+                        free = inode->i_acl;
+                        inode->i_acl = posix_acl_dup(acl);
                         break;
 
                 case ACL_TYPE_DEFAULT:
-                        free = SHMEM_I(inode)->i_default_acl;
-                        SHMEM_I(inode)->i_default_acl = posix_acl_dup(acl);
+                        free = inode->i_default_acl;
+                        inode->i_default_acl = posix_acl_dup(acl);
                         break;
         }
         spin_unlock(&inode->i_lock);
@@ -155,26 +155,9 @@ shmem_acl_init(struct inode *inode, struct inode *dir)
 }
 
 /**
- * shmem_acl_destroy_inode  -  destroy acls hanging off the in-memory inode
- *
- * This is done before destroying the actual inode.
- */
-
-void
-shmem_acl_destroy_inode(struct inode *inode)
-{
-        if (SHMEM_I(inode)->i_acl)
-                posix_acl_release(SHMEM_I(inode)->i_acl);
-        SHMEM_I(inode)->i_acl = NULL;
-        if (SHMEM_I(inode)->i_default_acl)
-                posix_acl_release(SHMEM_I(inode)->i_default_acl);
-        SHMEM_I(inode)->i_default_acl = NULL;
-}
-
-/**
  * shmem_check_acl  -  check_acl() callback for generic_permission()
  */
-static int
+int
 shmem_check_acl(struct inode *inode, int mask)
 {
         struct posix_acl *acl = shmem_get_acl(inode, ACL_TYPE_ACCESS);
@@ -186,12 +169,3 @@ shmem_check_acl(struct inode *inode, int mask)
         }
         return -EAGAIN;
 }
-
-/**
- * shmem_permission  -  permission() inode operation
- */
-int
-shmem_permission(struct inode *inode, int mask)
-{
-        return generic_permission(inode, mask, shmem_check_acl);
-}
diff --git a/mm/slab.c b/mm/slab.c
index 825c606f691d..7dfa481c96ba 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -102,16 +102,19 @@
 #include        <linux/cpu.h>
 #include        <linux/sysctl.h>
 #include        <linux/module.h>
+#include        <linux/kmemtrace.h>
 #include        <linux/rcupdate.h>
 #include        <linux/string.h>
 #include        <linux/uaccess.h>
 #include        <linux/nodemask.h>
+#include        <linux/kmemleak.h>
 #include        <linux/mempolicy.h>
 #include        <linux/mutex.h>
 #include        <linux/fault-inject.h>
 #include        <linux/rtmutex.h>
 #include        <linux/reciprocal_div.h>
 #include        <linux/debugobjects.h>
+#include        <linux/kmemcheck.h>
 
 #include        <asm/cacheflush.h>
 #include        <asm/tlbflush.h>
@@ -177,13 +180,13 @@
                          SLAB_STORE_USER | \
                          SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
                          SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
-                         SLAB_DEBUG_OBJECTS)
+                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
 #else
 # define CREATE_MASK    (SLAB_HWCACHE_ALIGN | \
                          SLAB_CACHE_DMA | \
                          SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
                          SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
-                         SLAB_DEBUG_OBJECTS)
+                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
 #endif
 
 /*
@@ -314,7 +317,7 @@ static int drain_freelist(struct kmem_cache *cache,
                         struct kmem_list3 *l3, int tofree);
 static void free_block(struct kmem_cache *cachep, void **objpp, int len,
                         int node);
-static int enable_cpucache(struct kmem_cache *cachep);
+static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
 static void cache_reap(struct work_struct *unused);
 
 /*
@@ -372,87 +375,6 @@ static void kmem_list3_init(struct kmem_list3 *parent)
         MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid);  \
         } while (0)
 
-/*
- * struct kmem_cache
- *
- * manages a cache.
- */
-
-struct kmem_cache {
-/* 1) per-cpu data, touched during every alloc/free */
-        struct array_cache *array[NR_CPUS];
-/* 2) Cache tunables. Protected by cache_chain_mutex */
-        unsigned int batchcount;
-        unsigned int limit;
-        unsigned int shared;
-
-        unsigned int buffer_size;
-        u32 reciprocal_buffer_size;
-/* 3) touched by every alloc & free from the backend */
-
-        unsigned int flags;             /* constant flags */
-        unsigned int num;               /* # of objs per slab */
-
-/* 4) cache_grow/shrink */
-        /* order of pgs per slab (2^n) */
-        unsigned int gfporder;
-
-        /* force GFP flags, e.g. GFP_DMA */
-        gfp_t gfpflags;
-
-        size_t colour;                  /* cache colouring range */
-        unsigned int colour_off;        /* colour offset */
-        struct kmem_cache *slabp_cache;
-        unsigned int slab_size;
-        unsigned int dflags;            /* dynamic flags */
-
-        /* constructor func */
-        void (*ctor)(void *obj);
-
-/* 5) cache creation/removal */
-        const char *name;
-        struct list_head next;
-
-/* 6) statistics */
-#if STATS
-        unsigned long num_active;
-        unsigned long num_allocations;
-        unsigned long high_mark;
-        unsigned long grown;
-        unsigned long reaped;
-        unsigned long errors;
-        unsigned long max_freeable;
-        unsigned long node_allocs;
-        unsigned long node_frees;
-        unsigned long node_overflow;
-        atomic_t allochit;
-        atomic_t allocmiss;
-        atomic_t freehit;
-        atomic_t freemiss;
-#endif
-#if DEBUG
-        /*
-         * If debugging is enabled, then the allocator can add additional
-         * fields and/or padding to every object. buffer_size contains the total
-         * object size including these internal fields, the following two
-         * variables contain the offset to the user object and its size.
-         */
-        int obj_offset;
-        int obj_size;
-#endif
-        /*
-         * We put nodelists[] at the end of kmem_cache, because we want to size
-         * this array to nr_node_ids slots instead of MAX_NUMNODES
-         * (see kmem_cache_init())
-         * We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache
-         * is statically defined, so we reserve the max number of nodes.
-         */
-        struct kmem_list3 *nodelists[MAX_NUMNODES];
-        /*
-         * Do not add fields after nodelists[]
-         */
-};
-
 #define CFLGS_OFF_SLAB          (0x80000000UL)
 #define OFF_SLAB(x)     ((x)->flags & CFLGS_OFF_SLAB)
 
@@ -568,6 +490,14 @@ static void **dbg_userword(struct kmem_cache *cachep, void *objp)
 
 #endif
 
+#ifdef CONFIG_KMEMTRACE
+size_t slab_buffer_size(struct kmem_cache *cachep)
+{
+        return cachep->buffer_size;
+}
+EXPORT_SYMBOL(slab_buffer_size);
+#endif
+
 /*
  * Do not go above this order unless 0 objects fit into the slab.
  */
@@ -743,6 +673,7 @@ static enum {
         NONE,
         PARTIAL_AC,
         PARTIAL_L3,
+        EARLY,
         FULL
 } g_cpucache_up;
 
@@ -751,7 +682,7 @@ static enum {
  */
 int slab_is_available(void)
 {
-        return g_cpucache_up == FULL;
+        return g_cpucache_up >= EARLY;
 }
 
 static DEFINE_PER_CPU(struct delayed_work, reap_work);
@@ -881,7 +812,6 @@ static void __slab_error(const char *function, struct kmem_cache *cachep,
   */
 
 static int use_alien_caches __read_mostly = 1;
-static int numa_platform __read_mostly = 1;
 static int __init noaliencache_setup(char *s)
 {
         use_alien_caches = 0;
@@ -949,12 +879,20 @@ static void __cpuinit start_cpu_timer(int cpu)
 }
 
 static struct array_cache *alloc_arraycache(int node, int entries,
-                                            int batchcount)
+                                            int batchcount, gfp_t gfp)
 {
         int memsize = sizeof(void *) * entries + sizeof(struct array_cache);
         struct array_cache *nc = NULL;
 
-        nc = kmalloc_node(memsize, GFP_KERNEL, node);
+        nc = kmalloc_node(memsize, gfp, node);
+        /*
+         * The array_cache structures contain pointers to free object.
+         * However, when such objects are allocated or transfered to another
+         * cache the pointers are not cleared and they could be counted as
+         * valid references during a kmemleak scan. Therefore, kmemleak must
+         * not scan such objects.
+         */
+        kmemleak_no_scan(nc);
         if (nc) {
                 nc->avail = 0;
                 nc->limit = entries;
@@ -994,7 +932,7 @@ static int transfer_objects(struct array_cache *to,
 #define drain_alien_cache(cachep, alien) do { } while (0)
 #define reap_alien(cachep, l3) do { } while (0)
 
-static inline struct array_cache **alloc_alien_cache(int node, int limit)
+static inline struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
 {
         return (struct array_cache **)BAD_ALIEN_MAGIC;
 }
@@ -1025,7 +963,7 @@ static inline void *____cache_alloc_node(struct kmem_cache *cachep,
 static void *____cache_alloc_node(struct kmem_cache *, gfp_t, int);
 static void *alternate_node_alloc(struct kmem_cache *, gfp_t);
 
-static struct array_cache **alloc_alien_cache(int node, int limit)
+static struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
 {
         struct array_cache **ac_ptr;
         int memsize = sizeof(void *) * nr_node_ids;
@@ -1033,14 +971,14 @@ static struct array_cache **alloc_alien_cache(int node, int limit)
 
         if (limit > 1)
                 limit = 12;
-        ac_ptr = kmalloc_node(memsize, GFP_KERNEL, node);
+        ac_ptr = kmalloc_node(memsize, gfp, node);
         if (ac_ptr) {
                 for_each_node(i) {
                         if (i == node || !node_online(i)) {
                                 ac_ptr[i] = NULL;
                                 continue;
                         }
-                        ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d);
+                        ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d, gfp);
                         if (!ac_ptr[i]) {
                                 for (i--; i >= 0; i--)
                                         kfree(ac_ptr[i]);
@@ -1160,7 +1098,7 @@ static void __cpuinit cpuup_canceled(long cpu)
         struct kmem_cache *cachep;
         struct kmem_list3 *l3 = NULL;
         int node = cpu_to_node(cpu);
-        node_to_cpumask_ptr(mask, node);
+        const struct cpumask *mask = cpumask_of_node(node);
 
         list_for_each_entry(cachep, &cache_chain, next) {
                 struct array_cache *nc;
@@ -1273,20 +1211,20 @@ static int __cpuinit cpuup_prepare(long cpu)
                 struct array_cache **alien = NULL;
 
                 nc = alloc_arraycache(node, cachep->limit,
-                                        cachep->batchcount);
+                                        cachep->batchcount, GFP_KERNEL);
                 if (!nc)
                         goto bad;
                 if (cachep->shared) {
                         shared = alloc_arraycache(node,
                                 cachep->shared * cachep->batchcount,
-                                0xbaadf00d);
+                                0xbaadf00d, GFP_KERNEL);
                         if (!shared) {
                                 kfree(nc);
                                 goto bad;
                         }
                 }
                 if (use_alien_caches) {
-                        alien = alloc_alien_cache(node, cachep->limit);
+                        alien = alloc_alien_cache(node, cachep->limit, GFP_KERNEL);
                         if (!alien) {
                                 kfree(shared);
                                 kfree(nc);
@@ -1390,10 +1328,9 @@ static void init_list(struct kmem_cache *cachep, struct kmem_list3 *list,
 {
         struct kmem_list3 *ptr;
 
-        ptr = kmalloc_node(sizeof(struct kmem_list3), GFP_KERNEL, nodeid);
+        ptr = kmalloc_node(sizeof(struct kmem_list3), GFP_NOWAIT, nodeid);
         BUG_ON(!ptr);
 
-        local_irq_disable();
         memcpy(ptr, list, sizeof(struct kmem_list3));
         /*
          * Do not assume that spinlocks can be initialized via memcpy:
@@ -1402,7 +1339,6 @@ static void init_list(struct kmem_cache *cachep, struct kmem_list3 *list,
 
         MAKE_ALL_LISTS(cachep, ptr, nodeid);
         cachep->nodelists[nodeid] = ptr;
-        local_irq_enable();
 }
 
 /*
@@ -1434,10 +1370,8 @@ void __init kmem_cache_init(void)
         int order;
         int node;
 
-        if (num_possible_nodes() == 1) {
+        if (num_possible_nodes() == 1)
                 use_alien_caches = 0;
-                numa_platform = 0;
-        }
 
         for (i = 0; i < NUM_INIT_LISTS; i++) {
                 kmem_list3_init(&initkmem_list3[i]);
@@ -1450,7 +1384,7 @@ void __init kmem_cache_init(void)
          * Fragmentation resistance on low memory - only use bigger
          * page orders on machines with more than 32MB of memory.
          */
-        if (num_physpages > (32 << 20) >> PAGE_SHIFT)
+        if (totalram_pages > (32 << 20) >> PAGE_SHIFT)
                 slab_break_gfp_order = BREAK_GFP_ORDER_HI;
 
         /* Bootstrap is tricky, because several objects are allocated
@@ -1566,9 +1500,8 @@ void __init kmem_cache_init(void)
         {
                 struct array_cache *ptr;
 
-                ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL);
+                ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
 
-                local_irq_disable();
                 BUG_ON(cpu_cache_get(&cache_cache) != &initarray_cache.cache);
                 memcpy(ptr, cpu_cache_get(&cache_cache),
                        sizeof(struct arraycache_init));
@@ -1578,11 +1511,9 @@ void __init kmem_cache_init(void)
                 spin_lock_init(&ptr->lock);
 
                 cache_cache.array[smp_processor_id()] = ptr;
-                local_irq_enable();
 
-                ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL);
+                ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
 
-                local_irq_disable();
                 BUG_ON(cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep)
                        != &initarray_generic.cache);
                 memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep),
@@ -1594,7 +1525,6 @@ void __init kmem_cache_init(void)
 
                 malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] =
                     ptr;
-                local_irq_enable();
         }
         /* 5) Replace the bootstrap kmem_list3's */
         {
@@ -1613,23 +1543,26 @@ void __init kmem_cache_init(void)
                 }
         }
 
-        /* 6) resize the head arrays to their final sizes */
-        {
-                struct kmem_cache *cachep;
-                mutex_lock(&cache_chain_mutex);
-                list_for_each_entry(cachep, &cache_chain, next)
-                        if (enable_cpucache(cachep))
-                                BUG();
-                mutex_unlock(&cache_chain_mutex);
-        }
+        g_cpucache_up = EARLY;
+}
 
-        /* Annotate slab for lockdep -- annotate the malloc caches */
-        init_lock_keys();
+void __init kmem_cache_init_late(void)
+{
+        struct kmem_cache *cachep;
 
+        /* 6) resize the head arrays to their final sizes */
+        mutex_lock(&cache_chain_mutex);
+        list_for_each_entry(cachep, &cache_chain, next)
+                if (enable_cpucache(cachep, GFP_NOWAIT))
+                        BUG();
+        mutex_unlock(&cache_chain_mutex);
 
         /* Done! */
         g_cpucache_up = FULL;
 
+        /* Annotate slab for lockdep -- annotate the malloc caches */
+        init_lock_keys();
+
         /*
          * Register a cpu startup notifier callback that initializes
          * cpu_cache_get for all new cpus
@@ -1680,7 +1613,7 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
         if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
                 flags |= __GFP_RECLAIMABLE;
 
-        page = alloc_pages_node(nodeid, flags, cachep->gfporder);
+        page = alloc_pages_exact_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
         if (!page)
                 return NULL;
 
@@ -1693,6 +1626,16 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
                         NR_SLAB_UNRECLAIMABLE, nr_pages);
         for (i = 0; i < nr_pages; i++)
                 __SetPageSlab(page + i);
+
+        if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
+                kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);
+
+                if (cachep->ctor)
+                        kmemcheck_mark_uninitialized_pages(page, nr_pages);
+                else
+                        kmemcheck_mark_unallocated_pages(page, nr_pages);
+        }
+
         return page_address(page);
 }
 
@@ -1705,6 +1648,8 @@ static void kmem_freepages(struct kmem_cache *cachep, void *addr)
         struct page *page = virt_to_page(addr);
         const unsigned long nr_freed = i;
 
+        kmemcheck_free_shadow(page, cachep->gfporder);
+
         if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
                 sub_zone_page_state(page_zone(page),
                                 NR_SLAB_RECLAIMABLE, nr_freed);
@@ -2055,10 +2000,10 @@ static size_t calculate_slab_order(struct kmem_cache *cachep,
         return left_over;
 }
 
-static int __init_refok setup_cpu_cache(struct kmem_cache *cachep)
+static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
 {
         if (g_cpucache_up == FULL)
-                return enable_cpucache(cachep);
+                return enable_cpucache(cachep, gfp);
 
         if (g_cpucache_up == NONE) {
                 /*
@@ -2080,7 +2025,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep)
                         g_cpucache_up = PARTIAL_AC;
         } else {
                 cachep->array[smp_processor_id()] =
-                        kmalloc(sizeof(struct arraycache_init), GFP_KERNEL);
+                        kmalloc(sizeof(struct arraycache_init), gfp);
 
                 if (g_cpucache_up == PARTIAL_AC) {
                         set_up_list3s(cachep, SIZE_L3);
@@ -2090,7 +2035,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep)
                         for_each_online_node(node) {
                                 cachep->nodelists[node] =
                                     kmalloc_node(sizeof(struct kmem_list3),
-                                                GFP_KERNEL, node);
+                                                gfp, node);
                                 BUG_ON(!cachep->nodelists[node]);
                                 kmem_list3_init(cachep->nodelists[node]);
                         }
@@ -2144,6 +2089,7 @@ kmem_cache_create (const char *name, size_t size, size_t align,
 {
         size_t left_over, slab_size, ralign;
         struct kmem_cache *cachep = NULL, *pc;
+        gfp_t gfp;
 
         /*
          * Sanity checks... these are all serious usage bugs.
@@ -2159,8 +2105,10 @@ kmem_cache_create (const char *name, size_t size, size_t align,
          * We use cache_chain_mutex to ensure a consistent view of
          * cpu_online_mask as well.  Please see cpuup_callback
          */
-        get_online_cpus();
-        mutex_lock(&cache_chain_mutex);
+        if (slab_is_available()) {
+                get_online_cpus();
+                mutex_lock(&cache_chain_mutex);
+        }
 
         list_for_each_entry(pc, &cache_chain, next) {
                 char tmp;
@@ -2269,8 +2217,13 @@ kmem_cache_create (const char *name, size_t size, size_t align,
          */
         align = ralign;
 
+        if (slab_is_available())
+                gfp = GFP_KERNEL;
+        else
+                gfp = GFP_NOWAIT;
+
         /* Get cache's description obj. */
-        cachep = kmem_cache_zalloc(&cache_cache, GFP_KERNEL);
+        cachep = kmem_cache_zalloc(&cache_cache, gfp);
         if (!cachep)
                 goto oops;
 
@@ -2344,6 +2297,15 @@ kmem_cache_create (const char *name, size_t size, size_t align,
                 /* really off slab. No need for manual alignment */
                 slab_size =
                     cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab);
+
+#ifdef CONFIG_PAGE_POISONING
+                /* If we're going to use the generic kernel_map_pages()
+                 * poisoning, then it's going to smash the contents of
+                 * the redzone and userword anyhow, so switch them off.
+                 */
+                if (size % PAGE_SIZE == 0 && flags & SLAB_POISON)
+                        flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
+#endif
         }
 
         cachep->colour_off = cache_line_size();
@@ -2373,7 +2335,7 @@ kmem_cache_create (const char *name, size_t size, size_t align,
         cachep->ctor = ctor;
         cachep->name = name;
 
-        if (setup_cpu_cache(cachep)) {
+        if (setup_cpu_cache(cachep, gfp)) {
                 __kmem_cache_destroy(cachep);
                 cachep = NULL;
                 goto oops;
@@ -2385,8 +2347,10 @@ oops:
         if (!cachep && (flags & SLAB_PANIC))
                 panic("kmem_cache_create(): failed to create slab `%s'\n",
                       name);
-        mutex_unlock(&cache_chain_mutex);
-        put_online_cpus();
+        if (slab_is_available()) {
+                mutex_unlock(&cache_chain_mutex);
+                put_online_cpus();
+        }
         return cachep;
 }
 EXPORT_SYMBOL(kmem_cache_create);
@@ -2583,7 +2547,7 @@ void kmem_cache_destroy(struct kmem_cache *cachep)
         }
 
         if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU))
-                synchronize_rcu();
+                rcu_barrier();
 
         __kmem_cache_destroy(cachep);
         mutex_unlock(&cache_chain_mutex);
@@ -2612,6 +2576,14 @@ static struct slab *alloc_slabmgmt(struct kmem_cache *cachep, void *objp,
                 /* Slab management obj is off-slab. */
                 slabp = kmem_cache_alloc_node(cachep->slabp_cache,
                                               local_flags, nodeid);
+                /*
+                 * If the first object in the slab is leaked (it's allocated
+                 * but no one has a reference to it), we want to make sure
+                 * kmemleak does not treat the ->s_mem pointer as a reference
+                 * to the object. Otherwise we will not report the leak.
+                 */
+                kmemleak_scan_area(slabp, offsetof(struct slab, list),
+                                   sizeof(struct list_head), local_flags);
                 if (!slabp)
                         return NULL;
         } else {
@@ -3132,6 +3104,12 @@ static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
                 STATS_INC_ALLOCMISS(cachep);
                 objp = cache_alloc_refill(cachep, flags);
         }
+        /*
+         * To avoid a false negative, if an object that is in one of the
+         * per-CPU caches is leaked, we need to make sure kmemleak doesn't
+         * treat the array pointers as a reference to the object.
+         */
+        kmemleak_erase(&ac->entry[ac->avail]);
         return objp;
 }
 
@@ -3210,7 +3188,7 @@ retry:
                 if (local_flags & __GFP_WAIT)
                         local_irq_enable();
                 kmem_flagcheck(cache, flags);
-                obj = kmem_getpages(cache, local_flags, -1);
+                obj = kmem_getpages(cache, local_flags, numa_node_id());
                 if (local_flags & __GFP_WAIT)
                         local_irq_disable();
                 if (obj) {
@@ -3318,6 +3296,8 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
         unsigned long save_flags;
         void *ptr;
 
+        flags &= gfp_allowed_mask;
+
         lockdep_trace_alloc(flags);
 
         if (slab_should_failslab(cachep, flags))
@@ -3351,6 +3331,11 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
   out:
         local_irq_restore(save_flags);
         ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);
+        kmemleak_alloc_recursive(ptr, obj_size(cachep), 1, cachep->flags,
+                                 flags);
+
+        if (likely(ptr))
+                kmemcheck_slab_alloc(cachep, flags, ptr, obj_size(cachep));
 
         if (unlikely((flags & __GFP_ZERO) && ptr))
                 memset(ptr, 0, obj_size(cachep));
@@ -3396,6 +3381,8 @@ __cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller)
         unsigned long save_flags;
         void *objp;
 
+        flags &= gfp_allowed_mask;
+
         lockdep_trace_alloc(flags);
 
         if (slab_should_failslab(cachep, flags))
@@ -3406,8 +3393,13 @@ __cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller)
         objp = __do_cache_alloc(cachep, flags);
         local_irq_restore(save_flags);
         objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
+        kmemleak_alloc_recursive(objp, obj_size(cachep), 1, cachep->flags,
+                                 flags);
         prefetchw(objp);
 
+        if (likely(objp))
+                kmemcheck_slab_alloc(cachep, flags, objp, obj_size(cachep));
+
         if (unlikely((flags & __GFP_ZERO) && objp))
                 memset(objp, 0, obj_size(cachep));
 
@@ -3521,8 +3513,11 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp)
         struct array_cache *ac = cpu_cache_get(cachep);
 
         check_irq_off();
+        kmemleak_free_recursive(objp, cachep->flags);
         objp = cache_free_debugcheck(cachep, objp, __builtin_return_address(0));
 
+        kmemcheck_slab_free(cachep, objp, obj_size(cachep));
+
         /*
          * Skip calling cache_free_alien() when the platform is not numa.
          * This will avoid cache misses that happen while accessing slabp (which
@@ -3530,7 +3525,7 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp)
          * variable to skip the call, which is mostly likely to be present in
          * the cache.
          */
-        if (numa_platform && cache_free_alien(cachep, objp))
+        if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))
                 return;
 
         if (likely(ac->avail < ac->limit)) {
@@ -3554,10 +3549,23 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp)
  */
 void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
 {
-        return __cache_alloc(cachep, flags, __builtin_return_address(0));
+        void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0));
+
+        trace_kmem_cache_alloc(_RET_IP_, ret,
+                               obj_size(cachep), cachep->buffer_size, flags);
+
+        return ret;
 }
 EXPORT_SYMBOL(kmem_cache_alloc);
 
+#ifdef CONFIG_KMEMTRACE
+void *kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags)
+{
+        return __cache_alloc(cachep, flags, __builtin_return_address(0));
+}
+EXPORT_SYMBOL(kmem_cache_alloc_notrace);
+#endif
+
 /**
  * kmem_ptr_validate - check if an untrusted pointer might be a slab entry.
  * @cachep: the cache we're checking against
@@ -3602,23 +3610,46 @@ out:
 #ifdef CONFIG_NUMA
 void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
 {
-        return __cache_alloc_node(cachep, flags, nodeid,
-                        __builtin_return_address(0));
+        void *ret = __cache_alloc_node(cachep, flags, nodeid,
+                                       __builtin_return_address(0));
+
+        trace_kmem_cache_alloc_node(_RET_IP_, ret,
+                                    obj_size(cachep), cachep->buffer_size,
+                                    flags, nodeid);
+
+        return ret;
 }
 EXPORT_SYMBOL(kmem_cache_alloc_node);
 
+#ifdef CONFIG_KMEMTRACE
+void *kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
+                                    gfp_t flags,
+                                    int nodeid)
+{
+        return __cache_alloc_node(cachep, flags, nodeid,
+                                  __builtin_return_address(0));
+}
+EXPORT_SYMBOL(kmem_cache_alloc_node_notrace);
+#endif
+
 static __always_inline void *
 __do_kmalloc_node(size_t size, gfp_t flags, int node, void *caller)
 {
         struct kmem_cache *cachep;
+        void *ret;
 
         cachep = kmem_find_general_cachep(size, flags);
         if (unlikely(ZERO_OR_NULL_PTR(cachep)))
                 return cachep;
-        return kmem_cache_alloc_node(cachep, flags, node);
+        ret = kmem_cache_alloc_node_notrace(cachep, flags, node);
+
+        trace_kmalloc_node((unsigned long) caller, ret,
+                           size, cachep->buffer_size, flags, node);
+
+        return ret;
 }
 
-#ifdef CONFIG_DEBUG_SLAB
+#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_KMEMTRACE)
 void *__kmalloc_node(size_t size, gfp_t flags, int node)
 {
         return __do_kmalloc_node(size, flags, node,
@@ -3651,6 +3682,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
                                           void *caller)
 {
         struct kmem_cache *cachep;
+        void *ret;
 
         /* If you want to save a few bytes .text space: replace
          * __ with kmem_.
@@ -3660,11 +3692,16 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
         cachep = __find_general_cachep(size, flags);
         if (unlikely(ZERO_OR_NULL_PTR(cachep)))
                 return cachep;
-        return __cache_alloc(cachep, flags, caller);
+        ret = __cache_alloc(cachep, flags, caller);
+
+        trace_kmalloc((unsigned long) caller, ret,
+                      size, cachep->buffer_size, flags);
+
+        return ret;
 }
 
 
-#ifdef CONFIG_DEBUG_SLAB
+#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_KMEMTRACE)
 void *__kmalloc(size_t size, gfp_t flags)
 {
         return __do_kmalloc(size, flags, __builtin_return_address(0));
@@ -3703,6 +3740,8 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp)
                 debug_check_no_obj_freed(objp, obj_size(cachep));
         __cache_free(cachep, objp);
         local_irq_restore(flags);
+
+        trace_kmem_cache_free(_RET_IP_, objp);
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
@@ -3720,6 +3759,8 @@ void kfree(const void *objp)
         struct kmem_cache *c;
         unsigned long flags;
 
+        trace_kfree(_RET_IP_, objp);
+
         if (unlikely(ZERO_OR_NULL_PTR(objp)))
                 return;
         local_irq_save(flags);
@@ -3747,7 +3788,7 @@ EXPORT_SYMBOL_GPL(kmem_cache_name);
 /*
  * This initializes kmem_list3 or resizes various caches for all nodes.
  */
-static int alloc_kmemlist(struct kmem_cache *cachep)
+static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp)
 {
         int node;
         struct kmem_list3 *l3;
@@ -3757,7 +3798,7 @@ static int alloc_kmemlist(struct kmem_cache *cachep)
         for_each_online_node(node) {
 
                 if (use_alien_caches) {
-                        new_alien = alloc_alien_cache(node, cachep->limit);
+                        new_alien = alloc_alien_cache(node, cachep->limit, gfp);
                         if (!new_alien)
                                 goto fail;
                 }
@@ -3766,7 +3807,7 @@ static int alloc_kmemlist(struct kmem_cache *cachep)
                 if (cachep->shared) {
                         new_shared = alloc_arraycache(node,
                                 cachep->shared*cachep->batchcount,
-                                        0xbaadf00d);
+                                        0xbaadf00d, gfp);
                         if (!new_shared) {
                                 free_alien_cache(new_alien);
                                 goto fail;
@@ -3795,7 +3836,7 @@ static int alloc_kmemlist(struct kmem_cache *cachep)
                         free_alien_cache(new_alien);
                         continue;
                 }
-                l3 = kmalloc_node(sizeof(struct kmem_list3), GFP_KERNEL, node);
+                l3 = kmalloc_node(sizeof(struct kmem_list3), gfp, node);
                 if (!l3) {
                         free_alien_cache(new_alien);
                         kfree(new_shared);
@@ -3851,18 +3892,18 @@ static void do_ccupdate_local(void *info)
 
 /* Always called with the cache_chain_mutex held */
 static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
-                                int batchcount, int shared)
+                                int batchcount, int shared, gfp_t gfp)
 {
         struct ccupdate_struct *new;
         int i;
 
-        new = kzalloc(sizeof(*new), GFP_KERNEL);
+        new = kzalloc(sizeof(*new), gfp);
         if (!new)
                 return -ENOMEM;
 
         for_each_online_cpu(i) {
                 new->new[i] = alloc_arraycache(cpu_to_node(i), limit,
-                                                batchcount);
+                                                batchcount, gfp);
                 if (!new->new[i]) {
                         for (i--; i >= 0; i--)
                                 kfree(new->new[i]);
@@ -3889,11 +3930,11 @@ static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
                 kfree(ccold);
         }
         kfree(new);
-        return alloc_kmemlist(cachep);
+        return alloc_kmemlist(cachep, gfp);
 }
 
 /* Called with cache_chain_mutex held always */
-static int enable_cpucache(struct kmem_cache *cachep)
+static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
 {
         int err;
         int limit, shared;
@@ -3939,7 +3980,7 @@ static int enable_cpucache(struct kmem_cache *cachep)
         if (limit > 32)
                 limit = 32;
 #endif
-        err = do_tune_cpucache(cachep, limit, (limit + 1) / 2, shared);
+        err = do_tune_cpucache(cachep, limit, (limit + 1) / 2, shared, gfp);
         if (err)
                 printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
                        cachep->name, -err);
@@ -3992,8 +4033,7 @@ static void cache_reap(struct work_struct *w)
         struct kmem_cache *searchp;
         struct kmem_list3 *l3;
         int node = numa_node_id();
-        struct delayed_work *work =
-                container_of(w, struct delayed_work, work);
+        struct delayed_work *work = to_delayed_work(w);
 
         if (!mutex_trylock(&cache_chain_mutex))
                 /* Give up. Setup the next iteration. */
@@ -4246,7 +4286,8 @@ ssize_t slabinfo_write(struct file *file, const char __user * buffer,
                                 res = 0;
                         } else {
                                 res = do_tune_cpucache(cachep, limit,
-                                                       batchcount, shared);
+                                                       batchcount, shared,
+                                                       GFP_KERNEL);
                         }
                         break;
                 }
diff --git a/mm/slob.c b/mm/slob.c
index 7a3411524dac..837ebd64cc34 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -46,7 +46,7 @@
  * NUMA support in SLOB is fairly simplistic, pushing most of the real
  * logic down to the page allocator, and simply doing the node accounting
  * on the upper levels. In the event that a node id is explicitly
- * provided, alloc_pages_node() with the specified node id is used
+ * provided, alloc_pages_exact_node() with the specified node id is used
  * instead. The common case (or when the node id isn't explicitly provided)
  * will default to the current node, as per numa_node_id().
  *
@@ -60,11 +60,14 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/mm.h>
+#include <linux/swap.h> /* struct reclaim_state */
 #include <linux/cache.h>
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/rcupdate.h>
 #include <linux/list.h>
+#include <linux/kmemtrace.h>
+#include <linux/kmemleak.h>
 #include <asm/atomic.h>
 
 /*
@@ -130,17 +133,17 @@ static LIST_HEAD(free_slob_large);
  */
 static inline int is_slob_page(struct slob_page *sp)
 {
-        return PageSlobPage((struct page *)sp);
+        return PageSlab((struct page *)sp);
 }
 
 static inline void set_slob_page(struct slob_page *sp)
 {
-        __SetPageSlobPage((struct page *)sp);
+        __SetPageSlab((struct page *)sp);
 }
 
 static inline void clear_slob_page(struct slob_page *sp)
 {
-        __ClearPageSlobPage((struct page *)sp);
+        __ClearPageSlab((struct page *)sp);
 }
 
 static inline struct slob_page *slob_page(const void *addr)
@@ -241,7 +244,7 @@ static void *slob_new_pages(gfp_t gfp, int order, int node)
 
 #ifdef CONFIG_NUMA
         if (node != -1)
-                page = alloc_pages_node(node, gfp, order);
+                page = alloc_pages_exact_node(node, gfp, order);
         else
 #endif
                 page = alloc_pages(gfp, order);
@@ -254,6 +257,8 @@ static void *slob_new_pages(gfp_t gfp, int order, int node)
 
 static void slob_free_pages(void *b, int order)
 {
+        if (current->reclaim_state)
+                current->reclaim_state->reclaimed_slab += 1 << order;
         free_pages((unsigned long)b, order);
 }
 
@@ -406,7 +411,7 @@ static void slob_free(void *block, int size)
                 spin_unlock_irqrestore(&slob_lock, flags);
                 clear_slob_page(sp);
                 free_slob_page(sp);
-                free_page((unsigned long)b);
+                slob_free_pages(b, 0);
                 return;
         }
 
@@ -474,6 +479,7 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node)
 {
         unsigned int *m;
         int align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
+        void *ret;
 
         lockdep_trace_alloc(gfp);
 
@@ -482,12 +488,16 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node)
                         return ZERO_SIZE_PTR;
 
                 m = slob_alloc(size + align, gfp, align, node);
+
                 if (!m)
                         return NULL;
                 *m = size;
-                return (void *)m + align;
+                ret = (void *)m + align;
+
+                trace_kmalloc_node(_RET_IP_, ret,
+                                   size, size + align, gfp, node);
         } else {
-                void *ret;
+                unsigned int order = get_order(size);
 
                 ret = slob_new_pages(gfp | __GFP_COMP, get_order(size), node);
                 if (ret) {
@@ -495,8 +505,13 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node)
                         page = virt_to_page(ret);
                         page->private = size;
                 }
-                return ret;
+
+                trace_kmalloc_node(_RET_IP_, ret,
+                                   size, PAGE_SIZE << order, gfp, node);
         }
+
+        kmemleak_alloc(ret, size, 1, gfp);
+        return ret;
 }
 EXPORT_SYMBOL(__kmalloc_node);
 
@@ -504,8 +519,11 @@ void kfree(const void *block)
 {
         struct slob_page *sp;
 
+        trace_kfree(_RET_IP_, block);
+
         if (unlikely(ZERO_OR_NULL_PTR(block)))
                 return;
+        kmemleak_free(block);
 
         sp = slob_page(block);
         if (is_slob_page(sp)) {
@@ -569,12 +587,16 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size,
         } else if (flags & SLAB_PANIC)
                 panic("Cannot create slab cache %s\n", name);
 
+        kmemleak_alloc(c, sizeof(struct kmem_cache), 1, GFP_KERNEL);
         return c;
 }
 EXPORT_SYMBOL(kmem_cache_create);
 
 void kmem_cache_destroy(struct kmem_cache *c)
 {
+        kmemleak_free(c);
+        if (c->flags & SLAB_DESTROY_BY_RCU)
+                rcu_barrier();
         slob_free(c, sizeof(struct kmem_cache));
 }
 EXPORT_SYMBOL(kmem_cache_destroy);
@@ -583,14 +605,22 @@ void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
 {
         void *b;
 
-        if (c->size < PAGE_SIZE)
+        if (c->size < PAGE_SIZE) {
                 b = slob_alloc(c->size, flags, c->align, node);
-        else
+                trace_kmem_cache_alloc_node(_RET_IP_, b, c->size,
+                                            SLOB_UNITS(c->size) * SLOB_UNIT,
+                                            flags, node);
+        } else {
                 b = slob_new_pages(flags, get_order(c->size), node);
+                trace_kmem_cache_alloc_node(_RET_IP_, b, c->size,
+                                            PAGE_SIZE << get_order(c->size),
+                                            flags, node);
+        }
 
         if (c->ctor)
                 c->ctor(b);
 
+        kmemleak_alloc_recursive(b, c->size, 1, c->flags, flags);
         return b;
 }
 EXPORT_SYMBOL(kmem_cache_alloc_node);
@@ -613,6 +643,7 @@ static void kmem_rcu_free(struct rcu_head *head)
 
 void kmem_cache_free(struct kmem_cache *c, void *b)
 {
+        kmemleak_free_recursive(b, c->flags);
         if (unlikely(c->flags & SLAB_DESTROY_BY_RCU)) {
                 struct slob_rcu *slob_rcu;
                 slob_rcu = b + (c->size - sizeof(struct slob_rcu));
@@ -622,6 +653,8 @@ void kmem_cache_free(struct kmem_cache *c, void *b)
         } else {
                 __kmem_cache_free(b, c->size);
         }
+
+        trace_kmem_cache_free(_RET_IP_, b);
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
@@ -659,3 +692,8 @@ void __init kmem_cache_init(void)
 {
         slob_ready = 1;
 }
+
+void __init kmem_cache_init_late(void)
+{
+        /* Nothing to do */
+}
diff --git a/mm/slub.c b/mm/slub.c
index c4ea9158c9fb..4996fc719552 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -9,6 +9,7 @@
  */
 
 #include <linux/mm.h>
+#include <linux/swap.h> /* struct reclaim_state */
 #include <linux/module.h>
 #include <linux/bit_spinlock.h>
 #include <linux/interrupt.h>
@@ -16,6 +17,8 @@
 #include <linux/slab.h>
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
+#include <linux/kmemtrace.h>
+#include <linux/kmemcheck.h>
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
 #include <linux/mempolicy.h>
@@ -138,13 +141,20 @@
                                 SLAB_POISON | SLAB_STORE_USER)
 
 /*
+ * Debugging flags that require metadata to be stored in the slab.  These get
+ * disabled when slub_debug=O is used and a cache's min order increases with
+ * metadata.
+ */
+#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
+
+/*
  * Set of flags that will prevent slab merging
  */
 #define SLUB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
-                SLAB_TRACE | SLAB_DESTROY_BY_RCU)
+                SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE)
 
 #define SLUB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
-                SLAB_CACHE_DMA)
+                SLAB_CACHE_DMA | SLAB_NOTRACK)
 
 #ifndef ARCH_KMALLOC_MINALIGN
 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
@@ -322,6 +332,7 @@ static int slub_debug;
 #endif
 
 static char *slub_debug_slabs;
+static int disable_higher_order_debug;
 
 /*
  * Object debugging
@@ -643,7 +654,7 @@ static int slab_pad_check(struct kmem_cache *s, struct page *page)
         slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
         print_section("Padding", end - remainder, remainder);
 
-        restore_bytes(s, "slab padding", POISON_INUSE, start, end);
+        restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end);
         return 0;
 }
 
@@ -830,6 +841,11 @@ static inline unsigned long slabs_node(struct kmem_cache *s, int node)
         return atomic_long_read(&n->nr_slabs);
 }
 
+static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
+{
+        return atomic_long_read(&n->nr_slabs);
+}
+
 static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
 {
         struct kmem_cache_node *n = get_node(s, node);
@@ -968,6 +984,15 @@ static int __init setup_slub_debug(char *str)
                  */
                 goto check_slabs;
 
+        if (tolower(*str) == 'o') {
+                /*
+                 * Avoid enabling debugging on caches if its minimum order
+                 * would increase as a result.
+                 */
+                disable_higher_order_debug = 1;
+                goto out;
+        }
+
         slub_debug = 0;
         if (*str == '-')
                 /*
@@ -1018,8 +1043,8 @@ static unsigned long kmem_cache_flags(unsigned long objsize,
          * Enable debugging if selected on the kernel commandline.
          */
         if (slub_debug && (!slub_debug_slabs ||
-            strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs)) == 0))
-                        flags |= slub_debug;
+                !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs))))
+                flags |= slub_debug;
 
         return flags;
 }
@@ -1046,8 +1071,12 @@ static inline unsigned long kmem_cache_flags(unsigned long objsize,
 }
 #define slub_debug 0
 
+#define disable_higher_order_debug 0
+
 static inline unsigned long slabs_node(struct kmem_cache *s, int node)
                                                         { return 0; }
+static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
+                                                        { return 0; }
 static inline void inc_slabs_node(struct kmem_cache *s, int node,
                                                         int objects) {}
 static inline void dec_slabs_node(struct kmem_cache *s, int node,
@@ -1062,6 +1091,8 @@ static inline struct page *alloc_slab_page(gfp_t flags, int node,
 {
         int order = oo_order(oo);
 
+        flags |= __GFP_NOTRACK;
+
         if (node == -1)
                 return alloc_pages(flags, order);
         else
@@ -1072,11 +1103,17 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
 {
         struct page *page;
         struct kmem_cache_order_objects oo = s->oo;
+        gfp_t alloc_gfp;
 
         flags |= s->allocflags;
 
-        page = alloc_slab_page(flags | __GFP_NOWARN | __GFP_NORETRY, node,
-                                                                        oo);
+        /*
+         * Let the initial higher-order allocation fail under memory pressure
+         * so we fall-back to the minimum order allocation.
+         */
+        alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
+
+        page = alloc_slab_page(alloc_gfp, node, oo);
         if (unlikely(!page)) {
                 oo = s->min;
                 /*
@@ -1089,6 +1126,23 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
 
                 stat(get_cpu_slab(s, raw_smp_processor_id()), ORDER_FALLBACK);
         }
+
+        if (kmemcheck_enabled
+                && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
+                int pages = 1 << oo_order(oo);
+
+                kmemcheck_alloc_shadow(page, oo_order(oo), flags, node);
+
+                /*
+                 * Objects from caches that have a constructor don't get
+                 * cleared when they're allocated, so we need to do it here.
+                 */
+                if (s->ctor)
+                        kmemcheck_mark_uninitialized_pages(page, pages);
+                else
+                        kmemcheck_mark_unallocated_pages(page, pages);
+        }
+
         page->objects = oo_objects(oo);
         mod_zone_page_state(page_zone(page),
                 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
@@ -1162,6 +1216,8 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
                 __ClearPageSlubDebug(page);
         }
 
+        kmemcheck_free_shadow(page, compound_order(page));
+
         mod_zone_page_state(page_zone(page),
                 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
                 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
@@ -1169,6 +1225,8 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
 
         __ClearPageSlab(page);
         reset_page_mapcount(page);
+        if (current->reclaim_state)
+                current->reclaim_state->reclaimed_slab += pages;
         __free_pages(page, order);
 }
 
@@ -1480,6 +1538,69 @@ static inline int node_match(struct kmem_cache_cpu *c, int node)
         return 1;
 }
 
+static int count_free(struct page *page)
+{
+        return page->objects - page->inuse;
+}
+
+static unsigned long count_partial(struct kmem_cache_node *n,
+                                        int (*get_count)(struct page *))
+{
+        unsigned long flags;
+        unsigned long x = 0;
+        struct page *page;
+
+        spin_lock_irqsave(&n->list_lock, flags);
+        list_for_each_entry(page, &n->partial, lru)
+                x += get_count(page);
+        spin_unlock_irqrestore(&n->list_lock, flags);
+        return x;
+}
+
+static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
+{
+#ifdef CONFIG_SLUB_DEBUG
+        return atomic_long_read(&n->total_objects);
+#else
+        return 0;
+#endif
+}
+
+static noinline void
+slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
+{
+        int node;
+
+        printk(KERN_WARNING
+                "SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n",
+                nid, gfpflags);
+        printk(KERN_WARNING "  cache: %s, object size: %d, buffer size: %d, "
+                "default order: %d, min order: %d\n", s->name, s->objsize,
+                s->size, oo_order(s->oo), oo_order(s->min));
+
+        if (oo_order(s->min) > get_order(s->objsize))
+                printk(KERN_WARNING "  %s debugging increased min order, use "
+                       "slub_debug=O to disable.\n", s->name);
+
+        for_each_online_node(node) {
+                struct kmem_cache_node *n = get_node(s, node);
+                unsigned long nr_slabs;
+                unsigned long nr_objs;
+                unsigned long nr_free;
+
+                if (!n)
+                        continue;
+
+                nr_free  = count_partial(n, count_free);
+                nr_slabs = node_nr_slabs(n);
+                nr_objs  = node_nr_objs(n);
+
+                printk(KERN_WARNING
+                        "  node %d: slabs: %ld, objs: %ld, free: %ld\n",
+                        node, nr_slabs, nr_objs, nr_free);
+        }
+}
+
 /*
  * Slow path. The lockless freelist is empty or we need to perform
  * debugging duties.
@@ -1561,6 +1682,8 @@ new_slab:
                 c->page = new;
                 goto load_freelist;
         }
+        if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
+                slab_out_of_memory(s, gfpflags, node);
         return NULL;
 debug:
         if (!alloc_debug_processing(s, c->page, object, addr))
@@ -1590,6 +1713,8 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
         unsigned long flags;
         unsigned int objsize;
 
+        gfpflags &= gfp_allowed_mask;
+
         lockdep_trace_alloc(gfpflags);
         might_sleep_if(gfpflags & __GFP_WAIT);
 
@@ -1613,23 +1738,53 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
         if (unlikely((gfpflags & __GFP_ZERO) && object))
                 memset(object, 0, objsize);
 
+        kmemcheck_slab_alloc(s, gfpflags, object, c->objsize);
+        kmemleak_alloc_recursive(object, objsize, 1, s->flags, gfpflags);
+
         return object;
 }
 
 void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
 {
-        return slab_alloc(s, gfpflags, -1, _RET_IP_);
+        void *ret = slab_alloc(s, gfpflags, -1, _RET_IP_);
+
+        trace_kmem_cache_alloc(_RET_IP_, ret, s->objsize, s->size, gfpflags);
+
+        return ret;
 }
 EXPORT_SYMBOL(kmem_cache_alloc);
 
+#ifdef CONFIG_KMEMTRACE
+void *kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags)
+{
+        return slab_alloc(s, gfpflags, -1, _RET_IP_);
+}
+EXPORT_SYMBOL(kmem_cache_alloc_notrace);
+#endif
+
 #ifdef CONFIG_NUMA
 void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
 {
-        return slab_alloc(s, gfpflags, node, _RET_IP_);
+        void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);
+
+        trace_kmem_cache_alloc_node(_RET_IP_, ret,
+                                    s->objsize, s->size, gfpflags, node);
+
+        return ret;
 }
 EXPORT_SYMBOL(kmem_cache_alloc_node);
 #endif
 
+#ifdef CONFIG_KMEMTRACE
+void *kmem_cache_alloc_node_notrace(struct kmem_cache *s,
+                                    gfp_t gfpflags,
+                                    int node)
+{
+        return slab_alloc(s, gfpflags, node, _RET_IP_);
+}
+EXPORT_SYMBOL(kmem_cache_alloc_node_notrace);
+#endif
+
 /*
  * Slow patch handling. This may still be called frequently since objects
  * have a longer lifetime than the cpu slabs in most processing loads.
@@ -1715,8 +1870,10 @@ static __always_inline void slab_free(struct kmem_cache *s,
         struct kmem_cache_cpu *c;
         unsigned long flags;
 
+        kmemleak_free_recursive(x, s->flags);
         local_irq_save(flags);
         c = get_cpu_slab(s, smp_processor_id());
+        kmemcheck_slab_free(s, object, c->objsize);
         debug_check_no_locks_freed(object, c->objsize);
         if (!(s->flags & SLAB_DEBUG_OBJECTS))
                 debug_check_no_obj_freed(object, c->objsize);
@@ -1737,6 +1894,8 @@ void kmem_cache_free(struct kmem_cache *s, void *x)
         page = virt_to_head_page(x);
 
         slab_free(s, page, x, _RET_IP_);
+
+        trace_kmem_cache_free(_RET_IP_, x);
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
@@ -1864,7 +2023,7 @@ static inline int calculate_order(int size)
                                 return order;
                         fraction /= 2;
                 }
-                min_objects --;
+                min_objects--;
         }
 
         /*
@@ -1879,7 +2038,7 @@ static inline int calculate_order(int size)
          * Doh this slab cannot be placed using slub_max_order.
          */
         order = slab_order(size, 1, MAX_ORDER, 1);
-        if (order <= MAX_ORDER)
+        if (order < MAX_ORDER)
                 return order;
         return -ENOSYS;
 }
@@ -1954,8 +2113,8 @@ init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s)
  */
 #define NR_KMEM_CACHE_CPU 100
 
-static DEFINE_PER_CPU(struct kmem_cache_cpu,
-                                kmem_cache_cpu)[NR_KMEM_CACHE_CPU];
+static DEFINE_PER_CPU(struct kmem_cache_cpu [NR_KMEM_CACHE_CPU],
+                      kmem_cache_cpu);
 
 static DEFINE_PER_CPU(struct kmem_cache_cpu *, kmem_cache_cpu_free);
 static DECLARE_BITMAP(kmem_cach_cpu_free_init_once, CONFIG_NR_CPUS);
@@ -2263,6 +2422,7 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
          * on bootup.
          */
         align = calculate_alignment(flags, align, s->objsize);
+        s->align = align;
 
         /*
          * SLUB stores one object immediately after another beginning from
@@ -2315,6 +2475,18 @@ static int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags,
 
         if (!calculate_sizes(s, -1))
                 goto error;
+        if (disable_higher_order_debug) {
+                /*
+                 * Disable debugging flags that store metadata if the min slab
+                 * order increased.
+                 */
+                if (get_order(s->size) > get_order(s->objsize)) {
+                        s->flags &= ~DEBUG_METADATA_FLAGS;
+                        s->offset = 0;
+                        if (!calculate_sizes(s, -1))
+                                goto error;
+                }
+        }
 
         /*
          * The larger the object size is, the more pages we want on the partial
@@ -2467,6 +2639,8 @@ void kmem_cache_destroy(struct kmem_cache *s)
                                 "still has objects.\n", s->name, __func__);
                         dump_stack();
                 }
+                if (s->flags & SLAB_DESTROY_BY_RCU)
+                        rcu_barrier();
                 sysfs_slab_remove(s);
         } else
                 up_write(&slub_lock);
@@ -2492,6 +2666,7 @@ __setup("slub_min_order=", setup_slub_min_order);
 static int __init setup_slub_max_order(char *str)
 {
         get_option(&str, &slub_max_order);
+        slub_max_order = min(slub_max_order, MAX_ORDER - 1);
 
         return 1;
 }
@@ -2523,13 +2698,16 @@ static struct kmem_cache *create_kmalloc_cache(struct kmem_cache *s,
         if (gfp_flags & SLUB_DMA)
                 flags = SLAB_CACHE_DMA;
 
-        down_write(&slub_lock);
+        /*
+         * This function is called with IRQs disabled during early-boot on
+         * single CPU so there's no need to take slub_lock here.
+         */
         if (!kmem_cache_open(s, gfp_flags, name, size, ARCH_KMALLOC_MINALIGN,
                                                                 flags, NULL))
                 goto panic;
 
         list_add(&s->list, &slab_caches);
-        up_write(&slub_lock);
+
         if (sysfs_slab_add(s))
                 goto panic;
         return s;
@@ -2562,6 +2740,7 @@ static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags)
         struct kmem_cache *s;
         char *text;
         size_t realsize;
+        unsigned long slabflags;
 
         s = kmalloc_caches_dma[index];
         if (s)
@@ -2583,9 +2762,18 @@ static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags)
                          (unsigned int)realsize);
         s = kmalloc(kmem_size, flags & ~SLUB_DMA);
 
+        /*
+         * Must defer sysfs creation to a workqueue because we don't know
+         * what context we are called from. Before sysfs comes up, we don't
+         * need to do anything because our sysfs initcall will start by
+         * adding all existing slabs to sysfs.
+         */
+        slabflags = SLAB_CACHE_DMA|SLAB_NOTRACK;
+        if (slab_state >= SYSFS)
+                slabflags |= __SYSFS_ADD_DEFERRED;
+
         if (!s || !text || !kmem_cache_open(s, flags, text,
-                        realsize, ARCH_KMALLOC_MINALIGN,
-                        SLAB_CACHE_DMA|__SYSFS_ADD_DEFERRED, NULL)) {
+                        realsize, ARCH_KMALLOC_MINALIGN, slabflags, NULL)) {
                 kfree(s);
                 kfree(text);
                 goto unlock_out;
@@ -2594,7 +2782,8 @@ static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags)
         list_add(&s->list, &slab_caches);
         kmalloc_caches_dma[index] = s;
 
-        schedule_work(&sysfs_add_work);
+        if (slab_state >= SYSFS)
+                schedule_work(&sysfs_add_work);
 
 unlock_out:
         up_write(&slub_lock);
@@ -2636,6 +2825,11 @@ static s8 size_index[24] = {
         2       /* 192 */
 };
 
+static inline int size_index_elem(size_t bytes)
+{
+        return (bytes - 1) / 8;
+}
+
 static struct kmem_cache *get_slab(size_t size, gfp_t flags)
 {
         int index;
@@ -2644,7 +2838,7 @@ static struct kmem_cache *get_slab(size_t size, gfp_t flags)
                 if (!size)
                         return ZERO_SIZE_PTR;
 
-                index = size_index[(size - 1) / 8];
+                index = size_index[size_index_elem(size)];
         } else
                 index = fls(size - 1);
 
@@ -2659,6 +2853,7 @@ static struct kmem_cache *get_slab(size_t size, gfp_t flags)
 void *__kmalloc(size_t size, gfp_t flags)
 {
         struct kmem_cache *s;
+        void *ret;
 
         if (unlikely(size > SLUB_MAX_SIZE))
                 return kmalloc_large(size, flags);
@@ -2668,35 +2863,54 @@ void *__kmalloc(size_t size, gfp_t flags)
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
 
-        return slab_alloc(s, flags, -1, _RET_IP_);
+        ret = slab_alloc(s, flags, -1, _RET_IP_);
+
+        trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
+
+        return ret;
 }
 EXPORT_SYMBOL(__kmalloc);
 
 static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
 {
-        struct page *page = alloc_pages_node(node, flags | __GFP_COMP,
-                                                get_order(size));
+        struct page *page;
+        void *ptr = NULL;
 
+        flags |= __GFP_COMP | __GFP_NOTRACK;
+        page = alloc_pages_node(node, flags, get_order(size));
         if (page)
-                return page_address(page);
-        else
-                return NULL;
+                ptr = page_address(page);
+
+        kmemleak_alloc(ptr, size, 1, flags);
+        return ptr;
 }
 
 #ifdef CONFIG_NUMA
 void *__kmalloc_node(size_t size, gfp_t flags, int node)
 {
         struct kmem_cache *s;
+        void *ret;
 
-        if (unlikely(size > SLUB_MAX_SIZE))
-                return kmalloc_large_node(size, flags, node);
+        if (unlikely(size > SLUB_MAX_SIZE)) {
+                ret = kmalloc_large_node(size, flags, node);
+
+                trace_kmalloc_node(_RET_IP_, ret,
+                                   size, PAGE_SIZE << get_order(size),
+                                   flags, node);
+
+                return ret;
+        }
 
         s = get_slab(size, flags);
 
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
 
-        return slab_alloc(s, flags, node, _RET_IP_);
+        ret = slab_alloc(s, flags, node, _RET_IP_);
+
+        trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
+
+        return ret;
 }
 EXPORT_SYMBOL(__kmalloc_node);
 #endif
@@ -2745,12 +2959,15 @@ void kfree(const void *x)
         struct page *page;
         void *object = (void *)x;
 
+        trace_kfree(_RET_IP_, x);
+
         if (unlikely(ZERO_OR_NULL_PTR(x)))
                 return;
 
         page = virt_to_head_page(x);
         if (unlikely(!PageSlab(page))) {
                 BUG_ON(!PageCompound(page));
+                kmemleak_free(x);
                 put_page(page);
                 return;
         }
@@ -2968,7 +3185,7 @@ void __init kmem_cache_init(void)
          * kmem_cache_open for slab_state == DOWN.
          */
         create_kmalloc_cache(&kmalloc_caches[0], "kmem_cache_node",
-                sizeof(struct kmem_cache_node), GFP_KERNEL);
+                sizeof(struct kmem_cache_node), GFP_NOWAIT);
         kmalloc_caches[0].refcount = -1;
         caches++;
 
@@ -2979,18 +3196,20 @@ void __init kmem_cache_init(void)
         slab_state = PARTIAL;
 
         /* Caches that are not of the two-to-the-power-of size */
-        if (KMALLOC_MIN_SIZE <= 64) {
+        if (KMALLOC_MIN_SIZE <= 32) {
                 create_kmalloc_cache(&kmalloc_caches[1],
-                                "kmalloc-96", 96, GFP_KERNEL);
+                                "kmalloc-96", 96, GFP_NOWAIT);
                 caches++;
+        }
+        if (KMALLOC_MIN_SIZE <= 64) {
                 create_kmalloc_cache(&kmalloc_caches[2],
-                                "kmalloc-192", 192, GFP_KERNEL);
+                                "kmalloc-192", 192, GFP_NOWAIT);
                 caches++;
         }
 
         for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
                 create_kmalloc_cache(&kmalloc_caches[i],
-                        "kmalloc", 1 << i, GFP_KERNEL);
+                        "kmalloc", 1 << i, GFP_NOWAIT);
                 caches++;
         }
 
@@ -3009,17 +3228,28 @@ void __init kmem_cache_init(void)
         BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
                 (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
 
-        for (i = 8; i < KMALLOC_MIN_SIZE; i += 8)
-                size_index[(i - 1) / 8] = KMALLOC_SHIFT_LOW;
+        for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
+                int elem = size_index_elem(i);
+                if (elem >= ARRAY_SIZE(size_index))
+                        break;
+                size_index[elem] = KMALLOC_SHIFT_LOW;
+        }
 
-        if (KMALLOC_MIN_SIZE == 128) {
+        if (KMALLOC_MIN_SIZE == 64) {
+                /*
+                 * The 96 byte size cache is not used if the alignment
+                 * is 64 byte.
+                 */
+                for (i = 64 + 8; i <= 96; i += 8)
+                        size_index[size_index_elem(i)] = 7;
+        } else if (KMALLOC_MIN_SIZE == 128) {
                 /*
                  * The 192 byte sized cache is not used if the alignment
                  * is 128 byte. Redirect kmalloc to use the 256 byte cache
                  * instead.
                  */
                 for (i = 128 + 8; i <= 192; i += 8)
-                        size_index[(i - 1) / 8] = 8;
+                        size_index[size_index_elem(i)] = 8;
         }
 
         slab_state = UP;
@@ -3027,7 +3257,7 @@ void __init kmem_cache_init(void)
         /* Provide the correct kmalloc names now that the caches are up */
         for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++)
                 kmalloc_caches[i]. name =
-                        kasprintf(GFP_KERNEL, "kmalloc-%d", 1 << i);
+                        kasprintf(GFP_NOWAIT, "kmalloc-%d", 1 << i);
 
 #ifdef CONFIG_SMP
         register_cpu_notifier(&slab_notifier);
@@ -3045,6 +3275,10 @@ void __init kmem_cache_init(void)
                 nr_cpu_ids, nr_node_ids);
 }
 
+void __init kmem_cache_init_late(void)
+{
+}
+
 /*
  * Find a mergeable slab cache
  */
@@ -3111,6 +3345,9 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size,
 {
         struct kmem_cache *s;
 
+        if (WARN_ON(!name))
+                return NULL;
+
         down_write(&slub_lock);
         s = find_mergeable(size, align, flags, name, ctor);
         if (s) {
@@ -3224,6 +3461,7 @@ static struct notifier_block __cpuinitdata slab_notifier = {
 void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
 {
         struct kmem_cache *s;
+        void *ret;
 
         if (unlikely(size > SLUB_MAX_SIZE))
                 return kmalloc_large(size, gfpflags);
@@ -3233,13 +3471,19 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
 
-        return slab_alloc(s, gfpflags, -1, caller);
+        ret = slab_alloc(s, gfpflags, -1, caller);
+
+        /* Honor the call site pointer we recieved. */
+        trace_kmalloc(caller, ret, size, s->size, gfpflags);
+
+        return ret;
 }
 
 void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
                                         int node, unsigned long caller)
 {
         struct kmem_cache *s;
+        void *ret;
 
         if (unlikely(size > SLUB_MAX_SIZE))
                 return kmalloc_large_node(size, gfpflags, node);
@@ -3249,24 +3493,15 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
 
-        return slab_alloc(s, gfpflags, node, caller);
-}
+        ret = slab_alloc(s, gfpflags, node, caller);
 
-#ifdef CONFIG_SLUB_DEBUG
-static unsigned long count_partial(struct kmem_cache_node *n,
-                                        int (*get_count)(struct page *))
-{
-        unsigned long flags;
-        unsigned long x = 0;
-        struct page *page;
+        /* Honor the call site pointer we recieved. */
+        trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
 
-        spin_lock_irqsave(&n->list_lock, flags);
-        list_for_each_entry(page, &n->partial, lru)
-                x += get_count(page);
-        spin_unlock_irqrestore(&n->list_lock, flags);
-        return x;
+        return ret;
 }
 
+#ifdef CONFIG_SLUB_DEBUG
 static int count_inuse(struct page *page)
 {
         return page->inuse;
@@ -3277,11 +3512,6 @@ static int count_total(struct page *page)
         return page->objects;
 }
 
-static int count_free(struct page *page)
-{
-        return page->objects - page->inuse;
-}
-
 static int validate_slab(struct kmem_cache *s, struct page *page,
                                                 unsigned long *map)
 {
@@ -3650,7 +3880,7 @@ static int list_locations(struct kmem_cache *s, char *buf,
                                                  to_cpumask(l->cpus));
                 }
 
-                if (num_online_nodes() > 1 && !nodes_empty(l->nodes) &&
+                if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
                                 len < PAGE_SIZE - 60) {
                         len += sprintf(buf + len, " nodes=");
                         len += nodelist_scnprintf(buf + len, PAGE_SIZE - len - 50,
@@ -4325,6 +4555,8 @@ static char *create_unique_id(struct kmem_cache *s)
                 *p++ = 'a';
         if (s->flags & SLAB_DEBUG_FREE)
                 *p++ = 'F';
+        if (!(s->flags & SLAB_NOTRACK))
+                *p++ = 't';
         if (p != name + 1)
                 *p++ = '-';
         p += sprintf(p, "%07d", s->size);
@@ -4367,8 +4599,11 @@ static int sysfs_slab_add(struct kmem_cache *s)
         }
 
         err = sysfs_create_group(&s->kobj, &slab_attr_group);
-        if (err)
+        if (err) {
+                kobject_del(&s->kobj);
+                kobject_put(&s->kobj);
                 return err;
+        }
         kobject_uevent(&s->kobj, KOBJ_ADD);
         if (!unmergeable) {
                 /* Setup first alias */
@@ -4550,7 +4785,7 @@ static const struct file_operations proc_slabinfo_operations = {
 
 static int __init slab_proc_init(void)
 {
-        proc_create("slabinfo",S_IWUSR|S_IRUGO,NULL,&proc_slabinfo_operations);
+        proc_create("slabinfo", S_IRUGO, NULL, &proc_slabinfo_operations);
         return 0;
 }
 module_init(slab_proc_init);
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index a13ea6401ae7..d9714bdcb4a3 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -48,8 +48,14 @@ void * __meminit vmemmap_alloc_block(unsigned long size, int node)
 {
         /* If the main allocator is up use that, fallback to bootmem. */
         if (slab_is_available()) {
-                struct page *page = alloc_pages_node(node,
+                struct page *page;
+
+                if (node_state(node, N_HIGH_MEMORY))
+                        page = alloc_pages_node(node,
                                 GFP_KERNEL | __GFP_ZERO, get_order(size));
+                else
+                        page = alloc_pages(GFP_KERNEL | __GFP_ZERO,
+                                get_order(size));
                 if (page)
                         return page_address(page);
                 return NULL;
diff --git a/mm/sparse.c b/mm/sparse.c
index da432d9f0ae8..6ce4aab69e99 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -62,9 +62,12 @@ static struct mem_section noinline __init_refok *sparse_index_alloc(int nid)
         unsigned long array_size = SECTIONS_PER_ROOT *
                                    sizeof(struct mem_section);
 
-        if (slab_is_available())
-                section = kmalloc_node(array_size, GFP_KERNEL, nid);
-        else
+        if (slab_is_available()) {
+                if (node_state(nid, N_HIGH_MEMORY))
+                        section = kmalloc_node(array_size, GFP_KERNEL, nid);
+                else
+                        section = kmalloc(array_size, GFP_KERNEL);
+        } else
                 section = alloc_bootmem_node(NODE_DATA(nid), array_size);
 
         if (section)
diff --git a/mm/swap.c b/mm/swap.c
index 6e83084c1f6c..308e57d8d7ed 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -118,7 +118,7 @@ static void pagevec_move_tail(struct pagevec *pvec)
                         spin_lock(&zone->lru_lock);
                 }
                 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
-                        int lru = page_is_file_cache(page);
+                        int lru = page_lru_base_type(page);
                         list_move_tail(&page->lru, &zone->lru[lru].list);
                         pgmoved++;
                 }
@@ -181,7 +181,7 @@ void activate_page(struct page *page)
         spin_lock_irq(&zone->lru_lock);
         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
                 int file = page_is_file_cache(page);
-                int lru = LRU_BASE + file;
+                int lru = page_lru_base_type(page);
                 del_page_from_lru_list(zone, page, lru);
 
                 SetPageActive(page);
@@ -189,7 +189,7 @@ void activate_page(struct page *page)
                 add_page_to_lru_list(zone, page, lru);
                 __count_vm_event(PGACTIVATE);
 
-                update_page_reclaim_stat(zone, page, !!file, 1);
+                update_page_reclaim_stat(zone, page, file, 1);
         }
         spin_unlock_irq(&zone->lru_lock);
 }
@@ -448,8 +448,8 @@ void pagevec_strip(struct pagevec *pvec)
         for (i = 0; i < pagevec_count(pvec); i++) {
                 struct page *page = pvec->pages[i];
 
-                if (PagePrivate(page) && trylock_page(page)) {
-                        if (PagePrivate(page))
+                if (page_has_private(page) && trylock_page(page)) {
+                        if (page_has_private(page))
                                 try_to_release_page(page, 0);
                         unlock_page(page);
                 }
@@ -491,55 +491,12 @@ unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping,
 
 EXPORT_SYMBOL(pagevec_lookup_tag);
 
-#ifdef CONFIG_SMP
-/*
- * We tolerate a little inaccuracy to avoid ping-ponging the counter between
- * CPUs
- */
-#define ACCT_THRESHOLD  max(16, NR_CPUS * 2)
-
-static DEFINE_PER_CPU(long, committed_space);
-
-void vm_acct_memory(long pages)
-{
-        long *local;
-
-        preempt_disable();
-        local = &__get_cpu_var(committed_space);
-        *local += pages;
-        if (*local > ACCT_THRESHOLD || *local < -ACCT_THRESHOLD) {
-                atomic_long_add(*local, &vm_committed_space);
-                *local = 0;
-        }
-        preempt_enable();
-}
-
-#ifdef CONFIG_HOTPLUG_CPU
-
-/* Drop the CPU's cached committed space back into the central pool. */
-static int cpu_swap_callback(struct notifier_block *nfb,
-                             unsigned long action,
-                             void *hcpu)
-{
-        long *committed;
-
-        committed = &per_cpu(committed_space, (long)hcpu);
-        if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
-                atomic_long_add(*committed, &vm_committed_space);
-                *committed = 0;
-                drain_cpu_pagevecs((long)hcpu);
-        }
-        return NOTIFY_OK;
-}
-#endif /* CONFIG_HOTPLUG_CPU */
-#endif /* CONFIG_SMP */
-
 /*
  * Perform any setup for the swap system
  */
 void __init swap_setup(void)
 {
-        unsigned long megs = num_physpages >> (20 - PAGE_SHIFT);
+        unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT);
 
 #ifdef CONFIG_SWAP
         bdi_init(swapper_space.backing_dev_info);
@@ -554,7 +511,4 @@ void __init swap_setup(void)
          * Right now other parts of the system means that we
          * _really_ don't want to cluster much more
          */
-#ifdef CONFIG_HOTPLUG_CPU
-        hotcpu_notifier(cpu_swap_callback, 0);
-#endif
 }
diff --git a/mm/swap_state.c b/mm/swap_state.c
index 3ecea98ecb45..6d1daeb1cb4a 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -34,6 +34,7 @@ static const struct address_space_operations swap_aops = {
 };
 
 static struct backing_dev_info swap_backing_dev_info = {
+        .name           = "swap",
         .capabilities   = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED,
         .unplug_io_fn   = swap_unplug_io_fn,
 };
@@ -66,10 +67,10 @@ void show_swap_cache_info(void)
 }
 
 /*
- * add_to_swap_cache resembles add_to_page_cache_locked on swapper_space,
+ * __add_to_swap_cache resembles add_to_page_cache_locked on swapper_space,
  * but sets SwapCache flag and private instead of mapping and index.
  */
-int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp_mask)
+static int __add_to_swap_cache(struct page *page, swp_entry_t entry)
 {
         int error;
 
@@ -77,28 +78,43 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp_mask)
         VM_BUG_ON(PageSwapCache(page));
         VM_BUG_ON(!PageSwapBacked(page));
 
+        page_cache_get(page);
+        SetPageSwapCache(page);
+        set_page_private(page, entry.val);
+
+        spin_lock_irq(&swapper_space.tree_lock);
+        error = radix_tree_insert(&swapper_space.page_tree, entry.val, page);
+        if (likely(!error)) {
+                total_swapcache_pages++;
+                __inc_zone_page_state(page, NR_FILE_PAGES);
+                INC_CACHE_INFO(add_total);
+        }
+        spin_unlock_irq(&swapper_space.tree_lock);
+
+        if (unlikely(error)) {
+                /*
+                 * Only the context which have set SWAP_HAS_CACHE flag
+                 * would call add_to_swap_cache().
+                 * So add_to_swap_cache() doesn't returns -EEXIST.
+                 */
+                VM_BUG_ON(error == -EEXIST);
+                set_page_private(page, 0UL);
+                ClearPageSwapCache(page);
+                page_cache_release(page);
+        }
+
+        return error;
+}
+
+
+int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp_mask)
+{
+        int error;
+
         error = radix_tree_preload(gfp_mask);
         if (!error) {
-                page_cache_get(page);
-                SetPageSwapCache(page);
-                set_page_private(page, entry.val);
-
-                spin_lock_irq(&swapper_space.tree_lock);
-                error = radix_tree_insert(&swapper_space.page_tree,
-                                                entry.val, page);
-                if (likely(!error)) {
-                        total_swapcache_pages++;
-                        __inc_zone_page_state(page, NR_FILE_PAGES);
-                        INC_CACHE_INFO(add_total);
-                }
-                spin_unlock_irq(&swapper_space.tree_lock);
+                error = __add_to_swap_cache(page, entry);
                 radix_tree_preload_end();
-
-                if (unlikely(error)) {
-                        set_page_private(page, 0UL);
-                        ClearPageSwapCache(page);
-                        page_cache_release(page);
-                }
         }
         return error;
 }
@@ -109,8 +125,6 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp_mask)
  */
 void __delete_from_swap_cache(struct page *page)
 {
-        swp_entry_t ent = {.val = page_private(page)};
-
         VM_BUG_ON(!PageLocked(page));
         VM_BUG_ON(!PageSwapCache(page));
         VM_BUG_ON(PageWriteback(page));
@@ -121,13 +135,11 @@ void __delete_from_swap_cache(struct page *page)
         total_swapcache_pages--;
         __dec_zone_page_state(page, NR_FILE_PAGES);
         INC_CACHE_INFO(del_total);
-        mem_cgroup_uncharge_swapcache(page, ent);
 }
 
 /**
  * add_to_swap - allocate swap space for a page
  * @page: page we want to move to swap
- * @gfp_mask: memory allocation flags
  *
  * Allocate swap space for the page and add the page to the
  * swap cache.  Caller needs to hold the page lock. 
@@ -140,38 +152,34 @@ int add_to_swap(struct page *page)
         VM_BUG_ON(!PageLocked(page));
         VM_BUG_ON(!PageUptodate(page));
 
-        for (;;) {
-                entry = get_swap_page();
-                if (!entry.val)
-                        return 0;
+        entry = get_swap_page();
+        if (!entry.val)
+                return 0;
 
+        /*
+         * Radix-tree node allocations from PF_MEMALLOC contexts could
+         * completely exhaust the page allocator. __GFP_NOMEMALLOC
+         * stops emergency reserves from being allocated.
+         *
+         * TODO: this could cause a theoretical memory reclaim
+         * deadlock in the swap out path.
+         */
+        /*
+         * Add it to the swap cache and mark it dirty
+         */
+        err = add_to_swap_cache(page, entry,
+                        __GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN);
+
+        if (!err) {     /* Success */
+                SetPageDirty(page);
+                return 1;
+        } else {        /* -ENOMEM radix-tree allocation failure */
                 /*
-                 * Radix-tree node allocations from PF_MEMALLOC contexts could
-                 * completely exhaust the page allocator. __GFP_NOMEMALLOC
-                 * stops emergency reserves from being allocated.
-                 *
-                 * TODO: this could cause a theoretical memory reclaim
-                 * deadlock in the swap out path.
-                 */
-                /*
-                 * Add it to the swap cache and mark it dirty
+                 * add_to_swap_cache() doesn't return -EEXIST, so we can safely
+                 * clear SWAP_HAS_CACHE flag.
                  */
-                err = add_to_swap_cache(page, entry,
-                                __GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN);
-
-                switch (err) {
-                case 0:                         /* Success */
-                        SetPageDirty(page);
-                        return 1;
-                case -EEXIST:
-                        /* Raced with "speculative" read_swap_cache_async */
-                        swap_free(entry);
-                        continue;
-                default:
-                        /* -ENOMEM radix-tree allocation failure */
-                        swap_free(entry);
-                        return 0;
-                }
+                swapcache_free(entry, NULL);
+                return 0;
         }
 }
 
@@ -191,7 +199,7 @@ void delete_from_swap_cache(struct page *page)
         __delete_from_swap_cache(page);
         spin_unlock_irq(&swapper_space.tree_lock);
 
-        swap_free(entry);
+        swapcache_free(entry, page);
         page_cache_release(page);
 }
 
@@ -293,33 +301,46 @@ struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
                 }
 
                 /*
-                 * Swap entry may have been freed since our caller observed it.
+                 * call radix_tree_preload() while we can wait.
                  */
-                if (!swap_duplicate(entry))
+                err = radix_tree_preload(gfp_mask & GFP_KERNEL);
+                if (err)
                         break;
 
                 /*
-                 * Associate the page with swap entry in the swap cache.
-                 * May fail (-EEXIST) if there is already a page associated
-                 * with this entry in the swap cache: added by a racing
-                 * read_swap_cache_async, or add_to_swap or shmem_writepage
-                 * re-using the just freed swap entry for an existing page.
-                 * May fail (-ENOMEM) if radix-tree node allocation failed.
+                 * Swap entry may have been freed since our caller observed it.
                  */
+                err = swapcache_prepare(entry);
+                if (err == -EEXIST) {   /* seems racy */
+                        radix_tree_preload_end();
+                        continue;
+                }
+                if (err) {              /* swp entry is obsolete ? */
+                        radix_tree_preload_end();
+                        break;
+                }
+
+                /* May fail (-ENOMEM) if radix-tree node allocation failed. */
                 __set_page_locked(new_page);
                 SetPageSwapBacked(new_page);
-                err = add_to_swap_cache(new_page, entry, gfp_mask & GFP_KERNEL);
+                err = __add_to_swap_cache(new_page, entry);
                 if (likely(!err)) {
+                        radix_tree_preload_end();
                         /*
                          * Initiate read into locked page and return.
                          */
                         lru_cache_add_anon(new_page);
-                        swap_readpage(NULL, new_page);
+                        swap_readpage(new_page);
                         return new_page;
                 }
+                radix_tree_preload_end();
                 ClearPageSwapBacked(new_page);
                 __clear_page_locked(new_page);
-                swap_free(entry);
+                /*
+                 * add_to_swap_cache() doesn't return -EEXIST, so we can safely
+                 * clear SWAP_HAS_CACHE flag.
+                 */
+                swapcache_free(entry, NULL);
         } while (err != -ENOMEM);
 
         if (new_page)
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 312fafe0ab6e..9c590eef7912 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -53,6 +53,59 @@ static struct swap_info_struct swap_info[MAX_SWAPFILES];
 
 static DEFINE_MUTEX(swapon_mutex);
 
+/* For reference count accounting in swap_map */
+/* enum for swap_map[] handling. internal use only */
+enum {
+        SWAP_MAP = 0,   /* ops for reference from swap users */
+        SWAP_CACHE,     /* ops for reference from swap cache */
+};
+
+static inline int swap_count(unsigned short ent)
+{
+        return ent & SWAP_COUNT_MASK;
+}
+
+static inline bool swap_has_cache(unsigned short ent)
+{
+        return !!(ent & SWAP_HAS_CACHE);
+}
+
+static inline unsigned short encode_swapmap(int count, bool has_cache)
+{
+        unsigned short ret = count;
+
+        if (has_cache)
+                return SWAP_HAS_CACHE | ret;
+        return ret;
+}
+
+/* returnes 1 if swap entry is freed */
+static int
+__try_to_reclaim_swap(struct swap_info_struct *si, unsigned long offset)
+{
+        int type = si - swap_info;
+        swp_entry_t entry = swp_entry(type, offset);
+        struct page *page;
+        int ret = 0;
+
+        page = find_get_page(&swapper_space, entry.val);
+        if (!page)
+                return 0;
+        /*
+         * This function is called from scan_swap_map() and it's called
+         * by vmscan.c at reclaiming pages. So, we hold a lock on a page, here.
+         * We have to use trylock for avoiding deadlock. This is a special
+         * case and you should use try_to_free_swap() with explicit lock_page()
+         * in usual operations.
+         */
+        if (trylock_page(page)) {
+                ret = try_to_free_swap(page);
+                unlock_page(page);
+        }
+        page_cache_release(page);
+        return ret;
+}
+
 /*
  * We need this because the bdev->unplug_fn can sleep and we cannot
  * hold swap_lock while calling the unplug_fn. And swap_lock
@@ -108,7 +161,8 @@ static int discard_swap(struct swap_info_struct *si)
                 }
 
                 err = blkdev_issue_discard(si->bdev, start_block,
-                                                nr_blocks, GFP_KERNEL);
+                                                nr_blocks, GFP_KERNEL,
+                                                DISCARD_FL_BARRIER);
                 if (err)
                         break;
 
@@ -147,7 +201,8 @@ static void discard_swap_cluster(struct swap_info_struct *si,
                         start_block <<= PAGE_SHIFT - 9;
                         nr_blocks <<= PAGE_SHIFT - 9;
                         if (blkdev_issue_discard(si->bdev, start_block,
-                                                        nr_blocks, GFP_NOIO))
+                                                        nr_blocks, GFP_NOIO,
+                                                        DISCARD_FL_BARRIER))
                                 break;
                 }
 
@@ -167,7 +222,8 @@ static int wait_for_discard(void *word)
 #define SWAPFILE_CLUSTER        256
 #define LATENCY_LIMIT           256
 
-static inline unsigned long scan_swap_map(struct swap_info_struct *si)
+static inline unsigned long scan_swap_map(struct swap_info_struct *si,
+                                          int cache)
 {
         unsigned long offset;
         unsigned long scan_base;
@@ -273,6 +329,19 @@ checks:
                 goto no_page;
         if (offset > si->highest_bit)
                 scan_base = offset = si->lowest_bit;
+
+        /* reuse swap entry of cache-only swap if not busy. */
+        if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
+                int swap_was_freed;
+                spin_unlock(&swap_lock);
+                swap_was_freed = __try_to_reclaim_swap(si, offset);
+                spin_lock(&swap_lock);
+                /* entry was freed successfully, try to use this again */
+                if (swap_was_freed)
+                        goto checks;
+                goto scan; /* check next one */
+        }
+
         if (si->swap_map[offset])
                 goto scan;
 
@@ -285,7 +354,10 @@ checks:
                 si->lowest_bit = si->max;
                 si->highest_bit = 0;
         }
-        si->swap_map[offset] = 1;
+        if (cache == SWAP_CACHE) /* at usual swap-out via vmscan.c */
+                si->swap_map[offset] = encode_swapmap(0, true);
+        else /* at suspend */
+                si->swap_map[offset] = encode_swapmap(1, false);
         si->cluster_next = offset + 1;
         si->flags -= SWP_SCANNING;
 
@@ -351,6 +423,10 @@ scan:
                         spin_lock(&swap_lock);
                         goto checks;
                 }
+                if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
+                        spin_lock(&swap_lock);
+                        goto checks;
+                }
                 if (unlikely(--latency_ration < 0)) {
                         cond_resched();
                         latency_ration = LATENCY_LIMIT;
@@ -362,6 +438,10 @@ scan:
                         spin_lock(&swap_lock);
                         goto checks;
                 }
+                if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
+                        spin_lock(&swap_lock);
+                        goto checks;
+                }
                 if (unlikely(--latency_ration < 0)) {
                         cond_resched();
                         latency_ration = LATENCY_LIMIT;
@@ -401,7 +481,8 @@ swp_entry_t get_swap_page(void)
                         continue;
 
                 swap_list.next = next;
-                offset = scan_swap_map(si);
+                /* This is called for allocating swap entry for cache */
+                offset = scan_swap_map(si, SWAP_CACHE);
                 if (offset) {
                         spin_unlock(&swap_lock);
                         return swp_entry(type, offset);
@@ -415,6 +496,7 @@ noswap:
         return (swp_entry_t) {0};
 }
 
+/* The only caller of this function is now susupend routine */
 swp_entry_t get_swap_page_of_type(int type)
 {
         struct swap_info_struct *si;
@@ -424,7 +506,8 @@ swp_entry_t get_swap_page_of_type(int type)
         si = swap_info + type;
         if (si->flags & SWP_WRITEOK) {
                 nr_swap_pages--;
-                offset = scan_swap_map(si);
+                /* This is called for allocating swap entry, not cache */
+                offset = scan_swap_map(si, SWAP_MAP);
                 if (offset) {
                         spin_unlock(&swap_lock);
                         return swp_entry(type, offset);
@@ -471,26 +554,40 @@ out:
         return NULL;
 }
 
-static int swap_entry_free(struct swap_info_struct *p, swp_entry_t ent)
+static int swap_entry_free(struct swap_info_struct *p,
+                           swp_entry_t ent, int cache)
 {
         unsigned long offset = swp_offset(ent);
-        int count = p->swap_map[offset];
-
-        if (count < SWAP_MAP_MAX) {
-                count--;
-                p->swap_map[offset] = count;
-                if (!count) {
-                        if (offset < p->lowest_bit)
-                                p->lowest_bit = offset;
-                        if (offset > p->highest_bit)
-                                p->highest_bit = offset;
-                        if (p->prio > swap_info[swap_list.next].prio)
-                                swap_list.next = p - swap_info;
-                        nr_swap_pages++;
-                        p->inuse_pages--;
-                        mem_cgroup_uncharge_swap(ent);
+        int count = swap_count(p->swap_map[offset]);
+        bool has_cache;
+
+        has_cache = swap_has_cache(p->swap_map[offset]);
+
+        if (cache == SWAP_MAP) { /* dropping usage count of swap */
+                if (count < SWAP_MAP_MAX) {
+                        count--;
+                        p->swap_map[offset] = encode_swapmap(count, has_cache);
                 }
+        } else { /* dropping swap cache flag */
+                VM_BUG_ON(!has_cache);
+                p->swap_map[offset] = encode_swapmap(count, false);
+
+        }
+        /* return code. */
+        count = p->swap_map[offset];
+        /* free if no reference */
+        if (!count) {
+                if (offset < p->lowest_bit)
+                        p->lowest_bit = offset;
+                if (offset > p->highest_bit)
+                        p->highest_bit = offset;
+                if (p->prio > swap_info[swap_list.next].prio)
+                        swap_list.next = p - swap_info;
+                nr_swap_pages++;
+                p->inuse_pages--;
         }
+        if (!swap_count(count))
+                mem_cgroup_uncharge_swap(ent);
         return count;
 }
 
@@ -504,9 +601,33 @@ void swap_free(swp_entry_t entry)
 
         p = swap_info_get(entry);
         if (p) {
-                swap_entry_free(p, entry);
+                swap_entry_free(p, entry, SWAP_MAP);
+                spin_unlock(&swap_lock);
+        }
+}
+
+/*
+ * Called after dropping swapcache to decrease refcnt to swap entries.
+ */
+void swapcache_free(swp_entry_t entry, struct page *page)
+{
+        struct swap_info_struct *p;
+        int ret;
+
+        p = swap_info_get(entry);
+        if (p) {
+                ret = swap_entry_free(p, entry, SWAP_CACHE);
+                if (page) {
+                        bool swapout;
+                        if (ret)
+                                swapout = true; /* the end of swap out */
+                        else
+                                swapout = false; /* no more swap users! */
+                        mem_cgroup_uncharge_swapcache(page, entry, swapout);
+                }
                 spin_unlock(&swap_lock);
         }
+        return;
 }
 
 /*
@@ -521,8 +642,7 @@ static inline int page_swapcount(struct page *page)
         entry.val = page_private(page);
         p = swap_info_get(entry);
         if (p) {
-                /* Subtract the 1 for the swap cache itself */
-                count = p->swap_map[swp_offset(entry)] - 1;
+                count = swap_count(p->swap_map[swp_offset(entry)]);
                 spin_unlock(&swap_lock);
         }
         return count;
@@ -579,12 +699,12 @@ int free_swap_and_cache(swp_entry_t entry)
         struct swap_info_struct *p;
         struct page *page = NULL;
 
-        if (is_migration_entry(entry))
+        if (non_swap_entry(entry))
                 return 1;
 
         p = swap_info_get(entry);
         if (p) {
-                if (swap_entry_free(p, entry) == 1) {
+                if (swap_entry_free(p, entry, SWAP_MAP) == SWAP_HAS_CACHE) {
                         page = find_get_page(&swapper_space, entry.val);
                         if (page && !trylock_page(page)) {
                                 page_cache_release(page);
@@ -635,7 +755,7 @@ int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p)
 
                 if (!bdev) {
                         if (bdev_p)
-                                *bdev_p = bdget(sis->bdev->bd_dev);
+                                *bdev_p = bdgrab(sis->bdev);
 
                         spin_unlock(&swap_lock);
                         return i;
@@ -647,7 +767,7 @@ int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p)
                                         struct swap_extent, list);
                         if (se->start_block == offset) {
                                 if (bdev_p)
-                                        *bdev_p = bdget(sis->bdev->bd_dev);
+                                        *bdev_p = bdgrab(sis->bdev);
 
                                 spin_unlock(&swap_lock);
                                 bdput(bdev);
@@ -891,7 +1011,7 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si,
                         i = 1;
                 }
                 count = si->swap_map[i];
-                if (count && count != SWAP_MAP_BAD)
+                if (count && swap_count(count) != SWAP_MAP_BAD)
                         break;
         }
         return i;
@@ -995,13 +1115,13 @@ static int try_to_unuse(unsigned int type)
                  */
                 shmem = 0;
                 swcount = *swap_map;
-                if (swcount > 1) {
+                if (swap_count(swcount)) {
                         if (start_mm == &init_mm)
                                 shmem = shmem_unuse(entry, page);
                         else
                                 retval = unuse_mm(start_mm, entry, page);
                 }
-                if (*swap_map > 1) {
+                if (swap_count(*swap_map)) {
                         int set_start_mm = (*swap_map >= swcount);
                         struct list_head *p = &start_mm->mmlist;
                         struct mm_struct *new_start_mm = start_mm;
@@ -1011,7 +1131,7 @@ static int try_to_unuse(unsigned int type)
                         atomic_inc(&new_start_mm->mm_users);
                         atomic_inc(&prev_mm->mm_users);
                         spin_lock(&mmlist_lock);
-                        while (*swap_map > 1 && !retval && !shmem &&
+                        while (swap_count(*swap_map) && !retval && !shmem &&
                                         (p = p->next) != &start_mm->mmlist) {
                                 mm = list_entry(p, struct mm_struct, mmlist);
                                 if (!atomic_inc_not_zero(&mm->mm_users))
@@ -1023,13 +1143,14 @@ static int try_to_unuse(unsigned int type)
                                 cond_resched();
 
                                 swcount = *swap_map;
-                                if (swcount <= 1)
+                                if (!swap_count(swcount)) /* any usage ? */
                                         ;
                                 else if (mm == &init_mm) {
                                         set_start_mm = 1;
                                         shmem = shmem_unuse(entry, page);
                                 } else
                                         retval = unuse_mm(mm, entry, page);
+
                                 if (set_start_mm && *swap_map < swcount) {
                                         mmput(new_start_mm);
                                         atomic_inc(&mm->mm_users);
@@ -1057,21 +1178,25 @@ static int try_to_unuse(unsigned int type)
                 }
 
                 /*
-                 * How could swap count reach 0x7fff when the maximum
-                 * pid is 0x7fff, and there's no way to repeat a swap
-                 * page within an mm (except in shmem, where it's the
-                 * shared object which takes the reference count)?
-                 * We believe SWAP_MAP_MAX cannot occur in Linux 2.4.
-                 *
+                 * How could swap count reach 0x7ffe ?
+                 * There's no way to repeat a swap page within an mm
+                 * (except in shmem, where it's the shared object which takes
+                 * the reference count)?
+                 * We believe SWAP_MAP_MAX cannot occur.(if occur, unsigned
+                 * short is too small....)
                  * If that's wrong, then we should worry more about
                  * exit_mmap() and do_munmap() cases described above:
                  * we might be resetting SWAP_MAP_MAX too early here.
                  * We know "Undead"s can happen, they're okay, so don't
                  * report them; but do report if we reset SWAP_MAP_MAX.
                  */
-                if (*swap_map == SWAP_MAP_MAX) {
+                /* We might release the lock_page() in unuse_mm(). */
+                if (!PageSwapCache(page) || page_private(page) != entry.val)
+                        goto retry;
+
+                if (swap_count(*swap_map) == SWAP_MAP_MAX) {
                         spin_lock(&swap_lock);
-                        *swap_map = 1;
+                        *swap_map = encode_swapmap(0, true);
                         spin_unlock(&swap_lock);
                         reset_overflow = 1;
                 }
@@ -1089,7 +1214,8 @@ static int try_to_unuse(unsigned int type)
                  * pages would be incorrect if swap supported "shared
                  * private" pages, but they are handled by tmpfs files.
                  */
-                if ((*swap_map > 1) && PageDirty(page) && PageSwapCache(page)) {
+                if (swap_count(*swap_map) &&
+                     PageDirty(page) && PageSwapCache(page)) {
                         struct writeback_control wbc = {
                                 .sync_mode = WB_SYNC_NONE,
                         };
@@ -1116,6 +1242,7 @@ static int try_to_unuse(unsigned int type)
                  * mark page dirty so shrink_page_list will preserve it.
                  */
                 SetPageDirty(page);
+retry:
                 unlock_page(page);
                 page_cache_release(page);
 
@@ -1447,9 +1574,9 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
         p->flags &= ~SWP_WRITEOK;
         spin_unlock(&swap_lock);
 
-        current->flags |= PF_SWAPOFF;
+        current->flags |= PF_OOM_ORIGIN;
         err = try_to_unuse(type);
-        current->flags &= ~PF_SWAPOFF;
+        current->flags &= ~PF_OOM_ORIGIN;
 
         if (err) {
                 /* re-insert swap space back into swap_list */
@@ -1846,12 +1973,14 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
                 goto bad_swap;
         }
 
-        if (blk_queue_nonrot(bdev_get_queue(p->bdev))) {
-                p->flags |= SWP_SOLIDSTATE;
-                p->cluster_next = 1 + (random32() % p->highest_bit);
+        if (p->bdev) {
+                if (blk_queue_nonrot(bdev_get_queue(p->bdev))) {
+                        p->flags |= SWP_SOLIDSTATE;
+                        p->cluster_next = 1 + (random32() % p->highest_bit);
+                }
+                if (discard_swap(p) == 0)
+                        p->flags |= SWP_DISCARDABLE;
         }
-        if (discard_swap(p) == 0)
-                p->flags |= SWP_DISCARDABLE;
 
         mutex_lock(&swapon_mutex);
         spin_lock(&swap_lock);
@@ -1942,15 +2071,23 @@ void si_swapinfo(struct sysinfo *val)
  *
  * Note: if swap_map[] reaches SWAP_MAP_MAX the entries are treated as
  * "permanent", but will be reclaimed by the next swapoff.
+ * Returns error code in following case.
+ * - success -> 0
+ * - swp_entry is invalid -> EINVAL
+ * - swp_entry is migration entry -> EINVAL
+ * - swap-cache reference is requested but there is already one. -> EEXIST
+ * - swap-cache reference is requested but the entry is not used. -> ENOENT
  */
-int swap_duplicate(swp_entry_t entry)
+static int __swap_duplicate(swp_entry_t entry, bool cache)
 {
         struct swap_info_struct * p;
         unsigned long offset, type;
-        int result = 0;
+        int result = -EINVAL;
+        int count;
+        bool has_cache;
 
-        if (is_migration_entry(entry))
-                return 1;
+        if (non_swap_entry(entry))
+                return -EINVAL;
 
         type = swp_type(entry);
         if (type >= nr_swapfiles)
@@ -1959,17 +2096,40 @@ int swap_duplicate(swp_entry_t entry)
         offset = swp_offset(entry);
 
         spin_lock(&swap_lock);
-        if (offset < p->max && p->swap_map[offset]) {
-                if (p->swap_map[offset] < SWAP_MAP_MAX - 1) {
-                        p->swap_map[offset]++;
-                        result = 1;
-                } else if (p->swap_map[offset] <= SWAP_MAP_MAX) {
+
+        if (unlikely(offset >= p->max))
+                goto unlock_out;
+
+        count = swap_count(p->swap_map[offset]);
+        has_cache = swap_has_cache(p->swap_map[offset]);
+
+        if (cache == SWAP_CACHE) { /* called for swapcache/swapin-readahead */
+
+                /* set SWAP_HAS_CACHE if there is no cache and entry is used */
+                if (!has_cache && count) {
+                        p->swap_map[offset] = encode_swapmap(count, true);
+                        result = 0;
+                } else if (has_cache) /* someone added cache */
+                        result = -EEXIST;
+                else if (!count) /* no users */
+                        result = -ENOENT;
+
+        } else if (count || has_cache) {
+                if (count < SWAP_MAP_MAX - 1) {
+                        p->swap_map[offset] = encode_swapmap(count + 1,
+                                                             has_cache);
+                        result = 0;
+                } else if (count <= SWAP_MAP_MAX) {
                         if (swap_overflow++ < 5)
-                                printk(KERN_WARNING "swap_dup: swap entry overflow\n");
-                        p->swap_map[offset] = SWAP_MAP_MAX;
-                        result = 1;
+                                printk(KERN_WARNING
+                                       "swap_dup: swap entry overflow\n");
+                        p->swap_map[offset] = encode_swapmap(SWAP_MAP_MAX,
+                                                              has_cache);
+                        result = 0;
                 }
-        }
+        } else
+                result = -ENOENT; /* unused swap entry */
+unlock_out:
         spin_unlock(&swap_lock);
 out:
         return result;
@@ -1978,6 +2138,27 @@ bad_file:
         printk(KERN_ERR "swap_dup: %s%08lx\n", Bad_file, entry.val);
         goto out;
 }
+/*
+ * increase reference count of swap entry by 1.
+ */
+void swap_duplicate(swp_entry_t entry)
+{
+        __swap_duplicate(entry, SWAP_MAP);
+}
+
+/*
+ * @entry: swap entry for which we allocate swap cache.
+ *
+ * Called when allocating swap cache for exising swap entry,
+ * This can return error codes. Returns 0 at success.
+ * -EBUSY means there is a swap cache.
+ * Note: return code is different from swap_duplicate().
+ */
+int swapcache_prepare(swp_entry_t entry)
+{
+        return __swap_duplicate(entry, SWAP_CACHE);
+}
+
 
 struct swap_info_struct *
 get_swap_info_struct(unsigned type)
@@ -2016,7 +2197,7 @@ int valid_swaphandles(swp_entry_t entry, unsigned long *offset)
                 /* Don't read in free or bad pages */
                 if (!si->swap_map[toff])
                         break;
-                if (si->swap_map[toff] == SWAP_MAP_BAD)
+                if (swap_count(si->swap_map[toff]) == SWAP_MAP_BAD)
                         break;
         }
         /* Count contiguous allocated slots below our target */
@@ -2024,7 +2205,7 @@ int valid_swaphandles(swp_entry_t entry, unsigned long *offset)
                 /* Don't read in free or bad pages */
                 if (!si->swap_map[toff])
                         break;
-                if (si->swap_map[toff] == SWAP_MAP_BAD)
+                if (swap_count(si->swap_map[toff]) == SWAP_MAP_BAD)
                         break;
         }
         spin_unlock(&swap_lock);
diff --git a/mm/thrash.c b/mm/thrash.c
index c4c5205a9c35..2372d4ed5dd8 100644
--- a/mm/thrash.c
+++ b/mm/thrash.c
@@ -26,47 +26,45 @@ static DEFINE_SPINLOCK(swap_token_lock);
 struct mm_struct *swap_token_mm;
 static unsigned int global_faults;
 
-void grab_swap_token(void)
+void grab_swap_token(struct mm_struct *mm)
 {
         int current_interval;
 
         global_faults++;
 
-        current_interval = global_faults - current->mm->faultstamp;
+        current_interval = global_faults - mm->faultstamp;
 
         if (!spin_trylock(&swap_token_lock))
                 return;
 
         /* First come first served */
         if (swap_token_mm == NULL) {
-                current->mm->token_priority = current->mm->token_priority + 2;
-                swap_token_mm = current->mm;
+                mm->token_priority = mm->token_priority + 2;
+                swap_token_mm = mm;
                 goto out;
         }
 
-        if (current->mm != swap_token_mm) {
-                if (current_interval < current->mm->last_interval)
-                        current->mm->token_priority++;
+        if (mm != swap_token_mm) {
+                if (current_interval < mm->last_interval)
+                        mm->token_priority++;
                 else {
-                        if (likely(current->mm->token_priority > 0))
-                                current->mm->token_priority--;
+                        if (likely(mm->token_priority > 0))
+                                mm->token_priority--;
                 }
                 /* Check if we deserve the token */
-                if (current->mm->token_priority >
-                                swap_token_mm->token_priority) {
-                        current->mm->token_priority += 2;
-                        swap_token_mm = current->mm;
+                if (mm->token_priority > swap_token_mm->token_priority) {
+                        mm->token_priority += 2;
+                        swap_token_mm = mm;
                 }
         } else {
                 /* Token holder came in again! */
-                current->mm->token_priority += 2;
+                mm->token_priority += 2;
         }
 
 out:
-        current->mm->faultstamp = global_faults;
-        current->mm->last_interval = current_interval;
+        mm->faultstamp = global_faults;
+        mm->last_interval = current_interval;
         spin_unlock(&swap_token_lock);
-return;
 }
 
 /* Called on process exit. */
diff --git a/mm/truncate.c b/mm/truncate.c
index 1229211104f8..450cebdabfc0 100644
--- a/mm/truncate.c
+++ b/mm/truncate.c
@@ -50,7 +50,7 @@ void do_invalidatepage(struct page *page, unsigned long offset)
 static inline void truncate_partial_page(struct page *page, unsigned partial)
 {
         zero_user_segment(page, partial, PAGE_CACHE_SIZE);
-        if (PagePrivate(page))
+        if (page_has_private(page))
                 do_invalidatepage(page, partial);
 }
 
@@ -93,13 +93,13 @@ EXPORT_SYMBOL(cancel_dirty_page);
  * its lock, b) when a concurrent invalidate_mapping_pages got there first and
  * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
  */
-static void
+static int
 truncate_complete_page(struct address_space *mapping, struct page *page)
 {
         if (page->mapping != mapping)
-                return;
+                return -EIO;
 
-        if (PagePrivate(page))
+        if (page_has_private(page))
                 do_invalidatepage(page, 0);
 
         cancel_dirty_page(page, PAGE_CACHE_SIZE);
@@ -108,6 +108,7 @@ truncate_complete_page(struct address_space *mapping, struct page *page)
         remove_from_page_cache(page);
         ClearPageMappedToDisk(page);
         page_cache_release(page);       /* pagecache ref */
+        return 0;
 }
 
 /*
@@ -126,7 +127,7 @@ invalidate_complete_page(struct address_space *mapping, struct page *page)
         if (page->mapping != mapping)
                 return 0;
 
-        if (PagePrivate(page) && !try_to_release_page(page, 0))
+        if (page_has_private(page) && !try_to_release_page(page, 0))
                 return 0;
 
         clear_page_mlock(page);
@@ -135,6 +136,51 @@ invalidate_complete_page(struct address_space *mapping, struct page *page)
         return ret;
 }
 
+int truncate_inode_page(struct address_space *mapping, struct page *page)
+{
+        if (page_mapped(page)) {
+                unmap_mapping_range(mapping,
+                                   (loff_t)page->index << PAGE_CACHE_SHIFT,
+                                   PAGE_CACHE_SIZE, 0);
+        }
+        return truncate_complete_page(mapping, page);
+}
+
+/*
+ * Used to get rid of pages on hardware memory corruption.
+ */
+int generic_error_remove_page(struct address_space *mapping, struct page *page)
+{
+        if (!mapping)
+                return -EINVAL;
+        /*
+         * Only punch for normal data pages for now.
+         * Handling other types like directories would need more auditing.
+         */
+        if (!S_ISREG(mapping->host->i_mode))
+                return -EIO;
+        return truncate_inode_page(mapping, page);
+}
+EXPORT_SYMBOL(generic_error_remove_page);
+
+/*
+ * Safely invalidate one page from its pagecache mapping.
+ * It only drops clean, unused pages. The page must be locked.
+ *
+ * Returns 1 if the page is successfully invalidated, otherwise 0.
+ */
+int invalidate_inode_page(struct page *page)
+{
+        struct address_space *mapping = page_mapping(page);
+        if (!mapping)
+                return 0;
+        if (PageDirty(page) || PageWriteback(page))
+                return 0;
+        if (page_mapped(page))
+                return 0;
+        return invalidate_complete_page(mapping, page);
+}
+
 /**
  * truncate_inode_pages - truncate range of pages specified by start & end byte offsets
  * @mapping: mapping to truncate
@@ -196,12 +242,7 @@ void truncate_inode_pages_range(struct address_space *mapping,
                                 unlock_page(page);
                                 continue;
                         }
-                        if (page_mapped(page)) {
-                                unmap_mapping_range(mapping,
-                                  (loff_t)page_index<<PAGE_CACHE_SHIFT,
-                                  PAGE_CACHE_SIZE, 0);
-                        }
-                        truncate_complete_page(mapping, page);
+                        truncate_inode_page(mapping, page);
                         unlock_page(page);
                 }
                 pagevec_release(&pvec);
@@ -238,15 +279,10 @@ void truncate_inode_pages_range(struct address_space *mapping,
                                 break;
                         lock_page(page);
                         wait_on_page_writeback(page);
-                        if (page_mapped(page)) {
-                                unmap_mapping_range(mapping,
-                                  (loff_t)page->index<<PAGE_CACHE_SHIFT,
-                                  PAGE_CACHE_SIZE, 0);
-                        }
+                        truncate_inode_page(mapping, page);
                         if (page->index > next)
                                 next = page->index;
                         next++;
-                        truncate_complete_page(mapping, page);
                         unlock_page(page);
                 }
                 pagevec_release(&pvec);
@@ -267,8 +303,21 @@ void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
 }
 EXPORT_SYMBOL(truncate_inode_pages);
 
-unsigned long __invalidate_mapping_pages(struct address_space *mapping,
-                                pgoff_t start, pgoff_t end, bool be_atomic)
+/**
+ * invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
+ * @mapping: the address_space which holds the pages to invalidate
+ * @start: the offset 'from' which to invalidate
+ * @end: the offset 'to' which to invalidate (inclusive)
+ *
+ * This function only removes the unlocked pages, if you want to
+ * remove all the pages of one inode, you must call truncate_inode_pages.
+ *
+ * invalidate_mapping_pages() will not block on IO activity. It will not
+ * invalidate pages which are dirty, locked, under writeback or mapped into
+ * pagetables.
+ */
+unsigned long invalidate_mapping_pages(struct address_space *mapping,
+                                       pgoff_t start, pgoff_t end)
 {
         struct pagevec pvec;
         pgoff_t next = start;
@@ -298,41 +347,17 @@ unsigned long __invalidate_mapping_pages(struct address_space *mapping,
                         if (lock_failed)
                                 continue;
 
-                        if (PageDirty(page) || PageWriteback(page))
-                                goto unlock;
-                        if (page_mapped(page))
-                                goto unlock;
-                        ret += invalidate_complete_page(mapping, page);
-unlock:
+                        ret += invalidate_inode_page(page);
+
                         unlock_page(page);
                         if (next > end)
                                 break;
                 }
                 pagevec_release(&pvec);
-                if (likely(!be_atomic))
-                        cond_resched();
+                cond_resched();
         }
         return ret;
 }
-
-/**
- * invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
- * @mapping: the address_space which holds the pages to invalidate
- * @start: the offset 'from' which to invalidate
- * @end: the offset 'to' which to invalidate (inclusive)
- *
- * This function only removes the unlocked pages, if you want to
- * remove all the pages of one inode, you must call truncate_inode_pages.
- *
- * invalidate_mapping_pages() will not block on IO activity. It will not
- * invalidate pages which are dirty, locked, under writeback or mapped into
- * pagetables.
- */
-unsigned long invalidate_mapping_pages(struct address_space *mapping,
-                                pgoff_t start, pgoff_t end)
-{
-        return __invalidate_mapping_pages(mapping, start, end, false);
-}
 EXPORT_SYMBOL(invalidate_mapping_pages);
 
 /*
@@ -348,7 +373,7 @@ invalidate_complete_page2(struct address_space *mapping, struct page *page)
         if (page->mapping != mapping)
                 return 0;
 
-        if (PagePrivate(page) && !try_to_release_page(page, GFP_KERNEL))
+        if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL))
                 return 0;
 
         spin_lock_irq(&mapping->tree_lock);
@@ -356,9 +381,10 @@ invalidate_complete_page2(struct address_space *mapping, struct page *page)
                 goto failed;
 
         clear_page_mlock(page);
-        BUG_ON(PagePrivate(page));
+        BUG_ON(page_has_private(page));
         __remove_from_page_cache(page);
         spin_unlock_irq(&mapping->tree_lock);
+        mem_cgroup_uncharge_cache_page(page);
         page_cache_release(page);       /* pagecache ref */
         return 1;
 failed:
@@ -471,3 +497,67 @@ int invalidate_inode_pages2(struct address_space *mapping)
         return invalidate_inode_pages2_range(mapping, 0, -1);
 }
 EXPORT_SYMBOL_GPL(invalidate_inode_pages2);
+
+/**
+ * truncate_pagecache - unmap and remove pagecache that has been truncated
+ * @inode: inode
+ * @old: old file offset
+ * @new: new file offset
+ *
+ * inode's new i_size must already be written before truncate_pagecache
+ * is called.
+ *
+ * This function should typically be called before the filesystem
+ * releases resources associated with the freed range (eg. deallocates
+ * blocks). This way, pagecache will always stay logically coherent
+ * with on-disk format, and the filesystem would not have to deal with
+ * situations such as writepage being called for a page that has already
+ * had its underlying blocks deallocated.
+ */
+void truncate_pagecache(struct inode *inode, loff_t old, loff_t new)
+{
+        if (new < old) {
+                struct address_space *mapping = inode->i_mapping;
+
+                /*
+                 * unmap_mapping_range is called twice, first simply for
+                 * efficiency so that truncate_inode_pages does fewer
+                 * single-page unmaps.  However after this first call, and
+                 * before truncate_inode_pages finishes, it is possible for
+                 * private pages to be COWed, which remain after
+                 * truncate_inode_pages finishes, hence the second
+                 * unmap_mapping_range call must be made for correctness.
+                 */
+                unmap_mapping_range(mapping, new + PAGE_SIZE - 1, 0, 1);
+                truncate_inode_pages(mapping, new);
+                unmap_mapping_range(mapping, new + PAGE_SIZE - 1, 0, 1);
+        }
+}
+EXPORT_SYMBOL(truncate_pagecache);
+
+/**
+ * vmtruncate - unmap mappings "freed" by truncate() syscall
+ * @inode: inode of the file used
+ * @offset: file offset to start truncating
+ *
+ * NOTE! We have to be ready to update the memory sharing
+ * between the file and the memory map for a potential last
+ * incomplete page.  Ugly, but necessary.
+ */
+int vmtruncate(struct inode *inode, loff_t offset)
+{
+        loff_t oldsize;
+        int error;
+
+        error = inode_newsize_ok(inode, offset);
+        if (error)
+                return error;
+        oldsize = inode->i_size;
+        i_size_write(inode, offset);
+        truncate_pagecache(inode, oldsize, offset);
+        if (inode->i_op->truncate)
+                inode->i_op->truncate(inode);
+
+        return error;
+}
+EXPORT_SYMBOL(vmtruncate);
diff --git a/mm/util.c b/mm/util.c
index 7c122e49f769..7c35ad95f927 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -6,6 +6,9 @@
 #include <linux/sched.h>
 #include <asm/uaccess.h>
 
+#define CREATE_TRACE_POINTS
+#include <trace/events/kmem.h>
+
 /**
  * kstrdup - allocate space for and copy an existing string
  * @s: the string to duplicate
@@ -165,6 +168,10 @@ EXPORT_SYMBOL(krealloc);
  *
  * The memory of the object @p points to is zeroed before freed.
  * If @p is %NULL, kzfree() does nothing.
+ *
+ * Note: this function zeroes the whole allocated buffer which can be a good
+ * deal bigger than the requested buffer size passed to kmalloc(). So be
+ * careful when using this function in performance sensitive code.
  */
 void kzfree(const void *p)
 {
@@ -222,6 +229,30 @@ void arch_pick_mmap_layout(struct mm_struct *mm)
 }
 #endif
 
+/**
+ * get_user_pages_fast() - pin user pages in memory
+ * @start:      starting user address
+ * @nr_pages:   number of pages from start to pin
+ * @write:      whether pages will be written to
+ * @pages:      array that receives pointers to the pages pinned.
+ *              Should be at least nr_pages long.
+ *
+ * Returns number of pages pinned. This may be fewer than the number
+ * requested. If nr_pages is 0 or negative, returns 0. If no pages
+ * were pinned, returns -errno.
+ *
+ * get_user_pages_fast provides equivalent functionality to get_user_pages,
+ * operating on current and current->mm, with force=0 and vma=NULL. However
+ * unlike get_user_pages, it must be called without mmap_sem held.
+ *
+ * get_user_pages_fast may take mmap_sem and page table locks, so no
+ * assumptions can be made about lack of locking. get_user_pages_fast is to be
+ * implemented in a way that is advantageous (vs get_user_pages()) when the
+ * user memory area is already faulted in and present in ptes. However if the
+ * pages have to be faulted in, it may turn out to be slightly slower so
+ * callers need to carefully consider what to use. On many architectures,
+ * get_user_pages_fast simply falls back to get_user_pages.
+ */
 int __attribute__((weak)) get_user_pages_fast(unsigned long start,
                                 int nr_pages, int write, struct page **pages)
 {
@@ -236,3 +267,11 @@ int __attribute__((weak)) get_user_pages_fast(unsigned long start,
         return ret;
 }
 EXPORT_SYMBOL_GPL(get_user_pages_fast);
+
+/* Tracepoints definitions. */
+EXPORT_TRACEPOINT_SYMBOL(kmalloc);
+EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
+EXPORT_TRACEPOINT_SYMBOL(kmalloc_node);
+EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);
+EXPORT_TRACEPOINT_SYMBOL(kfree);
+EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index fab19876b4d1..0f551a4a44cd 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -12,6 +12,7 @@
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/highmem.h>
+#include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/interrupt.h>
@@ -23,12 +24,12 @@
 #include <linux/rbtree.h>
 #include <linux/radix-tree.h>
 #include <linux/rcupdate.h>
-#include <linux/bootmem.h>
 #include <linux/pfn.h>
-
+#include <linux/kmemleak.h>
 #include <asm/atomic.h>
 #include <asm/uaccess.h>
 #include <asm/tlbflush.h>
+#include <asm/shmparam.h>
 
 
 /*** Page table manipulation functions ***/
@@ -168,11 +169,9 @@ static int vmap_page_range_noflush(unsigned long start, unsigned long end,
                 next = pgd_addr_end(addr, end);
                 err = vmap_pud_range(pgd, addr, next, prot, pages, &nr);
                 if (err)
-                        break;
+                        return err;
         } while (pgd++, addr = next, addr != end);
 
-        if (unlikely(err))
-                return err;
         return nr;
 }
 
@@ -186,7 +185,7 @@ static int vmap_page_range(unsigned long start, unsigned long end,
         return ret;
 }
 
-static inline int is_vmalloc_or_module_addr(const void *x)
+int is_vmalloc_or_module_addr(const void *x)
 {
         /*
          * ARM, x86-64 and sparc64 put modules in a special place,
@@ -265,6 +264,7 @@ struct vmap_area {
 static DEFINE_SPINLOCK(vmap_area_lock);
 static struct rb_root vmap_area_root = RB_ROOT;
 static LIST_HEAD(vmap_area_list);
+static unsigned long vmap_area_pcpu_hole;
 
 static struct vmap_area *__find_vmap_area(unsigned long addr)
 {
@@ -402,6 +402,7 @@ overflow:
                         printk(KERN_WARNING
                                 "vmap allocation for size %lu failed: "
                                 "use vmalloc=<size> to increase size.\n", size);
+                kfree(va);
                 return ERR_PTR(-EBUSY);
         }
 
@@ -430,6 +431,15 @@ static void __free_vmap_area(struct vmap_area *va)
         RB_CLEAR_NODE(&va->rb_node);
         list_del_rcu(&va->list);
 
+        /*
+         * Track the highest possible candidate for pcpu area
+         * allocation.  Areas outside of vmalloc area can be returned
+         * here too, consider only end addresses which fall inside
+         * vmalloc area proper.
+         */
+        if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END)
+                vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end);
+
         call_rcu(&va->rcu_head, rcu_free_va);
 }
 
@@ -1031,12 +1041,15 @@ void __init vmalloc_init(void)
 
         /* Import existing vmlist entries. */
         for (tmp = vmlist; tmp; tmp = tmp->next) {
-                va = alloc_bootmem(sizeof(struct vmap_area));
+                va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT);
                 va->flags = tmp->flags | VM_VM_AREA;
                 va->va_start = (unsigned long)tmp->addr;
                 va->va_end = va->va_start + tmp->size;
                 __insert_vmap_area(va);
         }
+
+        vmap_area_pcpu_hole = VMALLOC_END;
+
         vmap_initialized = true;
 }
 
@@ -1121,14 +1134,34 @@ EXPORT_SYMBOL_GPL(map_vm_area);
 DEFINE_RWLOCK(vmlist_lock);
 struct vm_struct *vmlist;
 
+static void insert_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
+                              unsigned long flags, void *caller)
+{
+        struct vm_struct *tmp, **p;
+
+        vm->flags = flags;
+        vm->addr = (void *)va->va_start;
+        vm->size = va->va_end - va->va_start;
+        vm->caller = caller;
+        va->private = vm;
+        va->flags |= VM_VM_AREA;
+
+        write_lock(&vmlist_lock);
+        for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
+                if (tmp->addr >= vm->addr)
+                        break;
+        }
+        vm->next = *p;
+        *p = vm;
+        write_unlock(&vmlist_lock);
+}
+
 static struct vm_struct *__get_vm_area_node(unsigned long size,
-                unsigned long flags, unsigned long start, unsigned long end,
-                int node, gfp_t gfp_mask, void *caller)
+                unsigned long align, unsigned long flags, unsigned long start,
+                unsigned long end, int node, gfp_t gfp_mask, void *caller)
 {
         static struct vmap_area *va;
         struct vm_struct *area;
-        struct vm_struct *tmp, **p;
-        unsigned long align = 1;
 
         BUG_ON(in_interrupt());
         if (flags & VM_IOREMAP) {
@@ -1146,7 +1179,7 @@ static struct vm_struct *__get_vm_area_node(unsigned long size,
         if (unlikely(!size))
                 return NULL;
 
-        area = kmalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
+        area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
         if (unlikely(!area))
                 return NULL;
 
@@ -1161,32 +1194,14 @@ static struct vm_struct *__get_vm_area_node(unsigned long size,
                 return NULL;
         }
 
-        area->flags = flags;
-        area->addr = (void *)va->va_start;
-        area->size = size;
-        area->pages = NULL;
-        area->nr_pages = 0;
-        area->phys_addr = 0;
-        area->caller = caller;
-        va->private = area;
-        va->flags |= VM_VM_AREA;
-
-        write_lock(&vmlist_lock);
-        for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
-                if (tmp->addr >= area->addr)
-                        break;
-        }
-        area->next = *p;
-        *p = area;
-        write_unlock(&vmlist_lock);
-
+        insert_vmalloc_vm(area, va, flags, caller);
         return area;
 }
 
 struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
                                 unsigned long start, unsigned long end)
 {
-        return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL,
+        return __get_vm_area_node(size, 1, flags, start, end, -1, GFP_KERNEL,
                                                 __builtin_return_address(0));
 }
 EXPORT_SYMBOL_GPL(__get_vm_area);
@@ -1195,7 +1210,7 @@ struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
                                        unsigned long start, unsigned long end,
                                        void *caller)
 {
-        return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL,
+        return __get_vm_area_node(size, 1, flags, start, end, -1, GFP_KERNEL,
                                   caller);
 }
 
@@ -1210,22 +1225,22 @@ struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
  */
 struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
 {
-        return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END,
+        return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
                                 -1, GFP_KERNEL, __builtin_return_address(0));
 }
 
 struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
                                 void *caller)
 {
-        return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END,
+        return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
                                                 -1, GFP_KERNEL, caller);
 }
 
 struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags,
                                    int node, gfp_t gfp_mask)
 {
-        return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node,
-                                  gfp_mask, __builtin_return_address(0));
+        return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
+                                  node, gfp_mask, __builtin_return_address(0));
 }
 
 static struct vm_struct *find_vm_area(const void *addr)
@@ -1255,17 +1270,21 @@ struct vm_struct *remove_vm_area(const void *addr)
         if (va && va->flags & VM_VM_AREA) {
                 struct vm_struct *vm = va->private;
                 struct vm_struct *tmp, **p;
-
-                vmap_debug_free_range(va->va_start, va->va_end);
-                free_unmap_vmap_area(va);
-                vm->size -= PAGE_SIZE;
-
+                /*
+                 * remove from list and disallow access to this vm_struct
+                 * before unmap. (address range confliction is maintained by
+                 * vmap.)
+                 */
                 write_lock(&vmlist_lock);
                 for (p = &vmlist; (tmp = *p) != vm; p = &tmp->next)
                         ;
                 *p = tmp->next;
                 write_unlock(&vmlist_lock);
 
+                vmap_debug_free_range(va->va_start, va->va_end);
+                free_unmap_vmap_area(va);
+                vm->size -= PAGE_SIZE;
+
                 return vm;
         }
         return NULL;
@@ -1326,6 +1345,9 @@ static void __vunmap(const void *addr, int deallocate_pages)
 void vfree(const void *addr)
 {
         BUG_ON(in_interrupt());
+
+        kmemleak_free(addr);
+
         __vunmap(addr, 1);
 }
 EXPORT_SYMBOL(vfree);
@@ -1364,7 +1386,7 @@ void *vmap(struct page **pages, unsigned int count,
 
         might_sleep();
 
-        if (count > num_physpages)
+        if (count > totalram_pages)
                 return NULL;
 
         area = get_vm_area_caller((count << PAGE_SHIFT), flags,
@@ -1381,7 +1403,8 @@ void *vmap(struct page **pages, unsigned int count,
 }
 EXPORT_SYMBOL(vmap);
 
-static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
+static void *__vmalloc_node(unsigned long size, unsigned long align,
+                            gfp_t gfp_mask, pgprot_t prot,
                             int node, void *caller);
 static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
                                  pgprot_t prot, int node, void *caller)
@@ -1395,7 +1418,7 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
         area->nr_pages = nr_pages;
         /* Please note that the recursion is strictly bounded. */
         if (array_size > PAGE_SIZE) {
-                pages = __vmalloc_node(array_size, gfp_mask | __GFP_ZERO,
+                pages = __vmalloc_node(array_size, 1, gfp_mask | __GFP_ZERO,
                                 PAGE_KERNEL, node, caller);
                 area->flags |= VM_VPAGES;
         } else {
@@ -1438,13 +1461,23 @@ fail:
 
 void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
 {
-        return __vmalloc_area_node(area, gfp_mask, prot, -1,
-                                        __builtin_return_address(0));
+        void *addr = __vmalloc_area_node(area, gfp_mask, prot, -1,
+                                         __builtin_return_address(0));
+
+        /*
+         * A ref_count = 3 is needed because the vm_struct and vmap_area
+         * structures allocated in the __get_vm_area_node() function contain
+         * references to the virtual address of the vmalloc'ed block.
+         */
+        kmemleak_alloc(addr, area->size - PAGE_SIZE, 3, gfp_mask);
+
+        return addr;
 }
 
 /**
  *      __vmalloc_node  -  allocate virtually contiguous memory
  *      @size:          allocation size
+ *      @align:         desired alignment
  *      @gfp_mask:      flags for the page level allocator
  *      @prot:          protection mask for the allocated pages
  *      @node:          node to use for allocation or -1
@@ -1454,27 +1487,39 @@ void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
  *      allocator with @gfp_mask flags.  Map them into contiguous
  *      kernel virtual space, using a pagetable protection of @prot.
  */
-static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
-                                                int node, void *caller)
+static void *__vmalloc_node(unsigned long size, unsigned long align,
+                            gfp_t gfp_mask, pgprot_t prot,
+                            int node, void *caller)
 {
         struct vm_struct *area;
+        void *addr;
+        unsigned long real_size = size;
 
         size = PAGE_ALIGN(size);
-        if (!size || (size >> PAGE_SHIFT) > num_physpages)
+        if (!size || (size >> PAGE_SHIFT) > totalram_pages)
                 return NULL;
 
-        area = __get_vm_area_node(size, VM_ALLOC, VMALLOC_START, VMALLOC_END,
-                                                node, gfp_mask, caller);
+        area = __get_vm_area_node(size, align, VM_ALLOC, VMALLOC_START,
+                                  VMALLOC_END, node, gfp_mask, caller);
 
         if (!area)
                 return NULL;
 
-        return __vmalloc_area_node(area, gfp_mask, prot, node, caller);
+        addr = __vmalloc_area_node(area, gfp_mask, prot, node, caller);
+
+        /*
+         * A ref_count = 3 is needed because the vm_struct and vmap_area
+         * structures allocated in the __get_vm_area_node() function contain
+         * references to the virtual address of the vmalloc'ed block.
+         */
+        kmemleak_alloc(addr, real_size, 3, gfp_mask);
+
+        return addr;
 }
 
 void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
 {
-        return __vmalloc_node(size, gfp_mask, prot, -1,
+        return __vmalloc_node(size, 1, gfp_mask, prot, -1,
                                 __builtin_return_address(0));
 }
 EXPORT_SYMBOL(__vmalloc);
@@ -1490,7 +1535,7 @@ EXPORT_SYMBOL(__vmalloc);
  */
 void *vmalloc(unsigned long size)
 {
-        return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
+        return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
                                         -1, __builtin_return_address(0));
 }
 EXPORT_SYMBOL(vmalloc);
@@ -1507,7 +1552,8 @@ void *vmalloc_user(unsigned long size)
         struct vm_struct *area;
         void *ret;
 
-        ret = __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
+        ret = __vmalloc_node(size, SHMLBA,
+                             GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
                              PAGE_KERNEL, -1, __builtin_return_address(0));
         if (ret) {
                 area = find_vm_area(ret);
@@ -1530,7 +1576,7 @@ EXPORT_SYMBOL(vmalloc_user);
  */
 void *vmalloc_node(unsigned long size, int node)
 {
-        return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
+        return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
                                         node, __builtin_return_address(0));
 }
 EXPORT_SYMBOL(vmalloc_node);
@@ -1553,7 +1599,7 @@ EXPORT_SYMBOL(vmalloc_node);
 
 void *vmalloc_exec(unsigned long size)
 {
-        return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC,
+        return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC,
                               -1, __builtin_return_address(0));
 }
 
@@ -1574,7 +1620,7 @@ void *vmalloc_exec(unsigned long size)
  */
 void *vmalloc_32(unsigned long size)
 {
-        return __vmalloc_node(size, GFP_VMALLOC32, PAGE_KERNEL,
+        return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL,
                               -1, __builtin_return_address(0));
 }
 EXPORT_SYMBOL(vmalloc_32);
@@ -1591,7 +1637,7 @@ void *vmalloc_32_user(unsigned long size)
         struct vm_struct *area;
         void *ret;
 
-        ret = __vmalloc_node(size, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL,
+        ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL,
                              -1, __builtin_return_address(0));
         if (ret) {
                 area = find_vm_area(ret);
@@ -1601,10 +1647,120 @@ void *vmalloc_32_user(unsigned long size)
 }
 EXPORT_SYMBOL(vmalloc_32_user);
 
+/*
+ * small helper routine , copy contents to buf from addr.
+ * If the page is not present, fill zero.
+ */
+
+static int aligned_vread(char *buf, char *addr, unsigned long count)
+{
+        struct page *p;
+        int copied = 0;
+
+        while (count) {
+                unsigned long offset, length;
+
+                offset = (unsigned long)addr & ~PAGE_MASK;
+                length = PAGE_SIZE - offset;
+                if (length > count)
+                        length = count;
+                p = vmalloc_to_page(addr);
+                /*
+                 * To do safe access to this _mapped_ area, we need
+                 * lock. But adding lock here means that we need to add
+                 * overhead of vmalloc()/vfree() calles for this _debug_
+                 * interface, rarely used. Instead of that, we'll use
+                 * kmap() and get small overhead in this access function.
+                 */
+                if (p) {
+                        /*
+                         * we can expect USER0 is not used (see vread/vwrite's
+                         * function description)
+                         */
+                        void *map = kmap_atomic(p, KM_USER0);
+                        memcpy(buf, map + offset, length);
+                        kunmap_atomic(map, KM_USER0);
+                } else
+                        memset(buf, 0, length);
+
+                addr += length;
+                buf += length;
+                copied += length;
+                count -= length;
+        }
+        return copied;
+}
+
+static int aligned_vwrite(char *buf, char *addr, unsigned long count)
+{
+        struct page *p;
+        int copied = 0;
+
+        while (count) {
+                unsigned long offset, length;
+
+                offset = (unsigned long)addr & ~PAGE_MASK;
+                length = PAGE_SIZE - offset;
+                if (length > count)
+                        length = count;
+                p = vmalloc_to_page(addr);
+                /*
+                 * To do safe access to this _mapped_ area, we need
+                 * lock. But adding lock here means that we need to add
+                 * overhead of vmalloc()/vfree() calles for this _debug_
+                 * interface, rarely used. Instead of that, we'll use
+                 * kmap() and get small overhead in this access function.
+                 */
+                if (p) {
+                        /*
+                         * we can expect USER0 is not used (see vread/vwrite's
+                         * function description)
+                         */
+                        void *map = kmap_atomic(p, KM_USER0);
+                        memcpy(map + offset, buf, length);
+                        kunmap_atomic(map, KM_USER0);
+                }
+                addr += length;
+                buf += length;
+                copied += length;
+                count -= length;
+        }
+        return copied;
+}
+
+/**
+ *      vread() -  read vmalloc area in a safe way.
+ *      @buf:           buffer for reading data
+ *      @addr:          vm address.
+ *      @count:         number of bytes to be read.
+ *
+ *      Returns # of bytes which addr and buf should be increased.
+ *      (same number to @count). Returns 0 if [addr...addr+count) doesn't
+ *      includes any intersect with alive vmalloc area.
+ *
+ *      This function checks that addr is a valid vmalloc'ed area, and
+ *      copy data from that area to a given buffer. If the given memory range
+ *      of [addr...addr+count) includes some valid address, data is copied to
+ *      proper area of @buf. If there are memory holes, they'll be zero-filled.
+ *      IOREMAP area is treated as memory hole and no copy is done.
+ *
+ *      If [addr...addr+count) doesn't includes any intersects with alive
+ *      vm_struct area, returns 0.
+ *      @buf should be kernel's buffer. Because this function uses KM_USER0,
+ *      the caller should guarantee KM_USER0 is not used.
+ *
+ *      Note: In usual ops, vread() is never necessary because the caller
+ *      should know vmalloc() area is valid and can use memcpy().
+ *      This is for routines which have to access vmalloc area without
+ *      any informaion, as /dev/kmem.
+ *
+ */
+
 long vread(char *buf, char *addr, unsigned long count)
 {
         struct vm_struct *tmp;
         char *vaddr, *buf_start = buf;
+        unsigned long buflen = count;
         unsigned long n;
 
         /* Don't allow overflow */
@@ -1612,7 +1768,7 @@ long vread(char *buf, char *addr, unsigned long count)
                 count = -(unsigned long) addr;
 
         read_lock(&vmlist_lock);
-        for (tmp = vmlist; tmp; tmp = tmp->next) {
+        for (tmp = vmlist; count && tmp; tmp = tmp->next) {
                 vaddr = (char *) tmp->addr;
                 if (addr >= vaddr + tmp->size - PAGE_SIZE)
                         continue;
@@ -1625,32 +1781,72 @@ long vread(char *buf, char *addr, unsigned long count)
                         count--;
                 }
                 n = vaddr + tmp->size - PAGE_SIZE - addr;
-                do {
-                        if (count == 0)
-                                goto finished;
-                        *buf = *addr;
-                        buf++;
-                        addr++;
-                        count--;
-                } while (--n > 0);
+                if (n > count)
+                        n = count;
+                if (!(tmp->flags & VM_IOREMAP))
+                        aligned_vread(buf, addr, n);
+                else /* IOREMAP area is treated as memory hole */
+                        memset(buf, 0, n);
+                buf += n;
+                addr += n;
+                count -= n;
         }
 finished:
         read_unlock(&vmlist_lock);
-        return buf - buf_start;
+
+        if (buf == buf_start)
+                return 0;
+        /* zero-fill memory holes */
+        if (buf != buf_start + buflen)
+                memset(buf, 0, buflen - (buf - buf_start));
+
+        return buflen;
 }
 
+/**
+ *      vwrite() -  write vmalloc area in a safe way.
+ *      @buf:           buffer for source data
+ *      @addr:          vm address.
+ *      @count:         number of bytes to be read.
+ *
+ *      Returns # of bytes which addr and buf should be incresed.
+ *      (same number to @count).
+ *      If [addr...addr+count) doesn't includes any intersect with valid
+ *      vmalloc area, returns 0.
+ *
+ *      This function checks that addr is a valid vmalloc'ed area, and
+ *      copy data from a buffer to the given addr. If specified range of
+ *      [addr...addr+count) includes some valid address, data is copied from
+ *      proper area of @buf. If there are memory holes, no copy to hole.
+ *      IOREMAP area is treated as memory hole and no copy is done.
+ *
+ *      If [addr...addr+count) doesn't includes any intersects with alive
+ *      vm_struct area, returns 0.
+ *      @buf should be kernel's buffer. Because this function uses KM_USER0,
+ *      the caller should guarantee KM_USER0 is not used.
+ *
+ *      Note: In usual ops, vwrite() is never necessary because the caller
+ *      should know vmalloc() area is valid and can use memcpy().
+ *      This is for routines which have to access vmalloc area without
+ *      any informaion, as /dev/kmem.
+ *
+ *      The caller should guarantee KM_USER1 is not used.
+ */
+
 long vwrite(char *buf, char *addr, unsigned long count)
 {
         struct vm_struct *tmp;
-        char *vaddr, *buf_start = buf;
-        unsigned long n;
+        char *vaddr;
+        unsigned long n, buflen;
+        int copied = 0;
 
         /* Don't allow overflow */
         if ((unsigned long) addr + count < count)
                 count = -(unsigned long) addr;
+        buflen = count;
 
         read_lock(&vmlist_lock);
-        for (tmp = vmlist; tmp; tmp = tmp->next) {
+        for (tmp = vmlist; count && tmp; tmp = tmp->next) {
                 vaddr = (char *) tmp->addr;
                 if (addr >= vaddr + tmp->size - PAGE_SIZE)
                         continue;
@@ -1662,18 +1858,21 @@ long vwrite(char *buf, char *addr, unsigned long count)
                         count--;
                 }
                 n = vaddr + tmp->size - PAGE_SIZE - addr;
-                do {
-                        if (count == 0)
-                                goto finished;
-                        *addr = *buf;
-                        buf++;
-                        addr++;
-                        count--;
-                } while (--n > 0);
+                if (n > count)
+                        n = count;
+                if (!(tmp->flags & VM_IOREMAP)) {
+                        aligned_vwrite(buf, addr, n);
+                        copied++;
+                }
+                buf += n;
+                addr += n;
+                count -= n;
         }
 finished:
         read_unlock(&vmlist_lock);
-        return buf - buf_start;
+        if (!copied)
+                return 0;
+        return buflen;
 }
 
 /**
@@ -1794,6 +1993,286 @@ void free_vm_area(struct vm_struct *area)
 }
 EXPORT_SYMBOL_GPL(free_vm_area);
 
+static struct vmap_area *node_to_va(struct rb_node *n)
+{
+        return n ? rb_entry(n, struct vmap_area, rb_node) : NULL;
+}
+
+/**
+ * pvm_find_next_prev - find the next and prev vmap_area surrounding @end
+ * @end: target address
+ * @pnext: out arg for the next vmap_area
+ * @pprev: out arg for the previous vmap_area
+ *
+ * Returns: %true if either or both of next and prev are found,
+ *          %false if no vmap_area exists
+ *
+ * Find vmap_areas end addresses of which enclose @end.  ie. if not
+ * NULL, *pnext->va_end > @end and *pprev->va_end <= @end.
+ */
+static bool pvm_find_next_prev(unsigned long end,
+                               struct vmap_area **pnext,
+                               struct vmap_area **pprev)
+{
+        struct rb_node *n = vmap_area_root.rb_node;
+        struct vmap_area *va = NULL;
+
+        while (n) {
+                va = rb_entry(n, struct vmap_area, rb_node);
+                if (end < va->va_end)
+                        n = n->rb_left;
+                else if (end > va->va_end)
+                        n = n->rb_right;
+                else
+                        break;
+        }
+
+        if (!va)
+                return false;
+
+        if (va->va_end > end) {
+                *pnext = va;
+                *pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
+        } else {
+                *pprev = va;
+                *pnext = node_to_va(rb_next(&(*pprev)->rb_node));
+        }
+        return true;
+}
+
+/**
+ * pvm_determine_end - find the highest aligned address between two vmap_areas
+ * @pnext: in/out arg for the next vmap_area
+ * @pprev: in/out arg for the previous vmap_area
+ * @align: alignment
+ *
+ * Returns: determined end address
+ *
+ * Find the highest aligned address between *@pnext and *@pprev below
+ * VMALLOC_END.  *@pnext and *@pprev are adjusted so that the aligned
+ * down address is between the end addresses of the two vmap_areas.
+ *
+ * Please note that the address returned by this function may fall
+ * inside *@pnext vmap_area.  The caller is responsible for checking
+ * that.
+ */
+static unsigned long pvm_determine_end(struct vmap_area **pnext,
+                                       struct vmap_area **pprev,
+                                       unsigned long align)
+{
+        const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
+        unsigned long addr;
+
+        if (*pnext)
+                addr = min((*pnext)->va_start & ~(align - 1), vmalloc_end);
+        else
+                addr = vmalloc_end;
+
+        while (*pprev && (*pprev)->va_end > addr) {
+                *pnext = *pprev;
+                *pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
+        }
+
+        return addr;
+}
+
+/**
+ * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
+ * @offsets: array containing offset of each area
+ * @sizes: array containing size of each area
+ * @nr_vms: the number of areas to allocate
+ * @align: alignment, all entries in @offsets and @sizes must be aligned to this
+ * @gfp_mask: allocation mask
+ *
+ * Returns: kmalloc'd vm_struct pointer array pointing to allocated
+ *          vm_structs on success, %NULL on failure
+ *
+ * Percpu allocator wants to use congruent vm areas so that it can
+ * maintain the offsets among percpu areas.  This function allocates
+ * congruent vmalloc areas for it.  These areas tend to be scattered
+ * pretty far, distance between two areas easily going up to
+ * gigabytes.  To avoid interacting with regular vmallocs, these areas
+ * are allocated from top.
+ *
+ * Despite its complicated look, this allocator is rather simple.  It
+ * does everything top-down and scans areas from the end looking for
+ * matching slot.  While scanning, if any of the areas overlaps with
+ * existing vmap_area, the base address is pulled down to fit the
+ * area.  Scanning is repeated till all the areas fit and then all
+ * necessary data structres are inserted and the result is returned.
+ */
+struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
+                                     const size_t *sizes, int nr_vms,
+                                     size_t align, gfp_t gfp_mask)
+{
+        const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
+        const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
+        struct vmap_area **vas, *prev, *next;
+        struct vm_struct **vms;
+        int area, area2, last_area, term_area;
+        unsigned long base, start, end, last_end;
+        bool purged = false;
+
+        gfp_mask &= GFP_RECLAIM_MASK;
+
+        /* verify parameters and allocate data structures */
+        BUG_ON(align & ~PAGE_MASK || !is_power_of_2(align));
+        for (last_area = 0, area = 0; area < nr_vms; area++) {
+                start = offsets[area];
+                end = start + sizes[area];
+
+                /* is everything aligned properly? */
+                BUG_ON(!IS_ALIGNED(offsets[area], align));
+                BUG_ON(!IS_ALIGNED(sizes[area], align));
+
+                /* detect the area with the highest address */
+                if (start > offsets[last_area])
+                        last_area = area;
+
+                for (area2 = 0; area2 < nr_vms; area2++) {
+                        unsigned long start2 = offsets[area2];
+                        unsigned long end2 = start2 + sizes[area2];
+
+                        if (area2 == area)
+                                continue;
+
+                        BUG_ON(start2 >= start && start2 < end);
+                        BUG_ON(end2 <= end && end2 > start);
+                }
+        }
+        last_end = offsets[last_area] + sizes[last_area];
+
+        if (vmalloc_end - vmalloc_start < last_end) {
+                WARN_ON(true);
+                return NULL;
+        }
+
+        vms = kzalloc(sizeof(vms[0]) * nr_vms, gfp_mask);
+        vas = kzalloc(sizeof(vas[0]) * nr_vms, gfp_mask);
+        if (!vas || !vms)
+                goto err_free;
+
+        for (area = 0; area < nr_vms; area++) {
+                vas[area] = kzalloc(sizeof(struct vmap_area), gfp_mask);
+                vms[area] = kzalloc(sizeof(struct vm_struct), gfp_mask);
+                if (!vas[area] || !vms[area])
+                        goto err_free;
+        }
+retry:
+        spin_lock(&vmap_area_lock);
+
+        /* start scanning - we scan from the top, begin with the last area */
+        area = term_area = last_area;
+        start = offsets[area];
+        end = start + sizes[area];
+
+        if (!pvm_find_next_prev(vmap_area_pcpu_hole, &next, &prev)) {
+                base = vmalloc_end - last_end;
+                goto found;
+        }
+        base = pvm_determine_end(&next, &prev, align) - end;
+
+        while (true) {
+                BUG_ON(next && next->va_end <= base + end);
+                BUG_ON(prev && prev->va_end > base + end);
+
+                /*
+                 * base might have underflowed, add last_end before
+                 * comparing.
+                 */
+                if (base + last_end < vmalloc_start + last_end) {
+                        spin_unlock(&vmap_area_lock);
+                        if (!purged) {
+                                purge_vmap_area_lazy();
+                                purged = true;
+                                goto retry;
+                        }
+                        goto err_free;
+                }
+
+                /*
+                 * If next overlaps, move base downwards so that it's
+                 * right below next and then recheck.
+                 */
+                if (next && next->va_start < base + end) {
+                        base = pvm_determine_end(&next, &prev, align) - end;
+                        term_area = area;
+                        continue;
+                }
+
+                /*
+                 * If prev overlaps, shift down next and prev and move
+                 * base so that it's right below new next and then
+                 * recheck.
+                 */
+                if (prev && prev->va_end > base + start)  {
+                        next = prev;
+                        prev = node_to_va(rb_prev(&next->rb_node));
+                        base = pvm_determine_end(&next, &prev, align) - end;
+                        term_area = area;
+                        continue;
+                }
+
+                /*
+                 * This area fits, move on to the previous one.  If
+                 * the previous one is the terminal one, we're done.
+                 */
+                area = (area + nr_vms - 1) % nr_vms;
+                if (area == term_area)
+                        break;
+                start = offsets[area];
+                end = start + sizes[area];
+                pvm_find_next_prev(base + end, &next, &prev);
+        }
+found:
+        /* we've found a fitting base, insert all va's */
+        for (area = 0; area < nr_vms; area++) {
+                struct vmap_area *va = vas[area];
+
+                va->va_start = base + offsets[area];
+                va->va_end = va->va_start + sizes[area];
+                __insert_vmap_area(va);
+        }
+
+        vmap_area_pcpu_hole = base + offsets[last_area];
+
+        spin_unlock(&vmap_area_lock);
+
+        /* insert all vm's */
+        for (area = 0; area < nr_vms; area++)
+                insert_vmalloc_vm(vms[area], vas[area], VM_ALLOC,
+                                  pcpu_get_vm_areas);
+
+        kfree(vas);
+        return vms;
+
+err_free:
+        for (area = 0; area < nr_vms; area++) {
+                if (vas)
+                        kfree(vas[area]);
+                if (vms)
+                        kfree(vms[area]);
+        }
+        kfree(vas);
+        kfree(vms);
+        return NULL;
+}
+
+/**
+ * pcpu_free_vm_areas - free vmalloc areas for percpu allocator
+ * @vms: vm_struct pointer array returned by pcpu_get_vm_areas()
+ * @nr_vms: the number of allocated areas
+ *
+ * Free vm_structs and the array allocated by pcpu_get_vm_areas().
+ */
+void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
+{
+        int i;
+
+        for (i = 0; i < nr_vms; i++)
+                free_vm_area(vms[i]);
+        kfree(vms);
+}
 
 #ifdef CONFIG_PROC_FS
 static void *s_start(struct seq_file *m, loff_t *pos)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 06e72693b458..777af57fd8c8 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -63,6 +63,9 @@ struct scan_control {
         /* Can mapped pages be reclaimed? */
         int may_unmap;
 
+        /* Can pages be swapped as part of reclaim? */
+        int may_swap;
+
         /* This context's SWAP_CLUSTER_MAX. If freeing memory for
          * suspend, we effectively ignore SWAP_CLUSTER_MAX.
          * In this context, it doesn't matter that we scan the
@@ -145,8 +148,8 @@ static struct zone_reclaim_stat *get_reclaim_stat(struct zone *zone,
         return &zone->reclaim_stat;
 }
 
-static unsigned long zone_nr_pages(struct zone *zone, struct scan_control *sc,
-                                   enum lru_list lru)
+static unsigned long zone_nr_lru_pages(struct zone *zone,
+                                struct scan_control *sc, enum lru_list lru)
 {
         if (!scanning_global_lru(sc))
                 return mem_cgroup_zone_nr_pages(sc->mem_cgroup, zone, lru);
@@ -283,7 +286,12 @@ static inline int page_mapping_inuse(struct page *page)
 
 static inline int is_page_cache_freeable(struct page *page)
 {
-        return page_count(page) - !!PagePrivate(page) == 2;
+        /*
+         * A freeable page cache page is referenced only by the caller
+         * that isolated the page, the page cache radix tree and
+         * optional buffer heads at page->private.
+         */
+        return page_count(page) - page_has_private(page) == 2;
 }
 
 static int may_write_to_queue(struct backing_dev_info *bdi)
@@ -358,7 +366,6 @@ static pageout_t pageout(struct page *page, struct address_space *mapping,
          * block, for some throttling. This happens by accident, because
          * swap_backing_dev_info is bust: it doesn't reflect the
          * congestion state of the swapdevs.  Easy to fix, if needed.
-         * See swapfile.c:page_queue_congested().
          */
         if (!is_page_cache_freeable(page))
                 return PAGE_KEEP;
@@ -367,7 +374,7 @@ static pageout_t pageout(struct page *page, struct address_space *mapping,
                  * Some data journaling orphaned pages can have
                  * page->mapping == NULL while being dirty with clean buffers.
                  */
-                if (PagePrivate(page)) {
+                if (page_has_private(page)) {
                         if (try_to_free_buffers(page)) {
                                 ClearPageDirty(page);
                                 printk("%s: orphaned page\n", __func__);
@@ -467,10 +474,11 @@ static int __remove_mapping(struct address_space *mapping, struct page *page)
                 swp_entry_t swap = { .val = page_private(page) };
                 __delete_from_swap_cache(page);
                 spin_unlock_irq(&mapping->tree_lock);
-                swap_free(swap);
+                swapcache_free(swap, page);
         } else {
                 __remove_from_page_cache(page);
                 spin_unlock_irq(&mapping->tree_lock);
+                mem_cgroup_uncharge_cache_page(page);
         }
 
         return 1;
@@ -509,7 +517,6 @@ int remove_mapping(struct address_space *mapping, struct page *page)
  *
  * lru_lock must not be held, interrupts must be enabled.
  */
-#ifdef CONFIG_UNEVICTABLE_LRU
 void putback_lru_page(struct page *page)
 {
         int lru;
@@ -528,7 +535,7 @@ redo:
                  * unevictable page on [in]active list.
                  * We know how to handle that.
                  */
-                lru = active + page_is_file_cache(page);
+                lru = active + page_lru_base_type(page);
                 lru_cache_add_lru(page, lru);
         } else {
                 /*
@@ -537,6 +544,16 @@ redo:
                  */
                 lru = LRU_UNEVICTABLE;
                 add_page_to_unevictable_list(page);
+                /*
+                 * When racing with an mlock clearing (page is
+                 * unlocked), make sure that if the other thread does
+                 * not observe our setting of PG_lru and fails
+                 * isolation, we see PG_mlocked cleared below and move
+                 * the page back to the evictable list.
+                 *
+                 * The other side is TestClearPageMlocked().
+                 */
+                smp_mb();
         }
 
         /*
@@ -563,20 +580,6 @@ redo:
         put_page(page);         /* drop ref from isolate */
 }
 
-#else /* CONFIG_UNEVICTABLE_LRU */
-
-void putback_lru_page(struct page *page)
-{
-        int lru;
-        VM_BUG_ON(PageLRU(page));
-
-        lru = !!TestClearPageActive(page) + page_is_file_cache(page);
-        lru_cache_add_lru(page, lru);
-        put_page(page);
-}
-#endif /* CONFIG_UNEVICTABLE_LRU */
-
-
 /*
  * shrink_page_list() returns the number of reclaimed pages
  */
@@ -588,6 +591,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
         struct pagevec freed_pvec;
         int pgactivate = 0;
         unsigned long nr_reclaimed = 0;
+        unsigned long vm_flags;
 
         cond_resched();
 
@@ -638,10 +642,16 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                                 goto keep_locked;
                 }
 
-                referenced = page_referenced(page, 1, sc->mem_cgroup);
-                /* In active use or really unfreeable?  Activate it. */
+                referenced = page_referenced(page, 1,
+                                                sc->mem_cgroup, &vm_flags);
+                /*
+                 * In active use or really unfreeable?  Activate it.
+                 * If page which have PG_mlocked lost isoltation race,
+                 * try_to_unmap moves it to unevictable list
+                 */
                 if (sc->order <= PAGE_ALLOC_COSTLY_ORDER &&
-                                        referenced && page_mapping_inuse(page))
+                                        referenced && page_mapping_inuse(page)
+                                        && !(vm_flags & VM_LOCKED))
                         goto activate_locked;
 
                 /*
@@ -663,7 +673,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                  * processes. Try to unmap it here.
                  */
                 if (page_mapped(page) && mapping) {
-                        switch (try_to_unmap(page, 0)) {
+                        switch (try_to_unmap(page, TTU_UNMAP)) {
                         case SWAP_FAIL:
                                 goto activate_locked;
                         case SWAP_AGAIN:
@@ -727,7 +737,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                  * process address space (page_count == 1) it can be freed.
                  * Otherwise, leave the page on the LRU so it is swappable.
                  */
-                if (PagePrivate(page)) {
+                if (page_has_private(page)) {
                         if (!try_to_release_page(page, sc->gfp_mask))
                                 goto activate_locked;
                         if (!mapping && page_count(page) == 1) {
@@ -825,7 +835,7 @@ int __isolate_lru_page(struct page *page, int mode, int file)
         if (mode != ISOLATE_BOTH && (!PageActive(page) != !mode))
                 return ret;
 
-        if (mode != ISOLATE_BOTH && (!page_is_file_cache(page) != !file))
+        if (mode != ISOLATE_BOTH && page_is_file_cache(page) != file)
                 return ret;
 
         /*
@@ -846,7 +856,6 @@ int __isolate_lru_page(struct page *page, int mode, int file)
                  */
                 ClearPageLRU(page);
                 ret = 0;
-                mem_cgroup_del_lru(page);
         }
 
         return ret;
@@ -894,12 +903,14 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
                 switch (__isolate_lru_page(page, mode, file)) {
                 case 0:
                         list_move(&page->lru, dst);
+                        mem_cgroup_del_lru(page);
                         nr_taken++;
                         break;
 
                 case -EBUSY:
                         /* else it is being freed elsewhere */
                         list_move(&page->lru, src);
+                        mem_cgroup_rotate_lru_list(page, page_lru(page));
                         continue;
 
                 default:
@@ -938,18 +949,21 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
                         /* Check that we have not crossed a zone boundary. */
                         if (unlikely(page_zone_id(cursor_page) != zone_id))
                                 continue;
-                        switch (__isolate_lru_page(cursor_page, mode, file)) {
-                        case 0:
+
+                        /*
+                         * If we don't have enough swap space, reclaiming of
+                         * anon page which don't already have a swap slot is
+                         * pointless.
+                         */
+                        if (nr_swap_pages <= 0 && PageAnon(cursor_page) &&
+                                        !PageSwapCache(cursor_page))
+                                continue;
+
+                        if (__isolate_lru_page(cursor_page, mode, file) == 0) {
                                 list_move(&cursor_page->lru, dst);
+                                mem_cgroup_del_lru(cursor_page);
                                 nr_taken++;
                                 scan++;
-                                break;
-
-                        case -EBUSY:
-                                /* else it is being freed elsewhere */
-                                list_move(&cursor_page->lru, src);
-                        default:
-                                break;  /* ! on LRU or wrong list */
                         }
                 }
         }
@@ -971,7 +985,7 @@ static unsigned long isolate_pages_global(unsigned long nr,
         if (file)
                 lru += LRU_FILE;
         return isolate_lru_pages(nr, &z->lru[lru].list, dst, scanned, order,
-                                                                mode, !!file);
+                                                                mode, file);
 }
 
 /*
@@ -986,7 +1000,7 @@ static unsigned long clear_active_flags(struct list_head *page_list,
         struct page *page;
 
         list_for_each_entry(page, page_list, lru) {
-                lru = page_is_file_cache(page);
+                lru = page_lru_base_type(page);
                 if (PageActive(page)) {
                         lru += LRU_ACTIVE;
                         ClearPageActive(page);
@@ -1044,6 +1058,31 @@ int isolate_lru_page(struct page *page)
 }
 
 /*
+ * Are there way too many processes in the direct reclaim path already?
+ */
+static int too_many_isolated(struct zone *zone, int file,
+                struct scan_control *sc)
+{
+        unsigned long inactive, isolated;
+
+        if (current_is_kswapd())
+                return 0;
+
+        if (!scanning_global_lru(sc))
+                return 0;
+
+        if (file) {
+                inactive = zone_page_state(zone, NR_INACTIVE_FILE);
+                isolated = zone_page_state(zone, NR_ISOLATED_FILE);
+        } else {
+                inactive = zone_page_state(zone, NR_INACTIVE_ANON);
+                isolated = zone_page_state(zone, NR_ISOLATED_ANON);
+        }
+
+        return isolated > inactive;
+}
+
+/*
  * shrink_inactive_list() is a helper for shrink_zone().  It returns the number
  * of reclaimed pages
  */
@@ -1056,6 +1095,27 @@ static unsigned long shrink_inactive_list(unsigned long max_scan,
         unsigned long nr_scanned = 0;
         unsigned long nr_reclaimed = 0;
         struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(zone, sc);
+        int lumpy_reclaim = 0;
+
+        while (unlikely(too_many_isolated(zone, file, sc))) {
+                congestion_wait(BLK_RW_ASYNC, HZ/10);
+
+                /* We are about to die and free our memory. Return now. */
+                if (fatal_signal_pending(current))
+                        return SWAP_CLUSTER_MAX;
+        }
+
+        /*
+         * If we need a large contiguous chunk of memory, or have
+         * trouble getting a small set of contiguous pages, we
+         * will reclaim both active and inactive pages.
+         *
+         * We use the same threshold as pageout congestion_wait below.
+         */
+        if (sc->order > PAGE_ALLOC_COSTLY_ORDER)
+                lumpy_reclaim = 1;
+        else if (sc->order && priority < DEF_PRIORITY - 2)
+                lumpy_reclaim = 1;
 
         pagevec_init(&pvec, 1);
 
@@ -1068,23 +1128,27 @@ static unsigned long shrink_inactive_list(unsigned long max_scan,
                 unsigned long nr_freed;
                 unsigned long nr_active;
                 unsigned int count[NR_LRU_LISTS] = { 0, };
-                int mode = ISOLATE_INACTIVE;
-
-                /*
-                 * If we need a large contiguous chunk of memory, or have
-                 * trouble getting a small set of contiguous pages, we
-                 * will reclaim both active and inactive pages.
-                 *
-                 * We use the same threshold as pageout congestion_wait below.
-                 */
-                if (sc->order > PAGE_ALLOC_COSTLY_ORDER)
-                        mode = ISOLATE_BOTH;
-                else if (sc->order && priority < DEF_PRIORITY - 2)
-                        mode = ISOLATE_BOTH;
+                int mode = lumpy_reclaim ? ISOLATE_BOTH : ISOLATE_INACTIVE;
+                unsigned long nr_anon;
+                unsigned long nr_file;
 
                 nr_taken = sc->isolate_pages(sc->swap_cluster_max,
                              &page_list, &nr_scan, sc->order, mode,
                                 zone, sc->mem_cgroup, 0, file);
+
+                if (scanning_global_lru(sc)) {
+                        zone->pages_scanned += nr_scan;
+                        if (current_is_kswapd())
+                                __count_zone_vm_events(PGSCAN_KSWAPD, zone,
+                                                       nr_scan);
+                        else
+                                __count_zone_vm_events(PGSCAN_DIRECT, zone,
+                                                       nr_scan);
+                }
+
+                if (nr_taken == 0)
+                        goto done;
+
                 nr_active = clear_active_flags(&page_list, count);
                 __count_vm_events(PGDEACTIVATE, nr_active);
 
@@ -1097,8 +1161,10 @@ static unsigned long shrink_inactive_list(unsigned long max_scan,
                 __mod_zone_page_state(zone, NR_INACTIVE_ANON,
                                                 -count[LRU_INACTIVE_ANON]);
 
-                if (scanning_global_lru(sc))
-                        zone->pages_scanned += nr_scan;
+                nr_anon = count[LRU_ACTIVE_ANON] + count[LRU_INACTIVE_ANON];
+                nr_file = count[LRU_ACTIVE_FILE] + count[LRU_INACTIVE_FILE];
+                __mod_zone_page_state(zone, NR_ISOLATED_ANON, nr_anon);
+                __mod_zone_page_state(zone, NR_ISOLATED_FILE, nr_file);
 
                 reclaim_stat->recent_scanned[0] += count[LRU_INACTIVE_ANON];
                 reclaim_stat->recent_scanned[0] += count[LRU_ACTIVE_ANON];
@@ -1117,8 +1183,8 @@ static unsigned long shrink_inactive_list(unsigned long max_scan,
                  * but that should be acceptable to the caller
                  */
                 if (nr_freed < nr_taken && !current_is_kswapd() &&
-                                        sc->order > PAGE_ALLOC_COSTLY_ORDER) {
-                        congestion_wait(WRITE, HZ/10);
+                    lumpy_reclaim) {
+                        congestion_wait(BLK_RW_ASYNC, HZ/10);
 
                         /*
                          * The attempt at page out may have made some
@@ -1132,18 +1198,12 @@ static unsigned long shrink_inactive_list(unsigned long max_scan,
                 }
 
                 nr_reclaimed += nr_freed;
+
                 local_irq_disable();
-                if (current_is_kswapd()) {
-                        __count_zone_vm_events(PGSCAN_KSWAPD, zone, nr_scan);
+                if (current_is_kswapd())
                         __count_vm_events(KSWAPD_STEAL, nr_freed);
-                } else if (scanning_global_lru(sc))
-                        __count_zone_vm_events(PGSCAN_DIRECT, zone, nr_scan);
-
                 __count_zone_vm_events(PGSTEAL, zone, nr_freed);
 
-                if (nr_taken == 0)
-                        goto done;
-
                 spin_lock(&zone->lru_lock);
                 /*
                  * Put back any unfreeable pages.
@@ -1162,8 +1222,8 @@ static unsigned long shrink_inactive_list(unsigned long max_scan,
                         SetPageLRU(page);
                         lru = page_lru(page);
                         add_page_to_lru_list(zone, page, lru);
-                        if (PageActive(page)) {
-                                int file = !!page_is_file_cache(page);
+                        if (is_active_lru(lru)) {
+                                int file = is_file_lru(lru);
                                 reclaim_stat->recent_rotated[file]++;
                         }
                         if (!pagevec_add(&pvec, page)) {
@@ -1172,10 +1232,13 @@ static unsigned long shrink_inactive_list(unsigned long max_scan,
                                 spin_lock_irq(&zone->lru_lock);
                         }
                 }
+                __mod_zone_page_state(zone, NR_ISOLATED_ANON, -nr_anon);
+                __mod_zone_page_state(zone, NR_ISOLATED_FILE, -nr_file);
+
         } while (nr_scanned < max_scan);
-        spin_unlock(&zone->lru_lock);
+
 done:
-        local_irq_enable();
+        spin_unlock_irq(&zone->lru_lock);
         pagevec_release(&pvec);
         return nr_reclaimed;
 }
@@ -1212,23 +1275,55 @@ static inline void note_zone_scanning_priority(struct zone *zone, int priority)
  * But we had to alter page->flags anyway.
  */
 
+static void move_active_pages_to_lru(struct zone *zone,
+                                     struct list_head *list,
+                                     enum lru_list lru)
+{
+        unsigned long pgmoved = 0;
+        struct pagevec pvec;
+        struct page *page;
+
+        pagevec_init(&pvec, 1);
+
+        while (!list_empty(list)) {
+                page = lru_to_page(list);
+
+                VM_BUG_ON(PageLRU(page));
+                SetPageLRU(page);
+
+                list_move(&page->lru, &zone->lru[lru].list);
+                mem_cgroup_add_lru_list(page, lru);
+                pgmoved++;
+
+                if (!pagevec_add(&pvec, page) || list_empty(list)) {
+                        spin_unlock_irq(&zone->lru_lock);
+                        if (buffer_heads_over_limit)
+                                pagevec_strip(&pvec);
+                        __pagevec_release(&pvec);
+                        spin_lock_irq(&zone->lru_lock);
+                }
+        }
+        __mod_zone_page_state(zone, NR_LRU_BASE + lru, pgmoved);
+        if (!is_active_lru(lru))
+                __count_vm_events(PGDEACTIVATE, pgmoved);
+}
 
 static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
                         struct scan_control *sc, int priority, int file)
 {
-        unsigned long pgmoved;
-        int pgdeactivate = 0;
+        unsigned long nr_taken;
         unsigned long pgscanned;
+        unsigned long vm_flags;
         LIST_HEAD(l_hold);      /* The pages which were snipped off */
+        LIST_HEAD(l_active);
         LIST_HEAD(l_inactive);
         struct page *page;
-        struct pagevec pvec;
-        enum lru_list lru;
         struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(zone, sc);
+        unsigned long nr_rotated = 0;
 
         lru_add_drain();
         spin_lock_irq(&zone->lru_lock);
-        pgmoved = sc->isolate_pages(nr_pages, &l_hold, &pgscanned, sc->order,
+        nr_taken = sc->isolate_pages(nr_pages, &l_hold, &pgscanned, sc->order,
                                         ISOLATE_ACTIVE, zone,
                                         sc->mem_cgroup, 1, file);
         /*
@@ -1238,15 +1333,16 @@ static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
         if (scanning_global_lru(sc)) {
                 zone->pages_scanned += pgscanned;
         }
-        reclaim_stat->recent_scanned[!!file] += pgmoved;
+        reclaim_stat->recent_scanned[file] += nr_taken;
 
+        __count_zone_vm_events(PGREFILL, zone, pgscanned);
         if (file)
-                __mod_zone_page_state(zone, NR_ACTIVE_FILE, -pgmoved);
+                __mod_zone_page_state(zone, NR_ACTIVE_FILE, -nr_taken);
         else
-                __mod_zone_page_state(zone, NR_ACTIVE_ANON, -pgmoved);
+                __mod_zone_page_state(zone, NR_ACTIVE_ANON, -nr_taken);
+        __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, nr_taken);
         spin_unlock_irq(&zone->lru_lock);
 
-        pgmoved = 0;
         while (!list_empty(&l_hold)) {
                 cond_resched();
                 page = lru_to_page(&l_hold);
@@ -1259,58 +1355,45 @@ static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
 
                 /* page_referenced clears PageReferenced */
                 if (page_mapping_inuse(page) &&
-                    page_referenced(page, 0, sc->mem_cgroup))
-                        pgmoved++;
+                    page_referenced(page, 0, sc->mem_cgroup, &vm_flags)) {
+                        nr_rotated++;
+                        /*
+                         * Identify referenced, file-backed active pages and
+                         * give them one more trip around the active list. So
+                         * that executable code get better chances to stay in
+                         * memory under moderate memory pressure.  Anon pages
+                         * are not likely to be evicted by use-once streaming
+                         * IO, plus JVM can create lots of anon VM_EXEC pages,
+                         * so we ignore them here.
+                         */
+                        if ((vm_flags & VM_EXEC) && page_is_file_cache(page)) {
+                                list_add(&page->lru, &l_active);
+                                continue;
+                        }
+                }
 
+                ClearPageActive(page);  /* we are de-activating */
                 list_add(&page->lru, &l_inactive);
         }
 
         /*
-         * Move the pages to the [file or anon] inactive list.
+         * Move pages back to the lru list.
          */
-        pagevec_init(&pvec, 1);
-        lru = LRU_BASE + file * LRU_FILE;
-
         spin_lock_irq(&zone->lru_lock);
         /*
-         * Count referenced pages from currently used mappings as
-         * rotated, even though they are moved to the inactive list.
-         * This helps balance scan pressure between file and anonymous
-         * pages in get_scan_ratio.
+         * Count referenced pages from currently used mappings as rotated,
+         * even though only some of them are actually re-activated.  This
+         * helps balance scan pressure between file and anonymous pages in
+         * get_scan_ratio.
          */
-        reclaim_stat->recent_rotated[!!file] += pgmoved;
+        reclaim_stat->recent_rotated[file] += nr_rotated;
 
-        pgmoved = 0;
-        while (!list_empty(&l_inactive)) {
-                page = lru_to_page(&l_inactive);
-                prefetchw_prev_lru_page(page, &l_inactive, flags);
-                VM_BUG_ON(PageLRU(page));
-                SetPageLRU(page);
-                VM_BUG_ON(!PageActive(page));
-                ClearPageActive(page);
-
-                list_move(&page->lru, &zone->lru[lru].list);
-                mem_cgroup_add_lru_list(page, lru);
-                pgmoved++;
-                if (!pagevec_add(&pvec, page)) {
-                        __mod_zone_page_state(zone, NR_LRU_BASE + lru, pgmoved);
-                        spin_unlock_irq(&zone->lru_lock);
-                        pgdeactivate += pgmoved;
-                        pgmoved = 0;
-                        if (buffer_heads_over_limit)
-                                pagevec_strip(&pvec);
-                        __pagevec_release(&pvec);
-                        spin_lock_irq(&zone->lru_lock);
-                }
-        }
-        __mod_zone_page_state(zone, NR_LRU_BASE + lru, pgmoved);
-        pgdeactivate += pgmoved;
-        __count_zone_vm_events(PGREFILL, zone, pgscanned);
-        __count_vm_events(PGDEACTIVATE, pgdeactivate);
+        move_active_pages_to_lru(zone, &l_active,
+                                                LRU_ACTIVE + file * LRU_FILE);
+        move_active_pages_to_lru(zone, &l_inactive,
+                                                LRU_BASE   + file * LRU_FILE);
+        __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken);
         spin_unlock_irq(&zone->lru_lock);
-        if (buffer_heads_over_limit)
-                pagevec_strip(&pvec);
-        pagevec_release(&pvec);
 }
 
 static int inactive_anon_is_low_global(struct zone *zone)
@@ -1345,12 +1428,48 @@ static int inactive_anon_is_low(struct zone *zone, struct scan_control *sc)
         return low;
 }
 
+static int inactive_file_is_low_global(struct zone *zone)
+{
+        unsigned long active, inactive;
+
+        active = zone_page_state(zone, NR_ACTIVE_FILE);
+        inactive = zone_page_state(zone, NR_INACTIVE_FILE);
+
+        return (active > inactive);
+}
+
+/**
+ * inactive_file_is_low - check if file pages need to be deactivated
+ * @zone: zone to check
+ * @sc:   scan control of this context
+ *
+ * When the system is doing streaming IO, memory pressure here
+ * ensures that active file pages get deactivated, until more
+ * than half of the file pages are on the inactive list.
+ *
+ * Once we get to that situation, protect the system's working
+ * set from being evicted by disabling active file page aging.
+ *
+ * This uses a different ratio than the anonymous pages, because
+ * the page cache uses a use-once replacement algorithm.
+ */
+static int inactive_file_is_low(struct zone *zone, struct scan_control *sc)
+{
+        int low;
+
+        if (scanning_global_lru(sc))
+                low = inactive_file_is_low_global(zone);
+        else
+                low = mem_cgroup_inactive_file_is_low(sc->mem_cgroup);
+        return low;
+}
+
 static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
         struct zone *zone, struct scan_control *sc, int priority)
 {
         int file = is_file_lru(lru);
 
-        if (lru == LRU_ACTIVE_FILE) {
+        if (lru == LRU_ACTIVE_FILE && inactive_file_is_low(zone, sc)) {
                 shrink_active_list(nr_to_scan, zone, sc, priority, file);
                 return 0;
         }
@@ -1379,23 +1498,16 @@ static void get_scan_ratio(struct zone *zone, struct scan_control *sc,
         unsigned long ap, fp;
         struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(zone, sc);
 
-        /* If we have no swap space, do not bother scanning anon pages. */
-        if (nr_swap_pages <= 0) {
-                percent[0] = 0;
-                percent[1] = 100;
-                return;
-        }
-
-        anon  = zone_nr_pages(zone, sc, LRU_ACTIVE_ANON) +
-                zone_nr_pages(zone, sc, LRU_INACTIVE_ANON);
-        file  = zone_nr_pages(zone, sc, LRU_ACTIVE_FILE) +
-                zone_nr_pages(zone, sc, LRU_INACTIVE_FILE);
+        anon  = zone_nr_lru_pages(zone, sc, LRU_ACTIVE_ANON) +
+                zone_nr_lru_pages(zone, sc, LRU_INACTIVE_ANON);
+        file  = zone_nr_lru_pages(zone, sc, LRU_ACTIVE_FILE) +
+                zone_nr_lru_pages(zone, sc, LRU_INACTIVE_FILE);
 
         if (scanning_global_lru(sc)) {
                 free  = zone_page_state(zone, NR_FREE_PAGES);
                 /* If we have very few page cache pages,
                    force-scan anon pages. */
-                if (unlikely(file + free <= zone->pages_high)) {
+                if (unlikely(file + free <= high_wmark_pages(zone))) {
                         percent[0] = 100;
                         percent[1] = 0;
                         return;
@@ -1450,6 +1562,26 @@ static void get_scan_ratio(struct zone *zone, struct scan_control *sc,
         percent[1] = 100 - percent[0];
 }
 
+/*
+ * Smallish @nr_to_scan's are deposited in @nr_saved_scan,
+ * until we collected @swap_cluster_max pages to scan.
+ */
+static unsigned long nr_scan_try_batch(unsigned long nr_to_scan,
+                                       unsigned long *nr_saved_scan,
+                                       unsigned long swap_cluster_max)
+{
+        unsigned long nr;
+
+        *nr_saved_scan += nr_to_scan;
+        nr = *nr_saved_scan;
+
+        if (nr >= swap_cluster_max)
+                *nr_saved_scan = 0;
+        else
+                nr = 0;
+
+        return nr;
+}
 
 /*
  * This is a basic per-zone page freer.  Used by both kswapd and direct reclaim.
@@ -1463,27 +1595,29 @@ static void shrink_zone(int priority, struct zone *zone,
         enum lru_list l;
         unsigned long nr_reclaimed = sc->nr_reclaimed;
         unsigned long swap_cluster_max = sc->swap_cluster_max;
+        struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(zone, sc);
+        int noswap = 0;
 
-        get_scan_ratio(zone, sc, percent);
+        /* If we have no swap space, do not bother scanning anon pages. */
+        if (!sc->may_swap || (nr_swap_pages <= 0)) {
+                noswap = 1;
+                percent[0] = 0;
+                percent[1] = 100;
+        } else
+                get_scan_ratio(zone, sc, percent);
 
         for_each_evictable_lru(l) {
                 int file = is_file_lru(l);
-                int scan;
+                unsigned long scan;
 
-                scan = zone_nr_pages(zone, sc, l);
-                if (priority) {
+                scan = zone_nr_lru_pages(zone, sc, l);
+                if (priority || noswap) {
                         scan >>= priority;
                         scan = (scan * percent[file]) / 100;
                 }
-                if (scanning_global_lru(sc)) {
-                        zone->lru[l].nr_scan += scan;
-                        nr[l] = zone->lru[l].nr_scan;
-                        if (nr[l] >= swap_cluster_max)
-                                zone->lru[l].nr_scan = 0;
-                        else
-                                nr[l] = 0;
-                } else
-                        nr[l] = scan;
+                nr[l] = nr_scan_try_batch(scan,
+                                          &reclaim_stat->nr_saved_scan[l],
+                                          swap_cluster_max);
         }
 
         while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
@@ -1516,7 +1650,7 @@ static void shrink_zone(int priority, struct zone *zone,
          * Even if we did not try to evict anon pages at all, we want to
          * rebalance the anon lru active/inactive ratio.
          */
-        if (inactive_anon_is_low(zone, sc))
+        if (inactive_anon_is_low(zone, sc) && nr_swap_pages > 0)
                 shrink_active_list(SWAP_CLUSTER_MAX, zone, sc, priority, 0);
 
         throttle_vm_writeout(sc->gfp_mask);
@@ -1527,11 +1661,13 @@ static void shrink_zone(int priority, struct zone *zone,
  * try to reclaim pages from zones which will satisfy the caller's allocation
  * request.
  *
- * We reclaim from a zone even if that zone is over pages_high.  Because:
+ * We reclaim from a zone even if that zone is over high_wmark_pages(zone).
+ * Because:
  * a) The caller may be trying to free *extra* pages to satisfy a higher-order
  *    allocation or
- * b) The zones may be over pages_high but they must go *over* pages_high to
- *    satisfy the `incremental min' zone defense algorithm.
+ * b) The target zone may be at high_wmark_pages(zone) but the lower zones
+ *    must go *over* high_wmark_pages(zone) to satisfy the `incremental min'
+ *    zone defense algorithm.
  *
  * If a zone is deemed to be full of pinned pages then just give it a light
  * scan then give up on it.
@@ -1583,10 +1719,10 @@ static void shrink_zones(int priority, struct zonelist *zonelist,
  *
  * If the caller is !__GFP_FS then the probability of a failure is reasonably
  * high - the zone may be full of dirty or under-writeback pages, which this
- * caller can't do much about.  We kick pdflush and take explicit naps in the
- * hope that some of these pages can be written.  But if the allocating task
- * holds filesystem locks which prevent writeout this might not work, and the
- * allocation attempt will fail.
+ * caller can't do much about.  We kick the writeback threads and take explicit
+ * naps in the hope that some of these pages can be written.  But if the
+ * allocating task holds filesystem locks which prevent writeout this might not
+ * work, and the allocation attempt will fail.
  *
  * returns:     0, if no pages reclaimed
  *              else, the number of pages reclaimed
@@ -1616,7 +1752,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
                         if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
                                 continue;
 
-                        lru_pages += zone_lru_pages(zone);
+                        lru_pages += zone_reclaimable_pages(zone);
                 }
         }
 
@@ -1651,13 +1787,13 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
                  */
                 if (total_scanned > sc->swap_cluster_max +
                                         sc->swap_cluster_max / 2) {
-                        wakeup_pdflush(laptop_mode ? 0 : total_scanned);
+                        wakeup_flusher_threads(laptop_mode ? 0 : total_scanned);
                         sc->may_writepage = 1;
                 }
 
                 /* Take a nap, wait for some writeback to complete */
                 if (sc->nr_scanned && priority < DEF_PRIORITY - 2)
-                        congestion_wait(WRITE, HZ/10);
+                        congestion_wait(BLK_RW_ASYNC, HZ/10);
         }
         /* top priority shrink_zones still had more to do? don't OOM, then */
         if (!sc->all_unreclaimable && scanning_global_lru(sc))
@@ -1697,6 +1833,7 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
                 .may_writepage = !laptop_mode,
                 .swap_cluster_max = SWAP_CLUSTER_MAX,
                 .may_unmap = 1,
+                .may_swap = 1,
                 .swappiness = vm_swappiness,
                 .order = order,
                 .mem_cgroup = NULL,
@@ -1709,14 +1846,49 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
 
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR
 
+unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *mem,
+                                                gfp_t gfp_mask, bool noswap,
+                                                unsigned int swappiness,
+                                                struct zone *zone, int nid)
+{
+        struct scan_control sc = {
+                .may_writepage = !laptop_mode,
+                .may_unmap = 1,
+                .may_swap = !noswap,
+                .swap_cluster_max = SWAP_CLUSTER_MAX,
+                .swappiness = swappiness,
+                .order = 0,
+                .mem_cgroup = mem,
+                .isolate_pages = mem_cgroup_isolate_pages,
+        };
+        nodemask_t nm  = nodemask_of_node(nid);
+
+        sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
+                        (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
+        sc.nodemask = &nm;
+        sc.nr_reclaimed = 0;
+        sc.nr_scanned = 0;
+        /*
+         * NOTE: Although we can get the priority field, using it
+         * here is not a good idea, since it limits the pages we can scan.
+         * if we don't reclaim here, the shrink_zone from balance_pgdat
+         * will pick up pages from other mem cgroup's as well. We hack
+         * the priority and make it zero.
+         */
+        shrink_zone(0, zone, &sc);
+        return sc.nr_reclaimed;
+}
+
 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *mem_cont,
                                            gfp_t gfp_mask,
                                            bool noswap,
                                            unsigned int swappiness)
 {
+        struct zonelist *zonelist;
         struct scan_control sc = {
                 .may_writepage = !laptop_mode,
                 .may_unmap = 1,
+                .may_swap = !noswap,
                 .swap_cluster_max = SWAP_CLUSTER_MAX,
                 .swappiness = swappiness,
                 .order = 0,
@@ -1724,10 +1896,6 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *mem_cont,
                 .isolate_pages = mem_cgroup_isolate_pages,
                 .nodemask = NULL, /* we don't care the placement */
         };
-        struct zonelist *zonelist;
-
-        if (noswap)
-                sc.may_unmap = 0;
 
         sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
                         (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
@@ -1738,7 +1906,7 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *mem_cont,
 
 /*
  * For kswapd, balance_pgdat() will work across all this node's zones until
- * they are all at pages_high.
+ * they are all at high_wmark_pages(zone).
  *
  * Returns the number of pages which were actually freed.
  *
@@ -1751,11 +1919,11 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *mem_cont,
  * the zone for when the problem goes away.
  *
  * kswapd scans the zones in the highmem->normal->dma direction.  It skips
- * zones which have free_pages > pages_high, but once a zone is found to have
- * free_pages <= pages_high, we scan that zone and the lower zones regardless
- * of the number of free pages in the lower zones.  This interoperates with
- * the page allocator fallback scheme to ensure that aging of pages is balanced
- * across the zones.
+ * zones which have free_pages > high_wmark_pages(zone), but once a zone is
+ * found to have free_pages <= high_wmark_pages(zone), we scan that zone and the
+ * lower zones regardless of the number of free pages in the lower zones. This
+ * interoperates with the page allocator fallback scheme to ensure that aging
+ * of pages is balanced across the zones.
  */
 static unsigned long balance_pgdat(pg_data_t *pgdat, int order)
 {
@@ -1767,6 +1935,7 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order)
         struct scan_control sc = {
                 .gfp_mask = GFP_KERNEL,
                 .may_unmap = 1,
+                .may_swap = 1,
                 .swap_cluster_max = SWAP_CLUSTER_MAX,
                 .swappiness = vm_swappiness,
                 .order = order,
@@ -1775,7 +1944,8 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order)
         };
         /*
          * temp_priority is used to remember the scanning priority at which
-         * this zone was successfully refilled to free_pages == pages_high.
+         * this zone was successfully refilled to
+         * free_pages == high_wmark_pages(zone).
          */
         int temp_priority[MAX_NR_ZONES];
 
@@ -1820,8 +1990,8 @@ loop_again:
                                 shrink_active_list(SWAP_CLUSTER_MAX, zone,
                                                         &sc, priority, 0);
 
-                        if (!zone_watermark_ok(zone, order, zone->pages_high,
-                                               0, 0)) {
+                        if (!zone_watermark_ok(zone, order,
+                                        high_wmark_pages(zone), 0, 0)) {
                                 end_zone = i;
                                 break;
                         }
@@ -1832,7 +2002,7 @@ loop_again:
                 for (i = 0; i <= end_zone; i++) {
                         struct zone *zone = pgdat->node_zones + i;
 
-                        lru_pages += zone_lru_pages(zone);
+                        lru_pages += zone_reclaimable_pages(zone);
                 }
 
                 /*
@@ -1847,6 +2017,7 @@ loop_again:
                 for (i = 0; i <= end_zone; i++) {
                         struct zone *zone = pgdat->node_zones + i;
                         int nr_slab;
+                        int nid, zid;
 
                         if (!populated_zone(zone))
                                 continue;
@@ -1855,18 +2026,27 @@ loop_again:
                                         priority != DEF_PRIORITY)
                                 continue;
 
-                        if (!zone_watermark_ok(zone, order, zone->pages_high,
-                                               end_zone, 0))
+                        if (!zone_watermark_ok(zone, order,
+                                        high_wmark_pages(zone), end_zone, 0))
                                 all_zones_ok = 0;
                         temp_priority[i] = priority;
                         sc.nr_scanned = 0;
                         note_zone_scanning_priority(zone, priority);
+
+                        nid = pgdat->node_id;
+                        zid = zone_idx(zone);
+                        /*
+                         * Call soft limit reclaim before calling shrink_zone.
+                         * For now we ignore the return value
+                         */
+                        mem_cgroup_soft_limit_reclaim(zone, order, sc.gfp_mask,
+                                                        nid, zid);
                         /*
                          * We put equal pressure on every zone, unless one
                          * zone has way too many pages free already.
                          */
-                        if (!zone_watermark_ok(zone, order, 8*zone->pages_high,
-                                                end_zone, 0))
+                        if (!zone_watermark_ok(zone, order,
+                                        8*high_wmark_pages(zone), end_zone, 0))
                                 shrink_zone(priority, zone, &sc);
                         reclaim_state->reclaimed_slab = 0;
                         nr_slab = shrink_slab(sc.nr_scanned, GFP_KERNEL,
@@ -1876,7 +2056,7 @@ loop_again:
                         if (zone_is_all_unreclaimable(zone))
                                 continue;
                         if (nr_slab == 0 && zone->pages_scanned >=
-                                                (zone_lru_pages(zone) * 6))
+                                        (zone_reclaimable_pages(zone) * 6))
                                         zone_set_flag(zone,
                                                       ZONE_ALL_UNRECLAIMABLE);
                         /*
@@ -1895,7 +2075,7 @@ loop_again:
                  * another pass across the zones.
                  */
                 if (total_scanned && priority < DEF_PRIORITY - 2)
-                        congestion_wait(WRITE, HZ/10);
+                        congestion_wait(BLK_RW_ASYNC, HZ/10);
 
                 /*
                  * We do this so kswapd doesn't build up large priorities for
@@ -1967,7 +2147,7 @@ static int kswapd(void *p)
         struct reclaim_state reclaim_state = {
                 .reclaimed_slab = 0,
         };
-        node_to_cpumask_ptr(cpumask, pgdat->node_id);
+        const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
 
         lockdep_set_current_reclaim_state(GFP_KERNEL);
 
@@ -2032,7 +2212,7 @@ void wakeup_kswapd(struct zone *zone, int order)
                 return;
 
         pgdat = zone->zone_pgdat;
-        if (zone_watermark_ok(zone, order, zone->pages_low, 0, 0))
+        if (zone_watermark_ok(zone, order, low_wmark_pages(zone), 0, 0))
                 return;
         if (pgdat->kswapd_max_order < order)
                 pgdat->kswapd_max_order = order;
@@ -2043,15 +2223,42 @@ void wakeup_kswapd(struct zone *zone, int order)
         wake_up_interruptible(&pgdat->kswapd_wait);
 }
 
-unsigned long global_lru_pages(void)
+/*
+ * The reclaimable count would be mostly accurate.
+ * The less reclaimable pages may be
+ * - mlocked pages, which will be moved to unevictable list when encountered
+ * - mapped pages, which may require several travels to be reclaimed
+ * - dirty pages, which is not "instantly" reclaimable
+ */
+unsigned long global_reclaimable_pages(void)
 {
-        return global_page_state(NR_ACTIVE_ANON)
-                + global_page_state(NR_ACTIVE_FILE)
-                + global_page_state(NR_INACTIVE_ANON)
-                + global_page_state(NR_INACTIVE_FILE);
+        int nr;
+
+        nr = global_page_state(NR_ACTIVE_FILE) +
+             global_page_state(NR_INACTIVE_FILE);
+
+        if (nr_swap_pages > 0)
+                nr += global_page_state(NR_ACTIVE_ANON) +
+                      global_page_state(NR_INACTIVE_ANON);
+
+        return nr;
+}
+
+unsigned long zone_reclaimable_pages(struct zone *zone)
+{
+        int nr;
+
+        nr = zone_page_state(zone, NR_ACTIVE_FILE) +
+             zone_page_state(zone, NR_INACTIVE_FILE);
+
+        if (nr_swap_pages > 0)
+                nr += zone_page_state(zone, NR_ACTIVE_ANON) +
+                      zone_page_state(zone, NR_INACTIVE_ANON);
+
+        return nr;
 }
 
-#ifdef CONFIG_PM
+#ifdef CONFIG_HIBERNATION
 /*
  * Helper function for shrink_all_memory().  Tries to reclaim 'nr_pages' pages
  * from LRU lists system-wide, for given pass and priority.
@@ -2063,6 +2270,7 @@ static void shrink_all_zones(unsigned long nr_pages, int prio,
 {
         struct zone *zone;
         unsigned long nr_reclaimed = 0;
+        struct zone_reclaim_stat *reclaim_stat;
 
         for_each_populated_zone(zone) {
                 enum lru_list l;
@@ -2079,22 +2287,25 @@ static void shrink_all_zones(unsigned long nr_pages, int prio,
                                                 l == LRU_ACTIVE_FILE))
                                 continue;
 
-                        zone->lru[l].nr_scan += (lru_pages >> prio) + 1;
-                        if (zone->lru[l].nr_scan >= nr_pages || pass > 3) {
+                        reclaim_stat = get_reclaim_stat(zone, sc);
+                        reclaim_stat->nr_saved_scan[l] +=
+                                                (lru_pages >> prio) + 1;
+                        if (reclaim_stat->nr_saved_scan[l]
+                                                >= nr_pages || pass > 3) {
                                 unsigned long nr_to_scan;
 
-                                zone->lru[l].nr_scan = 0;
+                                reclaim_stat->nr_saved_scan[l] = 0;
                                 nr_to_scan = min(nr_pages, lru_pages);
                                 nr_reclaimed += shrink_list(l, nr_to_scan, zone,
                                                                 sc, prio);
                                 if (nr_reclaimed >= nr_pages) {
-                                        sc->nr_reclaimed = nr_reclaimed;
+                                        sc->nr_reclaimed += nr_reclaimed;
                                         return;
                                 }
                         }
                 }
         }
-        sc->nr_reclaimed = nr_reclaimed;
+        sc->nr_reclaimed += nr_reclaimed;
 }
 
 /*
@@ -2115,11 +2326,12 @@ unsigned long shrink_all_memory(unsigned long nr_pages)
                 .may_unmap = 0,
                 .may_writepage = 1,
                 .isolate_pages = isolate_pages_global,
+                .nr_reclaimed = 0,
         };
 
         current->reclaim_state = &reclaim_state;
 
-        lru_pages = global_lru_pages();
+        lru_pages = global_reclaimable_pages();
         nr_slab = global_page_state(NR_SLAB_RECLAIMABLE);
         /* If slab caches are huge, it's better to hit them first */
         while (nr_slab >= lru_pages) {
@@ -2161,13 +2373,13 @@ unsigned long shrink_all_memory(unsigned long nr_pages)
 
                         reclaim_state.reclaimed_slab = 0;
                         shrink_slab(sc.nr_scanned, sc.gfp_mask,
-                                        global_lru_pages());
+                                    global_reclaimable_pages());
                         sc.nr_reclaimed += reclaim_state.reclaimed_slab;
                         if (sc.nr_reclaimed >= nr_pages)
                                 goto out;
 
                         if (sc.nr_scanned && prio < DEF_PRIORITY - 2)
-                                congestion_wait(WRITE, HZ / 10);
+                                congestion_wait(BLK_RW_ASYNC, HZ / 10);
                 }
         }
 
@@ -2178,7 +2390,8 @@ unsigned long shrink_all_memory(unsigned long nr_pages)
         if (!sc.nr_reclaimed) {
                 do {
                         reclaim_state.reclaimed_slab = 0;
-                        shrink_slab(nr_pages, sc.gfp_mask, global_lru_pages());
+                        shrink_slab(nr_pages, sc.gfp_mask,
+                                    global_reclaimable_pages());
                         sc.nr_reclaimed += reclaim_state.reclaimed_slab;
                 } while (sc.nr_reclaimed < nr_pages &&
                                 reclaim_state.reclaimed_slab > 0);
@@ -2190,7 +2403,7 @@ out:
 
         return sc.nr_reclaimed;
 }
-#endif
+#endif /* CONFIG_HIBERNATION */
 
 /* It's optimal to keep kswapds on the same CPUs as their memory, but
    not required for correctness.  So if the last cpu in a node goes
@@ -2204,7 +2417,9 @@ static int __devinit cpu_callback(struct notifier_block *nfb,
         if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) {
                 for_each_node_state(nid, N_HIGH_MEMORY) {
                         pg_data_t *pgdat = NODE_DATA(nid);
-                        node_to_cpumask_ptr(mask, pgdat->node_id);
+                        const struct cpumask *mask;
+
+                        mask = cpumask_of_node(pgdat->node_id);
 
                         if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
                                 /* One of our CPUs online: restore mask */
@@ -2282,6 +2497,48 @@ int sysctl_min_unmapped_ratio = 1;
  */
 int sysctl_min_slab_ratio = 5;
 
+static inline unsigned long zone_unmapped_file_pages(struct zone *zone)
+{
+        unsigned long file_mapped = zone_page_state(zone, NR_FILE_MAPPED);
+        unsigned long file_lru = zone_page_state(zone, NR_INACTIVE_FILE) +
+                zone_page_state(zone, NR_ACTIVE_FILE);
+
+        /*
+         * It's possible for there to be more file mapped pages than
+         * accounted for by the pages on the file LRU lists because
+         * tmpfs pages accounted for as ANON can also be FILE_MAPPED
+         */
+        return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
+}
+
+/* Work out how many page cache pages we can reclaim in this reclaim_mode */
+static long zone_pagecache_reclaimable(struct zone *zone)
+{
+        long nr_pagecache_reclaimable;
+        long delta = 0;
+
+        /*
+         * If RECLAIM_SWAP is set, then all file pages are considered
+         * potentially reclaimable. Otherwise, we have to worry about
+         * pages like swapcache and zone_unmapped_file_pages() provides
+         * a better estimate
+         */
+        if (zone_reclaim_mode & RECLAIM_SWAP)
+                nr_pagecache_reclaimable = zone_page_state(zone, NR_FILE_PAGES);
+        else
+                nr_pagecache_reclaimable = zone_unmapped_file_pages(zone);
+
+        /* If we can't clean pages, remove dirty pages from consideration */
+        if (!(zone_reclaim_mode & RECLAIM_WRITE))
+                delta += zone_page_state(zone, NR_FILE_DIRTY);
+
+        /* Watch for any possible underflows due to delta */
+        if (unlikely(delta > nr_pagecache_reclaimable))
+                delta = nr_pagecache_reclaimable;
+
+        return nr_pagecache_reclaimable - delta;
+}
+
 /*
  * Try to free up some pages from this zone through reclaim.
  */
@@ -2295,6 +2552,7 @@ static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
         struct scan_control sc = {
                 .may_writepage = !!(zone_reclaim_mode & RECLAIM_WRITE),
                 .may_unmap = !!(zone_reclaim_mode & RECLAIM_SWAP),
+                .may_swap = 1,
                 .swap_cluster_max = max_t(unsigned long, nr_pages,
                                         SWAP_CLUSTER_MAX),
                 .gfp_mask = gfp_mask,
@@ -2315,9 +2573,7 @@ static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
         reclaim_state.reclaimed_slab = 0;
         p->reclaim_state = &reclaim_state;
 
-        if (zone_page_state(zone, NR_FILE_PAGES) -
-                zone_page_state(zone, NR_FILE_MAPPED) >
-                zone->min_unmapped_pages) {
+        if (zone_pagecache_reclaimable(zone) > zone->min_unmapped_pages) {
                 /*
                  * Free memory by calling shrink zone with increasing
                  * priorities until we have enough memory freed.
@@ -2375,20 +2631,18 @@ int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
          * if less than a specified percentage of the zone is used by
          * unmapped file backed pages.
          */
-        if (zone_page_state(zone, NR_FILE_PAGES) -
-            zone_page_state(zone, NR_FILE_MAPPED) <= zone->min_unmapped_pages
-            && zone_page_state(zone, NR_SLAB_RECLAIMABLE)
-                        <= zone->min_slab_pages)
-                return 0;
+        if (zone_pagecache_reclaimable(zone) <= zone->min_unmapped_pages &&
+            zone_page_state(zone, NR_SLAB_RECLAIMABLE) <= zone->min_slab_pages)
+                return ZONE_RECLAIM_FULL;
 
         if (zone_is_all_unreclaimable(zone))
-                return 0;
+                return ZONE_RECLAIM_FULL;
 
         /*
          * Do not scan if the allocation should not be delayed.
          */
         if (!(gfp_mask & __GFP_WAIT) || (current->flags & PF_MEMALLOC))
-                        return 0;
+                return ZONE_RECLAIM_NOSCAN;
 
         /*
          * Only run zone reclaim on the local zone or on zones that do not
@@ -2398,18 +2652,21 @@ int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
          */
         node_id = zone_to_nid(zone);
         if (node_state(node_id, N_CPU) && node_id != numa_node_id())
-                return 0;
+                return ZONE_RECLAIM_NOSCAN;
 
         if (zone_test_and_set_flag(zone, ZONE_RECLAIM_LOCKED))
-                return 0;
+                return ZONE_RECLAIM_NOSCAN;
+
         ret = __zone_reclaim(zone, gfp_mask, order);
         zone_clear_flag(zone, ZONE_RECLAIM_LOCKED);
 
+        if (!ret)
+                count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
+
         return ret;
 }
 #endif
 
-#ifdef CONFIG_UNEVICTABLE_LRU
 /*
  * page_evictable - test whether a page is evictable
  * @page: the page to test
@@ -2454,7 +2711,7 @@ static void check_move_unevictable_page(struct page *page, struct zone *zone)
 retry:
         ClearPageUnevictable(page);
         if (page_evictable(page, NULL)) {
-                enum lru_list l = LRU_INACTIVE_ANON + page_is_file_cache(page);
+                enum lru_list l = page_lru_base_type(page);
 
                 __dec_zone_state(zone, NR_UNEVICTABLE);
                 list_move(&page->lru, &zone->lru[l].list);
@@ -2597,10 +2854,10 @@ static void scan_all_zones_unevictable_pages(void)
 unsigned long scan_unevictable_pages;
 
 int scan_unevictable_handler(struct ctl_table *table, int write,
-                           struct file *file, void __user *buffer,
+                           void __user *buffer,
                            size_t *length, loff_t *ppos)
 {
-        proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
+        proc_doulongvec_minmax(table, write, buffer, length, ppos);
 
         if (write && *(unsigned long *)table->data)
                 scan_all_zones_unevictable_pages();
@@ -2656,4 +2913,3 @@ void scan_unevictable_unregister_node(struct node *node)
         sysdev_remove_file(&node->sysdev, &attr_scan_unevictable_pages);
 }
 
-#endif
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 9826766f1274..c81321f9feec 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -509,22 +509,11 @@ static void pagetypeinfo_showblockcount_print(struct seq_file *m,
                         continue;
 
                 page = pfn_to_page(pfn);
-#ifdef CONFIG_ARCH_FLATMEM_HAS_HOLES
-                /*
-                 * Ordinarily, memory holes in flatmem still have a valid
-                 * memmap for the PFN range. However, an architecture for
-                 * embedded systems (e.g. ARM) can free up the memmap backing
-                 * holes to save memory on the assumption the memmap is
-                 * never used. The page_zone linkages are then broken even
-                 * though pfn_valid() returns true. Skip the page if the
-                 * linkages are broken. Even if this test passed, the impact
-                 * is that the counters for the movable type are off but
-                 * fragmentation monitoring is likely meaningless on small
-                 * systems.
-                 */
-                if (page_zone(page) != zone)
+
+                /* Watch for unexpected holes punched in the memmap */
+                if (!memmap_valid_within(pfn, page, zone))
                         continue;
-#endif
+
                 mtype = get_pageblock_migratetype(page);
 
                 if (mtype < MIGRATE_TYPES)
@@ -640,10 +629,8 @@ static const char * const vmstat_text[] = {
         "nr_active_anon",
         "nr_inactive_file",
         "nr_active_file",
-#ifdef CONFIG_UNEVICTABLE_LRU
         "nr_unevictable",
         "nr_mlock",
-#endif
         "nr_anon_pages",
         "nr_mapped",
         "nr_file_pages",
@@ -652,11 +639,14 @@ static const char * const vmstat_text[] = {
         "nr_slab_reclaimable",
         "nr_slab_unreclaimable",
         "nr_page_table_pages",
+        "nr_kernel_stack",
         "nr_unstable",
         "nr_bounce",
         "nr_vmscan_write",
         "nr_writeback_temp",
-
+        "nr_isolated_anon",
+        "nr_isolated_file",
+        "nr_shmem",
 #ifdef CONFIG_NUMA
         "numa_hit",
         "numa_miss",
@@ -686,6 +676,9 @@ static const char * const vmstat_text[] = {
         TEXTS_FOR_ZONES("pgscan_kswapd")
         TEXTS_FOR_ZONES("pgscan_direct")
 
+#ifdef CONFIG_NUMA
+        "zone_reclaim_failed",
+#endif
         "pginodesteal",
         "slabs_scanned",
         "kswapd_steal",
@@ -698,7 +691,6 @@ static const char * const vmstat_text[] = {
         "htlb_buddy_alloc_success",
         "htlb_buddy_alloc_fail",
 #endif
-#ifdef CONFIG_UNEVICTABLE_LRU
         "unevictable_pgs_culled",
         "unevictable_pgs_scanned",
         "unevictable_pgs_rescued",
@@ -708,7 +700,6 @@ static const char * const vmstat_text[] = {
         "unevictable_pgs_stranded",
         "unevictable_pgs_mlockfreed",
 #endif
-#endif
 };
 
 static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
@@ -721,18 +712,14 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
                    "\n        min      %lu"
                    "\n        low      %lu"
                    "\n        high     %lu"
-                   "\n        scanned  %lu (aa: %lu ia: %lu af: %lu if: %lu)"
+                   "\n        scanned  %lu"
                    "\n        spanned  %lu"
                    "\n        present  %lu",
                    zone_page_state(zone, NR_FREE_PAGES),
-                   zone->pages_min,
-                   zone->pages_low,
-                   zone->pages_high,
+                   min_wmark_pages(zone),
+                   low_wmark_pages(zone),
+                   high_wmark_pages(zone),
                    zone->pages_scanned,
-                   zone->lru[LRU_ACTIVE_ANON].nr_scan,
-                   zone->lru[LRU_INACTIVE_ANON].nr_scan,
-                   zone->lru[LRU_ACTIVE_FILE].nr_scan,
-                   zone->lru[LRU_INACTIVE_FILE].nr_scan,
                    zone->spanned_pages,
                    zone->present_pages);
 
@@ -891,7 +878,7 @@ static void vmstat_update(struct work_struct *w)
 {
         refresh_cpu_vm_stats(smp_processor_id());
         schedule_delayed_work(&__get_cpu_var(vmstat_work),
-                sysctl_stat_interval);
+                round_jiffies_relative(sysctl_stat_interval));
 }
 
 static void __cpuinit start_cpu_timer(int cpu)
@@ -899,7 +886,8 @@ static void __cpuinit start_cpu_timer(int cpu)
         struct delayed_work *vmstat_work = &per_cpu(vmstat_work, cpu);
 
         INIT_DELAYED_WORK_DEFERRABLE(vmstat_work, vmstat_update);
-        schedule_delayed_work_on(cpu, vmstat_work, HZ + cpu);
+        schedule_delayed_work_on(cpu, vmstat_work,
+                                 __round_jiffies_relative(HZ, cpu));
 }
 
 /*