1 files changed, 160 insertions, 74 deletions

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 4545d5944243..08f40a2f3fe0 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -73,7 +73,6 @@
 #include <linux/sched.h>
 #include <linux/nodemask.h>
 #include <linux/cpuset.h>
-#include <linux/gfp.h>
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/module.h>
@@ -85,10 +84,12 @@
 #include <linux/seq_file.h>
 #include <linux/proc_fs.h>
 #include <linux/migrate.h>
+#include <linux/ksm.h>
 #include <linux/rmap.h>
 #include <linux/security.h>
 #include <linux/syscalls.h>
 #include <linux/ctype.h>
+#include <linux/mm_inline.h>
 
 #include <asm/tlbflush.h>
 #include <asm/uaccess.h>
@@ -412,17 +413,11 @@ static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
                 if (!page)
                         continue;
                 /*
-                 * The check for PageReserved here is important to avoid
-                 * handling zero pages and other pages that may have been
-                 * marked special by the system.
-                 *
-                 * If the PageReserved would not be checked here then f.e.
-                 * the location of the zero page could have an influence
-                 * on MPOL_MF_STRICT, zero pages would be counted for
-                 * the per node stats, and there would be useless attempts
-                 * to put zero pages on the migration list.
+                 * vm_normal_page() filters out zero pages, but there might
+                 * still be PageReserved pages to skip, perhaps in a VDSO.
+                 * And we cannot move PageKsm pages sensibly or safely yet.
                  */
-                if (PageReserved(page))
+                if (PageReserved(page) || PageKsm(page))
                         continue;
                 nid = page_to_nid(page);
                 if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
@@ -567,24 +562,50 @@ static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new)
 }
 
 /* Step 2: apply policy to a range and do splits. */
-static int mbind_range(struct vm_area_struct *vma, unsigned long start,
-                       unsigned long end, struct mempolicy *new)
+static int mbind_range(struct mm_struct *mm, unsigned long start,
+                       unsigned long end, struct mempolicy *new_pol)
 {
         struct vm_area_struct *next;
-        int err;
+        struct vm_area_struct *prev;
+        struct vm_area_struct *vma;
+        int err = 0;
+        pgoff_t pgoff;
+        unsigned long vmstart;
+        unsigned long vmend;
 
-        err = 0;
-        for (; vma && vma->vm_start < end; vma = next) {
+        vma = find_vma_prev(mm, start, &prev);
+        if (!vma || vma->vm_start > start)
+                return -EFAULT;
+
+        for (; vma && vma->vm_start < end; prev = vma, vma = next) {
                 next = vma->vm_next;
-                if (vma->vm_start < start)
-                        err = split_vma(vma->vm_mm, vma, start, 1);
-                if (!err && vma->vm_end > end)
-                        err = split_vma(vma->vm_mm, vma, end, 0);
-                if (!err)
-                        err = policy_vma(vma, new);
+                vmstart = max(start, vma->vm_start);
+                vmend   = min(end, vma->vm_end);
+
+                pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
+                prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
+                                  vma->anon_vma, vma->vm_file, pgoff, new_pol);
+                if (prev) {
+                        vma = prev;
+                        next = vma->vm_next;
+                        continue;
+                }
+                if (vma->vm_start != vmstart) {
+                        err = split_vma(vma->vm_mm, vma, vmstart, 1);
+                        if (err)
+                                goto out;
+                }
+                if (vma->vm_end != vmend) {
+                        err = split_vma(vma->vm_mm, vma, vmend, 0);
+                        if (err)
+                                goto out;
+                }
+                err = policy_vma(vma, new_pol);
                 if (err)
-                        break;
+                        goto out;
         }
+
+ out:
         return err;
 }
 
@@ -784,9 +805,13 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask,
 
         err = 0;
         if (nmask) {
-                task_lock(current);
-                get_policy_nodemask(pol, nmask);
-                task_unlock(current);
+                if (mpol_store_user_nodemask(pol)) {
+                        *nmask = pol->w.user_nodemask;
+                } else {
+                        task_lock(current);
+                        get_policy_nodemask(pol, nmask);
+                        task_unlock(current);
+                }
         }
 
  out:
@@ -809,6 +834,8 @@ static void migrate_page_add(struct page *page, struct list_head *pagelist,
         if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) {
                 if (!isolate_lru_page(page)) {
                         list_add_tail(&page->lru, pagelist);
+                        inc_zone_page_state(page, NR_ISOLATED_ANON +
+                                            page_is_file_cache(page));
                 }
         }
 }
@@ -836,7 +863,7 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
                         flags | MPOL_MF_DISCONTIG_OK, &pagelist);
 
         if (!list_empty(&pagelist))
-                err = migrate_pages(&pagelist, new_node_page, dest);
+                err = migrate_pages(&pagelist, new_node_page, dest, 0);
 
         return err;
 }
@@ -864,36 +891,36 @@ int do_migrate_pages(struct mm_struct *mm,
         if (err)
                 goto out;
 
-/*
- * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
- * bit in 'to' is not also set in 'tmp'.  Clear the found 'source'
- * bit in 'tmp', and return that <source, dest> pair for migration.
- * The pair of nodemasks 'to' and 'from' define the map.
- *
- * If no pair of bits is found that way, fallback to picking some
- * pair of 'source' and 'dest' bits that are not the same.  If the
- * 'source' and 'dest' bits are the same, this represents a node
- * that will be migrating to itself, so no pages need move.
- *
- * If no bits are left in 'tmp', or if all remaining bits left
- * in 'tmp' correspond to the same bit in 'to', return false
- * (nothing left to migrate).
- *
- * This lets us pick a pair of nodes to migrate between, such that
- * if possible the dest node is not already occupied by some other
- * source node, minimizing the risk of overloading the memory on a
- * node that would happen if we migrated incoming memory to a node
- * before migrating outgoing memory source that same node.
- *
- * A single scan of tmp is sufficient.  As we go, we remember the
- * most recent <s, d> pair that moved (s != d).  If we find a pair
- * that not only moved, but what's better, moved to an empty slot
- * (d is not set in tmp), then we break out then, with that pair.
- * Otherwise when we finish scannng from_tmp, we at least have the
- * most recent <s, d> pair that moved.  If we get all the way through
- * the scan of tmp without finding any node that moved, much less
- * moved to an empty node, then there is nothing left worth migrating.
- */
+        /*
+         * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
+         * bit in 'to' is not also set in 'tmp'.  Clear the found 'source'
+         * bit in 'tmp', and return that <source, dest> pair for migration.
+         * The pair of nodemasks 'to' and 'from' define the map.
+         *
+         * If no pair of bits is found that way, fallback to picking some
+         * pair of 'source' and 'dest' bits that are not the same.  If the
+         * 'source' and 'dest' bits are the same, this represents a node
+         * that will be migrating to itself, so no pages need move.
+         *
+         * If no bits are left in 'tmp', or if all remaining bits left
+         * in 'tmp' correspond to the same bit in 'to', return false
+         * (nothing left to migrate).
+         *
+         * This lets us pick a pair of nodes to migrate between, such that
+         * if possible the dest node is not already occupied by some other
+         * source node, minimizing the risk of overloading the memory on a
+         * node that would happen if we migrated incoming memory to a node
+         * before migrating outgoing memory source that same node.
+         *
+         * A single scan of tmp is sufficient.  As we go, we remember the
+         * most recent <s, d> pair that moved (s != d).  If we find a pair
+         * that not only moved, but what's better, moved to an empty slot
+         * (d is not set in tmp), then we break out then, with that pair.
+         * Otherwise when we finish scannng from_tmp, we at least have the
+         * most recent <s, d> pair that moved.  If we get all the way through
+         * the scan of tmp without finding any node that moved, much less
+         * moved to an empty node, then there is nothing left worth migrating.
+         */
 
         tmp = *from_nodes;
         while (!nodes_empty(tmp)) {
@@ -1049,11 +1076,11 @@ static long do_mbind(unsigned long start, unsigned long len,
         if (!IS_ERR(vma)) {
                 int nr_failed = 0;
 
-                err = mbind_range(vma, start, end, new);
+                err = mbind_range(mm, start, end, new);
 
                 if (!list_empty(&pagelist))
                         nr_failed = migrate_pages(&pagelist, new_vma_page,
-                                                (unsigned long)vma);
+                                                (unsigned long)vma, 0);
 
                 if (!err && nr_failed && (flags & MPOL_MF_STRICT))
                         err = -EIO;
@@ -1565,6 +1592,53 @@ struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
         }
         return zl;
 }
+
+/*
+ * init_nodemask_of_mempolicy
+ *
+ * If the current task's mempolicy is "default" [NULL], return 'false'
+ * to indicate default policy.  Otherwise, extract the policy nodemask
+ * for 'bind' or 'interleave' policy into the argument nodemask, or
+ * initialize the argument nodemask to contain the single node for
+ * 'preferred' or 'local' policy and return 'true' to indicate presence
+ * of non-default mempolicy.
+ *
+ * We don't bother with reference counting the mempolicy [mpol_get/put]
+ * because the current task is examining it's own mempolicy and a task's
+ * mempolicy is only ever changed by the task itself.
+ *
+ * N.B., it is the caller's responsibility to free a returned nodemask.
+ */
+bool init_nodemask_of_mempolicy(nodemask_t *mask)
+{
+        struct mempolicy *mempolicy;
+        int nid;
+
+        if (!(mask && current->mempolicy))
+                return false;
+
+        mempolicy = current->mempolicy;
+        switch (mempolicy->mode) {
+        case MPOL_PREFERRED:
+                if (mempolicy->flags & MPOL_F_LOCAL)
+                        nid = numa_node_id();
+                else
+                        nid = mempolicy->v.preferred_node;
+                init_nodemask_of_node(mask, nid);
+                break;
+
+        case MPOL_BIND:
+                /* Fall through */
+        case MPOL_INTERLEAVE:
+                *mask =  mempolicy->v.nodes;
+                break;
+
+        default:
+                BUG();
+        }
+
+        return true;
+}
 #endif
 
 /* Allocate a page in interleaved policy.
@@ -1685,10 +1759,12 @@ struct mempolicy *__mpol_dup(struct mempolicy *old)
 
         if (!new)
                 return ERR_PTR(-ENOMEM);
+        rcu_read_lock();
         if (current_cpuset_is_being_rebound()) {
                 nodemask_t mems = cpuset_mems_allowed(current);
                 mpol_rebind_policy(old, &mems);
         }
+        rcu_read_unlock();
         *new = *old;
         atomic_set(&new->refcnt, 1);
         return new;
@@ -2122,8 +2198,8 @@ int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context)
                         char *rest = nodelist;
                         while (isdigit(*rest))
                                 rest++;
-                        if (!*rest)
-                                err = 0;
+                        if (*rest)
+                                goto out;
                 }
                 break;
         case MPOL_INTERLEAVE:
@@ -2132,7 +2208,6 @@ int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context)
                  */
                 if (!nodelist)
                         nodes = node_states[N_HIGH_MEMORY];
-                err = 0;
                 break;
         case MPOL_LOCAL:
                 /*
@@ -2142,11 +2217,19 @@ int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context)
                         goto out;
                 mode = MPOL_PREFERRED;
                 break;
-
-        /*
-         * case MPOL_BIND:    mpol_new() enforces non-empty nodemask.
-         * case MPOL_DEFAULT: mpol_new() enforces empty nodemask, ignores flags.
-         */
+        case MPOL_DEFAULT:
+                /*
+                 * Insist on a empty nodelist
+                 */
+                if (!nodelist)
+                        err = 0;
+                goto out;
+        case MPOL_BIND:
+                /*
+                 * Insist on a nodelist
+                 */
+                if (!nodelist)
+                        goto out;
         }
 
         mode_flags = 0;
@@ -2160,13 +2243,14 @@ int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context)
                 else if (!strcmp(flags, "relative"))
                         mode_flags |= MPOL_F_RELATIVE_NODES;
                 else
-                        err = 1;
+                        goto out;
         }
 
         new = mpol_new(mode, mode_flags, &nodes);
         if (IS_ERR(new))
-                err = 1;
-        else {
+                goto out;
+
+        {
                 int ret;
                 NODEMASK_SCRATCH(scratch);
                 if (scratch) {
@@ -2177,13 +2261,15 @@ int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context)
                         ret = -ENOMEM;
                 NODEMASK_SCRATCH_FREE(scratch);
                 if (ret) {
-                        err = 1;
                         mpol_put(new);
-                } else if (no_context) {
-                        /* save for contextualization */
-                        new->w.user_nodemask = nodes;
+                        goto out;
                 }
         }
+        err = 0;
+        if (no_context) {
+                /* save for contextualization */
+                new->w.user_nodemask = nodes;
+        }
 
 out:
         /* Restore string for error message */