4 files changed, 147 insertions, 65 deletions

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index ddffc74cdebe..e013b8e57d25 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -108,10 +108,12 @@ enum mem_cgroup_events_index {
 enum mem_cgroup_events_target {
         MEM_CGROUP_TARGET_THRESH,
         MEM_CGROUP_TARGET_SOFTLIMIT,
+        MEM_CGROUP_TARGET_NUMAINFO,
         MEM_CGROUP_NTARGETS,
 };
 #define THRESHOLDS_EVENTS_TARGET (128)
 #define SOFTLIMIT_EVENTS_TARGET (1024)
+#define NUMAINFO_EVENTS_TARGET  (1024)
 
 struct mem_cgroup_stat_cpu {
         long count[MEM_CGROUP_STAT_NSTATS];
@@ -237,7 +239,8 @@ struct mem_cgroup {
         int last_scanned_node;
 #if MAX_NUMNODES > 1
         nodemask_t      scan_nodes;
-        unsigned long   next_scan_node_update;
+        atomic_t        numainfo_events;
+        atomic_t        numainfo_updating;
 #endif
         /*
          * Should the accounting and control be hierarchical, per subtree?
@@ -577,15 +580,6 @@ static long mem_cgroup_read_stat(struct mem_cgroup *mem,
         return val;
 }
 
-static long mem_cgroup_local_usage(struct mem_cgroup *mem)
-{
-        long ret;
-
-        ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
-        ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
-        return ret;
-}
-
 static void mem_cgroup_swap_statistics(struct mem_cgroup *mem,
                                          bool charge)
 {
@@ -689,6 +683,9 @@ static void __mem_cgroup_target_update(struct mem_cgroup *mem, int target)
         case MEM_CGROUP_TARGET_SOFTLIMIT:
                 next = val + SOFTLIMIT_EVENTS_TARGET;
                 break;
+        case MEM_CGROUP_TARGET_NUMAINFO:
+                next = val + NUMAINFO_EVENTS_TARGET;
+                break;
         default:
                 return;
         }
@@ -707,11 +704,19 @@ static void memcg_check_events(struct mem_cgroup *mem, struct page *page)
                 mem_cgroup_threshold(mem);
                 __mem_cgroup_target_update(mem, MEM_CGROUP_TARGET_THRESH);
                 if (unlikely(__memcg_event_check(mem,
-                        MEM_CGROUP_TARGET_SOFTLIMIT))){
+                             MEM_CGROUP_TARGET_SOFTLIMIT))) {
                         mem_cgroup_update_tree(mem, page);
                         __mem_cgroup_target_update(mem,
-                                MEM_CGROUP_TARGET_SOFTLIMIT);
+                                                   MEM_CGROUP_TARGET_SOFTLIMIT);
+                }
+#if MAX_NUMNODES > 1
+                if (unlikely(__memcg_event_check(mem,
+                        MEM_CGROUP_TARGET_NUMAINFO))) {
+                        atomic_inc(&mem->numainfo_events);
+                        __mem_cgroup_target_update(mem,
+                                MEM_CGROUP_TARGET_NUMAINFO);
                 }
+#endif
         }
 }
 
@@ -1129,7 +1134,6 @@ unsigned long mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg,
         return MEM_CGROUP_ZSTAT(mz, lru);
 }
 
-#ifdef CONFIG_NUMA
 static unsigned long mem_cgroup_node_nr_file_lru_pages(struct mem_cgroup *memcg,
                                                         int nid)
 {
@@ -1141,6 +1145,17 @@ static unsigned long mem_cgroup_node_nr_file_lru_pages(struct mem_cgroup *memcg,
         return ret;
 }
 
+static unsigned long mem_cgroup_node_nr_anon_lru_pages(struct mem_cgroup *memcg,
+                                                        int nid)
+{
+        unsigned long ret;
+
+        ret = mem_cgroup_get_zonestat_node(memcg, nid, LRU_INACTIVE_ANON) +
+                mem_cgroup_get_zonestat_node(memcg, nid, LRU_ACTIVE_ANON);
+        return ret;
+}
+
+#if MAX_NUMNODES > 1
 static unsigned long mem_cgroup_nr_file_lru_pages(struct mem_cgroup *memcg)
 {
         u64 total = 0;
@@ -1152,17 +1167,6 @@ static unsigned long mem_cgroup_nr_file_lru_pages(struct mem_cgroup *memcg)
         return total;
 }
 
-static unsigned long mem_cgroup_node_nr_anon_lru_pages(struct mem_cgroup *memcg,
-                                                        int nid)
-{
-        unsigned long ret;
-
-        ret = mem_cgroup_get_zonestat_node(memcg, nid, LRU_INACTIVE_ANON) +
-                mem_cgroup_get_zonestat_node(memcg, nid, LRU_ACTIVE_ANON);
-
-        return ret;
-}
-
 static unsigned long mem_cgroup_nr_anon_lru_pages(struct mem_cgroup *memcg)
 {
         u64 total = 0;
@@ -1559,6 +1563,28 @@ mem_cgroup_select_victim(struct mem_cgroup *root_mem)
         return ret;
 }
 
+/**
+ * test_mem_cgroup_node_reclaimable
+ * @mem: the target memcg
+ * @nid: the node ID to be checked.
+ * @noswap : specify true here if the user wants flle only information.
+ *
+ * This function returns whether the specified memcg contains any
+ * reclaimable pages on a node. Returns true if there are any reclaimable
+ * pages in the node.
+ */
+static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *mem,
+                int nid, bool noswap)
+{
+        if (mem_cgroup_node_nr_file_lru_pages(mem, nid))
+                return true;
+        if (noswap || !total_swap_pages)
+                return false;
+        if (mem_cgroup_node_nr_anon_lru_pages(mem, nid))
+                return true;
+        return false;
+
+}
 #if MAX_NUMNODES > 1
 
 /*
@@ -1570,26 +1596,26 @@ mem_cgroup_select_victim(struct mem_cgroup *root_mem)
 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *mem)
 {
         int nid;
-
-        if (time_after(mem->next_scan_node_update, jiffies))
+        /*
+         * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
+         * pagein/pageout changes since the last update.
+         */
+        if (!atomic_read(&mem->numainfo_events))
+                return;
+        if (atomic_inc_return(&mem->numainfo_updating) > 1)
                 return;
 
-        mem->next_scan_node_update = jiffies + 10*HZ;
         /* make a nodemask where this memcg uses memory from */
         mem->scan_nodes = node_states[N_HIGH_MEMORY];
 
         for_each_node_mask(nid, node_states[N_HIGH_MEMORY]) {
 
-                if (mem_cgroup_get_zonestat_node(mem, nid, LRU_INACTIVE_FILE) ||
-                    mem_cgroup_get_zonestat_node(mem, nid, LRU_ACTIVE_FILE))
-                        continue;
-
-                if (total_swap_pages &&
-                    (mem_cgroup_get_zonestat_node(mem, nid, LRU_INACTIVE_ANON) ||
-                     mem_cgroup_get_zonestat_node(mem, nid, LRU_ACTIVE_ANON)))
-                        continue;
-                node_clear(nid, mem->scan_nodes);
+                if (!test_mem_cgroup_node_reclaimable(mem, nid, false))
+                        node_clear(nid, mem->scan_nodes);
         }
+
+        atomic_set(&mem->numainfo_events, 0);
+        atomic_set(&mem->numainfo_updating, 0);
 }
 
 /*
@@ -1627,11 +1653,51 @@ int mem_cgroup_select_victim_node(struct mem_cgroup *mem)
         return node;
 }
 
+/*
+ * Check all nodes whether it contains reclaimable pages or not.
+ * For quick scan, we make use of scan_nodes. This will allow us to skip
+ * unused nodes. But scan_nodes is lazily updated and may not cotain
+ * enough new information. We need to do double check.
+ */
+bool mem_cgroup_reclaimable(struct mem_cgroup *mem, bool noswap)
+{
+        int nid;
+
+        /*
+         * quick check...making use of scan_node.
+         * We can skip unused nodes.
+         */
+        if (!nodes_empty(mem->scan_nodes)) {
+                for (nid = first_node(mem->scan_nodes);
+                     nid < MAX_NUMNODES;
+                     nid = next_node(nid, mem->scan_nodes)) {
+
+                        if (test_mem_cgroup_node_reclaimable(mem, nid, noswap))
+                                return true;
+                }
+        }
+        /*
+         * Check rest of nodes.
+         */
+        for_each_node_state(nid, N_HIGH_MEMORY) {
+                if (node_isset(nid, mem->scan_nodes))
+                        continue;
+                if (test_mem_cgroup_node_reclaimable(mem, nid, noswap))
+                        return true;
+        }
+        return false;
+}
+
 #else
 int mem_cgroup_select_victim_node(struct mem_cgroup *mem)
 {
         return 0;
 }
+
+bool mem_cgroup_reclaimable(struct mem_cgroup *mem, bool noswap)
+{
+        return test_mem_cgroup_node_reclaimable(mem, 0, noswap);
+}
 #endif
 
 /*
@@ -1702,7 +1768,7 @@ static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
                                 }
                         }
                 }
-                if (!mem_cgroup_local_usage(victim)) {
+                if (!mem_cgroup_reclaimable(victim, noswap)) {
                         /* this cgroup's local usage == 0 */
                         css_put(&victim->css);
                         continue;
diff --git a/mm/memory.c b/mm/memory.c
index 40b7531ee8ba..9b8a01d941cb 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -305,6 +305,7 @@ int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
         if (batch->nr == batch->max) {
                 if (!tlb_next_batch(tlb))
                         return 0;
+                batch = tlb->active;
         }
         VM_BUG_ON(batch->nr > batch->max);
 
diff --git a/mm/nommu.c b/mm/nommu.c
index 1fd0c51b10a6..9edc897a3970 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -1813,10 +1813,13 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
         return NULL;
 }
 
-int remap_pfn_range(struct vm_area_struct *vma, unsigned long from,
-                unsigned long to, unsigned long size, pgprot_t prot)
+int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
+                unsigned long pfn, unsigned long size, pgprot_t prot)
 {
-        vma->vm_start = vma->vm_pgoff << PAGE_SHIFT;
+        if (addr != (pfn << PAGE_SHIFT))
+                return -EINVAL;
+
+        vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP;
         return 0;
 }
 EXPORT_SYMBOL(remap_pfn_range);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 4f49535d4cd3..d036e59d302b 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2310,7 +2310,8 @@ static bool pgdat_balanced(pg_data_t *pgdat, unsigned long balanced_pages,
         for (i = 0; i <= classzone_idx; i++)
                 present_pages += pgdat->node_zones[i].present_pages;
 
-        return balanced_pages > (present_pages >> 2);
+        /* A special case here: if zone has no page, we think it's balanced */
+        return balanced_pages >= (present_pages >> 2);
 }
 
 /* is kswapd sleeping prematurely? */
@@ -2326,7 +2327,7 @@ static bool sleeping_prematurely(pg_data_t *pgdat, int order, long remaining,
                 return true;
 
         /* Check the watermark levels */
-        for (i = 0; i < pgdat->nr_zones; i++) {
+        for (i = 0; i <= classzone_idx; i++) {
                 struct zone *zone = pgdat->node_zones + i;
 
                 if (!populated_zone(zone))
@@ -2344,7 +2345,7 @@ static bool sleeping_prematurely(pg_data_t *pgdat, int order, long remaining,
                 }
 
                 if (!zone_watermark_ok_safe(zone, order, high_wmark_pages(zone),
-                                                        classzone_idx, 0))
+                                                        i, 0))
                         all_zones_ok = false;
                 else
                         balanced += zone->present_pages;
@@ -2451,7 +2452,6 @@ loop_again:
                         if (!zone_watermark_ok_safe(zone, order,
                                         high_wmark_pages(zone), 0, 0)) {
                                 end_zone = i;
-                                *classzone_idx = i;
                                 break;
                         }
                 }
@@ -2510,18 +2510,18 @@ loop_again:
                                 KSWAPD_ZONE_BALANCE_GAP_RATIO);
                         if (!zone_watermark_ok_safe(zone, order,
                                         high_wmark_pages(zone) + balance_gap,
-                                        end_zone, 0))
+                                        end_zone, 0)) {
                                 shrink_zone(priority, zone, &sc);
-                        reclaim_state->reclaimed_slab = 0;
-                        nr_slab = shrink_slab(&shrink, sc.nr_scanned, lru_pages);
-                        sc.nr_reclaimed += reclaim_state->reclaimed_slab;
-                        total_scanned += sc.nr_scanned;
 
-                        if (zone->all_unreclaimable)
-                                continue;
-                        if (nr_slab == 0 &&
-                            !zone_reclaimable(zone))
-                                zone->all_unreclaimable = 1;
+                                reclaim_state->reclaimed_slab = 0;
+                                nr_slab = shrink_slab(&shrink, sc.nr_scanned, lru_pages);
+                                sc.nr_reclaimed += reclaim_state->reclaimed_slab;
+                                total_scanned += sc.nr_scanned;
+
+                                if (nr_slab == 0 && !zone_reclaimable(zone))
+                                        zone->all_unreclaimable = 1;
+                        }
+
                         /*
                          * If we've done a decent amount of scanning and
                          * the reclaim ratio is low, start doing writepage
@@ -2531,6 +2531,12 @@ loop_again:
                             total_scanned > sc.nr_reclaimed + sc.nr_reclaimed / 2)
                                 sc.may_writepage = 1;
 
+                        if (zone->all_unreclaimable) {
+                                if (end_zone && end_zone == i)
+                                        end_zone--;
+                                continue;
+                        }
+
                         if (!zone_watermark_ok_safe(zone, order,
                                         high_wmark_pages(zone), end_zone, 0)) {
                                 all_zones_ok = 0;
@@ -2709,8 +2715,8 @@ static void kswapd_try_to_sleep(pg_data_t *pgdat, int order, int classzone_idx)
  */
 static int kswapd(void *p)
 {
-        unsigned long order;
-        int classzone_idx;
+        unsigned long order, new_order;
+        int classzone_idx, new_classzone_idx;
         pg_data_t *pgdat = (pg_data_t*)p;
         struct task_struct *tsk = current;
 
@@ -2740,17 +2746,23 @@ static int kswapd(void *p)
         tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD;
         set_freezable();
 
-        order = 0;
-        classzone_idx = MAX_NR_ZONES - 1;
+        order = new_order = 0;
+        classzone_idx = new_classzone_idx = pgdat->nr_zones - 1;
         for ( ; ; ) {
-                unsigned long new_order;
-                int new_classzone_idx;
                 int ret;
 
-                new_order = pgdat->kswapd_max_order;
-                new_classzone_idx = pgdat->classzone_idx;
-                pgdat->kswapd_max_order = 0;
-                pgdat->classzone_idx = MAX_NR_ZONES - 1;
+                /*
+                 * If the last balance_pgdat was unsuccessful it's unlikely a
+                 * new request of a similar or harder type will succeed soon
+                 * so consider going to sleep on the basis we reclaimed at
+                 */
+                if (classzone_idx >= new_classzone_idx && order == new_order) {
+                        new_order = pgdat->kswapd_max_order;
+                        new_classzone_idx = pgdat->classzone_idx;
+                        pgdat->kswapd_max_order =  0;
+                        pgdat->classzone_idx = pgdat->nr_zones - 1;
+                }
+
                 if (order < new_order || classzone_idx > new_classzone_idx) {
                         /*
                          * Don't sleep if someone wants a larger 'order'
@@ -2763,7 +2775,7 @@ static int kswapd(void *p)
                         order = pgdat->kswapd_max_order;
                         classzone_idx = pgdat->classzone_idx;
                         pgdat->kswapd_max_order = 0;
-                        pgdat->classzone_idx = MAX_NR_ZONES - 1;
+                        pgdat->classzone_idx = pgdat->nr_zones - 1;
                 }
 
                 ret = try_to_freeze();