8 files changed, 114 insertions, 121 deletions

diff --git a/mm/filemap.c b/mm/filemap.c
index 87d62c44c3f0..796a5471b495 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -1416,7 +1416,7 @@ retry_find:
                  */
                 if (!did_readaround) {
                         majmin = VM_FAULT_MAJOR;
-                        inc_page_state(pgmajfault);
+                        count_vm_event(PGMAJFAULT);
                 }
                 did_readaround = 1;
                 ra_pages = max_sane_readahead(file->f_ra.ra_pages);
@@ -1487,7 +1487,7 @@ no_cached_page:
 page_not_uptodate:
         if (!did_readaround) {
                 majmin = VM_FAULT_MAJOR;
-                inc_page_state(pgmajfault);
+                count_vm_event(PGMAJFAULT);
         }
         lock_page(page);
 
diff --git a/mm/memory.c b/mm/memory.c
index 1a78791590fa..7e2a4b1580e3 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1951,7 +1951,7 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
 
                 /* Had to read the page from swap area: Major fault */
                 ret = VM_FAULT_MAJOR;
-                inc_page_state(pgmajfault);
+                count_vm_event(PGMAJFAULT);
                 grab_swap_token();
         }
 
@@ -2324,7 +2324,7 @@ int __handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
 
         __set_current_state(TASK_RUNNING);
 
-        inc_page_state(pgfault);
+        count_vm_event(PGFAULT);
 
         if (unlikely(is_vm_hugetlb_page(vma)))
                 return hugetlb_fault(mm, vma, address, write_access);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index d61671260f92..30b0b97ad023 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -456,7 +456,7 @@ static void __free_pages_ok(struct page *page, unsigned int order)
 
         kernel_map_pages(page, 1 << order, 0);
         local_irq_save(flags);
-        __mod_page_state(pgfree, 1 << order);
+        __count_vm_events(PGFREE, 1 << order);
         free_one_page(page_zone(page), page, order);
         local_irq_restore(flags);
 }
@@ -729,7 +729,7 @@ static void fastcall free_hot_cold_page(struct page *page, int cold)
 
         pcp = &zone_pcp(zone, get_cpu())->pcp[cold];
         local_irq_save(flags);
-        __inc_page_state(pgfree);
+        __count_vm_event(PGFREE);
         list_add(&page->lru, &pcp->list);
         pcp->count++;
         if (pcp->count >= pcp->high) {
@@ -805,7 +805,7 @@ again:
                         goto failed;
         }
 
-        __mod_page_state_zone(zone, pgalloc, 1 << order);
+        __count_zone_vm_events(PGALLOC, zone, 1 << order);
         zone_statistics(zonelist, zone);
         local_irq_restore(flags);
         put_cpu();
@@ -2101,24 +2101,11 @@ static int page_alloc_cpu_notify(struct notifier_block *self,
                                  unsigned long action, void *hcpu)
 {
         int cpu = (unsigned long)hcpu;
-        unsigned long *src, *dest;
 
         if (action == CPU_DEAD) {
-                int i;
-
                 local_irq_disable();
                 __drain_pages(cpu);
-
-                /* Add dead cpu's page_states to our own. */
-                dest = (unsigned long *)&__get_cpu_var(page_states);
-                src = (unsigned long *)&per_cpu(page_states, cpu);
-
-                for (i = 0; i < sizeof(struct page_state)/sizeof(unsigned long);
-                                i++) {
-                        dest[i] += src[i];
-                        src[i] = 0;
-                }
-
+                vm_events_fold_cpu(cpu);
                 local_irq_enable();
                 refresh_cpu_vm_stats(cpu);
         }
diff --git a/mm/page_io.c b/mm/page_io.c
index bb2b0d53889c..88029948d00a 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -101,7 +101,7 @@ int swap_writepage(struct page *page, struct writeback_control *wbc)
         }
         if (wbc->sync_mode == WB_SYNC_ALL)
                 rw |= (1 << BIO_RW_SYNC);
-        inc_page_state(pswpout);
+        count_vm_event(PSWPOUT);
         set_page_writeback(page);
         unlock_page(page);
         submit_bio(rw, bio);
@@ -123,7 +123,7 @@ int swap_readpage(struct file *file, struct page *page)
                 ret = -ENOMEM;
                 goto out;
         }
-        inc_page_state(pswpin);
+        count_vm_event(PSWPIN);
         submit_bio(READ, bio);
 out:
         return ret;
diff --git a/mm/shmem.c b/mm/shmem.c
index b14ff817d162..a9c09e0ba709 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -1045,12 +1045,12 @@ repeat:
                 swappage = lookup_swap_cache(swap);
                 if (!swappage) {
                         shmem_swp_unmap(entry);
-                        spin_unlock(&info->lock);
                         /* here we actually do the io */
                         if (type && *type == VM_FAULT_MINOR) {
-                                inc_page_state(pgmajfault);
+                                __count_vm_event(PGMAJFAULT);
                                 *type = VM_FAULT_MAJOR;
                         }
+                        spin_unlock(&info->lock);
                         swappage = shmem_swapin(info, swap, idx);
                         if (!swappage) {
                                 spin_lock(&info->lock);
diff --git a/mm/swap.c b/mm/swap.c
index 990868afc1c6..8fd095c4ae51 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -87,7 +87,7 @@ int rotate_reclaimable_page(struct page *page)
         spin_lock_irqsave(&zone->lru_lock, flags);
         if (PageLRU(page) && !PageActive(page)) {
                 list_move_tail(&page->lru, &zone->inactive_list);
-                inc_page_state(pgrotated);
+                __count_vm_event(PGROTATED);
         }
         if (!test_clear_page_writeback(page))
                 BUG();
@@ -107,7 +107,7 @@ void fastcall activate_page(struct page *page)
                 del_page_from_inactive_list(zone, page);
                 SetPageActive(page);
                 add_page_to_active_list(zone, page);
-                inc_page_state(pgactivate);
+                __count_vm_event(PGACTIVATE);
         }
         spin_unlock_irq(&zone->lru_lock);
 }
diff --git a/mm/vmscan.c b/mm/vmscan.c
index d6942436ac97..ff2ebe9458a3 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -215,7 +215,7 @@ unsigned long shrink_slab(unsigned long scanned, gfp_t gfp_mask,
                                 break;
                         if (shrink_ret < nr_before)
                                 ret += nr_before - shrink_ret;
-                        mod_page_state(slabs_scanned, this_scan);
+                        count_vm_events(SLABS_SCANNED, this_scan);
                         total_scan -= this_scan;
 
                         cond_resched();
@@ -569,7 +569,7 @@ keep:
         list_splice(&ret_pages, page_list);
         if (pagevec_count(&freed_pvec))
                 __pagevec_release_nonlru(&freed_pvec);
-        mod_page_state(pgactivate, pgactivate);
+        count_vm_events(PGACTIVATE, pgactivate);
         return nr_reclaimed;
 }
 
@@ -659,11 +659,11 @@ static unsigned long shrink_inactive_list(unsigned long max_scan,
                 nr_reclaimed += nr_freed;
                 local_irq_disable();
                 if (current_is_kswapd()) {
-                        __mod_page_state_zone(zone, pgscan_kswapd, nr_scan);
-                        __mod_page_state(kswapd_steal, nr_freed);
+                        __count_zone_vm_events(PGSCAN_KSWAPD, zone, nr_scan);
+                        __count_vm_events(KSWAPD_STEAL, nr_freed);
                 } else
-                        __mod_page_state_zone(zone, pgscan_direct, nr_scan);
-                __mod_page_state_zone(zone, pgsteal, nr_freed);
+                        __count_zone_vm_events(PGSCAN_DIRECT, zone, nr_scan);
+                __count_vm_events(PGACTIVATE, nr_freed);
 
                 if (nr_taken == 0)
                         goto done;
@@ -841,11 +841,10 @@ static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
                 }
         }
         zone->nr_active += pgmoved;
-        spin_unlock(&zone->lru_lock);
 
-        __mod_page_state_zone(zone, pgrefill, pgscanned);
-        __mod_page_state(pgdeactivate, pgdeactivate);
-        local_irq_enable();
+        __count_zone_vm_events(PGREFILL, zone, pgscanned);
+        __count_vm_events(PGDEACTIVATE, pgdeactivate);
+        spin_unlock_irq(&zone->lru_lock);
 
         pagevec_release(&pvec);
 }
@@ -977,7 +976,7 @@ unsigned long try_to_free_pages(struct zone **zones, gfp_t gfp_mask)
                 .swappiness = vm_swappiness,
         };
 
-        inc_page_state(allocstall);
+        count_vm_event(ALLOCSTALL);
 
         for (i = 0; zones[i] != NULL; i++) {
                 struct zone *zone = zones[i];
@@ -1074,7 +1073,7 @@ loop_again:
         total_scanned = 0;
         nr_reclaimed = 0;
         sc.may_writepage = !laptop_mode;
-        inc_page_state(pageoutrun);
+        count_vm_event(PAGEOUTRUN);
 
         for (i = 0; i < pgdat->nr_zones; i++) {
                 struct zone *zone = pgdat->node_zones + i;
diff --git a/mm/vmstat.c b/mm/vmstat.c
index ee7f89666250..73b83d67bab6 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -13,66 +13,6 @@
 #include <linux/mm.h>
 #include <linux/module.h>
 
-/*
- * Accumulate the page_state information across all CPUs.
- * The result is unavoidably approximate - it can change
- * during and after execution of this function.
- */
-DEFINE_PER_CPU(struct page_state, page_states) = {0};
-
-static void __get_page_state(struct page_state *ret, int nr, cpumask_t *cpumask)
-{
-        unsigned cpu;
-
-        memset(ret, 0, nr * sizeof(unsigned long));
-        cpus_and(*cpumask, *cpumask, cpu_online_map);
-
-        for_each_cpu_mask(cpu, *cpumask) {
-                unsigned long *in;
-                unsigned long *out;
-                unsigned off;
-                unsigned next_cpu;
-
-                in = (unsigned long *)&per_cpu(page_states, cpu);
-
-                next_cpu = next_cpu(cpu, *cpumask);
-                if (likely(next_cpu < NR_CPUS))
-                        prefetch(&per_cpu(page_states, next_cpu));
-
-                out = (unsigned long *)ret;
-                for (off = 0; off < nr; off++)
-                        *out++ += *in++;
-        }
-}
-
-void get_full_page_state(struct page_state *ret)
-{
-        cpumask_t mask = CPU_MASK_ALL;
-
-        __get_page_state(ret, sizeof(*ret) / sizeof(unsigned long), &mask);
-}
-
-void __mod_page_state_offset(unsigned long offset, unsigned long delta)
-{
-        void *ptr;
-
-        ptr = &__get_cpu_var(page_states);
-        *(unsigned long *)(ptr + offset) += delta;
-}
-EXPORT_SYMBOL(__mod_page_state_offset);
-
-void mod_page_state_offset(unsigned long offset, unsigned long delta)
-{
-        unsigned long flags;
-        void *ptr;
-
-        local_irq_save(flags);
-        ptr = &__get_cpu_var(page_states);
-        *(unsigned long *)(ptr + offset) += delta;
-        local_irq_restore(flags);
-}
-EXPORT_SYMBOL(mod_page_state_offset);
-
 void __get_zone_counts(unsigned long *active, unsigned long *inactive,
                         unsigned long *free, struct pglist_data *pgdat)
 {
@@ -106,6 +46,63 @@ void get_zone_counts(unsigned long *active,
         }
 }
 
+#ifdef CONFIG_VM_EVENT_COUNTERS
+DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
+EXPORT_PER_CPU_SYMBOL(vm_event_states);
+
+static void sum_vm_events(unsigned long *ret, cpumask_t *cpumask)
+{
+        int cpu = 0;
+        int i;
+
+        memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long));
+
+        cpu = first_cpu(*cpumask);
+        while (cpu < NR_CPUS) {
+                struct vm_event_state *this = &per_cpu(vm_event_states, cpu);
+
+                cpu = next_cpu(cpu, *cpumask);
+
+                if (cpu < NR_CPUS)
+                        prefetch(&per_cpu(vm_event_states, cpu));
+
+
+                for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
+                        ret[i] += this->event[i];
+        }
+}
+
+/*
+ * Accumulate the vm event counters across all CPUs.
+ * The result is unavoidably approximate - it can change
+ * during and after execution of this function.
+*/
+void all_vm_events(unsigned long *ret)
+{
+        sum_vm_events(ret, &cpu_online_map);
+}
+
+#ifdef CONFIG_HOTPLUG
+/*
+ * Fold the foreign cpu events into our own.
+ *
+ * This is adding to the events on one processor
+ * but keeps the global counts constant.
+ */
+void vm_events_fold_cpu(int cpu)
+{
+        struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu);
+        int i;
+
+        for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
+                count_vm_events(i, fold_state->event[i]);
+                fold_state->event[i] = 0;
+        }
+}
+#endif /* CONFIG_HOTPLUG */
+
+#endif /* CONFIG_VM_EVENT_COUNTERS */
+
 /*
  * Manage combined zone based / global counters
  *
@@ -405,16 +402,16 @@ static char *vmstat_text[] = {
         "numa_other",
 #endif
 
-        /* Event counters */
+#ifdef CONFIG_VM_EVENT_COUNTERS
         "pgpgin",
         "pgpgout",
         "pswpin",
         "pswpout",
 
-        "pgalloc_high",
-        "pgalloc_normal",
-        "pgalloc_dma32",
         "pgalloc_dma",
+        "pgalloc_dma32",
+        "pgalloc_normal",
+        "pgalloc_high",
 
         "pgfree",
         "pgactivate",
@@ -423,25 +420,25 @@ static char *vmstat_text[] = {
         "pgfault",
         "pgmajfault",
 
-        "pgrefill_high",
-        "pgrefill_normal",
-        "pgrefill_dma32",
         "pgrefill_dma",
+        "pgrefill_dma32",
+        "pgrefill_normal",
+        "pgrefill_high",
 
-        "pgsteal_high",
-        "pgsteal_normal",
-        "pgsteal_dma32",
         "pgsteal_dma",
+        "pgsteal_dma32",
+        "pgsteal_normal",
+        "pgsteal_high",
 
-        "pgscan_kswapd_high",
-        "pgscan_kswapd_normal",
-        "pgscan_kswapd_dma32",
         "pgscan_kswapd_dma",
+        "pgscan_kswapd_dma32",
+        "pgscan_kswapd_normal",
+        "pgscan_kswapd_high",
 
-        "pgscan_direct_high",
-        "pgscan_direct_normal",
-        "pgscan_direct_dma32",
         "pgscan_direct_dma",
+        "pgscan_direct_dma32",
+        "pgscan_direct_normal",
+        "pgscan_direct_high",
 
         "pginodesteal",
         "slabs_scanned",
@@ -451,6 +448,7 @@ static char *vmstat_text[] = {
         "allocstall",
 
         "pgrotated",
+#endif
 };
 
 /*
@@ -553,23 +551,32 @@ struct seq_operations zoneinfo_op = {
 static void *vmstat_start(struct seq_file *m, loff_t *pos)
 {
         unsigned long *v;
-        struct page_state *ps;
+#ifdef CONFIG_VM_EVENT_COUNTERS
+        unsigned long *e;
+#endif
         int i;
 
         if (*pos >= ARRAY_SIZE(vmstat_text))
                 return NULL;
 
+#ifdef CONFIG_VM_EVENT_COUNTERS
         v = kmalloc(NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long)
-                        + sizeof(*ps), GFP_KERNEL);
+                        + sizeof(struct vm_event_state), GFP_KERNEL);
+#else
+        v = kmalloc(NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long),
+                        GFP_KERNEL);
+#endif
         m->private = v;
         if (!v)
                 return ERR_PTR(-ENOMEM);
         for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
                 v[i] = global_page_state(i);
-        ps = (struct page_state *)(v + NR_VM_ZONE_STAT_ITEMS);
-        get_full_page_state(ps);
-        ps->pgpgin /= 2;                /* sectors -> kbytes */
-        ps->pgpgout /= 2;
+#ifdef CONFIG_VM_EVENT_COUNTERS
+        e = v + NR_VM_ZONE_STAT_ITEMS;
+        all_vm_events(e);
+        e[PGPGIN] /= 2;         /* sectors -> kbytes */
+        e[PGPGOUT] /= 2;
+#endif
         return v + *pos;
 }