6 files changed, 359 insertions, 5 deletions

diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index b487e227a1f7..47de9ee6bcd6 100644
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -70,6 +70,11 @@ config DMA_IS_DMA32
         bool
         default y
 
+config DMA_IS_NORMAL
+        bool
+        depends on IA64_SGI_SN2
+        default y
+
 choice
         prompt "System type"
         default IA64_GENERIC
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index d6120fa69116..543f9e411563 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -46,6 +46,9 @@ struct zone_padding {
 #define ZONE_PADDING(name)
 #endif
 
+enum zone_stat_item {
+        NR_VM_ZONE_STAT_ITEMS };
+
 struct per_cpu_pages {
         int count;              /* number of pages in the list */
         int high;               /* high watermark, emptying needed */
@@ -55,6 +58,10 @@ struct per_cpu_pages {
 
 struct per_cpu_pageset {
         struct per_cpu_pages pcp[2];    /* 0: hot.  1: cold */
+#ifdef CONFIG_SMP
+        s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
+#endif
+
 #ifdef CONFIG_NUMA
         unsigned long numa_hit;         /* allocated in intended node */
         unsigned long numa_miss;        /* allocated in non intended node */
@@ -165,6 +172,8 @@ struct zone {
         /* A count of how many reclaimers are scanning this zone */
         atomic_t                reclaim_in_progress;
 
+        /* Zone statistics */
+        atomic_long_t           vm_stat[NR_VM_ZONE_STAT_ITEMS];
         /*
          * timestamp (in jiffies) of the last zone reclaim that did not
          * result in freeing of pages. This is used to avoid repeated scans
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index 3ca0c1989fc2..3fd5c11e544a 100644
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -3,6 +3,9 @@
 
 #include <linux/types.h>
 #include <linux/percpu.h>
+#include <linux/config.h>
+#include <linux/mmzone.h>
+#include <asm/atomic.h>
 
 /*
  * Global page accounting.  One instance per CPU.  Only unsigned longs are
@@ -134,5 +137,129 @@ extern void __mod_page_state_offset(unsigned long offset, unsigned long delta);
 
 DECLARE_PER_CPU(struct page_state, page_states);
 
-#endif /* _LINUX_VMSTAT_H */
+/*
+ * Zone based page accounting with per cpu differentials.
+ */
+extern atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
+
+static inline void zone_page_state_add(long x, struct zone *zone,
+                                 enum zone_stat_item item)
+{
+        atomic_long_add(x, &zone->vm_stat[item]);
+        atomic_long_add(x, &vm_stat[item]);
+}
+
+static inline unsigned long global_page_state(enum zone_stat_item item)
+{
+        long x = atomic_long_read(&vm_stat[item]);
+#ifdef CONFIG_SMP
+        if (x < 0)
+                x = 0;
+#endif
+        return x;
+}
+
+static inline unsigned long zone_page_state(struct zone *zone,
+                                        enum zone_stat_item item)
+{
+        long x = atomic_long_read(&zone->vm_stat[item]);
+#ifdef CONFIG_SMP
+        if (x < 0)
+                x = 0;
+#endif
+        return x;
+}
+
+#ifdef CONFIG_NUMA
+/*
+ * Determine the per node value of a stat item. This function
+ * is called frequently in a NUMA machine, so try to be as
+ * frugal as possible.
+ */
+static inline unsigned long node_page_state(int node,
+                                 enum zone_stat_item item)
+{
+        struct zone *zones = NODE_DATA(node)->node_zones;
+
+        return
+#ifndef CONFIG_DMA_IS_NORMAL
+#if !defined(CONFIG_DMA_IS_DMA32) && BITS_PER_LONG >= 64
+                zone_page_state(&zones[ZONE_DMA32], item) +
+#endif
+                zone_page_state(&zones[ZONE_NORMAL], item) +
+#endif
+#ifdef CONFIG_HIGHMEM
+                zone_page_state(&zones[ZONE_HIGHMEM], item) +
+#endif
+                zone_page_state(&zones[ZONE_DMA], item);
+}
+#else
+#define node_page_state(node, item) global_page_state(item)
+#endif
+
+#define __add_zone_page_state(__z, __i, __d)    \
+                __mod_zone_page_state(__z, __i, __d)
+#define __sub_zone_page_state(__z, __i, __d)    \
+                __mod_zone_page_state(__z, __i,-(__d))
+
+#define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d)
+#define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d))
+
+static inline void zap_zone_vm_stats(struct zone *zone)
+{
+        memset(zone->vm_stat, 0, sizeof(zone->vm_stat));
+}
+
+#ifdef CONFIG_SMP
+void __mod_zone_page_state(struct zone *, enum zone_stat_item item, int);
+void __inc_zone_page_state(struct page *, enum zone_stat_item);
+void __dec_zone_page_state(struct page *, enum zone_stat_item);
 
+void mod_zone_page_state(struct zone *, enum zone_stat_item, int);
+void inc_zone_page_state(struct page *, enum zone_stat_item);
+void dec_zone_page_state(struct page *, enum zone_stat_item);
+
+extern void inc_zone_state(struct zone *, enum zone_stat_item);
+
+void refresh_cpu_vm_stats(int);
+void refresh_vm_stats(void);
+
+#else /* CONFIG_SMP */
+
+/*
+ * We do not maintain differentials in a single processor configuration.
+ * The functions directly modify the zone and global counters.
+ */
+static inline void __mod_zone_page_state(struct zone *zone,
+                        enum zone_stat_item item, int delta)
+{
+        zone_page_state_add(delta, zone, item);
+}
+
+static inline void __inc_zone_page_state(struct page *page,
+                        enum zone_stat_item item)
+{
+        atomic_long_inc(&page_zone(page)->vm_stat[item]);
+        atomic_long_inc(&vm_stat[item]);
+}
+
+static inline void __dec_zone_page_state(struct page *page,
+                        enum zone_stat_item item)
+{
+        atomic_long_dec(&page_zone(page)->vm_stat[item]);
+        atomic_long_dec(&vm_stat[item]);
+}
+
+/*
+ * We only use atomic operations to update counters. So there is no need to
+ * disable interrupts.
+ */
+#define inc_zone_page_state __inc_zone_page_state
+#define dec_zone_page_state __dec_zone_page_state
+#define mod_zone_page_state __mod_zone_page_state
+
+static inline void refresh_cpu_vm_stats(int cpu) { }
+static inline void refresh_vm_stats(void) { }
+#endif
+
+#endif /* _LINUX_VMSTAT_H */
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 87dc1297fe39..3a877fecc300 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2045,6 +2045,7 @@ static void __meminit free_area_init_core(struct pglist_data *pgdat,
                 zone->nr_scan_inactive = 0;
                 zone->nr_active = 0;
                 zone->nr_inactive = 0;
+                zap_zone_vm_stats(zone);
                 atomic_set(&zone->reclaim_in_progress, 0);
                 if (!size)
                         continue;
@@ -2147,6 +2148,7 @@ static int page_alloc_cpu_notify(struct notifier_block *self,
                 }
 
                 local_irq_enable();
+                refresh_cpu_vm_stats(cpu);
         }
         return NOTIFY_OK;
 }
diff --git a/mm/slab.c b/mm/slab.c
index 233e39d14caf..0c33820038cb 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -3763,6 +3763,7 @@ next:
         check_irq_on();
         mutex_unlock(&cache_chain_mutex);
         next_reap_node();
+        refresh_cpu_vm_stats(smp_processor_id());
         /* Set up the next iteration */
         schedule_delayed_work(&__get_cpu_var(reap_work), REAPTIMEOUT_CPUC);
 }
diff --git a/mm/vmstat.c b/mm/vmstat.c
index ad456202ff1a..210f9bbbb04f 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -3,10 +3,15 @@
  *
  *  Manages VM statistics
  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
+ *
+ *  zoned VM statistics
+ *  Copyright (C) 2006 Silicon Graphics, Inc.,
+ *              Christoph Lameter <christoph@lameter.com>
  */
 
 #include <linux/config.h>
 #include <linux/mm.h>
+#include <linux/module.h>
 
 /*
  * Accumulate the page_state information across all CPUs.
@@ -143,6 +148,197 @@ void get_zone_counts(unsigned long *active,
         }
 }
 
+/*
+ * Manage combined zone based / global counters
+ *
+ * vm_stat contains the global counters
+ */
+atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
+EXPORT_SYMBOL(vm_stat);
+
+#ifdef CONFIG_SMP
+
+#define STAT_THRESHOLD 32
+
+/*
+ * Determine pointer to currently valid differential byte given a zone and
+ * the item number.
+ *
+ * Preemption must be off
+ */
+static inline s8 *diff_pointer(struct zone *zone, enum zone_stat_item item)
+{
+        return &zone_pcp(zone, smp_processor_id())->vm_stat_diff[item];
+}
+
+/*
+ * For use when we know that interrupts are disabled.
+ */
+void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
+                                int delta)
+{
+        s8 *p;
+        long x;
+
+        p = diff_pointer(zone, item);
+        x = delta + *p;
+
+        if (unlikely(x > STAT_THRESHOLD || x < -STAT_THRESHOLD)) {
+                zone_page_state_add(x, zone, item);
+                x = 0;
+        }
+
+        *p = x;
+}
+EXPORT_SYMBOL(__mod_zone_page_state);
+
+/*
+ * For an unknown interrupt state
+ */
+void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
+                                        int delta)
+{
+        unsigned long flags;
+
+        local_irq_save(flags);
+        __mod_zone_page_state(zone, item, delta);
+        local_irq_restore(flags);
+}
+EXPORT_SYMBOL(mod_zone_page_state);
+
+/*
+ * Optimized increment and decrement functions.
+ *
+ * These are only for a single page and therefore can take a struct page *
+ * argument instead of struct zone *. This allows the inclusion of the code
+ * generated for page_zone(page) into the optimized functions.
+ *
+ * No overflow check is necessary and therefore the differential can be
+ * incremented or decremented in place which may allow the compilers to
+ * generate better code.
+ *
+ * The increment or decrement is known and therefore one boundary check can
+ * be omitted.
+ *
+ * Some processors have inc/dec instructions that are atomic vs an interrupt.
+ * However, the code must first determine the differential location in a zone
+ * based on the processor number and then inc/dec the counter. There is no
+ * guarantee without disabling preemption that the processor will not change
+ * in between and therefore the atomicity vs. interrupt cannot be exploited
+ * in a useful way here.
+ */
+void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
+{
+        struct zone *zone = page_zone(page);
+        s8 *p = diff_pointer(zone, item);
+
+        (*p)++;
+
+        if (unlikely(*p > STAT_THRESHOLD)) {
+                zone_page_state_add(*p, zone, item);
+                *p = 0;
+        }
+}
+EXPORT_SYMBOL(__inc_zone_page_state);
+
+void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
+{
+        struct zone *zone = page_zone(page);
+        s8 *p = diff_pointer(zone, item);
+
+        (*p)--;
+
+        if (unlikely(*p < -STAT_THRESHOLD)) {
+                zone_page_state_add(*p, zone, item);
+                *p = 0;
+        }
+}
+EXPORT_SYMBOL(__dec_zone_page_state);
+
+void inc_zone_page_state(struct page *page, enum zone_stat_item item)
+{
+        unsigned long flags;
+        struct zone *zone;
+        s8 *p;
+
+        zone = page_zone(page);
+        local_irq_save(flags);
+        p = diff_pointer(zone, item);
+
+        (*p)++;
+
+        if (unlikely(*p > STAT_THRESHOLD)) {
+                zone_page_state_add(*p, zone, item);
+                *p = 0;
+        }
+        local_irq_restore(flags);
+}
+EXPORT_SYMBOL(inc_zone_page_state);
+
+void dec_zone_page_state(struct page *page, enum zone_stat_item item)
+{
+        unsigned long flags;
+        struct zone *zone;
+        s8 *p;
+
+        zone = page_zone(page);
+        local_irq_save(flags);
+        p = diff_pointer(zone, item);
+
+        (*p)--;
+
+        if (unlikely(*p < -STAT_THRESHOLD)) {
+                zone_page_state_add(*p, zone, item);
+                *p = 0;
+        }
+        local_irq_restore(flags);
+}
+EXPORT_SYMBOL(dec_zone_page_state);
+
+/*
+ * Update the zone counters for one cpu.
+ */
+void refresh_cpu_vm_stats(int cpu)
+{
+        struct zone *zone;
+        int i;
+        unsigned long flags;
+
+        for_each_zone(zone) {
+                struct per_cpu_pageset *pcp;
+
+                pcp = zone_pcp(zone, cpu);
+
+                for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
+                        if (pcp->vm_stat_diff[i]) {
+                                local_irq_save(flags);
+                                zone_page_state_add(pcp->vm_stat_diff[i],
+                                        zone, i);
+                                pcp->vm_stat_diff[i] = 0;
+                                local_irq_restore(flags);
+                        }
+        }
+}
+
+static void __refresh_cpu_vm_stats(void *dummy)
+{
+        refresh_cpu_vm_stats(smp_processor_id());
+}
+
+/*
+ * Consolidate all counters.
+ *
+ * Note that the result is less inaccurate but still inaccurate
+ * if concurrent processes are allowed to run.
+ */
+void refresh_vm_stats(void)
+{
+        on_each_cpu(__refresh_cpu_vm_stats, NULL, 0, 1);
+}
+EXPORT_SYMBOL(refresh_vm_stats);
+
+#endif
+
 #ifdef CONFIG_PROC_FS
 
 #include <linux/seq_file.h>
@@ -204,6 +400,9 @@ struct seq_operations fragmentation_op = {
 };
 
 static char *vmstat_text[] = {
+        /* Zoned VM counters */
+
+        /* Page state */
         "nr_dirty",
         "nr_writeback",
         "nr_unstable",
@@ -297,6 +496,11 @@ static int zoneinfo_show(struct seq_file *m, void *arg)
                            zone->nr_scan_active, zone->nr_scan_inactive,
                            zone->spanned_pages,
                            zone->present_pages);
+
+                for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
+                        seq_printf(m, "\n    %-12s %lu", vmstat_text[i],
+                                        zone_page_state(zone, i));
+
                 seq_printf(m,
                            "\n        protection: (%lu",
                            zone->lowmem_reserve[0]);
@@ -368,19 +572,25 @@ struct seq_operations zoneinfo_op = {
 
 static void *vmstat_start(struct seq_file *m, loff_t *pos)
 {
+        unsigned long *v;
         struct page_state *ps;
+        int i;
 
         if (*pos >= ARRAY_SIZE(vmstat_text))
                 return NULL;
 
-        ps = kmalloc(sizeof(*ps), GFP_KERNEL);
-        m->private = ps;
-        if (!ps)
+        v = kmalloc(NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long)
+                        + sizeof(*ps), GFP_KERNEL);
+        m->private = v;
+        if (!v)
                 return ERR_PTR(-ENOMEM);
+        for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
+                v[i] = global_page_state(i);
+        ps = (struct page_state *)(v + NR_VM_ZONE_STAT_ITEMS);
         get_full_page_state(ps);
         ps->pgpgin /= 2;                /* sectors -> kbytes */
         ps->pgpgout /= 2;
-        return (unsigned long *)ps + *pos;
+        return v + *pos;
 }
 
 static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos)