fix PGFREE and NR_FREE_PAGES

12 files changed, 93 insertions, 23 deletions

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index d28f7ef8228d..750c0b64fa96 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -168,6 +168,7 @@ enum zone_stat_item {
         WORKINGSET_NODERECLAIM,
         NR_ANON_TRANSPARENT_HUGEPAGES,
         NR_FREE_CMA_PAGES,
+        NR_FREE_HC_PAGES,
         NR_VM_ZONE_STAT_ITEMS };
 
 /*
diff --git a/include/linux/slab.h b/include/linux/slab.h
index ffd24c830151..a899dda28def 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -87,6 +87,8 @@
 # define SLAB_FAILSLAB          0x00000000UL
 #endif
 
+#define SLAB_NO_MERGE           0x04000000UL    /* Do not merge with existing slab */
+
 /* The following flags affect the page allocator grouping pages by mobility */
 #define SLAB_RECLAIM_ACCOUNT    0x00020000UL            /* Objects are reclaimable */
 #define SLAB_TEMPORARY          SLAB_RECLAIM_ACCOUNT    /* Objects are short-lived */
diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h
index 33885118523c..9400aa1e9128 100644
--- a/include/linux/slub_def.h
+++ b/include/linux/slub_def.h
@@ -98,6 +98,8 @@ struct kmem_cache {
          */
         int remote_node_defrag_ratio;
 #endif
+        /* cpu id for higher-criticality slabs */
+        int cpu_id;
         struct kmem_cache_node *node[MAX_NUMNODES];
 };
 
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 9246d32dc973..3f5a9da27d7a 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -23,7 +23,7 @@
 
 enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
                 FOR_ALL_ZONES(PGALLOC),
-                PGFREE, PGACTIVATE, PGDEACTIVATE,
+                PGFREE, PGFREE_HC, PGACTIVATE, PGDEACTIVATE,
                 PGFAULT, PGMAJFAULT,
                 FOR_ALL_ZONES(PGREFILL),
                 FOR_ALL_ZONES(PGSTEAL_KSWAPD),
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index 82e7db7f7100..b6410f7efd74 100644
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -278,9 +278,12 @@ static inline void drain_zonestat(struct zone *zone,
 #endif          /* CONFIG_SMP */
 
 static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
-                                             int migratetype)
+                                             int migratetype, int part_no)
 {
-        __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
+        if (part_no == NR_CPUS)
+                __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
+        else
+                __mod_zone_page_state(zone, NR_FREE_HC_PAGES, nr_pages);
         if (is_migrate_cma(migratetype))
                 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
 }
diff --git a/litmus/page_dev.c b/litmus/page_dev.c
index 8e29e68ed89a..2894e93213d3 100644
--- a/litmus/page_dev.c
+++ b/litmus/page_dev.c
@@ -226,7 +226,7 @@ static struct ctl_table partition_table[] =
                 .maxlen         = sizeof(llc_partition[4]),

                 .extra1         = &dram_partition_min,

                 .extra2         = &dram_partition_max,

-        },      

+        },

         { }

 };

 

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 4611656df49a..e240fcd3039d 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -485,7 +485,7 @@ static inline void set_page_guard(struct zone *zone, struct page *page,
         INIT_LIST_HEAD(&page->lru);
         set_page_private(page, order);
         /* Guard pages are not available for any usage */
-        __mod_zone_freepage_state(zone, -(1 << order), migratetype);
+        __mod_zone_freepage_state(zone, -(1 << order), migratetype, bank_to_partition(page_bank(page)));
 }
 
 static inline void clear_page_guard(struct zone *zone, struct page *page,
@@ -501,7 +501,7 @@ static inline void clear_page_guard(struct zone *zone, struct page *page,
 
         set_page_private(page, 0);
         if (!is_migrate_isolate(migratetype))
-                __mod_zone_freepage_state(zone, (1 << order), migratetype);
+                __mod_zone_freepage_state(zone, (1 << order), migratetype, bank_to_partition(page_bank(page)));
 }
 #else
 struct page_ext_operations debug_guardpage_ops = { NULL, };
@@ -629,7 +629,7 @@ static inline void __free_one_page(struct page *page,
                          */
                         max_order = min(MAX_ORDER, pageblock_order + 1);
                 } else {
-                        __mod_zone_freepage_state(zone, 1 << order, migratetype);
+                        __mod_zone_freepage_state(zone, 1 << order, migratetype, parti_no);
                 }
 
                 page_idx = pfn & ((1 << max_order) - 1);
@@ -694,7 +694,7 @@ out:
                 if (is_migrate_isolate(migratetype)) {
                         max_order = min(MAX_PARTITIONED_ORDER, pageblock_order + 1);
                 } else {
-                        __mod_zone_freepage_state(zone, 1 << order, migratetype);
+                        __mod_zone_freepage_state(zone, 1 << order, migratetype, parti_no);
                 }
 
                 page_idx = pfn & ((1 << max_order) - 1);
@@ -927,7 +927,10 @@ static void __free_pages_ok(struct page *page, unsigned int order)
 
         migratetype = get_pfnblock_migratetype(page, pfn);
         local_irq_save(flags);
-        __count_vm_events(PGFREE, 1 << order);
+        if (bank_to_partition(page_bank(page)) == NR_CPUS)
+                __count_vm_events(PGFREE, 1 << order);
+        else if (bank_to_partition(page_bank(page)) < NR_CPUS)
+                __count_vm_events(PGFREE_HC, 1 << order);
         set_freepage_migratetype(page, migratetype);
         free_one_page(page_zone(page), page, pfn, order, migratetype);
         local_irq_restore(flags);
@@ -1097,6 +1100,7 @@ static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
 /* Kernel page coloring */
 
 /* build colored page list */
+#if 0
 static void build_colored_pages(struct zone *zone, struct page *page, int order)
 {
         int i, color, bank;
@@ -1167,6 +1171,7 @@ static inline struct page *get_colored_page(struct zone *zone, unsigned long req
         printk(KERN_INFO "color=%d, bank=%d allocated\n", color, bank);
         return page;
 }
+#endif
 
 /*
  * Go through the free lists for the given migratetype and remove
@@ -1181,8 +1186,11 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
         struct page *page;
         int cpu = raw_smp_processor_id();
 
-        if (order == 0 && color_req == 1) {
+        // if (order <= 2 && color_req == 1) {
+        /* The max. order of color_req is <= 2 */
+        if (color_req == 1) {
                 int found = 0;
+                printk(KERN_INFO "COLOR PAGE requested on CPU%d with order = %d\n", cpu, order);
                 /* Find a page of the appropriate size in the preferred list */
                 for (current_order = order; current_order < MAX_PARTITIONED_ORDER; ++current_order) {
                         area = &(zone->free_area_d[cpu][current_order]);
@@ -1555,7 +1563,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
                         __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
                                               -(1 << order));
         }
-        __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
+        __mod_zone_page_state(zone, NR_FREE_HC_PAGES, -(i << order));
         spin_unlock(&zone->lock);
         return i;
 }
@@ -1752,8 +1760,13 @@ void free_hot_cold_page(struct page *page, bool cold)
         migratetype = get_pfnblock_migratetype(page, pfn);
         set_freepage_migratetype(page, migratetype);
         local_irq_save(flags);
-        __count_vm_event(PGFREE);
-
+        
+        
+        if (bank_to_partition(page_bank(page)) == NR_CPUS)
+                __count_vm_event(PGFREE);
+        else if (bank_to_partition(page_bank(page)) < NR_CPUS)
+                __count_vm_event(PGFREE_HC);
+        
         /*
          * We only track unmovable, reclaimable and movable on pcp lists.
          * Free ISOLATE pages back to the allocator because they are being
@@ -1861,7 +1874,7 @@ int __isolate_free_page(struct page *page, unsigned int order)
                 if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
                         return 0;
 
-                __mod_zone_freepage_state(zone, -(1UL << order), mt);
+                __mod_zone_freepage_state(zone, -(1UL << order), mt, bank_to_partition(page_bank(page)));
         }
 
         /* Remove page from free list */
@@ -1966,7 +1979,7 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
                 if (!page)
                         goto failed;
                 __mod_zone_freepage_state(zone, -(1 << order),
-                                          get_freepage_migratetype(page));
+                                          get_freepage_migratetype(page), bank_to_partition(page_bank(page)));
         }
 
         __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order));
diff --git a/mm/page_isolation.c b/mm/page_isolation.c
index 303c908790ef..ed05910069ff 100644
--- a/mm/page_isolation.c
+++ b/mm/page_isolation.c
@@ -58,12 +58,13 @@ out:
         if (!ret) {
                 unsigned long nr_pages;
                 int migratetype = get_pageblock_migratetype(page);
+                int partno = bank_to_partition(page_bank(page));
 
                 set_pageblock_migratetype(page, MIGRATE_ISOLATE);
                 zone->nr_isolate_pageblock++;
                 nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE);
 
-                __mod_zone_freepage_state(zone, -nr_pages, migratetype);
+                __mod_zone_freepage_state(zone, -nr_pages, migratetype, partno);
         }
 
         spin_unlock_irqrestore(&zone->lock, flags);
@@ -117,8 +118,9 @@ void unset_migratetype_isolate(struct page *page, unsigned migratetype)
          * pageblock scanning for freepage moving.
          */
         if (!isolated_page) {
+                int partno = bank_to_partition(page_bank(page));
                 nr_pages = move_freepages_block(zone, page, migratetype);
-                __mod_zone_freepage_state(zone, nr_pages, migratetype);
+                __mod_zone_freepage_state(zone, nr_pages, migratetype, partno);
         }
         set_pageblock_migratetype(page, migratetype);
         zone->nr_isolate_pageblock--;
diff --git a/mm/slab.h b/mm/slab.h
index 4c3ac12dd644..48be14c4ec7b 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -114,7 +114,7 @@ static inline unsigned long kmem_cache_flags(unsigned long object_size,
 
 /* Legal flag mask for kmem_cache_create(), for various configurations */
 #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | SLAB_PANIC | \
-                         SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS )
+                         SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS | SLAB_NO_MERGE)
 
 #if defined(CONFIG_DEBUG_SLAB)
 #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
diff --git a/mm/slab_common.c b/mm/slab_common.c
index 999bb3424d44..dee018acaeaf 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -35,7 +35,7 @@ struct kmem_cache *kmem_cache;
  */
 #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
                 SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE | \
-                SLAB_FAILSLAB)
+                SLAB_FAILSLAB | SLAB_NO_MERGE)
 
 #define SLAB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
                 SLAB_CACHE_DMA | SLAB_NOTRACK)
@@ -703,7 +703,9 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
                 panic("Out of memory when creating slab %s\n", name);
 
         create_boot_cache(s, name, size, flags);
+
         list_add(&s->list, &slab_caches);
+        
         s->refcount = 1;
         return s;
 }
@@ -711,6 +713,11 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
 struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
 EXPORT_SYMBOL(kmalloc_caches);
 
+/* for per-cpu kmalloc objects */
+struct kmem_cache *hc_kmalloc_caches[NR_CPUS][KMALLOC_SHIFT_HIGH + 1];
+//struct kmem_cache *hc_kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
+EXPORT_SYMBOL(hc_kmalloc_caches);
+
 #ifdef CONFIG_ZONE_DMA
 struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
 EXPORT_SYMBOL(kmalloc_dma_caches);
@@ -790,7 +797,7 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
  */
 void __init create_kmalloc_caches(unsigned long flags)
 {
-        int i;
+        int i, cpu = 0;
 
         /*
          * Patch up the size_index table if we have strange large alignment
@@ -837,6 +844,7 @@ void __init create_kmalloc_caches(unsigned long flags)
                 if (!kmalloc_caches[i]) {
                         kmalloc_caches[i] = create_kmalloc_cache(NULL,
                                                         1 << i, flags);
+printk(KERN_INFO "KMALLOC-%d CACHE CREATED\n", 1<<i);
                 }
 
                 /*
@@ -844,11 +852,43 @@ void __init create_kmalloc_caches(unsigned long flags)
                  * These have to be created immediately after the
                  * earlier power of two caches
                  */
-                if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6)
+                if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6) {
                         kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);
+printk(KERN_INFO "KMALLOC-96 CACHE CREATED\n");
+                }
 
-                if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7)
+                if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7) {
                         kmalloc_caches[2] = create_kmalloc_cache(NULL, 192, flags);
+printk(KERN_INFO "KMALLOC-192 CACHE CREATED\n");
+                }
+        }
+
+/* per-cpu kmalloc caches */
+        printk(KERN_INFO "SLAB_STATE = %d\n", slab_state);
+        for (cpu = 0; cpu < NR_CPUS; cpu++) {
+        //cpu = 0;
+                for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
+                        char *n;
+                        n = kasprintf(GFP_NOWAIT, "hc%01d-kmalloc-%d", cpu, kmalloc_size(i));
+                        hc_kmalloc_caches[cpu][i] = create_kmalloc_cache(n, 1 << i, SLAB_NO_MERGE|flags);
+                        hc_kmalloc_caches[cpu][i]->cpu_id = cpu;
+                        printk(KERN_INFO "CPU%d HC-KMALLOC-%d CACHE CREATED\n", cpu, 1<<i);
+                        printk(KERN_INFO "HC-KMALLOC-%d slabs freelist=%p, pages=%p, partial=%p\n", 1<<i, hc_kmalloc_caches[cpu][i]->cpu_slab->freelist, hc_kmalloc_caches[cpu][i]->cpu_slab->page,hc_kmalloc_caches[cpu][i]->cpu_slab->partial);
+                        
+
+                        /*
+
+                        if (KMALLOC_MIN_SIZE <= 32 && !pc_kmalloc_caches[cpu][1] && i == 6) {
+                                pc_kmalloc_caches[cpu][1] = create_kmalloc_cache(NULL, 96, flags);
+        printk(KERN_INFO "PC-KMALLOC-96 CACHE CREATED\n");
+                        }
+
+                        if (KMALLOC_MIN_SIZE <= 64 && !pc_kmalloc_caches[cpu][2] && i == 7) {
+                                pc_kmalloc_caches[cpu][2] = create_kmalloc_cache(NULL, 192, flags);
+        printk(KERN_INFO "PC-KMALLOC-192 CACHE CREATED\n");
+                        }
+                        */
+                }
         }
 
         /* Kmalloc array is now usable */
diff --git a/mm/slub.c b/mm/slub.c
index 54c0876b43d5..2727a6fc403f 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2770,7 +2770,8 @@ EXPORT_SYMBOL(kmem_cache_free);
  * take the list_lock.
  */
 static int slub_min_order;
-static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
+//static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
+static int slub_max_order = 2;
 static int slub_min_objects;
 
 /*
@@ -5139,6 +5140,10 @@ static char *create_unique_id(struct kmem_cache *s)
          * are matched during merging to guarantee that the id is
          * unique.
          */
+        if (s->flags & SLAB_NO_MERGE) {
+                *p++ = 'n';
+                p += sprintf(p, "%01d", s->cpu_id);
+        }
         if (s->flags & SLAB_CACHE_DMA)
                 *p++ = 'd';
         if (s->flags & SLAB_RECLAIM_ACCOUNT)
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 4bbf65f7335b..0d748d23dc4c 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -739,6 +739,7 @@ const char * const vmstat_text[] = {
         "workingset_nodereclaim",
         "nr_anon_transparent_hugepages",
         "nr_free_cma",
+        "nr_free_hc_pages",
 
         /* enum writeback_stat_item counters */
         "nr_dirty_threshold",
@@ -754,6 +755,7 @@ const char * const vmstat_text[] = {
         TEXTS_FOR_ZONES("pgalloc")
 
         "pgfree",
+        "pgfree_hc",
         "pgactivate",
         "pgdeactivate",