From 84e36eb17cd9e5bb4bfb6cc4e29f55542fa69fde Mon Sep 17 00:00:00 2001 From: Namhoon Kim Date: Wed, 6 Sep 2017 08:50:44 -0400 Subject: fix PGFREE and NR_FREE_PAGES --- include/linux/mmzone.h | 1 + include/linux/slab.h | 2 ++ include/linux/slub_def.h | 2 ++ include/linux/vm_event_item.h | 2 +- include/linux/vmstat.h | 7 +++++-- litmus/page_dev.c | 2 +- mm/page_alloc.c | 35 +++++++++++++++++++++---------- mm/page_isolation.c | 6 ++++-- mm/slab.h | 2 +- mm/slab_common.c | 48 +++++++++++++++++++++++++++++++++++++++---- mm/slub.c | 7 ++++++- mm/vmstat.c | 2 ++ 12 files changed, 93 insertions(+), 23 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index d28f7ef8228d..750c0b64fa96 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -168,6 +168,7 @@ enum zone_stat_item { WORKINGSET_NODERECLAIM, NR_ANON_TRANSPARENT_HUGEPAGES, NR_FREE_CMA_PAGES, + NR_FREE_HC_PAGES, NR_VM_ZONE_STAT_ITEMS }; /* diff --git a/include/linux/slab.h b/include/linux/slab.h index ffd24c830151..a899dda28def 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -87,6 +87,8 @@ # define SLAB_FAILSLAB 0x00000000UL #endif +#define SLAB_NO_MERGE 0x04000000UL /* Do not merge with existing slab */ + /* The following flags affect the page allocator grouping pages by mobility */ #define SLAB_RECLAIM_ACCOUNT 0x00020000UL /* Objects are reclaimable */ #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 33885118523c..9400aa1e9128 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -98,6 +98,8 @@ struct kmem_cache { */ int remote_node_defrag_ratio; #endif + /* cpu id for higher-criticality slabs */ + int cpu_id; struct kmem_cache_node *node[MAX_NUMNODES]; }; diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index 9246d32dc973..3f5a9da27d7a 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -23,7 +23,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, FOR_ALL_ZONES(PGALLOC), - PGFREE, PGACTIVATE, PGDEACTIVATE, + PGFREE, PGFREE_HC, PGACTIVATE, PGDEACTIVATE, PGFAULT, PGMAJFAULT, FOR_ALL_ZONES(PGREFILL), FOR_ALL_ZONES(PGSTEAL_KSWAPD), diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h index 82e7db7f7100..b6410f7efd74 100644 --- a/include/linux/vmstat.h +++ b/include/linux/vmstat.h @@ -278,9 +278,12 @@ static inline void drain_zonestat(struct zone *zone, #endif /* CONFIG_SMP */ static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages, - int migratetype) + int migratetype, int part_no) { - __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages); + if (part_no == NR_CPUS) + __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages); + else + __mod_zone_page_state(zone, NR_FREE_HC_PAGES, nr_pages); if (is_migrate_cma(migratetype)) __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages); } diff --git a/litmus/page_dev.c b/litmus/page_dev.c index 8e29e68ed89a..2894e93213d3 100644 --- a/litmus/page_dev.c +++ b/litmus/page_dev.c @@ -226,7 +226,7 @@ static struct ctl_table partition_table[] = .maxlen = sizeof(llc_partition[4]), .extra1 = &dram_partition_min, .extra2 = &dram_partition_max, - }, + }, { } }; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 4611656df49a..e240fcd3039d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -485,7 +485,7 @@ static inline void set_page_guard(struct zone *zone, struct page *page, INIT_LIST_HEAD(&page->lru); set_page_private(page, order); /* Guard pages are not available for any usage */ - __mod_zone_freepage_state(zone, -(1 << order), migratetype); + __mod_zone_freepage_state(zone, -(1 << order), migratetype, bank_to_partition(page_bank(page))); } static inline void clear_page_guard(struct zone *zone, struct page *page, @@ -501,7 +501,7 @@ static inline void clear_page_guard(struct zone *zone, struct page *page, set_page_private(page, 0); if (!is_migrate_isolate(migratetype)) - __mod_zone_freepage_state(zone, (1 << order), migratetype); + __mod_zone_freepage_state(zone, (1 << order), migratetype, bank_to_partition(page_bank(page))); } #else struct page_ext_operations debug_guardpage_ops = { NULL, }; @@ -629,7 +629,7 @@ static inline void __free_one_page(struct page *page, */ max_order = min(MAX_ORDER, pageblock_order + 1); } else { - __mod_zone_freepage_state(zone, 1 << order, migratetype); + __mod_zone_freepage_state(zone, 1 << order, migratetype, parti_no); } page_idx = pfn & ((1 << max_order) - 1); @@ -694,7 +694,7 @@ out: if (is_migrate_isolate(migratetype)) { max_order = min(MAX_PARTITIONED_ORDER, pageblock_order + 1); } else { - __mod_zone_freepage_state(zone, 1 << order, migratetype); + __mod_zone_freepage_state(zone, 1 << order, migratetype, parti_no); } page_idx = pfn & ((1 << max_order) - 1); @@ -927,7 +927,10 @@ static void __free_pages_ok(struct page *page, unsigned int order) migratetype = get_pfnblock_migratetype(page, pfn); local_irq_save(flags); - __count_vm_events(PGFREE, 1 << order); + if (bank_to_partition(page_bank(page)) == NR_CPUS) + __count_vm_events(PGFREE, 1 << order); + else if (bank_to_partition(page_bank(page)) < NR_CPUS) + __count_vm_events(PGFREE_HC, 1 << order); set_freepage_migratetype(page, migratetype); free_one_page(page_zone(page), page, pfn, order, migratetype); local_irq_restore(flags); @@ -1097,6 +1100,7 @@ static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, /* Kernel page coloring */ /* build colored page list */ +#if 0 static void build_colored_pages(struct zone *zone, struct page *page, int order) { int i, color, bank; @@ -1167,6 +1171,7 @@ static inline struct page *get_colored_page(struct zone *zone, unsigned long req printk(KERN_INFO "color=%d, bank=%d allocated\n", color, bank); return page; } +#endif /* * Go through the free lists for the given migratetype and remove @@ -1181,8 +1186,11 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, struct page *page; int cpu = raw_smp_processor_id(); - if (order == 0 && color_req == 1) { + // if (order <= 2 && color_req == 1) { + /* The max. order of color_req is <= 2 */ + if (color_req == 1) { int found = 0; + printk(KERN_INFO "COLOR PAGE requested on CPU%d with order = %d\n", cpu, order); /* Find a page of the appropriate size in the preferred list */ for (current_order = order; current_order < MAX_PARTITIONED_ORDER; ++current_order) { area = &(zone->free_area_d[cpu][current_order]); @@ -1555,7 +1563,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, -(1 << order)); } - __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); + __mod_zone_page_state(zone, NR_FREE_HC_PAGES, -(i << order)); spin_unlock(&zone->lock); return i; } @@ -1752,8 +1760,13 @@ void free_hot_cold_page(struct page *page, bool cold) migratetype = get_pfnblock_migratetype(page, pfn); set_freepage_migratetype(page, migratetype); local_irq_save(flags); - __count_vm_event(PGFREE); - + + + if (bank_to_partition(page_bank(page)) == NR_CPUS) + __count_vm_event(PGFREE); + else if (bank_to_partition(page_bank(page)) < NR_CPUS) + __count_vm_event(PGFREE_HC); + /* * We only track unmovable, reclaimable and movable on pcp lists. * Free ISOLATE pages back to the allocator because they are being @@ -1861,7 +1874,7 @@ int __isolate_free_page(struct page *page, unsigned int order) if (!zone_watermark_ok(zone, 0, watermark, 0, 0)) return 0; - __mod_zone_freepage_state(zone, -(1UL << order), mt); + __mod_zone_freepage_state(zone, -(1UL << order), mt, bank_to_partition(page_bank(page))); } /* Remove page from free list */ @@ -1966,7 +1979,7 @@ struct page *buffered_rmqueue(struct zone *preferred_zone, if (!page) goto failed; __mod_zone_freepage_state(zone, -(1 << order), - get_freepage_migratetype(page)); + get_freepage_migratetype(page), bank_to_partition(page_bank(page))); } __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order)); diff --git a/mm/page_isolation.c b/mm/page_isolation.c index 303c908790ef..ed05910069ff 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -58,12 +58,13 @@ out: if (!ret) { unsigned long nr_pages; int migratetype = get_pageblock_migratetype(page); + int partno = bank_to_partition(page_bank(page)); set_pageblock_migratetype(page, MIGRATE_ISOLATE); zone->nr_isolate_pageblock++; nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE); - __mod_zone_freepage_state(zone, -nr_pages, migratetype); + __mod_zone_freepage_state(zone, -nr_pages, migratetype, partno); } spin_unlock_irqrestore(&zone->lock, flags); @@ -117,8 +118,9 @@ void unset_migratetype_isolate(struct page *page, unsigned migratetype) * pageblock scanning for freepage moving. */ if (!isolated_page) { + int partno = bank_to_partition(page_bank(page)); nr_pages = move_freepages_block(zone, page, migratetype); - __mod_zone_freepage_state(zone, nr_pages, migratetype); + __mod_zone_freepage_state(zone, nr_pages, migratetype, partno); } set_pageblock_migratetype(page, migratetype); zone->nr_isolate_pageblock--; diff --git a/mm/slab.h b/mm/slab.h index 4c3ac12dd644..48be14c4ec7b 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -114,7 +114,7 @@ static inline unsigned long kmem_cache_flags(unsigned long object_size, /* Legal flag mask for kmem_cache_create(), for various configurations */ #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | SLAB_PANIC | \ - SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS ) + SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS | SLAB_NO_MERGE) #if defined(CONFIG_DEBUG_SLAB) #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) diff --git a/mm/slab_common.c b/mm/slab_common.c index 999bb3424d44..dee018acaeaf 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -35,7 +35,7 @@ struct kmem_cache *kmem_cache; */ #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE | \ - SLAB_FAILSLAB) + SLAB_FAILSLAB | SLAB_NO_MERGE) #define SLAB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \ SLAB_CACHE_DMA | SLAB_NOTRACK) @@ -703,7 +703,9 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size, panic("Out of memory when creating slab %s\n", name); create_boot_cache(s, name, size, flags); + list_add(&s->list, &slab_caches); + s->refcount = 1; return s; } @@ -711,6 +713,11 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size, struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1]; EXPORT_SYMBOL(kmalloc_caches); +/* for per-cpu kmalloc objects */ +struct kmem_cache *hc_kmalloc_caches[NR_CPUS][KMALLOC_SHIFT_HIGH + 1]; +//struct kmem_cache *hc_kmalloc_caches[KMALLOC_SHIFT_HIGH + 1]; +EXPORT_SYMBOL(hc_kmalloc_caches); + #ifdef CONFIG_ZONE_DMA struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1]; EXPORT_SYMBOL(kmalloc_dma_caches); @@ -790,7 +797,7 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags) */ void __init create_kmalloc_caches(unsigned long flags) { - int i; + int i, cpu = 0; /* * Patch up the size_index table if we have strange large alignment @@ -837,6 +844,7 @@ void __init create_kmalloc_caches(unsigned long flags) if (!kmalloc_caches[i]) { kmalloc_caches[i] = create_kmalloc_cache(NULL, 1 << i, flags); +printk(KERN_INFO "KMALLOC-%d CACHE CREATED\n", 1<cpu_id = cpu; + printk(KERN_INFO "CPU%d HC-KMALLOC-%d CACHE CREATED\n", cpu, 1<cpu_slab->freelist, hc_kmalloc_caches[cpu][i]->cpu_slab->page,hc_kmalloc_caches[cpu][i]->cpu_slab->partial); + + + /* + + if (KMALLOC_MIN_SIZE <= 32 && !pc_kmalloc_caches[cpu][1] && i == 6) { + pc_kmalloc_caches[cpu][1] = create_kmalloc_cache(NULL, 96, flags); + printk(KERN_INFO "PC-KMALLOC-96 CACHE CREATED\n"); + } + + if (KMALLOC_MIN_SIZE <= 64 && !pc_kmalloc_caches[cpu][2] && i == 7) { + pc_kmalloc_caches[cpu][2] = create_kmalloc_cache(NULL, 192, flags); + printk(KERN_INFO "PC-KMALLOC-192 CACHE CREATED\n"); + } + */ + } } /* Kmalloc array is now usable */ diff --git a/mm/slub.c b/mm/slub.c index 54c0876b43d5..2727a6fc403f 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2770,7 +2770,8 @@ EXPORT_SYMBOL(kmem_cache_free); * take the list_lock. */ static int slub_min_order; -static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER; +//static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER; +static int slub_max_order = 2; static int slub_min_objects; /* @@ -5139,6 +5140,10 @@ static char *create_unique_id(struct kmem_cache *s) * are matched during merging to guarantee that the id is * unique. */ + if (s->flags & SLAB_NO_MERGE) { + *p++ = 'n'; + p += sprintf(p, "%01d", s->cpu_id); + } if (s->flags & SLAB_CACHE_DMA) *p++ = 'd'; if (s->flags & SLAB_RECLAIM_ACCOUNT) diff --git a/mm/vmstat.c b/mm/vmstat.c index 4bbf65f7335b..0d748d23dc4c 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -739,6 +739,7 @@ const char * const vmstat_text[] = { "workingset_nodereclaim", "nr_anon_transparent_hugepages", "nr_free_cma", + "nr_free_hc_pages", /* enum writeback_stat_item counters */ "nr_dirty_threshold", @@ -754,6 +755,7 @@ const char * const vmstat_text[] = { TEXTS_FOR_ZONES("pgalloc") "pgfree", + "pgfree_hc", "pgactivate", "pgdeactivate", -- cgit v1.2.2