From b0905375748cbc0fde1dfd7578bd4ff7ac47913b Mon Sep 17 00:00:00 2001 From: Namhoon Kim Date: Wed, 13 Sep 2017 05:48:56 -0400 Subject: Impl. Done. --- arch/arm/include/asm/dma-mapping.h | 7 +++ arch/arm/mm/dma-mapping.c | 15 +++++ drivers/media/usb/uvc/uvc_video.c | 4 +- drivers/usb/core/buffer.c | 1 + include/linux/mmzone.h | 4 +- include/linux/slab.h | 6 ++ litmus/page_dev.c | 1 + mm/page_alloc.c | 116 +++++++++++++++++++++++++++++-------- mm/slab_common.c | 6 +- mm/slub.c | 64 ++++++++++++-------- 10 files changed, 170 insertions(+), 54 deletions(-) diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index b52101d37ec7..fef26e00159d 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -219,6 +219,13 @@ static inline void *dma_alloc_attrs(struct device *dev, size_t size, void *cpu_addr; BUG_ON(!ops); +#ifdef CONFIG_SCHED_DEBUG_TRACE + if (flag&GFP_COLOR) { + printk(KERN_INFO "dma_alloc_attrs() \n"); + printk(KERN_INFO "func: %pF at address: %p\n", ops->alloc, ops->alloc); + } +#endif + cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs); debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr); return cpu_addr; diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index d0c70d97346b..e272fdcccc48 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -664,6 +664,11 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, size = PAGE_ALIGN(size); want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs); +#ifdef CONFIG_SCHED_DEBUG_TRACE + if (gfp&GFP_COLOR) + printk(KERN_INFO "__dma_alloc() for usb buffer\n"); +#endif + if (is_coherent || nommu()) addr = __alloc_simple_buffer(dev, size, gfp, &page); else if (!(gfp & __GFP_WAIT)) @@ -689,6 +694,16 @@ void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); void *memory; + if ((gfp&GFP_COLOR) && (size > PAGE_SIZE*4)) { +#ifdef CONFIG_SCHED_DEBUG_TRACE + printk(KERN_INFO "arm_dma_alloc(): original prot %08x\n", prot); +#endif + prot = pgprot_noncached(prot); +#ifdef CONFIG_SCHED_DEBUG_TRACE + printk(KERN_INFO "arm_dma_alloc(): set as uncacheable prot %08x\n", prot); +#endif + } + if (dma_alloc_from_coherent(dev, size, handle, &memory)) return memory; diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c index 20ccc9d315dc..454e6e83aa56 100644 --- a/drivers/media/usb/uvc/uvc_video.c +++ b/drivers/media/usb/uvc/uvc_video.c @@ -1406,10 +1406,10 @@ static int uvc_alloc_urb_buffers(struct uvc_streaming *stream, #ifndef CONFIG_DMA_NONCOHERENT stream->urb_buffer[i] = usb_alloc_coherent( stream->dev->udev, stream->urb_size, - gfp_flags | __GFP_NOWARN, &stream->urb_dma[i]); + gfp_flags | __GFP_NOWARN | GFP_COLOR, &stream->urb_dma[i]); #else stream->urb_buffer[i] = - kmalloc(stream->urb_size, gfp_flags | __GFP_NOWARN); + kmalloc(stream->urb_size, gfp_flags | __GFP_NOWARN | GFP_COLOR); #endif if (!stream->urb_buffer[i]) { uvc_free_urb_buffers(stream); diff --git a/drivers/usb/core/buffer.c b/drivers/usb/core/buffer.c index 506b969ea7fd..b9af51400214 100644 --- a/drivers/usb/core/buffer.c +++ b/drivers/usb/core/buffer.c @@ -128,6 +128,7 @@ void *hcd_buffer_alloc( if (size <= pool_max[i]) return dma_pool_alloc(hcd->pool[i], mem_flags, dma); } + return dma_alloc_coherent(hcd->self.controller, size, dma, mem_flags); } diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 750c0b64fa96..f0100050d1d6 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -44,7 +44,9 @@ #define CACHE_SHIFT 12 #define MAX_NUM_COLOR 16 #define MAX_NUM_BANK 8 -#define MAX_PARTITIONED_ORDER 3 +//#define MAX_PARTITIONED_ORDER 3 +#define MAX_PARTITIONED_ORDER 11 +#define MAX_CONTIG_ORDER 3 enum { MIGRATE_UNMOVABLE, diff --git a/include/linux/slab.h b/include/linux/slab.h index a899dda28def..6064df01e268 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -419,6 +419,12 @@ static __always_inline void *kmalloc_large(size_t size, gfp_t flags) */ static __always_inline void *kmalloc(size_t size, gfp_t flags) { + +#ifdef CONFIG_SCHED_DEBUG_TRACE + if (flags&GFP_COLOR) + printk(KERN_INFO "kmalloc() is called with GFP_COLOR\n"); +#endif + if (__builtin_constant_p(size)) { if (size > KMALLOC_MAX_CACHE_SIZE) return kmalloc_large(size, flags); diff --git a/litmus/page_dev.c b/litmus/page_dev.c index 2fd829b05a0a..ea5d5f5cb36d 100644 --- a/litmus/page_dev.c +++ b/litmus/page_dev.c @@ -6,6 +6,7 @@ */ #include +#include // This Address Decoding is used in imx6-sabredsd platform #define NUM_BANKS 8 diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 805c4f598b8c..90cf3ea441e0 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1032,6 +1032,41 @@ static inline void expand(struct zone *zone, struct page *page, } } +static inline void expand_middle(struct zone *zone, struct page *page, + int offset, int low, int high, struct free_area *area, + int migratetype) +{ + unsigned long size = 1 << high; + + while ((size>>1) > offset) { + area--; + high--; + size >>= 1; + VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); + + list_add(&page[size].lru, &area->free_list[migratetype]); + area->nr_free++; + set_page_order(&page[size], high); + } + area--; + high--; + size >>= 1; + VM_BUG_ON_PAGE(bad_range(zone, page), page); + list_add(&page[0].lru, &area->free_list[migratetype]); + area->nr_free++; + set_page_order(&page[0], high); + + if (offset == size) + return; + + area--; + high--; + VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); + list_add(&page[size].lru, &area->free_list[migratetype]); + area->nr_free++; + set_page_order(&page[size], high); +} + /* * This page is about to be returned from the page allocator */ @@ -1193,23 +1228,38 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, int found = 0; unsigned long s_pfn = zone->zone_start_pfn; unsigned long e_pfn = zone_end_pfn(zone); - printk(KERN_INFO "COLOR PAGE requested on CPU%d with order = %d migratetype = %d\n", cpu, order, migratetype); + TRACE("COLOR PAGE requested on CPU%d with order = %d migratetype = %d\n", cpu, order, migratetype); /* Find a page of the appropriate size in the preferred list */ for (current_order = order; current_order < MAX_PARTITIONED_ORDER; ++current_order) { + int offset = 0; area = &(zone->free_area_d[cpu][current_order]); if (list_empty(&area->free_list[migratetype])) { - printk(KERN_INFO "order %d list empty\n", current_order); + TRACE("order %d list empty\n", current_order); continue; } - + list_for_each_entry(page, &area->free_list[migratetype], lru) { - printk(KERN_INFO "__rmqueue_smallest list entry %p color %d\n", page, page_color(page)); - if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) { - found = 1; - break; + TRACE("__rmqueue_smallest list entry %p color %d pfn:%05lx\n", page, page_color(page), page_to_pfn(page)); + if (current_order < MAX_CONTIG_ORDER) { + if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) { + found = 1; + offset = 0; + break; + } + } else { // order >= 3 , must be uncacheable. + int size = 1 << current_order; + for (offset = 0; offset < size; offset += 4) { + if (is_in_llc_partition(&page[offset], cpu) && (page_to_pfn(&page[offset]) >= s_pfn && page_to_pfn(&page[offset]) < e_pfn)) { + found = 1; + break; + } + } + if (found) + break; } } - printk(KERN_INFO "__rmqueue_smallest LAST list entry %p\n", page); + + TRACE("__rmqueue_smallest LAST list entry %p\n", page); if (!found) return NULL; @@ -1219,7 +1269,7 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, area->nr_free--; expand(zone, page, order, current_order, area, migratetype); set_freepage_migratetype(page, migratetype); - printk(KERN_INFO "COLOR %d page return %p\n", page_color(page), page); + TRACE("COLOR %d page return %p\n", page_color(page), page); return page; } } else { @@ -1470,7 +1520,8 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype, /* Find the largest possible block of pages in the other list */ for (current_order = MAX_PARTITIONED_ORDER-1; current_order >= order && current_order <= MAX_PARTITIONED_ORDER-1; - --current_order) { + --current_order) { + int offset = 0; area = &(zone->free_area_d[cpu][current_order]); fallback_mt = find_suitable_fallback(area, current_order, start_migratetype, false, &can_steal); @@ -1478,13 +1529,26 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype, continue; list_for_each_entry(page, &area->free_list[fallback_mt], lru) { - printk(KERN_INFO "__rmqueue_falback list entry %p color %d\n", page, page_color(page)); - if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) { - found = 1; - break; + TRACE("__rmqueue_fallback list entry %p color %d pfn:%05lx\n", page, page_color(page), page_to_pfn(page)); + if (current_order < MAX_CONTIG_ORDER) { + if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) { + found = 1; + offset = 0; + break; + } + } else { // order >= 3 , must be uncacheable. + int size = 1 << current_order; + for (offset = 0; offset < size; offset += 4) { + if (is_in_llc_partition(&page[offset], cpu) && (page_to_pfn(&page[offset]) >= s_pfn && page_to_pfn(&page[offset]) < e_pfn)) { + found = 1; + break; + } + } + if (found) + break; } } - printk(KERN_INFO "__rmqueue_falback LAST list entry %p\n", page); + TRACE("__rmqueue_fallback LAST list entry %p\n", page); if (!found) return NULL; @@ -1497,8 +1561,8 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype, list_del(&page->lru); rmv_page_order(page); - expand(zone, page, order, current_order, area, - start_migratetype); + expand(zone, page, order, current_order, area, start_migratetype); + /* * The freepage_migratetype may differ from pageblock's * migratetype depending on the decisions in @@ -1512,7 +1576,7 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype, trace_mm_page_alloc_extfrag(page, order, current_order, start_migratetype, fallback_mt); - printk(KERN_INFO "__rmqueue_fallback(): CPU%d COLOR %d page return %p\n", cpu, page_color(page), page); + TRACE("__rmqueue_fallback(): CPU%d COLOR %d page return %p pfn:%05lx\n", cpu, page_color(page), page, page_to_pfn(page)); return page; } } else { @@ -1576,8 +1640,10 @@ retry_reserve: if (!page) { page = __rmqueue_fallback(zone, order, migratetype, color_req); +#ifdef CONFIG_SCHED_DEBUG_TRACE if (color_req) - printk(KERN_INFO "page received from __rmqueue_fallback()"); + TRACE("page received from __rmqueue_fallback()"); +#endif } /* @@ -2005,8 +2071,10 @@ struct page *buffered_rmqueue(struct zone *preferred_zone, bool cold = ((gfp_flags & __GFP_COLD) != 0); bool colored_req = ((gfp_flags & __GFP_COLOR) != 0); -if (colored_req) - printk(KERN_INFO "buffered_rmqueue(): colored_req received\n"); +#ifdef CONFIG_SCHED_DEBUG_TRACE + if (colored_req) + TRACE("buffered_rmqueue(): colored_req received\n"); +#endif if (likely(order == 0) && !colored_req) { struct per_cpu_pages *pcp; @@ -3158,8 +3226,10 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, .migratetype = gfpflags_to_migratetype(gfp_mask), }; -if (gfp_mask&GFP_COLOR) - printk(KERN_INFO "__alloc_pages_nodemask(): called gfp %08x gfp_allowed_mask %08x mt = %d\n", gfp_mask, gfp_allowed_mask, ac.migratetype); +#ifdef CONFIG_SCHED_DEBUG_TRACE + if (gfp_mask&GFP_COLOR) + TRACE("__alloc_pages_nodemask(): called gfp %08x gfp_allowed_mask %08x mt = %d\n", gfp_mask, gfp_allowed_mask, ac.migratetype); +#endif gfp_mask &= gfp_allowed_mask; diff --git a/mm/slab_common.c b/mm/slab_common.c index ff4d4c6f4129..bbd0ddc0b029 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -790,7 +790,7 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags) if (flags & GFP_COLOR) { int cpu = raw_smp_processor_id(); - printk(KERN_INFO "in kmalloc_slab index %d\n", index); + TRACE("in kmalloc_slab index %d\n", index); return hc_kmalloc_caches[cpu][index]; } else @@ -848,11 +848,9 @@ void __init create_kmalloc_caches(unsigned long flags) size_index[size_index_elem(i)] = 8; } for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) { - printk(KERN_INFO "KMALLOC i = %d\n", i); if (!kmalloc_caches[i]) { kmalloc_caches[i] = create_kmalloc_cache(NULL, 1 << i, flags); -printk(KERN_INFO "KMALLOC-%d CACHE CREATED\n", 1<oo; gfp_t alloc_gfp; -if (flags&GFP_COLOR) - printk(KERN_INFO "gfp_allowed_mask = %08x\n", gfp_allowed_mask); +#ifdef CONFIG_SCHED_DEBUG_TRACE + if (flags&GFP_COLOR) + printk(KERN_INFO "gfp_allowed_mask = %08x\n", gfp_allowed_mask); +#endif flags &= gfp_allowed_mask; @@ -1355,9 +1359,12 @@ if (flags&GFP_COLOR) * so we fall-back to the minimum order allocation. */ alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL; -if (flags&__GFP_COLOR) { - printk(KERN_INFO "allocate_slab with GFP_COLOR alloc_gfp = %08x\n", alloc_gfp); -} + +#ifdef CONFIG_SCHED_DEBUG_TRACE + if (flags&__GFP_COLOR) + printk(KERN_INFO "allocate_slab with GFP_COLOR alloc_gfp = %08x\n", alloc_gfp); +#endif + page = alloc_slab_page(s, alloc_gfp, node, oo); if (unlikely(!page)) { oo = s->min; @@ -2232,9 +2239,10 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, page = new_slab(s, flags, node); -if (flags&GFP_COLOR) { - printk(KERN_INFO "new_slab_objects(): gets page %p\n", page); -} +#ifdef CONFIG_SCHED_DEBUG_TRACE + if (flags&GFP_COLOR) + printk(KERN_INFO "new_slab_objects(): gets page %p\n", page); +#endif if (page) { c = raw_cpu_ptr(s->cpu_slab); @@ -2321,8 +2329,11 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, void *freelist; struct page *page; unsigned long flags; -if (gfpflags&GFP_COLOR) - printk(KERN_INFO "__slab_alloc slow_path\n"); + +#ifdef CONFIG_SCHED_DEBUG_TRACE + if (gfpflags&GFP_COLOR) + printk(KERN_INFO "__slab_alloc slow_path\n"); +#endif local_irq_save(flags); #ifdef CONFIG_PREEMPT @@ -2334,10 +2345,10 @@ if (gfpflags&GFP_COLOR) c = this_cpu_ptr(s->cpu_slab); #endif - -if (gfpflags&GFP_COLOR) { - printk(KERN_INFO "__slab_alloc : page %p, partial %p\n", c->page, c->partial); -} +#ifdef CONFIG_SCHED_DEBUG_TRACE + if (gfpflags&GFP_COLOR) + printk(KERN_INFO "__slab_alloc : page %p, partial %p\n", c->page, c->partial); +#endif page = c->page; if (!page) @@ -3328,22 +3339,29 @@ void *__kmalloc(size_t size, gfp_t flags) struct kmem_cache *s; void *ret; -if (flags & GFP_COLOR) { - printk(KERN_INFO "kmalloc size %d\n", size); -} +#ifdef CONFIG_SCHED_DEBUG_TRACE + if (flags & GFP_COLOR) + printk(KERN_INFO "kmalloc size %d\n", size); +#endif + if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) return kmalloc_large(size, flags); s = kmalloc_slab(size, flags); -if (flags & GFP_COLOR) { - printk(KERN_INFO "kmalloc_slab %p\n", s); -} + +#ifdef CONFIG_SCHED_DEBUG_TRACE + if (flags & GFP_COLOR) + printk(KERN_INFO "kmalloc_slab %p\n", s); +#endif + if (unlikely(ZERO_OR_NULL_PTR(s))) return s; -if (flags & GFP_COLOR) { - printk(KERN_INFO "slab_alloc calls!!\n"); -} +#ifdef CONFIG_SCHED_DEBUG_TRACE + if (flags & GFP_COLOR) + printk(KERN_INFO "slab_alloc calls!!\n"); +#endif + ret = slab_alloc(s, flags, _RET_IP_); trace_kmalloc(_RET_IP_, ret, size, s->size, flags); -- cgit v1.2.2