Impl. Done.

10 files changed, 170 insertions, 54 deletions

diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
index b52101d37ec7..fef26e00159d 100644
--- a/arch/arm/include/asm/dma-mapping.h
+++ b/arch/arm/include/asm/dma-mapping.h
@@ -219,6 +219,13 @@ static inline void *dma_alloc_attrs(struct device *dev, size_t size,
         void *cpu_addr;
         BUG_ON(!ops);
 
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+        if (flag&GFP_COLOR) {
+                printk(KERN_INFO "dma_alloc_attrs() \n");
+                printk(KERN_INFO "func: %pF at address: %p\n", ops->alloc, ops->alloc);
+        }
+#endif
+        
         cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
         debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr);
         return cpu_addr;
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index d0c70d97346b..e272fdcccc48 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -664,6 +664,11 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
         size = PAGE_ALIGN(size);
         want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs);
 
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+        if (gfp&GFP_COLOR)
+                printk(KERN_INFO "__dma_alloc() for usb buffer\n");
+#endif
+        
         if (is_coherent || nommu())
                 addr = __alloc_simple_buffer(dev, size, gfp, &page);
         else if (!(gfp & __GFP_WAIT))
@@ -689,6 +694,16 @@ void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
         pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
         void *memory;
 
+        if ((gfp&GFP_COLOR) && (size > PAGE_SIZE*4)) {
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+                printk(KERN_INFO "arm_dma_alloc(): original prot %08x\n", prot);
+#endif
+                prot = pgprot_noncached(prot);
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+                printk(KERN_INFO "arm_dma_alloc(): set as uncacheable prot %08x\n", prot);
+#endif
+        }
+        
         if (dma_alloc_from_coherent(dev, size, handle, &memory))
                 return memory;
 
diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c
index 20ccc9d315dc..454e6e83aa56 100644
--- a/drivers/media/usb/uvc/uvc_video.c
+++ b/drivers/media/usb/uvc/uvc_video.c
@@ -1406,10 +1406,10 @@ static int uvc_alloc_urb_buffers(struct uvc_streaming *stream,
 #ifndef CONFIG_DMA_NONCOHERENT
                         stream->urb_buffer[i] = usb_alloc_coherent(
                                 stream->dev->udev, stream->urb_size,
-                                gfp_flags | __GFP_NOWARN, &stream->urb_dma[i]);
+                                gfp_flags | __GFP_NOWARN | GFP_COLOR, &stream->urb_dma[i]);
 #else
                         stream->urb_buffer[i] =
-                            kmalloc(stream->urb_size, gfp_flags | __GFP_NOWARN);
+                            kmalloc(stream->urb_size, gfp_flags | __GFP_NOWARN | GFP_COLOR);
 #endif
                         if (!stream->urb_buffer[i]) {
                                 uvc_free_urb_buffers(stream);
diff --git a/drivers/usb/core/buffer.c b/drivers/usb/core/buffer.c
index 506b969ea7fd..b9af51400214 100644
--- a/drivers/usb/core/buffer.c
+++ b/drivers/usb/core/buffer.c
@@ -128,6 +128,7 @@ void *hcd_buffer_alloc(
                 if (size <= pool_max[i])
                         return dma_pool_alloc(hcd->pool[i], mem_flags, dma);
         }
+
         return dma_alloc_coherent(hcd->self.controller, size, dma, mem_flags);
 }
 
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 750c0b64fa96..f0100050d1d6 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -44,7 +44,9 @@
 #define CACHE_SHIFT                             12
 #define MAX_NUM_COLOR                   16
 #define MAX_NUM_BANK                    8
-#define MAX_PARTITIONED_ORDER   3
+//#define MAX_PARTITIONED_ORDER 3
+#define MAX_PARTITIONED_ORDER   11
+#define MAX_CONTIG_ORDER                3
 
 enum {
         MIGRATE_UNMOVABLE,
diff --git a/include/linux/slab.h b/include/linux/slab.h
index a899dda28def..6064df01e268 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -419,6 +419,12 @@ static __always_inline void *kmalloc_large(size_t size, gfp_t flags)
  */
 static __always_inline void *kmalloc(size_t size, gfp_t flags)
 {
+        
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+        if (flags&GFP_COLOR)
+                printk(KERN_INFO "kmalloc() is called with GFP_COLOR\n");
+#endif
+        
         if (__builtin_constant_p(size)) {
                 if (size > KMALLOC_MAX_CACHE_SIZE)
                         return kmalloc_large(size, flags);
diff --git a/litmus/page_dev.c b/litmus/page_dev.c
index 2fd829b05a0a..ea5d5f5cb36d 100644
--- a/litmus/page_dev.c
+++ b/litmus/page_dev.c
@@ -6,6 +6,7 @@
  */

  

 #include <litmus/page_dev.h>

+#include <litmus/debug_trace.h>

 

 // This Address Decoding is used in imx6-sabredsd platform

 #define NUM_BANKS       8

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 805c4f598b8c..90cf3ea441e0 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1032,6 +1032,41 @@ static inline void expand(struct zone *zone, struct page *page,
         }
 }
 
+static inline void expand_middle(struct zone *zone, struct page *page,
+        int offset, int low, int high, struct free_area *area,
+        int migratetype)
+{
+        unsigned long size = 1 << high;
+
+        while ((size>>1) > offset) {
+                area--;
+                high--;
+                size >>= 1;
+                VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
+
+                list_add(&page[size].lru, &area->free_list[migratetype]);
+                area->nr_free++;
+                set_page_order(&page[size], high);
+        }
+        area--;
+        high--;
+        size >>= 1;
+        VM_BUG_ON_PAGE(bad_range(zone, page), page);
+        list_add(&page[0].lru, &area->free_list[migratetype]);
+        area->nr_free++;
+        set_page_order(&page[0], high);
+        
+        if (offset == size)
+                return;
+        
+        area--;
+        high--;
+        VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
+        list_add(&page[size].lru, &area->free_list[migratetype]);
+        area->nr_free++;
+        set_page_order(&page[size], high);
+}
+
 /*
  * This page is about to be returned from the page allocator
  */
@@ -1193,23 +1228,38 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
                 int found = 0;
                 unsigned long s_pfn = zone->zone_start_pfn;
                 unsigned long e_pfn = zone_end_pfn(zone);
-                printk(KERN_INFO "COLOR PAGE requested on CPU%d with order = %d migratetype = %d\n", cpu, order, migratetype);
+                TRACE("COLOR PAGE requested on CPU%d with order = %d migratetype = %d\n", cpu, order, migratetype);
                 /* Find a page of the appropriate size in the preferred list */
                 for (current_order = order; current_order < MAX_PARTITIONED_ORDER; ++current_order) {
+                        int offset = 0;
                         area = &(zone->free_area_d[cpu][current_order]);
                         if (list_empty(&area->free_list[migratetype])) {
-                                printk(KERN_INFO "order %d list empty\n", current_order);
+                                TRACE("order %d list empty\n", current_order);
                                 continue;
                         }
-
+                        
                         list_for_each_entry(page, &area->free_list[migratetype], lru) {
-                                printk(KERN_INFO "__rmqueue_smallest list entry %p color %d\n", page, page_color(page));
-                                if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) {
-                                        found = 1;
-                                        break;
+                                TRACE("__rmqueue_smallest list entry %p color %d pfn:%05lx\n", page, page_color(page), page_to_pfn(page));
+                                if (current_order < MAX_CONTIG_ORDER) {
+                                        if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) {
+                                                found = 1;
+                                                offset = 0;
+                                                break;
+                                        }
+                                } else {        // order >= 3 , must be uncacheable.
+                                        int size = 1 << current_order;
+                                        for (offset = 0; offset < size; offset += 4) {
+                                                if (is_in_llc_partition(&page[offset], cpu) && (page_to_pfn(&page[offset]) >= s_pfn && page_to_pfn(&page[offset]) < e_pfn)) {
+                                                        found = 1;
+                                                        break;
+                                                }
+                                        }
+                                        if (found)
+                                                break;
                                 }
                         }
-                        printk(KERN_INFO "__rmqueue_smallest LAST list entry %p\n", page);
+                        
+                        TRACE("__rmqueue_smallest LAST list entry %p\n", page);
 
                         if (!found)
                                 return NULL;
@@ -1219,7 +1269,7 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
                         area->nr_free--;
                         expand(zone, page, order, current_order, area, migratetype);
                         set_freepage_migratetype(page, migratetype);
-                        printk(KERN_INFO "COLOR %d page return %p\n", page_color(page), page);
+                        TRACE("COLOR %d page return %p\n", page_color(page), page);
                         return page;
                 }
         } else {
@@ -1470,7 +1520,8 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype,
                 /* Find the largest possible block of pages in the other list */
                 for (current_order = MAX_PARTITIONED_ORDER-1;
                                         current_order >= order && current_order <= MAX_PARTITIONED_ORDER-1;
-                                        --current_order) {                                              
+                                        --current_order) {              
+                        int offset = 0;
                         area = &(zone->free_area_d[cpu][current_order]);
                         fallback_mt = find_suitable_fallback(area, current_order,
                                         start_migratetype, false, &can_steal);
@@ -1478,13 +1529,26 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype,
                                 continue;
         
                         list_for_each_entry(page, &area->free_list[fallback_mt], lru) {
-                                printk(KERN_INFO "__rmqueue_falback list entry %p color %d\n", page, page_color(page));
-                                if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) {
-                                        found = 1;
-                                        break;
+                                TRACE("__rmqueue_fallback list entry %p color %d pfn:%05lx\n", page, page_color(page), page_to_pfn(page));
+                                if (current_order < MAX_CONTIG_ORDER) {
+                                        if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) {
+                                                found = 1;
+                                                offset = 0;
+                                                break;
+                                        }
+                                } else {        // order >= 3 , must be uncacheable.
+                                        int size = 1 << current_order;
+                                        for (offset = 0; offset < size; offset += 4) {
+                                                if (is_in_llc_partition(&page[offset], cpu) && (page_to_pfn(&page[offset]) >= s_pfn && page_to_pfn(&page[offset]) < e_pfn)) {
+                                                        found = 1;
+                                                        break;
+                                                }
+                                        }
+                                        if (found)
+                                                break;
                                 }
                         }
-                        printk(KERN_INFO "__rmqueue_falback LAST list entry %p\n", page);
+                        TRACE("__rmqueue_fallback LAST list entry %p\n", page);
 
                         if (!found)
                                 return NULL;
@@ -1497,8 +1561,8 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype,
                         list_del(&page->lru);
                         rmv_page_order(page);
 
-                        expand(zone, page, order, current_order, area,
-                                                start_migratetype);
+                        expand(zone, page, order, current_order, area, start_migratetype);
+
                         /*
                          * The freepage_migratetype may differ from pageblock's
                          * migratetype depending on the decisions in
@@ -1512,7 +1576,7 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype,
                         trace_mm_page_alloc_extfrag(page, order, current_order,
                                 start_migratetype, fallback_mt);
                 
-                        printk(KERN_INFO "__rmqueue_fallback(): CPU%d COLOR %d page return %p\n", cpu, page_color(page), page);
+                        TRACE("__rmqueue_fallback(): CPU%d COLOR %d page return %p pfn:%05lx\n", cpu, page_color(page), page, page_to_pfn(page));
                         return page;
                 }
         } else {
@@ -1576,8 +1640,10 @@ retry_reserve:
 
                 if (!page) {
                         page = __rmqueue_fallback(zone, order, migratetype, color_req);
+#ifdef CONFIG_SCHED_DEBUG_TRACE
                         if (color_req)
-                                printk(KERN_INFO "page received from __rmqueue_fallback()");
+                                TRACE("page received from __rmqueue_fallback()");
+#endif                  
                 }
 
                 /*
@@ -2005,8 +2071,10 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
         bool cold = ((gfp_flags & __GFP_COLD) != 0);
         bool colored_req = ((gfp_flags & __GFP_COLOR) != 0);
 
-if (colored_req)
-        printk(KERN_INFO "buffered_rmqueue(): colored_req received\n");
+#ifdef CONFIG_SCHED_DEBUG_TRACE 
+        if (colored_req)
+                TRACE("buffered_rmqueue(): colored_req received\n");
+#endif
         
         if (likely(order == 0) && !colored_req) {
                 struct per_cpu_pages *pcp;
@@ -3158,8 +3226,10 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
                 .migratetype = gfpflags_to_migratetype(gfp_mask),
         };
 
-if (gfp_mask&GFP_COLOR)
-        printk(KERN_INFO "__alloc_pages_nodemask(): called gfp %08x gfp_allowed_mask %08x mt = %d\n", gfp_mask, gfp_allowed_mask, ac.migratetype);
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+        if (gfp_mask&GFP_COLOR)
+                TRACE("__alloc_pages_nodemask(): called gfp %08x gfp_allowed_mask %08x mt = %d\n", gfp_mask, gfp_allowed_mask, ac.migratetype);
+#endif
 
         gfp_mask &= gfp_allowed_mask;
 
diff --git a/mm/slab_common.c b/mm/slab_common.c
index ff4d4c6f4129..bbd0ddc0b029 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -790,7 +790,7 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
 
         if (flags & GFP_COLOR) {
                 int cpu = raw_smp_processor_id();
-                printk(KERN_INFO "in kmalloc_slab index %d\n", index);
+                TRACE("in kmalloc_slab index %d\n", index);
                 return hc_kmalloc_caches[cpu][index];
         }
         else
@@ -848,11 +848,9 @@ void __init create_kmalloc_caches(unsigned long flags)
                         size_index[size_index_elem(i)] = 8;
         }
         for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
-                printk(KERN_INFO "KMALLOC i = %d\n", i);
                 if (!kmalloc_caches[i]) {
                         kmalloc_caches[i] = create_kmalloc_cache(NULL,
                                                         1 << i, flags);
-printk(KERN_INFO "KMALLOC-%d CACHE CREATED\n", 1<<i);
                 }
 
                 /*
@@ -862,12 +860,10 @@ printk(KERN_INFO "KMALLOC-%d CACHE CREATED\n", 1<<i);
                  */
                 if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6) {
                         kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);
-printk(KERN_INFO "KMALLOC-96 CACHE CREATED\n");
                 }
 
                 if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7) {
                         kmalloc_caches[2] = create_kmalloc_cache(NULL, 192, flags);
-printk(KERN_INFO "KMALLOC-192 CACHE CREATED\n");
                 }
         }
 
diff --git a/mm/slub.c b/mm/slub.c
index 1a2858905c54..0e8ce1f912fb 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1321,8 +1321,10 @@ static inline struct page *alloc_slab_page(struct kmem_cache *s,
                 return NULL;
 
         if (node == NUMA_NO_NODE) {
+#ifdef CONFIG_SCHED_DEBUG_TRACE         
                 if (flags&GFP_COLOR)
                         printk(KERN_INFO "alloc_pages calls with GFP_COLOR order = %d\n", order);
+#endif          
                 page = alloc_pages(flags, order);
         }
         else
@@ -1340,8 +1342,10 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
         struct kmem_cache_order_objects oo = s->oo;
         gfp_t alloc_gfp;
 
-if (flags&GFP_COLOR)
-        printk(KERN_INFO "gfp_allowed_mask = %08x\n", gfp_allowed_mask);
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+        if (flags&GFP_COLOR)
+                printk(KERN_INFO "gfp_allowed_mask = %08x\n", gfp_allowed_mask);
+#endif
         
         flags &= gfp_allowed_mask;
 
@@ -1355,9 +1359,12 @@ if (flags&GFP_COLOR)
          * so we fall-back to the minimum order allocation.
          */
         alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
-if (flags&__GFP_COLOR) {
-        printk(KERN_INFO "allocate_slab with GFP_COLOR alloc_gfp = %08x\n", alloc_gfp);
-}
+
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+        if (flags&__GFP_COLOR)
+                printk(KERN_INFO "allocate_slab with GFP_COLOR alloc_gfp = %08x\n", alloc_gfp);
+#endif
+
         page = alloc_slab_page(s, alloc_gfp, node, oo);
         if (unlikely(!page)) {
                 oo = s->min;
@@ -2232,9 +2239,10 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
 
         page = new_slab(s, flags, node);
 
-if (flags&GFP_COLOR) {
-        printk(KERN_INFO "new_slab_objects(): gets page %p\n", page);
-}
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+        if (flags&GFP_COLOR)
+                printk(KERN_INFO "new_slab_objects(): gets page %p\n", page);
+#endif
 
         if (page) {
                 c = raw_cpu_ptr(s->cpu_slab);
@@ -2321,8 +2329,11 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
         void *freelist;
         struct page *page;
         unsigned long flags;
-if (gfpflags&GFP_COLOR)
-        printk(KERN_INFO "__slab_alloc slow_path\n");
+
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+        if (gfpflags&GFP_COLOR)
+                printk(KERN_INFO "__slab_alloc slow_path\n");
+#endif
 
         local_irq_save(flags);
 #ifdef CONFIG_PREEMPT
@@ -2334,10 +2345,10 @@ if (gfpflags&GFP_COLOR)
         c = this_cpu_ptr(s->cpu_slab);
 #endif
 
-
-if (gfpflags&GFP_COLOR) {
-        printk(KERN_INFO "__slab_alloc : page %p, partial %p\n", c->page, c->partial);
-}
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+        if (gfpflags&GFP_COLOR)
+                printk(KERN_INFO "__slab_alloc : page %p, partial %p\n", c->page, c->partial);
+#endif
 
         page = c->page;
         if (!page)
@@ -3328,22 +3339,29 @@ void *__kmalloc(size_t size, gfp_t flags)
         struct kmem_cache *s;
         void *ret;
 
-if (flags & GFP_COLOR) {
-        printk(KERN_INFO "kmalloc size %d\n", size);
-}
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+        if (flags & GFP_COLOR)
+                printk(KERN_INFO "kmalloc size %d\n", size);
+#endif
+
         if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
                 return kmalloc_large(size, flags);
 
         s = kmalloc_slab(size, flags);
-if (flags & GFP_COLOR) {
-        printk(KERN_INFO "kmalloc_slab %p\n", s);
-}
+        
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+        if (flags & GFP_COLOR)
+                printk(KERN_INFO "kmalloc_slab %p\n", s);
+#endif
+
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
 
-if (flags & GFP_COLOR) {
-        printk(KERN_INFO "slab_alloc calls!!\n");
-}
+#ifdef CONFIG_SCHED_DEBUG_TRACE
+        if (flags & GFP_COLOR)
+                printk(KERN_INFO "slab_alloc calls!!\n");
+#endif
+
         ret = slab_alloc(s, flags, _RET_IP_);
 
         trace_kmalloc(_RET_IP_, ret, size, s->size, flags);