6 files changed, 106 insertions, 27 deletions

diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index 052a5b6cc4d0..78d32a7be257 100644
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -397,6 +397,9 @@ static inline bool pm_suspended_storage(void)
 extern int alloc_contig_range(unsigned long start, unsigned long end);
 extern void free_contig_range(unsigned long pfn, unsigned nr_pages);
 
+/* CMA stuff */
+extern void init_cma_reserved_pageblock(struct page *page);
+
 #endif
 
 #endif /* __LINUX_GFP_H */
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index dff711509661..8c1335f3c3a3 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -35,13 +35,37 @@
  */
 #define PAGE_ALLOC_COSTLY_ORDER 3
 
-#define MIGRATE_UNMOVABLE     0
-#define MIGRATE_RECLAIMABLE   1
-#define MIGRATE_MOVABLE       2
-#define MIGRATE_PCPTYPES      3 /* the number of types on the pcp lists */
-#define MIGRATE_RESERVE       3
-#define MIGRATE_ISOLATE       4 /* can't allocate from here */
-#define MIGRATE_TYPES         5
+enum {
+        MIGRATE_UNMOVABLE,
+        MIGRATE_RECLAIMABLE,
+        MIGRATE_MOVABLE,
+        MIGRATE_PCPTYPES,       /* the number of types on the pcp lists */
+        MIGRATE_RESERVE = MIGRATE_PCPTYPES,
+#ifdef CONFIG_CMA
+        /*
+         * MIGRATE_CMA migration type is designed to mimic the way
+         * ZONE_MOVABLE works.  Only movable pages can be allocated
+         * from MIGRATE_CMA pageblocks and page allocator never
+         * implicitly change migration type of MIGRATE_CMA pageblock.
+         *
+         * The way to use it is to change migratetype of a range of
+         * pageblocks to MIGRATE_CMA which can be done by
+         * __free_pageblock_cma() function.  What is important though
+         * is that a range of pageblocks must be aligned to
+         * MAX_ORDER_NR_PAGES should biggest page be bigger then
+         * a single pageblock.
+         */
+        MIGRATE_CMA,
+#endif
+        MIGRATE_ISOLATE,        /* can't allocate from here */
+        MIGRATE_TYPES
+};
+
+#ifdef CONFIG_CMA
+#  define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA)
+#else
+#  define is_migrate_cma(migratetype) false
+#endif
 
 #define for_each_migratetype_order(order, type) \
         for (order = 0; order < MAX_ORDER; order++) \
diff --git a/mm/Kconfig b/mm/Kconfig
index e338407f1225..39220026c797 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -198,7 +198,7 @@ config COMPACTION
 config MIGRATION
         bool "Page migration"
         def_bool y
-        depends on NUMA || ARCH_ENABLE_MEMORY_HOTREMOVE || COMPACTION
+        depends on NUMA || ARCH_ENABLE_MEMORY_HOTREMOVE || COMPACTION || CMA
         help
           Allows the migration of the physical location of pages of processes
           while the virtual addresses are not changed. This is useful in
diff --git a/mm/compaction.c b/mm/compaction.c
index 7a92e418a187..da7d35ea5103 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -45,6 +45,11 @@ static void map_pages(struct list_head *list)
         }
 }
 
+static inline bool migrate_async_suitable(int migratetype)
+{
+        return is_migrate_cma(migratetype) || migratetype == MIGRATE_MOVABLE;
+}
+
 /*
  * Isolate free pages onto a private freelist. Caller must hold zone->lock.
  * If @strict is true, will abort returning 0 on any invalid PFNs or non-free
@@ -299,7 +304,7 @@ isolate_migratepages_range(struct zone *zone, struct compact_control *cc,
                  */
                 pageblock_nr = low_pfn >> pageblock_order;
                 if (!cc->sync && last_pageblock_nr != pageblock_nr &&
-                                get_pageblock_migratetype(page) != MIGRATE_MOVABLE) {
+                    !migrate_async_suitable(get_pageblock_migratetype(page))) {
                         low_pfn += pageblock_nr_pages;
                         low_pfn = ALIGN(low_pfn, pageblock_nr_pages) - 1;
                         last_pageblock_nr = pageblock_nr;
@@ -367,8 +372,8 @@ static bool suitable_migration_target(struct page *page)
         if (PageBuddy(page) && page_order(page) >= pageblock_order)
                 return true;
 
-        /* If the block is MIGRATE_MOVABLE, allow migration */
-        if (migratetype == MIGRATE_MOVABLE)
+        /* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */
+        if (migrate_async_suitable(migratetype))
                 return true;
 
         /* Otherwise skip the block */
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index d6b580c660f5..0869eb1e9461 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -750,6 +750,24 @@ void __meminit __free_pages_bootmem(struct page *page, unsigned int order)
         __free_pages(page, order);
 }
 
+#ifdef CONFIG_CMA
+/* Free whole pageblock and set it's migration type to MIGRATE_CMA. */
+void __init init_cma_reserved_pageblock(struct page *page)
+{
+        unsigned i = pageblock_nr_pages;
+        struct page *p = page;
+
+        do {
+                __ClearPageReserved(p);
+                set_page_count(p, 0);
+        } while (++p, --i);
+
+        set_page_refcounted(page);
+        set_pageblock_migratetype(page, MIGRATE_CMA);
+        __free_pages(page, pageblock_order);
+        totalram_pages += pageblock_nr_pages;
+}
+#endif
 
 /*
  * The order of subdivision here is critical for the IO subsystem.
@@ -875,10 +893,15 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
  * This array describes the order lists are fallen back to when
  * the free lists for the desirable migrate type are depleted
  */
-static int fallbacks[MIGRATE_TYPES][3] = {
-        [MIGRATE_UNMOVABLE]   = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE,   MIGRATE_RESERVE },
-        [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE,   MIGRATE_MOVABLE,   MIGRATE_RESERVE },
-        [MIGRATE_MOVABLE]     = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
+static int fallbacks[MIGRATE_TYPES][4] = {
+        [MIGRATE_UNMOVABLE]   = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE,     MIGRATE_RESERVE },
+        [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE,   MIGRATE_MOVABLE,     MIGRATE_RESERVE },
+#ifdef CONFIG_CMA
+        [MIGRATE_MOVABLE]     = { MIGRATE_CMA,         MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
+        [MIGRATE_CMA]         = { MIGRATE_RESERVE }, /* Never used */
+#else
+        [MIGRATE_MOVABLE]     = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE,   MIGRATE_RESERVE },
+#endif
         [MIGRATE_RESERVE]     = { MIGRATE_RESERVE }, /* Never used */
         [MIGRATE_ISOLATE]     = { MIGRATE_RESERVE }, /* Never used */
 };
@@ -995,11 +1018,18 @@ __rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
                          * pages to the preferred allocation list. If falling
                          * back for a reclaimable kernel allocation, be more
                          * aggressive about taking ownership of free pages
+                         *
+                         * On the other hand, never change migration
+                         * type of MIGRATE_CMA pageblocks nor move CMA
+                         * pages on different free lists. We don't
+                         * want unmovable pages to be allocated from
+                         * MIGRATE_CMA areas.
                          */
-                        if (unlikely(current_order >= (pageblock_order >> 1)) ||
-                                        start_migratetype == MIGRATE_RECLAIMABLE ||
-                                        page_group_by_mobility_disabled) {
-                                unsigned long pages;
+                        if (!is_migrate_cma(migratetype) &&
+                            (unlikely(current_order >= pageblock_order / 2) ||
+                             start_migratetype == MIGRATE_RECLAIMABLE ||
+                             page_group_by_mobility_disabled)) {
+                                int pages;
                                 pages = move_freepages_block(zone, page,
                                                                 start_migratetype);
 
@@ -1017,11 +1047,14 @@ __rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
                         rmv_page_order(page);
 
                         /* Take ownership for orders >= pageblock_order */
-                        if (current_order >= pageblock_order)
+                        if (current_order >= pageblock_order &&
+                            !is_migrate_cma(migratetype))
                                 change_pageblock_range(page, current_order,
                                                         start_migratetype);
 
-                        expand(zone, page, order, current_order, area, migratetype);
+                        expand(zone, page, order, current_order, area,
+                               is_migrate_cma(migratetype)
+                             ? migratetype : start_migratetype);
 
                         trace_mm_page_alloc_extfrag(page, order, current_order,
                                 start_migratetype, migratetype);
@@ -1072,7 +1105,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
                         unsigned long count, struct list_head *list,
                         int migratetype, int cold)
 {
-        int i;
+        int mt = migratetype, i;
 
         spin_lock(&zone->lock);
         for (i = 0; i < count; ++i) {
@@ -1093,7 +1126,12 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
                         list_add(&page->lru, list);
                 else
                         list_add_tail(&page->lru, list);
-                set_page_private(page, migratetype);
+                if (IS_ENABLED(CONFIG_CMA)) {
+                        mt = get_pageblock_migratetype(page);
+                        if (!is_migrate_cma(mt) && mt != MIGRATE_ISOLATE)
+                                mt = migratetype;
+                }
+                set_page_private(page, mt);
                 list = &page->lru;
         }
         __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
@@ -1373,8 +1411,12 @@ int split_free_page(struct page *page)
 
         if (order >= pageblock_order - 1) {
                 struct page *endpage = page + (1 << order) - 1;
-                for (; page < endpage; page += pageblock_nr_pages)
-                        set_pageblock_migratetype(page, MIGRATE_MOVABLE);
+                for (; page < endpage; page += pageblock_nr_pages) {
+                        int mt = get_pageblock_migratetype(page);
+                        if (mt != MIGRATE_ISOLATE && !is_migrate_cma(mt))
+                                set_pageblock_migratetype(page,
+                                                          MIGRATE_MOVABLE);
+                }
         }
 
         return 1 << order;
@@ -5414,14 +5456,16 @@ static int
 __count_immobile_pages(struct zone *zone, struct page *page, int count)
 {
         unsigned long pfn, iter, found;
+        int mt;
+
         /*
          * For avoiding noise data, lru_add_drain_all() should be called
          * If ZONE_MOVABLE, the zone never contains immobile pages
          */
         if (zone_idx(zone) == ZONE_MOVABLE)
                 return true;
-
-        if (get_pageblock_migratetype(page) == MIGRATE_MOVABLE)
+        mt = get_pageblock_migratetype(page);
+        if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt))
                 return true;
 
         pfn = page_to_pfn(page);
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 7db1b9bab492..0dad31dc1618 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -613,6 +613,9 @@ static char * const migratetype_names[MIGRATE_TYPES] = {
         "Reclaimable",
         "Movable",
         "Reserve",
+#ifdef CONFIG_CMA
+        "CMA",
+#endif
         "Isolate",
 };