3 files changed, 78 insertions, 29 deletions

diff --git a/mm/internal.h b/mm/internal.h
index 829304090b90..a4f90ba7068e 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -108,6 +108,31 @@ extern pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address);
 /*
  * in mm/page_alloc.c
  */
+
+/*
+ * Locate the struct page for both the matching buddy in our
+ * pair (buddy1) and the combined O(n+1) page they form (page).
+ *
+ * 1) Any buddy B1 will have an order O twin B2 which satisfies
+ * the following equation:
+ *     B2 = B1 ^ (1 << O)
+ * For example, if the starting buddy (buddy2) is #8 its order
+ * 1 buddy is #10:
+ *     B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
+ *
+ * 2) Any buddy B will have an order O+1 parent P which
+ * satisfies the following equation:
+ *     P = B & ~(1 << O)
+ *
+ * Assumption: *_mem_map is contiguous at least up to MAX_ORDER
+ */
+static inline unsigned long
+__find_buddy_index(unsigned long page_idx, unsigned int order)
+{
+        return page_idx ^ (1 << order);
+}
+
+extern int __isolate_free_page(struct page *page, unsigned int order);
 extern void __free_pages_bootmem(struct page *page, unsigned int order);
 extern void prep_compound_page(struct page *page, unsigned long order);
 #ifdef CONFIG_MEMORY_FAILURE
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 9f689f16b5aa..fd11b913779e 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -467,29 +467,6 @@ static inline void rmv_page_order(struct page *page)
 }
 
 /*
- * Locate the struct page for both the matching buddy in our
- * pair (buddy1) and the combined O(n+1) page they form (page).
- *
- * 1) Any buddy B1 will have an order O twin B2 which satisfies
- * the following equation:
- *     B2 = B1 ^ (1 << O)
- * For example, if the starting buddy (buddy2) is #8 its order
- * 1 buddy is #10:
- *     B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
- *
- * 2) Any buddy B will have an order O+1 parent P which
- * satisfies the following equation:
- *     P = B & ~(1 << O)
- *
- * Assumption: *_mem_map is contiguous at least up to MAX_ORDER
- */
-static inline unsigned long
-__find_buddy_index(unsigned long page_idx, unsigned int order)
-{
-        return page_idx ^ (1 << order);
-}
-
-/*
  * This function checks whether a page is free && is the buddy
  * we can do coalesce a page and its buddy if
  * (a) the buddy is not in a hole &&
@@ -569,6 +546,7 @@ static inline void __free_one_page(struct page *page,
         unsigned long combined_idx;
         unsigned long uninitialized_var(buddy_idx);
         struct page *buddy;
+        int max_order = MAX_ORDER;
 
         VM_BUG_ON(!zone_is_initialized(zone));
 
@@ -577,15 +555,24 @@ static inline void __free_one_page(struct page *page,
                         return;
 
         VM_BUG_ON(migratetype == -1);
-        if (!is_migrate_isolate(migratetype))
+        if (is_migrate_isolate(migratetype)) {
+                /*
+                 * We restrict max order of merging to prevent merge
+                 * between freepages on isolate pageblock and normal
+                 * pageblock. Without this, pageblock isolation
+                 * could cause incorrect freepage accounting.
+                 */
+                max_order = min(MAX_ORDER, pageblock_order + 1);
+        } else {
                 __mod_zone_freepage_state(zone, 1 << order, migratetype);
+        }
 
-        page_idx = pfn & ((1 << MAX_ORDER) - 1);
+        page_idx = pfn & ((1 << max_order) - 1);
 
         VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
         VM_BUG_ON_PAGE(bad_range(zone, page), page);
 
-        while (order < MAX_ORDER-1) {
+        while (order < max_order - 1) {
                 buddy_idx = __find_buddy_index(page_idx, order);
                 buddy = page + (buddy_idx - page_idx);
                 if (!page_is_buddy(page, buddy, order))
@@ -1486,7 +1473,7 @@ void split_page(struct page *page, unsigned int order)
 }
 EXPORT_SYMBOL_GPL(split_page);
 
-static int __isolate_free_page(struct page *page, unsigned int order)
+int __isolate_free_page(struct page *page, unsigned int order)
 {
         unsigned long watermark;
         struct zone *zone;
diff --git a/mm/page_isolation.c b/mm/page_isolation.c
index 1fa4a4d72ecc..c8778f7e208e 100644
--- a/mm/page_isolation.c
+++ b/mm/page_isolation.c
@@ -76,17 +76,54 @@ void unset_migratetype_isolate(struct page *page, unsigned migratetype)
 {
         struct zone *zone;
         unsigned long flags, nr_pages;
+        struct page *isolated_page = NULL;
+        unsigned int order;
+        unsigned long page_idx, buddy_idx;
+        struct page *buddy;
 
         zone = page_zone(page);
         spin_lock_irqsave(&zone->lock, flags);
         if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
                 goto out;
-        nr_pages = move_freepages_block(zone, page, migratetype);
-        __mod_zone_freepage_state(zone, nr_pages, migratetype);
+
+        /*
+         * Because freepage with more than pageblock_order on isolated
+         * pageblock is restricted to merge due to freepage counting problem,
+         * it is possible that there is free buddy page.
+         * move_freepages_block() doesn't care of merge so we need other
+         * approach in order to merge them. Isolation and free will make
+         * these pages to be merged.
+         */
+        if (PageBuddy(page)) {
+                order = page_order(page);
+                if (order >= pageblock_order) {
+                        page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
+                        buddy_idx = __find_buddy_index(page_idx, order);
+                        buddy = page + (buddy_idx - page_idx);
+
+                        if (!is_migrate_isolate_page(buddy)) {
+                                __isolate_free_page(page, order);
+                                set_page_refcounted(page);
+                                isolated_page = page;
+                        }
+                }
+        }
+
+        /*
+         * If we isolate freepage with more than pageblock_order, there
+         * should be no freepage in the range, so we could avoid costly
+         * pageblock scanning for freepage moving.
+         */
+        if (!isolated_page) {
+                nr_pages = move_freepages_block(zone, page, migratetype);
+                __mod_zone_freepage_state(zone, nr_pages, migratetype);
+        }
         set_pageblock_migratetype(page, migratetype);
         zone->nr_isolate_pageblock--;
 out:
         spin_unlock_irqrestore(&zone->lock, flags);
+        if (isolated_page)
+                __free_pages(isolated_page, order);
 }
 
 static inline struct page *