aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNamhoon Kim <namhoonk@cs.unc.edu>2017-09-12 15:57:53 -0400
committerNamhoon Kim <namhoonk@cs.unc.edu>2017-09-12 15:57:53 -0400
commit5cc515b26848d31deda903fc19e5534723cf4a35 (patch)
treef961ff143600cebee30a2f8a38795b3b31bfc4a4
parent84e36eb17cd9e5bb4bfb6cc4e29f55542fa69fde (diff)
ORDER 2 per-partition alloc
Diffstat
-rw-r--r--litmus/page_dev.c89
-rw-r--r--mm/page_alloc.c313
-rw-r--r--mm/slab_common.c37
-rw-r--r--mm/slub.c36
-rw-r--r--mm/vmstat.c5
5 files changed, 347 insertions, 133 deletions
diff --git a/litmus/page_dev.c b/litmus/page_dev.c
index 2894e93213d3..2fd829b05a0a 100644
--- a/litmus/page_dev.c
+++ b/litmus/page_dev.c
@@ -98,6 +98,81 @@ unsigned int llc_partition_min = 0;
98unsigned int dram_partition_max = 0x000000ff; 98unsigned int dram_partition_max = 0x000000ff;
99unsigned int dram_partition_min = 0; 99unsigned int dram_partition_min = 0;
100 100
101/* slabtest module */
102int buf_size = 0;
103int buf_num = 1;
104
105int slabtest_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos)
106{
107 int ret = 0, i;
108 int** testbuffer;
109 mutex_lock(&dev_mutex);
110 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
111
112 if (ret)
113 goto out;
114
115 if (write) {
116 int idx;
117 int n_data = buf_size/sizeof(int);
118
119 testbuffer = kmalloc(sizeof(int*)*buf_num, GFP_KERNEL|GFP_COLOR);
120
121 for (idx=0; idx<buf_num; idx++)
122 {
123 printk(KERN_INFO "kmalloc size %d, n_data %d\n", buf_size, n_data);
124 testbuffer[idx] = kmalloc(buf_size, GFP_KERNEL|GFP_COLOR);
125
126 if (!testbuffer[idx]) {
127 printk(KERN_ERR "kmalloc failed size = %d\n", buf_size);
128 goto out;
129 }
130 }
131
132
133 /* do test */
134 for (idx=0; idx<buf_num; idx++)
135 {
136 int t = 0;
137 printk(KERN_INFO "kmalloc size = %d n_data = %d\n", buf_size, n_data);
138 printk(KERN_INFO "write data to buffer\n");
139 for (i = 0; i < n_data; i++) {
140 testbuffer[idx][i] = i%27;
141 }
142 printk(KERN_INFO "read data from buffer\n");
143 for (i = 0; i < n_data; i++) {
144 t += testbuffer[idx][i];
145 //printk(KERN_INFO "[%d] = %d\n", i, testbuffer[idx][i]);
146 }
147 }
148
149 for (idx=0; idx<buf_num; idx++)
150 kfree(testbuffer[idx]);
151
152 kfree(testbuffer);
153 }
154out:
155 mutex_unlock(&dev_mutex);
156 return ret;
157}
158
159int num_buffer_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos)
160{
161 int ret = 0;
162 mutex_lock(&dev_mutex);
163 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
164
165 if (ret)
166 goto out;
167
168 if (write) {
169 printk(KERN_INFO "buf_num = %d\n", buf_num);
170 }
171out:
172 mutex_unlock(&dev_mutex);
173 return ret;
174}
175
101static struct ctl_table partition_table[] = 176static struct ctl_table partition_table[] =
102{ 177{
103 178
@@ -227,6 +302,20 @@ static struct ctl_table partition_table[] =
227 .extra1 = &dram_partition_min, 302 .extra1 = &dram_partition_min,
228 .extra2 = &dram_partition_max, 303 .extra2 = &dram_partition_max,
229 }, 304 },
305 {
306 .procname = "slabtest",
307 .mode = 0666,
308 .proc_handler = slabtest_handler,
309 .data = &buf_size,
310 .maxlen = sizeof(buf_size),
311 },
312 {
313 .procname = "num_buffer",
314 .mode = 0666,
315 .proc_handler = num_buffer_handler,
316 .data = &buf_num,
317 .maxlen = sizeof(buf_num),
318 },
230 { } 319 { }
231}; 320};
232 321
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index e240fcd3039d..2d49e44c7cb8 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -607,27 +607,19 @@ static inline void __free_one_page(struct page *page,
607 607
608 parti_no = bank_to_partition(page_bank(page)); 608 parti_no = bank_to_partition(page_bank(page));
609 BUG_ON(parti_no < 0 || parti_no > NR_CPUS); 609 BUG_ON(parti_no < 0 || parti_no > NR_CPUS);
610 if (parti_no < 0 || parti_no > NR_CPUS)
611 printk(KERN_ALERT "PART_NO %d\n", parti_no);
612 610
613 if (parti_no < NR_CPUS) 611 //if (parti_no < NR_CPUS)
614 printk(KERN_ALERT "pfn = %lx, part_no = %d order = %d\n", pfn, parti_no, order); 612 //printk(KERN_ALERT "pfn = %lx, part_no = %d order = %d\n", pfn, parti_no, order);
615 613
616 if (parti_no == NR_CPUS) { 614 if (parti_no < NR_CPUS) {
617 max_order = MAX_ORDER; 615 max_order = MAX_PARTITIONED_ORDER;
618 616
619 VM_BUG_ON(!zone_is_initialized(zone)); 617 VM_BUG_ON(!zone_is_initialized(zone));
620 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 618 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
621 619
622 VM_BUG_ON(migratetype == -1); 620 VM_BUG_ON(migratetype == -1);
623 if (is_migrate_isolate(migratetype)) { 621 if (is_migrate_isolate(migratetype)) {
624 /* 622 max_order = min(MAX_PARTITIONED_ORDER, pageblock_order + 1);
625 * We restrict max order of merging to prevent merge
626 * between freepages on isolate pageblock and normal
627 * pageblock. Without this, pageblock isolation
628 * could cause incorrect freepage accounting.
629 */
630 max_order = min(MAX_ORDER, pageblock_order + 1);
631 } else { 623 } else {
632 __mod_zone_freepage_state(zone, 1 << order, migratetype, parti_no); 624 __mod_zone_freepage_state(zone, 1 << order, migratetype, parti_no);
633 } 625 }
@@ -642,15 +634,12 @@ static inline void __free_one_page(struct page *page,
642 buddy = page + (buddy_idx - page_idx); 634 buddy = page + (buddy_idx - page_idx);
643 if (!page_is_buddy(page, buddy, order)) 635 if (!page_is_buddy(page, buddy, order))
644 break; 636 break;
645 /* 637
646 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
647 * merge with it and move up one order.
648 */
649 if (page_is_guard(buddy)) { 638 if (page_is_guard(buddy)) {
650 clear_page_guard(zone, buddy, order, migratetype); 639 clear_page_guard(zone, buddy, order, migratetype);
651 } else { 640 } else {
652 list_del(&buddy->lru); 641 list_del(&buddy->lru);
653 zone->free_area[order].nr_free--; 642 zone->free_area_d[parti_no][order].nr_free--;
654 rmv_page_order(buddy); 643 rmv_page_order(buddy);
655 } 644 }
656 combined_idx = buddy_idx & page_idx; 645 combined_idx = buddy_idx & page_idx;
@@ -660,15 +649,7 @@ static inline void __free_one_page(struct page *page,
660 } 649 }
661 set_page_order(page, order); 650 set_page_order(page, order);
662 651
663 /* 652 if ((order < MAX_PARTITIONED_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
664 * If this is not the largest possible page, check if the buddy
665 * of the next-highest order is free. If it is, it's possible
666 * that pages are being freed that will coalesce soon. In case,
667 * that is happening, add the free page to the tail of the list
668 * so it's less likely to be used soon and more likely to be merged
669 * as a higher order page
670 */
671 if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
672 struct page *higher_page, *higher_buddy; 653 struct page *higher_page, *higher_buddy;
673 combined_idx = buddy_idx & page_idx; 654 combined_idx = buddy_idx & page_idx;
674 higher_page = page + (combined_idx - page_idx); 655 higher_page = page + (combined_idx - page_idx);
@@ -676,23 +657,43 @@ static inline void __free_one_page(struct page *page,
676 higher_buddy = higher_page + (buddy_idx - combined_idx); 657 higher_buddy = higher_page + (buddy_idx - combined_idx);
677 if (page_is_buddy(higher_page, higher_buddy, order + 1)) { 658 if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
678 list_add_tail(&page->lru, 659 list_add_tail(&page->lru,
679 &zone->free_area[order].free_list[migratetype]); 660 &zone->free_area_d[parti_no][order].free_list[migratetype]);
680 goto out; 661 zone->free_area_d[parti_no][order].nr_free++;
662 return;
681 } 663 }
682 } 664 }
683 665
684 list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); 666 if (order >= MAX_PARTITIONED_ORDER) {
685out: 667 int n_idx = 0;
686 zone->free_area[order].nr_free++; 668 struct page *lower_page;
687 } else { 669 for (n_idx = 0 ; n_idx < (1 << (order - MAX_PARTITIONED_ORDER + 1)); n_idx++) {
688 max_order = MAX_PARTITIONED_ORDER; 670 lower_page = page + (n_idx << (MAX_PARTITIONED_ORDER - 1));
671 if (lower_page->flags & PAGE_FLAGS_CHECK_AT_PREP)
672 lower_page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
673 set_page_order(lower_page, MAX_PARTITIONED_ORDER-1);
674 list_add(&lower_page->lru, &zone->free_area_d[parti_no][MAX_PARTITIONED_ORDER-1].free_list[migratetype]);
675 zone->free_area_d[parti_no][MAX_PARTITIONED_ORDER-1].nr_free++;
676 }
677 } else {
678 list_add(&page->lru, &zone->free_area_d[parti_no][order].free_list[migratetype]);
679 zone->free_area_d[parti_no][order].nr_free++;
680 }
681 }
682 else {
683 max_order = MAX_ORDER;
689 684
690 VM_BUG_ON(!zone_is_initialized(zone)); 685 VM_BUG_ON(!zone_is_initialized(zone));
691 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 686 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
692 687
693 VM_BUG_ON(migratetype == -1); 688 VM_BUG_ON(migratetype == -1);
694 if (is_migrate_isolate(migratetype)) { 689 if (is_migrate_isolate(migratetype)) {
695 max_order = min(MAX_PARTITIONED_ORDER, pageblock_order + 1); 690 /*
691 * We restrict max order of merging to prevent merge
692 * between freepages on isolate pageblock and normal
693 * pageblock. Without this, pageblock isolation
694 * could cause incorrect freepage accounting.
695 */
696 max_order = min(MAX_ORDER, pageblock_order + 1);
696 } else { 697 } else {
697 __mod_zone_freepage_state(zone, 1 << order, migratetype, parti_no); 698 __mod_zone_freepage_state(zone, 1 << order, migratetype, parti_no);
698 } 699 }
@@ -707,12 +708,15 @@ out:
707 buddy = page + (buddy_idx - page_idx); 708 buddy = page + (buddy_idx - page_idx);
708 if (!page_is_buddy(page, buddy, order)) 709 if (!page_is_buddy(page, buddy, order))
709 break; 710 break;
710 711 /*
712 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
713 * merge with it and move up one order.
714 */
711 if (page_is_guard(buddy)) { 715 if (page_is_guard(buddy)) {
712 clear_page_guard(zone, buddy, order, migratetype); 716 clear_page_guard(zone, buddy, order, migratetype);
713 } else { 717 } else {
714 list_del(&buddy->lru); 718 list_del(&buddy->lru);
715 zone->free_area_d[parti_no][order].nr_free--; 719 zone->free_area[order].nr_free--;
716 rmv_page_order(buddy); 720 rmv_page_order(buddy);
717 } 721 }
718 combined_idx = buddy_idx & page_idx; 722 combined_idx = buddy_idx & page_idx;
@@ -722,7 +726,15 @@ out:
722 } 726 }
723 set_page_order(page, order); 727 set_page_order(page, order);
724 728
725 if ((order < MAX_PARTITIONED_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) { 729 /*
730 * If this is not the largest possible page, check if the buddy
731 * of the next-highest order is free. If it is, it's possible
732 * that pages are being freed that will coalesce soon. In case,
733 * that is happening, add the free page to the tail of the list
734 * so it's less likely to be used soon and more likely to be merged
735 * as a higher order page
736 */
737 if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
726 struct page *higher_page, *higher_buddy; 738 struct page *higher_page, *higher_buddy;
727 combined_idx = buddy_idx & page_idx; 739 combined_idx = buddy_idx & page_idx;
728 higher_page = page + (combined_idx - page_idx); 740 higher_page = page + (combined_idx - page_idx);
@@ -730,25 +742,14 @@ out:
730 higher_buddy = higher_page + (buddy_idx - combined_idx); 742 higher_buddy = higher_page + (buddy_idx - combined_idx);
731 if (page_is_buddy(higher_page, higher_buddy, order + 1)) { 743 if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
732 list_add_tail(&page->lru, 744 list_add_tail(&page->lru,
733 &zone->free_area_d[parti_no][order].free_list[migratetype]); 745 &zone->free_area[order].free_list[migratetype]);
734 zone->free_area_d[parti_no][order].nr_free++; 746 goto out;
735 return;
736 } 747 }
737 } 748 }
738 749
739 if (order >= MAX_PARTITIONED_ORDER) { 750 list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
740 int n_idx = 0; 751out:
741 struct page *lower_page; 752 zone->free_area[order].nr_free++;
742 for (n_idx = 0 ; n_idx < (1 << (order - MAX_PARTITIONED_ORDER + 1)); n_idx++) {
743 lower_page = page + (n_idx << (MAX_PARTITIONED_ORDER - 1));
744 set_page_order(lower_page, MAX_PARTITIONED_ORDER-1);
745 list_add(&lower_page->lru, &zone->free_area_d[parti_no][MAX_PARTITIONED_ORDER-1].free_list[migratetype]);
746 zone->free_area_d[parti_no][MAX_PARTITIONED_ORDER-1].nr_free++;
747 }
748 } else {
749 list_add(&page->lru, &zone->free_area_d[parti_no][order].free_list[migratetype]);
750 zone->free_area_d[parti_no][order].nr_free++;
751 }
752 } 753 }
753} 754}
754 755
@@ -1190,12 +1191,24 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
1190 /* The max. order of color_req is <= 2 */ 1191 /* The max. order of color_req is <= 2 */
1191 if (color_req == 1) { 1192 if (color_req == 1) {
1192 int found = 0; 1193 int found = 0;
1193 printk(KERN_INFO "COLOR PAGE requested on CPU%d with order = %d\n", cpu, order); 1194 unsigned long s_pfn = zone->zone_start_pfn;
1195 unsigned long e_pfn = zone_end_pfn(zone);
1196 printk(KERN_INFO "COLOR PAGE requested on CPU%d with order = %d migratetype = %d\n", cpu, order, migratetype);
1194 /* Find a page of the appropriate size in the preferred list */ 1197 /* Find a page of the appropriate size in the preferred list */
1195 for (current_order = order; current_order < MAX_PARTITIONED_ORDER; ++current_order) { 1198 for (current_order = order; current_order < MAX_PARTITIONED_ORDER; ++current_order) {
1196 area = &(zone->free_area_d[cpu][current_order]); 1199 area = &(zone->free_area_d[cpu][current_order]);
1197 if (list_empty(&area->free_list[migratetype])) 1200 if (list_empty(&area->free_list[migratetype])) {
1201 printk(KERN_INFO "order %d list empty\n", current_order);
1198 continue; 1202 continue;
1203 }
1204
1205
1206 {
1207 list_for_each_entry(page, &area->free_list[migratetype], lru) {
1208 printk(KERN_INFO "__rmqueue_smallest list entry %p color %d\n", page, page_color(page));
1209 }
1210 }
1211 printk(KERN_INFO "__rmqueue_smallest LAST list entry %p\n", page);
1199 1212
1200 page = list_entry(area->free_list[migratetype].next, 1213 page = list_entry(area->free_list[migratetype].next,
1201 struct page, lru); 1214 struct page, lru);
@@ -1204,14 +1217,17 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
1204 1217
1205 while(!found) { 1218 while(!found) {
1206 page = list_next_entry(page, lru); 1219 page = list_next_entry(page, lru);
1207 if (is_in_llc_partition(page, cpu)) 1220 if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn))
1208 found = 1; 1221 found = 1;
1209 } 1222 }
1223 BUG_ON(found == 0);
1224
1210 list_del(&page->lru); 1225 list_del(&page->lru);
1211 rmv_page_order(page); 1226 rmv_page_order(page);
1212 area->nr_free--; 1227 area->nr_free--;
1213 expand(zone, page, order, current_order, area, migratetype); 1228 expand(zone, page, order, current_order, area, migratetype);
1214 set_freepage_migratetype(page, migratetype); 1229 set_freepage_migratetype(page, migratetype);
1230 printk(KERN_INFO "COLOR %d page return %p\n", page_color(page), page);
1215 return page; 1231 return page;
1216 } 1232 }
1217 } else { 1233 } else {
@@ -1445,7 +1461,7 @@ int find_suitable_fallback(struct free_area *area, unsigned int order,
1445 1461
1446/* Remove an element from the buddy allocator from the fallback list */ 1462/* Remove an element from the buddy allocator from the fallback list */
1447static inline struct page * 1463static inline struct page *
1448__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype) 1464__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype, int color_req)
1449{ 1465{
1450 struct free_area *area; 1466 struct free_area *area;
1451 unsigned int current_order; 1467 unsigned int current_order;
@@ -1453,44 +1469,104 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
1453 int fallback_mt; 1469 int fallback_mt;
1454 bool can_steal; 1470 bool can_steal;
1455 1471
1456 /* Find the largest possible block of pages in the other list */ 1472 if (color_req == 1) {
1457 for (current_order = MAX_ORDER-1; 1473 int cpu = raw_smp_processor_id();
1458 current_order >= order && current_order <= MAX_ORDER-1; 1474 int found = 0;
1459 --current_order) { 1475 /* Find the largest possible block of pages in the other list */
1460 area = &(zone->free_area[current_order]); 1476 for (current_order = MAX_PARTITIONED_ORDER-1;
1461 fallback_mt = find_suitable_fallback(area, current_order, 1477 current_order >= order && current_order <= MAX_PARTITIONED_ORDER-1;
1462 start_migratetype, false, &can_steal); 1478 --current_order) {
1463 if (fallback_mt == -1) 1479 area = &(zone->free_area_d[cpu][current_order]);
1464 continue; 1480 fallback_mt = find_suitable_fallback(area, current_order,
1481 start_migratetype, false, &can_steal);
1482 if (fallback_mt == -1)
1483 continue;
1484
1485/*
1486 {
1487 list_for_each_entry(page, &area->free_list[fallback_mt], lru) {
1488 printk(KERN_INFO "__rmqueue_fallback list entry %p color %d\n", page, page_color(page));
1489 }
1490 }
1491*/
1492
1493 page = list_entry(area->free_list[fallback_mt].next,
1494 struct page, lru);
1495 if (is_in_llc_partition(page, cpu))
1496 found = 1;
1497
1498 while(!found) {
1499 page = list_next_entry(page, lru);
1500 if (is_in_llc_partition(page, cpu))
1501 found = 1;
1502 }
1503
1504 if (can_steal)
1505 steal_suitable_fallback(zone, page, start_migratetype);
1465 1506
1466 page = list_entry(area->free_list[fallback_mt].next, 1507 /* Remove the page from the freelists */
1467 struct page, lru); 1508 area->nr_free--;
1468 if (can_steal) 1509 list_del(&page->lru);
1469 steal_suitable_fallback(zone, page, start_migratetype); 1510 rmv_page_order(page);
1470 1511
1471 /* Remove the page from the freelists */ 1512 expand(zone, page, order, current_order, area,
1472 area->nr_free--; 1513 start_migratetype);
1473 list_del(&page->lru); 1514 /*
1474 rmv_page_order(page); 1515 * The freepage_migratetype may differ from pageblock's
1516 * migratetype depending on the decisions in
1517 * try_to_steal_freepages(). This is OK as long as it
1518 * does not differ for MIGRATE_CMA pageblocks. For CMA
1519 * we need to make sure unallocated pages flushed from
1520 * pcp lists are returned to the correct freelist.
1521 */
1522 set_freepage_migratetype(page, start_migratetype);
1475 1523
1476 expand(zone, page, order, current_order, area, 1524 trace_mm_page_alloc_extfrag(page, order, current_order,
1477 start_migratetype); 1525 start_migratetype, fallback_mt);
1478 /* 1526
1479 * The freepage_migratetype may differ from pageblock's 1527 printk(KERN_INFO "__rmqueue_fallback(): CPU%d COLOR %d page return %p\n", cpu, page_color(page), page);
1480 * migratetype depending on the decisions in 1528 return page;
1481 * try_to_steal_freepages(). This is OK as long as it 1529 }
1482 * does not differ for MIGRATE_CMA pageblocks. For CMA 1530 } else {
1483 * we need to make sure unallocated pages flushed from 1531 /* Find the largest possible block of pages in the other list */
1484 * pcp lists are returned to the correct freelist. 1532 for (current_order = MAX_ORDER-1;
1485 */ 1533 current_order >= order && current_order <= MAX_ORDER-1;
1486 set_freepage_migratetype(page, start_migratetype); 1534 --current_order) {
1535 area = &(zone->free_area[current_order]);
1536 fallback_mt = find_suitable_fallback(area, current_order,
1537 start_migratetype, false, &can_steal);
1538 if (fallback_mt == -1)
1539 continue;
1487 1540
1488 trace_mm_page_alloc_extfrag(page, order, current_order, 1541 page = list_entry(area->free_list[fallback_mt].next,
1489 start_migratetype, fallback_mt); 1542 struct page, lru);
1543 if (can_steal)
1544 steal_suitable_fallback(zone, page, start_migratetype);
1490 1545
1491 return page; 1546 /* Remove the page from the freelists */
1492 } 1547 area->nr_free--;
1548 list_del(&page->lru);
1549 rmv_page_order(page);
1550
1551 expand(zone, page, order, current_order, area,
1552 start_migratetype);
1553 /*
1554 * The freepage_migratetype may differ from pageblock's
1555 * migratetype depending on the decisions in
1556 * try_to_steal_freepages(). This is OK as long as it
1557 * does not differ for MIGRATE_CMA pageblocks. For CMA
1558 * we need to make sure unallocated pages flushed from
1559 * pcp lists are returned to the correct freelist.
1560 */
1561 set_freepage_migratetype(page, start_migratetype);
1562
1563 trace_mm_page_alloc_extfrag(page, order, current_order,
1564 start_migratetype, fallback_mt);
1493 1565
1566 return page;
1567 }
1568 }
1569
1494 return NULL; 1570 return NULL;
1495} 1571}
1496 1572
@@ -1510,8 +1586,11 @@ retry_reserve:
1510 if (migratetype == MIGRATE_MOVABLE) 1586 if (migratetype == MIGRATE_MOVABLE)
1511 page = __rmqueue_cma_fallback(zone, order); 1587 page = __rmqueue_cma_fallback(zone, order);
1512 1588
1513 if (!page) 1589 if (!page) {
1514 page = __rmqueue_fallback(zone, order, migratetype); 1590 page = __rmqueue_fallback(zone, order, migratetype, color_req);
1591 if (color_req)
1592 printk(KERN_INFO "page received from __rmqueue_fallback()");
1593 }
1515 1594
1516 /* 1595 /*
1517 * Use MIGRATE_RESERVE rather than fail an allocation. goto 1596 * Use MIGRATE_RESERVE rather than fail an allocation. goto
@@ -1541,7 +1620,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
1541 1620
1542 spin_lock(&zone->lock); 1621 spin_lock(&zone->lock);
1543 for (i = 0; i < count; ++i) { 1622 for (i = 0; i < count; ++i) {
1544 struct page *page = __rmqueue(zone, order, migratetype, 1); 1623 struct page *page = __rmqueue(zone, order, migratetype, 0);
1545 if (unlikely(page == NULL)) 1624 if (unlikely(page == NULL))
1546 break; 1625 break;
1547 1626
@@ -1563,7 +1642,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
1563 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 1642 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
1564 -(1 << order)); 1643 -(1 << order));
1565 } 1644 }
1566 __mod_zone_page_state(zone, NR_FREE_HC_PAGES, -(i << order)); 1645 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
1567 spin_unlock(&zone->lock); 1646 spin_unlock(&zone->lock);
1568 return i; 1647 return i;
1569} 1648}
@@ -1751,8 +1830,8 @@ void free_hot_cold_page(struct page *page, bool cold)
1751 unsigned long flags; 1830 unsigned long flags;
1752 unsigned long pfn = page_to_pfn(page); 1831 unsigned long pfn = page_to_pfn(page);
1753 int migratetype; 1832 int migratetype;
1754 unsigned int cpu; 1833 //unsigned int part_no;
1755 int is_local, is_in_pcp; 1834 //int is_local, is_hc_page;
1756 1835
1757 if (!free_pages_prepare(page, 0)) 1836 if (!free_pages_prepare(page, 0))
1758 return; 1837 return;
@@ -1765,7 +1844,8 @@ void free_hot_cold_page(struct page *page, bool cold)
1765 if (bank_to_partition(page_bank(page)) == NR_CPUS) 1844 if (bank_to_partition(page_bank(page)) == NR_CPUS)
1766 __count_vm_event(PGFREE); 1845 __count_vm_event(PGFREE);
1767 else if (bank_to_partition(page_bank(page)) < NR_CPUS) 1846 else if (bank_to_partition(page_bank(page)) < NR_CPUS)
1768 __count_vm_event(PGFREE_HC); 1847 //__count_vm_event(PGFREE_HC);
1848 BUG();
1769 1849
1770 /* 1850 /*
1771 * We only track unmovable, reclaimable and movable on pcp lists. 1851 * We only track unmovable, reclaimable and movable on pcp lists.
@@ -1782,18 +1862,18 @@ void free_hot_cold_page(struct page *page, bool cold)
1782 migratetype = MIGRATE_MOVABLE; 1862 migratetype = MIGRATE_MOVABLE;
1783 } 1863 }
1784 1864
1785 cpu = bank_to_partition(page_bank(page)); 1865 //part_no = bank_to_partition(page_bank(page));
1786 BUG_ON(cpu<0); 1866 //BUG_ON(part_no<0);
1787 1867
1788 if (cpu == smp_processor_id()) 1868 //if (part_no == smp_processor_id())
1789 is_local = 1; 1869 // is_local = 1;
1790 else 1870 //else
1791 is_local = 0; 1871 // is_local = 0;
1792 1872
1793 is_in_pcp = is_in_llc_partition(page, smp_processor_id()); 1873 //is_hc_page = is_in_llc_partition(page, smp_processor_id());
1794 if (cpu != NR_CPUS) 1874 //if (part_no != NR_CPUS)
1795 printk(KERN_ALERT "CPU%d Free order-0 page bank = %d, color = %d, is_local %d is_in_pcp %d\n", smp_processor_id(), page_bank(page), page_color(page), is_local, is_in_pcp); 1875 // printk(KERN_ALERT "CPU%d Free order-0 page bank = %d, color = %d, is_local %d is_hc_page %d\n", smp_processor_id(), page_bank(page), page_color(page), is_local, is_hc_page);
1796 if (is_local && is_in_pcp) { 1876 //if (!is_local || !is_hc_page) {
1797 pcp = &this_cpu_ptr(zone->pageset)->pcp; 1877 pcp = &this_cpu_ptr(zone->pageset)->pcp;
1798 if (!cold) 1878 if (!cold)
1799 list_add(&page->lru, &pcp->lists[migratetype]); 1879 list_add(&page->lru, &pcp->lists[migratetype]);
@@ -1805,9 +1885,9 @@ void free_hot_cold_page(struct page *page, bool cold)
1805 free_pcppages_bulk(zone, batch, pcp); 1885 free_pcppages_bulk(zone, batch, pcp);
1806 pcp->count -= batch; 1886 pcp->count -= batch;
1807 } 1887 }
1808 } else { 1888// } else {
1809 __free_page(page); 1889// __free_page(page);
1810 } 1890// }
1811out: 1891out:
1812 local_irq_restore(flags); 1892 local_irq_restore(flags);
1813} 1893}
@@ -1936,8 +2016,11 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
1936 struct page *page; 2016 struct page *page;
1937 bool cold = ((gfp_flags & __GFP_COLD) != 0); 2017 bool cold = ((gfp_flags & __GFP_COLD) != 0);
1938 bool colored_req = ((gfp_flags & __GFP_COLOR) != 0); 2018 bool colored_req = ((gfp_flags & __GFP_COLOR) != 0);
2019
2020if (colored_req)
2021 printk(KERN_INFO "buffered_rmqueue(): colored_req received\n");
1939 2022
1940 if (likely(order == 0) && colored_req) { 2023 if (likely(order == 0) && !colored_req) {
1941 struct per_cpu_pages *pcp; 2024 struct per_cpu_pages *pcp;
1942 struct list_head *list; 2025 struct list_head *list;
1943 2026
@@ -1974,7 +2057,7 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
1974 WARN_ON_ONCE(order > 1); 2057 WARN_ON_ONCE(order > 1);
1975 } 2058 }
1976 spin_lock_irqsave(&zone->lock, flags); 2059 spin_lock_irqsave(&zone->lock, flags);
1977 page = __rmqueue(zone, order, migratetype, 0); 2060 page = __rmqueue(zone, order, migratetype, colored_req);
1978 spin_unlock(&zone->lock); 2061 spin_unlock(&zone->lock);
1979 if (!page) 2062 if (!page)
1980 goto failed; 2063 goto failed;
@@ -3087,6 +3170,9 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
3087 .migratetype = gfpflags_to_migratetype(gfp_mask), 3170 .migratetype = gfpflags_to_migratetype(gfp_mask),
3088 }; 3171 };
3089 3172
3173if (gfp_mask&GFP_COLOR)
3174 printk(KERN_INFO "__alloc_pages_nodemask(): called gfp %08x gfp_allowed_mask %08x mt = %d\n", gfp_mask, gfp_allowed_mask, ac.migratetype);
3175
3090 gfp_mask &= gfp_allowed_mask; 3176 gfp_mask &= gfp_allowed_mask;
3091 3177
3092 lockdep_trace_alloc(gfp_mask); 3178 lockdep_trace_alloc(gfp_mask);
@@ -3181,8 +3267,9 @@ EXPORT_SYMBOL(get_zeroed_page);
3181 3267
3182void __free_pages(struct page *page, unsigned int order) 3268void __free_pages(struct page *page, unsigned int order)
3183{ 3269{
3270 int parti_no = bank_to_partition(page_bank(page));
3184 if (put_page_testzero(page)) { 3271 if (put_page_testzero(page)) {
3185 if (order == 0) 3272 if (order == 0 && parti_no == NR_CPUS)
3186 free_hot_cold_page(page, false); 3273 free_hot_cold_page(page, false);
3187 else 3274 else
3188 __free_pages_ok(page, order); 3275 __free_pages_ok(page, order);
diff --git a/mm/slab_common.c b/mm/slab_common.c
index dee018acaeaf..ff4d4c6f4129 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -787,7 +787,14 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
787 return kmalloc_dma_caches[index]; 787 return kmalloc_dma_caches[index];
788 788
789#endif 789#endif
790 return kmalloc_caches[index]; 790
791 if (flags & GFP_COLOR) {
792 int cpu = raw_smp_processor_id();
793 printk(KERN_INFO "in kmalloc_slab index %d\n", index);
794 return hc_kmalloc_caches[cpu][index];
795 }
796 else
797 return kmalloc_caches[index];
791} 798}
792 799
793/* 800/*
@@ -841,6 +848,7 @@ void __init create_kmalloc_caches(unsigned long flags)
841 size_index[size_index_elem(i)] = 8; 848 size_index[size_index_elem(i)] = 8;
842 } 849 }
843 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) { 850 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
851 printk(KERN_INFO "KMALLOC i = %d\n", i);
844 if (!kmalloc_caches[i]) { 852 if (!kmalloc_caches[i]) {
845 kmalloc_caches[i] = create_kmalloc_cache(NULL, 853 kmalloc_caches[i] = create_kmalloc_cache(NULL,
846 1 << i, flags); 854 1 << i, flags);
@@ -866,28 +874,29 @@ printk(KERN_INFO "KMALLOC-192 CACHE CREATED\n");
866/* per-cpu kmalloc caches */ 874/* per-cpu kmalloc caches */
867 printk(KERN_INFO "SLAB_STATE = %d\n", slab_state); 875 printk(KERN_INFO "SLAB_STATE = %d\n", slab_state);
868 for (cpu = 0; cpu < NR_CPUS; cpu++) { 876 for (cpu = 0; cpu < NR_CPUS; cpu++) {
869 //cpu = 0;
870 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) { 877 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
871 char *n; 878 char *n;
872 n = kasprintf(GFP_NOWAIT, "hc%01d-kmalloc-%d", cpu, kmalloc_size(i)); 879 n = kasprintf(GFP_NOWAIT, "cpu%01d-kmalloc-%d", cpu, kmalloc_size(i));
880
881 printk(KERN_INFO "HC-KMALLOC i = %d\n", i);
873 hc_kmalloc_caches[cpu][i] = create_kmalloc_cache(n, 1 << i, SLAB_NO_MERGE|flags); 882 hc_kmalloc_caches[cpu][i] = create_kmalloc_cache(n, 1 << i, SLAB_NO_MERGE|flags);
874 hc_kmalloc_caches[cpu][i]->cpu_id = cpu; 883 hc_kmalloc_caches[cpu][i]->cpu_id = cpu;
875 printk(KERN_INFO "CPU%d HC-KMALLOC-%d CACHE CREATED\n", cpu, 1<<i); 884 printk(KERN_INFO "CPU%d HC-KMALLOC-%d CACHE CREATED\n", cpu, 1<<i);
876 printk(KERN_INFO "HC-KMALLOC-%d slabs freelist=%p, pages=%p, partial=%p\n", 1<<i, hc_kmalloc_caches[cpu][i]->cpu_slab->freelist, hc_kmalloc_caches[cpu][i]->cpu_slab->page,hc_kmalloc_caches[cpu][i]->cpu_slab->partial); 885 printk(KERN_INFO "HC-KMALLOC-%d slabs freelist=%p, pages=%p, partial=%p\n", 1<<i, hc_kmalloc_caches[cpu][i]->cpu_slab->freelist, hc_kmalloc_caches[cpu][i]->cpu_slab->page,hc_kmalloc_caches[cpu][i]->cpu_slab->partial);
877
878
879 /*
880 886
881 if (KMALLOC_MIN_SIZE <= 32 && !pc_kmalloc_caches[cpu][1] && i == 6) { 887 if (KMALLOC_MIN_SIZE <= 32 && !hc_kmalloc_caches[cpu][1] && i == 6) {
882 pc_kmalloc_caches[cpu][1] = create_kmalloc_cache(NULL, 96, flags); 888 char *nm;
883 printk(KERN_INFO "PC-KMALLOC-96 CACHE CREATED\n"); 889 nm = kasprintf(GFP_NOWAIT, "cpu%01d-kmalloc-%d", cpu, kmalloc_size(1));
890 hc_kmalloc_caches[cpu][1] = create_kmalloc_cache(nm, 96, SLAB_NO_MERGE|flags);
891 printk(KERN_INFO "CPU%d HC-KMALLOC-96 CACHE CREATED\n", cpu);
884 } 892 }
885 893
886 if (KMALLOC_MIN_SIZE <= 64 && !pc_kmalloc_caches[cpu][2] && i == 7) { 894 if (KMALLOC_MIN_SIZE <= 64 && !hc_kmalloc_caches[cpu][2] && i == 7) {
887 pc_kmalloc_caches[cpu][2] = create_kmalloc_cache(NULL, 192, flags); 895 char *nm;
888 printk(KERN_INFO "PC-KMALLOC-192 CACHE CREATED\n"); 896 nm = kasprintf(GFP_NOWAIT, "cpu%01d-kmalloc-%d", cpu, kmalloc_size(2));
897 hc_kmalloc_caches[cpu][2] = create_kmalloc_cache(nm, 192, SLAB_NO_MERGE|flags);
898 printk(KERN_INFO "CPU%d HC-KMALLOC-192 CACHE CREATED\n", cpu);
889 } 899 }
890 */
891 } 900 }
892 } 901 }
893 902
diff --git a/mm/slub.c b/mm/slub.c
index 2727a6fc403f..1a2858905c54 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1320,8 +1320,11 @@ static inline struct page *alloc_slab_page(struct kmem_cache *s,
1320 if (memcg_charge_slab(s, flags, order)) 1320 if (memcg_charge_slab(s, flags, order))
1321 return NULL; 1321 return NULL;
1322 1322
1323 if (node == NUMA_NO_NODE) 1323 if (node == NUMA_NO_NODE) {
1324 if (flags&GFP_COLOR)
1325 printk(KERN_INFO "alloc_pages calls with GFP_COLOR order = %d\n", order);
1324 page = alloc_pages(flags, order); 1326 page = alloc_pages(flags, order);
1327 }
1325 else 1328 else
1326 page = alloc_pages_exact_node(node, flags, order); 1329 page = alloc_pages_exact_node(node, flags, order);
1327 1330
@@ -1337,6 +1340,9 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
1337 struct kmem_cache_order_objects oo = s->oo; 1340 struct kmem_cache_order_objects oo = s->oo;
1338 gfp_t alloc_gfp; 1341 gfp_t alloc_gfp;
1339 1342
1343if (flags&GFP_COLOR)
1344 printk(KERN_INFO "gfp_allowed_mask = %08x\n", gfp_allowed_mask);
1345
1340 flags &= gfp_allowed_mask; 1346 flags &= gfp_allowed_mask;
1341 1347
1342 if (flags & __GFP_WAIT) 1348 if (flags & __GFP_WAIT)
@@ -1349,7 +1355,9 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
1349 * so we fall-back to the minimum order allocation. 1355 * so we fall-back to the minimum order allocation.
1350 */ 1356 */
1351 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL; 1357 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
1352 1358if (flags&__GFP_COLOR) {
1359 printk(KERN_INFO "allocate_slab with GFP_COLOR alloc_gfp = %08x\n", alloc_gfp);
1360}
1353 page = alloc_slab_page(s, alloc_gfp, node, oo); 1361 page = alloc_slab_page(s, alloc_gfp, node, oo);
1354 if (unlikely(!page)) { 1362 if (unlikely(!page)) {
1355 oo = s->min; 1363 oo = s->min;
@@ -1419,7 +1427,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
1419 } 1427 }
1420 1428
1421 page = allocate_slab(s, 1429 page = allocate_slab(s,
1422 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); 1430 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK | GFP_COLOR), node);
1423 if (!page) 1431 if (!page)
1424 goto out; 1432 goto out;
1425 1433
@@ -2223,6 +2231,11 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
2223 return freelist; 2231 return freelist;
2224 2232
2225 page = new_slab(s, flags, node); 2233 page = new_slab(s, flags, node);
2234
2235if (flags&GFP_COLOR) {
2236 printk(KERN_INFO "new_slab_objects(): gets page %p\n", page);
2237}
2238
2226 if (page) { 2239 if (page) {
2227 c = raw_cpu_ptr(s->cpu_slab); 2240 c = raw_cpu_ptr(s->cpu_slab);
2228 if (c->page) 2241 if (c->page)
@@ -2308,6 +2321,8 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
2308 void *freelist; 2321 void *freelist;
2309 struct page *page; 2322 struct page *page;
2310 unsigned long flags; 2323 unsigned long flags;
2324if (gfpflags&GFP_COLOR)
2325 printk(KERN_INFO "__slab_alloc slow_path\n");
2311 2326
2312 local_irq_save(flags); 2327 local_irq_save(flags);
2313#ifdef CONFIG_PREEMPT 2328#ifdef CONFIG_PREEMPT
@@ -2319,6 +2334,11 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
2319 c = this_cpu_ptr(s->cpu_slab); 2334 c = this_cpu_ptr(s->cpu_slab);
2320#endif 2335#endif
2321 2336
2337
2338if (gfpflags&GFP_COLOR) {
2339 printk(KERN_INFO "__slab_alloc : page %p, partial %p\n", c->page, c->partial);
2340}
2341
2322 page = c->page; 2342 page = c->page;
2323 if (!page) 2343 if (!page)
2324 goto new_slab; 2344 goto new_slab;
@@ -3308,14 +3328,22 @@ void *__kmalloc(size_t size, gfp_t flags)
3308 struct kmem_cache *s; 3328 struct kmem_cache *s;
3309 void *ret; 3329 void *ret;
3310 3330
3331if (flags & GFP_COLOR) {
3332 printk(KERN_INFO "kmalloc size %d\n", size);
3333}
3311 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) 3334 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
3312 return kmalloc_large(size, flags); 3335 return kmalloc_large(size, flags);
3313 3336
3314 s = kmalloc_slab(size, flags); 3337 s = kmalloc_slab(size, flags);
3315 3338if (flags & GFP_COLOR) {
3339 printk(KERN_INFO "kmalloc_slab %p\n", s);
3340}
3316 if (unlikely(ZERO_OR_NULL_PTR(s))) 3341 if (unlikely(ZERO_OR_NULL_PTR(s)))
3317 return s; 3342 return s;
3318 3343
3344if (flags & GFP_COLOR) {
3345 printk(KERN_INFO "slab_alloc calls!!\n");
3346}
3319 ret = slab_alloc(s, flags, _RET_IP_); 3347 ret = slab_alloc(s, flags, _RET_IP_);
3320 3348
3321 trace_kmalloc(_RET_IP_, ret, size, s->size, flags); 3349 trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 0d748d23dc4c..5df6edb32512 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1312,8 +1312,8 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
1312 seq_printf(m, "\n vm stats threshold: %d", 1312 seq_printf(m, "\n vm stats threshold: %d",
1313 pageset->stat_threshold); 1313 pageset->stat_threshold);
1314#endif 1314#endif
1315 /* test */ 1315 /* pcp test */
1316 seq_printf(m, "\n"); 1316/* seq_printf(m, "\n");
1317 for (mtype = 0; mtype < MIGRATE_PCPTYPES; mtype++) { 1317 for (mtype = 0; mtype < MIGRATE_PCPTYPES; mtype++) {
1318 struct page *p; 1318 struct page *p;
1319 list_for_each_entry(p, &pageset->pcp.lists[mtype], lru) { 1319 list_for_each_entry(p, &pageset->pcp.lists[mtype], lru) {
@@ -1321,6 +1321,7 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
1321 seq_printf(m, "page bank = %d color = %d\n", page_bank(p), page_color(p)); 1321 seq_printf(m, "page bank = %d color = %d\n", page_bank(p), page_color(p));
1322 } 1322 }
1323 } 1323 }
1324*/
1324 } 1325 }
1325 seq_printf(m, 1326 seq_printf(m,
1326 "\n all_unreclaimable: %u" 1327 "\n all_unreclaimable: %u"
generated by cgit v1.2.2 (git 2.25.0) at 2024-06-02 12:16:54 -0400