aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--kernel/power/snapshot.c223
1 files changed, 21 insertions, 202 deletions
diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c
index 5b71caf43d32..ab1998adb0a9 100644
--- a/kernel/power/snapshot.c
+++ b/kernel/power/snapshot.c
@@ -267,18 +267,6 @@ static void *chain_alloc(struct chain_allocator *ca, unsigned int size)
267#define BM_BLOCK_SHIFT (PAGE_SHIFT + 3) 267#define BM_BLOCK_SHIFT (PAGE_SHIFT + 3)
268#define BM_BLOCK_MASK ((1UL << BM_BLOCK_SHIFT) - 1) 268#define BM_BLOCK_MASK ((1UL << BM_BLOCK_SHIFT) - 1)
269 269
270struct bm_block {
271 struct list_head hook; /* hook into a list of bitmap blocks */
272 unsigned long start_pfn; /* pfn represented by the first bit */
273 unsigned long end_pfn; /* pfn represented by the last bit plus 1 */
274 unsigned long *data; /* bitmap representing pages */
275};
276
277static inline unsigned long bm_block_bits(struct bm_block *bb)
278{
279 return bb->end_pfn - bb->start_pfn;
280}
281
282/* 270/*
283 * struct rtree_node is a wrapper struct to link the nodes 271 * struct rtree_node is a wrapper struct to link the nodes
284 * of the rtree together for easy linear iteration over 272 * of the rtree together for easy linear iteration over
@@ -307,9 +295,6 @@ struct mem_zone_bm_rtree {
307/* strcut bm_position is used for browsing memory bitmaps */ 295/* strcut bm_position is used for browsing memory bitmaps */
308 296
309struct bm_position { 297struct bm_position {
310 struct bm_block *block;
311 int bit;
312
313 struct mem_zone_bm_rtree *zone; 298 struct mem_zone_bm_rtree *zone;
314 struct rtree_node *node; 299 struct rtree_node *node;
315 unsigned long node_pfn; 300 unsigned long node_pfn;
@@ -318,7 +303,6 @@ struct bm_position {
318 303
319struct memory_bitmap { 304struct memory_bitmap {
320 struct list_head zones; 305 struct list_head zones;
321 struct list_head blocks; /* list of bitmap blocks */
322 struct linked_page *p_list; /* list of pages used to store zone 306 struct linked_page *p_list; /* list of pages used to store zone
323 * bitmap objects and bitmap block 307 * bitmap objects and bitmap block
324 * objects 308 * objects
@@ -490,9 +474,6 @@ static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,
490 474
491static void memory_bm_position_reset(struct memory_bitmap *bm) 475static void memory_bm_position_reset(struct memory_bitmap *bm)
492{ 476{
493 bm->cur.block = list_entry(bm->blocks.next, struct bm_block, hook);
494 bm->cur.bit = 0;
495
496 bm->cur.zone = list_entry(bm->zones.next, struct mem_zone_bm_rtree, 477 bm->cur.zone = list_entry(bm->zones.next, struct mem_zone_bm_rtree,
497 list); 478 list);
498 bm->cur.node = list_entry(bm->cur.zone->leaves.next, 479 bm->cur.node = list_entry(bm->cur.zone->leaves.next,
@@ -503,30 +484,6 @@ static void memory_bm_position_reset(struct memory_bitmap *bm)
503 484
504static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free); 485static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
505 486
506/**
507 * create_bm_block_list - create a list of block bitmap objects
508 * @pages - number of pages to track
509 * @list - list to put the allocated blocks into
510 * @ca - chain allocator to be used for allocating memory
511 */
512static int create_bm_block_list(unsigned long pages,
513 struct list_head *list,
514 struct chain_allocator *ca)
515{
516 unsigned int nr_blocks = DIV_ROUND_UP(pages, BM_BITS_PER_BLOCK);
517
518 while (nr_blocks-- > 0) {
519 struct bm_block *bb;
520
521 bb = chain_alloc(ca, sizeof(struct bm_block));
522 if (!bb)
523 return -ENOMEM;
524 list_add(&bb->hook, list);
525 }
526
527 return 0;
528}
529
530struct mem_extent { 487struct mem_extent {
531 struct list_head hook; 488 struct list_head hook;
532 unsigned long start; 489 unsigned long start;
@@ -618,7 +575,6 @@ memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed)
618 int error; 575 int error;
619 576
620 chain_init(&ca, gfp_mask, safe_needed); 577 chain_init(&ca, gfp_mask, safe_needed);
621 INIT_LIST_HEAD(&bm->blocks);
622 INIT_LIST_HEAD(&bm->zones); 578 INIT_LIST_HEAD(&bm->zones);
623 579
624 error = create_mem_extents(&mem_extents, gfp_mask); 580 error = create_mem_extents(&mem_extents, gfp_mask);
@@ -627,38 +583,13 @@ memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed)
627 583
628 list_for_each_entry(ext, &mem_extents, hook) { 584 list_for_each_entry(ext, &mem_extents, hook) {
629 struct mem_zone_bm_rtree *zone; 585 struct mem_zone_bm_rtree *zone;
630 struct bm_block *bb;
631 unsigned long pfn = ext->start;
632 unsigned long pages = ext->end - ext->start;
633
634 bb = list_entry(bm->blocks.prev, struct bm_block, hook);
635
636 error = create_bm_block_list(pages, bm->blocks.prev, &ca);
637 if (error)
638 goto Error;
639
640 list_for_each_entry_continue(bb, &bm->blocks, hook) {
641 bb->data = get_image_page(gfp_mask, safe_needed);
642 if (!bb->data) {
643 error = -ENOMEM;
644 goto Error;
645 }
646
647 bb->start_pfn = pfn;
648 if (pages >= BM_BITS_PER_BLOCK) {
649 pfn += BM_BITS_PER_BLOCK;
650 pages -= BM_BITS_PER_BLOCK;
651 } else {
652 /* This is executed only once in the loop */
653 pfn += pages;
654 }
655 bb->end_pfn = pfn;
656 }
657 586
658 zone = create_zone_bm_rtree(gfp_mask, safe_needed, &ca, 587 zone = create_zone_bm_rtree(gfp_mask, safe_needed, &ca,
659 ext->start, ext->end); 588 ext->start, ext->end);
660 if (!zone) 589 if (!zone) {
590 error = -ENOMEM;
661 goto Error; 591 goto Error;
592 }
662 list_add_tail(&zone->list, &bm->zones); 593 list_add_tail(&zone->list, &bm->zones);
663 } 594 }
664 595
@@ -680,11 +611,6 @@ memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed)
680static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free) 611static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free)
681{ 612{
682 struct mem_zone_bm_rtree *zone; 613 struct mem_zone_bm_rtree *zone;
683 struct bm_block *bb;
684
685 list_for_each_entry(bb, &bm->blocks, hook)
686 if (bb->data)
687 free_image_page(bb->data, clear_nosave_free);
688 614
689 list_for_each_entry(zone, &bm->zones, list) 615 list_for_each_entry(zone, &bm->zones, list)
690 free_zone_bm_rtree(zone, clear_nosave_free); 616 free_zone_bm_rtree(zone, clear_nosave_free);
@@ -692,55 +618,20 @@ static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free)
692 free_list_of_pages(bm->p_list, clear_nosave_free); 618 free_list_of_pages(bm->p_list, clear_nosave_free);
693 619
694 INIT_LIST_HEAD(&bm->zones); 620 INIT_LIST_HEAD(&bm->zones);
695 INIT_LIST_HEAD(&bm->blocks);
696} 621}
697 622
698/** 623/**
699 * memory_bm_find_bit - find the bit in the bitmap @bm that corresponds 624 * memory_bm_find_bit - Find the bit for pfn in the memory
700 * to given pfn. The cur_zone_bm member of @bm and the cur_block member 625 * bitmap
701 * of @bm->cur_zone_bm are updated.
702 */
703static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,
704 void **addr, unsigned int *bit_nr)
705{
706 struct bm_block *bb;
707
708 /*
709 * Check if the pfn corresponds to the current bitmap block and find
710 * the block where it fits if this is not the case.
711 */
712 bb = bm->cur.block;
713 if (pfn < bb->start_pfn)
714 list_for_each_entry_continue_reverse(bb, &bm->blocks, hook)
715 if (pfn >= bb->start_pfn)
716 break;
717
718 if (pfn >= bb->end_pfn)
719 list_for_each_entry_continue(bb, &bm->blocks, hook)
720 if (pfn >= bb->start_pfn && pfn < bb->end_pfn)
721 break;
722
723 if (&bb->hook == &bm->blocks)
724 return -EFAULT;
725
726 /* The block has been found */
727 bm->cur.block = bb;
728 pfn -= bb->start_pfn;
729 bm->cur.bit = pfn + 1;
730 *bit_nr = pfn;
731 *addr = bb->data;
732 return 0;
733}
734
735/*
736 * memory_rtree_find_bit - Find the bit for pfn in the memory
737 * bitmap
738 * 626 *
739 * Walks the radix tree to find the page which contains the bit for 627 * Find the bit in the bitmap @bm that corresponds to given pfn.
628 * The cur.zone, cur.block and cur.node_pfn member of @bm are
629 * updated.
630 * It walks the radix tree to find the page which contains the bit for
740 * pfn and returns the bit position in **addr and *bit_nr. 631 * pfn and returns the bit position in **addr and *bit_nr.
741 */ 632 */
742static int memory_rtree_find_bit(struct memory_bitmap *bm, unsigned long pfn, 633static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,
743 void **addr, unsigned int *bit_nr) 634 void **addr, unsigned int *bit_nr)
744{ 635{
745 struct mem_zone_bm_rtree *curr, *zone; 636 struct mem_zone_bm_rtree *curr, *zone;
746 struct rtree_node *node; 637 struct rtree_node *node;
@@ -808,10 +699,6 @@ static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn)
808 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 699 error = memory_bm_find_bit(bm, pfn, &addr, &bit);
809 BUG_ON(error); 700 BUG_ON(error);
810 set_bit(bit, addr); 701 set_bit(bit, addr);
811
812 error = memory_rtree_find_bit(bm, pfn, &addr, &bit);
813 BUG_ON(error);
814 set_bit(bit, addr);
815} 702}
816 703
817static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn) 704static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn)
@@ -823,12 +710,6 @@ static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn)
823 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 710 error = memory_bm_find_bit(bm, pfn, &addr, &bit);
824 if (!error) 711 if (!error)
825 set_bit(bit, addr); 712 set_bit(bit, addr);
826 else
827 return error;
828
829 error = memory_rtree_find_bit(bm, pfn, &addr, &bit);
830 if (!error)
831 set_bit(bit, addr);
832 713
833 return error; 714 return error;
834} 715}
@@ -842,10 +723,6 @@ static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn)
842 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 723 error = memory_bm_find_bit(bm, pfn, &addr, &bit);
843 BUG_ON(error); 724 BUG_ON(error);
844 clear_bit(bit, addr); 725 clear_bit(bit, addr);
845
846 error = memory_rtree_find_bit(bm, pfn, &addr, &bit);
847 BUG_ON(error);
848 clear_bit(bit, addr);
849} 726}
850 727
851static void memory_bm_clear_current(struct memory_bitmap *bm) 728static void memory_bm_clear_current(struct memory_bitmap *bm)
@@ -854,82 +731,25 @@ static void memory_bm_clear_current(struct memory_bitmap *bm)
854 731
855 bit = max(bm->cur.node_bit - 1, 0); 732 bit = max(bm->cur.node_bit - 1, 0);
856 clear_bit(bit, bm->cur.node->data); 733 clear_bit(bit, bm->cur.node->data);
857
858 bit = max(bm->cur.bit - 1, 0);
859 clear_bit(bit, bm->cur.block->data);
860} 734}
861 735
862static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn) 736static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)
863{ 737{
864 void *addr; 738 void *addr;
865 unsigned int bit; 739 unsigned int bit;
866 int error, error2; 740 int error;
867 int v;
868 741
869 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 742 error = memory_bm_find_bit(bm, pfn, &addr, &bit);
870 BUG_ON(error); 743 BUG_ON(error);
871 v = test_bit(bit, addr); 744 return test_bit(bit, addr);
872
873 error2 = memory_rtree_find_bit(bm, pfn, &addr, &bit);
874 BUG_ON(error2);
875
876 WARN_ON_ONCE(v != test_bit(bit, addr));
877
878 return v;
879} 745}
880 746
881static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn) 747static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn)
882{ 748{
883 void *addr; 749 void *addr;
884 unsigned int bit; 750 unsigned int bit;
885 int present;
886
887 present = !memory_bm_find_bit(bm, pfn, &addr, &bit);
888
889 WARN_ON_ONCE(present != !memory_rtree_find_bit(bm, pfn, &addr, &bit));
890 751
891 return present; 752 return !memory_bm_find_bit(bm, pfn, &addr, &bit);
892}
893
894/**
895 * memory_bm_next_pfn - find the pfn that corresponds to the next set bit
896 * in the bitmap @bm. If the pfn cannot be found, BM_END_OF_MAP is
897 * returned.
898 *
899 * It is required to run memory_bm_position_reset() before the first call to
900 * this function.
901 */
902
903static unsigned long memory_bm_rtree_next_pfn(struct memory_bitmap *bm);
904
905static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)
906{
907 unsigned long rtree_pfn;
908 struct bm_block *bb;
909 int bit;
910
911 rtree_pfn = memory_bm_rtree_next_pfn(bm);
912
913 bb = bm->cur.block;
914 do {
915 bit = bm->cur.bit;
916 bit = find_next_bit(bb->data, bm_block_bits(bb), bit);
917 if (bit < bm_block_bits(bb))
918 goto Return_pfn;
919
920 bb = list_entry(bb->hook.next, struct bm_block, hook);
921 bm->cur.block = bb;
922 bm->cur.bit = 0;
923 } while (&bb->hook != &bm->blocks);
924
925 memory_bm_position_reset(bm);
926 WARN_ON_ONCE(rtree_pfn != BM_END_OF_MAP);
927 return BM_END_OF_MAP;
928
929 Return_pfn:
930 WARN_ON_ONCE(bb->start_pfn + bit != rtree_pfn);
931 bm->cur.bit = bit + 1;
932 return bb->start_pfn + bit;
933} 753}
934 754
935/* 755/*
@@ -967,14 +787,17 @@ static bool rtree_next_node(struct memory_bitmap *bm)
967 return false; 787 return false;
968} 788}
969 789
970/* 790/**
971 * memory_bm_rtree_next_pfn - Find the next set bit 791 * memory_bm_rtree_next_pfn - Find the next set bit in the bitmap @bm
972 * 792 *
973 * Starting from the last returned position this function searches 793 * Starting from the last returned position this function searches
974 * for the next set bit in the memory bitmap and returns its 794 * for the next set bit in the memory bitmap and returns its
975 * number. If no more bit is set BM_END_OF_MAP is returned. 795 * number. If no more bit is set BM_END_OF_MAP is returned.
796 *
797 * It is required to run memory_bm_position_reset() before the
798 * first call to this function.
976 */ 799 */
977static unsigned long memory_bm_rtree_next_pfn(struct memory_bitmap *bm) 800static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)
978{ 801{
979 unsigned long bits, pfn, pages; 802 unsigned long bits, pfn, pages;
980 int bit; 803 int bit;
@@ -1216,11 +1039,7 @@ void free_basic_memory_bitmaps(void)
1216unsigned int snapshot_additional_pages(struct zone *zone) 1039unsigned int snapshot_additional_pages(struct zone *zone)
1217{ 1040{
1218 unsigned int rtree, nodes; 1041 unsigned int rtree, nodes;
1219 unsigned int res;
1220 1042
1221 res = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
1222 res += DIV_ROUND_UP(res * sizeof(struct bm_block),
1223 LINKED_PAGE_DATA_SIZE);
1224 rtree = nodes = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK); 1043 rtree = nodes = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
1225 rtree += DIV_ROUND_UP(rtree * sizeof(struct rtree_node), 1044 rtree += DIV_ROUND_UP(rtree * sizeof(struct rtree_node),
1226 LINKED_PAGE_DATA_SIZE); 1045 LINKED_PAGE_DATA_SIZE);
@@ -1229,7 +1048,7 @@ unsigned int snapshot_additional_pages(struct zone *zone)
1229 rtree += nodes; 1048 rtree += nodes;
1230 } 1049 }
1231 1050
1232 return 2 * (res + rtree); 1051 return 2 * rtree;
1233} 1052}
1234 1053
1235#ifdef CONFIG_HIGHMEM 1054#ifdef CONFIG_HIGHMEM