diff options
| -rw-r--r-- | kernel/power/snapshot.c | 223 |
1 files changed, 21 insertions, 202 deletions
diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index 5b71caf43d32..ab1998adb0a9 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c | |||
| @@ -267,18 +267,6 @@ static void *chain_alloc(struct chain_allocator *ca, unsigned int size) | |||
| 267 | #define BM_BLOCK_SHIFT (PAGE_SHIFT + 3) | 267 | #define BM_BLOCK_SHIFT (PAGE_SHIFT + 3) |
| 268 | #define BM_BLOCK_MASK ((1UL << BM_BLOCK_SHIFT) - 1) | 268 | #define BM_BLOCK_MASK ((1UL << BM_BLOCK_SHIFT) - 1) |
| 269 | 269 | ||
| 270 | struct bm_block { | ||
| 271 | struct list_head hook; /* hook into a list of bitmap blocks */ | ||
| 272 | unsigned long start_pfn; /* pfn represented by the first bit */ | ||
| 273 | unsigned long end_pfn; /* pfn represented by the last bit plus 1 */ | ||
| 274 | unsigned long *data; /* bitmap representing pages */ | ||
| 275 | }; | ||
| 276 | |||
| 277 | static inline unsigned long bm_block_bits(struct bm_block *bb) | ||
| 278 | { | ||
| 279 | return bb->end_pfn - bb->start_pfn; | ||
| 280 | } | ||
| 281 | |||
| 282 | /* | 270 | /* |
| 283 | * struct rtree_node is a wrapper struct to link the nodes | 271 | * struct rtree_node is a wrapper struct to link the nodes |
| 284 | * of the rtree together for easy linear iteration over | 272 | * of the rtree together for easy linear iteration over |
| @@ -307,9 +295,6 @@ struct mem_zone_bm_rtree { | |||
| 307 | /* strcut bm_position is used for browsing memory bitmaps */ | 295 | /* strcut bm_position is used for browsing memory bitmaps */ |
| 308 | 296 | ||
| 309 | struct bm_position { | 297 | struct bm_position { |
| 310 | struct bm_block *block; | ||
| 311 | int bit; | ||
| 312 | |||
| 313 | struct mem_zone_bm_rtree *zone; | 298 | struct mem_zone_bm_rtree *zone; |
| 314 | struct rtree_node *node; | 299 | struct rtree_node *node; |
| 315 | unsigned long node_pfn; | 300 | unsigned long node_pfn; |
| @@ -318,7 +303,6 @@ struct bm_position { | |||
| 318 | 303 | ||
| 319 | struct memory_bitmap { | 304 | struct memory_bitmap { |
| 320 | struct list_head zones; | 305 | struct list_head zones; |
| 321 | struct list_head blocks; /* list of bitmap blocks */ | ||
| 322 | struct linked_page *p_list; /* list of pages used to store zone | 306 | struct linked_page *p_list; /* list of pages used to store zone |
| 323 | * bitmap objects and bitmap block | 307 | * bitmap objects and bitmap block |
| 324 | * objects | 308 | * objects |
| @@ -490,9 +474,6 @@ static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone, | |||
| 490 | 474 | ||
| 491 | static void memory_bm_position_reset(struct memory_bitmap *bm) | 475 | static void memory_bm_position_reset(struct memory_bitmap *bm) |
| 492 | { | 476 | { |
| 493 | bm->cur.block = list_entry(bm->blocks.next, struct bm_block, hook); | ||
| 494 | bm->cur.bit = 0; | ||
| 495 | |||
| 496 | bm->cur.zone = list_entry(bm->zones.next, struct mem_zone_bm_rtree, | 477 | bm->cur.zone = list_entry(bm->zones.next, struct mem_zone_bm_rtree, |
| 497 | list); | 478 | list); |
| 498 | bm->cur.node = list_entry(bm->cur.zone->leaves.next, | 479 | bm->cur.node = list_entry(bm->cur.zone->leaves.next, |
| @@ -503,30 +484,6 @@ static void memory_bm_position_reset(struct memory_bitmap *bm) | |||
| 503 | 484 | ||
| 504 | static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free); | 485 | static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free); |
| 505 | 486 | ||
| 506 | /** | ||
| 507 | * create_bm_block_list - create a list of block bitmap objects | ||
| 508 | * @pages - number of pages to track | ||
| 509 | * @list - list to put the allocated blocks into | ||
| 510 | * @ca - chain allocator to be used for allocating memory | ||
| 511 | */ | ||
| 512 | static int create_bm_block_list(unsigned long pages, | ||
| 513 | struct list_head *list, | ||
| 514 | struct chain_allocator *ca) | ||
| 515 | { | ||
| 516 | unsigned int nr_blocks = DIV_ROUND_UP(pages, BM_BITS_PER_BLOCK); | ||
| 517 | |||
| 518 | while (nr_blocks-- > 0) { | ||
| 519 | struct bm_block *bb; | ||
| 520 | |||
| 521 | bb = chain_alloc(ca, sizeof(struct bm_block)); | ||
| 522 | if (!bb) | ||
| 523 | return -ENOMEM; | ||
| 524 | list_add(&bb->hook, list); | ||
| 525 | } | ||
| 526 | |||
| 527 | return 0; | ||
| 528 | } | ||
| 529 | |||
| 530 | struct mem_extent { | 487 | struct mem_extent { |
| 531 | struct list_head hook; | 488 | struct list_head hook; |
| 532 | unsigned long start; | 489 | unsigned long start; |
| @@ -618,7 +575,6 @@ memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed) | |||
| 618 | int error; | 575 | int error; |
| 619 | 576 | ||
| 620 | chain_init(&ca, gfp_mask, safe_needed); | 577 | chain_init(&ca, gfp_mask, safe_needed); |
| 621 | INIT_LIST_HEAD(&bm->blocks); | ||
| 622 | INIT_LIST_HEAD(&bm->zones); | 578 | INIT_LIST_HEAD(&bm->zones); |
| 623 | 579 | ||
| 624 | error = create_mem_extents(&mem_extents, gfp_mask); | 580 | error = create_mem_extents(&mem_extents, gfp_mask); |
| @@ -627,38 +583,13 @@ memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed) | |||
| 627 | 583 | ||
| 628 | list_for_each_entry(ext, &mem_extents, hook) { | 584 | list_for_each_entry(ext, &mem_extents, hook) { |
| 629 | struct mem_zone_bm_rtree *zone; | 585 | struct mem_zone_bm_rtree *zone; |
| 630 | struct bm_block *bb; | ||
| 631 | unsigned long pfn = ext->start; | ||
| 632 | unsigned long pages = ext->end - ext->start; | ||
| 633 | |||
| 634 | bb = list_entry(bm->blocks.prev, struct bm_block, hook); | ||
| 635 | |||
| 636 | error = create_bm_block_list(pages, bm->blocks.prev, &ca); | ||
| 637 | if (error) | ||
| 638 | goto Error; | ||
| 639 | |||
| 640 | list_for_each_entry_continue(bb, &bm->blocks, hook) { | ||
| 641 | bb->data = get_image_page(gfp_mask, safe_needed); | ||
| 642 | if (!bb->data) { | ||
| 643 | error = -ENOMEM; | ||
| 644 | goto Error; | ||
| 645 | } | ||
| 646 | |||
| 647 | bb->start_pfn = pfn; | ||
| 648 | if (pages >= BM_BITS_PER_BLOCK) { | ||
| 649 | pfn += BM_BITS_PER_BLOCK; | ||
| 650 | pages -= BM_BITS_PER_BLOCK; | ||
| 651 | } else { | ||
| 652 | /* This is executed only once in the loop */ | ||
| 653 | pfn += pages; | ||
| 654 | } | ||
| 655 | bb->end_pfn = pfn; | ||
| 656 | } | ||
| 657 | 586 | ||
| 658 | zone = create_zone_bm_rtree(gfp_mask, safe_needed, &ca, | 587 | zone = create_zone_bm_rtree(gfp_mask, safe_needed, &ca, |
| 659 | ext->start, ext->end); | 588 | ext->start, ext->end); |
| 660 | if (!zone) | 589 | if (!zone) { |
| 590 | error = -ENOMEM; | ||
| 661 | goto Error; | 591 | goto Error; |
| 592 | } | ||
| 662 | list_add_tail(&zone->list, &bm->zones); | 593 | list_add_tail(&zone->list, &bm->zones); |
| 663 | } | 594 | } |
| 664 | 595 | ||
| @@ -680,11 +611,6 @@ memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed) | |||
| 680 | static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free) | 611 | static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free) |
| 681 | { | 612 | { |
| 682 | struct mem_zone_bm_rtree *zone; | 613 | struct mem_zone_bm_rtree *zone; |
| 683 | struct bm_block *bb; | ||
| 684 | |||
| 685 | list_for_each_entry(bb, &bm->blocks, hook) | ||
| 686 | if (bb->data) | ||
| 687 | free_image_page(bb->data, clear_nosave_free); | ||
| 688 | 614 | ||
| 689 | list_for_each_entry(zone, &bm->zones, list) | 615 | list_for_each_entry(zone, &bm->zones, list) |
| 690 | free_zone_bm_rtree(zone, clear_nosave_free); | 616 | free_zone_bm_rtree(zone, clear_nosave_free); |
| @@ -692,55 +618,20 @@ static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free) | |||
| 692 | free_list_of_pages(bm->p_list, clear_nosave_free); | 618 | free_list_of_pages(bm->p_list, clear_nosave_free); |
| 693 | 619 | ||
| 694 | INIT_LIST_HEAD(&bm->zones); | 620 | INIT_LIST_HEAD(&bm->zones); |
| 695 | INIT_LIST_HEAD(&bm->blocks); | ||
| 696 | } | 621 | } |
| 697 | 622 | ||
| 698 | /** | 623 | /** |
| 699 | * memory_bm_find_bit - find the bit in the bitmap @bm that corresponds | 624 | * memory_bm_find_bit - Find the bit for pfn in the memory |
| 700 | * to given pfn. The cur_zone_bm member of @bm and the cur_block member | 625 | * bitmap |
| 701 | * of @bm->cur_zone_bm are updated. | ||
| 702 | */ | ||
| 703 | static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn, | ||
| 704 | void **addr, unsigned int *bit_nr) | ||
| 705 | { | ||
| 706 | struct bm_block *bb; | ||
| 707 | |||
| 708 | /* | ||
| 709 | * Check if the pfn corresponds to the current bitmap block and find | ||
| 710 | * the block where it fits if this is not the case. | ||
| 711 | */ | ||
| 712 | bb = bm->cur.block; | ||
| 713 | if (pfn < bb->start_pfn) | ||
| 714 | list_for_each_entry_continue_reverse(bb, &bm->blocks, hook) | ||
| 715 | if (pfn >= bb->start_pfn) | ||
| 716 | break; | ||
| 717 | |||
| 718 | if (pfn >= bb->end_pfn) | ||
| 719 | list_for_each_entry_continue(bb, &bm->blocks, hook) | ||
| 720 | if (pfn >= bb->start_pfn && pfn < bb->end_pfn) | ||
| 721 | break; | ||
| 722 | |||
| 723 | if (&bb->hook == &bm->blocks) | ||
| 724 | return -EFAULT; | ||
| 725 | |||
| 726 | /* The block has been found */ | ||
| 727 | bm->cur.block = bb; | ||
| 728 | pfn -= bb->start_pfn; | ||
| 729 | bm->cur.bit = pfn + 1; | ||
| 730 | *bit_nr = pfn; | ||
| 731 | *addr = bb->data; | ||
| 732 | return 0; | ||
| 733 | } | ||
| 734 | |||
| 735 | /* | ||
| 736 | * memory_rtree_find_bit - Find the bit for pfn in the memory | ||
| 737 | * bitmap | ||
| 738 | * | 626 | * |
| 739 | * Walks the radix tree to find the page which contains the bit for | 627 | * Find the bit in the bitmap @bm that corresponds to given pfn. |
| 628 | * The cur.zone, cur.block and cur.node_pfn member of @bm are | ||
| 629 | * updated. | ||
| 630 | * It walks the radix tree to find the page which contains the bit for | ||
| 740 | * pfn and returns the bit position in **addr and *bit_nr. | 631 | * pfn and returns the bit position in **addr and *bit_nr. |
| 741 | */ | 632 | */ |
| 742 | static int memory_rtree_find_bit(struct memory_bitmap *bm, unsigned long pfn, | 633 | static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn, |
| 743 | void **addr, unsigned int *bit_nr) | 634 | void **addr, unsigned int *bit_nr) |
| 744 | { | 635 | { |
| 745 | struct mem_zone_bm_rtree *curr, *zone; | 636 | struct mem_zone_bm_rtree *curr, *zone; |
| 746 | struct rtree_node *node; | 637 | struct rtree_node *node; |
| @@ -808,10 +699,6 @@ static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn) | |||
| 808 | error = memory_bm_find_bit(bm, pfn, &addr, &bit); | 699 | error = memory_bm_find_bit(bm, pfn, &addr, &bit); |
| 809 | BUG_ON(error); | 700 | BUG_ON(error); |
| 810 | set_bit(bit, addr); | 701 | set_bit(bit, addr); |
| 811 | |||
| 812 | error = memory_rtree_find_bit(bm, pfn, &addr, &bit); | ||
| 813 | BUG_ON(error); | ||
| 814 | set_bit(bit, addr); | ||
| 815 | } | 702 | } |
| 816 | 703 | ||
| 817 | static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn) | 704 | static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn) |
| @@ -823,12 +710,6 @@ static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn) | |||
| 823 | error = memory_bm_find_bit(bm, pfn, &addr, &bit); | 710 | error = memory_bm_find_bit(bm, pfn, &addr, &bit); |
| 824 | if (!error) | 711 | if (!error) |
| 825 | set_bit(bit, addr); | 712 | set_bit(bit, addr); |
| 826 | else | ||
| 827 | return error; | ||
| 828 | |||
| 829 | error = memory_rtree_find_bit(bm, pfn, &addr, &bit); | ||
| 830 | if (!error) | ||
| 831 | set_bit(bit, addr); | ||
| 832 | 713 | ||
| 833 | return error; | 714 | return error; |
| 834 | } | 715 | } |
| @@ -842,10 +723,6 @@ static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn) | |||
| 842 | error = memory_bm_find_bit(bm, pfn, &addr, &bit); | 723 | error = memory_bm_find_bit(bm, pfn, &addr, &bit); |
| 843 | BUG_ON(error); | 724 | BUG_ON(error); |
| 844 | clear_bit(bit, addr); | 725 | clear_bit(bit, addr); |
| 845 | |||
| 846 | error = memory_rtree_find_bit(bm, pfn, &addr, &bit); | ||
| 847 | BUG_ON(error); | ||
| 848 | clear_bit(bit, addr); | ||
| 849 | } | 726 | } |
| 850 | 727 | ||
| 851 | static void memory_bm_clear_current(struct memory_bitmap *bm) | 728 | static void memory_bm_clear_current(struct memory_bitmap *bm) |
| @@ -854,82 +731,25 @@ static void memory_bm_clear_current(struct memory_bitmap *bm) | |||
| 854 | 731 | ||
| 855 | bit = max(bm->cur.node_bit - 1, 0); | 732 | bit = max(bm->cur.node_bit - 1, 0); |
| 856 | clear_bit(bit, bm->cur.node->data); | 733 | clear_bit(bit, bm->cur.node->data); |
| 857 | |||
| 858 | bit = max(bm->cur.bit - 1, 0); | ||
| 859 | clear_bit(bit, bm->cur.block->data); | ||
| 860 | } | 734 | } |
| 861 | 735 | ||
| 862 | static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn) | 736 | static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn) |
| 863 | { | 737 | { |
| 864 | void *addr; | 738 | void *addr; |
| 865 | unsigned int bit; | 739 | unsigned int bit; |
| 866 | int error, error2; | 740 | int error; |
| 867 | int v; | ||
| 868 | 741 | ||
| 869 | error = memory_bm_find_bit(bm, pfn, &addr, &bit); | 742 | error = memory_bm_find_bit(bm, pfn, &addr, &bit); |
| 870 | BUG_ON(error); | 743 | BUG_ON(error); |
| 871 | v = test_bit(bit, addr); | 744 | return test_bit(bit, addr); |
| 872 | |||
| 873 | error2 = memory_rtree_find_bit(bm, pfn, &addr, &bit); | ||
| 874 | BUG_ON(error2); | ||
| 875 | |||
| 876 | WARN_ON_ONCE(v != test_bit(bit, addr)); | ||
| 877 | |||
| 878 | return v; | ||
| 879 | } | 745 | } |
| 880 | 746 | ||
| 881 | static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn) | 747 | static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn) |
| 882 | { | 748 | { |
| 883 | void *addr; | 749 | void *addr; |
| 884 | unsigned int bit; | 750 | unsigned int bit; |
| 885 | int present; | ||
| 886 | |||
| 887 | present = !memory_bm_find_bit(bm, pfn, &addr, &bit); | ||
| 888 | |||
| 889 | WARN_ON_ONCE(present != !memory_rtree_find_bit(bm, pfn, &addr, &bit)); | ||
| 890 | 751 | ||
| 891 | return present; | 752 | return !memory_bm_find_bit(bm, pfn, &addr, &bit); |
| 892 | } | ||
| 893 | |||
| 894 | /** | ||
| 895 | * memory_bm_next_pfn - find the pfn that corresponds to the next set bit | ||
| 896 | * in the bitmap @bm. If the pfn cannot be found, BM_END_OF_MAP is | ||
| 897 | * returned. | ||
| 898 | * | ||
| 899 | * It is required to run memory_bm_position_reset() before the first call to | ||
| 900 | * this function. | ||
| 901 | */ | ||
| 902 | |||
| 903 | static unsigned long memory_bm_rtree_next_pfn(struct memory_bitmap *bm); | ||
| 904 | |||
| 905 | static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm) | ||
| 906 | { | ||
| 907 | unsigned long rtree_pfn; | ||
| 908 | struct bm_block *bb; | ||
| 909 | int bit; | ||
| 910 | |||
| 911 | rtree_pfn = memory_bm_rtree_next_pfn(bm); | ||
| 912 | |||
| 913 | bb = bm->cur.block; | ||
| 914 | do { | ||
| 915 | bit = bm->cur.bit; | ||
| 916 | bit = find_next_bit(bb->data, bm_block_bits(bb), bit); | ||
| 917 | if (bit < bm_block_bits(bb)) | ||
| 918 | goto Return_pfn; | ||
| 919 | |||
| 920 | bb = list_entry(bb->hook.next, struct bm_block, hook); | ||
| 921 | bm->cur.block = bb; | ||
| 922 | bm->cur.bit = 0; | ||
| 923 | } while (&bb->hook != &bm->blocks); | ||
| 924 | |||
| 925 | memory_bm_position_reset(bm); | ||
| 926 | WARN_ON_ONCE(rtree_pfn != BM_END_OF_MAP); | ||
| 927 | return BM_END_OF_MAP; | ||
| 928 | |||
| 929 | Return_pfn: | ||
| 930 | WARN_ON_ONCE(bb->start_pfn + bit != rtree_pfn); | ||
| 931 | bm->cur.bit = bit + 1; | ||
| 932 | return bb->start_pfn + bit; | ||
| 933 | } | 753 | } |
| 934 | 754 | ||
| 935 | /* | 755 | /* |
| @@ -967,14 +787,17 @@ static bool rtree_next_node(struct memory_bitmap *bm) | |||
| 967 | return false; | 787 | return false; |
| 968 | } | 788 | } |
| 969 | 789 | ||
| 970 | /* | 790 | /** |
| 971 | * memory_bm_rtree_next_pfn - Find the next set bit | 791 | * memory_bm_rtree_next_pfn - Find the next set bit in the bitmap @bm |
| 972 | * | 792 | * |
| 973 | * Starting from the last returned position this function searches | 793 | * Starting from the last returned position this function searches |
| 974 | * for the next set bit in the memory bitmap and returns its | 794 | * for the next set bit in the memory bitmap and returns its |
| 975 | * number. If no more bit is set BM_END_OF_MAP is returned. | 795 | * number. If no more bit is set BM_END_OF_MAP is returned. |
| 796 | * | ||
| 797 | * It is required to run memory_bm_position_reset() before the | ||
| 798 | * first call to this function. | ||
| 976 | */ | 799 | */ |
| 977 | static unsigned long memory_bm_rtree_next_pfn(struct memory_bitmap *bm) | 800 | static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm) |
| 978 | { | 801 | { |
| 979 | unsigned long bits, pfn, pages; | 802 | unsigned long bits, pfn, pages; |
| 980 | int bit; | 803 | int bit; |
| @@ -1216,11 +1039,7 @@ void free_basic_memory_bitmaps(void) | |||
| 1216 | unsigned int snapshot_additional_pages(struct zone *zone) | 1039 | unsigned int snapshot_additional_pages(struct zone *zone) |
| 1217 | { | 1040 | { |
| 1218 | unsigned int rtree, nodes; | 1041 | unsigned int rtree, nodes; |
| 1219 | unsigned int res; | ||
| 1220 | 1042 | ||
| 1221 | res = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK); | ||
| 1222 | res += DIV_ROUND_UP(res * sizeof(struct bm_block), | ||
| 1223 | LINKED_PAGE_DATA_SIZE); | ||
| 1224 | rtree = nodes = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK); | 1043 | rtree = nodes = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK); |
| 1225 | rtree += DIV_ROUND_UP(rtree * sizeof(struct rtree_node), | 1044 | rtree += DIV_ROUND_UP(rtree * sizeof(struct rtree_node), |
| 1226 | LINKED_PAGE_DATA_SIZE); | 1045 | LINKED_PAGE_DATA_SIZE); |
| @@ -1229,7 +1048,7 @@ unsigned int snapshot_additional_pages(struct zone *zone) | |||
| 1229 | rtree += nodes; | 1048 | rtree += nodes; |
| 1230 | } | 1049 | } |
| 1231 | 1050 | ||
| 1232 | return 2 * (res + rtree); | 1051 | return 2 * rtree; |
| 1233 | } | 1052 | } |
| 1234 | 1053 | ||
| 1235 | #ifdef CONFIG_HIGHMEM | 1054 | #ifdef CONFIG_HIGHMEM |
