aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/mm/init_64.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/mm/init_64.c')
-rw-r--r--arch/x86/mm/init_64.c60
1 files changed, 28 insertions, 32 deletions
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 8b72923f1d35..af11a2890235 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -800,17 +800,11 @@ int arch_add_memory(int nid, u64 start, u64 size, struct vmem_altmap *altmap,
800 800
801#define PAGE_INUSE 0xFD 801#define PAGE_INUSE 0xFD
802 802
803static void __meminit free_pagetable(struct page *page, int order, 803static void __meminit free_pagetable(struct page *page, int order)
804 struct vmem_altmap *altmap)
805{ 804{
806 unsigned long magic; 805 unsigned long magic;
807 unsigned int nr_pages = 1 << order; 806 unsigned int nr_pages = 1 << order;
808 807
809 if (altmap) {
810 vmem_altmap_free(altmap, nr_pages);
811 return;
812 }
813
814 /* bootmem page has reserved flag */ 808 /* bootmem page has reserved flag */
815 if (PageReserved(page)) { 809 if (PageReserved(page)) {
816 __ClearPageReserved(page); 810 __ClearPageReserved(page);
@@ -826,9 +820,17 @@ static void __meminit free_pagetable(struct page *page, int order,
826 free_pages((unsigned long)page_address(page), order); 820 free_pages((unsigned long)page_address(page), order);
827} 821}
828 822
829static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd, 823static void __meminit free_hugepage_table(struct page *page,
830 struct vmem_altmap *altmap) 824 struct vmem_altmap *altmap)
831{ 825{
826 if (altmap)
827 vmem_altmap_free(altmap, PMD_SIZE / PAGE_SIZE);
828 else
829 free_pagetable(page, get_order(PMD_SIZE));
830}
831
832static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd)
833{
832 pte_t *pte; 834 pte_t *pte;
833 int i; 835 int i;
834 836
@@ -839,14 +841,13 @@ static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd,
839 } 841 }
840 842
841 /* free a pte talbe */ 843 /* free a pte talbe */
842 free_pagetable(pmd_page(*pmd), 0, altmap); 844 free_pagetable(pmd_page(*pmd), 0);
843 spin_lock(&init_mm.page_table_lock); 845 spin_lock(&init_mm.page_table_lock);
844 pmd_clear(pmd); 846 pmd_clear(pmd);
845 spin_unlock(&init_mm.page_table_lock); 847 spin_unlock(&init_mm.page_table_lock);
846} 848}
847 849
848static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud, 850static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud)
849 struct vmem_altmap *altmap)
850{ 851{
851 pmd_t *pmd; 852 pmd_t *pmd;
852 int i; 853 int i;
@@ -858,14 +859,13 @@ static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud,
858 } 859 }
859 860
860 /* free a pmd talbe */ 861 /* free a pmd talbe */
861 free_pagetable(pud_page(*pud), 0, altmap); 862 free_pagetable(pud_page(*pud), 0);
862 spin_lock(&init_mm.page_table_lock); 863 spin_lock(&init_mm.page_table_lock);
863 pud_clear(pud); 864 pud_clear(pud);
864 spin_unlock(&init_mm.page_table_lock); 865 spin_unlock(&init_mm.page_table_lock);
865} 866}
866 867
867static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d, 868static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d)
868 struct vmem_altmap *altmap)
869{ 869{
870 pud_t *pud; 870 pud_t *pud;
871 int i; 871 int i;
@@ -877,7 +877,7 @@ static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d,
877 } 877 }
878 878
879 /* free a pud talbe */ 879 /* free a pud talbe */
880 free_pagetable(p4d_page(*p4d), 0, altmap); 880 free_pagetable(p4d_page(*p4d), 0);
881 spin_lock(&init_mm.page_table_lock); 881 spin_lock(&init_mm.page_table_lock);
882 p4d_clear(p4d); 882 p4d_clear(p4d);
883 spin_unlock(&init_mm.page_table_lock); 883 spin_unlock(&init_mm.page_table_lock);
@@ -885,7 +885,7 @@ static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d,
885 885
886static void __meminit 886static void __meminit
887remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end, 887remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end,
888 struct vmem_altmap *altmap, bool direct) 888 bool direct)
889{ 889{
890 unsigned long next, pages = 0; 890 unsigned long next, pages = 0;
891 pte_t *pte; 891 pte_t *pte;
@@ -916,7 +916,7 @@ remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end,
916 * freed when offlining, or simplely not in use. 916 * freed when offlining, or simplely not in use.
917 */ 917 */
918 if (!direct) 918 if (!direct)
919 free_pagetable(pte_page(*pte), 0, altmap); 919 free_pagetable(pte_page(*pte), 0);
920 920
921 spin_lock(&init_mm.page_table_lock); 921 spin_lock(&init_mm.page_table_lock);
922 pte_clear(&init_mm, addr, pte); 922 pte_clear(&init_mm, addr, pte);
@@ -939,7 +939,7 @@ remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end,
939 939
940 page_addr = page_address(pte_page(*pte)); 940 page_addr = page_address(pte_page(*pte));
941 if (!memchr_inv(page_addr, PAGE_INUSE, PAGE_SIZE)) { 941 if (!memchr_inv(page_addr, PAGE_INUSE, PAGE_SIZE)) {
942 free_pagetable(pte_page(*pte), 0, altmap); 942 free_pagetable(pte_page(*pte), 0);
943 943
944 spin_lock(&init_mm.page_table_lock); 944 spin_lock(&init_mm.page_table_lock);
945 pte_clear(&init_mm, addr, pte); 945 pte_clear(&init_mm, addr, pte);
@@ -974,9 +974,8 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
974 if (IS_ALIGNED(addr, PMD_SIZE) && 974 if (IS_ALIGNED(addr, PMD_SIZE) &&
975 IS_ALIGNED(next, PMD_SIZE)) { 975 IS_ALIGNED(next, PMD_SIZE)) {
976 if (!direct) 976 if (!direct)
977 free_pagetable(pmd_page(*pmd), 977 free_hugepage_table(pmd_page(*pmd),
978 get_order(PMD_SIZE), 978 altmap);
979 altmap);
980 979
981 spin_lock(&init_mm.page_table_lock); 980 spin_lock(&init_mm.page_table_lock);
982 pmd_clear(pmd); 981 pmd_clear(pmd);
@@ -989,9 +988,8 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
989 page_addr = page_address(pmd_page(*pmd)); 988 page_addr = page_address(pmd_page(*pmd));
990 if (!memchr_inv(page_addr, PAGE_INUSE, 989 if (!memchr_inv(page_addr, PAGE_INUSE,
991 PMD_SIZE)) { 990 PMD_SIZE)) {
992 free_pagetable(pmd_page(*pmd), 991 free_hugepage_table(pmd_page(*pmd),
993 get_order(PMD_SIZE), 992 altmap);
994 altmap);
995 993
996 spin_lock(&init_mm.page_table_lock); 994 spin_lock(&init_mm.page_table_lock);
997 pmd_clear(pmd); 995 pmd_clear(pmd);
@@ -1003,8 +1001,8 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
1003 } 1001 }
1004 1002
1005 pte_base = (pte_t *)pmd_page_vaddr(*pmd); 1003 pte_base = (pte_t *)pmd_page_vaddr(*pmd);
1006 remove_pte_table(pte_base, addr, next, altmap, direct); 1004 remove_pte_table(pte_base, addr, next, direct);
1007 free_pte_table(pte_base, pmd, altmap); 1005 free_pte_table(pte_base, pmd);
1008 } 1006 }
1009 1007
1010 /* Call free_pmd_table() in remove_pud_table(). */ 1008 /* Call free_pmd_table() in remove_pud_table(). */
@@ -1033,8 +1031,7 @@ remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,
1033 IS_ALIGNED(next, PUD_SIZE)) { 1031 IS_ALIGNED(next, PUD_SIZE)) {
1034 if (!direct) 1032 if (!direct)
1035 free_pagetable(pud_page(*pud), 1033 free_pagetable(pud_page(*pud),
1036 get_order(PUD_SIZE), 1034 get_order(PUD_SIZE));
1037 altmap);
1038 1035
1039 spin_lock(&init_mm.page_table_lock); 1036 spin_lock(&init_mm.page_table_lock);
1040 pud_clear(pud); 1037 pud_clear(pud);
@@ -1048,8 +1045,7 @@ remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,
1048 if (!memchr_inv(page_addr, PAGE_INUSE, 1045 if (!memchr_inv(page_addr, PAGE_INUSE,
1049 PUD_SIZE)) { 1046 PUD_SIZE)) {
1050 free_pagetable(pud_page(*pud), 1047 free_pagetable(pud_page(*pud),
1051 get_order(PUD_SIZE), 1048 get_order(PUD_SIZE));
1052 altmap);
1053 1049
1054 spin_lock(&init_mm.page_table_lock); 1050 spin_lock(&init_mm.page_table_lock);
1055 pud_clear(pud); 1051 pud_clear(pud);
@@ -1062,7 +1058,7 @@ remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,
1062 1058
1063 pmd_base = pmd_offset(pud, 0); 1059 pmd_base = pmd_offset(pud, 0);
1064 remove_pmd_table(pmd_base, addr, next, direct, altmap); 1060 remove_pmd_table(pmd_base, addr, next, direct, altmap);
1065 free_pmd_table(pmd_base, pud, altmap); 1061 free_pmd_table(pmd_base, pud);
1066 } 1062 }
1067 1063
1068 if (direct) 1064 if (direct)
@@ -1094,7 +1090,7 @@ remove_p4d_table(p4d_t *p4d_start, unsigned long addr, unsigned long end,
1094 * to adapt for boot-time switching between 4 and 5 level page tables. 1090 * to adapt for boot-time switching between 4 and 5 level page tables.
1095 */ 1091 */
1096 if (CONFIG_PGTABLE_LEVELS == 5) 1092 if (CONFIG_PGTABLE_LEVELS == 5)
1097 free_pud_table(pud_base, p4d, altmap); 1093 free_pud_table(pud_base, p4d);
1098 } 1094 }
1099 1095
1100 if (direct) 1096 if (direct)
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-30 22:07:48 -0400