aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/x86/mm/init.c328
-rw-r--r--arch/x86/mm/init_32.c308
-rw-r--r--arch/x86/mm/init_64.c314
3 files changed, 342 insertions, 608 deletions
diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c
index cc7fe660f33d..3a21b136da24 100644
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -2,10 +2,338 @@
2#include <linux/swap.h> 2#include <linux/swap.h>
3 3
4#include <asm/cacheflush.h> 4#include <asm/cacheflush.h>
5#include <asm/e820.h>
5#include <asm/page.h> 6#include <asm/page.h>
6#include <asm/page_types.h> 7#include <asm/page_types.h>
7#include <asm/sections.h> 8#include <asm/sections.h>
8#include <asm/system.h> 9#include <asm/system.h>
10#include <asm/tlbflush.h>
11
12#ifdef CONFIG_X86_32
13extern void __init early_ioremap_page_table_range_init(void);
14extern void __init kernel_physical_mapping_init(unsigned long start_pfn,
15 unsigned long end_pfn,
16 int use_pse);
17#endif
18
19#ifdef CONFIG_X86_64
20extern unsigned long __meminit
21kernel_physical_mapping_init(unsigned long start,
22 unsigned long end,
23 unsigned long page_size_mask);
24#endif
25
26unsigned long __initdata table_start;
27unsigned long __meminitdata table_end;
28unsigned long __meminitdata table_top;
29
30int after_bootmem;
31
32int direct_gbpages
33#ifdef CONFIG_DIRECT_GBPAGES
34 = 1
35#endif
36;
37
38static void __init find_early_table_space(unsigned long end, int use_pse,
39 int use_gbpages)
40{
41 unsigned long puds, pmds, ptes, tables, start;
42
43 puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
44 tables = roundup(puds * sizeof(pud_t), PAGE_SIZE);
45
46 if (use_gbpages) {
47 unsigned long extra;
48
49 extra = end - ((end>>PUD_SHIFT) << PUD_SHIFT);
50 pmds = (extra + PMD_SIZE - 1) >> PMD_SHIFT;
51 } else
52 pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
53
54 tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE);
55
56 if (use_pse) {
57 unsigned long extra;
58
59 extra = end - ((end>>PMD_SHIFT) << PMD_SHIFT);
60#ifdef CONFIG_X86_32
61 extra += PMD_SIZE;
62#endif
63 ptes = (extra + PAGE_SIZE - 1) >> PAGE_SHIFT;
64 } else
65 ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT;
66
67 tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE);
68
69#ifdef CONFIG_X86_32
70 /* for fixmap */
71 tables += roundup(__end_of_fixed_addresses * sizeof(pte_t), PAGE_SIZE);
72#endif
73
74 /*
75 * RED-PEN putting page tables only on node 0 could
76 * cause a hotspot and fill up ZONE_DMA. The page tables
77 * need roughly 0.5KB per GB.
78 */
79#ifdef CONFIG_X86_32
80 start = 0x7000;
81 table_start = find_e820_area(start, max_pfn_mapped<<PAGE_SHIFT,
82 tables, PAGE_SIZE);
83#else /* CONFIG_X86_64 */
84 start = 0x8000;
85 table_start = find_e820_area(start, end, tables, PAGE_SIZE);
86#endif
87 if (table_start == -1UL)
88 panic("Cannot find space for the kernel page tables");
89
90 table_start >>= PAGE_SHIFT;
91 table_end = table_start;
92 table_top = table_start + (tables >> PAGE_SHIFT);
93
94 printk(KERN_DEBUG "kernel direct mapping tables up to %lx @ %lx-%lx\n",
95 end, table_start << PAGE_SHIFT, table_top << PAGE_SHIFT);
96}
97
98struct map_range {
99 unsigned long start;
100 unsigned long end;
101 unsigned page_size_mask;
102};
103
104#ifdef CONFIG_X86_32
105#define NR_RANGE_MR 3
106#else /* CONFIG_X86_64 */
107#define NR_RANGE_MR 5
108#endif
109
110static int save_mr(struct map_range *mr, int nr_range,
111 unsigned long start_pfn, unsigned long end_pfn,
112 unsigned long page_size_mask)
113{
114 if (start_pfn < end_pfn) {
115 if (nr_range >= NR_RANGE_MR)
116 panic("run out of range for init_memory_mapping\n");
117 mr[nr_range].start = start_pfn<<PAGE_SHIFT;
118 mr[nr_range].end = end_pfn<<PAGE_SHIFT;
119 mr[nr_range].page_size_mask = page_size_mask;
120 nr_range++;
121 }
122
123 return nr_range;
124}
125
126#ifdef CONFIG_X86_64
127static void __init init_gbpages(void)
128{
129 if (direct_gbpages && cpu_has_gbpages)
130 printk(KERN_INFO "Using GB pages for direct mapping\n");
131 else
132 direct_gbpages = 0;
133}
134#else
135static inline void init_gbpages(void)
136{
137}
138#endif
139
140/*
141 * Setup the direct mapping of the physical memory at PAGE_OFFSET.
142 * This runs before bootmem is initialized and gets pages directly from
143 * the physical memory. To access them they are temporarily mapped.
144 */
145unsigned long __init_refok init_memory_mapping(unsigned long start,
146 unsigned long end)
147{
148 unsigned long page_size_mask = 0;
149 unsigned long start_pfn, end_pfn;
150 unsigned long pos;
151 unsigned long ret;
152
153 struct map_range mr[NR_RANGE_MR];
154 int nr_range, i;
155 int use_pse, use_gbpages;
156
157 printk(KERN_INFO "init_memory_mapping: %016lx-%016lx\n", start, end);
158
159 if (!after_bootmem)
160 init_gbpages();
161
162#ifdef CONFIG_DEBUG_PAGEALLOC
163 /*
164 * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
165 * This will simplify cpa(), which otherwise needs to support splitting
166 * large pages into small in interrupt context, etc.
167 */
168 use_pse = use_gbpages = 0;
169#else
170 use_pse = cpu_has_pse;
171 use_gbpages = direct_gbpages;
172#endif
173
174#ifdef CONFIG_X86_32
175#ifdef CONFIG_X86_PAE
176 set_nx();
177 if (nx_enabled)
178 printk(KERN_INFO "NX (Execute Disable) protection: active\n");
179#endif
180
181 /* Enable PSE if available */
182 if (cpu_has_pse)
183 set_in_cr4(X86_CR4_PSE);
184
185 /* Enable PGE if available */
186 if (cpu_has_pge) {
187 set_in_cr4(X86_CR4_PGE);
188 __supported_pte_mask |= _PAGE_GLOBAL;
189 }
190#endif
191
192 if (use_gbpages)
193 page_size_mask |= 1 << PG_LEVEL_1G;
194 if (use_pse)
195 page_size_mask |= 1 << PG_LEVEL_2M;
196
197 memset(mr, 0, sizeof(mr));
198 nr_range = 0;
199
200 /* head if not big page alignment ? */
201 start_pfn = start >> PAGE_SHIFT;
202 pos = start_pfn << PAGE_SHIFT;
203#ifdef CONFIG_X86_32
204 /*
205 * Don't use a large page for the first 2/4MB of memory
206 * because there are often fixed size MTRRs in there
207 * and overlapping MTRRs into large pages can cause
208 * slowdowns.
209 */
210 if (pos == 0)
211 end_pfn = 1<<(PMD_SHIFT - PAGE_SHIFT);
212 else
213 end_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
214 << (PMD_SHIFT - PAGE_SHIFT);
215#else /* CONFIG_X86_64 */
216 end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT)
217 << (PMD_SHIFT - PAGE_SHIFT);
218#endif
219 if (end_pfn > (end >> PAGE_SHIFT))
220 end_pfn = end >> PAGE_SHIFT;
221 if (start_pfn < end_pfn) {
222 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
223 pos = end_pfn << PAGE_SHIFT;
224 }
225
226 /* big page (2M) range */
227 start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
228 << (PMD_SHIFT - PAGE_SHIFT);
229#ifdef CONFIG_X86_32
230 end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
231#else /* CONFIG_X86_64 */
232 end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
233 << (PUD_SHIFT - PAGE_SHIFT);
234 if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)))
235 end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT));
236#endif
237
238 if (start_pfn < end_pfn) {
239 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
240 page_size_mask & (1<<PG_LEVEL_2M));
241 pos = end_pfn << PAGE_SHIFT;
242 }
243
244#ifdef CONFIG_X86_64
245 /* big page (1G) range */
246 start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
247 << (PUD_SHIFT - PAGE_SHIFT);
248 end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT);
249 if (start_pfn < end_pfn) {
250 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
251 page_size_mask &
252 ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G)));
253 pos = end_pfn << PAGE_SHIFT;
254 }
255
256 /* tail is not big page (1G) alignment */
257 start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
258 << (PMD_SHIFT - PAGE_SHIFT);
259 end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
260 if (start_pfn < end_pfn) {
261 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
262 page_size_mask & (1<<PG_LEVEL_2M));
263 pos = end_pfn << PAGE_SHIFT;
264 }
265#endif
266
267 /* tail is not big page (2M) alignment */
268 start_pfn = pos>>PAGE_SHIFT;
269 end_pfn = end>>PAGE_SHIFT;
270 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
271
272 /* try to merge same page size and continuous */
273 for (i = 0; nr_range > 1 && i < nr_range - 1; i++) {
274 unsigned long old_start;
275 if (mr[i].end != mr[i+1].start ||
276 mr[i].page_size_mask != mr[i+1].page_size_mask)
277 continue;
278 /* move it */
279 old_start = mr[i].start;
280 memmove(&mr[i], &mr[i+1],
281 (nr_range - 1 - i) * sizeof(struct map_range));
282 mr[i--].start = old_start;
283 nr_range--;
284 }
285
286 for (i = 0; i < nr_range; i++)
287 printk(KERN_DEBUG " %010lx - %010lx page %s\n",
288 mr[i].start, mr[i].end,
289 (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":(
290 (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k"));
291
292 /*
293 * Find space for the kernel direct mapping tables.
294 *
295 * Later we should allocate these tables in the local node of the
296 * memory mapped. Unfortunately this is done currently before the
297 * nodes are discovered.
298 */
299 if (!after_bootmem)
300 find_early_table_space(end, use_pse, use_gbpages);
301
302#ifdef CONFIG_X86_32
303 for (i = 0; i < nr_range; i++)
304 kernel_physical_mapping_init(
305 mr[i].start >> PAGE_SHIFT,
306 mr[i].end >> PAGE_SHIFT,
307 mr[i].page_size_mask == (1<<PG_LEVEL_2M));
308 ret = end;
309#else /* CONFIG_X86_64 */
310 for (i = 0; i < nr_range; i++)
311 ret = kernel_physical_mapping_init(mr[i].start, mr[i].end,
312 mr[i].page_size_mask);
313#endif
314
315#ifdef CONFIG_X86_32
316 early_ioremap_page_table_range_init();
317
318 load_cr3(swapper_pg_dir);
319#endif
320
321#ifdef CONFIG_X86_64
322 if (!after_bootmem)
323 mmu_cr4_features = read_cr4();
324#endif
325 __flush_tlb_all();
326
327 if (!after_bootmem && table_end > table_start)
328 reserve_early(table_start << PAGE_SHIFT,
329 table_end << PAGE_SHIFT, "PGTABLE");
330
331 if (!after_bootmem)
332 early_memtest(start, end);
333
334 return ret >> PAGE_SHIFT;
335}
336
9 337
10/* 338/*
11 * devmem_is_allowed() checks to see if /dev/mem access to a certain address 339 * devmem_is_allowed() checks to see if /dev/mem access to a certain address
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index cd3c24b490a1..187522a0c66b 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -59,13 +59,9 @@ unsigned long highstart_pfn, highend_pfn;
59static noinline int do_test_wp_bit(void); 59static noinline int do_test_wp_bit(void);
60 60
61 61
62static unsigned long __initdata table_start; 62extern unsigned long __initdata table_start;
63static unsigned long __meminitdata table_end; 63extern unsigned long __meminitdata table_end;
64static unsigned long __meminitdata table_top; 64extern unsigned long __meminitdata table_top;
65
66int after_bootmem;
67
68int direct_gbpages;
69 65
70static __init void *alloc_low_page(void) 66static __init void *alloc_low_page(void)
71{ 67{
@@ -227,9 +223,9 @@ static inline int is_kernel_text(unsigned long addr)
227 * of max_low_pfn pages, by creating page tables starting from address 223 * of max_low_pfn pages, by creating page tables starting from address
228 * PAGE_OFFSET: 224 * PAGE_OFFSET:
229 */ 225 */
230static void __init kernel_physical_mapping_init(unsigned long start_pfn, 226void __init kernel_physical_mapping_init(unsigned long start_pfn,
231 unsigned long end_pfn, 227 unsigned long end_pfn,
232 int use_pse) 228 int use_pse)
233{ 229{
234 pgd_t *pgd_base = swapper_pg_dir; 230 pgd_t *pgd_base = swapper_pg_dir;
235 int pgd_idx, pmd_idx, pte_ofs; 231 int pgd_idx, pmd_idx, pte_ofs;
@@ -509,7 +505,7 @@ void __init native_pagetable_setup_done(pgd_t *base)
509 * be partially populated, and so it avoids stomping on any existing 505 * be partially populated, and so it avoids stomping on any existing
510 * mappings. 506 * mappings.
511 */ 507 */
512static void __init early_ioremap_page_table_range_init(void) 508void __init early_ioremap_page_table_range_init(void)
513{ 509{
514 pgd_t *pgd_base = swapper_pg_dir; 510 pgd_t *pgd_base = swapper_pg_dir;
515 unsigned long vaddr, end; 511 unsigned long vaddr, end;
@@ -834,296 +830,6 @@ void __init setup_bootmem_allocator(void)
834 after_bootmem = 1; 830 after_bootmem = 1;
835} 831}
836 832
837static void __init find_early_table_space(unsigned long end, int use_pse,
838 int use_gbpages)
839{
840 unsigned long puds, pmds, ptes, tables, start;
841
842 puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
843 tables = roundup(puds * sizeof(pud_t), PAGE_SIZE);
844
845 if (use_gbpages) {
846 unsigned long extra;
847
848 extra = end - ((end>>PUD_SHIFT) << PUD_SHIFT);
849 pmds = (extra + PMD_SIZE - 1) >> PMD_SHIFT;
850 } else
851 pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
852
853 tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE);
854
855 if (use_pse) {
856 unsigned long extra;
857
858 extra = end - ((end>>PMD_SHIFT) << PMD_SHIFT);
859#ifdef CONFIG_X86_32
860 extra += PMD_SIZE;
861#endif
862 ptes = (extra + PAGE_SIZE - 1) >> PAGE_SHIFT;
863 } else
864 ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT;
865
866 tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE);
867
868#ifdef CONFIG_X86_32
869 /* for fixmap */
870 tables += roundup(__end_of_fixed_addresses * sizeof(pte_t), PAGE_SIZE);
871#endif
872
873 /*
874 * RED-PEN putting page tables only on node 0 could
875 * cause a hotspot and fill up ZONE_DMA. The page tables
876 * need roughly 0.5KB per GB.
877 */
878#ifdef CONFIG_X86_32
879 start = 0x7000;
880 table_start = find_e820_area(start, max_pfn_mapped<<PAGE_SHIFT,
881 tables, PAGE_SIZE);
882#else /* CONFIG_X86_64 */
883 start = 0x8000;
884 table_start = find_e820_area(start, end, tables, PAGE_SIZE);
885#endif
886 if (table_start == -1UL)
887 panic("Cannot find space for the kernel page tables");
888
889 table_start >>= PAGE_SHIFT;
890 table_end = table_start;
891 table_top = table_start + (tables >> PAGE_SHIFT);
892
893 printk(KERN_DEBUG "kernel direct mapping tables up to %lx @ %lx-%lx\n",
894 end, table_start << PAGE_SHIFT, table_top << PAGE_SHIFT);
895}
896
897struct map_range {
898 unsigned long start;
899 unsigned long end;
900 unsigned page_size_mask;
901};
902
903#ifdef CONFIG_X86_32
904#define NR_RANGE_MR 3
905#else /* CONFIG_X86_64 */
906#define NR_RANGE_MR 5
907#endif
908
909static int save_mr(struct map_range *mr, int nr_range,
910 unsigned long start_pfn, unsigned long end_pfn,
911 unsigned long page_size_mask)
912{
913 if (start_pfn < end_pfn) {
914 if (nr_range >= NR_RANGE_MR)
915 panic("run out of range for init_memory_mapping\n");
916 mr[nr_range].start = start_pfn<<PAGE_SHIFT;
917 mr[nr_range].end = end_pfn<<PAGE_SHIFT;
918 mr[nr_range].page_size_mask = page_size_mask;
919 nr_range++;
920 }
921
922 return nr_range;
923}
924
925static inline void init_gbpages(void)
926{
927}
928
929/*
930 * Setup the direct mapping of the physical memory at PAGE_OFFSET.
931 * This runs before bootmem is initialized and gets pages directly from
932 * the physical memory. To access them they are temporarily mapped.
933 */
934unsigned long __init_refok init_memory_mapping(unsigned long start,
935 unsigned long end)
936{
937 unsigned long page_size_mask = 0;
938 unsigned long start_pfn, end_pfn;
939 unsigned long pos;
940 unsigned long ret;
941
942 struct map_range mr[NR_RANGE_MR];
943 int nr_range, i;
944 int use_pse, use_gbpages;
945
946 printk(KERN_INFO "init_memory_mapping: %016lx-%016lx\n", start, end);
947
948 if (!after_bootmem)
949 init_gbpages();
950
951#ifdef CONFIG_DEBUG_PAGEALLOC
952 /*
953 * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
954 * This will simplify cpa(), which otherwise needs to support splitting
955 * large pages into small in interrupt context, etc.
956 */
957 use_pse = use_gbpages = 0;
958#else
959 use_pse = cpu_has_pse;
960 use_gbpages = direct_gbpages;
961#endif
962
963#ifdef CONFIG_X86_32
964#ifdef CONFIG_X86_PAE
965 set_nx();
966 if (nx_enabled)
967 printk(KERN_INFO "NX (Execute Disable) protection: active\n");
968#endif
969
970 /* Enable PSE if available */
971 if (cpu_has_pse)
972 set_in_cr4(X86_CR4_PSE);
973
974 /* Enable PGE if available */
975 if (cpu_has_pge) {
976 set_in_cr4(X86_CR4_PGE);
977 __supported_pte_mask |= _PAGE_GLOBAL;
978 }
979#endif
980
981 if (use_gbpages)
982 page_size_mask |= 1 << PG_LEVEL_1G;
983 if (use_pse)
984 page_size_mask |= 1 << PG_LEVEL_2M;
985
986 memset(mr, 0, sizeof(mr));
987 nr_range = 0;
988
989 /* head if not big page alignment ? */
990 start_pfn = start >> PAGE_SHIFT;
991 pos = start_pfn << PAGE_SHIFT;
992#ifdef CONFIG_X86_32
993 /*
994 * Don't use a large page for the first 2/4MB of memory
995 * because there are often fixed size MTRRs in there
996 * and overlapping MTRRs into large pages can cause
997 * slowdowns.
998 */
999 if (pos == 0)
1000 end_pfn = 1<<(PMD_SHIFT - PAGE_SHIFT);
1001 else
1002 end_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
1003 << (PMD_SHIFT - PAGE_SHIFT);
1004#else /* CONFIG_X86_64 */
1005 end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT)
1006 << (PMD_SHIFT - PAGE_SHIFT);
1007#endif
1008 if (end_pfn > (end >> PAGE_SHIFT))
1009 end_pfn = end >> PAGE_SHIFT;
1010 if (start_pfn < end_pfn) {
1011 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
1012 pos = end_pfn << PAGE_SHIFT;
1013 }
1014
1015 /* big page (2M) range */
1016 start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
1017 << (PMD_SHIFT - PAGE_SHIFT);
1018#ifdef CONFIG_X86_32
1019 end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
1020#else /* CONFIG_X86_64 */
1021 end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
1022 << (PUD_SHIFT - PAGE_SHIFT);
1023 if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)))
1024 end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT));
1025#endif
1026
1027 if (start_pfn < end_pfn) {
1028 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
1029 page_size_mask & (1<<PG_LEVEL_2M));
1030 pos = end_pfn << PAGE_SHIFT;
1031 }
1032
1033#ifdef CONFIG_X86_64
1034 /* big page (1G) range */
1035 start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
1036 << (PUD_SHIFT - PAGE_SHIFT);
1037 end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT);
1038 if (start_pfn < end_pfn) {
1039 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
1040 page_size_mask &
1041 ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G)));
1042 pos = end_pfn << PAGE_SHIFT;
1043 }
1044
1045 /* tail is not big page (1G) alignment */
1046 start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
1047 << (PMD_SHIFT - PAGE_SHIFT);
1048 end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
1049 if (start_pfn < end_pfn) {
1050 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
1051 page_size_mask & (1<<PG_LEVEL_2M));
1052 pos = end_pfn << PAGE_SHIFT;
1053 }
1054#endif
1055
1056 /* tail is not big page (2M) alignment */
1057 start_pfn = pos>>PAGE_SHIFT;
1058 end_pfn = end>>PAGE_SHIFT;
1059 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
1060
1061 /* try to merge same page size and continuous */
1062 for (i = 0; nr_range > 1 && i < nr_range - 1; i++) {
1063 unsigned long old_start;
1064 if (mr[i].end != mr[i+1].start ||
1065 mr[i].page_size_mask != mr[i+1].page_size_mask)
1066 continue;
1067 /* move it */
1068 old_start = mr[i].start;
1069 memmove(&mr[i], &mr[i+1],
1070 (nr_range - 1 - i) * sizeof(struct map_range));
1071 mr[i--].start = old_start;
1072 nr_range--;
1073 }
1074
1075 for (i = 0; i < nr_range; i++)
1076 printk(KERN_DEBUG " %010lx - %010lx page %s\n",
1077 mr[i].start, mr[i].end,
1078 (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":(
1079 (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k"));
1080
1081 /*
1082 * Find space for the kernel direct mapping tables.
1083 *
1084 * Later we should allocate these tables in the local node of the
1085 * memory mapped. Unfortunately this is done currently before the
1086 * nodes are discovered.
1087 */
1088 if (!after_bootmem)
1089 find_early_table_space(end, use_pse, use_gbpages);
1090
1091#ifdef CONFIG_X86_32
1092 for (i = 0; i < nr_range; i++)
1093 kernel_physical_mapping_init(
1094 mr[i].start >> PAGE_SHIFT,
1095 mr[i].end >> PAGE_SHIFT,
1096 mr[i].page_size_mask == (1<<PG_LEVEL_2M));
1097 ret = end;
1098#else /* CONFIG_X86_64 */
1099 for (i = 0; i < nr_range; i++)
1100 ret = kernel_physical_mapping_init(mr[i].start, mr[i].end,
1101 mr[i].page_size_mask);
1102#endif
1103
1104#ifdef CONFIG_X86_32
1105 early_ioremap_page_table_range_init();
1106
1107 load_cr3(swapper_pg_dir);
1108#endif
1109
1110#ifdef CONFIG_X86_64
1111 if (!after_bootmem)
1112 mmu_cr4_features = read_cr4();
1113#endif
1114 __flush_tlb_all();
1115
1116 if (!after_bootmem && table_end > table_start)
1117 reserve_early(table_start << PAGE_SHIFT,
1118 table_end << PAGE_SHIFT, "PGTABLE");
1119
1120 if (!after_bootmem)
1121 early_memtest(start, end);
1122
1123 return ret >> PAGE_SHIFT;
1124}
1125
1126
1127/* 833/*
1128 * paging_init() sets up the page tables - note that the first 8MB are 834 * paging_init() sets up the page tables - note that the first 8MB are
1129 * already mapped by head.S. 835 * already mapped by head.S.
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index d101990e4635..a32fe0756088 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -61,12 +61,6 @@ static unsigned long dma_reserve __initdata;
61 61
62DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); 62DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
63 63
64int direct_gbpages
65#ifdef CONFIG_DIRECT_GBPAGES
66 = 1
67#endif
68;
69
70static int __init parse_direct_gbpages_off(char *arg) 64static int __init parse_direct_gbpages_off(char *arg)
71{ 65{
72 direct_gbpages = 0; 66 direct_gbpages = 0;
@@ -87,8 +81,6 @@ early_param("gbpages", parse_direct_gbpages_on);
87 * around without checking the pgd every time. 81 * around without checking the pgd every time.
88 */ 82 */
89 83
90int after_bootmem;
91
92pteval_t __supported_pte_mask __read_mostly = ~_PAGE_IOMAP; 84pteval_t __supported_pte_mask __read_mostly = ~_PAGE_IOMAP;
93EXPORT_SYMBOL_GPL(__supported_pte_mask); 85EXPORT_SYMBOL_GPL(__supported_pte_mask);
94 86
@@ -291,9 +283,9 @@ void __init cleanup_highmap(void)
291 } 283 }
292} 284}
293 285
294static unsigned long __initdata table_start; 286extern unsigned long __initdata table_start;
295static unsigned long __meminitdata table_end; 287extern unsigned long __meminitdata table_end;
296static unsigned long __meminitdata table_top; 288extern unsigned long __meminitdata table_top;
297 289
298static __ref void *alloc_low_page(unsigned long *phys) 290static __ref void *alloc_low_page(unsigned long *phys)
299{ 291{
@@ -547,77 +539,10 @@ phys_pud_update(pgd_t *pgd, unsigned long addr, unsigned long end,
547 return phys_pud_init(pud, addr, end, page_size_mask); 539 return phys_pud_init(pud, addr, end, page_size_mask);
548} 540}
549 541
550static void __init find_early_table_space(unsigned long end, int use_pse, 542unsigned long __meminit
551 int use_gbpages) 543kernel_physical_mapping_init(unsigned long start,
552{ 544 unsigned long end,
553 unsigned long puds, pmds, ptes, tables, start; 545 unsigned long page_size_mask)
554
555 puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
556 tables = roundup(puds * sizeof(pud_t), PAGE_SIZE);
557
558 if (use_gbpages) {
559 unsigned long extra;
560
561 extra = end - ((end>>PUD_SHIFT) << PUD_SHIFT);
562 pmds = (extra + PMD_SIZE - 1) >> PMD_SHIFT;
563 } else
564 pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
565
566 tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE);
567
568 if (use_pse) {
569 unsigned long extra;
570
571 extra = end - ((end>>PMD_SHIFT) << PMD_SHIFT);
572#ifdef CONFIG_X86_32
573 extra += PMD_SIZE;
574#endif
575 ptes = (extra + PAGE_SIZE - 1) >> PAGE_SHIFT;
576 } else
577 ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT;
578
579 tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE);
580
581#ifdef CONFIG_X86_32
582 /* for fixmap */
583 tables += roundup(__end_of_fixed_addresses * sizeof(pte_t), PAGE_SIZE);
584#endif
585
586 /*
587 * RED-PEN putting page tables only on node 0 could
588 * cause a hotspot and fill up ZONE_DMA. The page tables
589 * need roughly 0.5KB per GB.
590 */
591#ifdef CONFIG_X86_32
592 start = 0x7000;
593 table_start = find_e820_area(start, max_pfn_mapped<<PAGE_SHIFT,
594 tables, PAGE_SIZE);
595#else /* CONFIG_X86_64 */
596 start = 0x8000;
597 table_start = find_e820_area(start, end, tables, PAGE_SIZE);
598#endif
599 if (table_start == -1UL)
600 panic("Cannot find space for the kernel page tables");
601
602 table_start >>= PAGE_SHIFT;
603 table_end = table_start;
604 table_top = table_start + (tables >> PAGE_SHIFT);
605
606 printk(KERN_DEBUG "kernel direct mapping tables up to %lx @ %lx-%lx\n",
607 end, table_start << PAGE_SHIFT, table_top << PAGE_SHIFT);
608}
609
610static void __init init_gbpages(void)
611{
612 if (direct_gbpages && cpu_has_gbpages)
613 printk(KERN_INFO "Using GB pages for direct mapping\n");
614 else
615 direct_gbpages = 0;
616}
617
618static unsigned long __meminit kernel_physical_mapping_init(unsigned long start,
619 unsigned long end,
620 unsigned long page_size_mask)
621{ 546{
622 547
623 unsigned long next, last_map_addr = end; 548 unsigned long next, last_map_addr = end;
@@ -654,231 +579,6 @@ static unsigned long __meminit kernel_physical_mapping_init(unsigned long start,
654 return last_map_addr; 579 return last_map_addr;
655} 580}
656 581
657struct map_range {
658 unsigned long start;
659 unsigned long end;
660 unsigned page_size_mask;
661};
662
663#ifdef CONFIG_X86_32
664#define NR_RANGE_MR 3
665#else /* CONFIG_X86_64 */
666#define NR_RANGE_MR 5
667#endif
668
669static int save_mr(struct map_range *mr, int nr_range,
670 unsigned long start_pfn, unsigned long end_pfn,
671 unsigned long page_size_mask)
672{
673 if (start_pfn < end_pfn) {
674 if (nr_range >= NR_RANGE_MR)
675 panic("run out of range for init_memory_mapping\n");
676 mr[nr_range].start = start_pfn<<PAGE_SHIFT;
677 mr[nr_range].end = end_pfn<<PAGE_SHIFT;
678 mr[nr_range].page_size_mask = page_size_mask;
679 nr_range++;
680 }
681
682 return nr_range;
683}
684
685/*
686 * Setup the direct mapping of the physical memory at PAGE_OFFSET.
687 * This runs before bootmem is initialized and gets pages directly from
688 * the physical memory. To access them they are temporarily mapped.
689 */
690unsigned long __init_refok init_memory_mapping(unsigned long start,
691 unsigned long end)
692{
693 unsigned long page_size_mask = 0;
694 unsigned long start_pfn, end_pfn;
695 unsigned long pos;
696 unsigned long ret;
697
698 struct map_range mr[NR_RANGE_MR];
699 int nr_range, i;
700 int use_pse, use_gbpages;
701
702 printk(KERN_INFO "init_memory_mapping: %016lx-%016lx\n", start, end);
703
704 if (!after_bootmem)
705 init_gbpages();
706
707#ifdef CONFIG_DEBUG_PAGEALLOC
708 /*
709 * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
710 * This will simplify cpa(), which otherwise needs to support splitting
711 * large pages into small in interrupt context, etc.
712 */
713 use_pse = use_gbpages = 0;
714#else
715 use_pse = cpu_has_pse;
716 use_gbpages = direct_gbpages;
717#endif
718
719#ifdef CONFIG_X86_32
720#ifdef CONFIG_X86_PAE
721 set_nx();
722 if (nx_enabled)
723 printk(KERN_INFO "NX (Execute Disable) protection: active\n");
724#endif
725
726 /* Enable PSE if available */
727 if (cpu_has_pse)
728 set_in_cr4(X86_CR4_PSE);
729
730 /* Enable PGE if available */
731 if (cpu_has_pge) {
732 set_in_cr4(X86_CR4_PGE);
733 __supported_pte_mask |= _PAGE_GLOBAL;
734 }
735#endif
736
737 if (use_gbpages)
738 page_size_mask |= 1 << PG_LEVEL_1G;
739 if (use_pse)
740 page_size_mask |= 1 << PG_LEVEL_2M;
741
742 memset(mr, 0, sizeof(mr));
743 nr_range = 0;
744
745 /* head if not big page alignment ? */
746 start_pfn = start >> PAGE_SHIFT;
747 pos = start_pfn << PAGE_SHIFT;
748#ifdef CONFIG_X86_32
749 /*
750 * Don't use a large page for the first 2/4MB of memory
751 * because there are often fixed size MTRRs in there
752 * and overlapping MTRRs into large pages can cause
753 * slowdowns.
754 */
755 if (pos == 0)
756 end_pfn = 1<<(PMD_SHIFT - PAGE_SHIFT);
757 else
758 end_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
759 << (PMD_SHIFT - PAGE_SHIFT);
760#else /* CONFIG_X86_64 */
761 end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT)
762 << (PMD_SHIFT - PAGE_SHIFT);
763#endif
764 if (end_pfn > (end >> PAGE_SHIFT))
765 end_pfn = end >> PAGE_SHIFT;
766 if (start_pfn < end_pfn) {
767 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
768 pos = end_pfn << PAGE_SHIFT;
769 }
770
771 /* big page (2M) range */
772 start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
773 << (PMD_SHIFT - PAGE_SHIFT);
774#ifdef CONFIG_X86_32
775 end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
776#else /* CONFIG_X86_64 */
777 end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
778 << (PUD_SHIFT - PAGE_SHIFT);
779 if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)))
780 end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT));
781#endif
782
783 if (start_pfn < end_pfn) {
784 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
785 page_size_mask & (1<<PG_LEVEL_2M));
786 pos = end_pfn << PAGE_SHIFT;
787 }
788
789#ifdef CONFIG_X86_64
790 /* big page (1G) range */
791 start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
792 << (PUD_SHIFT - PAGE_SHIFT);
793 end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT);
794 if (start_pfn < end_pfn) {
795 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
796 page_size_mask &
797 ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G)));
798 pos = end_pfn << PAGE_SHIFT;
799 }
800
801 /* tail is not big page (1G) alignment */
802 start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
803 << (PMD_SHIFT - PAGE_SHIFT);
804 end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
805 if (start_pfn < end_pfn) {
806 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
807 page_size_mask & (1<<PG_LEVEL_2M));
808 pos = end_pfn << PAGE_SHIFT;
809 }
810#endif
811
812 /* tail is not big page (2M) alignment */
813 start_pfn = pos>>PAGE_SHIFT;
814 end_pfn = end>>PAGE_SHIFT;
815 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
816
817 /* try to merge same page size and continuous */
818 for (i = 0; nr_range > 1 && i < nr_range - 1; i++) {
819 unsigned long old_start;
820 if (mr[i].end != mr[i+1].start ||
821 mr[i].page_size_mask != mr[i+1].page_size_mask)
822 continue;
823 /* move it */
824 old_start = mr[i].start;
825 memmove(&mr[i], &mr[i+1],
826 (nr_range - 1 - i) * sizeof(struct map_range));
827 mr[i--].start = old_start;
828 nr_range--;
829 }
830
831 for (i = 0; i < nr_range; i++)
832 printk(KERN_DEBUG " %010lx - %010lx page %s\n",
833 mr[i].start, mr[i].end,
834 (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":(
835 (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k"));
836
837 /*
838 * Find space for the kernel direct mapping tables.
839 *
840 * Later we should allocate these tables in the local node of the
841 * memory mapped. Unfortunately this is done currently before the
842 * nodes are discovered.
843 */
844 if (!after_bootmem)
845 find_early_table_space(end, use_pse, use_gbpages);
846
847#ifdef CONFIG_X86_32
848 for (i = 0; i < nr_range; i++)
849 kernel_physical_mapping_init(
850 mr[i].start >> PAGE_SHIFT,
851 mr[i].end >> PAGE_SHIFT,
852 mr[i].page_size_mask == (1<<PG_LEVEL_2M));
853 ret = end;
854#else /* CONFIG_X86_64 */
855 for (i = 0; i < nr_range; i++)
856 ret = kernel_physical_mapping_init(mr[i].start, mr[i].end,
857 mr[i].page_size_mask);
858#endif
859
860#ifdef CONFIG_X86_32
861 early_ioremap_page_table_range_init();
862
863 load_cr3(swapper_pg_dir);
864#endif
865
866#ifdef CONFIG_X86_64
867 if (!after_bootmem)
868 mmu_cr4_features = read_cr4();
869#endif
870 __flush_tlb_all();
871
872 if (!after_bootmem && table_end > table_start)
873 reserve_early(table_start << PAGE_SHIFT,
874 table_end << PAGE_SHIFT, "PGTABLE");
875
876 if (!after_bootmem)
877 early_memtest(start, end);
878
879 return ret >> PAGE_SHIFT;
880}
881
882#ifndef CONFIG_NUMA 582#ifndef CONFIG_NUMA
883void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn) 583void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn)
884{ 584{
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-05 09:18:31 -0400