aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/x86/mm/init_32.c65
-rw-r--r--arch/x86/mm/init_64.c99
2 files changed, 144 insertions, 20 deletions
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index d37f29376b0c..9b5f7d7049d0 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -194,11 +194,30 @@ static void __init kernel_physical_mapping_init(pgd_t *pgd_base,
194 pgd_t *pgd; 194 pgd_t *pgd;
195 pmd_t *pmd; 195 pmd_t *pmd;
196 pte_t *pte; 196 pte_t *pte;
197 unsigned pages_2m = 0, pages_4k = 0; 197 unsigned pages_2m, pages_4k;
198 int mapping_iter;
199
200 /*
201 * First iteration will setup identity mapping using large/small pages
202 * based on use_pse, with other attributes same as set by
203 * the early code in head_32.S
204 *
205 * Second iteration will setup the appropriate attributes (NX, GLOBAL..)
206 * as desired for the kernel identity mapping.
207 *
208 * This two pass mechanism conforms to the TLB app note which says:
209 *
210 * "Software should not write to a paging-structure entry in a way
211 * that would change, for any linear address, both the page size
212 * and either the page frame or attributes."
213 */
214 mapping_iter = 1;
198 215
199 if (!cpu_has_pse) 216 if (!cpu_has_pse)
200 use_pse = 0; 217 use_pse = 0;
201 218
219repeat:
220 pages_2m = pages_4k = 0;
202 pfn = start_pfn; 221 pfn = start_pfn;
203 pgd_idx = pgd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET); 222 pgd_idx = pgd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
204 pgd = pgd_base + pgd_idx; 223 pgd = pgd_base + pgd_idx;
@@ -224,6 +243,13 @@ static void __init kernel_physical_mapping_init(pgd_t *pgd_base,
224 if (use_pse) { 243 if (use_pse) {
225 unsigned int addr2; 244 unsigned int addr2;
226 pgprot_t prot = PAGE_KERNEL_LARGE; 245 pgprot_t prot = PAGE_KERNEL_LARGE;
246 /*
247 * first pass will use the same initial
248 * identity mapping attribute + _PAGE_PSE.
249 */
250 pgprot_t init_prot =
251 __pgprot(PTE_IDENT_ATTR |
252 _PAGE_PSE);
227 253
228 addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE + 254 addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE +
229 PAGE_OFFSET + PAGE_SIZE-1; 255 PAGE_OFFSET + PAGE_SIZE-1;
@@ -233,7 +259,10 @@ static void __init kernel_physical_mapping_init(pgd_t *pgd_base,
233 prot = PAGE_KERNEL_LARGE_EXEC; 259 prot = PAGE_KERNEL_LARGE_EXEC;
234 260
235 pages_2m++; 261 pages_2m++;
236 set_pmd(pmd, pfn_pmd(pfn, prot)); 262 if (mapping_iter == 1)
263 set_pmd(pmd, pfn_pmd(pfn, init_prot));
264 else
265 set_pmd(pmd, pfn_pmd(pfn, prot));
237 266
238 pfn += PTRS_PER_PTE; 267 pfn += PTRS_PER_PTE;
239 continue; 268 continue;
@@ -245,17 +274,43 @@ static void __init kernel_physical_mapping_init(pgd_t *pgd_base,
245 for (; pte_ofs < PTRS_PER_PTE && pfn < end_pfn; 274 for (; pte_ofs < PTRS_PER_PTE && pfn < end_pfn;
246 pte++, pfn++, pte_ofs++, addr += PAGE_SIZE) { 275 pte++, pfn++, pte_ofs++, addr += PAGE_SIZE) {
247 pgprot_t prot = PAGE_KERNEL; 276 pgprot_t prot = PAGE_KERNEL;
277 /*
278 * first pass will use the same initial
279 * identity mapping attribute.
280 */
281 pgprot_t init_prot = __pgprot(PTE_IDENT_ATTR);
248 282
249 if (is_kernel_text(addr)) 283 if (is_kernel_text(addr))
250 prot = PAGE_KERNEL_EXEC; 284 prot = PAGE_KERNEL_EXEC;
251 285
252 pages_4k++; 286 pages_4k++;
253 set_pte(pte, pfn_pte(pfn, prot)); 287 if (mapping_iter == 1)
288 set_pte(pte, pfn_pte(pfn, init_prot));
289 else
290 set_pte(pte, pfn_pte(pfn, prot));
254 } 291 }
255 } 292 }
256 } 293 }
257 update_page_count(PG_LEVEL_2M, pages_2m); 294 if (mapping_iter == 1) {
258 update_page_count(PG_LEVEL_4K, pages_4k); 295 /*
296 * update direct mapping page count only in the first
297 * iteration.
298 */
299 update_page_count(PG_LEVEL_2M, pages_2m);
300 update_page_count(PG_LEVEL_4K, pages_4k);
301
302 /*
303 * local global flush tlb, which will flush the previous
304 * mappings present in both small and large page TLB's.
305 */
306 __flush_tlb_all();
307
308 /*
309 * Second iteration will set the actual desired PTE attributes.
310 */
311 mapping_iter = 2;
312 goto repeat;
313 }
259} 314}
260 315
261/* 316/*
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index d3746efb060d..1ba945eb6282 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -270,6 +270,8 @@ static __ref void unmap_low_page(void *adr)
270 early_iounmap(adr, PAGE_SIZE); 270 early_iounmap(adr, PAGE_SIZE);
271} 271}
272 272
273static int physical_mapping_iter;
274
273static unsigned long __meminit 275static unsigned long __meminit
274phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end) 276phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end)
275{ 277{
@@ -290,16 +292,19 @@ phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end)
290 } 292 }
291 293
292 if (pte_val(*pte)) 294 if (pte_val(*pte))
293 continue; 295 goto repeat_set_pte;
294 296
295 if (0) 297 if (0)
296 printk(" pte=%p addr=%lx pte=%016lx\n", 298 printk(" pte=%p addr=%lx pte=%016lx\n",
297 pte, addr, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL).pte); 299 pte, addr, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL).pte);
300 pages++;
301repeat_set_pte:
298 set_pte(pte, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL)); 302 set_pte(pte, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL));
299 last_map_addr = (addr & PAGE_MASK) + PAGE_SIZE; 303 last_map_addr = (addr & PAGE_MASK) + PAGE_SIZE;
300 pages++;
301 } 304 }
302 update_page_count(PG_LEVEL_4K, pages); 305
306 if (physical_mapping_iter == 1)
307 update_page_count(PG_LEVEL_4K, pages);
303 308
304 return last_map_addr; 309 return last_map_addr;
305} 310}
@@ -318,7 +323,6 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
318{ 323{
319 unsigned long pages = 0; 324 unsigned long pages = 0;
320 unsigned long last_map_addr = end; 325 unsigned long last_map_addr = end;
321 unsigned long start = address;
322 326
323 int i = pmd_index(address); 327 int i = pmd_index(address);
324 328
@@ -341,15 +345,14 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
341 last_map_addr = phys_pte_update(pmd, address, 345 last_map_addr = phys_pte_update(pmd, address,
342 end); 346 end);
343 spin_unlock(&init_mm.page_table_lock); 347 spin_unlock(&init_mm.page_table_lock);
348 continue;
344 } 349 }
345 /* Count entries we're using from level2_ident_pgt */ 350 goto repeat_set_pte;
346 if (start == 0)
347 pages++;
348 continue;
349 } 351 }
350 352
351 if (page_size_mask & (1<<PG_LEVEL_2M)) { 353 if (page_size_mask & (1<<PG_LEVEL_2M)) {
352 pages++; 354 pages++;
355repeat_set_pte:
353 spin_lock(&init_mm.page_table_lock); 356 spin_lock(&init_mm.page_table_lock);
354 set_pte((pte_t *)pmd, 357 set_pte((pte_t *)pmd,
355 pfn_pte(address >> PAGE_SHIFT, PAGE_KERNEL_LARGE)); 358 pfn_pte(address >> PAGE_SHIFT, PAGE_KERNEL_LARGE));
@@ -366,7 +369,8 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
366 pmd_populate_kernel(&init_mm, pmd, __va(pte_phys)); 369 pmd_populate_kernel(&init_mm, pmd, __va(pte_phys));
367 spin_unlock(&init_mm.page_table_lock); 370 spin_unlock(&init_mm.page_table_lock);
368 } 371 }
369 update_page_count(PG_LEVEL_2M, pages); 372 if (physical_mapping_iter == 1)
373 update_page_count(PG_LEVEL_2M, pages);
370 return last_map_addr; 374 return last_map_addr;
371} 375}
372 376
@@ -405,14 +409,18 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,
405 } 409 }
406 410
407 if (pud_val(*pud)) { 411 if (pud_val(*pud)) {
408 if (!pud_large(*pud)) 412 if (!pud_large(*pud)) {
409 last_map_addr = phys_pmd_update(pud, addr, end, 413 last_map_addr = phys_pmd_update(pud, addr, end,
410 page_size_mask); 414 page_size_mask);
411 continue; 415 continue;
416 }
417
418 goto repeat_set_pte;
412 } 419 }
413 420
414 if (page_size_mask & (1<<PG_LEVEL_1G)) { 421 if (page_size_mask & (1<<PG_LEVEL_1G)) {
415 pages++; 422 pages++;
423repeat_set_pte:
416 spin_lock(&init_mm.page_table_lock); 424 spin_lock(&init_mm.page_table_lock);
417 set_pte((pte_t *)pud, 425 set_pte((pte_t *)pud,
418 pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL_LARGE)); 426 pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL_LARGE));
@@ -430,7 +438,9 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,
430 spin_unlock(&init_mm.page_table_lock); 438 spin_unlock(&init_mm.page_table_lock);
431 } 439 }
432 __flush_tlb_all(); 440 __flush_tlb_all();
433 update_page_count(PG_LEVEL_1G, pages); 441
442 if (physical_mapping_iter == 1)
443 update_page_count(PG_LEVEL_1G, pages);
434 444
435 return last_map_addr; 445 return last_map_addr;
436} 446}
@@ -494,15 +504,54 @@ static void __init init_gbpages(void)
494 direct_gbpages = 0; 504 direct_gbpages = 0;
495} 505}
496 506
507static int is_kernel(unsigned long pfn)
508{
509 unsigned long pg_addresss = pfn << PAGE_SHIFT;
510
511 if (pg_addresss >= (unsigned long) __pa(_text) &&
512 pg_addresss <= (unsigned long) __pa(_end))
513 return 1;
514
515 return 0;
516}
517
497static unsigned long __init kernel_physical_mapping_init(unsigned long start, 518static unsigned long __init kernel_physical_mapping_init(unsigned long start,
498 unsigned long end, 519 unsigned long end,
499 unsigned long page_size_mask) 520 unsigned long page_size_mask)
500{ 521{
501 522
502 unsigned long next, last_map_addr = end; 523 unsigned long next, last_map_addr;
524 u64 cached_supported_pte_mask = __supported_pte_mask;
525 unsigned long cache_start = start;
526 unsigned long cache_end = end;
527
528 /*
529 * First iteration will setup identity mapping using large/small pages
530 * based on page_size_mask, with other attributes same as set by
531 * the early code in head_64.S
532 *
533 * Second iteration will setup the appropriate attributes
534 * as desired for the kernel identity mapping.
535 *
536 * This two pass mechanism conforms to the TLB app note which says:
537 *
538 * "Software should not write to a paging-structure entry in a way
539 * that would change, for any linear address, both the page size
540 * and either the page frame or attributes."
541 *
542 * For now, only difference between very early PTE attributes used in
543 * head_64.S and here is _PAGE_NX.
544 */
545 BUILD_BUG_ON((__PAGE_KERNEL_LARGE & ~__PAGE_KERNEL_IDENT_LARGE_EXEC)
546 != _PAGE_NX);
547 __supported_pte_mask &= ~(_PAGE_NX);
548 physical_mapping_iter = 1;
503 549
504 start = (unsigned long)__va(start); 550repeat:
505 end = (unsigned long)__va(end); 551 last_map_addr = cache_end;
552
553 start = (unsigned long)__va(cache_start);
554 end = (unsigned long)__va(cache_end);
506 555
507 for (; start < end; start = next) { 556 for (; start < end; start = next) {
508 pgd_t *pgd = pgd_offset_k(start); 557 pgd_t *pgd = pgd_offset_k(start);
@@ -514,11 +563,21 @@ static unsigned long __init kernel_physical_mapping_init(unsigned long start,
514 next = end; 563 next = end;
515 564
516 if (pgd_val(*pgd)) { 565 if (pgd_val(*pgd)) {
566 /*
567 * Static identity mappings will be overwritten
568 * with run-time mappings. For example, this allows
569 * the static 0-1GB identity mapping to be mapped
570 * non-executable with this.
571 */
572 if (is_kernel(pte_pfn(*((pte_t *) pgd))))
573 goto realloc;
574
517 last_map_addr = phys_pud_update(pgd, __pa(start), 575 last_map_addr = phys_pud_update(pgd, __pa(start),
518 __pa(end), page_size_mask); 576 __pa(end), page_size_mask);
519 continue; 577 continue;
520 } 578 }
521 579
580realloc:
522 pud = alloc_low_page(&pud_phys); 581 pud = alloc_low_page(&pud_phys);
523 last_map_addr = phys_pud_init(pud, __pa(start), __pa(next), 582 last_map_addr = phys_pud_init(pud, __pa(start), __pa(next),
524 page_size_mask); 583 page_size_mask);
@@ -528,6 +587,16 @@ static unsigned long __init kernel_physical_mapping_init(unsigned long start,
528 pgd_populate(&init_mm, pgd, __va(pud_phys)); 587 pgd_populate(&init_mm, pgd, __va(pud_phys));
529 spin_unlock(&init_mm.page_table_lock); 588 spin_unlock(&init_mm.page_table_lock);
530 } 589 }
590 __flush_tlb_all();
591
592 if (physical_mapping_iter == 1) {
593 physical_mapping_iter = 2;
594 /*
595 * Second iteration will set the actual desired PTE attributes.
596 */
597 __supported_pte_mask = cached_supported_pte_mask;
598 goto repeat;
599 }
531 600
532 return last_map_addr; 601 return last_map_addr;
533} 602}
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-15 17:39:04 -0400