diff options
Diffstat (limited to 'arch/powerpc/include/asm/pgtable-ppc64.h')
-rw-r--r-- | arch/powerpc/include/asm/pgtable-ppc64.h | 241 |
1 files changed, 210 insertions, 31 deletions
diff --git a/arch/powerpc/include/asm/pgtable-ppc64.h b/arch/powerpc/include/asm/pgtable-ppc64.h index e3d55f6f24fe..46db09414a10 100644 --- a/arch/powerpc/include/asm/pgtable-ppc64.h +++ b/arch/powerpc/include/asm/pgtable-ppc64.h | |||
@@ -10,6 +10,7 @@ | |||
10 | #else | 10 | #else |
11 | #include <asm/pgtable-ppc64-4k.h> | 11 | #include <asm/pgtable-ppc64-4k.h> |
12 | #endif | 12 | #endif |
13 | #include <asm/barrier.h> | ||
13 | 14 | ||
14 | #define FIRST_USER_ADDRESS 0 | 15 | #define FIRST_USER_ADDRESS 0 |
15 | 16 | ||
@@ -20,7 +21,11 @@ | |||
20 | PUD_INDEX_SIZE + PGD_INDEX_SIZE + PAGE_SHIFT) | 21 | PUD_INDEX_SIZE + PGD_INDEX_SIZE + PAGE_SHIFT) |
21 | #define PGTABLE_RANGE (ASM_CONST(1) << PGTABLE_EADDR_SIZE) | 22 | #define PGTABLE_RANGE (ASM_CONST(1) << PGTABLE_EADDR_SIZE) |
22 | 23 | ||
23 | 24 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | |
25 | #define PMD_CACHE_INDEX (PMD_INDEX_SIZE + 1) | ||
26 | #else | ||
27 | #define PMD_CACHE_INDEX PMD_INDEX_SIZE | ||
28 | #endif | ||
24 | /* | 29 | /* |
25 | * Define the address range of the kernel non-linear virtual area | 30 | * Define the address range of the kernel non-linear virtual area |
26 | */ | 31 | */ |
@@ -150,7 +155,7 @@ | |||
150 | #define pmd_present(pmd) (pmd_val(pmd) != 0) | 155 | #define pmd_present(pmd) (pmd_val(pmd) != 0) |
151 | #define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0) | 156 | #define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0) |
152 | #define pmd_page_vaddr(pmd) (pmd_val(pmd) & ~PMD_MASKED_BITS) | 157 | #define pmd_page_vaddr(pmd) (pmd_val(pmd) & ~PMD_MASKED_BITS) |
153 | #define pmd_page(pmd) virt_to_page(pmd_page_vaddr(pmd)) | 158 | extern struct page *pmd_page(pmd_t pmd); |
154 | 159 | ||
155 | #define pud_set(pudp, pudval) (pud_val(*(pudp)) = (pudval)) | 160 | #define pud_set(pudp, pudval) (pud_val(*(pudp)) = (pudval)) |
156 | #define pud_none(pud) (!pud_val(pud)) | 161 | #define pud_none(pud) (!pud_val(pud)) |
@@ -339,43 +344,217 @@ static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry) | |||
339 | 344 | ||
340 | void pgtable_cache_add(unsigned shift, void (*ctor)(void *)); | 345 | void pgtable_cache_add(unsigned shift, void (*ctor)(void *)); |
341 | void pgtable_cache_init(void); | 346 | void pgtable_cache_init(void); |
347 | #endif /* __ASSEMBLY__ */ | ||
348 | |||
349 | /* | ||
350 | * THP pages can't be special. So use the _PAGE_SPECIAL | ||
351 | */ | ||
352 | #define _PAGE_SPLITTING _PAGE_SPECIAL | ||
353 | |||
354 | /* | ||
355 | * We need to differentiate between explicit huge page and THP huge | ||
356 | * page, since THP huge page also need to track real subpage details | ||
357 | */ | ||
358 | #define _PAGE_THP_HUGE _PAGE_4K_PFN | ||
342 | 359 | ||
343 | /* | 360 | /* |
344 | * find_linux_pte returns the address of a linux pte for a given | 361 | * set of bits not changed in pmd_modify. |
345 | * effective address and directory. If not found, it returns zero. | ||
346 | */ | 362 | */ |
347 | static inline pte_t *find_linux_pte(pgd_t *pgdir, unsigned long ea) | 363 | #define _HPAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | \ |
364 | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_SPLITTING | \ | ||
365 | _PAGE_THP_HUGE) | ||
366 | |||
367 | #ifndef __ASSEMBLY__ | ||
368 | /* | ||
369 | * The linux hugepage PMD now include the pmd entries followed by the address | ||
370 | * to the stashed pgtable_t. The stashed pgtable_t contains the hpte bits. | ||
371 | * [ 1 bit secondary | 3 bit hidx | 1 bit valid | 000]. We use one byte per | ||
372 | * each HPTE entry. With 16MB hugepage and 64K HPTE we need 256 entries and | ||
373 | * with 4K HPTE we need 4096 entries. Both will fit in a 4K pgtable_t. | ||
374 | * | ||
375 | * The last three bits are intentionally left to zero. This memory location | ||
376 | * are also used as normal page PTE pointers. So if we have any pointers | ||
377 | * left around while we collapse a hugepage, we need to make sure | ||
378 | * _PAGE_PRESENT and _PAGE_FILE bits of that are zero when we look at them | ||
379 | */ | ||
380 | static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index) | ||
348 | { | 381 | { |
349 | pgd_t *pg; | 382 | return (hpte_slot_array[index] >> 3) & 0x1; |
350 | pud_t *pu; | ||
351 | pmd_t *pm; | ||
352 | pte_t *pt = NULL; | ||
353 | |||
354 | pg = pgdir + pgd_index(ea); | ||
355 | if (!pgd_none(*pg)) { | ||
356 | pu = pud_offset(pg, ea); | ||
357 | if (!pud_none(*pu)) { | ||
358 | pm = pmd_offset(pu, ea); | ||
359 | if (pmd_present(*pm)) | ||
360 | pt = pte_offset_kernel(pm, ea); | ||
361 | } | ||
362 | } | ||
363 | return pt; | ||
364 | } | 383 | } |
365 | 384 | ||
366 | #ifdef CONFIG_HUGETLB_PAGE | 385 | static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array, |
367 | pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, | 386 | int index) |
368 | unsigned *shift); | ||
369 | #else | ||
370 | static inline pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, | ||
371 | unsigned *shift) | ||
372 | { | 387 | { |
373 | if (shift) | 388 | return hpte_slot_array[index] >> 4; |
374 | *shift = 0; | ||
375 | return find_linux_pte(pgdir, ea); | ||
376 | } | 389 | } |
377 | #endif /* !CONFIG_HUGETLB_PAGE */ | ||
378 | 390 | ||
379 | #endif /* __ASSEMBLY__ */ | 391 | static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array, |
392 | unsigned int index, unsigned int hidx) | ||
393 | { | ||
394 | hpte_slot_array[index] = hidx << 4 | 0x1 << 3; | ||
395 | } | ||
380 | 396 | ||
397 | static inline char *get_hpte_slot_array(pmd_t *pmdp) | ||
398 | { | ||
399 | /* | ||
400 | * The hpte hindex is stored in the pgtable whose address is in the | ||
401 | * second half of the PMD | ||
402 | * | ||
403 | * Order this load with the test for pmd_trans_huge in the caller | ||
404 | */ | ||
405 | smp_rmb(); | ||
406 | return *(char **)(pmdp + PTRS_PER_PMD); | ||
407 | |||
408 | |||
409 | } | ||
410 | |||
411 | extern void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr, | ||
412 | pmd_t *pmdp); | ||
413 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | ||
414 | extern pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot); | ||
415 | extern pmd_t mk_pmd(struct page *page, pgprot_t pgprot); | ||
416 | extern pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot); | ||
417 | extern void set_pmd_at(struct mm_struct *mm, unsigned long addr, | ||
418 | pmd_t *pmdp, pmd_t pmd); | ||
419 | extern void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, | ||
420 | pmd_t *pmd); | ||
421 | |||
422 | static inline int pmd_trans_huge(pmd_t pmd) | ||
423 | { | ||
424 | /* | ||
425 | * leaf pte for huge page, bottom two bits != 00 | ||
426 | */ | ||
427 | return (pmd_val(pmd) & 0x3) && (pmd_val(pmd) & _PAGE_THP_HUGE); | ||
428 | } | ||
429 | |||
430 | static inline int pmd_large(pmd_t pmd) | ||
431 | { | ||
432 | /* | ||
433 | * leaf pte for huge page, bottom two bits != 00 | ||
434 | */ | ||
435 | if (pmd_trans_huge(pmd)) | ||
436 | return pmd_val(pmd) & _PAGE_PRESENT; | ||
437 | return 0; | ||
438 | } | ||
439 | |||
440 | static inline int pmd_trans_splitting(pmd_t pmd) | ||
441 | { | ||
442 | if (pmd_trans_huge(pmd)) | ||
443 | return pmd_val(pmd) & _PAGE_SPLITTING; | ||
444 | return 0; | ||
445 | } | ||
446 | |||
447 | extern int has_transparent_hugepage(void); | ||
448 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ | ||
449 | |||
450 | static inline pte_t pmd_pte(pmd_t pmd) | ||
451 | { | ||
452 | return __pte(pmd_val(pmd)); | ||
453 | } | ||
454 | |||
455 | static inline pmd_t pte_pmd(pte_t pte) | ||
456 | { | ||
457 | return __pmd(pte_val(pte)); | ||
458 | } | ||
459 | |||
460 | static inline pte_t *pmdp_ptep(pmd_t *pmd) | ||
461 | { | ||
462 | return (pte_t *)pmd; | ||
463 | } | ||
464 | |||
465 | #define pmd_pfn(pmd) pte_pfn(pmd_pte(pmd)) | ||
466 | #define pmd_young(pmd) pte_young(pmd_pte(pmd)) | ||
467 | #define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd))) | ||
468 | #define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd))) | ||
469 | #define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd))) | ||
470 | #define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd))) | ||
471 | #define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd))) | ||
472 | |||
473 | #define __HAVE_ARCH_PMD_WRITE | ||
474 | #define pmd_write(pmd) pte_write(pmd_pte(pmd)) | ||
475 | |||
476 | static inline pmd_t pmd_mkhuge(pmd_t pmd) | ||
477 | { | ||
478 | /* Do nothing, mk_pmd() does this part. */ | ||
479 | return pmd; | ||
480 | } | ||
481 | |||
482 | static inline pmd_t pmd_mknotpresent(pmd_t pmd) | ||
483 | { | ||
484 | pmd_val(pmd) &= ~_PAGE_PRESENT; | ||
485 | return pmd; | ||
486 | } | ||
487 | |||
488 | static inline pmd_t pmd_mksplitting(pmd_t pmd) | ||
489 | { | ||
490 | pmd_val(pmd) |= _PAGE_SPLITTING; | ||
491 | return pmd; | ||
492 | } | ||
493 | |||
494 | #define __HAVE_ARCH_PMD_SAME | ||
495 | static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) | ||
496 | { | ||
497 | return (((pmd_val(pmd_a) ^ pmd_val(pmd_b)) & ~_PAGE_HPTEFLAGS) == 0); | ||
498 | } | ||
499 | |||
500 | #define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS | ||
501 | extern int pmdp_set_access_flags(struct vm_area_struct *vma, | ||
502 | unsigned long address, pmd_t *pmdp, | ||
503 | pmd_t entry, int dirty); | ||
504 | |||
505 | extern unsigned long pmd_hugepage_update(struct mm_struct *mm, | ||
506 | unsigned long addr, | ||
507 | pmd_t *pmdp, unsigned long clr); | ||
508 | |||
509 | static inline int __pmdp_test_and_clear_young(struct mm_struct *mm, | ||
510 | unsigned long addr, pmd_t *pmdp) | ||
511 | { | ||
512 | unsigned long old; | ||
513 | |||
514 | if ((pmd_val(*pmdp) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0) | ||
515 | return 0; | ||
516 | old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED); | ||
517 | return ((old & _PAGE_ACCESSED) != 0); | ||
518 | } | ||
519 | |||
520 | #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG | ||
521 | extern int pmdp_test_and_clear_young(struct vm_area_struct *vma, | ||
522 | unsigned long address, pmd_t *pmdp); | ||
523 | #define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH | ||
524 | extern int pmdp_clear_flush_young(struct vm_area_struct *vma, | ||
525 | unsigned long address, pmd_t *pmdp); | ||
526 | |||
527 | #define __HAVE_ARCH_PMDP_GET_AND_CLEAR | ||
528 | extern pmd_t pmdp_get_and_clear(struct mm_struct *mm, | ||
529 | unsigned long addr, pmd_t *pmdp); | ||
530 | |||
531 | #define __HAVE_ARCH_PMDP_CLEAR_FLUSH | ||
532 | extern pmd_t pmdp_clear_flush(struct vm_area_struct *vma, unsigned long address, | ||
533 | pmd_t *pmdp); | ||
534 | |||
535 | #define __HAVE_ARCH_PMDP_SET_WRPROTECT | ||
536 | static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr, | ||
537 | pmd_t *pmdp) | ||
538 | { | ||
539 | |||
540 | if ((pmd_val(*pmdp) & _PAGE_RW) == 0) | ||
541 | return; | ||
542 | |||
543 | pmd_hugepage_update(mm, addr, pmdp, _PAGE_RW); | ||
544 | } | ||
545 | |||
546 | #define __HAVE_ARCH_PMDP_SPLITTING_FLUSH | ||
547 | extern void pmdp_splitting_flush(struct vm_area_struct *vma, | ||
548 | unsigned long address, pmd_t *pmdp); | ||
549 | |||
550 | #define __HAVE_ARCH_PGTABLE_DEPOSIT | ||
551 | extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, | ||
552 | pgtable_t pgtable); | ||
553 | #define __HAVE_ARCH_PGTABLE_WITHDRAW | ||
554 | extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); | ||
555 | |||
556 | #define __HAVE_ARCH_PMDP_INVALIDATE | ||
557 | extern void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, | ||
558 | pmd_t *pmdp); | ||
559 | #endif /* __ASSEMBLY__ */ | ||
381 | #endif /* _ASM_POWERPC_PGTABLE_PPC64_H_ */ | 560 | #endif /* _ASM_POWERPC_PGTABLE_PPC64_H_ */ |