aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/mm
diff options
context:
space:
mode:
authorPaul Mackerras <paulus@samba.org>2007-10-11 06:37:10 -0400
committerPaul Mackerras <paulus@samba.org>2007-10-12 00:05:17 -0400
commit1189be6508d45183013ddb82b18f4934193de274 (patch)
tree58924481b4de56699e4a884dce8dc601e71cf7d1 /arch/powerpc/mm
parent287e5d6fcccfa38b953cebe307e1ddfd32363355 (diff)
[POWERPC] Use 1TB segments
This makes the kernel use 1TB segments for all kernel mappings and for user addresses of 1TB and above, on machines which support them (currently POWER5+, POWER6 and PA6T). We detect that the machine supports 1TB segments by looking at the ibm,processor-segment-sizes property in the device tree. We don't currently use 1TB segments for user addresses < 1T, since that would effectively prevent 32-bit processes from using huge pages unless we also had a way to revert to using 256MB segments. That would be possible but would involve extra complications (such as keeping track of which segment size was used when HPTEs were inserted) and is not addressed here. Parts of this patch were originally written by Ben Herrenschmidt. Signed-off-by: Paul Mackerras <paulus@samba.org>
Diffstat (limited to 'arch/powerpc/mm')
-rw-r--r--arch/powerpc/mm/hash_low_64.S73
-rw-r--r--arch/powerpc/mm/hash_native_64.c92
-rw-r--r--arch/powerpc/mm/hash_utils_64.c114
-rw-r--r--arch/powerpc/mm/hugetlbpage.c13
-rw-r--r--arch/powerpc/mm/pgtable_64.c4
-rw-r--r--arch/powerpc/mm/slb.c67
-rw-r--r--arch/powerpc/mm/slb_low.S37
-rw-r--r--arch/powerpc/mm/stab.c6
-rw-r--r--arch/powerpc/mm/tlb_64.c20
9 files changed, 293 insertions, 133 deletions
diff --git a/arch/powerpc/mm/hash_low_64.S b/arch/powerpc/mm/hash_low_64.S
index 35eabfb50723..ad253b959030 100644
--- a/arch/powerpc/mm/hash_low_64.S
+++ b/arch/powerpc/mm/hash_low_64.S
@@ -54,7 +54,7 @@
54 54
55/* 55/*
56 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, 56 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
57 * pte_t *ptep, unsigned long trap, int local) 57 * pte_t *ptep, unsigned long trap, int local, int ssize)
58 * 58 *
59 * Adds a 4K page to the hash table in a segment of 4K pages only 59 * Adds a 4K page to the hash table in a segment of 4K pages only
60 */ 60 */
@@ -66,6 +66,7 @@ _GLOBAL(__hash_page_4K)
66 /* Save all params that we need after a function call */ 66 /* Save all params that we need after a function call */
67 std r6,STK_PARM(r6)(r1) 67 std r6,STK_PARM(r6)(r1)
68 std r8,STK_PARM(r8)(r1) 68 std r8,STK_PARM(r8)(r1)
69 std r9,STK_PARM(r9)(r1)
69 70
70 /* Add _PAGE_PRESENT to access */ 71 /* Add _PAGE_PRESENT to access */
71 ori r4,r4,_PAGE_PRESENT 72 ori r4,r4,_PAGE_PRESENT
@@ -117,6 +118,10 @@ _GLOBAL(__hash_page_4K)
117 * r4 (access) is re-useable, we use it for the new HPTE flags 118 * r4 (access) is re-useable, we use it for the new HPTE flags
118 */ 119 */
119 120
121BEGIN_FTR_SECTION
122 cmpdi r9,0 /* check segment size */
123 bne 3f
124END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
120 /* Calc va and put it in r29 */ 125 /* Calc va and put it in r29 */
121 rldicr r29,r5,28,63-28 126 rldicr r29,r5,28,63-28
122 rldicl r3,r3,0,36 127 rldicl r3,r3,0,36
@@ -126,9 +131,20 @@ _GLOBAL(__hash_page_4K)
126 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ 131 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
127 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ 132 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
128 xor r28,r5,r0 133 xor r28,r5,r0
134 b 4f
135
1363: /* Calc VA and hash in r29 and r28 for 1T segment */
137 sldi r29,r5,40 /* vsid << 40 */
138 clrldi r3,r3,24 /* ea & 0xffffffffff */
139 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
140 clrldi r5,r5,40 /* vsid & 0xffffff */
141 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
142 xor r28,r28,r5
143 or r29,r3,r29 /* VA */
144 xor r28,r28,r0 /* hash */
129 145
130 /* Convert linux PTE bits into HW equivalents */ 146 /* Convert linux PTE bits into HW equivalents */
131 andi. r3,r30,0x1fe /* Get basic set of flags */ 1474: andi. r3,r30,0x1fe /* Get basic set of flags */
132 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ 148 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
133 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 149 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
134 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ 150 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
@@ -183,6 +199,7 @@ htab_insert_pte:
183 mr r4,r29 /* Retreive va */ 199 mr r4,r29 /* Retreive va */
184 li r7,0 /* !bolted, !secondary */ 200 li r7,0 /* !bolted, !secondary */
185 li r8,MMU_PAGE_4K /* page size */ 201 li r8,MMU_PAGE_4K /* page size */
202 ld r9,STK_PARM(r9)(r1) /* segment size */
186_GLOBAL(htab_call_hpte_insert1) 203_GLOBAL(htab_call_hpte_insert1)
187 bl . /* Patched by htab_finish_init() */ 204 bl . /* Patched by htab_finish_init() */
188 cmpdi 0,r3,0 205 cmpdi 0,r3,0
@@ -205,6 +222,7 @@ _GLOBAL(htab_call_hpte_insert1)
205 mr r4,r29 /* Retreive va */ 222 mr r4,r29 /* Retreive va */
206 li r7,HPTE_V_SECONDARY /* !bolted, secondary */ 223 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
207 li r8,MMU_PAGE_4K /* page size */ 224 li r8,MMU_PAGE_4K /* page size */
225 ld r9,STK_PARM(r9)(r1) /* segment size */
208_GLOBAL(htab_call_hpte_insert2) 226_GLOBAL(htab_call_hpte_insert2)
209 bl . /* Patched by htab_finish_init() */ 227 bl . /* Patched by htab_finish_init() */
210 cmpdi 0,r3,0 228 cmpdi 0,r3,0
@@ -273,7 +291,8 @@ htab_modify_pte:
273 /* Call ppc_md.hpte_updatepp */ 291 /* Call ppc_md.hpte_updatepp */
274 mr r5,r29 /* va */ 292 mr r5,r29 /* va */
275 li r6,MMU_PAGE_4K /* page size */ 293 li r6,MMU_PAGE_4K /* page size */
276 ld r7,STK_PARM(r8)(r1) /* get "local" param */ 294 ld r7,STK_PARM(r9)(r1) /* segment size */
295 ld r8,STK_PARM(r8)(r1) /* get "local" param */
277_GLOBAL(htab_call_hpte_updatepp) 296_GLOBAL(htab_call_hpte_updatepp)
278 bl . /* Patched by htab_finish_init() */ 297 bl . /* Patched by htab_finish_init() */
279 298
@@ -325,6 +344,7 @@ _GLOBAL(__hash_page_4K)
325 /* Save all params that we need after a function call */ 344 /* Save all params that we need after a function call */
326 std r6,STK_PARM(r6)(r1) 345 std r6,STK_PARM(r6)(r1)
327 std r8,STK_PARM(r8)(r1) 346 std r8,STK_PARM(r8)(r1)
347 std r9,STK_PARM(r9)(r1)
328 348
329 /* Add _PAGE_PRESENT to access */ 349 /* Add _PAGE_PRESENT to access */
330 ori r4,r4,_PAGE_PRESENT 350 ori r4,r4,_PAGE_PRESENT
@@ -383,18 +403,33 @@ _GLOBAL(__hash_page_4K)
383 /* Load the hidx index */ 403 /* Load the hidx index */
384 rldicl r25,r3,64-12,60 404 rldicl r25,r3,64-12,60
385 405
406BEGIN_FTR_SECTION
407 cmpdi r9,0 /* check segment size */
408 bne 3f
409END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
386 /* Calc va and put it in r29 */ 410 /* Calc va and put it in r29 */
387 rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */ 411 rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */
388 rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */ 412 rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */
389 or r29,r3,r29 /* r29 = va 413 or r29,r3,r29 /* r29 = va */
390 414
391 /* Calculate hash value for primary slot and store it in r28 */ 415 /* Calculate hash value for primary slot and store it in r28 */
392 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ 416 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
393 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ 417 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
394 xor r28,r5,r0 418 xor r28,r5,r0
419 b 4f
420
4213: /* Calc VA and hash in r29 and r28 for 1T segment */
422 sldi r29,r5,40 /* vsid << 40 */
423 clrldi r3,r3,24 /* ea & 0xffffffffff */
424 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
425 clrldi r5,r5,40 /* vsid & 0xffffff */
426 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
427 xor r28,r28,r5
428 or r29,r3,r29 /* VA */
429 xor r28,r28,r0 /* hash */
395 430
396 /* Convert linux PTE bits into HW equivalents */ 431 /* Convert linux PTE bits into HW equivalents */
397 andi. r3,r30,0x1fe /* Get basic set of flags */ 4324: andi. r3,r30,0x1fe /* Get basic set of flags */
398 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ 433 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
399 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 434 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
400 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ 435 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
@@ -462,6 +497,7 @@ htab_special_pfn:
462 mr r4,r29 /* Retreive va */ 497 mr r4,r29 /* Retreive va */
463 li r7,0 /* !bolted, !secondary */ 498 li r7,0 /* !bolted, !secondary */
464 li r8,MMU_PAGE_4K /* page size */ 499 li r8,MMU_PAGE_4K /* page size */
500 ld r9,STK_PARM(r9)(r1) /* segment size */
465_GLOBAL(htab_call_hpte_insert1) 501_GLOBAL(htab_call_hpte_insert1)
466 bl . /* patched by htab_finish_init() */ 502 bl . /* patched by htab_finish_init() */
467 cmpdi 0,r3,0 503 cmpdi 0,r3,0
@@ -488,6 +524,7 @@ _GLOBAL(htab_call_hpte_insert1)
488 mr r4,r29 /* Retreive va */ 524 mr r4,r29 /* Retreive va */
489 li r7,HPTE_V_SECONDARY /* !bolted, secondary */ 525 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
490 li r8,MMU_PAGE_4K /* page size */ 526 li r8,MMU_PAGE_4K /* page size */
527 ld r9,STK_PARM(r9)(r1) /* segment size */
491_GLOBAL(htab_call_hpte_insert2) 528_GLOBAL(htab_call_hpte_insert2)
492 bl . /* patched by htab_finish_init() */ 529 bl . /* patched by htab_finish_init() */
493 cmpdi 0,r3,0 530 cmpdi 0,r3,0
@@ -586,7 +623,8 @@ htab_modify_pte:
586 /* Call ppc_md.hpte_updatepp */ 623 /* Call ppc_md.hpte_updatepp */
587 mr r5,r29 /* va */ 624 mr r5,r29 /* va */
588 li r6,MMU_PAGE_4K /* page size */ 625 li r6,MMU_PAGE_4K /* page size */
589 ld r7,STK_PARM(r8)(r1) /* get "local" param */ 626 ld r7,STK_PARM(r9)(r1) /* segment size */
627 ld r8,STK_PARM(r8)(r1) /* get "local" param */
590_GLOBAL(htab_call_hpte_updatepp) 628_GLOBAL(htab_call_hpte_updatepp)
591 bl . /* patched by htab_finish_init() */ 629 bl . /* patched by htab_finish_init() */
592 630
@@ -634,6 +672,7 @@ _GLOBAL(__hash_page_64K)
634 /* Save all params that we need after a function call */ 672 /* Save all params that we need after a function call */
635 std r6,STK_PARM(r6)(r1) 673 std r6,STK_PARM(r6)(r1)
636 std r8,STK_PARM(r8)(r1) 674 std r8,STK_PARM(r8)(r1)
675 std r9,STK_PARM(r9)(r1)
637 676
638 /* Add _PAGE_PRESENT to access */ 677 /* Add _PAGE_PRESENT to access */
639 ori r4,r4,_PAGE_PRESENT 678 ori r4,r4,_PAGE_PRESENT
@@ -690,6 +729,10 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
690 * r4 (access) is re-useable, we use it for the new HPTE flags 729 * r4 (access) is re-useable, we use it for the new HPTE flags
691 */ 730 */
692 731
732BEGIN_FTR_SECTION
733 cmpdi r9,0 /* check segment size */
734 bne 3f
735END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
693 /* Calc va and put it in r29 */ 736 /* Calc va and put it in r29 */
694 rldicr r29,r5,28,63-28 737 rldicr r29,r5,28,63-28
695 rldicl r3,r3,0,36 738 rldicl r3,r3,0,36
@@ -699,9 +742,20 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
699 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ 742 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
700 rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */ 743 rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */
701 xor r28,r5,r0 744 xor r28,r5,r0
745 b 4f
746
7473: /* Calc VA and hash in r29 and r28 for 1T segment */
748 sldi r29,r5,40 /* vsid << 40 */
749 clrldi r3,r3,24 /* ea & 0xffffffffff */
750 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
751 clrldi r5,r5,40 /* vsid & 0xffffff */
752 rldicl r0,r3,64-16,40 /* (ea >> 16) & 0xffffff */
753 xor r28,r28,r5
754 or r29,r3,r29 /* VA */
755 xor r28,r28,r0 /* hash */
702 756
703 /* Convert linux PTE bits into HW equivalents */ 757 /* Convert linux PTE bits into HW equivalents */
704 andi. r3,r30,0x1fe /* Get basic set of flags */ 7584: andi. r3,r30,0x1fe /* Get basic set of flags */
705 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ 759 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
706 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 760 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
707 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ 761 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
@@ -756,6 +810,7 @@ ht64_insert_pte:
756 mr r4,r29 /* Retreive va */ 810 mr r4,r29 /* Retreive va */
757 li r7,0 /* !bolted, !secondary */ 811 li r7,0 /* !bolted, !secondary */
758 li r8,MMU_PAGE_64K 812 li r8,MMU_PAGE_64K
813 ld r9,STK_PARM(r9)(r1) /* segment size */
759_GLOBAL(ht64_call_hpte_insert1) 814_GLOBAL(ht64_call_hpte_insert1)
760 bl . /* patched by htab_finish_init() */ 815 bl . /* patched by htab_finish_init() */
761 cmpdi 0,r3,0 816 cmpdi 0,r3,0
@@ -778,6 +833,7 @@ _GLOBAL(ht64_call_hpte_insert1)
778 mr r4,r29 /* Retreive va */ 833 mr r4,r29 /* Retreive va */
779 li r7,HPTE_V_SECONDARY /* !bolted, secondary */ 834 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
780 li r8,MMU_PAGE_64K 835 li r8,MMU_PAGE_64K
836 ld r9,STK_PARM(r9)(r1) /* segment size */
781_GLOBAL(ht64_call_hpte_insert2) 837_GLOBAL(ht64_call_hpte_insert2)
782 bl . /* patched by htab_finish_init() */ 838 bl . /* patched by htab_finish_init() */
783 cmpdi 0,r3,0 839 cmpdi 0,r3,0
@@ -846,7 +902,8 @@ ht64_modify_pte:
846 /* Call ppc_md.hpte_updatepp */ 902 /* Call ppc_md.hpte_updatepp */
847 mr r5,r29 /* va */ 903 mr r5,r29 /* va */
848 li r6,MMU_PAGE_64K 904 li r6,MMU_PAGE_64K
849 ld r7,STK_PARM(r8)(r1) /* get "local" param */ 905 ld r7,STK_PARM(r9)(r1) /* segment size */
906 ld r8,STK_PARM(r8)(r1) /* get "local" param */
850_GLOBAL(ht64_call_hpte_updatepp) 907_GLOBAL(ht64_call_hpte_updatepp)
851 bl . /* patched by htab_finish_init() */ 908 bl . /* patched by htab_finish_init() */
852 909
diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index 6ba9b47e55af..34e5c0b219b9 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -38,7 +38,7 @@
38 38
39static DEFINE_SPINLOCK(native_tlbie_lock); 39static DEFINE_SPINLOCK(native_tlbie_lock);
40 40
41static inline void __tlbie(unsigned long va, unsigned int psize) 41static inline void __tlbie(unsigned long va, int psize, int ssize)
42{ 42{
43 unsigned int penc; 43 unsigned int penc;
44 44
@@ -48,18 +48,20 @@ static inline void __tlbie(unsigned long va, unsigned int psize)
48 switch (psize) { 48 switch (psize) {
49 case MMU_PAGE_4K: 49 case MMU_PAGE_4K:
50 va &= ~0xffful; 50 va &= ~0xffful;
51 va |= ssize << 8;
51 asm volatile("tlbie %0,0" : : "r" (va) : "memory"); 52 asm volatile("tlbie %0,0" : : "r" (va) : "memory");
52 break; 53 break;
53 default: 54 default:
54 penc = mmu_psize_defs[psize].penc; 55 penc = mmu_psize_defs[psize].penc;
55 va &= ~((1ul << mmu_psize_defs[psize].shift) - 1); 56 va &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
56 va |= penc << 12; 57 va |= penc << 12;
58 va |= ssize << 8;
57 asm volatile("tlbie %0,1" : : "r" (va) : "memory"); 59 asm volatile("tlbie %0,1" : : "r" (va) : "memory");
58 break; 60 break;
59 } 61 }
60} 62}
61 63
62static inline void __tlbiel(unsigned long va, unsigned int psize) 64static inline void __tlbiel(unsigned long va, int psize, int ssize)
63{ 65{
64 unsigned int penc; 66 unsigned int penc;
65 67
@@ -69,6 +71,7 @@ static inline void __tlbiel(unsigned long va, unsigned int psize)
69 switch (psize) { 71 switch (psize) {
70 case MMU_PAGE_4K: 72 case MMU_PAGE_4K:
71 va &= ~0xffful; 73 va &= ~0xffful;
74 va |= ssize << 8;
72 asm volatile(".long 0x7c000224 | (%0 << 11) | (0 << 21)" 75 asm volatile(".long 0x7c000224 | (%0 << 11) | (0 << 21)"
73 : : "r"(va) : "memory"); 76 : : "r"(va) : "memory");
74 break; 77 break;
@@ -76,6 +79,7 @@ static inline void __tlbiel(unsigned long va, unsigned int psize)
76 penc = mmu_psize_defs[psize].penc; 79 penc = mmu_psize_defs[psize].penc;
77 va &= ~((1ul << mmu_psize_defs[psize].shift) - 1); 80 va &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
78 va |= penc << 12; 81 va |= penc << 12;
82 va |= ssize << 8;
79 asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)" 83 asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)"
80 : : "r"(va) : "memory"); 84 : : "r"(va) : "memory");
81 break; 85 break;
@@ -83,7 +87,7 @@ static inline void __tlbiel(unsigned long va, unsigned int psize)
83 87
84} 88}
85 89
86static inline void tlbie(unsigned long va, int psize, int local) 90static inline void tlbie(unsigned long va, int psize, int ssize, int local)
87{ 91{
88 unsigned int use_local = local && cpu_has_feature(CPU_FTR_TLBIEL); 92 unsigned int use_local = local && cpu_has_feature(CPU_FTR_TLBIEL);
89 int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE); 93 int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE);
@@ -94,10 +98,10 @@ static inline void tlbie(unsigned long va, int psize, int local)
94 spin_lock(&native_tlbie_lock); 98 spin_lock(&native_tlbie_lock);
95 asm volatile("ptesync": : :"memory"); 99 asm volatile("ptesync": : :"memory");
96 if (use_local) { 100 if (use_local) {
97 __tlbiel(va, psize); 101 __tlbiel(va, psize, ssize);
98 asm volatile("ptesync": : :"memory"); 102 asm volatile("ptesync": : :"memory");
99 } else { 103 } else {
100 __tlbie(va, psize); 104 __tlbie(va, psize, ssize);
101 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 105 asm volatile("eieio; tlbsync; ptesync": : :"memory");
102 } 106 }
103 if (lock_tlbie && !use_local) 107 if (lock_tlbie && !use_local)
@@ -126,7 +130,7 @@ static inline void native_unlock_hpte(struct hash_pte *hptep)
126 130
127static long native_hpte_insert(unsigned long hpte_group, unsigned long va, 131static long native_hpte_insert(unsigned long hpte_group, unsigned long va,
128 unsigned long pa, unsigned long rflags, 132 unsigned long pa, unsigned long rflags,
129 unsigned long vflags, int psize) 133 unsigned long vflags, int psize, int ssize)
130{ 134{
131 struct hash_pte *hptep = htab_address + hpte_group; 135 struct hash_pte *hptep = htab_address + hpte_group;
132 unsigned long hpte_v, hpte_r; 136 unsigned long hpte_v, hpte_r;
@@ -153,7 +157,7 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long va,
153 if (i == HPTES_PER_GROUP) 157 if (i == HPTES_PER_GROUP)
154 return -1; 158 return -1;
155 159
156 hpte_v = hpte_encode_v(va, psize) | vflags | HPTE_V_VALID; 160 hpte_v = hpte_encode_v(va, psize, ssize) | vflags | HPTE_V_VALID;
157 hpte_r = hpte_encode_r(pa, psize) | rflags; 161 hpte_r = hpte_encode_r(pa, psize) | rflags;
158 162
159 if (!(vflags & HPTE_V_BOLTED)) { 163 if (!(vflags & HPTE_V_BOLTED)) {
@@ -215,13 +219,14 @@ static long native_hpte_remove(unsigned long hpte_group)
215} 219}
216 220
217static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, 221static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
218 unsigned long va, int psize, int local) 222 unsigned long va, int psize, int ssize,
223 int local)
219{ 224{
220 struct hash_pte *hptep = htab_address + slot; 225 struct hash_pte *hptep = htab_address + slot;
221 unsigned long hpte_v, want_v; 226 unsigned long hpte_v, want_v;
222 int ret = 0; 227 int ret = 0;
223 228
224 want_v = hpte_encode_v(va, psize); 229 want_v = hpte_encode_v(va, psize, ssize);
225 230
226 DBG_LOW(" update(va=%016lx, avpnv=%016lx, hash=%016lx, newpp=%x)", 231 DBG_LOW(" update(va=%016lx, avpnv=%016lx, hash=%016lx, newpp=%x)",
227 va, want_v & HPTE_V_AVPN, slot, newpp); 232 va, want_v & HPTE_V_AVPN, slot, newpp);
@@ -243,39 +248,32 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
243 native_unlock_hpte(hptep); 248 native_unlock_hpte(hptep);
244 249
245 /* Ensure it is out of the tlb too. */ 250 /* Ensure it is out of the tlb too. */
246 tlbie(va, psize, local); 251 tlbie(va, psize, ssize, local);
247 252
248 return ret; 253 return ret;
249} 254}
250 255
251static long native_hpte_find(unsigned long va, int psize) 256static long native_hpte_find(unsigned long va, int psize, int ssize)
252{ 257{
253 struct hash_pte *hptep; 258 struct hash_pte *hptep;
254 unsigned long hash; 259 unsigned long hash;
255 unsigned long i, j; 260 unsigned long i;
256 long slot; 261 long slot;
257 unsigned long want_v, hpte_v; 262 unsigned long want_v, hpte_v;
258 263
259 hash = hpt_hash(va, mmu_psize_defs[psize].shift); 264 hash = hpt_hash(va, mmu_psize_defs[psize].shift, ssize);
260 want_v = hpte_encode_v(va, psize); 265 want_v = hpte_encode_v(va, psize, ssize);
261 266
262 for (j = 0; j < 2; j++) { 267 /* Bolted mappings are only ever in the primary group */
263 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 268 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
264 for (i = 0; i < HPTES_PER_GROUP; i++) { 269 for (i = 0; i < HPTES_PER_GROUP; i++) {
265 hptep = htab_address + slot; 270 hptep = htab_address + slot;
266 hpte_v = hptep->v; 271 hpte_v = hptep->v;
267 272
268 if (HPTE_V_COMPARE(hpte_v, want_v) 273 if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
269 && (hpte_v & HPTE_V_VALID) 274 /* HPTE matches */
270 && ( !!(hpte_v & HPTE_V_SECONDARY) == j)) { 275 return slot;
271 /* HPTE matches */ 276 ++slot;
272 if (j)
273 slot = -slot;
274 return slot;
275 }
276 ++slot;
277 }
278 hash = ~hash;
279 } 277 }
280 278
281 return -1; 279 return -1;
@@ -289,16 +287,16 @@ static long native_hpte_find(unsigned long va, int psize)
289 * No need to lock here because we should be the only user. 287 * No need to lock here because we should be the only user.
290 */ 288 */
291static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, 289static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
292 int psize) 290 int psize, int ssize)
293{ 291{
294 unsigned long vsid, va; 292 unsigned long vsid, va;
295 long slot; 293 long slot;
296 struct hash_pte *hptep; 294 struct hash_pte *hptep;
297 295
298 vsid = get_kernel_vsid(ea); 296 vsid = get_kernel_vsid(ea, ssize);
299 va = (vsid << 28) | (ea & 0x0fffffff); 297 va = hpt_va(ea, vsid, ssize);
300 298
301 slot = native_hpte_find(va, psize); 299 slot = native_hpte_find(va, psize, ssize);
302 if (slot == -1) 300 if (slot == -1)
303 panic("could not find page to bolt\n"); 301 panic("could not find page to bolt\n");
304 hptep = htab_address + slot; 302 hptep = htab_address + slot;
@@ -308,11 +306,11 @@ static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
308 (newpp & (HPTE_R_PP | HPTE_R_N)); 306 (newpp & (HPTE_R_PP | HPTE_R_N));
309 307
310 /* Ensure it is out of the tlb too. */ 308 /* Ensure it is out of the tlb too. */
311 tlbie(va, psize, 0); 309 tlbie(va, psize, ssize, 0);
312} 310}
313 311
314static void native_hpte_invalidate(unsigned long slot, unsigned long va, 312static void native_hpte_invalidate(unsigned long slot, unsigned long va,
315 int psize, int local) 313 int psize, int ssize, int local)
316{ 314{
317 struct hash_pte *hptep = htab_address + slot; 315 struct hash_pte *hptep = htab_address + slot;
318 unsigned long hpte_v; 316 unsigned long hpte_v;
@@ -323,7 +321,7 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long va,
323 321
324 DBG_LOW(" invalidate(va=%016lx, hash: %x)\n", va, slot); 322 DBG_LOW(" invalidate(va=%016lx, hash: %x)\n", va, slot);
325 323
326 want_v = hpte_encode_v(va, psize); 324 want_v = hpte_encode_v(va, psize, ssize);
327 native_lock_hpte(hptep); 325 native_lock_hpte(hptep);
328 hpte_v = hptep->v; 326 hpte_v = hptep->v;
329 327
@@ -335,7 +333,7 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long va,
335 hptep->v = 0; 333 hptep->v = 0;
336 334
337 /* Invalidate the TLB */ 335 /* Invalidate the TLB */
338 tlbie(va, psize, local); 336 tlbie(va, psize, ssize, local);
339 337
340 local_irq_restore(flags); 338 local_irq_restore(flags);
341} 339}
@@ -345,7 +343,7 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long va,
345#define LP_MASK(i) ((0xFF >> (i)) << LP_SHIFT) 343#define LP_MASK(i) ((0xFF >> (i)) << LP_SHIFT)
346 344
347static void hpte_decode(struct hash_pte *hpte, unsigned long slot, 345static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
348 int *psize, unsigned long *va) 346 int *psize, int *ssize, unsigned long *va)
349{ 347{
350 unsigned long hpte_r = hpte->r; 348 unsigned long hpte_r = hpte->r;
351 unsigned long hpte_v = hpte->v; 349 unsigned long hpte_v = hpte->v;
@@ -401,6 +399,7 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
401 399
402 *va = avpn; 400 *va = avpn;
403 *psize = size; 401 *psize = size;
402 *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
404} 403}
405 404
406/* 405/*
@@ -417,7 +416,7 @@ static void native_hpte_clear(void)
417 struct hash_pte *hptep = htab_address; 416 struct hash_pte *hptep = htab_address;
418 unsigned long hpte_v, va; 417 unsigned long hpte_v, va;
419 unsigned long pteg_count; 418 unsigned long pteg_count;
420 int psize; 419 int psize, ssize;
421 420
422 pteg_count = htab_hash_mask + 1; 421 pteg_count = htab_hash_mask + 1;
423 422
@@ -443,9 +442,9 @@ static void native_hpte_clear(void)
443 * already hold the native_tlbie_lock. 442 * already hold the native_tlbie_lock.
444 */ 443 */
445 if (hpte_v & HPTE_V_VALID) { 444 if (hpte_v & HPTE_V_VALID) {
446 hpte_decode(hptep, slot, &psize, &va); 445 hpte_decode(hptep, slot, &psize, &ssize, &va);
447 hptep->v = 0; 446 hptep->v = 0;
448 __tlbie(va, psize); 447 __tlbie(va, psize, ssize);
449 } 448 }
450 } 449 }
451 450
@@ -468,6 +467,7 @@ static void native_flush_hash_range(unsigned long number, int local)
468 real_pte_t pte; 467 real_pte_t pte;
469 struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); 468 struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
470 unsigned long psize = batch->psize; 469 unsigned long psize = batch->psize;
470 int ssize = batch->ssize;
471 int i; 471 int i;
472 472
473 local_irq_save(flags); 473 local_irq_save(flags);
@@ -477,14 +477,14 @@ static void native_flush_hash_range(unsigned long number, int local)
477 pte = batch->pte[i]; 477 pte = batch->pte[i];
478 478
479 pte_iterate_hashed_subpages(pte, psize, va, index, shift) { 479 pte_iterate_hashed_subpages(pte, psize, va, index, shift) {
480 hash = hpt_hash(va, shift); 480 hash = hpt_hash(va, shift, ssize);
481 hidx = __rpte_to_hidx(pte, index); 481 hidx = __rpte_to_hidx(pte, index);
482 if (hidx & _PTEIDX_SECONDARY) 482 if (hidx & _PTEIDX_SECONDARY)
483 hash = ~hash; 483 hash = ~hash;
484 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 484 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
485 slot += hidx & _PTEIDX_GROUP_IX; 485 slot += hidx & _PTEIDX_GROUP_IX;
486 hptep = htab_address + slot; 486 hptep = htab_address + slot;
487 want_v = hpte_encode_v(va, psize); 487 want_v = hpte_encode_v(va, psize, ssize);
488 native_lock_hpte(hptep); 488 native_lock_hpte(hptep);
489 hpte_v = hptep->v; 489 hpte_v = hptep->v;
490 if (!HPTE_V_COMPARE(hpte_v, want_v) || 490 if (!HPTE_V_COMPARE(hpte_v, want_v) ||
@@ -504,7 +504,7 @@ static void native_flush_hash_range(unsigned long number, int local)
504 504
505 pte_iterate_hashed_subpages(pte, psize, va, index, 505 pte_iterate_hashed_subpages(pte, psize, va, index,
506 shift) { 506 shift) {
507 __tlbiel(va, psize); 507 __tlbiel(va, psize, ssize);
508 } pte_iterate_hashed_end(); 508 } pte_iterate_hashed_end();
509 } 509 }
510 asm volatile("ptesync":::"memory"); 510 asm volatile("ptesync":::"memory");
@@ -521,7 +521,7 @@ static void native_flush_hash_range(unsigned long number, int local)
521 521
522 pte_iterate_hashed_subpages(pte, psize, va, index, 522 pte_iterate_hashed_subpages(pte, psize, va, index,
523 shift) { 523 shift) {
524 __tlbie(va, psize); 524 __tlbie(va, psize, ssize);
525 } pte_iterate_hashed_end(); 525 } pte_iterate_hashed_end();
526 } 526 }
527 asm volatile("eieio; tlbsync; ptesync":::"memory"); 527 asm volatile("eieio; tlbsync; ptesync":::"memory");
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index d525f2eba313..611ad084b7e7 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -93,6 +93,8 @@ int mmu_linear_psize = MMU_PAGE_4K;
93int mmu_virtual_psize = MMU_PAGE_4K; 93int mmu_virtual_psize = MMU_PAGE_4K;
94int mmu_vmalloc_psize = MMU_PAGE_4K; 94int mmu_vmalloc_psize = MMU_PAGE_4K;
95int mmu_io_psize = MMU_PAGE_4K; 95int mmu_io_psize = MMU_PAGE_4K;
96int mmu_kernel_ssize = MMU_SEGSIZE_256M;
97int mmu_highuser_ssize = MMU_SEGSIZE_256M;
96#ifdef CONFIG_HUGETLB_PAGE 98#ifdef CONFIG_HUGETLB_PAGE
97int mmu_huge_psize = MMU_PAGE_16M; 99int mmu_huge_psize = MMU_PAGE_16M;
98unsigned int HPAGE_SHIFT; 100unsigned int HPAGE_SHIFT;
@@ -145,7 +147,8 @@ struct mmu_psize_def mmu_psize_defaults_gp[] = {
145 147
146 148
147int htab_bolt_mapping(unsigned long vstart, unsigned long vend, 149int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
148 unsigned long pstart, unsigned long mode, int psize) 150 unsigned long pstart, unsigned long mode,
151 int psize, int ssize)
149{ 152{
150 unsigned long vaddr, paddr; 153 unsigned long vaddr, paddr;
151 unsigned int step, shift; 154 unsigned int step, shift;
@@ -158,8 +161,8 @@ int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
158 for (vaddr = vstart, paddr = pstart; vaddr < vend; 161 for (vaddr = vstart, paddr = pstart; vaddr < vend;
159 vaddr += step, paddr += step) { 162 vaddr += step, paddr += step) {
160 unsigned long hash, hpteg; 163 unsigned long hash, hpteg;
161 unsigned long vsid = get_kernel_vsid(vaddr); 164 unsigned long vsid = get_kernel_vsid(vaddr, ssize);
162 unsigned long va = (vsid << 28) | (vaddr & 0x0fffffff); 165 unsigned long va = hpt_va(vaddr, vsid, ssize);
163 166
164 tmp_mode = mode; 167 tmp_mode = mode;
165 168
@@ -167,14 +170,14 @@ int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
167 if (!in_kernel_text(vaddr)) 170 if (!in_kernel_text(vaddr))
168 tmp_mode = mode | HPTE_R_N; 171 tmp_mode = mode | HPTE_R_N;
169 172
170 hash = hpt_hash(va, shift); 173 hash = hpt_hash(va, shift, ssize);
171 hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP); 174 hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP);
172 175
173 DBG("htab_bolt_mapping: calling %p\n", ppc_md.hpte_insert); 176 DBG("htab_bolt_mapping: calling %p\n", ppc_md.hpte_insert);
174 177
175 BUG_ON(!ppc_md.hpte_insert); 178 BUG_ON(!ppc_md.hpte_insert);
176 ret = ppc_md.hpte_insert(hpteg, va, paddr, 179 ret = ppc_md.hpte_insert(hpteg, va, paddr,
177 tmp_mode, HPTE_V_BOLTED, psize); 180 tmp_mode, HPTE_V_BOLTED, psize, ssize);
178 181
179 if (ret < 0) 182 if (ret < 0)
180 break; 183 break;
@@ -186,6 +189,37 @@ int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
186 return ret < 0 ? ret : 0; 189 return ret < 0 ? ret : 0;
187} 190}
188 191
192static int __init htab_dt_scan_seg_sizes(unsigned long node,
193 const char *uname, int depth,
194 void *data)
195{
196 char *type = of_get_flat_dt_prop(node, "device_type", NULL);
197 u32 *prop;
198 unsigned long size = 0;
199
200 /* We are scanning "cpu" nodes only */
201 if (type == NULL || strcmp(type, "cpu") != 0)
202 return 0;
203
204 prop = (u32 *)of_get_flat_dt_prop(node, "ibm,processor-segment-sizes",
205 &size);
206 if (prop == NULL)
207 return 0;
208 for (; size >= 4; size -= 4, ++prop) {
209 if (prop[0] == 40) {
210 DBG("1T segment support detected\n");
211 cur_cpu_spec->cpu_features |= CPU_FTR_1T_SEGMENT;
212 }
213 return 1;
214 }
215 return 0;
216}
217
218static void __init htab_init_seg_sizes(void)
219{
220 of_scan_flat_dt(htab_dt_scan_seg_sizes, NULL);
221}
222
189static int __init htab_dt_scan_page_sizes(unsigned long node, 223static int __init htab_dt_scan_page_sizes(unsigned long node,
190 const char *uname, int depth, 224 const char *uname, int depth,
191 void *data) 225 void *data)
@@ -265,7 +299,6 @@ static int __init htab_dt_scan_page_sizes(unsigned long node,
265 return 0; 299 return 0;
266} 300}
267 301
268
269static void __init htab_init_page_sizes(void) 302static void __init htab_init_page_sizes(void)
270{ 303{
271 int rc; 304 int rc;
@@ -398,7 +431,7 @@ void create_section_mapping(unsigned long start, unsigned long end)
398{ 431{
399 BUG_ON(htab_bolt_mapping(start, end, __pa(start), 432 BUG_ON(htab_bolt_mapping(start, end, __pa(start),
400 _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_COHERENT | PP_RWXX, 433 _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_COHERENT | PP_RWXX,
401 mmu_linear_psize)); 434 mmu_linear_psize, mmu_kernel_ssize));
402} 435}
403#endif /* CONFIG_MEMORY_HOTPLUG */ 436#endif /* CONFIG_MEMORY_HOTPLUG */
404 437
@@ -449,9 +482,18 @@ void __init htab_initialize(void)
449 482
450 DBG(" -> htab_initialize()\n"); 483 DBG(" -> htab_initialize()\n");
451 484
485 /* Initialize segment sizes */
486 htab_init_seg_sizes();
487
452 /* Initialize page sizes */ 488 /* Initialize page sizes */
453 htab_init_page_sizes(); 489 htab_init_page_sizes();
454 490
491 if (cpu_has_feature(CPU_FTR_1T_SEGMENT)) {
492 mmu_kernel_ssize = MMU_SEGSIZE_1T;
493 mmu_highuser_ssize = MMU_SEGSIZE_1T;
494 printk(KERN_INFO "Using 1TB segments\n");
495 }
496
455 /* 497 /*
456 * Calculate the required size of the htab. We want the number of 498 * Calculate the required size of the htab. We want the number of
457 * PTEGs to equal one half the number of real pages. 499 * PTEGs to equal one half the number of real pages.
@@ -523,18 +565,20 @@ void __init htab_initialize(void)
523 if (base != dart_tablebase) 565 if (base != dart_tablebase)
524 BUG_ON(htab_bolt_mapping(base, dart_tablebase, 566 BUG_ON(htab_bolt_mapping(base, dart_tablebase,
525 __pa(base), mode_rw, 567 __pa(base), mode_rw,
526 mmu_linear_psize)); 568 mmu_linear_psize,
569 mmu_kernel_ssize));
527 if ((base + size) > dart_table_end) 570 if ((base + size) > dart_table_end)
528 BUG_ON(htab_bolt_mapping(dart_tablebase+16*MB, 571 BUG_ON(htab_bolt_mapping(dart_tablebase+16*MB,
529 base + size, 572 base + size,
530 __pa(dart_table_end), 573 __pa(dart_table_end),
531 mode_rw, 574 mode_rw,
532 mmu_linear_psize)); 575 mmu_linear_psize,
576 mmu_kernel_ssize));
533 continue; 577 continue;
534 } 578 }
535#endif /* CONFIG_U3_DART */ 579#endif /* CONFIG_U3_DART */
536 BUG_ON(htab_bolt_mapping(base, base + size, __pa(base), 580 BUG_ON(htab_bolt_mapping(base, base + size, __pa(base),
537 mode_rw, mmu_linear_psize)); 581 mode_rw, mmu_linear_psize, mmu_kernel_ssize));
538 } 582 }
539 583
540 /* 584 /*
@@ -553,7 +597,7 @@ void __init htab_initialize(void)
553 597
554 BUG_ON(htab_bolt_mapping(tce_alloc_start, tce_alloc_end, 598 BUG_ON(htab_bolt_mapping(tce_alloc_start, tce_alloc_end,
555 __pa(tce_alloc_start), mode_rw, 599 __pa(tce_alloc_start), mode_rw,
556 mmu_linear_psize)); 600 mmu_linear_psize, mmu_kernel_ssize));
557 } 601 }
558 602
559 htab_finish_init(); 603 htab_finish_init();
@@ -621,7 +665,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
621 pte_t *ptep; 665 pte_t *ptep;
622 cpumask_t tmp; 666 cpumask_t tmp;
623 int rc, user_region = 0, local = 0; 667 int rc, user_region = 0, local = 0;
624 int psize; 668 int psize, ssize;
625 669
626 DBG_LOW("hash_page(ea=%016lx, access=%lx, trap=%lx\n", 670 DBG_LOW("hash_page(ea=%016lx, access=%lx, trap=%lx\n",
627 ea, access, trap); 671 ea, access, trap);
@@ -640,20 +684,22 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
640 DBG_LOW(" user region with no mm !\n"); 684 DBG_LOW(" user region with no mm !\n");
641 return 1; 685 return 1;
642 } 686 }
643 vsid = get_vsid(mm->context.id, ea);
644#ifdef CONFIG_PPC_MM_SLICES 687#ifdef CONFIG_PPC_MM_SLICES
645 psize = get_slice_psize(mm, ea); 688 psize = get_slice_psize(mm, ea);
646#else 689#else
647 psize = mm->context.user_psize; 690 psize = mm->context.user_psize;
648#endif 691#endif
692 ssize = user_segment_size(ea);
693 vsid = get_vsid(mm->context.id, ea, ssize);
649 break; 694 break;
650 case VMALLOC_REGION_ID: 695 case VMALLOC_REGION_ID:
651 mm = &init_mm; 696 mm = &init_mm;
652 vsid = get_kernel_vsid(ea); 697 vsid = get_kernel_vsid(ea, mmu_kernel_ssize);
653 if (ea < VMALLOC_END) 698 if (ea < VMALLOC_END)
654 psize = mmu_vmalloc_psize; 699 psize = mmu_vmalloc_psize;
655 else 700 else
656 psize = mmu_io_psize; 701 psize = mmu_io_psize;
702 ssize = mmu_kernel_ssize;
657 break; 703 break;
658 default: 704 default:
659 /* Not a valid range 705 /* Not a valid range
@@ -758,10 +804,10 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
758 804
759#ifdef CONFIG_PPC_HAS_HASH_64K 805#ifdef CONFIG_PPC_HAS_HASH_64K
760 if (psize == MMU_PAGE_64K) 806 if (psize == MMU_PAGE_64K)
761 rc = __hash_page_64K(ea, access, vsid, ptep, trap, local); 807 rc = __hash_page_64K(ea, access, vsid, ptep, trap, local, ssize);
762 else 808 else
763#endif /* CONFIG_PPC_HAS_HASH_64K */ 809#endif /* CONFIG_PPC_HAS_HASH_64K */
764 rc = __hash_page_4K(ea, access, vsid, ptep, trap, local); 810 rc = __hash_page_4K(ea, access, vsid, ptep, trap, local, ssize);
765 811
766#ifndef CONFIG_PPC_64K_PAGES 812#ifndef CONFIG_PPC_64K_PAGES
767 DBG_LOW(" o-pte: %016lx\n", pte_val(*ptep)); 813 DBG_LOW(" o-pte: %016lx\n", pte_val(*ptep));
@@ -783,6 +829,7 @@ void hash_preload(struct mm_struct *mm, unsigned long ea,
783 cpumask_t mask; 829 cpumask_t mask;
784 unsigned long flags; 830 unsigned long flags;
785 int local = 0; 831 int local = 0;
832 int ssize;
786 833
787 BUG_ON(REGION_ID(ea) != USER_REGION_ID); 834 BUG_ON(REGION_ID(ea) != USER_REGION_ID);
788 835
@@ -815,7 +862,8 @@ void hash_preload(struct mm_struct *mm, unsigned long ea,
815#endif /* CONFIG_PPC_64K_PAGES */ 862#endif /* CONFIG_PPC_64K_PAGES */
816 863
817 /* Get VSID */ 864 /* Get VSID */
818 vsid = get_vsid(mm->context.id, ea); 865 ssize = user_segment_size(ea);
866 vsid = get_vsid(mm->context.id, ea, ssize);
819 867
820 /* Hash doesn't like irqs */ 868 /* Hash doesn't like irqs */
821 local_irq_save(flags); 869 local_irq_save(flags);
@@ -828,28 +876,29 @@ void hash_preload(struct mm_struct *mm, unsigned long ea,
828 /* Hash it in */ 876 /* Hash it in */
829#ifdef CONFIG_PPC_HAS_HASH_64K 877#ifdef CONFIG_PPC_HAS_HASH_64K
830 if (mm->context.user_psize == MMU_PAGE_64K) 878 if (mm->context.user_psize == MMU_PAGE_64K)
831 __hash_page_64K(ea, access, vsid, ptep, trap, local); 879 __hash_page_64K(ea, access, vsid, ptep, trap, local, ssize);
832 else 880 else
833#endif /* CONFIG_PPC_HAS_HASH_64K */ 881#endif /* CONFIG_PPC_HAS_HASH_64K */
834 __hash_page_4K(ea, access, vsid, ptep, trap, local); 882 __hash_page_4K(ea, access, vsid, ptep, trap, local, ssize);
835 883
836 local_irq_restore(flags); 884 local_irq_restore(flags);
837} 885}
838 886
839void flush_hash_page(unsigned long va, real_pte_t pte, int psize, int local) 887void flush_hash_page(unsigned long va, real_pte_t pte, int psize, int ssize,
888 int local)
840{ 889{
841 unsigned long hash, index, shift, hidx, slot; 890 unsigned long hash, index, shift, hidx, slot;
842 891
843 DBG_LOW("flush_hash_page(va=%016x)\n", va); 892 DBG_LOW("flush_hash_page(va=%016x)\n", va);
844 pte_iterate_hashed_subpages(pte, psize, va, index, shift) { 893 pte_iterate_hashed_subpages(pte, psize, va, index, shift) {
845 hash = hpt_hash(va, shift); 894 hash = hpt_hash(va, shift, ssize);
846 hidx = __rpte_to_hidx(pte, index); 895 hidx = __rpte_to_hidx(pte, index);
847 if (hidx & _PTEIDX_SECONDARY) 896 if (hidx & _PTEIDX_SECONDARY)
848 hash = ~hash; 897 hash = ~hash;
849 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 898 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
850 slot += hidx & _PTEIDX_GROUP_IX; 899 slot += hidx & _PTEIDX_GROUP_IX;
851 DBG_LOW(" sub %d: hash=%x, hidx=%x\n", index, slot, hidx); 900 DBG_LOW(" sub %d: hash=%x, hidx=%x\n", index, slot, hidx);
852 ppc_md.hpte_invalidate(slot, va, psize, local); 901 ppc_md.hpte_invalidate(slot, va, psize, ssize, local);
853 } pte_iterate_hashed_end(); 902 } pte_iterate_hashed_end();
854} 903}
855 904
@@ -864,7 +913,7 @@ void flush_hash_range(unsigned long number, int local)
864 913
865 for (i = 0; i < number; i++) 914 for (i = 0; i < number; i++)
866 flush_hash_page(batch->vaddr[i], batch->pte[i], 915 flush_hash_page(batch->vaddr[i], batch->pte[i],
867 batch->psize, local); 916 batch->psize, batch->ssize, local);
868 } 917 }
869} 918}
870 919
@@ -890,17 +939,19 @@ void low_hash_fault(struct pt_regs *regs, unsigned long address)
890#ifdef CONFIG_DEBUG_PAGEALLOC 939#ifdef CONFIG_DEBUG_PAGEALLOC
891static void kernel_map_linear_page(unsigned long vaddr, unsigned long lmi) 940static void kernel_map_linear_page(unsigned long vaddr, unsigned long lmi)
892{ 941{
893 unsigned long hash, hpteg, vsid = get_kernel_vsid(vaddr); 942 unsigned long hash, hpteg;
894 unsigned long va = (vsid << 28) | (vaddr & 0x0fffffff); 943 unsigned long vsid = get_kernel_vsid(vaddr, mmu_kernel_ssize);
944 unsigned long va = hpt_va(vaddr, vsid, mmu_kernel_ssize);
895 unsigned long mode = _PAGE_ACCESSED | _PAGE_DIRTY | 945 unsigned long mode = _PAGE_ACCESSED | _PAGE_DIRTY |
896 _PAGE_COHERENT | PP_RWXX | HPTE_R_N; 946 _PAGE_COHERENT | PP_RWXX | HPTE_R_N;
897 int ret; 947 int ret;
898 948
899 hash = hpt_hash(va, PAGE_SHIFT); 949 hash = hpt_hash(va, PAGE_SHIFT, mmu_kernel_ssize);
900 hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP); 950 hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP);
901 951
902 ret = ppc_md.hpte_insert(hpteg, va, __pa(vaddr), 952 ret = ppc_md.hpte_insert(hpteg, va, __pa(vaddr),
903 mode, HPTE_V_BOLTED, mmu_linear_psize); 953 mode, HPTE_V_BOLTED,
954 mmu_linear_psize, mmu_kernel_ssize);
904 BUG_ON (ret < 0); 955 BUG_ON (ret < 0);
905 spin_lock(&linear_map_hash_lock); 956 spin_lock(&linear_map_hash_lock);
906 BUG_ON(linear_map_hash_slots[lmi] & 0x80); 957 BUG_ON(linear_map_hash_slots[lmi] & 0x80);
@@ -910,10 +961,11 @@ static void kernel_map_linear_page(unsigned long vaddr, unsigned long lmi)
910 961
911static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi) 962static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi)
912{ 963{
913 unsigned long hash, hidx, slot, vsid = get_kernel_vsid(vaddr); 964 unsigned long hash, hidx, slot;
914 unsigned long va = (vsid << 28) | (vaddr & 0x0fffffff); 965 unsigned long vsid = get_kernel_vsid(vaddr, mmu_kernel_ssize);
966 unsigned long va = hpt_va(vaddr, vsid, mmu_kernel_ssize);
915 967
916 hash = hpt_hash(va, PAGE_SHIFT); 968 hash = hpt_hash(va, PAGE_SHIFT, mmu_kernel_ssize);
917 spin_lock(&linear_map_hash_lock); 969 spin_lock(&linear_map_hash_lock);
918 BUG_ON(!(linear_map_hash_slots[lmi] & 0x80)); 970 BUG_ON(!(linear_map_hash_slots[lmi] & 0x80));
919 hidx = linear_map_hash_slots[lmi] & 0x7f; 971 hidx = linear_map_hash_slots[lmi] & 0x7f;
@@ -923,7 +975,7 @@ static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi)
923 hash = ~hash; 975 hash = ~hash;
924 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 976 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
925 slot += hidx & _PTEIDX_GROUP_IX; 977 slot += hidx & _PTEIDX_GROUP_IX;
926 ppc_md.hpte_invalidate(slot, va, mmu_linear_psize, 0); 978 ppc_md.hpte_invalidate(slot, va, mmu_linear_psize, mmu_kernel_ssize, 0);
927} 979}
928 980
929void kernel_map_pages(struct page *page, int numpages, int enable) 981void kernel_map_pages(struct page *page, int numpages, int enable)
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index ba5f12a60467..08f0d9ff7712 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -403,11 +403,12 @@ int hash_huge_page(struct mm_struct *mm, unsigned long access,
403 unsigned long va, rflags, pa; 403 unsigned long va, rflags, pa;
404 long slot; 404 long slot;
405 int err = 1; 405 int err = 1;
406 int ssize = user_segment_size(ea);
406 407
407 ptep = huge_pte_offset(mm, ea); 408 ptep = huge_pte_offset(mm, ea);
408 409
409 /* Search the Linux page table for a match with va */ 410 /* Search the Linux page table for a match with va */
410 va = (vsid << 28) | (ea & 0x0fffffff); 411 va = hpt_va(ea, vsid, ssize);
411 412
412 /* 413 /*
413 * If no pte found or not present, send the problem up to 414 * If no pte found or not present, send the problem up to
@@ -458,19 +459,19 @@ int hash_huge_page(struct mm_struct *mm, unsigned long access,
458 /* There MIGHT be an HPTE for this pte */ 459 /* There MIGHT be an HPTE for this pte */
459 unsigned long hash, slot; 460 unsigned long hash, slot;
460 461
461 hash = hpt_hash(va, HPAGE_SHIFT); 462 hash = hpt_hash(va, HPAGE_SHIFT, ssize);
462 if (old_pte & _PAGE_F_SECOND) 463 if (old_pte & _PAGE_F_SECOND)
463 hash = ~hash; 464 hash = ~hash;
464 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 465 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
465 slot += (old_pte & _PAGE_F_GIX) >> 12; 466 slot += (old_pte & _PAGE_F_GIX) >> 12;
466 467
467 if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_huge_psize, 468 if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_huge_psize,
468 local) == -1) 469 ssize, local) == -1)
469 old_pte &= ~_PAGE_HPTEFLAGS; 470 old_pte &= ~_PAGE_HPTEFLAGS;
470 } 471 }
471 472
472 if (likely(!(old_pte & _PAGE_HASHPTE))) { 473 if (likely(!(old_pte & _PAGE_HASHPTE))) {
473 unsigned long hash = hpt_hash(va, HPAGE_SHIFT); 474 unsigned long hash = hpt_hash(va, HPAGE_SHIFT, ssize);
474 unsigned long hpte_group; 475 unsigned long hpte_group;
475 476
476 pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT; 477 pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
@@ -489,7 +490,7 @@ repeat:
489 490
490 /* Insert into the hash table, primary slot */ 491 /* Insert into the hash table, primary slot */
491 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0, 492 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0,
492 mmu_huge_psize); 493 mmu_huge_psize, ssize);
493 494
494 /* Primary is full, try the secondary */ 495 /* Primary is full, try the secondary */
495 if (unlikely(slot == -1)) { 496 if (unlikely(slot == -1)) {
@@ -497,7 +498,7 @@ repeat:
497 HPTES_PER_GROUP) & ~0x7UL; 498 HPTES_PER_GROUP) & ~0x7UL;
498 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 499 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags,
499 HPTE_V_SECONDARY, 500 HPTE_V_SECONDARY,
500 mmu_huge_psize); 501 mmu_huge_psize, ssize);
501 if (slot == -1) { 502 if (slot == -1) {
502 if (mftb() & 0x1) 503 if (mftb() & 0x1)
503 hpte_group = ((hash & htab_hash_mask) * 504 hpte_group = ((hash & htab_hash_mask) *
diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index 60fd52cd270f..3ef0ad2f9ca0 100644
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -87,8 +87,8 @@ static int map_io_page(unsigned long ea, unsigned long pa, int flags)
87 * entry in the hardware page table. 87 * entry in the hardware page table.
88 * 88 *
89 */ 89 */
90 if (htab_bolt_mapping(ea, (unsigned long)ea + PAGE_SIZE, 90 if (htab_bolt_mapping(ea, ea + PAGE_SIZE, pa, flags,
91 pa, flags, mmu_io_psize)) { 91 mmu_io_psize, mmu_kernel_ssize)) {
92 printk(KERN_ERR "Failed to do bolted mapping IO " 92 printk(KERN_ERR "Failed to do bolted mapping IO "
93 "memory at %016lx !\n", pa); 93 "memory at %016lx !\n", pa);
94 return -ENOMEM; 94 return -ENOMEM;
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index 4bee1cfa9dea..6c164cec9d2c 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -43,17 +43,26 @@ static void slb_allocate(unsigned long ea)
43 slb_allocate_realmode(ea); 43 slb_allocate_realmode(ea);
44} 44}
45 45
46static inline unsigned long mk_esid_data(unsigned long ea, unsigned long slot) 46static inline unsigned long mk_esid_data(unsigned long ea, int ssize,
47 unsigned long slot)
47{ 48{
48 return (ea & ESID_MASK) | SLB_ESID_V | slot; 49 unsigned long mask;
50
51 mask = (ssize == MMU_SEGSIZE_256M)? ESID_MASK: ESID_MASK_1T;
52 return (ea & mask) | SLB_ESID_V | slot;
49} 53}
50 54
51static inline unsigned long mk_vsid_data(unsigned long ea, unsigned long flags) 55#define slb_vsid_shift(ssize) \
56 ((ssize) == MMU_SEGSIZE_256M? SLB_VSID_SHIFT: SLB_VSID_SHIFT_1T)
57
58static inline unsigned long mk_vsid_data(unsigned long ea, int ssize,
59 unsigned long flags)
52{ 60{
53 return (get_kernel_vsid(ea) << SLB_VSID_SHIFT) | flags; 61 return (get_kernel_vsid(ea, ssize) << slb_vsid_shift(ssize)) | flags |
62 ((unsigned long) ssize << SLB_VSID_SSIZE_SHIFT);
54} 63}
55 64
56static inline void slb_shadow_update(unsigned long ea, 65static inline void slb_shadow_update(unsigned long ea, int ssize,
57 unsigned long flags, 66 unsigned long flags,
58 unsigned long entry) 67 unsigned long entry)
59{ 68{
@@ -63,8 +72,8 @@ static inline void slb_shadow_update(unsigned long ea,
63 * we only update the current CPU's SLB shadow buffer. 72 * we only update the current CPU's SLB shadow buffer.
64 */ 73 */
65 get_slb_shadow()->save_area[entry].esid = 0; 74 get_slb_shadow()->save_area[entry].esid = 0;
66 get_slb_shadow()->save_area[entry].vsid = mk_vsid_data(ea, flags); 75 get_slb_shadow()->save_area[entry].vsid = mk_vsid_data(ea, ssize, flags);
67 get_slb_shadow()->save_area[entry].esid = mk_esid_data(ea, entry); 76 get_slb_shadow()->save_area[entry].esid = mk_esid_data(ea, ssize, entry);
68} 77}
69 78
70static inline void slb_shadow_clear(unsigned long entry) 79static inline void slb_shadow_clear(unsigned long entry)
@@ -72,7 +81,8 @@ static inline void slb_shadow_clear(unsigned long entry)
72 get_slb_shadow()->save_area[entry].esid = 0; 81 get_slb_shadow()->save_area[entry].esid = 0;
73} 82}
74 83
75static inline void create_shadowed_slbe(unsigned long ea, unsigned long flags, 84static inline void create_shadowed_slbe(unsigned long ea, int ssize,
85 unsigned long flags,
76 unsigned long entry) 86 unsigned long entry)
77{ 87{
78 /* 88 /*
@@ -80,11 +90,11 @@ static inline void create_shadowed_slbe(unsigned long ea, unsigned long flags,
80 * we don't get a stale entry here if we get preempted by PHYP 90 * we don't get a stale entry here if we get preempted by PHYP
81 * between these two statements. 91 * between these two statements.
82 */ 92 */
83 slb_shadow_update(ea, flags, entry); 93 slb_shadow_update(ea, ssize, flags, entry);
84 94
85 asm volatile("slbmte %0,%1" : 95 asm volatile("slbmte %0,%1" :
86 : "r" (mk_vsid_data(ea, flags)), 96 : "r" (mk_vsid_data(ea, ssize, flags)),
87 "r" (mk_esid_data(ea, entry)) 97 "r" (mk_esid_data(ea, ssize, entry))
88 : "memory" ); 98 : "memory" );
89} 99}
90 100
@@ -93,7 +103,7 @@ void slb_flush_and_rebolt(void)
93 /* If you change this make sure you change SLB_NUM_BOLTED 103 /* If you change this make sure you change SLB_NUM_BOLTED
94 * appropriately too. */ 104 * appropriately too. */
95 unsigned long linear_llp, vmalloc_llp, lflags, vflags; 105 unsigned long linear_llp, vmalloc_llp, lflags, vflags;
96 unsigned long ksp_esid_data; 106 unsigned long ksp_esid_data, ksp_vsid_data;
97 107
98 WARN_ON(!irqs_disabled()); 108 WARN_ON(!irqs_disabled());
99 109
@@ -102,13 +112,15 @@ void slb_flush_and_rebolt(void)
102 lflags = SLB_VSID_KERNEL | linear_llp; 112 lflags = SLB_VSID_KERNEL | linear_llp;
103 vflags = SLB_VSID_KERNEL | vmalloc_llp; 113 vflags = SLB_VSID_KERNEL | vmalloc_llp;
104 114
105 ksp_esid_data = mk_esid_data(get_paca()->kstack, 2); 115 ksp_esid_data = mk_esid_data(get_paca()->kstack, mmu_kernel_ssize, 2);
106 if ((ksp_esid_data & ESID_MASK) == PAGE_OFFSET) { 116 if ((ksp_esid_data & ~0xfffffffUL) <= PAGE_OFFSET) {
107 ksp_esid_data &= ~SLB_ESID_V; 117 ksp_esid_data &= ~SLB_ESID_V;
118 ksp_vsid_data = 0;
108 slb_shadow_clear(2); 119 slb_shadow_clear(2);
109 } else { 120 } else {
110 /* Update stack entry; others don't change */ 121 /* Update stack entry; others don't change */
111 slb_shadow_update(get_paca()->kstack, lflags, 2); 122 slb_shadow_update(get_paca()->kstack, mmu_kernel_ssize, lflags, 2);
123 ksp_vsid_data = get_slb_shadow()->save_area[2].vsid;
112 } 124 }
113 125
114 /* We need to do this all in asm, so we're sure we don't touch 126 /* We need to do this all in asm, so we're sure we don't touch
@@ -120,9 +132,9 @@ void slb_flush_and_rebolt(void)
120 /* Slot 2 - kernel stack */ 132 /* Slot 2 - kernel stack */
121 "slbmte %2,%3\n" 133 "slbmte %2,%3\n"
122 "isync" 134 "isync"
123 :: "r"(mk_vsid_data(VMALLOC_START, vflags)), 135 :: "r"(mk_vsid_data(VMALLOC_START, mmu_kernel_ssize, vflags)),
124 "r"(mk_esid_data(VMALLOC_START, 1)), 136 "r"(mk_esid_data(VMALLOC_START, mmu_kernel_ssize, 1)),
125 "r"(mk_vsid_data(ksp_esid_data, lflags)), 137 "r"(ksp_vsid_data),
126 "r"(ksp_esid_data) 138 "r"(ksp_esid_data)
127 : "memory"); 139 : "memory");
128} 140}
@@ -132,7 +144,7 @@ void slb_vmalloc_update(void)
132 unsigned long vflags; 144 unsigned long vflags;
133 145
134 vflags = SLB_VSID_KERNEL | mmu_psize_defs[mmu_vmalloc_psize].sllp; 146 vflags = SLB_VSID_KERNEL | mmu_psize_defs[mmu_vmalloc_psize].sllp;
135 slb_shadow_update(VMALLOC_START, vflags, 1); 147 slb_shadow_update(VMALLOC_START, mmu_kernel_ssize, vflags, 1);
136 slb_flush_and_rebolt(); 148 slb_flush_and_rebolt();
137} 149}
138 150
@@ -140,7 +152,7 @@ void slb_vmalloc_update(void)
140void switch_slb(struct task_struct *tsk, struct mm_struct *mm) 152void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
141{ 153{
142 unsigned long offset = get_paca()->slb_cache_ptr; 154 unsigned long offset = get_paca()->slb_cache_ptr;
143 unsigned long esid_data = 0; 155 unsigned long slbie_data = 0;
144 unsigned long pc = KSTK_EIP(tsk); 156 unsigned long pc = KSTK_EIP(tsk);
145 unsigned long stack = KSTK_ESP(tsk); 157 unsigned long stack = KSTK_ESP(tsk);
146 unsigned long unmapped_base; 158 unsigned long unmapped_base;
@@ -149,9 +161,12 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
149 int i; 161 int i;
150 asm volatile("isync" : : : "memory"); 162 asm volatile("isync" : : : "memory");
151 for (i = 0; i < offset; i++) { 163 for (i = 0; i < offset; i++) {
152 esid_data = ((unsigned long)get_paca()->slb_cache[i] 164 slbie_data = (unsigned long)get_paca()->slb_cache[i]
153 << SID_SHIFT) | SLBIE_C; 165 << SID_SHIFT; /* EA */
154 asm volatile("slbie %0" : : "r" (esid_data)); 166 slbie_data |= user_segment_size(slbie_data)
167 << SLBIE_SSIZE_SHIFT;
168 slbie_data |= SLBIE_C; /* C set for user addresses */
169 asm volatile("slbie %0" : : "r" (slbie_data));
155 } 170 }
156 asm volatile("isync" : : : "memory"); 171 asm volatile("isync" : : : "memory");
157 } else { 172 } else {
@@ -160,7 +175,7 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
160 175
161 /* Workaround POWER5 < DD2.1 issue */ 176 /* Workaround POWER5 < DD2.1 issue */
162 if (offset == 1 || offset > SLB_CACHE_ENTRIES) 177 if (offset == 1 || offset > SLB_CACHE_ENTRIES)
163 asm volatile("slbie %0" : : "r" (esid_data)); 178 asm volatile("slbie %0" : : "r" (slbie_data));
164 179
165 get_paca()->slb_cache_ptr = 0; 180 get_paca()->slb_cache_ptr = 0;
166 get_paca()->context = mm->context; 181 get_paca()->context = mm->context;
@@ -243,9 +258,9 @@ void slb_initialize(void)
243 asm volatile("isync":::"memory"); 258 asm volatile("isync":::"memory");
244 asm volatile("slbmte %0,%0"::"r" (0) : "memory"); 259 asm volatile("slbmte %0,%0"::"r" (0) : "memory");
245 asm volatile("isync; slbia; isync":::"memory"); 260 asm volatile("isync; slbia; isync":::"memory");
246 create_shadowed_slbe(PAGE_OFFSET, lflags, 0); 261 create_shadowed_slbe(PAGE_OFFSET, mmu_kernel_ssize, lflags, 0);
247 262
248 create_shadowed_slbe(VMALLOC_START, vflags, 1); 263 create_shadowed_slbe(VMALLOC_START, mmu_kernel_ssize, vflags, 1);
249 264
250 /* We don't bolt the stack for the time being - we're in boot, 265 /* We don't bolt the stack for the time being - we're in boot,
251 * so the stack is in the bolted segment. By the time it goes 266 * so the stack is in the bolted segment. By the time it goes
diff --git a/arch/powerpc/mm/slb_low.S b/arch/powerpc/mm/slb_low.S
index cd1a93d4948c..1328a81a84aa 100644
--- a/arch/powerpc/mm/slb_low.S
+++ b/arch/powerpc/mm/slb_low.S
@@ -57,7 +57,10 @@ _GLOBAL(slb_allocate_realmode)
57 */ 57 */
58_GLOBAL(slb_miss_kernel_load_linear) 58_GLOBAL(slb_miss_kernel_load_linear)
59 li r11,0 59 li r11,0
60BEGIN_FTR_SECTION
60 b slb_finish_load 61 b slb_finish_load
62END_FTR_SECTION_IFCLR(CPU_FTR_1T_SEGMENT)
63 b slb_finish_load_1T
61 64
621: /* vmalloc/ioremap mapping encoding bits, the "li" instructions below 651: /* vmalloc/ioremap mapping encoding bits, the "li" instructions below
63 * will be patched by the kernel at boot 66 * will be patched by the kernel at boot
@@ -68,13 +71,16 @@ BEGIN_FTR_SECTION
68 cmpldi r11,(VMALLOC_SIZE >> 28) - 1 71 cmpldi r11,(VMALLOC_SIZE >> 28) - 1
69 bgt 5f 72 bgt 5f
70 lhz r11,PACAVMALLOCSLLP(r13) 73 lhz r11,PACAVMALLOCSLLP(r13)
71 b slb_finish_load 74 b 6f
725: 755:
73END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE) 76END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
74_GLOBAL(slb_miss_kernel_load_io) 77_GLOBAL(slb_miss_kernel_load_io)
75 li r11,0 78 li r11,0
796:
80BEGIN_FTR_SECTION
76 b slb_finish_load 81 b slb_finish_load
77 82END_FTR_SECTION_IFCLR(CPU_FTR_1T_SEGMENT)
83 b slb_finish_load_1T
78 84
790: /* user address: proto-VSID = context << 15 | ESID. First check 850: /* user address: proto-VSID = context << 15 | ESID. First check
80 * if the address is within the boundaries of the user region 86 * if the address is within the boundaries of the user region
@@ -122,7 +128,13 @@ _GLOBAL(slb_miss_kernel_load_io)
122#endif /* CONFIG_PPC_MM_SLICES */ 128#endif /* CONFIG_PPC_MM_SLICES */
123 129
124 ld r9,PACACONTEXTID(r13) 130 ld r9,PACACONTEXTID(r13)
131BEGIN_FTR_SECTION
132 cmpldi r10,0x1000
133END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
125 rldimi r10,r9,USER_ESID_BITS,0 134 rldimi r10,r9,USER_ESID_BITS,0
135BEGIN_FTR_SECTION
136 bge slb_finish_load_1T
137END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
126 b slb_finish_load 138 b slb_finish_load
127 139
1288: /* invalid EA */ 1408: /* invalid EA */
@@ -188,7 +200,7 @@ _GLOBAL(slb_allocate_user)
188 * r3 = EA, r10 = proto-VSID, r11 = flags, clobbers r9, cr7 = <> PAGE_OFFSET 200 * r3 = EA, r10 = proto-VSID, r11 = flags, clobbers r9, cr7 = <> PAGE_OFFSET
189 */ 201 */
190slb_finish_load: 202slb_finish_load:
191 ASM_VSID_SCRAMBLE(r10,r9) 203 ASM_VSID_SCRAMBLE(r10,r9,256M)
192 rldimi r11,r10,SLB_VSID_SHIFT,16 /* combine VSID and flags */ 204 rldimi r11,r10,SLB_VSID_SHIFT,16 /* combine VSID and flags */
193 205
194 /* r3 = EA, r11 = VSID data */ 206 /* r3 = EA, r11 = VSID data */
@@ -213,7 +225,7 @@ BEGIN_FW_FTR_SECTION
213END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES) 225END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
214#endif /* CONFIG_PPC_ISERIES */ 226#endif /* CONFIG_PPC_ISERIES */
215 227
216 ld r10,PACASTABRR(r13) 2287: ld r10,PACASTABRR(r13)
217 addi r10,r10,1 229 addi r10,r10,1
218 /* use a cpu feature mask if we ever change our slb size */ 230 /* use a cpu feature mask if we ever change our slb size */
219 cmpldi r10,SLB_NUM_ENTRIES 231 cmpldi r10,SLB_NUM_ENTRIES
@@ -259,3 +271,20 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
259 crclr 4*cr0+eq /* set result to "success" */ 271 crclr 4*cr0+eq /* set result to "success" */
260 blr 272 blr
261 273
274/*
275 * Finish loading of a 1T SLB entry (for the kernel linear mapping) and return.
276 * We assume legacy iSeries will never have 1T segments.
277 *
278 * r3 = EA, r10 = proto-VSID, r11 = flags, clobbers r9
279 */
280slb_finish_load_1T:
281 srdi r10,r10,40-28 /* get 1T ESID */
282 ASM_VSID_SCRAMBLE(r10,r9,1T)
283 rldimi r11,r10,SLB_VSID_SHIFT_1T,16 /* combine VSID and flags */
284 li r10,MMU_SEGSIZE_1T
285 rldimi r11,r10,SLB_VSID_SSIZE_SHIFT,0 /* insert segment size */
286
287 /* r3 = EA, r11 = VSID data */
288 clrrdi r3,r3,SID_SHIFT_1T /* clear out non-ESID bits */
289 b 7b
290
diff --git a/arch/powerpc/mm/stab.c b/arch/powerpc/mm/stab.c
index 28492bbdee8e..9e85bda76216 100644
--- a/arch/powerpc/mm/stab.c
+++ b/arch/powerpc/mm/stab.c
@@ -122,12 +122,12 @@ static int __ste_allocate(unsigned long ea, struct mm_struct *mm)
122 122
123 /* Kernel or user address? */ 123 /* Kernel or user address? */
124 if (is_kernel_addr(ea)) { 124 if (is_kernel_addr(ea)) {
125 vsid = get_kernel_vsid(ea); 125 vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M);
126 } else { 126 } else {
127 if ((ea >= TASK_SIZE_USER64) || (! mm)) 127 if ((ea >= TASK_SIZE_USER64) || (! mm))
128 return 1; 128 return 1;
129 129
130 vsid = get_vsid(mm->context.id, ea); 130 vsid = get_vsid(mm->context.id, ea, MMU_SEGSIZE_256M);
131 } 131 }
132 132
133 stab_entry = make_ste(get_paca()->stab_addr, GET_ESID(ea), vsid); 133 stab_entry = make_ste(get_paca()->stab_addr, GET_ESID(ea), vsid);
@@ -261,7 +261,7 @@ void __init stabs_alloc(void)
261 */ 261 */
262void stab_initialize(unsigned long stab) 262void stab_initialize(unsigned long stab)
263{ 263{
264 unsigned long vsid = get_kernel_vsid(PAGE_OFFSET); 264 unsigned long vsid = get_kernel_vsid(PAGE_OFFSET, MMU_SEGSIZE_256M);
265 unsigned long stabreal; 265 unsigned long stabreal;
266 266
267 asm volatile("isync; slbia; isync":::"memory"); 267 asm volatile("isync; slbia; isync":::"memory");
diff --git a/arch/powerpc/mm/tlb_64.c b/arch/powerpc/mm/tlb_64.c
index cbd34fc813ee..eafbca52bff9 100644
--- a/arch/powerpc/mm/tlb_64.c
+++ b/arch/powerpc/mm/tlb_64.c
@@ -132,6 +132,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
132 struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); 132 struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
133 unsigned long vsid, vaddr; 133 unsigned long vsid, vaddr;
134 unsigned int psize; 134 unsigned int psize;
135 int ssize;
135 real_pte_t rpte; 136 real_pte_t rpte;
136 int i; 137 int i;
137 138
@@ -161,11 +162,14 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
161 162
162 /* Build full vaddr */ 163 /* Build full vaddr */
163 if (!is_kernel_addr(addr)) { 164 if (!is_kernel_addr(addr)) {
164 vsid = get_vsid(mm->context.id, addr); 165 ssize = user_segment_size(addr);
166 vsid = get_vsid(mm->context.id, addr, ssize);
165 WARN_ON(vsid == 0); 167 WARN_ON(vsid == 0);
166 } else 168 } else {
167 vsid = get_kernel_vsid(addr); 169 vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
168 vaddr = (vsid << 28 ) | (addr & 0x0fffffff); 170 ssize = mmu_kernel_ssize;
171 }
172 vaddr = hpt_va(addr, vsid, ssize);
169 rpte = __real_pte(__pte(pte), ptep); 173 rpte = __real_pte(__pte(pte), ptep);
170 174
171 /* 175 /*
@@ -175,7 +179,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
175 * and decide to use local invalidates instead... 179 * and decide to use local invalidates instead...
176 */ 180 */
177 if (!batch->active) { 181 if (!batch->active) {
178 flush_hash_page(vaddr, rpte, psize, 0); 182 flush_hash_page(vaddr, rpte, psize, ssize, 0);
179 return; 183 return;
180 } 184 }
181 185
@@ -189,13 +193,15 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
189 * We also need to ensure only one page size is present in a given 193 * We also need to ensure only one page size is present in a given
190 * batch 194 * batch
191 */ 195 */
192 if (i != 0 && (mm != batch->mm || batch->psize != psize)) { 196 if (i != 0 && (mm != batch->mm || batch->psize != psize ||
197 batch->ssize != ssize)) {
193 __flush_tlb_pending(batch); 198 __flush_tlb_pending(batch);
194 i = 0; 199 i = 0;
195 } 200 }
196 if (i == 0) { 201 if (i == 0) {
197 batch->mm = mm; 202 batch->mm = mm;
198 batch->psize = psize; 203 batch->psize = psize;
204 batch->ssize = ssize;
199 } 205 }
200 batch->pte[i] = rpte; 206 batch->pte[i] = rpte;
201 batch->vaddr[i] = vaddr; 207 batch->vaddr[i] = vaddr;
@@ -222,7 +228,7 @@ void __flush_tlb_pending(struct ppc64_tlb_batch *batch)
222 local = 1; 228 local = 1;
223 if (i == 1) 229 if (i == 1)
224 flush_hash_page(batch->vaddr[0], batch->pte[0], 230 flush_hash_page(batch->vaddr[0], batch->pte[0],
225 batch->psize, local); 231 batch->psize, batch->ssize, local);
226 else 232 else
227 flush_hash_range(i, local); 233 flush_hash_range(i, local);
228 batch->index = 0; 234 batch->index = 0;
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-21 08:58:34 -0500