aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/powerpc/mm')
-rw-r--r--arch/powerpc/mm/44x_mmu.c13
-rw-r--r--arch/powerpc/mm/fault.c6
-rw-r--r--arch/powerpc/mm/gup.c12
-rw-r--r--arch/powerpc/mm/hash_native_64.c6
-rw-r--r--arch/powerpc/mm/hash_utils_64.c6
-rw-r--r--arch/powerpc/mm/hugetlbpage.c21
-rw-r--r--arch/powerpc/mm/init_32.c32
-rw-r--r--arch/powerpc/mm/init_64.c16
-rw-r--r--arch/powerpc/mm/mem.c38
-rw-r--r--arch/powerpc/mm/mmu_context_hash64.c12
-rw-r--r--arch/powerpc/mm/numa.c3
-rw-r--r--arch/powerpc/mm/tlb_hash32.c4
-rw-r--r--arch/powerpc/mm/tlb_low_64e.S206
-rw-r--r--arch/powerpc/mm/tlb_nohash.c64
14 files changed, 336 insertions, 103 deletions
diff --git a/arch/powerpc/mm/44x_mmu.c b/arch/powerpc/mm/44x_mmu.c
index 024acab588f..f60e006d90c 100644
--- a/arch/powerpc/mm/44x_mmu.c
+++ b/arch/powerpc/mm/44x_mmu.c
@@ -186,10 +186,11 @@ void __init MMU_init_hw(void)
186unsigned long __init mmu_mapin_ram(unsigned long top) 186unsigned long __init mmu_mapin_ram(unsigned long top)
187{ 187{
188 unsigned long addr; 188 unsigned long addr;
189 unsigned long memstart = memstart_addr & ~(PPC_PIN_SIZE - 1);
189 190
190 /* Pin in enough TLBs to cover any lowmem not covered by the 191 /* Pin in enough TLBs to cover any lowmem not covered by the
191 * initial 256M mapping established in head_44x.S */ 192 * initial 256M mapping established in head_44x.S */
192 for (addr = PPC_PIN_SIZE; addr < lowmem_end_addr; 193 for (addr = memstart + PPC_PIN_SIZE; addr < lowmem_end_addr;
193 addr += PPC_PIN_SIZE) { 194 addr += PPC_PIN_SIZE) {
194 if (mmu_has_feature(MMU_FTR_TYPE_47x)) 195 if (mmu_has_feature(MMU_FTR_TYPE_47x))
195 ppc47x_pin_tlb(addr + PAGE_OFFSET, addr); 196 ppc47x_pin_tlb(addr + PAGE_OFFSET, addr);
@@ -218,19 +219,25 @@ unsigned long __init mmu_mapin_ram(unsigned long top)
218void setup_initial_memory_limit(phys_addr_t first_memblock_base, 219void setup_initial_memory_limit(phys_addr_t first_memblock_base,
219 phys_addr_t first_memblock_size) 220 phys_addr_t first_memblock_size)
220{ 221{
222 u64 size;
223
224#ifndef CONFIG_RELOCATABLE
221 /* We don't currently support the first MEMBLOCK not mapping 0 225 /* We don't currently support the first MEMBLOCK not mapping 0
222 * physical on those processors 226 * physical on those processors
223 */ 227 */
224 BUG_ON(first_memblock_base != 0); 228 BUG_ON(first_memblock_base != 0);
229#endif
225 230
226 /* 44x has a 256M TLB entry pinned at boot */ 231 /* 44x has a 256M TLB entry pinned at boot */
227 memblock_set_current_limit(min_t(u64, first_memblock_size, PPC_PIN_SIZE)); 232 size = (min_t(u64, first_memblock_size, PPC_PIN_SIZE));
233 memblock_set_current_limit(first_memblock_base + size);
228} 234}
229 235
230#ifdef CONFIG_SMP 236#ifdef CONFIG_SMP
231void __cpuinit mmu_init_secondary(int cpu) 237void __cpuinit mmu_init_secondary(int cpu)
232{ 238{
233 unsigned long addr; 239 unsigned long addr;
240 unsigned long memstart = memstart_addr & ~(PPC_PIN_SIZE - 1);
234 241
235 /* Pin in enough TLBs to cover any lowmem not covered by the 242 /* Pin in enough TLBs to cover any lowmem not covered by the
236 * initial 256M mapping established in head_44x.S 243 * initial 256M mapping established in head_44x.S
@@ -241,7 +248,7 @@ void __cpuinit mmu_init_secondary(int cpu)
241 * stack. current (r2) isn't initialized, smp_processor_id() 248 * stack. current (r2) isn't initialized, smp_processor_id()
242 * will not work, current thread info isn't accessible, ... 249 * will not work, current thread info isn't accessible, ...
243 */ 250 */
244 for (addr = PPC_PIN_SIZE; addr < lowmem_end_addr; 251 for (addr = memstart + PPC_PIN_SIZE; addr < lowmem_end_addr;
245 addr += PPC_PIN_SIZE) { 252 addr += PPC_PIN_SIZE) {
246 if (mmu_has_feature(MMU_FTR_TYPE_47x)) 253 if (mmu_has_feature(MMU_FTR_TYPE_47x))
247 ppc47x_pin_tlb(addr + PAGE_OFFSET, addr); 254 ppc47x_pin_tlb(addr + PAGE_OFFSET, addr);
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index ad35f66c69e..5efe8c96d37 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -174,7 +174,7 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
174 die("Weird page fault", regs, SIGSEGV); 174 die("Weird page fault", regs, SIGSEGV);
175 } 175 }
176 176
177 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); 177 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
178 178
179 /* When running in the kernel we expect faults to occur only to 179 /* When running in the kernel we expect faults to occur only to
180 * addresses in user space. All other faults represent errors in the 180 * addresses in user space. All other faults represent errors in the
@@ -320,7 +320,7 @@ good_area:
320 } 320 }
321 if (ret & VM_FAULT_MAJOR) { 321 if (ret & VM_FAULT_MAJOR) {
322 current->maj_flt++; 322 current->maj_flt++;
323 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, 323 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
324 regs, address); 324 regs, address);
325#ifdef CONFIG_PPC_SMLPAR 325#ifdef CONFIG_PPC_SMLPAR
326 if (firmware_has_feature(FW_FEATURE_CMO)) { 326 if (firmware_has_feature(FW_FEATURE_CMO)) {
@@ -331,7 +331,7 @@ good_area:
331#endif 331#endif
332 } else { 332 } else {
333 current->min_flt++; 333 current->min_flt++;
334 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, 334 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
335 regs, address); 335 regs, address);
336 } 336 }
337 up_read(&mm->mmap_sem); 337 up_read(&mm->mmap_sem);
diff --git a/arch/powerpc/mm/gup.c b/arch/powerpc/mm/gup.c
index fec13200868..d7efdbf640c 100644
--- a/arch/powerpc/mm/gup.c
+++ b/arch/powerpc/mm/gup.c
@@ -16,16 +16,6 @@
16 16
17#ifdef __HAVE_ARCH_PTE_SPECIAL 17#ifdef __HAVE_ARCH_PTE_SPECIAL
18 18
19static inline void get_huge_page_tail(struct page *page)
20{
21 /*
22 * __split_huge_page_refcount() cannot run
23 * from under us.
24 */
25 VM_BUG_ON(atomic_read(&page->_count) < 0);
26 atomic_inc(&page->_count);
27}
28
29/* 19/*
30 * The performance critical leaf functions are made noinline otherwise gcc 20 * The performance critical leaf functions are made noinline otherwise gcc
31 * inlines everything into a single function which results in too much 21 * inlines everything into a single function which results in too much
@@ -57,8 +47,6 @@ static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,
57 put_page(page); 47 put_page(page);
58 return 0; 48 return 0;
59 } 49 }
60 if (PageTail(page))
61 get_huge_page_tail(page);
62 pages[*nr] = page; 50 pages[*nr] = page;
63 (*nr)++; 51 (*nr)++;
64 52
diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index dfd764896db..90039bc6411 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -37,7 +37,7 @@
37 37
38#define HPTE_LOCK_BIT 3 38#define HPTE_LOCK_BIT 3
39 39
40static DEFINE_RAW_SPINLOCK(native_tlbie_lock); 40DEFINE_RAW_SPINLOCK(native_tlbie_lock);
41 41
42static inline void __tlbie(unsigned long va, int psize, int ssize) 42static inline void __tlbie(unsigned long va, int psize, int ssize)
43{ 43{
@@ -51,7 +51,7 @@ static inline void __tlbie(unsigned long va, int psize, int ssize)
51 va &= ~0xffful; 51 va &= ~0xffful;
52 va |= ssize << 8; 52 va |= ssize << 8;
53 asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2) 53 asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2)
54 : : "r" (va), "r"(0), "i" (CPU_FTR_HVMODE_206) 54 : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
55 : "memory"); 55 : "memory");
56 break; 56 break;
57 default: 57 default:
@@ -61,7 +61,7 @@ static inline void __tlbie(unsigned long va, int psize, int ssize)
61 va |= ssize << 8; 61 va |= ssize << 8;
62 va |= 1; /* L */ 62 va |= 1; /* L */
63 asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2) 63 asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2)
64 : : "r" (va), "r"(0), "i" (CPU_FTR_HVMODE_206) 64 : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
65 : "memory"); 65 : "memory");
66 break; 66 break;
67 } 67 }
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 26b2872b3d0..07f9e9f0d87 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -534,11 +534,11 @@ static unsigned long __init htab_get_table_size(void)
534} 534}
535 535
536#ifdef CONFIG_MEMORY_HOTPLUG 536#ifdef CONFIG_MEMORY_HOTPLUG
537void create_section_mapping(unsigned long start, unsigned long end) 537int create_section_mapping(unsigned long start, unsigned long end)
538{ 538{
539 BUG_ON(htab_bolt_mapping(start, end, __pa(start), 539 return htab_bolt_mapping(start, end, __pa(start),
540 pgprot_val(PAGE_KERNEL), mmu_linear_psize, 540 pgprot_val(PAGE_KERNEL), mmu_linear_psize,
541 mmu_kernel_ssize)); 541 mmu_kernel_ssize);
542} 542}
543 543
544int remove_section_mapping(unsigned long start, unsigned long end) 544int remove_section_mapping(unsigned long start, unsigned long end)
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index 0b9a5c1901b..da5eb388570 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -390,7 +390,7 @@ static noinline int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long add
390{ 390{
391 unsigned long mask; 391 unsigned long mask;
392 unsigned long pte_end; 392 unsigned long pte_end;
393 struct page *head, *page; 393 struct page *head, *page, *tail;
394 pte_t pte; 394 pte_t pte;
395 int refs; 395 int refs;
396 396
@@ -413,6 +413,7 @@ static noinline int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long add
413 head = pte_page(pte); 413 head = pte_page(pte);
414 414
415 page = head + ((addr & (sz-1)) >> PAGE_SHIFT); 415 page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
416 tail = page;
416 do { 417 do {
417 VM_BUG_ON(compound_head(page) != head); 418 VM_BUG_ON(compound_head(page) != head);
418 pages[*nr] = page; 419 pages[*nr] = page;
@@ -428,10 +429,20 @@ static noinline int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long add
428 429
429 if (unlikely(pte_val(pte) != pte_val(*ptep))) { 430 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
430 /* Could be optimized better */ 431 /* Could be optimized better */
431 while (*nr) { 432 *nr -= refs;
432 put_page(page); 433 while (refs--)
433 (*nr)--; 434 put_page(head);
434 } 435 return 0;
436 }
437
438 /*
439 * Any tail page need their mapcount reference taken before we
440 * return.
441 */
442 while (refs--) {
443 if (PageTail(tail))
444 get_huge_page_tail(tail);
445 tail++;
435 } 446 }
436 447
437 return 1; 448 return 1;
diff --git a/arch/powerpc/mm/init_32.c b/arch/powerpc/mm/init_32.c
index 5de0f254dbb..c77fef56dad 100644
--- a/arch/powerpc/mm/init_32.c
+++ b/arch/powerpc/mm/init_32.c
@@ -191,38 +191,6 @@ void __init *early_get_page(void)
191 return __va(memblock_alloc(PAGE_SIZE, PAGE_SIZE)); 191 return __va(memblock_alloc(PAGE_SIZE, PAGE_SIZE));
192} 192}
193 193
194/* Free up now-unused memory */
195static void free_sec(unsigned long start, unsigned long end, const char *name)
196{
197 unsigned long cnt = 0;
198
199 while (start < end) {
200 ClearPageReserved(virt_to_page(start));
201 init_page_count(virt_to_page(start));
202 free_page(start);
203 cnt++;
204 start += PAGE_SIZE;
205 }
206 if (cnt) {
207 printk(" %ldk %s", cnt << (PAGE_SHIFT - 10), name);
208 totalram_pages += cnt;
209 }
210}
211
212void free_initmem(void)
213{
214#define FREESEC(TYPE) \
215 free_sec((unsigned long)(&__ ## TYPE ## _begin), \
216 (unsigned long)(&__ ## TYPE ## _end), \
217 #TYPE);
218
219 printk ("Freeing unused kernel memory:");
220 FREESEC(init);
221 printk("\n");
222 ppc_md.progress = NULL;
223#undef FREESEC
224}
225
226#ifdef CONFIG_8xx /* No 8xx specific .c file to put that in ... */ 194#ifdef CONFIG_8xx /* No 8xx specific .c file to put that in ... */
227void setup_initial_memory_limit(phys_addr_t first_memblock_base, 195void setup_initial_memory_limit(phys_addr_t first_memblock_base,
228 phys_addr_t first_memblock_size) 196 phys_addr_t first_memblock_size)
diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c
index f6dbb4c20e6..e94b57fb79a 100644
--- a/arch/powerpc/mm/init_64.c
+++ b/arch/powerpc/mm/init_64.c
@@ -83,22 +83,6 @@ EXPORT_SYMBOL_GPL(memstart_addr);
83phys_addr_t kernstart_addr; 83phys_addr_t kernstart_addr;
84EXPORT_SYMBOL_GPL(kernstart_addr); 84EXPORT_SYMBOL_GPL(kernstart_addr);
85 85
86void free_initmem(void)
87{
88 unsigned long addr;
89
90 addr = (unsigned long)__init_begin;
91 for (; addr < (unsigned long)__init_end; addr += PAGE_SIZE) {
92 memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
93 ClearPageReserved(virt_to_page(addr));
94 init_page_count(virt_to_page(addr));
95 free_page(addr);
96 totalram_pages++;
97 }
98 printk ("Freeing unused kernel memory: %luk freed\n",
99 ((unsigned long)__init_end - (unsigned long)__init_begin) >> 10);
100}
101
102static void pgd_ctor(void *addr) 86static void pgd_ctor(void *addr)
103{ 87{
104 memset(addr, 0, PGD_TABLE_SIZE); 88 memset(addr, 0, PGD_TABLE_SIZE);
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 29d4dde65c4..95985f286e3 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -123,7 +123,8 @@ int arch_add_memory(int nid, u64 start, u64 size)
123 pgdata = NODE_DATA(nid); 123 pgdata = NODE_DATA(nid);
124 124
125 start = (unsigned long)__va(start); 125 start = (unsigned long)__va(start);
126 create_section_mapping(start, start + size); 126 if (create_section_mapping(start, start + size))
127 return -EINVAL;
127 128
128 /* this should work for most non-highmem platforms */ 129 /* this should work for most non-highmem platforms */
129 zone = pgdata->node_zones; 130 zone = pgdata->node_zones;
@@ -249,7 +250,7 @@ static int __init mark_nonram_nosave(void)
249 */ 250 */
250void __init paging_init(void) 251void __init paging_init(void)
251{ 252{
252 unsigned long total_ram = memblock_phys_mem_size(); 253 unsigned long long total_ram = memblock_phys_mem_size();
253 phys_addr_t top_of_ram = memblock_end_of_DRAM(); 254 phys_addr_t top_of_ram = memblock_end_of_DRAM();
254 unsigned long max_zone_pfns[MAX_NR_ZONES]; 255 unsigned long max_zone_pfns[MAX_NR_ZONES];
255 256
@@ -269,7 +270,7 @@ void __init paging_init(void)
269 kmap_prot = PAGE_KERNEL; 270 kmap_prot = PAGE_KERNEL;
270#endif /* CONFIG_HIGHMEM */ 271#endif /* CONFIG_HIGHMEM */
271 272
272 printk(KERN_DEBUG "Top of RAM: 0x%llx, Total RAM: 0x%lx\n", 273 printk(KERN_DEBUG "Top of RAM: 0x%llx, Total RAM: 0x%llx\n",
273 (unsigned long long)top_of_ram, total_ram); 274 (unsigned long long)top_of_ram, total_ram);
274 printk(KERN_DEBUG "Memory hole size: %ldMB\n", 275 printk(KERN_DEBUG "Memory hole size: %ldMB\n",
275 (long int)((top_of_ram - total_ram) >> 20)); 276 (long int)((top_of_ram - total_ram) >> 20));
@@ -337,8 +338,9 @@ void __init mem_init(void)
337 338
338 highmem_mapnr = lowmem_end_addr >> PAGE_SHIFT; 339 highmem_mapnr = lowmem_end_addr >> PAGE_SHIFT;
339 for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) { 340 for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) {
341 phys_addr_t paddr = (phys_addr_t)pfn << PAGE_SHIFT;
340 struct page *page = pfn_to_page(pfn); 342 struct page *page = pfn_to_page(pfn);
341 if (memblock_is_reserved(pfn << PAGE_SHIFT)) 343 if (memblock_is_reserved(paddr))
342 continue; 344 continue;
343 ClearPageReserved(page); 345 ClearPageReserved(page);
344 init_page_count(page); 346 init_page_count(page);
@@ -352,6 +354,15 @@ void __init mem_init(void)
352 } 354 }
353#endif /* CONFIG_HIGHMEM */ 355#endif /* CONFIG_HIGHMEM */
354 356
357#if defined(CONFIG_PPC_FSL_BOOK3E) && !defined(CONFIG_SMP)
358 /*
359 * If smp is enabled, next_tlbcam_idx is initialized in the cpu up
360 * functions.... do it here for the non-smp case.
361 */
362 per_cpu(next_tlbcam_idx, smp_processor_id()) =
363 (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1;
364#endif
365
355 printk(KERN_INFO "Memory: %luk/%luk available (%luk kernel code, " 366 printk(KERN_INFO "Memory: %luk/%luk available (%luk kernel code, "
356 "%luk reserved, %luk data, %luk bss, %luk init)\n", 367 "%luk reserved, %luk data, %luk bss, %luk init)\n",
357 nr_free_pages() << (PAGE_SHIFT-10), 368 nr_free_pages() << (PAGE_SHIFT-10),
@@ -382,6 +393,25 @@ void __init mem_init(void)
382 mem_init_done = 1; 393 mem_init_done = 1;
383} 394}
384 395
396void free_initmem(void)
397{
398 unsigned long addr;
399
400 ppc_md.progress = ppc_printk_progress;
401
402 addr = (unsigned long)__init_begin;
403 for (; addr < (unsigned long)__init_end; addr += PAGE_SIZE) {
404 memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
405 ClearPageReserved(virt_to_page(addr));
406 init_page_count(virt_to_page(addr));
407 free_page(addr);
408 totalram_pages++;
409 }
410 pr_info("Freeing unused kernel memory: %luk freed\n",
411 ((unsigned long)__init_end -
412 (unsigned long)__init_begin) >> 10);
413}
414
385#ifdef CONFIG_BLK_DEV_INITRD 415#ifdef CONFIG_BLK_DEV_INITRD
386void __init free_initrd_mem(unsigned long start, unsigned long end) 416void __init free_initrd_mem(unsigned long start, unsigned long end)
387{ 417{
diff --git a/arch/powerpc/mm/mmu_context_hash64.c b/arch/powerpc/mm/mmu_context_hash64.c
index 3bafc3deca6..4ff587e3825 100644
--- a/arch/powerpc/mm/mmu_context_hash64.c
+++ b/arch/powerpc/mm/mmu_context_hash64.c
@@ -136,8 +136,8 @@ int use_cop(unsigned long acop, struct mm_struct *mm)
136 if (!mm || !acop) 136 if (!mm || !acop)
137 return -EINVAL; 137 return -EINVAL;
138 138
139 /* We need to make sure mm_users doesn't change */ 139 /* The page_table_lock ensures mm_users won't change under us */
140 down_read(&mm->mmap_sem); 140 spin_lock(&mm->page_table_lock);
141 spin_lock(mm->context.cop_lockp); 141 spin_lock(mm->context.cop_lockp);
142 142
143 if (mm->context.cop_pid == COP_PID_NONE) { 143 if (mm->context.cop_pid == COP_PID_NONE) {
@@ -164,7 +164,7 @@ int use_cop(unsigned long acop, struct mm_struct *mm)
164 164
165out: 165out:
166 spin_unlock(mm->context.cop_lockp); 166 spin_unlock(mm->context.cop_lockp);
167 up_read(&mm->mmap_sem); 167 spin_unlock(&mm->page_table_lock);
168 168
169 return ret; 169 return ret;
170} 170}
@@ -185,8 +185,8 @@ void drop_cop(unsigned long acop, struct mm_struct *mm)
185 if (WARN_ON_ONCE(!mm)) 185 if (WARN_ON_ONCE(!mm))
186 return; 186 return;
187 187
188 /* We need to make sure mm_users doesn't change */ 188 /* The page_table_lock ensures mm_users won't change under us */
189 down_read(&mm->mmap_sem); 189 spin_lock(&mm->page_table_lock);
190 spin_lock(mm->context.cop_lockp); 190 spin_lock(mm->context.cop_lockp);
191 191
192 mm->context.acop &= ~acop; 192 mm->context.acop &= ~acop;
@@ -213,7 +213,7 @@ void drop_cop(unsigned long acop, struct mm_struct *mm)
213 } 213 }
214 214
215 spin_unlock(mm->context.cop_lockp); 215 spin_unlock(mm->context.cop_lockp);
216 up_read(&mm->mmap_sem); 216 spin_unlock(&mm->page_table_lock);
217} 217}
218EXPORT_SYMBOL_GPL(drop_cop); 218EXPORT_SYMBOL_GPL(drop_cop);
219 219
diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
index 2164006fe17..2c1ae7a5fb5 100644
--- a/arch/powerpc/mm/numa.c
+++ b/arch/powerpc/mm/numa.c
@@ -1214,11 +1214,12 @@ int hot_add_node_scn_to_nid(unsigned long scn_addr)
1214 break; 1214 break;
1215 } 1215 }
1216 1216
1217 of_node_put(memory);
1218 if (nid >= 0) 1217 if (nid >= 0)
1219 break; 1218 break;
1220 } 1219 }
1221 1220
1221 of_node_put(memory);
1222
1222 return nid; 1223 return nid;
1223} 1224}
1224 1225
diff --git a/arch/powerpc/mm/tlb_hash32.c b/arch/powerpc/mm/tlb_hash32.c
index 27b863c1494..9a445f64acc 100644
--- a/arch/powerpc/mm/tlb_hash32.c
+++ b/arch/powerpc/mm/tlb_hash32.c
@@ -177,3 +177,7 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
177 flush_range(vma->vm_mm, start, end); 177 flush_range(vma->vm_mm, start, end);
178} 178}
179EXPORT_SYMBOL(flush_tlb_range); 179EXPORT_SYMBOL(flush_tlb_range);
180
181void __init early_init_mmu(void)
182{
183}
diff --git a/arch/powerpc/mm/tlb_low_64e.S b/arch/powerpc/mm/tlb_low_64e.S
index af089220941..4ebb34bc01d 100644
--- a/arch/powerpc/mm/tlb_low_64e.S
+++ b/arch/powerpc/mm/tlb_low_64e.S
@@ -30,6 +30,212 @@
30#define VPTE_PGD_SHIFT (VPTE_PUD_SHIFT + PUD_INDEX_SIZE) 30#define VPTE_PGD_SHIFT (VPTE_PUD_SHIFT + PUD_INDEX_SIZE)
31#define VPTE_INDEX_SIZE (VPTE_PGD_SHIFT + PGD_INDEX_SIZE) 31#define VPTE_INDEX_SIZE (VPTE_PGD_SHIFT + PGD_INDEX_SIZE)
32 32
33/**********************************************************************
34 * *
35 * TLB miss handling for Book3E with a bolted linear mapping *
36 * No virtual page table, no nested TLB misses *
37 * *
38 **********************************************************************/
39
40.macro tlb_prolog_bolted addr
41 mtspr SPRN_SPRG_TLB_SCRATCH,r13
42 mfspr r13,SPRN_SPRG_PACA
43 std r10,PACA_EXTLB+EX_TLB_R10(r13)
44 mfcr r10
45 std r11,PACA_EXTLB+EX_TLB_R11(r13)
46 std r16,PACA_EXTLB+EX_TLB_R16(r13)
47 mfspr r16,\addr /* get faulting address */
48 std r14,PACA_EXTLB+EX_TLB_R14(r13)
49 ld r14,PACAPGD(r13)
50 std r15,PACA_EXTLB+EX_TLB_R15(r13)
51 std r10,PACA_EXTLB+EX_TLB_CR(r13)
52 TLB_MISS_PROLOG_STATS_BOLTED
53.endm
54
55.macro tlb_epilog_bolted
56 ld r14,PACA_EXTLB+EX_TLB_CR(r13)
57 ld r10,PACA_EXTLB+EX_TLB_R10(r13)
58 ld r11,PACA_EXTLB+EX_TLB_R11(r13)
59 mtcr r14
60 ld r14,PACA_EXTLB+EX_TLB_R14(r13)
61 ld r15,PACA_EXTLB+EX_TLB_R15(r13)
62 TLB_MISS_RESTORE_STATS_BOLTED
63 ld r16,PACA_EXTLB+EX_TLB_R16(r13)
64 mfspr r13,SPRN_SPRG_TLB_SCRATCH
65.endm
66
67/* Data TLB miss */
68 START_EXCEPTION(data_tlb_miss_bolted)
69 tlb_prolog_bolted SPRN_DEAR
70
71 /* We need _PAGE_PRESENT and _PAGE_ACCESSED set */
72
73 /* We do the user/kernel test for the PID here along with the RW test
74 */
75 /* We pre-test some combination of permissions to avoid double
76 * faults:
77 *
78 * We move the ESR:ST bit into the position of _PAGE_BAP_SW in the PTE
79 * ESR_ST is 0x00800000
80 * _PAGE_BAP_SW is 0x00000010
81 * So the shift is >> 19. This tests for supervisor writeability.
82 * If the page happens to be supervisor writeable and not user
83 * writeable, we will take a new fault later, but that should be
84 * a rare enough case.
85 *
86 * We also move ESR_ST in _PAGE_DIRTY position
87 * _PAGE_DIRTY is 0x00001000 so the shift is >> 11
88 *
89 * MAS1 is preset for all we need except for TID that needs to
90 * be cleared for kernel translations
91 */
92
93 mfspr r11,SPRN_ESR
94
95 srdi r15,r16,60 /* get region */
96 rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
97 bne- dtlb_miss_fault_bolted
98
99 rlwinm r10,r11,32-19,27,27
100 rlwimi r10,r11,32-16,19,19
101 cmpwi r15,0
102 ori r10,r10,_PAGE_PRESENT
103 oris r11,r10,_PAGE_ACCESSED@h
104
105 TLB_MISS_STATS_SAVE_INFO_BOLTED
106 bne tlb_miss_kernel_bolted
107
108tlb_miss_common_bolted:
109/*
110 * This is the guts of the TLB miss handler for bolted-linear.
111 * We are entered with:
112 *
113 * r16 = faulting address
114 * r15 = crap (free to use)
115 * r14 = page table base
116 * r13 = PACA
117 * r11 = PTE permission mask
118 * r10 = crap (free to use)
119 */
120 rldicl r15,r16,64-PGDIR_SHIFT+3,64-PGD_INDEX_SIZE-3
121 cmpldi cr0,r14,0
122 clrrdi r15,r15,3
123 beq tlb_miss_fault_bolted
124
125BEGIN_MMU_FTR_SECTION
126 /* Set the TLB reservation and search for existing entry. Then load
127 * the entry.
128 */
129 PPC_TLBSRX_DOT(0,r16)
130 ldx r14,r14,r15
131 beq normal_tlb_miss_done
132MMU_FTR_SECTION_ELSE
133 ldx r14,r14,r15
134ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_USE_TLBRSRV)
135
136#ifndef CONFIG_PPC_64K_PAGES
137 rldicl r15,r16,64-PUD_SHIFT+3,64-PUD_INDEX_SIZE-3
138 clrrdi r15,r15,3
139
140 cmpldi cr0,r14,0
141 beq tlb_miss_fault_bolted
142
143 ldx r14,r14,r15
144#endif /* CONFIG_PPC_64K_PAGES */
145
146 rldicl r15,r16,64-PMD_SHIFT+3,64-PMD_INDEX_SIZE-3
147 clrrdi r15,r15,3
148
149 cmpldi cr0,r14,0
150 beq tlb_miss_fault_bolted
151
152 ldx r14,r14,r15
153
154 rldicl r15,r16,64-PAGE_SHIFT+3,64-PTE_INDEX_SIZE-3
155 clrrdi r15,r15,3
156
157 cmpldi cr0,r14,0
158 beq tlb_miss_fault_bolted
159
160 ldx r14,r14,r15
161
162 /* Check if required permissions are met */
163 andc. r15,r11,r14
164 rldicr r15,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT
165 bne- tlb_miss_fault_bolted
166
167 /* Now we build the MAS:
168 *
169 * MAS 0 : Fully setup with defaults in MAS4 and TLBnCFG
170 * MAS 1 : Almost fully setup
171 * - PID already updated by caller if necessary
172 * - TSIZE need change if !base page size, not
173 * yet implemented for now
174 * MAS 2 : Defaults not useful, need to be redone
175 * MAS 3+7 : Needs to be done
176 */
177 clrrdi r11,r16,12 /* Clear low crap in EA */
178 clrldi r15,r15,12 /* Clear crap at the top */
179 rlwimi r11,r14,32-19,27,31 /* Insert WIMGE */
180 rlwimi r15,r14,32-8,22,25 /* Move in U bits */
181 mtspr SPRN_MAS2,r11
182 andi. r11,r14,_PAGE_DIRTY
183 rlwimi r15,r14,32-2,26,31 /* Move in BAP bits */
184
185 /* Mask out SW and UW if !DIRTY (XXX optimize this !) */
186 bne 1f
187 li r11,MAS3_SW|MAS3_UW
188 andc r15,r15,r11
1891:
190 mtspr SPRN_MAS7_MAS3,r15
191 tlbwe
192
193 TLB_MISS_STATS_X(MMSTAT_TLB_MISS_NORM_OK)
194 tlb_epilog_bolted
195 rfi
196
197itlb_miss_kernel_bolted:
198 li r11,_PAGE_PRESENT|_PAGE_BAP_SX /* Base perm */
199 oris r11,r11,_PAGE_ACCESSED@h
200tlb_miss_kernel_bolted:
201 mfspr r10,SPRN_MAS1
202 ld r14,PACA_KERNELPGD(r13)
203 cmpldi cr0,r15,8 /* Check for vmalloc region */
204 rlwinm r10,r10,0,16,1 /* Clear TID */
205 mtspr SPRN_MAS1,r10
206 beq+ tlb_miss_common_bolted
207
208tlb_miss_fault_bolted:
209 /* We need to check if it was an instruction miss */
210 andi. r10,r11,_PAGE_EXEC|_PAGE_BAP_SX
211 bne itlb_miss_fault_bolted
212dtlb_miss_fault_bolted:
213 TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
214 tlb_epilog_bolted
215 b exc_data_storage_book3e
216itlb_miss_fault_bolted:
217 TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
218 tlb_epilog_bolted
219 b exc_instruction_storage_book3e
220
221/* Instruction TLB miss */
222 START_EXCEPTION(instruction_tlb_miss_bolted)
223 tlb_prolog_bolted SPRN_SRR0
224
225 rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
226 srdi r15,r16,60 /* get region */
227 TLB_MISS_STATS_SAVE_INFO_BOLTED
228 bne- itlb_miss_fault_bolted
229
230 li r11,_PAGE_PRESENT|_PAGE_EXEC /* Base perm */
231
232 /* We do the user/kernel test for the PID here along with the RW test
233 */
234
235 cmpldi cr0,r15,0 /* Check for user region */
236 oris r11,r11,_PAGE_ACCESSED@h
237 beq tlb_miss_common_bolted
238 b itlb_miss_kernel_bolted
33 239
34/********************************************************************** 240/**********************************************************************
35 * * 241 * *
diff --git a/arch/powerpc/mm/tlb_nohash.c b/arch/powerpc/mm/tlb_nohash.c
index 0bdad3aecc6..d32ec643c23 100644
--- a/arch/powerpc/mm/tlb_nohash.c
+++ b/arch/powerpc/mm/tlb_nohash.c
@@ -35,6 +35,7 @@
35#include <linux/preempt.h> 35#include <linux/preempt.h>
36#include <linux/spinlock.h> 36#include <linux/spinlock.h>
37#include <linux/memblock.h> 37#include <linux/memblock.h>
38#include <linux/of_fdt.h>
38 39
39#include <asm/tlbflush.h> 40#include <asm/tlbflush.h>
40#include <asm/tlb.h> 41#include <asm/tlb.h>
@@ -102,6 +103,12 @@ unsigned long linear_map_top; /* Top of linear mapping */
102 103
103#endif /* CONFIG_PPC64 */ 104#endif /* CONFIG_PPC64 */
104 105
106#ifdef CONFIG_PPC_FSL_BOOK3E
107/* next_tlbcam_idx is used to round-robin tlbcam entry assignment */
108DEFINE_PER_CPU(int, next_tlbcam_idx);
109EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx);
110#endif
111
105/* 112/*
106 * Base TLB flushing operations: 113 * Base TLB flushing operations:
107 * 114 *
@@ -266,6 +273,17 @@ EXPORT_SYMBOL(flush_tlb_page);
266 273
267#endif /* CONFIG_SMP */ 274#endif /* CONFIG_SMP */
268 275
276#ifdef CONFIG_PPC_47x
277void __init early_init_mmu_47x(void)
278{
279#ifdef CONFIG_SMP
280 unsigned long root = of_get_flat_dt_root();
281 if (of_get_flat_dt_prop(root, "cooperative-partition", NULL))
282 mmu_clear_feature(MMU_FTR_USE_TLBIVAX_BCAST);
283#endif /* CONFIG_SMP */
284}
285#endif /* CONFIG_PPC_47x */
286
269/* 287/*
270 * Flush kernel TLB entries in the given range 288 * Flush kernel TLB entries in the given range
271 */ 289 */
@@ -443,14 +461,27 @@ static void setup_page_sizes(void)
443 } 461 }
444} 462}
445 463
446static void setup_mmu_htw(void) 464static void __patch_exception(int exc, unsigned long addr)
447{ 465{
448 extern unsigned int interrupt_base_book3e; 466 extern unsigned int interrupt_base_book3e;
449 extern unsigned int exc_data_tlb_miss_htw_book3e; 467 unsigned int *ibase = &interrupt_base_book3e;
450 extern unsigned int exc_instruction_tlb_miss_htw_book3e; 468
469 /* Our exceptions vectors start with a NOP and -then- a branch
470 * to deal with single stepping from userspace which stops on
471 * the second instruction. Thus we need to patch the second
472 * instruction of the exception, not the first one
473 */
451 474
452 unsigned int *ibase = &interrupt_base_book3e; 475 patch_branch(ibase + (exc / 4) + 1, addr, 0);
476}
477
478#define patch_exception(exc, name) do { \
479 extern unsigned int name; \
480 __patch_exception((exc), (unsigned long)&name); \
481} while (0)
453 482
483static void setup_mmu_htw(void)
484{
454 /* Check if HW tablewalk is present, and if yes, enable it by: 485 /* Check if HW tablewalk is present, and if yes, enable it by:
455 * 486 *
456 * - patching the TLB miss handlers to branch to the 487 * - patching the TLB miss handlers to branch to the
@@ -462,19 +493,12 @@ static void setup_mmu_htw(void)
462 493
463 if ((tlb0cfg & TLBnCFG_IND) && 494 if ((tlb0cfg & TLBnCFG_IND) &&
464 (tlb0cfg & TLBnCFG_PT)) { 495 (tlb0cfg & TLBnCFG_PT)) {
465 /* Our exceptions vectors start with a NOP and -then- a branch 496 patch_exception(0x1c0, exc_data_tlb_miss_htw_book3e);
466 * to deal with single stepping from userspace which stops on 497 patch_exception(0x1e0, exc_instruction_tlb_miss_htw_book3e);
467 * the second instruction. Thus we need to patch the second
468 * instruction of the exception, not the first one
469 */
470 patch_branch(ibase + (0x1c0 / 4) + 1,
471 (unsigned long)&exc_data_tlb_miss_htw_book3e, 0);
472 patch_branch(ibase + (0x1e0 / 4) + 1,
473 (unsigned long)&exc_instruction_tlb_miss_htw_book3e, 0);
474 book3e_htw_enabled = 1; 498 book3e_htw_enabled = 1;
475 } 499 }
476 pr_info("MMU: Book3E Page Tables %s\n", 500 pr_info("MMU: Book3E HW tablewalk %s\n",
477 book3e_htw_enabled ? "Enabled" : "Disabled"); 501 book3e_htw_enabled ? "enabled" : "not supported");
478} 502}
479 503
480/* 504/*
@@ -549,6 +573,9 @@ static void __early_init_mmu(int boot_cpu)
549 /* limit memory so we dont have linear faults */ 573 /* limit memory so we dont have linear faults */
550 memblock_enforce_memory_limit(linear_map_top); 574 memblock_enforce_memory_limit(linear_map_top);
551 memblock_analyze(); 575 memblock_analyze();
576
577 patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e);
578 patch_exception(0x1e0, exc_instruction_tlb_miss_bolted_book3e);
552 } 579 }
553#endif 580#endif
554 581
@@ -584,4 +611,11 @@ void setup_initial_memory_limit(phys_addr_t first_memblock_base,
584 /* Finally limit subsequent allocations */ 611 /* Finally limit subsequent allocations */
585 memblock_set_current_limit(first_memblock_base + ppc64_rma_size); 612 memblock_set_current_limit(first_memblock_base + ppc64_rma_size);
586} 613}
614#else /* ! CONFIG_PPC64 */
615void __init early_init_mmu(void)
616{
617#ifdef CONFIG_PPC_47x
618 early_init_mmu_47x();
619#endif
620}
587#endif /* CONFIG_PPC64 */ 621#endif /* CONFIG_PPC64 */
generated by cgit v1.2.2 (git 2.25.0) at 2025-07-24 03:14:00 -0400