aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/mm
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2016-07-31 00:01:36 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2016-07-31 00:01:36 -0400
commitbad60e6f259a01cf9f29a1ef8d435ab6c60b2de9 (patch)
treec9aaa8166735659761239c117af2b11b022bc6cb /arch/powerpc/mm
parentdd0f0cf58af70dc9267409f113bea772d57f675c (diff)
parent719dbb2df78fc9a40e28392b07cd715bfc5a665c (diff)
Merge tag 'powerpc-4.8-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux
Pull powerpc updates from Michael Ellerman: "Highlights: - PowerNV PCI hotplug support. - Lots more Power9 support. - eBPF JIT support on ppc64le. - Lots of cxl updates. - Boot code consolidation. Bug fixes: - Fix spin_unlock_wait() from Boqun Feng - Fix stack pointer corruption in __tm_recheckpoint() from Michael Neuling - Fix multiple bugs in memory_hotplug_max() from Bharata B Rao - mm: Ensure "special" zones are empty from Oliver O'Halloran - ftrace: Separate the heuristics for checking call sites from Michael Ellerman - modules: Never restore r2 for a mprofile-kernel style mcount() call from Michael Ellerman - Fix endianness when reading TCEs from Alexey Kardashevskiy - start rtasd before PCI probing from Greg Kurz - PCI: rpaphp: Fix slot registration for multiple slots under a PHB from Tyrel Datwyler - powerpc/mm: Add memory barrier in __hugepte_alloc() from Sukadev Bhattiprolu Cleanups & fixes: - Drop support for MPIC in pseries from Rashmica Gupta - Define and use PPC64_ELF_ABI_v2/v1 from Michael Ellerman - Remove unused symbols in asm-offsets.c from Rashmica Gupta - Fix SRIOV not building without EEH enabled from Russell Currey - Remove kretprobe_trampoline_holder from Thiago Jung Bauermann - Reduce log level of PCI I/O space warning from Benjamin Herrenschmidt - Add array bounds checking to crash_shutdown_handlers from Suraj Jitindar Singh - Avoid -maltivec when using clang integrated assembler from Anton Blanchard - Fix array overrun in ppc_rtas() syscall from Andrew Donnellan - Fix error return value in cmm_mem_going_offline() from Rasmus Villemoes - export cpu_to_core_id() from Mauricio Faria de Oliveira - Remove old symbols from defconfigs from Andrew Donnellan - Update obsolete comments in setup_32.c about entry conditions from Benjamin Herrenschmidt - Add comment explaining the purpose of setup_kdump_trampoline() from Benjamin Herrenschmidt - Merge the RELOCATABLE config entries for ppc32 and ppc64 from Kevin Hao - Remove RELOCATABLE_PPC32 from Kevin Hao - Fix .long's in tlb-radix.c to more meaningful from Balbir Singh Minor cleanups & fixes: - Andrew Donnellan, Anna-Maria Gleixner, Anton Blanchard, Benjamin Herrenschmidt, Bharata B Rao, Christophe Leroy, Colin Ian King, Geliang Tang, Greg Kurz, Madhavan Srinivasan, Michael Ellerman, Michael Ellerman, Stephen Rothwell, Stewart Smith. Freescale updates from Scott: - "Highlights include more 8xx optimizations, device tree updates, and MVME7100 support." PowerNV PCI hotplug from Gavin Shan: - PCI: Add pcibios_setup_bridge() - Override pcibios_setup_bridge() - Remove PCI_RESET_DELAY_US - Move pnv_pci_ioda_setup_opal_tce_kill() around - Increase PE# capacity - Allocate PE# in reverse order - Create PEs in pcibios_setup_bridge() - Setup PE for root bus - Extend PCI bridge resources - Make pnv_ioda_deconfigure_pe() visible - Dynamically release PE - Update bridge windows on PCI plug - Delay populating pdn - Support PCI slot ID - Use PCI slot reset infrastructure - Introduce pnv_pci_get_slot_id() - Functions to get/set PCI slot state - PCI/hotplug: PowerPC PowerNV PCI hotplug driver - Print correct PHB type names Power9 idle support from Shreyas B. Prabhu: - set power_save func after the idle states are initialized - Use PNV_THREAD_WINKLE macro while requesting for winkle - make hypervisor state restore a function - Rename idle_power7.S to idle_book3s.S - Rename reusable idle functions to hardware agnostic names - Make pnv_powersave_common more generic - abstraction for saving SPRs before entering deep idle states - Add platform support for stop instruction - cpuidle/powernv: Use CPUIDLE_STATE_MAX instead of MAX_POWERNV_IDLE_STATES - cpuidle/powernv: cleanup cpuidle-powernv.c - cpuidle/powernv: Add support for POWER ISA v3 idle states - Use deepest stop state when cpu is offlined Power9 PMU from Madhavan Srinivasan: - factor out power8 pmu macros and defines - factor out power8 pmu functions - factor out power8 __init_pmu code - Add power9 event list macros for generic and cache events - Power9 PMU support - Export Power9 generic and cache events to sysfs Power9 preliminary interrupt & PCI support from Benjamin Herrenschmidt: - Add XICS emulation APIs - Move a few exception common handlers to make room - Add support for HV virtualization interrupts - Add mechanism to force a replay of interrupts - Add ICP OPAL backend - Discover IODA3 PHBs - pci: Remove obsolete SW invalidate - opal: Add real mode call wrappers - Rename TCE invalidation calls - Remove SWINV constants and obsolete TCE code - Rework accessing the TCE invalidate register - Fallback to OPAL for TCE invalidations - Use the device-tree to get available range of M64's - Check status of a PHB before using it - pci: Don't try to allocate resources that will be reassigned Other Power9: - Send SIGBUS on unaligned copy and paste from Chris Smart - Large Decrementer support from Oliver O'Halloran - Load Monitor Register Support from Jack Miller Performance improvements from Anton Blanchard: - Avoid load hit store in __giveup_fpu() and __giveup_altivec() - Avoid load hit store in setup_sigcontext() - Remove assembly versions of strcpy, strcat, strlen and strcmp - Align hot loops of some string functions eBPF JIT from Naveen N. Rao: - Fix/enhance 32-bit Load Immediate implementation - Optimize 64-bit Immediate loads - Introduce rotate immediate instructions - A few cleanups - Isolate classic BPF JIT specifics into a separate header - Implement JIT compiler for extended BPF Operator Panel driver from Suraj Jitindar Singh: - devicetree/bindings: Add binding for operator panel on FSP machines - Add inline function to get rc from an ASYNC_COMP opal_msg - Add driver for operator panel on FSP machines Sparse fixes from Daniel Axtens: - make some things static - Introduce asm-prototypes.h - Include headers containing prototypes - Use #ifdef __BIG_ENDIAN__ #else for REG_BYTE - kvm: Clarify __user annotations - Pass endianness to sparse - Make ppc_md.{halt, restart} __noreturn MM fixes & cleanups from Aneesh Kumar K.V: - radix: Update LPCR HR bit as per ISA - use _raw variant of page table accessors - Compile out radix related functions if RADIX_MMU is disabled - Clear top 16 bits of va only on older cpus - Print formation regarding the the MMU mode - hash: Update SDR1 size encoding as documented in ISA 3.0 - radix: Update PID switch sequence - radix: Update machine call back to support new HCALL. - radix: Add LPID based tlb flush helpers - radix: Add a kernel command line to disable radix - Cleanup LPCR defines Boot code consolidation from Benjamin Herrenschmidt: - Move epapr_paravirt_early_init() to early_init_devtree() - cell: Don't use flat device-tree after boot - ge_imp3a: Don't use the flat device-tree after boot - mpc85xx_ds: Don't use the flat device-tree after boot - mpc85xx_rdb: Don't use the flat device-tree after boot - Don't test for machine type in rtas_initialize() - Don't test for machine type in smp_setup_cpu_maps() - dt: Add of_device_compatible_match() - Factor do_feature_fixup calls - Move 64-bit feature fixup earlier - Move 64-bit memory reserves to setup_arch() - Use a cachable DART - Move FW feature probing out of pseries probe() - Put exception configuration in a common place - Remove early allocation of the SMU command buffer - Move MMU backend selection out of platform code - pasemi: Remove IOBMAP allocation from platform probe() - mm/hash: Don't use machine_is() early during boot - Don't test for machine type to detect HEA special case - pmac: Remove spurrious machine type test - Move hash table ops to a separate structure - Ensure that ppc_md is empty before probing for machine type - Move 64-bit probe_machine() to later in the boot process - Move 32-bit probe() machine to later in the boot process - Get rid of ppc_md.init_early() - Move the boot time info banner to a separate function - Move setting of {i,d}cache_bsize to initialize_cache_info() - Move the content of setup_system() to setup_arch() - Move cache info inits to a separate function - Re-order the call to smp_setup_cpu_maps() - Re-order setup_panic() - Make a few boot functions __init - Merge 32-bit and 64-bit setup_arch() Other new features: - tty/hvc: Use IRQF_SHARED for OPAL hvc consoles from Sam Mendoza-Jonas - tty/hvc: Use opal irqchip interface if available from Sam Mendoza-Jonas - powerpc: Add module autoloading based on CPU features from Alastair D'Silva - crypto: vmx - Convert to CPU feature based module autoloading from Alastair D'Silva - Wake up kopald polling thread before waiting for events from Benjamin Herrenschmidt - xmon: Dump ISA 2.06 SPRs from Michael Ellerman - xmon: Dump ISA 2.07 SPRs from Michael Ellerman - Add a parameter to disable 1TB segs from Oliver O'Halloran - powerpc/boot: Add OPAL console to epapr wrappers from Oliver O'Halloran - Assign fixed PHB number based on device-tree properties from Guilherme G. Piccoli - pseries: Add pseries hotplug workqueue from John Allen - pseries: Add support for hotplug interrupt source from John Allen - pseries: Use kernel hotplug queue for PowerVM hotplug events from John Allen - pseries: Move property cloning into its own routine from Nathan Fontenot - pseries: Dynamic add entires to associativity lookup array from Nathan Fontenot - pseries: Auto-online hotplugged memory from Nathan Fontenot - pseries: Remove call to memblock_add() from Nathan Fontenot cxl: - Add set and get private data to context struct from Michael Neuling - make base more explicitly non-modular from Paul Gortmaker - Use for_each_compatible_node() macro from Wei Yongjun - Frederic Barrat - Abstract the differences between the PSL and XSL - Make vPHB device node match adapter's - Philippe Bergheaud - Add mechanism for delivering AFU driver specific events - Ignore CAPI adapters misplaced in switched slots - Refine slice error debug messages - Andrew Donnellan - static-ify variables to fix sparse warnings - PCI/hotplug: pnv_php: export symbols and move struct types needed by cxl - PCI/hotplug: pnv_php: handle OPAL_PCI_SLOT_OFFLINE power state - Add cxl_check_and_switch_mode() API to switch bi-modal cards - remove dead Kconfig options - fix potential NULL dereference in free_adapter() - Ian Munsie - Update process element after allocating interrupts - Add support for CAPP DMA mode - Fix allowing bogus AFU descriptors with 0 maximum processes - Fix allocating a minimum of 2 pages for the SPA - Fix bug where AFU disable operation had no effect - Workaround XSL bug that does not clear the RA bit after a reset - Fix NULL pointer dereference on kernel contexts with no AFU interrupts - powerpc/powernv: Split cxl code out into a separate file - Add cxl_slot_is_supported API - Enable bus mastering for devices using CAPP DMA mode - Move cxl_afu_get / cxl_afu_put to base - Allow a default context to be associated with an external pci_dev - Do not create vPHB if there are no AFU configuration records - powerpc/powernv: Add support for the cxl kernel api on the real phb - Add support for using the kernel API with a real PHB - Add kernel APIs to get & set the max irqs per context - Add preliminary workaround for CX4 interrupt limitation - Add support for interrupts on the Mellanox CX4 - Workaround PE=0 hardware limitation in Mellanox CX4 - powerpc/powernv: Fix pci-cxl.c build when CONFIG_MODULES=n selftests: - Test unaligned copy and paste from Chris Smart - Load Monitor Register Tests from Jack Miller - Cyril Bur - exec() with suspended transaction - Use signed long to read perf_event_paranoid - Fix usage message in context_switch - Fix generation of vector instructions/types in context_switch - Michael Ellerman - Use "Delta" rather than "Error" in normal output - Import Anton's mmap & futex micro benchmarks - Add a test for PROT_SAO" * tag 'powerpc-4.8-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux: (263 commits) powerpc/mm: Parenthesise IS_ENABLED() in if condition tty/hvc: Use opal irqchip interface if available tty/hvc: Use IRQF_SHARED for OPAL hvc consoles selftests/powerpc: exec() with suspended transaction powerpc: Improve comment explaining why we modify VRSAVE powerpc/mm: Drop unused externs for hpte_init_beat[_v3]() powerpc/mm: Rename hpte_init_lpar() and move the fallback to a header powerpc/mm: Fix build break when PPC_NATIVE=n crypto: vmx - Convert to CPU feature based module autoloading powerpc: Add module autoloading based on CPU features powerpc/powernv/ioda: Fix endianness when reading TCEs powerpc/mm: Add memory barrier in __hugepte_alloc() powerpc/modules: Never restore r2 for a mprofile-kernel style mcount() call powerpc/ftrace: Separate the heuristics for checking call sites powerpc: Merge 32-bit and 64-bit setup_arch() powerpc/64: Make a few boot functions __init powerpc: Re-order setup_panic() powerpc: Re-order the call to smp_setup_cpu_maps() powerpc/32: Move cache info inits to a separate function powerpc/64: Move the content of setup_system() to setup_arch() ...
Diffstat (limited to 'arch/powerpc/mm')
-rw-r--r--arch/powerpc/mm/8xx_mmu.c131
-rw-r--r--arch/powerpc/mm/hash64_4k.c18
-rw-r--r--arch/powerpc/mm/hash64_64k.c39
-rw-r--r--arch/powerpc/mm/hash_native_64.c36
-rw-r--r--arch/powerpc/mm/hash_utils_64.c153
-rw-r--r--arch/powerpc/mm/hugepage-hash64.c17
-rw-r--r--arch/powerpc/mm/hugetlbpage-hash64.c4
-rw-r--r--arch/powerpc/mm/hugetlbpage.c7
-rw-r--r--arch/powerpc/mm/init_32.c5
-rw-r--r--arch/powerpc/mm/mem.c18
-rw-r--r--arch/powerpc/mm/mmu_context_book3s64.c5
-rw-r--r--arch/powerpc/mm/mmu_decl.h3
-rw-r--r--arch/powerpc/mm/numa.c36
-rw-r--r--arch/powerpc/mm/pgtable-radix.c17
-rw-r--r--arch/powerpc/mm/tlb-radix.c65
15 files changed, 361 insertions, 193 deletions
diff --git a/arch/powerpc/mm/8xx_mmu.c b/arch/powerpc/mm/8xx_mmu.c
index 949100577db5..6c5025e81236 100644
--- a/arch/powerpc/mm/8xx_mmu.c
+++ b/arch/powerpc/mm/8xx_mmu.c
@@ -13,62 +13,115 @@
13 */ 13 */
14 14
15#include <linux/memblock.h> 15#include <linux/memblock.h>
16#include <asm/fixmap.h>
17#include <asm/code-patching.h>
16 18
17#include "mmu_decl.h" 19#include "mmu_decl.h"
18 20
21#define IMMR_SIZE (FIX_IMMR_SIZE << PAGE_SHIFT)
22
19extern int __map_without_ltlbs; 23extern int __map_without_ltlbs;
24
20/* 25/*
21 * MMU_init_hw does the chip-specific initialization of the MMU hardware. 26 * Return PA for this VA if it is in IMMR area, or 0
22 */ 27 */
23void __init MMU_init_hw(void) 28phys_addr_t v_block_mapped(unsigned long va)
24{ 29{
25 /* Nothing to do for the time being but keep it similar to other PPC */ 30 unsigned long p = PHYS_IMMR_BASE;
31
32 if (__map_without_ltlbs)
33 return 0;
34 if (va >= VIRT_IMMR_BASE && va < VIRT_IMMR_BASE + IMMR_SIZE)
35 return p + va - VIRT_IMMR_BASE;
36 return 0;
37}
38
39/*
40 * Return VA for a given PA or 0 if not mapped
41 */
42unsigned long p_block_mapped(phys_addr_t pa)
43{
44 unsigned long p = PHYS_IMMR_BASE;
45
46 if (__map_without_ltlbs)
47 return 0;
48 if (pa >= p && pa < p + IMMR_SIZE)
49 return VIRT_IMMR_BASE + pa - p;
50 return 0;
26} 51}
27 52
28#define LARGE_PAGE_SIZE_4M (1<<22)
29#define LARGE_PAGE_SIZE_8M (1<<23) 53#define LARGE_PAGE_SIZE_8M (1<<23)
30#define LARGE_PAGE_SIZE_64M (1<<26)
31 54
32unsigned long __init mmu_mapin_ram(unsigned long top) 55/*
56 * MMU_init_hw does the chip-specific initialization of the MMU hardware.
57 */
58void __init MMU_init_hw(void)
33{ 59{
34 unsigned long v, s, mapped; 60 /* PIN up to the 3 first 8Mb after IMMR in DTLB table */
35 phys_addr_t p; 61#ifdef CONFIG_PIN_TLB
62 unsigned long ctr = mfspr(SPRN_MD_CTR) & 0xfe000000;
63 unsigned long flags = 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY;
64#ifdef CONFIG_PIN_TLB_IMMR
65 int i = 29;
66#else
67 int i = 28;
68#endif
69 unsigned long addr = 0;
70 unsigned long mem = total_lowmem;
71
72 for (; i < 32 && mem >= LARGE_PAGE_SIZE_8M; i++) {
73 mtspr(SPRN_MD_CTR, ctr | (i << 8));
74 mtspr(SPRN_MD_EPN, (unsigned long)__va(addr) | MD_EVALID);
75 mtspr(SPRN_MD_TWC, MD_PS8MEG | MD_SVALID);
76 mtspr(SPRN_MD_RPN, addr | flags | _PAGE_PRESENT);
77 addr += LARGE_PAGE_SIZE_8M;
78 mem -= LARGE_PAGE_SIZE_8M;
79 }
80#endif
81}
36 82
37 v = KERNELBASE; 83static void mmu_mapin_immr(void)
38 p = 0; 84{
39 s = top; 85 unsigned long p = PHYS_IMMR_BASE;
86 unsigned long v = VIRT_IMMR_BASE;
87 unsigned long f = pgprot_val(PAGE_KERNEL_NCG);
88 int offset;
40 89
41 if (__map_without_ltlbs) 90 for (offset = 0; offset < IMMR_SIZE; offset += PAGE_SIZE)
42 return 0; 91 map_page(v + offset, p + offset, f);
92}
43 93
44#ifdef CONFIG_PPC_4K_PAGES 94/* Address of instructions to patch */
45 while (s >= LARGE_PAGE_SIZE_8M) { 95#ifndef CONFIG_PIN_TLB_IMMR
46 pmd_t *pmdp; 96extern unsigned int DTLBMiss_jmp;
47 unsigned long val = p | MD_PS8MEG; 97#endif
98extern unsigned int DTLBMiss_cmp, FixupDAR_cmp;
48 99
49 pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v); 100void mmu_patch_cmp_limit(unsigned int *addr, unsigned long mapped)
50 *pmdp++ = __pmd(val); 101{
51 *pmdp++ = __pmd(val + LARGE_PAGE_SIZE_4M); 102 unsigned int instr = *addr;
52 103
53 v += LARGE_PAGE_SIZE_8M; 104 instr &= 0xffff0000;
54 p += LARGE_PAGE_SIZE_8M; 105 instr |= (unsigned long)__va(mapped) >> 16;
55 s -= LARGE_PAGE_SIZE_8M; 106 patch_instruction(addr, instr);
56 } 107}
57#else /* CONFIG_PPC_16K_PAGES */
58 while (s >= LARGE_PAGE_SIZE_64M) {
59 pmd_t *pmdp;
60 unsigned long val = p | MD_PS8MEG;
61 108
62 pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v); 109unsigned long __init mmu_mapin_ram(unsigned long top)
63 *pmdp++ = __pmd(val); 110{
111 unsigned long mapped;
64 112
65 v += LARGE_PAGE_SIZE_64M; 113 if (__map_without_ltlbs) {
66 p += LARGE_PAGE_SIZE_64M; 114 mapped = 0;
67 s -= LARGE_PAGE_SIZE_64M; 115 mmu_mapin_immr();
68 } 116#ifndef CONFIG_PIN_TLB_IMMR
117 patch_instruction(&DTLBMiss_jmp, PPC_INST_NOP);
69#endif 118#endif
119 } else {
120 mapped = top & ~(LARGE_PAGE_SIZE_8M - 1);
121 }
70 122
71 mapped = top - s; 123 mmu_patch_cmp_limit(&DTLBMiss_cmp, mapped);
124 mmu_patch_cmp_limit(&FixupDAR_cmp, mapped);
72 125
73 /* If the size of RAM is not an exact power of two, we may not 126 /* If the size of RAM is not an exact power of two, we may not
74 * have covered RAM in its entirety with 8 MiB 127 * have covered RAM in its entirety with 8 MiB
@@ -77,7 +130,8 @@ unsigned long __init mmu_mapin_ram(unsigned long top)
77 * coverage with normal-sized pages (or other reasons) do not 130 * coverage with normal-sized pages (or other reasons) do not
78 * attempt to allocate outside the allowed range. 131 * attempt to allocate outside the allowed range.
79 */ 132 */
80 memblock_set_current_limit(mapped); 133 if (mapped)
134 memblock_set_current_limit(mapped);
81 135
82 return mapped; 136 return mapped;
83} 137}
@@ -90,13 +144,8 @@ void setup_initial_memory_limit(phys_addr_t first_memblock_base,
90 */ 144 */
91 BUG_ON(first_memblock_base != 0); 145 BUG_ON(first_memblock_base != 0);
92 146
93#ifdef CONFIG_PIN_TLB
94 /* 8xx can only access 24MB at the moment */ 147 /* 8xx can only access 24MB at the moment */
95 memblock_set_current_limit(min_t(u64, first_memblock_size, 0x01800000)); 148 memblock_set_current_limit(min_t(u64, first_memblock_size, 0x01800000));
96#else
97 /* 8xx can only access 8MB at the moment */
98 memblock_set_current_limit(min_t(u64, first_memblock_size, 0x00800000));
99#endif
100} 149}
101 150
102/* 151/*
diff --git a/arch/powerpc/mm/hash64_4k.c b/arch/powerpc/mm/hash64_4k.c
index 6333b273d2d5..42c702b3be1f 100644
--- a/arch/powerpc/mm/hash64_4k.c
+++ b/arch/powerpc/mm/hash64_4k.c
@@ -70,8 +70,8 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
70 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 70 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
71 slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT; 71 slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;
72 72
73 if (ppc_md.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_4K, 73 if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_4K,
74 MMU_PAGE_4K, ssize, flags) == -1) 74 MMU_PAGE_4K, ssize, flags) == -1)
75 old_pte &= ~_PAGE_HPTEFLAGS; 75 old_pte &= ~_PAGE_HPTEFLAGS;
76 } 76 }
77 77
@@ -84,21 +84,23 @@ repeat:
84 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; 84 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
85 85
86 /* Insert into the hash table, primary slot */ 86 /* Insert into the hash table, primary slot */
87 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0, 87 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
88 MMU_PAGE_4K, MMU_PAGE_4K, ssize); 88 MMU_PAGE_4K, MMU_PAGE_4K, ssize);
89 /* 89 /*
90 * Primary is full, try the secondary 90 * Primary is full, try the secondary
91 */ 91 */
92 if (unlikely(slot == -1)) { 92 if (unlikely(slot == -1)) {
93 hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; 93 hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
94 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, 94 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
95 rflags, HPTE_V_SECONDARY, 95 rflags,
96 MMU_PAGE_4K, MMU_PAGE_4K, ssize); 96 HPTE_V_SECONDARY,
97 MMU_PAGE_4K,
98 MMU_PAGE_4K, ssize);
97 if (slot == -1) { 99 if (slot == -1) {
98 if (mftb() & 0x1) 100 if (mftb() & 0x1)
99 hpte_group = ((hash & htab_hash_mask) * 101 hpte_group = ((hash & htab_hash_mask) *
100 HPTES_PER_GROUP) & ~0x7UL; 102 HPTES_PER_GROUP) & ~0x7UL;
101 ppc_md.hpte_remove(hpte_group); 103 mmu_hash_ops.hpte_remove(hpte_group);
102 /* 104 /*
103 * FIXME!! Should be try the group from which we removed ? 105 * FIXME!! Should be try the group from which we removed ?
104 */ 106 */
diff --git a/arch/powerpc/mm/hash64_64k.c b/arch/powerpc/mm/hash64_64k.c
index 16644e1f4e6b..3bbbea07378c 100644
--- a/arch/powerpc/mm/hash64_64k.c
+++ b/arch/powerpc/mm/hash64_64k.c
@@ -133,9 +133,9 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
133 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 133 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
134 slot += hidx & _PTEIDX_GROUP_IX; 134 slot += hidx & _PTEIDX_GROUP_IX;
135 135
136 ret = ppc_md.hpte_updatepp(slot, rflags, vpn, 136 ret = mmu_hash_ops.hpte_updatepp(slot, rflags, vpn,
137 MMU_PAGE_4K, MMU_PAGE_4K, 137 MMU_PAGE_4K, MMU_PAGE_4K,
138 ssize, flags); 138 ssize, flags);
139 /* 139 /*
140 *if we failed because typically the HPTE wasn't really here 140 *if we failed because typically the HPTE wasn't really here
141 * we try an insertion. 141 * we try an insertion.
@@ -166,21 +166,22 @@ repeat:
166 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; 166 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
167 167
168 /* Insert into the hash table, primary slot */ 168 /* Insert into the hash table, primary slot */
169 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0, 169 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
170 MMU_PAGE_4K, MMU_PAGE_4K, ssize); 170 MMU_PAGE_4K, MMU_PAGE_4K, ssize);
171 /* 171 /*
172 * Primary is full, try the secondary 172 * Primary is full, try the secondary
173 */ 173 */
174 if (unlikely(slot == -1)) { 174 if (unlikely(slot == -1)) {
175 hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; 175 hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
176 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, 176 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
177 rflags, HPTE_V_SECONDARY, 177 rflags, HPTE_V_SECONDARY,
178 MMU_PAGE_4K, MMU_PAGE_4K, ssize); 178 MMU_PAGE_4K, MMU_PAGE_4K,
179 ssize);
179 if (slot == -1) { 180 if (slot == -1) {
180 if (mftb() & 0x1) 181 if (mftb() & 0x1)
181 hpte_group = ((hash & htab_hash_mask) * 182 hpte_group = ((hash & htab_hash_mask) *
182 HPTES_PER_GROUP) & ~0x7UL; 183 HPTES_PER_GROUP) & ~0x7UL;
183 ppc_md.hpte_remove(hpte_group); 184 mmu_hash_ops.hpte_remove(hpte_group);
184 /* 185 /*
185 * FIXME!! Should be try the group from which we removed ? 186 * FIXME!! Should be try the group from which we removed ?
186 */ 187 */
@@ -272,8 +273,9 @@ int __hash_page_64K(unsigned long ea, unsigned long access,
272 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 273 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
273 slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT; 274 slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;
274 275
275 if (ppc_md.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_64K, 276 if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_64K,
276 MMU_PAGE_64K, ssize, flags) == -1) 277 MMU_PAGE_64K, ssize,
278 flags) == -1)
277 old_pte &= ~_PAGE_HPTEFLAGS; 279 old_pte &= ~_PAGE_HPTEFLAGS;
278 } 280 }
279 281
@@ -286,21 +288,24 @@ repeat:
286 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; 288 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
287 289
288 /* Insert into the hash table, primary slot */ 290 /* Insert into the hash table, primary slot */
289 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0, 291 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
290 MMU_PAGE_64K, MMU_PAGE_64K, ssize); 292 MMU_PAGE_64K, MMU_PAGE_64K,
293 ssize);
291 /* 294 /*
292 * Primary is full, try the secondary 295 * Primary is full, try the secondary
293 */ 296 */
294 if (unlikely(slot == -1)) { 297 if (unlikely(slot == -1)) {
295 hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; 298 hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
296 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, 299 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
297 rflags, HPTE_V_SECONDARY, 300 rflags,
298 MMU_PAGE_64K, MMU_PAGE_64K, ssize); 301 HPTE_V_SECONDARY,
302 MMU_PAGE_64K,
303 MMU_PAGE_64K, ssize);
299 if (slot == -1) { 304 if (slot == -1) {
300 if (mftb() & 0x1) 305 if (mftb() & 0x1)
301 hpte_group = ((hash & htab_hash_mask) * 306 hpte_group = ((hash & htab_hash_mask) *
302 HPTES_PER_GROUP) & ~0x7UL; 307 HPTES_PER_GROUP) & ~0x7UL;
303 ppc_md.hpte_remove(hpte_group); 308 mmu_hash_ops.hpte_remove(hpte_group);
304 /* 309 /*
305 * FIXME!! Should be try the group from which we removed ? 310 * FIXME!! Should be try the group from which we removed ?
306 */ 311 */
diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index f8a871a72985..88ce7d212320 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -55,7 +55,7 @@ static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
55 * We need 14 to 65 bits of va for a tlibe of 4K page 55 * We need 14 to 65 bits of va for a tlibe of 4K page
56 * With vpn we ignore the lower VPN_SHIFT bits already. 56 * With vpn we ignore the lower VPN_SHIFT bits already.
57 * And top two bits are already ignored because we can 57 * And top two bits are already ignored because we can
58 * only accomadate 76 bits in a 64 bit vpn with a VPN_SHIFT 58 * only accomodate 76 bits in a 64 bit vpn with a VPN_SHIFT
59 * of 12. 59 * of 12.
60 */ 60 */
61 va = vpn << VPN_SHIFT; 61 va = vpn << VPN_SHIFT;
@@ -64,7 +64,8 @@ static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
64 * Older versions of the architecture (2.02 and earler) require the 64 * Older versions of the architecture (2.02 and earler) require the
65 * masking of the top 16 bits. 65 * masking of the top 16 bits.
66 */ 66 */
67 va &= ~(0xffffULL << 48); 67 if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA))
68 va &= ~(0xffffULL << 48);
68 69
69 switch (psize) { 70 switch (psize) {
70 case MMU_PAGE_4K: 71 case MMU_PAGE_4K:
@@ -113,7 +114,8 @@ static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize)
113 * Older versions of the architecture (2.02 and earler) require the 114 * Older versions of the architecture (2.02 and earler) require the
114 * masking of the top 16 bits. 115 * masking of the top 16 bits.
115 */ 116 */
116 va &= ~(0xffffULL << 48); 117 if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA))
118 va &= ~(0xffffULL << 48);
117 119
118 switch (psize) { 120 switch (psize) {
119 case MMU_PAGE_4K: 121 case MMU_PAGE_4K:
@@ -605,7 +607,7 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
605 * crashdump and all bets are off anyway. 607 * crashdump and all bets are off anyway.
606 * 608 *
607 * TODO: add batching support when enabled. remember, no dynamic memory here, 609 * TODO: add batching support when enabled. remember, no dynamic memory here,
608 * athough there is the control page available... 610 * although there is the control page available...
609 */ 611 */
610static void native_hpte_clear(void) 612static void native_hpte_clear(void)
611{ 613{
@@ -723,23 +725,29 @@ static void native_flush_hash_range(unsigned long number, int local)
723 local_irq_restore(flags); 725 local_irq_restore(flags);
724} 726}
725 727
726static int native_update_partition_table(u64 patb1) 728static int native_register_proc_table(unsigned long base, unsigned long page_size,
729 unsigned long table_size)
727{ 730{
731 unsigned long patb1 = base << 25; /* VSID */
732
733 patb1 |= (page_size << 5); /* sllp */
734 patb1 |= table_size;
735
728 partition_tb->patb1 = cpu_to_be64(patb1); 736 partition_tb->patb1 = cpu_to_be64(patb1);
729 return 0; 737 return 0;
730} 738}
731 739
732void __init hpte_init_native(void) 740void __init hpte_init_native(void)
733{ 741{
734 ppc_md.hpte_invalidate = native_hpte_invalidate; 742 mmu_hash_ops.hpte_invalidate = native_hpte_invalidate;
735 ppc_md.hpte_updatepp = native_hpte_updatepp; 743 mmu_hash_ops.hpte_updatepp = native_hpte_updatepp;
736 ppc_md.hpte_updateboltedpp = native_hpte_updateboltedpp; 744 mmu_hash_ops.hpte_updateboltedpp = native_hpte_updateboltedpp;
737 ppc_md.hpte_insert = native_hpte_insert; 745 mmu_hash_ops.hpte_insert = native_hpte_insert;
738 ppc_md.hpte_remove = native_hpte_remove; 746 mmu_hash_ops.hpte_remove = native_hpte_remove;
739 ppc_md.hpte_clear_all = native_hpte_clear; 747 mmu_hash_ops.hpte_clear_all = native_hpte_clear;
740 ppc_md.flush_hash_range = native_flush_hash_range; 748 mmu_hash_ops.flush_hash_range = native_flush_hash_range;
741 ppc_md.hugepage_invalidate = native_hugepage_invalidate; 749 mmu_hash_ops.hugepage_invalidate = native_hugepage_invalidate;
742 750
743 if (cpu_has_feature(CPU_FTR_ARCH_300)) 751 if (cpu_has_feature(CPU_FTR_ARCH_300))
744 ppc_md.update_partition_table = native_update_partition_table; 752 ppc_md.register_process_table = native_register_proc_table;
745} 753}
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 2971ea18c768..b78b5d211278 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -34,6 +34,7 @@
34#include <linux/signal.h> 34#include <linux/signal.h>
35#include <linux/memblock.h> 35#include <linux/memblock.h>
36#include <linux/context_tracking.h> 36#include <linux/context_tracking.h>
37#include <linux/libfdt.h>
37 38
38#include <asm/processor.h> 39#include <asm/processor.h>
39#include <asm/pgtable.h> 40#include <asm/pgtable.h>
@@ -58,6 +59,7 @@
58#include <asm/firmware.h> 59#include <asm/firmware.h>
59#include <asm/tm.h> 60#include <asm/tm.h>
60#include <asm/trace.h> 61#include <asm/trace.h>
62#include <asm/ps3.h>
61 63
62#ifdef DEBUG 64#ifdef DEBUG
63#define DBG(fmt...) udbg_printf(fmt) 65#define DBG(fmt...) udbg_printf(fmt)
@@ -87,10 +89,6 @@
87 * 89 *
88 */ 90 */
89 91
90#ifdef CONFIG_U3_DART
91extern unsigned long dart_tablebase;
92#endif /* CONFIG_U3_DART */
93
94static unsigned long _SDR1; 92static unsigned long _SDR1;
95struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT]; 93struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT];
96EXPORT_SYMBOL_GPL(mmu_psize_defs); 94EXPORT_SYMBOL_GPL(mmu_psize_defs);
@@ -120,6 +118,8 @@ static u8 *linear_map_hash_slots;
120static unsigned long linear_map_hash_count; 118static unsigned long linear_map_hash_count;
121static DEFINE_SPINLOCK(linear_map_hash_lock); 119static DEFINE_SPINLOCK(linear_map_hash_lock);
122#endif /* CONFIG_DEBUG_PAGEALLOC */ 120#endif /* CONFIG_DEBUG_PAGEALLOC */
121struct mmu_hash_ops mmu_hash_ops;
122EXPORT_SYMBOL(mmu_hash_ops);
123 123
124/* There are definitions of page sizes arrays to be used when none 124/* There are definitions of page sizes arrays to be used when none
125 * is provided by the firmware. 125 * is provided by the firmware.
@@ -278,9 +278,10 @@ int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
278 hash = hpt_hash(vpn, shift, ssize); 278 hash = hpt_hash(vpn, shift, ssize);
279 hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP); 279 hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP);
280 280
281 BUG_ON(!ppc_md.hpte_insert); 281 BUG_ON(!mmu_hash_ops.hpte_insert);
282 ret = ppc_md.hpte_insert(hpteg, vpn, paddr, tprot, 282 ret = mmu_hash_ops.hpte_insert(hpteg, vpn, paddr, tprot,
283 HPTE_V_BOLTED, psize, psize, ssize); 283 HPTE_V_BOLTED, psize, psize,
284 ssize);
284 285
285 if (ret < 0) 286 if (ret < 0)
286 break; 287 break;
@@ -305,11 +306,11 @@ int htab_remove_mapping(unsigned long vstart, unsigned long vend,
305 shift = mmu_psize_defs[psize].shift; 306 shift = mmu_psize_defs[psize].shift;
306 step = 1 << shift; 307 step = 1 << shift;
307 308
308 if (!ppc_md.hpte_removebolted) 309 if (!mmu_hash_ops.hpte_removebolted)
309 return -ENODEV; 310 return -ENODEV;
310 311
311 for (vaddr = vstart; vaddr < vend; vaddr += step) { 312 for (vaddr = vstart; vaddr < vend; vaddr += step) {
312 rc = ppc_md.hpte_removebolted(vaddr, psize, ssize); 313 rc = mmu_hash_ops.hpte_removebolted(vaddr, psize, ssize);
313 if (rc == -ENOENT) { 314 if (rc == -ENOENT) {
314 ret = -ENOENT; 315 ret = -ENOENT;
315 continue; 316 continue;
@@ -321,6 +322,15 @@ int htab_remove_mapping(unsigned long vstart, unsigned long vend,
321 return ret; 322 return ret;
322} 323}
323 324
325static bool disable_1tb_segments = false;
326
327static int __init parse_disable_1tb_segments(char *p)
328{
329 disable_1tb_segments = true;
330 return 0;
331}
332early_param("disable_1tb_segments", parse_disable_1tb_segments);
333
324static int __init htab_dt_scan_seg_sizes(unsigned long node, 334static int __init htab_dt_scan_seg_sizes(unsigned long node,
325 const char *uname, int depth, 335 const char *uname, int depth,
326 void *data) 336 void *data)
@@ -339,6 +349,12 @@ static int __init htab_dt_scan_seg_sizes(unsigned long node,
339 for (; size >= 4; size -= 4, ++prop) { 349 for (; size >= 4; size -= 4, ++prop) {
340 if (be32_to_cpu(prop[0]) == 40) { 350 if (be32_to_cpu(prop[0]) == 40) {
341 DBG("1T segment support detected\n"); 351 DBG("1T segment support detected\n");
352
353 if (disable_1tb_segments) {
354 DBG("1T segments disabled by command line\n");
355 break;
356 }
357
342 cur_cpu_spec->mmu_features |= MMU_FTR_1T_SEGMENT; 358 cur_cpu_spec->mmu_features |= MMU_FTR_1T_SEGMENT;
343 return 1; 359 return 1;
344 } 360 }
@@ -514,7 +530,8 @@ static bool might_have_hea(void)
514 * we will never see an HEA ethernet device. 530 * we will never see an HEA ethernet device.
515 */ 531 */
516#ifdef CONFIG_IBMEBUS 532#ifdef CONFIG_IBMEBUS
517 return !cpu_has_feature(CPU_FTR_ARCH_207S); 533 return !cpu_has_feature(CPU_FTR_ARCH_207S) &&
534 !firmware_has_feature(FW_FEATURE_SPLPAR);
518#else 535#else
519 return false; 536 return false;
520#endif 537#endif
@@ -580,7 +597,7 @@ found:
580 * would stop us accessing the HEA ethernet. So if we 597 * would stop us accessing the HEA ethernet. So if we
581 * have the chance of ever seeing one, stay at 4k. 598 * have the chance of ever seeing one, stay at 4k.
582 */ 599 */
583 if (!might_have_hea() || !machine_is(pseries)) 600 if (!might_have_hea())
584 mmu_io_psize = MMU_PAGE_64K; 601 mmu_io_psize = MMU_PAGE_64K;
585 } else 602 } else
586 mmu_ci_restrictions = 1; 603 mmu_ci_restrictions = 1;
@@ -699,10 +716,9 @@ int remove_section_mapping(unsigned long start, unsigned long end)
699#endif /* CONFIG_MEMORY_HOTPLUG */ 716#endif /* CONFIG_MEMORY_HOTPLUG */
700 717
701static void __init hash_init_partition_table(phys_addr_t hash_table, 718static void __init hash_init_partition_table(phys_addr_t hash_table,
702 unsigned long pteg_count) 719 unsigned long htab_size)
703{ 720{
704 unsigned long ps_field; 721 unsigned long ps_field;
705 unsigned long htab_size;
706 unsigned long patb_size = 1UL << PATB_SIZE_SHIFT; 722 unsigned long patb_size = 1UL << PATB_SIZE_SHIFT;
707 723
708 /* 724 /*
@@ -710,7 +726,7 @@ static void __init hash_init_partition_table(phys_addr_t hash_table,
710 * We can ignore that for lpid 0 726 * We can ignore that for lpid 0
711 */ 727 */
712 ps_field = 0; 728 ps_field = 0;
713 htab_size = __ilog2(pteg_count) - 11; 729 htab_size = __ilog2(htab_size) - 18;
714 730
715 BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 24), "Partition table size too large."); 731 BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 24), "Partition table size too large.");
716 partition_tb = __va(memblock_alloc_base(patb_size, patb_size, 732 partition_tb = __va(memblock_alloc_base(patb_size, patb_size,
@@ -724,7 +740,7 @@ static void __init hash_init_partition_table(phys_addr_t hash_table,
724 * For now UPRT is 0 for us. 740 * For now UPRT is 0 for us.
725 */ 741 */
726 partition_tb->patb1 = 0; 742 partition_tb->patb1 = 0;
727 DBG("Partition table %p\n", partition_tb); 743 pr_info("Partition table %p\n", partition_tb);
728 /* 744 /*
729 * update partition table control register, 745 * update partition table control register,
730 * 64 K size. 746 * 64 K size.
@@ -738,7 +754,7 @@ static void __init htab_initialize(void)
738 unsigned long table; 754 unsigned long table;
739 unsigned long pteg_count; 755 unsigned long pteg_count;
740 unsigned long prot; 756 unsigned long prot;
741 unsigned long base = 0, size = 0, limit; 757 unsigned long base = 0, size = 0;
742 struct memblock_region *reg; 758 struct memblock_region *reg;
743 759
744 DBG(" -> htab_initialize()\n"); 760 DBG(" -> htab_initialize()\n");
@@ -764,7 +780,8 @@ static void __init htab_initialize(void)
764 780
765 htab_hash_mask = pteg_count - 1; 781 htab_hash_mask = pteg_count - 1;
766 782
767 if (firmware_has_feature(FW_FEATURE_LPAR)) { 783 if (firmware_has_feature(FW_FEATURE_LPAR) ||
784 firmware_has_feature(FW_FEATURE_PS3_LV1)) {
768 /* Using a hypervisor which owns the htab */ 785 /* Using a hypervisor which owns the htab */
769 htab_address = NULL; 786 htab_address = NULL;
770 _SDR1 = 0; 787 _SDR1 = 0;
@@ -775,20 +792,26 @@ static void __init htab_initialize(void)
775 * Clear the htab if firmware assisted dump is active so 792 * Clear the htab if firmware assisted dump is active so
776 * that we dont end up using old mappings. 793 * that we dont end up using old mappings.
777 */ 794 */
778 if (is_fadump_active() && ppc_md.hpte_clear_all) 795 if (is_fadump_active() && mmu_hash_ops.hpte_clear_all)
779 ppc_md.hpte_clear_all(); 796 mmu_hash_ops.hpte_clear_all();
780#endif 797#endif
781 } else { 798 } else {
782 /* Find storage for the HPT. Must be contiguous in 799 unsigned long limit = MEMBLOCK_ALLOC_ANYWHERE;
783 * the absolute address space. On cell we want it to be 800
784 * in the first 2 Gig so we can use it for IOMMU hacks. 801#ifdef CONFIG_PPC_CELL
802 /*
803 * Cell may require the hash table down low when using the
804 * Axon IOMMU in order to fit the dynamic region over it, see
805 * comments in cell/iommu.c
785 */ 806 */
786 if (machine_is(cell)) 807 if (fdt_subnode_offset(initial_boot_params, 0, "axon") > 0) {
787 limit = 0x80000000; 808 limit = 0x80000000;
788 else 809 pr_info("Hash table forced below 2G for Axon IOMMU\n");
789 limit = MEMBLOCK_ALLOC_ANYWHERE; 810 }
811#endif /* CONFIG_PPC_CELL */
790 812
791 table = memblock_alloc_base(htab_size_bytes, htab_size_bytes, limit); 813 table = memblock_alloc_base(htab_size_bytes, htab_size_bytes,
814 limit);
792 815
793 DBG("Hash table allocated at %lx, size: %lx\n", table, 816 DBG("Hash table allocated at %lx, size: %lx\n", table,
794 htab_size_bytes); 817 htab_size_bytes);
@@ -796,7 +819,7 @@ static void __init htab_initialize(void)
796 htab_address = __va(table); 819 htab_address = __va(table);
797 820
798 /* htab absolute addr + encoded htabsize */ 821 /* htab absolute addr + encoded htabsize */
799 _SDR1 = table + __ilog2(pteg_count) - 11; 822 _SDR1 = table + __ilog2(htab_size_bytes) - 18;
800 823
801 /* Initialize the HPT with no entries */ 824 /* Initialize the HPT with no entries */
802 memset((void *)table, 0, htab_size_bytes); 825 memset((void *)table, 0, htab_size_bytes);
@@ -805,7 +828,7 @@ static void __init htab_initialize(void)
805 /* Set SDR1 */ 828 /* Set SDR1 */
806 mtspr(SPRN_SDR1, _SDR1); 829 mtspr(SPRN_SDR1, _SDR1);
807 else 830 else
808 hash_init_partition_table(table, pteg_count); 831 hash_init_partition_table(table, htab_size_bytes);
809 } 832 }
810 833
811 prot = pgprot_val(PAGE_KERNEL); 834 prot = pgprot_val(PAGE_KERNEL);
@@ -832,34 +855,6 @@ static void __init htab_initialize(void)
832 DBG("creating mapping for region: %lx..%lx (prot: %lx)\n", 855 DBG("creating mapping for region: %lx..%lx (prot: %lx)\n",
833 base, size, prot); 856 base, size, prot);
834 857
835#ifdef CONFIG_U3_DART
836 /* Do not map the DART space. Fortunately, it will be aligned
837 * in such a way that it will not cross two memblock regions and
838 * will fit within a single 16Mb page.
839 * The DART space is assumed to be a full 16Mb region even if
840 * we only use 2Mb of that space. We will use more of it later
841 * for AGP GART. We have to use a full 16Mb large page.
842 */
843 DBG("DART base: %lx\n", dart_tablebase);
844
845 if (dart_tablebase != 0 && dart_tablebase >= base
846 && dart_tablebase < (base + size)) {
847 unsigned long dart_table_end = dart_tablebase + 16 * MB;
848 if (base != dart_tablebase)
849 BUG_ON(htab_bolt_mapping(base, dart_tablebase,
850 __pa(base), prot,
851 mmu_linear_psize,
852 mmu_kernel_ssize));
853 if ((base + size) > dart_table_end)
854 BUG_ON(htab_bolt_mapping(dart_tablebase+16*MB,
855 base + size,
856 __pa(dart_table_end),
857 prot,
858 mmu_linear_psize,
859 mmu_kernel_ssize));
860 continue;
861 }
862#endif /* CONFIG_U3_DART */
863 BUG_ON(htab_bolt_mapping(base, base + size, __pa(base), 858 BUG_ON(htab_bolt_mapping(base, base + size, __pa(base),
864 prot, mmu_linear_psize, mmu_kernel_ssize)); 859 prot, mmu_linear_psize, mmu_kernel_ssize));
865 } 860 }
@@ -926,12 +921,24 @@ void __init hash__early_init_mmu(void)
926 pci_io_base = ISA_IO_BASE; 921 pci_io_base = ISA_IO_BASE;
927#endif 922#endif
928 923
924 /* Select appropriate backend */
925 if (firmware_has_feature(FW_FEATURE_PS3_LV1))
926 ps3_early_mm_init();
927 else if (firmware_has_feature(FW_FEATURE_LPAR))
928 hpte_init_pseries();
929 else if (IS_ENABLED(CONFIG_PPC_NATIVE))
930 hpte_init_native();
931
932 if (!mmu_hash_ops.hpte_insert)
933 panic("hash__early_init_mmu: No MMU hash ops defined!\n");
934
929 /* Initialize the MMU Hash table and create the linear mapping 935 /* Initialize the MMU Hash table and create the linear mapping
930 * of memory. Has to be done before SLB initialization as this is 936 * of memory. Has to be done before SLB initialization as this is
931 * currently where the page size encoding is obtained. 937 * currently where the page size encoding is obtained.
932 */ 938 */
933 htab_initialize(); 939 htab_initialize();
934 940
941 pr_info("Initializing hash mmu with SLB\n");
935 /* Initialize SLB management */ 942 /* Initialize SLB management */
936 slb_initialize(); 943 slb_initialize();
937} 944}
@@ -1474,7 +1481,8 @@ void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize,
1474 * We use same base page size and actual psize, because we don't 1481 * We use same base page size and actual psize, because we don't
1475 * use these functions for hugepage 1482 * use these functions for hugepage
1476 */ 1483 */
1477 ppc_md.hpte_invalidate(slot, vpn, psize, psize, ssize, local); 1484 mmu_hash_ops.hpte_invalidate(slot, vpn, psize, psize,
1485 ssize, local);
1478 } pte_iterate_hashed_end(); 1486 } pte_iterate_hashed_end();
1479 1487
1480#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1488#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -1515,9 +1523,9 @@ void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
1515 if (!hpte_slot_array) 1523 if (!hpte_slot_array)
1516 return; 1524 return;
1517 1525
1518 if (ppc_md.hugepage_invalidate) { 1526 if (mmu_hash_ops.hugepage_invalidate) {
1519 ppc_md.hugepage_invalidate(vsid, s_addr, hpte_slot_array, 1527 mmu_hash_ops.hugepage_invalidate(vsid, s_addr, hpte_slot_array,
1520 psize, ssize, local); 1528 psize, ssize, local);
1521 goto tm_abort; 1529 goto tm_abort;
1522 } 1530 }
1523 /* 1531 /*
@@ -1544,8 +1552,8 @@ void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
1544 1552
1545 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 1553 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
1546 slot += hidx & _PTEIDX_GROUP_IX; 1554 slot += hidx & _PTEIDX_GROUP_IX;
1547 ppc_md.hpte_invalidate(slot, vpn, psize, 1555 mmu_hash_ops.hpte_invalidate(slot, vpn, psize,
1548 MMU_PAGE_16M, ssize, local); 1556 MMU_PAGE_16M, ssize, local);
1549 } 1557 }
1550tm_abort: 1558tm_abort:
1551#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1559#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -1569,8 +1577,8 @@ tm_abort:
1569 1577
1570void flush_hash_range(unsigned long number, int local) 1578void flush_hash_range(unsigned long number, int local)
1571{ 1579{
1572 if (ppc_md.flush_hash_range) 1580 if (mmu_hash_ops.flush_hash_range)
1573 ppc_md.flush_hash_range(number, local); 1581 mmu_hash_ops.flush_hash_range(number, local);
1574 else { 1582 else {
1575 int i; 1583 int i;
1576 struct ppc64_tlb_batch *batch = 1584 struct ppc64_tlb_batch *batch =
@@ -1615,22 +1623,22 @@ repeat:
1615 HPTES_PER_GROUP) & ~0x7UL; 1623 HPTES_PER_GROUP) & ~0x7UL;
1616 1624
1617 /* Insert into the hash table, primary slot */ 1625 /* Insert into the hash table, primary slot */
1618 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, vflags, 1626 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, vflags,
1619 psize, psize, ssize); 1627 psize, psize, ssize);
1620 1628
1621 /* Primary is full, try the secondary */ 1629 /* Primary is full, try the secondary */
1622 if (unlikely(slot == -1)) { 1630 if (unlikely(slot == -1)) {
1623 hpte_group = ((~hash & htab_hash_mask) * 1631 hpte_group = ((~hash & htab_hash_mask) *
1624 HPTES_PER_GROUP) & ~0x7UL; 1632 HPTES_PER_GROUP) & ~0x7UL;
1625 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 1633 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags,
1626 vflags | HPTE_V_SECONDARY, 1634 vflags | HPTE_V_SECONDARY,
1627 psize, psize, ssize); 1635 psize, psize, ssize);
1628 if (slot == -1) { 1636 if (slot == -1) {
1629 if (mftb() & 0x1) 1637 if (mftb() & 0x1)
1630 hpte_group = ((hash & htab_hash_mask) * 1638 hpte_group = ((hash & htab_hash_mask) *
1631 HPTES_PER_GROUP)&~0x7UL; 1639 HPTES_PER_GROUP)&~0x7UL;
1632 1640
1633 ppc_md.hpte_remove(hpte_group); 1641 mmu_hash_ops.hpte_remove(hpte_group);
1634 goto repeat; 1642 goto repeat;
1635 } 1643 }
1636 } 1644 }
@@ -1680,8 +1688,9 @@ static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi)
1680 hash = ~hash; 1688 hash = ~hash;
1681 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 1689 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
1682 slot += hidx & _PTEIDX_GROUP_IX; 1690 slot += hidx & _PTEIDX_GROUP_IX;
1683 ppc_md.hpte_invalidate(slot, vpn, mmu_linear_psize, mmu_linear_psize, 1691 mmu_hash_ops.hpte_invalidate(slot, vpn, mmu_linear_psize,
1684 mmu_kernel_ssize, 0); 1692 mmu_linear_psize,
1693 mmu_kernel_ssize, 0);
1685} 1694}
1686 1695
1687void __kernel_map_pages(struct page *page, int numpages, int enable) 1696void __kernel_map_pages(struct page *page, int numpages, int enable)
diff --git a/arch/powerpc/mm/hugepage-hash64.c b/arch/powerpc/mm/hugepage-hash64.c
index ba3fc229468a..f20d16f849c5 100644
--- a/arch/powerpc/mm/hugepage-hash64.c
+++ b/arch/powerpc/mm/hugepage-hash64.c
@@ -103,8 +103,8 @@ int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid,
103 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 103 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
104 slot += hidx & _PTEIDX_GROUP_IX; 104 slot += hidx & _PTEIDX_GROUP_IX;
105 105
106 ret = ppc_md.hpte_updatepp(slot, rflags, vpn, 106 ret = mmu_hash_ops.hpte_updatepp(slot, rflags, vpn,
107 psize, lpsize, ssize, flags); 107 psize, lpsize, ssize, flags);
108 /* 108 /*
109 * We failed to update, try to insert a new entry. 109 * We failed to update, try to insert a new entry.
110 */ 110 */
@@ -131,23 +131,24 @@ repeat:
131 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; 131 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
132 132
133 /* Insert into the hash table, primary slot */ 133 /* Insert into the hash table, primary slot */
134 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0, 134 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
135 psize, lpsize, ssize); 135 psize, lpsize, ssize);
136 /* 136 /*
137 * Primary is full, try the secondary 137 * Primary is full, try the secondary
138 */ 138 */
139 if (unlikely(slot == -1)) { 139 if (unlikely(slot == -1)) {
140 hpte_group = ((~hash & htab_hash_mask) * 140 hpte_group = ((~hash & htab_hash_mask) *
141 HPTES_PER_GROUP) & ~0x7UL; 141 HPTES_PER_GROUP) & ~0x7UL;
142 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, 142 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
143 rflags, HPTE_V_SECONDARY, 143 rflags,
144 psize, lpsize, ssize); 144 HPTE_V_SECONDARY,
145 psize, lpsize, ssize);
145 if (slot == -1) { 146 if (slot == -1) {
146 if (mftb() & 0x1) 147 if (mftb() & 0x1)
147 hpte_group = ((hash & htab_hash_mask) * 148 hpte_group = ((hash & htab_hash_mask) *
148 HPTES_PER_GROUP) & ~0x7UL; 149 HPTES_PER_GROUP) & ~0x7UL;
149 150
150 ppc_md.hpte_remove(hpte_group); 151 mmu_hash_ops.hpte_remove(hpte_group);
151 goto repeat; 152 goto repeat;
152 } 153 }
153 } 154 }
diff --git a/arch/powerpc/mm/hugetlbpage-hash64.c b/arch/powerpc/mm/hugetlbpage-hash64.c
index 3058560b6121..d5026f3800b6 100644
--- a/arch/powerpc/mm/hugetlbpage-hash64.c
+++ b/arch/powerpc/mm/hugetlbpage-hash64.c
@@ -79,8 +79,8 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
79 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 79 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
80 slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT; 80 slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;
81 81
82 if (ppc_md.hpte_updatepp(slot, rflags, vpn, mmu_psize, 82 if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, mmu_psize,
83 mmu_psize, ssize, flags) == -1) 83 mmu_psize, ssize, flags) == -1)
84 old_pte &= ~_PAGE_HPTEFLAGS; 84 old_pte &= ~_PAGE_HPTEFLAGS;
85 } 85 }
86 86
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index 119d18611500..7372ee13eb1e 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -81,6 +81,13 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
81 if (! new) 81 if (! new)
82 return -ENOMEM; 82 return -ENOMEM;
83 83
84 /*
85 * Make sure other cpus find the hugepd set only after a
86 * properly initialized page table is visible to them.
87 * For more details look for comment in __pte_alloc().
88 */
89 smp_wmb();
90
84 spin_lock(&mm->page_table_lock); 91 spin_lock(&mm->page_table_lock);
85#ifdef CONFIG_PPC_FSL_BOOK3E 92#ifdef CONFIG_PPC_FSL_BOOK3E
86 /* 93 /*
diff --git a/arch/powerpc/mm/init_32.c b/arch/powerpc/mm/init_32.c
index c899fe340bbd..448685fbf27c 100644
--- a/arch/powerpc/mm/init_32.c
+++ b/arch/powerpc/mm/init_32.c
@@ -64,7 +64,7 @@ EXPORT_SYMBOL(memstart_addr);
64phys_addr_t kernstart_addr; 64phys_addr_t kernstart_addr;
65EXPORT_SYMBOL(kernstart_addr); 65EXPORT_SYMBOL(kernstart_addr);
66 66
67#ifdef CONFIG_RELOCATABLE_PPC32 67#ifdef CONFIG_RELOCATABLE
68/* Used in __va()/__pa() */ 68/* Used in __va()/__pa() */
69long long virt_phys_offset; 69long long virt_phys_offset;
70EXPORT_SYMBOL(virt_phys_offset); 70EXPORT_SYMBOL(virt_phys_offset);
@@ -80,9 +80,6 @@ EXPORT_SYMBOL(agp_special_page);
80 80
81void MMU_init(void); 81void MMU_init(void);
82 82
83/* XXX should be in current.h -- paulus */
84extern struct task_struct *current_set[NR_CPUS];
85
86/* 83/*
87 * this tells the system to map all of ram with the segregs 84 * this tells the system to map all of ram with the segregs
88 * (i.e. page tables) instead of the bats. 85 * (i.e. page tables) instead of the bats.
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 2fd57fa48429..5f844337de21 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -116,6 +116,16 @@ int memory_add_physaddr_to_nid(u64 start)
116} 116}
117#endif 117#endif
118 118
119int __weak create_section_mapping(unsigned long start, unsigned long end)
120{
121 return -ENODEV;
122}
123
124int __weak remove_section_mapping(unsigned long start, unsigned long end)
125{
126 return -ENODEV;
127}
128
119int arch_add_memory(int nid, u64 start, u64 size, bool for_device) 129int arch_add_memory(int nid, u64 start, u64 size, bool for_device)
120{ 130{
121 struct pglist_data *pgdata; 131 struct pglist_data *pgdata;
@@ -239,8 +249,14 @@ static int __init mark_nonram_nosave(void)
239 249
240static bool zone_limits_final; 250static bool zone_limits_final;
241 251
252/*
253 * The memory zones past TOP_ZONE are managed by generic mm code.
254 * These should be set to zero since that's what every other
255 * architecture does.
256 */
242static unsigned long max_zone_pfns[MAX_NR_ZONES] = { 257static unsigned long max_zone_pfns[MAX_NR_ZONES] = {
243 [0 ... MAX_NR_ZONES - 1] = ~0UL 258 [0 ... TOP_ZONE ] = ~0UL,
259 [TOP_ZONE + 1 ... MAX_NR_ZONES - 1] = 0
244}; 260};
245 261
246/* 262/*
diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c
index 196222227e82..b114f8b93ec9 100644
--- a/arch/powerpc/mm/mmu_context_book3s64.c
+++ b/arch/powerpc/mm/mmu_context_book3s64.c
@@ -181,7 +181,10 @@ void destroy_context(struct mm_struct *mm)
181#ifdef CONFIG_PPC_RADIX_MMU 181#ifdef CONFIG_PPC_RADIX_MMU
182void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next) 182void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
183{ 183{
184 mtspr(SPRN_PID, next->context.id);
185 asm volatile("isync": : :"memory"); 184 asm volatile("isync": : :"memory");
185 mtspr(SPRN_PID, next->context.id);
186 asm volatile("isync \n"
187 PPC_SLBIA(0x7)
188 : : :"memory");
186} 189}
187#endif 190#endif
diff --git a/arch/powerpc/mm/mmu_decl.h b/arch/powerpc/mm/mmu_decl.h
index 6af65327c993..f988db655e5b 100644
--- a/arch/powerpc/mm/mmu_decl.h
+++ b/arch/powerpc/mm/mmu_decl.h
@@ -154,9 +154,10 @@ struct tlbcam {
154}; 154};
155#endif 155#endif
156 156
157#if defined(CONFIG_6xx) || defined(CONFIG_FSL_BOOKE) 157#if defined(CONFIG_6xx) || defined(CONFIG_FSL_BOOKE) || defined(CONFIG_PPC_8xx)
158/* 6xx have BATS */ 158/* 6xx have BATS */
159/* FSL_BOOKE have TLBCAM */ 159/* FSL_BOOKE have TLBCAM */
160/* 8xx have LTLB */
160phys_addr_t v_block_mapped(unsigned long va); 161phys_addr_t v_block_mapped(unsigned long va);
161unsigned long p_block_mapped(phys_addr_t pa); 162unsigned long p_block_mapped(phys_addr_t pa);
162#else 163#else
diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
index 6dc07ddbfd04..75b9cd6150cc 100644
--- a/arch/powerpc/mm/numa.c
+++ b/arch/powerpc/mm/numa.c
@@ -1153,18 +1153,34 @@ int hot_add_scn_to_nid(unsigned long scn_addr)
1153 1153
1154static u64 hot_add_drconf_memory_max(void) 1154static u64 hot_add_drconf_memory_max(void)
1155{ 1155{
1156 struct device_node *memory = NULL; 1156 struct device_node *memory = NULL;
1157 unsigned int drconf_cell_cnt = 0; 1157 struct device_node *dn = NULL;
1158 u64 lmb_size = 0; 1158 unsigned int drconf_cell_cnt = 0;
1159 u64 lmb_size = 0;
1159 const __be32 *dm = NULL; 1160 const __be32 *dm = NULL;
1161 const __be64 *lrdr = NULL;
1162 struct of_drconf_cell drmem;
1163
1164 dn = of_find_node_by_path("/rtas");
1165 if (dn) {
1166 lrdr = of_get_property(dn, "ibm,lrdr-capacity", NULL);
1167 of_node_put(dn);
1168 if (lrdr)
1169 return be64_to_cpup(lrdr);
1170 }
1171
1172 memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
1173 if (memory) {
1174 drconf_cell_cnt = of_get_drconf_memory(memory, &dm);
1175 lmb_size = of_get_lmb_size(memory);
1160 1176
1161 memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); 1177 /* Advance to the last cell, each cell has 6 32 bit integers */
1162 if (memory) { 1178 dm += (drconf_cell_cnt - 1) * 6;
1163 drconf_cell_cnt = of_get_drconf_memory(memory, &dm); 1179 read_drconf_cell(&drmem, &dm);
1164 lmb_size = of_get_lmb_size(memory); 1180 of_node_put(memory);
1165 of_node_put(memory); 1181 return drmem.base_addr + lmb_size;
1166 } 1182 }
1167 return lmb_size * drconf_cell_cnt; 1183 return 0;
1168} 1184}
1169 1185
1170/* 1186/*
diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c
index 7931e1496f0d..003ff48a11b6 100644
--- a/arch/powerpc/mm/pgtable-radix.c
+++ b/arch/powerpc/mm/pgtable-radix.c
@@ -21,8 +21,11 @@
21 21
22#include <trace/events/thp.h> 22#include <trace/events/thp.h>
23 23
24static int native_update_partition_table(u64 patb1) 24static int native_register_process_table(unsigned long base, unsigned long pg_sz,
25 unsigned long table_size)
25{ 26{
27 unsigned long patb1 = base | table_size | PATB_GR;
28
26 partition_tb->patb1 = cpu_to_be64(patb1); 29 partition_tb->patb1 = cpu_to_be64(patb1);
27 return 0; 30 return 0;
28} 31}
@@ -168,7 +171,7 @@ redo:
168 * of process table here. But our linear mapping also enable us to use 171 * of process table here. But our linear mapping also enable us to use
169 * physical address here. 172 * physical address here.
170 */ 173 */
171 ppc_md.update_partition_table(__pa(process_tb) | (PRTB_SIZE_SHIFT - 12) | PATB_GR); 174 ppc_md.register_process_table(__pa(process_tb), 0, PRTB_SIZE_SHIFT - 12);
172 pr_info("Process table %p and radix root for kernel: %p\n", process_tb, init_mm.pgd); 175 pr_info("Process table %p and radix root for kernel: %p\n", process_tb, init_mm.pgd);
173} 176}
174 177
@@ -182,7 +185,8 @@ static void __init radix_init_partition_table(void)
182 partition_tb = early_alloc_pgtable(1UL << PATB_SIZE_SHIFT); 185 partition_tb = early_alloc_pgtable(1UL << PATB_SIZE_SHIFT);
183 partition_tb->patb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | 186 partition_tb->patb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) |
184 RADIX_PGD_INDEX_SIZE | PATB_HR); 187 RADIX_PGD_INDEX_SIZE | PATB_HR);
185 printk("Partition table %p\n", partition_tb); 188 pr_info("Initializing Radix MMU\n");
189 pr_info("Partition table %p\n", partition_tb);
186 190
187 memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE); 191 memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
188 /* 192 /*
@@ -194,7 +198,7 @@ static void __init radix_init_partition_table(void)
194 198
195void __init radix_init_native(void) 199void __init radix_init_native(void)
196{ 200{
197 ppc_md.update_partition_table = native_update_partition_table; 201 ppc_md.register_process_table = native_register_process_table;
198} 202}
199 203
200static int __init get_idx_from_shift(unsigned int shift) 204static int __init get_idx_from_shift(unsigned int shift)
@@ -341,8 +345,9 @@ void __init radix__early_init_mmu(void)
341 345
342 radix_init_page_sizes(); 346 radix_init_page_sizes();
343 if (!firmware_has_feature(FW_FEATURE_LPAR)) { 347 if (!firmware_has_feature(FW_FEATURE_LPAR)) {
348 radix_init_native();
344 lpcr = mfspr(SPRN_LPCR); 349 lpcr = mfspr(SPRN_LPCR);
345 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT); 350 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
346 radix_init_partition_table(); 351 radix_init_partition_table();
347 } 352 }
348 353
@@ -357,7 +362,7 @@ void radix__early_init_mmu_secondary(void)
357 */ 362 */
358 if (!firmware_has_feature(FW_FEATURE_LPAR)) { 363 if (!firmware_has_feature(FW_FEATURE_LPAR)) {
359 lpcr = mfspr(SPRN_LPCR); 364 lpcr = mfspr(SPRN_LPCR);
360 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT); 365 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
361 366
362 mtspr(SPRN_PTCR, 367 mtspr(SPRN_PTCR,
363 __pa(partition_tb) | (PATB_SIZE_SHIFT - 12)); 368 __pa(partition_tb) | (PATB_SIZE_SHIFT - 12));
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index ab2f60e812e2..e1f22700fb16 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -12,6 +12,7 @@
12#include <linux/mm.h> 12#include <linux/mm.h>
13#include <linux/hugetlb.h> 13#include <linux/hugetlb.h>
14#include <linux/memblock.h> 14#include <linux/memblock.h>
15#include <asm/ppc-opcode.h>
15 16
16#include <asm/tlb.h> 17#include <asm/tlb.h>
17#include <asm/tlbflush.h> 18#include <asm/tlbflush.h>
@@ -34,8 +35,7 @@ static inline void __tlbiel_pid(unsigned long pid, int set,
34 r = 1; /* raidx format */ 35 r = 1; /* raidx format */
35 36
36 asm volatile("ptesync": : :"memory"); 37 asm volatile("ptesync": : :"memory");
37 asm volatile(".long 0x7c000224 | (%0 << 11) | (%1 << 16) |" 38 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
38 "(%2 << 17) | (%3 << 18) | (%4 << 21)"
39 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 39 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
40 asm volatile("ptesync": : :"memory"); 40 asm volatile("ptesync": : :"memory");
41} 41}
@@ -63,8 +63,7 @@ static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
63 r = 1; /* raidx format */ 63 r = 1; /* raidx format */
64 64
65 asm volatile("ptesync": : :"memory"); 65 asm volatile("ptesync": : :"memory");
66 asm volatile(".long 0x7c000264 | (%0 << 11) | (%1 << 16) |" 66 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
67 "(%2 << 17) | (%3 << 18) | (%4 << 21)"
68 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 67 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
69 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 68 asm volatile("eieio; tlbsync; ptesync": : :"memory");
70} 69}
@@ -81,8 +80,7 @@ static inline void _tlbiel_va(unsigned long va, unsigned long pid,
81 r = 1; /* raidx format */ 80 r = 1; /* raidx format */
82 81
83 asm volatile("ptesync": : :"memory"); 82 asm volatile("ptesync": : :"memory");
84 asm volatile(".long 0x7c000224 | (%0 << 11) | (%1 << 16) |" 83 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
85 "(%2 << 17) | (%3 << 18) | (%4 << 21)"
86 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 84 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
87 asm volatile("ptesync": : :"memory"); 85 asm volatile("ptesync": : :"memory");
88} 86}
@@ -99,8 +97,7 @@ static inline void _tlbie_va(unsigned long va, unsigned long pid,
99 r = 1; /* raidx format */ 97 r = 1; /* raidx format */
100 98
101 asm volatile("ptesync": : :"memory"); 99 asm volatile("ptesync": : :"memory");
102 asm volatile(".long 0x7c000264 | (%0 << 11) | (%1 << 16) |" 100 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
103 "(%2 << 17) | (%3 << 18) | (%4 << 21)"
104 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 101 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
105 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 102 asm volatile("eieio; tlbsync; ptesync": : :"memory");
106} 103}
@@ -285,9 +282,61 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
285} 282}
286EXPORT_SYMBOL(radix__flush_tlb_range); 283EXPORT_SYMBOL(radix__flush_tlb_range);
287 284
285static int radix_get_mmu_psize(int page_size)
286{
287 int psize;
288
289 if (page_size == (1UL << mmu_psize_defs[mmu_virtual_psize].shift))
290 psize = mmu_virtual_psize;
291 else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_2M].shift))
292 psize = MMU_PAGE_2M;
293 else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_1G].shift))
294 psize = MMU_PAGE_1G;
295 else
296 return -1;
297 return psize;
298}
288 299
289void radix__tlb_flush(struct mmu_gather *tlb) 300void radix__tlb_flush(struct mmu_gather *tlb)
290{ 301{
291 struct mm_struct *mm = tlb->mm; 302 struct mm_struct *mm = tlb->mm;
292 radix__flush_tlb_mm(mm); 303 radix__flush_tlb_mm(mm);
293} 304}
305
306void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
307 unsigned long page_size)
308{
309 unsigned long rb,rs,prs,r;
310 unsigned long ap;
311 unsigned long ric = RIC_FLUSH_TLB;
312
313 ap = mmu_get_ap(radix_get_mmu_psize(page_size));
314 rb = gpa & ~(PPC_BITMASK(52, 63));
315 rb |= ap << PPC_BITLSHIFT(58);
316 rs = lpid & ((1UL << 32) - 1);
317 prs = 0; /* process scoped */
318 r = 1; /* raidx format */
319
320 asm volatile("ptesync": : :"memory");
321 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
322 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
323 asm volatile("eieio; tlbsync; ptesync": : :"memory");
324}
325EXPORT_SYMBOL(radix__flush_tlb_lpid_va);
326
327void radix__flush_tlb_lpid(unsigned long lpid)
328{
329 unsigned long rb,rs,prs,r;
330 unsigned long ric = RIC_FLUSH_ALL;
331
332 rb = 0x2 << PPC_BITLSHIFT(53); /* IS = 2 */
333 rs = lpid & ((1UL << 32) - 1);
334 prs = 0; /* partition scoped */
335 r = 1; /* raidx format */
336
337 asm volatile("ptesync": : :"memory");
338 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
339 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
340 asm volatile("eieio; tlbsync; ptesync": : :"memory");
341}
342EXPORT_SYMBOL(radix__flush_tlb_lpid);
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-03 06:49:53 -0400