aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMichael Ellerman <mpe@ellerman.id.au>2018-07-19 00:37:57 -0400
committerMichael Ellerman <mpe@ellerman.id.au>2018-07-19 00:37:57 -0400
commitce57c6610cc2d7cde61fc005a2d2090bce46fc73 (patch)
treebe1230fb1773e3de7fb7f6a0adde68ca0468273e
parent9c3250a127901c93b23723dc5859c71a0d068fb8 (diff)
parenta68bd1267b7286b1687905651b404e765046de25 (diff)
Merge branch 'topic/ppc-kvm' into next
Merge in some commits we're sharing with the KVM tree. I manually propagated the change from commit d3d4ffaae439 ("powerpc/powernv/ioda2: Reduce upper limit for DMA window size") into pci-ioda-tce.c. Conflicts: arch/powerpc/include/asm/cputable.h arch/powerpc/platforms/powernv/pci-ioda.c arch/powerpc/platforms/powernv/pci.h
Diffstat
-rw-r--r--arch/powerpc/include/asm/book3s/64/hugetlb.h20
-rw-r--r--arch/powerpc/include/asm/book3s/64/pgtable.h5
-rw-r--r--arch/powerpc/include/asm/book3s/64/radix.h35
-rw-r--r--arch/powerpc/include/asm/book3s/64/tlbflush-radix.h2
-rw-r--r--arch/powerpc/include/asm/cputable.h19
-rw-r--r--arch/powerpc/include/asm/iommu.h11
-rw-r--r--arch/powerpc/include/asm/paca.h5
-rw-r--r--arch/powerpc/kernel/asm-offsets.c1
-rw-r--r--arch/powerpc/kernel/cputable.c19
-rw-r--r--arch/powerpc/kernel/dt_cpu_ftrs.c4
-rw-r--r--arch/powerpc/kernel/exceptions-64s.S4
-rw-r--r--arch/powerpc/kernel/idle_book3s.S50
-rw-r--r--arch/powerpc/kernel/process.c10
-rw-r--r--arch/powerpc/kvm/book3s_64_mmu_radix.c15
-rw-r--r--arch/powerpc/kvm/book3s_64_vio.c11
-rw-r--r--arch/powerpc/kvm/book3s_64_vio_hv.c18
-rw-r--r--arch/powerpc/kvm/book3s_hv.c10
-rw-r--r--arch/powerpc/kvm/book3s_hv_rmhandlers.S16
-rw-r--r--arch/powerpc/kvm/book3s_xive_template.c39
-rw-r--r--arch/powerpc/mm/hash_utils_64.c30
-rw-r--r--arch/powerpc/mm/hugetlbpage.c9
-rw-r--r--arch/powerpc/mm/mmu_context_book3s64.c12
-rw-r--r--arch/powerpc/mm/pgtable-radix.c60
-rw-r--r--arch/powerpc/mm/tlb-radix.c18
-rw-r--r--arch/powerpc/perf/core-book3s.c34
-rw-r--r--arch/powerpc/perf/isa207-common.c12
-rw-r--r--arch/powerpc/perf/isa207-common.h5
-rw-r--r--arch/powerpc/perf/power9-pmu.c54
-rw-r--r--arch/powerpc/platforms/powernv/Makefile2
-rw-r--r--arch/powerpc/platforms/powernv/idle.c28
-rw-r--r--arch/powerpc/platforms/powernv/pci-ioda-tce.c399
-rw-r--r--arch/powerpc/platforms/powernv/pci-ioda.c184
-rw-r--r--arch/powerpc/platforms/powernv/pci.c158
-rw-r--r--arch/powerpc/platforms/powernv/pci.h44
-rw-r--r--arch/powerpc/platforms/powernv/smp.c27
-rw-r--r--arch/powerpc/sysdev/xive/common.c8
-rw-r--r--arch/powerpc/xmon/xmon.c1
-rw-r--r--drivers/misc/cxl/cxl.h8
-rw-r--r--drivers/misc/cxl/cxllib.c4
-rw-r--r--drivers/misc/cxl/pci.c41
-rw-r--r--drivers/vfio/vfio_iommu_spapr_tce.c65
41 files changed, 546 insertions, 951 deletions
diff --git a/arch/powerpc/include/asm/book3s/64/hugetlb.h b/arch/powerpc/include/asm/book3s/64/hugetlb.h
index c459f937d484..50888388a359 100644
--- a/arch/powerpc/include/asm/book3s/64/hugetlb.h
+++ b/arch/powerpc/include/asm/book3s/64/hugetlb.h
@@ -32,26 +32,6 @@ static inline int hstate_get_psize(struct hstate *hstate)
32 } 32 }
33} 33}
34 34
35#define arch_make_huge_pte arch_make_huge_pte
36static inline pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
37 struct page *page, int writable)
38{
39 unsigned long page_shift;
40
41 if (!cpu_has_feature(CPU_FTR_POWER9_DD1))
42 return entry;
43
44 page_shift = huge_page_shift(hstate_vma(vma));
45 /*
46 * We don't support 1G hugetlb pages yet.
47 */
48 VM_WARN_ON(page_shift == mmu_psize_defs[MMU_PAGE_1G].shift);
49 if (page_shift == mmu_psize_defs[MMU_PAGE_2M].shift)
50 return __pte(pte_val(entry) | R_PAGE_LARGE);
51 else
52 return entry;
53}
54
55#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE 35#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
56static inline bool gigantic_page_supported(void) 36static inline bool gigantic_page_supported(void)
57{ 37{
diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
index 42aafba7a308..676118743a06 100644
--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -479,9 +479,8 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
479{ 479{
480 if (full && radix_enabled()) { 480 if (full && radix_enabled()) {
481 /* 481 /*
482 * Let's skip the DD1 style pte update here. We know that 482 * We know that this is a full mm pte clear and
483 * this is a full mm pte clear and hence can be sure there is 483 * hence can be sure there is no parallel set_pte.
484 * no parallel set_pte.
485 */ 484 */
486 return radix__ptep_get_and_clear_full(mm, addr, ptep, full); 485 return radix__ptep_get_and_clear_full(mm, addr, ptep, full);
487 } 486 }
diff --git a/arch/powerpc/include/asm/book3s/64/radix.h b/arch/powerpc/include/asm/book3s/64/radix.h
index ef9f96742ce1..3ab3f7aef022 100644
--- a/arch/powerpc/include/asm/book3s/64/radix.h
+++ b/arch/powerpc/include/asm/book3s/64/radix.h
@@ -12,12 +12,6 @@
12#include <asm/book3s/64/radix-4k.h> 12#include <asm/book3s/64/radix-4k.h>
13#endif 13#endif
14 14
15/*
16 * For P9 DD1 only, we need to track whether the pte's huge.
17 */
18#define R_PAGE_LARGE _RPAGE_RSV1
19
20
21#ifndef __ASSEMBLY__ 15#ifndef __ASSEMBLY__
22#include <asm/book3s/64/tlbflush-radix.h> 16#include <asm/book3s/64/tlbflush-radix.h>
23#include <asm/cpu_has_feature.h> 17#include <asm/cpu_has_feature.h>
@@ -154,20 +148,7 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm,
154{ 148{
155 unsigned long old_pte; 149 unsigned long old_pte;
156 150
157 if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { 151 old_pte = __radix_pte_update(ptep, clr, set);
158
159 unsigned long new_pte;
160
161 old_pte = __radix_pte_update(ptep, ~0ul, 0);
162 /*
163 * new value of pte
164 */
165 new_pte = (old_pte | set) & ~clr;
166 radix__flush_tlb_pte_p9_dd1(old_pte, mm, addr);
167 if (new_pte)
168 __radix_pte_update(ptep, 0, new_pte);
169 } else
170 old_pte = __radix_pte_update(ptep, clr, set);
171 if (!huge) 152 if (!huge)
172 assert_pte_locked(mm, addr); 153 assert_pte_locked(mm, addr);
173 154
@@ -253,8 +234,6 @@ static inline int radix__pmd_trans_huge(pmd_t pmd)
253 234
254static inline pmd_t radix__pmd_mkhuge(pmd_t pmd) 235static inline pmd_t radix__pmd_mkhuge(pmd_t pmd)
255{ 236{
256 if (cpu_has_feature(CPU_FTR_POWER9_DD1))
257 return __pmd(pmd_val(pmd) | _PAGE_PTE | R_PAGE_LARGE);
258 return __pmd(pmd_val(pmd) | _PAGE_PTE); 237 return __pmd(pmd_val(pmd) | _PAGE_PTE);
259} 238}
260 239
@@ -285,18 +264,14 @@ static inline unsigned long radix__get_tree_size(void)
285 unsigned long rts_field; 264 unsigned long rts_field;
286 /* 265 /*
287 * We support 52 bits, hence: 266 * We support 52 bits, hence:
288 * DD1 52-28 = 24, 0b11000 267 * bits 52 - 31 = 21, 0b10101
289 * Others 52-31 = 21, 0b10101
290 * RTS encoding details 268 * RTS encoding details
291 * bits 0 - 3 of rts -> bits 6 - 8 unsigned long 269 * bits 0 - 3 of rts -> bits 6 - 8 unsigned long
292 * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long 270 * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long
293 */ 271 */
294 if (cpu_has_feature(CPU_FTR_POWER9_DD1)) 272 rts_field = (0x5UL << 5); /* 6 - 8 bits */
295 rts_field = (0x3UL << 61); 273 rts_field |= (0x2UL << 61);
296 else { 274
297 rts_field = (0x5UL << 5); /* 6 - 8 bits */
298 rts_field |= (0x2UL << 61);
299 }
300 return rts_field; 275 return rts_field;
301} 276}
302 277
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index ef5c3f2994c9..1154a6dc6d26 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -48,8 +48,6 @@ extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmad
48extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr); 48extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
49extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr); 49extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr);
50extern void radix__flush_tlb_all(void); 50extern void radix__flush_tlb_all(void);
51extern void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm,
52 unsigned long address);
53 51
54extern void radix__flush_tlb_lpid_page(unsigned int lpid, 52extern void radix__flush_tlb_lpid_page(unsigned int lpid,
55 unsigned long addr, 53 unsigned long addr,
diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h
index 7711ebfb7d5f..82816a2043b9 100644
--- a/arch/powerpc/include/asm/cputable.h
+++ b/arch/powerpc/include/asm/cputable.h
@@ -210,7 +210,6 @@ static inline void cpu_feature_keys_init(void) { }
210#define CPU_FTR_DAWR LONG_ASM_CONST(0x0000008000000000) 210#define CPU_FTR_DAWR LONG_ASM_CONST(0x0000008000000000)
211#define CPU_FTR_DABRX LONG_ASM_CONST(0x0000010000000000) 211#define CPU_FTR_DABRX LONG_ASM_CONST(0x0000010000000000)
212#define CPU_FTR_PMAO_BUG LONG_ASM_CONST(0x0000020000000000) 212#define CPU_FTR_PMAO_BUG LONG_ASM_CONST(0x0000020000000000)
213#define CPU_FTR_POWER9_DD1 LONG_ASM_CONST(0x0000040000000000)
214#define CPU_FTR_POWER9_DD2_1 LONG_ASM_CONST(0x0000080000000000) 213#define CPU_FTR_POWER9_DD2_1 LONG_ASM_CONST(0x0000080000000000)
215#define CPU_FTR_P9_TM_HV_ASSIST LONG_ASM_CONST(0x0000100000000000) 214#define CPU_FTR_P9_TM_HV_ASSIST LONG_ASM_CONST(0x0000100000000000)
216#define CPU_FTR_P9_TM_XER_SO_BUG LONG_ASM_CONST(0x0000200000000000) 215#define CPU_FTR_P9_TM_XER_SO_BUG LONG_ASM_CONST(0x0000200000000000)
@@ -463,8 +462,6 @@ static inline void cpu_feature_keys_init(void) { }
463 CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_ARCH_207S | \ 462 CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_ARCH_207S | \
464 CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | CPU_FTR_PKEY | \ 463 CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | CPU_FTR_PKEY | \
465 CPU_FTR_P9_TLBIE_BUG | CPU_FTR_P9_TIDR) 464 CPU_FTR_P9_TLBIE_BUG | CPU_FTR_P9_TIDR)
466#define CPU_FTRS_POWER9_DD1 ((CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD1) & \
467 (~CPU_FTR_SAO))
468#define CPU_FTRS_POWER9_DD2_0 CPU_FTRS_POWER9 465#define CPU_FTRS_POWER9_DD2_0 CPU_FTRS_POWER9
469#define CPU_FTRS_POWER9_DD2_1 (CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD2_1) 466#define CPU_FTRS_POWER9_DD2_1 (CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD2_1)
470#define CPU_FTRS_POWER9_DD2_2 (CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD2_1 | \ 467#define CPU_FTRS_POWER9_DD2_2 (CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD2_1 | \
@@ -488,16 +485,14 @@ static inline void cpu_feature_keys_init(void) { }
488#define CPU_FTRS_POSSIBLE \ 485#define CPU_FTRS_POSSIBLE \
489 (CPU_FTRS_POWER7 | CPU_FTRS_POWER8E | CPU_FTRS_POWER8 | \ 486 (CPU_FTRS_POWER7 | CPU_FTRS_POWER8E | CPU_FTRS_POWER8 | \
490 CPU_FTR_ALTIVEC_COMP | CPU_FTR_VSX_COMP | CPU_FTRS_POWER9 | \ 487 CPU_FTR_ALTIVEC_COMP | CPU_FTR_VSX_COMP | CPU_FTRS_POWER9 | \
491 CPU_FTRS_POWER9_DD1 | CPU_FTRS_POWER9_DD2_1 | \ 488 CPU_FTRS_POWER9_DD2_1 | CPU_FTRS_POWER9_DD2_2)
492 CPU_FTRS_POWER9_DD2_2)
493#else 489#else
494#define CPU_FTRS_POSSIBLE \ 490#define CPU_FTRS_POSSIBLE \
495 (CPU_FTRS_PPC970 | CPU_FTRS_POWER5 | \ 491 (CPU_FTRS_PPC970 | CPU_FTRS_POWER5 | \
496 CPU_FTRS_POWER6 | CPU_FTRS_POWER7 | CPU_FTRS_POWER8E | \ 492 CPU_FTRS_POWER6 | CPU_FTRS_POWER7 | CPU_FTRS_POWER8E | \
497 CPU_FTRS_POWER8 | CPU_FTRS_CELL | CPU_FTRS_PA6T | \ 493 CPU_FTRS_POWER8 | CPU_FTRS_CELL | CPU_FTRS_PA6T | \
498 CPU_FTR_VSX_COMP | CPU_FTR_ALTIVEC_COMP | CPU_FTRS_POWER9 | \ 494 CPU_FTR_VSX_COMP | CPU_FTR_ALTIVEC_COMP | CPU_FTRS_POWER9 | \
499 CPU_FTRS_POWER9_DD1 | CPU_FTRS_POWER9_DD2_1 | \ 495 CPU_FTRS_POWER9_DD2_1 | CPU_FTRS_POWER9_DD2_2)
500 CPU_FTRS_POWER9_DD2_2)
501#endif /* CONFIG_CPU_LITTLE_ENDIAN */ 496#endif /* CONFIG_CPU_LITTLE_ENDIAN */
502#endif 497#endif
503#else 498#else
@@ -565,17 +560,15 @@ enum {
565#ifdef CONFIG_CPU_LITTLE_ENDIAN 560#ifdef CONFIG_CPU_LITTLE_ENDIAN
566#define CPU_FTRS_ALWAYS \ 561#define CPU_FTRS_ALWAYS \
567 (CPU_FTRS_POSSIBLE & ~CPU_FTR_HVMODE & CPU_FTRS_POWER7 & \ 562 (CPU_FTRS_POSSIBLE & ~CPU_FTR_HVMODE & CPU_FTRS_POWER7 & \
568 CPU_FTRS_POWER8E & CPU_FTRS_POWER8 & \ 563 CPU_FTRS_POWER8E & CPU_FTRS_POWER8 & CPU_FTRS_POWER9 & \
569 CPU_FTRS_POWER9 & CPU_FTRS_POWER9_DD1 & CPU_FTRS_POWER9_DD2_1 & \ 564 CPU_FTRS_POWER9_DD2_1 & CPU_FTRS_DT_CPU_BASE)
570 CPU_FTRS_DT_CPU_BASE)
571#else 565#else
572#define CPU_FTRS_ALWAYS \ 566#define CPU_FTRS_ALWAYS \
573 (CPU_FTRS_PPC970 & CPU_FTRS_POWER5 & \ 567 (CPU_FTRS_PPC970 & CPU_FTRS_POWER5 & \
574 CPU_FTRS_POWER6 & CPU_FTRS_POWER7 & CPU_FTRS_CELL & \ 568 CPU_FTRS_POWER6 & CPU_FTRS_POWER7 & CPU_FTRS_CELL & \
575 CPU_FTRS_PA6T & CPU_FTRS_POWER8 & CPU_FTRS_POWER8E & \ 569 CPU_FTRS_PA6T & CPU_FTRS_POWER8 & CPU_FTRS_POWER8E & \
576 ~CPU_FTR_HVMODE & CPU_FTRS_POSSIBLE & \ 570 ~CPU_FTR_HVMODE & CPU_FTRS_POSSIBLE & CPU_FTRS_POWER9 & \
577 CPU_FTRS_POWER9 & CPU_FTRS_POWER9_DD1 & CPU_FTRS_POWER9_DD2_1 & \ 571 CPU_FTRS_POWER9_DD2_1 & CPU_FTRS_DT_CPU_BASE)
578 CPU_FTRS_DT_CPU_BASE)
579#endif /* CONFIG_CPU_LITTLE_ENDIAN */ 572#endif /* CONFIG_CPU_LITTLE_ENDIAN */
580#endif 573#endif
581#else 574#else
diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h
index 20febe0b7f32..daa3ee5d7ad2 100644
--- a/arch/powerpc/include/asm/iommu.h
+++ b/arch/powerpc/include/asm/iommu.h
@@ -69,6 +69,8 @@ struct iommu_table_ops {
69 long index, 69 long index,
70 unsigned long *hpa, 70 unsigned long *hpa,
71 enum dma_data_direction *direction); 71 enum dma_data_direction *direction);
72
73 __be64 *(*useraddrptr)(struct iommu_table *tbl, long index, bool alloc);
72#endif 74#endif
73 void (*clear)(struct iommu_table *tbl, 75 void (*clear)(struct iommu_table *tbl,
74 long index, long npages); 76 long index, long npages);
@@ -117,15 +119,16 @@ struct iommu_table {
117 unsigned long *it_map; /* A simple allocation bitmap for now */ 119 unsigned long *it_map; /* A simple allocation bitmap for now */
118 unsigned long it_page_shift;/* table iommu page size */ 120 unsigned long it_page_shift;/* table iommu page size */
119 struct list_head it_group_list;/* List of iommu_table_group_link */ 121 struct list_head it_group_list;/* List of iommu_table_group_link */
120 unsigned long *it_userspace; /* userspace view of the table */ 122 __be64 *it_userspace; /* userspace view of the table */
121 struct iommu_table_ops *it_ops; 123 struct iommu_table_ops *it_ops;
122 struct kref it_kref; 124 struct kref it_kref;
125 int it_nid;
123}; 126};
124 127
128#define IOMMU_TABLE_USERSPACE_ENTRY_RM(tbl, entry) \
129 ((tbl)->it_ops->useraddrptr((tbl), (entry), false))
125#define IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry) \ 130#define IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry) \
126 ((tbl)->it_userspace ? \ 131 ((tbl)->it_ops->useraddrptr((tbl), (entry), true))
127 &((tbl)->it_userspace[(entry) - (tbl)->it_offset]) : \
128 NULL)
129 132
130/* Pure 2^n version of get_order */ 133/* Pure 2^n version of get_order */
131static inline __attribute_const__ 134static inline __attribute_const__
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 6d34bd71139d..4e9cede5a7e7 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -187,11 +187,6 @@ struct paca_struct {
187 u8 subcore_sibling_mask; 187 u8 subcore_sibling_mask;
188 /* Flag to request this thread not to stop */ 188 /* Flag to request this thread not to stop */
189 atomic_t dont_stop; 189 atomic_t dont_stop;
190 /*
191 * Pointer to an array which contains pointer
192 * to the sibling threads' paca.
193 */
194 struct paca_struct **thread_sibling_pacas;
195 /* The PSSCR value that the kernel requested before going to stop */ 190 /* The PSSCR value that the kernel requested before going to stop */
196 u64 requested_psscr; 191 u64 requested_psscr;
197 192
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 0a0544335950..89cf15566c4e 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -766,7 +766,6 @@ int main(void)
766 OFFSET(PACA_THREAD_IDLE_STATE, paca_struct, thread_idle_state); 766 OFFSET(PACA_THREAD_IDLE_STATE, paca_struct, thread_idle_state);
767 OFFSET(PACA_THREAD_MASK, paca_struct, thread_mask); 767 OFFSET(PACA_THREAD_MASK, paca_struct, thread_mask);
768 OFFSET(PACA_SUBCORE_SIBLING_MASK, paca_struct, subcore_sibling_mask); 768 OFFSET(PACA_SUBCORE_SIBLING_MASK, paca_struct, subcore_sibling_mask);
769 OFFSET(PACA_SIBLING_PACA_PTRS, paca_struct, thread_sibling_pacas);
770 OFFSET(PACA_REQ_PSSCR, paca_struct, requested_psscr); 769 OFFSET(PACA_REQ_PSSCR, paca_struct, requested_psscr);
771 OFFSET(PACA_DONT_STOP, paca_struct, dont_stop); 770 OFFSET(PACA_DONT_STOP, paca_struct, dont_stop);
772#define STOP_SPR(x, f) OFFSET(x, paca_struct, stop_sprs.f) 771#define STOP_SPR(x, f) OFFSET(x, paca_struct, stop_sprs.f)
diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c
index 9169ffdbab08..2da01340c84c 100644
--- a/arch/powerpc/kernel/cputable.c
+++ b/arch/powerpc/kernel/cputable.c
@@ -466,25 +466,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
466 .machine_check_early = __machine_check_early_realmode_p8, 466 .machine_check_early = __machine_check_early_realmode_p8,
467 .platform = "power8", 467 .platform = "power8",
468 }, 468 },
469 { /* Power9 DD1*/
470 .pvr_mask = 0xffffff00,
471 .pvr_value = 0x004e0100,
472 .cpu_name = "POWER9 (raw)",
473 .cpu_features = CPU_FTRS_POWER9_DD1,
474 .cpu_user_features = COMMON_USER_POWER9,
475 .cpu_user_features2 = COMMON_USER2_POWER9,
476 .mmu_features = MMU_FTRS_POWER9,
477 .icache_bsize = 128,
478 .dcache_bsize = 128,
479 .num_pmcs = 6,
480 .pmc_type = PPC_PMC_IBM,
481 .oprofile_cpu_type = "ppc64/power9",
482 .oprofile_type = PPC_OPROFILE_INVALID,
483 .cpu_setup = __setup_cpu_power9,
484 .cpu_restore = __restore_cpu_power9,
485 .machine_check_early = __machine_check_early_realmode_p9,
486 .platform = "power9",
487 },
488 { /* Power9 DD2.0 */ 469 { /* Power9 DD2.0 */
489 .pvr_mask = 0xffffefff, 470 .pvr_mask = 0xffffefff,
490 .pvr_value = 0x004e0200, 471 .pvr_value = 0x004e0200,
diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c
index 96dd3d871986..f432054234a4 100644
--- a/arch/powerpc/kernel/dt_cpu_ftrs.c
+++ b/arch/powerpc/kernel/dt_cpu_ftrs.c
@@ -701,9 +701,7 @@ static __init void cpufeatures_cpu_quirks(void)
701 /* 701 /*
702 * Not all quirks can be derived from the cpufeatures device tree. 702 * Not all quirks can be derived from the cpufeatures device tree.
703 */ 703 */
704 if ((version & 0xffffff00) == 0x004e0100) 704 if ((version & 0xffffefff) == 0x004e0200)
705 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD1;
706 else if ((version & 0xffffefff) == 0x004e0200)
707 ; /* DD2.0 has no feature flag */ 705 ; /* DD2.0 has no feature flag */
708 else if ((version & 0xffffefff) == 0x004e0201) 706 else if ((version & 0xffffefff) == 0x004e0201)
709 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 707 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1;
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 285c6465324a..76a14702cb9c 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -276,9 +276,7 @@ BEGIN_FTR_SECTION
276 * 276 *
277 * This interrupt can wake directly from idle. If that is the case, 277 * This interrupt can wake directly from idle. If that is the case,
278 * the machine check is handled then the idle wakeup code is called 278 * the machine check is handled then the idle wakeup code is called
279 * to restore state. In that case, the POWER9 DD1 idle PACA workaround 279 * to restore state.
280 * is not applied in the early machine check code, which will cause
281 * bugs.
282 */ 280 */
283 mr r11,r1 /* Save r1 */ 281 mr r11,r1 /* Save r1 */
284 lhz r10,PACA_IN_MCE(r13) 282 lhz r10,PACA_IN_MCE(r13)
diff --git a/arch/powerpc/kernel/idle_book3s.S b/arch/powerpc/kernel/idle_book3s.S
index e734f6e45abc..d85d5515a091 100644
--- a/arch/powerpc/kernel/idle_book3s.S
+++ b/arch/powerpc/kernel/idle_book3s.S
@@ -467,43 +467,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG)
467#endif 467#endif
468 468
469/* 469/*
470 * On waking up from stop 0,1,2 with ESL=1 on POWER9 DD1,
471 * HSPRG0 will be set to the HSPRG0 value of one of the
472 * threads in this core. Thus the value we have in r13
473 * may not be this thread's paca pointer.
474 *
475 * Fortunately, the TIR remains invariant. Since this thread's
476 * paca pointer is recorded in all its sibling's paca, we can
477 * correctly recover this thread's paca pointer if we
478 * know the index of this thread in the core.
479 *
480 * This index can be obtained from the TIR.
481 *
482 * i.e, thread's position in the core = TIR.
483 * If this value is i, then this thread's paca is
484 * paca->thread_sibling_pacas[i].
485 */
486power9_dd1_recover_paca:
487 mfspr r4, SPRN_TIR
488 /*
489 * Since each entry in thread_sibling_pacas is 8 bytes
490 * we need to left-shift by 3 bits. Thus r4 = i * 8
491 */
492 sldi r4, r4, 3
493 /* Get &paca->thread_sibling_pacas[0] in r5 */
494 ld r5, PACA_SIBLING_PACA_PTRS(r13)
495 /* Load paca->thread_sibling_pacas[i] into r13 */
496 ldx r13, r4, r5
497 SET_PACA(r13)
498 /*
499 * Indicate that we have lost NVGPR state
500 * which needs to be restored from the stack.
501 */
502 li r3, 1
503 stb r3,PACA_NAPSTATELOST(r13)
504 blr
505
506/*
507 * Called from machine check handler for powersave wakeups. 470 * Called from machine check handler for powersave wakeups.
508 * Low level machine check processing has already been done. Now just 471 * Low level machine check processing has already been done. Now just
509 * go through the wake up path to get everything in order. 472 * go through the wake up path to get everything in order.
@@ -537,9 +500,6 @@ pnv_powersave_wakeup:
537 ld r2, PACATOC(r13) 500 ld r2, PACATOC(r13)
538 501
539BEGIN_FTR_SECTION 502BEGIN_FTR_SECTION
540BEGIN_FTR_SECTION_NESTED(70)
541 bl power9_dd1_recover_paca
542END_FTR_SECTION_NESTED_IFSET(CPU_FTR_POWER9_DD1, 70)
543 bl pnv_restore_hyp_resource_arch300 503 bl pnv_restore_hyp_resource_arch300
544FTR_SECTION_ELSE 504FTR_SECTION_ELSE
545 bl pnv_restore_hyp_resource_arch207 505 bl pnv_restore_hyp_resource_arch207
@@ -602,22 +562,12 @@ END_FTR_SECTION_IFCLR(CPU_FTR_POWER9_DD2_1)
602 LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state) 562 LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
603 ld r4,ADDROFF(pnv_first_deep_stop_state)(r5) 563 ld r4,ADDROFF(pnv_first_deep_stop_state)(r5)
604 564
605BEGIN_FTR_SECTION_NESTED(71)
606 /*
607 * Assume that we are waking up from the state
608 * same as the Requested Level (RL) in the PSSCR
609 * which are Bits 60-63
610 */
611 ld r5,PACA_REQ_PSSCR(r13)
612 rldicl r5,r5,0,60
613FTR_SECTION_ELSE_NESTED(71)
614 /* 565 /*
615 * 0-3 bits correspond to Power-Saving Level Status 566 * 0-3 bits correspond to Power-Saving Level Status
616 * which indicates the idle state we are waking up from 567 * which indicates the idle state we are waking up from
617 */ 568 */
618 mfspr r5, SPRN_PSSCR 569 mfspr r5, SPRN_PSSCR
619 rldicl r5,r5,4,60 570 rldicl r5,r5,4,60
620ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_POWER9_DD1, 71)
621 li r0, 0 /* clear requested_psscr to say we're awake */ 571 li r0, 0 /* clear requested_psscr to say we're awake */
622 std r0, PACA_REQ_PSSCR(r13) 572 std r0, PACA_REQ_PSSCR(r13)
623 cmpd cr4,r5,r4 573 cmpd cr4,r5,r4
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 9ef4aea9fffe..27f0caee55ea 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -1250,17 +1250,9 @@ struct task_struct *__switch_to(struct task_struct *prev,
1250 * mappings. If the new process has the foreign real address 1250 * mappings. If the new process has the foreign real address
1251 * mappings, we must issue a cp_abort to clear any state and 1251 * mappings, we must issue a cp_abort to clear any state and
1252 * prevent snooping, corruption or a covert channel. 1252 * prevent snooping, corruption or a covert channel.
1253 *
1254 * DD1 allows paste into normal system memory so we do an
1255 * unpaired copy, rather than cp_abort, to clear the buffer,
1256 * since cp_abort is quite expensive.
1257 */ 1253 */
1258 if (current_thread_info()->task->thread.used_vas) { 1254 if (current_thread_info()->task->thread.used_vas)
1259 asm volatile(PPC_CP_ABORT); 1255 asm volatile(PPC_CP_ABORT);
1260 } else if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
1261 asm volatile(PPC_COPY(%0, %1)
1262 : : "r"(dummy_copy_buffer), "r"(0));
1263 }
1264 } 1256 }
1265#endif /* CONFIG_PPC_BOOK3S_64 */ 1257#endif /* CONFIG_PPC_BOOK3S_64 */
1266 1258
diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c
index 176f911ee983..0af1c0aea1fe 100644
--- a/arch/powerpc/kvm/book3s_64_mmu_radix.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c
@@ -66,10 +66,7 @@ int kvmppc_mmu_radix_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
66 bits = root & RPDS_MASK; 66 bits = root & RPDS_MASK;
67 root = root & RPDB_MASK; 67 root = root & RPDB_MASK;
68 68
69 /* P9 DD1 interprets RTS (radix tree size) differently */
70 offset = rts + 31; 69 offset = rts + 31;
71 if (cpu_has_feature(CPU_FTR_POWER9_DD1))
72 offset -= 3;
73 70
74 /* current implementations only support 52-bit space */ 71 /* current implementations only support 52-bit space */
75 if (offset != 52) 72 if (offset != 52)
@@ -160,17 +157,7 @@ static unsigned long kvmppc_radix_update_pte(struct kvm *kvm, pte_t *ptep,
160 unsigned long clr, unsigned long set, 157 unsigned long clr, unsigned long set,
161 unsigned long addr, unsigned int shift) 158 unsigned long addr, unsigned int shift)
162{ 159{
163 unsigned long old = 0; 160 return __radix_pte_update(ptep, clr, set);
164
165 if (!(clr & _PAGE_PRESENT) && cpu_has_feature(CPU_FTR_POWER9_DD1) &&
166 pte_present(*ptep)) {
167 /* have to invalidate it first */
168 old = __radix_pte_update(ptep, _PAGE_PRESENT, 0);
169 kvmppc_radix_tlbie_page(kvm, addr, shift);
170 set |= _PAGE_PRESENT;
171 old &= _PAGE_PRESENT;
172 }
173 return __radix_pte_update(ptep, clr, set) | old;
174} 161}
175 162
176void kvmppc_radix_set_pte_at(struct kvm *kvm, unsigned long addr, 163void kvmppc_radix_set_pte_at(struct kvm *kvm, unsigned long addr,
diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c
index d066e37551ec..8b9aaf24b0a2 100644
--- a/arch/powerpc/kvm/book3s_64_vio.c
+++ b/arch/powerpc/kvm/book3s_64_vio.c
@@ -378,19 +378,19 @@ static long kvmppc_tce_iommu_mapped_dec(struct kvm *kvm,
378{ 378{
379 struct mm_iommu_table_group_mem_t *mem = NULL; 379 struct mm_iommu_table_group_mem_t *mem = NULL;
380 const unsigned long pgsize = 1ULL << tbl->it_page_shift; 380 const unsigned long pgsize = 1ULL << tbl->it_page_shift;
381 unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); 381 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry);
382 382
383 if (!pua) 383 if (!pua)
384 /* it_userspace allocation might be delayed */ 384 /* it_userspace allocation might be delayed */
385 return H_TOO_HARD; 385 return H_TOO_HARD;
386 386
387 mem = mm_iommu_lookup(kvm->mm, *pua, pgsize); 387 mem = mm_iommu_lookup(kvm->mm, be64_to_cpu(*pua), pgsize);
388 if (!mem) 388 if (!mem)
389 return H_TOO_HARD; 389 return H_TOO_HARD;
390 390
391 mm_iommu_mapped_dec(mem); 391 mm_iommu_mapped_dec(mem);
392 392
393 *pua = 0; 393 *pua = cpu_to_be64(0);
394 394
395 return H_SUCCESS; 395 return H_SUCCESS;
396} 396}
@@ -437,7 +437,8 @@ long kvmppc_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl,
437 enum dma_data_direction dir) 437 enum dma_data_direction dir)
438{ 438{
439 long ret; 439 long ret;
440 unsigned long hpa, *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); 440 unsigned long hpa;
441 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry);
441 struct mm_iommu_table_group_mem_t *mem; 442 struct mm_iommu_table_group_mem_t *mem;
442 443
443 if (!pua) 444 if (!pua)
@@ -464,7 +465,7 @@ long kvmppc_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl,
464 if (dir != DMA_NONE) 465 if (dir != DMA_NONE)
465 kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry); 466 kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry);
466 467
467 *pua = ua; 468 *pua = cpu_to_be64(ua);
468 469
469 return 0; 470 return 0;
470} 471}
diff --git a/arch/powerpc/kvm/book3s_64_vio_hv.c b/arch/powerpc/kvm/book3s_64_vio_hv.c
index 925fc316a104..d4bcd1b17b09 100644
--- a/arch/powerpc/kvm/book3s_64_vio_hv.c
+++ b/arch/powerpc/kvm/book3s_64_vio_hv.c
@@ -200,23 +200,19 @@ static long kvmppc_rm_tce_iommu_mapped_dec(struct kvm *kvm,
200{ 200{
201 struct mm_iommu_table_group_mem_t *mem = NULL; 201 struct mm_iommu_table_group_mem_t *mem = NULL;
202 const unsigned long pgsize = 1ULL << tbl->it_page_shift; 202 const unsigned long pgsize = 1ULL << tbl->it_page_shift;
203 unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); 203 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RM(tbl, entry);
204 204
205 if (!pua) 205 if (!pua)
206 /* it_userspace allocation might be delayed */ 206 /* it_userspace allocation might be delayed */
207 return H_TOO_HARD; 207 return H_TOO_HARD;
208 208
209 pua = (void *) vmalloc_to_phys(pua); 209 mem = mm_iommu_lookup_rm(kvm->mm, be64_to_cpu(*pua), pgsize);
210 if (WARN_ON_ONCE_RM(!pua))
211 return H_HARDWARE;
212
213 mem = mm_iommu_lookup_rm(kvm->mm, *pua, pgsize);
214 if (!mem) 210 if (!mem)
215 return H_TOO_HARD; 211 return H_TOO_HARD;
216 212
217 mm_iommu_mapped_dec(mem); 213 mm_iommu_mapped_dec(mem);
218 214
219 *pua = 0; 215 *pua = cpu_to_be64(0);
220 216
221 return H_SUCCESS; 217 return H_SUCCESS;
222} 218}
@@ -268,7 +264,7 @@ static long kvmppc_rm_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl,
268{ 264{
269 long ret; 265 long ret;
270 unsigned long hpa = 0; 266 unsigned long hpa = 0;
271 unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); 267 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RM(tbl, entry);
272 struct mm_iommu_table_group_mem_t *mem; 268 struct mm_iommu_table_group_mem_t *mem;
273 269
274 if (!pua) 270 if (!pua)
@@ -282,10 +278,6 @@ static long kvmppc_rm_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl,
282 if (WARN_ON_ONCE_RM(mm_iommu_ua_to_hpa_rm(mem, ua, &hpa))) 278 if (WARN_ON_ONCE_RM(mm_iommu_ua_to_hpa_rm(mem, ua, &hpa)))
283 return H_HARDWARE; 279 return H_HARDWARE;
284 280
285 pua = (void *) vmalloc_to_phys(pua);
286 if (WARN_ON_ONCE_RM(!pua))
287 return H_HARDWARE;
288
289 if (WARN_ON_ONCE_RM(mm_iommu_mapped_inc(mem))) 281 if (WARN_ON_ONCE_RM(mm_iommu_mapped_inc(mem)))
290 return H_CLOSED; 282 return H_CLOSED;
291 283
@@ -302,7 +294,7 @@ static long kvmppc_rm_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl,
302 if (dir != DMA_NONE) 294 if (dir != DMA_NONE)
303 kvmppc_rm_tce_iommu_mapped_dec(kvm, tbl, entry); 295 kvmppc_rm_tce_iommu_mapped_dec(kvm, tbl, entry);
304 296
305 *pua = ua; 297 *pua = cpu_to_be64(ua);
306 298
307 return 0; 299 return 0;
308} 300}
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index de686b340f4a..b568582120a3 100644
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -1693,14 +1693,6 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
1693 r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len); 1693 r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len);
1694 break; 1694 break;
1695 case KVM_REG_PPC_TB_OFFSET: 1695 case KVM_REG_PPC_TB_OFFSET:
1696 /*
1697 * POWER9 DD1 has an erratum where writing TBU40 causes
1698 * the timebase to lose ticks. So we don't let the
1699 * timebase offset be changed on P9 DD1. (It is
1700 * initialized to zero.)
1701 */
1702 if (cpu_has_feature(CPU_FTR_POWER9_DD1))
1703 break;
1704 /* round up to multiple of 2^24 */ 1696 /* round up to multiple of 2^24 */
1705 vcpu->arch.vcore->tb_offset = 1697 vcpu->arch.vcore->tb_offset =
1706 ALIGN(set_reg_val(id, *val), 1UL << 24); 1698 ALIGN(set_reg_val(id, *val), 1UL << 24);
@@ -2026,8 +2018,6 @@ static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm,
2026 /* 2018 /*
2027 * Set the default HFSCR for the guest from the host value. 2019 * Set the default HFSCR for the guest from the host value.
2028 * This value is only used on POWER9. 2020 * This value is only used on POWER9.
2029 * On POWER9 DD1, TM doesn't work, so we make sure to
2030 * prevent the guest from using it.
2031 * On POWER9, we want to virtualize the doorbell facility, so we 2021 * On POWER9, we want to virtualize the doorbell facility, so we
2032 * turn off the HFSCR bit, which causes those instructions to trap. 2022 * turn off the HFSCR bit, which causes those instructions to trap.
2033 */ 2023 */
diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
index 153988d878e8..6e4554b273f1 100644
--- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
+++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
@@ -917,9 +917,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_DAWR)
917 mtspr SPRN_PID, r7 917 mtspr SPRN_PID, r7
918 mtspr SPRN_WORT, r8 918 mtspr SPRN_WORT, r8
919BEGIN_FTR_SECTION 919BEGIN_FTR_SECTION
920 PPC_INVALIDATE_ERAT
921END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1)
922BEGIN_FTR_SECTION
923 /* POWER8-only registers */ 920 /* POWER8-only registers */
924 ld r5, VCPU_TCSCR(r4) 921 ld r5, VCPU_TCSCR(r4)
925 ld r6, VCPU_ACOP(r4) 922 ld r6, VCPU_ACOP(r4)
@@ -1912,7 +1909,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
1912 ld r5, VCPU_KVM(r9) 1909 ld r5, VCPU_KVM(r9)
1913 lbz r0, KVM_RADIX(r5) 1910 lbz r0, KVM_RADIX(r5)
1914 cmpwi cr2, r0, 0 1911 cmpwi cr2, r0, 0
1915 beq cr2, 4f 1912 beq cr2, 2f
1916 1913
1917 /* 1914 /*
1918 * Radix: do eieio; tlbsync; ptesync sequence in case we 1915 * Radix: do eieio; tlbsync; ptesync sequence in case we
@@ -1952,11 +1949,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
1952 bdnz 1b 1949 bdnz 1b
1953 ptesync 1950 ptesync
1954 1951
19552: /* Flush the ERAT on radix P9 DD1 guest exit */ 19522:
1956BEGIN_FTR_SECTION
1957 PPC_INVALIDATE_ERAT
1958END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1)
19594:
1960#endif /* CONFIG_PPC_RADIX_MMU */ 1953#endif /* CONFIG_PPC_RADIX_MMU */
1961 1954
1962 /* 1955 /*
@@ -3367,11 +3360,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
3367 mtspr SPRN_CIABR, r0 3360 mtspr SPRN_CIABR, r0
3368 mtspr SPRN_DAWRX, r0 3361 mtspr SPRN_DAWRX, r0
3369 3362
3370 /* Flush the ERAT on radix P9 DD1 guest exit */
3371BEGIN_FTR_SECTION
3372 PPC_INVALIDATE_ERAT
3373END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1)
3374
3375BEGIN_MMU_FTR_SECTION 3363BEGIN_MMU_FTR_SECTION
3376 b 4f 3364 b 4f
3377END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 3365END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
diff --git a/arch/powerpc/kvm/book3s_xive_template.c b/arch/powerpc/kvm/book3s_xive_template.c
index 6e41ba7ec8f4..4171ede8722b 100644
--- a/arch/powerpc/kvm/book3s_xive_template.c
+++ b/arch/powerpc/kvm/book3s_xive_template.c
@@ -25,18 +25,6 @@ static void GLUE(X_PFX,ack_pending)(struct kvmppc_xive_vcpu *xc)
25 */ 25 */
26 eieio(); 26 eieio();
27 27
28 /*
29 * DD1 bug workaround: If PIPR is less favored than CPPR
30 * ignore the interrupt or we might incorrectly lose an IPB
31 * bit.
32 */
33 if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
34 __be64 qw1 = __x_readq(__x_tima + TM_QW1_OS);
35 u8 pipr = be64_to_cpu(qw1) & 0xff;
36 if (pipr >= xc->hw_cppr)
37 return;
38 }
39
40 /* Perform the acknowledge OS to register cycle. */ 28 /* Perform the acknowledge OS to register cycle. */
41 ack = be16_to_cpu(__x_readw(__x_tima + TM_SPC_ACK_OS_REG)); 29 ack = be16_to_cpu(__x_readw(__x_tima + TM_SPC_ACK_OS_REG));
42 30
@@ -89,8 +77,15 @@ static void GLUE(X_PFX,source_eoi)(u32 hw_irq, struct xive_irq_data *xd)
89 /* If the XIVE supports the new "store EOI facility, use it */ 77 /* If the XIVE supports the new "store EOI facility, use it */
90 if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI) 78 if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI)
91 __x_writeq(0, __x_eoi_page(xd) + XIVE_ESB_STORE_EOI); 79 __x_writeq(0, __x_eoi_page(xd) + XIVE_ESB_STORE_EOI);
92 else if (hw_irq && xd->flags & XIVE_IRQ_FLAG_EOI_FW) { 80 else if (hw_irq && xd->flags & XIVE_IRQ_FLAG_EOI_FW)
93 opal_int_eoi(hw_irq); 81 opal_int_eoi(hw_irq);
82 else if (xd->flags & XIVE_IRQ_FLAG_LSI) {
83 /*
84 * For LSIs the HW EOI cycle is used rather than PQ bits,
85 * as they are automatically re-triggred in HW when still
86 * pending.
87 */
88 __x_readq(__x_eoi_page(xd) + XIVE_ESB_LOAD_EOI);
94 } else { 89 } else {
95 uint64_t eoi_val; 90 uint64_t eoi_val;
96 91
@@ -102,20 +97,12 @@ static void GLUE(X_PFX,source_eoi)(u32 hw_irq, struct xive_irq_data *xd)
102 * 97 *
103 * This allows us to then do a re-trigger if Q was set 98 * This allows us to then do a re-trigger if Q was set
104 * rather than synthetizing an interrupt in software 99 * rather than synthetizing an interrupt in software
105 *
106 * For LSIs, using the HW EOI cycle works around a problem
107 * on P9 DD1 PHBs where the other ESB accesses don't work
108 * properly.
109 */ 100 */
110 if (xd->flags & XIVE_IRQ_FLAG_LSI) 101 eoi_val = GLUE(X_PFX,esb_load)(xd, XIVE_ESB_SET_PQ_00);
111 __x_readq(__x_eoi_page(xd) + XIVE_ESB_LOAD_EOI); 102
112 else { 103 /* Re-trigger if needed */
113 eoi_val = GLUE(X_PFX,esb_load)(xd, XIVE_ESB_SET_PQ_00); 104 if ((eoi_val & 1) && __x_trig_page(xd))
114 105 __x_writeq(0, __x_trig_page(xd));
115 /* Re-trigger if needed */
116 if ((eoi_val & 1) && __x_trig_page(xd))
117 __x_writeq(0, __x_trig_page(xd));
118 }
119 } 106 }
120} 107}
121 108
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 8318716e5075..5a72e980e25a 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -808,31 +808,6 @@ int hash__remove_section_mapping(unsigned long start, unsigned long end)
808} 808}
809#endif /* CONFIG_MEMORY_HOTPLUG */ 809#endif /* CONFIG_MEMORY_HOTPLUG */
810 810
811static void update_hid_for_hash(void)
812{
813 unsigned long hid0;
814 unsigned long rb = 3UL << PPC_BITLSHIFT(53); /* IS = 3 */
815
816 asm volatile("ptesync": : :"memory");
817 /* prs = 0, ric = 2, rs = 0, r = 1 is = 3 */
818 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
819 : : "r"(rb), "i"(0), "i"(0), "i"(2), "r"(0) : "memory");
820 asm volatile("eieio; tlbsync; ptesync; isync; slbia": : :"memory");
821 trace_tlbie(0, 0, rb, 0, 2, 0, 0);
822
823 /*
824 * now switch the HID
825 */
826 hid0 = mfspr(SPRN_HID0);
827 hid0 &= ~HID0_POWER9_RADIX;
828 mtspr(SPRN_HID0, hid0);
829 asm volatile("isync": : :"memory");
830
831 /* Wait for it to happen */
832 while ((mfspr(SPRN_HID0) & HID0_POWER9_RADIX))
833 cpu_relax();
834}
835
836static void __init hash_init_partition_table(phys_addr_t hash_table, 811static void __init hash_init_partition_table(phys_addr_t hash_table,
837 unsigned long htab_size) 812 unsigned long htab_size)
838{ 813{
@@ -845,8 +820,6 @@ static void __init hash_init_partition_table(phys_addr_t hash_table,
845 htab_size = __ilog2(htab_size) - 18; 820 htab_size = __ilog2(htab_size) - 18;
846 mmu_partition_table_set_entry(0, hash_table | htab_size, 0); 821 mmu_partition_table_set_entry(0, hash_table | htab_size, 0);
847 pr_info("Partition table %p\n", partition_tb); 822 pr_info("Partition table %p\n", partition_tb);
848 if (cpu_has_feature(CPU_FTR_POWER9_DD1))
849 update_hid_for_hash();
850} 823}
851 824
852static void __init htab_initialize(void) 825static void __init htab_initialize(void)
@@ -1077,9 +1050,6 @@ void hash__early_init_mmu_secondary(void)
1077 /* Initialize hash table for that CPU */ 1050 /* Initialize hash table for that CPU */
1078 if (!firmware_has_feature(FW_FEATURE_LPAR)) { 1051 if (!firmware_has_feature(FW_FEATURE_LPAR)) {
1079 1052
1080 if (cpu_has_feature(CPU_FTR_POWER9_DD1))
1081 update_hid_for_hash();
1082
1083 if (!cpu_has_feature(CPU_FTR_ARCH_300)) 1053 if (!cpu_has_feature(CPU_FTR_ARCH_300))
1084 mtspr(SPRN_SDR1, _SDR1); 1054 mtspr(SPRN_SDR1, _SDR1);
1085 else 1055 else
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index 8a9a49c13865..f425b5b37d58 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -621,15 +621,12 @@ static int __init add_huge_page_size(unsigned long long size)
621 * firmware we only add hugetlb support for page sizes that can be 621 * firmware we only add hugetlb support for page sizes that can be
622 * supported by linux page table layout. 622 * supported by linux page table layout.
623 * For now we have 623 * For now we have
624 * Radix: 2M 624 * Radix: 2M and 1G
625 * Hash: 16M and 16G 625 * Hash: 16M and 16G
626 */ 626 */
627 if (radix_enabled()) { 627 if (radix_enabled()) {
628 if (mmu_psize != MMU_PAGE_2M) { 628 if (mmu_psize != MMU_PAGE_2M && mmu_psize != MMU_PAGE_1G)
629 if (cpu_has_feature(CPU_FTR_POWER9_DD1) || 629 return -EINVAL;
630 (mmu_psize != MMU_PAGE_1G))
631 return -EINVAL;
632 }
633 } else { 630 } else {
634 if (mmu_psize != MMU_PAGE_16M && mmu_psize != MMU_PAGE_16G) 631 if (mmu_psize != MMU_PAGE_16M && mmu_psize != MMU_PAGE_16G)
635 return -EINVAL; 632 return -EINVAL;
diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c
index f3d4b4a0e561..39e9ef0eb78b 100644
--- a/arch/powerpc/mm/mmu_context_book3s64.c
+++ b/arch/powerpc/mm/mmu_context_book3s64.c
@@ -273,15 +273,7 @@ void arch_exit_mmap(struct mm_struct *mm)
273#ifdef CONFIG_PPC_RADIX_MMU 273#ifdef CONFIG_PPC_RADIX_MMU
274void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next) 274void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
275{ 275{
276 276 mtspr(SPRN_PID, next->context.id);
277 if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { 277 isync();
278 isync();
279 mtspr(SPRN_PID, next->context.id);
280 isync();
281 asm volatile(PPC_INVALIDATE_ERAT : : :"memory");
282 } else {
283 mtspr(SPRN_PID, next->context.id);
284 isync();
285 }
286} 278}
287#endif 279#endif
diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c
index 96f68c5aa1f5..bba168d02235 100644
--- a/arch/powerpc/mm/pgtable-radix.c
+++ b/arch/powerpc/mm/pgtable-radix.c
@@ -226,16 +226,6 @@ void radix__mark_rodata_ro(void)
226{ 226{
227 unsigned long start, end; 227 unsigned long start, end;
228 228
229 /*
230 * mark_rodata_ro() will mark itself as !writable at some point.
231 * Due to DD1 workaround in radix__pte_update(), we'll end up with
232 * an invalid pte and the system will crash quite severly.
233 */
234 if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
235 pr_warn("Warning: Unable to mark rodata read only on P9 DD1\n");
236 return;
237 }
238
239 start = (unsigned long)_stext; 229 start = (unsigned long)_stext;
240 end = (unsigned long)__init_begin; 230 end = (unsigned long)__init_begin;
241 231
@@ -533,35 +523,6 @@ found:
533 return; 523 return;
534} 524}
535 525
536static void update_hid_for_radix(void)
537{
538 unsigned long hid0;
539 unsigned long rb = 3UL << PPC_BITLSHIFT(53); /* IS = 3 */
540
541 asm volatile("ptesync": : :"memory");
542 /* prs = 0, ric = 2, rs = 0, r = 1 is = 3 */
543 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
544 : : "r"(rb), "i"(1), "i"(0), "i"(2), "r"(0) : "memory");
545 /* prs = 1, ric = 2, rs = 0, r = 1 is = 3 */
546 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
547 : : "r"(rb), "i"(1), "i"(1), "i"(2), "r"(0) : "memory");
548 asm volatile("eieio; tlbsync; ptesync; isync; slbia": : :"memory");
549 trace_tlbie(0, 0, rb, 0, 2, 0, 1);
550 trace_tlbie(0, 0, rb, 0, 2, 1, 1);
551
552 /*
553 * now switch the HID
554 */
555 hid0 = mfspr(SPRN_HID0);
556 hid0 |= HID0_POWER9_RADIX;
557 mtspr(SPRN_HID0, hid0);
558 asm volatile("isync": : :"memory");
559
560 /* Wait for it to happen */
561 while (!(mfspr(SPRN_HID0) & HID0_POWER9_RADIX))
562 cpu_relax();
563}
564
565static void radix_init_amor(void) 526static void radix_init_amor(void)
566{ 527{
567 /* 528 /*
@@ -576,22 +537,12 @@ static void radix_init_amor(void)
576 537
577static void radix_init_iamr(void) 538static void radix_init_iamr(void)
578{ 539{
579 unsigned long iamr;
580
581 /*
582 * The IAMR should set to 0 on DD1.
583 */
584 if (cpu_has_feature(CPU_FTR_POWER9_DD1))
585 iamr = 0;
586 else
587 iamr = (1ul << 62);
588
589 /* 540 /*
590 * Radix always uses key0 of the IAMR to determine if an access is 541 * Radix always uses key0 of the IAMR to determine if an access is
591 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction 542 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction
592 * fetch. 543 * fetch.
593 */ 544 */
594 mtspr(SPRN_IAMR, iamr); 545 mtspr(SPRN_IAMR, (1ul << 62));
595} 546}
596 547
597void __init radix__early_init_mmu(void) 548void __init radix__early_init_mmu(void)
@@ -644,8 +595,6 @@ void __init radix__early_init_mmu(void)
644 595
645 if (!firmware_has_feature(FW_FEATURE_LPAR)) { 596 if (!firmware_has_feature(FW_FEATURE_LPAR)) {
646 radix_init_native(); 597 radix_init_native();
647 if (cpu_has_feature(CPU_FTR_POWER9_DD1))
648 update_hid_for_radix();
649 lpcr = mfspr(SPRN_LPCR); 598 lpcr = mfspr(SPRN_LPCR);
650 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 599 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
651 radix_init_partition_table(); 600 radix_init_partition_table();
@@ -671,10 +620,6 @@ void radix__early_init_mmu_secondary(void)
671 * update partition table control register and UPRT 620 * update partition table control register and UPRT
672 */ 621 */
673 if (!firmware_has_feature(FW_FEATURE_LPAR)) { 622 if (!firmware_has_feature(FW_FEATURE_LPAR)) {
674
675 if (cpu_has_feature(CPU_FTR_POWER9_DD1))
676 update_hid_for_radix();
677
678 lpcr = mfspr(SPRN_LPCR); 623 lpcr = mfspr(SPRN_LPCR);
679 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 624 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
680 625
@@ -1095,8 +1040,7 @@ void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
1095 * To avoid NMMU hang while relaxing access, we need mark 1040 * To avoid NMMU hang while relaxing access, we need mark
1096 * the pte invalid in between. 1041 * the pte invalid in between.
1097 */ 1042 */
1098 if (cpu_has_feature(CPU_FTR_POWER9_DD1) || 1043 if (atomic_read(&mm->context.copros) > 0) {
1099 atomic_read(&mm->context.copros) > 0) {
1100 unsigned long old_pte, new_pte; 1044 unsigned long old_pte, new_pte;
1101 1045
1102 old_pte = __radix_pte_update(ptep, ~0, 0); 1046 old_pte = __radix_pte_update(ptep, ~0, 0);
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index 1135b43a597c..fef3e1eb3a19 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -1048,24 +1048,6 @@ void radix__flush_tlb_all(void)
1048 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 1048 asm volatile("eieio; tlbsync; ptesync": : :"memory");
1049} 1049}
1050 1050
1051void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm,
1052 unsigned long address)
1053{
1054 /*
1055 * We track page size in pte only for DD1, So we can
1056 * call this only on DD1.
1057 */
1058 if (!cpu_has_feature(CPU_FTR_POWER9_DD1)) {
1059 VM_WARN_ON(1);
1060 return;
1061 }
1062
1063 if (old_pte & R_PAGE_LARGE)
1064 radix__flush_tlb_page_psize(mm, address, MMU_PAGE_2M);
1065 else
1066 radix__flush_tlb_page_psize(mm, address, mmu_virtual_psize);
1067}
1068
1069#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 1051#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
1070extern void radix_kvm_prefetch_workaround(struct mm_struct *mm) 1052extern void radix_kvm_prefetch_workaround(struct mm_struct *mm)
1071{ 1053{
diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c
index 3f66fcf8ad99..01f92c4a9f02 100644
--- a/arch/powerpc/perf/core-book3s.c
+++ b/arch/powerpc/perf/core-book3s.c
@@ -128,10 +128,6 @@ static inline void power_pmu_bhrb_disable(struct perf_event *event) {}
128static void power_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) {} 128static void power_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) {}
129static inline void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw) {} 129static inline void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw) {}
130static void pmao_restore_workaround(bool ebb) { } 130static void pmao_restore_workaround(bool ebb) { }
131static bool use_ic(u64 event)
132{
133 return false;
134}
135#endif /* CONFIG_PPC32 */ 131#endif /* CONFIG_PPC32 */
136 132
137static bool regs_use_siar(struct pt_regs *regs) 133static bool regs_use_siar(struct pt_regs *regs)
@@ -714,14 +710,6 @@ static void pmao_restore_workaround(bool ebb)
714 mtspr(SPRN_PMC6, pmcs[5]); 710 mtspr(SPRN_PMC6, pmcs[5]);
715} 711}
716 712
717static bool use_ic(u64 event)
718{
719 if (cpu_has_feature(CPU_FTR_POWER9_DD1) &&
720 (event == 0x200f2 || event == 0x300f2))
721 return true;
722
723 return false;
724}
725#endif /* CONFIG_PPC64 */ 713#endif /* CONFIG_PPC64 */
726 714
727static void perf_event_interrupt(struct pt_regs *regs); 715static void perf_event_interrupt(struct pt_regs *regs);
@@ -1046,7 +1034,6 @@ static u64 check_and_compute_delta(u64 prev, u64 val)
1046static void power_pmu_read(struct perf_event *event) 1034static void power_pmu_read(struct perf_event *event)
1047{ 1035{
1048 s64 val, delta, prev; 1036 s64 val, delta, prev;
1049 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
1050 1037
1051 if (event->hw.state & PERF_HES_STOPPED) 1038 if (event->hw.state & PERF_HES_STOPPED)
1052 return; 1039 return;
@@ -1056,13 +1043,6 @@ static void power_pmu_read(struct perf_event *event)
1056 1043
1057 if (is_ebb_event(event)) { 1044 if (is_ebb_event(event)) {
1058 val = read_pmc(event->hw.idx); 1045 val = read_pmc(event->hw.idx);
1059 if (use_ic(event->attr.config)) {
1060 val = mfspr(SPRN_IC);
1061 if (val > cpuhw->ic_init)
1062 val = val - cpuhw->ic_init;
1063 else
1064 val = val + (0 - cpuhw->ic_init);
1065 }
1066 local64_set(&event->hw.prev_count, val); 1046 local64_set(&event->hw.prev_count, val);
1067 return; 1047 return;
1068 } 1048 }
@@ -1076,13 +1056,6 @@ static void power_pmu_read(struct perf_event *event)
1076 prev = local64_read(&event->hw.prev_count); 1056 prev = local64_read(&event->hw.prev_count);
1077 barrier(); 1057 barrier();
1078 val = read_pmc(event->hw.idx); 1058 val = read_pmc(event->hw.idx);
1079 if (use_ic(event->attr.config)) {
1080 val = mfspr(SPRN_IC);
1081 if (val > cpuhw->ic_init)
1082 val = val - cpuhw->ic_init;
1083 else
1084 val = val + (0 - cpuhw->ic_init);
1085 }
1086 delta = check_and_compute_delta(prev, val); 1059 delta = check_and_compute_delta(prev, val);
1087 if (!delta) 1060 if (!delta)
1088 return; 1061 return;
@@ -1535,13 +1508,6 @@ nocheck:
1535 event->attr.branch_sample_type); 1508 event->attr.branch_sample_type);
1536 } 1509 }
1537 1510
1538 /*
1539 * Workaround for POWER9 DD1 to use the Instruction Counter
1540 * register value for instruction counting
1541 */
1542 if (use_ic(event->attr.config))
1543 cpuhw->ic_init = mfspr(SPRN_IC);
1544
1545 perf_pmu_enable(event->pmu); 1511 perf_pmu_enable(event->pmu);
1546 local_irq_restore(flags); 1512 local_irq_restore(flags);
1547 return ret; 1513 return ret;
diff --git a/arch/powerpc/perf/isa207-common.c b/arch/powerpc/perf/isa207-common.c
index 2efee3f196f5..177de814286f 100644
--- a/arch/powerpc/perf/isa207-common.c
+++ b/arch/powerpc/perf/isa207-common.c
@@ -59,7 +59,7 @@ static bool is_event_valid(u64 event)
59{ 59{
60 u64 valid_mask = EVENT_VALID_MASK; 60 u64 valid_mask = EVENT_VALID_MASK;
61 61
62 if (cpu_has_feature(CPU_FTR_ARCH_300) && !cpu_has_feature(CPU_FTR_POWER9_DD1)) 62 if (cpu_has_feature(CPU_FTR_ARCH_300))
63 valid_mask = p9_EVENT_VALID_MASK; 63 valid_mask = p9_EVENT_VALID_MASK;
64 64
65 return !(event & ~valid_mask); 65 return !(event & ~valid_mask);
@@ -86,8 +86,6 @@ static void mmcra_sdar_mode(u64 event, unsigned long *mmcra)
86 * Incase of Power9: 86 * Incase of Power9:
87 * Marked event: MMCRA[SDAR_MODE] will be set to 0b00 ('No Updates'), 87 * Marked event: MMCRA[SDAR_MODE] will be set to 0b00 ('No Updates'),
88 * or if group already have any marked events. 88 * or if group already have any marked events.
89 * Non-Marked events (for DD1):
90 * MMCRA[SDAR_MODE] will be set to 0b01
91 * For rest 89 * For rest
92 * MMCRA[SDAR_MODE] will be set from event code. 90 * MMCRA[SDAR_MODE] will be set from event code.
93 * If sdar_mode from event is zero, default to 0b01. Hardware 91 * If sdar_mode from event is zero, default to 0b01. Hardware
@@ -96,7 +94,7 @@ static void mmcra_sdar_mode(u64 event, unsigned long *mmcra)
96 if (cpu_has_feature(CPU_FTR_ARCH_300)) { 94 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
97 if (is_event_marked(event) || (*mmcra & MMCRA_SAMPLE_ENABLE)) 95 if (is_event_marked(event) || (*mmcra & MMCRA_SAMPLE_ENABLE))
98 *mmcra &= MMCRA_SDAR_MODE_NO_UPDATES; 96 *mmcra &= MMCRA_SDAR_MODE_NO_UPDATES;
99 else if (!cpu_has_feature(CPU_FTR_POWER9_DD1) && p9_SDAR_MODE(event)) 97 else if (p9_SDAR_MODE(event))
100 *mmcra |= p9_SDAR_MODE(event) << MMCRA_SDAR_MODE_SHIFT; 98 *mmcra |= p9_SDAR_MODE(event) << MMCRA_SDAR_MODE_SHIFT;
101 else 99 else
102 *mmcra |= MMCRA_SDAR_MODE_DCACHE; 100 *mmcra |= MMCRA_SDAR_MODE_DCACHE;
@@ -106,7 +104,7 @@ static void mmcra_sdar_mode(u64 event, unsigned long *mmcra)
106 104
107static u64 thresh_cmp_val(u64 value) 105static u64 thresh_cmp_val(u64 value)
108{ 106{
109 if (cpu_has_feature(CPU_FTR_ARCH_300) && !cpu_has_feature(CPU_FTR_POWER9_DD1)) 107 if (cpu_has_feature(CPU_FTR_ARCH_300))
110 return value << p9_MMCRA_THR_CMP_SHIFT; 108 return value << p9_MMCRA_THR_CMP_SHIFT;
111 109
112 return value << MMCRA_THR_CMP_SHIFT; 110 return value << MMCRA_THR_CMP_SHIFT;
@@ -114,7 +112,7 @@ static u64 thresh_cmp_val(u64 value)
114 112
115static unsigned long combine_from_event(u64 event) 113static unsigned long combine_from_event(u64 event)
116{ 114{
117 if (cpu_has_feature(CPU_FTR_ARCH_300) && !cpu_has_feature(CPU_FTR_POWER9_DD1)) 115 if (cpu_has_feature(CPU_FTR_ARCH_300))
118 return p9_EVENT_COMBINE(event); 116 return p9_EVENT_COMBINE(event);
119 117
120 return EVENT_COMBINE(event); 118 return EVENT_COMBINE(event);
@@ -122,7 +120,7 @@ static unsigned long combine_from_event(u64 event)
122 120
123static unsigned long combine_shift(unsigned long pmc) 121static unsigned long combine_shift(unsigned long pmc)
124{ 122{
125 if (cpu_has_feature(CPU_FTR_ARCH_300) && !cpu_has_feature(CPU_FTR_POWER9_DD1)) 123 if (cpu_has_feature(CPU_FTR_ARCH_300))
126 return p9_MMCR1_COMBINE_SHIFT(pmc); 124 return p9_MMCR1_COMBINE_SHIFT(pmc);
127 125
128 return MMCR1_COMBINE_SHIFT(pmc); 126 return MMCR1_COMBINE_SHIFT(pmc);
diff --git a/arch/powerpc/perf/isa207-common.h b/arch/powerpc/perf/isa207-common.h
index 6a0b586c935a..0028f4b9490d 100644
--- a/arch/powerpc/perf/isa207-common.h
+++ b/arch/powerpc/perf/isa207-common.h
@@ -158,11 +158,6 @@
158 CNST_PMC_VAL(1) | CNST_PMC_VAL(2) | CNST_PMC_VAL(3) | \ 158 CNST_PMC_VAL(1) | CNST_PMC_VAL(2) | CNST_PMC_VAL(3) | \
159 CNST_PMC_VAL(4) | CNST_PMC_VAL(5) | CNST_PMC_VAL(6) | CNST_NC_VAL 159 CNST_PMC_VAL(4) | CNST_PMC_VAL(5) | CNST_PMC_VAL(6) | CNST_NC_VAL
160 160
161/*
162 * Lets restrict use of PMC5 for instruction counting.
163 */
164#define P9_DD1_TEST_ADDER (ISA207_TEST_ADDER | CNST_PMC_VAL(5))
165
166/* Bits in MMCR1 for PowerISA v2.07 */ 161/* Bits in MMCR1 for PowerISA v2.07 */
167#define MMCR1_UNIT_SHIFT(pmc) (60 - (4 * ((pmc) - 1))) 162#define MMCR1_UNIT_SHIFT(pmc) (60 - (4 * ((pmc) - 1)))
168#define MMCR1_COMBINE_SHIFT(pmc) (35 - ((pmc) - 1)) 163#define MMCR1_COMBINE_SHIFT(pmc) (35 - ((pmc) - 1))
diff --git a/arch/powerpc/perf/power9-pmu.c b/arch/powerpc/perf/power9-pmu.c
index 2ca0b33b4efb..e012b1030a5b 100644
--- a/arch/powerpc/perf/power9-pmu.c
+++ b/arch/powerpc/perf/power9-pmu.c
@@ -219,12 +219,6 @@ static struct attribute_group power9_pmu_events_group = {
219 .attrs = power9_events_attr, 219 .attrs = power9_events_attr,
220}; 220};
221 221
222static const struct attribute_group *power9_isa207_pmu_attr_groups[] = {
223 &isa207_pmu_format_group,
224 &power9_pmu_events_group,
225 NULL,
226};
227
228PMU_FORMAT_ATTR(event, "config:0-51"); 222PMU_FORMAT_ATTR(event, "config:0-51");
229PMU_FORMAT_ATTR(pmcxsel, "config:0-7"); 223PMU_FORMAT_ATTR(pmcxsel, "config:0-7");
230PMU_FORMAT_ATTR(mark, "config:8"); 224PMU_FORMAT_ATTR(mark, "config:8");
@@ -267,17 +261,6 @@ static const struct attribute_group *power9_pmu_attr_groups[] = {
267 NULL, 261 NULL,
268}; 262};
269 263
270static int power9_generic_events_dd1[] = {
271 [PERF_COUNT_HW_CPU_CYCLES] = PM_CYC,
272 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = PM_ICT_NOSLOT_CYC,
273 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = PM_CMPLU_STALL,
274 [PERF_COUNT_HW_INSTRUCTIONS] = PM_INST_DISP,
275 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = PM_BR_CMPL_ALT,
276 [PERF_COUNT_HW_BRANCH_MISSES] = PM_BR_MPRED_CMPL,
277 [PERF_COUNT_HW_CACHE_REFERENCES] = PM_LD_REF_L1,
278 [PERF_COUNT_HW_CACHE_MISSES] = PM_LD_MISS_L1_FIN,
279};
280
281static int power9_generic_events[] = { 264static int power9_generic_events[] = {
282 [PERF_COUNT_HW_CPU_CYCLES] = PM_CYC, 265 [PERF_COUNT_HW_CPU_CYCLES] = PM_CYC,
283 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = PM_ICT_NOSLOT_CYC, 266 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = PM_ICT_NOSLOT_CYC,
@@ -439,25 +422,6 @@ static int power9_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
439 422
440#undef C 423#undef C
441 424
442static struct power_pmu power9_isa207_pmu = {
443 .name = "POWER9",
444 .n_counter = MAX_PMU_COUNTERS,
445 .add_fields = ISA207_ADD_FIELDS,
446 .test_adder = P9_DD1_TEST_ADDER,
447 .compute_mmcr = isa207_compute_mmcr,
448 .config_bhrb = power9_config_bhrb,
449 .bhrb_filter_map = power9_bhrb_filter_map,
450 .get_constraint = isa207_get_constraint,
451 .get_alternatives = power9_get_alternatives,
452 .disable_pmc = isa207_disable_pmc,
453 .flags = PPMU_NO_SIAR | PPMU_ARCH_207S,
454 .n_generic = ARRAY_SIZE(power9_generic_events_dd1),
455 .generic_events = power9_generic_events_dd1,
456 .cache_events = &power9_cache_events,
457 .attr_groups = power9_isa207_pmu_attr_groups,
458 .bhrb_nr = 32,
459};
460
461static struct power_pmu power9_pmu = { 425static struct power_pmu power9_pmu = {
462 .name = "POWER9", 426 .name = "POWER9",
463 .n_counter = MAX_PMU_COUNTERS, 427 .n_counter = MAX_PMU_COUNTERS,
@@ -500,23 +464,7 @@ static int __init init_power9_pmu(void)
500 } 464 }
501 } 465 }
502 466
503 if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { 467 rc = register_power_pmu(&power9_pmu);
504 /*
505 * Since PM_INST_CMPL may not provide right counts in all
506 * sampling scenarios in power9 DD1, instead use PM_INST_DISP.
507 */
508 EVENT_VAR(PM_INST_CMPL, _g).id = PM_INST_DISP;
509 /*
510 * Power9 DD1 should use PM_BR_CMPL_ALT event code for
511 * "branches" to provide correct counter value.
512 */
513 EVENT_VAR(PM_BR_CMPL, _g).id = PM_BR_CMPL_ALT;
514 EVENT_VAR(PM_BR_CMPL, _c).id = PM_BR_CMPL_ALT;
515 rc = register_power_pmu(&power9_isa207_pmu);
516 } else {
517 rc = register_power_pmu(&power9_pmu);
518 }
519
520 if (rc) 468 if (rc)
521 return rc; 469 return rc;
522 470
diff --git a/arch/powerpc/platforms/powernv/Makefile b/arch/powerpc/platforms/powernv/Makefile
index 703a350a7f4e..b540ce8eec55 100644
--- a/arch/powerpc/platforms/powernv/Makefile
+++ b/arch/powerpc/platforms/powernv/Makefile
@@ -6,7 +6,7 @@ obj-y += opal-msglog.o opal-hmi.o opal-power.o opal-irqchip.o
6obj-y += opal-kmsg.o opal-powercap.o opal-psr.o opal-sensor-groups.o 6obj-y += opal-kmsg.o opal-powercap.o opal-psr.o opal-sensor-groups.o
7 7
8obj-$(CONFIG_SMP) += smp.o subcore.o subcore-asm.o 8obj-$(CONFIG_SMP) += smp.o subcore.o subcore-asm.o
9obj-$(CONFIG_PCI) += pci.o pci-ioda.o npu-dma.o 9obj-$(CONFIG_PCI) += pci.o pci-ioda.o npu-dma.o pci-ioda-tce.o
10obj-$(CONFIG_CXL_BASE) += pci-cxl.o 10obj-$(CONFIG_CXL_BASE) += pci-cxl.o
11obj-$(CONFIG_EEH) += eeh-powernv.o 11obj-$(CONFIG_EEH) += eeh-powernv.o
12obj-$(CONFIG_PPC_SCOM) += opal-xscom.o 12obj-$(CONFIG_PPC_SCOM) += opal-xscom.o
diff --git a/arch/powerpc/platforms/powernv/idle.c b/arch/powerpc/platforms/powernv/idle.c
index 1c5d0675b43c..12f13acee1f6 100644
--- a/arch/powerpc/platforms/powernv/idle.c
+++ b/arch/powerpc/platforms/powernv/idle.c
@@ -177,11 +177,6 @@ static void pnv_alloc_idle_core_states(void)
177 paca_ptrs[cpu]->core_idle_state_ptr = core_idle_state; 177 paca_ptrs[cpu]->core_idle_state_ptr = core_idle_state;
178 paca_ptrs[cpu]->thread_idle_state = PNV_THREAD_RUNNING; 178 paca_ptrs[cpu]->thread_idle_state = PNV_THREAD_RUNNING;
179 paca_ptrs[cpu]->thread_mask = 1 << j; 179 paca_ptrs[cpu]->thread_mask = 1 << j;
180 if (!cpu_has_feature(CPU_FTR_POWER9_DD1))
181 continue;
182 paca_ptrs[cpu]->thread_sibling_pacas =
183 kmalloc_node(paca_ptr_array_size,
184 GFP_KERNEL, node);
185 } 180 }
186 } 181 }
187 182
@@ -805,29 +800,6 @@ static int __init pnv_init_idle_states(void)
805 800
806 pnv_alloc_idle_core_states(); 801 pnv_alloc_idle_core_states();
807 802
808 /*
809 * For each CPU, record its PACA address in each of it's
810 * sibling thread's PACA at the slot corresponding to this
811 * CPU's index in the core.
812 */
813 if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
814 int cpu;
815
816 pr_info("powernv: idle: Saving PACA pointers of all CPUs in their thread sibling PACA\n");
817 for_each_present_cpu(cpu) {
818 int base_cpu = cpu_first_thread_sibling(cpu);
819 int idx = cpu_thread_in_core(cpu);
820 int i;
821
822 for (i = 0; i < threads_per_core; i++) {
823 int j = base_cpu + i;
824
825 paca_ptrs[j]->thread_sibling_pacas[idx] =
826 paca_ptrs[cpu];
827 }
828 }
829 }
830
831 if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED) 803 if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED)
832 ppc_md.power_save = power7_idle; 804 ppc_md.power_save = power7_idle;
833 805
diff --git a/arch/powerpc/platforms/powernv/pci-ioda-tce.c b/arch/powerpc/platforms/powernv/pci-ioda-tce.c
new file mode 100644
index 000000000000..6c5db1acbe8d
--- /dev/null
+++ b/arch/powerpc/platforms/powernv/pci-ioda-tce.c
@@ -0,0 +1,399 @@
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * TCE helpers for IODA PCI/PCIe on PowerNV platforms
4 *
5 * Copyright 2018 IBM Corp.
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 */
12
13#include <linux/kernel.h>
14#include <linux/iommu.h>
15
16#include <asm/iommu.h>
17#include <asm/tce.h>
18#include "pci.h"
19
20void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
21 void *tce_mem, u64 tce_size,
22 u64 dma_offset, unsigned int page_shift)
23{
24 tbl->it_blocksize = 16;
25 tbl->it_base = (unsigned long)tce_mem;
26 tbl->it_page_shift = page_shift;
27 tbl->it_offset = dma_offset >> tbl->it_page_shift;
28 tbl->it_index = 0;
29 tbl->it_size = tce_size >> 3;
30 tbl->it_busno = 0;
31 tbl->it_type = TCE_PCI;
32}
33
34static __be64 *pnv_alloc_tce_level(int nid, unsigned int shift)
35{
36 struct page *tce_mem = NULL;
37 __be64 *addr;
38
39 tce_mem = alloc_pages_node(nid, GFP_KERNEL, shift - PAGE_SHIFT);
40 if (!tce_mem) {
41 pr_err("Failed to allocate a TCE memory, level shift=%d\n",
42 shift);
43 return NULL;
44 }
45 addr = page_address(tce_mem);
46 memset(addr, 0, 1UL << shift);
47
48 return addr;
49}
50
51static __be64 *pnv_tce(struct iommu_table *tbl, bool user, long idx, bool alloc)
52{
53 __be64 *tmp = user ? tbl->it_userspace : (__be64 *) tbl->it_base;
54 int level = tbl->it_indirect_levels;
55 const long shift = ilog2(tbl->it_level_size);
56 unsigned long mask = (tbl->it_level_size - 1) << (level * shift);
57
58 while (level) {
59 int n = (idx & mask) >> (level * shift);
60 unsigned long tce;
61
62 if (tmp[n] == 0) {
63 __be64 *tmp2;
64
65 if (!alloc)
66 return NULL;
67
68 tmp2 = pnv_alloc_tce_level(tbl->it_nid,
69 ilog2(tbl->it_level_size) + 3);
70 if (!tmp2)
71 return NULL;
72
73 tmp[n] = cpu_to_be64(__pa(tmp2) |
74 TCE_PCI_READ | TCE_PCI_WRITE);
75 }
76 tce = be64_to_cpu(tmp[n]);
77
78 tmp = __va(tce & ~(TCE_PCI_READ | TCE_PCI_WRITE));
79 idx &= ~mask;
80 mask >>= shift;
81 --level;
82 }
83
84 return tmp + idx;
85}
86
87int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
88 unsigned long uaddr, enum dma_data_direction direction,
89 unsigned long attrs)
90{
91 u64 proto_tce = iommu_direction_to_tce_perm(direction);
92 u64 rpn = __pa(uaddr) >> tbl->it_page_shift;
93 long i;
94
95 if (proto_tce & TCE_PCI_WRITE)
96 proto_tce |= TCE_PCI_READ;
97
98 for (i = 0; i < npages; i++) {
99 unsigned long newtce = proto_tce |
100 ((rpn + i) << tbl->it_page_shift);
101 unsigned long idx = index - tbl->it_offset + i;
102
103 *(pnv_tce(tbl, false, idx, true)) = cpu_to_be64(newtce);
104 }
105
106 return 0;
107}
108
109#ifdef CONFIG_IOMMU_API
110int pnv_tce_xchg(struct iommu_table *tbl, long index,
111 unsigned long *hpa, enum dma_data_direction *direction,
112 bool alloc)
113{
114 u64 proto_tce = iommu_direction_to_tce_perm(*direction);
115 unsigned long newtce = *hpa | proto_tce, oldtce;
116 unsigned long idx = index - tbl->it_offset;
117 __be64 *ptce = NULL;
118
119 BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl));
120
121 if (*direction == DMA_NONE) {
122 ptce = pnv_tce(tbl, false, idx, false);
123 if (!ptce) {
124 *hpa = 0;
125 return 0;
126 }
127 }
128
129 if (!ptce) {
130 ptce = pnv_tce(tbl, false, idx, alloc);
131 if (!ptce)
132 return alloc ? H_HARDWARE : H_TOO_HARD;
133 }
134
135 if (newtce & TCE_PCI_WRITE)
136 newtce |= TCE_PCI_READ;
137
138 oldtce = be64_to_cpu(xchg(ptce, cpu_to_be64(newtce)));
139 *hpa = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE);
140 *direction = iommu_tce_direction(oldtce);
141
142 return 0;
143}
144
145__be64 *pnv_tce_useraddrptr(struct iommu_table *tbl, long index, bool alloc)
146{
147 if (WARN_ON_ONCE(!tbl->it_userspace))
148 return NULL;
149
150 return pnv_tce(tbl, true, index - tbl->it_offset, alloc);
151}
152#endif
153
154void pnv_tce_free(struct iommu_table *tbl, long index, long npages)
155{
156 long i;
157
158 for (i = 0; i < npages; i++) {
159 unsigned long idx = index - tbl->it_offset + i;
160 __be64 *ptce = pnv_tce(tbl, false, idx, false);
161
162 if (ptce)
163 *ptce = cpu_to_be64(0);
164 }
165}
166
167unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
168{
169 __be64 *ptce = pnv_tce(tbl, false, index - tbl->it_offset, false);
170
171 if (!ptce)
172 return 0;
173
174 return be64_to_cpu(*ptce);
175}
176
177static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr,
178 unsigned long size, unsigned int levels)
179{
180 const unsigned long addr_ul = (unsigned long) addr &
181 ~(TCE_PCI_READ | TCE_PCI_WRITE);
182
183 if (levels) {
184 long i;
185 u64 *tmp = (u64 *) addr_ul;
186
187 for (i = 0; i < size; ++i) {
188 unsigned long hpa = be64_to_cpu(tmp[i]);
189
190 if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE)))
191 continue;
192
193 pnv_pci_ioda2_table_do_free_pages(__va(hpa), size,
194 levels - 1);
195 }
196 }
197
198 free_pages(addr_ul, get_order(size << 3));
199}
200
201void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl)
202{
203 const unsigned long size = tbl->it_indirect_levels ?
204 tbl->it_level_size : tbl->it_size;
205
206 if (!tbl->it_size)
207 return;
208
209 pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size,
210 tbl->it_indirect_levels);
211 if (tbl->it_userspace) {
212 pnv_pci_ioda2_table_do_free_pages(tbl->it_userspace, size,
213 tbl->it_indirect_levels);
214 }
215}
216
217static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned int shift,
218 unsigned int levels, unsigned long limit,
219 unsigned long *current_offset, unsigned long *total_allocated)
220{
221 __be64 *addr, *tmp;
222 unsigned long allocated = 1UL << shift;
223 unsigned int entries = 1UL << (shift - 3);
224 long i;
225
226 addr = pnv_alloc_tce_level(nid, shift);
227 *total_allocated += allocated;
228
229 --levels;
230 if (!levels) {
231 *current_offset += allocated;
232 return addr;
233 }
234
235 for (i = 0; i < entries; ++i) {
236 tmp = pnv_pci_ioda2_table_do_alloc_pages(nid, shift,
237 levels, limit, current_offset, total_allocated);
238 if (!tmp)
239 break;
240
241 addr[i] = cpu_to_be64(__pa(tmp) |
242 TCE_PCI_READ | TCE_PCI_WRITE);
243
244 if (*current_offset >= limit)
245 break;
246 }
247
248 return addr;
249}
250
251long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
252 __u32 page_shift, __u64 window_size, __u32 levels,
253 bool alloc_userspace_copy, struct iommu_table *tbl)
254{
255 void *addr, *uas = NULL;
256 unsigned long offset = 0, level_shift, total_allocated = 0;
257 unsigned long total_allocated_uas = 0;
258 const unsigned int window_shift = ilog2(window_size);
259 unsigned int entries_shift = window_shift - page_shift;
260 unsigned int table_shift = max_t(unsigned int, entries_shift + 3,
261 PAGE_SHIFT);
262 const unsigned long tce_table_size = 1UL << table_shift;
263 unsigned int tmplevels = levels;
264
265 if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS))
266 return -EINVAL;
267
268 if (!is_power_of_2(window_size))
269 return -EINVAL;
270
271 if (alloc_userspace_copy && (window_size > (1ULL << 32)))
272 tmplevels = 1;
273
274 /* Adjust direct table size from window_size and levels */
275 entries_shift = (entries_shift + levels - 1) / levels;
276 level_shift = entries_shift + 3;
277 level_shift = max_t(unsigned int, level_shift, PAGE_SHIFT);
278
279 if ((level_shift - 3) * levels + page_shift >= 60)
280 return -EINVAL;
281
282 /* Allocate TCE table */
283 addr = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift,
284 tmplevels, tce_table_size, &offset, &total_allocated);
285
286 /* addr==NULL means that the first level allocation failed */
287 if (!addr)
288 return -ENOMEM;
289
290 /*
291 * First level was allocated but some lower level failed as
292 * we did not allocate as much as we wanted,
293 * release partially allocated table.
294 */
295 if (tmplevels == levels && offset < tce_table_size)
296 goto free_tces_exit;
297
298 /* Allocate userspace view of the TCE table */
299 if (alloc_userspace_copy) {
300 offset = 0;
301 uas = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift,
302 levels, tce_table_size, &offset,
303 &total_allocated_uas);
304 if (!uas)
305 goto free_tces_exit;
306 if (tmplevels == levels && (offset < tce_table_size ||
307 total_allocated_uas != total_allocated))
308 goto free_uas_exit;
309 }
310
311 /* Setup linux iommu table */
312 pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset,
313 page_shift);
314 tbl->it_level_size = 1ULL << (level_shift - 3);
315 tbl->it_indirect_levels = levels - 1;
316 tbl->it_allocated_size = total_allocated;
317 tbl->it_userspace = uas;
318 tbl->it_nid = nid;
319
320 pr_debug("Created TCE table: ws=%08llx ts=%lx @%08llx base=%lx uas=%p levels=%d/%d\n",
321 window_size, tce_table_size, bus_offset, tbl->it_base,
322 tbl->it_userspace, tmplevels, levels);
323
324 return 0;
325
326free_uas_exit:
327 pnv_pci_ioda2_table_do_free_pages(uas,
328 1ULL << (level_shift - 3), levels - 1);
329free_tces_exit:
330 pnv_pci_ioda2_table_do_free_pages(addr,
331 1ULL << (level_shift - 3), levels - 1);
332
333 return -ENOMEM;
334}
335
336static void pnv_iommu_table_group_link_free(struct rcu_head *head)
337{
338 struct iommu_table_group_link *tgl = container_of(head,
339 struct iommu_table_group_link, rcu);
340
341 kfree(tgl);
342}
343
344void pnv_pci_unlink_table_and_group(struct iommu_table *tbl,
345 struct iommu_table_group *table_group)
346{
347 long i;
348 bool found;
349 struct iommu_table_group_link *tgl;
350
351 if (!tbl || !table_group)
352 return;
353
354 /* Remove link to a group from table's list of attached groups */
355 found = false;
356 list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
357 if (tgl->table_group == table_group) {
358 list_del_rcu(&tgl->next);
359 call_rcu(&tgl->rcu, pnv_iommu_table_group_link_free);
360 found = true;
361 break;
362 }
363 }
364 if (WARN_ON(!found))
365 return;
366
367 /* Clean a pointer to iommu_table in iommu_table_group::tables[] */
368 found = false;
369 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
370 if (table_group->tables[i] == tbl) {
371 table_group->tables[i] = NULL;
372 found = true;
373 break;
374 }
375 }
376 WARN_ON(!found);
377}
378
379long pnv_pci_link_table_and_group(int node, int num,
380 struct iommu_table *tbl,
381 struct iommu_table_group *table_group)
382{
383 struct iommu_table_group_link *tgl = NULL;
384
385 if (WARN_ON(!tbl || !table_group))
386 return -EINVAL;
387
388 tgl = kzalloc_node(sizeof(struct iommu_table_group_link), GFP_KERNEL,
389 node);
390 if (!tgl)
391 return -ENOMEM;
392
393 tgl->table_group = table_group;
394 list_add_rcu(&tgl->next, &tbl->it_group_list);
395
396 table_group->tables[num] = tbl;
397
398 return 0;
399}
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index e7f2bee89fdd..cb5d8ee23acb 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -51,12 +51,8 @@
51#define PNV_IODA1_M64_SEGS 8 /* Segments per M64 BAR */ 51#define PNV_IODA1_M64_SEGS 8 /* Segments per M64 BAR */
52#define PNV_IODA1_DMA32_SEGSIZE 0x10000000 52#define PNV_IODA1_DMA32_SEGSIZE 0x10000000
53 53
54#define POWERNV_IOMMU_DEFAULT_LEVELS 1
55#define POWERNV_IOMMU_MAX_LEVELS 5
56
57static const char * const pnv_phb_names[] = { "IODA1", "IODA2", "NPU_NVLINK", 54static const char * const pnv_phb_names[] = { "IODA1", "IODA2", "NPU_NVLINK",
58 "NPU_OCAPI" }; 55 "NPU_OCAPI" };
59static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl);
60 56
61void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level, 57void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
62 const char *fmt, ...) 58 const char *fmt, ...)
@@ -2007,7 +2003,7 @@ static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index,
2007static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index, 2003static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index,
2008 unsigned long *hpa, enum dma_data_direction *direction) 2004 unsigned long *hpa, enum dma_data_direction *direction)
2009{ 2005{
2010 long ret = pnv_tce_xchg(tbl, index, hpa, direction); 2006 long ret = pnv_tce_xchg(tbl, index, hpa, direction, true);
2011 2007
2012 if (!ret) 2008 if (!ret)
2013 pnv_pci_p7ioc_tce_invalidate(tbl, index, 1, false); 2009 pnv_pci_p7ioc_tce_invalidate(tbl, index, 1, false);
@@ -2018,7 +2014,7 @@ static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index,
2018static int pnv_ioda1_tce_xchg_rm(struct iommu_table *tbl, long index, 2014static int pnv_ioda1_tce_xchg_rm(struct iommu_table *tbl, long index,
2019 unsigned long *hpa, enum dma_data_direction *direction) 2015 unsigned long *hpa, enum dma_data_direction *direction)
2020{ 2016{
2021 long ret = pnv_tce_xchg(tbl, index, hpa, direction); 2017 long ret = pnv_tce_xchg(tbl, index, hpa, direction, false);
2022 2018
2023 if (!ret) 2019 if (!ret)
2024 pnv_pci_p7ioc_tce_invalidate(tbl, index, 1, true); 2020 pnv_pci_p7ioc_tce_invalidate(tbl, index, 1, true);
@@ -2040,6 +2036,7 @@ static struct iommu_table_ops pnv_ioda1_iommu_ops = {
2040#ifdef CONFIG_IOMMU_API 2036#ifdef CONFIG_IOMMU_API
2041 .exchange = pnv_ioda1_tce_xchg, 2037 .exchange = pnv_ioda1_tce_xchg,
2042 .exchange_rm = pnv_ioda1_tce_xchg_rm, 2038 .exchange_rm = pnv_ioda1_tce_xchg_rm,
2039 .useraddrptr = pnv_tce_useraddrptr,
2043#endif 2040#endif
2044 .clear = pnv_ioda1_tce_free, 2041 .clear = pnv_ioda1_tce_free,
2045 .get = pnv_tce_get, 2042 .get = pnv_tce_get,
@@ -2171,7 +2168,7 @@ static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,
2171static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index, 2168static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index,
2172 unsigned long *hpa, enum dma_data_direction *direction) 2169 unsigned long *hpa, enum dma_data_direction *direction)
2173{ 2170{
2174 long ret = pnv_tce_xchg(tbl, index, hpa, direction); 2171 long ret = pnv_tce_xchg(tbl, index, hpa, direction, true);
2175 2172
2176 if (!ret) 2173 if (!ret)
2177 pnv_pci_ioda2_tce_invalidate(tbl, index, 1, false); 2174 pnv_pci_ioda2_tce_invalidate(tbl, index, 1, false);
@@ -2182,7 +2179,7 @@ static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index,
2182static int pnv_ioda2_tce_xchg_rm(struct iommu_table *tbl, long index, 2179static int pnv_ioda2_tce_xchg_rm(struct iommu_table *tbl, long index,
2183 unsigned long *hpa, enum dma_data_direction *direction) 2180 unsigned long *hpa, enum dma_data_direction *direction)
2184{ 2181{
2185 long ret = pnv_tce_xchg(tbl, index, hpa, direction); 2182 long ret = pnv_tce_xchg(tbl, index, hpa, direction, false);
2186 2183
2187 if (!ret) 2184 if (!ret)
2188 pnv_pci_ioda2_tce_invalidate(tbl, index, 1, true); 2185 pnv_pci_ioda2_tce_invalidate(tbl, index, 1, true);
@@ -2199,20 +2196,16 @@ static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
2199 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false); 2196 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
2200} 2197}
2201 2198
2202static void pnv_ioda2_table_free(struct iommu_table *tbl)
2203{
2204 pnv_pci_ioda2_table_free_pages(tbl);
2205}
2206
2207static struct iommu_table_ops pnv_ioda2_iommu_ops = { 2199static struct iommu_table_ops pnv_ioda2_iommu_ops = {
2208 .set = pnv_ioda2_tce_build, 2200 .set = pnv_ioda2_tce_build,
2209#ifdef CONFIG_IOMMU_API 2201#ifdef CONFIG_IOMMU_API
2210 .exchange = pnv_ioda2_tce_xchg, 2202 .exchange = pnv_ioda2_tce_xchg,
2211 .exchange_rm = pnv_ioda2_tce_xchg_rm, 2203 .exchange_rm = pnv_ioda2_tce_xchg_rm,
2204 .useraddrptr = pnv_tce_useraddrptr,
2212#endif 2205#endif
2213 .clear = pnv_ioda2_tce_free, 2206 .clear = pnv_ioda2_tce_free,
2214 .get = pnv_tce_get, 2207 .get = pnv_tce_get,
2215 .free = pnv_ioda2_table_free, 2208 .free = pnv_pci_ioda2_table_free_pages,
2216}; 2209};
2217 2210
2218static int pnv_pci_ioda_dev_dma_weight(struct pci_dev *dev, void *data) 2211static int pnv_pci_ioda_dev_dma_weight(struct pci_dev *dev, void *data)
@@ -2462,13 +2455,9 @@ void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable)
2462 pe->tce_bypass_enabled = enable; 2455 pe->tce_bypass_enabled = enable;
2463} 2456}
2464 2457
2465static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
2466 __u32 page_shift, __u64 window_size, __u32 levels,
2467 struct iommu_table *tbl);
2468
2469static long pnv_pci_ioda2_create_table(struct iommu_table_group *table_group, 2458static long pnv_pci_ioda2_create_table(struct iommu_table_group *table_group,
2470 int num, __u32 page_shift, __u64 window_size, __u32 levels, 2459 int num, __u32 page_shift, __u64 window_size, __u32 levels,
2471 struct iommu_table **ptbl) 2460 bool alloc_userspace_copy, struct iommu_table **ptbl)
2472{ 2461{
2473 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe, 2462 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2474 table_group); 2463 table_group);
@@ -2485,7 +2474,7 @@ static long pnv_pci_ioda2_create_table(struct iommu_table_group *table_group,
2485 2474
2486 ret = pnv_pci_ioda2_table_alloc_pages(nid, 2475 ret = pnv_pci_ioda2_table_alloc_pages(nid,
2487 bus_offset, page_shift, window_size, 2476 bus_offset, page_shift, window_size,
2488 levels, tbl); 2477 levels, alloc_userspace_copy, tbl);
2489 if (ret) { 2478 if (ret) {
2490 iommu_tce_table_put(tbl); 2479 iommu_tce_table_put(tbl);
2491 return ret; 2480 return ret;
@@ -2518,7 +2507,7 @@ static long pnv_pci_ioda2_setup_default_config(struct pnv_ioda_pe *pe)
2518 rc = pnv_pci_ioda2_create_table(&pe->table_group, 0, 2507 rc = pnv_pci_ioda2_create_table(&pe->table_group, 0,
2519 IOMMU_PAGE_SHIFT_4K, 2508 IOMMU_PAGE_SHIFT_4K,
2520 window_size, 2509 window_size,
2521 POWERNV_IOMMU_DEFAULT_LEVELS, &tbl); 2510 POWERNV_IOMMU_DEFAULT_LEVELS, false, &tbl);
2522 if (rc) { 2511 if (rc) {
2523 pe_err(pe, "Failed to create 32-bit TCE table, err %ld", 2512 pe_err(pe, "Failed to create 32-bit TCE table, err %ld",
2524 rc); 2513 rc);
@@ -2605,7 +2594,16 @@ static unsigned long pnv_pci_ioda2_get_table_size(__u32 page_shift,
2605 tce_table_size, direct_table_size); 2594 tce_table_size, direct_table_size);
2606 } 2595 }
2607 2596
2608 return bytes; 2597 return bytes + bytes; /* one for HW table, one for userspace copy */
2598}
2599
2600static long pnv_pci_ioda2_create_table_userspace(
2601 struct iommu_table_group *table_group,
2602 int num, __u32 page_shift, __u64 window_size, __u32 levels,
2603 struct iommu_table **ptbl)
2604{
2605 return pnv_pci_ioda2_create_table(table_group,
2606 num, page_shift, window_size, levels, true, ptbl);
2609} 2607}
2610 2608
2611static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group) 2609static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
@@ -2634,7 +2632,7 @@ static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
2634 2632
2635static struct iommu_table_group_ops pnv_pci_ioda2_ops = { 2633static struct iommu_table_group_ops pnv_pci_ioda2_ops = {
2636 .get_table_size = pnv_pci_ioda2_get_table_size, 2634 .get_table_size = pnv_pci_ioda2_get_table_size,
2637 .create_table = pnv_pci_ioda2_create_table, 2635 .create_table = pnv_pci_ioda2_create_table_userspace,
2638 .set_window = pnv_pci_ioda2_set_window, 2636 .set_window = pnv_pci_ioda2_set_window,
2639 .unset_window = pnv_pci_ioda2_unset_window, 2637 .unset_window = pnv_pci_ioda2_unset_window,
2640 .take_ownership = pnv_ioda2_take_ownership, 2638 .take_ownership = pnv_ioda2_take_ownership,
@@ -2739,7 +2737,7 @@ static void pnv_ioda2_npu_take_ownership(struct iommu_table_group *table_group)
2739 2737
2740static struct iommu_table_group_ops pnv_pci_ioda2_npu_ops = { 2738static struct iommu_table_group_ops pnv_pci_ioda2_npu_ops = {
2741 .get_table_size = pnv_pci_ioda2_get_table_size, 2739 .get_table_size = pnv_pci_ioda2_get_table_size,
2742 .create_table = pnv_pci_ioda2_create_table, 2740 .create_table = pnv_pci_ioda2_create_table_userspace,
2743 .set_window = pnv_pci_ioda2_npu_set_window, 2741 .set_window = pnv_pci_ioda2_npu_set_window,
2744 .unset_window = pnv_pci_ioda2_npu_unset_window, 2742 .unset_window = pnv_pci_ioda2_npu_unset_window,
2745 .take_ownership = pnv_ioda2_npu_take_ownership, 2743 .take_ownership = pnv_ioda2_npu_take_ownership,
@@ -2773,144 +2771,6 @@ static void pnv_pci_ioda_setup_iommu_api(void)
2773static void pnv_pci_ioda_setup_iommu_api(void) { }; 2771static void pnv_pci_ioda_setup_iommu_api(void) { };
2774#endif 2772#endif
2775 2773
2776static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned shift,
2777 unsigned levels, unsigned long limit,
2778 unsigned long *current_offset, unsigned long *total_allocated)
2779{
2780 struct page *tce_mem = NULL;
2781 __be64 *addr, *tmp;
2782 unsigned order = max_t(unsigned, shift, PAGE_SHIFT) - PAGE_SHIFT;
2783 unsigned long allocated = 1UL << (order + PAGE_SHIFT);
2784 unsigned entries = 1UL << (shift - 3);
2785 long i;
2786
2787 tce_mem = alloc_pages_node(nid, GFP_KERNEL, order);
2788 if (!tce_mem) {
2789 pr_err("Failed to allocate a TCE memory, order=%d\n", order);
2790 return NULL;
2791 }
2792 addr = page_address(tce_mem);
2793 memset(addr, 0, allocated);
2794 *total_allocated += allocated;
2795
2796 --levels;
2797 if (!levels) {
2798 *current_offset += allocated;
2799 return addr;
2800 }
2801
2802 for (i = 0; i < entries; ++i) {
2803 tmp = pnv_pci_ioda2_table_do_alloc_pages(nid, shift,
2804 levels, limit, current_offset, total_allocated);
2805 if (!tmp)
2806 break;
2807
2808 addr[i] = cpu_to_be64(__pa(tmp) |
2809 TCE_PCI_READ | TCE_PCI_WRITE);
2810
2811 if (*current_offset >= limit)
2812 break;
2813 }
2814
2815 return addr;
2816}
2817
2818static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr,
2819 unsigned long size, unsigned level);
2820
2821static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
2822 __u32 page_shift, __u64 window_size, __u32 levels,
2823 struct iommu_table *tbl)
2824{
2825 void *addr;
2826 unsigned long offset = 0, level_shift, total_allocated = 0;
2827 const unsigned window_shift = ilog2(window_size);
2828 unsigned entries_shift = window_shift - page_shift;
2829 unsigned table_shift = max_t(unsigned, entries_shift + 3, PAGE_SHIFT);
2830 const unsigned long tce_table_size = 1UL << table_shift;
2831
2832 if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS))
2833 return -EINVAL;
2834
2835 if (!is_power_of_2(window_size))
2836 return -EINVAL;
2837
2838 /* Adjust direct table size from window_size and levels */
2839 entries_shift = (entries_shift + levels - 1) / levels;
2840 level_shift = entries_shift + 3;
2841 level_shift = max_t(unsigned, level_shift, PAGE_SHIFT);
2842
2843 if ((level_shift - 3) * levels + page_shift >= 55)
2844 return -EINVAL;
2845
2846 /* Allocate TCE table */
2847 addr = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift,
2848 levels, tce_table_size, &offset, &total_allocated);
2849
2850 /* addr==NULL means that the first level allocation failed */
2851 if (!addr)
2852 return -ENOMEM;
2853
2854 /*
2855 * First level was allocated but some lower level failed as
2856 * we did not allocate as much as we wanted,
2857 * release partially allocated table.
2858 */
2859 if (offset < tce_table_size) {
2860 pnv_pci_ioda2_table_do_free_pages(addr,
2861 1ULL << (level_shift - 3), levels - 1);
2862 return -ENOMEM;
2863 }
2864
2865 /* Setup linux iommu table */
2866 pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset,
2867 page_shift);
2868 tbl->it_level_size = 1ULL << (level_shift - 3);
2869 tbl->it_indirect_levels = levels - 1;
2870 tbl->it_allocated_size = total_allocated;
2871
2872 pr_devel("Created TCE table: ws=%08llx ts=%lx @%08llx\n",
2873 window_size, tce_table_size, bus_offset);
2874
2875 return 0;
2876}
2877
2878static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr,
2879 unsigned long size, unsigned level)
2880{
2881 const unsigned long addr_ul = (unsigned long) addr &
2882 ~(TCE_PCI_READ | TCE_PCI_WRITE);
2883
2884 if (level) {
2885 long i;
2886 u64 *tmp = (u64 *) addr_ul;
2887
2888 for (i = 0; i < size; ++i) {
2889 unsigned long hpa = be64_to_cpu(tmp[i]);
2890
2891 if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE)))
2892 continue;
2893
2894 pnv_pci_ioda2_table_do_free_pages(__va(hpa), size,
2895 level - 1);
2896 }
2897 }
2898
2899 free_pages(addr_ul, get_order(size << 3));
2900}
2901
2902static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl)
2903{
2904 const unsigned long size = tbl->it_indirect_levels ?
2905 tbl->it_level_size : tbl->it_size;
2906
2907 if (!tbl->it_size)
2908 return;
2909
2910 pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size,
2911 tbl->it_indirect_levels);
2912}
2913
2914static unsigned long pnv_ioda_parse_tce_sizes(struct pnv_phb *phb) 2774static unsigned long pnv_ioda_parse_tce_sizes(struct pnv_phb *phb)
2915{ 2775{
2916 struct pci_controller *hose = phb->hose; 2776 struct pci_controller *hose = phb->hose;
diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c
index b265ecc0836a..13aef2323bbc 100644
--- a/arch/powerpc/platforms/powernv/pci.c
+++ b/arch/powerpc/platforms/powernv/pci.c
@@ -802,85 +802,6 @@ struct pci_ops pnv_pci_ops = {
802 .write = pnv_pci_write_config, 802 .write = pnv_pci_write_config,
803}; 803};
804 804
805static __be64 *pnv_tce(struct iommu_table *tbl, long idx)
806{
807 __be64 *tmp = ((__be64 *)tbl->it_base);
808 int level = tbl->it_indirect_levels;
809 const long shift = ilog2(tbl->it_level_size);
810 unsigned long mask = (tbl->it_level_size - 1) << (level * shift);
811
812 while (level) {
813 int n = (idx & mask) >> (level * shift);
814 unsigned long tce = be64_to_cpu(tmp[n]);
815
816 tmp = __va(tce & ~(TCE_PCI_READ | TCE_PCI_WRITE));
817 idx &= ~mask;
818 mask >>= shift;
819 --level;
820 }
821
822 return tmp + idx;
823}
824
825int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
826 unsigned long uaddr, enum dma_data_direction direction,
827 unsigned long attrs)
828{
829 u64 proto_tce = iommu_direction_to_tce_perm(direction);
830 u64 rpn = __pa(uaddr) >> tbl->it_page_shift;
831 long i;
832
833 if (proto_tce & TCE_PCI_WRITE)
834 proto_tce |= TCE_PCI_READ;
835
836 for (i = 0; i < npages; i++) {
837 unsigned long newtce = proto_tce |
838 ((rpn + i) << tbl->it_page_shift);
839 unsigned long idx = index - tbl->it_offset + i;
840
841 *(pnv_tce(tbl, idx)) = cpu_to_be64(newtce);
842 }
843
844 return 0;
845}
846
847#ifdef CONFIG_IOMMU_API
848int pnv_tce_xchg(struct iommu_table *tbl, long index,
849 unsigned long *hpa, enum dma_data_direction *direction)
850{
851 u64 proto_tce = iommu_direction_to_tce_perm(*direction);
852 unsigned long newtce = *hpa | proto_tce, oldtce;
853 unsigned long idx = index - tbl->it_offset;
854
855 BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl));
856
857 if (newtce & TCE_PCI_WRITE)
858 newtce |= TCE_PCI_READ;
859
860 oldtce = be64_to_cpu(xchg(pnv_tce(tbl, idx), cpu_to_be64(newtce)));
861 *hpa = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE);
862 *direction = iommu_tce_direction(oldtce);
863
864 return 0;
865}
866#endif
867
868void pnv_tce_free(struct iommu_table *tbl, long index, long npages)
869{
870 long i;
871
872 for (i = 0; i < npages; i++) {
873 unsigned long idx = index - tbl->it_offset + i;
874
875 *(pnv_tce(tbl, idx)) = cpu_to_be64(0);
876 }
877}
878
879unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
880{
881 return be64_to_cpu(*(pnv_tce(tbl, index - tbl->it_offset)));
882}
883
884struct iommu_table *pnv_pci_table_alloc(int nid) 805struct iommu_table *pnv_pci_table_alloc(int nid)
885{ 806{
886 struct iommu_table *tbl; 807 struct iommu_table *tbl;
@@ -895,85 +816,6 @@ struct iommu_table *pnv_pci_table_alloc(int nid)
895 return tbl; 816 return tbl;
896} 817}
897 818
898long pnv_pci_link_table_and_group(int node, int num,
899 struct iommu_table *tbl,
900 struct iommu_table_group *table_group)
901{
902 struct iommu_table_group_link *tgl = NULL;
903
904 if (WARN_ON(!tbl || !table_group))
905 return -EINVAL;
906
907 tgl = kzalloc_node(sizeof(struct iommu_table_group_link), GFP_KERNEL,
908 node);
909 if (!tgl)
910 return -ENOMEM;
911
912 tgl->table_group = table_group;
913 list_add_rcu(&tgl->next, &tbl->it_group_list);
914
915 table_group->tables[num] = tbl;
916
917 return 0;
918}
919
920static void pnv_iommu_table_group_link_free(struct rcu_head *head)
921{
922 struct iommu_table_group_link *tgl = container_of(head,
923 struct iommu_table_group_link, rcu);
924
925 kfree(tgl);
926}
927
928void pnv_pci_unlink_table_and_group(struct iommu_table *tbl,
929 struct iommu_table_group *table_group)
930{
931 long i;
932 bool found;
933 struct iommu_table_group_link *tgl;
934
935 if (!tbl || !table_group)
936 return;
937
938 /* Remove link to a group from table's list of attached groups */
939 found = false;
940 list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
941 if (tgl->table_group == table_group) {
942 list_del_rcu(&tgl->next);
943 call_rcu(&tgl->rcu, pnv_iommu_table_group_link_free);
944 found = true;
945 break;
946 }
947 }
948 if (WARN_ON(!found))
949 return;
950
951 /* Clean a pointer to iommu_table in iommu_table_group::tables[] */
952 found = false;
953 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
954 if (table_group->tables[i] == tbl) {
955 table_group->tables[i] = NULL;
956 found = true;
957 break;
958 }
959 }
960 WARN_ON(!found);
961}
962
963void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
964 void *tce_mem, u64 tce_size,
965 u64 dma_offset, unsigned page_shift)
966{
967 tbl->it_blocksize = 16;
968 tbl->it_base = (unsigned long)tce_mem;
969 tbl->it_page_shift = page_shift;
970 tbl->it_offset = dma_offset >> tbl->it_page_shift;
971 tbl->it_index = 0;
972 tbl->it_size = tce_size >> 3;
973 tbl->it_busno = 0;
974 tbl->it_type = TCE_PCI;
975}
976
977void pnv_pci_dma_dev_setup(struct pci_dev *pdev) 819void pnv_pci_dma_dev_setup(struct pci_dev *pdev)
978{ 820{
979 struct pci_controller *hose = pci_bus_to_host(pdev->bus); 821 struct pci_controller *hose = pci_bus_to_host(pdev->bus);
diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
index 44dfbc37f547..8b37b28e3831 100644
--- a/arch/powerpc/platforms/powernv/pci.h
+++ b/arch/powerpc/platforms/powernv/pci.h
@@ -197,13 +197,6 @@ struct pnv_phb {
197}; 197};
198 198
199extern struct pci_ops pnv_pci_ops; 199extern struct pci_ops pnv_pci_ops;
200extern int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
201 unsigned long uaddr, enum dma_data_direction direction,
202 unsigned long attrs);
203extern void pnv_tce_free(struct iommu_table *tbl, long index, long npages);
204extern int pnv_tce_xchg(struct iommu_table *tbl, long index,
205 unsigned long *hpa, enum dma_data_direction *direction);
206extern unsigned long pnv_tce_get(struct iommu_table *tbl, long index);
207 200
208void pnv_pci_dump_phb_diag_data(struct pci_controller *hose, 201void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
209 unsigned char *log_buff); 202 unsigned char *log_buff);
@@ -213,14 +206,6 @@ int pnv_pci_cfg_write(struct pci_dn *pdn,
213 int where, int size, u32 val); 206 int where, int size, u32 val);
214extern struct iommu_table *pnv_pci_table_alloc(int nid); 207extern struct iommu_table *pnv_pci_table_alloc(int nid);
215 208
216extern long pnv_pci_link_table_and_group(int node, int num,
217 struct iommu_table *tbl,
218 struct iommu_table_group *table_group);
219extern void pnv_pci_unlink_table_and_group(struct iommu_table *tbl,
220 struct iommu_table_group *table_group);
221extern void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
222 void *tce_mem, u64 tce_size,
223 u64 dma_offset, unsigned page_shift);
224extern void pnv_pci_init_ioda_hub(struct device_node *np); 209extern void pnv_pci_init_ioda_hub(struct device_node *np);
225extern void pnv_pci_init_ioda2_phb(struct device_node *np); 210extern void pnv_pci_init_ioda2_phb(struct device_node *np);
226extern void pnv_pci_init_npu_phb(struct device_node *np); 211extern void pnv_pci_init_npu_phb(struct device_node *np);
@@ -257,4 +242,33 @@ extern void pnv_npu_take_ownership(struct pnv_ioda_pe *npe);
257extern void pnv_npu_release_ownership(struct pnv_ioda_pe *npe); 242extern void pnv_npu_release_ownership(struct pnv_ioda_pe *npe);
258extern int pnv_npu2_init(struct pnv_phb *phb); 243extern int pnv_npu2_init(struct pnv_phb *phb);
259 244
245/* pci-ioda-tce.c */
246#define POWERNV_IOMMU_DEFAULT_LEVELS 1
247#define POWERNV_IOMMU_MAX_LEVELS 5
248
249extern int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
250 unsigned long uaddr, enum dma_data_direction direction,
251 unsigned long attrs);
252extern void pnv_tce_free(struct iommu_table *tbl, long index, long npages);
253extern int pnv_tce_xchg(struct iommu_table *tbl, long index,
254 unsigned long *hpa, enum dma_data_direction *direction,
255 bool alloc);
256extern __be64 *pnv_tce_useraddrptr(struct iommu_table *tbl, long index,
257 bool alloc);
258extern unsigned long pnv_tce_get(struct iommu_table *tbl, long index);
259
260extern long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
261 __u32 page_shift, __u64 window_size, __u32 levels,
262 bool alloc_userspace_copy, struct iommu_table *tbl);
263extern void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl);
264
265extern long pnv_pci_link_table_and_group(int node, int num,
266 struct iommu_table *tbl,
267 struct iommu_table_group *table_group);
268extern void pnv_pci_unlink_table_and_group(struct iommu_table *tbl,
269 struct iommu_table_group *table_group);
270extern void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
271 void *tce_mem, u64 tce_size,
272 u64 dma_offset, unsigned int page_shift);
273
260#endif /* __POWERNV_PCI_H */ 274#endif /* __POWERNV_PCI_H */
diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c
index b80909957792..0d354e19ef92 100644
--- a/arch/powerpc/platforms/powernv/smp.c
+++ b/arch/powerpc/platforms/powernv/smp.c
@@ -283,23 +283,6 @@ static void pnv_cause_ipi(int cpu)
283 ic_cause_ipi(cpu); 283 ic_cause_ipi(cpu);
284} 284}
285 285
286static void pnv_p9_dd1_cause_ipi(int cpu)
287{
288 int this_cpu = get_cpu();
289
290 /*
291 * POWER9 DD1 has a global addressed msgsnd, but for now we restrict
292 * IPIs to same core, because it requires additional synchronization
293 * for inter-core doorbells which we do not implement.
294 */
295 if (cpumask_test_cpu(cpu, cpu_sibling_mask(this_cpu)))
296 doorbell_global_ipi(cpu);
297 else
298 ic_cause_ipi(cpu);
299
300 put_cpu();
301}
302
303static void __init pnv_smp_probe(void) 286static void __init pnv_smp_probe(void)
304{ 287{
305 if (xive_enabled()) 288 if (xive_enabled())
@@ -311,14 +294,10 @@ static void __init pnv_smp_probe(void)
311 ic_cause_ipi = smp_ops->cause_ipi; 294 ic_cause_ipi = smp_ops->cause_ipi;
312 WARN_ON(!ic_cause_ipi); 295 WARN_ON(!ic_cause_ipi);
313 296
314 if (cpu_has_feature(CPU_FTR_ARCH_300)) { 297 if (cpu_has_feature(CPU_FTR_ARCH_300))
315 if (cpu_has_feature(CPU_FTR_POWER9_DD1)) 298 smp_ops->cause_ipi = doorbell_global_ipi;
316 smp_ops->cause_ipi = pnv_p9_dd1_cause_ipi; 299 else
317 else
318 smp_ops->cause_ipi = doorbell_global_ipi;
319 } else {
320 smp_ops->cause_ipi = pnv_cause_ipi; 300 smp_ops->cause_ipi = pnv_cause_ipi;
321 }
322 } 301 }
323} 302}
324 303
diff --git a/arch/powerpc/sysdev/xive/common.c b/arch/powerpc/sysdev/xive/common.c
index 3459015092fa..4758173df426 100644
--- a/arch/powerpc/sysdev/xive/common.c
+++ b/arch/powerpc/sysdev/xive/common.c
@@ -319,7 +319,7 @@ void xive_do_source_eoi(u32 hw_irq, struct xive_irq_data *xd)
319 * The FW told us to call it. This happens for some 319 * The FW told us to call it. This happens for some
320 * interrupt sources that need additional HW whacking 320 * interrupt sources that need additional HW whacking
321 * beyond the ESB manipulation. For example LPC interrupts 321 * beyond the ESB manipulation. For example LPC interrupts
322 * on P9 DD1.0 need a latch to be clared in the LPC bridge 322 * on P9 DD1.0 needed a latch to be clared in the LPC bridge
323 * itself. The Firmware will take care of it. 323 * itself. The Firmware will take care of it.
324 */ 324 */
325 if (WARN_ON_ONCE(!xive_ops->eoi)) 325 if (WARN_ON_ONCE(!xive_ops->eoi))
@@ -337,9 +337,9 @@ void xive_do_source_eoi(u32 hw_irq, struct xive_irq_data *xd)
337 * This allows us to then do a re-trigger if Q was set 337 * This allows us to then do a re-trigger if Q was set
338 * rather than synthesizing an interrupt in software 338 * rather than synthesizing an interrupt in software
339 * 339 *
340 * For LSIs, using the HW EOI cycle works around a problem 340 * For LSIs the HW EOI cycle is used rather than PQ bits,
341 * on P9 DD1 PHBs where the other ESB accesses don't work 341 * as they are automatically re-triggred in HW when still
342 * properly. 342 * pending.
343 */ 343 */
344 if (xd->flags & XIVE_IRQ_FLAG_LSI) 344 if (xd->flags & XIVE_IRQ_FLAG_LSI)
345 xive_esb_read(xd, XIVE_ESB_LOAD_EOI); 345 xive_esb_read(xd, XIVE_ESB_LOAD_EOI);
diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
index 45e3d0ec1246..b7dd683b0470 100644
--- a/arch/powerpc/xmon/xmon.c
+++ b/arch/powerpc/xmon/xmon.c
@@ -2429,7 +2429,6 @@ static void dump_one_paca(int cpu)
2429 DUMP(p, thread_idle_state, "%#-*x"); 2429 DUMP(p, thread_idle_state, "%#-*x");
2430 DUMP(p, thread_mask, "%#-*x"); 2430 DUMP(p, thread_mask, "%#-*x");
2431 DUMP(p, subcore_sibling_mask, "%#-*x"); 2431 DUMP(p, subcore_sibling_mask, "%#-*x");
2432 DUMP(p, thread_sibling_pacas, "%-*px");
2433 DUMP(p, requested_psscr, "%#-*llx"); 2432 DUMP(p, requested_psscr, "%#-*llx");
2434 DUMP(p, stop_sprs.pid, "%#-*llx"); 2433 DUMP(p, stop_sprs.pid, "%#-*llx");
2435 DUMP(p, stop_sprs.ldbar, "%#-*llx"); 2434 DUMP(p, stop_sprs.ldbar, "%#-*llx");
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index 44bcfafbb579..d1d927ccb589 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -850,14 +850,6 @@ static inline bool cxl_is_power9(void)
850 return false; 850 return false;
851} 851}
852 852
853static inline bool cxl_is_power9_dd1(void)
854{
855 if ((pvr_version_is(PVR_POWER9)) &&
856 cpu_has_feature(CPU_FTR_POWER9_DD1))
857 return true;
858 return false;
859}
860
861ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf, 853ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
862 loff_t off, size_t count); 854 loff_t off, size_t count);
863 855
diff --git a/drivers/misc/cxl/cxllib.c b/drivers/misc/cxl/cxllib.c
index 0bc7c31cf739..5a3f91255258 100644
--- a/drivers/misc/cxl/cxllib.c
+++ b/drivers/misc/cxl/cxllib.c
@@ -102,10 +102,6 @@ int cxllib_get_xsl_config(struct pci_dev *dev, struct cxllib_xsl_config *cfg)
102 rc = cxl_get_xsl9_dsnctl(dev, capp_unit_id, &cfg->dsnctl); 102 rc = cxl_get_xsl9_dsnctl(dev, capp_unit_id, &cfg->dsnctl);
103 if (rc) 103 if (rc)
104 return rc; 104 return rc;
105 if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
106 /* workaround for DD1 - nbwind = capiind */
107 cfg->dsnctl |= ((u64)0x02 << (63-47));
108 }
109 105
110 cfg->version = CXL_XSL_CONFIG_CURRENT_VERSION; 106 cfg->version = CXL_XSL_CONFIG_CURRENT_VERSION;
111 cfg->log_bar_size = CXL_CAPI_WINDOW_LOG_SIZE; 107 cfg->log_bar_size = CXL_CAPI_WINDOW_LOG_SIZE;
diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
index 6dfb4ed345d3..b66d832d3233 100644
--- a/drivers/misc/cxl/pci.c
+++ b/drivers/misc/cxl/pci.c
@@ -463,23 +463,21 @@ int cxl_get_xsl9_dsnctl(struct pci_dev *dev, u64 capp_unit_id, u64 *reg)
463 /* nMMU_ID Defaults to: b’000001001’*/ 463 /* nMMU_ID Defaults to: b’000001001’*/
464 xsl_dsnctl |= ((u64)0x09 << (63-28)); 464 xsl_dsnctl |= ((u64)0x09 << (63-28));
465 465
466 if (!(cxl_is_power9_dd1())) { 466 /*
467 /* 467 * Used to identify CAPI packets which should be sorted into
468 * Used to identify CAPI packets which should be sorted into 468 * the Non-Blocking queues by the PHB. This field should match
469 * the Non-Blocking queues by the PHB. This field should match 469 * the PHB PBL_NBW_CMPM register
470 * the PHB PBL_NBW_CMPM register 470 * nbwind=0x03, bits [57:58], must include capi indicator.
471 * nbwind=0x03, bits [57:58], must include capi indicator. 471 * Not supported on P9 DD1.
472 * Not supported on P9 DD1. 472 */
473 */ 473 xsl_dsnctl |= (nbwind << (63-55));
474 xsl_dsnctl |= (nbwind << (63-55));
475 474
476 /* 475 /*
477 * Upper 16b address bits of ASB_Notify messages sent to the 476 * Upper 16b address bits of ASB_Notify messages sent to the
478 * system. Need to match the PHB’s ASN Compare/Mask Register. 477 * system. Need to match the PHB’s ASN Compare/Mask Register.
479 * Not supported on P9 DD1. 478 * Not supported on P9 DD1.
480 */ 479 */
481 xsl_dsnctl |= asnind; 480 xsl_dsnctl |= asnind;
482 }
483 481
484 *reg = xsl_dsnctl; 482 *reg = xsl_dsnctl;
485 return 0; 483 return 0;
@@ -537,15 +535,8 @@ static int init_implementation_adapter_regs_psl9(struct cxl *adapter,
537 /* Snoop machines */ 535 /* Snoop machines */
538 cxl_p1_write(adapter, CXL_PSL9_APCDEDALLOC, 0x800F000200000000ULL); 536 cxl_p1_write(adapter, CXL_PSL9_APCDEDALLOC, 0x800F000200000000ULL);
539 537
540 if (cxl_is_power9_dd1()) { 538 /* Enable NORST and DD2 features */
541 /* Disabling deadlock counter CAR */ 539 cxl_p1_write(adapter, CXL_PSL9_DEBUG, 0xC000000000000000ULL);
542 cxl_p1_write(adapter, CXL_PSL9_GP_CT, 0x0020000000000001ULL);
543 /* Enable NORST */
544 cxl_p1_write(adapter, CXL_PSL9_DEBUG, 0x8000000000000000ULL);
545 } else {
546 /* Enable NORST and DD2 features */
547 cxl_p1_write(adapter, CXL_PSL9_DEBUG, 0xC000000000000000ULL);
548 }
549 540
550 /* 541 /*
551 * Check if PSL has data-cache. We need to flush adapter datacache 542 * Check if PSL has data-cache. We need to flush adapter datacache
diff --git a/drivers/vfio/vfio_iommu_spapr_tce.c b/drivers/vfio/vfio_iommu_spapr_tce.c
index 759a5bdd40e1..11a4c194d6e3 100644
--- a/drivers/vfio/vfio_iommu_spapr_tce.c
+++ b/drivers/vfio/vfio_iommu_spapr_tce.c
@@ -211,44 +211,6 @@ static long tce_iommu_register_pages(struct tce_container *container,
211 return 0; 211 return 0;
212} 212}
213 213
214static long tce_iommu_userspace_view_alloc(struct iommu_table *tbl,
215 struct mm_struct *mm)
216{
217 unsigned long cb = _ALIGN_UP(sizeof(tbl->it_userspace[0]) *
218 tbl->it_size, PAGE_SIZE);
219 unsigned long *uas;
220 long ret;
221
222 BUG_ON(tbl->it_userspace);
223
224 ret = try_increment_locked_vm(mm, cb >> PAGE_SHIFT);
225 if (ret)
226 return ret;
227
228 uas = vzalloc(cb);
229 if (!uas) {
230 decrement_locked_vm(mm, cb >> PAGE_SHIFT);
231 return -ENOMEM;
232 }
233 tbl->it_userspace = uas;
234
235 return 0;
236}
237
238static void tce_iommu_userspace_view_free(struct iommu_table *tbl,
239 struct mm_struct *mm)
240{
241 unsigned long cb = _ALIGN_UP(sizeof(tbl->it_userspace[0]) *
242 tbl->it_size, PAGE_SIZE);
243
244 if (!tbl->it_userspace)
245 return;
246
247 vfree(tbl->it_userspace);
248 tbl->it_userspace = NULL;
249 decrement_locked_vm(mm, cb >> PAGE_SHIFT);
250}
251
252static bool tce_page_is_contained(struct page *page, unsigned page_shift) 214static bool tce_page_is_contained(struct page *page, unsigned page_shift)
253{ 215{
254 /* 216 /*
@@ -482,20 +444,20 @@ static void tce_iommu_unuse_page_v2(struct tce_container *container,
482 struct mm_iommu_table_group_mem_t *mem = NULL; 444 struct mm_iommu_table_group_mem_t *mem = NULL;
483 int ret; 445 int ret;
484 unsigned long hpa = 0; 446 unsigned long hpa = 0;
485 unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); 447 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry);
486 448
487 if (!pua) 449 if (!pua)
488 return; 450 return;
489 451
490 ret = tce_iommu_prereg_ua_to_hpa(container, *pua, IOMMU_PAGE_SIZE(tbl), 452 ret = tce_iommu_prereg_ua_to_hpa(container, be64_to_cpu(*pua),
491 &hpa, &mem); 453 IOMMU_PAGE_SIZE(tbl), &hpa, &mem);
492 if (ret) 454 if (ret)
493 pr_debug("%s: tce %lx at #%lx was not cached, ret=%d\n", 455 pr_debug("%s: tce %llx at #%lx was not cached, ret=%d\n",
494 __func__, *pua, entry, ret); 456 __func__, be64_to_cpu(*pua), entry, ret);
495 if (mem) 457 if (mem)
496 mm_iommu_mapped_dec(mem); 458 mm_iommu_mapped_dec(mem);
497 459
498 *pua = 0; 460 *pua = cpu_to_be64(0);
499} 461}
500 462
501static int tce_iommu_clear(struct tce_container *container, 463static int tce_iommu_clear(struct tce_container *container,
@@ -599,16 +561,9 @@ static long tce_iommu_build_v2(struct tce_container *container,
599 unsigned long hpa; 561 unsigned long hpa;
600 enum dma_data_direction dirtmp; 562 enum dma_data_direction dirtmp;
601 563
602 if (!tbl->it_userspace) {
603 ret = tce_iommu_userspace_view_alloc(tbl, container->mm);
604 if (ret)
605 return ret;
606 }
607
608 for (i = 0; i < pages; ++i) { 564 for (i = 0; i < pages; ++i) {
609 struct mm_iommu_table_group_mem_t *mem = NULL; 565 struct mm_iommu_table_group_mem_t *mem = NULL;
610 unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, 566 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry + i);
611 entry + i);
612 567
613 ret = tce_iommu_prereg_ua_to_hpa(container, 568 ret = tce_iommu_prereg_ua_to_hpa(container,
614 tce, IOMMU_PAGE_SIZE(tbl), &hpa, &mem); 569 tce, IOMMU_PAGE_SIZE(tbl), &hpa, &mem);
@@ -642,7 +597,7 @@ static long tce_iommu_build_v2(struct tce_container *container,
642 if (dirtmp != DMA_NONE) 597 if (dirtmp != DMA_NONE)
643 tce_iommu_unuse_page_v2(container, tbl, entry + i); 598 tce_iommu_unuse_page_v2(container, tbl, entry + i);
644 599
645 *pua = tce; 600 *pua = cpu_to_be64(tce);
646 601
647 tce += IOMMU_PAGE_SIZE(tbl); 602 tce += IOMMU_PAGE_SIZE(tbl);
648 } 603 }
@@ -676,7 +631,7 @@ static long tce_iommu_create_table(struct tce_container *container,
676 page_shift, window_size, levels, ptbl); 631 page_shift, window_size, levels, ptbl);
677 632
678 WARN_ON(!ret && !(*ptbl)->it_ops->free); 633 WARN_ON(!ret && !(*ptbl)->it_ops->free);
679 WARN_ON(!ret && ((*ptbl)->it_allocated_size != table_size)); 634 WARN_ON(!ret && ((*ptbl)->it_allocated_size > table_size));
680 635
681 return ret; 636 return ret;
682} 637}
@@ -686,7 +641,6 @@ static void tce_iommu_free_table(struct tce_container *container,
686{ 641{
687 unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT; 642 unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;
688 643
689 tce_iommu_userspace_view_free(tbl, container->mm);
690 iommu_tce_table_put(tbl); 644 iommu_tce_table_put(tbl);
691 decrement_locked_vm(container->mm, pages); 645 decrement_locked_vm(container->mm, pages);
692} 646}
@@ -1201,7 +1155,6 @@ static void tce_iommu_release_ownership(struct tce_container *container,
1201 continue; 1155 continue;
1202 1156
1203 tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size); 1157 tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
1204 tce_iommu_userspace_view_free(tbl, container->mm);
1205 if (tbl->it_map) 1158 if (tbl->it_map)
1206 iommu_release_ownership(tbl); 1159 iommu_release_ownership(tbl);
1207 1160
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-24 22:37:14 -0400