aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm/mm')
-rw-r--r--arch/arm/mm/Kconfig51
-rw-r--r--arch/arm/mm/Makefile3
-rw-r--r--arch/arm/mm/alignment.c19
-rw-r--r--arch/arm/mm/cache-feroceon-l2.c1
-rw-r--r--arch/arm/mm/cache-l2x0.c1498
-rw-r--r--arch/arm/mm/cache-v7.S12
-rw-r--r--arch/arm/mm/dma-mapping.c11
-rw-r--r--arch/arm/mm/flush.c33
-rw-r--r--arch/arm/mm/highmem.c33
-rw-r--r--arch/arm/mm/init.c81
-rw-r--r--arch/arm/mm/ioremap.c9
-rw-r--r--arch/arm/mm/l2c-common.c20
-rw-r--r--arch/arm/mm/l2c-l2x0-resume.S58
-rw-r--r--arch/arm/mm/mm.h4
-rw-r--r--arch/arm/mm/mmu.c238
-rw-r--r--arch/arm/mm/nommu.c66
-rw-r--r--arch/arm/mm/proc-v7-3level.S18
-rw-r--r--arch/arm/mm/proc-v7.S39
18 files changed, 1447 insertions, 747 deletions
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 5bf7c3c3b301..eda0dd0ab97b 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -897,6 +897,57 @@ config CACHE_PL310
897 This option enables optimisations for the PL310 cache 897 This option enables optimisations for the PL310 cache
898 controller. 898 controller.
899 899
900config PL310_ERRATA_588369
901 bool "PL310 errata: Clean & Invalidate maintenance operations do not invalidate clean lines"
902 depends on CACHE_L2X0
903 help
904 The PL310 L2 cache controller implements three types of Clean &
905 Invalidate maintenance operations: by Physical Address
906 (offset 0x7F0), by Index/Way (0x7F8) and by Way (0x7FC).
907 They are architecturally defined to behave as the execution of a
908 clean operation followed immediately by an invalidate operation,
909 both performing to the same memory location. This functionality
910 is not correctly implemented in PL310 as clean lines are not
911 invalidated as a result of these operations.
912
913config PL310_ERRATA_727915
914 bool "PL310 errata: Background Clean & Invalidate by Way operation can cause data corruption"
915 depends on CACHE_L2X0
916 help
917 PL310 implements the Clean & Invalidate by Way L2 cache maintenance
918 operation (offset 0x7FC). This operation runs in background so that
919 PL310 can handle normal accesses while it is in progress. Under very
920 rare circumstances, due to this erratum, write data can be lost when
921 PL310 treats a cacheable write transaction during a Clean &
922 Invalidate by Way operation.
923
924config PL310_ERRATA_753970
925 bool "PL310 errata: cache sync operation may be faulty"
926 depends on CACHE_PL310
927 help
928 This option enables the workaround for the 753970 PL310 (r3p0) erratum.
929
930 Under some condition the effect of cache sync operation on
931 the store buffer still remains when the operation completes.
932 This means that the store buffer is always asked to drain and
933 this prevents it from merging any further writes. The workaround
934 is to replace the normal offset of cache sync operation (0x730)
935 by another offset targeting an unmapped PL310 register 0x740.
936 This has the same effect as the cache sync operation: store buffer
937 drain and waiting for all buffers empty.
938
939config PL310_ERRATA_769419
940 bool "PL310 errata: no automatic Store Buffer drain"
941 depends on CACHE_L2X0
942 help
943 On revisions of the PL310 prior to r3p2, the Store Buffer does
944 not automatically drain. This can cause normal, non-cacheable
945 writes to be retained when the memory system is idle, leading
946 to suboptimal I/O performance for drivers using coherent DMA.
947 This option adds a write barrier to the cpu_idle loop so that,
948 on systems with an outer cache, the store buffer is drained
949 explicitly.
950
900config CACHE_TAUROS2 951config CACHE_TAUROS2
901 bool "Enable the Tauros2 L2 cache controller" 952 bool "Enable the Tauros2 L2 cache controller"
902 depends on (ARCH_DOVE || ARCH_MMP || CPU_PJ4) 953 depends on (ARCH_DOVE || ARCH_MMP || CPU_PJ4)
diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile
index 7f39ce2f841f..91da64de440f 100644
--- a/arch/arm/mm/Makefile
+++ b/arch/arm/mm/Makefile
@@ -95,7 +95,8 @@ obj-$(CONFIG_CPU_V7M) += proc-v7m.o
95AFLAGS_proc-v6.o :=-Wa,-march=armv6 95AFLAGS_proc-v6.o :=-Wa,-march=armv6
96AFLAGS_proc-v7.o :=-Wa,-march=armv7-a 96AFLAGS_proc-v7.o :=-Wa,-march=armv7-a
97 97
98obj-$(CONFIG_OUTER_CACHE) += l2c-common.o
98obj-$(CONFIG_CACHE_FEROCEON_L2) += cache-feroceon-l2.o 99obj-$(CONFIG_CACHE_FEROCEON_L2) += cache-feroceon-l2.o
99obj-$(CONFIG_CACHE_L2X0) += cache-l2x0.o 100obj-$(CONFIG_CACHE_L2X0) += cache-l2x0.o l2c-l2x0-resume.o
100obj-$(CONFIG_CACHE_XSC3L2) += cache-xsc3l2.o 101obj-$(CONFIG_CACHE_XSC3L2) += cache-xsc3l2.o
101obj-$(CONFIG_CACHE_TAUROS2) += cache-tauros2.o 102obj-$(CONFIG_CACHE_TAUROS2) += cache-tauros2.o
diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c
index 924036473b16..b8cb1a2688a0 100644
--- a/arch/arm/mm/alignment.c
+++ b/arch/arm/mm/alignment.c
@@ -28,6 +28,7 @@
28#include <asm/opcodes.h> 28#include <asm/opcodes.h>
29 29
30#include "fault.h" 30#include "fault.h"
31#include "mm.h"
31 32
32/* 33/*
33 * 32-bit misaligned trap handler (c) 1998 San Mehat (CCC) -July 1998 34 * 32-bit misaligned trap handler (c) 1998 San Mehat (CCC) -July 1998
@@ -81,6 +82,7 @@ static unsigned long ai_word;
81static unsigned long ai_dword; 82static unsigned long ai_dword;
82static unsigned long ai_multi; 83static unsigned long ai_multi;
83static int ai_usermode; 84static int ai_usermode;
85static unsigned long cr_no_alignment;
84 86
85core_param(alignment, ai_usermode, int, 0600); 87core_param(alignment, ai_usermode, int, 0600);
86 88
@@ -91,7 +93,7 @@ core_param(alignment, ai_usermode, int, 0600);
91/* Return true if and only if the ARMv6 unaligned access model is in use. */ 93/* Return true if and only if the ARMv6 unaligned access model is in use. */
92static bool cpu_is_v6_unaligned(void) 94static bool cpu_is_v6_unaligned(void)
93{ 95{
94 return cpu_architecture() >= CPU_ARCH_ARMv6 && (cr_alignment & CR_U); 96 return cpu_architecture() >= CPU_ARCH_ARMv6 && get_cr() & CR_U;
95} 97}
96 98
97static int safe_usermode(int new_usermode, bool warn) 99static int safe_usermode(int new_usermode, bool warn)
@@ -949,6 +951,13 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
949 return 0; 951 return 0;
950} 952}
951 953
954static int __init noalign_setup(char *__unused)
955{
956 set_cr(__clear_cr(CR_A));
957 return 1;
958}
959__setup("noalign", noalign_setup);
960
952/* 961/*
953 * This needs to be done after sysctl_init, otherwise sys/ will be 962 * This needs to be done after sysctl_init, otherwise sys/ will be
954 * overwritten. Actually, this shouldn't be in sys/ at all since 963 * overwritten. Actually, this shouldn't be in sys/ at all since
@@ -966,14 +975,12 @@ static int __init alignment_init(void)
966 return -ENOMEM; 975 return -ENOMEM;
967#endif 976#endif
968 977
969#ifdef CONFIG_CPU_CP15
970 if (cpu_is_v6_unaligned()) { 978 if (cpu_is_v6_unaligned()) {
971 cr_alignment &= ~CR_A; 979 set_cr(__clear_cr(CR_A));
972 cr_no_alignment &= ~CR_A;
973 set_cr(cr_alignment);
974 ai_usermode = safe_usermode(ai_usermode, false); 980 ai_usermode = safe_usermode(ai_usermode, false);
975 } 981 }
976#endif 982
983 cr_no_alignment = get_cr() & ~CR_A;
977 984
978 hook_fault_code(FAULT_CODE_ALIGNMENT, do_alignment, SIGBUS, BUS_ADRALN, 985 hook_fault_code(FAULT_CODE_ALIGNMENT, do_alignment, SIGBUS, BUS_ADRALN,
979 "alignment exception"); 986 "alignment exception");
diff --git a/arch/arm/mm/cache-feroceon-l2.c b/arch/arm/mm/cache-feroceon-l2.c
index dc814a548056..e028a7f2ebcc 100644
--- a/arch/arm/mm/cache-feroceon-l2.c
+++ b/arch/arm/mm/cache-feroceon-l2.c
@@ -350,7 +350,6 @@ void __init feroceon_l2_init(int __l2_wt_override)
350 outer_cache.inv_range = feroceon_l2_inv_range; 350 outer_cache.inv_range = feroceon_l2_inv_range;
351 outer_cache.clean_range = feroceon_l2_clean_range; 351 outer_cache.clean_range = feroceon_l2_clean_range;
352 outer_cache.flush_range = feroceon_l2_flush_range; 352 outer_cache.flush_range = feroceon_l2_flush_range;
353 outer_cache.inv_all = l2_inv_all;
354 353
355 enable_l2(); 354 enable_l2();
356 355
diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c
index 7abde2ce8973..efc5cabf70e0 100644
--- a/arch/arm/mm/cache-l2x0.c
+++ b/arch/arm/mm/cache-l2x0.c
@@ -16,18 +16,33 @@
16 * along with this program; if not, write to the Free Software 16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */ 18 */
19#include <linux/cpu.h>
19#include <linux/err.h> 20#include <linux/err.h>
20#include <linux/init.h> 21#include <linux/init.h>
22#include <linux/smp.h>
21#include <linux/spinlock.h> 23#include <linux/spinlock.h>
22#include <linux/io.h> 24#include <linux/io.h>
23#include <linux/of.h> 25#include <linux/of.h>
24#include <linux/of_address.h> 26#include <linux/of_address.h>
25 27
26#include <asm/cacheflush.h> 28#include <asm/cacheflush.h>
29#include <asm/cp15.h>
30#include <asm/cputype.h>
27#include <asm/hardware/cache-l2x0.h> 31#include <asm/hardware/cache-l2x0.h>
28#include "cache-tauros3.h" 32#include "cache-tauros3.h"
29#include "cache-aurora-l2.h" 33#include "cache-aurora-l2.h"
30 34
35struct l2c_init_data {
36 const char *type;
37 unsigned way_size_0;
38 unsigned num_lock;
39 void (*of_parse)(const struct device_node *, u32 *, u32 *);
40 void (*enable)(void __iomem *, u32, unsigned);
41 void (*fixup)(void __iomem *, u32, struct outer_cache_fns *);
42 void (*save)(void __iomem *);
43 struct outer_cache_fns outer_cache;
44};
45
31#define CACHE_LINE_SIZE 32 46#define CACHE_LINE_SIZE 32
32 47
33static void __iomem *l2x0_base; 48static void __iomem *l2x0_base;
@@ -36,96 +51,116 @@ static u32 l2x0_way_mask; /* Bitmask of active ways */
36static u32 l2x0_size; 51static u32 l2x0_size;
37static unsigned long sync_reg_offset = L2X0_CACHE_SYNC; 52static unsigned long sync_reg_offset = L2X0_CACHE_SYNC;
38 53
39/* Aurora don't have the cache ID register available, so we have to
40 * pass it though the device tree */
41static u32 cache_id_part_number_from_dt;
42
43struct l2x0_regs l2x0_saved_regs; 54struct l2x0_regs l2x0_saved_regs;
44 55
45struct l2x0_of_data { 56/*
46 void (*setup)(const struct device_node *, u32 *, u32 *); 57 * Common code for all cache controllers.
47 void (*save)(void); 58 */
48 struct outer_cache_fns outer_cache; 59static inline void l2c_wait_mask(void __iomem *reg, unsigned long mask)
49};
50
51static bool of_init = false;
52
53static inline void cache_wait_way(void __iomem *reg, unsigned long mask)
54{ 60{
55 /* wait for cache operation by line or way to complete */ 61 /* wait for cache operation by line or way to complete */
56 while (readl_relaxed(reg) & mask) 62 while (readl_relaxed(reg) & mask)
57 cpu_relax(); 63 cpu_relax();
58} 64}
59 65
60#ifdef CONFIG_CACHE_PL310 66/*
61static inline void cache_wait(void __iomem *reg, unsigned long mask) 67 * By default, we write directly to secure registers. Platforms must
68 * override this if they are running non-secure.
69 */
70static void l2c_write_sec(unsigned long val, void __iomem *base, unsigned reg)
62{ 71{
63 /* cache operations by line are atomic on PL310 */ 72 if (val == readl_relaxed(base + reg))
73 return;
74 if (outer_cache.write_sec)
75 outer_cache.write_sec(val, reg);
76 else
77 writel_relaxed(val, base + reg);
64} 78}
65#else
66#define cache_wait cache_wait_way
67#endif
68 79
69static inline void cache_sync(void) 80/*
81 * This should only be called when we have a requirement that the
82 * register be written due to a work-around, as platforms running
83 * in non-secure mode may not be able to access this register.
84 */
85static inline void l2c_set_debug(void __iomem *base, unsigned long val)
70{ 86{
71 void __iomem *base = l2x0_base; 87 l2c_write_sec(val, base, L2X0_DEBUG_CTRL);
72
73 writel_relaxed(0, base + sync_reg_offset);
74 cache_wait(base + L2X0_CACHE_SYNC, 1);
75} 88}
76 89
77static inline void l2x0_clean_line(unsigned long addr) 90static void __l2c_op_way(void __iomem *reg)
78{ 91{
79 void __iomem *base = l2x0_base; 92 writel_relaxed(l2x0_way_mask, reg);
80 cache_wait(base + L2X0_CLEAN_LINE_PA, 1); 93 l2c_wait_mask(reg, l2x0_way_mask);
81 writel_relaxed(addr, base + L2X0_CLEAN_LINE_PA);
82} 94}
83 95
84static inline void l2x0_inv_line(unsigned long addr) 96static inline void l2c_unlock(void __iomem *base, unsigned num)
85{ 97{
86 void __iomem *base = l2x0_base; 98 unsigned i;
87 cache_wait(base + L2X0_INV_LINE_PA, 1); 99
88 writel_relaxed(addr, base + L2X0_INV_LINE_PA); 100 for (i = 0; i < num; i++) {
101 writel_relaxed(0, base + L2X0_LOCKDOWN_WAY_D_BASE +
102 i * L2X0_LOCKDOWN_STRIDE);
103 writel_relaxed(0, base + L2X0_LOCKDOWN_WAY_I_BASE +
104 i * L2X0_LOCKDOWN_STRIDE);
105 }
89} 106}
90 107
91#if defined(CONFIG_PL310_ERRATA_588369) || defined(CONFIG_PL310_ERRATA_727915) 108/*
92static inline void debug_writel(unsigned long val) 109 * Enable the L2 cache controller. This function must only be
110 * called when the cache controller is known to be disabled.
111 */
112static void l2c_enable(void __iomem *base, u32 aux, unsigned num_lock)
93{ 113{
94 if (outer_cache.set_debug) 114 unsigned long flags;
95 outer_cache.set_debug(val); 115
116 l2c_write_sec(aux, base, L2X0_AUX_CTRL);
117
118 l2c_unlock(base, num_lock);
119
120 local_irq_save(flags);
121 __l2c_op_way(base + L2X0_INV_WAY);
122 writel_relaxed(0, base + sync_reg_offset);
123 l2c_wait_mask(base + sync_reg_offset, 1);
124 local_irq_restore(flags);
125
126 l2c_write_sec(L2X0_CTRL_EN, base, L2X0_CTRL);
96} 127}
97 128
98static void pl310_set_debug(unsigned long val) 129static void l2c_disable(void)
99{ 130{
100 writel_relaxed(val, l2x0_base + L2X0_DEBUG_CTRL); 131 void __iomem *base = l2x0_base;
132
133 outer_cache.flush_all();
134 l2c_write_sec(0, base, L2X0_CTRL);
135 dsb(st);
101} 136}
102#else 137
103/* Optimised out for non-errata case */ 138#ifdef CONFIG_CACHE_PL310
104static inline void debug_writel(unsigned long val) 139static inline void cache_wait(void __iomem *reg, unsigned long mask)
105{ 140{
141 /* cache operations by line are atomic on PL310 */
106} 142}
107 143#else
108#define pl310_set_debug NULL 144#define cache_wait l2c_wait_mask
109#endif 145#endif
110 146
111#ifdef CONFIG_PL310_ERRATA_588369 147static inline void cache_sync(void)
112static inline void l2x0_flush_line(unsigned long addr)
113{ 148{
114 void __iomem *base = l2x0_base; 149 void __iomem *base = l2x0_base;
115 150
116 /* Clean by PA followed by Invalidate by PA */ 151 writel_relaxed(0, base + sync_reg_offset);
117 cache_wait(base + L2X0_CLEAN_LINE_PA, 1); 152 cache_wait(base + L2X0_CACHE_SYNC, 1);
118 writel_relaxed(addr, base + L2X0_CLEAN_LINE_PA);
119 cache_wait(base + L2X0_INV_LINE_PA, 1);
120 writel_relaxed(addr, base + L2X0_INV_LINE_PA);
121} 153}
122#else
123 154
124static inline void l2x0_flush_line(unsigned long addr) 155#if defined(CONFIG_PL310_ERRATA_588369) || defined(CONFIG_PL310_ERRATA_727915)
156static inline void debug_writel(unsigned long val)
157{
158 l2c_set_debug(l2x0_base, val);
159}
160#else
161/* Optimised out for non-errata case */
162static inline void debug_writel(unsigned long val)
125{ 163{
126 void __iomem *base = l2x0_base;
127 cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
128 writel_relaxed(addr, base + L2X0_CLEAN_INV_LINE_PA);
129} 164}
130#endif 165#endif
131 166
@@ -141,8 +176,7 @@ static void l2x0_cache_sync(void)
141static void __l2x0_flush_all(void) 176static void __l2x0_flush_all(void)
142{ 177{
143 debug_writel(0x03); 178 debug_writel(0x03);
144 writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_CLEAN_INV_WAY); 179 __l2c_op_way(l2x0_base + L2X0_CLEAN_INV_WAY);
145 cache_wait_way(l2x0_base + L2X0_CLEAN_INV_WAY, l2x0_way_mask);
146 cache_sync(); 180 cache_sync();
147 debug_writel(0x00); 181 debug_writel(0x00);
148} 182}
@@ -157,275 +191,883 @@ static void l2x0_flush_all(void)
157 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 191 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
158} 192}
159 193
160static void l2x0_clean_all(void) 194static void l2x0_disable(void)
161{ 195{
162 unsigned long flags; 196 unsigned long flags;
163 197
164 /* clean all ways */
165 raw_spin_lock_irqsave(&l2x0_lock, flags); 198 raw_spin_lock_irqsave(&l2x0_lock, flags);
166 writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_CLEAN_WAY); 199 __l2x0_flush_all();
167 cache_wait_way(l2x0_base + L2X0_CLEAN_WAY, l2x0_way_mask); 200 l2c_write_sec(0, l2x0_base, L2X0_CTRL);
168 cache_sync(); 201 dsb(st);
169 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 202 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
170} 203}
171 204
172static void l2x0_inv_all(void) 205static void l2c_save(void __iomem *base)
173{ 206{
174 unsigned long flags; 207 l2x0_saved_regs.aux_ctrl = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
208}
175 209
176 /* invalidate all ways */ 210/*
177 raw_spin_lock_irqsave(&l2x0_lock, flags); 211 * L2C-210 specific code.
178 /* Invalidating when L2 is enabled is a nono */ 212 *
179 BUG_ON(readl(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN); 213 * The L2C-2x0 PA, set/way and sync operations are atomic, but we must
180 writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_INV_WAY); 214 * ensure that no background operation is running. The way operations
181 cache_wait_way(l2x0_base + L2X0_INV_WAY, l2x0_way_mask); 215 * are all background tasks.
182 cache_sync(); 216 *
183 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 217 * While a background operation is in progress, any new operation is
218 * ignored (unspecified whether this causes an error.) Thankfully, not
219 * used on SMP.
220 *
221 * Never has a different sync register other than L2X0_CACHE_SYNC, but
222 * we use sync_reg_offset here so we can share some of this with L2C-310.
223 */
224static void __l2c210_cache_sync(void __iomem *base)
225{
226 writel_relaxed(0, base + sync_reg_offset);
184} 227}
185 228
186static void l2x0_inv_range(unsigned long start, unsigned long end) 229static void __l2c210_op_pa_range(void __iomem *reg, unsigned long start,
230 unsigned long end)
231{
232 while (start < end) {
233 writel_relaxed(start, reg);
234 start += CACHE_LINE_SIZE;
235 }
236}
237
238static void l2c210_inv_range(unsigned long start, unsigned long end)
187{ 239{
188 void __iomem *base = l2x0_base; 240 void __iomem *base = l2x0_base;
189 unsigned long flags;
190 241
191 raw_spin_lock_irqsave(&l2x0_lock, flags);
192 if (start & (CACHE_LINE_SIZE - 1)) { 242 if (start & (CACHE_LINE_SIZE - 1)) {
193 start &= ~(CACHE_LINE_SIZE - 1); 243 start &= ~(CACHE_LINE_SIZE - 1);
194 debug_writel(0x03); 244 writel_relaxed(start, base + L2X0_CLEAN_INV_LINE_PA);
195 l2x0_flush_line(start);
196 debug_writel(0x00);
197 start += CACHE_LINE_SIZE; 245 start += CACHE_LINE_SIZE;
198 } 246 }
199 247
200 if (end & (CACHE_LINE_SIZE - 1)) { 248 if (end & (CACHE_LINE_SIZE - 1)) {
201 end &= ~(CACHE_LINE_SIZE - 1); 249 end &= ~(CACHE_LINE_SIZE - 1);
202 debug_writel(0x03); 250 writel_relaxed(end, base + L2X0_CLEAN_INV_LINE_PA);
203 l2x0_flush_line(end);
204 debug_writel(0x00);
205 } 251 }
206 252
253 __l2c210_op_pa_range(base + L2X0_INV_LINE_PA, start, end);
254 __l2c210_cache_sync(base);
255}
256
257static void l2c210_clean_range(unsigned long start, unsigned long end)
258{
259 void __iomem *base = l2x0_base;
260
261 start &= ~(CACHE_LINE_SIZE - 1);
262 __l2c210_op_pa_range(base + L2X0_CLEAN_LINE_PA, start, end);
263 __l2c210_cache_sync(base);
264}
265
266static void l2c210_flush_range(unsigned long start, unsigned long end)
267{
268 void __iomem *base = l2x0_base;
269
270 start &= ~(CACHE_LINE_SIZE - 1);
271 __l2c210_op_pa_range(base + L2X0_CLEAN_INV_LINE_PA, start, end);
272 __l2c210_cache_sync(base);
273}
274
275static void l2c210_flush_all(void)
276{
277 void __iomem *base = l2x0_base;
278
279 BUG_ON(!irqs_disabled());
280
281 __l2c_op_way(base + L2X0_CLEAN_INV_WAY);
282 __l2c210_cache_sync(base);
283}
284
285static void l2c210_sync(void)
286{
287 __l2c210_cache_sync(l2x0_base);
288}
289
290static void l2c210_resume(void)
291{
292 void __iomem *base = l2x0_base;
293
294 if (!(readl_relaxed(base + L2X0_CTRL) & L2X0_CTRL_EN))
295 l2c_enable(base, l2x0_saved_regs.aux_ctrl, 1);
296}
297
298static const struct l2c_init_data l2c210_data __initconst = {
299 .type = "L2C-210",
300 .way_size_0 = SZ_8K,
301 .num_lock = 1,
302 .enable = l2c_enable,
303 .save = l2c_save,
304 .outer_cache = {
305 .inv_range = l2c210_inv_range,
306 .clean_range = l2c210_clean_range,
307 .flush_range = l2c210_flush_range,
308 .flush_all = l2c210_flush_all,
309 .disable = l2c_disable,
310 .sync = l2c210_sync,
311 .resume = l2c210_resume,
312 },
313};
314
315/*
316 * L2C-220 specific code.
317 *
318 * All operations are background operations: they have to be waited for.
319 * Conflicting requests generate a slave error (which will cause an
320 * imprecise abort.) Never uses sync_reg_offset, so we hard-code the
321 * sync register here.
322 *
323 * However, we can re-use the l2c210_resume call.
324 */
325static inline void __l2c220_cache_sync(void __iomem *base)
326{
327 writel_relaxed(0, base + L2X0_CACHE_SYNC);
328 l2c_wait_mask(base + L2X0_CACHE_SYNC, 1);
329}
330
331static void l2c220_op_way(void __iomem *base, unsigned reg)
332{
333 unsigned long flags;
334
335 raw_spin_lock_irqsave(&l2x0_lock, flags);
336 __l2c_op_way(base + reg);
337 __l2c220_cache_sync(base);
338 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
339}
340
341static unsigned long l2c220_op_pa_range(void __iomem *reg, unsigned long start,
342 unsigned long end, unsigned long flags)
343{
344 raw_spinlock_t *lock = &l2x0_lock;
345
207 while (start < end) { 346 while (start < end) {
208 unsigned long blk_end = start + min(end - start, 4096UL); 347 unsigned long blk_end = start + min(end - start, 4096UL);
209 348
210 while (start < blk_end) { 349 while (start < blk_end) {
211 l2x0_inv_line(start); 350 l2c_wait_mask(reg, 1);
351 writel_relaxed(start, reg);
212 start += CACHE_LINE_SIZE; 352 start += CACHE_LINE_SIZE;
213 } 353 }
214 354
215 if (blk_end < end) { 355 if (blk_end < end) {
216 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 356 raw_spin_unlock_irqrestore(lock, flags);
217 raw_spin_lock_irqsave(&l2x0_lock, flags); 357 raw_spin_lock_irqsave(lock, flags);
218 } 358 }
219 } 359 }
220 cache_wait(base + L2X0_INV_LINE_PA, 1); 360
221 cache_sync(); 361 return flags;
222 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
223} 362}
224 363
225static void l2x0_clean_range(unsigned long start, unsigned long end) 364static void l2c220_inv_range(unsigned long start, unsigned long end)
226{ 365{
227 void __iomem *base = l2x0_base; 366 void __iomem *base = l2x0_base;
228 unsigned long flags; 367 unsigned long flags;
229 368
230 if ((end - start) >= l2x0_size) {
231 l2x0_clean_all();
232 return;
233 }
234
235 raw_spin_lock_irqsave(&l2x0_lock, flags); 369 raw_spin_lock_irqsave(&l2x0_lock, flags);
236 start &= ~(CACHE_LINE_SIZE - 1); 370 if ((start | end) & (CACHE_LINE_SIZE - 1)) {
237 while (start < end) { 371 if (start & (CACHE_LINE_SIZE - 1)) {
238 unsigned long blk_end = start + min(end - start, 4096UL); 372 start &= ~(CACHE_LINE_SIZE - 1);
239 373 writel_relaxed(start, base + L2X0_CLEAN_INV_LINE_PA);
240 while (start < blk_end) {
241 l2x0_clean_line(start);
242 start += CACHE_LINE_SIZE; 374 start += CACHE_LINE_SIZE;
243 } 375 }
244 376
245 if (blk_end < end) { 377 if (end & (CACHE_LINE_SIZE - 1)) {
246 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 378 end &= ~(CACHE_LINE_SIZE - 1);
247 raw_spin_lock_irqsave(&l2x0_lock, flags); 379 l2c_wait_mask(base + L2X0_CLEAN_INV_LINE_PA, 1);
380 writel_relaxed(end, base + L2X0_CLEAN_INV_LINE_PA);
248 } 381 }
249 } 382 }
250 cache_wait(base + L2X0_CLEAN_LINE_PA, 1); 383
251 cache_sync(); 384 flags = l2c220_op_pa_range(base + L2X0_INV_LINE_PA,
385 start, end, flags);
386 l2c_wait_mask(base + L2X0_INV_LINE_PA, 1);
387 __l2c220_cache_sync(base);
252 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 388 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
253} 389}
254 390
255static void l2x0_flush_range(unsigned long start, unsigned long end) 391static void l2c220_clean_range(unsigned long start, unsigned long end)
256{ 392{
257 void __iomem *base = l2x0_base; 393 void __iomem *base = l2x0_base;
258 unsigned long flags; 394 unsigned long flags;
259 395
396 start &= ~(CACHE_LINE_SIZE - 1);
260 if ((end - start) >= l2x0_size) { 397 if ((end - start) >= l2x0_size) {
261 l2x0_flush_all(); 398 l2c220_op_way(base, L2X0_CLEAN_WAY);
262 return; 399 return;
263 } 400 }
264 401
265 raw_spin_lock_irqsave(&l2x0_lock, flags); 402 raw_spin_lock_irqsave(&l2x0_lock, flags);
403 flags = l2c220_op_pa_range(base + L2X0_CLEAN_LINE_PA,
404 start, end, flags);
405 l2c_wait_mask(base + L2X0_CLEAN_INV_LINE_PA, 1);
406 __l2c220_cache_sync(base);
407 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
408}
409
410static void l2c220_flush_range(unsigned long start, unsigned long end)
411{
412 void __iomem *base = l2x0_base;
413 unsigned long flags;
414
266 start &= ~(CACHE_LINE_SIZE - 1); 415 start &= ~(CACHE_LINE_SIZE - 1);
416 if ((end - start) >= l2x0_size) {
417 l2c220_op_way(base, L2X0_CLEAN_INV_WAY);
418 return;
419 }
420
421 raw_spin_lock_irqsave(&l2x0_lock, flags);
422 flags = l2c220_op_pa_range(base + L2X0_CLEAN_INV_LINE_PA,
423 start, end, flags);
424 l2c_wait_mask(base + L2X0_CLEAN_INV_LINE_PA, 1);
425 __l2c220_cache_sync(base);
426 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
427}
428
429static void l2c220_flush_all(void)
430{
431 l2c220_op_way(l2x0_base, L2X0_CLEAN_INV_WAY);
432}
433
434static void l2c220_sync(void)
435{
436 unsigned long flags;
437
438 raw_spin_lock_irqsave(&l2x0_lock, flags);
439 __l2c220_cache_sync(l2x0_base);
440 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
441}
442
443static void l2c220_enable(void __iomem *base, u32 aux, unsigned num_lock)
444{
445 /*
446 * Always enable non-secure access to the lockdown registers -
447 * we write to them as part of the L2C enable sequence so they
448 * need to be accessible.
449 */
450 aux |= L220_AUX_CTRL_NS_LOCKDOWN;
451
452 l2c_enable(base, aux, num_lock);
453}
454
455static const struct l2c_init_data l2c220_data = {
456 .type = "L2C-220",
457 .way_size_0 = SZ_8K,
458 .num_lock = 1,
459 .enable = l2c220_enable,
460 .save = l2c_save,
461 .outer_cache = {
462 .inv_range = l2c220_inv_range,
463 .clean_range = l2c220_clean_range,
464 .flush_range = l2c220_flush_range,
465 .flush_all = l2c220_flush_all,
466 .disable = l2c_disable,
467 .sync = l2c220_sync,
468 .resume = l2c210_resume,
469 },
470};
471
472/*
473 * L2C-310 specific code.
474 *
475 * Very similar to L2C-210, the PA, set/way and sync operations are atomic,
476 * and the way operations are all background tasks. However, issuing an
477 * operation while a background operation is in progress results in a
478 * SLVERR response. We can reuse:
479 *
480 * __l2c210_cache_sync (using sync_reg_offset)
481 * l2c210_sync
482 * l2c210_inv_range (if 588369 is not applicable)
483 * l2c210_clean_range
484 * l2c210_flush_range (if 588369 is not applicable)
485 * l2c210_flush_all (if 727915 is not applicable)
486 *
487 * Errata:
488 * 588369: PL310 R0P0->R1P0, fixed R2P0.
489 * Affects: all clean+invalidate operations
490 * clean and invalidate skips the invalidate step, so we need to issue
491 * separate operations. We also require the above debug workaround
492 * enclosing this code fragment on affected parts. On unaffected parts,
493 * we must not use this workaround without the debug register writes
494 * to avoid exposing a problem similar to 727915.
495 *
496 * 727915: PL310 R2P0->R3P0, fixed R3P1.
497 * Affects: clean+invalidate by way
498 * clean and invalidate by way runs in the background, and a store can
499 * hit the line between the clean operation and invalidate operation,
500 * resulting in the store being lost.
501 *
502 * 752271: PL310 R3P0->R3P1-50REL0, fixed R3P2.
503 * Affects: 8x64-bit (double fill) line fetches
504 * double fill line fetches can fail to cause dirty data to be evicted
505 * from the cache before the new data overwrites the second line.
506 *
507 * 753970: PL310 R3P0, fixed R3P1.
508 * Affects: sync
509 * prevents merging writes after the sync operation, until another L2C
510 * operation is performed (or a number of other conditions.)
511 *
512 * 769419: PL310 R0P0->R3P1, fixed R3P2.
513 * Affects: store buffer
514 * store buffer is not automatically drained.
515 */
516static void l2c310_inv_range_erratum(unsigned long start, unsigned long end)
517{
518 void __iomem *base = l2x0_base;
519
520 if ((start | end) & (CACHE_LINE_SIZE - 1)) {
521 unsigned long flags;
522
523 /* Erratum 588369 for both clean+invalidate operations */
524 raw_spin_lock_irqsave(&l2x0_lock, flags);
525 l2c_set_debug(base, 0x03);
526
527 if (start & (CACHE_LINE_SIZE - 1)) {
528 start &= ~(CACHE_LINE_SIZE - 1);
529 writel_relaxed(start, base + L2X0_CLEAN_LINE_PA);
530 writel_relaxed(start, base + L2X0_INV_LINE_PA);
531 start += CACHE_LINE_SIZE;
532 }
533
534 if (end & (CACHE_LINE_SIZE - 1)) {
535 end &= ~(CACHE_LINE_SIZE - 1);
536 writel_relaxed(end, base + L2X0_CLEAN_LINE_PA);
537 writel_relaxed(end, base + L2X0_INV_LINE_PA);
538 }
539
540 l2c_set_debug(base, 0x00);
541 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
542 }
543
544 __l2c210_op_pa_range(base + L2X0_INV_LINE_PA, start, end);
545 __l2c210_cache_sync(base);
546}
547
548static void l2c310_flush_range_erratum(unsigned long start, unsigned long end)
549{
550 raw_spinlock_t *lock = &l2x0_lock;
551 unsigned long flags;
552 void __iomem *base = l2x0_base;
553
554 raw_spin_lock_irqsave(lock, flags);
267 while (start < end) { 555 while (start < end) {
268 unsigned long blk_end = start + min(end - start, 4096UL); 556 unsigned long blk_end = start + min(end - start, 4096UL);
269 557
270 debug_writel(0x03); 558 l2c_set_debug(base, 0x03);
271 while (start < blk_end) { 559 while (start < blk_end) {
272 l2x0_flush_line(start); 560 writel_relaxed(start, base + L2X0_CLEAN_LINE_PA);
561 writel_relaxed(start, base + L2X0_INV_LINE_PA);
273 start += CACHE_LINE_SIZE; 562 start += CACHE_LINE_SIZE;
274 } 563 }
275 debug_writel(0x00); 564 l2c_set_debug(base, 0x00);
276 565
277 if (blk_end < end) { 566 if (blk_end < end) {
278 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 567 raw_spin_unlock_irqrestore(lock, flags);
279 raw_spin_lock_irqsave(&l2x0_lock, flags); 568 raw_spin_lock_irqsave(lock, flags);
280 } 569 }
281 } 570 }
282 cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); 571 raw_spin_unlock_irqrestore(lock, flags);
283 cache_sync(); 572 __l2c210_cache_sync(base);
284 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
285} 573}
286 574
287static void l2x0_disable(void) 575static void l2c310_flush_all_erratum(void)
288{ 576{
577 void __iomem *base = l2x0_base;
289 unsigned long flags; 578 unsigned long flags;
290 579
291 raw_spin_lock_irqsave(&l2x0_lock, flags); 580 raw_spin_lock_irqsave(&l2x0_lock, flags);
292 __l2x0_flush_all(); 581 l2c_set_debug(base, 0x03);
293 writel_relaxed(0, l2x0_base + L2X0_CTRL); 582 __l2c_op_way(base + L2X0_CLEAN_INV_WAY);
294 dsb(st); 583 l2c_set_debug(base, 0x00);
584 __l2c210_cache_sync(base);
295 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 585 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
296} 586}
297 587
298static void l2x0_unlock(u32 cache_id) 588static void __init l2c310_save(void __iomem *base)
299{ 589{
300 int lockregs; 590 unsigned revision;
301 int i;
302 591
303 switch (cache_id & L2X0_CACHE_ID_PART_MASK) { 592 l2c_save(base);
304 case L2X0_CACHE_ID_PART_L310: 593
305 lockregs = 8; 594 l2x0_saved_regs.tag_latency = readl_relaxed(base +
306 break; 595 L310_TAG_LATENCY_CTRL);
307 case AURORA_CACHE_ID: 596 l2x0_saved_regs.data_latency = readl_relaxed(base +
308 lockregs = 4; 597 L310_DATA_LATENCY_CTRL);
598 l2x0_saved_regs.filter_end = readl_relaxed(base +
599 L310_ADDR_FILTER_END);
600 l2x0_saved_regs.filter_start = readl_relaxed(base +
601 L310_ADDR_FILTER_START);
602
603 revision = readl_relaxed(base + L2X0_CACHE_ID) &
604 L2X0_CACHE_ID_RTL_MASK;
605
606 /* From r2p0, there is Prefetch offset/control register */
607 if (revision >= L310_CACHE_ID_RTL_R2P0)
608 l2x0_saved_regs.prefetch_ctrl = readl_relaxed(base +
609 L310_PREFETCH_CTRL);
610
611 /* From r3p0, there is Power control register */
612 if (revision >= L310_CACHE_ID_RTL_R3P0)
613 l2x0_saved_regs.pwr_ctrl = readl_relaxed(base +
614 L310_POWER_CTRL);
615}
616
617static void l2c310_resume(void)
618{
619 void __iomem *base = l2x0_base;
620
621 if (!(readl_relaxed(base + L2X0_CTRL) & L2X0_CTRL_EN)) {
622 unsigned revision;
623
624 /* restore pl310 setup */
625 writel_relaxed(l2x0_saved_regs.tag_latency,
626 base + L310_TAG_LATENCY_CTRL);
627 writel_relaxed(l2x0_saved_regs.data_latency,
628 base + L310_DATA_LATENCY_CTRL);
629 writel_relaxed(l2x0_saved_regs.filter_end,
630 base + L310_ADDR_FILTER_END);
631 writel_relaxed(l2x0_saved_regs.filter_start,
632 base + L310_ADDR_FILTER_START);
633
634 revision = readl_relaxed(base + L2X0_CACHE_ID) &
635 L2X0_CACHE_ID_RTL_MASK;
636
637 if (revision >= L310_CACHE_ID_RTL_R2P0)
638 l2c_write_sec(l2x0_saved_regs.prefetch_ctrl, base,
639 L310_PREFETCH_CTRL);
640 if (revision >= L310_CACHE_ID_RTL_R3P0)
641 l2c_write_sec(l2x0_saved_regs.pwr_ctrl, base,
642 L310_POWER_CTRL);
643
644 l2c_enable(base, l2x0_saved_regs.aux_ctrl, 8);
645
646 /* Re-enable full-line-of-zeros for Cortex-A9 */
647 if (l2x0_saved_regs.aux_ctrl & L310_AUX_CTRL_FULL_LINE_ZERO)
648 set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1));
649 }
650}
651
652static int l2c310_cpu_enable_flz(struct notifier_block *nb, unsigned long act, void *data)
653{
654 switch (act & ~CPU_TASKS_FROZEN) {
655 case CPU_STARTING:
656 set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1));
309 break; 657 break;
310 default: 658 case CPU_DYING:
311 /* L210 and unknown types */ 659 set_auxcr(get_auxcr() & ~(BIT(3) | BIT(2) | BIT(1)));
312 lockregs = 1;
313 break; 660 break;
314 } 661 }
662 return NOTIFY_OK;
663}
315 664
316 for (i = 0; i < lockregs; i++) { 665static void __init l2c310_enable(void __iomem *base, u32 aux, unsigned num_lock)
317 writel_relaxed(0x0, l2x0_base + L2X0_LOCKDOWN_WAY_D_BASE + 666{
318 i * L2X0_LOCKDOWN_STRIDE); 667 unsigned rev = readl_relaxed(base + L2X0_CACHE_ID) & L2X0_CACHE_ID_PART_MASK;
319 writel_relaxed(0x0, l2x0_base + L2X0_LOCKDOWN_WAY_I_BASE + 668 bool cortex_a9 = read_cpuid_part_number() == ARM_CPU_PART_CORTEX_A9;
320 i * L2X0_LOCKDOWN_STRIDE); 669
670 if (rev >= L310_CACHE_ID_RTL_R2P0) {
671 if (cortex_a9) {
672 aux |= L310_AUX_CTRL_EARLY_BRESP;
673 pr_info("L2C-310 enabling early BRESP for Cortex-A9\n");
674 } else if (aux & L310_AUX_CTRL_EARLY_BRESP) {
675 pr_warn("L2C-310 early BRESP only supported with Cortex-A9\n");
676 aux &= ~L310_AUX_CTRL_EARLY_BRESP;
677 }
678 }
679
680 if (cortex_a9) {
681 u32 aux_cur = readl_relaxed(base + L2X0_AUX_CTRL);
682 u32 acr = get_auxcr();
683
684 pr_debug("Cortex-A9 ACR=0x%08x\n", acr);
685
686 if (acr & BIT(3) && !(aux_cur & L310_AUX_CTRL_FULL_LINE_ZERO))
687 pr_err("L2C-310: full line of zeros enabled in Cortex-A9 but not L2C-310 - invalid\n");
688
689 if (aux & L310_AUX_CTRL_FULL_LINE_ZERO && !(acr & BIT(3)))
690 pr_err("L2C-310: enabling full line of zeros but not enabled in Cortex-A9\n");
691
692 if (!(aux & L310_AUX_CTRL_FULL_LINE_ZERO) && !outer_cache.write_sec) {
693 aux |= L310_AUX_CTRL_FULL_LINE_ZERO;
694 pr_info("L2C-310 full line of zeros enabled for Cortex-A9\n");
695 }
696 } else if (aux & (L310_AUX_CTRL_FULL_LINE_ZERO | L310_AUX_CTRL_EARLY_BRESP)) {
697 pr_err("L2C-310: disabling Cortex-A9 specific feature bits\n");
698 aux &= ~(L310_AUX_CTRL_FULL_LINE_ZERO | L310_AUX_CTRL_EARLY_BRESP);
699 }
700
701 if (aux & (L310_AUX_CTRL_DATA_PREFETCH | L310_AUX_CTRL_INSTR_PREFETCH)) {
702 u32 prefetch = readl_relaxed(base + L310_PREFETCH_CTRL);
703
704 pr_info("L2C-310 %s%s prefetch enabled, offset %u lines\n",
705 aux & L310_AUX_CTRL_INSTR_PREFETCH ? "I" : "",
706 aux & L310_AUX_CTRL_DATA_PREFETCH ? "D" : "",
707 1 + (prefetch & L310_PREFETCH_CTRL_OFFSET_MASK));
708 }
709
710 /* r3p0 or later has power control register */
711 if (rev >= L310_CACHE_ID_RTL_R3P0) {
712 u32 power_ctrl;
713
714 l2c_write_sec(L310_DYNAMIC_CLK_GATING_EN | L310_STNDBY_MODE_EN,
715 base, L310_POWER_CTRL);
716 power_ctrl = readl_relaxed(base + L310_POWER_CTRL);
717 pr_info("L2C-310 dynamic clock gating %sabled, standby mode %sabled\n",
718 power_ctrl & L310_DYNAMIC_CLK_GATING_EN ? "en" : "dis",
719 power_ctrl & L310_STNDBY_MODE_EN ? "en" : "dis");
720 }
721
722 /*
723 * Always enable non-secure access to the lockdown registers -
724 * we write to them as part of the L2C enable sequence so they
725 * need to be accessible.
726 */
727 aux |= L310_AUX_CTRL_NS_LOCKDOWN;
728
729 l2c_enable(base, aux, num_lock);
730
731 if (aux & L310_AUX_CTRL_FULL_LINE_ZERO) {
732 set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1));
733 cpu_notifier(l2c310_cpu_enable_flz, 0);
321 } 734 }
322} 735}
323 736
324void __init l2x0_init(void __iomem *base, u32 aux_val, u32 aux_mask) 737static void __init l2c310_fixup(void __iomem *base, u32 cache_id,
738 struct outer_cache_fns *fns)
325{ 739{
326 u32 aux; 740 unsigned revision = cache_id & L2X0_CACHE_ID_RTL_MASK;
327 u32 cache_id; 741 const char *errata[8];
328 u32 way_size = 0; 742 unsigned n = 0;
329 int ways; 743
330 int way_size_shift = L2X0_WAY_SIZE_SHIFT; 744 if (IS_ENABLED(CONFIG_PL310_ERRATA_588369) &&
331 const char *type; 745 revision < L310_CACHE_ID_RTL_R2P0 &&
746 /* For bcm compatibility */
747 fns->inv_range == l2c210_inv_range) {
748 fns->inv_range = l2c310_inv_range_erratum;
749 fns->flush_range = l2c310_flush_range_erratum;
750 errata[n++] = "588369";
751 }
332 752
333 l2x0_base = base; 753 if (IS_ENABLED(CONFIG_PL310_ERRATA_727915) &&
334 if (cache_id_part_number_from_dt) 754 revision >= L310_CACHE_ID_RTL_R2P0 &&
335 cache_id = cache_id_part_number_from_dt; 755 revision < L310_CACHE_ID_RTL_R3P1) {
336 else 756 fns->flush_all = l2c310_flush_all_erratum;
337 cache_id = readl_relaxed(l2x0_base + L2X0_CACHE_ID); 757 errata[n++] = "727915";
338 aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL); 758 }
759
760 if (revision >= L310_CACHE_ID_RTL_R3P0 &&
761 revision < L310_CACHE_ID_RTL_R3P2) {
762 u32 val = readl_relaxed(base + L310_PREFETCH_CTRL);
763 /* I don't think bit23 is required here... but iMX6 does so */
764 if (val & (BIT(30) | BIT(23))) {
765 val &= ~(BIT(30) | BIT(23));
766 l2c_write_sec(val, base, L310_PREFETCH_CTRL);
767 errata[n++] = "752271";
768 }
769 }
770
771 if (IS_ENABLED(CONFIG_PL310_ERRATA_753970) &&
772 revision == L310_CACHE_ID_RTL_R3P0) {
773 sync_reg_offset = L2X0_DUMMY_REG;
774 errata[n++] = "753970";
775 }
776
777 if (IS_ENABLED(CONFIG_PL310_ERRATA_769419))
778 errata[n++] = "769419";
779
780 if (n) {
781 unsigned i;
339 782
783 pr_info("L2C-310 errat%s", n > 1 ? "a" : "um");
784 for (i = 0; i < n; i++)
785 pr_cont(" %s", errata[i]);
786 pr_cont(" enabled\n");
787 }
788}
789
790static void l2c310_disable(void)
791{
792 /*
793 * If full-line-of-zeros is enabled, we must first disable it in the
794 * Cortex-A9 auxiliary control register before disabling the L2 cache.
795 */
796 if (l2x0_saved_regs.aux_ctrl & L310_AUX_CTRL_FULL_LINE_ZERO)
797 set_auxcr(get_auxcr() & ~(BIT(3) | BIT(2) | BIT(1)));
798
799 l2c_disable();
800}
801
802static const struct l2c_init_data l2c310_init_fns __initconst = {
803 .type = "L2C-310",
804 .way_size_0 = SZ_8K,
805 .num_lock = 8,
806 .enable = l2c310_enable,
807 .fixup = l2c310_fixup,
808 .save = l2c310_save,
809 .outer_cache = {
810 .inv_range = l2c210_inv_range,
811 .clean_range = l2c210_clean_range,
812 .flush_range = l2c210_flush_range,
813 .flush_all = l2c210_flush_all,
814 .disable = l2c310_disable,
815 .sync = l2c210_sync,
816 .resume = l2c310_resume,
817 },
818};
819
820static void __init __l2c_init(const struct l2c_init_data *data,
821 u32 aux_val, u32 aux_mask, u32 cache_id)
822{
823 struct outer_cache_fns fns;
824 unsigned way_size_bits, ways;
825 u32 aux, old_aux;
826
827 /*
828 * Sanity check the aux values. aux_mask is the bits we preserve
829 * from reading the hardware register, and aux_val is the bits we
830 * set.
831 */
832 if (aux_val & aux_mask)
833 pr_alert("L2C: platform provided aux values permit register corruption.\n");
834
835 old_aux = aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
340 aux &= aux_mask; 836 aux &= aux_mask;
341 aux |= aux_val; 837 aux |= aux_val;
342 838
839 if (old_aux != aux)
840 pr_warn("L2C: DT/platform modifies aux control register: 0x%08x -> 0x%08x\n",
841 old_aux, aux);
842
343 /* Determine the number of ways */ 843 /* Determine the number of ways */
344 switch (cache_id & L2X0_CACHE_ID_PART_MASK) { 844 switch (cache_id & L2X0_CACHE_ID_PART_MASK) {
345 case L2X0_CACHE_ID_PART_L310: 845 case L2X0_CACHE_ID_PART_L310:
846 if ((aux_val | ~aux_mask) & (L2C_AUX_CTRL_WAY_SIZE_MASK | L310_AUX_CTRL_ASSOCIATIVITY_16))
847 pr_warn("L2C: DT/platform tries to modify or specify cache size\n");
346 if (aux & (1 << 16)) 848 if (aux & (1 << 16))
347 ways = 16; 849 ways = 16;
348 else 850 else
349 ways = 8; 851 ways = 8;
350 type = "L310";
351#ifdef CONFIG_PL310_ERRATA_753970
352 /* Unmapped register. */
353 sync_reg_offset = L2X0_DUMMY_REG;
354#endif
355 if ((cache_id & L2X0_CACHE_ID_RTL_MASK) <= L2X0_CACHE_ID_RTL_R3P0)
356 outer_cache.set_debug = pl310_set_debug;
357 break; 852 break;
853
358 case L2X0_CACHE_ID_PART_L210: 854 case L2X0_CACHE_ID_PART_L210:
855 case L2X0_CACHE_ID_PART_L220:
359 ways = (aux >> 13) & 0xf; 856 ways = (aux >> 13) & 0xf;
360 type = "L210";
361 break; 857 break;
362 858
363 case AURORA_CACHE_ID: 859 case AURORA_CACHE_ID:
364 sync_reg_offset = AURORA_SYNC_REG;
365 ways = (aux >> 13) & 0xf; 860 ways = (aux >> 13) & 0xf;
366 ways = 2 << ((ways + 1) >> 2); 861 ways = 2 << ((ways + 1) >> 2);
367 way_size_shift = AURORA_WAY_SIZE_SHIFT;
368 type = "Aurora";
369 break; 862 break;
863
370 default: 864 default:
371 /* Assume unknown chips have 8 ways */ 865 /* Assume unknown chips have 8 ways */
372 ways = 8; 866 ways = 8;
373 type = "L2x0 series";
374 break; 867 break;
375 } 868 }
376 869
377 l2x0_way_mask = (1 << ways) - 1; 870 l2x0_way_mask = (1 << ways) - 1;
378 871
379 /* 872 /*
380 * L2 cache Size = Way size * Number of ways 873 * way_size_0 is the size that a way_size value of zero would be
874 * given the calculation: way_size = way_size_0 << way_size_bits.
875 * So, if way_size_bits=0 is reserved, but way_size_bits=1 is 16k,
876 * then way_size_0 would be 8k.
877 *
878 * L2 cache size = number of ways * way size.
381 */ 879 */
382 way_size = (aux & L2X0_AUX_CTRL_WAY_SIZE_MASK) >> 17; 880 way_size_bits = (aux & L2C_AUX_CTRL_WAY_SIZE_MASK) >>
383 way_size = 1 << (way_size + way_size_shift); 881 L2C_AUX_CTRL_WAY_SIZE_SHIFT;
882 l2x0_size = ways * (data->way_size_0 << way_size_bits);
384 883
385 l2x0_size = ways * way_size * SZ_1K; 884 fns = data->outer_cache;
885 fns.write_sec = outer_cache.write_sec;
886 if (data->fixup)
887 data->fixup(l2x0_base, cache_id, &fns);
386 888
387 /* 889 /*
388 * Check if l2x0 controller is already enabled. 890 * Check if l2x0 controller is already enabled. If we are booting
389 * If you are booting from non-secure mode 891 * in non-secure mode accessing the below registers will fault.
390 * accessing the below registers will fault.
391 */ 892 */
392 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) { 893 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN))
393 /* Make sure that I&D is not locked down when starting */ 894 data->enable(l2x0_base, aux, data->num_lock);
394 l2x0_unlock(cache_id);
395 895
396 /* l2x0 controller is disabled */ 896 outer_cache = fns;
397 writel_relaxed(aux, l2x0_base + L2X0_AUX_CTRL);
398 897
399 l2x0_inv_all(); 898 /*
400 899 * It is strange to save the register state before initialisation,
401 /* enable L2X0 */ 900 * but hey, this is what the DT implementations decided to do.
402 writel_relaxed(L2X0_CTRL_EN, l2x0_base + L2X0_CTRL); 901 */
403 } 902 if (data->save)
903 data->save(l2x0_base);
404 904
405 /* Re-read it in case some bits are reserved. */ 905 /* Re-read it in case some bits are reserved. */
406 aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL); 906 aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
407 907
408 /* Save the value for resuming. */ 908 pr_info("%s cache controller enabled, %d ways, %d kB\n",
409 l2x0_saved_regs.aux_ctrl = aux; 909 data->type, ways, l2x0_size >> 10);
910 pr_info("%s: CACHE_ID 0x%08x, AUX_CTRL 0x%08x\n",
911 data->type, cache_id, aux);
912}
913
914void __init l2x0_init(void __iomem *base, u32 aux_val, u32 aux_mask)
915{
916 const struct l2c_init_data *data;
917 u32 cache_id;
918
919 l2x0_base = base;
920
921 cache_id = readl_relaxed(base + L2X0_CACHE_ID);
922
923 switch (cache_id & L2X0_CACHE_ID_PART_MASK) {
924 default:
925 case L2X0_CACHE_ID_PART_L210:
926 data = &l2c210_data;
927 break;
410 928
411 if (!of_init) { 929 case L2X0_CACHE_ID_PART_L220:
412 outer_cache.inv_range = l2x0_inv_range; 930 data = &l2c220_data;
413 outer_cache.clean_range = l2x0_clean_range; 931 break;
414 outer_cache.flush_range = l2x0_flush_range; 932
415 outer_cache.sync = l2x0_cache_sync; 933 case L2X0_CACHE_ID_PART_L310:
416 outer_cache.flush_all = l2x0_flush_all; 934 data = &l2c310_init_fns;
417 outer_cache.inv_all = l2x0_inv_all; 935 break;
418 outer_cache.disable = l2x0_disable;
419 } 936 }
420 937
421 pr_info("%s cache controller enabled\n", type); 938 __l2c_init(data, aux_val, aux_mask, cache_id);
422 pr_info("l2x0: %d ways, CACHE_ID 0x%08x, AUX_CTRL 0x%08x, Cache size: %d kB\n",
423 ways, cache_id, aux, l2x0_size >> 10);
424} 939}
425 940
426#ifdef CONFIG_OF 941#ifdef CONFIG_OF
427static int l2_wt_override; 942static int l2_wt_override;
428 943
944/* Aurora don't have the cache ID register available, so we have to
945 * pass it though the device tree */
946static u32 cache_id_part_number_from_dt;
947
948static void __init l2x0_of_parse(const struct device_node *np,
949 u32 *aux_val, u32 *aux_mask)
950{
951 u32 data[2] = { 0, 0 };
952 u32 tag = 0;
953 u32 dirty = 0;
954 u32 val = 0, mask = 0;
955
956 of_property_read_u32(np, "arm,tag-latency", &tag);
957 if (tag) {
958 mask |= L2X0_AUX_CTRL_TAG_LATENCY_MASK;
959 val |= (tag - 1) << L2X0_AUX_CTRL_TAG_LATENCY_SHIFT;
960 }
961
962 of_property_read_u32_array(np, "arm,data-latency",
963 data, ARRAY_SIZE(data));
964 if (data[0] && data[1]) {
965 mask |= L2X0_AUX_CTRL_DATA_RD_LATENCY_MASK |
966 L2X0_AUX_CTRL_DATA_WR_LATENCY_MASK;
967 val |= ((data[0] - 1) << L2X0_AUX_CTRL_DATA_RD_LATENCY_SHIFT) |
968 ((data[1] - 1) << L2X0_AUX_CTRL_DATA_WR_LATENCY_SHIFT);
969 }
970
971 of_property_read_u32(np, "arm,dirty-latency", &dirty);
972 if (dirty) {
973 mask |= L2X0_AUX_CTRL_DIRTY_LATENCY_MASK;
974 val |= (dirty - 1) << L2X0_AUX_CTRL_DIRTY_LATENCY_SHIFT;
975 }
976
977 *aux_val &= ~mask;
978 *aux_val |= val;
979 *aux_mask &= ~mask;
980}
981
982static const struct l2c_init_data of_l2c210_data __initconst = {
983 .type = "L2C-210",
984 .way_size_0 = SZ_8K,
985 .num_lock = 1,
986 .of_parse = l2x0_of_parse,
987 .enable = l2c_enable,
988 .save = l2c_save,
989 .outer_cache = {
990 .inv_range = l2c210_inv_range,
991 .clean_range = l2c210_clean_range,
992 .flush_range = l2c210_flush_range,
993 .flush_all = l2c210_flush_all,
994 .disable = l2c_disable,
995 .sync = l2c210_sync,
996 .resume = l2c210_resume,
997 },
998};
999
1000static const struct l2c_init_data of_l2c220_data __initconst = {
1001 .type = "L2C-220",
1002 .way_size_0 = SZ_8K,
1003 .num_lock = 1,
1004 .of_parse = l2x0_of_parse,
1005 .enable = l2c220_enable,
1006 .save = l2c_save,
1007 .outer_cache = {
1008 .inv_range = l2c220_inv_range,
1009 .clean_range = l2c220_clean_range,
1010 .flush_range = l2c220_flush_range,
1011 .flush_all = l2c220_flush_all,
1012 .disable = l2c_disable,
1013 .sync = l2c220_sync,
1014 .resume = l2c210_resume,
1015 },
1016};
1017
1018static void __init l2c310_of_parse(const struct device_node *np,
1019 u32 *aux_val, u32 *aux_mask)
1020{
1021 u32 data[3] = { 0, 0, 0 };
1022 u32 tag[3] = { 0, 0, 0 };
1023 u32 filter[2] = { 0, 0 };
1024
1025 of_property_read_u32_array(np, "arm,tag-latency", tag, ARRAY_SIZE(tag));
1026 if (tag[0] && tag[1] && tag[2])
1027 writel_relaxed(
1028 L310_LATENCY_CTRL_RD(tag[0] - 1) |
1029 L310_LATENCY_CTRL_WR(tag[1] - 1) |
1030 L310_LATENCY_CTRL_SETUP(tag[2] - 1),
1031 l2x0_base + L310_TAG_LATENCY_CTRL);
1032
1033 of_property_read_u32_array(np, "arm,data-latency",
1034 data, ARRAY_SIZE(data));
1035 if (data[0] && data[1] && data[2])
1036 writel_relaxed(
1037 L310_LATENCY_CTRL_RD(data[0] - 1) |
1038 L310_LATENCY_CTRL_WR(data[1] - 1) |
1039 L310_LATENCY_CTRL_SETUP(data[2] - 1),
1040 l2x0_base + L310_DATA_LATENCY_CTRL);
1041
1042 of_property_read_u32_array(np, "arm,filter-ranges",
1043 filter, ARRAY_SIZE(filter));
1044 if (filter[1]) {
1045 writel_relaxed(ALIGN(filter[0] + filter[1], SZ_1M),
1046 l2x0_base + L310_ADDR_FILTER_END);
1047 writel_relaxed((filter[0] & ~(SZ_1M - 1)) | L310_ADDR_FILTER_EN,
1048 l2x0_base + L310_ADDR_FILTER_START);
1049 }
1050}
1051
1052static const struct l2c_init_data of_l2c310_data __initconst = {
1053 .type = "L2C-310",
1054 .way_size_0 = SZ_8K,
1055 .num_lock = 8,
1056 .of_parse = l2c310_of_parse,
1057 .enable = l2c310_enable,
1058 .fixup = l2c310_fixup,
1059 .save = l2c310_save,
1060 .outer_cache = {
1061 .inv_range = l2c210_inv_range,
1062 .clean_range = l2c210_clean_range,
1063 .flush_range = l2c210_flush_range,
1064 .flush_all = l2c210_flush_all,
1065 .disable = l2c310_disable,
1066 .sync = l2c210_sync,
1067 .resume = l2c310_resume,
1068 },
1069};
1070
429/* 1071/*
430 * Note that the end addresses passed to Linux primitives are 1072 * Note that the end addresses passed to Linux primitives are
431 * noninclusive, while the hardware cache range operations use 1073 * noninclusive, while the hardware cache range operations use
@@ -524,6 +1166,100 @@ static void aurora_flush_range(unsigned long start, unsigned long end)
524 } 1166 }
525} 1167}
526 1168
1169static void aurora_save(void __iomem *base)
1170{
1171 l2x0_saved_regs.ctrl = readl_relaxed(base + L2X0_CTRL);
1172 l2x0_saved_regs.aux_ctrl = readl_relaxed(base + L2X0_AUX_CTRL);
1173}
1174
1175static void aurora_resume(void)
1176{
1177 void __iomem *base = l2x0_base;
1178
1179 if (!(readl(base + L2X0_CTRL) & L2X0_CTRL_EN)) {
1180 writel_relaxed(l2x0_saved_regs.aux_ctrl, base + L2X0_AUX_CTRL);
1181 writel_relaxed(l2x0_saved_regs.ctrl, base + L2X0_CTRL);
1182 }
1183}
1184
1185/*
1186 * For Aurora cache in no outer mode, enable via the CP15 coprocessor
1187 * broadcasting of cache commands to L2.
1188 */
1189static void __init aurora_enable_no_outer(void __iomem *base, u32 aux,
1190 unsigned num_lock)
1191{
1192 u32 u;
1193
1194 asm volatile("mrc p15, 1, %0, c15, c2, 0" : "=r" (u));
1195 u |= AURORA_CTRL_FW; /* Set the FW bit */
1196 asm volatile("mcr p15, 1, %0, c15, c2, 0" : : "r" (u));
1197
1198 isb();
1199
1200 l2c_enable(base, aux, num_lock);
1201}
1202
1203static void __init aurora_fixup(void __iomem *base, u32 cache_id,
1204 struct outer_cache_fns *fns)
1205{
1206 sync_reg_offset = AURORA_SYNC_REG;
1207}
1208
1209static void __init aurora_of_parse(const struct device_node *np,
1210 u32 *aux_val, u32 *aux_mask)
1211{
1212 u32 val = AURORA_ACR_REPLACEMENT_TYPE_SEMIPLRU;
1213 u32 mask = AURORA_ACR_REPLACEMENT_MASK;
1214
1215 of_property_read_u32(np, "cache-id-part",
1216 &cache_id_part_number_from_dt);
1217
1218 /* Determine and save the write policy */
1219 l2_wt_override = of_property_read_bool(np, "wt-override");
1220
1221 if (l2_wt_override) {
1222 val |= AURORA_ACR_FORCE_WRITE_THRO_POLICY;
1223 mask |= AURORA_ACR_FORCE_WRITE_POLICY_MASK;
1224 }
1225
1226 *aux_val &= ~mask;
1227 *aux_val |= val;
1228 *aux_mask &= ~mask;
1229}
1230
1231static const struct l2c_init_data of_aurora_with_outer_data __initconst = {
1232 .type = "Aurora",
1233 .way_size_0 = SZ_4K,
1234 .num_lock = 4,
1235 .of_parse = aurora_of_parse,
1236 .enable = l2c_enable,
1237 .fixup = aurora_fixup,
1238 .save = aurora_save,
1239 .outer_cache = {
1240 .inv_range = aurora_inv_range,
1241 .clean_range = aurora_clean_range,
1242 .flush_range = aurora_flush_range,
1243 .flush_all = l2x0_flush_all,
1244 .disable = l2x0_disable,
1245 .sync = l2x0_cache_sync,
1246 .resume = aurora_resume,
1247 },
1248};
1249
1250static const struct l2c_init_data of_aurora_no_outer_data __initconst = {
1251 .type = "Aurora",
1252 .way_size_0 = SZ_4K,
1253 .num_lock = 4,
1254 .of_parse = aurora_of_parse,
1255 .enable = aurora_enable_no_outer,
1256 .fixup = aurora_fixup,
1257 .save = aurora_save,
1258 .outer_cache = {
1259 .resume = aurora_resume,
1260 },
1261};
1262
527/* 1263/*
528 * For certain Broadcom SoCs, depending on the address range, different offsets 1264 * For certain Broadcom SoCs, depending on the address range, different offsets
529 * need to be added to the address before passing it to L2 for 1265 * need to be added to the address before passing it to L2 for
@@ -588,16 +1324,16 @@ static void bcm_inv_range(unsigned long start, unsigned long end)
588 1324
589 /* normal case, no cross section between start and end */ 1325 /* normal case, no cross section between start and end */
590 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) { 1326 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) {
591 l2x0_inv_range(new_start, new_end); 1327 l2c210_inv_range(new_start, new_end);
592 return; 1328 return;
593 } 1329 }
594 1330
595 /* They cross sections, so it can only be a cross from section 1331 /* They cross sections, so it can only be a cross from section
596 * 2 to section 3 1332 * 2 to section 3
597 */ 1333 */
598 l2x0_inv_range(new_start, 1334 l2c210_inv_range(new_start,
599 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1)); 1335 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));
600 l2x0_inv_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR), 1336 l2c210_inv_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR),
601 new_end); 1337 new_end);
602} 1338}
603 1339
@@ -610,26 +1346,21 @@ static void bcm_clean_range(unsigned long start, unsigned long end)
610 if (unlikely(end <= start)) 1346 if (unlikely(end <= start))
611 return; 1347 return;
612 1348
613 if ((end - start) >= l2x0_size) {
614 l2x0_clean_all();
615 return;
616 }
617
618 new_start = bcm_l2_phys_addr(start); 1349 new_start = bcm_l2_phys_addr(start);
619 new_end = bcm_l2_phys_addr(end); 1350 new_end = bcm_l2_phys_addr(end);
620 1351
621 /* normal case, no cross section between start and end */ 1352 /* normal case, no cross section between start and end */
622 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) { 1353 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) {
623 l2x0_clean_range(new_start, new_end); 1354 l2c210_clean_range(new_start, new_end);
624 return; 1355 return;
625 } 1356 }
626 1357
627 /* They cross sections, so it can only be a cross from section 1358 /* They cross sections, so it can only be a cross from section
628 * 2 to section 3 1359 * 2 to section 3
629 */ 1360 */
630 l2x0_clean_range(new_start, 1361 l2c210_clean_range(new_start,
631 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1)); 1362 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));
632 l2x0_clean_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR), 1363 l2c210_clean_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR),
633 new_end); 1364 new_end);
634} 1365}
635 1366
@@ -643,7 +1374,7 @@ static void bcm_flush_range(unsigned long start, unsigned long end)
643 return; 1374 return;
644 1375
645 if ((end - start) >= l2x0_size) { 1376 if ((end - start) >= l2x0_size) {
646 l2x0_flush_all(); 1377 outer_cache.flush_all();
647 return; 1378 return;
648 } 1379 }
649 1380
@@ -652,283 +1383,67 @@ static void bcm_flush_range(unsigned long start, unsigned long end)
652 1383
653 /* normal case, no cross section between start and end */ 1384 /* normal case, no cross section between start and end */
654 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) { 1385 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) {
655 l2x0_flush_range(new_start, new_end); 1386 l2c210_flush_range(new_start, new_end);
656 return; 1387 return;
657 } 1388 }
658 1389
659 /* They cross sections, so it can only be a cross from section 1390 /* They cross sections, so it can only be a cross from section
660 * 2 to section 3 1391 * 2 to section 3
661 */ 1392 */
662 l2x0_flush_range(new_start, 1393 l2c210_flush_range(new_start,
663 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1)); 1394 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));
664 l2x0_flush_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR), 1395 l2c210_flush_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR),
665 new_end); 1396 new_end);
666} 1397}
667 1398
668static void __init l2x0_of_setup(const struct device_node *np, 1399/* Broadcom L2C-310 start from ARMs R3P2 or later, and require no fixups */
669 u32 *aux_val, u32 *aux_mask) 1400static const struct l2c_init_data of_bcm_l2x0_data __initconst = {
670{ 1401 .type = "BCM-L2C-310",
671 u32 data[2] = { 0, 0 }; 1402 .way_size_0 = SZ_8K,
672 u32 tag = 0; 1403 .num_lock = 8,
673 u32 dirty = 0; 1404 .of_parse = l2c310_of_parse,
674 u32 val = 0, mask = 0; 1405 .enable = l2c310_enable,
675 1406 .save = l2c310_save,
676 of_property_read_u32(np, "arm,tag-latency", &tag); 1407 .outer_cache = {
677 if (tag) { 1408 .inv_range = bcm_inv_range,
678 mask |= L2X0_AUX_CTRL_TAG_LATENCY_MASK; 1409 .clean_range = bcm_clean_range,
679 val |= (tag - 1) << L2X0_AUX_CTRL_TAG_LATENCY_SHIFT; 1410 .flush_range = bcm_flush_range,
680 } 1411 .flush_all = l2c210_flush_all,
681 1412 .disable = l2c310_disable,
682 of_property_read_u32_array(np, "arm,data-latency", 1413 .sync = l2c210_sync,
683 data, ARRAY_SIZE(data)); 1414 .resume = l2c310_resume,
684 if (data[0] && data[1]) { 1415 },
685 mask |= L2X0_AUX_CTRL_DATA_RD_LATENCY_MASK | 1416};
686 L2X0_AUX_CTRL_DATA_WR_LATENCY_MASK;
687 val |= ((data[0] - 1) << L2X0_AUX_CTRL_DATA_RD_LATENCY_SHIFT) |
688 ((data[1] - 1) << L2X0_AUX_CTRL_DATA_WR_LATENCY_SHIFT);
689 }
690
691 of_property_read_u32(np, "arm,dirty-latency", &dirty);
692 if (dirty) {
693 mask |= L2X0_AUX_CTRL_DIRTY_LATENCY_MASK;
694 val |= (dirty - 1) << L2X0_AUX_CTRL_DIRTY_LATENCY_SHIFT;
695 }
696
697 *aux_val &= ~mask;
698 *aux_val |= val;
699 *aux_mask &= ~mask;
700}
701
702static void __init pl310_of_setup(const struct device_node *np,
703 u32 *aux_val, u32 *aux_mask)
704{
705 u32 data[3] = { 0, 0, 0 };
706 u32 tag[3] = { 0, 0, 0 };
707 u32 filter[2] = { 0, 0 };
708
709 of_property_read_u32_array(np, "arm,tag-latency", tag, ARRAY_SIZE(tag));
710 if (tag[0] && tag[1] && tag[2])
711 writel_relaxed(
712 ((tag[0] - 1) << L2X0_LATENCY_CTRL_RD_SHIFT) |
713 ((tag[1] - 1) << L2X0_LATENCY_CTRL_WR_SHIFT) |
714 ((tag[2] - 1) << L2X0_LATENCY_CTRL_SETUP_SHIFT),
715 l2x0_base + L2X0_TAG_LATENCY_CTRL);
716
717 of_property_read_u32_array(np, "arm,data-latency",
718 data, ARRAY_SIZE(data));
719 if (data[0] && data[1] && data[2])
720 writel_relaxed(
721 ((data[0] - 1) << L2X0_LATENCY_CTRL_RD_SHIFT) |
722 ((data[1] - 1) << L2X0_LATENCY_CTRL_WR_SHIFT) |
723 ((data[2] - 1) << L2X0_LATENCY_CTRL_SETUP_SHIFT),
724 l2x0_base + L2X0_DATA_LATENCY_CTRL);
725
726 of_property_read_u32_array(np, "arm,filter-ranges",
727 filter, ARRAY_SIZE(filter));
728 if (filter[1]) {
729 writel_relaxed(ALIGN(filter[0] + filter[1], SZ_1M),
730 l2x0_base + L2X0_ADDR_FILTER_END);
731 writel_relaxed((filter[0] & ~(SZ_1M - 1)) | L2X0_ADDR_FILTER_EN,
732 l2x0_base + L2X0_ADDR_FILTER_START);
733 }
734}
735
736static void __init pl310_save(void)
737{
738 u32 l2x0_revision = readl_relaxed(l2x0_base + L2X0_CACHE_ID) &
739 L2X0_CACHE_ID_RTL_MASK;
740
741 l2x0_saved_regs.tag_latency = readl_relaxed(l2x0_base +
742 L2X0_TAG_LATENCY_CTRL);
743 l2x0_saved_regs.data_latency = readl_relaxed(l2x0_base +
744 L2X0_DATA_LATENCY_CTRL);
745 l2x0_saved_regs.filter_end = readl_relaxed(l2x0_base +
746 L2X0_ADDR_FILTER_END);
747 l2x0_saved_regs.filter_start = readl_relaxed(l2x0_base +
748 L2X0_ADDR_FILTER_START);
749
750 if (l2x0_revision >= L2X0_CACHE_ID_RTL_R2P0) {
751 /*
752 * From r2p0, there is Prefetch offset/control register
753 */
754 l2x0_saved_regs.prefetch_ctrl = readl_relaxed(l2x0_base +
755 L2X0_PREFETCH_CTRL);
756 /*
757 * From r3p0, there is Power control register
758 */
759 if (l2x0_revision >= L2X0_CACHE_ID_RTL_R3P0)
760 l2x0_saved_regs.pwr_ctrl = readl_relaxed(l2x0_base +
761 L2X0_POWER_CTRL);
762 }
763}
764 1417
765static void aurora_save(void) 1418static void __init tauros3_save(void __iomem *base)
766{ 1419{
767 l2x0_saved_regs.ctrl = readl_relaxed(l2x0_base + L2X0_CTRL); 1420 l2c_save(base);
768 l2x0_saved_regs.aux_ctrl = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
769}
770 1421
771static void __init tauros3_save(void)
772{
773 l2x0_saved_regs.aux2_ctrl = 1422 l2x0_saved_regs.aux2_ctrl =
774 readl_relaxed(l2x0_base + TAUROS3_AUX2_CTRL); 1423 readl_relaxed(base + TAUROS3_AUX2_CTRL);
775 l2x0_saved_regs.prefetch_ctrl = 1424 l2x0_saved_regs.prefetch_ctrl =
776 readl_relaxed(l2x0_base + L2X0_PREFETCH_CTRL); 1425 readl_relaxed(base + L310_PREFETCH_CTRL);
777}
778
779static void l2x0_resume(void)
780{
781 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) {
782 /* restore aux ctrl and enable l2 */
783 l2x0_unlock(readl_relaxed(l2x0_base + L2X0_CACHE_ID));
784
785 writel_relaxed(l2x0_saved_regs.aux_ctrl, l2x0_base +
786 L2X0_AUX_CTRL);
787
788 l2x0_inv_all();
789
790 writel_relaxed(L2X0_CTRL_EN, l2x0_base + L2X0_CTRL);
791 }
792}
793
794static void pl310_resume(void)
795{
796 u32 l2x0_revision;
797
798 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) {
799 /* restore pl310 setup */
800 writel_relaxed(l2x0_saved_regs.tag_latency,
801 l2x0_base + L2X0_TAG_LATENCY_CTRL);
802 writel_relaxed(l2x0_saved_regs.data_latency,
803 l2x0_base + L2X0_DATA_LATENCY_CTRL);
804 writel_relaxed(l2x0_saved_regs.filter_end,
805 l2x0_base + L2X0_ADDR_FILTER_END);
806 writel_relaxed(l2x0_saved_regs.filter_start,
807 l2x0_base + L2X0_ADDR_FILTER_START);
808
809 l2x0_revision = readl_relaxed(l2x0_base + L2X0_CACHE_ID) &
810 L2X0_CACHE_ID_RTL_MASK;
811
812 if (l2x0_revision >= L2X0_CACHE_ID_RTL_R2P0) {
813 writel_relaxed(l2x0_saved_regs.prefetch_ctrl,
814 l2x0_base + L2X0_PREFETCH_CTRL);
815 if (l2x0_revision >= L2X0_CACHE_ID_RTL_R3P0)
816 writel_relaxed(l2x0_saved_regs.pwr_ctrl,
817 l2x0_base + L2X0_POWER_CTRL);
818 }
819 }
820
821 l2x0_resume();
822}
823
824static void aurora_resume(void)
825{
826 if (!(readl(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) {
827 writel_relaxed(l2x0_saved_regs.aux_ctrl,
828 l2x0_base + L2X0_AUX_CTRL);
829 writel_relaxed(l2x0_saved_regs.ctrl, l2x0_base + L2X0_CTRL);
830 }
831} 1426}
832 1427
833static void tauros3_resume(void) 1428static void tauros3_resume(void)
834{ 1429{
835 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) { 1430 void __iomem *base = l2x0_base;
1431
1432 if (!(readl_relaxed(base + L2X0_CTRL) & L2X0_CTRL_EN)) {
836 writel_relaxed(l2x0_saved_regs.aux2_ctrl, 1433 writel_relaxed(l2x0_saved_regs.aux2_ctrl,
837 l2x0_base + TAUROS3_AUX2_CTRL); 1434 base + TAUROS3_AUX2_CTRL);
838 writel_relaxed(l2x0_saved_regs.prefetch_ctrl, 1435 writel_relaxed(l2x0_saved_regs.prefetch_ctrl,
839 l2x0_base + L2X0_PREFETCH_CTRL); 1436 base + L310_PREFETCH_CTRL);
840 }
841 1437
842 l2x0_resume(); 1438 l2c_enable(base, l2x0_saved_regs.aux_ctrl, 8);
843}
844
845static void __init aurora_broadcast_l2_commands(void)
846{
847 __u32 u;
848 /* Enable Broadcasting of cache commands to L2*/
849 __asm__ __volatile__("mrc p15, 1, %0, c15, c2, 0" : "=r"(u));
850 u |= AURORA_CTRL_FW; /* Set the FW bit */
851 __asm__ __volatile__("mcr p15, 1, %0, c15, c2, 0\n" : : "r"(u));
852 isb();
853}
854
855static void __init aurora_of_setup(const struct device_node *np,
856 u32 *aux_val, u32 *aux_mask)
857{
858 u32 val = AURORA_ACR_REPLACEMENT_TYPE_SEMIPLRU;
859 u32 mask = AURORA_ACR_REPLACEMENT_MASK;
860
861 of_property_read_u32(np, "cache-id-part",
862 &cache_id_part_number_from_dt);
863
864 /* Determine and save the write policy */
865 l2_wt_override = of_property_read_bool(np, "wt-override");
866
867 if (l2_wt_override) {
868 val |= AURORA_ACR_FORCE_WRITE_THRO_POLICY;
869 mask |= AURORA_ACR_FORCE_WRITE_POLICY_MASK;
870 } 1439 }
871
872 *aux_val &= ~mask;
873 *aux_val |= val;
874 *aux_mask &= ~mask;
875} 1440}
876 1441
877static const struct l2x0_of_data pl310_data = { 1442static const struct l2c_init_data of_tauros3_data __initconst = {
878 .setup = pl310_of_setup, 1443 .type = "Tauros3",
879 .save = pl310_save, 1444 .way_size_0 = SZ_8K,
880 .outer_cache = { 1445 .num_lock = 8,
881 .resume = pl310_resume, 1446 .enable = l2c_enable,
882 .inv_range = l2x0_inv_range,
883 .clean_range = l2x0_clean_range,
884 .flush_range = l2x0_flush_range,
885 .sync = l2x0_cache_sync,
886 .flush_all = l2x0_flush_all,
887 .inv_all = l2x0_inv_all,
888 .disable = l2x0_disable,
889 },
890};
891
892static const struct l2x0_of_data l2x0_data = {
893 .setup = l2x0_of_setup,
894 .save = NULL,
895 .outer_cache = {
896 .resume = l2x0_resume,
897 .inv_range = l2x0_inv_range,
898 .clean_range = l2x0_clean_range,
899 .flush_range = l2x0_flush_range,
900 .sync = l2x0_cache_sync,
901 .flush_all = l2x0_flush_all,
902 .inv_all = l2x0_inv_all,
903 .disable = l2x0_disable,
904 },
905};
906
907static const struct l2x0_of_data aurora_with_outer_data = {
908 .setup = aurora_of_setup,
909 .save = aurora_save,
910 .outer_cache = {
911 .resume = aurora_resume,
912 .inv_range = aurora_inv_range,
913 .clean_range = aurora_clean_range,
914 .flush_range = aurora_flush_range,
915 .sync = l2x0_cache_sync,
916 .flush_all = l2x0_flush_all,
917 .inv_all = l2x0_inv_all,
918 .disable = l2x0_disable,
919 },
920};
921
922static const struct l2x0_of_data aurora_no_outer_data = {
923 .setup = aurora_of_setup,
924 .save = aurora_save,
925 .outer_cache = {
926 .resume = aurora_resume,
927 },
928};
929
930static const struct l2x0_of_data tauros3_data = {
931 .setup = NULL,
932 .save = tauros3_save, 1447 .save = tauros3_save,
933 /* Tauros3 broadcasts L1 cache operations to L2 */ 1448 /* Tauros3 broadcasts L1 cache operations to L2 */
934 .outer_cache = { 1449 .outer_cache = {
@@ -936,43 +1451,26 @@ static const struct l2x0_of_data tauros3_data = {
936 }, 1451 },
937}; 1452};
938 1453
939static const struct l2x0_of_data bcm_l2x0_data = { 1454#define L2C_ID(name, fns) { .compatible = name, .data = (void *)&fns }
940 .setup = pl310_of_setup,
941 .save = pl310_save,
942 .outer_cache = {
943 .resume = pl310_resume,
944 .inv_range = bcm_inv_range,
945 .clean_range = bcm_clean_range,
946 .flush_range = bcm_flush_range,
947 .sync = l2x0_cache_sync,
948 .flush_all = l2x0_flush_all,
949 .inv_all = l2x0_inv_all,
950 .disable = l2x0_disable,
951 },
952};
953
954static const struct of_device_id l2x0_ids[] __initconst = { 1455static const struct of_device_id l2x0_ids[] __initconst = {
955 { .compatible = "arm,l210-cache", .data = (void *)&l2x0_data }, 1456 L2C_ID("arm,l210-cache", of_l2c210_data),
956 { .compatible = "arm,l220-cache", .data = (void *)&l2x0_data }, 1457 L2C_ID("arm,l220-cache", of_l2c220_data),
957 { .compatible = "arm,pl310-cache", .data = (void *)&pl310_data }, 1458 L2C_ID("arm,pl310-cache", of_l2c310_data),
958 { .compatible = "bcm,bcm11351-a2-pl310-cache", /* deprecated name */ 1459 L2C_ID("brcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data),
959 .data = (void *)&bcm_l2x0_data}, 1460 L2C_ID("marvell,aurora-outer-cache", of_aurora_with_outer_data),
960 { .compatible = "brcm,bcm11351-a2-pl310-cache", 1461 L2C_ID("marvell,aurora-system-cache", of_aurora_no_outer_data),
961 .data = (void *)&bcm_l2x0_data}, 1462 L2C_ID("marvell,tauros3-cache", of_tauros3_data),
962 { .compatible = "marvell,aurora-outer-cache", 1463 /* Deprecated IDs */
963 .data = (void *)&aurora_with_outer_data}, 1464 L2C_ID("bcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data),
964 { .compatible = "marvell,aurora-system-cache",
965 .data = (void *)&aurora_no_outer_data},
966 { .compatible = "marvell,tauros3-cache",
967 .data = (void *)&tauros3_data },
968 {} 1465 {}
969}; 1466};
970 1467
971int __init l2x0_of_init(u32 aux_val, u32 aux_mask) 1468int __init l2x0_of_init(u32 aux_val, u32 aux_mask)
972{ 1469{
1470 const struct l2c_init_data *data;
973 struct device_node *np; 1471 struct device_node *np;
974 const struct l2x0_of_data *data;
975 struct resource res; 1472 struct resource res;
1473 u32 cache_id, old_aux;
976 1474
977 np = of_find_matching_node(NULL, l2x0_ids); 1475 np = of_find_matching_node(NULL, l2x0_ids);
978 if (!np) 1476 if (!np)
@@ -989,23 +1487,29 @@ int __init l2x0_of_init(u32 aux_val, u32 aux_mask)
989 1487
990 data = of_match_node(l2x0_ids, np)->data; 1488 data = of_match_node(l2x0_ids, np)->data;
991 1489
992 /* L2 configuration can only be changed if the cache is disabled */ 1490 old_aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
993 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) { 1491 if (old_aux != ((old_aux & aux_mask) | aux_val)) {
994 if (data->setup) 1492 pr_warn("L2C: platform modifies aux control register: 0x%08x -> 0x%08x\n",
995 data->setup(np, &aux_val, &aux_mask); 1493 old_aux, (old_aux & aux_mask) | aux_val);
996 1494 } else if (aux_mask != ~0U && aux_val != 0) {
997 /* For aurora cache in no outer mode select the 1495 pr_alert("L2C: platform provided aux values match the hardware, so have no effect. Please remove them.\n");
998 * correct mode using the coprocessor*/
999 if (data == &aurora_no_outer_data)
1000 aurora_broadcast_l2_commands();
1001 } 1496 }
1002 1497
1003 if (data->save) 1498 /* All L2 caches are unified, so this property should be specified */
1004 data->save(); 1499 if (!of_property_read_bool(np, "cache-unified"))
1500 pr_err("L2C: device tree omits to specify unified cache\n");
1501
1502 /* L2 configuration can only be changed if the cache is disabled */
1503 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN))
1504 if (data->of_parse)
1505 data->of_parse(np, &aux_val, &aux_mask);
1506
1507 if (cache_id_part_number_from_dt)
1508 cache_id = cache_id_part_number_from_dt;
1509 else
1510 cache_id = readl_relaxed(l2x0_base + L2X0_CACHE_ID);
1005 1511
1006 of_init = true; 1512 __l2c_init(data, aux_val, aux_mask, cache_id);
1007 memcpy(&outer_cache, &data->outer_cache, sizeof(outer_cache));
1008 l2x0_init(l2x0_base, aux_val, aux_mask);
1009 1513
1010 return 0; 1514 return 0;
1011} 1515}
diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S
index 778bcf88ee79..615c99e38ba1 100644
--- a/arch/arm/mm/cache-v7.S
+++ b/arch/arm/mm/cache-v7.S
@@ -59,7 +59,7 @@ ENTRY(v7_invalidate_l1)
59 bgt 2b 59 bgt 2b
60 cmp r2, #0 60 cmp r2, #0
61 bgt 1b 61 bgt 1b
62 dsb 62 dsb st
63 isb 63 isb
64 mov pc, lr 64 mov pc, lr
65ENDPROC(v7_invalidate_l1) 65ENDPROC(v7_invalidate_l1)
@@ -166,7 +166,7 @@ skip:
166finished: 166finished:
167 mov r10, #0 @ swith back to cache level 0 167 mov r10, #0 @ swith back to cache level 0
168 mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr 168 mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr
169 dsb 169 dsb st
170 isb 170 isb
171 mov pc, lr 171 mov pc, lr
172ENDPROC(v7_flush_dcache_all) 172ENDPROC(v7_flush_dcache_all)
@@ -335,7 +335,7 @@ ENTRY(v7_flush_kern_dcache_area)
335 add r0, r0, r2 335 add r0, r0, r2
336 cmp r0, r1 336 cmp r0, r1
337 blo 1b 337 blo 1b
338 dsb 338 dsb st
339 mov pc, lr 339 mov pc, lr
340ENDPROC(v7_flush_kern_dcache_area) 340ENDPROC(v7_flush_kern_dcache_area)
341 341
@@ -368,7 +368,7 @@ v7_dma_inv_range:
368 add r0, r0, r2 368 add r0, r0, r2
369 cmp r0, r1 369 cmp r0, r1
370 blo 1b 370 blo 1b
371 dsb 371 dsb st
372 mov pc, lr 372 mov pc, lr
373ENDPROC(v7_dma_inv_range) 373ENDPROC(v7_dma_inv_range)
374 374
@@ -390,7 +390,7 @@ v7_dma_clean_range:
390 add r0, r0, r2 390 add r0, r0, r2
391 cmp r0, r1 391 cmp r0, r1
392 blo 1b 392 blo 1b
393 dsb 393 dsb st
394 mov pc, lr 394 mov pc, lr
395ENDPROC(v7_dma_clean_range) 395ENDPROC(v7_dma_clean_range)
396 396
@@ -412,7 +412,7 @@ ENTRY(v7_dma_flush_range)
412 add r0, r0, r2 412 add r0, r0, r2
413 cmp r0, r1 413 cmp r0, r1
414 blo 1b 414 blo 1b
415 dsb 415 dsb st
416 mov pc, lr 416 mov pc, lr
417ENDPROC(v7_dma_flush_range) 417ENDPROC(v7_dma_flush_range)
418 418
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index 5bef858568e6..4c88935654ca 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -885,7 +885,7 @@ static void dma_cache_maint_page(struct page *page, unsigned long offset,
885static void __dma_page_cpu_to_dev(struct page *page, unsigned long off, 885static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
886 size_t size, enum dma_data_direction dir) 886 size_t size, enum dma_data_direction dir)
887{ 887{
888 unsigned long paddr; 888 phys_addr_t paddr;
889 889
890 dma_cache_maint_page(page, off, size, dir, dmac_map_area); 890 dma_cache_maint_page(page, off, size, dir, dmac_map_area);
891 891
@@ -901,14 +901,15 @@ static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
901static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, 901static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
902 size_t size, enum dma_data_direction dir) 902 size_t size, enum dma_data_direction dir)
903{ 903{
904 unsigned long paddr = page_to_phys(page) + off; 904 phys_addr_t paddr = page_to_phys(page) + off;
905 905
906 /* FIXME: non-speculating: not required */ 906 /* FIXME: non-speculating: not required */
907 /* don't bother invalidating if DMA to device */ 907 /* in any case, don't bother invalidating if DMA to device */
908 if (dir != DMA_TO_DEVICE) 908 if (dir != DMA_TO_DEVICE) {
909 outer_inv_range(paddr, paddr + size); 909 outer_inv_range(paddr, paddr + size);
910 910
911 dma_cache_maint_page(page, off, size, dir, dmac_unmap_area); 911 dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
912 }
912 913
913 /* 914 /*
914 * Mark the D-cache clean for these pages to avoid extra flushing. 915 * Mark the D-cache clean for these pages to avoid extra flushing.
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index 3387e60e4ea3..43d54f5b26b9 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -104,17 +104,20 @@ void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsig
104#define flush_icache_alias(pfn,vaddr,len) do { } while (0) 104#define flush_icache_alias(pfn,vaddr,len) do { } while (0)
105#endif 105#endif
106 106
107#define FLAG_PA_IS_EXEC 1
108#define FLAG_PA_CORE_IN_MM 2
109
107static void flush_ptrace_access_other(void *args) 110static void flush_ptrace_access_other(void *args)
108{ 111{
109 __flush_icache_all(); 112 __flush_icache_all();
110} 113}
111 114
112static 115static inline
113void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, 116void __flush_ptrace_access(struct page *page, unsigned long uaddr, void *kaddr,
114 unsigned long uaddr, void *kaddr, unsigned long len) 117 unsigned long len, unsigned int flags)
115{ 118{
116 if (cache_is_vivt()) { 119 if (cache_is_vivt()) {
117 if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) { 120 if (flags & FLAG_PA_CORE_IN_MM) {
118 unsigned long addr = (unsigned long)kaddr; 121 unsigned long addr = (unsigned long)kaddr;
119 __cpuc_coherent_kern_range(addr, addr + len); 122 __cpuc_coherent_kern_range(addr, addr + len);
120 } 123 }
@@ -128,7 +131,7 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
128 } 131 }
129 132
130 /* VIPT non-aliasing D-cache */ 133 /* VIPT non-aliasing D-cache */
131 if (vma->vm_flags & VM_EXEC) { 134 if (flags & FLAG_PA_IS_EXEC) {
132 unsigned long addr = (unsigned long)kaddr; 135 unsigned long addr = (unsigned long)kaddr;
133 if (icache_is_vipt_aliasing()) 136 if (icache_is_vipt_aliasing())
134 flush_icache_alias(page_to_pfn(page), uaddr, len); 137 flush_icache_alias(page_to_pfn(page), uaddr, len);
@@ -140,6 +143,26 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
140 } 143 }
141} 144}
142 145
146static
147void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
148 unsigned long uaddr, void *kaddr, unsigned long len)
149{
150 unsigned int flags = 0;
151 if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm)))
152 flags |= FLAG_PA_CORE_IN_MM;
153 if (vma->vm_flags & VM_EXEC)
154 flags |= FLAG_PA_IS_EXEC;
155 __flush_ptrace_access(page, uaddr, kaddr, len, flags);
156}
157
158void flush_uprobe_xol_access(struct page *page, unsigned long uaddr,
159 void *kaddr, unsigned long len)
160{
161 unsigned int flags = FLAG_PA_CORE_IN_MM|FLAG_PA_IS_EXEC;
162
163 __flush_ptrace_access(page, uaddr, kaddr, len, flags);
164}
165
143/* 166/*
144 * Copy user data from/to a page which is mapped into a different 167 * Copy user data from/to a page which is mapped into a different
145 * processes address space. Really, we want to allow our "user 168 * processes address space. Really, we want to allow our "user
diff --git a/arch/arm/mm/highmem.c b/arch/arm/mm/highmem.c
index 21b9e1bf9b77..45aeaaca9052 100644
--- a/arch/arm/mm/highmem.c
+++ b/arch/arm/mm/highmem.c
@@ -18,6 +18,21 @@
18#include <asm/tlbflush.h> 18#include <asm/tlbflush.h>
19#include "mm.h" 19#include "mm.h"
20 20
21pte_t *fixmap_page_table;
22
23static inline void set_fixmap_pte(int idx, pte_t pte)
24{
25 unsigned long vaddr = __fix_to_virt(idx);
26 set_pte_ext(fixmap_page_table + idx, pte, 0);
27 local_flush_tlb_kernel_page(vaddr);
28}
29
30static inline pte_t get_fixmap_pte(unsigned long vaddr)
31{
32 unsigned long idx = __virt_to_fix(vaddr);
33 return *(fixmap_page_table + idx);
34}
35
21void *kmap(struct page *page) 36void *kmap(struct page *page)
22{ 37{
23 might_sleep(); 38 might_sleep();
@@ -63,20 +78,20 @@ void *kmap_atomic(struct page *page)
63 type = kmap_atomic_idx_push(); 78 type = kmap_atomic_idx_push();
64 79
65 idx = type + KM_TYPE_NR * smp_processor_id(); 80 idx = type + KM_TYPE_NR * smp_processor_id();
66 vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); 81 vaddr = __fix_to_virt(idx);
67#ifdef CONFIG_DEBUG_HIGHMEM 82#ifdef CONFIG_DEBUG_HIGHMEM
68 /* 83 /*
69 * With debugging enabled, kunmap_atomic forces that entry to 0. 84 * With debugging enabled, kunmap_atomic forces that entry to 0.
70 * Make sure it was indeed properly unmapped. 85 * Make sure it was indeed properly unmapped.
71 */ 86 */
72 BUG_ON(!pte_none(get_top_pte(vaddr))); 87 BUG_ON(!pte_none(*(fixmap_page_table + idx)));
73#endif 88#endif
74 /* 89 /*
75 * When debugging is off, kunmap_atomic leaves the previous mapping 90 * When debugging is off, kunmap_atomic leaves the previous mapping
76 * in place, so the contained TLB flush ensures the TLB is updated 91 * in place, so the contained TLB flush ensures the TLB is updated
77 * with the new mapping. 92 * with the new mapping.
78 */ 93 */
79 set_top_pte(vaddr, mk_pte(page, kmap_prot)); 94 set_fixmap_pte(idx, mk_pte(page, kmap_prot));
80 95
81 return (void *)vaddr; 96 return (void *)vaddr;
82} 97}
@@ -94,8 +109,8 @@ void __kunmap_atomic(void *kvaddr)
94 if (cache_is_vivt()) 109 if (cache_is_vivt())
95 __cpuc_flush_dcache_area((void *)vaddr, PAGE_SIZE); 110 __cpuc_flush_dcache_area((void *)vaddr, PAGE_SIZE);
96#ifdef CONFIG_DEBUG_HIGHMEM 111#ifdef CONFIG_DEBUG_HIGHMEM
97 BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx)); 112 BUG_ON(vaddr != __fix_to_virt(idx));
98 set_top_pte(vaddr, __pte(0)); 113 set_fixmap_pte(idx, __pte(0));
99#else 114#else
100 (void) idx; /* to kill a warning */ 115 (void) idx; /* to kill a warning */
101#endif 116#endif
@@ -117,11 +132,11 @@ void *kmap_atomic_pfn(unsigned long pfn)
117 132
118 type = kmap_atomic_idx_push(); 133 type = kmap_atomic_idx_push();
119 idx = type + KM_TYPE_NR * smp_processor_id(); 134 idx = type + KM_TYPE_NR * smp_processor_id();
120 vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); 135 vaddr = __fix_to_virt(idx);
121#ifdef CONFIG_DEBUG_HIGHMEM 136#ifdef CONFIG_DEBUG_HIGHMEM
122 BUG_ON(!pte_none(get_top_pte(vaddr))); 137 BUG_ON(!pte_none(*(fixmap_page_table + idx)));
123#endif 138#endif
124 set_top_pte(vaddr, pfn_pte(pfn, kmap_prot)); 139 set_fixmap_pte(idx, pfn_pte(pfn, kmap_prot));
125 140
126 return (void *)vaddr; 141 return (void *)vaddr;
127} 142}
@@ -133,5 +148,5 @@ struct page *kmap_atomic_to_page(const void *ptr)
133 if (vaddr < FIXADDR_START) 148 if (vaddr < FIXADDR_START)
134 return virt_to_page(ptr); 149 return virt_to_page(ptr);
135 150
136 return pte_page(get_top_pte(vaddr)); 151 return pte_page(get_fixmap_pte(vaddr));
137} 152}
diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index 928d596d9ab4..659c75d808dc 100644
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -23,6 +23,7 @@
23#include <linux/dma-contiguous.h> 23#include <linux/dma-contiguous.h>
24#include <linux/sizes.h> 24#include <linux/sizes.h>
25 25
26#include <asm/cp15.h>
26#include <asm/mach-types.h> 27#include <asm/mach-types.h>
27#include <asm/memblock.h> 28#include <asm/memblock.h>
28#include <asm/prom.h> 29#include <asm/prom.h>
@@ -36,6 +37,14 @@
36 37
37#include "mm.h" 38#include "mm.h"
38 39
40#ifdef CONFIG_CPU_CP15_MMU
41unsigned long __init __clear_cr(unsigned long mask)
42{
43 cr_alignment = cr_alignment & ~mask;
44 return cr_alignment;
45}
46#endif
47
39static phys_addr_t phys_initrd_start __initdata = 0; 48static phys_addr_t phys_initrd_start __initdata = 0;
40static unsigned long phys_initrd_size __initdata = 0; 49static unsigned long phys_initrd_size __initdata = 0;
41 50
@@ -81,24 +90,21 @@ __tagtable(ATAG_INITRD2, parse_tag_initrd2);
81 * initialization functions, as well as show_mem() for the skipping 90 * initialization functions, as well as show_mem() for the skipping
82 * of holes in the memory map. It is populated by arm_add_memory(). 91 * of holes in the memory map. It is populated by arm_add_memory().
83 */ 92 */
84struct meminfo meminfo;
85
86void show_mem(unsigned int filter) 93void show_mem(unsigned int filter)
87{ 94{
88 int free = 0, total = 0, reserved = 0; 95 int free = 0, total = 0, reserved = 0;
89 int shared = 0, cached = 0, slab = 0, i; 96 int shared = 0, cached = 0, slab = 0;
90 struct meminfo * mi = &meminfo; 97 struct memblock_region *reg;
91 98
92 printk("Mem-info:\n"); 99 printk("Mem-info:\n");
93 show_free_areas(filter); 100 show_free_areas(filter);
94 101
95 for_each_bank (i, mi) { 102 for_each_memblock (memory, reg) {
96 struct membank *bank = &mi->bank[i];
97 unsigned int pfn1, pfn2; 103 unsigned int pfn1, pfn2;
98 struct page *page, *end; 104 struct page *page, *end;
99 105
100 pfn1 = bank_pfn_start(bank); 106 pfn1 = memblock_region_memory_base_pfn(reg);
101 pfn2 = bank_pfn_end(bank); 107 pfn2 = memblock_region_memory_end_pfn(reg);
102 108
103 page = pfn_to_page(pfn1); 109 page = pfn_to_page(pfn1);
104 end = pfn_to_page(pfn2 - 1) + 1; 110 end = pfn_to_page(pfn2 - 1) + 1;
@@ -115,8 +121,9 @@ void show_mem(unsigned int filter)
115 free++; 121 free++;
116 else 122 else
117 shared += page_count(page) - 1; 123 shared += page_count(page) - 1;
118 page++; 124 pfn1++;
119 } while (page < end); 125 page = pfn_to_page(pfn1);
126 } while (pfn1 < pfn2);
120 } 127 }
121 128
122 printk("%d pages of RAM\n", total); 129 printk("%d pages of RAM\n", total);
@@ -130,16 +137,9 @@ void show_mem(unsigned int filter)
130static void __init find_limits(unsigned long *min, unsigned long *max_low, 137static void __init find_limits(unsigned long *min, unsigned long *max_low,
131 unsigned long *max_high) 138 unsigned long *max_high)
132{ 139{
133 struct meminfo *mi = &meminfo; 140 *max_low = PFN_DOWN(memblock_get_current_limit());
134 int i; 141 *min = PFN_UP(memblock_start_of_DRAM());
135 142 *max_high = PFN_DOWN(memblock_end_of_DRAM());
136 /* This assumes the meminfo array is properly sorted */
137 *min = bank_pfn_start(&mi->bank[0]);
138 for_each_bank (i, mi)
139 if (mi->bank[i].highmem)
140 break;
141 *max_low = bank_pfn_end(&mi->bank[i - 1]);
142 *max_high = bank_pfn_end(&mi->bank[mi->nr_banks - 1]);
143} 143}
144 144
145#ifdef CONFIG_ZONE_DMA 145#ifdef CONFIG_ZONE_DMA
@@ -274,14 +274,8 @@ phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align)
274 return phys; 274 return phys;
275} 275}
276 276
277void __init arm_memblock_init(struct meminfo *mi, 277void __init arm_memblock_init(const struct machine_desc *mdesc)
278 const struct machine_desc *mdesc)
279{ 278{
280 int i;
281
282 for (i = 0; i < mi->nr_banks; i++)
283 memblock_add(mi->bank[i].start, mi->bank[i].size);
284
285 /* Register the kernel text, kernel data and initrd with memblock. */ 279 /* Register the kernel text, kernel data and initrd with memblock. */
286#ifdef CONFIG_XIP_KERNEL 280#ifdef CONFIG_XIP_KERNEL
287 memblock_reserve(__pa(_sdata), _end - _sdata); 281 memblock_reserve(__pa(_sdata), _end - _sdata);
@@ -412,54 +406,53 @@ free_memmap(unsigned long start_pfn, unsigned long end_pfn)
412/* 406/*
413 * The mem_map array can get very big. Free the unused area of the memory map. 407 * The mem_map array can get very big. Free the unused area of the memory map.
414 */ 408 */
415static void __init free_unused_memmap(struct meminfo *mi) 409static void __init free_unused_memmap(void)
416{ 410{
417 unsigned long bank_start, prev_bank_end = 0; 411 unsigned long start, prev_end = 0;
418 unsigned int i; 412 struct memblock_region *reg;
419 413
420 /* 414 /*
421 * This relies on each bank being in address order. 415 * This relies on each bank being in address order.
422 * The banks are sorted previously in bootmem_init(). 416 * The banks are sorted previously in bootmem_init().
423 */ 417 */
424 for_each_bank(i, mi) { 418 for_each_memblock(memory, reg) {
425 struct membank *bank = &mi->bank[i]; 419 start = memblock_region_memory_base_pfn(reg);
426
427 bank_start = bank_pfn_start(bank);
428 420
429#ifdef CONFIG_SPARSEMEM 421#ifdef CONFIG_SPARSEMEM
430 /* 422 /*
431 * Take care not to free memmap entries that don't exist 423 * Take care not to free memmap entries that don't exist
432 * due to SPARSEMEM sections which aren't present. 424 * due to SPARSEMEM sections which aren't present.
433 */ 425 */
434 bank_start = min(bank_start, 426 start = min(start,
435 ALIGN(prev_bank_end, PAGES_PER_SECTION)); 427 ALIGN(prev_end, PAGES_PER_SECTION));
436#else 428#else
437 /* 429 /*
438 * Align down here since the VM subsystem insists that the 430 * Align down here since the VM subsystem insists that the
439 * memmap entries are valid from the bank start aligned to 431 * memmap entries are valid from the bank start aligned to
440 * MAX_ORDER_NR_PAGES. 432 * MAX_ORDER_NR_PAGES.
441 */ 433 */
442 bank_start = round_down(bank_start, MAX_ORDER_NR_PAGES); 434 start = round_down(start, MAX_ORDER_NR_PAGES);
443#endif 435#endif
444 /* 436 /*
445 * If we had a previous bank, and there is a space 437 * If we had a previous bank, and there is a space
446 * between the current bank and the previous, free it. 438 * between the current bank and the previous, free it.
447 */ 439 */
448 if (prev_bank_end && prev_bank_end < bank_start) 440 if (prev_end && prev_end < start)
449 free_memmap(prev_bank_end, bank_start); 441 free_memmap(prev_end, start);
450 442
451 /* 443 /*
452 * Align up here since the VM subsystem insists that the 444 * Align up here since the VM subsystem insists that the
453 * memmap entries are valid from the bank end aligned to 445 * memmap entries are valid from the bank end aligned to
454 * MAX_ORDER_NR_PAGES. 446 * MAX_ORDER_NR_PAGES.
455 */ 447 */
456 prev_bank_end = ALIGN(bank_pfn_end(bank), MAX_ORDER_NR_PAGES); 448 prev_end = ALIGN(memblock_region_memory_end_pfn(reg),
449 MAX_ORDER_NR_PAGES);
457 } 450 }
458 451
459#ifdef CONFIG_SPARSEMEM 452#ifdef CONFIG_SPARSEMEM
460 if (!IS_ALIGNED(prev_bank_end, PAGES_PER_SECTION)) 453 if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION))
461 free_memmap(prev_bank_end, 454 free_memmap(prev_end,
462 ALIGN(prev_bank_end, PAGES_PER_SECTION)); 455 ALIGN(prev_end, PAGES_PER_SECTION));
463#endif 456#endif
464} 457}
465 458
@@ -535,7 +528,7 @@ void __init mem_init(void)
535 set_max_mapnr(pfn_to_page(max_pfn) - mem_map); 528 set_max_mapnr(pfn_to_page(max_pfn) - mem_map);
536 529
537 /* this will put all unused low memory onto the freelists */ 530 /* this will put all unused low memory onto the freelists */
538 free_unused_memmap(&meminfo); 531 free_unused_memmap();
539 free_all_bootmem(); 532 free_all_bootmem();
540 533
541#ifdef CONFIG_SA1111 534#ifdef CONFIG_SA1111
diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c
index f9c32ba73544..d1e5ad7ab3bc 100644
--- a/arch/arm/mm/ioremap.c
+++ b/arch/arm/mm/ioremap.c
@@ -438,6 +438,13 @@ void __arm_iounmap(volatile void __iomem *io_addr)
438EXPORT_SYMBOL(__arm_iounmap); 438EXPORT_SYMBOL(__arm_iounmap);
439 439
440#ifdef CONFIG_PCI 440#ifdef CONFIG_PCI
441static int pci_ioremap_mem_type = MT_DEVICE;
442
443void pci_ioremap_set_mem_type(int mem_type)
444{
445 pci_ioremap_mem_type = mem_type;
446}
447
441int pci_ioremap_io(unsigned int offset, phys_addr_t phys_addr) 448int pci_ioremap_io(unsigned int offset, phys_addr_t phys_addr)
442{ 449{
443 BUG_ON(offset + SZ_64K > IO_SPACE_LIMIT); 450 BUG_ON(offset + SZ_64K > IO_SPACE_LIMIT);
@@ -445,7 +452,7 @@ int pci_ioremap_io(unsigned int offset, phys_addr_t phys_addr)
445 return ioremap_page_range(PCI_IO_VIRT_BASE + offset, 452 return ioremap_page_range(PCI_IO_VIRT_BASE + offset,
446 PCI_IO_VIRT_BASE + offset + SZ_64K, 453 PCI_IO_VIRT_BASE + offset + SZ_64K,
447 phys_addr, 454 phys_addr,
448 __pgprot(get_mem_type(MT_DEVICE)->prot_pte)); 455 __pgprot(get_mem_type(pci_ioremap_mem_type)->prot_pte));
449} 456}
450EXPORT_SYMBOL_GPL(pci_ioremap_io); 457EXPORT_SYMBOL_GPL(pci_ioremap_io);
451#endif 458#endif
diff --git a/arch/arm/mm/l2c-common.c b/arch/arm/mm/l2c-common.c
new file mode 100644
index 000000000000..10a3cf28c362
--- /dev/null
+++ b/arch/arm/mm/l2c-common.c
@@ -0,0 +1,20 @@
1/*
2 * Copyright (C) 2010 ARM Ltd.
3 * Written by Catalin Marinas <catalin.marinas@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 */
9#include <linux/bug.h>
10#include <linux/smp.h>
11#include <asm/outercache.h>
12
13void outer_disable(void)
14{
15 WARN_ON(!irqs_disabled());
16 WARN_ON(num_online_cpus() > 1);
17
18 if (outer_cache.disable)
19 outer_cache.disable();
20}
diff --git a/arch/arm/mm/l2c-l2x0-resume.S b/arch/arm/mm/l2c-l2x0-resume.S
new file mode 100644
index 000000000000..99b05f21a59a
--- /dev/null
+++ b/arch/arm/mm/l2c-l2x0-resume.S
@@ -0,0 +1,58 @@
1/*
2 * L2C-310 early resume code. This can be used by platforms to restore
3 * the settings of their L2 cache controller before restoring the
4 * processor state.
5 *
6 * This code can only be used to if you are running in the secure world.
7 */
8#include <linux/linkage.h>
9#include <asm/hardware/cache-l2x0.h>
10
11 .text
12
13ENTRY(l2c310_early_resume)
14 adr r0, 1f
15 ldr r2, [r0]
16 add r0, r2, r0
17
18 ldmia r0, {r1, r2, r3, r4, r5, r6, r7, r8}
19 @ r1 = phys address of L2C-310 controller
20 @ r2 = aux_ctrl
21 @ r3 = tag_latency
22 @ r4 = data_latency
23 @ r5 = filter_start
24 @ r6 = filter_end
25 @ r7 = prefetch_ctrl
26 @ r8 = pwr_ctrl
27
28 @ Check that the address has been initialised
29 teq r1, #0
30 moveq pc, lr
31
32 @ The prefetch and power control registers are revision dependent
33 @ and can be written whether or not the L2 cache is enabled
34 ldr r0, [r1, #L2X0_CACHE_ID]
35 and r0, r0, #L2X0_CACHE_ID_RTL_MASK
36 cmp r0, #L310_CACHE_ID_RTL_R2P0
37 strcs r7, [r1, #L310_PREFETCH_CTRL]
38 cmp r0, #L310_CACHE_ID_RTL_R3P0
39 strcs r8, [r1, #L310_POWER_CTRL]
40
41 @ Don't setup the L2 cache if it is already enabled
42 ldr r0, [r1, #L2X0_CTRL]
43 tst r0, #L2X0_CTRL_EN
44 movne pc, lr
45
46 str r3, [r1, #L310_TAG_LATENCY_CTRL]
47 str r4, [r1, #L310_DATA_LATENCY_CTRL]
48 str r6, [r1, #L310_ADDR_FILTER_END]
49 str r5, [r1, #L310_ADDR_FILTER_START]
50
51 str r2, [r1, #L2X0_AUX_CTRL]
52 mov r9, #L2X0_CTRL_EN
53 str r9, [r1, #L2X0_CTRL]
54 mov pc, lr
55ENDPROC(l2c310_early_resume)
56
57 .align
581: .long l2x0_saved_regs - .
diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h
index 7ea641b7aa7d..ce727d47275c 100644
--- a/arch/arm/mm/mm.h
+++ b/arch/arm/mm/mm.h
@@ -2,6 +2,8 @@
2#include <linux/list.h> 2#include <linux/list.h>
3#include <linux/vmalloc.h> 3#include <linux/vmalloc.h>
4 4
5#include <asm/pgtable.h>
6
5/* the upper-most page table pointer */ 7/* the upper-most page table pointer */
6extern pmd_t *top_pmd; 8extern pmd_t *top_pmd;
7 9
@@ -93,3 +95,5 @@ extern phys_addr_t arm_lowmem_limit;
93void __init bootmem_init(void); 95void __init bootmem_init(void);
94void arm_mm_memblock_reserve(void); 96void arm_mm_memblock_reserve(void);
95void dma_contiguous_remap(void); 97void dma_contiguous_remap(void);
98
99unsigned long __clear_cr(unsigned long mask);
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index b68c6b22e1c8..ab14b79b03f0 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -35,6 +35,7 @@
35#include <asm/mach/arch.h> 35#include <asm/mach/arch.h>
36#include <asm/mach/map.h> 36#include <asm/mach/map.h>
37#include <asm/mach/pci.h> 37#include <asm/mach/pci.h>
38#include <asm/fixmap.h>
38 39
39#include "mm.h" 40#include "mm.h"
40#include "tcm.h" 41#include "tcm.h"
@@ -117,28 +118,54 @@ static struct cachepolicy cache_policies[] __initdata = {
117}; 118};
118 119
119#ifdef CONFIG_CPU_CP15 120#ifdef CONFIG_CPU_CP15
121static unsigned long initial_pmd_value __initdata = 0;
122
120/* 123/*
121 * These are useful for identifying cache coherency 124 * Initialise the cache_policy variable with the initial state specified
122 * problems by allowing the cache or the cache and 125 * via the "pmd" value. This is used to ensure that on ARMv6 and later,
123 * writebuffer to be turned off. (Note: the write 126 * the C code sets the page tables up with the same policy as the head
124 * buffer should not be on and the cache off). 127 * assembly code, which avoids an illegal state where the TLBs can get
128 * confused. See comments in early_cachepolicy() for more information.
125 */ 129 */
126static int __init early_cachepolicy(char *p) 130void __init init_default_cache_policy(unsigned long pmd)
127{ 131{
128 int i; 132 int i;
129 133
134 initial_pmd_value = pmd;
135
136 pmd &= PMD_SECT_TEX(1) | PMD_SECT_BUFFERABLE | PMD_SECT_CACHEABLE;
137
138 for (i = 0; i < ARRAY_SIZE(cache_policies); i++)
139 if (cache_policies[i].pmd == pmd) {
140 cachepolicy = i;
141 break;
142 }
143
144 if (i == ARRAY_SIZE(cache_policies))
145 pr_err("ERROR: could not find cache policy\n");
146}
147
148/*
149 * These are useful for identifying cache coherency problems by allowing
150 * the cache or the cache and writebuffer to be turned off. (Note: the
151 * write buffer should not be on and the cache off).
152 */
153static int __init early_cachepolicy(char *p)
154{
155 int i, selected = -1;
156
130 for (i = 0; i < ARRAY_SIZE(cache_policies); i++) { 157 for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
131 int len = strlen(cache_policies[i].policy); 158 int len = strlen(cache_policies[i].policy);
132 159
133 if (memcmp(p, cache_policies[i].policy, len) == 0) { 160 if (memcmp(p, cache_policies[i].policy, len) == 0) {
134 cachepolicy = i; 161 selected = i;
135 cr_alignment &= ~cache_policies[i].cr_mask;
136 cr_no_alignment &= ~cache_policies[i].cr_mask;
137 break; 162 break;
138 } 163 }
139 } 164 }
140 if (i == ARRAY_SIZE(cache_policies)) 165
141 printk(KERN_ERR "ERROR: unknown or unsupported cache policy\n"); 166 if (selected == -1)
167 pr_err("ERROR: unknown or unsupported cache policy\n");
168
142 /* 169 /*
143 * This restriction is partly to do with the way we boot; it is 170 * This restriction is partly to do with the way we boot; it is
144 * unpredictable to have memory mapped using two different sets of 171 * unpredictable to have memory mapped using two different sets of
@@ -146,12 +173,18 @@ static int __init early_cachepolicy(char *p)
146 * change these attributes once the initial assembly has setup the 173 * change these attributes once the initial assembly has setup the
147 * page tables. 174 * page tables.
148 */ 175 */
149 if (cpu_architecture() >= CPU_ARCH_ARMv6) { 176 if (cpu_architecture() >= CPU_ARCH_ARMv6 && selected != cachepolicy) {
150 printk(KERN_WARNING "Only cachepolicy=writeback supported on ARMv6 and later\n"); 177 pr_warn("Only cachepolicy=%s supported on ARMv6 and later\n",
151 cachepolicy = CPOLICY_WRITEBACK; 178 cache_policies[cachepolicy].policy);
179 return 0;
180 }
181
182 if (selected != cachepolicy) {
183 unsigned long cr = __clear_cr(cache_policies[selected].cr_mask);
184 cachepolicy = selected;
185 flush_cache_all();
186 set_cr(cr);
152 } 187 }
153 flush_cache_all();
154 set_cr(cr_alignment);
155 return 0; 188 return 0;
156} 189}
157early_param("cachepolicy", early_cachepolicy); 190early_param("cachepolicy", early_cachepolicy);
@@ -186,35 +219,6 @@ static int __init early_ecc(char *p)
186early_param("ecc", early_ecc); 219early_param("ecc", early_ecc);
187#endif 220#endif
188 221
189static int __init noalign_setup(char *__unused)
190{
191 cr_alignment &= ~CR_A;
192 cr_no_alignment &= ~CR_A;
193 set_cr(cr_alignment);
194 return 1;
195}
196__setup("noalign", noalign_setup);
197
198#ifndef CONFIG_SMP
199void adjust_cr(unsigned long mask, unsigned long set)
200{
201 unsigned long flags;
202
203 mask &= ~CR_A;
204
205 set &= mask;
206
207 local_irq_save(flags);
208
209 cr_no_alignment = (cr_no_alignment & ~mask) | set;
210 cr_alignment = (cr_alignment & ~mask) | set;
211
212 set_cr((get_cr() & ~mask) | set);
213
214 local_irq_restore(flags);
215}
216#endif
217
218#else /* ifdef CONFIG_CPU_CP15 */ 222#else /* ifdef CONFIG_CPU_CP15 */
219 223
220static int __init early_cachepolicy(char *p) 224static int __init early_cachepolicy(char *p)
@@ -414,8 +418,17 @@ static void __init build_mem_type_table(void)
414 cachepolicy = CPOLICY_WRITEBACK; 418 cachepolicy = CPOLICY_WRITEBACK;
415 ecc_mask = 0; 419 ecc_mask = 0;
416 } 420 }
417 if (is_smp()) 421
418 cachepolicy = CPOLICY_WRITEALLOC; 422 if (is_smp()) {
423 if (cachepolicy != CPOLICY_WRITEALLOC) {
424 pr_warn("Forcing write-allocate cache policy for SMP\n");
425 cachepolicy = CPOLICY_WRITEALLOC;
426 }
427 if (!(initial_pmd_value & PMD_SECT_S)) {
428 pr_warn("Forcing shared mappings for SMP\n");
429 initial_pmd_value |= PMD_SECT_S;
430 }
431 }
419 432
420 /* 433 /*
421 * Strip out features not present on earlier architectures. 434 * Strip out features not present on earlier architectures.
@@ -539,11 +552,12 @@ static void __init build_mem_type_table(void)
539 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; 552 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
540#endif 553#endif
541 554
542 if (is_smp()) { 555 /*
543 /* 556 * If the initial page tables were created with the S bit
544 * Mark memory with the "shared" attribute 557 * set, then we need to do the same here for the same
545 * for SMP systems 558 * reasons given in early_cachepolicy().
546 */ 559 */
560 if (initial_pmd_value & PMD_SECT_S) {
547 user_pgprot |= L_PTE_SHARED; 561 user_pgprot |= L_PTE_SHARED;
548 kern_pgprot |= L_PTE_SHARED; 562 kern_pgprot |= L_PTE_SHARED;
549 vecs_pgprot |= L_PTE_SHARED; 563 vecs_pgprot |= L_PTE_SHARED;
@@ -1061,74 +1075,47 @@ phys_addr_t arm_lowmem_limit __initdata = 0;
1061void __init sanity_check_meminfo(void) 1075void __init sanity_check_meminfo(void)
1062{ 1076{
1063 phys_addr_t memblock_limit = 0; 1077 phys_addr_t memblock_limit = 0;
1064 int i, j, highmem = 0; 1078 int highmem = 0;
1065 phys_addr_t vmalloc_limit = __pa(vmalloc_min - 1) + 1; 1079 phys_addr_t vmalloc_limit = __pa(vmalloc_min - 1) + 1;
1080 struct memblock_region *reg;
1066 1081
1067 for (i = 0, j = 0; i < meminfo.nr_banks; i++) { 1082 for_each_memblock(memory, reg) {
1068 struct membank *bank = &meminfo.bank[j]; 1083 phys_addr_t block_start = reg->base;
1069 phys_addr_t size_limit; 1084 phys_addr_t block_end = reg->base + reg->size;
1070 1085 phys_addr_t size_limit = reg->size;
1071 *bank = meminfo.bank[i];
1072 size_limit = bank->size;
1073 1086
1074 if (bank->start >= vmalloc_limit) 1087 if (reg->base >= vmalloc_limit)
1075 highmem = 1; 1088 highmem = 1;
1076 else 1089 else
1077 size_limit = vmalloc_limit - bank->start; 1090 size_limit = vmalloc_limit - reg->base;
1078 1091
1079 bank->highmem = highmem;
1080 1092
1081#ifdef CONFIG_HIGHMEM 1093 if (!IS_ENABLED(CONFIG_HIGHMEM) || cache_is_vipt_aliasing()) {
1082 /* 1094
1083 * Split those memory banks which are partially overlapping 1095 if (highmem) {
1084 * the vmalloc area greatly simplifying things later. 1096 pr_notice("Ignoring RAM at %pa-%pa (!CONFIG_HIGHMEM)\n",
1085 */ 1097 &block_start, &block_end);
1086 if (!highmem && bank->size > size_limit) { 1098 memblock_remove(reg->base, reg->size);
1087 if (meminfo.nr_banks >= NR_BANKS) { 1099 continue;
1088 printk(KERN_CRIT "NR_BANKS too low, "
1089 "ignoring high memory\n");
1090 } else {
1091 memmove(bank + 1, bank,
1092 (meminfo.nr_banks - i) * sizeof(*bank));
1093 meminfo.nr_banks++;
1094 i++;
1095 bank[1].size -= size_limit;
1096 bank[1].start = vmalloc_limit;
1097 bank[1].highmem = highmem = 1;
1098 j++;
1099 } 1100 }
1100 bank->size = size_limit;
1101 }
1102#else
1103 /*
1104 * Highmem banks not allowed with !CONFIG_HIGHMEM.
1105 */
1106 if (highmem) {
1107 printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx "
1108 "(!CONFIG_HIGHMEM).\n",
1109 (unsigned long long)bank->start,
1110 (unsigned long long)bank->start + bank->size - 1);
1111 continue;
1112 }
1113 1101
1114 /* 1102 if (reg->size > size_limit) {
1115 * Check whether this memory bank would partially overlap 1103 phys_addr_t overlap_size = reg->size - size_limit;
1116 * the vmalloc area. 1104
1117 */ 1105 pr_notice("Truncating RAM at %pa-%pa to -%pa",
1118 if (bank->size > size_limit) { 1106 &block_start, &block_end, &vmalloc_limit);
1119 printk(KERN_NOTICE "Truncating RAM at %.8llx-%.8llx " 1107 memblock_remove(vmalloc_limit, overlap_size);
1120 "to -%.8llx (vmalloc region overlap).\n", 1108 block_end = vmalloc_limit;
1121 (unsigned long long)bank->start, 1109 }
1122 (unsigned long long)bank->start + bank->size - 1,
1123 (unsigned long long)bank->start + size_limit - 1);
1124 bank->size = size_limit;
1125 } 1110 }
1126#endif
1127 if (!bank->highmem) {
1128 phys_addr_t bank_end = bank->start + bank->size;
1129 1111
1130 if (bank_end > arm_lowmem_limit) 1112 if (!highmem) {
1131 arm_lowmem_limit = bank_end; 1113 if (block_end > arm_lowmem_limit) {
1114 if (reg->size > size_limit)
1115 arm_lowmem_limit = vmalloc_limit;
1116 else
1117 arm_lowmem_limit = block_end;
1118 }
1132 1119
1133 /* 1120 /*
1134 * Find the first non-section-aligned page, and point 1121 * Find the first non-section-aligned page, and point
@@ -1144,35 +1131,15 @@ void __init sanity_check_meminfo(void)
1144 * occurs before any free memory is mapped. 1131 * occurs before any free memory is mapped.
1145 */ 1132 */
1146 if (!memblock_limit) { 1133 if (!memblock_limit) {
1147 if (!IS_ALIGNED(bank->start, SECTION_SIZE)) 1134 if (!IS_ALIGNED(block_start, SECTION_SIZE))
1148 memblock_limit = bank->start; 1135 memblock_limit = block_start;
1149 else if (!IS_ALIGNED(bank_end, SECTION_SIZE)) 1136 else if (!IS_ALIGNED(block_end, SECTION_SIZE))
1150 memblock_limit = bank_end; 1137 memblock_limit = arm_lowmem_limit;
1151 } 1138 }
1152 }
1153 j++;
1154 }
1155#ifdef CONFIG_HIGHMEM
1156 if (highmem) {
1157 const char *reason = NULL;
1158 1139
1159 if (cache_is_vipt_aliasing()) {
1160 /*
1161 * Interactions between kmap and other mappings
1162 * make highmem support with aliasing VIPT caches
1163 * rather difficult.
1164 */
1165 reason = "with VIPT aliasing cache";
1166 }
1167 if (reason) {
1168 printk(KERN_CRIT "HIGHMEM is not supported %s, ignoring high memory\n",
1169 reason);
1170 while (j > 0 && meminfo.bank[j - 1].highmem)
1171 j--;
1172 } 1140 }
1173 } 1141 }
1174#endif 1142
1175 meminfo.nr_banks = j;
1176 high_memory = __va(arm_lowmem_limit - 1) + 1; 1143 high_memory = __va(arm_lowmem_limit - 1) + 1;
1177 1144
1178 /* 1145 /*
@@ -1359,6 +1326,9 @@ static void __init kmap_init(void)
1359#ifdef CONFIG_HIGHMEM 1326#ifdef CONFIG_HIGHMEM
1360 pkmap_page_table = early_pte_alloc(pmd_off_k(PKMAP_BASE), 1327 pkmap_page_table = early_pte_alloc(pmd_off_k(PKMAP_BASE),
1361 PKMAP_BASE, _PAGE_KERNEL_TABLE); 1328 PKMAP_BASE, _PAGE_KERNEL_TABLE);
1329
1330 fixmap_page_table = early_pte_alloc(pmd_off_k(FIXADDR_START),
1331 FIXADDR_START, _PAGE_KERNEL_TABLE);
1362#endif 1332#endif
1363} 1333}
1364 1334
@@ -1461,7 +1431,7 @@ void __init early_paging_init(const struct machine_desc *mdesc,
1461 * just complicate the code. 1431 * just complicate the code.
1462 */ 1432 */
1463 flush_cache_louis(); 1433 flush_cache_louis();
1464 dsb(); 1434 dsb(ishst);
1465 isb(); 1435 isb();
1466 1436
1467 /* remap level 1 table */ 1437 /* remap level 1 table */
diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c
index 55764a7ef1f0..da1874f9f8cf 100644
--- a/arch/arm/mm/nommu.c
+++ b/arch/arm/mm/nommu.c
@@ -88,30 +88,35 @@ static unsigned long irbar_read(void)
88void __init sanity_check_meminfo_mpu(void) 88void __init sanity_check_meminfo_mpu(void)
89{ 89{
90 int i; 90 int i;
91 struct membank *bank = meminfo.bank;
92 phys_addr_t phys_offset = PHYS_OFFSET; 91 phys_addr_t phys_offset = PHYS_OFFSET;
93 phys_addr_t aligned_region_size, specified_mem_size, rounded_mem_size; 92 phys_addr_t aligned_region_size, specified_mem_size, rounded_mem_size;
94 93 struct memblock_region *reg;
95 /* Initially only use memory continuous from PHYS_OFFSET */ 94 bool first = true;
96 if (bank_phys_start(&bank[0]) != phys_offset) 95 phys_addr_t mem_start;
97 panic("First memory bank must be contiguous from PHYS_OFFSET"); 96 phys_addr_t mem_end;
98 97
99 /* Banks have already been sorted by start address */ 98 for_each_memblock(memory, reg) {
100 for (i = 1; i < meminfo.nr_banks; i++) { 99 if (first) {
101 if (bank[i].start <= bank_phys_end(&bank[0]) && 100 /*
102 bank_phys_end(&bank[i]) > bank_phys_end(&bank[0])) { 101 * Initially only use memory continuous from
103 bank[0].size = bank_phys_end(&bank[i]) - bank[0].start; 102 * PHYS_OFFSET */
103 if (reg->base != phys_offset)
104 panic("First memory bank must be contiguous from PHYS_OFFSET");
105
106 mem_start = reg->base;
107 mem_end = reg->base + reg->size;
108 specified_mem_size = reg->size;
109 first = false;
104 } else { 110 } else {
105 pr_notice("Ignoring RAM after 0x%.8lx. " 111 /*
106 "First non-contiguous (ignored) bank start: 0x%.8lx\n", 112 * memblock auto merges contiguous blocks, remove
107 (unsigned long)bank_phys_end(&bank[0]), 113 * all blocks afterwards
108 (unsigned long)bank_phys_start(&bank[i])); 114 */
109 break; 115 pr_notice("Ignoring RAM after %pa, memory at %pa ignored\n",
116 &mem_start, &reg->base);
117 memblock_remove(reg->base, reg->size);
110 } 118 }
111 } 119 }
112 /* All contiguous banks are now merged in to the first bank */
113 meminfo.nr_banks = 1;
114 specified_mem_size = bank[0].size;
115 120
116 /* 121 /*
117 * MPU has curious alignment requirements: Size must be power of 2, and 122 * MPU has curious alignment requirements: Size must be power of 2, and
@@ -128,23 +133,24 @@ void __init sanity_check_meminfo_mpu(void)
128 */ 133 */
129 aligned_region_size = (phys_offset - 1) ^ (phys_offset); 134 aligned_region_size = (phys_offset - 1) ^ (phys_offset);
130 /* Find the max power-of-two sized region that fits inside our bank */ 135 /* Find the max power-of-two sized region that fits inside our bank */
131 rounded_mem_size = (1 << __fls(bank[0].size)) - 1; 136 rounded_mem_size = (1 << __fls(specified_mem_size)) - 1;
132 137
133 /* The actual region size is the smaller of the two */ 138 /* The actual region size is the smaller of the two */
134 aligned_region_size = aligned_region_size < rounded_mem_size 139 aligned_region_size = aligned_region_size < rounded_mem_size
135 ? aligned_region_size + 1 140 ? aligned_region_size + 1
136 : rounded_mem_size + 1; 141 : rounded_mem_size + 1;
137 142
138 if (aligned_region_size != specified_mem_size) 143 if (aligned_region_size != specified_mem_size) {
139 pr_warn("Truncating memory from 0x%.8lx to 0x%.8lx (MPU region constraints)", 144 pr_warn("Truncating memory from %pa to %pa (MPU region constraints)",
140 (unsigned long)specified_mem_size, 145 &specified_mem_size, &aligned_region_size);
141 (unsigned long)aligned_region_size); 146 memblock_remove(mem_start + aligned_region_size,
147 specified_mem_size - aligned_round_size);
148
149 mem_end = mem_start + aligned_region_size;
150 }
142 151
143 meminfo.bank[0].size = aligned_region_size; 152 pr_debug("MPU Region from %pa size %pa (end %pa))\n",
144 pr_debug("MPU Region from 0x%.8lx size 0x%.8lx (end 0x%.8lx))\n", 153 &phys_offset, &aligned_region_size, &mem_end);
145 (unsigned long)phys_offset,
146 (unsigned long)aligned_region_size,
147 (unsigned long)bank_phys_end(&bank[0]));
148 154
149} 155}
150 156
@@ -292,7 +298,7 @@ void __init sanity_check_meminfo(void)
292{ 298{
293 phys_addr_t end; 299 phys_addr_t end;
294 sanity_check_meminfo_mpu(); 300 sanity_check_meminfo_mpu();
295 end = bank_phys_end(&meminfo.bank[meminfo.nr_banks - 1]); 301 end = memblock_end_of_DRAM();
296 high_memory = __va(end - 1) + 1; 302 high_memory = __va(end - 1) + 1;
297} 303}
298 304
diff --git a/arch/arm/mm/proc-v7-3level.S b/arch/arm/mm/proc-v7-3level.S
index 01a719e18bb0..22e3ad63500c 100644
--- a/arch/arm/mm/proc-v7-3level.S
+++ b/arch/arm/mm/proc-v7-3level.S
@@ -64,6 +64,14 @@ ENTRY(cpu_v7_switch_mm)
64 mov pc, lr 64 mov pc, lr
65ENDPROC(cpu_v7_switch_mm) 65ENDPROC(cpu_v7_switch_mm)
66 66
67#ifdef __ARMEB__
68#define rl r3
69#define rh r2
70#else
71#define rl r2
72#define rh r3
73#endif
74
67/* 75/*
68 * cpu_v7_set_pte_ext(ptep, pte) 76 * cpu_v7_set_pte_ext(ptep, pte)
69 * 77 *
@@ -73,13 +81,13 @@ ENDPROC(cpu_v7_switch_mm)
73 */ 81 */
74ENTRY(cpu_v7_set_pte_ext) 82ENTRY(cpu_v7_set_pte_ext)
75#ifdef CONFIG_MMU 83#ifdef CONFIG_MMU
76 tst r2, #L_PTE_VALID 84 tst rl, #L_PTE_VALID
77 beq 1f 85 beq 1f
78 tst r3, #1 << (57 - 32) @ L_PTE_NONE 86 tst rh, #1 << (57 - 32) @ L_PTE_NONE
79 bicne r2, #L_PTE_VALID 87 bicne rl, #L_PTE_VALID
80 bne 1f 88 bne 1f
81 tst r3, #1 << (55 - 32) @ L_PTE_DIRTY 89 tst rh, #1 << (55 - 32) @ L_PTE_DIRTY
82 orreq r2, #L_PTE_RDONLY 90 orreq rl, #L_PTE_RDONLY
831: strd r2, r3, [r0] 911: strd r2, r3, [r0]
84 ALT_SMP(W(nop)) 92 ALT_SMP(W(nop))
85 ALT_UP (mcr p15, 0, r0, c7, c10, 1) @ flush_pte 93 ALT_UP (mcr p15, 0, r0, c7, c10, 1) @ flush_pte
diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S
index 195731d3813b..3db2c2f04a30 100644
--- a/arch/arm/mm/proc-v7.S
+++ b/arch/arm/mm/proc-v7.S
@@ -169,9 +169,31 @@ ENDPROC(cpu_pj4b_do_idle)
169 globl_equ cpu_pj4b_do_idle, cpu_v7_do_idle 169 globl_equ cpu_pj4b_do_idle, cpu_v7_do_idle
170#endif 170#endif
171 globl_equ cpu_pj4b_dcache_clean_area, cpu_v7_dcache_clean_area 171 globl_equ cpu_pj4b_dcache_clean_area, cpu_v7_dcache_clean_area
172 globl_equ cpu_pj4b_do_suspend, cpu_v7_do_suspend 172#ifdef CONFIG_ARM_CPU_SUSPEND
173 globl_equ cpu_pj4b_do_resume, cpu_v7_do_resume 173ENTRY(cpu_pj4b_do_suspend)
174 globl_equ cpu_pj4b_suspend_size, cpu_v7_suspend_size 174 stmfd sp!, {r6 - r10}
175 mrc p15, 1, r6, c15, c1, 0 @ save CP15 - extra features
176 mrc p15, 1, r7, c15, c2, 0 @ save CP15 - Aux Func Modes Ctrl 0
177 mrc p15, 1, r8, c15, c1, 2 @ save CP15 - Aux Debug Modes Ctrl 2
178 mrc p15, 1, r9, c15, c1, 1 @ save CP15 - Aux Debug Modes Ctrl 1
179 mrc p15, 0, r10, c9, c14, 0 @ save CP15 - PMC
180 stmia r0!, {r6 - r10}
181 ldmfd sp!, {r6 - r10}
182 b cpu_v7_do_suspend
183ENDPROC(cpu_pj4b_do_suspend)
184
185ENTRY(cpu_pj4b_do_resume)
186 ldmia r0!, {r6 - r10}
187 mcr p15, 1, r6, c15, c1, 0 @ save CP15 - extra features
188 mcr p15, 1, r7, c15, c2, 0 @ save CP15 - Aux Func Modes Ctrl 0
189 mcr p15, 1, r8, c15, c1, 2 @ save CP15 - Aux Debug Modes Ctrl 2
190 mcr p15, 1, r9, c15, c1, 1 @ save CP15 - Aux Debug Modes Ctrl 1
191 mcr p15, 0, r10, c9, c14, 0 @ save CP15 - PMC
192 b cpu_v7_do_resume
193ENDPROC(cpu_pj4b_do_resume)
194#endif
195.globl cpu_pj4b_suspend_size
196.equ cpu_pj4b_suspend_size, 4 * 14
175 197
176#endif 198#endif
177 199
@@ -194,6 +216,7 @@ __v7_cr7mp_setup:
194__v7_ca7mp_setup: 216__v7_ca7mp_setup:
195__v7_ca12mp_setup: 217__v7_ca12mp_setup:
196__v7_ca15mp_setup: 218__v7_ca15mp_setup:
219__v7_ca17mp_setup:
197 mov r10, #0 220 mov r10, #0
1981: 2211:
199#ifdef CONFIG_SMP 222#ifdef CONFIG_SMP
@@ -505,6 +528,16 @@ __v7_ca15mp_proc_info:
505 .size __v7_ca15mp_proc_info, . - __v7_ca15mp_proc_info 528 .size __v7_ca15mp_proc_info, . - __v7_ca15mp_proc_info
506 529
507 /* 530 /*
531 * ARM Ltd. Cortex A17 processor.
532 */
533 .type __v7_ca17mp_proc_info, #object
534__v7_ca17mp_proc_info:
535 .long 0x410fc0e0
536 .long 0xff0ffff0
537 __v7_proc __v7_ca17mp_setup
538 .size __v7_ca17mp_proc_info, . - __v7_ca17mp_proc_info
539
540 /*
508 * Qualcomm Inc. Krait processors. 541 * Qualcomm Inc. Krait processors.
509 */ 542 */
510 .type __krait_proc_info, #object 543 .type __krait_proc_info, #object
generated by cgit v1.2.2 (git 2.25.0) at 2024-09-24 02:31:24 -0400