aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm/mm/cache-l2x0.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm/mm/cache-l2x0.c')
-rw-r--r--arch/arm/mm/cache-l2x0.c1498
1 files changed, 1001 insertions, 497 deletions
diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c
index 7abde2ce8973..efc5cabf70e0 100644
--- a/arch/arm/mm/cache-l2x0.c
+++ b/arch/arm/mm/cache-l2x0.c
@@ -16,18 +16,33 @@
16 * along with this program; if not, write to the Free Software 16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */ 18 */
19#include <linux/cpu.h>
19#include <linux/err.h> 20#include <linux/err.h>
20#include <linux/init.h> 21#include <linux/init.h>
22#include <linux/smp.h>
21#include <linux/spinlock.h> 23#include <linux/spinlock.h>
22#include <linux/io.h> 24#include <linux/io.h>
23#include <linux/of.h> 25#include <linux/of.h>
24#include <linux/of_address.h> 26#include <linux/of_address.h>
25 27
26#include <asm/cacheflush.h> 28#include <asm/cacheflush.h>
29#include <asm/cp15.h>
30#include <asm/cputype.h>
27#include <asm/hardware/cache-l2x0.h> 31#include <asm/hardware/cache-l2x0.h>
28#include "cache-tauros3.h" 32#include "cache-tauros3.h"
29#include "cache-aurora-l2.h" 33#include "cache-aurora-l2.h"
30 34
35struct l2c_init_data {
36 const char *type;
37 unsigned way_size_0;
38 unsigned num_lock;
39 void (*of_parse)(const struct device_node *, u32 *, u32 *);
40 void (*enable)(void __iomem *, u32, unsigned);
41 void (*fixup)(void __iomem *, u32, struct outer_cache_fns *);
42 void (*save)(void __iomem *);
43 struct outer_cache_fns outer_cache;
44};
45
31#define CACHE_LINE_SIZE 32 46#define CACHE_LINE_SIZE 32
32 47
33static void __iomem *l2x0_base; 48static void __iomem *l2x0_base;
@@ -36,96 +51,116 @@ static u32 l2x0_way_mask; /* Bitmask of active ways */
36static u32 l2x0_size; 51static u32 l2x0_size;
37static unsigned long sync_reg_offset = L2X0_CACHE_SYNC; 52static unsigned long sync_reg_offset = L2X0_CACHE_SYNC;
38 53
39/* Aurora don't have the cache ID register available, so we have to
40 * pass it though the device tree */
41static u32 cache_id_part_number_from_dt;
42
43struct l2x0_regs l2x0_saved_regs; 54struct l2x0_regs l2x0_saved_regs;
44 55
45struct l2x0_of_data { 56/*
46 void (*setup)(const struct device_node *, u32 *, u32 *); 57 * Common code for all cache controllers.
47 void (*save)(void); 58 */
48 struct outer_cache_fns outer_cache; 59static inline void l2c_wait_mask(void __iomem *reg, unsigned long mask)
49};
50
51static bool of_init = false;
52
53static inline void cache_wait_way(void __iomem *reg, unsigned long mask)
54{ 60{
55 /* wait for cache operation by line or way to complete */ 61 /* wait for cache operation by line or way to complete */
56 while (readl_relaxed(reg) & mask) 62 while (readl_relaxed(reg) & mask)
57 cpu_relax(); 63 cpu_relax();
58} 64}
59 65
60#ifdef CONFIG_CACHE_PL310 66/*
61static inline void cache_wait(void __iomem *reg, unsigned long mask) 67 * By default, we write directly to secure registers. Platforms must
68 * override this if they are running non-secure.
69 */
70static void l2c_write_sec(unsigned long val, void __iomem *base, unsigned reg)
62{ 71{
63 /* cache operations by line are atomic on PL310 */ 72 if (val == readl_relaxed(base + reg))
73 return;
74 if (outer_cache.write_sec)
75 outer_cache.write_sec(val, reg);
76 else
77 writel_relaxed(val, base + reg);
64} 78}
65#else
66#define cache_wait cache_wait_way
67#endif
68 79
69static inline void cache_sync(void) 80/*
81 * This should only be called when we have a requirement that the
82 * register be written due to a work-around, as platforms running
83 * in non-secure mode may not be able to access this register.
84 */
85static inline void l2c_set_debug(void __iomem *base, unsigned long val)
70{ 86{
71 void __iomem *base = l2x0_base; 87 l2c_write_sec(val, base, L2X0_DEBUG_CTRL);
72
73 writel_relaxed(0, base + sync_reg_offset);
74 cache_wait(base + L2X0_CACHE_SYNC, 1);
75} 88}
76 89
77static inline void l2x0_clean_line(unsigned long addr) 90static void __l2c_op_way(void __iomem *reg)
78{ 91{
79 void __iomem *base = l2x0_base; 92 writel_relaxed(l2x0_way_mask, reg);
80 cache_wait(base + L2X0_CLEAN_LINE_PA, 1); 93 l2c_wait_mask(reg, l2x0_way_mask);
81 writel_relaxed(addr, base + L2X0_CLEAN_LINE_PA);
82} 94}
83 95
84static inline void l2x0_inv_line(unsigned long addr) 96static inline void l2c_unlock(void __iomem *base, unsigned num)
85{ 97{
86 void __iomem *base = l2x0_base; 98 unsigned i;
87 cache_wait(base + L2X0_INV_LINE_PA, 1); 99
88 writel_relaxed(addr, base + L2X0_INV_LINE_PA); 100 for (i = 0; i < num; i++) {
101 writel_relaxed(0, base + L2X0_LOCKDOWN_WAY_D_BASE +
102 i * L2X0_LOCKDOWN_STRIDE);
103 writel_relaxed(0, base + L2X0_LOCKDOWN_WAY_I_BASE +
104 i * L2X0_LOCKDOWN_STRIDE);
105 }
89} 106}
90 107
91#if defined(CONFIG_PL310_ERRATA_588369) || defined(CONFIG_PL310_ERRATA_727915) 108/*
92static inline void debug_writel(unsigned long val) 109 * Enable the L2 cache controller. This function must only be
110 * called when the cache controller is known to be disabled.
111 */
112static void l2c_enable(void __iomem *base, u32 aux, unsigned num_lock)
93{ 113{
94 if (outer_cache.set_debug) 114 unsigned long flags;
95 outer_cache.set_debug(val); 115
116 l2c_write_sec(aux, base, L2X0_AUX_CTRL);
117
118 l2c_unlock(base, num_lock);
119
120 local_irq_save(flags);
121 __l2c_op_way(base + L2X0_INV_WAY);
122 writel_relaxed(0, base + sync_reg_offset);
123 l2c_wait_mask(base + sync_reg_offset, 1);
124 local_irq_restore(flags);
125
126 l2c_write_sec(L2X0_CTRL_EN, base, L2X0_CTRL);
96} 127}
97 128
98static void pl310_set_debug(unsigned long val) 129static void l2c_disable(void)
99{ 130{
100 writel_relaxed(val, l2x0_base + L2X0_DEBUG_CTRL); 131 void __iomem *base = l2x0_base;
132
133 outer_cache.flush_all();
134 l2c_write_sec(0, base, L2X0_CTRL);
135 dsb(st);
101} 136}
102#else 137
103/* Optimised out for non-errata case */ 138#ifdef CONFIG_CACHE_PL310
104static inline void debug_writel(unsigned long val) 139static inline void cache_wait(void __iomem *reg, unsigned long mask)
105{ 140{
141 /* cache operations by line are atomic on PL310 */
106} 142}
107 143#else
108#define pl310_set_debug NULL 144#define cache_wait l2c_wait_mask
109#endif 145#endif
110 146
111#ifdef CONFIG_PL310_ERRATA_588369 147static inline void cache_sync(void)
112static inline void l2x0_flush_line(unsigned long addr)
113{ 148{
114 void __iomem *base = l2x0_base; 149 void __iomem *base = l2x0_base;
115 150
116 /* Clean by PA followed by Invalidate by PA */ 151 writel_relaxed(0, base + sync_reg_offset);
117 cache_wait(base + L2X0_CLEAN_LINE_PA, 1); 152 cache_wait(base + L2X0_CACHE_SYNC, 1);
118 writel_relaxed(addr, base + L2X0_CLEAN_LINE_PA);
119 cache_wait(base + L2X0_INV_LINE_PA, 1);
120 writel_relaxed(addr, base + L2X0_INV_LINE_PA);
121} 153}
122#else
123 154
124static inline void l2x0_flush_line(unsigned long addr) 155#if defined(CONFIG_PL310_ERRATA_588369) || defined(CONFIG_PL310_ERRATA_727915)
156static inline void debug_writel(unsigned long val)
157{
158 l2c_set_debug(l2x0_base, val);
159}
160#else
161/* Optimised out for non-errata case */
162static inline void debug_writel(unsigned long val)
125{ 163{
126 void __iomem *base = l2x0_base;
127 cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
128 writel_relaxed(addr, base + L2X0_CLEAN_INV_LINE_PA);
129} 164}
130#endif 165#endif
131 166
@@ -141,8 +176,7 @@ static void l2x0_cache_sync(void)
141static void __l2x0_flush_all(void) 176static void __l2x0_flush_all(void)
142{ 177{
143 debug_writel(0x03); 178 debug_writel(0x03);
144 writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_CLEAN_INV_WAY); 179 __l2c_op_way(l2x0_base + L2X0_CLEAN_INV_WAY);
145 cache_wait_way(l2x0_base + L2X0_CLEAN_INV_WAY, l2x0_way_mask);
146 cache_sync(); 180 cache_sync();
147 debug_writel(0x00); 181 debug_writel(0x00);
148} 182}
@@ -157,275 +191,883 @@ static void l2x0_flush_all(void)
157 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 191 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
158} 192}
159 193
160static void l2x0_clean_all(void) 194static void l2x0_disable(void)
161{ 195{
162 unsigned long flags; 196 unsigned long flags;
163 197
164 /* clean all ways */
165 raw_spin_lock_irqsave(&l2x0_lock, flags); 198 raw_spin_lock_irqsave(&l2x0_lock, flags);
166 writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_CLEAN_WAY); 199 __l2x0_flush_all();
167 cache_wait_way(l2x0_base + L2X0_CLEAN_WAY, l2x0_way_mask); 200 l2c_write_sec(0, l2x0_base, L2X0_CTRL);
168 cache_sync(); 201 dsb(st);
169 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 202 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
170} 203}
171 204
172static void l2x0_inv_all(void) 205static void l2c_save(void __iomem *base)
173{ 206{
174 unsigned long flags; 207 l2x0_saved_regs.aux_ctrl = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
208}
175 209
176 /* invalidate all ways */ 210/*
177 raw_spin_lock_irqsave(&l2x0_lock, flags); 211 * L2C-210 specific code.
178 /* Invalidating when L2 is enabled is a nono */ 212 *
179 BUG_ON(readl(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN); 213 * The L2C-2x0 PA, set/way and sync operations are atomic, but we must
180 writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_INV_WAY); 214 * ensure that no background operation is running. The way operations
181 cache_wait_way(l2x0_base + L2X0_INV_WAY, l2x0_way_mask); 215 * are all background tasks.
182 cache_sync(); 216 *
183 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 217 * While a background operation is in progress, any new operation is
218 * ignored (unspecified whether this causes an error.) Thankfully, not
219 * used on SMP.
220 *
221 * Never has a different sync register other than L2X0_CACHE_SYNC, but
222 * we use sync_reg_offset here so we can share some of this with L2C-310.
223 */
224static void __l2c210_cache_sync(void __iomem *base)
225{
226 writel_relaxed(0, base + sync_reg_offset);
184} 227}
185 228
186static void l2x0_inv_range(unsigned long start, unsigned long end) 229static void __l2c210_op_pa_range(void __iomem *reg, unsigned long start,
230 unsigned long end)
231{
232 while (start < end) {
233 writel_relaxed(start, reg);
234 start += CACHE_LINE_SIZE;
235 }
236}
237
238static void l2c210_inv_range(unsigned long start, unsigned long end)
187{ 239{
188 void __iomem *base = l2x0_base; 240 void __iomem *base = l2x0_base;
189 unsigned long flags;
190 241
191 raw_spin_lock_irqsave(&l2x0_lock, flags);
192 if (start & (CACHE_LINE_SIZE - 1)) { 242 if (start & (CACHE_LINE_SIZE - 1)) {
193 start &= ~(CACHE_LINE_SIZE - 1); 243 start &= ~(CACHE_LINE_SIZE - 1);
194 debug_writel(0x03); 244 writel_relaxed(start, base + L2X0_CLEAN_INV_LINE_PA);
195 l2x0_flush_line(start);
196 debug_writel(0x00);
197 start += CACHE_LINE_SIZE; 245 start += CACHE_LINE_SIZE;
198 } 246 }
199 247
200 if (end & (CACHE_LINE_SIZE - 1)) { 248 if (end & (CACHE_LINE_SIZE - 1)) {
201 end &= ~(CACHE_LINE_SIZE - 1); 249 end &= ~(CACHE_LINE_SIZE - 1);
202 debug_writel(0x03); 250 writel_relaxed(end, base + L2X0_CLEAN_INV_LINE_PA);
203 l2x0_flush_line(end);
204 debug_writel(0x00);
205 } 251 }
206 252
253 __l2c210_op_pa_range(base + L2X0_INV_LINE_PA, start, end);
254 __l2c210_cache_sync(base);
255}
256
257static void l2c210_clean_range(unsigned long start, unsigned long end)
258{
259 void __iomem *base = l2x0_base;
260
261 start &= ~(CACHE_LINE_SIZE - 1);
262 __l2c210_op_pa_range(base + L2X0_CLEAN_LINE_PA, start, end);
263 __l2c210_cache_sync(base);
264}
265
266static void l2c210_flush_range(unsigned long start, unsigned long end)
267{
268 void __iomem *base = l2x0_base;
269
270 start &= ~(CACHE_LINE_SIZE - 1);
271 __l2c210_op_pa_range(base + L2X0_CLEAN_INV_LINE_PA, start, end);
272 __l2c210_cache_sync(base);
273}
274
275static void l2c210_flush_all(void)
276{
277 void __iomem *base = l2x0_base;
278
279 BUG_ON(!irqs_disabled());
280
281 __l2c_op_way(base + L2X0_CLEAN_INV_WAY);
282 __l2c210_cache_sync(base);
283}
284
285static void l2c210_sync(void)
286{
287 __l2c210_cache_sync(l2x0_base);
288}
289
290static void l2c210_resume(void)
291{
292 void __iomem *base = l2x0_base;
293
294 if (!(readl_relaxed(base + L2X0_CTRL) & L2X0_CTRL_EN))
295 l2c_enable(base, l2x0_saved_regs.aux_ctrl, 1);
296}
297
298static const struct l2c_init_data l2c210_data __initconst = {
299 .type = "L2C-210",
300 .way_size_0 = SZ_8K,
301 .num_lock = 1,
302 .enable = l2c_enable,
303 .save = l2c_save,
304 .outer_cache = {
305 .inv_range = l2c210_inv_range,
306 .clean_range = l2c210_clean_range,
307 .flush_range = l2c210_flush_range,
308 .flush_all = l2c210_flush_all,
309 .disable = l2c_disable,
310 .sync = l2c210_sync,
311 .resume = l2c210_resume,
312 },
313};
314
315/*
316 * L2C-220 specific code.
317 *
318 * All operations are background operations: they have to be waited for.
319 * Conflicting requests generate a slave error (which will cause an
320 * imprecise abort.) Never uses sync_reg_offset, so we hard-code the
321 * sync register here.
322 *
323 * However, we can re-use the l2c210_resume call.
324 */
325static inline void __l2c220_cache_sync(void __iomem *base)
326{
327 writel_relaxed(0, base + L2X0_CACHE_SYNC);
328 l2c_wait_mask(base + L2X0_CACHE_SYNC, 1);
329}
330
331static void l2c220_op_way(void __iomem *base, unsigned reg)
332{
333 unsigned long flags;
334
335 raw_spin_lock_irqsave(&l2x0_lock, flags);
336 __l2c_op_way(base + reg);
337 __l2c220_cache_sync(base);
338 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
339}
340
341static unsigned long l2c220_op_pa_range(void __iomem *reg, unsigned long start,
342 unsigned long end, unsigned long flags)
343{
344 raw_spinlock_t *lock = &l2x0_lock;
345
207 while (start < end) { 346 while (start < end) {
208 unsigned long blk_end = start + min(end - start, 4096UL); 347 unsigned long blk_end = start + min(end - start, 4096UL);
209 348
210 while (start < blk_end) { 349 while (start < blk_end) {
211 l2x0_inv_line(start); 350 l2c_wait_mask(reg, 1);
351 writel_relaxed(start, reg);
212 start += CACHE_LINE_SIZE; 352 start += CACHE_LINE_SIZE;
213 } 353 }
214 354
215 if (blk_end < end) { 355 if (blk_end < end) {
216 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 356 raw_spin_unlock_irqrestore(lock, flags);
217 raw_spin_lock_irqsave(&l2x0_lock, flags); 357 raw_spin_lock_irqsave(lock, flags);
218 } 358 }
219 } 359 }
220 cache_wait(base + L2X0_INV_LINE_PA, 1); 360
221 cache_sync(); 361 return flags;
222 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
223} 362}
224 363
225static void l2x0_clean_range(unsigned long start, unsigned long end) 364static void l2c220_inv_range(unsigned long start, unsigned long end)
226{ 365{
227 void __iomem *base = l2x0_base; 366 void __iomem *base = l2x0_base;
228 unsigned long flags; 367 unsigned long flags;
229 368
230 if ((end - start) >= l2x0_size) {
231 l2x0_clean_all();
232 return;
233 }
234
235 raw_spin_lock_irqsave(&l2x0_lock, flags); 369 raw_spin_lock_irqsave(&l2x0_lock, flags);
236 start &= ~(CACHE_LINE_SIZE - 1); 370 if ((start | end) & (CACHE_LINE_SIZE - 1)) {
237 while (start < end) { 371 if (start & (CACHE_LINE_SIZE - 1)) {
238 unsigned long blk_end = start + min(end - start, 4096UL); 372 start &= ~(CACHE_LINE_SIZE - 1);
239 373 writel_relaxed(start, base + L2X0_CLEAN_INV_LINE_PA);
240 while (start < blk_end) {
241 l2x0_clean_line(start);
242 start += CACHE_LINE_SIZE; 374 start += CACHE_LINE_SIZE;
243 } 375 }
244 376
245 if (blk_end < end) { 377 if (end & (CACHE_LINE_SIZE - 1)) {
246 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 378 end &= ~(CACHE_LINE_SIZE - 1);
247 raw_spin_lock_irqsave(&l2x0_lock, flags); 379 l2c_wait_mask(base + L2X0_CLEAN_INV_LINE_PA, 1);
380 writel_relaxed(end, base + L2X0_CLEAN_INV_LINE_PA);
248 } 381 }
249 } 382 }
250 cache_wait(base + L2X0_CLEAN_LINE_PA, 1); 383
251 cache_sync(); 384 flags = l2c220_op_pa_range(base + L2X0_INV_LINE_PA,
385 start, end, flags);
386 l2c_wait_mask(base + L2X0_INV_LINE_PA, 1);
387 __l2c220_cache_sync(base);
252 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 388 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
253} 389}
254 390
255static void l2x0_flush_range(unsigned long start, unsigned long end) 391static void l2c220_clean_range(unsigned long start, unsigned long end)
256{ 392{
257 void __iomem *base = l2x0_base; 393 void __iomem *base = l2x0_base;
258 unsigned long flags; 394 unsigned long flags;
259 395
396 start &= ~(CACHE_LINE_SIZE - 1);
260 if ((end - start) >= l2x0_size) { 397 if ((end - start) >= l2x0_size) {
261 l2x0_flush_all(); 398 l2c220_op_way(base, L2X0_CLEAN_WAY);
262 return; 399 return;
263 } 400 }
264 401
265 raw_spin_lock_irqsave(&l2x0_lock, flags); 402 raw_spin_lock_irqsave(&l2x0_lock, flags);
403 flags = l2c220_op_pa_range(base + L2X0_CLEAN_LINE_PA,
404 start, end, flags);
405 l2c_wait_mask(base + L2X0_CLEAN_INV_LINE_PA, 1);
406 __l2c220_cache_sync(base);
407 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
408}
409
410static void l2c220_flush_range(unsigned long start, unsigned long end)
411{
412 void __iomem *base = l2x0_base;
413 unsigned long flags;
414
266 start &= ~(CACHE_LINE_SIZE - 1); 415 start &= ~(CACHE_LINE_SIZE - 1);
416 if ((end - start) >= l2x0_size) {
417 l2c220_op_way(base, L2X0_CLEAN_INV_WAY);
418 return;
419 }
420
421 raw_spin_lock_irqsave(&l2x0_lock, flags);
422 flags = l2c220_op_pa_range(base + L2X0_CLEAN_INV_LINE_PA,
423 start, end, flags);
424 l2c_wait_mask(base + L2X0_CLEAN_INV_LINE_PA, 1);
425 __l2c220_cache_sync(base);
426 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
427}
428
429static void l2c220_flush_all(void)
430{
431 l2c220_op_way(l2x0_base, L2X0_CLEAN_INV_WAY);
432}
433
434static void l2c220_sync(void)
435{
436 unsigned long flags;
437
438 raw_spin_lock_irqsave(&l2x0_lock, flags);
439 __l2c220_cache_sync(l2x0_base);
440 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
441}
442
443static void l2c220_enable(void __iomem *base, u32 aux, unsigned num_lock)
444{
445 /*
446 * Always enable non-secure access to the lockdown registers -
447 * we write to them as part of the L2C enable sequence so they
448 * need to be accessible.
449 */
450 aux |= L220_AUX_CTRL_NS_LOCKDOWN;
451
452 l2c_enable(base, aux, num_lock);
453}
454
455static const struct l2c_init_data l2c220_data = {
456 .type = "L2C-220",
457 .way_size_0 = SZ_8K,
458 .num_lock = 1,
459 .enable = l2c220_enable,
460 .save = l2c_save,
461 .outer_cache = {
462 .inv_range = l2c220_inv_range,
463 .clean_range = l2c220_clean_range,
464 .flush_range = l2c220_flush_range,
465 .flush_all = l2c220_flush_all,
466 .disable = l2c_disable,
467 .sync = l2c220_sync,
468 .resume = l2c210_resume,
469 },
470};
471
472/*
473 * L2C-310 specific code.
474 *
475 * Very similar to L2C-210, the PA, set/way and sync operations are atomic,
476 * and the way operations are all background tasks. However, issuing an
477 * operation while a background operation is in progress results in a
478 * SLVERR response. We can reuse:
479 *
480 * __l2c210_cache_sync (using sync_reg_offset)
481 * l2c210_sync
482 * l2c210_inv_range (if 588369 is not applicable)
483 * l2c210_clean_range
484 * l2c210_flush_range (if 588369 is not applicable)
485 * l2c210_flush_all (if 727915 is not applicable)
486 *
487 * Errata:
488 * 588369: PL310 R0P0->R1P0, fixed R2P0.
489 * Affects: all clean+invalidate operations
490 * clean and invalidate skips the invalidate step, so we need to issue
491 * separate operations. We also require the above debug workaround
492 * enclosing this code fragment on affected parts. On unaffected parts,
493 * we must not use this workaround without the debug register writes
494 * to avoid exposing a problem similar to 727915.
495 *
496 * 727915: PL310 R2P0->R3P0, fixed R3P1.
497 * Affects: clean+invalidate by way
498 * clean and invalidate by way runs in the background, and a store can
499 * hit the line between the clean operation and invalidate operation,
500 * resulting in the store being lost.
501 *
502 * 752271: PL310 R3P0->R3P1-50REL0, fixed R3P2.
503 * Affects: 8x64-bit (double fill) line fetches
504 * double fill line fetches can fail to cause dirty data to be evicted
505 * from the cache before the new data overwrites the second line.
506 *
507 * 753970: PL310 R3P0, fixed R3P1.
508 * Affects: sync
509 * prevents merging writes after the sync operation, until another L2C
510 * operation is performed (or a number of other conditions.)
511 *
512 * 769419: PL310 R0P0->R3P1, fixed R3P2.
513 * Affects: store buffer
514 * store buffer is not automatically drained.
515 */
516static void l2c310_inv_range_erratum(unsigned long start, unsigned long end)
517{
518 void __iomem *base = l2x0_base;
519
520 if ((start | end) & (CACHE_LINE_SIZE - 1)) {
521 unsigned long flags;
522
523 /* Erratum 588369 for both clean+invalidate operations */
524 raw_spin_lock_irqsave(&l2x0_lock, flags);
525 l2c_set_debug(base, 0x03);
526
527 if (start & (CACHE_LINE_SIZE - 1)) {
528 start &= ~(CACHE_LINE_SIZE - 1);
529 writel_relaxed(start, base + L2X0_CLEAN_LINE_PA);
530 writel_relaxed(start, base + L2X0_INV_LINE_PA);
531 start += CACHE_LINE_SIZE;
532 }
533
534 if (end & (CACHE_LINE_SIZE - 1)) {
535 end &= ~(CACHE_LINE_SIZE - 1);
536 writel_relaxed(end, base + L2X0_CLEAN_LINE_PA);
537 writel_relaxed(end, base + L2X0_INV_LINE_PA);
538 }
539
540 l2c_set_debug(base, 0x00);
541 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
542 }
543
544 __l2c210_op_pa_range(base + L2X0_INV_LINE_PA, start, end);
545 __l2c210_cache_sync(base);
546}
547
548static void l2c310_flush_range_erratum(unsigned long start, unsigned long end)
549{
550 raw_spinlock_t *lock = &l2x0_lock;
551 unsigned long flags;
552 void __iomem *base = l2x0_base;
553
554 raw_spin_lock_irqsave(lock, flags);
267 while (start < end) { 555 while (start < end) {
268 unsigned long blk_end = start + min(end - start, 4096UL); 556 unsigned long blk_end = start + min(end - start, 4096UL);
269 557
270 debug_writel(0x03); 558 l2c_set_debug(base, 0x03);
271 while (start < blk_end) { 559 while (start < blk_end) {
272 l2x0_flush_line(start); 560 writel_relaxed(start, base + L2X0_CLEAN_LINE_PA);
561 writel_relaxed(start, base + L2X0_INV_LINE_PA);
273 start += CACHE_LINE_SIZE; 562 start += CACHE_LINE_SIZE;
274 } 563 }
275 debug_writel(0x00); 564 l2c_set_debug(base, 0x00);
276 565
277 if (blk_end < end) { 566 if (blk_end < end) {
278 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 567 raw_spin_unlock_irqrestore(lock, flags);
279 raw_spin_lock_irqsave(&l2x0_lock, flags); 568 raw_spin_lock_irqsave(lock, flags);
280 } 569 }
281 } 570 }
282 cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); 571 raw_spin_unlock_irqrestore(lock, flags);
283 cache_sync(); 572 __l2c210_cache_sync(base);
284 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
285} 573}
286 574
287static void l2x0_disable(void) 575static void l2c310_flush_all_erratum(void)
288{ 576{
577 void __iomem *base = l2x0_base;
289 unsigned long flags; 578 unsigned long flags;
290 579
291 raw_spin_lock_irqsave(&l2x0_lock, flags); 580 raw_spin_lock_irqsave(&l2x0_lock, flags);
292 __l2x0_flush_all(); 581 l2c_set_debug(base, 0x03);
293 writel_relaxed(0, l2x0_base + L2X0_CTRL); 582 __l2c_op_way(base + L2X0_CLEAN_INV_WAY);
294 dsb(st); 583 l2c_set_debug(base, 0x00);
584 __l2c210_cache_sync(base);
295 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 585 raw_spin_unlock_irqrestore(&l2x0_lock, flags);
296} 586}
297 587
298static void l2x0_unlock(u32 cache_id) 588static void __init l2c310_save(void __iomem *base)
299{ 589{
300 int lockregs; 590 unsigned revision;
301 int i;
302 591
303 switch (cache_id & L2X0_CACHE_ID_PART_MASK) { 592 l2c_save(base);
304 case L2X0_CACHE_ID_PART_L310: 593
305 lockregs = 8; 594 l2x0_saved_regs.tag_latency = readl_relaxed(base +
306 break; 595 L310_TAG_LATENCY_CTRL);
307 case AURORA_CACHE_ID: 596 l2x0_saved_regs.data_latency = readl_relaxed(base +
308 lockregs = 4; 597 L310_DATA_LATENCY_CTRL);
598 l2x0_saved_regs.filter_end = readl_relaxed(base +
599 L310_ADDR_FILTER_END);
600 l2x0_saved_regs.filter_start = readl_relaxed(base +
601 L310_ADDR_FILTER_START);
602
603 revision = readl_relaxed(base + L2X0_CACHE_ID) &
604 L2X0_CACHE_ID_RTL_MASK;
605
606 /* From r2p0, there is Prefetch offset/control register */
607 if (revision >= L310_CACHE_ID_RTL_R2P0)
608 l2x0_saved_regs.prefetch_ctrl = readl_relaxed(base +
609 L310_PREFETCH_CTRL);
610
611 /* From r3p0, there is Power control register */
612 if (revision >= L310_CACHE_ID_RTL_R3P0)
613 l2x0_saved_regs.pwr_ctrl = readl_relaxed(base +
614 L310_POWER_CTRL);
615}
616
617static void l2c310_resume(void)
618{
619 void __iomem *base = l2x0_base;
620
621 if (!(readl_relaxed(base + L2X0_CTRL) & L2X0_CTRL_EN)) {
622 unsigned revision;
623
624 /* restore pl310 setup */
625 writel_relaxed(l2x0_saved_regs.tag_latency,
626 base + L310_TAG_LATENCY_CTRL);
627 writel_relaxed(l2x0_saved_regs.data_latency,
628 base + L310_DATA_LATENCY_CTRL);
629 writel_relaxed(l2x0_saved_regs.filter_end,
630 base + L310_ADDR_FILTER_END);
631 writel_relaxed(l2x0_saved_regs.filter_start,
632 base + L310_ADDR_FILTER_START);
633
634 revision = readl_relaxed(base + L2X0_CACHE_ID) &
635 L2X0_CACHE_ID_RTL_MASK;
636
637 if (revision >= L310_CACHE_ID_RTL_R2P0)
638 l2c_write_sec(l2x0_saved_regs.prefetch_ctrl, base,
639 L310_PREFETCH_CTRL);
640 if (revision >= L310_CACHE_ID_RTL_R3P0)
641 l2c_write_sec(l2x0_saved_regs.pwr_ctrl, base,
642 L310_POWER_CTRL);
643
644 l2c_enable(base, l2x0_saved_regs.aux_ctrl, 8);
645
646 /* Re-enable full-line-of-zeros for Cortex-A9 */
647 if (l2x0_saved_regs.aux_ctrl & L310_AUX_CTRL_FULL_LINE_ZERO)
648 set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1));
649 }
650}
651
652static int l2c310_cpu_enable_flz(struct notifier_block *nb, unsigned long act, void *data)
653{
654 switch (act & ~CPU_TASKS_FROZEN) {
655 case CPU_STARTING:
656 set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1));
309 break; 657 break;
310 default: 658 case CPU_DYING:
311 /* L210 and unknown types */ 659 set_auxcr(get_auxcr() & ~(BIT(3) | BIT(2) | BIT(1)));
312 lockregs = 1;
313 break; 660 break;
314 } 661 }
662 return NOTIFY_OK;
663}
315 664
316 for (i = 0; i < lockregs; i++) { 665static void __init l2c310_enable(void __iomem *base, u32 aux, unsigned num_lock)
317 writel_relaxed(0x0, l2x0_base + L2X0_LOCKDOWN_WAY_D_BASE + 666{
318 i * L2X0_LOCKDOWN_STRIDE); 667 unsigned rev = readl_relaxed(base + L2X0_CACHE_ID) & L2X0_CACHE_ID_PART_MASK;
319 writel_relaxed(0x0, l2x0_base + L2X0_LOCKDOWN_WAY_I_BASE + 668 bool cortex_a9 = read_cpuid_part_number() == ARM_CPU_PART_CORTEX_A9;
320 i * L2X0_LOCKDOWN_STRIDE); 669
670 if (rev >= L310_CACHE_ID_RTL_R2P0) {
671 if (cortex_a9) {
672 aux |= L310_AUX_CTRL_EARLY_BRESP;
673 pr_info("L2C-310 enabling early BRESP for Cortex-A9\n");
674 } else if (aux & L310_AUX_CTRL_EARLY_BRESP) {
675 pr_warn("L2C-310 early BRESP only supported with Cortex-A9\n");
676 aux &= ~L310_AUX_CTRL_EARLY_BRESP;
677 }
678 }
679
680 if (cortex_a9) {
681 u32 aux_cur = readl_relaxed(base + L2X0_AUX_CTRL);
682 u32 acr = get_auxcr();
683
684 pr_debug("Cortex-A9 ACR=0x%08x\n", acr);
685
686 if (acr & BIT(3) && !(aux_cur & L310_AUX_CTRL_FULL_LINE_ZERO))
687 pr_err("L2C-310: full line of zeros enabled in Cortex-A9 but not L2C-310 - invalid\n");
688
689 if (aux & L310_AUX_CTRL_FULL_LINE_ZERO && !(acr & BIT(3)))
690 pr_err("L2C-310: enabling full line of zeros but not enabled in Cortex-A9\n");
691
692 if (!(aux & L310_AUX_CTRL_FULL_LINE_ZERO) && !outer_cache.write_sec) {
693 aux |= L310_AUX_CTRL_FULL_LINE_ZERO;
694 pr_info("L2C-310 full line of zeros enabled for Cortex-A9\n");
695 }
696 } else if (aux & (L310_AUX_CTRL_FULL_LINE_ZERO | L310_AUX_CTRL_EARLY_BRESP)) {
697 pr_err("L2C-310: disabling Cortex-A9 specific feature bits\n");
698 aux &= ~(L310_AUX_CTRL_FULL_LINE_ZERO | L310_AUX_CTRL_EARLY_BRESP);
699 }
700
701 if (aux & (L310_AUX_CTRL_DATA_PREFETCH | L310_AUX_CTRL_INSTR_PREFETCH)) {
702 u32 prefetch = readl_relaxed(base + L310_PREFETCH_CTRL);
703
704 pr_info("L2C-310 %s%s prefetch enabled, offset %u lines\n",
705 aux & L310_AUX_CTRL_INSTR_PREFETCH ? "I" : "",
706 aux & L310_AUX_CTRL_DATA_PREFETCH ? "D" : "",
707 1 + (prefetch & L310_PREFETCH_CTRL_OFFSET_MASK));
708 }
709
710 /* r3p0 or later has power control register */
711 if (rev >= L310_CACHE_ID_RTL_R3P0) {
712 u32 power_ctrl;
713
714 l2c_write_sec(L310_DYNAMIC_CLK_GATING_EN | L310_STNDBY_MODE_EN,
715 base, L310_POWER_CTRL);
716 power_ctrl = readl_relaxed(base + L310_POWER_CTRL);
717 pr_info("L2C-310 dynamic clock gating %sabled, standby mode %sabled\n",
718 power_ctrl & L310_DYNAMIC_CLK_GATING_EN ? "en" : "dis",
719 power_ctrl & L310_STNDBY_MODE_EN ? "en" : "dis");
720 }
721
722 /*
723 * Always enable non-secure access to the lockdown registers -
724 * we write to them as part of the L2C enable sequence so they
725 * need to be accessible.
726 */
727 aux |= L310_AUX_CTRL_NS_LOCKDOWN;
728
729 l2c_enable(base, aux, num_lock);
730
731 if (aux & L310_AUX_CTRL_FULL_LINE_ZERO) {
732 set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1));
733 cpu_notifier(l2c310_cpu_enable_flz, 0);
321 } 734 }
322} 735}
323 736
324void __init l2x0_init(void __iomem *base, u32 aux_val, u32 aux_mask) 737static void __init l2c310_fixup(void __iomem *base, u32 cache_id,
738 struct outer_cache_fns *fns)
325{ 739{
326 u32 aux; 740 unsigned revision = cache_id & L2X0_CACHE_ID_RTL_MASK;
327 u32 cache_id; 741 const char *errata[8];
328 u32 way_size = 0; 742 unsigned n = 0;
329 int ways; 743
330 int way_size_shift = L2X0_WAY_SIZE_SHIFT; 744 if (IS_ENABLED(CONFIG_PL310_ERRATA_588369) &&
331 const char *type; 745 revision < L310_CACHE_ID_RTL_R2P0 &&
746 /* For bcm compatibility */
747 fns->inv_range == l2c210_inv_range) {
748 fns->inv_range = l2c310_inv_range_erratum;
749 fns->flush_range = l2c310_flush_range_erratum;
750 errata[n++] = "588369";
751 }
332 752
333 l2x0_base = base; 753 if (IS_ENABLED(CONFIG_PL310_ERRATA_727915) &&
334 if (cache_id_part_number_from_dt) 754 revision >= L310_CACHE_ID_RTL_R2P0 &&
335 cache_id = cache_id_part_number_from_dt; 755 revision < L310_CACHE_ID_RTL_R3P1) {
336 else 756 fns->flush_all = l2c310_flush_all_erratum;
337 cache_id = readl_relaxed(l2x0_base + L2X0_CACHE_ID); 757 errata[n++] = "727915";
338 aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL); 758 }
759
760 if (revision >= L310_CACHE_ID_RTL_R3P0 &&
761 revision < L310_CACHE_ID_RTL_R3P2) {
762 u32 val = readl_relaxed(base + L310_PREFETCH_CTRL);
763 /* I don't think bit23 is required here... but iMX6 does so */
764 if (val & (BIT(30) | BIT(23))) {
765 val &= ~(BIT(30) | BIT(23));
766 l2c_write_sec(val, base, L310_PREFETCH_CTRL);
767 errata[n++] = "752271";
768 }
769 }
770
771 if (IS_ENABLED(CONFIG_PL310_ERRATA_753970) &&
772 revision == L310_CACHE_ID_RTL_R3P0) {
773 sync_reg_offset = L2X0_DUMMY_REG;
774 errata[n++] = "753970";
775 }
776
777 if (IS_ENABLED(CONFIG_PL310_ERRATA_769419))
778 errata[n++] = "769419";
779
780 if (n) {
781 unsigned i;
339 782
783 pr_info("L2C-310 errat%s", n > 1 ? "a" : "um");
784 for (i = 0; i < n; i++)
785 pr_cont(" %s", errata[i]);
786 pr_cont(" enabled\n");
787 }
788}
789
790static void l2c310_disable(void)
791{
792 /*
793 * If full-line-of-zeros is enabled, we must first disable it in the
794 * Cortex-A9 auxiliary control register before disabling the L2 cache.
795 */
796 if (l2x0_saved_regs.aux_ctrl & L310_AUX_CTRL_FULL_LINE_ZERO)
797 set_auxcr(get_auxcr() & ~(BIT(3) | BIT(2) | BIT(1)));
798
799 l2c_disable();
800}
801
802static const struct l2c_init_data l2c310_init_fns __initconst = {
803 .type = "L2C-310",
804 .way_size_0 = SZ_8K,
805 .num_lock = 8,
806 .enable = l2c310_enable,
807 .fixup = l2c310_fixup,
808 .save = l2c310_save,
809 .outer_cache = {
810 .inv_range = l2c210_inv_range,
811 .clean_range = l2c210_clean_range,
812 .flush_range = l2c210_flush_range,
813 .flush_all = l2c210_flush_all,
814 .disable = l2c310_disable,
815 .sync = l2c210_sync,
816 .resume = l2c310_resume,
817 },
818};
819
820static void __init __l2c_init(const struct l2c_init_data *data,
821 u32 aux_val, u32 aux_mask, u32 cache_id)
822{
823 struct outer_cache_fns fns;
824 unsigned way_size_bits, ways;
825 u32 aux, old_aux;
826
827 /*
828 * Sanity check the aux values. aux_mask is the bits we preserve
829 * from reading the hardware register, and aux_val is the bits we
830 * set.
831 */
832 if (aux_val & aux_mask)
833 pr_alert("L2C: platform provided aux values permit register corruption.\n");
834
835 old_aux = aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
340 aux &= aux_mask; 836 aux &= aux_mask;
341 aux |= aux_val; 837 aux |= aux_val;
342 838
839 if (old_aux != aux)
840 pr_warn("L2C: DT/platform modifies aux control register: 0x%08x -> 0x%08x\n",
841 old_aux, aux);
842
343 /* Determine the number of ways */ 843 /* Determine the number of ways */
344 switch (cache_id & L2X0_CACHE_ID_PART_MASK) { 844 switch (cache_id & L2X0_CACHE_ID_PART_MASK) {
345 case L2X0_CACHE_ID_PART_L310: 845 case L2X0_CACHE_ID_PART_L310:
846 if ((aux_val | ~aux_mask) & (L2C_AUX_CTRL_WAY_SIZE_MASK | L310_AUX_CTRL_ASSOCIATIVITY_16))
847 pr_warn("L2C: DT/platform tries to modify or specify cache size\n");
346 if (aux & (1 << 16)) 848 if (aux & (1 << 16))
347 ways = 16; 849 ways = 16;
348 else 850 else
349 ways = 8; 851 ways = 8;
350 type = "L310";
351#ifdef CONFIG_PL310_ERRATA_753970
352 /* Unmapped register. */
353 sync_reg_offset = L2X0_DUMMY_REG;
354#endif
355 if ((cache_id & L2X0_CACHE_ID_RTL_MASK) <= L2X0_CACHE_ID_RTL_R3P0)
356 outer_cache.set_debug = pl310_set_debug;
357 break; 852 break;
853
358 case L2X0_CACHE_ID_PART_L210: 854 case L2X0_CACHE_ID_PART_L210:
855 case L2X0_CACHE_ID_PART_L220:
359 ways = (aux >> 13) & 0xf; 856 ways = (aux >> 13) & 0xf;
360 type = "L210";
361 break; 857 break;
362 858
363 case AURORA_CACHE_ID: 859 case AURORA_CACHE_ID:
364 sync_reg_offset = AURORA_SYNC_REG;
365 ways = (aux >> 13) & 0xf; 860 ways = (aux >> 13) & 0xf;
366 ways = 2 << ((ways + 1) >> 2); 861 ways = 2 << ((ways + 1) >> 2);
367 way_size_shift = AURORA_WAY_SIZE_SHIFT;
368 type = "Aurora";
369 break; 862 break;
863
370 default: 864 default:
371 /* Assume unknown chips have 8 ways */ 865 /* Assume unknown chips have 8 ways */
372 ways = 8; 866 ways = 8;
373 type = "L2x0 series";
374 break; 867 break;
375 } 868 }
376 869
377 l2x0_way_mask = (1 << ways) - 1; 870 l2x0_way_mask = (1 << ways) - 1;
378 871
379 /* 872 /*
380 * L2 cache Size = Way size * Number of ways 873 * way_size_0 is the size that a way_size value of zero would be
874 * given the calculation: way_size = way_size_0 << way_size_bits.
875 * So, if way_size_bits=0 is reserved, but way_size_bits=1 is 16k,
876 * then way_size_0 would be 8k.
877 *
878 * L2 cache size = number of ways * way size.
381 */ 879 */
382 way_size = (aux & L2X0_AUX_CTRL_WAY_SIZE_MASK) >> 17; 880 way_size_bits = (aux & L2C_AUX_CTRL_WAY_SIZE_MASK) >>
383 way_size = 1 << (way_size + way_size_shift); 881 L2C_AUX_CTRL_WAY_SIZE_SHIFT;
882 l2x0_size = ways * (data->way_size_0 << way_size_bits);
384 883
385 l2x0_size = ways * way_size * SZ_1K; 884 fns = data->outer_cache;
885 fns.write_sec = outer_cache.write_sec;
886 if (data->fixup)
887 data->fixup(l2x0_base, cache_id, &fns);
386 888
387 /* 889 /*
388 * Check if l2x0 controller is already enabled. 890 * Check if l2x0 controller is already enabled. If we are booting
389 * If you are booting from non-secure mode 891 * in non-secure mode accessing the below registers will fault.
390 * accessing the below registers will fault.
391 */ 892 */
392 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) { 893 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN))
393 /* Make sure that I&D is not locked down when starting */ 894 data->enable(l2x0_base, aux, data->num_lock);
394 l2x0_unlock(cache_id);
395 895
396 /* l2x0 controller is disabled */ 896 outer_cache = fns;
397 writel_relaxed(aux, l2x0_base + L2X0_AUX_CTRL);
398 897
399 l2x0_inv_all(); 898 /*
400 899 * It is strange to save the register state before initialisation,
401 /* enable L2X0 */ 900 * but hey, this is what the DT implementations decided to do.
402 writel_relaxed(L2X0_CTRL_EN, l2x0_base + L2X0_CTRL); 901 */
403 } 902 if (data->save)
903 data->save(l2x0_base);
404 904
405 /* Re-read it in case some bits are reserved. */ 905 /* Re-read it in case some bits are reserved. */
406 aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL); 906 aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
407 907
408 /* Save the value for resuming. */ 908 pr_info("%s cache controller enabled, %d ways, %d kB\n",
409 l2x0_saved_regs.aux_ctrl = aux; 909 data->type, ways, l2x0_size >> 10);
910 pr_info("%s: CACHE_ID 0x%08x, AUX_CTRL 0x%08x\n",
911 data->type, cache_id, aux);
912}
913
914void __init l2x0_init(void __iomem *base, u32 aux_val, u32 aux_mask)
915{
916 const struct l2c_init_data *data;
917 u32 cache_id;
918
919 l2x0_base = base;
920
921 cache_id = readl_relaxed(base + L2X0_CACHE_ID);
922
923 switch (cache_id & L2X0_CACHE_ID_PART_MASK) {
924 default:
925 case L2X0_CACHE_ID_PART_L210:
926 data = &l2c210_data;
927 break;
410 928
411 if (!of_init) { 929 case L2X0_CACHE_ID_PART_L220:
412 outer_cache.inv_range = l2x0_inv_range; 930 data = &l2c220_data;
413 outer_cache.clean_range = l2x0_clean_range; 931 break;
414 outer_cache.flush_range = l2x0_flush_range; 932
415 outer_cache.sync = l2x0_cache_sync; 933 case L2X0_CACHE_ID_PART_L310:
416 outer_cache.flush_all = l2x0_flush_all; 934 data = &l2c310_init_fns;
417 outer_cache.inv_all = l2x0_inv_all; 935 break;
418 outer_cache.disable = l2x0_disable;
419 } 936 }
420 937
421 pr_info("%s cache controller enabled\n", type); 938 __l2c_init(data, aux_val, aux_mask, cache_id);
422 pr_info("l2x0: %d ways, CACHE_ID 0x%08x, AUX_CTRL 0x%08x, Cache size: %d kB\n",
423 ways, cache_id, aux, l2x0_size >> 10);
424} 939}
425 940
426#ifdef CONFIG_OF 941#ifdef CONFIG_OF
427static int l2_wt_override; 942static int l2_wt_override;
428 943
944/* Aurora don't have the cache ID register available, so we have to
945 * pass it though the device tree */
946static u32 cache_id_part_number_from_dt;
947
948static void __init l2x0_of_parse(const struct device_node *np,
949 u32 *aux_val, u32 *aux_mask)
950{
951 u32 data[2] = { 0, 0 };
952 u32 tag = 0;
953 u32 dirty = 0;
954 u32 val = 0, mask = 0;
955
956 of_property_read_u32(np, "arm,tag-latency", &tag);
957 if (tag) {
958 mask |= L2X0_AUX_CTRL_TAG_LATENCY_MASK;
959 val |= (tag - 1) << L2X0_AUX_CTRL_TAG_LATENCY_SHIFT;
960 }
961
962 of_property_read_u32_array(np, "arm,data-latency",
963 data, ARRAY_SIZE(data));
964 if (data[0] && data[1]) {
965 mask |= L2X0_AUX_CTRL_DATA_RD_LATENCY_MASK |
966 L2X0_AUX_CTRL_DATA_WR_LATENCY_MASK;
967 val |= ((data[0] - 1) << L2X0_AUX_CTRL_DATA_RD_LATENCY_SHIFT) |
968 ((data[1] - 1) << L2X0_AUX_CTRL_DATA_WR_LATENCY_SHIFT);
969 }
970
971 of_property_read_u32(np, "arm,dirty-latency", &dirty);
972 if (dirty) {
973 mask |= L2X0_AUX_CTRL_DIRTY_LATENCY_MASK;
974 val |= (dirty - 1) << L2X0_AUX_CTRL_DIRTY_LATENCY_SHIFT;
975 }
976
977 *aux_val &= ~mask;
978 *aux_val |= val;
979 *aux_mask &= ~mask;
980}
981
982static const struct l2c_init_data of_l2c210_data __initconst = {
983 .type = "L2C-210",
984 .way_size_0 = SZ_8K,
985 .num_lock = 1,
986 .of_parse = l2x0_of_parse,
987 .enable = l2c_enable,
988 .save = l2c_save,
989 .outer_cache = {
990 .inv_range = l2c210_inv_range,
991 .clean_range = l2c210_clean_range,
992 .flush_range = l2c210_flush_range,
993 .flush_all = l2c210_flush_all,
994 .disable = l2c_disable,
995 .sync = l2c210_sync,
996 .resume = l2c210_resume,
997 },
998};
999
1000static const struct l2c_init_data of_l2c220_data __initconst = {
1001 .type = "L2C-220",
1002 .way_size_0 = SZ_8K,
1003 .num_lock = 1,
1004 .of_parse = l2x0_of_parse,
1005 .enable = l2c220_enable,
1006 .save = l2c_save,
1007 .outer_cache = {
1008 .inv_range = l2c220_inv_range,
1009 .clean_range = l2c220_clean_range,
1010 .flush_range = l2c220_flush_range,
1011 .flush_all = l2c220_flush_all,
1012 .disable = l2c_disable,
1013 .sync = l2c220_sync,
1014 .resume = l2c210_resume,
1015 },
1016};
1017
1018static void __init l2c310_of_parse(const struct device_node *np,
1019 u32 *aux_val, u32 *aux_mask)
1020{
1021 u32 data[3] = { 0, 0, 0 };
1022 u32 tag[3] = { 0, 0, 0 };
1023 u32 filter[2] = { 0, 0 };
1024
1025 of_property_read_u32_array(np, "arm,tag-latency", tag, ARRAY_SIZE(tag));
1026 if (tag[0] && tag[1] && tag[2])
1027 writel_relaxed(
1028 L310_LATENCY_CTRL_RD(tag[0] - 1) |
1029 L310_LATENCY_CTRL_WR(tag[1] - 1) |
1030 L310_LATENCY_CTRL_SETUP(tag[2] - 1),
1031 l2x0_base + L310_TAG_LATENCY_CTRL);
1032
1033 of_property_read_u32_array(np, "arm,data-latency",
1034 data, ARRAY_SIZE(data));
1035 if (data[0] && data[1] && data[2])
1036 writel_relaxed(
1037 L310_LATENCY_CTRL_RD(data[0] - 1) |
1038 L310_LATENCY_CTRL_WR(data[1] - 1) |
1039 L310_LATENCY_CTRL_SETUP(data[2] - 1),
1040 l2x0_base + L310_DATA_LATENCY_CTRL);
1041
1042 of_property_read_u32_array(np, "arm,filter-ranges",
1043 filter, ARRAY_SIZE(filter));
1044 if (filter[1]) {
1045 writel_relaxed(ALIGN(filter[0] + filter[1], SZ_1M),
1046 l2x0_base + L310_ADDR_FILTER_END);
1047 writel_relaxed((filter[0] & ~(SZ_1M - 1)) | L310_ADDR_FILTER_EN,
1048 l2x0_base + L310_ADDR_FILTER_START);
1049 }
1050}
1051
1052static const struct l2c_init_data of_l2c310_data __initconst = {
1053 .type = "L2C-310",
1054 .way_size_0 = SZ_8K,
1055 .num_lock = 8,
1056 .of_parse = l2c310_of_parse,
1057 .enable = l2c310_enable,
1058 .fixup = l2c310_fixup,
1059 .save = l2c310_save,
1060 .outer_cache = {
1061 .inv_range = l2c210_inv_range,
1062 .clean_range = l2c210_clean_range,
1063 .flush_range = l2c210_flush_range,
1064 .flush_all = l2c210_flush_all,
1065 .disable = l2c310_disable,
1066 .sync = l2c210_sync,
1067 .resume = l2c310_resume,
1068 },
1069};
1070
429/* 1071/*
430 * Note that the end addresses passed to Linux primitives are 1072 * Note that the end addresses passed to Linux primitives are
431 * noninclusive, while the hardware cache range operations use 1073 * noninclusive, while the hardware cache range operations use
@@ -524,6 +1166,100 @@ static void aurora_flush_range(unsigned long start, unsigned long end)
524 } 1166 }
525} 1167}
526 1168
1169static void aurora_save(void __iomem *base)
1170{
1171 l2x0_saved_regs.ctrl = readl_relaxed(base + L2X0_CTRL);
1172 l2x0_saved_regs.aux_ctrl = readl_relaxed(base + L2X0_AUX_CTRL);
1173}
1174
1175static void aurora_resume(void)
1176{
1177 void __iomem *base = l2x0_base;
1178
1179 if (!(readl(base + L2X0_CTRL) & L2X0_CTRL_EN)) {
1180 writel_relaxed(l2x0_saved_regs.aux_ctrl, base + L2X0_AUX_CTRL);
1181 writel_relaxed(l2x0_saved_regs.ctrl, base + L2X0_CTRL);
1182 }
1183}
1184
1185/*
1186 * For Aurora cache in no outer mode, enable via the CP15 coprocessor
1187 * broadcasting of cache commands to L2.
1188 */
1189static void __init aurora_enable_no_outer(void __iomem *base, u32 aux,
1190 unsigned num_lock)
1191{
1192 u32 u;
1193
1194 asm volatile("mrc p15, 1, %0, c15, c2, 0" : "=r" (u));
1195 u |= AURORA_CTRL_FW; /* Set the FW bit */
1196 asm volatile("mcr p15, 1, %0, c15, c2, 0" : : "r" (u));
1197
1198 isb();
1199
1200 l2c_enable(base, aux, num_lock);
1201}
1202
1203static void __init aurora_fixup(void __iomem *base, u32 cache_id,
1204 struct outer_cache_fns *fns)
1205{
1206 sync_reg_offset = AURORA_SYNC_REG;
1207}
1208
1209static void __init aurora_of_parse(const struct device_node *np,
1210 u32 *aux_val, u32 *aux_mask)
1211{
1212 u32 val = AURORA_ACR_REPLACEMENT_TYPE_SEMIPLRU;
1213 u32 mask = AURORA_ACR_REPLACEMENT_MASK;
1214
1215 of_property_read_u32(np, "cache-id-part",
1216 &cache_id_part_number_from_dt);
1217
1218 /* Determine and save the write policy */
1219 l2_wt_override = of_property_read_bool(np, "wt-override");
1220
1221 if (l2_wt_override) {
1222 val |= AURORA_ACR_FORCE_WRITE_THRO_POLICY;
1223 mask |= AURORA_ACR_FORCE_WRITE_POLICY_MASK;
1224 }
1225
1226 *aux_val &= ~mask;
1227 *aux_val |= val;
1228 *aux_mask &= ~mask;
1229}
1230
1231static const struct l2c_init_data of_aurora_with_outer_data __initconst = {
1232 .type = "Aurora",
1233 .way_size_0 = SZ_4K,
1234 .num_lock = 4,
1235 .of_parse = aurora_of_parse,
1236 .enable = l2c_enable,
1237 .fixup = aurora_fixup,
1238 .save = aurora_save,
1239 .outer_cache = {
1240 .inv_range = aurora_inv_range,
1241 .clean_range = aurora_clean_range,
1242 .flush_range = aurora_flush_range,
1243 .flush_all = l2x0_flush_all,
1244 .disable = l2x0_disable,
1245 .sync = l2x0_cache_sync,
1246 .resume = aurora_resume,
1247 },
1248};
1249
1250static const struct l2c_init_data of_aurora_no_outer_data __initconst = {
1251 .type = "Aurora",
1252 .way_size_0 = SZ_4K,
1253 .num_lock = 4,
1254 .of_parse = aurora_of_parse,
1255 .enable = aurora_enable_no_outer,
1256 .fixup = aurora_fixup,
1257 .save = aurora_save,
1258 .outer_cache = {
1259 .resume = aurora_resume,
1260 },
1261};
1262
527/* 1263/*
528 * For certain Broadcom SoCs, depending on the address range, different offsets 1264 * For certain Broadcom SoCs, depending on the address range, different offsets
529 * need to be added to the address before passing it to L2 for 1265 * need to be added to the address before passing it to L2 for
@@ -588,16 +1324,16 @@ static void bcm_inv_range(unsigned long start, unsigned long end)
588 1324
589 /* normal case, no cross section between start and end */ 1325 /* normal case, no cross section between start and end */
590 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) { 1326 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) {
591 l2x0_inv_range(new_start, new_end); 1327 l2c210_inv_range(new_start, new_end);
592 return; 1328 return;
593 } 1329 }
594 1330
595 /* They cross sections, so it can only be a cross from section 1331 /* They cross sections, so it can only be a cross from section
596 * 2 to section 3 1332 * 2 to section 3
597 */ 1333 */
598 l2x0_inv_range(new_start, 1334 l2c210_inv_range(new_start,
599 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1)); 1335 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));
600 l2x0_inv_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR), 1336 l2c210_inv_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR),
601 new_end); 1337 new_end);
602} 1338}
603 1339
@@ -610,26 +1346,21 @@ static void bcm_clean_range(unsigned long start, unsigned long end)
610 if (unlikely(end <= start)) 1346 if (unlikely(end <= start))
611 return; 1347 return;
612 1348
613 if ((end - start) >= l2x0_size) {
614 l2x0_clean_all();
615 return;
616 }
617
618 new_start = bcm_l2_phys_addr(start); 1349 new_start = bcm_l2_phys_addr(start);
619 new_end = bcm_l2_phys_addr(end); 1350 new_end = bcm_l2_phys_addr(end);
620 1351
621 /* normal case, no cross section between start and end */ 1352 /* normal case, no cross section between start and end */
622 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) { 1353 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) {
623 l2x0_clean_range(new_start, new_end); 1354 l2c210_clean_range(new_start, new_end);
624 return; 1355 return;
625 } 1356 }
626 1357
627 /* They cross sections, so it can only be a cross from section 1358 /* They cross sections, so it can only be a cross from section
628 * 2 to section 3 1359 * 2 to section 3
629 */ 1360 */
630 l2x0_clean_range(new_start, 1361 l2c210_clean_range(new_start,
631 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1)); 1362 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));
632 l2x0_clean_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR), 1363 l2c210_clean_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR),
633 new_end); 1364 new_end);
634} 1365}
635 1366
@@ -643,7 +1374,7 @@ static void bcm_flush_range(unsigned long start, unsigned long end)
643 return; 1374 return;
644 1375
645 if ((end - start) >= l2x0_size) { 1376 if ((end - start) >= l2x0_size) {
646 l2x0_flush_all(); 1377 outer_cache.flush_all();
647 return; 1378 return;
648 } 1379 }
649 1380
@@ -652,283 +1383,67 @@ static void bcm_flush_range(unsigned long start, unsigned long end)
652 1383
653 /* normal case, no cross section between start and end */ 1384 /* normal case, no cross section between start and end */
654 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) { 1385 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) {
655 l2x0_flush_range(new_start, new_end); 1386 l2c210_flush_range(new_start, new_end);
656 return; 1387 return;
657 } 1388 }
658 1389
659 /* They cross sections, so it can only be a cross from section 1390 /* They cross sections, so it can only be a cross from section
660 * 2 to section 3 1391 * 2 to section 3
661 */ 1392 */
662 l2x0_flush_range(new_start, 1393 l2c210_flush_range(new_start,
663 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1)); 1394 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));
664 l2x0_flush_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR), 1395 l2c210_flush_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR),
665 new_end); 1396 new_end);
666} 1397}
667 1398
668static void __init l2x0_of_setup(const struct device_node *np, 1399/* Broadcom L2C-310 start from ARMs R3P2 or later, and require no fixups */
669 u32 *aux_val, u32 *aux_mask) 1400static const struct l2c_init_data of_bcm_l2x0_data __initconst = {
670{ 1401 .type = "BCM-L2C-310",
671 u32 data[2] = { 0, 0 }; 1402 .way_size_0 = SZ_8K,
672 u32 tag = 0; 1403 .num_lock = 8,
673 u32 dirty = 0; 1404 .of_parse = l2c310_of_parse,
674 u32 val = 0, mask = 0; 1405 .enable = l2c310_enable,
675 1406 .save = l2c310_save,
676 of_property_read_u32(np, "arm,tag-latency", &tag); 1407 .outer_cache = {
677 if (tag) { 1408 .inv_range = bcm_inv_range,
678 mask |= L2X0_AUX_CTRL_TAG_LATENCY_MASK; 1409 .clean_range = bcm_clean_range,
679 val |= (tag - 1) << L2X0_AUX_CTRL_TAG_LATENCY_SHIFT; 1410 .flush_range = bcm_flush_range,
680 } 1411 .flush_all = l2c210_flush_all,
681 1412 .disable = l2c310_disable,
682 of_property_read_u32_array(np, "arm,data-latency", 1413 .sync = l2c210_sync,
683 data, ARRAY_SIZE(data)); 1414 .resume = l2c310_resume,
684 if (data[0] && data[1]) { 1415 },
685 mask |= L2X0_AUX_CTRL_DATA_RD_LATENCY_MASK | 1416};
686 L2X0_AUX_CTRL_DATA_WR_LATENCY_MASK;
687 val |= ((data[0] - 1) << L2X0_AUX_CTRL_DATA_RD_LATENCY_SHIFT) |
688 ((data[1] - 1) << L2X0_AUX_CTRL_DATA_WR_LATENCY_SHIFT);
689 }
690
691 of_property_read_u32(np, "arm,dirty-latency", &dirty);
692 if (dirty) {
693 mask |= L2X0_AUX_CTRL_DIRTY_LATENCY_MASK;
694 val |= (dirty - 1) << L2X0_AUX_CTRL_DIRTY_LATENCY_SHIFT;
695 }
696
697 *aux_val &= ~mask;
698 *aux_val |= val;
699 *aux_mask &= ~mask;
700}
701
702static void __init pl310_of_setup(const struct device_node *np,
703 u32 *aux_val, u32 *aux_mask)
704{
705 u32 data[3] = { 0, 0, 0 };
706 u32 tag[3] = { 0, 0, 0 };
707 u32 filter[2] = { 0, 0 };
708
709 of_property_read_u32_array(np, "arm,tag-latency", tag, ARRAY_SIZE(tag));
710 if (tag[0] && tag[1] && tag[2])
711 writel_relaxed(
712 ((tag[0] - 1) << L2X0_LATENCY_CTRL_RD_SHIFT) |
713 ((tag[1] - 1) << L2X0_LATENCY_CTRL_WR_SHIFT) |
714 ((tag[2] - 1) << L2X0_LATENCY_CTRL_SETUP_SHIFT),
715 l2x0_base + L2X0_TAG_LATENCY_CTRL);
716
717 of_property_read_u32_array(np, "arm,data-latency",
718 data, ARRAY_SIZE(data));
719 if (data[0] && data[1] && data[2])
720 writel_relaxed(
721 ((data[0] - 1) << L2X0_LATENCY_CTRL_RD_SHIFT) |
722 ((data[1] - 1) << L2X0_LATENCY_CTRL_WR_SHIFT) |
723 ((data[2] - 1) << L2X0_LATENCY_CTRL_SETUP_SHIFT),
724 l2x0_base + L2X0_DATA_LATENCY_CTRL);
725
726 of_property_read_u32_array(np, "arm,filter-ranges",
727 filter, ARRAY_SIZE(filter));
728 if (filter[1]) {
729 writel_relaxed(ALIGN(filter[0] + filter[1], SZ_1M),
730 l2x0_base + L2X0_ADDR_FILTER_END);
731 writel_relaxed((filter[0] & ~(SZ_1M - 1)) | L2X0_ADDR_FILTER_EN,
732 l2x0_base + L2X0_ADDR_FILTER_START);
733 }
734}
735
736static void __init pl310_save(void)
737{
738 u32 l2x0_revision = readl_relaxed(l2x0_base + L2X0_CACHE_ID) &
739 L2X0_CACHE_ID_RTL_MASK;
740
741 l2x0_saved_regs.tag_latency = readl_relaxed(l2x0_base +
742 L2X0_TAG_LATENCY_CTRL);
743 l2x0_saved_regs.data_latency = readl_relaxed(l2x0_base +
744 L2X0_DATA_LATENCY_CTRL);
745 l2x0_saved_regs.filter_end = readl_relaxed(l2x0_base +
746 L2X0_ADDR_FILTER_END);
747 l2x0_saved_regs.filter_start = readl_relaxed(l2x0_base +
748 L2X0_ADDR_FILTER_START);
749
750 if (l2x0_revision >= L2X0_CACHE_ID_RTL_R2P0) {
751 /*
752 * From r2p0, there is Prefetch offset/control register
753 */
754 l2x0_saved_regs.prefetch_ctrl = readl_relaxed(l2x0_base +
755 L2X0_PREFETCH_CTRL);
756 /*
757 * From r3p0, there is Power control register
758 */
759 if (l2x0_revision >= L2X0_CACHE_ID_RTL_R3P0)
760 l2x0_saved_regs.pwr_ctrl = readl_relaxed(l2x0_base +
761 L2X0_POWER_CTRL);
762 }
763}
764 1417
765static void aurora_save(void) 1418static void __init tauros3_save(void __iomem *base)
766{ 1419{
767 l2x0_saved_regs.ctrl = readl_relaxed(l2x0_base + L2X0_CTRL); 1420 l2c_save(base);
768 l2x0_saved_regs.aux_ctrl = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
769}
770 1421
771static void __init tauros3_save(void)
772{
773 l2x0_saved_regs.aux2_ctrl = 1422 l2x0_saved_regs.aux2_ctrl =
774 readl_relaxed(l2x0_base + TAUROS3_AUX2_CTRL); 1423 readl_relaxed(base + TAUROS3_AUX2_CTRL);
775 l2x0_saved_regs.prefetch_ctrl = 1424 l2x0_saved_regs.prefetch_ctrl =
776 readl_relaxed(l2x0_base + L2X0_PREFETCH_CTRL); 1425 readl_relaxed(base + L310_PREFETCH_CTRL);
777}
778
779static void l2x0_resume(void)
780{
781 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) {
782 /* restore aux ctrl and enable l2 */
783 l2x0_unlock(readl_relaxed(l2x0_base + L2X0_CACHE_ID));
784
785 writel_relaxed(l2x0_saved_regs.aux_ctrl, l2x0_base +
786 L2X0_AUX_CTRL);
787
788 l2x0_inv_all();
789
790 writel_relaxed(L2X0_CTRL_EN, l2x0_base + L2X0_CTRL);
791 }
792}
793
794static void pl310_resume(void)
795{
796 u32 l2x0_revision;
797
798 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) {
799 /* restore pl310 setup */
800 writel_relaxed(l2x0_saved_regs.tag_latency,
801 l2x0_base + L2X0_TAG_LATENCY_CTRL);
802 writel_relaxed(l2x0_saved_regs.data_latency,
803 l2x0_base + L2X0_DATA_LATENCY_CTRL);
804 writel_relaxed(l2x0_saved_regs.filter_end,
805 l2x0_base + L2X0_ADDR_FILTER_END);
806 writel_relaxed(l2x0_saved_regs.filter_start,
807 l2x0_base + L2X0_ADDR_FILTER_START);
808
809 l2x0_revision = readl_relaxed(l2x0_base + L2X0_CACHE_ID) &
810 L2X0_CACHE_ID_RTL_MASK;
811
812 if (l2x0_revision >= L2X0_CACHE_ID_RTL_R2P0) {
813 writel_relaxed(l2x0_saved_regs.prefetch_ctrl,
814 l2x0_base + L2X0_PREFETCH_CTRL);
815 if (l2x0_revision >= L2X0_CACHE_ID_RTL_R3P0)
816 writel_relaxed(l2x0_saved_regs.pwr_ctrl,
817 l2x0_base + L2X0_POWER_CTRL);
818 }
819 }
820
821 l2x0_resume();
822}
823
824static void aurora_resume(void)
825{
826 if (!(readl(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) {
827 writel_relaxed(l2x0_saved_regs.aux_ctrl,
828 l2x0_base + L2X0_AUX_CTRL);
829 writel_relaxed(l2x0_saved_regs.ctrl, l2x0_base + L2X0_CTRL);
830 }
831} 1426}
832 1427
833static void tauros3_resume(void) 1428static void tauros3_resume(void)
834{ 1429{
835 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) { 1430 void __iomem *base = l2x0_base;
1431
1432 if (!(readl_relaxed(base + L2X0_CTRL) & L2X0_CTRL_EN)) {
836 writel_relaxed(l2x0_saved_regs.aux2_ctrl, 1433 writel_relaxed(l2x0_saved_regs.aux2_ctrl,
837 l2x0_base + TAUROS3_AUX2_CTRL); 1434 base + TAUROS3_AUX2_CTRL);
838 writel_relaxed(l2x0_saved_regs.prefetch_ctrl, 1435 writel_relaxed(l2x0_saved_regs.prefetch_ctrl,
839 l2x0_base + L2X0_PREFETCH_CTRL); 1436 base + L310_PREFETCH_CTRL);
840 }
841 1437
842 l2x0_resume(); 1438 l2c_enable(base, l2x0_saved_regs.aux_ctrl, 8);
843}
844
845static void __init aurora_broadcast_l2_commands(void)
846{
847 __u32 u;
848 /* Enable Broadcasting of cache commands to L2*/
849 __asm__ __volatile__("mrc p15, 1, %0, c15, c2, 0" : "=r"(u));
850 u |= AURORA_CTRL_FW; /* Set the FW bit */
851 __asm__ __volatile__("mcr p15, 1, %0, c15, c2, 0\n" : : "r"(u));
852 isb();
853}
854
855static void __init aurora_of_setup(const struct device_node *np,
856 u32 *aux_val, u32 *aux_mask)
857{
858 u32 val = AURORA_ACR_REPLACEMENT_TYPE_SEMIPLRU;
859 u32 mask = AURORA_ACR_REPLACEMENT_MASK;
860
861 of_property_read_u32(np, "cache-id-part",
862 &cache_id_part_number_from_dt);
863
864 /* Determine and save the write policy */
865 l2_wt_override = of_property_read_bool(np, "wt-override");
866
867 if (l2_wt_override) {
868 val |= AURORA_ACR_FORCE_WRITE_THRO_POLICY;
869 mask |= AURORA_ACR_FORCE_WRITE_POLICY_MASK;
870 } 1439 }
871
872 *aux_val &= ~mask;
873 *aux_val |= val;
874 *aux_mask &= ~mask;
875} 1440}
876 1441
877static const struct l2x0_of_data pl310_data = { 1442static const struct l2c_init_data of_tauros3_data __initconst = {
878 .setup = pl310_of_setup, 1443 .type = "Tauros3",
879 .save = pl310_save, 1444 .way_size_0 = SZ_8K,
880 .outer_cache = { 1445 .num_lock = 8,
881 .resume = pl310_resume, 1446 .enable = l2c_enable,
882 .inv_range = l2x0_inv_range,
883 .clean_range = l2x0_clean_range,
884 .flush_range = l2x0_flush_range,
885 .sync = l2x0_cache_sync,
886 .flush_all = l2x0_flush_all,
887 .inv_all = l2x0_inv_all,
888 .disable = l2x0_disable,
889 },
890};
891
892static const struct l2x0_of_data l2x0_data = {
893 .setup = l2x0_of_setup,
894 .save = NULL,
895 .outer_cache = {
896 .resume = l2x0_resume,
897 .inv_range = l2x0_inv_range,
898 .clean_range = l2x0_clean_range,
899 .flush_range = l2x0_flush_range,
900 .sync = l2x0_cache_sync,
901 .flush_all = l2x0_flush_all,
902 .inv_all = l2x0_inv_all,
903 .disable = l2x0_disable,
904 },
905};
906
907static const struct l2x0_of_data aurora_with_outer_data = {
908 .setup = aurora_of_setup,
909 .save = aurora_save,
910 .outer_cache = {
911 .resume = aurora_resume,
912 .inv_range = aurora_inv_range,
913 .clean_range = aurora_clean_range,
914 .flush_range = aurora_flush_range,
915 .sync = l2x0_cache_sync,
916 .flush_all = l2x0_flush_all,
917 .inv_all = l2x0_inv_all,
918 .disable = l2x0_disable,
919 },
920};
921
922static const struct l2x0_of_data aurora_no_outer_data = {
923 .setup = aurora_of_setup,
924 .save = aurora_save,
925 .outer_cache = {
926 .resume = aurora_resume,
927 },
928};
929
930static const struct l2x0_of_data tauros3_data = {
931 .setup = NULL,
932 .save = tauros3_save, 1447 .save = tauros3_save,
933 /* Tauros3 broadcasts L1 cache operations to L2 */ 1448 /* Tauros3 broadcasts L1 cache operations to L2 */
934 .outer_cache = { 1449 .outer_cache = {
@@ -936,43 +1451,26 @@ static const struct l2x0_of_data tauros3_data = {
936 }, 1451 },
937}; 1452};
938 1453
939static const struct l2x0_of_data bcm_l2x0_data = { 1454#define L2C_ID(name, fns) { .compatible = name, .data = (void *)&fns }
940 .setup = pl310_of_setup,
941 .save = pl310_save,
942 .outer_cache = {
943 .resume = pl310_resume,
944 .inv_range = bcm_inv_range,
945 .clean_range = bcm_clean_range,
946 .flush_range = bcm_flush_range,
947 .sync = l2x0_cache_sync,
948 .flush_all = l2x0_flush_all,
949 .inv_all = l2x0_inv_all,
950 .disable = l2x0_disable,
951 },
952};
953
954static const struct of_device_id l2x0_ids[] __initconst = { 1455static const struct of_device_id l2x0_ids[] __initconst = {
955 { .compatible = "arm,l210-cache", .data = (void *)&l2x0_data }, 1456 L2C_ID("arm,l210-cache", of_l2c210_data),
956 { .compatible = "arm,l220-cache", .data = (void *)&l2x0_data }, 1457 L2C_ID("arm,l220-cache", of_l2c220_data),
957 { .compatible = "arm,pl310-cache", .data = (void *)&pl310_data }, 1458 L2C_ID("arm,pl310-cache", of_l2c310_data),
958 { .compatible = "bcm,bcm11351-a2-pl310-cache", /* deprecated name */ 1459 L2C_ID("brcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data),
959 .data = (void *)&bcm_l2x0_data}, 1460 L2C_ID("marvell,aurora-outer-cache", of_aurora_with_outer_data),
960 { .compatible = "brcm,bcm11351-a2-pl310-cache", 1461 L2C_ID("marvell,aurora-system-cache", of_aurora_no_outer_data),
961 .data = (void *)&bcm_l2x0_data}, 1462 L2C_ID("marvell,tauros3-cache", of_tauros3_data),
962 { .compatible = "marvell,aurora-outer-cache", 1463 /* Deprecated IDs */
963 .data = (void *)&aurora_with_outer_data}, 1464 L2C_ID("bcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data),
964 { .compatible = "marvell,aurora-system-cache",
965 .data = (void *)&aurora_no_outer_data},
966 { .compatible = "marvell,tauros3-cache",
967 .data = (void *)&tauros3_data },
968 {} 1465 {}
969}; 1466};
970 1467
971int __init l2x0_of_init(u32 aux_val, u32 aux_mask) 1468int __init l2x0_of_init(u32 aux_val, u32 aux_mask)
972{ 1469{
1470 const struct l2c_init_data *data;
973 struct device_node *np; 1471 struct device_node *np;
974 const struct l2x0_of_data *data;
975 struct resource res; 1472 struct resource res;
1473 u32 cache_id, old_aux;
976 1474
977 np = of_find_matching_node(NULL, l2x0_ids); 1475 np = of_find_matching_node(NULL, l2x0_ids);
978 if (!np) 1476 if (!np)
@@ -989,23 +1487,29 @@ int __init l2x0_of_init(u32 aux_val, u32 aux_mask)
989 1487
990 data = of_match_node(l2x0_ids, np)->data; 1488 data = of_match_node(l2x0_ids, np)->data;
991 1489
992 /* L2 configuration can only be changed if the cache is disabled */ 1490 old_aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
993 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) { 1491 if (old_aux != ((old_aux & aux_mask) | aux_val)) {
994 if (data->setup) 1492 pr_warn("L2C: platform modifies aux control register: 0x%08x -> 0x%08x\n",
995 data->setup(np, &aux_val, &aux_mask); 1493 old_aux, (old_aux & aux_mask) | aux_val);
996 1494 } else if (aux_mask != ~0U && aux_val != 0) {
997 /* For aurora cache in no outer mode select the 1495 pr_alert("L2C: platform provided aux values match the hardware, so have no effect. Please remove them.\n");
998 * correct mode using the coprocessor*/
999 if (data == &aurora_no_outer_data)
1000 aurora_broadcast_l2_commands();
1001 } 1496 }
1002 1497
1003 if (data->save) 1498 /* All L2 caches are unified, so this property should be specified */
1004 data->save(); 1499 if (!of_property_read_bool(np, "cache-unified"))
1500 pr_err("L2C: device tree omits to specify unified cache\n");
1501
1502 /* L2 configuration can only be changed if the cache is disabled */
1503 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN))
1504 if (data->of_parse)
1505 data->of_parse(np, &aux_val, &aux_mask);
1506
1507 if (cache_id_part_number_from_dt)
1508 cache_id = cache_id_part_number_from_dt;
1509 else
1510 cache_id = readl_relaxed(l2x0_base + L2X0_CACHE_ID);
1005 1511
1006 of_init = true; 1512 __l2c_init(data, aux_val, aux_mask, cache_id);
1007 memcpy(&outer_cache, &data->outer_cache, sizeof(outer_cache));
1008 l2x0_init(l2x0_base, aux_val, aux_mask);
1009 1513
1010 return 0; 1514 return 0;
1011} 1515}
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-24 21:00:09 -0500