aboutsummaryrefslogtreecommitdiffstats
path: root/arch/mips/kernel/smtc.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/mips/kernel/smtc.c')
-rw-r--r--arch/mips/kernel/smtc.c1322
1 files changed, 1322 insertions, 0 deletions
diff --git a/arch/mips/kernel/smtc.c b/arch/mips/kernel/smtc.c
new file mode 100644
index 000000000000..2e8e52c135e6
--- /dev/null
+++ b/arch/mips/kernel/smtc.c
@@ -0,0 +1,1322 @@
1/* Copyright (C) 2004 Mips Technologies, Inc */
2
3#include <linux/kernel.h>
4#include <linux/sched.h>
5#include <linux/cpumask.h>
6#include <linux/interrupt.h>
7
8#include <asm/cpu.h>
9#include <asm/processor.h>
10#include <asm/atomic.h>
11#include <asm/system.h>
12#include <asm/hardirq.h>
13#include <asm/hazards.h>
14#include <asm/mmu_context.h>
15#include <asm/smp.h>
16#include <asm/mipsregs.h>
17#include <asm/cacheflush.h>
18#include <asm/time.h>
19#include <asm/addrspace.h>
20#include <asm/smtc.h>
21#include <asm/smtc_ipi.h>
22#include <asm/smtc_proc.h>
23
24/*
25 * This file should be built into the kernel only if CONFIG_MIPS_MT_SMTC is set.
26 */
27
28/*
29 * MIPSCPU_INT_BASE is identically defined in both
30 * asm-mips/mips-boards/maltaint.h and asm-mips/mips-boards/simint.h,
31 * but as yet there's no properly organized include structure that
32 * will ensure that the right *int.h file will be included for a
33 * given platform build.
34 */
35
36#define MIPSCPU_INT_BASE 16
37
38#define MIPS_CPU_IPI_IRQ 1
39
40#define LOCK_MT_PRA() \
41 local_irq_save(flags); \
42 mtflags = dmt()
43
44#define UNLOCK_MT_PRA() \
45 emt(mtflags); \
46 local_irq_restore(flags)
47
48#define LOCK_CORE_PRA() \
49 local_irq_save(flags); \
50 mtflags = dvpe()
51
52#define UNLOCK_CORE_PRA() \
53 evpe(mtflags); \
54 local_irq_restore(flags)
55
56/*
57 * Data structures purely associated with SMTC parallelism
58 */
59
60
61/*
62 * Table for tracking ASIDs whose lifetime is prolonged.
63 */
64
65asiduse smtc_live_asid[MAX_SMTC_TLBS][MAX_SMTC_ASIDS];
66
67/*
68 * Clock interrupt "latch" buffers, per "CPU"
69 */
70
71unsigned int ipi_timer_latch[NR_CPUS];
72
73/*
74 * Number of InterProcessor Interupt (IPI) message buffers to allocate
75 */
76
77#define IPIBUF_PER_CPU 4
78
79struct smtc_ipi_q IPIQ[NR_CPUS];
80struct smtc_ipi_q freeIPIq;
81
82
83/* Forward declarations */
84
85void ipi_decode(struct pt_regs *, struct smtc_ipi *);
86void post_direct_ipi(int cpu, struct smtc_ipi *pipi);
87void setup_cross_vpe_interrupts(void);
88void init_smtc_stats(void);
89
90/* Global SMTC Status */
91
92unsigned int smtc_status = 0;
93
94/* Boot command line configuration overrides */
95
96static int vpelimit = 0;
97static int tclimit = 0;
98static int ipibuffers = 0;
99static int nostlb = 0;
100static int asidmask = 0;
101unsigned long smtc_asid_mask = 0xff;
102
103static int __init maxvpes(char *str)
104{
105 get_option(&str, &vpelimit);
106 return 1;
107}
108
109static int __init maxtcs(char *str)
110{
111 get_option(&str, &tclimit);
112 return 1;
113}
114
115static int __init ipibufs(char *str)
116{
117 get_option(&str, &ipibuffers);
118 return 1;
119}
120
121static int __init stlb_disable(char *s)
122{
123 nostlb = 1;
124 return 1;
125}
126
127static int __init asidmask_set(char *str)
128{
129 get_option(&str, &asidmask);
130 switch(asidmask) {
131 case 0x1:
132 case 0x3:
133 case 0x7:
134 case 0xf:
135 case 0x1f:
136 case 0x3f:
137 case 0x7f:
138 case 0xff:
139 smtc_asid_mask = (unsigned long)asidmask;
140 break;
141 default:
142 printk("ILLEGAL ASID mask 0x%x from command line\n", asidmask);
143 }
144 return 1;
145}
146
147__setup("maxvpes=", maxvpes);
148__setup("maxtcs=", maxtcs);
149__setup("ipibufs=", ipibufs);
150__setup("nostlb", stlb_disable);
151__setup("asidmask=", asidmask_set);
152
153/* Enable additional debug checks before going into CPU idle loop */
154#define SMTC_IDLE_HOOK_DEBUG
155
156#ifdef SMTC_IDLE_HOOK_DEBUG
157
158static int hang_trig = 0;
159
160static int __init hangtrig_enable(char *s)
161{
162 hang_trig = 1;
163 return 1;
164}
165
166
167__setup("hangtrig", hangtrig_enable);
168
169#define DEFAULT_BLOCKED_IPI_LIMIT 32
170
171static int timerq_limit = DEFAULT_BLOCKED_IPI_LIMIT;
172
173static int __init tintq(char *str)
174{
175 get_option(&str, &timerq_limit);
176 return 1;
177}
178
179__setup("tintq=", tintq);
180
181int imstuckcount[2][8];
182/* vpemask represents IM/IE bits of per-VPE Status registers, low-to-high */
183int vpemask[2][8] = {{0,1,1,0,0,0,0,1},{0,1,0,0,0,0,0,1}};
184int tcnoprog[NR_CPUS];
185static atomic_t idle_hook_initialized = {0};
186static int clock_hang_reported[NR_CPUS];
187
188#endif /* SMTC_IDLE_HOOK_DEBUG */
189
190/* Initialize shared TLB - the should probably migrate to smtc_setup_cpus() */
191
192void __init sanitize_tlb_entries(void)
193{
194 printk("Deprecated sanitize_tlb_entries() invoked\n");
195}
196
197
198/*
199 * Configure shared TLB - VPC configuration bit must be set by caller
200 */
201
202void smtc_configure_tlb(void)
203{
204 int i,tlbsiz,vpes;
205 unsigned long mvpconf0;
206 unsigned long config1val;
207
208 /* Set up ASID preservation table */
209 for (vpes=0; vpes<MAX_SMTC_TLBS; vpes++) {
210 for(i = 0; i < MAX_SMTC_ASIDS; i++) {
211 smtc_live_asid[vpes][i] = 0;
212 }
213 }
214 mvpconf0 = read_c0_mvpconf0();
215
216 if ((vpes = ((mvpconf0 & MVPCONF0_PVPE)
217 >> MVPCONF0_PVPE_SHIFT) + 1) > 1) {
218 /* If we have multiple VPEs, try to share the TLB */
219 if ((mvpconf0 & MVPCONF0_TLBS) && !nostlb) {
220 /*
221 * If TLB sizing is programmable, shared TLB
222 * size is the total available complement.
223 * Otherwise, we have to take the sum of all
224 * static VPE TLB entries.
225 */
226 if ((tlbsiz = ((mvpconf0 & MVPCONF0_PTLBE)
227 >> MVPCONF0_PTLBE_SHIFT)) == 0) {
228 /*
229 * If there's more than one VPE, there had better
230 * be more than one TC, because we need one to bind
231 * to each VPE in turn to be able to read
232 * its configuration state!
233 */
234 settc(1);
235 /* Stop the TC from doing anything foolish */
236 write_tc_c0_tchalt(TCHALT_H);
237 mips_ihb();
238 /* No need to un-Halt - that happens later anyway */
239 for (i=0; i < vpes; i++) {
240 write_tc_c0_tcbind(i);
241 /*
242 * To be 100% sure we're really getting the right
243 * information, we exit the configuration state
244 * and do an IHB after each rebinding.
245 */
246 write_c0_mvpcontrol(
247 read_c0_mvpcontrol() & ~ MVPCONTROL_VPC );
248 mips_ihb();
249 /*
250 * Only count if the MMU Type indicated is TLB
251 */
252 if(((read_vpe_c0_config() & MIPS_CONF_MT) >> 7) == 1) {
253 config1val = read_vpe_c0_config1();
254 tlbsiz += ((config1val >> 25) & 0x3f) + 1;
255 }
256
257 /* Put core back in configuration state */
258 write_c0_mvpcontrol(
259 read_c0_mvpcontrol() | MVPCONTROL_VPC );
260 mips_ihb();
261 }
262 }
263 write_c0_mvpcontrol(read_c0_mvpcontrol() | MVPCONTROL_STLB);
264
265 /*
266 * Setup kernel data structures to use software total,
267 * rather than read the per-VPE Config1 value. The values
268 * for "CPU 0" gets copied to all the other CPUs as part
269 * of their initialization in smtc_cpu_setup().
270 */
271
272 tlbsiz = tlbsiz & 0x3f; /* MIPS32 limits TLB indices to 64 */
273 cpu_data[0].tlbsize = tlbsiz;
274 smtc_status |= SMTC_TLB_SHARED;
275
276 printk("TLB of %d entry pairs shared by %d VPEs\n",
277 tlbsiz, vpes);
278 } else {
279 printk("WARNING: TLB Not Sharable on SMTC Boot!\n");
280 }
281 }
282}
283
284
285/*
286 * Incrementally build the CPU map out of constituent MIPS MT cores,
287 * using the specified available VPEs and TCs. Plaform code needs
288 * to ensure that each MIPS MT core invokes this routine on reset,
289 * one at a time(!).
290 *
291 * This version of the build_cpu_map and prepare_cpus routines assumes
292 * that *all* TCs of a MIPS MT core will be used for Linux, and that
293 * they will be spread across *all* available VPEs (to minimise the
294 * loss of efficiency due to exception service serialization).
295 * An improved version would pick up configuration information and
296 * possibly leave some TCs/VPEs as "slave" processors.
297 *
298 * Use c0_MVPConf0 to find out how many TCs are available, setting up
299 * phys_cpu_present_map and the logical/physical mappings.
300 */
301
302int __init mipsmt_build_cpu_map(int start_cpu_slot)
303{
304 int i, ntcs;
305
306 /*
307 * The CPU map isn't actually used for anything at this point,
308 * so it's not clear what else we should do apart from set
309 * everything up so that "logical" = "physical".
310 */
311 ntcs = ((read_c0_mvpconf0() & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
312 for (i=start_cpu_slot; i<NR_CPUS && i<ntcs; i++) {
313 cpu_set(i, phys_cpu_present_map);
314 __cpu_number_map[i] = i;
315 __cpu_logical_map[i] = i;
316 }
317 /* Initialize map of CPUs with FPUs */
318 cpus_clear(mt_fpu_cpumask);
319
320 /* One of those TC's is the one booting, and not a secondary... */
321 printk("%i available secondary CPU TC(s)\n", i - 1);
322
323 return i;
324}
325
326/*
327 * Common setup before any secondaries are started
328 * Make sure all CPU's are in a sensible state before we boot any of the
329 * secondaries.
330 *
331 * For MIPS MT "SMTC" operation, we set up all TCs, spread as evenly
332 * as possible across the available VPEs.
333 */
334
335static void smtc_tc_setup(int vpe, int tc, int cpu)
336{
337 settc(tc);
338 write_tc_c0_tchalt(TCHALT_H);
339 mips_ihb();
340 write_tc_c0_tcstatus((read_tc_c0_tcstatus()
341 & ~(TCSTATUS_TKSU | TCSTATUS_DA | TCSTATUS_IXMT))
342 | TCSTATUS_A);
343 write_tc_c0_tccontext(0);
344 /* Bind tc to vpe */
345 write_tc_c0_tcbind(vpe);
346 /* In general, all TCs should have the same cpu_data indications */
347 memcpy(&cpu_data[cpu], &cpu_data[0], sizeof(struct cpuinfo_mips));
348 /* For 34Kf, start with TC/CPU 0 as sole owner of single FPU context */
349 if (cpu_data[0].cputype == CPU_34K)
350 cpu_data[cpu].options &= ~MIPS_CPU_FPU;
351 cpu_data[cpu].vpe_id = vpe;
352 cpu_data[cpu].tc_id = tc;
353}
354
355
356void mipsmt_prepare_cpus(void)
357{
358 int i, vpe, tc, ntc, nvpe, tcpervpe, slop, cpu;
359 unsigned long flags;
360 unsigned long val;
361 int nipi;
362 struct smtc_ipi *pipi;
363
364 /* disable interrupts so we can disable MT */
365 local_irq_save(flags);
366 /* disable MT so we can configure */
367 dvpe();
368 dmt();
369
370 freeIPIq.lock = SPIN_LOCK_UNLOCKED;
371
372 /*
373 * We probably don't have as many VPEs as we do SMP "CPUs",
374 * but it's possible - and in any case we'll never use more!
375 */
376 for (i=0; i<NR_CPUS; i++) {
377 IPIQ[i].head = IPIQ[i].tail = NULL;
378 IPIQ[i].lock = SPIN_LOCK_UNLOCKED;
379 IPIQ[i].depth = 0;
380 ipi_timer_latch[i] = 0;
381 }
382
383 /* cpu_data index starts at zero */
384 cpu = 0;
385 cpu_data[cpu].vpe_id = 0;
386 cpu_data[cpu].tc_id = 0;
387 cpu++;
388
389 /* Report on boot-time options */
390 mips_mt_set_cpuoptions ();
391 if (vpelimit > 0)
392 printk("Limit of %d VPEs set\n", vpelimit);
393 if (tclimit > 0)
394 printk("Limit of %d TCs set\n", tclimit);
395 if (nostlb) {
396 printk("Shared TLB Use Inhibited - UNSAFE for Multi-VPE Operation\n");
397 }
398 if (asidmask)
399 printk("ASID mask value override to 0x%x\n", asidmask);
400
401 /* Temporary */
402#ifdef SMTC_IDLE_HOOK_DEBUG
403 if (hang_trig)
404 printk("Logic Analyser Trigger on suspected TC hang\n");
405#endif /* SMTC_IDLE_HOOK_DEBUG */
406
407 /* Put MVPE's into 'configuration state' */
408 write_c0_mvpcontrol( read_c0_mvpcontrol() | MVPCONTROL_VPC );
409
410 val = read_c0_mvpconf0();
411 nvpe = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1;
412 if (vpelimit > 0 && nvpe > vpelimit)
413 nvpe = vpelimit;
414 ntc = ((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
415 if (ntc > NR_CPUS)
416 ntc = NR_CPUS;
417 if (tclimit > 0 && ntc > tclimit)
418 ntc = tclimit;
419 tcpervpe = ntc / nvpe;
420 slop = ntc % nvpe; /* Residual TCs, < NVPE */
421
422 /* Set up shared TLB */
423 smtc_configure_tlb();
424
425 for (tc = 0, vpe = 0 ; (vpe < nvpe) && (tc < ntc) ; vpe++) {
426 /*
427 * Set the MVP bits.
428 */
429 settc(tc);
430 write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_MVP);
431 if (vpe != 0)
432 printk(", ");
433 printk("VPE %d: TC", vpe);
434 for (i = 0; i < tcpervpe; i++) {
435 /*
436 * TC 0 is bound to VPE 0 at reset,
437 * and is presumably executing this
438 * code. Leave it alone!
439 */
440 if (tc != 0) {
441 smtc_tc_setup(vpe,tc, cpu);
442 cpu++;
443 }
444 printk(" %d", tc);
445 tc++;
446 }
447 if (slop) {
448 if (tc != 0) {
449 smtc_tc_setup(vpe,tc, cpu);
450 cpu++;
451 }
452 printk(" %d", tc);
453 tc++;
454 slop--;
455 }
456 if (vpe != 0) {
457 /*
458 * Clear any stale software interrupts from VPE's Cause
459 */
460 write_vpe_c0_cause(0);
461
462 /*
463 * Clear ERL/EXL of VPEs other than 0
464 * and set restricted interrupt enable/mask.
465 */
466 write_vpe_c0_status((read_vpe_c0_status()
467 & ~(ST0_BEV | ST0_ERL | ST0_EXL | ST0_IM))
468 | (STATUSF_IP0 | STATUSF_IP1 | STATUSF_IP7
469 | ST0_IE));
470 /*
471 * set config to be the same as vpe0,
472 * particularly kseg0 coherency alg
473 */
474 write_vpe_c0_config(read_c0_config());
475 /* Clear any pending timer interrupt */
476 write_vpe_c0_compare(0);
477 /* Propagate Config7 */
478 write_vpe_c0_config7(read_c0_config7());
479 }
480 /* enable multi-threading within VPE */
481 write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() | VPECONTROL_TE);
482 /* enable the VPE */
483 write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA);
484 }
485
486 /*
487 * Pull any physically present but unused TCs out of circulation.
488 */
489 while (tc < (((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1)) {
490 cpu_clear(tc, phys_cpu_present_map);
491 cpu_clear(tc, cpu_present_map);
492 tc++;
493 }
494
495 /* release config state */
496 write_c0_mvpcontrol( read_c0_mvpcontrol() & ~ MVPCONTROL_VPC );
497
498 printk("\n");
499
500 /* Set up coprocessor affinity CPU mask(s) */
501
502 for (tc = 0; tc < ntc; tc++) {
503 if(cpu_data[tc].options & MIPS_CPU_FPU)
504 cpu_set(tc, mt_fpu_cpumask);
505 }
506
507 /* set up ipi interrupts... */
508
509 /* If we have multiple VPEs running, set up the cross-VPE interrupt */
510
511 if (nvpe > 1)
512 setup_cross_vpe_interrupts();
513
514 /* Set up queue of free IPI "messages". */
515 nipi = NR_CPUS * IPIBUF_PER_CPU;
516 if (ipibuffers > 0)
517 nipi = ipibuffers;
518
519 pipi = kmalloc(nipi *sizeof(struct smtc_ipi), GFP_KERNEL);
520 if (pipi == NULL)
521 panic("kmalloc of IPI message buffers failed\n");
522 else
523 printk("IPI buffer pool of %d buffers\n", nipi);
524 for (i = 0; i < nipi; i++) {
525 smtc_ipi_nq(&freeIPIq, pipi);
526 pipi++;
527 }
528
529 /* Arm multithreading and enable other VPEs - but all TCs are Halted */
530 emt(EMT_ENABLE);
531 evpe(EVPE_ENABLE);
532 local_irq_restore(flags);
533 /* Initialize SMTC /proc statistics/diagnostics */
534 init_smtc_stats();
535}
536
537
538/*
539 * Setup the PC, SP, and GP of a secondary processor and start it
540 * running!
541 * smp_bootstrap is the place to resume from
542 * __KSTK_TOS(idle) is apparently the stack pointer
543 * (unsigned long)idle->thread_info the gp
544 *
545 */
546void smtc_boot_secondary(int cpu, struct task_struct *idle)
547{
548 extern u32 kernelsp[NR_CPUS];
549 long flags;
550 int mtflags;
551
552 LOCK_MT_PRA();
553 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) {
554 dvpe();
555 }
556 settc(cpu_data[cpu].tc_id);
557
558 /* pc */
559 write_tc_c0_tcrestart((unsigned long)&smp_bootstrap);
560
561 /* stack pointer */
562 kernelsp[cpu] = __KSTK_TOS(idle);
563 write_tc_gpr_sp(__KSTK_TOS(idle));
564
565 /* global pointer */
566 write_tc_gpr_gp((unsigned long)idle->thread_info);
567
568 smtc_status |= SMTC_MTC_ACTIVE;
569 write_tc_c0_tchalt(0);
570 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) {
571 evpe(EVPE_ENABLE);
572 }
573 UNLOCK_MT_PRA();
574}
575
576void smtc_init_secondary(void)
577{
578 /*
579 * Start timer on secondary VPEs if necessary.
580 * mips_timer_setup should already have been invoked by init/main
581 * on "boot" TC. Like per_cpu_trap_init() hack, this assumes that
582 * SMTC init code assigns TCs consdecutively and in ascending order
583 * to across available VPEs.
584 */
585 if(((read_c0_tcbind() & TCBIND_CURTC) != 0)
586 && ((read_c0_tcbind() & TCBIND_CURVPE)
587 != cpu_data[smp_processor_id() - 1].vpe_id)){
588 write_c0_compare (read_c0_count() + mips_hpt_frequency/HZ);
589 }
590
591 local_irq_enable();
592}
593
594void smtc_smp_finish(void)
595{
596 printk("TC %d going on-line as CPU %d\n",
597 cpu_data[smp_processor_id()].tc_id, smp_processor_id());
598}
599
600void smtc_cpus_done(void)
601{
602}
603
604/*
605 * Support for SMTC-optimized driver IRQ registration
606 */
607
608/*
609 * SMTC Kernel needs to manipulate low-level CPU interrupt mask
610 * in do_IRQ. These are passed in setup_irq_smtc() and stored
611 * in this table.
612 */
613
614int setup_irq_smtc(unsigned int irq, struct irqaction * new,
615 unsigned long hwmask)
616{
617 irq_hwmask[irq] = hwmask;
618
619 return setup_irq(irq, new);
620}
621
622/*
623 * IPI model for SMTC is tricky, because interrupts aren't TC-specific.
624 * Within a VPE one TC can interrupt another by different approaches.
625 * The easiest to get right would probably be to make all TCs except
626 * the target IXMT and set a software interrupt, but an IXMT-based
627 * scheme requires that a handler must run before a new IPI could
628 * be sent, which would break the "broadcast" loops in MIPS MT.
629 * A more gonzo approach within a VPE is to halt the TC, extract
630 * its Restart, Status, and a couple of GPRs, and program the Restart
631 * address to emulate an interrupt.
632 *
633 * Within a VPE, one can be confident that the target TC isn't in
634 * a critical EXL state when halted, since the write to the Halt
635 * register could not have issued on the writing thread if the
636 * halting thread had EXL set. So k0 and k1 of the target TC
637 * can be used by the injection code. Across VPEs, one can't
638 * be certain that the target TC isn't in a critical exception
639 * state. So we try a two-step process of sending a software
640 * interrupt to the target VPE, which either handles the event
641 * itself (if it was the target) or injects the event within
642 * the VPE.
643 */
644
645void smtc_ipi_qdump(void)
646{
647 int i;
648
649 for (i = 0; i < NR_CPUS ;i++) {
650 printk("IPIQ[%d]: head = 0x%x, tail = 0x%x, depth = %d\n",
651 i, (unsigned)IPIQ[i].head, (unsigned)IPIQ[i].tail,
652 IPIQ[i].depth);
653 }
654}
655
656/*
657 * The standard atomic.h primitives don't quite do what we want
658 * here: We need an atomic add-and-return-previous-value (which
659 * could be done with atomic_add_return and a decrement) and an
660 * atomic set/zero-and-return-previous-value (which can't really
661 * be done with the atomic.h primitives). And since this is
662 * MIPS MT, we can assume that we have LL/SC.
663 */
664static __inline__ int atomic_postincrement(unsigned int *pv)
665{
666 unsigned long result;
667
668 unsigned long temp;
669
670 __asm__ __volatile__(
671 "1: ll %0, %2 \n"
672 " addu %1, %0, 1 \n"
673 " sc %1, %2 \n"
674 " beqz %1, 1b \n"
675 " sync \n"
676 : "=&r" (result), "=&r" (temp), "=m" (*pv)
677 : "m" (*pv)
678 : "memory");
679
680 return result;
681}
682
683/* No longer used in IPI dispatch, but retained for future recycling */
684
685static __inline__ int atomic_postclear(unsigned int *pv)
686{
687 unsigned long result;
688
689 unsigned long temp;
690
691 __asm__ __volatile__(
692 "1: ll %0, %2 \n"
693 " or %1, $0, $0 \n"
694 " sc %1, %2 \n"
695 " beqz %1, 1b \n"
696 " sync \n"
697 : "=&r" (result), "=&r" (temp), "=m" (*pv)
698 : "m" (*pv)
699 : "memory");
700
701 return result;
702}
703
704
705void smtc_send_ipi(int cpu, int type, unsigned int action)
706{
707 int tcstatus;
708 struct smtc_ipi *pipi;
709 long flags;
710 int mtflags;
711
712 if (cpu == smp_processor_id()) {
713 printk("Cannot Send IPI to self!\n");
714 return;
715 }
716 /* Set up a descriptor, to be delivered either promptly or queued */
717 pipi = smtc_ipi_dq(&freeIPIq);
718 if (pipi == NULL) {
719 bust_spinlocks(1);
720 mips_mt_regdump(dvpe());
721 panic("IPI Msg. Buffers Depleted\n");
722 }
723 pipi->type = type;
724 pipi->arg = (void *)action;
725 pipi->dest = cpu;
726 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) {
727 /* If not on same VPE, enqueue and send cross-VPE interupt */
728 smtc_ipi_nq(&IPIQ[cpu], pipi);
729 LOCK_CORE_PRA();
730 settc(cpu_data[cpu].tc_id);
731 write_vpe_c0_cause(read_vpe_c0_cause() | C_SW1);
732 UNLOCK_CORE_PRA();
733 } else {
734 /*
735 * Not sufficient to do a LOCK_MT_PRA (dmt) here,
736 * since ASID shootdown on the other VPE may
737 * collide with this operation.
738 */
739 LOCK_CORE_PRA();
740 settc(cpu_data[cpu].tc_id);
741 /* Halt the targeted TC */
742 write_tc_c0_tchalt(TCHALT_H);
743 mips_ihb();
744
745 /*
746 * Inspect TCStatus - if IXMT is set, we have to queue
747 * a message. Otherwise, we set up the "interrupt"
748 * of the other TC
749 */
750 tcstatus = read_tc_c0_tcstatus();
751
752 if ((tcstatus & TCSTATUS_IXMT) != 0) {
753 /*
754 * Spin-waiting here can deadlock,
755 * so we queue the message for the target TC.
756 */
757 write_tc_c0_tchalt(0);
758 UNLOCK_CORE_PRA();
759 /* Try to reduce redundant timer interrupt messages */
760 if(type == SMTC_CLOCK_TICK) {
761 if(atomic_postincrement(&ipi_timer_latch[cpu])!=0) {
762 smtc_ipi_nq(&freeIPIq, pipi);
763 return;
764 }
765 }
766 smtc_ipi_nq(&IPIQ[cpu], pipi);
767 } else {
768 post_direct_ipi(cpu, pipi);
769 write_tc_c0_tchalt(0);
770 UNLOCK_CORE_PRA();
771 }
772 }
773}
774
775/*
776 * Send IPI message to Halted TC, TargTC/TargVPE already having been set
777 */
778void post_direct_ipi(int cpu, struct smtc_ipi *pipi)
779{
780 struct pt_regs *kstack;
781 unsigned long tcstatus;
782 unsigned long tcrestart;
783 extern u32 kernelsp[NR_CPUS];
784 extern void __smtc_ipi_vector(void);
785
786 /* Extract Status, EPC from halted TC */
787 tcstatus = read_tc_c0_tcstatus();
788 tcrestart = read_tc_c0_tcrestart();
789 /* If TCRestart indicates a WAIT instruction, advance the PC */
790 if ((tcrestart & 0x80000000)
791 && ((*(unsigned int *)tcrestart & 0xfe00003f) == 0x42000020)) {
792 tcrestart += 4;
793 }
794 /*
795 * Save on TC's future kernel stack
796 *
797 * CU bit of Status is indicator that TC was
798 * already running on a kernel stack...
799 */
800 if(tcstatus & ST0_CU0) {
801 /* Note that this "- 1" is pointer arithmetic */
802 kstack = ((struct pt_regs *)read_tc_gpr_sp()) - 1;
803 } else {
804 kstack = ((struct pt_regs *)kernelsp[cpu]) - 1;
805 }
806
807 kstack->cp0_epc = (long)tcrestart;
808 /* Save TCStatus */
809 kstack->cp0_tcstatus = tcstatus;
810 /* Pass token of operation to be performed kernel stack pad area */
811 kstack->pad0[4] = (unsigned long)pipi;
812 /* Pass address of function to be called likewise */
813 kstack->pad0[5] = (unsigned long)&ipi_decode;
814 /* Set interrupt exempt and kernel mode */
815 tcstatus |= TCSTATUS_IXMT;
816 tcstatus &= ~TCSTATUS_TKSU;
817 write_tc_c0_tcstatus(tcstatus);
818 ehb();
819 /* Set TC Restart address to be SMTC IPI vector */
820 write_tc_c0_tcrestart(__smtc_ipi_vector);
821}
822
823void ipi_resched_interrupt(struct pt_regs *regs)
824{
825 /* Return from interrupt should be enough to cause scheduler check */
826}
827
828
829void ipi_call_interrupt(struct pt_regs *regs)
830{
831 /* Invoke generic function invocation code in smp.c */
832 smp_call_function_interrupt();
833}
834
835void ipi_decode(struct pt_regs *regs, struct smtc_ipi *pipi)
836{
837 void *arg_copy = pipi->arg;
838 int type_copy = pipi->type;
839 int dest_copy = pipi->dest;
840
841 smtc_ipi_nq(&freeIPIq, pipi);
842 switch (type_copy) {
843 case SMTC_CLOCK_TICK:
844 /* Invoke Clock "Interrupt" */
845 ipi_timer_latch[dest_copy] = 0;
846#ifdef SMTC_IDLE_HOOK_DEBUG
847 clock_hang_reported[dest_copy] = 0;
848#endif /* SMTC_IDLE_HOOK_DEBUG */
849 local_timer_interrupt(0, NULL, regs);
850 break;
851 case LINUX_SMP_IPI:
852 switch ((int)arg_copy) {
853 case SMP_RESCHEDULE_YOURSELF:
854 ipi_resched_interrupt(regs);
855 break;
856 case SMP_CALL_FUNCTION:
857 ipi_call_interrupt(regs);
858 break;
859 default:
860 printk("Impossible SMTC IPI Argument 0x%x\n",
861 (int)arg_copy);
862 break;
863 }
864 break;
865 default:
866 printk("Impossible SMTC IPI Type 0x%x\n", type_copy);
867 break;
868 }
869}
870
871void deferred_smtc_ipi(struct pt_regs *regs)
872{
873 struct smtc_ipi *pipi;
874 unsigned long flags;
875/* DEBUG */
876 int q = smp_processor_id();
877
878 /*
879 * Test is not atomic, but much faster than a dequeue,
880 * and the vast majority of invocations will have a null queue.
881 */
882 if(IPIQ[q].head != NULL) {
883 while((pipi = smtc_ipi_dq(&IPIQ[q])) != NULL) {
884 /* ipi_decode() should be called with interrupts off */
885 local_irq_save(flags);
886 ipi_decode(regs, pipi);
887 local_irq_restore(flags);
888 }
889 }
890}
891
892/*
893 * Send clock tick to all TCs except the one executing the funtion
894 */
895
896void smtc_timer_broadcast(int vpe)
897{
898 int cpu;
899 int myTC = cpu_data[smp_processor_id()].tc_id;
900 int myVPE = cpu_data[smp_processor_id()].vpe_id;
901
902 smtc_cpu_stats[smp_processor_id()].timerints++;
903
904 for_each_online_cpu(cpu) {
905 if (cpu_data[cpu].vpe_id == myVPE &&
906 cpu_data[cpu].tc_id != myTC)
907 smtc_send_ipi(cpu, SMTC_CLOCK_TICK, 0);
908 }
909}
910
911/*
912 * Cross-VPE interrupts in the SMTC prototype use "software interrupts"
913 * set via cross-VPE MTTR manipulation of the Cause register. It would be
914 * in some regards preferable to have external logic for "doorbell" hardware
915 * interrupts.
916 */
917
918static int cpu_ipi_irq = MIPSCPU_INT_BASE + MIPS_CPU_IPI_IRQ;
919
920static irqreturn_t ipi_interrupt(int irq, void *dev_idm, struct pt_regs *regs)
921{
922 int my_vpe = cpu_data[smp_processor_id()].vpe_id;
923 int my_tc = cpu_data[smp_processor_id()].tc_id;
924 int cpu;
925 struct smtc_ipi *pipi;
926 unsigned long tcstatus;
927 int sent;
928 long flags;
929 unsigned int mtflags;
930 unsigned int vpflags;
931
932 /*
933 * So long as cross-VPE interrupts are done via
934 * MFTR/MTTR read-modify-writes of Cause, we need
935 * to stop other VPEs whenever the local VPE does
936 * anything similar.
937 */
938 local_irq_save(flags);
939 vpflags = dvpe();
940 clear_c0_cause(0x100 << MIPS_CPU_IPI_IRQ);
941 set_c0_status(0x100 << MIPS_CPU_IPI_IRQ);
942 irq_enable_hazard();
943 evpe(vpflags);
944 local_irq_restore(flags);
945
946 /*
947 * Cross-VPE Interrupt handler: Try to directly deliver IPIs
948 * queued for TCs on this VPE other than the current one.
949 * Return-from-interrupt should cause us to drain the queue
950 * for the current TC, so we ought not to have to do it explicitly here.
951 */
952
953 for_each_online_cpu(cpu) {
954 if (cpu_data[cpu].vpe_id != my_vpe)
955 continue;
956
957 pipi = smtc_ipi_dq(&IPIQ[cpu]);
958 if (pipi != NULL) {
959 if (cpu_data[cpu].tc_id != my_tc) {
960 sent = 0;
961 LOCK_MT_PRA();
962 settc(cpu_data[cpu].tc_id);
963 write_tc_c0_tchalt(TCHALT_H);
964 mips_ihb();
965 tcstatus = read_tc_c0_tcstatus();
966 if ((tcstatus & TCSTATUS_IXMT) == 0) {
967 post_direct_ipi(cpu, pipi);
968 sent = 1;
969 }
970 write_tc_c0_tchalt(0);
971 UNLOCK_MT_PRA();
972 if (!sent) {
973 smtc_ipi_req(&IPIQ[cpu], pipi);
974 }
975 } else {
976 /*
977 * ipi_decode() should be called
978 * with interrupts off
979 */
980 local_irq_save(flags);
981 ipi_decode(regs, pipi);
982 local_irq_restore(flags);
983 }
984 }
985 }
986
987 return IRQ_HANDLED;
988}
989
990static void ipi_irq_dispatch(struct pt_regs *regs)
991{
992 do_IRQ(cpu_ipi_irq, regs);
993}
994
995static struct irqaction irq_ipi;
996
997void setup_cross_vpe_interrupts(void)
998{
999 if (!cpu_has_vint)
1000 panic("SMTC Kernel requires Vectored Interupt support");
1001
1002 set_vi_handler(MIPS_CPU_IPI_IRQ, ipi_irq_dispatch);
1003
1004 irq_ipi.handler = ipi_interrupt;
1005 irq_ipi.flags = SA_INTERRUPT;
1006 irq_ipi.name = "SMTC_IPI";
1007
1008 setup_irq_smtc(cpu_ipi_irq, &irq_ipi, (0x100 << MIPS_CPU_IPI_IRQ));
1009
1010 irq_desc[cpu_ipi_irq].status |= IRQ_PER_CPU;
1011}
1012
1013/*
1014 * SMTC-specific hacks invoked from elsewhere in the kernel.
1015 */
1016
1017void smtc_idle_loop_hook(void)
1018{
1019#ifdef SMTC_IDLE_HOOK_DEBUG
1020 int im;
1021 int flags;
1022 int mtflags;
1023 int bit;
1024 int vpe;
1025 int tc;
1026 int hook_ntcs;
1027 /*
1028 * printk within DMT-protected regions can deadlock,
1029 * so buffer diagnostic messages for later output.
1030 */
1031 char *pdb_msg;
1032 char id_ho_db_msg[768]; /* worst-case use should be less than 700 */
1033
1034 if (atomic_read(&idle_hook_initialized) == 0) { /* fast test */
1035 if (atomic_add_return(1, &idle_hook_initialized) == 1) {
1036 int mvpconf0;
1037 /* Tedious stuff to just do once */
1038 mvpconf0 = read_c0_mvpconf0();
1039 hook_ntcs = ((mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
1040 if (hook_ntcs > NR_CPUS)
1041 hook_ntcs = NR_CPUS;
1042 for (tc = 0; tc < hook_ntcs; tc++) {
1043 tcnoprog[tc] = 0;
1044 clock_hang_reported[tc] = 0;
1045 }
1046 for (vpe = 0; vpe < 2; vpe++)
1047 for (im = 0; im < 8; im++)
1048 imstuckcount[vpe][im] = 0;
1049 printk("Idle loop test hook initialized for %d TCs\n", hook_ntcs);
1050 atomic_set(&idle_hook_initialized, 1000);
1051 } else {
1052 /* Someone else is initializing in parallel - let 'em finish */
1053 while (atomic_read(&idle_hook_initialized) < 1000)
1054 ;
1055 }
1056 }
1057
1058 /* Have we stupidly left IXMT set somewhere? */
1059 if (read_c0_tcstatus() & 0x400) {
1060 write_c0_tcstatus(read_c0_tcstatus() & ~0x400);
1061 ehb();
1062 printk("Dangling IXMT in cpu_idle()\n");
1063 }
1064
1065 /* Have we stupidly left an IM bit turned off? */
1066#define IM_LIMIT 2000
1067 local_irq_save(flags);
1068 mtflags = dmt();
1069 pdb_msg = &id_ho_db_msg[0];
1070 im = read_c0_status();
1071 vpe = cpu_data[smp_processor_id()].vpe_id;
1072 for (bit = 0; bit < 8; bit++) {
1073 /*
1074 * In current prototype, I/O interrupts
1075 * are masked for VPE > 0
1076 */
1077 if (vpemask[vpe][bit]) {
1078 if (!(im & (0x100 << bit)))
1079 imstuckcount[vpe][bit]++;
1080 else
1081 imstuckcount[vpe][bit] = 0;
1082 if (imstuckcount[vpe][bit] > IM_LIMIT) {
1083 set_c0_status(0x100 << bit);
1084 ehb();
1085 imstuckcount[vpe][bit] = 0;
1086 pdb_msg += sprintf(pdb_msg,
1087 "Dangling IM %d fixed for VPE %d\n", bit,
1088 vpe);
1089 }
1090 }
1091 }
1092
1093 /*
1094 * Now that we limit outstanding timer IPIs, check for hung TC
1095 */
1096 for (tc = 0; tc < NR_CPUS; tc++) {
1097 /* Don't check ourself - we'll dequeue IPIs just below */
1098 if ((tc != smp_processor_id()) &&
1099 ipi_timer_latch[tc] > timerq_limit) {
1100 if (clock_hang_reported[tc] == 0) {
1101 pdb_msg += sprintf(pdb_msg,
1102 "TC %d looks hung with timer latch at %d\n",
1103 tc, ipi_timer_latch[tc]);
1104 clock_hang_reported[tc]++;
1105 }
1106 }
1107 }
1108 emt(mtflags);
1109 local_irq_restore(flags);
1110 if (pdb_msg != &id_ho_db_msg[0])
1111 printk("CPU%d: %s", smp_processor_id(), id_ho_db_msg);
1112#endif /* SMTC_IDLE_HOOK_DEBUG */
1113 /*
1114 * To the extent that we've ever turned interrupts off,
1115 * we may have accumulated deferred IPIs. This is subtle.
1116 * If we use the smtc_ipi_qdepth() macro, we'll get an
1117 * exact number - but we'll also disable interrupts
1118 * and create a window of failure where a new IPI gets
1119 * queued after we test the depth but before we re-enable
1120 * interrupts. So long as IXMT never gets set, however,
1121 * we should be OK: If we pick up something and dispatch
1122 * it here, that's great. If we see nothing, but concurrent
1123 * with this operation, another TC sends us an IPI, IXMT
1124 * is clear, and we'll handle it as a real pseudo-interrupt
1125 * and not a pseudo-pseudo interrupt.
1126 */
1127 if (IPIQ[smp_processor_id()].depth > 0) {
1128 struct smtc_ipi *pipi;
1129 extern void self_ipi(struct smtc_ipi *);
1130
1131 if ((pipi = smtc_ipi_dq(&IPIQ[smp_processor_id()])) != NULL) {
1132 self_ipi(pipi);
1133 smtc_cpu_stats[smp_processor_id()].selfipis++;
1134 }
1135 }
1136}
1137
1138void smtc_soft_dump(void)
1139{
1140 int i;
1141
1142 printk("Counter Interrupts taken per CPU (TC)\n");
1143 for (i=0; i < NR_CPUS; i++) {
1144 printk("%d: %ld\n", i, smtc_cpu_stats[i].timerints);
1145 }
1146 printk("Self-IPI invocations:\n");
1147 for (i=0; i < NR_CPUS; i++) {
1148 printk("%d: %ld\n", i, smtc_cpu_stats[i].selfipis);
1149 }
1150 smtc_ipi_qdump();
1151 printk("Timer IPI Backlogs:\n");
1152 for (i=0; i < NR_CPUS; i++) {
1153 printk("%d: %d\n", i, ipi_timer_latch[i]);
1154 }
1155 printk("%d Recoveries of \"stolen\" FPU\n",
1156 atomic_read(&smtc_fpu_recoveries));
1157}
1158
1159
1160/*
1161 * TLB management routines special to SMTC
1162 */
1163
1164void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu)
1165{
1166 unsigned long flags, mtflags, tcstat, prevhalt, asid;
1167 int tlb, i;
1168
1169 /*
1170 * It would be nice to be able to use a spinlock here,
1171 * but this is invoked from within TLB flush routines
1172 * that protect themselves with DVPE, so if a lock is
1173 * held by another TC, it'll never be freed.
1174 *
1175 * DVPE/DMT must not be done with interrupts enabled,
1176 * so even so most callers will already have disabled
1177 * them, let's be really careful...
1178 */
1179
1180 local_irq_save(flags);
1181 if (smtc_status & SMTC_TLB_SHARED) {
1182 mtflags = dvpe();
1183 tlb = 0;
1184 } else {
1185 mtflags = dmt();
1186 tlb = cpu_data[cpu].vpe_id;
1187 }
1188 asid = asid_cache(cpu);
1189
1190 do {
1191 if (!((asid += ASID_INC) & ASID_MASK) ) {
1192 if (cpu_has_vtag_icache)
1193 flush_icache_all();
1194 /* Traverse all online CPUs (hack requires contigous range) */
1195 for (i = 0; i < num_online_cpus(); i++) {
1196 /*
1197 * We don't need to worry about our own CPU, nor those of
1198 * CPUs who don't share our TLB.
1199 */
1200 if ((i != smp_processor_id()) &&
1201 ((smtc_status & SMTC_TLB_SHARED) ||
1202 (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))) {
1203 settc(cpu_data[i].tc_id);
1204 prevhalt = read_tc_c0_tchalt() & TCHALT_H;
1205 if (!prevhalt) {
1206 write_tc_c0_tchalt(TCHALT_H);
1207 mips_ihb();
1208 }
1209 tcstat = read_tc_c0_tcstatus();
1210 smtc_live_asid[tlb][(tcstat & ASID_MASK)] |= (asiduse)(0x1 << i);
1211 if (!prevhalt)
1212 write_tc_c0_tchalt(0);
1213 }
1214 }
1215 if (!asid) /* fix version if needed */
1216 asid = ASID_FIRST_VERSION;
1217 local_flush_tlb_all(); /* start new asid cycle */
1218 }
1219 } while (smtc_live_asid[tlb][(asid & ASID_MASK)]);
1220
1221 /*
1222 * SMTC shares the TLB within VPEs and possibly across all VPEs.
1223 */
1224 for (i = 0; i < num_online_cpus(); i++) {
1225 if ((smtc_status & SMTC_TLB_SHARED) ||
1226 (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))
1227 cpu_context(i, mm) = asid_cache(i) = asid;
1228 }
1229
1230 if (smtc_status & SMTC_TLB_SHARED)
1231 evpe(mtflags);
1232 else
1233 emt(mtflags);
1234 local_irq_restore(flags);
1235}
1236
1237/*
1238 * Invoked from macros defined in mmu_context.h
1239 * which must already have disabled interrupts
1240 * and done a DVPE or DMT as appropriate.
1241 */
1242
1243void smtc_flush_tlb_asid(unsigned long asid)
1244{
1245 int entry;
1246 unsigned long ehi;
1247
1248 entry = read_c0_wired();
1249
1250 /* Traverse all non-wired entries */
1251 while (entry < current_cpu_data.tlbsize) {
1252 write_c0_index(entry);
1253 ehb();
1254 tlb_read();
1255 ehb();
1256 ehi = read_c0_entryhi();
1257 if((ehi & ASID_MASK) == asid) {
1258 /*
1259 * Invalidate only entries with specified ASID,
1260 * makiing sure all entries differ.
1261 */
1262 write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1)));
1263 write_c0_entrylo0(0);
1264 write_c0_entrylo1(0);
1265 mtc0_tlbw_hazard();
1266 tlb_write_indexed();
1267 }
1268 entry++;
1269 }
1270 write_c0_index(PARKED_INDEX);
1271 tlbw_use_hazard();
1272}
1273
1274/*
1275 * Support for single-threading cache flush operations.
1276 */
1277
1278int halt_state_save[NR_CPUS];
1279
1280/*
1281 * To really, really be sure that nothing is being done
1282 * by other TCs, halt them all. This code assumes that
1283 * a DVPE has already been done, so while their Halted
1284 * state is theoretically architecturally unstable, in
1285 * practice, it's not going to change while we're looking
1286 * at it.
1287 */
1288
1289void smtc_cflush_lockdown(void)
1290{
1291 int cpu;
1292
1293 for_each_online_cpu(cpu) {
1294 if (cpu != smp_processor_id()) {
1295 settc(cpu_data[cpu].tc_id);
1296 halt_state_save[cpu] = read_tc_c0_tchalt();
1297 write_tc_c0_tchalt(TCHALT_H);
1298 }
1299 }
1300 mips_ihb();
1301}
1302
1303/* It would be cheating to change the cpu_online states during a flush! */
1304
1305void smtc_cflush_release(void)
1306{
1307 int cpu;
1308
1309 /*
1310 * Start with a hazard barrier to ensure
1311 * that all CACHE ops have played through.
1312 */
1313 mips_ihb();
1314
1315 for_each_online_cpu(cpu) {
1316 if (cpu != smp_processor_id()) {
1317 settc(cpu_data[cpu].tc_id);
1318 write_tc_c0_tchalt(halt_state_save[cpu]);
1319 }
1320 }
1321 mips_ihb();
1322}
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-11 19:04:05 -0500