aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/x86/kernel/smpboot.c588
-rw-r--r--arch/x86/kernel/smpboot_32.c532
-rw-r--r--arch/x86/kernel/smpboot_64.c515
-rw-r--r--include/asm-x86/smp.h3
4 files changed, 594 insertions, 1044 deletions
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 5bff87e99898..69c17965f48d 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -4,14 +4,42 @@
4#include <linux/sched.h> 4#include <linux/sched.h>
5#include <linux/percpu.h> 5#include <linux/percpu.h>
6#include <linux/bootmem.h> 6#include <linux/bootmem.h>
7#include <linux/err.h>
8#include <linux/nmi.h>
7 9
10#include <asm/desc.h>
8#include <asm/nmi.h> 11#include <asm/nmi.h>
9#include <asm/irq.h> 12#include <asm/irq.h>
10#include <asm/smp.h> 13#include <asm/smp.h>
11#include <asm/cpu.h> 14#include <asm/cpu.h>
12#include <asm/numa.h> 15#include <asm/numa.h>
16#include <asm/pgtable.h>
17#include <asm/tlbflush.h>
18#include <asm/mtrr.h>
19#include <asm/nmi.h>
20#include <linux/mc146818rtc.h>
13 21
14#include <mach_apic.h> 22#include <mach_apic.h>
23#include <mach_wakecpu.h>
24#include <smpboot_hooks.h>
25
26/* Store all idle threads, this can be reused instead of creating
27* a new thread. Also avoids complicated thread destroy functionality
28* for idle threads.
29*/
30#ifdef CONFIG_HOTPLUG_CPU
31/*
32 * Needed only for CONFIG_HOTPLUG_CPU because __cpuinitdata is
33 * removed after init for !CONFIG_HOTPLUG_CPU.
34 */
35static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);
36#define get_idle_for_cpu(x) (per_cpu(idle_thread_array, x))
37#define set_idle_for_cpu(x, p) (per_cpu(idle_thread_array, x) = (p))
38#else
39struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
40#define get_idle_for_cpu(x) (idle_thread_array[(x)])
41#define set_idle_for_cpu(x, p) (idle_thread_array[(x)] = (p))
42#endif
15 43
16/* Number of siblings per CPU package */ 44/* Number of siblings per CPU package */
17int smp_num_siblings = 1; 45int smp_num_siblings = 1;
@@ -41,6 +69,8 @@ EXPORT_PER_CPU_SYMBOL(cpu_core_map);
41DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info); 69DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
42EXPORT_PER_CPU_SYMBOL(cpu_info); 70EXPORT_PER_CPU_SYMBOL(cpu_info);
43 71
72static atomic_t init_deasserted;
73
44/* ready for x86_64, no harm for x86, since it will overwrite after alloc */ 74/* ready for x86_64, no harm for x86, since it will overwrite after alloc */
45unsigned char *trampoline_base = __va(SMP_TRAMPOLINE_BASE); 75unsigned char *trampoline_base = __va(SMP_TRAMPOLINE_BASE);
46 76
@@ -110,6 +140,96 @@ void unmap_cpu_to_logical_apicid(int cpu)
110#define map_cpu_to_logical_apicid() do {} while (0) 140#define map_cpu_to_logical_apicid() do {} while (0)
111#endif 141#endif
112 142
143/*
144 * Report back to the Boot Processor.
145 * Running on AP.
146 */
147void __cpuinit smp_callin(void)
148{
149 int cpuid, phys_id;
150 unsigned long timeout;
151
152 /*
153 * If waken up by an INIT in an 82489DX configuration
154 * we may get here before an INIT-deassert IPI reaches
155 * our local APIC. We have to wait for the IPI or we'll
156 * lock up on an APIC access.
157 */
158 wait_for_init_deassert(&init_deasserted);
159
160 /*
161 * (This works even if the APIC is not enabled.)
162 */
163 phys_id = GET_APIC_ID(apic_read(APIC_ID));
164 cpuid = smp_processor_id();
165 if (cpu_isset(cpuid, cpu_callin_map)) {
166 panic("%s: phys CPU#%d, CPU#%d already present??\n", __func__,
167 phys_id, cpuid);
168 }
169 Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
170
171 /*
172 * STARTUP IPIs are fragile beasts as they might sometimes
173 * trigger some glue motherboard logic. Complete APIC bus
174 * silence for 1 second, this overestimates the time the
175 * boot CPU is spending to send the up to 2 STARTUP IPIs
176 * by a factor of two. This should be enough.
177 */
178
179 /*
180 * Waiting 2s total for startup (udelay is not yet working)
181 */
182 timeout = jiffies + 2*HZ;
183 while (time_before(jiffies, timeout)) {
184 /*
185 * Has the boot CPU finished it's STARTUP sequence?
186 */
187 if (cpu_isset(cpuid, cpu_callout_map))
188 break;
189 cpu_relax();
190 }
191
192 if (!time_before(jiffies, timeout)) {
193 panic("%s: CPU%d started up but did not get a callout!\n",
194 __func__, cpuid);
195 }
196
197 /*
198 * the boot CPU has finished the init stage and is spinning
199 * on callin_map until we finish. We are free to set up this
200 * CPU, first the APIC. (this is probably redundant on most
201 * boards)
202 */
203
204 Dprintk("CALLIN, before setup_local_APIC().\n");
205 smp_callin_clear_local_apic();
206 setup_local_APIC();
207 end_local_APIC_setup();
208 map_cpu_to_logical_apicid();
209
210 /*
211 * Get our bogomips.
212 *
213 * Need to enable IRQs because it can take longer and then
214 * the NMI watchdog might kill us.
215 */
216 local_irq_enable();
217 calibrate_delay();
218 local_irq_disable();
219 Dprintk("Stack at about %p\n", &cpuid);
220
221 /*
222 * Save our processor parameters
223 */
224 smp_store_cpu_info(cpuid);
225
226 /*
227 * Allow the master to continue.
228 */
229 cpu_set(cpuid, cpu_callin_map);
230}
231
232
113static void __cpuinit smp_apply_quirks(struct cpuinfo_x86 *c) 233static void __cpuinit smp_apply_quirks(struct cpuinfo_x86 *c)
114{ 234{
115#ifdef CONFIG_X86_32 235#ifdef CONFIG_X86_32
@@ -327,6 +447,474 @@ void impress_friends(void)
327 Dprintk("Before bogocount - setting activated=1.\n"); 447 Dprintk("Before bogocount - setting activated=1.\n");
328} 448}
329 449
450static inline void __inquire_remote_apic(int apicid)
451{
452 unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
453 char *names[] = { "ID", "VERSION", "SPIV" };
454 int timeout;
455 u32 status;
456
457 printk(KERN_INFO "Inquiring remote APIC #%d...\n", apicid);
458
459 for (i = 0; i < ARRAY_SIZE(regs); i++) {
460 printk(KERN_INFO "... APIC #%d %s: ", apicid, names[i]);
461
462 /*
463 * Wait for idle.
464 */
465 status = safe_apic_wait_icr_idle();
466 if (status)
467 printk(KERN_CONT
468 "a previous APIC delivery may have failed\n");
469
470 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(apicid));
471 apic_write_around(APIC_ICR, APIC_DM_REMRD | regs[i]);
472
473 timeout = 0;
474 do {
475 udelay(100);
476 status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
477 } while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);
478
479 switch (status) {
480 case APIC_ICR_RR_VALID:
481 status = apic_read(APIC_RRR);
482 printk(KERN_CONT "%08x\n", status);
483 break;
484 default:
485 printk(KERN_CONT "failed\n");
486 }
487 }
488}
489
490#ifdef WAKE_SECONDARY_VIA_NMI
491/*
492 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
493 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
494 * won't ... remember to clear down the APIC, etc later.
495 */
496static int __devinit
497wakeup_secondary_cpu(int logical_apicid, unsigned long start_eip)
498{
499 unsigned long send_status, accept_status = 0;
500 int maxlvt;
501
502 /* Target chip */
503 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(logical_apicid));
504
505 /* Boot on the stack */
506 /* Kick the second */
507 apic_write_around(APIC_ICR, APIC_DM_NMI | APIC_DEST_LOGICAL);
508
509 Dprintk("Waiting for send to finish...\n");
510 send_status = safe_apic_wait_icr_idle();
511
512 /*
513 * Give the other CPU some time to accept the IPI.
514 */
515 udelay(200);
516 /*
517 * Due to the Pentium erratum 3AP.
518 */
519 maxlvt = lapic_get_maxlvt();
520 if (maxlvt > 3) {
521 apic_read_around(APIC_SPIV);
522 apic_write(APIC_ESR, 0);
523 }
524 accept_status = (apic_read(APIC_ESR) & 0xEF);
525 Dprintk("NMI sent.\n");
526
527 if (send_status)
528 printk(KERN_ERR "APIC never delivered???\n");
529 if (accept_status)
530 printk(KERN_ERR "APIC delivery error (%lx).\n", accept_status);
531
532 return (send_status | accept_status);
533}
534#endif /* WAKE_SECONDARY_VIA_NMI */
535
536extern void start_secondary(void *unused);
537#ifdef WAKE_SECONDARY_VIA_INIT
538static int __devinit
539wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
540{
541 unsigned long send_status, accept_status = 0;
542 int maxlvt, num_starts, j;
543
544 /*
545 * Be paranoid about clearing APIC errors.
546 */
547 if (APIC_INTEGRATED(apic_version[phys_apicid])) {
548 apic_read_around(APIC_SPIV);
549 apic_write(APIC_ESR, 0);
550 apic_read(APIC_ESR);
551 }
552
553 Dprintk("Asserting INIT.\n");
554
555 /*
556 * Turn INIT on target chip
557 */
558 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
559
560 /*
561 * Send IPI
562 */
563 apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_INT_ASSERT
564 | APIC_DM_INIT);
565
566 Dprintk("Waiting for send to finish...\n");
567 send_status = safe_apic_wait_icr_idle();
568
569 mdelay(10);
570
571 Dprintk("Deasserting INIT.\n");
572
573 /* Target chip */
574 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
575
576 /* Send IPI */
577 apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_DM_INIT);
578
579 Dprintk("Waiting for send to finish...\n");
580 send_status = safe_apic_wait_icr_idle();
581
582 mb();
583 atomic_set(&init_deasserted, 1);
584
585 /*
586 * Should we send STARTUP IPIs ?
587 *
588 * Determine this based on the APIC version.
589 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
590 */
591 if (APIC_INTEGRATED(apic_version[phys_apicid]))
592 num_starts = 2;
593 else
594 num_starts = 0;
595
596 /*
597 * Paravirt / VMI wants a startup IPI hook here to set up the
598 * target processor state.
599 */
600 startup_ipi_hook(phys_apicid, (unsigned long) start_secondary,
601#ifdef CONFIG_X86_64
602 (unsigned long)init_rsp);
603#else
604 (unsigned long)stack_start.sp);
605#endif
606
607 /*
608 * Run STARTUP IPI loop.
609 */
610 Dprintk("#startup loops: %d.\n", num_starts);
611
612 maxlvt = lapic_get_maxlvt();
613
614 for (j = 1; j <= num_starts; j++) {
615 Dprintk("Sending STARTUP #%d.\n", j);
616 apic_read_around(APIC_SPIV);
617 apic_write(APIC_ESR, 0);
618 apic_read(APIC_ESR);
619 Dprintk("After apic_write.\n");
620
621 /*
622 * STARTUP IPI
623 */
624
625 /* Target chip */
626 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
627
628 /* Boot on the stack */
629 /* Kick the second */
630 apic_write_around(APIC_ICR, APIC_DM_STARTUP
631 | (start_eip >> 12));
632
633 /*
634 * Give the other CPU some time to accept the IPI.
635 */
636 udelay(300);
637
638 Dprintk("Startup point 1.\n");
639
640 Dprintk("Waiting for send to finish...\n");
641 send_status = safe_apic_wait_icr_idle();
642
643 /*
644 * Give the other CPU some time to accept the IPI.
645 */
646 udelay(200);
647 /*
648 * Due to the Pentium erratum 3AP.
649 */
650 if (maxlvt > 3) {
651 apic_read_around(APIC_SPIV);
652 apic_write(APIC_ESR, 0);
653 }
654 accept_status = (apic_read(APIC_ESR) & 0xEF);
655 if (send_status || accept_status)
656 break;
657 }
658 Dprintk("After Startup.\n");
659
660 if (send_status)
661 printk(KERN_ERR "APIC never delivered???\n");
662 if (accept_status)
663 printk(KERN_ERR "APIC delivery error (%lx).\n", accept_status);
664
665 return (send_status | accept_status);
666}
667#endif /* WAKE_SECONDARY_VIA_INIT */
668
669struct create_idle {
670 struct work_struct work;
671 struct task_struct *idle;
672 struct completion done;
673 int cpu;
674};
675
676static void __cpuinit do_fork_idle(struct work_struct *work)
677{
678 struct create_idle *c_idle =
679 container_of(work, struct create_idle, work);
680
681 c_idle->idle = fork_idle(c_idle->cpu);
682 complete(&c_idle->done);
683}
684
685static int __cpuinit do_boot_cpu(int apicid, int cpu)
686/*
687 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
688 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
689 * Returns zero if CPU booted OK, else error code from wakeup_secondary_cpu.
690 */
691{
692 unsigned long boot_error = 0;
693 int timeout;
694 unsigned long start_ip;
695 unsigned short nmi_high = 0, nmi_low = 0;
696 struct create_idle c_idle = {
697 .cpu = cpu,
698 .done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done),
699 };
700 INIT_WORK(&c_idle.work, do_fork_idle);
701#ifdef CONFIG_X86_64
702 /* allocate memory for gdts of secondary cpus. Hotplug is considered */
703 if (!cpu_gdt_descr[cpu].address &&
704 !(cpu_gdt_descr[cpu].address = get_zeroed_page(GFP_KERNEL))) {
705 printk(KERN_ERR "Failed to allocate GDT for CPU %d\n", cpu);
706 return -1;
707 }
708
709 /* Allocate node local memory for AP pdas */
710 if (cpu_pda(cpu) == &boot_cpu_pda[cpu]) {
711 struct x8664_pda *newpda, *pda;
712 int node = cpu_to_node(cpu);
713 pda = cpu_pda(cpu);
714 newpda = kmalloc_node(sizeof(struct x8664_pda), GFP_ATOMIC,
715 node);
716 if (newpda) {
717 memcpy(newpda, pda, sizeof(struct x8664_pda));
718 cpu_pda(cpu) = newpda;
719 } else
720 printk(KERN_ERR
721 "Could not allocate node local PDA for CPU %d on node %d\n",
722 cpu, node);
723 }
724#endif
725
726 alternatives_smp_switch(1);
727
728 c_idle.idle = get_idle_for_cpu(cpu);
729
730 /*
731 * We can't use kernel_thread since we must avoid to
732 * reschedule the child.
733 */
734 if (c_idle.idle) {
735 c_idle.idle->thread.sp = (unsigned long) (((struct pt_regs *)
736 (THREAD_SIZE + task_stack_page(c_idle.idle))) - 1);
737 init_idle(c_idle.idle, cpu);
738 goto do_rest;
739 }
740
741 if (!keventd_up() || current_is_keventd())
742 c_idle.work.func(&c_idle.work);
743 else {
744 schedule_work(&c_idle.work);
745 wait_for_completion(&c_idle.done);
746 }
747
748 if (IS_ERR(c_idle.idle)) {
749 printk("failed fork for CPU %d\n", cpu);
750 return PTR_ERR(c_idle.idle);
751 }
752
753 set_idle_for_cpu(cpu, c_idle.idle);
754do_rest:
755#ifdef CONFIG_X86_32
756 per_cpu(current_task, cpu) = c_idle.idle;
757 init_gdt(cpu);
758 early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
759 c_idle.idle->thread.ip = (unsigned long) start_secondary;
760 /* Stack for startup_32 can be just as for start_secondary onwards */
761 stack_start.sp = (void *) c_idle.idle->thread.sp;
762 irq_ctx_init(cpu);
763#else
764 cpu_pda(cpu)->pcurrent = c_idle.idle;
765 init_rsp = c_idle.idle->thread.sp;
766 load_sp0(&per_cpu(init_tss, cpu), &c_idle.idle->thread);
767 initial_code = (unsigned long)start_secondary;
768 clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
769#endif
770
771 /* start_ip had better be page-aligned! */
772 start_ip = setup_trampoline();
773
774 /* So we see what's up */
775 printk(KERN_INFO "Booting processor %d/%d ip %lx\n",
776 cpu, apicid, start_ip);
777
778 /*
779 * This grunge runs the startup process for
780 * the targeted processor.
781 */
782
783 atomic_set(&init_deasserted, 0);
784
785 Dprintk("Setting warm reset code and vector.\n");
786
787 store_NMI_vector(&nmi_high, &nmi_low);
788
789 smpboot_setup_warm_reset_vector(start_ip);
790 /*
791 * Be paranoid about clearing APIC errors.
792 */
793 apic_write(APIC_ESR, 0);
794 apic_read(APIC_ESR);
795
796
797 /*
798 * Starting actual IPI sequence...
799 */
800 boot_error = wakeup_secondary_cpu(apicid, start_ip);
801
802 if (!boot_error) {
803 /*
804 * allow APs to start initializing.
805 */
806 Dprintk("Before Callout %d.\n", cpu);
807 cpu_set(cpu, cpu_callout_map);
808 Dprintk("After Callout %d.\n", cpu);
809
810 /*
811 * Wait 5s total for a response
812 */
813 for (timeout = 0; timeout < 50000; timeout++) {
814 if (cpu_isset(cpu, cpu_callin_map))
815 break; /* It has booted */
816 udelay(100);
817 }
818
819 if (cpu_isset(cpu, cpu_callin_map)) {
820 /* number CPUs logically, starting from 1 (BSP is 0) */
821 Dprintk("OK.\n");
822 printk(KERN_INFO "CPU%d: ", cpu);
823 print_cpu_info(&cpu_data(cpu));
824 Dprintk("CPU has booted.\n");
825 } else {
826 boot_error = 1;
827 if (*((volatile unsigned char *)trampoline_base)
828 == 0xA5)
829 /* trampoline started but...? */
830 printk(KERN_ERR "Stuck ??\n");
831 else
832 /* trampoline code not run */
833 printk(KERN_ERR "Not responding.\n");
834 inquire_remote_apic(apicid);
835 }
836 }
837
838 if (boot_error) {
839 /* Try to put things back the way they were before ... */
840 unmap_cpu_to_logical_apicid(cpu);
841#ifdef CONFIG_X86_64
842 clear_node_cpumask(cpu); /* was set by numa_add_cpu */
843#endif
844 cpu_clear(cpu, cpu_callout_map); /* was set by do_boot_cpu() */
845 cpu_clear(cpu, cpu_initialized); /* was set by cpu_init() */
846 cpu_clear(cpu, cpu_possible_map);
847 cpu_clear(cpu, cpu_present_map);
848 per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID;
849 }
850
851 /* mark "stuck" area as not stuck */
852 *((volatile unsigned long *)trampoline_base) = 0;
853
854 return boot_error;
855}
856
857int __cpuinit native_cpu_up(unsigned int cpu)
858{
859 int apicid = cpu_present_to_apicid(cpu);
860 unsigned long flags;
861 int err;
862
863 WARN_ON(irqs_disabled());
864
865 Dprintk("++++++++++++++++++++=_---CPU UP %u\n", cpu);
866
867 if (apicid == BAD_APICID || apicid == boot_cpu_physical_apicid ||
868 !physid_isset(apicid, phys_cpu_present_map)) {
869 printk(KERN_ERR "%s: bad cpu %d\n", __func__, cpu);
870 return -EINVAL;
871 }
872
873 /*
874 * Already booted CPU?
875 */
876 if (cpu_isset(cpu, cpu_callin_map)) {
877 Dprintk("do_boot_cpu %d Already started\n", cpu);
878 return -ENOSYS;
879 }
880
881 /*
882 * Save current MTRR state in case it was changed since early boot
883 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
884 */
885 mtrr_save_state();
886
887 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
888
889#ifdef CONFIG_X86_32
890 /* init low mem mapping */
891 clone_pgd_range(swapper_pg_dir, swapper_pg_dir + USER_PGD_PTRS,
892 min_t(unsigned long, KERNEL_PGD_PTRS, USER_PGD_PTRS));
893 flush_tlb_all();
894#endif
895
896 err = do_boot_cpu(apicid, cpu);
897 if (err < 0) {
898 Dprintk("do_boot_cpu failed %d\n", err);
899 return err;
900 }
901
902 /*
903 * Check TSC synchronization with the AP (keep irqs disabled
904 * while doing so):
905 */
906 local_irq_save(flags);
907 check_tsc_sync_source(cpu);
908 local_irq_restore(flags);
909
910 while (!cpu_isset(cpu, cpu_online_map)) {
911 cpu_relax();
912 touch_nmi_watchdog();
913 }
914
915 return 0;
916}
917
330#ifdef CONFIG_HOTPLUG_CPU 918#ifdef CONFIG_HOTPLUG_CPU
331void remove_siblinginfo(int cpu) 919void remove_siblinginfo(int cpu)
332{ 920{
diff --git a/arch/x86/kernel/smpboot_32.c b/arch/x86/kernel/smpboot_32.c
index ae25927f08c1..e82eeb2fdfef 100644
--- a/arch/x86/kernel/smpboot_32.c
+++ b/arch/x86/kernel/smpboot_32.c
@@ -80,114 +80,12 @@ extern void unmap_cpu_to_logical_apicid(int cpu);
80/* State of each CPU. */ 80/* State of each CPU. */
81DEFINE_PER_CPU(int, cpu_state) = { 0 }; 81DEFINE_PER_CPU(int, cpu_state) = { 0 };
82 82
83/* Store all idle threads, this can be reused instead of creating 83extern void smp_callin(void);
84* a new thread. Also avoids complicated thread destroy functionality
85* for idle threads.
86*/
87#ifdef CONFIG_HOTPLUG_CPU
88/*
89 * Needed only for CONFIG_HOTPLUG_CPU because __cpuinitdata is
90 * removed after init for !CONFIG_HOTPLUG_CPU.
91 */
92static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);
93#define get_idle_for_cpu(x) (per_cpu(idle_thread_array, x))
94#define set_idle_for_cpu(x, p) (per_cpu(idle_thread_array, x) = (p))
95#else
96struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
97#define get_idle_for_cpu(x) (idle_thread_array[(x)])
98#define set_idle_for_cpu(x, p) (idle_thread_array[(x)] = (p))
99#endif
100
101static atomic_t init_deasserted;
102
103static void __cpuinit smp_callin(void)
104{
105 int cpuid, phys_id;
106 unsigned long timeout;
107
108 /*
109 * If waken up by an INIT in an 82489DX configuration
110 * we may get here before an INIT-deassert IPI reaches
111 * our local APIC. We have to wait for the IPI or we'll
112 * lock up on an APIC access.
113 */
114 wait_for_init_deassert(&init_deasserted);
115
116 /*
117 * (This works even if the APIC is not enabled.)
118 */
119 phys_id = GET_APIC_ID(apic_read(APIC_ID));
120 cpuid = smp_processor_id();
121 if (cpu_isset(cpuid, cpu_callin_map)) {
122 printk("huh, phys CPU#%d, CPU#%d already present??\n",
123 phys_id, cpuid);
124 BUG();
125 }
126 Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
127
128 /*
129 * STARTUP IPIs are fragile beasts as they might sometimes
130 * trigger some glue motherboard logic. Complete APIC bus
131 * silence for 1 second, this overestimates the time the
132 * boot CPU is spending to send the up to 2 STARTUP IPIs
133 * by a factor of two. This should be enough.
134 */
135
136 /*
137 * Waiting 2s total for startup (udelay is not yet working)
138 */
139 timeout = jiffies + 2*HZ;
140 while (time_before(jiffies, timeout)) {
141 /*
142 * Has the boot CPU finished it's STARTUP sequence?
143 */
144 if (cpu_isset(cpuid, cpu_callout_map))
145 break;
146 cpu_relax();
147 }
148
149 if (!time_before(jiffies, timeout)) {
150 printk("BUG: CPU%d started up but did not get a callout!\n",
151 cpuid);
152 BUG();
153 }
154
155 /*
156 * the boot CPU has finished the init stage and is spinning
157 * on callin_map until we finish. We are free to set up this
158 * CPU, first the APIC. (this is probably redundant on most
159 * boards)
160 */
161
162 Dprintk("CALLIN, before setup_local_APIC().\n");
163 smp_callin_clear_local_apic();
164 setup_local_APIC();
165 end_local_APIC_setup();
166 map_cpu_to_logical_apicid();
167
168 /*
169 * Get our bogomips.
170 */
171 local_irq_enable();
172 calibrate_delay();
173 local_irq_disable();
174 Dprintk("Stack at about %p\n",&cpuid);
175
176 /*
177 * Save our processor parameters
178 */
179 smp_store_cpu_info(cpuid);
180
181 /*
182 * Allow the master to continue.
183 */
184 cpu_set(cpuid, cpu_callin_map);
185}
186 84
187/* 85/*
188 * Activate a secondary processor. 86 * Activate a secondary processor.
189 */ 87 */
190static void __cpuinit start_secondary(void *unused) 88void __cpuinit start_secondary(void *unused)
191{ 89{
192 /* 90 /*
193 * Don't put *anything* before cpu_init(), SMP booting is too 91 * Don't put *anything* before cpu_init(), SMP booting is too
@@ -257,373 +155,6 @@ void __devinit initialize_secondary(void)
257 :"m" (current->thread.sp),"m" (current->thread.ip)); 155 :"m" (current->thread.sp),"m" (current->thread.ip));
258} 156}
259 157
260static inline void __inquire_remote_apic(int apicid)
261{
262 unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
263 char *names[] = { "ID", "VERSION", "SPIV" };
264 int timeout;
265 u32 status;
266
267 printk(KERN_INFO "Inquiring remote APIC #%d...\n", apicid);
268
269 for (i = 0; i < ARRAY_SIZE(regs); i++) {
270 printk(KERN_INFO "... APIC #%d %s: ", apicid, names[i]);
271
272 /*
273 * Wait for idle.
274 */
275 status = safe_apic_wait_icr_idle();
276 if (status)
277 printk(KERN_CONT
278 "a previous APIC delivery may have failed\n");
279
280 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(apicid));
281 apic_write_around(APIC_ICR, APIC_DM_REMRD | regs[i]);
282
283 timeout = 0;
284 do {
285 udelay(100);
286 status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
287 } while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);
288
289 switch (status) {
290 case APIC_ICR_RR_VALID:
291 status = apic_read(APIC_RRR);
292 printk(KERN_CONT "%08x\n", status);
293 break;
294 default:
295 printk(KERN_CONT "failed\n");
296 }
297 }
298}
299
300#ifdef WAKE_SECONDARY_VIA_NMI
301/*
302 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
303 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
304 * won't ... remember to clear down the APIC, etc later.
305 */
306static int __devinit
307wakeup_secondary_cpu(int logical_apicid, unsigned long start_eip)
308{
309 unsigned long send_status, accept_status = 0;
310 int maxlvt;
311
312 /* Target chip */
313 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(logical_apicid));
314
315 /* Boot on the stack */
316 /* Kick the second */
317 apic_write_around(APIC_ICR, APIC_DM_NMI | APIC_DEST_LOGICAL);
318
319 Dprintk("Waiting for send to finish...\n");
320 send_status = safe_apic_wait_icr_idle();
321
322 /*
323 * Give the other CPU some time to accept the IPI.
324 */
325 udelay(200);
326 /*
327 * Due to the Pentium erratum 3AP.
328 */
329 maxlvt = lapic_get_maxlvt();
330 if (maxlvt > 3) {
331 apic_read_around(APIC_SPIV);
332 apic_write(APIC_ESR, 0);
333 }
334 accept_status = (apic_read(APIC_ESR) & 0xEF);
335 Dprintk("NMI sent.\n");
336
337 if (send_status)
338 printk("APIC never delivered???\n");
339 if (accept_status)
340 printk("APIC delivery error (%lx).\n", accept_status);
341
342 return (send_status | accept_status);
343}
344#endif /* WAKE_SECONDARY_VIA_NMI */
345
346#ifdef WAKE_SECONDARY_VIA_INIT
347static int __devinit
348wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
349{
350 unsigned long send_status, accept_status = 0;
351 int maxlvt, num_starts, j;
352
353 /*
354 * Be paranoid about clearing APIC errors.
355 */
356 if (APIC_INTEGRATED(apic_version[phys_apicid])) {
357 apic_read_around(APIC_SPIV);
358 apic_write(APIC_ESR, 0);
359 apic_read(APIC_ESR);
360 }
361
362 Dprintk("Asserting INIT.\n");
363
364 /*
365 * Turn INIT on target chip
366 */
367 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
368
369 /*
370 * Send IPI
371 */
372 apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_INT_ASSERT
373 | APIC_DM_INIT);
374
375 Dprintk("Waiting for send to finish...\n");
376 send_status = safe_apic_wait_icr_idle();
377
378 mdelay(10);
379
380 Dprintk("Deasserting INIT.\n");
381
382 /* Target chip */
383 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
384
385 /* Send IPI */
386 apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_DM_INIT);
387
388 Dprintk("Waiting for send to finish...\n");
389 send_status = safe_apic_wait_icr_idle();
390
391 mb();
392 atomic_set(&init_deasserted, 1);
393
394 /*
395 * Should we send STARTUP IPIs ?
396 *
397 * Determine this based on the APIC version.
398 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
399 */
400 if (APIC_INTEGRATED(apic_version[phys_apicid]))
401 num_starts = 2;
402 else
403 num_starts = 0;
404
405 /*
406 * Paravirt / VMI wants a startup IPI hook here to set up the
407 * target processor state.
408 */
409 startup_ipi_hook(phys_apicid, (unsigned long) start_secondary,
410 (unsigned long) stack_start.sp);
411
412 /*
413 * Run STARTUP IPI loop.
414 */
415 Dprintk("#startup loops: %d.\n", num_starts);
416
417 maxlvt = lapic_get_maxlvt();
418
419 for (j = 1; j <= num_starts; j++) {
420 Dprintk("Sending STARTUP #%d.\n",j);
421 apic_read_around(APIC_SPIV);
422 apic_write(APIC_ESR, 0);
423 apic_read(APIC_ESR);
424 Dprintk("After apic_write.\n");
425
426 /*
427 * STARTUP IPI
428 */
429
430 /* Target chip */
431 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
432
433 /* Boot on the stack */
434 /* Kick the second */
435 apic_write_around(APIC_ICR, APIC_DM_STARTUP
436 | (start_eip >> 12));
437
438 /*
439 * Give the other CPU some time to accept the IPI.
440 */
441 udelay(300);
442
443 Dprintk("Startup point 1.\n");
444
445 Dprintk("Waiting for send to finish...\n");
446 send_status = safe_apic_wait_icr_idle();
447
448 /*
449 * Give the other CPU some time to accept the IPI.
450 */
451 udelay(200);
452 /*
453 * Due to the Pentium erratum 3AP.
454 */
455 if (maxlvt > 3) {
456 apic_read_around(APIC_SPIV);
457 apic_write(APIC_ESR, 0);
458 }
459 accept_status = (apic_read(APIC_ESR) & 0xEF);
460 if (send_status || accept_status)
461 break;
462 }
463 Dprintk("After Startup.\n");
464
465 if (send_status)
466 printk("APIC never delivered???\n");
467 if (accept_status)
468 printk("APIC delivery error (%lx).\n", accept_status);
469
470 return (send_status | accept_status);
471}
472#endif /* WAKE_SECONDARY_VIA_INIT */
473
474extern cpumask_t cpu_initialized;
475
476struct create_idle {
477 struct work_struct work;
478 struct task_struct *idle;
479 struct completion done;
480 int cpu;
481};
482
483static void __cpuinit do_fork_idle(struct work_struct *work)
484{
485 struct create_idle *c_idle =
486 container_of(work, struct create_idle, work);
487
488 c_idle->idle = fork_idle(c_idle->cpu);
489 complete(&c_idle->done);
490}
491static int __cpuinit do_boot_cpu(int apicid, int cpu)
492/*
493 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
494 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
495 * Returns zero if CPU booted OK, else error code from wakeup_secondary_cpu.
496 */
497{
498 unsigned long boot_error = 0;
499 int timeout;
500 unsigned long start_eip;
501 unsigned short nmi_high = 0, nmi_low = 0;
502 struct create_idle c_idle = {
503 .cpu = cpu,
504 .done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done),
505 };
506 INIT_WORK(&c_idle.work, do_fork_idle);
507
508 alternatives_smp_switch(1);
509
510 c_idle.idle = get_idle_for_cpu(cpu);
511
512 /*
513 * We can't use kernel_thread since we must avoid to
514 * reschedule the child.
515 */
516 if (c_idle.idle) {
517 c_idle.idle->thread.sp = (unsigned long) (((struct pt_regs *)
518 (THREAD_SIZE + task_stack_page(c_idle.idle))) - 1);
519 init_idle(c_idle.idle, cpu);
520 goto do_rest;
521 }
522
523 if (!keventd_up() || current_is_keventd())
524 c_idle.work.func(&c_idle.work);
525 else {
526 schedule_work(&c_idle.work);
527 wait_for_completion(&c_idle.done);
528 }
529
530 if (IS_ERR(c_idle.idle)) {
531 printk(KERN_ERR "failed fork for CPU %d\n", cpu);
532 return PTR_ERR(c_idle.idle);
533 }
534
535 set_idle_for_cpu(cpu, c_idle.idle);
536do_rest:
537 per_cpu(current_task, cpu) = c_idle.idle;
538 init_gdt(cpu);
539 early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
540
541 c_idle.idle->thread.ip = (unsigned long) start_secondary;
542 /* start_eip had better be page-aligned! */
543 start_eip = setup_trampoline();
544
545 /* So we see what's up */
546 printk("Booting processor %d/%d ip %lx\n", cpu, apicid, start_eip);
547 /* Stack for startup_32 can be just as for start_secondary onwards */
548 stack_start.sp = (void *) c_idle.idle->thread.sp;
549
550 irq_ctx_init(cpu);
551
552 /*
553 * This grunge runs the startup process for
554 * the targeted processor.
555 */
556
557 atomic_set(&init_deasserted, 0);
558
559 Dprintk("Setting warm reset code and vector.\n");
560
561 store_NMI_vector(&nmi_high, &nmi_low);
562
563 smpboot_setup_warm_reset_vector(start_eip);
564 /*
565 * Be paranoid about clearing APIC errors.
566 */
567 apic_write(APIC_ESR, 0);
568 apic_read(APIC_ESR);
569
570
571 /*
572 * Starting actual IPI sequence...
573 */
574 boot_error = wakeup_secondary_cpu(apicid, start_eip);
575
576 if (!boot_error) {
577 /*
578 * allow APs to start initializing.
579 */
580 Dprintk("Before Callout %d.\n", cpu);
581 cpu_set(cpu, cpu_callout_map);
582 Dprintk("After Callout %d.\n", cpu);
583
584 /*
585 * Wait 5s total for a response
586 */
587 for (timeout = 0; timeout < 50000; timeout++) {
588 if (cpu_isset(cpu, cpu_callin_map))
589 break; /* It has booted */
590 udelay(100);
591 }
592
593 if (cpu_isset(cpu, cpu_callin_map)) {
594 /* number CPUs logically, starting from 1 (BSP is 0) */
595 Dprintk("OK.\n");
596 printk("CPU%d: ", cpu);
597 print_cpu_info(&cpu_data(cpu));
598 Dprintk("CPU has booted.\n");
599 } else {
600 boot_error= 1;
601 if (*((volatile unsigned char *)trampoline_base)
602 == 0xA5)
603 /* trampoline started but...? */
604 printk("Stuck ??\n");
605 else
606 /* trampoline code not run */
607 printk("Not responding.\n");
608 inquire_remote_apic(apicid);
609 }
610 }
611
612 if (boot_error) {
613 /* Try to put things back the way they were before ... */
614 unmap_cpu_to_logical_apicid(cpu);
615 cpu_clear(cpu, cpu_callout_map); /* was set here (do_boot_cpu()) */
616 cpu_clear(cpu, cpu_initialized); /* was set by cpu_init() */
617 cpu_clear(cpu, cpu_possible_map);
618 per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID;
619 }
620
621 /* mark "stuck" area as not stuck */
622 *((volatile unsigned long *)trampoline_base) = 0;
623
624 return boot_error;
625}
626
627#ifdef CONFIG_HOTPLUG_CPU 158#ifdef CONFIG_HOTPLUG_CPU
628void cpu_exit_clear(void) 159void cpu_exit_clear(void)
629{ 160{
@@ -774,65 +305,6 @@ void __init native_smp_prepare_boot_cpu(void)
774 __get_cpu_var(cpu_state) = CPU_ONLINE; 305 __get_cpu_var(cpu_state) = CPU_ONLINE;
775} 306}
776 307
777int __cpuinit native_cpu_up(unsigned int cpu)
778{
779 int apicid = cpu_present_to_apicid(cpu);
780 unsigned long flags;
781 int err;
782
783 WARN_ON(irqs_disabled());
784
785 Dprintk("++++++++++++++++++++=_---CPU UP %u\n", cpu);
786
787 if (apicid == BAD_APICID || apicid == boot_cpu_physical_apicid ||
788 !physid_isset(apicid, phys_cpu_present_map)) {
789 printk(KERN_ERR "%s: bad cpu %d\n", __func__, cpu);
790 return -EINVAL;
791 }
792
793 /*
794 * Already booted CPU?
795 */
796 if (cpu_isset(cpu, cpu_callin_map)) {
797 Dprintk("do_boot_cpu %d Already started\n", cpu);
798 return -ENOSYS;
799 }
800
801 /*
802 * Save current MTRR state in case it was changed since early boot
803 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
804 */
805 mtrr_save_state();
806
807 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
808
809 /* init low mem mapping */
810 clone_pgd_range(swapper_pg_dir, swapper_pg_dir + USER_PGD_PTRS,
811 min_t(unsigned long, KERNEL_PGD_PTRS, USER_PGD_PTRS));
812 flush_tlb_all();
813
814 err = do_boot_cpu(apicid, cpu);
815 if (err < 0) {
816 Dprintk("do_boot_cpu failed %d\n", err);
817 return err;
818 }
819
820 /*
821 * Check TSC synchronization with the AP (keep irqs disabled
822 * while doing so):
823 */
824 local_irq_save(flags);
825 check_tsc_sync_source(cpu);
826 local_irq_restore(flags);
827
828 while (!cpu_isset(cpu, cpu_online_map)) {
829 cpu_relax();
830 touch_nmi_watchdog();
831 }
832
833 return 0;
834}
835
836extern void impress_friends(void); 308extern void impress_friends(void);
837extern void smp_checks(void); 309extern void smp_checks(void);
838 310
diff --git a/arch/x86/kernel/smpboot_64.c b/arch/x86/kernel/smpboot_64.c
index 420ae4a33548..71f13b15bd89 100644
--- a/arch/x86/kernel/smpboot_64.c
+++ b/arch/x86/kernel/smpboot_64.c
@@ -71,119 +71,7 @@ int smp_threads_ready;
71/* State of each CPU */ 71/* State of each CPU */
72DEFINE_PER_CPU(int, cpu_state) = { 0 }; 72DEFINE_PER_CPU(int, cpu_state) = { 0 };
73 73
74/* 74extern void smp_callin(void);
75 * Store all idle threads, this can be reused instead of creating
76 * a new thread. Also avoids complicated thread destroy functionality
77 * for idle threads.
78 */
79#ifdef CONFIG_HOTPLUG_CPU
80/*
81 * Needed only for CONFIG_HOTPLUG_CPU because __cpuinitdata is
82 * removed after init for !CONFIG_HOTPLUG_CPU.
83 */
84static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);
85#define get_idle_for_cpu(x) (per_cpu(idle_thread_array, x))
86#define set_idle_for_cpu(x,p) (per_cpu(idle_thread_array, x) = (p))
87#else
88struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
89#define get_idle_for_cpu(x) (idle_thread_array[(x)])
90#define set_idle_for_cpu(x,p) (idle_thread_array[(x)] = (p))
91#endif
92
93static atomic_t init_deasserted __cpuinitdata;
94
95#define smp_callin_clear_local_apic() do {} while (0)
96#define map_cpu_to_logical_apicid() do {} while (0)
97
98/*
99 * Report back to the Boot Processor.
100 * Running on AP.
101 */
102void __cpuinit smp_callin(void)
103{
104 int cpuid, phys_id;
105 unsigned long timeout;
106
107 /*
108 * If waken up by an INIT in an 82489DX configuration
109 * we may get here before an INIT-deassert IPI reaches
110 * our local APIC. We have to wait for the IPI or we'll
111 * lock up on an APIC access.
112 */
113 wait_for_init_deassert(&init_deasserted);
114
115 /*
116 * (This works even if the APIC is not enabled.)
117 */
118 phys_id = GET_APIC_ID(apic_read(APIC_ID));
119 cpuid = smp_processor_id();
120 if (cpu_isset(cpuid, cpu_callin_map)) {
121 panic("smp_callin: phys CPU#%d, CPU#%d already present??\n",
122 phys_id, cpuid);
123 }
124 Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
125
126 /*
127 * STARTUP IPIs are fragile beasts as they might sometimes
128 * trigger some glue motherboard logic. Complete APIC bus
129 * silence for 1 second, this overestimates the time the
130 * boot CPU is spending to send the up to 2 STARTUP IPIs
131 * by a factor of two. This should be enough.
132 */
133
134 /*
135 * Waiting 2s total for startup (udelay is not yet working)
136 */
137 timeout = jiffies + 2*HZ;
138 while (time_before(jiffies, timeout)) {
139 /*
140 * Has the boot CPU finished it's STARTUP sequence?
141 */
142 if (cpu_isset(cpuid, cpu_callout_map))
143 break;
144 cpu_relax();
145 }
146
147 if (!time_before(jiffies, timeout)) {
148 panic("smp_callin: CPU%d started up but did not get a callout!\n",
149 cpuid);
150 }
151
152 /*
153 * the boot CPU has finished the init stage and is spinning
154 * on callin_map until we finish. We are free to set up this
155 * CPU, first the APIC. (this is probably redundant on most
156 * boards)
157 */
158
159 Dprintk("CALLIN, before setup_local_APIC().\n");
160 smp_callin_clear_local_apic();
161 setup_local_APIC();
162 end_local_APIC_setup();
163 map_cpu_to_logical_apicid();
164
165 /*
166 * Get our bogomips.
167 *
168 * Need to enable IRQs because it can take longer and then
169 * the NMI watchdog might kill us.
170 */
171 local_irq_enable();
172 calibrate_delay();
173 local_irq_disable();
174 Dprintk("Stack at about %p\n",&cpuid);
175
176 /*
177 * Save our processor parameters
178 */
179 smp_store_cpu_info(cpuid);
180
181 /*
182 * Allow the master to continue.
183 */
184 cpu_set(cpuid, cpu_callin_map);
185}
186
187/* 75/*
188 * Setup code on secondary processor (after comming out of the trampoline) 76 * Setup code on secondary processor (after comming out of the trampoline)
189 */ 77 */
@@ -246,349 +134,6 @@ void __cpuinit start_secondary(void)
246 cpu_idle(); 134 cpu_idle();
247} 135}
248 136
249extern volatile unsigned long init_rsp;
250extern void (*initial_code)(void);
251
252#ifdef APIC_DEBUG
253static void __inquire_remote_apic(int apicid)
254{
255 unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
256 char *names[] = { "ID", "VERSION", "SPIV" };
257 int timeout;
258 u32 status;
259
260 printk(KERN_INFO "Inquiring remote APIC #%d...\n", apicid);
261
262 for (i = 0; i < ARRAY_SIZE(regs); i++) {
263 printk(KERN_INFO "... APIC #%d %s: ", apicid, names[i]);
264
265 /*
266 * Wait for idle.
267 */
268 status = safe_apic_wait_icr_idle();
269 if (status)
270 printk(KERN_CONT
271 "a previous APIC delivery may have failed\n");
272
273 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(apicid));
274 apic_write_around(APIC_ICR, APIC_DM_REMRD | regs[i]);
275
276 timeout = 0;
277 do {
278 udelay(100);
279 status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
280 } while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);
281
282 switch (status) {
283 case APIC_ICR_RR_VALID:
284 status = apic_read(APIC_RRR);
285 printk(KERN_CONT "%08x\n", status);
286 break;
287 default:
288 printk(KERN_CONT "failed\n");
289 }
290 }
291}
292#endif
293
294/*
295 * Kick the secondary to wake up.
296 */
297static int __cpuinit wakeup_secondary_cpu(int phys_apicid,
298 unsigned int start_rip)
299{
300 unsigned long send_status, accept_status = 0;
301 int maxlvt, num_starts, j;
302
303 /*
304 * Be paranoid about clearing APIC errors.
305 */
306 if (APIC_INTEGRATED(apic_version[phys_apicid])) {
307 apic_read_around(APIC_SPIV);
308 apic_write(APIC_ESR, 0);
309 apic_read(APIC_ESR);
310 }
311
312 Dprintk("Asserting INIT.\n");
313
314 /*
315 * Turn INIT on target chip
316 */
317 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
318
319 /*
320 * Send IPI
321 */
322 apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_INT_ASSERT
323 | APIC_DM_INIT);
324
325 Dprintk("Waiting for send to finish...\n");
326 send_status = safe_apic_wait_icr_idle();
327
328 mdelay(10);
329
330 Dprintk("Deasserting INIT.\n");
331
332 /* Target chip */
333 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
334
335 /* Send IPI */
336 apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_DM_INIT);
337
338 Dprintk("Waiting for send to finish...\n");
339 send_status = safe_apic_wait_icr_idle();
340
341 mb();
342 atomic_set(&init_deasserted, 1);
343
344 if (APIC_INTEGRATED(apic_version[phys_apicid]))
345 num_starts = 2;
346 else
347 num_starts = 0;
348
349 /*
350 * Paravirt / VMI wants a startup IPI hook here to set up the
351 * target processor state.
352 */
353 startup_ipi_hook(phys_apicid, (unsigned long) start_secondary,
354 (unsigned long) init_rsp);
355
356
357 /*
358 * Run STARTUP IPI loop.
359 */
360 Dprintk("#startup loops: %d.\n", num_starts);
361
362 maxlvt = lapic_get_maxlvt();
363
364 for (j = 1; j <= num_starts; j++) {
365 Dprintk("Sending STARTUP #%d.\n",j);
366 apic_read_around(APIC_SPIV);
367 apic_write(APIC_ESR, 0);
368 apic_read(APIC_ESR);
369 Dprintk("After apic_write.\n");
370
371 /*
372 * STARTUP IPI
373 */
374
375 /* Target chip */
376 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
377
378 /* Boot on the stack */
379 /* Kick the second */
380 apic_write_around(APIC_ICR, APIC_DM_STARTUP | (start_rip>>12));
381
382 /*
383 * Give the other CPU some time to accept the IPI.
384 */
385 udelay(300);
386
387 Dprintk("Startup point 1.\n");
388
389 Dprintk("Waiting for send to finish...\n");
390 send_status = safe_apic_wait_icr_idle();
391
392 /*
393 * Give the other CPU some time to accept the IPI.
394 */
395 udelay(200);
396 /*
397 * Due to the Pentium erratum 3AP.
398 */
399 if (maxlvt > 3) {
400 apic_read_around(APIC_SPIV);
401 apic_write(APIC_ESR, 0);
402 }
403 accept_status = (apic_read(APIC_ESR) & 0xEF);
404 if (send_status || accept_status)
405 break;
406 }
407 Dprintk("After Startup.\n");
408
409 if (send_status)
410 printk(KERN_ERR "APIC never delivered???\n");
411 if (accept_status)
412 printk(KERN_ERR "APIC delivery error (%lx).\n", accept_status);
413
414 return (send_status | accept_status);
415}
416
417struct create_idle {
418 struct work_struct work;
419 struct task_struct *idle;
420 struct completion done;
421 int cpu;
422};
423
424static void __cpuinit do_fork_idle(struct work_struct *work)
425{
426 struct create_idle *c_idle =
427 container_of(work, struct create_idle, work);
428
429 c_idle->idle = fork_idle(c_idle->cpu);
430 complete(&c_idle->done);
431}
432
433/*
434 * Boot one CPU.
435 */
436static int __cpuinit do_boot_cpu(int cpu, int apicid)
437{
438 unsigned long boot_error = 0;
439 int timeout;
440 unsigned long start_rip;
441 struct create_idle c_idle = {
442 .cpu = cpu,
443 .done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done),
444 };
445 INIT_WORK(&c_idle.work, do_fork_idle);
446
447 /* allocate memory for gdts of secondary cpus. Hotplug is considered */
448 if (!cpu_gdt_descr[cpu].address &&
449 !(cpu_gdt_descr[cpu].address = get_zeroed_page(GFP_KERNEL))) {
450 printk(KERN_ERR "Failed to allocate GDT for CPU %d\n", cpu);
451 return -1;
452 }
453
454 /* Allocate node local memory for AP pdas */
455 if (cpu_pda(cpu) == &boot_cpu_pda[cpu]) {
456 struct x8664_pda *newpda, *pda;
457 int node = cpu_to_node(cpu);
458 pda = cpu_pda(cpu);
459 newpda = kmalloc_node(sizeof (struct x8664_pda), GFP_ATOMIC,
460 node);
461 if (newpda) {
462 memcpy(newpda, pda, sizeof (struct x8664_pda));
463 cpu_pda(cpu) = newpda;
464 } else
465 printk(KERN_ERR
466 "Could not allocate node local PDA for CPU %d on node %d\n",
467 cpu, node);
468 }
469
470 alternatives_smp_switch(1);
471
472 c_idle.idle = get_idle_for_cpu(cpu);
473
474 if (c_idle.idle) {
475 c_idle.idle->thread.sp = (unsigned long) (((struct pt_regs *)
476 (THREAD_SIZE + task_stack_page(c_idle.idle))) - 1);
477 init_idle(c_idle.idle, cpu);
478 goto do_rest;
479 }
480
481 /*
482 * During cold boot process, keventd thread is not spun up yet.
483 * When we do cpu hot-add, we create idle threads on the fly, we should
484 * not acquire any attributes from the calling context. Hence the clean
485 * way to create kernel_threads() is to do that from keventd().
486 * We do the current_is_keventd() due to the fact that ACPI notifier
487 * was also queuing to keventd() and when the caller is already running
488 * in context of keventd(), we would end up with locking up the keventd
489 * thread.
490 */
491 if (!keventd_up() || current_is_keventd())
492 c_idle.work.func(&c_idle.work);
493 else {
494 schedule_work(&c_idle.work);
495 wait_for_completion(&c_idle.done);
496 }
497
498 if (IS_ERR(c_idle.idle)) {
499 printk("failed fork for CPU %d\n", cpu);
500 return PTR_ERR(c_idle.idle);
501 }
502
503 set_idle_for_cpu(cpu, c_idle.idle);
504
505do_rest:
506
507 cpu_pda(cpu)->pcurrent = c_idle.idle;
508
509 start_rip = setup_trampoline();
510
511 init_rsp = c_idle.idle->thread.sp;
512 load_sp0(&per_cpu(init_tss, cpu), &c_idle.idle->thread);
513 initial_code = start_secondary;
514 clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
515
516 printk(KERN_INFO "Booting processor %d/%d APIC 0x%x\n", cpu,
517 cpus_weight(cpu_present_map),
518 apicid);
519
520 /*
521 * This grunge runs the startup process for
522 * the targeted processor.
523 */
524
525 atomic_set(&init_deasserted, 0);
526
527 Dprintk("Setting warm reset code and vector.\n");
528
529 smpboot_setup_warm_reset_vector(start_rip);
530 /*
531 * Be paranoid about clearing APIC errors.
532 */
533 apic_write(APIC_ESR, 0);
534 apic_read(APIC_ESR);
535
536 /*
537 * Starting actual IPI sequence...
538 */
539 boot_error = wakeup_secondary_cpu(apicid, start_rip);
540
541 if (!boot_error) {
542 /*
543 * allow APs to start initializing.
544 */
545 Dprintk("Before Callout %d.\n", cpu);
546 cpu_set(cpu, cpu_callout_map);
547 Dprintk("After Callout %d.\n", cpu);
548
549 /*
550 * Wait 5s total for a response
551 */
552 for (timeout = 0; timeout < 50000; timeout++) {
553 if (cpu_isset(cpu, cpu_callin_map))
554 break; /* It has booted */
555 udelay(100);
556 }
557
558 if (cpu_isset(cpu, cpu_callin_map)) {
559 /* number CPUs logically, starting from 1 (BSP is 0) */
560 Dprintk("CPU has booted.\n");
561 printk(KERN_INFO "CPU%d: ", cpu);
562 print_cpu_info(&cpu_data(cpu));
563 } else {
564 boot_error = 1;
565 if (*((volatile unsigned char *)trampoline_base)
566 == 0xA5)
567 /* trampoline started but...? */
568 printk("Stuck ??\n");
569 else
570 /* trampoline code not run */
571 printk("Not responding.\n");
572#ifdef APIC_DEBUG
573 inquire_remote_apic(apicid);
574#endif
575 }
576 }
577 if (boot_error) {
578 cpu_clear(cpu, cpu_callout_map); /* was set here (do_boot_cpu()) */
579 clear_bit(cpu, (unsigned long *)&cpu_initialized); /* was set by cpu_init() */
580 clear_node_cpumask(cpu); /* was set by numa_add_cpu */
581 cpu_clear(cpu, cpu_present_map);
582 cpu_clear(cpu, cpu_possible_map);
583 per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID;
584 }
585
586 /* mark "stuck" area as not stuck */
587 *((volatile unsigned long *)trampoline_base) = 0;
588
589 return boot_error;
590}
591
592cycles_t cacheflush_time; 137cycles_t cacheflush_time;
593unsigned long cache_decay_ticks; 138unsigned long cache_decay_ticks;
594 139
@@ -745,64 +290,6 @@ void __init native_smp_prepare_boot_cpu(void)
745 per_cpu(cpu_state, me) = CPU_ONLINE; 290 per_cpu(cpu_state, me) = CPU_ONLINE;
746} 291}
747 292
748/*
749 * Entry point to boot a CPU.
750 */
751int __cpuinit native_cpu_up(unsigned int cpu)
752{
753 int apicid = cpu_present_to_apicid(cpu);
754 unsigned long flags;
755 int err;
756
757 WARN_ON(irqs_disabled());
758
759 Dprintk("++++++++++++++++++++=_---CPU UP %u\n", cpu);
760
761 if (apicid == BAD_APICID || apicid == boot_cpu_physical_apicid ||
762 !physid_isset(apicid, phys_cpu_present_map)) {
763 printk("__cpu_up: bad cpu %d\n", cpu);
764 return -EINVAL;
765 }
766
767 /*
768 * Already booted CPU?
769 */
770 if (cpu_isset(cpu, cpu_callin_map)) {
771 Dprintk("do_boot_cpu %d Already started\n", cpu);
772 return -ENOSYS;
773 }
774
775 /*
776 * Save current MTRR state in case it was changed since early boot
777 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
778 */
779 mtrr_save_state();
780
781 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
782 /* Boot it! */
783 err = do_boot_cpu(cpu, apicid);
784 if (err < 0) {
785 Dprintk("do_boot_cpu failed %d\n", err);
786 return err;
787 }
788
789 /* Unleash the CPU! */
790 Dprintk("waiting for cpu %d\n", cpu);
791
792 /*
793 * Make sure and check TSC sync:
794 */
795 local_irq_save(flags);
796 check_tsc_sync_source(cpu);
797 local_irq_restore(flags);
798
799 while (!cpu_isset(cpu, cpu_online_map))
800 cpu_relax();
801 err = 0;
802
803 return err;
804}
805
806extern void impress_friends(void); 293extern void impress_friends(void);
807extern void smp_checks(void); 294extern void smp_checks(void);
808 295
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index 2ad2f4ffe498..ef26911dc22a 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -9,6 +9,7 @@ extern cpumask_t cpu_callout_map;
9 9
10extern int smp_num_siblings; 10extern int smp_num_siblings;
11extern unsigned int num_processors; 11extern unsigned int num_processors;
12extern cpumask_t cpu_initialized;
12 13
13extern u16 x86_cpu_to_apicid_init[]; 14extern u16 x86_cpu_to_apicid_init[];
14extern u16 x86_bios_cpu_apicid_init[]; 15extern u16 x86_bios_cpu_apicid_init[];
@@ -34,6 +35,8 @@ extern struct {
34 unsigned short ss; 35 unsigned short ss;
35} stack_start; 36} stack_start;
36 37
38extern unsigned long init_rsp;
39extern unsigned long initial_code;
37 40
38struct smp_ops { 41struct smp_ops {
39 void (*smp_prepare_boot_cpu)(void); 42 void (*smp_prepare_boot_cpu)(void);
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-27 00:31:12 -0400