aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/Kconfig3
-rw-r--r--arch/alpha/Kconfig1
-rw-r--r--arch/alpha/kernel/core_marvel.c6
-rw-r--r--arch/alpha/kernel/process.c2
-rw-r--r--arch/alpha/kernel/smp.c180
-rw-r--r--arch/alpha/oprofile/common.c6
-rw-r--r--arch/arm/Kconfig1
-rw-r--r--arch/arm/kernel/smp.c163
-rw-r--r--arch/arm/oprofile/op_model_mpcore.c2
-rw-r--r--arch/arm/vfp/vfpmodule.c2
-rw-r--r--arch/cris/arch-v32/kernel/smp.c5
-rw-r--r--arch/ia64/Kconfig1
-rw-r--r--arch/ia64/kernel/mca.c6
-rw-r--r--arch/ia64/kernel/palinfo.c2
-rw-r--r--arch/ia64/kernel/perfmon.c6
-rw-r--r--arch/ia64/kernel/process.c2
-rw-r--r--arch/ia64/kernel/smp.c254
-rw-r--r--arch/ia64/kernel/smpboot.c6
-rw-r--r--arch/ia64/kernel/uncached.c5
-rw-r--r--arch/ia64/sn/kernel/sn2/sn_hwperf.c2
-rw-r--r--arch/m32r/Kconfig1
-rw-r--r--arch/m32r/kernel/m32r_ksyms.c3
-rw-r--r--arch/m32r/kernel/smp.c132
-rw-r--r--arch/m32r/kernel/traps.c3
-rw-r--r--arch/mips/Kconfig1
-rw-r--r--arch/mips/kernel/irq-rm9000.c4
-rw-r--r--arch/mips/kernel/smp.c149
-rw-r--r--arch/mips/kernel/smtc.c1
-rw-r--r--arch/mips/mm/c-r4k.c18
-rw-r--r--arch/mips/oprofile/common.c6
-rw-r--r--arch/mips/pmc-sierra/yosemite/prom.c2
-rw-r--r--arch/mips/sibyte/cfe/setup.c2
-rw-r--r--arch/mips/sibyte/sb1250/prom.c2
-rw-r--r--arch/parisc/Kconfig1
-rw-r--r--arch/parisc/kernel/cache.c6
-rw-r--r--arch/parisc/kernel/smp.c136
-rw-r--r--arch/parisc/mm/init.c2
-rw-r--r--arch/powerpc/Kconfig1
-rw-r--r--arch/powerpc/kernel/machine_kexec_64.c2
-rw-r--r--arch/powerpc/kernel/rtas.c2
-rw-r--r--arch/powerpc/kernel/smp.c234
-rw-r--r--arch/powerpc/kernel/tau_6xx.c4
-rw-r--r--arch/powerpc/kernel/time.c2
-rw-r--r--arch/powerpc/mm/slice.c2
-rw-r--r--arch/powerpc/mm/tlb_64.c2
-rw-r--r--arch/powerpc/oprofile/common.c6
-rw-r--r--arch/powerpc/platforms/cell/interrupt.c1
-rw-r--r--arch/powerpc/platforms/ps3/smp.c7
-rw-r--r--arch/powerpc/platforms/pseries/xics.c6
-rw-r--r--arch/powerpc/sysdev/mpic.c2
-rw-r--r--arch/s390/appldata/appldata_base.c4
-rw-r--r--arch/s390/kernel/smp.c22
-rw-r--r--arch/s390/kernel/time.c6
-rw-r--r--arch/sh/Kconfig1
-rw-r--r--arch/sh/kernel/smp.c62
-rw-r--r--arch/sparc64/kernel/smp.c13
-rw-r--r--arch/sparc64/mm/hugetlbpage.c2
-rw-r--r--arch/um/kernel/smp.c3
-rw-r--r--arch/x86/Kconfig1
-rw-r--r--arch/x86/kernel/apic_32.c4
-rw-r--r--arch/x86/kernel/cpu/mcheck/mce_64.c6
-rw-r--r--arch/x86/kernel/cpu/mcheck/non-fatal.c2
-rw-r--r--arch/x86/kernel/cpu/mtrr/main.c4
-rw-r--r--arch/x86/kernel/cpu/perfctr-watchdog.c4
-rw-r--r--arch/x86/kernel/cpuid.c2
-rw-r--r--arch/x86/kernel/entry_64.S3
-rw-r--r--arch/x86/kernel/io_apic_32.c2
-rw-r--r--arch/x86/kernel/io_apic_64.c2
-rw-r--r--arch/x86/kernel/irqinit_64.c4
-rw-r--r--arch/x86/kernel/ldt.c2
-rw-r--r--arch/x86/kernel/nmi.c6
-rw-r--r--arch/x86/kernel/process.c2
-rw-r--r--arch/x86/kernel/smp.c158
-rw-r--r--arch/x86/kernel/smpboot.c4
-rw-r--r--arch/x86/kernel/smpcommon.c56
-rw-r--r--arch/x86/kernel/tlb_32.c2
-rw-r--r--arch/x86/kernel/tlb_64.c2
-rw-r--r--arch/x86/kernel/vsyscall_64.c4
-rw-r--r--arch/x86/kvm/vmx.c4
-rw-r--r--arch/x86/kvm/x86.c2
-rw-r--r--arch/x86/lib/msr-on-cpu.c8
-rw-r--r--arch/x86/mach-voyager/voyager_smp.c98
-rw-r--r--arch/x86/mm/pageattr.c4
-rw-r--r--arch/x86/oprofile/nmi_int.c10
-rw-r--r--arch/x86/xen/enlighten.c4
-rw-r--r--arch/x86/xen/mmu.c2
-rw-r--r--arch/x86/xen/smp.c135
-rw-r--r--arch/x86/xen/xen-ops.h9
88 files changed, 373 insertions, 1681 deletions
diff --git a/arch/Kconfig b/arch/Kconfig
index 3ea332b009e5..ad89a33d8c6e 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -39,3 +39,6 @@ config HAVE_KRETPROBES
39 39
40config HAVE_DMA_ATTRS 40config HAVE_DMA_ATTRS
41 def_bool n 41 def_bool n
42
43config USE_GENERIC_SMP_HELPERS
44 def_bool n
diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig
index 729cdbdf8036..dbe8c280fea9 100644
--- a/arch/alpha/Kconfig
+++ b/arch/alpha/Kconfig
@@ -528,6 +528,7 @@ config ARCH_MAY_HAVE_PC_FDC
528config SMP 528config SMP
529 bool "Symmetric multi-processing support" 529 bool "Symmetric multi-processing support"
530 depends on ALPHA_SABLE || ALPHA_LYNX || ALPHA_RAWHIDE || ALPHA_DP264 || ALPHA_WILDFIRE || ALPHA_TITAN || ALPHA_GENERIC || ALPHA_SHARK || ALPHA_MARVEL 530 depends on ALPHA_SABLE || ALPHA_LYNX || ALPHA_RAWHIDE || ALPHA_DP264 || ALPHA_WILDFIRE || ALPHA_TITAN || ALPHA_GENERIC || ALPHA_SHARK || ALPHA_MARVEL
531 select USE_GENERIC_SMP_HELPERS
531 ---help--- 532 ---help---
532 This enables support for systems with more than one CPU. If you have 533 This enables support for systems with more than one CPU. If you have
533 a system with only one CPU, like most personal computers, say N. If 534 a system with only one CPU, like most personal computers, say N. If
diff --git a/arch/alpha/kernel/core_marvel.c b/arch/alpha/kernel/core_marvel.c
index b04f1feb1dda..04dcc5e5d4c1 100644
--- a/arch/alpha/kernel/core_marvel.c
+++ b/arch/alpha/kernel/core_marvel.c
@@ -660,9 +660,9 @@ __marvel_rtc_io(u8 b, unsigned long addr, int write)
660 660
661#ifdef CONFIG_SMP 661#ifdef CONFIG_SMP
662 if (smp_processor_id() != boot_cpuid) 662 if (smp_processor_id() != boot_cpuid)
663 smp_call_function_on_cpu(__marvel_access_rtc, 663 smp_call_function_single(boot_cpuid,
664 &rtc_access, 1, 1, 664 __marvel_access_rtc,
665 cpumask_of_cpu(boot_cpuid)); 665 &rtc_access, 1);
666 else 666 else
667 __marvel_access_rtc(&rtc_access); 667 __marvel_access_rtc(&rtc_access);
668#else 668#else
diff --git a/arch/alpha/kernel/process.c b/arch/alpha/kernel/process.c
index 96ed82fd9eef..351407e07e71 100644
--- a/arch/alpha/kernel/process.c
+++ b/arch/alpha/kernel/process.c
@@ -160,7 +160,7 @@ common_shutdown(int mode, char *restart_cmd)
160 struct halt_info args; 160 struct halt_info args;
161 args.mode = mode; 161 args.mode = mode;
162 args.restart_cmd = restart_cmd; 162 args.restart_cmd = restart_cmd;
163 on_each_cpu(common_shutdown_1, &args, 1, 0); 163 on_each_cpu(common_shutdown_1, &args, 0);
164} 164}
165 165
166void 166void
diff --git a/arch/alpha/kernel/smp.c b/arch/alpha/kernel/smp.c
index 2525692db0ab..83df541650fc 100644
--- a/arch/alpha/kernel/smp.c
+++ b/arch/alpha/kernel/smp.c
@@ -62,6 +62,7 @@ static struct {
62enum ipi_message_type { 62enum ipi_message_type {
63 IPI_RESCHEDULE, 63 IPI_RESCHEDULE,
64 IPI_CALL_FUNC, 64 IPI_CALL_FUNC,
65 IPI_CALL_FUNC_SINGLE,
65 IPI_CPU_STOP, 66 IPI_CPU_STOP,
66}; 67};
67 68
@@ -558,51 +559,6 @@ send_ipi_message(cpumask_t to_whom, enum ipi_message_type operation)
558 wripir(i); 559 wripir(i);
559} 560}
560 561
561/* Structure and data for smp_call_function. This is designed to
562 minimize static memory requirements. Plus it looks cleaner. */
563
564struct smp_call_struct {
565 void (*func) (void *info);
566 void *info;
567 long wait;
568 atomic_t unstarted_count;
569 atomic_t unfinished_count;
570};
571
572static struct smp_call_struct *smp_call_function_data;
573
574/* Atomicly drop data into a shared pointer. The pointer is free if
575 it is initially locked. If retry, spin until free. */
576
577static int
578pointer_lock (void *lock, void *data, int retry)
579{
580 void *old, *tmp;
581
582 mb();
583 again:
584 /* Compare and swap with zero. */
585 asm volatile (
586 "1: ldq_l %0,%1\n"
587 " mov %3,%2\n"
588 " bne %0,2f\n"
589 " stq_c %2,%1\n"
590 " beq %2,1b\n"
591 "2:"
592 : "=&r"(old), "=m"(*(void **)lock), "=&r"(tmp)
593 : "r"(data)
594 : "memory");
595
596 if (old == 0)
597 return 0;
598 if (! retry)
599 return -EBUSY;
600
601 while (*(void **)lock)
602 barrier();
603 goto again;
604}
605
606void 562void
607handle_ipi(struct pt_regs *regs) 563handle_ipi(struct pt_regs *regs)
608{ 564{
@@ -632,31 +588,12 @@ handle_ipi(struct pt_regs *regs)
632 break; 588 break;
633 589
634 case IPI_CALL_FUNC: 590 case IPI_CALL_FUNC:
635 { 591 generic_smp_call_function_interrupt();
636 struct smp_call_struct *data; 592 break;
637 void (*func)(void *info); 593
638 void *info; 594 case IPI_CALL_FUNC_SINGLE:
639 int wait; 595 generic_smp_call_function_single_interrupt();
640
641 data = smp_call_function_data;
642 func = data->func;
643 info = data->info;
644 wait = data->wait;
645
646 /* Notify the sending CPU that the data has been
647 received, and execution is about to begin. */
648 mb();
649 atomic_dec (&data->unstarted_count);
650
651 /* At this point the structure may be gone unless
652 wait is true. */
653 (*func)(info);
654
655 /* Notify the sending CPU that the task is done. */
656 mb();
657 if (wait) atomic_dec (&data->unfinished_count);
658 break; 596 break;
659 }
660 597
661 case IPI_CPU_STOP: 598 case IPI_CPU_STOP:
662 halt(); 599 halt();
@@ -700,102 +637,15 @@ smp_send_stop(void)
700 send_ipi_message(to_whom, IPI_CPU_STOP); 637 send_ipi_message(to_whom, IPI_CPU_STOP);
701} 638}
702 639
703/* 640void arch_send_call_function_ipi(cpumask_t mask)
704 * Run a function on all other CPUs.
705 * <func> The function to run. This must be fast and non-blocking.
706 * <info> An arbitrary pointer to pass to the function.
707 * <retry> If true, keep retrying until ready.
708 * <wait> If true, wait until function has completed on other CPUs.
709 * [RETURNS] 0 on success, else a negative status code.
710 *
711 * Does not return until remote CPUs are nearly ready to execute <func>
712 * or are or have executed.
713 * You must not call this function with disabled interrupts or from a
714 * hardware interrupt handler or from a bottom half handler.
715 */
716
717int
718smp_call_function_on_cpu (void (*func) (void *info), void *info, int retry,
719 int wait, cpumask_t to_whom)
720{ 641{
721 struct smp_call_struct data; 642 send_ipi_message(mask, IPI_CALL_FUNC);
722 unsigned long timeout;
723 int num_cpus_to_call;
724
725 /* Can deadlock when called with interrupts disabled */
726 WARN_ON(irqs_disabled());
727
728 data.func = func;
729 data.info = info;
730 data.wait = wait;
731
732 cpu_clear(smp_processor_id(), to_whom);
733 num_cpus_to_call = cpus_weight(to_whom);
734
735 atomic_set(&data.unstarted_count, num_cpus_to_call);
736 atomic_set(&data.unfinished_count, num_cpus_to_call);
737
738 /* Acquire the smp_call_function_data mutex. */
739 if (pointer_lock(&smp_call_function_data, &data, retry))
740 return -EBUSY;
741
742 /* Send a message to the requested CPUs. */
743 send_ipi_message(to_whom, IPI_CALL_FUNC);
744
745 /* Wait for a minimal response. */
746 timeout = jiffies + HZ;
747 while (atomic_read (&data.unstarted_count) > 0
748 && time_before (jiffies, timeout))
749 barrier();
750
751 /* If there's no response yet, log a message but allow a longer
752 * timeout period -- if we get a response this time, log
753 * a message saying when we got it..
754 */
755 if (atomic_read(&data.unstarted_count) > 0) {
756 long start_time = jiffies;
757 printk(KERN_ERR "%s: initial timeout -- trying long wait\n",
758 __func__);
759 timeout = jiffies + 30 * HZ;
760 while (atomic_read(&data.unstarted_count) > 0
761 && time_before(jiffies, timeout))
762 barrier();
763 if (atomic_read(&data.unstarted_count) <= 0) {
764 long delta = jiffies - start_time;
765 printk(KERN_ERR
766 "%s: response %ld.%ld seconds into long wait\n",
767 __func__, delta / HZ,
768 (100 * (delta - ((delta / HZ) * HZ))) / HZ);
769 }
770 }
771
772 /* We either got one or timed out -- clear the lock. */
773 mb();
774 smp_call_function_data = NULL;
775
776 /*
777 * If after both the initial and long timeout periods we still don't
778 * have a response, something is very wrong...
779 */
780 BUG_ON(atomic_read (&data.unstarted_count) > 0);
781
782 /* Wait for a complete response, if needed. */
783 if (wait) {
784 while (atomic_read (&data.unfinished_count) > 0)
785 barrier();
786 }
787
788 return 0;
789} 643}
790EXPORT_SYMBOL(smp_call_function_on_cpu);
791 644
792int 645void arch_send_call_function_single_ipi(int cpu)
793smp_call_function (void (*func) (void *info), void *info, int retry, int wait)
794{ 646{
795 return smp_call_function_on_cpu (func, info, retry, wait, 647 send_ipi_message(cpumask_of_cpu(cpu), IPI_CALL_FUNC_SINGLE);
796 cpu_online_map);
797} 648}
798EXPORT_SYMBOL(smp_call_function);
799 649
800static void 650static void
801ipi_imb(void *ignored) 651ipi_imb(void *ignored)
@@ -807,7 +657,7 @@ void
807smp_imb(void) 657smp_imb(void)
808{ 658{
809 /* Must wait other processors to flush their icache before continue. */ 659 /* Must wait other processors to flush their icache before continue. */
810 if (on_each_cpu(ipi_imb, NULL, 1, 1)) 660 if (on_each_cpu(ipi_imb, NULL, 1))
811 printk(KERN_CRIT "smp_imb: timed out\n"); 661 printk(KERN_CRIT "smp_imb: timed out\n");
812} 662}
813EXPORT_SYMBOL(smp_imb); 663EXPORT_SYMBOL(smp_imb);
@@ -823,7 +673,7 @@ flush_tlb_all(void)
823{ 673{
824 /* Although we don't have any data to pass, we do want to 674 /* Although we don't have any data to pass, we do want to
825 synchronize with the other processors. */ 675 synchronize with the other processors. */
826 if (on_each_cpu(ipi_flush_tlb_all, NULL, 1, 1)) { 676 if (on_each_cpu(ipi_flush_tlb_all, NULL, 1)) {
827 printk(KERN_CRIT "flush_tlb_all: timed out\n"); 677 printk(KERN_CRIT "flush_tlb_all: timed out\n");
828 } 678 }
829} 679}
@@ -860,7 +710,7 @@ flush_tlb_mm(struct mm_struct *mm)
860 } 710 }
861 } 711 }
862 712
863 if (smp_call_function(ipi_flush_tlb_mm, mm, 1, 1)) { 713 if (smp_call_function(ipi_flush_tlb_mm, mm, 1)) {
864 printk(KERN_CRIT "flush_tlb_mm: timed out\n"); 714 printk(KERN_CRIT "flush_tlb_mm: timed out\n");
865 } 715 }
866 716
@@ -913,7 +763,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
913 data.mm = mm; 763 data.mm = mm;
914 data.addr = addr; 764 data.addr = addr;
915 765
916 if (smp_call_function(ipi_flush_tlb_page, &data, 1, 1)) { 766 if (smp_call_function(ipi_flush_tlb_page, &data, 1)) {
917 printk(KERN_CRIT "flush_tlb_page: timed out\n"); 767 printk(KERN_CRIT "flush_tlb_page: timed out\n");
918 } 768 }
919 769
@@ -965,7 +815,7 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
965 } 815 }
966 } 816 }
967 817
968 if (smp_call_function(ipi_flush_icache_page, mm, 1, 1)) { 818 if (smp_call_function(ipi_flush_icache_page, mm, 1)) {
969 printk(KERN_CRIT "flush_icache_page: timed out\n"); 819 printk(KERN_CRIT "flush_icache_page: timed out\n");
970 } 820 }
971 821
diff --git a/arch/alpha/oprofile/common.c b/arch/alpha/oprofile/common.c
index 9fc0eeb4f0ab..7c3d5ec6ec67 100644
--- a/arch/alpha/oprofile/common.c
+++ b/arch/alpha/oprofile/common.c
@@ -65,7 +65,7 @@ op_axp_setup(void)
65 model->reg_setup(&reg, ctr, &sys); 65 model->reg_setup(&reg, ctr, &sys);
66 66
67 /* Configure the registers on all cpus. */ 67 /* Configure the registers on all cpus. */
68 (void)smp_call_function(model->cpu_setup, &reg, 0, 1); 68 (void)smp_call_function(model->cpu_setup, &reg, 1);
69 model->cpu_setup(&reg); 69 model->cpu_setup(&reg);
70 return 0; 70 return 0;
71} 71}
@@ -86,7 +86,7 @@ op_axp_cpu_start(void *dummy)
86static int 86static int
87op_axp_start(void) 87op_axp_start(void)
88{ 88{
89 (void)smp_call_function(op_axp_cpu_start, NULL, 0, 1); 89 (void)smp_call_function(op_axp_cpu_start, NULL, 1);
90 op_axp_cpu_start(NULL); 90 op_axp_cpu_start(NULL);
91 return 0; 91 return 0;
92} 92}
@@ -101,7 +101,7 @@ op_axp_cpu_stop(void *dummy)
101static void 101static void
102op_axp_stop(void) 102op_axp_stop(void)
103{ 103{
104 (void)smp_call_function(op_axp_cpu_stop, NULL, 0, 1); 104 (void)smp_call_function(op_axp_cpu_stop, NULL, 1);
105 op_axp_cpu_stop(NULL); 105 op_axp_cpu_stop(NULL);
106} 106}
107 107
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 258f1369fb0c..c7ad324ddf2c 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -701,6 +701,7 @@ source "kernel/time/Kconfig"
701config SMP 701config SMP
702 bool "Symmetric Multi-Processing (EXPERIMENTAL)" 702 bool "Symmetric Multi-Processing (EXPERIMENTAL)"
703 depends on EXPERIMENTAL && (REALVIEW_EB_ARM11MP || MACH_REALVIEW_PB11MP) 703 depends on EXPERIMENTAL && (REALVIEW_EB_ARM11MP || MACH_REALVIEW_PB11MP)
704 select USE_GENERIC_SMP_HELPERS
704 help 705 help
705 This enables support for systems with more than one CPU. If you have 706 This enables support for systems with more than one CPU. If you have
706 a system with only one CPU, like most personal computers, say N. If 707 a system with only one CPU, like most personal computers, say N. If
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index eefae1de334c..5a7c09564d13 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -68,20 +68,10 @@ enum ipi_msg_type {
68 IPI_TIMER, 68 IPI_TIMER,
69 IPI_RESCHEDULE, 69 IPI_RESCHEDULE,
70 IPI_CALL_FUNC, 70 IPI_CALL_FUNC,
71 IPI_CALL_FUNC_SINGLE,
71 IPI_CPU_STOP, 72 IPI_CPU_STOP,
72}; 73};
73 74
74struct smp_call_struct {
75 void (*func)(void *info);
76 void *info;
77 int wait;
78 cpumask_t pending;
79 cpumask_t unfinished;
80};
81
82static struct smp_call_struct * volatile smp_call_function_data;
83static DEFINE_SPINLOCK(smp_call_function_lock);
84
85int __cpuinit __cpu_up(unsigned int cpu) 75int __cpuinit __cpu_up(unsigned int cpu)
86{ 76{
87 struct cpuinfo_arm *ci = &per_cpu(cpu_data, cpu); 77 struct cpuinfo_arm *ci = &per_cpu(cpu_data, cpu);
@@ -366,114 +356,15 @@ static void send_ipi_message(cpumask_t callmap, enum ipi_msg_type msg)
366 local_irq_restore(flags); 356 local_irq_restore(flags);
367} 357}
368 358
369/* 359void arch_send_call_function_ipi(cpumask_t mask)
370 * You must not call this function with disabled interrupts, from a
371 * hardware interrupt handler, nor from a bottom half handler.
372 */
373static int smp_call_function_on_cpu(void (*func)(void *info), void *info,
374 int retry, int wait, cpumask_t callmap)
375{
376 struct smp_call_struct data;
377 unsigned long timeout;
378 int ret = 0;
379
380 data.func = func;
381 data.info = info;
382 data.wait = wait;
383
384 cpu_clear(smp_processor_id(), callmap);
385 if (cpus_empty(callmap))
386 goto out;
387
388 data.pending = callmap;
389 if (wait)
390 data.unfinished = callmap;
391
392 /*
393 * try to get the mutex on smp_call_function_data
394 */
395 spin_lock(&smp_call_function_lock);
396 smp_call_function_data = &data;
397
398 send_ipi_message(callmap, IPI_CALL_FUNC);
399
400 timeout = jiffies + HZ;
401 while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
402 barrier();
403
404 /*
405 * did we time out?
406 */
407 if (!cpus_empty(data.pending)) {
408 /*
409 * this may be causing our panic - report it
410 */
411 printk(KERN_CRIT
412 "CPU%u: smp_call_function timeout for %p(%p)\n"
413 " callmap %lx pending %lx, %swait\n",
414 smp_processor_id(), func, info, *cpus_addr(callmap),
415 *cpus_addr(data.pending), wait ? "" : "no ");
416
417 /*
418 * TRACE
419 */
420 timeout = jiffies + (5 * HZ);
421 while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
422 barrier();
423
424 if (cpus_empty(data.pending))
425 printk(KERN_CRIT " RESOLVED\n");
426 else
427 printk(KERN_CRIT " STILL STUCK\n");
428 }
429
430 /*
431 * whatever happened, we're done with the data, so release it
432 */
433 smp_call_function_data = NULL;
434 spin_unlock(&smp_call_function_lock);
435
436 if (!cpus_empty(data.pending)) {
437 ret = -ETIMEDOUT;
438 goto out;
439 }
440
441 if (wait)
442 while (!cpus_empty(data.unfinished))
443 barrier();
444 out:
445
446 return 0;
447}
448
449int smp_call_function(void (*func)(void *info), void *info, int retry,
450 int wait)
451{ 360{
452 return smp_call_function_on_cpu(func, info, retry, wait, 361 send_ipi_message(mask, IPI_CALL_FUNC);
453 cpu_online_map);
454} 362}
455EXPORT_SYMBOL_GPL(smp_call_function);
456 363
457int smp_call_function_single(int cpu, void (*func)(void *info), void *info, 364void arch_send_call_function_single_ipi(int cpu)
458 int retry, int wait)
459{ 365{
460 /* prevent preemption and reschedule on another processor */ 366 send_ipi_message(cpumask_of_cpu(cpu), IPI_CALL_FUNC_SINGLE);
461 int current_cpu = get_cpu();
462 int ret = 0;
463
464 if (cpu == current_cpu) {
465 local_irq_disable();
466 func(info);
467 local_irq_enable();
468 } else
469 ret = smp_call_function_on_cpu(func, info, retry, wait,
470 cpumask_of_cpu(cpu));
471
472 put_cpu();
473
474 return ret;
475} 367}
476EXPORT_SYMBOL_GPL(smp_call_function_single);
477 368
478void show_ipi_list(struct seq_file *p) 369void show_ipi_list(struct seq_file *p)
479{ 370{
@@ -521,27 +412,6 @@ asmlinkage void __exception do_local_timer(struct pt_regs *regs)
521} 412}
522#endif 413#endif
523 414
524/*
525 * ipi_call_function - handle IPI from smp_call_function()
526 *
527 * Note that we copy data out of the cross-call structure and then
528 * let the caller know that we're here and have done with their data
529 */
530static void ipi_call_function(unsigned int cpu)
531{
532 struct smp_call_struct *data = smp_call_function_data;
533 void (*func)(void *info) = data->func;
534 void *info = data->info;
535 int wait = data->wait;
536
537 cpu_clear(cpu, data->pending);
538
539 func(info);
540
541 if (wait)
542 cpu_clear(cpu, data->unfinished);
543}
544
545static DEFINE_SPINLOCK(stop_lock); 415static DEFINE_SPINLOCK(stop_lock);
546 416
547/* 417/*
@@ -611,7 +481,11 @@ asmlinkage void __exception do_IPI(struct pt_regs *regs)
611 break; 481 break;
612 482
613 case IPI_CALL_FUNC: 483 case IPI_CALL_FUNC:
614 ipi_call_function(cpu); 484 generic_smp_call_function_interrupt();
485 break;
486
487 case IPI_CALL_FUNC_SINGLE:
488 generic_smp_call_function_single_interrupt();
615 break; 489 break;
616 490
617 case IPI_CPU_STOP: 491 case IPI_CPU_STOP:
@@ -662,14 +536,13 @@ int setup_profiling_timer(unsigned int multiplier)
662} 536}
663 537
664static int 538static int
665on_each_cpu_mask(void (*func)(void *), void *info, int retry, int wait, 539on_each_cpu_mask(void (*func)(void *), void *info, int wait, cpumask_t mask)
666 cpumask_t mask)
667{ 540{
668 int ret = 0; 541 int ret = 0;
669 542
670 preempt_disable(); 543 preempt_disable();
671 544
672 ret = smp_call_function_on_cpu(func, info, retry, wait, mask); 545 ret = smp_call_function_mask(mask, func, info, wait);
673 if (cpu_isset(smp_processor_id(), mask)) 546 if (cpu_isset(smp_processor_id(), mask))
674 func(info); 547 func(info);
675 548
@@ -731,14 +604,14 @@ static inline void ipi_flush_tlb_kernel_range(void *arg)
731 604
732void flush_tlb_all(void) 605void flush_tlb_all(void)
733{ 606{
734 on_each_cpu(ipi_flush_tlb_all, NULL, 1, 1); 607 on_each_cpu(ipi_flush_tlb_all, NULL, 1);
735} 608}
736 609
737void flush_tlb_mm(struct mm_struct *mm) 610void flush_tlb_mm(struct mm_struct *mm)
738{ 611{
739 cpumask_t mask = mm->cpu_vm_mask; 612 cpumask_t mask = mm->cpu_vm_mask;
740 613
741 on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, 1, mask); 614 on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, mask);
742} 615}
743 616
744void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) 617void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
@@ -749,7 +622,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
749 ta.ta_vma = vma; 622 ta.ta_vma = vma;
750 ta.ta_start = uaddr; 623 ta.ta_start = uaddr;
751 624
752 on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, 1, mask); 625 on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, mask);
753} 626}
754 627
755void flush_tlb_kernel_page(unsigned long kaddr) 628void flush_tlb_kernel_page(unsigned long kaddr)
@@ -758,7 +631,7 @@ void flush_tlb_kernel_page(unsigned long kaddr)
758 631
759 ta.ta_start = kaddr; 632 ta.ta_start = kaddr;
760 633
761 on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1, 1); 634 on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
762} 635}
763 636
764void flush_tlb_range(struct vm_area_struct *vma, 637void flush_tlb_range(struct vm_area_struct *vma,
@@ -771,7 +644,7 @@ void flush_tlb_range(struct vm_area_struct *vma,
771 ta.ta_start = start; 644 ta.ta_start = start;
772 ta.ta_end = end; 645 ta.ta_end = end;
773 646
774 on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, 1, mask); 647 on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, mask);
775} 648}
776 649
777void flush_tlb_kernel_range(unsigned long start, unsigned long end) 650void flush_tlb_kernel_range(unsigned long start, unsigned long end)
@@ -781,5 +654,5 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
781 ta.ta_start = start; 654 ta.ta_start = start;
782 ta.ta_end = end; 655 ta.ta_end = end;
783 656
784 on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1, 1); 657 on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
785} 658}
diff --git a/arch/arm/oprofile/op_model_mpcore.c b/arch/arm/oprofile/op_model_mpcore.c
index 74fae6045650..4458705021e0 100644
--- a/arch/arm/oprofile/op_model_mpcore.c
+++ b/arch/arm/oprofile/op_model_mpcore.c
@@ -201,7 +201,7 @@ static int em_call_function(int (*fn)(void))
201 data.ret = 0; 201 data.ret = 0;
202 202
203 preempt_disable(); 203 preempt_disable();
204 smp_call_function(em_func, &data, 1, 1); 204 smp_call_function(em_func, &data, 1);
205 em_func(&data); 205 em_func(&data);
206 preempt_enable(); 206 preempt_enable();
207 207
diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c
index 32455c633f1c..c0d2c9bb952b 100644
--- a/arch/arm/vfp/vfpmodule.c
+++ b/arch/arm/vfp/vfpmodule.c
@@ -352,7 +352,7 @@ static int __init vfp_init(void)
352 else if (vfpsid & FPSID_NODOUBLE) { 352 else if (vfpsid & FPSID_NODOUBLE) {
353 printk("no double precision support\n"); 353 printk("no double precision support\n");
354 } else { 354 } else {
355 smp_call_function(vfp_enable, NULL, 1, 1); 355 smp_call_function(vfp_enable, NULL, 1);
356 356
357 VFP_arch = (vfpsid & FPSID_ARCH_MASK) >> FPSID_ARCH_BIT; /* Extract the architecture version */ 357 VFP_arch = (vfpsid & FPSID_ARCH_MASK) >> FPSID_ARCH_BIT; /* Extract the architecture version */
358 printk("implementor %02x architecture %d part %02x variant %x rev %x\n", 358 printk("implementor %02x architecture %d part %02x variant %x rev %x\n",
diff --git a/arch/cris/arch-v32/kernel/smp.c b/arch/cris/arch-v32/kernel/smp.c
index a9c3334e46c9..952a24b2f5a9 100644
--- a/arch/cris/arch-v32/kernel/smp.c
+++ b/arch/cris/arch-v32/kernel/smp.c
@@ -194,7 +194,7 @@ void stop_this_cpu(void* dummy)
194/* Other calls */ 194/* Other calls */
195void smp_send_stop(void) 195void smp_send_stop(void)
196{ 196{
197 smp_call_function(stop_this_cpu, NULL, 1, 0); 197 smp_call_function(stop_this_cpu, NULL, 0);
198} 198}
199 199
200int setup_profiling_timer(unsigned int multiplier) 200int setup_profiling_timer(unsigned int multiplier)
@@ -316,8 +316,7 @@ int send_ipi(int vector, int wait, cpumask_t cpu_mask)
316 * You must not call this function with disabled interrupts or from a 316 * You must not call this function with disabled interrupts or from a
317 * hardware interrupt handler or from a bottom half handler. 317 * hardware interrupt handler or from a bottom half handler.
318 */ 318 */
319int smp_call_function(void (*func)(void *info), void *info, 319int smp_call_function(void (*func)(void *info), void *info, int wait)
320 int nonatomic, int wait)
321{ 320{
322 cpumask_t cpu_mask = CPU_MASK_ALL; 321 cpumask_t cpu_mask = CPU_MASK_ALL;
323 struct call_data_struct data; 322 struct call_data_struct data;
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index 16be41446b5b..18bcc10903b4 100644
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -303,6 +303,7 @@ config VIRT_CPU_ACCOUNTING
303 303
304config SMP 304config SMP
305 bool "Symmetric multi-processing support" 305 bool "Symmetric multi-processing support"
306 select USE_GENERIC_SMP_HELPERS
306 help 307 help
307 This enables support for systems with more than one CPU. If you have 308 This enables support for systems with more than one CPU. If you have
308 a system with only one CPU, say N. If you have a system with more 309 a system with only one CPU, say N. If you have a system with more
diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c
index 705176b434b3..7dd96c127177 100644
--- a/arch/ia64/kernel/mca.c
+++ b/arch/ia64/kernel/mca.c
@@ -707,7 +707,7 @@ ia64_mca_cmc_vector_enable (void *dummy)
707static void 707static void
708ia64_mca_cmc_vector_disable_keventd(struct work_struct *unused) 708ia64_mca_cmc_vector_disable_keventd(struct work_struct *unused)
709{ 709{
710 on_each_cpu(ia64_mca_cmc_vector_disable, NULL, 1, 0); 710 on_each_cpu(ia64_mca_cmc_vector_disable, NULL, 0);
711} 711}
712 712
713/* 713/*
@@ -719,7 +719,7 @@ ia64_mca_cmc_vector_disable_keventd(struct work_struct *unused)
719static void 719static void
720ia64_mca_cmc_vector_enable_keventd(struct work_struct *unused) 720ia64_mca_cmc_vector_enable_keventd(struct work_struct *unused)
721{ 721{
722 on_each_cpu(ia64_mca_cmc_vector_enable, NULL, 1, 0); 722 on_each_cpu(ia64_mca_cmc_vector_enable, NULL, 0);
723} 723}
724 724
725/* 725/*
@@ -1881,7 +1881,7 @@ static int __cpuinit mca_cpu_callback(struct notifier_block *nfb,
1881 case CPU_ONLINE: 1881 case CPU_ONLINE:
1882 case CPU_ONLINE_FROZEN: 1882 case CPU_ONLINE_FROZEN:
1883 smp_call_function_single(hotcpu, ia64_mca_cmc_vector_adjust, 1883 smp_call_function_single(hotcpu, ia64_mca_cmc_vector_adjust,
1884 NULL, 1, 0); 1884 NULL, 0);
1885 break; 1885 break;
1886 } 1886 }
1887 return NOTIFY_OK; 1887 return NOTIFY_OK;
diff --git a/arch/ia64/kernel/palinfo.c b/arch/ia64/kernel/palinfo.c
index 9dc00f7fe10e..e5c57f413ca2 100644
--- a/arch/ia64/kernel/palinfo.c
+++ b/arch/ia64/kernel/palinfo.c
@@ -921,7 +921,7 @@ int palinfo_handle_smp(pal_func_cpu_u_t *f, char *page)
921 921
922 922
923 /* will send IPI to other CPU and wait for completion of remote call */ 923 /* will send IPI to other CPU and wait for completion of remote call */
924 if ((ret=smp_call_function_single(f->req_cpu, palinfo_smp_call, &ptr, 0, 1))) { 924 if ((ret=smp_call_function_single(f->req_cpu, palinfo_smp_call, &ptr, 1))) {
925 printk(KERN_ERR "palinfo: remote CPU call from %d to %d on function %d: " 925 printk(KERN_ERR "palinfo: remote CPU call from %d to %d on function %d: "
926 "error %d\n", smp_processor_id(), f->req_cpu, f->func_id, ret); 926 "error %d\n", smp_processor_id(), f->req_cpu, f->func_id, ret);
927 return 0; 927 return 0;
diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c
index 7714a97b0104..19d4493c6193 100644
--- a/arch/ia64/kernel/perfmon.c
+++ b/arch/ia64/kernel/perfmon.c
@@ -1820,7 +1820,7 @@ pfm_syswide_cleanup_other_cpu(pfm_context_t *ctx)
1820 int ret; 1820 int ret;
1821 1821
1822 DPRINT(("calling CPU%d for cleanup\n", ctx->ctx_cpu)); 1822 DPRINT(("calling CPU%d for cleanup\n", ctx->ctx_cpu));
1823 ret = smp_call_function_single(ctx->ctx_cpu, pfm_syswide_force_stop, ctx, 0, 1); 1823 ret = smp_call_function_single(ctx->ctx_cpu, pfm_syswide_force_stop, ctx, 1);
1824 DPRINT(("called CPU%d for cleanup ret=%d\n", ctx->ctx_cpu, ret)); 1824 DPRINT(("called CPU%d for cleanup ret=%d\n", ctx->ctx_cpu, ret));
1825} 1825}
1826#endif /* CONFIG_SMP */ 1826#endif /* CONFIG_SMP */
@@ -6508,7 +6508,7 @@ pfm_install_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
6508 } 6508 }
6509 6509
6510 /* save the current system wide pmu states */ 6510 /* save the current system wide pmu states */
6511 ret = on_each_cpu(pfm_alt_save_pmu_state, NULL, 0, 1); 6511 ret = on_each_cpu(pfm_alt_save_pmu_state, NULL, 1);
6512 if (ret) { 6512 if (ret) {
6513 DPRINT(("on_each_cpu() failed: %d\n", ret)); 6513 DPRINT(("on_each_cpu() failed: %d\n", ret));
6514 goto cleanup_reserve; 6514 goto cleanup_reserve;
@@ -6553,7 +6553,7 @@ pfm_remove_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
6553 6553
6554 pfm_alt_intr_handler = NULL; 6554 pfm_alt_intr_handler = NULL;
6555 6555
6556 ret = on_each_cpu(pfm_alt_restore_pmu_state, NULL, 0, 1); 6556 ret = on_each_cpu(pfm_alt_restore_pmu_state, NULL, 1);
6557 if (ret) { 6557 if (ret) {
6558 DPRINT(("on_each_cpu() failed: %d\n", ret)); 6558 DPRINT(("on_each_cpu() failed: %d\n", ret));
6559 } 6559 }
diff --git a/arch/ia64/kernel/process.c b/arch/ia64/kernel/process.c
index a3a34b4eb038..fabaf08d9a69 100644
--- a/arch/ia64/kernel/process.c
+++ b/arch/ia64/kernel/process.c
@@ -286,7 +286,7 @@ void cpu_idle_wait(void)
286{ 286{
287 smp_mb(); 287 smp_mb();
288 /* kick all the CPUs so that they exit out of pm_idle */ 288 /* kick all the CPUs so that they exit out of pm_idle */
289 smp_call_function(do_nothing, NULL, 0, 1); 289 smp_call_function(do_nothing, NULL, 1);
290} 290}
291EXPORT_SYMBOL_GPL(cpu_idle_wait); 291EXPORT_SYMBOL_GPL(cpu_idle_wait);
292 292
diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c
index 983296f1c813..3676468612b6 100644
--- a/arch/ia64/kernel/smp.c
+++ b/arch/ia64/kernel/smp.c
@@ -60,25 +60,9 @@ static struct local_tlb_flush_counts {
60 60
61static DEFINE_PER_CPU(unsigned int, shadow_flush_counts[NR_CPUS]) ____cacheline_aligned; 61static DEFINE_PER_CPU(unsigned int, shadow_flush_counts[NR_CPUS]) ____cacheline_aligned;
62 62
63
64/*
65 * Structure and data for smp_call_function(). This is designed to minimise static memory
66 * requirements. It also looks cleaner.
67 */
68static __cacheline_aligned DEFINE_SPINLOCK(call_lock);
69
70struct call_data_struct {
71 void (*func) (void *info);
72 void *info;
73 long wait;
74 atomic_t started;
75 atomic_t finished;
76};
77
78static volatile struct call_data_struct *call_data;
79
80#define IPI_CALL_FUNC 0 63#define IPI_CALL_FUNC 0
81#define IPI_CPU_STOP 1 64#define IPI_CPU_STOP 1
65#define IPI_CALL_FUNC_SINGLE 2
82#define IPI_KDUMP_CPU_STOP 3 66#define IPI_KDUMP_CPU_STOP 3
83 67
84/* This needs to be cacheline aligned because it is written to by *other* CPUs. */ 68/* This needs to be cacheline aligned because it is written to by *other* CPUs. */
@@ -86,43 +70,6 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(u64, ipi_operation);
86 70
87extern void cpu_halt (void); 71extern void cpu_halt (void);
88 72
89void
90lock_ipi_calllock(void)
91{
92 spin_lock_irq(&call_lock);
93}
94
95void
96unlock_ipi_calllock(void)
97{
98 spin_unlock_irq(&call_lock);
99}
100
101static inline void
102handle_call_data(void)
103{
104 struct call_data_struct *data;
105 void (*func)(void *info);
106 void *info;
107 int wait;
108
109 /* release the 'pointer lock' */
110 data = (struct call_data_struct *)call_data;
111 func = data->func;
112 info = data->info;
113 wait = data->wait;
114
115 mb();
116 atomic_inc(&data->started);
117 /* At this point the structure may be gone unless wait is true. */
118 (*func)(info);
119
120 /* Notify the sending CPU that the task is done. */
121 mb();
122 if (wait)
123 atomic_inc(&data->finished);
124}
125
126static void 73static void
127stop_this_cpu(void) 74stop_this_cpu(void)
128{ 75{
@@ -163,13 +110,15 @@ handle_IPI (int irq, void *dev_id)
163 ops &= ~(1 << which); 110 ops &= ~(1 << which);
164 111
165 switch (which) { 112 switch (which) {
166 case IPI_CALL_FUNC:
167 handle_call_data();
168 break;
169
170 case IPI_CPU_STOP: 113 case IPI_CPU_STOP:
171 stop_this_cpu(); 114 stop_this_cpu();
172 break; 115 break;
116 case IPI_CALL_FUNC:
117 generic_smp_call_function_interrupt();
118 break;
119 case IPI_CALL_FUNC_SINGLE:
120 generic_smp_call_function_single_interrupt();
121 break;
173#ifdef CONFIG_KEXEC 122#ifdef CONFIG_KEXEC
174 case IPI_KDUMP_CPU_STOP: 123 case IPI_KDUMP_CPU_STOP:
175 unw_init_running(kdump_cpu_freeze, NULL); 124 unw_init_running(kdump_cpu_freeze, NULL);
@@ -187,6 +136,8 @@ handle_IPI (int irq, void *dev_id)
187 return IRQ_HANDLED; 136 return IRQ_HANDLED;
188} 137}
189 138
139
140
190/* 141/*
191 * Called with preemption disabled. 142 * Called with preemption disabled.
192 */ 143 */
@@ -334,7 +285,7 @@ smp_flush_tlb_cpumask(cpumask_t xcpumask)
334void 285void
335smp_flush_tlb_all (void) 286smp_flush_tlb_all (void)
336{ 287{
337 on_each_cpu((void (*)(void *))local_flush_tlb_all, NULL, 1, 1); 288 on_each_cpu((void (*)(void *))local_flush_tlb_all, NULL, 1);
338} 289}
339 290
340void 291void
@@ -357,193 +308,18 @@ smp_flush_tlb_mm (struct mm_struct *mm)
357 * anyhow, and once a CPU is interrupted, the cost of local_flush_tlb_all() is 308 * anyhow, and once a CPU is interrupted, the cost of local_flush_tlb_all() is
358 * rather trivial. 309 * rather trivial.
359 */ 310 */
360 on_each_cpu((void (*)(void *))local_finish_flush_tlb_mm, mm, 1, 1); 311 on_each_cpu((void (*)(void *))local_finish_flush_tlb_mm, mm, 1);
361} 312}
362 313
363/* 314void arch_send_call_function_single_ipi(int cpu)
364 * Run a function on a specific CPU
365 * <func> The function to run. This must be fast and non-blocking.
366 * <info> An arbitrary pointer to pass to the function.
367 * <nonatomic> Currently unused.
368 * <wait> If true, wait until function has completed on other CPUs.
369 * [RETURNS] 0 on success, else a negative status code.
370 *
371 * Does not return until the remote CPU is nearly ready to execute <func>
372 * or is or has executed.
373 */
374
375int
376smp_call_function_single (int cpuid, void (*func) (void *info), void *info, int nonatomic,
377 int wait)
378{
379 struct call_data_struct data;
380 int cpus = 1;
381 int me = get_cpu(); /* prevent preemption and reschedule on another processor */
382
383 if (cpuid == me) {
384 local_irq_disable();
385 func(info);
386 local_irq_enable();
387 put_cpu();
388 return 0;
389 }
390
391 data.func = func;
392 data.info = info;
393 atomic_set(&data.started, 0);
394 data.wait = wait;
395 if (wait)
396 atomic_set(&data.finished, 0);
397
398 spin_lock_bh(&call_lock);
399
400 call_data = &data;
401 mb(); /* ensure store to call_data precedes setting of IPI_CALL_FUNC */
402 send_IPI_single(cpuid, IPI_CALL_FUNC);
403
404 /* Wait for response */
405 while (atomic_read(&data.started) != cpus)
406 cpu_relax();
407
408 if (wait)
409 while (atomic_read(&data.finished) != cpus)
410 cpu_relax();
411 call_data = NULL;
412
413 spin_unlock_bh(&call_lock);
414 put_cpu();
415 return 0;
416}
417EXPORT_SYMBOL(smp_call_function_single);
418
419/**
420 * smp_call_function_mask(): Run a function on a set of other CPUs.
421 * <mask> The set of cpus to run on. Must not include the current cpu.
422 * <func> The function to run. This must be fast and non-blocking.
423 * <info> An arbitrary pointer to pass to the function.
424 * <wait> If true, wait (atomically) until function
425 * has completed on other CPUs.
426 *
427 * Returns 0 on success, else a negative status code.
428 *
429 * If @wait is true, then returns once @func has returned; otherwise
430 * it returns just before the target cpu calls @func.
431 *
432 * You must not call this function with disabled interrupts or from a
433 * hardware interrupt handler or from a bottom half handler.
434 */
435int smp_call_function_mask(cpumask_t mask,
436 void (*func)(void *), void *info,
437 int wait)
438{ 315{
439 struct call_data_struct data; 316 send_IPI_single(cpu, IPI_CALL_FUNC_SINGLE);
440 cpumask_t allbutself;
441 int cpus;
442
443 spin_lock(&call_lock);
444 allbutself = cpu_online_map;
445 cpu_clear(smp_processor_id(), allbutself);
446
447 cpus_and(mask, mask, allbutself);
448 cpus = cpus_weight(mask);
449 if (!cpus) {
450 spin_unlock(&call_lock);
451 return 0;
452 }
453
454 /* Can deadlock when called with interrupts disabled */
455 WARN_ON(irqs_disabled());
456
457 data.func = func;
458 data.info = info;
459 atomic_set(&data.started, 0);
460 data.wait = wait;
461 if (wait)
462 atomic_set(&data.finished, 0);
463
464 call_data = &data;
465 mb(); /* ensure store to call_data precedes setting of IPI_CALL_FUNC*/
466
467 /* Send a message to other CPUs */
468 if (cpus_equal(mask, allbutself))
469 send_IPI_allbutself(IPI_CALL_FUNC);
470 else
471 send_IPI_mask(mask, IPI_CALL_FUNC);
472
473 /* Wait for response */
474 while (atomic_read(&data.started) != cpus)
475 cpu_relax();
476
477 if (wait)
478 while (atomic_read(&data.finished) != cpus)
479 cpu_relax();
480 call_data = NULL;
481
482 spin_unlock(&call_lock);
483 return 0;
484
485} 317}
486EXPORT_SYMBOL(smp_call_function_mask);
487 318
488/* 319void arch_send_call_function_ipi(cpumask_t mask)
489 * this function sends a 'generic call function' IPI to all other CPUs
490 * in the system.
491 */
492
493/*
494 * [SUMMARY] Run a function on all other CPUs.
495 * <func> The function to run. This must be fast and non-blocking.
496 * <info> An arbitrary pointer to pass to the function.
497 * <nonatomic> currently unused.
498 * <wait> If true, wait (atomically) until function has completed on other CPUs.
499 * [RETURNS] 0 on success, else a negative status code.
500 *
501 * Does not return until remote CPUs are nearly ready to execute <func> or are or have
502 * executed.
503 *
504 * You must not call this function with disabled interrupts or from a
505 * hardware interrupt handler or from a bottom half handler.
506 */
507int
508smp_call_function (void (*func) (void *info), void *info, int nonatomic, int wait)
509{ 320{
510 struct call_data_struct data; 321 send_IPI_mask(mask, IPI_CALL_FUNC);
511 int cpus;
512
513 spin_lock(&call_lock);
514 cpus = num_online_cpus() - 1;
515 if (!cpus) {
516 spin_unlock(&call_lock);
517 return 0;
518 }
519
520 /* Can deadlock when called with interrupts disabled */
521 WARN_ON(irqs_disabled());
522
523 data.func = func;
524 data.info = info;
525 atomic_set(&data.started, 0);
526 data.wait = wait;
527 if (wait)
528 atomic_set(&data.finished, 0);
529
530 call_data = &data;
531 mb(); /* ensure store to call_data precedes setting of IPI_CALL_FUNC */
532 send_IPI_allbutself(IPI_CALL_FUNC);
533
534 /* Wait for response */
535 while (atomic_read(&data.started) != cpus)
536 cpu_relax();
537
538 if (wait)
539 while (atomic_read(&data.finished) != cpus)
540 cpu_relax();
541 call_data = NULL;
542
543 spin_unlock(&call_lock);
544 return 0;
545} 322}
546EXPORT_SYMBOL(smp_call_function);
547 323
548/* 324/*
549 * this function calls the 'stop' function on all other CPUs in the system. 325 * this function calls the 'stop' function on all other CPUs in the system.
diff --git a/arch/ia64/kernel/smpboot.c b/arch/ia64/kernel/smpboot.c
index d7ad42b77d41..9d1d429c6c59 100644
--- a/arch/ia64/kernel/smpboot.c
+++ b/arch/ia64/kernel/smpboot.c
@@ -317,7 +317,7 @@ ia64_sync_itc (unsigned int master)
317 317
318 go[MASTER] = 1; 318 go[MASTER] = 1;
319 319
320 if (smp_call_function_single(master, sync_master, NULL, 1, 0) < 0) { 320 if (smp_call_function_single(master, sync_master, NULL, 0) < 0) {
321 printk(KERN_ERR "sync_itc: failed to get attention of CPU %u!\n", master); 321 printk(KERN_ERR "sync_itc: failed to get attention of CPU %u!\n", master);
322 return; 322 return;
323 } 323 }
@@ -395,14 +395,14 @@ smp_callin (void)
395 395
396 fix_b0_for_bsp(); 396 fix_b0_for_bsp();
397 397
398 lock_ipi_calllock(); 398 ipi_call_lock_irq();
399 spin_lock(&vector_lock); 399 spin_lock(&vector_lock);
400 /* Setup the per cpu irq handling data structures */ 400 /* Setup the per cpu irq handling data structures */
401 __setup_vector_irq(cpuid); 401 __setup_vector_irq(cpuid);
402 cpu_set(cpuid, cpu_online_map); 402 cpu_set(cpuid, cpu_online_map);
403 per_cpu(cpu_state, cpuid) = CPU_ONLINE; 403 per_cpu(cpu_state, cpuid) = CPU_ONLINE;
404 spin_unlock(&vector_lock); 404 spin_unlock(&vector_lock);
405 unlock_ipi_calllock(); 405 ipi_call_unlock_irq();
406 406
407 smp_setup_percpu_timer(); 407 smp_setup_percpu_timer();
408 408
diff --git a/arch/ia64/kernel/uncached.c b/arch/ia64/kernel/uncached.c
index e77995a6e3ed..8eff8c1d40a6 100644
--- a/arch/ia64/kernel/uncached.c
+++ b/arch/ia64/kernel/uncached.c
@@ -123,8 +123,7 @@ static int uncached_add_chunk(struct uncached_pool *uc_pool, int nid)
123 status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL); 123 status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL);
124 if (status == PAL_VISIBILITY_OK_REMOTE_NEEDED) { 124 if (status == PAL_VISIBILITY_OK_REMOTE_NEEDED) {
125 atomic_set(&uc_pool->status, 0); 125 atomic_set(&uc_pool->status, 0);
126 status = smp_call_function(uncached_ipi_visibility, uc_pool, 126 status = smp_call_function(uncached_ipi_visibility, uc_pool, 1);
127 0, 1);
128 if (status || atomic_read(&uc_pool->status)) 127 if (status || atomic_read(&uc_pool->status))
129 goto failed; 128 goto failed;
130 } else if (status != PAL_VISIBILITY_OK) 129 } else if (status != PAL_VISIBILITY_OK)
@@ -146,7 +145,7 @@ static int uncached_add_chunk(struct uncached_pool *uc_pool, int nid)
146 if (status != PAL_STATUS_SUCCESS) 145 if (status != PAL_STATUS_SUCCESS)
147 goto failed; 146 goto failed;
148 atomic_set(&uc_pool->status, 0); 147 atomic_set(&uc_pool->status, 0);
149 status = smp_call_function(uncached_ipi_mc_drain, uc_pool, 0, 1); 148 status = smp_call_function(uncached_ipi_mc_drain, uc_pool, 1);
150 if (status || atomic_read(&uc_pool->status)) 149 if (status || atomic_read(&uc_pool->status))
151 goto failed; 150 goto failed;
152 151
diff --git a/arch/ia64/sn/kernel/sn2/sn_hwperf.c b/arch/ia64/sn/kernel/sn2/sn_hwperf.c
index 8cc0c4753d89..636588e7e068 100644
--- a/arch/ia64/sn/kernel/sn2/sn_hwperf.c
+++ b/arch/ia64/sn/kernel/sn2/sn_hwperf.c
@@ -629,7 +629,7 @@ static int sn_hwperf_op_cpu(struct sn_hwperf_op_info *op_info)
629 if (use_ipi) { 629 if (use_ipi) {
630 /* use an interprocessor interrupt to call SAL */ 630 /* use an interprocessor interrupt to call SAL */
631 smp_call_function_single(cpu, sn_hwperf_call_sal, 631 smp_call_function_single(cpu, sn_hwperf_call_sal,
632 op_info, 1, 1); 632 op_info, 1);
633 } 633 }
634 else { 634 else {
635 /* migrate the task before calling SAL */ 635 /* migrate the task before calling SAL */
diff --git a/arch/m32r/Kconfig b/arch/m32r/Kconfig
index de153de2ea9f..a5f864c445b2 100644
--- a/arch/m32r/Kconfig
+++ b/arch/m32r/Kconfig
@@ -296,6 +296,7 @@ config PREEMPT
296 296
297config SMP 297config SMP
298 bool "Symmetric multi-processing support" 298 bool "Symmetric multi-processing support"
299 select USE_GENERIC_SMP_HELPERS
299 ---help--- 300 ---help---
300 This enables support for systems with more than one CPU. If you have 301 This enables support for systems with more than one CPU. If you have
301 a system with only one CPU, like most personal computers, say N. If 302 a system with only one CPU, like most personal computers, say N. If
diff --git a/arch/m32r/kernel/m32r_ksyms.c b/arch/m32r/kernel/m32r_ksyms.c
index e6709fe950ba..16bcb189a383 100644
--- a/arch/m32r/kernel/m32r_ksyms.c
+++ b/arch/m32r/kernel/m32r_ksyms.c
@@ -43,9 +43,6 @@ EXPORT_SYMBOL(dcache_dummy);
43#endif 43#endif
44EXPORT_SYMBOL(cpu_data); 44EXPORT_SYMBOL(cpu_data);
45 45
46/* Global SMP stuff */
47EXPORT_SYMBOL(smp_call_function);
48
49/* TLB flushing */ 46/* TLB flushing */
50EXPORT_SYMBOL(smp_flush_tlb_page); 47EXPORT_SYMBOL(smp_flush_tlb_page);
51#endif 48#endif
diff --git a/arch/m32r/kernel/smp.c b/arch/m32r/kernel/smp.c
index c837bc13b015..7577f971ea4e 100644
--- a/arch/m32r/kernel/smp.c
+++ b/arch/m32r/kernel/smp.c
@@ -35,22 +35,6 @@
35/*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ 35/*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
36 36
37/* 37/*
38 * Structure and data for smp_call_function(). This is designed to minimise
39 * static memory requirements. It also looks cleaner.
40 */
41static DEFINE_SPINLOCK(call_lock);
42
43struct call_data_struct {
44 void (*func) (void *info);
45 void *info;
46 atomic_t started;
47 atomic_t finished;
48 int wait;
49} __attribute__ ((__aligned__(SMP_CACHE_BYTES)));
50
51static struct call_data_struct *call_data;
52
53/*
54 * For flush_cache_all() 38 * For flush_cache_all()
55 */ 39 */
56static DEFINE_SPINLOCK(flushcache_lock); 40static DEFINE_SPINLOCK(flushcache_lock);
@@ -96,9 +80,6 @@ void smp_invalidate_interrupt(void);
96void smp_send_stop(void); 80void smp_send_stop(void);
97static void stop_this_cpu(void *); 81static void stop_this_cpu(void *);
98 82
99int smp_call_function(void (*) (void *), void *, int, int);
100void smp_call_function_interrupt(void);
101
102void smp_send_timer(void); 83void smp_send_timer(void);
103void smp_ipi_timer_interrupt(struct pt_regs *); 84void smp_ipi_timer_interrupt(struct pt_regs *);
104void smp_local_timer_interrupt(void); 85void smp_local_timer_interrupt(void);
@@ -231,7 +212,7 @@ void smp_flush_tlb_all(void)
231 local_irq_save(flags); 212 local_irq_save(flags);
232 __flush_tlb_all(); 213 __flush_tlb_all();
233 local_irq_restore(flags); 214 local_irq_restore(flags);
234 smp_call_function(flush_tlb_all_ipi, NULL, 1, 1); 215 smp_call_function(flush_tlb_all_ipi, NULL, 1);
235 preempt_enable(); 216 preempt_enable();
236} 217}
237 218
@@ -524,7 +505,7 @@ void smp_invalidate_interrupt(void)
524 *==========================================================================*/ 505 *==========================================================================*/
525void smp_send_stop(void) 506void smp_send_stop(void)
526{ 507{
527 smp_call_function(stop_this_cpu, NULL, 1, 0); 508 smp_call_function(stop_this_cpu, NULL, 0);
528} 509}
529 510
530/*==========================================================================* 511/*==========================================================================*
@@ -565,86 +546,14 @@ static void stop_this_cpu(void *dummy)
565 for ( ; ; ); 546 for ( ; ; );
566} 547}
567 548
568/*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ 549void arch_send_call_function_ipi(cpumask_t mask)
569/* Call function Routines */
570/*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
571
572/*==========================================================================*
573 * Name: smp_call_function
574 *
575 * Description: This routine sends a 'CALL_FUNCTION_IPI' to all other CPUs
576 * in the system.
577 *
578 * Born on Date: 2002.02.05
579 *
580 * Arguments: *func - The function to run. This must be fast and
581 * non-blocking.
582 * *info - An arbitrary pointer to pass to the function.
583 * nonatomic - currently unused.
584 * wait - If true, wait (atomically) until function has
585 * completed on other CPUs.
586 *
587 * Returns: 0 on success, else a negative status code. Does not return
588 * until remote CPUs are nearly ready to execute <<func>> or
589 * are or have executed.
590 *
591 * Cautions: You must not call this function with disabled interrupts or
592 * from a hardware interrupt handler, you may call it from a
593 * bottom half handler.
594 *
595 * Modification log:
596 * Date Who Description
597 * ---------- --- --------------------------------------------------------
598 *
599 *==========================================================================*/
600int smp_call_function(void (*func) (void *info), void *info, int nonatomic,
601 int wait)
602{ 550{
603 struct call_data_struct data; 551 send_IPI_mask(mask, CALL_FUNCTION_IPI, 0);
604 int cpus; 552}
605
606#ifdef DEBUG_SMP
607 unsigned long flags;
608 __save_flags(flags);
609 if (!(flags & 0x0040)) /* Interrupt Disable NONONO */
610 BUG();
611#endif /* DEBUG_SMP */
612
613 /* Holding any lock stops cpus from going down. */
614 spin_lock(&call_lock);
615 cpus = num_online_cpus() - 1;
616
617 if (!cpus) {
618 spin_unlock(&call_lock);
619 return 0;
620 }
621
622 /* Can deadlock when called with interrupts disabled */
623 WARN_ON(irqs_disabled());
624
625 data.func = func;
626 data.info = info;
627 atomic_set(&data.started, 0);
628 data.wait = wait;
629 if (wait)
630 atomic_set(&data.finished, 0);
631
632 call_data = &data;
633 mb();
634
635 /* Send a message to all other CPUs and wait for them to respond */
636 send_IPI_allbutself(CALL_FUNCTION_IPI, 0);
637
638 /* Wait for response */
639 while (atomic_read(&data.started) != cpus)
640 barrier();
641
642 if (wait)
643 while (atomic_read(&data.finished) != cpus)
644 barrier();
645 spin_unlock(&call_lock);
646 553
647 return 0; 554void arch_send_call_function_single_ipi(int cpu)
555{
556 send_IPI_mask(cpumask_of_cpu(cpu), CALL_FUNC_SINGLE_IPI, 0);
648} 557}
649 558
650/*==========================================================================* 559/*==========================================================================*
@@ -666,27 +575,16 @@ int smp_call_function(void (*func) (void *info), void *info, int nonatomic,
666 *==========================================================================*/ 575 *==========================================================================*/
667void smp_call_function_interrupt(void) 576void smp_call_function_interrupt(void)
668{ 577{
669 void (*func) (void *info) = call_data->func;
670 void *info = call_data->info;
671 int wait = call_data->wait;
672
673 /*
674 * Notify initiating CPU that I've grabbed the data and am
675 * about to execute the function
676 */
677 mb();
678 atomic_inc(&call_data->started);
679 /*
680 * At this point the info structure may be out of scope unless wait==1
681 */
682 irq_enter(); 578 irq_enter();
683 (*func)(info); 579 generic_smp_call_function_interrupt();
684 irq_exit(); 580 irq_exit();
581}
685 582
686 if (wait) { 583void smp_call_function_single_interrupt(void)
687 mb(); 584{
688 atomic_inc(&call_data->finished); 585 irq_enter();
689 } 586 generic_smp_call_function_single_interrupt();
587 irq_exit();
690} 588}
691 589
692/*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ 590/*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
diff --git a/arch/m32r/kernel/traps.c b/arch/m32r/kernel/traps.c
index 89ba4a0b5d51..46159a4e644b 100644
--- a/arch/m32r/kernel/traps.c
+++ b/arch/m32r/kernel/traps.c
@@ -40,6 +40,7 @@ extern void smp_invalidate_interrupt(void);
40extern void smp_call_function_interrupt(void); 40extern void smp_call_function_interrupt(void);
41extern void smp_ipi_timer_interrupt(void); 41extern void smp_ipi_timer_interrupt(void);
42extern void smp_flush_cache_all_interrupt(void); 42extern void smp_flush_cache_all_interrupt(void);
43extern void smp_call_function_single_interrupt(void);
43 44
44/* 45/*
45 * for Boot AP function 46 * for Boot AP function
@@ -103,7 +104,7 @@ void set_eit_vector_entries(void)
103 eit_vector[186] = (unsigned long)smp_call_function_interrupt; 104 eit_vector[186] = (unsigned long)smp_call_function_interrupt;
104 eit_vector[187] = (unsigned long)smp_ipi_timer_interrupt; 105 eit_vector[187] = (unsigned long)smp_ipi_timer_interrupt;
105 eit_vector[188] = (unsigned long)smp_flush_cache_all_interrupt; 106 eit_vector[188] = (unsigned long)smp_flush_cache_all_interrupt;
106 eit_vector[189] = 0; 107 eit_vector[189] = (unsigned long)smp_call_function_single_interrupt;
107 eit_vector[190] = 0; 108 eit_vector[190] = 0;
108 eit_vector[191] = 0; 109 eit_vector[191] = 0;
109#endif 110#endif
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index 24c5dee91768..d2be3ffca280 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -1763,6 +1763,7 @@ config SMP
1763 bool "Multi-Processing support" 1763 bool "Multi-Processing support"
1764 depends on SYS_SUPPORTS_SMP 1764 depends on SYS_SUPPORTS_SMP
1765 select IRQ_PER_CPU 1765 select IRQ_PER_CPU
1766 select USE_GENERIC_SMP_HELPERS
1766 help 1767 help
1767 This enables support for systems with more than one CPU. If you have 1768 This enables support for systems with more than one CPU. If you have
1768 a system with only one CPU, like most personal computers, say N. If 1769 a system with only one CPU, like most personal computers, say N. If
diff --git a/arch/mips/kernel/irq-rm9000.c b/arch/mips/kernel/irq-rm9000.c
index ed9febe63d72..b47e4615ec12 100644
--- a/arch/mips/kernel/irq-rm9000.c
+++ b/arch/mips/kernel/irq-rm9000.c
@@ -49,7 +49,7 @@ static void local_rm9k_perfcounter_irq_startup(void *args)
49 49
50static unsigned int rm9k_perfcounter_irq_startup(unsigned int irq) 50static unsigned int rm9k_perfcounter_irq_startup(unsigned int irq)
51{ 51{
52 on_each_cpu(local_rm9k_perfcounter_irq_startup, (void *) irq, 0, 1); 52 on_each_cpu(local_rm9k_perfcounter_irq_startup, (void *) irq, 1);
53 53
54 return 0; 54 return 0;
55} 55}
@@ -66,7 +66,7 @@ static void local_rm9k_perfcounter_irq_shutdown(void *args)
66 66
67static void rm9k_perfcounter_irq_shutdown(unsigned int irq) 67static void rm9k_perfcounter_irq_shutdown(unsigned int irq)
68{ 68{
69 on_each_cpu(local_rm9k_perfcounter_irq_shutdown, (void *) irq, 0, 1); 69 on_each_cpu(local_rm9k_perfcounter_irq_shutdown, (void *) irq, 1);
70} 70}
71 71
72static struct irq_chip rm9k_irq_controller = { 72static struct irq_chip rm9k_irq_controller = {
diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c
index cdf87a9dd4ba..4410f172b8ab 100644
--- a/arch/mips/kernel/smp.c
+++ b/arch/mips/kernel/smp.c
@@ -131,148 +131,29 @@ asmlinkage __cpuinit void start_secondary(void)
131 cpu_idle(); 131 cpu_idle();
132} 132}
133 133
134DEFINE_SPINLOCK(smp_call_lock); 134void arch_send_call_function_ipi(cpumask_t mask)
135
136struct call_data_struct *call_data;
137
138/*
139 * Run a function on all other CPUs.
140 *
141 * <mask> cpuset_t of all processors to run the function on.
142 * <func> The function to run. This must be fast and non-blocking.
143 * <info> An arbitrary pointer to pass to the function.
144 * <retry> If true, keep retrying until ready.
145 * <wait> If true, wait until function has completed on other CPUs.
146 * [RETURNS] 0 on success, else a negative status code.
147 *
148 * Does not return until remote CPUs are nearly ready to execute <func>
149 * or are or have executed.
150 *
151 * You must not call this function with disabled interrupts or from a
152 * hardware interrupt handler or from a bottom half handler:
153 *
154 * CPU A CPU B
155 * Disable interrupts
156 * smp_call_function()
157 * Take call_lock
158 * Send IPIs
159 * Wait for all cpus to acknowledge IPI
160 * CPU A has not responded, spin waiting
161 * for cpu A to respond, holding call_lock
162 * smp_call_function()
163 * Spin waiting for call_lock
164 * Deadlock Deadlock
165 */
166int smp_call_function_mask(cpumask_t mask, void (*func) (void *info),
167 void *info, int retry, int wait)
168{ 135{
169 struct call_data_struct data;
170 int cpu = smp_processor_id();
171 int cpus;
172
173 /*
174 * Can die spectacularly if this CPU isn't yet marked online
175 */
176 BUG_ON(!cpu_online(cpu));
177
178 cpu_clear(cpu, mask);
179 cpus = cpus_weight(mask);
180 if (!cpus)
181 return 0;
182
183 /* Can deadlock when called with interrupts disabled */
184 WARN_ON(irqs_disabled());
185
186 data.func = func;
187 data.info = info;
188 atomic_set(&data.started, 0);
189 data.wait = wait;
190 if (wait)
191 atomic_set(&data.finished, 0);
192
193 spin_lock(&smp_call_lock);
194 call_data = &data;
195 smp_mb();
196
197 /* Send a message to all other CPUs and wait for them to respond */
198 mp_ops->send_ipi_mask(mask, SMP_CALL_FUNCTION); 136 mp_ops->send_ipi_mask(mask, SMP_CALL_FUNCTION);
199
200 /* Wait for response */
201 /* FIXME: lock-up detection, backtrace on lock-up */
202 while (atomic_read(&data.started) != cpus)
203 barrier();
204
205 if (wait)
206 while (atomic_read(&data.finished) != cpus)
207 barrier();
208 call_data = NULL;
209 spin_unlock(&smp_call_lock);
210
211 return 0;
212} 137}
213 138
214int smp_call_function(void (*func) (void *info), void *info, int retry, 139/*
215 int wait) 140 * We reuse the same vector for the single IPI
141 */
142void arch_send_call_function_single_ipi(int cpu)
216{ 143{
217 return smp_call_function_mask(cpu_online_map, func, info, retry, wait); 144 mp_ops->send_ipi_mask(cpumask_of_cpu(cpu), SMP_CALL_FUNCTION);
218} 145}
219EXPORT_SYMBOL(smp_call_function);
220 146
147/*
148 * Call into both interrupt handlers, as we share the IPI for them
149 */
221void smp_call_function_interrupt(void) 150void smp_call_function_interrupt(void)
222{ 151{
223 void (*func) (void *info) = call_data->func;
224 void *info = call_data->info;
225 int wait = call_data->wait;
226
227 /*
228 * Notify initiating CPU that I've grabbed the data and am
229 * about to execute the function.
230 */
231 smp_mb();
232 atomic_inc(&call_data->started);
233
234 /*
235 * At this point the info structure may be out of scope unless wait==1.
236 */
237 irq_enter(); 152 irq_enter();
238 (*func)(info); 153 generic_smp_call_function_single_interrupt();
154 generic_smp_call_function_interrupt();
239 irq_exit(); 155 irq_exit();
240
241 if (wait) {
242 smp_mb();
243 atomic_inc(&call_data->finished);
244 }
245}
246
247int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
248 int retry, int wait)
249{
250 int ret, me;
251
252 /*
253 * Can die spectacularly if this CPU isn't yet marked online
254 */
255 if (!cpu_online(cpu))
256 return 0;
257
258 me = get_cpu();
259 BUG_ON(!cpu_online(me));
260
261 if (cpu == me) {
262 local_irq_disable();
263 func(info);
264 local_irq_enable();
265 put_cpu();
266 return 0;
267 }
268
269 ret = smp_call_function_mask(cpumask_of_cpu(cpu), func, info, retry,
270 wait);
271
272 put_cpu();
273 return 0;
274} 156}
275EXPORT_SYMBOL(smp_call_function_single);
276 157
277static void stop_this_cpu(void *dummy) 158static void stop_this_cpu(void *dummy)
278{ 159{
@@ -286,7 +167,7 @@ static void stop_this_cpu(void *dummy)
286 167
287void smp_send_stop(void) 168void smp_send_stop(void)
288{ 169{
289 smp_call_function(stop_this_cpu, NULL, 1, 0); 170 smp_call_function(stop_this_cpu, NULL, 0);
290} 171}
291 172
292void __init smp_cpus_done(unsigned int max_cpus) 173void __init smp_cpus_done(unsigned int max_cpus)
@@ -365,7 +246,7 @@ static void flush_tlb_all_ipi(void *info)
365 246
366void flush_tlb_all(void) 247void flush_tlb_all(void)
367{ 248{
368 on_each_cpu(flush_tlb_all_ipi, NULL, 1, 1); 249 on_each_cpu(flush_tlb_all_ipi, NULL, 1);
369} 250}
370 251
371static void flush_tlb_mm_ipi(void *mm) 252static void flush_tlb_mm_ipi(void *mm)
@@ -385,7 +266,7 @@ static void flush_tlb_mm_ipi(void *mm)
385static inline void smp_on_other_tlbs(void (*func) (void *info), void *info) 266static inline void smp_on_other_tlbs(void (*func) (void *info), void *info)
386{ 267{
387#ifndef CONFIG_MIPS_MT_SMTC 268#ifndef CONFIG_MIPS_MT_SMTC
388 smp_call_function(func, info, 1, 1); 269 smp_call_function(func, info, 1);
389#endif 270#endif
390} 271}
391 272
@@ -485,7 +366,7 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
485 .addr2 = end, 366 .addr2 = end,
486 }; 367 };
487 368
488 on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1, 1); 369 on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1);
489} 370}
490 371
491static void flush_tlb_page_ipi(void *info) 372static void flush_tlb_page_ipi(void *info)
diff --git a/arch/mips/kernel/smtc.c b/arch/mips/kernel/smtc.c
index 3e863186cd22..a516286532ab 100644
--- a/arch/mips/kernel/smtc.c
+++ b/arch/mips/kernel/smtc.c
@@ -877,7 +877,6 @@ static void ipi_resched_interrupt(void)
877 /* Return from interrupt should be enough to cause scheduler check */ 877 /* Return from interrupt should be enough to cause scheduler check */
878} 878}
879 879
880
881static void ipi_call_interrupt(void) 880static void ipi_call_interrupt(void)
882{ 881{
883 /* Invoke generic function invocation code in smp.c */ 882 /* Invoke generic function invocation code in smp.c */
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
index 27096751ddce..71df3390c07b 100644
--- a/arch/mips/mm/c-r4k.c
+++ b/arch/mips/mm/c-r4k.c
@@ -43,12 +43,12 @@
43 * primary cache. 43 * primary cache.
44 */ 44 */
45static inline void r4k_on_each_cpu(void (*func) (void *info), void *info, 45static inline void r4k_on_each_cpu(void (*func) (void *info), void *info,
46 int retry, int wait) 46 int wait)
47{ 47{
48 preempt_disable(); 48 preempt_disable();
49 49
50#if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC) 50#if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC)
51 smp_call_function(func, info, retry, wait); 51 smp_call_function(func, info, wait);
52#endif 52#endif
53 func(info); 53 func(info);
54 preempt_enable(); 54 preempt_enable();
@@ -350,7 +350,7 @@ static inline void local_r4k___flush_cache_all(void * args)
350 350
351static void r4k___flush_cache_all(void) 351static void r4k___flush_cache_all(void)
352{ 352{
353 r4k_on_each_cpu(local_r4k___flush_cache_all, NULL, 1, 1); 353 r4k_on_each_cpu(local_r4k___flush_cache_all, NULL, 1);
354} 354}
355 355
356static inline int has_valid_asid(const struct mm_struct *mm) 356static inline int has_valid_asid(const struct mm_struct *mm)
@@ -397,7 +397,7 @@ static void r4k_flush_cache_range(struct vm_area_struct *vma,
397 int exec = vma->vm_flags & VM_EXEC; 397 int exec = vma->vm_flags & VM_EXEC;
398 398
399 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) 399 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc))
400 r4k_on_each_cpu(local_r4k_flush_cache_range, vma, 1, 1); 400 r4k_on_each_cpu(local_r4k_flush_cache_range, vma, 1);
401} 401}
402 402
403static inline void local_r4k_flush_cache_mm(void * args) 403static inline void local_r4k_flush_cache_mm(void * args)
@@ -429,7 +429,7 @@ static void r4k_flush_cache_mm(struct mm_struct *mm)
429 if (!cpu_has_dc_aliases) 429 if (!cpu_has_dc_aliases)
430 return; 430 return;
431 431
432 r4k_on_each_cpu(local_r4k_flush_cache_mm, mm, 1, 1); 432 r4k_on_each_cpu(local_r4k_flush_cache_mm, mm, 1);
433} 433}
434 434
435struct flush_cache_page_args { 435struct flush_cache_page_args {
@@ -521,7 +521,7 @@ static void r4k_flush_cache_page(struct vm_area_struct *vma,
521 args.addr = addr; 521 args.addr = addr;
522 args.pfn = pfn; 522 args.pfn = pfn;
523 523
524 r4k_on_each_cpu(local_r4k_flush_cache_page, &args, 1, 1); 524 r4k_on_each_cpu(local_r4k_flush_cache_page, &args, 1);
525} 525}
526 526
527static inline void local_r4k_flush_data_cache_page(void * addr) 527static inline void local_r4k_flush_data_cache_page(void * addr)
@@ -535,7 +535,7 @@ static void r4k_flush_data_cache_page(unsigned long addr)
535 local_r4k_flush_data_cache_page((void *)addr); 535 local_r4k_flush_data_cache_page((void *)addr);
536 else 536 else
537 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr, 537 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr,
538 1, 1); 538 1);
539} 539}
540 540
541struct flush_icache_range_args { 541struct flush_icache_range_args {
@@ -571,7 +571,7 @@ static void r4k_flush_icache_range(unsigned long start, unsigned long end)
571 args.start = start; 571 args.start = start;
572 args.end = end; 572 args.end = end;
573 573
574 r4k_on_each_cpu(local_r4k_flush_icache_range, &args, 1, 1); 574 r4k_on_each_cpu(local_r4k_flush_icache_range, &args, 1);
575 instruction_hazard(); 575 instruction_hazard();
576} 576}
577 577
@@ -672,7 +672,7 @@ static void local_r4k_flush_cache_sigtramp(void * arg)
672 672
673static void r4k_flush_cache_sigtramp(unsigned long addr) 673static void r4k_flush_cache_sigtramp(unsigned long addr)
674{ 674{
675 r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr, 1, 1); 675 r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr, 1);
676} 676}
677 677
678static void r4k_flush_icache_all(void) 678static void r4k_flush_icache_all(void)
diff --git a/arch/mips/oprofile/common.c b/arch/mips/oprofile/common.c
index b5f6f71b27bc..dd2fbd6645c1 100644
--- a/arch/mips/oprofile/common.c
+++ b/arch/mips/oprofile/common.c
@@ -27,7 +27,7 @@ static int op_mips_setup(void)
27 model->reg_setup(ctr); 27 model->reg_setup(ctr);
28 28
29 /* Configure the registers on all cpus. */ 29 /* Configure the registers on all cpus. */
30 on_each_cpu(model->cpu_setup, NULL, 0, 1); 30 on_each_cpu(model->cpu_setup, NULL, 1);
31 31
32 return 0; 32 return 0;
33} 33}
@@ -58,7 +58,7 @@ static int op_mips_create_files(struct super_block * sb, struct dentry * root)
58 58
59static int op_mips_start(void) 59static int op_mips_start(void)
60{ 60{
61 on_each_cpu(model->cpu_start, NULL, 0, 1); 61 on_each_cpu(model->cpu_start, NULL, 1);
62 62
63 return 0; 63 return 0;
64} 64}
@@ -66,7 +66,7 @@ static int op_mips_start(void)
66static void op_mips_stop(void) 66static void op_mips_stop(void)
67{ 67{
68 /* Disable performance monitoring for all counters. */ 68 /* Disable performance monitoring for all counters. */
69 on_each_cpu(model->cpu_stop, NULL, 0, 1); 69 on_each_cpu(model->cpu_stop, NULL, 1);
70} 70}
71 71
72int __init oprofile_arch_init(struct oprofile_operations *ops) 72int __init oprofile_arch_init(struct oprofile_operations *ops)
diff --git a/arch/mips/pmc-sierra/yosemite/prom.c b/arch/mips/pmc-sierra/yosemite/prom.c
index 35dc435846a6..cf4c868715ac 100644
--- a/arch/mips/pmc-sierra/yosemite/prom.c
+++ b/arch/mips/pmc-sierra/yosemite/prom.c
@@ -64,7 +64,7 @@ static void prom_exit(void)
64#ifdef CONFIG_SMP 64#ifdef CONFIG_SMP
65 if (smp_processor_id()) 65 if (smp_processor_id())
66 /* CPU 1 */ 66 /* CPU 1 */
67 smp_call_function(prom_cpu0_exit, NULL, 1, 1); 67 smp_call_function(prom_cpu0_exit, NULL, 1);
68#endif 68#endif
69 prom_cpu0_exit(NULL); 69 prom_cpu0_exit(NULL);
70} 70}
diff --git a/arch/mips/sibyte/cfe/setup.c b/arch/mips/sibyte/cfe/setup.c
index 33fce826f8bf..fd9604d5555a 100644
--- a/arch/mips/sibyte/cfe/setup.c
+++ b/arch/mips/sibyte/cfe/setup.c
@@ -74,7 +74,7 @@ static void __noreturn cfe_linux_exit(void *arg)
74 if (!reboot_smp) { 74 if (!reboot_smp) {
75 /* Get CPU 0 to do the cfe_exit */ 75 /* Get CPU 0 to do the cfe_exit */
76 reboot_smp = 1; 76 reboot_smp = 1;
77 smp_call_function(cfe_linux_exit, arg, 1, 0); 77 smp_call_function(cfe_linux_exit, arg, 0);
78 } 78 }
79 } else { 79 } else {
80 printk("Passing control back to CFE...\n"); 80 printk("Passing control back to CFE...\n");
diff --git a/arch/mips/sibyte/sb1250/prom.c b/arch/mips/sibyte/sb1250/prom.c
index cf8f6b3de86c..65b1af66b674 100644
--- a/arch/mips/sibyte/sb1250/prom.c
+++ b/arch/mips/sibyte/sb1250/prom.c
@@ -66,7 +66,7 @@ static void prom_linux_exit(void)
66{ 66{
67#ifdef CONFIG_SMP 67#ifdef CONFIG_SMP
68 if (smp_processor_id()) { 68 if (smp_processor_id()) {
69 smp_call_function(prom_cpu0_exit, NULL, 1, 1); 69 smp_call_function(prom_cpu0_exit, NULL, 1);
70 } 70 }
71#endif 71#endif
72 while(1); 72 while(1);
diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig
index bc7a19da6245..a7d4fd353c2b 100644
--- a/arch/parisc/Kconfig
+++ b/arch/parisc/Kconfig
@@ -199,6 +199,7 @@ endchoice
199 199
200config SMP 200config SMP
201 bool "Symmetric multi-processing support" 201 bool "Symmetric multi-processing support"
202 select USE_GENERIC_SMP_HELPERS
202 ---help--- 203 ---help---
203 This enables support for systems with more than one CPU. If you have 204 This enables support for systems with more than one CPU. If you have
204 a system with only one CPU, like most personal computers, say N. If 205 a system with only one CPU, like most personal computers, say N. If
diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c
index e10d25d2d9c9..5259d8c20676 100644
--- a/arch/parisc/kernel/cache.c
+++ b/arch/parisc/kernel/cache.c
@@ -51,12 +51,12 @@ static struct pdc_btlb_info btlb_info __read_mostly;
51void 51void
52flush_data_cache(void) 52flush_data_cache(void)
53{ 53{
54 on_each_cpu(flush_data_cache_local, NULL, 1, 1); 54 on_each_cpu(flush_data_cache_local, NULL, 1);
55} 55}
56void 56void
57flush_instruction_cache(void) 57flush_instruction_cache(void)
58{ 58{
59 on_each_cpu(flush_instruction_cache_local, NULL, 1, 1); 59 on_each_cpu(flush_instruction_cache_local, NULL, 1);
60} 60}
61#endif 61#endif
62 62
@@ -515,7 +515,7 @@ static void cacheflush_h_tmp_function(void *dummy)
515 515
516void flush_cache_all(void) 516void flush_cache_all(void)
517{ 517{
518 on_each_cpu(cacheflush_h_tmp_function, NULL, 1, 1); 518 on_each_cpu(cacheflush_h_tmp_function, NULL, 1);
519} 519}
520 520
521void flush_cache_mm(struct mm_struct *mm) 521void flush_cache_mm(struct mm_struct *mm)
diff --git a/arch/parisc/kernel/smp.c b/arch/parisc/kernel/smp.c
index 85fc7754ec25..d47f3975c9c6 100644
--- a/arch/parisc/kernel/smp.c
+++ b/arch/parisc/kernel/smp.c
@@ -84,19 +84,11 @@ EXPORT_SYMBOL(cpu_possible_map);
84 84
85DEFINE_PER_CPU(spinlock_t, ipi_lock) = SPIN_LOCK_UNLOCKED; 85DEFINE_PER_CPU(spinlock_t, ipi_lock) = SPIN_LOCK_UNLOCKED;
86 86
87struct smp_call_struct {
88 void (*func) (void *info);
89 void *info;
90 long wait;
91 atomic_t unstarted_count;
92 atomic_t unfinished_count;
93};
94static volatile struct smp_call_struct *smp_call_function_data;
95
96enum ipi_message_type { 87enum ipi_message_type {
97 IPI_NOP=0, 88 IPI_NOP=0,
98 IPI_RESCHEDULE=1, 89 IPI_RESCHEDULE=1,
99 IPI_CALL_FUNC, 90 IPI_CALL_FUNC,
91 IPI_CALL_FUNC_SINGLE,
100 IPI_CPU_START, 92 IPI_CPU_START,
101 IPI_CPU_STOP, 93 IPI_CPU_STOP,
102 IPI_CPU_TEST 94 IPI_CPU_TEST
@@ -187,33 +179,12 @@ ipi_interrupt(int irq, void *dev_id)
187 179
188 case IPI_CALL_FUNC: 180 case IPI_CALL_FUNC:
189 smp_debug(100, KERN_DEBUG "CPU%d IPI_CALL_FUNC\n", this_cpu); 181 smp_debug(100, KERN_DEBUG "CPU%d IPI_CALL_FUNC\n", this_cpu);
190 { 182 generic_smp_call_function_interrupt();
191 volatile struct smp_call_struct *data; 183 break;
192 void (*func)(void *info); 184
193 void *info; 185 case IPI_CALL_FUNC_SINGLE:
194 int wait; 186 smp_debug(100, KERN_DEBUG "CPU%d IPI_CALL_FUNC_SINGLE\n", this_cpu);
195 187 generic_smp_call_function_single_interrupt();
196 data = smp_call_function_data;
197 func = data->func;
198 info = data->info;
199 wait = data->wait;
200
201 mb();
202 atomic_dec ((atomic_t *)&data->unstarted_count);
203
204 /* At this point, *data can't
205 * be relied upon.
206 */
207
208 (*func)(info);
209
210 /* Notify the sending CPU that the
211 * task is done.
212 */
213 mb();
214 if (wait)
215 atomic_dec ((atomic_t *)&data->unfinished_count);
216 }
217 break; 188 break;
218 189
219 case IPI_CPU_START: 190 case IPI_CPU_START:
@@ -256,6 +227,14 @@ ipi_send(int cpu, enum ipi_message_type op)
256 spin_unlock_irqrestore(lock, flags); 227 spin_unlock_irqrestore(lock, flags);
257} 228}
258 229
230static void
231send_IPI_mask(cpumask_t mask, enum ipi_message_type op)
232{
233 int cpu;
234
235 for_each_cpu_mask(cpu, mask)
236 ipi_send(cpu, op);
237}
259 238
260static inline void 239static inline void
261send_IPI_single(int dest_cpu, enum ipi_message_type op) 240send_IPI_single(int dest_cpu, enum ipi_message_type op)
@@ -295,86 +274,15 @@ smp_send_all_nop(void)
295 send_IPI_allbutself(IPI_NOP); 274 send_IPI_allbutself(IPI_NOP);
296} 275}
297 276
298 277void arch_send_call_function_ipi(cpumask_t mask)
299/**
300 * Run a function on all other CPUs.
301 * <func> The function to run. This must be fast and non-blocking.
302 * <info> An arbitrary pointer to pass to the function.
303 * <retry> If true, keep retrying until ready.
304 * <wait> If true, wait until function has completed on other CPUs.
305 * [RETURNS] 0 on success, else a negative status code.
306 *
307 * Does not return until remote CPUs are nearly ready to execute <func>
308 * or have executed.
309 */
310
311int
312smp_call_function (void (*func) (void *info), void *info, int retry, int wait)
313{ 278{
314 struct smp_call_struct data; 279 send_IPI_mask(mask, IPI_CALL_FUNC);
315 unsigned long timeout;
316 static DEFINE_SPINLOCK(lock);
317 int retries = 0;
318
319 if (num_online_cpus() < 2)
320 return 0;
321
322 /* Can deadlock when called with interrupts disabled */
323 WARN_ON(irqs_disabled());
324
325 /* can also deadlock if IPIs are disabled */
326 WARN_ON((get_eiem() & (1UL<<(CPU_IRQ_MAX - IPI_IRQ))) == 0);
327
328
329 data.func = func;
330 data.info = info;
331 data.wait = wait;
332 atomic_set(&data.unstarted_count, num_online_cpus() - 1);
333 atomic_set(&data.unfinished_count, num_online_cpus() - 1);
334
335 if (retry) {
336 spin_lock (&lock);
337 while (smp_call_function_data != 0)
338 barrier();
339 }
340 else {
341 spin_lock (&lock);
342 if (smp_call_function_data) {
343 spin_unlock (&lock);
344 return -EBUSY;
345 }
346 }
347
348 smp_call_function_data = &data;
349 spin_unlock (&lock);
350
351 /* Send a message to all other CPUs and wait for them to respond */
352 send_IPI_allbutself(IPI_CALL_FUNC);
353
354 retry:
355 /* Wait for response */
356 timeout = jiffies + HZ;
357 while ( (atomic_read (&data.unstarted_count) > 0) &&
358 time_before (jiffies, timeout) )
359 barrier ();
360
361 if (atomic_read (&data.unstarted_count) > 0) {
362 printk(KERN_CRIT "SMP CALL FUNCTION TIMED OUT! (cpu=%d), try %d\n",
363 smp_processor_id(), ++retries);
364 goto retry;
365 }
366 /* We either got one or timed out. Release the lock */
367
368 mb();
369 smp_call_function_data = NULL;
370
371 while (wait && atomic_read (&data.unfinished_count) > 0)
372 barrier ();
373
374 return 0;
375} 280}
376 281
377EXPORT_SYMBOL(smp_call_function); 282void arch_send_call_function_single_ipi(int cpu)
283{
284 send_IPI_single(cpu, IPI_CALL_FUNC_SINGLE);
285}
378 286
379/* 287/*
380 * Flush all other CPU's tlb and then mine. Do this with on_each_cpu() 288 * Flush all other CPU's tlb and then mine. Do this with on_each_cpu()
@@ -384,7 +292,7 @@ EXPORT_SYMBOL(smp_call_function);
384void 292void
385smp_flush_tlb_all(void) 293smp_flush_tlb_all(void)
386{ 294{
387 on_each_cpu(flush_tlb_all_local, NULL, 1, 1); 295 on_each_cpu(flush_tlb_all_local, NULL, 1);
388} 296}
389 297
390/* 298/*
diff --git a/arch/parisc/mm/init.c b/arch/parisc/mm/init.c
index ce0da689a89d..b4d6c8777ed0 100644
--- a/arch/parisc/mm/init.c
+++ b/arch/parisc/mm/init.c
@@ -1053,7 +1053,7 @@ void flush_tlb_all(void)
1053 do_recycle++; 1053 do_recycle++;
1054 } 1054 }
1055 spin_unlock(&sid_lock); 1055 spin_unlock(&sid_lock);
1056 on_each_cpu(flush_tlb_all_local, NULL, 1, 1); 1056 on_each_cpu(flush_tlb_all_local, NULL, 1);
1057 if (do_recycle) { 1057 if (do_recycle) {
1058 spin_lock(&sid_lock); 1058 spin_lock(&sid_lock);
1059 recycle_sids(recycle_ndirty,recycle_dirty_array); 1059 recycle_sids(recycle_ndirty,recycle_dirty_array);
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index a5e9912e2d37..20eacf2a8424 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -111,6 +111,7 @@ config PPC
111 select HAVE_KPROBES 111 select HAVE_KPROBES
112 select HAVE_KRETPROBES 112 select HAVE_KRETPROBES
113 select HAVE_LMB 113 select HAVE_LMB
114 select USE_GENERIC_SMP_HELPERS if SMP
114 select HAVE_OPROFILE 115 select HAVE_OPROFILE
115 116
116config EARLY_PRINTK 117config EARLY_PRINTK
diff --git a/arch/powerpc/kernel/machine_kexec_64.c b/arch/powerpc/kernel/machine_kexec_64.c
index 704375bda73a..b732b5f8e356 100644
--- a/arch/powerpc/kernel/machine_kexec_64.c
+++ b/arch/powerpc/kernel/machine_kexec_64.c
@@ -172,7 +172,7 @@ static void kexec_prepare_cpus(void)
172{ 172{
173 int my_cpu, i, notified=-1; 173 int my_cpu, i, notified=-1;
174 174
175 smp_call_function(kexec_smp_down, NULL, 0, /* wait */0); 175 smp_call_function(kexec_smp_down, NULL, /* wait */0);
176 my_cpu = get_cpu(); 176 my_cpu = get_cpu();
177 177
178 /* check the others cpus are now down (via paca hw cpu id == -1) */ 178 /* check the others cpus are now down (via paca hw cpu id == -1) */
diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c
index 34843c318419..647f3e8677dc 100644
--- a/arch/powerpc/kernel/rtas.c
+++ b/arch/powerpc/kernel/rtas.c
@@ -747,7 +747,7 @@ static int rtas_ibm_suspend_me(struct rtas_args *args)
747 /* Call function on all CPUs. One of us will make the 747 /* Call function on all CPUs. One of us will make the
748 * rtas call 748 * rtas call
749 */ 749 */
750 if (on_each_cpu(rtas_percpu_suspend_me, &data, 1, 0)) 750 if (on_each_cpu(rtas_percpu_suspend_me, &data, 0))
751 data.error = -EINVAL; 751 data.error = -EINVAL;
752 752
753 wait_for_completion(&done); 753 wait_for_completion(&done);
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index 1457aa0a08f1..5191b46a611e 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -72,12 +72,8 @@ struct smp_ops_t *smp_ops;
72 72
73static volatile unsigned int cpu_callin_map[NR_CPUS]; 73static volatile unsigned int cpu_callin_map[NR_CPUS];
74 74
75void smp_call_function_interrupt(void);
76
77int smt_enabled_at_boot = 1; 75int smt_enabled_at_boot = 1;
78 76
79static int ipi_fail_ok;
80
81static void (*crash_ipi_function_ptr)(struct pt_regs *) = NULL; 77static void (*crash_ipi_function_ptr)(struct pt_regs *) = NULL;
82 78
83#ifdef CONFIG_PPC64 79#ifdef CONFIG_PPC64
@@ -99,12 +95,15 @@ void smp_message_recv(int msg)
99{ 95{
100 switch(msg) { 96 switch(msg) {
101 case PPC_MSG_CALL_FUNCTION: 97 case PPC_MSG_CALL_FUNCTION:
102 smp_call_function_interrupt(); 98 generic_smp_call_function_interrupt();
103 break; 99 break;
104 case PPC_MSG_RESCHEDULE: 100 case PPC_MSG_RESCHEDULE:
105 /* XXX Do we have to do this? */ 101 /* XXX Do we have to do this? */
106 set_need_resched(); 102 set_need_resched();
107 break; 103 break;
104 case PPC_MSG_CALL_FUNC_SINGLE:
105 generic_smp_call_function_single_interrupt();
106 break;
108 case PPC_MSG_DEBUGGER_BREAK: 107 case PPC_MSG_DEBUGGER_BREAK:
109 if (crash_ipi_function_ptr) { 108 if (crash_ipi_function_ptr) {
110 crash_ipi_function_ptr(get_irq_regs()); 109 crash_ipi_function_ptr(get_irq_regs());
@@ -128,6 +127,19 @@ void smp_send_reschedule(int cpu)
128 smp_ops->message_pass(cpu, PPC_MSG_RESCHEDULE); 127 smp_ops->message_pass(cpu, PPC_MSG_RESCHEDULE);
129} 128}
130 129
130void arch_send_call_function_single_ipi(int cpu)
131{
132 smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNC_SINGLE);
133}
134
135void arch_send_call_function_ipi(cpumask_t mask)
136{
137 unsigned int cpu;
138
139 for_each_cpu_mask(cpu, mask)
140 smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNCTION);
141}
142
131#ifdef CONFIG_DEBUGGER 143#ifdef CONFIG_DEBUGGER
132void smp_send_debugger_break(int cpu) 144void smp_send_debugger_break(int cpu)
133{ 145{
@@ -154,215 +166,9 @@ static void stop_this_cpu(void *dummy)
154 ; 166 ;
155} 167}
156 168
157/*
158 * Structure and data for smp_call_function(). This is designed to minimise
159 * static memory requirements. It also looks cleaner.
160 * Stolen from the i386 version.
161 */
162static __cacheline_aligned_in_smp DEFINE_SPINLOCK(call_lock);
163
164static struct call_data_struct {
165 void (*func) (void *info);
166 void *info;
167 atomic_t started;
168 atomic_t finished;
169 int wait;
170} *call_data;
171
172/* delay of at least 8 seconds */
173#define SMP_CALL_TIMEOUT 8
174
175/*
176 * These functions send a 'generic call function' IPI to other online
177 * CPUS in the system.
178 *
179 * [SUMMARY] Run a function on other CPUs.
180 * <func> The function to run. This must be fast and non-blocking.
181 * <info> An arbitrary pointer to pass to the function.
182 * <nonatomic> currently unused.
183 * <wait> If true, wait (atomically) until function has completed on other CPUs.
184 * [RETURNS] 0 on success, else a negative status code. Does not return until
185 * remote CPUs are nearly ready to execute <<func>> or are or have executed.
186 * <map> is a cpu map of the cpus to send IPI to.
187 *
188 * You must not call this function with disabled interrupts or from a
189 * hardware interrupt handler or from a bottom half handler.
190 */
191static int __smp_call_function_map(void (*func) (void *info), void *info,
192 int nonatomic, int wait, cpumask_t map)
193{
194 struct call_data_struct data;
195 int ret = -1, num_cpus;
196 int cpu;
197 u64 timeout;
198
199 if (unlikely(smp_ops == NULL))
200 return ret;
201
202 data.func = func;
203 data.info = info;
204 atomic_set(&data.started, 0);
205 data.wait = wait;
206 if (wait)
207 atomic_set(&data.finished, 0);
208
209 /* remove 'self' from the map */
210 if (cpu_isset(smp_processor_id(), map))
211 cpu_clear(smp_processor_id(), map);
212
213 /* sanity check the map, remove any non-online processors. */
214 cpus_and(map, map, cpu_online_map);
215
216 num_cpus = cpus_weight(map);
217 if (!num_cpus)
218 goto done;
219
220 call_data = &data;
221 smp_wmb();
222 /* Send a message to all CPUs in the map */
223 for_each_cpu_mask(cpu, map)
224 smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNCTION);
225
226 timeout = get_tb() + (u64) SMP_CALL_TIMEOUT * tb_ticks_per_sec;
227
228 /* Wait for indication that they have received the message */
229 while (atomic_read(&data.started) != num_cpus) {
230 HMT_low();
231 if (get_tb() >= timeout) {
232 printk("smp_call_function on cpu %d: other cpus not "
233 "responding (%d)\n", smp_processor_id(),
234 atomic_read(&data.started));
235 if (!ipi_fail_ok)
236 debugger(NULL);
237 goto out;
238 }
239 }
240
241 /* optionally wait for the CPUs to complete */
242 if (wait) {
243 while (atomic_read(&data.finished) != num_cpus) {
244 HMT_low();
245 if (get_tb() >= timeout) {
246 printk("smp_call_function on cpu %d: other "
247 "cpus not finishing (%d/%d)\n",
248 smp_processor_id(),
249 atomic_read(&data.finished),
250 atomic_read(&data.started));
251 debugger(NULL);
252 goto out;
253 }
254 }
255 }
256
257 done:
258 ret = 0;
259
260 out:
261 call_data = NULL;
262 HMT_medium();
263 return ret;
264}
265
266static int __smp_call_function(void (*func)(void *info), void *info,
267 int nonatomic, int wait)
268{
269 int ret;
270 spin_lock(&call_lock);
271 ret =__smp_call_function_map(func, info, nonatomic, wait,
272 cpu_online_map);
273 spin_unlock(&call_lock);
274 return ret;
275}
276
277int smp_call_function(void (*func) (void *info), void *info, int nonatomic,
278 int wait)
279{
280 /* Can deadlock when called with interrupts disabled */
281 WARN_ON(irqs_disabled());
282
283 return __smp_call_function(func, info, nonatomic, wait);
284}
285EXPORT_SYMBOL(smp_call_function);
286
287int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
288 int nonatomic, int wait)
289{
290 cpumask_t map = CPU_MASK_NONE;
291 int ret = 0;
292
293 /* Can deadlock when called with interrupts disabled */
294 WARN_ON(irqs_disabled());
295
296 if (!cpu_online(cpu))
297 return -EINVAL;
298
299 cpu_set(cpu, map);
300 if (cpu != get_cpu()) {
301 spin_lock(&call_lock);
302 ret = __smp_call_function_map(func, info, nonatomic, wait, map);
303 spin_unlock(&call_lock);
304 } else {
305 local_irq_disable();
306 func(info);
307 local_irq_enable();
308 }
309 put_cpu();
310 return ret;
311}
312EXPORT_SYMBOL(smp_call_function_single);
313
314void smp_send_stop(void) 169void smp_send_stop(void)
315{ 170{
316 int nolock; 171 smp_call_function(stop_this_cpu, NULL, 0);
317
318 /* It's OK to fail sending the IPI, since the alternative is to
319 * be stuck forever waiting on the other CPU to take the interrupt.
320 *
321 * It's better to at least continue and go through reboot, since this
322 * function is usually called at panic or reboot time in the first
323 * place.
324 */
325 ipi_fail_ok = 1;
326
327 /* Don't deadlock in case we got called through panic */
328 nolock = !spin_trylock(&call_lock);
329 __smp_call_function_map(stop_this_cpu, NULL, 1, 0, cpu_online_map);
330 if (!nolock)
331 spin_unlock(&call_lock);
332}
333
334void smp_call_function_interrupt(void)
335{
336 void (*func) (void *info);
337 void *info;
338 int wait;
339
340 /* call_data will be NULL if the sender timed out while
341 * waiting on us to receive the call.
342 */
343 if (!call_data)
344 return;
345
346 func = call_data->func;
347 info = call_data->info;
348 wait = call_data->wait;
349
350 if (!wait)
351 smp_mb__before_atomic_inc();
352
353 /*
354 * Notify initiating CPU that I've grabbed the data and am
355 * about to execute the function
356 */
357 atomic_inc(&call_data->started);
358 /*
359 * At this point the info structure may be out of scope unless wait==1
360 */
361 (*func)(info);
362 if (wait) {
363 smp_mb__before_atomic_inc();
364 atomic_inc(&call_data->finished);
365 }
366} 172}
367 173
368extern struct gettimeofday_struct do_gtod; 174extern struct gettimeofday_struct do_gtod;
@@ -596,9 +402,9 @@ int __devinit start_secondary(void *unused)
596 402
597 secondary_cpu_time_init(); 403 secondary_cpu_time_init();
598 404
599 spin_lock(&call_lock); 405 ipi_call_lock();
600 cpu_set(cpu, cpu_online_map); 406 cpu_set(cpu, cpu_online_map);
601 spin_unlock(&call_lock); 407 ipi_call_unlock();
602 408
603 local_irq_enable(); 409 local_irq_enable();
604 410
diff --git a/arch/powerpc/kernel/tau_6xx.c b/arch/powerpc/kernel/tau_6xx.c
index 368a4934f7ee..c3a56d65c5a9 100644
--- a/arch/powerpc/kernel/tau_6xx.c
+++ b/arch/powerpc/kernel/tau_6xx.c
@@ -192,7 +192,7 @@ static void tau_timeout_smp(unsigned long unused)
192 192
193 /* schedule ourselves to be run again */ 193 /* schedule ourselves to be run again */
194 mod_timer(&tau_timer, jiffies + shrink_timer) ; 194 mod_timer(&tau_timer, jiffies + shrink_timer) ;
195 on_each_cpu(tau_timeout, NULL, 1, 0); 195 on_each_cpu(tau_timeout, NULL, 0);
196} 196}
197 197
198/* 198/*
@@ -234,7 +234,7 @@ int __init TAU_init(void)
234 tau_timer.expires = jiffies + shrink_timer; 234 tau_timer.expires = jiffies + shrink_timer;
235 add_timer(&tau_timer); 235 add_timer(&tau_timer);
236 236
237 on_each_cpu(TAU_init_smp, NULL, 1, 0); 237 on_each_cpu(TAU_init_smp, NULL, 0);
238 238
239 printk("Thermal assist unit "); 239 printk("Thermal assist unit ");
240#ifdef CONFIG_TAU_INT 240#ifdef CONFIG_TAU_INT
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index 73401e83739a..f1a38a6c1e2d 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -322,7 +322,7 @@ void snapshot_timebases(void)
322{ 322{
323 if (!cpu_has_feature(CPU_FTR_PURR)) 323 if (!cpu_has_feature(CPU_FTR_PURR))
324 return; 324 return;
325 on_each_cpu(snapshot_tb_and_purr, NULL, 0, 1); 325 on_each_cpu(snapshot_tb_and_purr, NULL, 1);
326} 326}
327 327
328/* 328/*
diff --git a/arch/powerpc/mm/slice.c b/arch/powerpc/mm/slice.c
index ad928edafb0a..2bd12d965db1 100644
--- a/arch/powerpc/mm/slice.c
+++ b/arch/powerpc/mm/slice.c
@@ -218,7 +218,7 @@ static void slice_convert(struct mm_struct *mm, struct slice_mask mask, int psiz
218 mb(); 218 mb();
219 219
220 /* XXX this is sub-optimal but will do for now */ 220 /* XXX this is sub-optimal but will do for now */
221 on_each_cpu(slice_flush_segments, mm, 0, 1); 221 on_each_cpu(slice_flush_segments, mm, 1);
222#ifdef CONFIG_SPU_BASE 222#ifdef CONFIG_SPU_BASE
223 spu_flush_all_slbs(mm); 223 spu_flush_all_slbs(mm);
224#endif 224#endif
diff --git a/arch/powerpc/mm/tlb_64.c b/arch/powerpc/mm/tlb_64.c
index e2d867ce1c7e..69ad829a7fa3 100644
--- a/arch/powerpc/mm/tlb_64.c
+++ b/arch/powerpc/mm/tlb_64.c
@@ -66,7 +66,7 @@ static void pgtable_free_now(pgtable_free_t pgf)
66{ 66{
67 pte_freelist_forced_free++; 67 pte_freelist_forced_free++;
68 68
69 smp_call_function(pte_free_smp_sync, NULL, 0, 1); 69 smp_call_function(pte_free_smp_sync, NULL, 1);
70 70
71 pgtable_free(pgf); 71 pgtable_free(pgf);
72} 72}
diff --git a/arch/powerpc/oprofile/common.c b/arch/powerpc/oprofile/common.c
index 4908dc98f9ca..17807acb05d9 100644
--- a/arch/powerpc/oprofile/common.c
+++ b/arch/powerpc/oprofile/common.c
@@ -65,7 +65,7 @@ static int op_powerpc_setup(void)
65 65
66 /* Configure the registers on all cpus. If an error occurs on one 66 /* Configure the registers on all cpus. If an error occurs on one
67 * of the cpus, op_per_cpu_rc will be set to the error */ 67 * of the cpus, op_per_cpu_rc will be set to the error */
68 on_each_cpu(op_powerpc_cpu_setup, NULL, 0, 1); 68 on_each_cpu(op_powerpc_cpu_setup, NULL, 1);
69 69
70out: if (op_per_cpu_rc) { 70out: if (op_per_cpu_rc) {
71 /* error on setup release the performance counter hardware */ 71 /* error on setup release the performance counter hardware */
@@ -100,7 +100,7 @@ static int op_powerpc_start(void)
100 if (model->global_start) 100 if (model->global_start)
101 return model->global_start(ctr); 101 return model->global_start(ctr);
102 if (model->start) { 102 if (model->start) {
103 on_each_cpu(op_powerpc_cpu_start, NULL, 0, 1); 103 on_each_cpu(op_powerpc_cpu_start, NULL, 1);
104 return op_per_cpu_rc; 104 return op_per_cpu_rc;
105 } 105 }
106 return -EIO; /* No start function is defined for this 106 return -EIO; /* No start function is defined for this
@@ -115,7 +115,7 @@ static inline void op_powerpc_cpu_stop(void *dummy)
115static void op_powerpc_stop(void) 115static void op_powerpc_stop(void)
116{ 116{
117 if (model->stop) 117 if (model->stop)
118 on_each_cpu(op_powerpc_cpu_stop, NULL, 0, 1); 118 on_each_cpu(op_powerpc_cpu_stop, NULL, 1);
119 if (model->global_stop) 119 if (model->global_stop)
120 model->global_stop(); 120 model->global_stop();
121} 121}
diff --git a/arch/powerpc/platforms/cell/interrupt.c b/arch/powerpc/platforms/cell/interrupt.c
index 5bf7df146022..2d5bb22d6c09 100644
--- a/arch/powerpc/platforms/cell/interrupt.c
+++ b/arch/powerpc/platforms/cell/interrupt.c
@@ -218,6 +218,7 @@ void iic_request_IPIs(void)
218{ 218{
219 iic_request_ipi(PPC_MSG_CALL_FUNCTION, "IPI-call"); 219 iic_request_ipi(PPC_MSG_CALL_FUNCTION, "IPI-call");
220 iic_request_ipi(PPC_MSG_RESCHEDULE, "IPI-resched"); 220 iic_request_ipi(PPC_MSG_RESCHEDULE, "IPI-resched");
221 iic_request_ipi(PPC_MSG_CALL_FUNC_SINGLE, "IPI-call-single");
221#ifdef CONFIG_DEBUGGER 222#ifdef CONFIG_DEBUGGER
222 iic_request_ipi(PPC_MSG_DEBUGGER_BREAK, "IPI-debug"); 223 iic_request_ipi(PPC_MSG_DEBUGGER_BREAK, "IPI-debug");
223#endif /* CONFIG_DEBUGGER */ 224#endif /* CONFIG_DEBUGGER */
diff --git a/arch/powerpc/platforms/ps3/smp.c b/arch/powerpc/platforms/ps3/smp.c
index f0b12f212363..a0927a3bacb7 100644
--- a/arch/powerpc/platforms/ps3/smp.c
+++ b/arch/powerpc/platforms/ps3/smp.c
@@ -105,9 +105,10 @@ static void __init ps3_smp_setup_cpu(int cpu)
105 * to index needs to be setup. 105 * to index needs to be setup.
106 */ 106 */
107 107
108 BUILD_BUG_ON(PPC_MSG_CALL_FUNCTION != 0); 108 BUILD_BUG_ON(PPC_MSG_CALL_FUNCTION != 0);
109 BUILD_BUG_ON(PPC_MSG_RESCHEDULE != 1); 109 BUILD_BUG_ON(PPC_MSG_RESCHEDULE != 1);
110 BUILD_BUG_ON(PPC_MSG_DEBUGGER_BREAK != 3); 110 BUILD_BUG_ON(PPC_MSG_CALL_FUNC_SINGLE != 2);
111 BUILD_BUG_ON(PPC_MSG_DEBUGGER_BREAK != 3);
111 112
112 for (i = 0; i < MSG_COUNT; i++) { 113 for (i = 0; i < MSG_COUNT; i++) {
113 result = ps3_event_receive_port_setup(cpu, &virqs[i]); 114 result = ps3_event_receive_port_setup(cpu, &virqs[i]);
diff --git a/arch/powerpc/platforms/pseries/xics.c b/arch/powerpc/platforms/pseries/xics.c
index ebebc28fe895..0fc830f576f5 100644
--- a/arch/powerpc/platforms/pseries/xics.c
+++ b/arch/powerpc/platforms/pseries/xics.c
@@ -383,13 +383,11 @@ static irqreturn_t xics_ipi_dispatch(int cpu)
383 mb(); 383 mb();
384 smp_message_recv(PPC_MSG_RESCHEDULE); 384 smp_message_recv(PPC_MSG_RESCHEDULE);
385 } 385 }
386#if 0 386 if (test_and_clear_bit(PPC_MSG_CALL_FUNC_SINGLE,
387 if (test_and_clear_bit(PPC_MSG_MIGRATE_TASK,
388 &xics_ipi_message[cpu].value)) { 387 &xics_ipi_message[cpu].value)) {
389 mb(); 388 mb();
390 smp_message_recv(PPC_MSG_MIGRATE_TASK); 389 smp_message_recv(PPC_MSG_CALL_FUNC_SINGLE);
391 } 390 }
392#endif
393#if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC) 391#if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC)
394 if (test_and_clear_bit(PPC_MSG_DEBUGGER_BREAK, 392 if (test_and_clear_bit(PPC_MSG_DEBUGGER_BREAK,
395 &xics_ipi_message[cpu].value)) { 393 &xics_ipi_message[cpu].value)) {
diff --git a/arch/powerpc/sysdev/mpic.c b/arch/powerpc/sysdev/mpic.c
index 7680001676a6..6c90c95b454e 100644
--- a/arch/powerpc/sysdev/mpic.c
+++ b/arch/powerpc/sysdev/mpic.c
@@ -1494,7 +1494,7 @@ void mpic_request_ipis(void)
1494 static char *ipi_names[] = { 1494 static char *ipi_names[] = {
1495 "IPI0 (call function)", 1495 "IPI0 (call function)",
1496 "IPI1 (reschedule)", 1496 "IPI1 (reschedule)",
1497 "IPI2 (unused)", 1497 "IPI2 (call function single)",
1498 "IPI3 (debugger break)", 1498 "IPI3 (debugger break)",
1499 }; 1499 };
1500 BUG_ON(mpic == NULL); 1500 BUG_ON(mpic == NULL);
diff --git a/arch/s390/appldata/appldata_base.c b/arch/s390/appldata/appldata_base.c
index 9cb3d92447a3..a7f8979fb925 100644
--- a/arch/s390/appldata/appldata_base.c
+++ b/arch/s390/appldata/appldata_base.c
@@ -203,7 +203,7 @@ __appldata_vtimer_setup(int cmd)
203 per_cpu(appldata_timer, i).expires = per_cpu_interval; 203 per_cpu(appldata_timer, i).expires = per_cpu_interval;
204 smp_call_function_single(i, add_virt_timer_periodic, 204 smp_call_function_single(i, add_virt_timer_periodic,
205 &per_cpu(appldata_timer, i), 205 &per_cpu(appldata_timer, i),
206 0, 1); 206 1);
207 } 207 }
208 appldata_timer_active = 1; 208 appldata_timer_active = 1;
209 break; 209 break;
@@ -228,7 +228,7 @@ __appldata_vtimer_setup(int cmd)
228 args.timer = &per_cpu(appldata_timer, i); 228 args.timer = &per_cpu(appldata_timer, i);
229 args.expires = per_cpu_interval; 229 args.expires = per_cpu_interval;
230 smp_call_function_single(i, __appldata_mod_vtimer_wrap, 230 smp_call_function_single(i, __appldata_mod_vtimer_wrap,
231 &args, 0, 1); 231 &args, 1);
232 } 232 }
233 } 233 }
234} 234}
diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index 5d4fa4b1c74c..b6781030cfbd 100644
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -109,7 +109,7 @@ static void do_call_function(void)
109} 109}
110 110
111static void __smp_call_function_map(void (*func) (void *info), void *info, 111static void __smp_call_function_map(void (*func) (void *info), void *info,
112 int nonatomic, int wait, cpumask_t map) 112 int wait, cpumask_t map)
113{ 113{
114 struct call_data_struct data; 114 struct call_data_struct data;
115 int cpu, local = 0; 115 int cpu, local = 0;
@@ -162,7 +162,6 @@ out:
162 * smp_call_function: 162 * smp_call_function:
163 * @func: the function to run; this must be fast and non-blocking 163 * @func: the function to run; this must be fast and non-blocking
164 * @info: an arbitrary pointer to pass to the function 164 * @info: an arbitrary pointer to pass to the function
165 * @nonatomic: unused
166 * @wait: if true, wait (atomically) until function has completed on other CPUs 165 * @wait: if true, wait (atomically) until function has completed on other CPUs
167 * 166 *
168 * Run a function on all other CPUs. 167 * Run a function on all other CPUs.
@@ -170,15 +169,14 @@ out:
170 * You must not call this function with disabled interrupts, from a 169 * You must not call this function with disabled interrupts, from a
171 * hardware interrupt handler or from a bottom half. 170 * hardware interrupt handler or from a bottom half.
172 */ 171 */
173int smp_call_function(void (*func) (void *info), void *info, int nonatomic, 172int smp_call_function(void (*func) (void *info), void *info, int wait)
174 int wait)
175{ 173{
176 cpumask_t map; 174 cpumask_t map;
177 175
178 spin_lock(&call_lock); 176 spin_lock(&call_lock);
179 map = cpu_online_map; 177 map = cpu_online_map;
180 cpu_clear(smp_processor_id(), map); 178 cpu_clear(smp_processor_id(), map);
181 __smp_call_function_map(func, info, nonatomic, wait, map); 179 __smp_call_function_map(func, info, wait, map);
182 spin_unlock(&call_lock); 180 spin_unlock(&call_lock);
183 return 0; 181 return 0;
184} 182}
@@ -189,7 +187,6 @@ EXPORT_SYMBOL(smp_call_function);
189 * @cpu: the CPU where func should run 187 * @cpu: the CPU where func should run
190 * @func: the function to run; this must be fast and non-blocking 188 * @func: the function to run; this must be fast and non-blocking
191 * @info: an arbitrary pointer to pass to the function 189 * @info: an arbitrary pointer to pass to the function
192 * @nonatomic: unused
193 * @wait: if true, wait (atomically) until function has completed on other CPUs 190 * @wait: if true, wait (atomically) until function has completed on other CPUs
194 * 191 *
195 * Run a function on one processor. 192 * Run a function on one processor.
@@ -198,11 +195,10 @@ EXPORT_SYMBOL(smp_call_function);
198 * hardware interrupt handler or from a bottom half. 195 * hardware interrupt handler or from a bottom half.
199 */ 196 */
200int smp_call_function_single(int cpu, void (*func) (void *info), void *info, 197int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
201 int nonatomic, int wait) 198 int wait)
202{ 199{
203 spin_lock(&call_lock); 200 spin_lock(&call_lock);
204 __smp_call_function_map(func, info, nonatomic, wait, 201 __smp_call_function_map(func, info, wait, cpumask_of_cpu(cpu));
205 cpumask_of_cpu(cpu));
206 spin_unlock(&call_lock); 202 spin_unlock(&call_lock);
207 return 0; 203 return 0;
208} 204}
@@ -228,7 +224,7 @@ int smp_call_function_mask(cpumask_t mask, void (*func)(void *), void *info,
228{ 224{
229 spin_lock(&call_lock); 225 spin_lock(&call_lock);
230 cpu_clear(smp_processor_id(), mask); 226 cpu_clear(smp_processor_id(), mask);
231 __smp_call_function_map(func, info, 0, wait, mask); 227 __smp_call_function_map(func, info, wait, mask);
232 spin_unlock(&call_lock); 228 spin_unlock(&call_lock);
233 return 0; 229 return 0;
234} 230}
@@ -303,7 +299,7 @@ static void smp_ptlb_callback(void *info)
303 299
304void smp_ptlb_all(void) 300void smp_ptlb_all(void)
305{ 301{
306 on_each_cpu(smp_ptlb_callback, NULL, 0, 1); 302 on_each_cpu(smp_ptlb_callback, NULL, 1);
307} 303}
308EXPORT_SYMBOL(smp_ptlb_all); 304EXPORT_SYMBOL(smp_ptlb_all);
309#endif /* ! CONFIG_64BIT */ 305#endif /* ! CONFIG_64BIT */
@@ -351,7 +347,7 @@ void smp_ctl_set_bit(int cr, int bit)
351 memset(&parms.orvals, 0, sizeof(parms.orvals)); 347 memset(&parms.orvals, 0, sizeof(parms.orvals));
352 memset(&parms.andvals, 0xff, sizeof(parms.andvals)); 348 memset(&parms.andvals, 0xff, sizeof(parms.andvals));
353 parms.orvals[cr] = 1 << bit; 349 parms.orvals[cr] = 1 << bit;
354 on_each_cpu(smp_ctl_bit_callback, &parms, 0, 1); 350 on_each_cpu(smp_ctl_bit_callback, &parms, 1);
355} 351}
356EXPORT_SYMBOL(smp_ctl_set_bit); 352EXPORT_SYMBOL(smp_ctl_set_bit);
357 353
@@ -365,7 +361,7 @@ void smp_ctl_clear_bit(int cr, int bit)
365 memset(&parms.orvals, 0, sizeof(parms.orvals)); 361 memset(&parms.orvals, 0, sizeof(parms.orvals));
366 memset(&parms.andvals, 0xff, sizeof(parms.andvals)); 362 memset(&parms.andvals, 0xff, sizeof(parms.andvals));
367 parms.andvals[cr] = ~(1L << bit); 363 parms.andvals[cr] = ~(1L << bit);
368 on_each_cpu(smp_ctl_bit_callback, &parms, 0, 1); 364 on_each_cpu(smp_ctl_bit_callback, &parms, 1);
369} 365}
370EXPORT_SYMBOL(smp_ctl_clear_bit); 366EXPORT_SYMBOL(smp_ctl_clear_bit);
371 367
diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c
index 7418bebb547f..8051e9326dfc 100644
--- a/arch/s390/kernel/time.c
+++ b/arch/s390/kernel/time.c
@@ -707,7 +707,7 @@ static int etr_sync_clock(struct etr_aib *aib, int port)
707 */ 707 */
708 memset(&etr_sync, 0, sizeof(etr_sync)); 708 memset(&etr_sync, 0, sizeof(etr_sync));
709 preempt_disable(); 709 preempt_disable();
710 smp_call_function(clock_sync_cpu_start, &etr_sync, 0, 0); 710 smp_call_function(clock_sync_cpu_start, &etr_sync, 0);
711 local_irq_disable(); 711 local_irq_disable();
712 enable_sync_clock(); 712 enable_sync_clock();
713 713
@@ -746,7 +746,7 @@ static int etr_sync_clock(struct etr_aib *aib, int port)
746 rc = -EAGAIN; 746 rc = -EAGAIN;
747 } 747 }
748 local_irq_enable(); 748 local_irq_enable();
749 smp_call_function(clock_sync_cpu_end, NULL, 0, 0); 749 smp_call_function(clock_sync_cpu_end, NULL, 0);
750 preempt_enable(); 750 preempt_enable();
751 return rc; 751 return rc;
752} 752}
@@ -926,7 +926,7 @@ static void etr_work_fn(struct work_struct *work)
926 if (!eacr.ea) { 926 if (!eacr.ea) {
927 /* Both ports offline. Reset everything. */ 927 /* Both ports offline. Reset everything. */
928 eacr.dp = eacr.es = eacr.sl = 0; 928 eacr.dp = eacr.es = eacr.sl = 0;
929 on_each_cpu(disable_sync_clock, NULL, 0, 1); 929 on_each_cpu(disable_sync_clock, NULL, 1);
930 del_timer_sync(&etr_timer); 930 del_timer_sync(&etr_timer);
931 etr_update_eacr(eacr); 931 etr_update_eacr(eacr);
932 clear_bit(CLOCK_SYNC_ETR, &clock_sync_flags); 932 clear_bit(CLOCK_SYNC_ETR, &clock_sync_flags);
diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig
index 9a854c8e5274..3e7384f4619c 100644
--- a/arch/sh/Kconfig
+++ b/arch/sh/Kconfig
@@ -688,6 +688,7 @@ config CRASH_DUMP
688config SMP 688config SMP
689 bool "Symmetric multi-processing support" 689 bool "Symmetric multi-processing support"
690 depends on SYS_SUPPORTS_SMP 690 depends on SYS_SUPPORTS_SMP
691 select USE_GENERIC_SMP_HELPERS
691 ---help--- 692 ---help---
692 This enables support for systems with more than one CPU. If you have 693 This enables support for systems with more than one CPU. If you have
693 a system with only one CPU, like most personal computers, say N. If 694 a system with only one CPU, like most personal computers, say N. If
diff --git a/arch/sh/kernel/smp.c b/arch/sh/kernel/smp.c
index 5d039d168f57..60c50841143e 100644
--- a/arch/sh/kernel/smp.c
+++ b/arch/sh/kernel/smp.c
@@ -36,13 +36,6 @@ EXPORT_SYMBOL(cpu_possible_map);
36cpumask_t cpu_online_map; 36cpumask_t cpu_online_map;
37EXPORT_SYMBOL(cpu_online_map); 37EXPORT_SYMBOL(cpu_online_map);
38 38
39static atomic_t cpus_booted = ATOMIC_INIT(0);
40
41/*
42 * Run specified function on a particular processor.
43 */
44void __smp_call_function(unsigned int cpu);
45
46static inline void __init smp_store_cpu_info(unsigned int cpu) 39static inline void __init smp_store_cpu_info(unsigned int cpu)
47{ 40{
48 struct sh_cpuinfo *c = cpu_data + cpu; 41 struct sh_cpuinfo *c = cpu_data + cpu;
@@ -175,45 +168,20 @@ static void stop_this_cpu(void *unused)
175 168
176void smp_send_stop(void) 169void smp_send_stop(void)
177{ 170{
178 smp_call_function(stop_this_cpu, 0, 1, 0); 171 smp_call_function(stop_this_cpu, 0, 0);
179} 172}
180 173
181struct smp_fn_call_struct smp_fn_call = { 174void arch_send_call_function_ipi(cpumask_t mask)
182 .lock = __SPIN_LOCK_UNLOCKED(smp_fn_call.lock),
183 .finished = ATOMIC_INIT(0),
184};
185
186/*
187 * The caller of this wants the passed function to run on every cpu. If wait
188 * is set, wait until all cpus have finished the function before returning.
189 * The lock is here to protect the call structure.
190 * You must not call this function with disabled interrupts or from a
191 * hardware interrupt handler or from a bottom half handler.
192 */
193int smp_call_function(void (*func)(void *info), void *info, int retry, int wait)
194{ 175{
195 unsigned int nr_cpus = atomic_read(&cpus_booted); 176 int cpu;
196 int i;
197
198 /* Can deadlock when called with interrupts disabled */
199 WARN_ON(irqs_disabled());
200
201 spin_lock(&smp_fn_call.lock);
202
203 atomic_set(&smp_fn_call.finished, 0);
204 smp_fn_call.fn = func;
205 smp_fn_call.data = info;
206
207 for (i = 0; i < nr_cpus; i++)
208 if (i != smp_processor_id())
209 plat_send_ipi(i, SMP_MSG_FUNCTION);
210
211 if (wait)
212 while (atomic_read(&smp_fn_call.finished) != (nr_cpus - 1));
213 177
214 spin_unlock(&smp_fn_call.lock); 178 for_each_cpu_mask(cpu, mask)
179 plat_send_ipi(cpu, SMP_MSG_FUNCTION);
180}
215 181
216 return 0; 182void arch_send_call_function_single_ipi(int cpu)
183{
184 plat_send_ipi(cpu, SMP_MSG_FUNCTION_SINGLE);
217} 185}
218 186
219/* Not really SMP stuff ... */ 187/* Not really SMP stuff ... */
@@ -229,7 +197,7 @@ static void flush_tlb_all_ipi(void *info)
229 197
230void flush_tlb_all(void) 198void flush_tlb_all(void)
231{ 199{
232 on_each_cpu(flush_tlb_all_ipi, 0, 1, 1); 200 on_each_cpu(flush_tlb_all_ipi, 0, 1);
233} 201}
234 202
235static void flush_tlb_mm_ipi(void *mm) 203static void flush_tlb_mm_ipi(void *mm)
@@ -255,7 +223,7 @@ void flush_tlb_mm(struct mm_struct *mm)
255 preempt_disable(); 223 preempt_disable();
256 224
257 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { 225 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
258 smp_call_function(flush_tlb_mm_ipi, (void *)mm, 1, 1); 226 smp_call_function(flush_tlb_mm_ipi, (void *)mm, 1);
259 } else { 227 } else {
260 int i; 228 int i;
261 for (i = 0; i < num_online_cpus(); i++) 229 for (i = 0; i < num_online_cpus(); i++)
@@ -292,7 +260,7 @@ void flush_tlb_range(struct vm_area_struct *vma,
292 fd.vma = vma; 260 fd.vma = vma;
293 fd.addr1 = start; 261 fd.addr1 = start;
294 fd.addr2 = end; 262 fd.addr2 = end;
295 smp_call_function(flush_tlb_range_ipi, (void *)&fd, 1, 1); 263 smp_call_function(flush_tlb_range_ipi, (void *)&fd, 1);
296 } else { 264 } else {
297 int i; 265 int i;
298 for (i = 0; i < num_online_cpus(); i++) 266 for (i = 0; i < num_online_cpus(); i++)
@@ -316,7 +284,7 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
316 284
317 fd.addr1 = start; 285 fd.addr1 = start;
318 fd.addr2 = end; 286 fd.addr2 = end;
319 on_each_cpu(flush_tlb_kernel_range_ipi, (void *)&fd, 1, 1); 287 on_each_cpu(flush_tlb_kernel_range_ipi, (void *)&fd, 1);
320} 288}
321 289
322static void flush_tlb_page_ipi(void *info) 290static void flush_tlb_page_ipi(void *info)
@@ -335,7 +303,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
335 303
336 fd.vma = vma; 304 fd.vma = vma;
337 fd.addr1 = page; 305 fd.addr1 = page;
338 smp_call_function(flush_tlb_page_ipi, (void *)&fd, 1, 1); 306 smp_call_function(flush_tlb_page_ipi, (void *)&fd, 1);
339 } else { 307 } else {
340 int i; 308 int i;
341 for (i = 0; i < num_online_cpus(); i++) 309 for (i = 0; i < num_online_cpus(); i++)
@@ -359,6 +327,6 @@ void flush_tlb_one(unsigned long asid, unsigned long vaddr)
359 fd.addr1 = asid; 327 fd.addr1 = asid;
360 fd.addr2 = vaddr; 328 fd.addr2 = vaddr;
361 329
362 smp_call_function(flush_tlb_one_ipi, (void *)&fd, 1, 1); 330 smp_call_function(flush_tlb_one_ipi, (void *)&fd, 1);
363 local_flush_tlb_one(asid, vaddr); 331 local_flush_tlb_one(asid, vaddr);
364} 332}
diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c
index fa63c68a1819..c099d96f1239 100644
--- a/arch/sparc64/kernel/smp.c
+++ b/arch/sparc64/kernel/smp.c
@@ -807,7 +807,6 @@ extern unsigned long xcall_call_function;
807 * smp_call_function(): Run a function on all other CPUs. 807 * smp_call_function(): Run a function on all other CPUs.
808 * @func: The function to run. This must be fast and non-blocking. 808 * @func: The function to run. This must be fast and non-blocking.
809 * @info: An arbitrary pointer to pass to the function. 809 * @info: An arbitrary pointer to pass to the function.
810 * @nonatomic: currently unused.
811 * @wait: If true, wait (atomically) until function has completed on other CPUs. 810 * @wait: If true, wait (atomically) until function has completed on other CPUs.
812 * 811 *
813 * Returns 0 on success, else a negative status code. Does not return until 812 * Returns 0 on success, else a negative status code. Does not return until
@@ -816,8 +815,8 @@ extern unsigned long xcall_call_function;
816 * You must not call this function with disabled interrupts or from a 815 * You must not call this function with disabled interrupts or from a
817 * hardware interrupt handler or from a bottom half handler. 816 * hardware interrupt handler or from a bottom half handler.
818 */ 817 */
819static int smp_call_function_mask(void (*func)(void *info), void *info, 818static int sparc64_smp_call_function_mask(void (*func)(void *info), void *info,
820 int nonatomic, int wait, cpumask_t mask) 819 int wait, cpumask_t mask)
821{ 820{
822 struct call_data_struct data; 821 struct call_data_struct data;
823 int cpus; 822 int cpus;
@@ -852,11 +851,9 @@ out_unlock:
852 return 0; 851 return 0;
853} 852}
854 853
855int smp_call_function(void (*func)(void *info), void *info, 854int smp_call_function(void (*func)(void *info), void *info, int wait)
856 int nonatomic, int wait)
857{ 855{
858 return smp_call_function_mask(func, info, nonatomic, wait, 856 return sparc64_smp_call_function_mask(func, info, wait, cpu_online_map);
859 cpu_online_map);
860} 857}
861 858
862void smp_call_function_client(int irq, struct pt_regs *regs) 859void smp_call_function_client(int irq, struct pt_regs *regs)
@@ -893,7 +890,7 @@ static void tsb_sync(void *info)
893 890
894void smp_tsb_sync(struct mm_struct *mm) 891void smp_tsb_sync(struct mm_struct *mm)
895{ 892{
896 smp_call_function_mask(tsb_sync, mm, 0, 1, mm->cpu_vm_mask); 893 sparc64_smp_call_function_mask(tsb_sync, mm, 1, mm->cpu_vm_mask);
897} 894}
898 895
899extern unsigned long xcall_flush_tlb_mm; 896extern unsigned long xcall_flush_tlb_mm;
diff --git a/arch/sparc64/mm/hugetlbpage.c b/arch/sparc64/mm/hugetlbpage.c
index 6cfab2e4d340..ebefd2a14375 100644
--- a/arch/sparc64/mm/hugetlbpage.c
+++ b/arch/sparc64/mm/hugetlbpage.c
@@ -344,7 +344,7 @@ void hugetlb_prefault_arch_hook(struct mm_struct *mm)
344 * also executing in this address space. 344 * also executing in this address space.
345 */ 345 */
346 mm->context.sparc64_ctx_val = ctx; 346 mm->context.sparc64_ctx_val = ctx;
347 on_each_cpu(context_reload, mm, 0, 0); 347 on_each_cpu(context_reload, mm, 0);
348 } 348 }
349 spin_unlock(&ctx_alloc_lock); 349 spin_unlock(&ctx_alloc_lock);
350 } 350 }
diff --git a/arch/um/kernel/smp.c b/arch/um/kernel/smp.c
index e1062ec36d40..be2d50c3aa95 100644
--- a/arch/um/kernel/smp.c
+++ b/arch/um/kernel/smp.c
@@ -214,8 +214,7 @@ void smp_call_function_slave(int cpu)
214 atomic_inc(&scf_finished); 214 atomic_inc(&scf_finished);
215} 215}
216 216
217int smp_call_function(void (*_func)(void *info), void *_info, int nonatomic, 217int smp_call_function(void (*_func)(void *info), void *_info, int wait)
218 int wait)
219{ 218{
220 int cpus = num_online_cpus() - 1; 219 int cpus = num_online_cpus() - 1;
221 int i; 220 int i;
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 2642b4bf41b9..96e0c2ebc388 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -170,6 +170,7 @@ config GENERIC_PENDING_IRQ
170config X86_SMP 170config X86_SMP
171 bool 171 bool
172 depends on SMP && ((X86_32 && !X86_VOYAGER) || X86_64) 172 depends on SMP && ((X86_32 && !X86_VOYAGER) || X86_64)
173 select USE_GENERIC_SMP_HELPERS
173 default y 174 default y
174 175
175config X86_32_SMP 176config X86_32_SMP
diff --git a/arch/x86/kernel/apic_32.c b/arch/x86/kernel/apic_32.c
index 3e58b676d23b..a437d027f20b 100644
--- a/arch/x86/kernel/apic_32.c
+++ b/arch/x86/kernel/apic_32.c
@@ -1340,6 +1340,10 @@ void __init smp_intr_init(void)
1340 1340
1341 /* IPI for generic function call */ 1341 /* IPI for generic function call */
1342 alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt); 1342 alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
1343
1344 /* IPI for single call function */
1345 set_intr_gate(CALL_FUNCTION_SINGLE_VECTOR,
1346 call_function_single_interrupt);
1343} 1347}
1344#endif 1348#endif
1345 1349
diff --git a/arch/x86/kernel/cpu/mcheck/mce_64.c b/arch/x86/kernel/cpu/mcheck/mce_64.c
index 987410745182..c4a7ec31394c 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_64.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_64.c
@@ -364,7 +364,7 @@ static void mcheck_check_cpu(void *info)
364 364
365static void mcheck_timer(struct work_struct *work) 365static void mcheck_timer(struct work_struct *work)
366{ 366{
367 on_each_cpu(mcheck_check_cpu, NULL, 1, 1); 367 on_each_cpu(mcheck_check_cpu, NULL, 1);
368 368
369 /* 369 /*
370 * Alert userspace if needed. If we logged an MCE, reduce the 370 * Alert userspace if needed. If we logged an MCE, reduce the
@@ -621,7 +621,7 @@ static ssize_t mce_read(struct file *filp, char __user *ubuf, size_t usize,
621 * Collect entries that were still getting written before the 621 * Collect entries that were still getting written before the
622 * synchronize. 622 * synchronize.
623 */ 623 */
624 on_each_cpu(collect_tscs, cpu_tsc, 1, 1); 624 on_each_cpu(collect_tscs, cpu_tsc, 1);
625 for (i = next; i < MCE_LOG_LEN; i++) { 625 for (i = next; i < MCE_LOG_LEN; i++) {
626 if (mcelog.entry[i].finished && 626 if (mcelog.entry[i].finished &&
627 mcelog.entry[i].tsc < cpu_tsc[mcelog.entry[i].cpu]) { 627 mcelog.entry[i].tsc < cpu_tsc[mcelog.entry[i].cpu]) {
@@ -746,7 +746,7 @@ static void mce_restart(void)
746 if (next_interval) 746 if (next_interval)
747 cancel_delayed_work(&mcheck_work); 747 cancel_delayed_work(&mcheck_work);
748 /* Timer race is harmless here */ 748 /* Timer race is harmless here */
749 on_each_cpu(mce_init, NULL, 1, 1); 749 on_each_cpu(mce_init, NULL, 1);
750 next_interval = check_interval * HZ; 750 next_interval = check_interval * HZ;
751 if (next_interval) 751 if (next_interval)
752 schedule_delayed_work(&mcheck_work, 752 schedule_delayed_work(&mcheck_work,
diff --git a/arch/x86/kernel/cpu/mcheck/non-fatal.c b/arch/x86/kernel/cpu/mcheck/non-fatal.c
index 00ccb6c14ec2..cc1fccdd31e0 100644
--- a/arch/x86/kernel/cpu/mcheck/non-fatal.c
+++ b/arch/x86/kernel/cpu/mcheck/non-fatal.c
@@ -59,7 +59,7 @@ static DECLARE_DELAYED_WORK(mce_work, mce_work_fn);
59 59
60static void mce_work_fn(struct work_struct *work) 60static void mce_work_fn(struct work_struct *work)
61{ 61{
62 on_each_cpu(mce_checkregs, NULL, 1, 1); 62 on_each_cpu(mce_checkregs, NULL, 1);
63 schedule_delayed_work(&mce_work, round_jiffies_relative(MCE_RATE)); 63 schedule_delayed_work(&mce_work, round_jiffies_relative(MCE_RATE));
64} 64}
65 65
diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c
index 105afe12beb0..6f23969c8faf 100644
--- a/arch/x86/kernel/cpu/mtrr/main.c
+++ b/arch/x86/kernel/cpu/mtrr/main.c
@@ -223,7 +223,7 @@ static void set_mtrr(unsigned int reg, unsigned long base,
223 atomic_set(&data.gate,0); 223 atomic_set(&data.gate,0);
224 224
225 /* Start the ball rolling on other CPUs */ 225 /* Start the ball rolling on other CPUs */
226 if (smp_call_function(ipi_handler, &data, 1, 0) != 0) 226 if (smp_call_function(ipi_handler, &data, 0) != 0)
227 panic("mtrr: timed out waiting for other CPUs\n"); 227 panic("mtrr: timed out waiting for other CPUs\n");
228 228
229 local_irq_save(flags); 229 local_irq_save(flags);
@@ -1682,7 +1682,7 @@ void mtrr_ap_init(void)
1682 */ 1682 */
1683void mtrr_save_state(void) 1683void mtrr_save_state(void)
1684{ 1684{
1685 smp_call_function_single(0, mtrr_save_fixed_ranges, NULL, 1, 1); 1685 smp_call_function_single(0, mtrr_save_fixed_ranges, NULL, 1);
1686} 1686}
1687 1687
1688static int __init mtrr_init_finialize(void) 1688static int __init mtrr_init_finialize(void)
diff --git a/arch/x86/kernel/cpu/perfctr-watchdog.c b/arch/x86/kernel/cpu/perfctr-watchdog.c
index 2e9bef6e3aa3..6d4bdc02388a 100644
--- a/arch/x86/kernel/cpu/perfctr-watchdog.c
+++ b/arch/x86/kernel/cpu/perfctr-watchdog.c
@@ -189,7 +189,7 @@ void disable_lapic_nmi_watchdog(void)
189 if (atomic_read(&nmi_active) <= 0) 189 if (atomic_read(&nmi_active) <= 0)
190 return; 190 return;
191 191
192 on_each_cpu(stop_apic_nmi_watchdog, NULL, 0, 1); 192 on_each_cpu(stop_apic_nmi_watchdog, NULL, 1);
193 193
194 if (wd_ops) 194 if (wd_ops)
195 wd_ops->unreserve(); 195 wd_ops->unreserve();
@@ -213,7 +213,7 @@ void enable_lapic_nmi_watchdog(void)
213 return; 213 return;
214 } 214 }
215 215
216 on_each_cpu(setup_apic_nmi_watchdog, NULL, 0, 1); 216 on_each_cpu(setup_apic_nmi_watchdog, NULL, 1);
217 touch_nmi_watchdog(); 217 touch_nmi_watchdog();
218} 218}
219 219
diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c
index 71f1c2654bec..2de5fa2bbf77 100644
--- a/arch/x86/kernel/cpuid.c
+++ b/arch/x86/kernel/cpuid.c
@@ -96,7 +96,7 @@ static ssize_t cpuid_read(struct file *file, char __user *buf,
96 for (; count; count -= 16) { 96 for (; count; count -= 16) {
97 cmd.eax = pos; 97 cmd.eax = pos;
98 cmd.ecx = pos >> 32; 98 cmd.ecx = pos >> 32;
99 smp_call_function_single(cpu, cpuid_smp_cpuid, &cmd, 1, 1); 99 smp_call_function_single(cpu, cpuid_smp_cpuid, &cmd, 1);
100 if (copy_to_user(tmp, &cmd, 16)) 100 if (copy_to_user(tmp, &cmd, 16))
101 return -EFAULT; 101 return -EFAULT;
102 tmp += 16; 102 tmp += 16;
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index ba41bf42748d..ae63e584c340 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -816,6 +816,9 @@ END(invalidate_interrupt\num)
816ENTRY(call_function_interrupt) 816ENTRY(call_function_interrupt)
817 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt 817 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
818END(call_function_interrupt) 818END(call_function_interrupt)
819ENTRY(call_function_single_interrupt)
820 apicinterrupt CALL_FUNCTION_SINGLE_VECTOR,smp_call_function_single_interrupt
821END(call_function_single_interrupt)
819ENTRY(irq_move_cleanup_interrupt) 822ENTRY(irq_move_cleanup_interrupt)
820 apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt 823 apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
821END(irq_move_cleanup_interrupt) 824END(irq_move_cleanup_interrupt)
diff --git a/arch/x86/kernel/io_apic_32.c b/arch/x86/kernel/io_apic_32.c
index 603261a5885c..558abf4c796a 100644
--- a/arch/x86/kernel/io_apic_32.c
+++ b/arch/x86/kernel/io_apic_32.c
@@ -1569,7 +1569,7 @@ void /*__init*/ print_local_APIC(void *dummy)
1569 1569
1570void print_all_local_APICs(void) 1570void print_all_local_APICs(void)
1571{ 1571{
1572 on_each_cpu(print_local_APIC, NULL, 1, 1); 1572 on_each_cpu(print_local_APIC, NULL, 1);
1573} 1573}
1574 1574
1575void /*__init*/ print_PIC(void) 1575void /*__init*/ print_PIC(void)
diff --git a/arch/x86/kernel/io_apic_64.c b/arch/x86/kernel/io_apic_64.c
index b16ef029cf88..6510cde36b35 100644
--- a/arch/x86/kernel/io_apic_64.c
+++ b/arch/x86/kernel/io_apic_64.c
@@ -1160,7 +1160,7 @@ void __apicdebuginit print_local_APIC(void * dummy)
1160 1160
1161void print_all_local_APICs (void) 1161void print_all_local_APICs (void)
1162{ 1162{
1163 on_each_cpu(print_local_APIC, NULL, 1, 1); 1163 on_each_cpu(print_local_APIC, NULL, 1);
1164} 1164}
1165 1165
1166void __apicdebuginit print_PIC(void) 1166void __apicdebuginit print_PIC(void)
diff --git a/arch/x86/kernel/irqinit_64.c b/arch/x86/kernel/irqinit_64.c
index 31f49e8f46a7..0373e88de95a 100644
--- a/arch/x86/kernel/irqinit_64.c
+++ b/arch/x86/kernel/irqinit_64.c
@@ -199,6 +199,10 @@ void __init native_init_IRQ(void)
199 /* IPI for generic function call */ 199 /* IPI for generic function call */
200 alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt); 200 alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
201 201
202 /* IPI for generic single function call */
203 alloc_intr_gate(CALL_FUNCTION_SINGLE_VECTOR,
204 call_function_single_interrupt);
205
202 /* Low priority IPI to cleanup after moving an irq */ 206 /* Low priority IPI to cleanup after moving an irq */
203 set_intr_gate(IRQ_MOVE_CLEANUP_VECTOR, irq_move_cleanup_interrupt); 207 set_intr_gate(IRQ_MOVE_CLEANUP_VECTOR, irq_move_cleanup_interrupt);
204#endif 208#endif
diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c
index 21f2bae98c15..a8449571858a 100644
--- a/arch/x86/kernel/ldt.c
+++ b/arch/x86/kernel/ldt.c
@@ -68,7 +68,7 @@ static int alloc_ldt(mm_context_t *pc, int mincount, int reload)
68 load_LDT(pc); 68 load_LDT(pc);
69 mask = cpumask_of_cpu(smp_processor_id()); 69 mask = cpumask_of_cpu(smp_processor_id());
70 if (!cpus_equal(current->mm->cpu_vm_mask, mask)) 70 if (!cpus_equal(current->mm->cpu_vm_mask, mask))
71 smp_call_function(flush_ldt, current->mm, 1, 1); 71 smp_call_function(flush_ldt, current->mm, 1);
72 preempt_enable(); 72 preempt_enable();
73#else 73#else
74 load_LDT(pc); 74 load_LDT(pc);
diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index 716b89284be0..ec024b3baad0 100644
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -130,7 +130,7 @@ int __init check_nmi_watchdog(void)
130 130
131#ifdef CONFIG_SMP 131#ifdef CONFIG_SMP
132 if (nmi_watchdog == NMI_LOCAL_APIC) 132 if (nmi_watchdog == NMI_LOCAL_APIC)
133 smp_call_function(nmi_cpu_busy, (void *)&endflag, 0, 0); 133 smp_call_function(nmi_cpu_busy, (void *)&endflag, 0);
134#endif 134#endif
135 135
136 for_each_possible_cpu(cpu) 136 for_each_possible_cpu(cpu)
@@ -272,7 +272,7 @@ static void __acpi_nmi_enable(void *__unused)
272void acpi_nmi_enable(void) 272void acpi_nmi_enable(void)
273{ 273{
274 if (atomic_read(&nmi_active) && nmi_watchdog == NMI_IO_APIC) 274 if (atomic_read(&nmi_active) && nmi_watchdog == NMI_IO_APIC)
275 on_each_cpu(__acpi_nmi_enable, NULL, 0, 1); 275 on_each_cpu(__acpi_nmi_enable, NULL, 1);
276} 276}
277 277
278static void __acpi_nmi_disable(void *__unused) 278static void __acpi_nmi_disable(void *__unused)
@@ -286,7 +286,7 @@ static void __acpi_nmi_disable(void *__unused)
286void acpi_nmi_disable(void) 286void acpi_nmi_disable(void)
287{ 287{
288 if (atomic_read(&nmi_active) && nmi_watchdog == NMI_IO_APIC) 288 if (atomic_read(&nmi_active) && nmi_watchdog == NMI_IO_APIC)
289 on_each_cpu(__acpi_nmi_disable, NULL, 0, 1); 289 on_each_cpu(__acpi_nmi_disable, NULL, 1);
290} 290}
291 291
292void setup_apic_nmi_watchdog(void *unused) 292void setup_apic_nmi_watchdog(void *unused)
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 4061d63aabe7..7dceea947232 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -132,7 +132,7 @@ void cpu_idle_wait(void)
132{ 132{
133 smp_mb(); 133 smp_mb();
134 /* kick all the CPUs so that they exit out of pm_idle */ 134 /* kick all the CPUs so that they exit out of pm_idle */
135 smp_call_function(do_nothing, NULL, 0, 1); 135 smp_call_function(do_nothing, NULL, 1);
136} 136}
137EXPORT_SYMBOL_GPL(cpu_idle_wait); 137EXPORT_SYMBOL_GPL(cpu_idle_wait);
138 138
diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
index 0cb7aadc87cd..361b7a4c640c 100644
--- a/arch/x86/kernel/smp.c
+++ b/arch/x86/kernel/smp.c
@@ -121,132 +121,23 @@ static void native_smp_send_reschedule(int cpu)
121 send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR); 121 send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
122} 122}
123 123
124/* 124void native_send_call_func_single_ipi(int cpu)
125 * Structure and data for smp_call_function(). This is designed to minimise
126 * static memory requirements. It also looks cleaner.
127 */
128static DEFINE_SPINLOCK(call_lock);
129
130struct call_data_struct {
131 void (*func) (void *info);
132 void *info;
133 atomic_t started;
134 atomic_t finished;
135 int wait;
136};
137
138void lock_ipi_call_lock(void)
139{ 125{
140 spin_lock_irq(&call_lock); 126 send_IPI_mask(cpumask_of_cpu(cpu), CALL_FUNCTION_SINGLE_VECTOR);
141}
142
143void unlock_ipi_call_lock(void)
144{
145 spin_unlock_irq(&call_lock);
146}
147
148static struct call_data_struct *call_data;
149
150static void __smp_call_function(void (*func) (void *info), void *info,
151 int nonatomic, int wait)
152{
153 struct call_data_struct data;
154 int cpus = num_online_cpus() - 1;
155
156 if (!cpus)
157 return;
158
159 data.func = func;
160 data.info = info;
161 atomic_set(&data.started, 0);
162 data.wait = wait;
163 if (wait)
164 atomic_set(&data.finished, 0);
165
166 call_data = &data;
167 mb();
168
169 /* Send a message to all other CPUs and wait for them to respond */
170 send_IPI_allbutself(CALL_FUNCTION_VECTOR);
171
172 /* Wait for response */
173 while (atomic_read(&data.started) != cpus)
174 cpu_relax();
175
176 if (wait)
177 while (atomic_read(&data.finished) != cpus)
178 cpu_relax();
179} 127}
180 128
181 129void native_send_call_func_ipi(cpumask_t mask)
182/**
183 * smp_call_function_mask(): Run a function on a set of other CPUs.
184 * @mask: The set of cpus to run on. Must not include the current cpu.
185 * @func: The function to run. This must be fast and non-blocking.
186 * @info: An arbitrary pointer to pass to the function.
187 * @wait: If true, wait (atomically) until function has completed on other CPUs.
188 *
189 * Returns 0 on success, else a negative status code.
190 *
191 * If @wait is true, then returns once @func has returned; otherwise
192 * it returns just before the target cpu calls @func.
193 *
194 * You must not call this function with disabled interrupts or from a
195 * hardware interrupt handler or from a bottom half handler.
196 */
197static int
198native_smp_call_function_mask(cpumask_t mask,
199 void (*func)(void *), void *info,
200 int wait)
201{ 130{
202 struct call_data_struct data;
203 cpumask_t allbutself; 131 cpumask_t allbutself;
204 int cpus;
205
206 /* Can deadlock when called with interrupts disabled */
207 WARN_ON(irqs_disabled());
208
209 /* Holding any lock stops cpus from going down. */
210 spin_lock(&call_lock);
211 132
212 allbutself = cpu_online_map; 133 allbutself = cpu_online_map;
213 cpu_clear(smp_processor_id(), allbutself); 134 cpu_clear(smp_processor_id(), allbutself);
214 135
215 cpus_and(mask, mask, allbutself);
216 cpus = cpus_weight(mask);
217
218 if (!cpus) {
219 spin_unlock(&call_lock);
220 return 0;
221 }
222
223 data.func = func;
224 data.info = info;
225 atomic_set(&data.started, 0);
226 data.wait = wait;
227 if (wait)
228 atomic_set(&data.finished, 0);
229
230 call_data = &data;
231 wmb();
232
233 /* Send a message to other CPUs */
234 if (cpus_equal(mask, allbutself) && 136 if (cpus_equal(mask, allbutself) &&
235 cpus_equal(cpu_online_map, cpu_callout_map)) 137 cpus_equal(cpu_online_map, cpu_callout_map))
236 send_IPI_allbutself(CALL_FUNCTION_VECTOR); 138 send_IPI_allbutself(CALL_FUNCTION_VECTOR);
237 else 139 else
238 send_IPI_mask(mask, CALL_FUNCTION_VECTOR); 140 send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
239
240 /* Wait for response */
241 while (atomic_read(&data.started) != cpus)
242 cpu_relax();
243
244 if (wait)
245 while (atomic_read(&data.finished) != cpus)
246 cpu_relax();
247 spin_unlock(&call_lock);
248
249 return 0;
250} 141}
251 142
252static void stop_this_cpu(void *dummy) 143static void stop_this_cpu(void *dummy)
@@ -268,18 +159,13 @@ static void stop_this_cpu(void *dummy)
268 159
269static void native_smp_send_stop(void) 160static void native_smp_send_stop(void)
270{ 161{
271 int nolock;
272 unsigned long flags; 162 unsigned long flags;
273 163
274 if (reboot_force) 164 if (reboot_force)
275 return; 165 return;
276 166
277 /* Don't deadlock on the call lock in panic */ 167 smp_call_function(stop_this_cpu, NULL, 0);
278 nolock = !spin_trylock(&call_lock);
279 local_irq_save(flags); 168 local_irq_save(flags);
280 __smp_call_function(stop_this_cpu, NULL, 0, 0);
281 if (!nolock)
282 spin_unlock(&call_lock);
283 disable_local_APIC(); 169 disable_local_APIC();
284 local_irq_restore(flags); 170 local_irq_restore(flags);
285} 171}
@@ -301,33 +187,28 @@ void smp_reschedule_interrupt(struct pt_regs *regs)
301 187
302void smp_call_function_interrupt(struct pt_regs *regs) 188void smp_call_function_interrupt(struct pt_regs *regs)
303{ 189{
304 void (*func) (void *info) = call_data->func;
305 void *info = call_data->info;
306 int wait = call_data->wait;
307
308 ack_APIC_irq(); 190 ack_APIC_irq();
309 /*
310 * Notify initiating CPU that I've grabbed the data and am
311 * about to execute the function
312 */
313 mb();
314 atomic_inc(&call_data->started);
315 /*
316 * At this point the info structure may be out of scope unless wait==1
317 */
318 irq_enter(); 191 irq_enter();
319 (*func)(info); 192 generic_smp_call_function_interrupt();
320#ifdef CONFIG_X86_32 193#ifdef CONFIG_X86_32
321 __get_cpu_var(irq_stat).irq_call_count++; 194 __get_cpu_var(irq_stat).irq_call_count++;
322#else 195#else
323 add_pda(irq_call_count, 1); 196 add_pda(irq_call_count, 1);
324#endif 197#endif
325 irq_exit(); 198 irq_exit();
199}
326 200
327 if (wait) { 201void smp_call_function_single_interrupt(struct pt_regs *regs)
328 mb(); 202{
329 atomic_inc(&call_data->finished); 203 ack_APIC_irq();
330 } 204 irq_enter();
205 generic_smp_call_function_single_interrupt();
206#ifdef CONFIG_X86_32
207 __get_cpu_var(irq_stat).irq_call_count++;
208#else
209 add_pda(irq_call_count, 1);
210#endif
211 irq_exit();
331} 212}
332 213
333struct smp_ops smp_ops = { 214struct smp_ops smp_ops = {
@@ -338,7 +219,8 @@ struct smp_ops smp_ops = {
338 219
339 .smp_send_stop = native_smp_send_stop, 220 .smp_send_stop = native_smp_send_stop,
340 .smp_send_reschedule = native_smp_send_reschedule, 221 .smp_send_reschedule = native_smp_send_reschedule,
341 .smp_call_function_mask = native_smp_call_function_mask, 222
223 .send_call_func_ipi = native_send_call_func_ipi,
224 .send_call_func_single_ipi = native_send_call_func_single_ipi,
342}; 225};
343EXPORT_SYMBOL_GPL(smp_ops); 226EXPORT_SYMBOL_GPL(smp_ops);
344
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index f35c2d8016ac..687376ab07e8 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -327,12 +327,12 @@ static void __cpuinit start_secondary(void *unused)
327 * lock helps us to not include this cpu in a currently in progress 327 * lock helps us to not include this cpu in a currently in progress
328 * smp_call_function(). 328 * smp_call_function().
329 */ 329 */
330 lock_ipi_call_lock(); 330 ipi_call_lock_irq();
331#ifdef CONFIG_X86_IO_APIC 331#ifdef CONFIG_X86_IO_APIC
332 setup_vector_irq(smp_processor_id()); 332 setup_vector_irq(smp_processor_id());
333#endif 333#endif
334 cpu_set(smp_processor_id(), cpu_online_map); 334 cpu_set(smp_processor_id(), cpu_online_map);
335 unlock_ipi_call_lock(); 335 ipi_call_unlock_irq();
336 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; 336 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
337 337
338 setup_secondary_clock(); 338 setup_secondary_clock();
diff --git a/arch/x86/kernel/smpcommon.c b/arch/x86/kernel/smpcommon.c
index 3449064d141a..99941b37eca0 100644
--- a/arch/x86/kernel/smpcommon.c
+++ b/arch/x86/kernel/smpcommon.c
@@ -25,59 +25,3 @@ __cpuinit void init_gdt(int cpu)
25 per_cpu(cpu_number, cpu) = cpu; 25 per_cpu(cpu_number, cpu) = cpu;
26} 26}
27#endif 27#endif
28
29/**
30 * smp_call_function(): Run a function on all other CPUs.
31 * @func: The function to run. This must be fast and non-blocking.
32 * @info: An arbitrary pointer to pass to the function.
33 * @nonatomic: Unused.
34 * @wait: If true, wait (atomically) until function has completed on other CPUs.
35 *
36 * Returns 0 on success, else a negative status code.
37 *
38 * If @wait is true, then returns once @func has returned; otherwise
39 * it returns just before the target cpu calls @func.
40 *
41 * You must not call this function with disabled interrupts or from a
42 * hardware interrupt handler or from a bottom half handler.
43 */
44int smp_call_function(void (*func) (void *info), void *info, int nonatomic,
45 int wait)
46{
47 return smp_call_function_mask(cpu_online_map, func, info, wait);
48}
49EXPORT_SYMBOL(smp_call_function);
50
51/**
52 * smp_call_function_single - Run a function on a specific CPU
53 * @cpu: The target CPU. Cannot be the calling CPU.
54 * @func: The function to run. This must be fast and non-blocking.
55 * @info: An arbitrary pointer to pass to the function.
56 * @nonatomic: Unused.
57 * @wait: If true, wait until function has completed on other CPUs.
58 *
59 * Returns 0 on success, else a negative status code.
60 *
61 * If @wait is true, then returns once @func has returned; otherwise
62 * it returns just before the target cpu calls @func.
63 */
64int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
65 int nonatomic, int wait)
66{
67 /* prevent preemption and reschedule on another processor */
68 int ret;
69 int me = get_cpu();
70 if (cpu == me) {
71 local_irq_disable();
72 func(info);
73 local_irq_enable();
74 put_cpu();
75 return 0;
76 }
77
78 ret = smp_call_function_mask(cpumask_of_cpu(cpu), func, info, wait);
79
80 put_cpu();
81 return ret;
82}
83EXPORT_SYMBOL(smp_call_function_single);
diff --git a/arch/x86/kernel/tlb_32.c b/arch/x86/kernel/tlb_32.c
index 9bb2363851af..fec1ecedc9b7 100644
--- a/arch/x86/kernel/tlb_32.c
+++ b/arch/x86/kernel/tlb_32.c
@@ -238,6 +238,6 @@ static void do_flush_tlb_all(void *info)
238 238
239void flush_tlb_all(void) 239void flush_tlb_all(void)
240{ 240{
241 on_each_cpu(do_flush_tlb_all, NULL, 1, 1); 241 on_each_cpu(do_flush_tlb_all, NULL, 1);
242} 242}
243 243
diff --git a/arch/x86/kernel/tlb_64.c b/arch/x86/kernel/tlb_64.c
index 5039d0f097a2..dcbf7a1159ea 100644
--- a/arch/x86/kernel/tlb_64.c
+++ b/arch/x86/kernel/tlb_64.c
@@ -275,5 +275,5 @@ static void do_flush_tlb_all(void *info)
275 275
276void flush_tlb_all(void) 276void flush_tlb_all(void)
277{ 277{
278 on_each_cpu(do_flush_tlb_all, NULL, 1, 1); 278 on_each_cpu(do_flush_tlb_all, NULL, 1);
279} 279}
diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c
index e50740d32314..0b8b6690a86d 100644
--- a/arch/x86/kernel/vsyscall_64.c
+++ b/arch/x86/kernel/vsyscall_64.c
@@ -279,7 +279,7 @@ cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
279{ 279{
280 long cpu = (long)arg; 280 long cpu = (long)arg;
281 if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) 281 if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
282 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 0, 1); 282 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 1);
283 return NOTIFY_DONE; 283 return NOTIFY_DONE;
284} 284}
285 285
@@ -302,7 +302,7 @@ static int __init vsyscall_init(void)
302#ifdef CONFIG_SYSCTL 302#ifdef CONFIG_SYSCTL
303 register_sysctl_table(kernel_root_table2); 303 register_sysctl_table(kernel_root_table2);
304#endif 304#endif
305 on_each_cpu(cpu_vsyscall_init, NULL, 0, 1); 305 on_each_cpu(cpu_vsyscall_init, NULL, 1);
306 hotcpu_notifier(cpu_vsyscall_notifier, 0); 306 hotcpu_notifier(cpu_vsyscall_notifier, 0);
307 return 0; 307 return 0;
308} 308}
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 540e95179074..10ce6ee4c491 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -335,7 +335,7 @@ static void vcpu_clear(struct vcpu_vmx *vmx)
335{ 335{
336 if (vmx->vcpu.cpu == -1) 336 if (vmx->vcpu.cpu == -1)
337 return; 337 return;
338 smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 0, 1); 338 smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 1);
339 vmx->launched = 0; 339 vmx->launched = 0;
340} 340}
341 341
@@ -2968,7 +2968,7 @@ static void vmx_free_vmcs(struct kvm_vcpu *vcpu)
2968 struct vcpu_vmx *vmx = to_vmx(vcpu); 2968 struct vcpu_vmx *vmx = to_vmx(vcpu);
2969 2969
2970 if (vmx->vmcs) { 2970 if (vmx->vmcs) {
2971 on_each_cpu(__vcpu_clear, vmx, 0, 1); 2971 on_each_cpu(__vcpu_clear, vmx, 1);
2972 free_vmcs(vmx->vmcs); 2972 free_vmcs(vmx->vmcs);
2973 vmx->vmcs = NULL; 2973 vmx->vmcs = NULL;
2974 } 2974 }
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 63a77caa59f1..0faa2546b1cd 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -4044,6 +4044,6 @@ void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
4044 * So need not to call smp_call_function_single() in that case. 4044 * So need not to call smp_call_function_single() in that case.
4045 */ 4045 */
4046 if (vcpu->guest_mode && vcpu->cpu != cpu) 4046 if (vcpu->guest_mode && vcpu->cpu != cpu)
4047 smp_call_function_single(ipi_pcpu, vcpu_kick_intr, vcpu, 0, 0); 4047 smp_call_function_single(ipi_pcpu, vcpu_kick_intr, vcpu, 0);
4048 put_cpu(); 4048 put_cpu();
4049} 4049}
diff --git a/arch/x86/lib/msr-on-cpu.c b/arch/x86/lib/msr-on-cpu.c
index 57d043fa893e..d5a2b39f882b 100644
--- a/arch/x86/lib/msr-on-cpu.c
+++ b/arch/x86/lib/msr-on-cpu.c
@@ -30,10 +30,10 @@ static int _rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h, int safe)
30 30
31 rv.msr_no = msr_no; 31 rv.msr_no = msr_no;
32 if (safe) { 32 if (safe) {
33 smp_call_function_single(cpu, __rdmsr_safe_on_cpu, &rv, 0, 1); 33 smp_call_function_single(cpu, __rdmsr_safe_on_cpu, &rv, 1);
34 err = rv.err; 34 err = rv.err;
35 } else { 35 } else {
36 smp_call_function_single(cpu, __rdmsr_on_cpu, &rv, 0, 1); 36 smp_call_function_single(cpu, __rdmsr_on_cpu, &rv, 1);
37 } 37 }
38 *l = rv.l; 38 *l = rv.l;
39 *h = rv.h; 39 *h = rv.h;
@@ -64,10 +64,10 @@ static int _wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h, int safe)
64 rv.l = l; 64 rv.l = l;
65 rv.h = h; 65 rv.h = h;
66 if (safe) { 66 if (safe) {
67 smp_call_function_single(cpu, __wrmsr_safe_on_cpu, &rv, 0, 1); 67 smp_call_function_single(cpu, __wrmsr_safe_on_cpu, &rv, 1);
68 err = rv.err; 68 err = rv.err;
69 } else { 69 } else {
70 smp_call_function_single(cpu, __wrmsr_on_cpu, &rv, 0, 1); 70 smp_call_function_single(cpu, __wrmsr_on_cpu, &rv, 1);
71 } 71 }
72 72
73 return err; 73 return err;
diff --git a/arch/x86/mach-voyager/voyager_smp.c b/arch/x86/mach-voyager/voyager_smp.c
index 8dedd01e909f..ee0fba092157 100644
--- a/arch/x86/mach-voyager/voyager_smp.c
+++ b/arch/x86/mach-voyager/voyager_smp.c
@@ -950,94 +950,24 @@ static void smp_stop_cpu_function(void *dummy)
950 halt(); 950 halt();
951} 951}
952 952
953static DEFINE_SPINLOCK(call_lock);
954
955struct call_data_struct {
956 void (*func) (void *info);
957 void *info;
958 volatile unsigned long started;
959 volatile unsigned long finished;
960 int wait;
961};
962
963static struct call_data_struct *call_data;
964
965/* execute a thread on a new CPU. The function to be called must be 953/* execute a thread on a new CPU. The function to be called must be
966 * previously set up. This is used to schedule a function for 954 * previously set up. This is used to schedule a function for
967 * execution on all CPUs - set up the function then broadcast a 955 * execution on all CPUs - set up the function then broadcast a
968 * function_interrupt CPI to come here on each CPU */ 956 * function_interrupt CPI to come here on each CPU */
969static void smp_call_function_interrupt(void) 957static void smp_call_function_interrupt(void)
970{ 958{
971 void (*func) (void *info) = call_data->func;
972 void *info = call_data->info;
973 /* must take copy of wait because call_data may be replaced
974 * unless the function is waiting for us to finish */
975 int wait = call_data->wait;
976 __u8 cpu = smp_processor_id();
977
978 /*
979 * Notify initiating CPU that I've grabbed the data and am
980 * about to execute the function
981 */
982 mb();
983 if (!test_and_clear_bit(cpu, &call_data->started)) {
984 /* If the bit wasn't set, this could be a replay */
985 printk(KERN_WARNING "VOYAGER SMP: CPU %d received call funtion"
986 " with no call pending\n", cpu);
987 return;
988 }
989 /*
990 * At this point the info structure may be out of scope unless wait==1
991 */
992 irq_enter(); 959 irq_enter();
993 (*func) (info); 960 generic_smp_call_function_interrupt();
994 __get_cpu_var(irq_stat).irq_call_count++; 961 __get_cpu_var(irq_stat).irq_call_count++;
995 irq_exit(); 962 irq_exit();
996 if (wait) {
997 mb();
998 clear_bit(cpu, &call_data->finished);
999 }
1000} 963}
1001 964
1002static int 965static void smp_call_function_single_interrupt(void)
1003voyager_smp_call_function_mask(cpumask_t cpumask,
1004 void (*func) (void *info), void *info, int wait)
1005{ 966{
1006 struct call_data_struct data; 967 irq_enter();
1007 u32 mask = cpus_addr(cpumask)[0]; 968 generic_smp_call_function_single_interrupt();
1008 969 __get_cpu_var(irq_stat).irq_call_count++;
1009 mask &= ~(1 << smp_processor_id()); 970 irq_exit();
1010
1011 if (!mask)
1012 return 0;
1013
1014 /* Can deadlock when called with interrupts disabled */
1015 WARN_ON(irqs_disabled());
1016
1017 data.func = func;
1018 data.info = info;
1019 data.started = mask;
1020 data.wait = wait;
1021 if (wait)
1022 data.finished = mask;
1023
1024 spin_lock(&call_lock);
1025 call_data = &data;
1026 wmb();
1027 /* Send a message to all other CPUs and wait for them to respond */
1028 send_CPI(mask, VIC_CALL_FUNCTION_CPI);
1029
1030 /* Wait for response */
1031 while (data.started)
1032 barrier();
1033
1034 if (wait)
1035 while (data.finished)
1036 barrier();
1037
1038 spin_unlock(&call_lock);
1039
1040 return 0;
1041} 971}
1042 972
1043/* Sorry about the name. In an APIC based system, the APICs 973/* Sorry about the name. In an APIC based system, the APICs
@@ -1094,6 +1024,12 @@ void smp_qic_call_function_interrupt(struct pt_regs *regs)
1094 smp_call_function_interrupt(); 1024 smp_call_function_interrupt();
1095} 1025}
1096 1026
1027void smp_qic_call_function_single_interrupt(struct pt_regs *regs)
1028{
1029 ack_QIC_CPI(QIC_CALL_FUNCTION_SINGLE_CPI);
1030 smp_call_function_single_interrupt();
1031}
1032
1097void smp_vic_cpi_interrupt(struct pt_regs *regs) 1033void smp_vic_cpi_interrupt(struct pt_regs *regs)
1098{ 1034{
1099 struct pt_regs *old_regs = set_irq_regs(regs); 1035 struct pt_regs *old_regs = set_irq_regs(regs);
@@ -1114,6 +1050,8 @@ void smp_vic_cpi_interrupt(struct pt_regs *regs)
1114 smp_enable_irq_interrupt(); 1050 smp_enable_irq_interrupt();
1115 if (test_and_clear_bit(VIC_CALL_FUNCTION_CPI, &vic_cpi_mailbox[cpu])) 1051 if (test_and_clear_bit(VIC_CALL_FUNCTION_CPI, &vic_cpi_mailbox[cpu]))
1116 smp_call_function_interrupt(); 1052 smp_call_function_interrupt();
1053 if (test_and_clear_bit(VIC_CALL_FUNCTION_SINGLE_CPI, &vic_cpi_mailbox[cpu]))
1054 smp_call_function_single_interrupt();
1117 set_irq_regs(old_regs); 1055 set_irq_regs(old_regs);
1118} 1056}
1119 1057
@@ -1129,7 +1067,7 @@ static void do_flush_tlb_all(void *info)
1129/* flush the TLB of every active CPU in the system */ 1067/* flush the TLB of every active CPU in the system */
1130void flush_tlb_all(void) 1068void flush_tlb_all(void)
1131{ 1069{
1132 on_each_cpu(do_flush_tlb_all, 0, 1, 1); 1070 on_each_cpu(do_flush_tlb_all, 0, 1);
1133} 1071}
1134 1072
1135/* send a reschedule CPI to one CPU by physical CPU number*/ 1073/* send a reschedule CPI to one CPU by physical CPU number*/
@@ -1161,7 +1099,7 @@ int safe_smp_processor_id(void)
1161/* broadcast a halt to all other CPUs */ 1099/* broadcast a halt to all other CPUs */
1162static void voyager_smp_send_stop(void) 1100static void voyager_smp_send_stop(void)
1163{ 1101{
1164 smp_call_function(smp_stop_cpu_function, NULL, 1, 1); 1102 smp_call_function(smp_stop_cpu_function, NULL, 1);
1165} 1103}
1166 1104
1167/* this function is triggered in time.c when a clock tick fires 1105/* this function is triggered in time.c when a clock tick fires
@@ -1848,5 +1786,7 @@ struct smp_ops smp_ops = {
1848 1786
1849 .smp_send_stop = voyager_smp_send_stop, 1787 .smp_send_stop = voyager_smp_send_stop,
1850 .smp_send_reschedule = voyager_smp_send_reschedule, 1788 .smp_send_reschedule = voyager_smp_send_reschedule,
1851 .smp_call_function_mask = voyager_smp_call_function_mask, 1789
1790 .send_call_func_ipi = native_send_call_func_ipi,
1791 .send_call_func_single_ipi = native_send_call_func_single_ipi,
1852}; 1792};
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index 47f4e2e4a096..65c6e46bf059 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -141,7 +141,7 @@ static void cpa_flush_all(unsigned long cache)
141{ 141{
142 BUG_ON(irqs_disabled()); 142 BUG_ON(irqs_disabled());
143 143
144 on_each_cpu(__cpa_flush_all, (void *) cache, 1, 1); 144 on_each_cpu(__cpa_flush_all, (void *) cache, 1);
145} 145}
146 146
147static void __cpa_flush_range(void *arg) 147static void __cpa_flush_range(void *arg)
@@ -162,7 +162,7 @@ static void cpa_flush_range(unsigned long start, int numpages, int cache)
162 BUG_ON(irqs_disabled()); 162 BUG_ON(irqs_disabled());
163 WARN_ON(PAGE_ALIGN(start) != start); 163 WARN_ON(PAGE_ALIGN(start) != start);
164 164
165 on_each_cpu(__cpa_flush_range, NULL, 1, 1); 165 on_each_cpu(__cpa_flush_range, NULL, 1);
166 166
167 if (!cache) 167 if (!cache)
168 return; 168 return;
diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c
index 2b6ad5b9f9d5..7f3329b55d2e 100644
--- a/arch/x86/oprofile/nmi_int.c
+++ b/arch/x86/oprofile/nmi_int.c
@@ -218,8 +218,8 @@ static int nmi_setup(void)
218 } 218 }
219 219
220 } 220 }
221 on_each_cpu(nmi_save_registers, NULL, 0, 1); 221 on_each_cpu(nmi_save_registers, NULL, 1);
222 on_each_cpu(nmi_cpu_setup, NULL, 0, 1); 222 on_each_cpu(nmi_cpu_setup, NULL, 1);
223 nmi_enabled = 1; 223 nmi_enabled = 1;
224 return 0; 224 return 0;
225} 225}
@@ -271,7 +271,7 @@ static void nmi_shutdown(void)
271{ 271{
272 struct op_msrs *msrs = &get_cpu_var(cpu_msrs); 272 struct op_msrs *msrs = &get_cpu_var(cpu_msrs);
273 nmi_enabled = 0; 273 nmi_enabled = 0;
274 on_each_cpu(nmi_cpu_shutdown, NULL, 0, 1); 274 on_each_cpu(nmi_cpu_shutdown, NULL, 1);
275 unregister_die_notifier(&profile_exceptions_nb); 275 unregister_die_notifier(&profile_exceptions_nb);
276 model->shutdown(msrs); 276 model->shutdown(msrs);
277 free_msrs(); 277 free_msrs();
@@ -286,7 +286,7 @@ static void nmi_cpu_start(void *dummy)
286 286
287static int nmi_start(void) 287static int nmi_start(void)
288{ 288{
289 on_each_cpu(nmi_cpu_start, NULL, 0, 1); 289 on_each_cpu(nmi_cpu_start, NULL, 1);
290 return 0; 290 return 0;
291} 291}
292 292
@@ -298,7 +298,7 @@ static void nmi_cpu_stop(void *dummy)
298 298
299static void nmi_stop(void) 299static void nmi_stop(void)
300{ 300{
301 on_each_cpu(nmi_cpu_stop, NULL, 0, 1); 301 on_each_cpu(nmi_cpu_stop, NULL, 1);
302} 302}
303 303
304struct op_counter_config counter_config[OP_MAX_COUNTER]; 304struct op_counter_config counter_config[OP_MAX_COUNTER];
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index dcd4e51f2f16..bb508456ef52 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -1214,7 +1214,9 @@ static const struct smp_ops xen_smp_ops __initdata = {
1214 1214
1215 .smp_send_stop = xen_smp_send_stop, 1215 .smp_send_stop = xen_smp_send_stop,
1216 .smp_send_reschedule = xen_smp_send_reschedule, 1216 .smp_send_reschedule = xen_smp_send_reschedule,
1217 .smp_call_function_mask = xen_smp_call_function_mask, 1217
1218 .send_call_func_ipi = xen_smp_send_call_function_ipi,
1219 .send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
1218}; 1220};
1219#endif /* CONFIG_SMP */ 1221#endif /* CONFIG_SMP */
1220 1222
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 42b3b9ed641d..ff0aa74afaa1 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -796,7 +796,7 @@ static void drop_mm_ref(struct mm_struct *mm)
796 } 796 }
797 797
798 if (!cpus_empty(mask)) 798 if (!cpus_empty(mask))
799 xen_smp_call_function_mask(mask, drop_other_mm_ref, mm, 1); 799 smp_call_function_mask(mask, drop_other_mm_ref, mm, 1);
800} 800}
801#else 801#else
802static void drop_mm_ref(struct mm_struct *mm) 802static void drop_mm_ref(struct mm_struct *mm)
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index d2e3c20127d7..233156f39b7f 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -36,27 +36,14 @@
36#include "mmu.h" 36#include "mmu.h"
37 37
38cpumask_t xen_cpu_initialized_map; 38cpumask_t xen_cpu_initialized_map;
39static DEFINE_PER_CPU(int, resched_irq) = -1;
40static DEFINE_PER_CPU(int, callfunc_irq) = -1;
41static DEFINE_PER_CPU(int, debug_irq) = -1;
42
43/*
44 * Structure and data for smp_call_function(). This is designed to minimise
45 * static memory requirements. It also looks cleaner.
46 */
47static DEFINE_SPINLOCK(call_lock);
48 39
49struct call_data_struct { 40static DEFINE_PER_CPU(int, resched_irq);
50 void (*func) (void *info); 41static DEFINE_PER_CPU(int, callfunc_irq);
51 void *info; 42static DEFINE_PER_CPU(int, callfuncsingle_irq);
52 atomic_t started; 43static DEFINE_PER_CPU(int, debug_irq) = -1;
53 atomic_t finished;
54 int wait;
55};
56 44
57static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id); 45static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
58 46static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
59static struct call_data_struct *call_data;
60 47
61/* 48/*
62 * Reschedule call back. Nothing to do, 49 * Reschedule call back. Nothing to do,
@@ -128,6 +115,17 @@ static int xen_smp_intr_init(unsigned int cpu)
128 goto fail; 115 goto fail;
129 per_cpu(debug_irq, cpu) = rc; 116 per_cpu(debug_irq, cpu) = rc;
130 117
118 callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
119 rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
120 cpu,
121 xen_call_function_single_interrupt,
122 IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
123 callfunc_name,
124 NULL);
125 if (rc < 0)
126 goto fail;
127 per_cpu(callfuncsingle_irq, cpu) = rc;
128
131 return 0; 129 return 0;
132 130
133 fail: 131 fail:
@@ -137,6 +135,9 @@ static int xen_smp_intr_init(unsigned int cpu)
137 unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL); 135 unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL);
138 if (per_cpu(debug_irq, cpu) >= 0) 136 if (per_cpu(debug_irq, cpu) >= 0)
139 unbind_from_irqhandler(per_cpu(debug_irq, cpu), NULL); 137 unbind_from_irqhandler(per_cpu(debug_irq, cpu), NULL);
138 if (per_cpu(callfuncsingle_irq, cpu) >= 0)
139 unbind_from_irqhandler(per_cpu(callfuncsingle_irq, cpu), NULL);
140
140 return rc; 141 return rc;
141} 142}
142 143
@@ -336,7 +337,7 @@ static void stop_self(void *v)
336 337
337void xen_smp_send_stop(void) 338void xen_smp_send_stop(void)
338{ 339{
339 smp_call_function(stop_self, NULL, 0, 0); 340 smp_call_function(stop_self, NULL, 0);
340} 341}
341 342
342void xen_smp_send_reschedule(int cpu) 343void xen_smp_send_reschedule(int cpu)
@@ -344,7 +345,6 @@ void xen_smp_send_reschedule(int cpu)
344 xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR); 345 xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
345} 346}
346 347
347
348static void xen_send_IPI_mask(cpumask_t mask, enum ipi_vector vector) 348static void xen_send_IPI_mask(cpumask_t mask, enum ipi_vector vector)
349{ 349{
350 unsigned cpu; 350 unsigned cpu;
@@ -355,83 +355,42 @@ static void xen_send_IPI_mask(cpumask_t mask, enum ipi_vector vector)
355 xen_send_IPI_one(cpu, vector); 355 xen_send_IPI_one(cpu, vector);
356} 356}
357 357
358void xen_smp_send_call_function_ipi(cpumask_t mask)
359{
360 int cpu;
361
362 xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
363
364 /* Make sure other vcpus get a chance to run if they need to. */
365 for_each_cpu_mask(cpu, mask) {
366 if (xen_vcpu_stolen(cpu)) {
367 HYPERVISOR_sched_op(SCHEDOP_yield, 0);
368 break;
369 }
370 }
371}
372
373void xen_smp_send_call_function_single_ipi(int cpu)
374{
375 xen_send_IPI_mask(cpumask_of_cpu(cpu), XEN_CALL_FUNCTION_SINGLE_VECTOR);
376}
377
358static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id) 378static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
359{ 379{
360 void (*func) (void *info) = call_data->func;
361 void *info = call_data->info;
362 int wait = call_data->wait;
363
364 /*
365 * Notify initiating CPU that I've grabbed the data and am
366 * about to execute the function
367 */
368 mb();
369 atomic_inc(&call_data->started);
370 /*
371 * At this point the info structure may be out of scope unless wait==1
372 */
373 irq_enter(); 380 irq_enter();
374 (*func)(info); 381 generic_smp_call_function_interrupt();
375 __get_cpu_var(irq_stat).irq_call_count++; 382 __get_cpu_var(irq_stat).irq_call_count++;
376 irq_exit(); 383 irq_exit();
377 384
378 if (wait) {
379 mb(); /* commit everything before setting finished */
380 atomic_inc(&call_data->finished);
381 }
382
383 return IRQ_HANDLED; 385 return IRQ_HANDLED;
384} 386}
385 387
386int xen_smp_call_function_mask(cpumask_t mask, void (*func)(void *), 388static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
387 void *info, int wait)
388{ 389{
389 struct call_data_struct data; 390 irq_enter();
390 int cpus, cpu; 391 generic_smp_call_function_single_interrupt();
391 bool yield; 392 __get_cpu_var(irq_stat).irq_call_count++;
392 393 irq_exit();
393 /* Holding any lock stops cpus from going down. */
394 spin_lock(&call_lock);
395
396 cpu_clear(smp_processor_id(), mask);
397
398 cpus = cpus_weight(mask);
399 if (!cpus) {
400 spin_unlock(&call_lock);
401 return 0;
402 }
403
404 /* Can deadlock when called with interrupts disabled */
405 WARN_ON(irqs_disabled());
406
407 data.func = func;
408 data.info = info;
409 atomic_set(&data.started, 0);
410 data.wait = wait;
411 if (wait)
412 atomic_set(&data.finished, 0);
413
414 call_data = &data;
415 mb(); /* write everything before IPI */
416
417 /* Send a message to other CPUs and wait for them to respond */
418 xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
419
420 /* Make sure other vcpus get a chance to run if they need to. */
421 yield = false;
422 for_each_cpu_mask(cpu, mask)
423 if (xen_vcpu_stolen(cpu))
424 yield = true;
425
426 if (yield)
427 HYPERVISOR_sched_op(SCHEDOP_yield, 0);
428
429 /* Wait for response */
430 while (atomic_read(&data.started) != cpus ||
431 (wait && atomic_read(&data.finished) != cpus))
432 cpu_relax();
433
434 spin_unlock(&call_lock);
435 394
436 return 0; 395 return IRQ_HANDLED;
437} 396}
diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h
index d852ddbb3448..6f4b1045c1c2 100644
--- a/arch/x86/xen/xen-ops.h
+++ b/arch/x86/xen/xen-ops.h
@@ -55,13 +55,8 @@ void xen_smp_cpus_done(unsigned int max_cpus);
55 55
56void xen_smp_send_stop(void); 56void xen_smp_send_stop(void);
57void xen_smp_send_reschedule(int cpu); 57void xen_smp_send_reschedule(int cpu);
58int xen_smp_call_function (void (*func) (void *info), void *info, int nonatomic, 58void xen_smp_send_call_function_ipi(cpumask_t mask);
59 int wait); 59void xen_smp_send_call_function_single_ipi(int cpu);
60int xen_smp_call_function_single(int cpu, void (*func) (void *info), void *info,
61 int nonatomic, int wait);
62
63int xen_smp_call_function_mask(cpumask_t mask, void (*func)(void *),
64 void *info, int wait);
65 60
66extern cpumask_t xen_cpu_initialized_map; 61extern cpumask_t xen_cpu_initialized_map;
67 62