aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/s390/include/asm/lowcore.h100
-rw-r--r--arch/s390/include/asm/sigp.h132
-rw-r--r--arch/s390/include/asm/smp.h63
-rw-r--r--arch/s390/include/asm/vdso.h4
-rw-r--r--arch/s390/kernel/Makefile2
-rw-r--r--arch/s390/kernel/asm-offsets.c17
-rw-r--r--arch/s390/kernel/entry.S81
-rw-r--r--arch/s390/kernel/entry.h9
-rw-r--r--arch/s390/kernel/entry64.S72
-rw-r--r--arch/s390/kernel/ipl.c21
-rw-r--r--arch/s390/kernel/machine_kexec.c43
-rw-r--r--arch/s390/kernel/setup.c56
-rw-r--r--arch/s390/kernel/smp.c1083
-rw-r--r--arch/s390/kernel/switch_cpu.S58
-rw-r--r--arch/s390/kernel/switch_cpu64.S51
-rw-r--r--arch/s390/kernel/swsusp_asm64.S12
-rw-r--r--arch/s390/kernel/topology.c8
-rw-r--r--arch/s390/kernel/vdso.c28
-rw-r--r--arch/s390/kernel/vtime.c3
-rw-r--r--arch/s390/lib/spinlock.c30
-rw-r--r--drivers/s390/char/sclp_quiesce.c1
-rw-r--r--drivers/s390/char/zcore.c1
22 files changed, 739 insertions, 1136 deletions
diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h
index 0831449e87a3..4e69563bc956 100644
--- a/arch/s390/include/asm/lowcore.h
+++ b/arch/s390/include/asm/lowcore.h
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright IBM Corp. 1999,2010 2 * Copyright IBM Corp. 1999,2012
3 * Author(s): Hartmut Penner <hp@de.ibm.com>, 3 * Author(s): Hartmut Penner <hp@de.ibm.com>,
4 * Martin Schwidefsky <schwidefsky@de.ibm.com>, 4 * Martin Schwidefsky <schwidefsky@de.ibm.com>,
5 * Denis Joseph Barrow, 5 * Denis Joseph Barrow,
@@ -12,14 +12,6 @@
12#include <asm/ptrace.h> 12#include <asm/ptrace.h>
13#include <asm/cpu.h> 13#include <asm/cpu.h>
14 14
15void restart_int_handler(void);
16void ext_int_handler(void);
17void system_call(void);
18void pgm_check_handler(void);
19void mcck_int_handler(void);
20void io_int_handler(void);
21void psw_restart_int_handler(void);
22
23#ifdef CONFIG_32BIT 15#ifdef CONFIG_32BIT
24 16
25#define LC_ORDER 0 17#define LC_ORDER 0
@@ -117,32 +109,37 @@ struct _lowcore {
117 __u64 steal_timer; /* 0x0288 */ 109 __u64 steal_timer; /* 0x0288 */
118 __u64 last_update_timer; /* 0x0290 */ 110 __u64 last_update_timer; /* 0x0290 */
119 __u64 last_update_clock; /* 0x0298 */ 111 __u64 last_update_clock; /* 0x0298 */
112 __u64 int_clock; /* 0x02a0 */
113 __u64 mcck_clock; /* 0x02a8 */
114 __u64 clock_comparator; /* 0x02b0 */
120 115
121 /* Current process. */ 116 /* Current process. */
122 __u32 current_task; /* 0x02a0 */ 117 __u32 current_task; /* 0x02b8 */
123 __u32 thread_info; /* 0x02a4 */ 118 __u32 thread_info; /* 0x02bc */
124 __u32 kernel_stack; /* 0x02a8 */ 119 __u32 kernel_stack; /* 0x02c0 */
120
121 /* Interrupt, panic and restart stack. */
122 __u32 async_stack; /* 0x02c4 */
123 __u32 panic_stack; /* 0x02c8 */
124 __u32 restart_stack; /* 0x02cc */
125 125
126 /* Interrupt and panic stack. */ 126 /* Restart function and parameter. */
127 __u32 async_stack; /* 0x02ac */ 127 __u32 restart_fn; /* 0x02d0 */
128 __u32 panic_stack; /* 0x02b0 */ 128 __u32 restart_data; /* 0x02d4 */
129 __u32 restart_source; /* 0x02d8 */
129 130
130 /* Address space pointer. */ 131 /* Address space pointer. */
131 __u32 kernel_asce; /* 0x02b4 */ 132 __u32 kernel_asce; /* 0x02dc */
132 __u32 user_asce; /* 0x02b8 */ 133 __u32 user_asce; /* 0x02e0 */
133 __u32 current_pid; /* 0x02bc */ 134 __u32 current_pid; /* 0x02e4 */
134 135
135 /* SMP info area */ 136 /* SMP info area */
136 __u32 cpu_nr; /* 0x02c0 */ 137 __u32 cpu_nr; /* 0x02e8 */
137 __u32 softirq_pending; /* 0x02c4 */ 138 __u32 softirq_pending; /* 0x02ec */
138 __u32 percpu_offset; /* 0x02c8 */ 139 __u32 percpu_offset; /* 0x02f0 */
139 __u32 ext_call_fast; /* 0x02cc */ 140 __u32 machine_flags; /* 0x02f4 */
140 __u64 int_clock; /* 0x02d0 */ 141 __u32 ftrace_func; /* 0x02f8 */
141 __u64 mcck_clock; /* 0x02d8 */ 142 __u8 pad_0x02fc[0x0300-0x02fc]; /* 0x02fc */
142 __u64 clock_comparator; /* 0x02e0 */
143 __u32 machine_flags; /* 0x02e8 */
144 __u32 ftrace_func; /* 0x02ec */
145 __u8 pad_0x02f8[0x0300-0x02f0]; /* 0x02f0 */
146 143
147 /* Interrupt response block */ 144 /* Interrupt response block */
148 __u8 irb[64]; /* 0x0300 */ 145 __u8 irb[64]; /* 0x0300 */
@@ -254,34 +251,39 @@ struct _lowcore {
254 __u64 steal_timer; /* 0x02e0 */ 251 __u64 steal_timer; /* 0x02e0 */
255 __u64 last_update_timer; /* 0x02e8 */ 252 __u64 last_update_timer; /* 0x02e8 */
256 __u64 last_update_clock; /* 0x02f0 */ 253 __u64 last_update_clock; /* 0x02f0 */
254 __u64 int_clock; /* 0x02f8 */
255 __u64 mcck_clock; /* 0x0300 */
256 __u64 clock_comparator; /* 0x0308 */
257 257
258 /* Current process. */ 258 /* Current process. */
259 __u64 current_task; /* 0x02f8 */ 259 __u64 current_task; /* 0x0310 */
260 __u64 thread_info; /* 0x0300 */ 260 __u64 thread_info; /* 0x0318 */
261 __u64 kernel_stack; /* 0x0308 */ 261 __u64 kernel_stack; /* 0x0320 */
262
263 /* Interrupt, panic and restart stack. */
264 __u64 async_stack; /* 0x0328 */
265 __u64 panic_stack; /* 0x0330 */
266 __u64 restart_stack; /* 0x0338 */
262 267
263 /* Interrupt and panic stack. */ 268 /* Restart function and parameter. */
264 __u64 async_stack; /* 0x0310 */ 269 __u64 restart_fn; /* 0x0340 */
265 __u64 panic_stack; /* 0x0318 */ 270 __u64 restart_data; /* 0x0348 */
271 __u64 restart_source; /* 0x0350 */
266 272
267 /* Address space pointer. */ 273 /* Address space pointer. */
268 __u64 kernel_asce; /* 0x0320 */ 274 __u64 kernel_asce; /* 0x0358 */
269 __u64 user_asce; /* 0x0328 */ 275 __u64 user_asce; /* 0x0360 */
270 __u64 current_pid; /* 0x0330 */ 276 __u64 current_pid; /* 0x0368 */
271 277
272 /* SMP info area */ 278 /* SMP info area */
273 __u32 cpu_nr; /* 0x0338 */ 279 __u32 cpu_nr; /* 0x0370 */
274 __u32 softirq_pending; /* 0x033c */ 280 __u32 softirq_pending; /* 0x0374 */
275 __u64 percpu_offset; /* 0x0340 */ 281 __u64 percpu_offset; /* 0x0378 */
276 __u64 ext_call_fast; /* 0x0348 */ 282 __u64 vdso_per_cpu_data; /* 0x0380 */
277 __u64 int_clock; /* 0x0350 */ 283 __u64 machine_flags; /* 0x0388 */
278 __u64 mcck_clock; /* 0x0358 */ 284 __u64 ftrace_func; /* 0x0390 */
279 __u64 clock_comparator; /* 0x0360 */ 285 __u64 gmap; /* 0x0398 */
280 __u64 vdso_per_cpu_data; /* 0x0368 */ 286 __u8 pad_0x03a0[0x0400-0x03a0]; /* 0x03a0 */
281 __u64 machine_flags; /* 0x0370 */
282 __u64 ftrace_func; /* 0x0378 */
283 __u64 gmap; /* 0x0380 */
284 __u8 pad_0x0388[0x0400-0x0388]; /* 0x0388 */
285 287
286 /* Interrupt response block. */ 288 /* Interrupt response block. */
287 __u8 irb[64]; /* 0x0400 */ 289 __u8 irb[64]; /* 0x0400 */
diff --git a/arch/s390/include/asm/sigp.h b/arch/s390/include/asm/sigp.h
deleted file mode 100644
index 7040b8567cd0..000000000000
--- a/arch/s390/include/asm/sigp.h
+++ /dev/null
@@ -1,132 +0,0 @@
1/*
2 * Routines and structures for signalling other processors.
3 *
4 * Copyright IBM Corp. 1999,2010
5 * Author(s): Denis Joseph Barrow,
6 * Martin Schwidefsky <schwidefsky@de.ibm.com>,
7 * Heiko Carstens <heiko.carstens@de.ibm.com>,
8 */
9
10#ifndef __ASM_SIGP_H
11#define __ASM_SIGP_H
12
13#include <asm/system.h>
14
15/* Get real cpu address from logical cpu number. */
16extern unsigned short __cpu_logical_map[];
17
18static inline int cpu_logical_map(int cpu)
19{
20#ifdef CONFIG_SMP
21 return __cpu_logical_map[cpu];
22#else
23 return stap();
24#endif
25}
26
27enum {
28 sigp_sense = 1,
29 sigp_external_call = 2,
30 sigp_emergency_signal = 3,
31 sigp_start = 4,
32 sigp_stop = 5,
33 sigp_restart = 6,
34 sigp_stop_and_store_status = 9,
35 sigp_initial_cpu_reset = 11,
36 sigp_cpu_reset = 12,
37 sigp_set_prefix = 13,
38 sigp_store_status_at_address = 14,
39 sigp_store_extended_status_at_address = 15,
40 sigp_set_architecture = 18,
41 sigp_conditional_emergency_signal = 19,
42 sigp_sense_running = 21,
43};
44
45enum {
46 sigp_order_code_accepted = 0,
47 sigp_status_stored = 1,
48 sigp_busy = 2,
49 sigp_not_operational = 3,
50};
51
52/*
53 * Definitions for external call.
54 */
55enum {
56 ec_schedule = 0,
57 ec_call_function,
58 ec_call_function_single,
59 ec_stop_cpu,
60};
61
62/*
63 * Signal processor.
64 */
65static inline int raw_sigp(u16 cpu, int order)
66{
67 register unsigned long reg1 asm ("1") = 0;
68 int ccode;
69
70 asm volatile(
71 " sigp %1,%2,0(%3)\n"
72 " ipm %0\n"
73 " srl %0,28\n"
74 : "=d" (ccode)
75 : "d" (reg1), "d" (cpu),
76 "a" (order) : "cc" , "memory");
77 return ccode;
78}
79
80/*
81 * Signal processor with parameter.
82 */
83static inline int raw_sigp_p(u32 parameter, u16 cpu, int order)
84{
85 register unsigned int reg1 asm ("1") = parameter;
86 int ccode;
87
88 asm volatile(
89 " sigp %1,%2,0(%3)\n"
90 " ipm %0\n"
91 " srl %0,28\n"
92 : "=d" (ccode)
93 : "d" (reg1), "d" (cpu),
94 "a" (order) : "cc" , "memory");
95 return ccode;
96}
97
98/*
99 * Signal processor with parameter and return status.
100 */
101static inline int raw_sigp_ps(u32 *status, u32 parm, u16 cpu, int order)
102{
103 register unsigned int reg1 asm ("1") = parm;
104 int ccode;
105
106 asm volatile(
107 " sigp %1,%2,0(%3)\n"
108 " ipm %0\n"
109 " srl %0,28\n"
110 : "=d" (ccode), "+d" (reg1)
111 : "d" (cpu), "a" (order)
112 : "cc" , "memory");
113 *status = reg1;
114 return ccode;
115}
116
117static inline int sigp(int cpu, int order)
118{
119 return raw_sigp(cpu_logical_map(cpu), order);
120}
121
122static inline int sigp_p(u32 parameter, int cpu, int order)
123{
124 return raw_sigp_p(parameter, cpu_logical_map(cpu), order);
125}
126
127static inline int sigp_ps(u32 *status, u32 parm, int cpu, int order)
128{
129 return raw_sigp_ps(status, parm, cpu_logical_map(cpu), order);
130}
131
132#endif /* __ASM_SIGP_H */
diff --git a/arch/s390/include/asm/smp.h b/arch/s390/include/asm/smp.h
index c32e9123b40c..797f78729680 100644
--- a/arch/s390/include/asm/smp.h
+++ b/arch/s390/include/asm/smp.h
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright IBM Corp. 1999,2009 2 * Copyright IBM Corp. 1999,2012
3 * Author(s): Denis Joseph Barrow, 3 * Author(s): Denis Joseph Barrow,
4 * Martin Schwidefsky <schwidefsky@de.ibm.com>, 4 * Martin Schwidefsky <schwidefsky@de.ibm.com>,
5 * Heiko Carstens <heiko.carstens@de.ibm.com>, 5 * Heiko Carstens <heiko.carstens@de.ibm.com>,
@@ -10,71 +10,52 @@
10#ifdef CONFIG_SMP 10#ifdef CONFIG_SMP
11 11
12#include <asm/system.h> 12#include <asm/system.h>
13#include <asm/sigp.h>
14
15extern void machine_restart_smp(char *);
16extern void machine_halt_smp(void);
17extern void machine_power_off_smp(void);
18 13
19#define raw_smp_processor_id() (S390_lowcore.cpu_nr) 14#define raw_smp_processor_id() (S390_lowcore.cpu_nr)
20 15
21extern int __cpu_disable (void);
22extern void __cpu_die (unsigned int cpu);
23extern int __cpu_up (unsigned int cpu);
24
25extern struct mutex smp_cpu_state_mutex; 16extern struct mutex smp_cpu_state_mutex;
17extern struct save_area *zfcpdump_save_areas[NR_CPUS + 1];
18
19extern int __cpu_up(unsigned int cpu);
26 20
27extern void arch_send_call_function_single_ipi(int cpu); 21extern void arch_send_call_function_single_ipi(int cpu);
28extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); 22extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);
29 23
30extern struct save_area *zfcpdump_save_areas[NR_CPUS + 1]; 24extern void smp_call_online_cpu(void (*func)(void *), void *);
31 25extern void smp_call_ipl_cpu(void (*func)(void *), void *);
32extern void smp_switch_to_ipl_cpu(void (*func)(void *), void *);
33extern void smp_switch_to_cpu(void (*)(void *), void *, unsigned long sp,
34 int from, int to);
35extern void smp_restart_with_online_cpu(void);
36extern void smp_restart_cpu(void);
37 26
38/* 27extern int smp_find_processor_id(u16 address);
39 * returns 1 if (virtual) cpu is scheduled 28extern int smp_store_status(int cpu);
40 * returns 0 otherwise 29extern int smp_vcpu_scheduled(int cpu);
41 */ 30extern void smp_yield_cpu(int cpu);
42static inline int smp_vcpu_scheduled(int cpu) 31extern void smp_yield(void);
43{ 32extern void smp_stop_cpu(void);
44 u32 status;
45
46 switch (sigp_ps(&status, 0, cpu, sigp_sense_running)) {
47 case sigp_status_stored:
48 /* Check for running status */
49 if (status & 0x400)
50 return 0;
51 break;
52 case sigp_not_operational:
53 return 0;
54 default:
55 break;
56 }
57 return 1;
58}
59 33
60#else /* CONFIG_SMP */ 34#else /* CONFIG_SMP */
61 35
62static inline void smp_switch_to_ipl_cpu(void (*func)(void *), void *data) 36static inline void smp_call_ipl_cpu(void (*func)(void *), void *data)
63{ 37{
64 func(data); 38 func(data);
65} 39}
66 40
67static inline void smp_restart_with_online_cpu(void) 41static inline void smp_call_online_cpu(void (*func)(void *), void *data)
68{ 42{
43 func(data);
69} 44}
70 45
71#define smp_vcpu_scheduled (1) 46static inline int smp_find_processor_id(int address) { return 0; }
47static inline int smp_vcpu_scheduled(int cpu) { return 1; }
48static inline void smp_yield_cpu(int cpu) { }
49static inline void smp_yield(void) { }
50static inline void smp_stop_cpu(void) { }
72 51
73#endif /* CONFIG_SMP */ 52#endif /* CONFIG_SMP */
74 53
75#ifdef CONFIG_HOTPLUG_CPU 54#ifdef CONFIG_HOTPLUG_CPU
76extern int smp_rescan_cpus(void); 55extern int smp_rescan_cpus(void);
77extern void __noreturn cpu_die(void); 56extern void __noreturn cpu_die(void);
57extern void __cpu_die(unsigned int cpu);
58extern int __cpu_disable(void);
78#else 59#else
79static inline int smp_rescan_cpus(void) { return 0; } 60static inline int smp_rescan_cpus(void) { return 0; }
80static inline void cpu_die(void) { } 61static inline void cpu_die(void) { }
diff --git a/arch/s390/include/asm/vdso.h b/arch/s390/include/asm/vdso.h
index 533f35751aeb..c4a11cfad3c8 100644
--- a/arch/s390/include/asm/vdso.h
+++ b/arch/s390/include/asm/vdso.h
@@ -40,8 +40,8 @@ struct vdso_per_cpu_data {
40extern struct vdso_data *vdso_data; 40extern struct vdso_data *vdso_data;
41 41
42#ifdef CONFIG_64BIT 42#ifdef CONFIG_64BIT
43int vdso_alloc_per_cpu(int cpu, struct _lowcore *lowcore); 43int vdso_alloc_per_cpu(struct _lowcore *lowcore);
44void vdso_free_per_cpu(int cpu, struct _lowcore *lowcore); 44void vdso_free_per_cpu(struct _lowcore *lowcore);
45#endif 45#endif
46 46
47#endif /* __ASSEMBLY__ */ 47#endif /* __ASSEMBLY__ */
diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile
index 7d9ec924e7e7..d0a48268eb27 100644
--- a/arch/s390/kernel/Makefile
+++ b/arch/s390/kernel/Makefile
@@ -34,8 +34,6 @@ extra-y += $(if $(CONFIG_64BIT),head64.o,head31.o)
34obj-$(CONFIG_MODULES) += s390_ksyms.o module.o 34obj-$(CONFIG_MODULES) += s390_ksyms.o module.o
35obj-$(CONFIG_SMP) += smp.o 35obj-$(CONFIG_SMP) += smp.o
36obj-$(CONFIG_SCHED_BOOK) += topology.o 36obj-$(CONFIG_SCHED_BOOK) += topology.o
37obj-$(CONFIG_SMP) += $(if $(CONFIG_64BIT),switch_cpu64.o, \
38 switch_cpu.o)
39obj-$(CONFIG_HIBERNATION) += suspend.o swsusp_asm64.o 37obj-$(CONFIG_HIBERNATION) += suspend.o swsusp_asm64.o
40obj-$(CONFIG_AUDIT) += audit.o 38obj-$(CONFIG_AUDIT) += audit.o
41compat-obj-$(CONFIG_AUDIT) += compat_audit.o 39compat-obj-$(CONFIG_AUDIT) += compat_audit.o
diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c
index 530ae0e8e38f..aeeaf896be9b 100644
--- a/arch/s390/kernel/asm-offsets.c
+++ b/arch/s390/kernel/asm-offsets.c
@@ -9,8 +9,8 @@
9#include <linux/kbuild.h> 9#include <linux/kbuild.h>
10#include <linux/sched.h> 10#include <linux/sched.h>
11#include <asm/vdso.h> 11#include <asm/vdso.h>
12#include <asm/sigp.h>
13#include <asm/pgtable.h> 12#include <asm/pgtable.h>
13#include <asm/system.h>
14 14
15/* 15/*
16 * Make sure that the compiler is new enough. We want a compiler that 16 * Make sure that the compiler is new enough. We want a compiler that
@@ -70,12 +70,6 @@ int main(void)
70 DEFINE(__CLOCK_MONOTONIC, CLOCK_MONOTONIC); 70 DEFINE(__CLOCK_MONOTONIC, CLOCK_MONOTONIC);
71 DEFINE(__CLOCK_REALTIME_RES, MONOTONIC_RES_NSEC); 71 DEFINE(__CLOCK_REALTIME_RES, MONOTONIC_RES_NSEC);
72 BLANK(); 72 BLANK();
73 /* constants for SIGP */
74 DEFINE(__SIGP_STOP, sigp_stop);
75 DEFINE(__SIGP_RESTART, sigp_restart);
76 DEFINE(__SIGP_SENSE, sigp_sense);
77 DEFINE(__SIGP_INITIAL_CPU_RESET, sigp_initial_cpu_reset);
78 BLANK();
79 /* lowcore offsets */ 73 /* lowcore offsets */
80 DEFINE(__LC_EXT_PARAMS, offsetof(struct _lowcore, ext_params)); 74 DEFINE(__LC_EXT_PARAMS, offsetof(struct _lowcore, ext_params));
81 DEFINE(__LC_EXT_CPU_ADDR, offsetof(struct _lowcore, ext_cpu_addr)); 75 DEFINE(__LC_EXT_CPU_ADDR, offsetof(struct _lowcore, ext_cpu_addr));
@@ -95,20 +89,19 @@ int main(void)
95 DEFINE(__LC_IO_INT_WORD, offsetof(struct _lowcore, io_int_word)); 89 DEFINE(__LC_IO_INT_WORD, offsetof(struct _lowcore, io_int_word));
96 DEFINE(__LC_STFL_FAC_LIST, offsetof(struct _lowcore, stfl_fac_list)); 90 DEFINE(__LC_STFL_FAC_LIST, offsetof(struct _lowcore, stfl_fac_list));
97 DEFINE(__LC_MCCK_CODE, offsetof(struct _lowcore, mcck_interruption_code)); 91 DEFINE(__LC_MCCK_CODE, offsetof(struct _lowcore, mcck_interruption_code));
98 DEFINE(__LC_DUMP_REIPL, offsetof(struct _lowcore, ipib));
99 BLANK();
100 DEFINE(__LC_RST_NEW_PSW, offsetof(struct _lowcore, restart_psw));
101 DEFINE(__LC_RST_OLD_PSW, offsetof(struct _lowcore, restart_old_psw)); 92 DEFINE(__LC_RST_OLD_PSW, offsetof(struct _lowcore, restart_old_psw));
102 DEFINE(__LC_EXT_OLD_PSW, offsetof(struct _lowcore, external_old_psw)); 93 DEFINE(__LC_EXT_OLD_PSW, offsetof(struct _lowcore, external_old_psw));
103 DEFINE(__LC_SVC_OLD_PSW, offsetof(struct _lowcore, svc_old_psw)); 94 DEFINE(__LC_SVC_OLD_PSW, offsetof(struct _lowcore, svc_old_psw));
104 DEFINE(__LC_PGM_OLD_PSW, offsetof(struct _lowcore, program_old_psw)); 95 DEFINE(__LC_PGM_OLD_PSW, offsetof(struct _lowcore, program_old_psw));
105 DEFINE(__LC_MCK_OLD_PSW, offsetof(struct _lowcore, mcck_old_psw)); 96 DEFINE(__LC_MCK_OLD_PSW, offsetof(struct _lowcore, mcck_old_psw));
106 DEFINE(__LC_IO_OLD_PSW, offsetof(struct _lowcore, io_old_psw)); 97 DEFINE(__LC_IO_OLD_PSW, offsetof(struct _lowcore, io_old_psw));
98 DEFINE(__LC_RST_NEW_PSW, offsetof(struct _lowcore, restart_psw));
107 DEFINE(__LC_EXT_NEW_PSW, offsetof(struct _lowcore, external_new_psw)); 99 DEFINE(__LC_EXT_NEW_PSW, offsetof(struct _lowcore, external_new_psw));
108 DEFINE(__LC_SVC_NEW_PSW, offsetof(struct _lowcore, svc_new_psw)); 100 DEFINE(__LC_SVC_NEW_PSW, offsetof(struct _lowcore, svc_new_psw));
109 DEFINE(__LC_PGM_NEW_PSW, offsetof(struct _lowcore, program_new_psw)); 101 DEFINE(__LC_PGM_NEW_PSW, offsetof(struct _lowcore, program_new_psw));
110 DEFINE(__LC_MCK_NEW_PSW, offsetof(struct _lowcore, mcck_new_psw)); 102 DEFINE(__LC_MCK_NEW_PSW, offsetof(struct _lowcore, mcck_new_psw));
111 DEFINE(__LC_IO_NEW_PSW, offsetof(struct _lowcore, io_new_psw)); 103 DEFINE(__LC_IO_NEW_PSW, offsetof(struct _lowcore, io_new_psw));
104 BLANK();
112 DEFINE(__LC_SAVE_AREA_SYNC, offsetof(struct _lowcore, save_area_sync)); 105 DEFINE(__LC_SAVE_AREA_SYNC, offsetof(struct _lowcore, save_area_sync));
113 DEFINE(__LC_SAVE_AREA_ASYNC, offsetof(struct _lowcore, save_area_async)); 106 DEFINE(__LC_SAVE_AREA_ASYNC, offsetof(struct _lowcore, save_area_async));
114 DEFINE(__LC_SAVE_AREA_RESTART, offsetof(struct _lowcore, save_area_restart)); 107 DEFINE(__LC_SAVE_AREA_RESTART, offsetof(struct _lowcore, save_area_restart));
@@ -129,12 +122,16 @@ int main(void)
129 DEFINE(__LC_KERNEL_STACK, offsetof(struct _lowcore, kernel_stack)); 122 DEFINE(__LC_KERNEL_STACK, offsetof(struct _lowcore, kernel_stack));
130 DEFINE(__LC_ASYNC_STACK, offsetof(struct _lowcore, async_stack)); 123 DEFINE(__LC_ASYNC_STACK, offsetof(struct _lowcore, async_stack));
131 DEFINE(__LC_PANIC_STACK, offsetof(struct _lowcore, panic_stack)); 124 DEFINE(__LC_PANIC_STACK, offsetof(struct _lowcore, panic_stack));
125 DEFINE(__LC_RESTART_STACK, offsetof(struct _lowcore, restart_stack));
126 DEFINE(__LC_RESTART_FN, offsetof(struct _lowcore, restart_fn));
132 DEFINE(__LC_USER_ASCE, offsetof(struct _lowcore, user_asce)); 127 DEFINE(__LC_USER_ASCE, offsetof(struct _lowcore, user_asce));
133 DEFINE(__LC_INT_CLOCK, offsetof(struct _lowcore, int_clock)); 128 DEFINE(__LC_INT_CLOCK, offsetof(struct _lowcore, int_clock));
134 DEFINE(__LC_MCCK_CLOCK, offsetof(struct _lowcore, mcck_clock)); 129 DEFINE(__LC_MCCK_CLOCK, offsetof(struct _lowcore, mcck_clock));
135 DEFINE(__LC_MACHINE_FLAGS, offsetof(struct _lowcore, machine_flags)); 130 DEFINE(__LC_MACHINE_FLAGS, offsetof(struct _lowcore, machine_flags));
136 DEFINE(__LC_FTRACE_FUNC, offsetof(struct _lowcore, ftrace_func)); 131 DEFINE(__LC_FTRACE_FUNC, offsetof(struct _lowcore, ftrace_func));
137 DEFINE(__LC_IRB, offsetof(struct _lowcore, irb)); 132 DEFINE(__LC_IRB, offsetof(struct _lowcore, irb));
133 DEFINE(__LC_DUMP_REIPL, offsetof(struct _lowcore, ipib));
134 BLANK();
138 DEFINE(__LC_CPU_TIMER_SAVE_AREA, offsetof(struct _lowcore, cpu_timer_save_area)); 135 DEFINE(__LC_CPU_TIMER_SAVE_AREA, offsetof(struct _lowcore, cpu_timer_save_area));
139 DEFINE(__LC_CLOCK_COMP_SAVE_AREA, offsetof(struct _lowcore, clock_comp_save_area)); 136 DEFINE(__LC_CLOCK_COMP_SAVE_AREA, offsetof(struct _lowcore, clock_comp_save_area));
140 DEFINE(__LC_PSW_SAVE_AREA, offsetof(struct _lowcore, psw_save_area)); 137 DEFINE(__LC_PSW_SAVE_AREA, offsetof(struct _lowcore, psw_save_area));
diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index 5f437b830da5..6143521a4fff 100644
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -2,7 +2,7 @@
2 * arch/s390/kernel/entry.S 2 * arch/s390/kernel/entry.S
3 * S390 low-level entry points. 3 * S390 low-level entry points.
4 * 4 *
5 * Copyright (C) IBM Corp. 1999,2006 5 * Copyright (C) IBM Corp. 1999,2012
6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7 * Hartmut Penner (hp@de.ibm.com), 7 * Hartmut Penner (hp@de.ibm.com),
8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com), 8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
@@ -691,77 +691,30 @@ mcck_panic:
6910: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 6910: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
692 j mcck_skip 692 j mcck_skip
693 693
694/*
695 * Restart interruption handler, kick starter for additional CPUs
696 */
697#ifdef CONFIG_SMP
698 __CPUINIT
699ENTRY(restart_int_handler)
700 basr %r1,0
701restart_base:
702 spt restart_vtime-restart_base(%r1)
703 stck __LC_LAST_UPDATE_CLOCK
704 mvc __LC_LAST_UPDATE_TIMER(8),restart_vtime-restart_base(%r1)
705 mvc __LC_EXIT_TIMER(8),restart_vtime-restart_base(%r1)
706 l %r15,__LC_GPREGS_SAVE_AREA+60 # load ksp
707 lctl %c0,%c15,__LC_CREGS_SAVE_AREA # get new ctl regs
708 lam %a0,%a15,__LC_AREGS_SAVE_AREA
709 lm %r6,%r15,__SF_GPRS(%r15)# load registers from clone
710 l %r1,__LC_THREAD_INFO
711 mvc __LC_USER_TIMER(8),__TI_user_timer(%r1)
712 mvc __LC_SYSTEM_TIMER(8),__TI_system_timer(%r1)
713 xc __LC_STEAL_TIMER(8),__LC_STEAL_TIMER
714 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
715 basr %r14,0
716 l %r14,restart_addr-.(%r14)
717 basr %r14,%r14 # call start_secondary
718restart_addr:
719 .long start_secondary
720 .align 8
721restart_vtime:
722 .long 0x7fffffff,0xffffffff
723 .previous
724#else
725/*
726 * If we do not run with SMP enabled, let the new CPU crash ...
727 */
728ENTRY(restart_int_handler)
729 basr %r1,0
730restart_base:
731 lpsw restart_crash-restart_base(%r1)
732 .align 8
733restart_crash:
734 .long 0x000a0000,0x00000000
735restart_go:
736#endif
737
738# 694#
739# PSW restart interrupt handler 695# PSW restart interrupt handler
740# 696#
741ENTRY(psw_restart_int_handler) 697ENTRY(restart_int_handler)
742 st %r15,__LC_SAVE_AREA_RESTART 698 st %r15,__LC_SAVE_AREA_RESTART
743 basr %r15,0 699 l %r15,__LC_RESTART_STACK
7440: l %r15,.Lrestart_stack-0b(%r15) # load restart stack
745 l %r15,0(%r15)
746 ahi %r15,-__PT_SIZE # create pt_regs on stack 700 ahi %r15,-__PT_SIZE # create pt_regs on stack
701 xc 0(__PT_SIZE,%r15),0(%r15)
747 stm %r0,%r14,__PT_R0(%r15) 702 stm %r0,%r14,__PT_R0(%r15)
748 mvc __PT_R15(4,%r15),__LC_SAVE_AREA_RESTART 703 mvc __PT_R15(4,%r15),__LC_SAVE_AREA_RESTART
749 mvc __PT_PSW(8,%r15),__LC_RST_OLD_PSW # store restart old psw 704 mvc __PT_PSW(8,%r15),__LC_RST_OLD_PSW # store restart old psw
750 ahi %r15,-STACK_FRAME_OVERHEAD 705 ahi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack
751 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 706 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
752 basr %r14,0 707 lm %r1,%r3,__LC_RESTART_FN # load fn, parm & source cpu
7531: l %r14,.Ldo_restart-1b(%r14) 708 ltr %r3,%r3 # test source cpu address
754 basr %r14,%r14 709 jm 1f # negative -> skip source stop
755 basr %r14,0 # load disabled wait PSW if 7100: sigp %r4,%r3,1 # sigp sense to source cpu
7562: lpsw restart_psw_crash-2b(%r14) # do_restart returns 711 brc 10,0b # wait for status stored
757 .align 4 7121: basr %r14,%r1 # call function
758.Ldo_restart: 713 stap __SF_EMPTY(%r15) # store cpu address
759 .long do_restart 714 lh %r3,__SF_EMPTY(%r15)
760.Lrestart_stack: 7152: sigp %r4,%r3,5 # sigp stop to current cpu
761 .long restart_stack 716 brc 2,2b
762 .align 8 7173: j 3b
763restart_psw_crash:
764 .long 0x000a0000,0x00000000 + restart_psw_crash
765 718
766 .section .kprobes.text, "ax" 719 .section .kprobes.text, "ax"
767 720
diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h
index bf538aaf407d..92b1617d0c95 100644
--- a/arch/s390/kernel/entry.h
+++ b/arch/s390/kernel/entry.h
@@ -9,6 +9,14 @@
9extern void (*pgm_check_table[128])(struct pt_regs *); 9extern void (*pgm_check_table[128])(struct pt_regs *);
10extern void *restart_stack; 10extern void *restart_stack;
11 11
12void system_call(void);
13void pgm_check_handler(void);
14void ext_int_handler(void);
15void io_int_handler(void);
16void mcck_int_handler(void);
17void restart_int_handler(void);
18void restart_call_handler(void);
19
12asmlinkage long do_syscall_trace_enter(struct pt_regs *regs); 20asmlinkage long do_syscall_trace_enter(struct pt_regs *regs);
13asmlinkage void do_syscall_trace_exit(struct pt_regs *regs); 21asmlinkage void do_syscall_trace_exit(struct pt_regs *regs);
14 22
@@ -26,7 +34,6 @@ void do_notify_resume(struct pt_regs *regs);
26 34
27void do_extint(struct pt_regs *regs, unsigned int, unsigned int, unsigned long); 35void do_extint(struct pt_regs *regs, unsigned int, unsigned int, unsigned long);
28void do_restart(void); 36void do_restart(void);
29int __cpuinit start_secondary(void *cpuvoid);
30void __init startup_init(void); 37void __init startup_init(void);
31void die(struct pt_regs *regs, const char *str); 38void die(struct pt_regs *regs, const char *str);
32 39
diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S
index bacbd2848d40..e33789a45752 100644
--- a/arch/s390/kernel/entry64.S
+++ b/arch/s390/kernel/entry64.S
@@ -2,7 +2,7 @@
2 * arch/s390/kernel/entry64.S 2 * arch/s390/kernel/entry64.S
3 * S390 low-level entry points. 3 * S390 low-level entry points.
4 * 4 *
5 * Copyright (C) IBM Corp. 1999,2010 5 * Copyright (C) IBM Corp. 1999,2012
6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7 * Hartmut Penner (hp@de.ibm.com), 7 * Hartmut Penner (hp@de.ibm.com),
8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com), 8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
@@ -713,68 +713,30 @@ mcck_panic:
7130: aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 7130: aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
714 j mcck_skip 714 j mcck_skip
715 715
716/*
717 * Restart interruption handler, kick starter for additional CPUs
718 */
719#ifdef CONFIG_SMP
720 __CPUINIT
721ENTRY(restart_int_handler)
722 basr %r1,0
723restart_base:
724 spt restart_vtime-restart_base(%r1)
725 stck __LC_LAST_UPDATE_CLOCK
726 mvc __LC_LAST_UPDATE_TIMER(8),restart_vtime-restart_base(%r1)
727 mvc __LC_EXIT_TIMER(8),restart_vtime-restart_base(%r1)
728 lghi %r10,__LC_GPREGS_SAVE_AREA
729 lg %r15,120(%r10) # load ksp
730 lghi %r10,__LC_CREGS_SAVE_AREA
731 lctlg %c0,%c15,0(%r10) # get new ctl regs
732 lghi %r10,__LC_AREGS_SAVE_AREA
733 lam %a0,%a15,0(%r10)
734 lmg %r6,%r15,__SF_GPRS(%r15)# load registers from clone
735 lg %r1,__LC_THREAD_INFO
736 mvc __LC_USER_TIMER(8),__TI_user_timer(%r1)
737 mvc __LC_SYSTEM_TIMER(8),__TI_system_timer(%r1)
738 xc __LC_STEAL_TIMER(8),__LC_STEAL_TIMER
739 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
740 brasl %r14,start_secondary
741 .align 8
742restart_vtime:
743 .long 0x7fffffff,0xffffffff
744 .previous
745#else
746/*
747 * If we do not run with SMP enabled, let the new CPU crash ...
748 */
749ENTRY(restart_int_handler)
750 basr %r1,0
751restart_base:
752 lpswe restart_crash-restart_base(%r1)
753 .align 8
754restart_crash:
755 .long 0x000a0000,0x00000000,0x00000000,0x00000000
756restart_go:
757#endif
758
759# 716#
760# PSW restart interrupt handler 717# PSW restart interrupt handler
761# 718#
762ENTRY(psw_restart_int_handler) 719ENTRY(restart_int_handler)
763 stg %r15,__LC_SAVE_AREA_RESTART 720 stg %r15,__LC_SAVE_AREA_RESTART
764 larl %r15,restart_stack # load restart stack 721 lg %r15,__LC_RESTART_STACK
765 lg %r15,0(%r15)
766 aghi %r15,-__PT_SIZE # create pt_regs on stack 722 aghi %r15,-__PT_SIZE # create pt_regs on stack
723 xc 0(__PT_SIZE,%r15),0(%r15)
767 stmg %r0,%r14,__PT_R0(%r15) 724 stmg %r0,%r14,__PT_R0(%r15)
768 mvc __PT_R15(8,%r15),__LC_SAVE_AREA_RESTART 725 mvc __PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
769 mvc __PT_PSW(16,%r15),__LC_RST_OLD_PSW # store restart old psw 726 mvc __PT_PSW(16,%r15),__LC_RST_OLD_PSW # store restart old psw
770 aghi %r15,-STACK_FRAME_OVERHEAD 727 aghi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack
771 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 728 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
772 brasl %r14,do_restart 729 lmg %r1,%r3,__LC_RESTART_FN # load fn, parm & source cpu
773 larl %r14,restart_psw_crash # load disabled wait PSW if 730 ltgr %r3,%r3 # test source cpu address
774 lpswe 0(%r14) # do_restart returns 731 jm 1f # negative -> skip source stop
775 .align 8 7320: sigp %r4,%r3,1 # sigp sense to source cpu
776restart_psw_crash: 733 brc 10,0b # wait for status stored
777 .quad 0x0002000080000000,0x0000000000000000 + restart_psw_crash 7341: basr %r14,%r1 # call function
735 stap __SF_EMPTY(%r15) # store cpu address
736 llgh %r3,__SF_EMPTY(%r15)
7372: sigp %r4,%r3,5 # sigp stop to current cpu
738 brc 2,2b
7393: j 3b
778 740
779 .section .kprobes.text, "ax" 741 .section .kprobes.text, "ax"
780 742
diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c
index affa8e68124a..e5a72a2b0c56 100644
--- a/arch/s390/kernel/ipl.c
+++ b/arch/s390/kernel/ipl.c
@@ -2,7 +2,7 @@
2 * arch/s390/kernel/ipl.c 2 * arch/s390/kernel/ipl.c
3 * ipl/reipl/dump support for Linux on s390. 3 * ipl/reipl/dump support for Linux on s390.
4 * 4 *
5 * Copyright IBM Corp. 2005,2007 5 * Copyright IBM Corp. 2005,2012
6 * Author(s): Michael Holzheu <holzheu@de.ibm.com> 6 * Author(s): Michael Holzheu <holzheu@de.ibm.com>
7 * Heiko Carstens <heiko.carstens@de.ibm.com> 7 * Heiko Carstens <heiko.carstens@de.ibm.com>
8 * Volker Sameske <sameske@de.ibm.com> 8 * Volker Sameske <sameske@de.ibm.com>
@@ -25,7 +25,6 @@
25#include <asm/ebcdic.h> 25#include <asm/ebcdic.h>
26#include <asm/reset.h> 26#include <asm/reset.h>
27#include <asm/sclp.h> 27#include <asm/sclp.h>
28#include <asm/sigp.h>
29#include <asm/checksum.h> 28#include <asm/checksum.h>
30#include "entry.h" 29#include "entry.h"
31 30
@@ -571,7 +570,7 @@ static void __ipl_run(void *unused)
571 570
572static void ipl_run(struct shutdown_trigger *trigger) 571static void ipl_run(struct shutdown_trigger *trigger)
573{ 572{
574 smp_switch_to_ipl_cpu(__ipl_run, NULL); 573 smp_call_ipl_cpu(__ipl_run, NULL);
575} 574}
576 575
577static int __init ipl_init(void) 576static int __init ipl_init(void)
@@ -1101,7 +1100,7 @@ static void __reipl_run(void *unused)
1101 1100
1102static void reipl_run(struct shutdown_trigger *trigger) 1101static void reipl_run(struct shutdown_trigger *trigger)
1103{ 1102{
1104 smp_switch_to_ipl_cpu(__reipl_run, NULL); 1103 smp_call_ipl_cpu(__reipl_run, NULL);
1105} 1104}
1106 1105
1107static void reipl_block_ccw_init(struct ipl_parameter_block *ipb) 1106static void reipl_block_ccw_init(struct ipl_parameter_block *ipb)
@@ -1421,7 +1420,7 @@ static void dump_run(struct shutdown_trigger *trigger)
1421 if (dump_method == DUMP_METHOD_NONE) 1420 if (dump_method == DUMP_METHOD_NONE)
1422 return; 1421 return;
1423 smp_send_stop(); 1422 smp_send_stop();
1424 smp_switch_to_ipl_cpu(__dump_run, NULL); 1423 smp_call_ipl_cpu(__dump_run, NULL);
1425} 1424}
1426 1425
1427static int __init dump_ccw_init(void) 1426static int __init dump_ccw_init(void)
@@ -1623,9 +1622,7 @@ static void stop_run(struct shutdown_trigger *trigger)
1623 if (strcmp(trigger->name, ON_PANIC_STR) == 0 || 1622 if (strcmp(trigger->name, ON_PANIC_STR) == 0 ||
1624 strcmp(trigger->name, ON_RESTART_STR) == 0) 1623 strcmp(trigger->name, ON_RESTART_STR) == 0)
1625 disabled_wait((unsigned long) __builtin_return_address(0)); 1624 disabled_wait((unsigned long) __builtin_return_address(0));
1626 while (sigp(smp_processor_id(), sigp_stop) == sigp_busy) 1625 smp_stop_cpu();
1627 cpu_relax();
1628 for (;;);
1629} 1626}
1630 1627
1631static struct shutdown_action stop_action = {SHUTDOWN_ACTION_STOP_STR, 1628static struct shutdown_action stop_action = {SHUTDOWN_ACTION_STOP_STR,
@@ -1738,9 +1735,8 @@ static ssize_t on_restart_store(struct kobject *kobj,
1738static struct kobj_attribute on_restart_attr = 1735static struct kobj_attribute on_restart_attr =
1739 __ATTR(on_restart, 0644, on_restart_show, on_restart_store); 1736 __ATTR(on_restart, 0644, on_restart_show, on_restart_store);
1740 1737
1741void do_restart(void) 1738static void __do_restart(void *ignore)
1742{ 1739{
1743 smp_restart_with_online_cpu();
1744 smp_send_stop(); 1740 smp_send_stop();
1745#ifdef CONFIG_CRASH_DUMP 1741#ifdef CONFIG_CRASH_DUMP
1746 crash_kexec(NULL); 1742 crash_kexec(NULL);
@@ -1749,6 +1745,11 @@ void do_restart(void)
1749 stop_run(&on_restart_trigger); 1745 stop_run(&on_restart_trigger);
1750} 1746}
1751 1747
1748void do_restart(void)
1749{
1750 smp_call_online_cpu(__do_restart, NULL);
1751}
1752
1752/* on halt */ 1753/* on halt */
1753 1754
1754static struct shutdown_trigger on_halt_trigger = {ON_HALT_STR, &stop_action}; 1755static struct shutdown_trigger on_halt_trigger = {ON_HALT_STR, &stop_action};
diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c
index 47b168fb29c4..bf6fbc03ebaf 100644
--- a/arch/s390/kernel/machine_kexec.c
+++ b/arch/s390/kernel/machine_kexec.c
@@ -49,50 +49,21 @@ static void add_elf_notes(int cpu)
49} 49}
50 50
51/* 51/*
52 * Store status of next available physical CPU
53 */
54static int store_status_next(int start_cpu, int this_cpu)
55{
56 struct save_area *sa = (void *) 4608 + store_prefix();
57 int cpu, rc;
58
59 for (cpu = start_cpu; cpu < 65536; cpu++) {
60 if (cpu == this_cpu)
61 continue;
62 do {
63 rc = raw_sigp(cpu, sigp_stop_and_store_status);
64 } while (rc == sigp_busy);
65 if (rc != sigp_order_code_accepted)
66 continue;
67 if (sa->pref_reg)
68 return cpu;
69 }
70 return -1;
71}
72
73/*
74 * Initialize CPU ELF notes 52 * Initialize CPU ELF notes
75 */ 53 */
76void setup_regs(void) 54void setup_regs(void)
77{ 55{
78 unsigned long sa = S390_lowcore.prefixreg_save_area + SAVE_AREA_BASE; 56 unsigned long sa = S390_lowcore.prefixreg_save_area + SAVE_AREA_BASE;
79 int cpu, this_cpu, phys_cpu = 0, first = 1; 57 int cpu, this_cpu;
80 58
81 this_cpu = stap(); 59 this_cpu = smp_find_processor_id(stap());
82 60 add_elf_notes(this_cpu);
83 if (!S390_lowcore.prefixreg_save_area)
84 first = 0;
85 for_each_online_cpu(cpu) { 61 for_each_online_cpu(cpu) {
86 if (first) { 62 if (cpu == this_cpu)
87 add_elf_notes(cpu); 63 continue;
88 first = 0; 64 if (smp_store_status(cpu))
89 continue; 65 continue;
90 }
91 phys_cpu = store_status_next(phys_cpu, this_cpu);
92 if (phys_cpu == -1)
93 break;
94 add_elf_notes(cpu); 66 add_elf_notes(cpu);
95 phys_cpu++;
96 } 67 }
97 /* Copy dump CPU store status info to absolute zero */ 68 /* Copy dump CPU store status info to absolute zero */
98 memcpy((void *) SAVE_AREA_BASE, (void *) sa, sizeof(struct save_area)); 69 memcpy((void *) SAVE_AREA_BASE, (void *) sa, sizeof(struct save_area));
@@ -255,5 +226,5 @@ void machine_kexec(struct kimage *image)
255 return; 226 return;
256 tracer_disable(); 227 tracer_disable();
257 smp_send_stop(); 228 smp_send_stop();
258 smp_switch_to_ipl_cpu(__machine_kexec, image); 229 smp_call_ipl_cpu(__machine_kexec, image);
259} 230}
diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c
index 778c100fe314..9a3edb5f2c92 100644
--- a/arch/s390/kernel/setup.c
+++ b/arch/s390/kernel/setup.c
@@ -2,7 +2,7 @@
2 * arch/s390/kernel/setup.c 2 * arch/s390/kernel/setup.c
3 * 3 *
4 * S390 version 4 * S390 version
5 * Copyright (C) IBM Corp. 1999,2010 5 * Copyright (C) IBM Corp. 1999,2012
6 * Author(s): Hartmut Penner (hp@de.ibm.com), 6 * Author(s): Hartmut Penner (hp@de.ibm.com),
7 * Martin Schwidefsky (schwidefsky@de.ibm.com) 7 * Martin Schwidefsky (schwidefsky@de.ibm.com)
8 * 8 *
@@ -62,6 +62,7 @@
62#include <asm/ebcdic.h> 62#include <asm/ebcdic.h>
63#include <asm/kvm_virtio.h> 63#include <asm/kvm_virtio.h>
64#include <asm/diag.h> 64#include <asm/diag.h>
65#include "entry.h"
65 66
66long psw_kernel_bits = PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_ASC_PRIMARY | 67long psw_kernel_bits = PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_ASC_PRIMARY |
67 PSW_MASK_EA | PSW_MASK_BA; 68 PSW_MASK_EA | PSW_MASK_BA;
@@ -351,8 +352,9 @@ static void setup_addressing_mode(void)
351 } 352 }
352} 353}
353 354
354static void __init 355void *restart_stack __attribute__((__section__(".data")));
355setup_lowcore(void) 356
357static void __init setup_lowcore(void)
356{ 358{
357 struct _lowcore *lc; 359 struct _lowcore *lc;
358 360
@@ -363,7 +365,7 @@ setup_lowcore(void)
363 lc = __alloc_bootmem_low(LC_PAGES * PAGE_SIZE, LC_PAGES * PAGE_SIZE, 0); 365 lc = __alloc_bootmem_low(LC_PAGES * PAGE_SIZE, LC_PAGES * PAGE_SIZE, 0);
364 lc->restart_psw.mask = psw_kernel_bits; 366 lc->restart_psw.mask = psw_kernel_bits;
365 lc->restart_psw.addr = 367 lc->restart_psw.addr =
366 PSW_ADDR_AMODE | (unsigned long) psw_restart_int_handler; 368 PSW_ADDR_AMODE | (unsigned long) restart_int_handler;
367 lc->external_new_psw.mask = psw_kernel_bits | 369 lc->external_new_psw.mask = psw_kernel_bits |
368 PSW_MASK_DAT | PSW_MASK_MCHECK; 370 PSW_MASK_DAT | PSW_MASK_MCHECK;
369 lc->external_new_psw.addr = 371 lc->external_new_psw.addr =
@@ -412,6 +414,24 @@ setup_lowcore(void)
412 lc->last_update_timer = S390_lowcore.last_update_timer; 414 lc->last_update_timer = S390_lowcore.last_update_timer;
413 lc->last_update_clock = S390_lowcore.last_update_clock; 415 lc->last_update_clock = S390_lowcore.last_update_clock;
414 lc->ftrace_func = S390_lowcore.ftrace_func; 416 lc->ftrace_func = S390_lowcore.ftrace_func;
417
418 restart_stack = __alloc_bootmem(ASYNC_SIZE, ASYNC_SIZE, 0);
419 restart_stack += ASYNC_SIZE;
420
421 /*
422 * Set up PSW restart to call ipl.c:do_restart(). Copy the relevant
423 * restart data to the absolute zero lowcore. This is necesary if
424 * PSW restart is done on an offline CPU that has lowcore zero.
425 */
426 lc->restart_stack = (unsigned long) restart_stack;
427 lc->restart_fn = (unsigned long) do_restart;
428 lc->restart_data = 0;
429 lc->restart_source = -1UL;
430 memcpy(&S390_lowcore.restart_stack, &lc->restart_stack,
431 4*sizeof(unsigned long));
432 copy_to_absolute_zero(&S390_lowcore.restart_psw,
433 &lc->restart_psw, sizeof(psw_t));
434
415 set_prefix((u32)(unsigned long) lc); 435 set_prefix((u32)(unsigned long) lc);
416 lowcore_ptr[0] = lc; 436 lowcore_ptr[0] = lc;
417} 437}
@@ -572,27 +592,6 @@ static void __init setup_memory_end(void)
572 } 592 }
573} 593}
574 594
575void *restart_stack __attribute__((__section__(".data")));
576
577/*
578 * Setup new PSW and allocate stack for PSW restart interrupt
579 */
580static void __init setup_restart_psw(void)
581{
582 psw_t psw;
583
584 restart_stack = __alloc_bootmem(ASYNC_SIZE, ASYNC_SIZE, 0);
585 restart_stack += ASYNC_SIZE;
586
587 /*
588 * Setup restart PSW for absolute zero lowcore. This is necesary
589 * if PSW restart is done on an offline CPU that has lowcore zero
590 */
591 psw.mask = PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_MASK_EA | PSW_MASK_BA;
592 psw.addr = PSW_ADDR_AMODE | (unsigned long) psw_restart_int_handler;
593 copy_to_absolute_zero(&S390_lowcore.restart_psw, &psw, sizeof(psw));
594}
595
596static void __init setup_vmcoreinfo(void) 595static void __init setup_vmcoreinfo(void)
597{ 596{
598#ifdef CONFIG_KEXEC 597#ifdef CONFIG_KEXEC
@@ -782,8 +781,7 @@ static void __init reserve_crashkernel(void)
782#endif 781#endif
783} 782}
784 783
785static void __init 784static void __init setup_memory(void)
786setup_memory(void)
787{ 785{
788 unsigned long bootmap_size; 786 unsigned long bootmap_size;
789 unsigned long start_pfn, end_pfn; 787 unsigned long start_pfn, end_pfn;
@@ -1014,8 +1012,7 @@ static void __init setup_hwcaps(void)
1014 * was printed. 1012 * was printed.
1015 */ 1013 */
1016 1014
1017void __init 1015void __init setup_arch(char **cmdline_p)
1018setup_arch(char **cmdline_p)
1019{ 1016{
1020 /* 1017 /*
1021 * print what head.S has found out about the machine 1018 * print what head.S has found out about the machine
@@ -1068,7 +1065,6 @@ setup_arch(char **cmdline_p)
1068 setup_memory(); 1065 setup_memory();
1069 setup_resources(); 1066 setup_resources();
1070 setup_vmcoreinfo(); 1067 setup_vmcoreinfo();
1071 setup_restart_psw();
1072 setup_lowcore(); 1068 setup_lowcore();
1073 1069
1074 cpu_init(); 1070 cpu_init();
diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index 2398ce6b15ae..6db8526a602d 100644
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -1,23 +1,18 @@
1/* 1/*
2 * arch/s390/kernel/smp.c 2 * SMP related functions
3 * 3 *
4 * Copyright IBM Corp. 1999, 2009 4 * Copyright IBM Corp. 1999,2012
5 * Author(s): Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com), 5 * Author(s): Denis Joseph Barrow,
6 * Martin Schwidefsky (schwidefsky@de.ibm.com) 6 * Martin Schwidefsky <schwidefsky@de.ibm.com>,
7 * Heiko Carstens (heiko.carstens@de.ibm.com) 7 * Heiko Carstens <heiko.carstens@de.ibm.com>,
8 * 8 *
9 * based on other smp stuff by 9 * based on other smp stuff by
10 * (c) 1995 Alan Cox, CymruNET Ltd <alan@cymru.net> 10 * (c) 1995 Alan Cox, CymruNET Ltd <alan@cymru.net>
11 * (c) 1998 Ingo Molnar 11 * (c) 1998 Ingo Molnar
12 * 12 *
13 * We work with logical cpu numbering everywhere we can. The only 13 * The code outside of smp.c uses logical cpu numbers, only smp.c does
14 * functions using the real cpu address (got from STAP) are the sigp 14 * the translation of logical to physical cpu ids. All new code that
15 * functions. For all other functions we use the identity mapping. 15 * operates on physical cpu numbers needs to go into smp.c.
16 * That means that cpu_number_map[i] == i for every cpu. cpu_number_map is
17 * used e.g. to find the idle task belonging to a logical cpu. Every array
18 * in the kernel is sorted by the logical cpu number and not by the physical
19 * one which is causing all the confusion with __cpu_logical_map and
20 * cpu_number_map in other architectures.
21 */ 16 */
22 17
23#define KMSG_COMPONENT "cpu" 18#define KMSG_COMPONENT "cpu"
@@ -31,140 +26,381 @@
31#include <linux/spinlock.h> 26#include <linux/spinlock.h>
32#include <linux/kernel_stat.h> 27#include <linux/kernel_stat.h>
33#include <linux/delay.h> 28#include <linux/delay.h>
34#include <linux/cache.h>
35#include <linux/interrupt.h> 29#include <linux/interrupt.h>
36#include <linux/irqflags.h> 30#include <linux/irqflags.h>
37#include <linux/cpu.h> 31#include <linux/cpu.h>
38#include <linux/timex.h>
39#include <linux/bootmem.h>
40#include <linux/slab.h> 32#include <linux/slab.h>
41#include <linux/crash_dump.h> 33#include <linux/crash_dump.h>
42#include <asm/asm-offsets.h> 34#include <asm/asm-offsets.h>
43#include <asm/ipl.h> 35#include <asm/ipl.h>
44#include <asm/setup.h> 36#include <asm/setup.h>
45#include <asm/sigp.h>
46#include <asm/pgalloc.h>
47#include <asm/irq.h> 37#include <asm/irq.h>
48#include <asm/cpcmd.h>
49#include <asm/tlbflush.h> 38#include <asm/tlbflush.h>
50#include <asm/timer.h> 39#include <asm/timer.h>
51#include <asm/lowcore.h> 40#include <asm/lowcore.h>
52#include <asm/sclp.h> 41#include <asm/sclp.h>
53#include <asm/cputime.h>
54#include <asm/vdso.h> 42#include <asm/vdso.h>
55#include <asm/cpu.h>
56#include "entry.h" 43#include "entry.h"
57 44
58/* logical cpu to cpu address */ 45enum {
59unsigned short __cpu_logical_map[NR_CPUS]; 46 sigp_sense = 1,
47 sigp_external_call = 2,
48 sigp_emergency_signal = 3,
49 sigp_start = 4,
50 sigp_stop = 5,
51 sigp_restart = 6,
52 sigp_stop_and_store_status = 9,
53 sigp_initial_cpu_reset = 11,
54 sigp_cpu_reset = 12,
55 sigp_set_prefix = 13,
56 sigp_store_status_at_address = 14,
57 sigp_store_extended_status_at_address = 15,
58 sigp_set_architecture = 18,
59 sigp_conditional_emergency_signal = 19,
60 sigp_sense_running = 21,
61};
60 62
61static struct task_struct *current_set[NR_CPUS]; 63enum {
64 sigp_order_code_accepted = 0,
65 sigp_status_stored = 1,
66 sigp_busy = 2,
67 sigp_not_operational = 3,
68};
62 69
63static u8 smp_cpu_type; 70enum {
64static int smp_use_sigp_detection; 71 ec_schedule = 0,
72 ec_call_function,
73 ec_call_function_single,
74 ec_stop_cpu,
75};
65 76
66enum s390_cpu_state { 77enum {
67 CPU_STATE_STANDBY, 78 CPU_STATE_STANDBY,
68 CPU_STATE_CONFIGURED, 79 CPU_STATE_CONFIGURED,
69}; 80};
70 81
82struct pcpu {
83 struct cpu cpu;
84 struct task_struct *idle; /* idle process for the cpu */
85 struct _lowcore *lowcore; /* lowcore page(s) for the cpu */
86 unsigned long async_stack; /* async stack for the cpu */
87 unsigned long panic_stack; /* panic stack for the cpu */
88 unsigned long ec_mask; /* bit mask for ec_xxx functions */
89 int state; /* physical cpu state */
90 u32 status; /* last status received via sigp */
91 u16 address; /* physical cpu address */
92};
93
94static u8 boot_cpu_type;
95static u16 boot_cpu_address;
96static struct pcpu pcpu_devices[NR_CPUS];
97
71DEFINE_MUTEX(smp_cpu_state_mutex); 98DEFINE_MUTEX(smp_cpu_state_mutex);
72static int smp_cpu_state[NR_CPUS];
73 99
74static DEFINE_PER_CPU(struct cpu, cpu_devices); 100/*
101 * Signal processor helper functions.
102 */
103static inline int __pcpu_sigp(u16 addr, u8 order, u32 parm, u32 *status)
104{
105 register unsigned int reg1 asm ("1") = parm;
106 int cc;
75 107
76static void smp_ext_bitcall(int, int); 108 asm volatile(
109 " sigp %1,%2,0(%3)\n"
110 " ipm %0\n"
111 " srl %0,28\n"
112 : "=d" (cc), "+d" (reg1) : "d" (addr), "a" (order) : "cc");
113 if (status && cc == 1)
114 *status = reg1;
115 return cc;
116}
77 117
78static int raw_cpu_stopped(int cpu) 118static inline int __pcpu_sigp_relax(u16 addr, u8 order, u32 parm, u32 *status)
79{ 119{
80 u32 status; 120 int cc;
81 121
82 switch (raw_sigp_ps(&status, 0, cpu, sigp_sense)) { 122 while (1) {
83 case sigp_status_stored: 123 cc = __pcpu_sigp(addr, order, parm, status);
84 /* Check for stopped and check stop state */ 124 if (cc != sigp_busy)
85 if (status & 0x50) 125 return cc;
86 return 1; 126 cpu_relax();
87 break;
88 default:
89 break;
90 } 127 }
91 return 0;
92} 128}
93 129
94static inline int cpu_stopped(int cpu) 130static int pcpu_sigp_retry(struct pcpu *pcpu, u8 order, u32 parm)
95{ 131{
96 return raw_cpu_stopped(cpu_logical_map(cpu)); 132 int cc, retry;
133
134 for (retry = 0; ; retry++) {
135 cc = __pcpu_sigp(pcpu->address, order, parm, &pcpu->status);
136 if (cc != sigp_busy)
137 break;
138 if (retry >= 3)
139 udelay(10);
140 }
141 return cc;
142}
143
144static inline int pcpu_stopped(struct pcpu *pcpu)
145{
146 if (__pcpu_sigp(pcpu->address, sigp_sense,
147 0, &pcpu->status) != sigp_status_stored)
148 return 0;
149 /* Check for stopped and check stop state */
150 return !!(pcpu->status & 0x50);
151}
152
153static inline int pcpu_running(struct pcpu *pcpu)
154{
155 if (__pcpu_sigp(pcpu->address, sigp_sense_running,
156 0, &pcpu->status) != sigp_status_stored)
157 return 1;
158 /* Check for running status */
159 return !(pcpu->status & 0x400);
97} 160}
98 161
99/* 162/*
100 * Ensure that PSW restart is done on an online CPU 163 * Find struct pcpu by cpu address.
101 */ 164 */
102void smp_restart_with_online_cpu(void) 165static struct pcpu *pcpu_find_address(const struct cpumask *mask, int address)
103{ 166{
104 int cpu; 167 int cpu;
105 168
106 for_each_online_cpu(cpu) { 169 for_each_cpu(cpu, mask)
107 if (stap() == __cpu_logical_map[cpu]) { 170 if (pcpu_devices[cpu].address == address)
108 /* We are online: Enable DAT again and return */ 171 return pcpu_devices + cpu;
109 __load_psw_mask(psw_kernel_bits | PSW_MASK_DAT); 172 return NULL;
110 return; 173}
111 } 174
175static void pcpu_ec_call(struct pcpu *pcpu, int ec_bit)
176{
177 int order;
178
179 set_bit(ec_bit, &pcpu->ec_mask);
180 order = pcpu_running(pcpu) ?
181 sigp_external_call : sigp_emergency_signal;
182 pcpu_sigp_retry(pcpu, order, 0);
183}
184
185static int __cpuinit pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu)
186{
187 struct _lowcore *lc;
188
189 if (pcpu != &pcpu_devices[0]) {
190 pcpu->lowcore = (struct _lowcore *)
191 __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
192 pcpu->async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
193 pcpu->panic_stack = __get_free_page(GFP_KERNEL);
194 if (!pcpu->lowcore || !pcpu->panic_stack || !pcpu->async_stack)
195 goto out;
112 } 196 }
113 /* We are not online: Do PSW restart on an online CPU */ 197 lc = pcpu->lowcore;
114 while (sigp(cpu, sigp_restart) == sigp_busy) 198 memcpy(lc, &S390_lowcore, 512);
115 cpu_relax(); 199 memset((char *) lc + 512, 0, sizeof(*lc) - 512);
116 /* And stop ourself */ 200 lc->async_stack = pcpu->async_stack + ASYNC_SIZE;
117 while (raw_sigp(stap(), sigp_stop) == sigp_busy) 201 lc->panic_stack = pcpu->panic_stack + PAGE_SIZE;
118 cpu_relax(); 202 lc->cpu_nr = cpu;
119 for (;;); 203#ifndef CONFIG_64BIT
204 if (MACHINE_HAS_IEEE) {
205 lc->extended_save_area_addr = get_zeroed_page(GFP_KERNEL);
206 if (!lc->extended_save_area_addr)
207 goto out;
208 }
209#else
210 if (vdso_alloc_per_cpu(lc))
211 goto out;
212#endif
213 lowcore_ptr[cpu] = lc;
214 pcpu_sigp_retry(pcpu, sigp_set_prefix, (u32)(unsigned long) lc);
215 return 0;
216out:
217 if (pcpu != &pcpu_devices[0]) {
218 free_page(pcpu->panic_stack);
219 free_pages(pcpu->async_stack, ASYNC_ORDER);
220 free_pages((unsigned long) pcpu->lowcore, LC_ORDER);
221 }
222 return -ENOMEM;
120} 223}
121 224
122void smp_switch_to_ipl_cpu(void (*func)(void *), void *data) 225static void pcpu_free_lowcore(struct pcpu *pcpu)
123{ 226{
124 struct _lowcore *lc, *current_lc; 227 pcpu_sigp_retry(pcpu, sigp_set_prefix, 0);
125 struct stack_frame *sf; 228 lowcore_ptr[pcpu - pcpu_devices] = NULL;
126 struct pt_regs *regs; 229#ifndef CONFIG_64BIT
127 unsigned long sp; 230 if (MACHINE_HAS_IEEE) {
128 231 struct _lowcore *lc = pcpu->lowcore;
129 if (smp_processor_id() == 0) 232
130 func(data); 233 free_page((unsigned long) lc->extended_save_area_addr);
131 __load_psw_mask(PSW_DEFAULT_KEY | PSW_MASK_BASE | 234 lc->extended_save_area_addr = 0;
132 PSW_MASK_EA | PSW_MASK_BA); 235 }
133 /* Disable lowcore protection */ 236#else
134 __ctl_clear_bit(0, 28); 237 vdso_free_per_cpu(pcpu->lowcore);
135 current_lc = lowcore_ptr[smp_processor_id()]; 238#endif
136 lc = lowcore_ptr[0]; 239 if (pcpu != &pcpu_devices[0]) {
137 if (!lc) 240 free_page(pcpu->panic_stack);
138 lc = current_lc; 241 free_pages(pcpu->async_stack, ASYNC_ORDER);
139 lc->restart_psw.mask = 242 free_pages((unsigned long) pcpu->lowcore, LC_ORDER);
140 PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_MASK_EA | PSW_MASK_BA; 243 }
141 lc->restart_psw.addr = PSW_ADDR_AMODE | (unsigned long) smp_restart_cpu; 244}
142 if (!cpu_online(0)) 245
143 smp_switch_to_cpu(func, data, 0, stap(), __cpu_logical_map[0]); 246static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu)
144 while (sigp(0, sigp_stop_and_store_status) == sigp_busy) 247{
145 cpu_relax(); 248 struct _lowcore *lc = pcpu->lowcore;
146 sp = lc->panic_stack; 249
147 sp -= sizeof(struct pt_regs); 250 atomic_inc(&init_mm.context.attach_count);
148 regs = (struct pt_regs *) sp; 251 lc->cpu_nr = cpu;
149 memcpy(&regs->gprs, &current_lc->gpregs_save_area, sizeof(regs->gprs)); 252 lc->percpu_offset = __per_cpu_offset[cpu];
150 regs->psw = current_lc->psw_save_area; 253 lc->kernel_asce = S390_lowcore.kernel_asce;
151 sp -= STACK_FRAME_OVERHEAD; 254 lc->machine_flags = S390_lowcore.machine_flags;
152 sf = (struct stack_frame *) sp; 255 lc->ftrace_func = S390_lowcore.ftrace_func;
153 sf->back_chain = 0; 256 lc->user_timer = lc->system_timer = lc->steal_timer = 0;
154 smp_switch_to_cpu(func, data, sp, stap(), __cpu_logical_map[0]); 257 __ctl_store(lc->cregs_save_area, 0, 15);
258 save_access_regs((unsigned int *) lc->access_regs_save_area);
259 memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list,
260 MAX_FACILITY_BIT/8);
261}
262
263static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk)
264{
265 struct _lowcore *lc = pcpu->lowcore;
266 struct thread_info *ti = task_thread_info(tsk);
267
268 lc->kernel_stack = (unsigned long) task_stack_page(tsk) + THREAD_SIZE;
269 lc->thread_info = (unsigned long) task_thread_info(tsk);
270 lc->current_task = (unsigned long) tsk;
271 lc->user_timer = ti->user_timer;
272 lc->system_timer = ti->system_timer;
273 lc->steal_timer = 0;
274}
275
276static void pcpu_start_fn(struct pcpu *pcpu, void (*func)(void *), void *data)
277{
278 struct _lowcore *lc = pcpu->lowcore;
279
280 lc->restart_stack = lc->kernel_stack;
281 lc->restart_fn = (unsigned long) func;
282 lc->restart_data = (unsigned long) data;
283 lc->restart_source = -1UL;
284 pcpu_sigp_retry(pcpu, sigp_restart, 0);
285}
286
287/*
288 * Call function via PSW restart on pcpu and stop the current cpu.
289 */
290static void pcpu_delegate(struct pcpu *pcpu, void (*func)(void *),
291 void *data, unsigned long stack)
292{
293 struct _lowcore *lc = pcpu->lowcore;
294 unsigned short this_cpu;
295
296 __load_psw_mask(psw_kernel_bits);
297 this_cpu = stap();
298 if (pcpu->address == this_cpu)
299 func(data); /* should not return */
300 /* Stop target cpu (if func returns this stops the current cpu). */
301 pcpu_sigp_retry(pcpu, sigp_stop, 0);
302 /* Restart func on the target cpu and stop the current cpu. */
303 lc->restart_stack = stack;
304 lc->restart_fn = (unsigned long) func;
305 lc->restart_data = (unsigned long) data;
306 lc->restart_source = (unsigned long) this_cpu;
307 asm volatile(
308 "0: sigp 0,%0,6 # sigp restart to target cpu\n"
309 " brc 2,0b # busy, try again\n"
310 "1: sigp 0,%1,5 # sigp stop to current cpu\n"
311 " brc 2,1b # busy, try again\n"
312 : : "d" (pcpu->address), "d" (this_cpu) : "0", "1", "cc");
313 for (;;) ;
314}
315
316/*
317 * Call function on an online CPU.
318 */
319void smp_call_online_cpu(void (*func)(void *), void *data)
320{
321 struct pcpu *pcpu;
322
323 /* Use the current cpu if it is online. */
324 pcpu = pcpu_find_address(cpu_online_mask, stap());
325 if (!pcpu)
326 /* Use the first online cpu. */
327 pcpu = pcpu_devices + cpumask_first(cpu_online_mask);
328 pcpu_delegate(pcpu, func, data, (unsigned long) restart_stack);
329}
330
331/*
332 * Call function on the ipl CPU.
333 */
334void smp_call_ipl_cpu(void (*func)(void *), void *data)
335{
336 pcpu_delegate(&pcpu_devices[0], func, data, pcpu_devices->panic_stack);
337}
338
339int smp_find_processor_id(u16 address)
340{
341 int cpu;
342
343 for_each_present_cpu(cpu)
344 if (pcpu_devices[cpu].address == address)
345 return cpu;
346 return -1;
155} 347}
156 348
157static void smp_stop_cpu(void) 349int smp_vcpu_scheduled(int cpu)
158{ 350{
159 while (sigp(smp_processor_id(), sigp_stop) == sigp_busy) 351 return pcpu_running(pcpu_devices + cpu);
352}
353
354void smp_yield(void)
355{
356 if (MACHINE_HAS_DIAG44)
357 asm volatile("diag 0,0,0x44");
358}
359
360void smp_yield_cpu(int cpu)
361{
362 if (MACHINE_HAS_DIAG9C)
363 asm volatile("diag %0,0,0x9c"
364 : : "d" (pcpu_devices[cpu].address));
365 else if (MACHINE_HAS_DIAG44)
366 asm volatile("diag 0,0,0x44");
367}
368
369/*
370 * Send cpus emergency shutdown signal. This gives the cpus the
371 * opportunity to complete outstanding interrupts.
372 */
373void smp_emergency_stop(cpumask_t *cpumask)
374{
375 u64 end;
376 int cpu;
377
378 end = get_clock() + (1000000UL << 12);
379 for_each_cpu(cpu, cpumask) {
380 struct pcpu *pcpu = pcpu_devices + cpu;
381 set_bit(ec_stop_cpu, &pcpu->ec_mask);
382 while (__pcpu_sigp(pcpu->address, sigp_emergency_signal,
383 0, NULL) == sigp_busy &&
384 get_clock() < end)
385 cpu_relax();
386 }
387 while (get_clock() < end) {
388 for_each_cpu(cpu, cpumask)
389 if (pcpu_stopped(pcpu_devices + cpu))
390 cpumask_clear_cpu(cpu, cpumask);
391 if (cpumask_empty(cpumask))
392 break;
160 cpu_relax(); 393 cpu_relax();
394 }
161} 395}
162 396
397/*
398 * Stop all cpus but the current one.
399 */
163void smp_send_stop(void) 400void smp_send_stop(void)
164{ 401{
165 cpumask_t cpumask; 402 cpumask_t cpumask;
166 int cpu; 403 int cpu;
167 u64 end;
168 404
169 /* Disable all interrupts/machine checks */ 405 /* Disable all interrupts/machine checks */
170 __load_psw_mask(psw_kernel_bits | PSW_MASK_DAT); 406 __load_psw_mask(psw_kernel_bits | PSW_MASK_DAT);
@@ -173,56 +409,46 @@ void smp_send_stop(void)
173 cpumask_copy(&cpumask, cpu_online_mask); 409 cpumask_copy(&cpumask, cpu_online_mask);
174 cpumask_clear_cpu(smp_processor_id(), &cpumask); 410 cpumask_clear_cpu(smp_processor_id(), &cpumask);
175 411
176 if (oops_in_progress) { 412 if (oops_in_progress)
177 /* 413 smp_emergency_stop(&cpumask);
178 * Give the other cpus the opportunity to complete
179 * outstanding interrupts before stopping them.
180 */
181 end = get_clock() + (1000000UL << 12);
182 for_each_cpu(cpu, &cpumask) {
183 set_bit(ec_stop_cpu, (unsigned long *)
184 &lowcore_ptr[cpu]->ext_call_fast);
185 while (sigp(cpu, sigp_emergency_signal) == sigp_busy &&
186 get_clock() < end)
187 cpu_relax();
188 }
189 while (get_clock() < end) {
190 for_each_cpu(cpu, &cpumask)
191 if (cpu_stopped(cpu))
192 cpumask_clear_cpu(cpu, &cpumask);
193 if (cpumask_empty(&cpumask))
194 break;
195 cpu_relax();
196 }
197 }
198 414
199 /* stop all processors */ 415 /* stop all processors */
200 for_each_cpu(cpu, &cpumask) { 416 for_each_cpu(cpu, &cpumask) {
201 while (sigp(cpu, sigp_stop) == sigp_busy) 417 struct pcpu *pcpu = pcpu_devices + cpu;
202 cpu_relax(); 418 pcpu_sigp_retry(pcpu, sigp_stop, 0);
203 while (!cpu_stopped(cpu)) 419 while (!pcpu_stopped(pcpu))
204 cpu_relax(); 420 cpu_relax();
205 } 421 }
206} 422}
207 423
208/* 424/*
425 * Stop the current cpu.
426 */
427void smp_stop_cpu(void)
428{
429 pcpu_sigp_retry(pcpu_devices + smp_processor_id(), sigp_stop, 0);
430 for (;;) ;
431}
432
433/*
209 * This is the main routine where commands issued by other 434 * This is the main routine where commands issued by other
210 * cpus are handled. 435 * cpus are handled.
211 */ 436 */
212
213static void do_ext_call_interrupt(unsigned int ext_int_code, 437static void do_ext_call_interrupt(unsigned int ext_int_code,
214 unsigned int param32, unsigned long param64) 438 unsigned int param32, unsigned long param64)
215{ 439{
216 unsigned long bits; 440 unsigned long bits;
441 int cpu;
217 442
443 cpu = smp_processor_id();
218 if ((ext_int_code & 0xffff) == 0x1202) 444 if ((ext_int_code & 0xffff) == 0x1202)
219 kstat_cpu(smp_processor_id()).irqs[EXTINT_EXC]++; 445 kstat_cpu(cpu).irqs[EXTINT_EXC]++;
220 else 446 else
221 kstat_cpu(smp_processor_id()).irqs[EXTINT_EMS]++; 447 kstat_cpu(cpu).irqs[EXTINT_EMS]++;
222 /* 448 /*
223 * handle bit signal external calls 449 * handle bit signal external calls
224 */ 450 */
225 bits = xchg(&S390_lowcore.ext_call_fast, 0); 451 bits = xchg(&pcpu_devices[cpu].ec_mask, 0);
226 452
227 if (test_bit(ec_stop_cpu, &bits)) 453 if (test_bit(ec_stop_cpu, &bits))
228 smp_stop_cpu(); 454 smp_stop_cpu();
@@ -238,38 +464,17 @@ static void do_ext_call_interrupt(unsigned int ext_int_code,
238 464
239} 465}
240 466
241/*
242 * Send an external call sigp to another cpu and return without waiting
243 * for its completion.
244 */
245static void smp_ext_bitcall(int cpu, int sig)
246{
247 int order;
248
249 /*
250 * Set signaling bit in lowcore of target cpu and kick it
251 */
252 set_bit(sig, (unsigned long *) &lowcore_ptr[cpu]->ext_call_fast);
253 while (1) {
254 order = smp_vcpu_scheduled(cpu) ?
255 sigp_external_call : sigp_emergency_signal;
256 if (sigp(cpu, order) != sigp_busy)
257 break;
258 udelay(10);
259 }
260}
261
262void arch_send_call_function_ipi_mask(const struct cpumask *mask) 467void arch_send_call_function_ipi_mask(const struct cpumask *mask)
263{ 468{
264 int cpu; 469 int cpu;
265 470
266 for_each_cpu(cpu, mask) 471 for_each_cpu(cpu, mask)
267 smp_ext_bitcall(cpu, ec_call_function); 472 pcpu_ec_call(pcpu_devices + cpu, ec_call_function);
268} 473}
269 474
270void arch_send_call_function_single_ipi(int cpu) 475void arch_send_call_function_single_ipi(int cpu)
271{ 476{
272 smp_ext_bitcall(cpu, ec_call_function_single); 477 pcpu_ec_call(pcpu_devices + cpu, ec_call_function_single);
273} 478}
274 479
275#ifndef CONFIG_64BIT 480#ifndef CONFIG_64BIT
@@ -295,15 +500,16 @@ EXPORT_SYMBOL(smp_ptlb_all);
295 */ 500 */
296void smp_send_reschedule(int cpu) 501void smp_send_reschedule(int cpu)
297{ 502{
298 smp_ext_bitcall(cpu, ec_schedule); 503 pcpu_ec_call(pcpu_devices + cpu, ec_schedule);
299} 504}
300 505
301/* 506/*
302 * parameter area for the set/clear control bit callbacks 507 * parameter area for the set/clear control bit callbacks
303 */ 508 */
304struct ec_creg_mask_parms { 509struct ec_creg_mask_parms {
305 unsigned long orvals[16]; 510 unsigned long orval;
306 unsigned long andvals[16]; 511 unsigned long andval;
512 int cr;
307}; 513};
308 514
309/* 515/*
@@ -313,11 +519,9 @@ static void smp_ctl_bit_callback(void *info)
313{ 519{
314 struct ec_creg_mask_parms *pp = info; 520 struct ec_creg_mask_parms *pp = info;
315 unsigned long cregs[16]; 521 unsigned long cregs[16];
316 int i;
317 522
318 __ctl_store(cregs, 0, 15); 523 __ctl_store(cregs, 0, 15);
319 for (i = 0; i <= 15; i++) 524 cregs[pp->cr] = (cregs[pp->cr] & pp->andval) | pp->orval;
320 cregs[i] = (cregs[i] & pp->andvals[i]) | pp->orvals[i];
321 __ctl_load(cregs, 0, 15); 525 __ctl_load(cregs, 0, 15);
322} 526}
323 527
@@ -326,11 +530,8 @@ static void smp_ctl_bit_callback(void *info)
326 */ 530 */
327void smp_ctl_set_bit(int cr, int bit) 531void smp_ctl_set_bit(int cr, int bit)
328{ 532{
329 struct ec_creg_mask_parms parms; 533 struct ec_creg_mask_parms parms = { 1UL << bit, -1UL, cr };
330 534
331 memset(&parms.orvals, 0, sizeof(parms.orvals));
332 memset(&parms.andvals, 0xff, sizeof(parms.andvals));
333 parms.orvals[cr] = 1UL << bit;
334 on_each_cpu(smp_ctl_bit_callback, &parms, 1); 535 on_each_cpu(smp_ctl_bit_callback, &parms, 1);
335} 536}
336EXPORT_SYMBOL(smp_ctl_set_bit); 537EXPORT_SYMBOL(smp_ctl_set_bit);
@@ -340,216 +541,175 @@ EXPORT_SYMBOL(smp_ctl_set_bit);
340 */ 541 */
341void smp_ctl_clear_bit(int cr, int bit) 542void smp_ctl_clear_bit(int cr, int bit)
342{ 543{
343 struct ec_creg_mask_parms parms; 544 struct ec_creg_mask_parms parms = { 0, ~(1UL << bit), cr };
344 545
345 memset(&parms.orvals, 0, sizeof(parms.orvals));
346 memset(&parms.andvals, 0xff, sizeof(parms.andvals));
347 parms.andvals[cr] = ~(1UL << bit);
348 on_each_cpu(smp_ctl_bit_callback, &parms, 1); 546 on_each_cpu(smp_ctl_bit_callback, &parms, 1);
349} 547}
350EXPORT_SYMBOL(smp_ctl_clear_bit); 548EXPORT_SYMBOL(smp_ctl_clear_bit);
351 549
352#if defined(CONFIG_ZFCPDUMP) || defined(CONFIG_CRASH_DUMP) 550#if defined(CONFIG_ZFCPDUMP) || defined(CONFIG_CRASH_DUMP)
353 551
354static void __init smp_get_save_area(unsigned int cpu, unsigned int phy_cpu) 552struct save_area *zfcpdump_save_areas[NR_CPUS + 1];
553EXPORT_SYMBOL_GPL(zfcpdump_save_areas);
554
555static void __init smp_get_save_area(int cpu, u16 address)
355{ 556{
356 if (ipl_info.type != IPL_TYPE_FCP_DUMP && !OLDMEM_BASE) 557 void *lc = pcpu_devices[0].lowcore;
357 return; 558 struct save_area *save_area;
559
358 if (is_kdump_kernel()) 560 if (is_kdump_kernel())
359 return; 561 return;
562 if (!OLDMEM_BASE && (address == boot_cpu_address ||
563 ipl_info.type != IPL_TYPE_FCP_DUMP))
564 return;
360 if (cpu >= NR_CPUS) { 565 if (cpu >= NR_CPUS) {
361 pr_warning("CPU %i exceeds the maximum %i and is excluded from " 566 pr_warning("CPU %i exceeds the maximum %i and is excluded "
362 "the dump\n", cpu, NR_CPUS - 1); 567 "from the dump\n", cpu, NR_CPUS - 1);
363 return; 568 return;
364 } 569 }
365 zfcpdump_save_areas[cpu] = kmalloc(sizeof(struct save_area), GFP_KERNEL); 570 save_area = kmalloc(sizeof(struct save_area), GFP_KERNEL);
366 while (raw_sigp(phy_cpu, sigp_stop_and_store_status) == sigp_busy) 571 if (!save_area)
367 cpu_relax(); 572 panic("could not allocate memory for save area\n");
368 memcpy_real(zfcpdump_save_areas[cpu], 573 zfcpdump_save_areas[cpu] = save_area;
369 (void *)(unsigned long) store_prefix() + SAVE_AREA_BASE, 574#ifdef CONFIG_CRASH_DUMP
370 sizeof(struct save_area)); 575 if (address == boot_cpu_address) {
576 /* Copy the registers of the boot cpu. */
577 copy_oldmem_page(1, (void *) save_area, sizeof(*save_area),
578 SAVE_AREA_BASE - PAGE_SIZE, 0);
579 return;
580 }
581#endif
582 /* Get the registers of a non-boot cpu. */
583 __pcpu_sigp_relax(address, sigp_stop_and_store_status, 0, NULL);
584 memcpy_real(save_area, lc + SAVE_AREA_BASE, sizeof(*save_area));
371} 585}
372 586
373struct save_area *zfcpdump_save_areas[NR_CPUS + 1]; 587int smp_store_status(int cpu)
374EXPORT_SYMBOL_GPL(zfcpdump_save_areas);
375
376#else
377
378static inline void smp_get_save_area(unsigned int cpu, unsigned int phy_cpu) { }
379
380#endif /* CONFIG_ZFCPDUMP */
381
382static int cpu_known(int cpu_id)
383{ 588{
384 int cpu; 589 struct pcpu *pcpu;
385 590
386 for_each_present_cpu(cpu) { 591 pcpu = pcpu_devices + cpu;
387 if (__cpu_logical_map[cpu] == cpu_id) 592 if (__pcpu_sigp_relax(pcpu->address, sigp_stop_and_store_status,
388 return 1; 593 0, NULL) != sigp_order_code_accepted)
389 } 594 return -EIO;
390 return 0; 595 return 0;
391} 596}
392 597
393static int smp_rescan_cpus_sigp(cpumask_t avail) 598#else /* CONFIG_ZFCPDUMP || CONFIG_CRASH_DUMP */
394{
395 int cpu_id, logical_cpu;
396 599
397 logical_cpu = cpumask_first(&avail); 600static inline void smp_get_save_area(int cpu, u16 address) { }
398 if (logical_cpu >= nr_cpu_ids) 601
399 return 0; 602#endif /* CONFIG_ZFCPDUMP || CONFIG_CRASH_DUMP */
400 for (cpu_id = 0; cpu_id <= MAX_CPU_ADDRESS; cpu_id++) {
401 if (cpu_known(cpu_id))
402 continue;
403 __cpu_logical_map[logical_cpu] = cpu_id;
404 cpu_set_polarization(logical_cpu, POLARIZATION_UNKNOWN);
405 if (!cpu_stopped(logical_cpu))
406 continue;
407 set_cpu_present(logical_cpu, true);
408 smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
409 logical_cpu = cpumask_next(logical_cpu, &avail);
410 if (logical_cpu >= nr_cpu_ids)
411 break;
412 }
413 return 0;
414}
415 603
416static int smp_rescan_cpus_sclp(cpumask_t avail) 604static struct sclp_cpu_info *smp_get_cpu_info(void)
417{ 605{
606 static int use_sigp_detection;
418 struct sclp_cpu_info *info; 607 struct sclp_cpu_info *info;
419 int cpu_id, logical_cpu, cpu; 608 int address;
420 int rc; 609
421 610 info = kzalloc(sizeof(*info), GFP_KERNEL);
422 logical_cpu = cpumask_first(&avail); 611 if (info && (use_sigp_detection || sclp_get_cpu_info(info))) {
423 if (logical_cpu >= nr_cpu_ids) 612 use_sigp_detection = 1;
424 return 0; 613 for (address = 0; address <= MAX_CPU_ADDRESS; address++) {
425 info = kmalloc(sizeof(*info), GFP_KERNEL); 614 if (__pcpu_sigp_relax(address, sigp_sense, 0, NULL) ==
426 if (!info) 615 sigp_not_operational)
427 return -ENOMEM; 616 continue;
428 rc = sclp_get_cpu_info(info); 617 info->cpu[info->configured].address = address;
429 if (rc) 618 info->configured++;
430 goto out; 619 }
431 for (cpu = 0; cpu < info->combined; cpu++) { 620 info->combined = info->configured;
432 if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
433 continue;
434 cpu_id = info->cpu[cpu].address;
435 if (cpu_known(cpu_id))
436 continue;
437 __cpu_logical_map[logical_cpu] = cpu_id;
438 cpu_set_polarization(logical_cpu, POLARIZATION_UNKNOWN);
439 set_cpu_present(logical_cpu, true);
440 if (cpu >= info->configured)
441 smp_cpu_state[logical_cpu] = CPU_STATE_STANDBY;
442 else
443 smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
444 logical_cpu = cpumask_next(logical_cpu, &avail);
445 if (logical_cpu >= nr_cpu_ids)
446 break;
447 } 621 }
448out: 622 return info;
449 kfree(info);
450 return rc;
451} 623}
452 624
453static int __smp_rescan_cpus(void) 625static int __devinit smp_add_present_cpu(int cpu);
626
627static int __devinit __smp_rescan_cpus(struct sclp_cpu_info *info,
628 int sysfs_add)
454{ 629{
630 struct pcpu *pcpu;
455 cpumask_t avail; 631 cpumask_t avail;
632 int cpu, nr, i;
456 633
634 nr = 0;
457 cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask); 635 cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask);
458 if (smp_use_sigp_detection) 636 cpu = cpumask_first(&avail);
459 return smp_rescan_cpus_sigp(avail); 637 for (i = 0; (i < info->combined) && (cpu < nr_cpu_ids); i++) {
460 else 638 if (info->has_cpu_type && info->cpu[i].type != boot_cpu_type)
461 return smp_rescan_cpus_sclp(avail); 639 continue;
640 if (pcpu_find_address(cpu_present_mask, info->cpu[i].address))
641 continue;
642 pcpu = pcpu_devices + cpu;
643 pcpu->address = info->cpu[i].address;
644 pcpu->state = (cpu >= info->configured) ?
645 CPU_STATE_STANDBY : CPU_STATE_CONFIGURED;
646 cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
647 set_cpu_present(cpu, true);
648 if (sysfs_add && smp_add_present_cpu(cpu) != 0)
649 set_cpu_present(cpu, false);
650 else
651 nr++;
652 cpu = cpumask_next(cpu, &avail);
653 }
654 return nr;
462} 655}
463 656
464static void __init smp_detect_cpus(void) 657static void __init smp_detect_cpus(void)
465{ 658{
466 unsigned int cpu, c_cpus, s_cpus; 659 unsigned int cpu, c_cpus, s_cpus;
467 struct sclp_cpu_info *info; 660 struct sclp_cpu_info *info;
468 u16 boot_cpu_addr, cpu_addr;
469 661
470 c_cpus = 1; 662 info = smp_get_cpu_info();
471 s_cpus = 0;
472 boot_cpu_addr = __cpu_logical_map[0];
473 info = kmalloc(sizeof(*info), GFP_KERNEL);
474 if (!info) 663 if (!info)
475 panic("smp_detect_cpus failed to allocate memory\n"); 664 panic("smp_detect_cpus failed to allocate memory\n");
476#ifdef CONFIG_CRASH_DUMP
477 if (OLDMEM_BASE && !is_kdump_kernel()) {
478 struct save_area *save_area;
479
480 save_area = kmalloc(sizeof(*save_area), GFP_KERNEL);
481 if (!save_area)
482 panic("could not allocate memory for save area\n");
483 copy_oldmem_page(1, (void *) save_area, sizeof(*save_area),
484 0x200, 0);
485 zfcpdump_save_areas[0] = save_area;
486 }
487#endif
488 /* Use sigp detection algorithm if sclp doesn't work. */
489 if (sclp_get_cpu_info(info)) {
490 smp_use_sigp_detection = 1;
491 for (cpu = 0; cpu <= MAX_CPU_ADDRESS; cpu++) {
492 if (cpu == boot_cpu_addr)
493 continue;
494 if (!raw_cpu_stopped(cpu))
495 continue;
496 smp_get_save_area(c_cpus, cpu);
497 c_cpus++;
498 }
499 goto out;
500 }
501
502 if (info->has_cpu_type) { 665 if (info->has_cpu_type) {
503 for (cpu = 0; cpu < info->combined; cpu++) { 666 for (cpu = 0; cpu < info->combined; cpu++) {
504 if (info->cpu[cpu].address == boot_cpu_addr) { 667 if (info->cpu[cpu].address != boot_cpu_address)
505 smp_cpu_type = info->cpu[cpu].type; 668 continue;
506 break; 669 /* The boot cpu dictates the cpu type. */
507 } 670 boot_cpu_type = info->cpu[cpu].type;
671 break;
508 } 672 }
509 } 673 }
510 674 c_cpus = s_cpus = 0;
511 for (cpu = 0; cpu < info->combined; cpu++) { 675 for (cpu = 0; cpu < info->combined; cpu++) {
512 if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type) 676 if (info->has_cpu_type && info->cpu[cpu].type != boot_cpu_type)
513 continue; 677 continue;
514 cpu_addr = info->cpu[cpu].address; 678 if (cpu < info->configured) {
515 if (cpu_addr == boot_cpu_addr) 679 smp_get_save_area(c_cpus, info->cpu[cpu].address);
516 continue; 680 c_cpus++;
517 if (!raw_cpu_stopped(cpu_addr)) { 681 } else
518 s_cpus++; 682 s_cpus++;
519 continue;
520 }
521 smp_get_save_area(c_cpus, cpu_addr);
522 c_cpus++;
523 } 683 }
524out:
525 kfree(info);
526 pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus); 684 pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
527 get_online_cpus(); 685 get_online_cpus();
528 __smp_rescan_cpus(); 686 __smp_rescan_cpus(info, 0);
529 put_online_cpus(); 687 put_online_cpus();
688 kfree(info);
530} 689}
531 690
532/* 691/*
533 * Activate a secondary processor. 692 * Activate a secondary processor.
534 */ 693 */
535int __cpuinit start_secondary(void *cpuvoid) 694static void __cpuinit smp_start_secondary(void *cpuvoid)
536{ 695{
696 S390_lowcore.last_update_clock = get_clock();
697 S390_lowcore.restart_stack = (unsigned long) restart_stack;
698 S390_lowcore.restart_fn = (unsigned long) do_restart;
699 S390_lowcore.restart_data = 0;
700 S390_lowcore.restart_source = -1UL;
701 restore_access_regs(S390_lowcore.access_regs_save_area);
702 __ctl_load(S390_lowcore.cregs_save_area, 0, 15);
703 __load_psw_mask(psw_kernel_bits | PSW_MASK_DAT);
537 cpu_init(); 704 cpu_init();
538 preempt_disable(); 705 preempt_disable();
539 init_cpu_timer(); 706 init_cpu_timer();
540 init_cpu_vtimer(); 707 init_cpu_vtimer();
541 pfault_init(); 708 pfault_init();
542
543 notify_cpu_starting(smp_processor_id()); 709 notify_cpu_starting(smp_processor_id());
544 ipi_call_lock(); 710 ipi_call_lock();
545 set_cpu_online(smp_processor_id(), true); 711 set_cpu_online(smp_processor_id(), true);
546 ipi_call_unlock(); 712 ipi_call_unlock();
547 __ctl_clear_bit(0, 28); /* Disable lowcore protection */
548 S390_lowcore.restart_psw.mask =
549 PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_MASK_EA | PSW_MASK_BA;
550 S390_lowcore.restart_psw.addr =
551 PSW_ADDR_AMODE | (unsigned long) psw_restart_int_handler;
552 __ctl_set_bit(0, 28); /* Enable lowcore protection */
553 /* 713 /*
554 * Wait until the cpu which brought this one up marked it 714 * Wait until the cpu which brought this one up marked it
555 * active before enabling interrupts. 715 * active before enabling interrupts.
@@ -559,7 +719,6 @@ int __cpuinit start_secondary(void *cpuvoid)
559 local_irq_enable(); 719 local_irq_enable();
560 /* cpu_idle will call schedule for us */ 720 /* cpu_idle will call schedule for us */
561 cpu_idle(); 721 cpu_idle();
562 return 0;
563} 722}
564 723
565struct create_idle { 724struct create_idle {
@@ -578,82 +737,20 @@ static void __cpuinit smp_fork_idle(struct work_struct *work)
578 complete(&c_idle->done); 737 complete(&c_idle->done);
579} 738}
580 739
581static int __cpuinit smp_alloc_lowcore(int cpu)
582{
583 unsigned long async_stack, panic_stack;
584 struct _lowcore *lowcore;
585
586 lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
587 if (!lowcore)
588 return -ENOMEM;
589 async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
590 panic_stack = __get_free_page(GFP_KERNEL);
591 if (!panic_stack || !async_stack)
592 goto out;
593 memcpy(lowcore, &S390_lowcore, 512);
594 memset((char *)lowcore + 512, 0, sizeof(*lowcore) - 512);
595 lowcore->async_stack = async_stack + ASYNC_SIZE;
596 lowcore->panic_stack = panic_stack + PAGE_SIZE;
597 lowcore->restart_psw.mask =
598 PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_MASK_EA | PSW_MASK_BA;
599 lowcore->restart_psw.addr =
600 PSW_ADDR_AMODE | (unsigned long) restart_int_handler;
601 if (user_mode != HOME_SPACE_MODE)
602 lowcore->restart_psw.mask |= PSW_ASC_HOME;
603#ifndef CONFIG_64BIT
604 if (MACHINE_HAS_IEEE) {
605 unsigned long save_area;
606
607 save_area = get_zeroed_page(GFP_KERNEL);
608 if (!save_area)
609 goto out;
610 lowcore->extended_save_area_addr = (u32) save_area;
611 }
612#else
613 if (vdso_alloc_per_cpu(cpu, lowcore))
614 goto out;
615#endif
616 lowcore_ptr[cpu] = lowcore;
617 return 0;
618
619out:
620 free_page(panic_stack);
621 free_pages(async_stack, ASYNC_ORDER);
622 free_pages((unsigned long) lowcore, LC_ORDER);
623 return -ENOMEM;
624}
625
626static void smp_free_lowcore(int cpu)
627{
628 struct _lowcore *lowcore;
629
630 lowcore = lowcore_ptr[cpu];
631#ifndef CONFIG_64BIT
632 if (MACHINE_HAS_IEEE)
633 free_page((unsigned long) lowcore->extended_save_area_addr);
634#else
635 vdso_free_per_cpu(cpu, lowcore);
636#endif
637 free_page(lowcore->panic_stack - PAGE_SIZE);
638 free_pages(lowcore->async_stack - ASYNC_SIZE, ASYNC_ORDER);
639 free_pages((unsigned long) lowcore, LC_ORDER);
640 lowcore_ptr[cpu] = NULL;
641}
642
643/* Upping and downing of CPUs */ 740/* Upping and downing of CPUs */
644int __cpuinit __cpu_up(unsigned int cpu) 741int __cpuinit __cpu_up(unsigned int cpu)
645{ 742{
646 struct _lowcore *cpu_lowcore;
647 struct create_idle c_idle; 743 struct create_idle c_idle;
648 struct task_struct *idle; 744 struct pcpu *pcpu;
649 struct stack_frame *sf; 745 int rc;
650 u32 lowcore;
651 int ccode;
652 746
653 if (smp_cpu_state[cpu] != CPU_STATE_CONFIGURED) 747 pcpu = pcpu_devices + cpu;
748 if (pcpu->state != CPU_STATE_CONFIGURED)
654 return -EIO; 749 return -EIO;
655 idle = current_set[cpu]; 750 if (pcpu_sigp_retry(pcpu, sigp_initial_cpu_reset, 0) !=
656 if (!idle) { 751 sigp_order_code_accepted)
752 return -EIO;
753 if (!pcpu->idle) {
657 c_idle.done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done); 754 c_idle.done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done);
658 INIT_WORK_ONSTACK(&c_idle.work, smp_fork_idle); 755 INIT_WORK_ONSTACK(&c_idle.work, smp_fork_idle);
659 c_idle.cpu = cpu; 756 c_idle.cpu = cpu;
@@ -661,68 +758,28 @@ int __cpuinit __cpu_up(unsigned int cpu)
661 wait_for_completion(&c_idle.done); 758 wait_for_completion(&c_idle.done);
662 if (IS_ERR(c_idle.idle)) 759 if (IS_ERR(c_idle.idle))
663 return PTR_ERR(c_idle.idle); 760 return PTR_ERR(c_idle.idle);
664 idle = c_idle.idle; 761 pcpu->idle = c_idle.idle;
665 current_set[cpu] = c_idle.idle;
666 } 762 }
667 init_idle(idle, cpu); 763 init_idle(pcpu->idle, cpu);
668 if (smp_alloc_lowcore(cpu)) 764 rc = pcpu_alloc_lowcore(pcpu, cpu);
669 return -ENOMEM; 765 if (rc)
670 do { 766 return rc;
671 ccode = sigp(cpu, sigp_initial_cpu_reset); 767 pcpu_prepare_secondary(pcpu, cpu);
672 if (ccode == sigp_busy) 768 pcpu_attach_task(pcpu, pcpu->idle);
673 udelay(10); 769 pcpu_start_fn(pcpu, smp_start_secondary, NULL);
674 if (ccode == sigp_not_operational)
675 goto err_out;
676 } while (ccode == sigp_busy);
677
678 lowcore = (u32)(unsigned long)lowcore_ptr[cpu];
679 while (sigp_p(lowcore, cpu, sigp_set_prefix) == sigp_busy)
680 udelay(10);
681
682 cpu_lowcore = lowcore_ptr[cpu];
683 cpu_lowcore->kernel_stack = (unsigned long)
684 task_stack_page(idle) + THREAD_SIZE;
685 cpu_lowcore->thread_info = (unsigned long) task_thread_info(idle);
686 sf = (struct stack_frame *) (cpu_lowcore->kernel_stack
687 - sizeof(struct pt_regs)
688 - sizeof(struct stack_frame));
689 memset(sf, 0, sizeof(struct stack_frame));
690 sf->gprs[9] = (unsigned long) sf;
691 cpu_lowcore->gpregs_save_area[15] = (unsigned long) sf;
692 __ctl_store(cpu_lowcore->cregs_save_area, 0, 15);
693 atomic_inc(&init_mm.context.attach_count);
694 asm volatile(
695 " stam 0,15,0(%0)"
696 : : "a" (&cpu_lowcore->access_regs_save_area) : "memory");
697 cpu_lowcore->percpu_offset = __per_cpu_offset[cpu];
698 cpu_lowcore->current_task = (unsigned long) idle;
699 cpu_lowcore->cpu_nr = cpu;
700 cpu_lowcore->kernel_asce = S390_lowcore.kernel_asce;
701 cpu_lowcore->machine_flags = S390_lowcore.machine_flags;
702 cpu_lowcore->ftrace_func = S390_lowcore.ftrace_func;
703 memcpy(cpu_lowcore->stfle_fac_list, S390_lowcore.stfle_fac_list,
704 MAX_FACILITY_BIT/8);
705 eieio();
706
707 while (sigp(cpu, sigp_restart) == sigp_busy)
708 udelay(10);
709
710 while (!cpu_online(cpu)) 770 while (!cpu_online(cpu))
711 cpu_relax(); 771 cpu_relax();
712 return 0; 772 return 0;
713
714err_out:
715 smp_free_lowcore(cpu);
716 return -EIO;
717} 773}
718 774
719static int __init setup_possible_cpus(char *s) 775static int __init setup_possible_cpus(char *s)
720{ 776{
721 int pcpus, cpu; 777 int max, cpu;
722 778
723 pcpus = simple_strtoul(s, NULL, 0); 779 if (kstrtoint(s, 0, &max) < 0)
780 return 0;
724 init_cpu_possible(cpumask_of(0)); 781 init_cpu_possible(cpumask_of(0));
725 for (cpu = 1; cpu < pcpus && cpu < nr_cpu_ids; cpu++) 782 for (cpu = 1; cpu < max && cpu < nr_cpu_ids; cpu++)
726 set_cpu_possible(cpu, true); 783 set_cpu_possible(cpu, true);
727 return 0; 784 return 0;
728} 785}
@@ -732,113 +789,67 @@ early_param("possible_cpus", setup_possible_cpus);
732 789
733int __cpu_disable(void) 790int __cpu_disable(void)
734{ 791{
735 struct ec_creg_mask_parms cr_parms; 792 unsigned long cregs[16];
736 int cpu = smp_processor_id();
737
738 set_cpu_online(cpu, false);
739 793
740 /* Disable pfault pseudo page faults on this cpu. */ 794 set_cpu_online(smp_processor_id(), false);
795 /* Disable pseudo page faults on this cpu. */
741 pfault_fini(); 796 pfault_fini();
742 797 /* Disable interrupt sources via control register. */
743 memset(&cr_parms.orvals, 0, sizeof(cr_parms.orvals)); 798 __ctl_store(cregs, 0, 15);
744 memset(&cr_parms.andvals, 0xff, sizeof(cr_parms.andvals)); 799 cregs[0] &= ~0x0000ee70UL; /* disable all external interrupts */
745 800 cregs[6] &= ~0xff000000UL; /* disable all I/O interrupts */
746 /* disable all external interrupts */ 801 cregs[14] &= ~0x1f000000UL; /* disable most machine checks */
747 cr_parms.orvals[0] = 0; 802 __ctl_load(cregs, 0, 15);
748 cr_parms.andvals[0] = ~(1 << 15 | 1 << 14 | 1 << 13 | 1 << 11 |
749 1 << 10 | 1 << 9 | 1 << 6 | 1 << 5 |
750 1 << 4);
751 /* disable all I/O interrupts */
752 cr_parms.orvals[6] = 0;
753 cr_parms.andvals[6] = ~(1 << 31 | 1 << 30 | 1 << 29 | 1 << 28 |
754 1 << 27 | 1 << 26 | 1 << 25 | 1 << 24);
755 /* disable most machine checks */
756 cr_parms.orvals[14] = 0;
757 cr_parms.andvals[14] = ~(1 << 28 | 1 << 27 | 1 << 26 |
758 1 << 25 | 1 << 24);
759
760 smp_ctl_bit_callback(&cr_parms);
761
762 return 0; 803 return 0;
763} 804}
764 805
765void __cpu_die(unsigned int cpu) 806void __cpu_die(unsigned int cpu)
766{ 807{
808 struct pcpu *pcpu;
809
767 /* Wait until target cpu is down */ 810 /* Wait until target cpu is down */
768 while (!cpu_stopped(cpu)) 811 pcpu = pcpu_devices + cpu;
812 while (!pcpu_stopped(pcpu))
769 cpu_relax(); 813 cpu_relax();
770 while (sigp_p(0, cpu, sigp_set_prefix) == sigp_busy) 814 pcpu_free_lowcore(pcpu);
771 udelay(10);
772 smp_free_lowcore(cpu);
773 atomic_dec(&init_mm.context.attach_count); 815 atomic_dec(&init_mm.context.attach_count);
774} 816}
775 817
776void __noreturn cpu_die(void) 818void __noreturn cpu_die(void)
777{ 819{
778 idle_task_exit(); 820 idle_task_exit();
779 while (sigp(smp_processor_id(), sigp_stop) == sigp_busy) 821 pcpu_sigp_retry(pcpu_devices + smp_processor_id(), sigp_stop, 0);
780 cpu_relax(); 822 for (;;) ;
781 for (;;);
782} 823}
783 824
784#endif /* CONFIG_HOTPLUG_CPU */ 825#endif /* CONFIG_HOTPLUG_CPU */
785 826
786void __init smp_prepare_cpus(unsigned int max_cpus) 827void __init smp_prepare_cpus(unsigned int max_cpus)
787{ 828{
788#ifndef CONFIG_64BIT
789 unsigned long save_area = 0;
790#endif
791 unsigned long async_stack, panic_stack;
792 struct _lowcore *lowcore;
793
794 smp_detect_cpus();
795
796 /* request the 0x1201 emergency signal external interrupt */ 829 /* request the 0x1201 emergency signal external interrupt */
797 if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0) 830 if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0)
798 panic("Couldn't request external interrupt 0x1201"); 831 panic("Couldn't request external interrupt 0x1201");
799 /* request the 0x1202 external call external interrupt */ 832 /* request the 0x1202 external call external interrupt */
800 if (register_external_interrupt(0x1202, do_ext_call_interrupt) != 0) 833 if (register_external_interrupt(0x1202, do_ext_call_interrupt) != 0)
801 panic("Couldn't request external interrupt 0x1202"); 834 panic("Couldn't request external interrupt 0x1202");
802 835 smp_detect_cpus();
803 /* Reallocate current lowcore, but keep its contents. */
804 lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
805 panic_stack = __get_free_page(GFP_KERNEL);
806 async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
807 BUG_ON(!lowcore || !panic_stack || !async_stack);
808#ifndef CONFIG_64BIT
809 if (MACHINE_HAS_IEEE)
810 save_area = get_zeroed_page(GFP_KERNEL);
811#endif
812 local_irq_disable();
813 local_mcck_disable();
814 lowcore_ptr[smp_processor_id()] = lowcore;
815 *lowcore = S390_lowcore;
816 lowcore->panic_stack = panic_stack + PAGE_SIZE;
817 lowcore->async_stack = async_stack + ASYNC_SIZE;
818#ifndef CONFIG_64BIT
819 if (MACHINE_HAS_IEEE)
820 lowcore->extended_save_area_addr = (u32) save_area;
821#endif
822 set_prefix((u32)(unsigned long) lowcore);
823 local_mcck_enable();
824 local_irq_enable();
825#ifdef CONFIG_64BIT
826 if (vdso_alloc_per_cpu(smp_processor_id(), &S390_lowcore))
827 BUG();
828#endif
829} 836}
830 837
831void __init smp_prepare_boot_cpu(void) 838void __init smp_prepare_boot_cpu(void)
832{ 839{
833 BUG_ON(smp_processor_id() != 0); 840 struct pcpu *pcpu = pcpu_devices;
834 841
835 current_thread_info()->cpu = 0; 842 boot_cpu_address = stap();
836 set_cpu_present(0, true); 843 pcpu->idle = current;
837 set_cpu_online(0, true); 844 pcpu->state = CPU_STATE_CONFIGURED;
845 pcpu->address = boot_cpu_address;
846 pcpu->lowcore = (struct _lowcore *)(unsigned long) store_prefix();
847 pcpu->async_stack = S390_lowcore.async_stack - ASYNC_SIZE;
848 pcpu->panic_stack = S390_lowcore.panic_stack - PAGE_SIZE;
838 S390_lowcore.percpu_offset = __per_cpu_offset[0]; 849 S390_lowcore.percpu_offset = __per_cpu_offset[0];
839 current_set[0] = current;
840 smp_cpu_state[0] = CPU_STATE_CONFIGURED;
841 cpu_set_polarization(0, POLARIZATION_UNKNOWN); 850 cpu_set_polarization(0, POLARIZATION_UNKNOWN);
851 set_cpu_present(0, true);
852 set_cpu_online(0, true);
842} 853}
843 854
844void __init smp_cpus_done(unsigned int max_cpus) 855void __init smp_cpus_done(unsigned int max_cpus)
@@ -848,7 +859,6 @@ void __init smp_cpus_done(unsigned int max_cpus)
848void __init smp_setup_processor_id(void) 859void __init smp_setup_processor_id(void)
849{ 860{
850 S390_lowcore.cpu_nr = 0; 861 S390_lowcore.cpu_nr = 0;
851 __cpu_logical_map[0] = stap();
852} 862}
853 863
854/* 864/*
@@ -864,56 +874,57 @@ int setup_profiling_timer(unsigned int multiplier)
864 874
865#ifdef CONFIG_HOTPLUG_CPU 875#ifdef CONFIG_HOTPLUG_CPU
866static ssize_t cpu_configure_show(struct device *dev, 876static ssize_t cpu_configure_show(struct device *dev,
867 struct device_attribute *attr, char *buf) 877 struct device_attribute *attr, char *buf)
868{ 878{
869 ssize_t count; 879 ssize_t count;
870 880
871 mutex_lock(&smp_cpu_state_mutex); 881 mutex_lock(&smp_cpu_state_mutex);
872 count = sprintf(buf, "%d\n", smp_cpu_state[dev->id]); 882 count = sprintf(buf, "%d\n", pcpu_devices[dev->id].state);
873 mutex_unlock(&smp_cpu_state_mutex); 883 mutex_unlock(&smp_cpu_state_mutex);
874 return count; 884 return count;
875} 885}
876 886
877static ssize_t cpu_configure_store(struct device *dev, 887static ssize_t cpu_configure_store(struct device *dev,
878 struct device_attribute *attr, 888 struct device_attribute *attr,
879 const char *buf, size_t count) 889 const char *buf, size_t count)
880{ 890{
881 int cpu = dev->id; 891 struct pcpu *pcpu;
882 int val, rc; 892 int cpu, val, rc;
883 char delim; 893 char delim;
884 894
885 if (sscanf(buf, "%d %c", &val, &delim) != 1) 895 if (sscanf(buf, "%d %c", &val, &delim) != 1)
886 return -EINVAL; 896 return -EINVAL;
887 if (val != 0 && val != 1) 897 if (val != 0 && val != 1)
888 return -EINVAL; 898 return -EINVAL;
889
890 get_online_cpus(); 899 get_online_cpus();
891 mutex_lock(&smp_cpu_state_mutex); 900 mutex_lock(&smp_cpu_state_mutex);
892 rc = -EBUSY; 901 rc = -EBUSY;
893 /* disallow configuration changes of online cpus and cpu 0 */ 902 /* disallow configuration changes of online cpus and cpu 0 */
903 cpu = dev->id;
894 if (cpu_online(cpu) || cpu == 0) 904 if (cpu_online(cpu) || cpu == 0)
895 goto out; 905 goto out;
906 pcpu = pcpu_devices + cpu;
896 rc = 0; 907 rc = 0;
897 switch (val) { 908 switch (val) {
898 case 0: 909 case 0:
899 if (smp_cpu_state[cpu] == CPU_STATE_CONFIGURED) { 910 if (pcpu->state != CPU_STATE_CONFIGURED)
900 rc = sclp_cpu_deconfigure(__cpu_logical_map[cpu]); 911 break;
901 if (!rc) { 912 rc = sclp_cpu_deconfigure(pcpu->address);
902 smp_cpu_state[cpu] = CPU_STATE_STANDBY; 913 if (rc)
903 cpu_set_polarization(cpu, POLARIZATION_UNKNOWN); 914 break;
904 topology_expect_change(); 915 pcpu->state = CPU_STATE_STANDBY;
905 } 916 cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
906 } 917 topology_expect_change();
907 break; 918 break;
908 case 1: 919 case 1:
909 if (smp_cpu_state[cpu] == CPU_STATE_STANDBY) { 920 if (pcpu->state != CPU_STATE_STANDBY)
910 rc = sclp_cpu_configure(__cpu_logical_map[cpu]); 921 break;
911 if (!rc) { 922 rc = sclp_cpu_configure(pcpu->address);
912 smp_cpu_state[cpu] = CPU_STATE_CONFIGURED; 923 if (rc)
913 cpu_set_polarization(cpu, POLARIZATION_UNKNOWN); 924 break;
914 topology_expect_change(); 925 pcpu->state = CPU_STATE_CONFIGURED;
915 } 926 cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
916 } 927 topology_expect_change();
917 break; 928 break;
918 default: 929 default:
919 break; 930 break;
@@ -929,7 +940,7 @@ static DEVICE_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
929static ssize_t show_cpu_address(struct device *dev, 940static ssize_t show_cpu_address(struct device *dev,
930 struct device_attribute *attr, char *buf) 941 struct device_attribute *attr, char *buf)
931{ 942{
932 return sprintf(buf, "%d\n", __cpu_logical_map[dev->id]); 943 return sprintf(buf, "%d\n", pcpu_devices[dev->id].address);
933} 944}
934static DEVICE_ATTR(address, 0444, show_cpu_address, NULL); 945static DEVICE_ATTR(address, 0444, show_cpu_address, NULL);
935 946
@@ -1021,7 +1032,7 @@ static int __cpuinit smp_cpu_notify(struct notifier_block *self,
1021 unsigned long action, void *hcpu) 1032 unsigned long action, void *hcpu)
1022{ 1033{
1023 unsigned int cpu = (unsigned int)(long)hcpu; 1034 unsigned int cpu = (unsigned int)(long)hcpu;
1024 struct cpu *c = &per_cpu(cpu_devices, cpu); 1035 struct cpu *c = &pcpu_devices[cpu].cpu;
1025 struct device *s = &c->dev; 1036 struct device *s = &c->dev;
1026 struct s390_idle_data *idle; 1037 struct s390_idle_data *idle;
1027 int err = 0; 1038 int err = 0;
@@ -1047,7 +1058,7 @@ static struct notifier_block __cpuinitdata smp_cpu_nb = {
1047 1058
1048static int __devinit smp_add_present_cpu(int cpu) 1059static int __devinit smp_add_present_cpu(int cpu)
1049{ 1060{
1050 struct cpu *c = &per_cpu(cpu_devices, cpu); 1061 struct cpu *c = &pcpu_devices[cpu].cpu;
1051 struct device *s = &c->dev; 1062 struct device *s = &c->dev;
1052 int rc; 1063 int rc;
1053 1064
@@ -1085,29 +1096,21 @@ out:
1085 1096
1086int __ref smp_rescan_cpus(void) 1097int __ref smp_rescan_cpus(void)
1087{ 1098{
1088 cpumask_t newcpus; 1099 struct sclp_cpu_info *info;
1089 int cpu; 1100 int nr;
1090 int rc;
1091 1101
1102 info = smp_get_cpu_info();
1103 if (!info)
1104 return -ENOMEM;
1092 get_online_cpus(); 1105 get_online_cpus();
1093 mutex_lock(&smp_cpu_state_mutex); 1106 mutex_lock(&smp_cpu_state_mutex);
1094 cpumask_copy(&newcpus, cpu_present_mask); 1107 nr = __smp_rescan_cpus(info, 1);
1095 rc = __smp_rescan_cpus();
1096 if (rc)
1097 goto out;
1098 cpumask_andnot(&newcpus, cpu_present_mask, &newcpus);
1099 for_each_cpu(cpu, &newcpus) {
1100 rc = smp_add_present_cpu(cpu);
1101 if (rc)
1102 set_cpu_present(cpu, false);
1103 }
1104 rc = 0;
1105out:
1106 mutex_unlock(&smp_cpu_state_mutex); 1108 mutex_unlock(&smp_cpu_state_mutex);
1107 put_online_cpus(); 1109 put_online_cpus();
1108 if (!cpumask_empty(&newcpus)) 1110 kfree(info);
1111 if (nr)
1109 topology_schedule_update(); 1112 topology_schedule_update();
1110 return rc; 1113 return 0;
1111} 1114}
1112 1115
1113static ssize_t __ref rescan_store(struct device *dev, 1116static ssize_t __ref rescan_store(struct device *dev,
diff --git a/arch/s390/kernel/switch_cpu.S b/arch/s390/kernel/switch_cpu.S
deleted file mode 100644
index bfe070bc7659..000000000000
--- a/arch/s390/kernel/switch_cpu.S
+++ /dev/null
@@ -1,58 +0,0 @@
1/*
2 * 31-bit switch cpu code
3 *
4 * Copyright IBM Corp. 2009
5 *
6 */
7
8#include <linux/linkage.h>
9#include <asm/asm-offsets.h>
10#include <asm/ptrace.h>
11
12# smp_switch_to_cpu switches to destination cpu and executes the passed function
13# Parameter: %r2 - function to call
14# %r3 - function parameter
15# %r4 - stack poiner
16# %r5 - current cpu
17# %r6 - destination cpu
18
19 .section .text
20ENTRY(smp_switch_to_cpu)
21 stm %r6,%r15,__SF_GPRS(%r15)
22 lr %r1,%r15
23 ahi %r15,-STACK_FRAME_OVERHEAD
24 st %r1,__SF_BACKCHAIN(%r15)
25 basr %r13,0
260: la %r1,.gprregs_addr-0b(%r13)
27 l %r1,0(%r1)
28 stm %r0,%r15,0(%r1)
291: sigp %r0,%r6,__SIGP_RESTART /* start destination CPU */
30 brc 2,1b /* busy, try again */
312: sigp %r0,%r5,__SIGP_STOP /* stop current CPU */
32 brc 2,2b /* busy, try again */
333: j 3b
34
35ENTRY(smp_restart_cpu)
36 basr %r13,0
370: la %r1,.gprregs_addr-0b(%r13)
38 l %r1,0(%r1)
39 lm %r0,%r15,0(%r1)
401: sigp %r0,%r5,__SIGP_SENSE /* Wait for calling CPU */
41 brc 10,1b /* busy, accepted (status 0), running */
42 tmll %r0,0x40 /* Test if calling CPU is stopped */
43 jz 1b
44 ltr %r4,%r4 /* New stack ? */
45 jz 1f
46 lr %r15,%r4
471: lr %r14,%r2 /* r14: Function to call */
48 lr %r2,%r3 /* r2 : Parameter for function*/
49 basr %r14,%r14 /* Call function */
50
51.gprregs_addr:
52 .long .gprregs
53
54 .section .data,"aw",@progbits
55.gprregs:
56 .rept 16
57 .long 0
58 .endr
diff --git a/arch/s390/kernel/switch_cpu64.S b/arch/s390/kernel/switch_cpu64.S
deleted file mode 100644
index fcc42d799e41..000000000000
--- a/arch/s390/kernel/switch_cpu64.S
+++ /dev/null
@@ -1,51 +0,0 @@
1/*
2 * 64-bit switch cpu code
3 *
4 * Copyright IBM Corp. 2009
5 *
6 */
7
8#include <linux/linkage.h>
9#include <asm/asm-offsets.h>
10#include <asm/ptrace.h>
11
12# smp_switch_to_cpu switches to destination cpu and executes the passed function
13# Parameter: %r2 - function to call
14# %r3 - function parameter
15# %r4 - stack poiner
16# %r5 - current cpu
17# %r6 - destination cpu
18
19 .section .text
20ENTRY(smp_switch_to_cpu)
21 stmg %r6,%r15,__SF_GPRS(%r15)
22 lgr %r1,%r15
23 aghi %r15,-STACK_FRAME_OVERHEAD
24 stg %r1,__SF_BACKCHAIN(%r15)
25 larl %r1,.gprregs
26 stmg %r0,%r15,0(%r1)
271: sigp %r0,%r6,__SIGP_RESTART /* start destination CPU */
28 brc 2,1b /* busy, try again */
292: sigp %r0,%r5,__SIGP_STOP /* stop current CPU */
30 brc 2,2b /* busy, try again */
313: j 3b
32
33ENTRY(smp_restart_cpu)
34 larl %r1,.gprregs
35 lmg %r0,%r15,0(%r1)
361: sigp %r0,%r5,__SIGP_SENSE /* Wait for calling CPU */
37 brc 10,1b /* busy, accepted (status 0), running */
38 tmll %r0,0x40 /* Test if calling CPU is stopped */
39 jz 1b
40 ltgr %r4,%r4 /* New stack ? */
41 jz 1f
42 lgr %r15,%r4
431: lgr %r14,%r2 /* r14: Function to call */
44 lgr %r2,%r3 /* r2 : Parameter for function*/
45 basr %r14,%r14 /* Call function */
46
47 .section .data,"aw",@progbits
48.gprregs:
49 .rept 16
50 .quad 0
51 .endr
diff --git a/arch/s390/kernel/swsusp_asm64.S b/arch/s390/kernel/swsusp_asm64.S
index 2ef39d1519a9..ad3c79eceed7 100644
--- a/arch/s390/kernel/swsusp_asm64.S
+++ b/arch/s390/kernel/swsusp_asm64.S
@@ -179,9 +179,9 @@ pgm_check_entry:
179 larl %r4,.Lrestart_suspend_psw /* Set new restart PSW */ 179 larl %r4,.Lrestart_suspend_psw /* Set new restart PSW */
180 mvc __LC_RST_NEW_PSW(16,%r0),0(%r4) 180 mvc __LC_RST_NEW_PSW(16,%r0),0(%r4)
1813: 1813:
182 sigp %r9,%r1,__SIGP_INITIAL_CPU_RESET 182 sigp %r9,%r1,11 /* sigp initial cpu reset */
183 brc 8,4f /* accepted */ 183 brc 8,4f /* accepted */
184 brc 2,3b /* busy, try again */ 184 brc 2,3b /* busy, try again */
185 185
186 /* Suspend CPU not available -> panic */ 186 /* Suspend CPU not available -> panic */
187 larl %r15,init_thread_union 187 larl %r15,init_thread_union
@@ -196,10 +196,10 @@ pgm_check_entry:
196 lpsw 0(%r3) 196 lpsw 0(%r3)
1974: 1974:
198 /* Switch to suspend CPU */ 198 /* Switch to suspend CPU */
199 sigp %r9,%r1,__SIGP_RESTART /* start suspend CPU */ 199 sigp %r9,%r1,6 /* sigp restart to suspend CPU */
200 brc 2,4b /* busy, try again */ 200 brc 2,4b /* busy, try again */
2015: 2015:
202 sigp %r9,%r2,__SIGP_STOP /* stop resume (current) CPU */ 202 sigp %r9,%r2,5 /* sigp stop to current resume CPU */
203 brc 2,5b /* busy, try again */ 203 brc 2,5b /* busy, try again */
2046: j 6b 2046: j 6b
205 205
@@ -207,7 +207,7 @@ restart_suspend:
207 larl %r1,.Lresume_cpu 207 larl %r1,.Lresume_cpu
208 llgh %r2,0(%r1) 208 llgh %r2,0(%r1)
2097: 2097:
210 sigp %r9,%r2,__SIGP_SENSE /* Wait for resume CPU */ 210 sigp %r9,%r2,1 /* sigp sense, wait for resume CPU */
211 brc 8,7b /* accepted, status 0, still running */ 211 brc 8,7b /* accepted, status 0, still running */
212 brc 2,7b /* busy, try again */ 212 brc 2,7b /* busy, try again */
213 tmll %r9,0x40 /* Test if resume CPU is stopped */ 213 tmll %r9,0x40 /* Test if resume CPU is stopped */
diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c
index 7370a41948ca..4f8dc942257c 100644
--- a/arch/s390/kernel/topology.c
+++ b/arch/s390/kernel/topology.c
@@ -79,12 +79,12 @@ static struct mask_info *add_cpus_to_mask(struct topology_cpu *tl_cpu,
79 cpu < TOPOLOGY_CPU_BITS; 79 cpu < TOPOLOGY_CPU_BITS;
80 cpu = find_next_bit(&tl_cpu->mask[0], TOPOLOGY_CPU_BITS, cpu + 1)) 80 cpu = find_next_bit(&tl_cpu->mask[0], TOPOLOGY_CPU_BITS, cpu + 1))
81 { 81 {
82 unsigned int rcpu, lcpu; 82 unsigned int rcpu;
83 int lcpu;
83 84
84 rcpu = TOPOLOGY_CPU_BITS - 1 - cpu + tl_cpu->origin; 85 rcpu = TOPOLOGY_CPU_BITS - 1 - cpu + tl_cpu->origin;
85 for_each_present_cpu(lcpu) { 86 lcpu = smp_find_processor_id(rcpu);
86 if (cpu_logical_map(lcpu) != rcpu) 87 if (lcpu >= 0) {
87 continue;
88 cpumask_set_cpu(lcpu, &book->mask); 88 cpumask_set_cpu(lcpu, &book->mask);
89 cpu_book_id[lcpu] = book->id; 89 cpu_book_id[lcpu] = book->id;
90 cpumask_set_cpu(lcpu, &core->mask); 90 cpumask_set_cpu(lcpu, &core->mask);
diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c
index d73630b4fe1d..e704a9965f90 100644
--- a/arch/s390/kernel/vdso.c
+++ b/arch/s390/kernel/vdso.c
@@ -89,18 +89,11 @@ static void vdso_init_data(struct vdso_data *vd)
89 89
90#ifdef CONFIG_64BIT 90#ifdef CONFIG_64BIT
91/* 91/*
92 * Setup per cpu vdso data page.
93 */
94static void vdso_init_per_cpu_data(int cpu, struct vdso_per_cpu_data *vpcd)
95{
96}
97
98/*
99 * Allocate/free per cpu vdso data. 92 * Allocate/free per cpu vdso data.
100 */ 93 */
101#define SEGMENT_ORDER 2 94#define SEGMENT_ORDER 2
102 95
103int vdso_alloc_per_cpu(int cpu, struct _lowcore *lowcore) 96int vdso_alloc_per_cpu(struct _lowcore *lowcore)
104{ 97{
105 unsigned long segment_table, page_table, page_frame; 98 unsigned long segment_table, page_table, page_frame;
106 u32 *psal, *aste; 99 u32 *psal, *aste;
@@ -139,7 +132,6 @@ int vdso_alloc_per_cpu(int cpu, struct _lowcore *lowcore)
139 aste[4] = (u32)(addr_t) psal; 132 aste[4] = (u32)(addr_t) psal;
140 lowcore->vdso_per_cpu_data = page_frame; 133 lowcore->vdso_per_cpu_data = page_frame;
141 134
142 vdso_init_per_cpu_data(cpu, (struct vdso_per_cpu_data *) page_frame);
143 return 0; 135 return 0;
144 136
145out: 137out:
@@ -149,7 +141,7 @@ out:
149 return -ENOMEM; 141 return -ENOMEM;
150} 142}
151 143
152void vdso_free_per_cpu(int cpu, struct _lowcore *lowcore) 144void vdso_free_per_cpu(struct _lowcore *lowcore)
153{ 145{
154 unsigned long segment_table, page_table, page_frame; 146 unsigned long segment_table, page_table, page_frame;
155 u32 *psal, *aste; 147 u32 *psal, *aste;
@@ -168,19 +160,15 @@ void vdso_free_per_cpu(int cpu, struct _lowcore *lowcore)
168 free_pages(segment_table, SEGMENT_ORDER); 160 free_pages(segment_table, SEGMENT_ORDER);
169} 161}
170 162
171static void __vdso_init_cr5(void *dummy) 163static void vdso_init_cr5(void)
172{ 164{
173 unsigned long cr5; 165 unsigned long cr5;
174 166
167 if (user_mode == HOME_SPACE_MODE || !vdso_enabled)
168 return;
175 cr5 = offsetof(struct _lowcore, paste); 169 cr5 = offsetof(struct _lowcore, paste);
176 __ctl_load(cr5, 5, 5); 170 __ctl_load(cr5, 5, 5);
177} 171}
178
179static void vdso_init_cr5(void)
180{
181 if (user_mode != HOME_SPACE_MODE && vdso_enabled)
182 on_each_cpu(__vdso_init_cr5, NULL, 1);
183}
184#endif /* CONFIG_64BIT */ 172#endif /* CONFIG_64BIT */
185 173
186/* 174/*
@@ -322,10 +310,8 @@ static int __init vdso_init(void)
322 } 310 }
323 vdso64_pagelist[vdso64_pages - 1] = virt_to_page(vdso_data); 311 vdso64_pagelist[vdso64_pages - 1] = virt_to_page(vdso_data);
324 vdso64_pagelist[vdso64_pages] = NULL; 312 vdso64_pagelist[vdso64_pages] = NULL;
325#ifndef CONFIG_SMP 313 if (vdso_alloc_per_cpu(&S390_lowcore))
326 if (vdso_alloc_per_cpu(0, &S390_lowcore))
327 BUG(); 314 BUG();
328#endif
329 vdso_init_cr5(); 315 vdso_init_cr5();
330#endif /* CONFIG_64BIT */ 316#endif /* CONFIG_64BIT */
331 317
@@ -335,7 +321,7 @@ static int __init vdso_init(void)
335 321
336 return 0; 322 return 0;
337} 323}
338arch_initcall(vdso_init); 324early_initcall(vdso_init);
339 325
340int in_gate_area_no_mm(unsigned long addr) 326int in_gate_area_no_mm(unsigned long addr)
341{ 327{
diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c
index bb48977f5469..7bacee9a546f 100644
--- a/arch/s390/kernel/vtime.c
+++ b/arch/s390/kernel/vtime.c
@@ -570,6 +570,9 @@ void init_cpu_vtimer(void)
570 570
571 /* enable cpu timer interrupts */ 571 /* enable cpu timer interrupts */
572 __ctl_set_bit(0,10); 572 __ctl_set_bit(0,10);
573
574 /* set initial cpu timer */
575 set_vtimer(0x7fffffffffffffffULL);
573} 576}
574 577
575static int __cpuinit s390_nohz_notify(struct notifier_block *self, 578static int __cpuinit s390_nohz_notify(struct notifier_block *self,
diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c
index 91754ffb9203..093eb694d9c1 100644
--- a/arch/s390/lib/spinlock.c
+++ b/arch/s390/lib/spinlock.c
@@ -10,6 +10,7 @@
10#include <linux/module.h> 10#include <linux/module.h>
11#include <linux/spinlock.h> 11#include <linux/spinlock.h>
12#include <linux/init.h> 12#include <linux/init.h>
13#include <linux/smp.h>
13#include <asm/io.h> 14#include <asm/io.h>
14 15
15int spin_retry = 1000; 16int spin_retry = 1000;
@@ -24,21 +25,6 @@ static int __init spin_retry_setup(char *str)
24} 25}
25__setup("spin_retry=", spin_retry_setup); 26__setup("spin_retry=", spin_retry_setup);
26 27
27static inline void _raw_yield(void)
28{
29 if (MACHINE_HAS_DIAG44)
30 asm volatile("diag 0,0,0x44");
31}
32
33static inline void _raw_yield_cpu(int cpu)
34{
35 if (MACHINE_HAS_DIAG9C)
36 asm volatile("diag %0,0,0x9c"
37 : : "d" (cpu_logical_map(cpu)));
38 else
39 _raw_yield();
40}
41
42void arch_spin_lock_wait(arch_spinlock_t *lp) 28void arch_spin_lock_wait(arch_spinlock_t *lp)
43{ 29{
44 int count = spin_retry; 30 int count = spin_retry;
@@ -60,7 +46,7 @@ void arch_spin_lock_wait(arch_spinlock_t *lp)
60 } 46 }
61 owner = lp->owner_cpu; 47 owner = lp->owner_cpu;
62 if (owner) 48 if (owner)
63 _raw_yield_cpu(~owner); 49 smp_yield_cpu(~owner);
64 if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0) 50 if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0)
65 return; 51 return;
66 } 52 }
@@ -91,7 +77,7 @@ void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
91 } 77 }
92 owner = lp->owner_cpu; 78 owner = lp->owner_cpu;
93 if (owner) 79 if (owner)
94 _raw_yield_cpu(~owner); 80 smp_yield_cpu(~owner);
95 local_irq_disable(); 81 local_irq_disable();
96 if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0) 82 if (_raw_compare_and_swap(&lp->owner_cpu, 0, cpu) == 0)
97 return; 83 return;
@@ -121,7 +107,7 @@ void arch_spin_relax(arch_spinlock_t *lock)
121 if (cpu != 0) { 107 if (cpu != 0) {
122 if (MACHINE_IS_VM || MACHINE_IS_KVM || 108 if (MACHINE_IS_VM || MACHINE_IS_KVM ||
123 !smp_vcpu_scheduled(~cpu)) 109 !smp_vcpu_scheduled(~cpu))
124 _raw_yield_cpu(~cpu); 110 smp_yield_cpu(~cpu);
125 } 111 }
126} 112}
127EXPORT_SYMBOL(arch_spin_relax); 113EXPORT_SYMBOL(arch_spin_relax);
@@ -133,7 +119,7 @@ void _raw_read_lock_wait(arch_rwlock_t *rw)
133 119
134 while (1) { 120 while (1) {
135 if (count-- <= 0) { 121 if (count-- <= 0) {
136 _raw_yield(); 122 smp_yield();
137 count = spin_retry; 123 count = spin_retry;
138 } 124 }
139 if (!arch_read_can_lock(rw)) 125 if (!arch_read_can_lock(rw))
@@ -153,7 +139,7 @@ void _raw_read_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
153 local_irq_restore(flags); 139 local_irq_restore(flags);
154 while (1) { 140 while (1) {
155 if (count-- <= 0) { 141 if (count-- <= 0) {
156 _raw_yield(); 142 smp_yield();
157 count = spin_retry; 143 count = spin_retry;
158 } 144 }
159 if (!arch_read_can_lock(rw)) 145 if (!arch_read_can_lock(rw))
@@ -188,7 +174,7 @@ void _raw_write_lock_wait(arch_rwlock_t *rw)
188 174
189 while (1) { 175 while (1) {
190 if (count-- <= 0) { 176 if (count-- <= 0) {
191 _raw_yield(); 177 smp_yield();
192 count = spin_retry; 178 count = spin_retry;
193 } 179 }
194 if (!arch_write_can_lock(rw)) 180 if (!arch_write_can_lock(rw))
@@ -206,7 +192,7 @@ void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
206 local_irq_restore(flags); 192 local_irq_restore(flags);
207 while (1) { 193 while (1) {
208 if (count-- <= 0) { 194 if (count-- <= 0) {
209 _raw_yield(); 195 smp_yield();
210 count = spin_retry; 196 count = spin_retry;
211 } 197 }
212 if (!arch_write_can_lock(rw)) 198 if (!arch_write_can_lock(rw))
diff --git a/drivers/s390/char/sclp_quiesce.c b/drivers/s390/char/sclp_quiesce.c
index 87fc0ac11e67..69df137310bc 100644
--- a/drivers/s390/char/sclp_quiesce.c
+++ b/drivers/s390/char/sclp_quiesce.c
@@ -15,7 +15,6 @@
15#include <linux/reboot.h> 15#include <linux/reboot.h>
16#include <linux/atomic.h> 16#include <linux/atomic.h>
17#include <asm/ptrace.h> 17#include <asm/ptrace.h>
18#include <asm/sigp.h>
19#include <asm/smp.h> 18#include <asm/smp.h>
20 19
21#include "sclp.h" 20#include "sclp.h"
diff --git a/drivers/s390/char/zcore.c b/drivers/s390/char/zcore.c
index 1b6d9247fdc7..3303d66b2794 100644
--- a/drivers/s390/char/zcore.c
+++ b/drivers/s390/char/zcore.c
@@ -21,7 +21,6 @@
21#include <asm/ipl.h> 21#include <asm/ipl.h>
22#include <asm/sclp.h> 22#include <asm/sclp.h>
23#include <asm/setup.h> 23#include <asm/setup.h>
24#include <asm/sigp.h>
25#include <asm/uaccess.h> 24#include <asm/uaccess.h>
26#include <asm/debug.h> 25#include <asm/debug.h>
27#include <asm/processor.h> 26#include <asm/processor.h>
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-29 23:19:48 -0400