aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/s390/include/asm/cputime.h9
-rw-r--r--arch/s390/include/asm/timer.h4
-rw-r--r--arch/s390/kernel/asm-offsets.c8
-rw-r--r--arch/s390/kernel/entry.S76
-rw-r--r--arch/s390/kernel/entry.h5
-rw-r--r--arch/s390/kernel/entry64.S65
-rw-r--r--arch/s390/kernel/irq.c2
-rw-r--r--arch/s390/kernel/nmi.c2
-rw-r--r--arch/s390/kernel/process.c7
-rw-r--r--arch/s390/kernel/smp.c46
-rw-r--r--arch/s390/kernel/vtime.c163
-rw-r--r--arch/s390/lib/delay.c31
-rw-r--r--drivers/s390/cio/cio.c2
13 files changed, 210 insertions, 210 deletions
diff --git a/arch/s390/include/asm/cputime.h b/arch/s390/include/asm/cputime.h
index c23c3900c304..24ef186a1c4f 100644
--- a/arch/s390/include/asm/cputime.h
+++ b/arch/s390/include/asm/cputime.h
@@ -170,24 +170,17 @@ struct s390_idle_data {
170 unsigned int sequence; 170 unsigned int sequence;
171 unsigned long long idle_count; 171 unsigned long long idle_count;
172 unsigned long long idle_enter; 172 unsigned long long idle_enter;
173 unsigned long long idle_exit;
173 unsigned long long idle_time; 174 unsigned long long idle_time;
174 int nohz_delay; 175 int nohz_delay;
175}; 176};
176 177
177DECLARE_PER_CPU(struct s390_idle_data, s390_idle); 178DECLARE_PER_CPU(struct s390_idle_data, s390_idle);
178 179
179void vtime_start_cpu(__u64 int_clock, __u64 enter_timer);
180cputime64_t s390_get_idle_time(int cpu); 180cputime64_t s390_get_idle_time(int cpu);
181 181
182#define arch_idle_time(cpu) s390_get_idle_time(cpu) 182#define arch_idle_time(cpu) s390_get_idle_time(cpu)
183 183
184static inline void s390_idle_check(struct pt_regs *regs, __u64 int_clock,
185 __u64 enter_timer)
186{
187 if (regs->psw.mask & PSW_MASK_WAIT)
188 vtime_start_cpu(int_clock, enter_timer);
189}
190
191static inline int s390_nohz_delay(int cpu) 184static inline int s390_nohz_delay(int cpu)
192{ 185{
193 return __get_cpu_var(s390_idle).nohz_delay != 0; 186 return __get_cpu_var(s390_idle).nohz_delay != 0;
diff --git a/arch/s390/include/asm/timer.h b/arch/s390/include/asm/timer.h
index 814243cafdfe..e63069ba39e3 100644
--- a/arch/s390/include/asm/timer.h
+++ b/arch/s390/include/asm/timer.h
@@ -33,8 +33,8 @@ struct vtimer_queue {
33 spinlock_t lock; 33 spinlock_t lock;
34 __u64 timer; /* last programmed timer */ 34 __u64 timer; /* last programmed timer */
35 __u64 elapsed; /* elapsed time of timer expire values */ 35 __u64 elapsed; /* elapsed time of timer expire values */
36 __u64 idle; /* temp var for idle */ 36 __u64 idle_enter; /* cpu timer on idle enter */
37 int do_spt; /* =1: reprogram cpu timer in idle */ 37 __u64 idle_exit; /* cpu timer on idle exit */
38}; 38};
39 39
40extern void init_virt_timer(struct vtimer_list *timer); 40extern void init_virt_timer(struct vtimer_list *timer);
diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c
index aeeaf896be9b..ed8c913db79e 100644
--- a/arch/s390/kernel/asm-offsets.c
+++ b/arch/s390/kernel/asm-offsets.c
@@ -8,6 +8,8 @@
8 8
9#include <linux/kbuild.h> 9#include <linux/kbuild.h>
10#include <linux/sched.h> 10#include <linux/sched.h>
11#include <asm/cputime.h>
12#include <asm/timer.h>
11#include <asm/vdso.h> 13#include <asm/vdso.h>
12#include <asm/pgtable.h> 14#include <asm/pgtable.h>
13#include <asm/system.h> 15#include <asm/system.h>
@@ -70,6 +72,12 @@ int main(void)
70 DEFINE(__CLOCK_MONOTONIC, CLOCK_MONOTONIC); 72 DEFINE(__CLOCK_MONOTONIC, CLOCK_MONOTONIC);
71 DEFINE(__CLOCK_REALTIME_RES, MONOTONIC_RES_NSEC); 73 DEFINE(__CLOCK_REALTIME_RES, MONOTONIC_RES_NSEC);
72 BLANK(); 74 BLANK();
75 /* idle data offsets */
76 DEFINE(__IDLE_ENTER, offsetof(struct s390_idle_data, idle_enter));
77 DEFINE(__IDLE_EXIT, offsetof(struct s390_idle_data, idle_exit));
78 /* vtimer queue offsets */
79 DEFINE(__VQ_IDLE_ENTER, offsetof(struct vtimer_queue, idle_enter));
80 DEFINE(__VQ_IDLE_EXIT, offsetof(struct vtimer_queue, idle_exit));
73 /* lowcore offsets */ 81 /* lowcore offsets */
74 DEFINE(__LC_EXT_PARAMS, offsetof(struct _lowcore, ext_params)); 82 DEFINE(__LC_EXT_PARAMS, offsetof(struct _lowcore, ext_params));
75 DEFINE(__LC_EXT_CPU_ADDR, offsetof(struct _lowcore, ext_cpu_addr)); 83 DEFINE(__LC_EXT_CPU_ADDR, offsetof(struct _lowcore, ext_cpu_addr));
diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index 6143521a4fff..74ee563fe62b 100644
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -105,14 +105,14 @@ STACK_SIZE = 1 << STACK_SHIFT
105 105
106 .macro ADD64 high,low,timer 106 .macro ADD64 high,low,timer
107 al \high,\timer 107 al \high,\timer
108 al \low,\timer+4 108 al \low,4+\timer
109 brc 12,.+8 109 brc 12,.+8
110 ahi \high,1 110 ahi \high,1
111 .endm 111 .endm
112 112
113 .macro SUB64 high,low,timer 113 .macro SUB64 high,low,timer
114 sl \high,\timer 114 sl \high,\timer
115 sl \low,\timer+4 115 sl \low,4+\timer
116 brc 3,.+8 116 brc 3,.+8
117 ahi \high,-1 117 ahi \high,-1
118 .endm 118 .endm
@@ -471,7 +471,6 @@ io_tif:
471 jnz io_work # there is work to do (signals etc.) 471 jnz io_work # there is work to do (signals etc.)
472io_restore: 472io_restore:
473 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11) 473 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11)
474 ni __LC_RETURN_PSW+1,0xfd # clean wait state bit
475 stpt __LC_EXIT_TIMER 474 stpt __LC_EXIT_TIMER
476 lm %r0,%r15,__PT_R0(%r11) 475 lm %r0,%r15,__PT_R0(%r11)
477 lpsw __LC_RETURN_PSW 476 lpsw __LC_RETURN_PSW
@@ -612,6 +611,26 @@ ext_skip:
612 basr %r14,%r1 # call do_extint 611 basr %r14,%r1 # call do_extint
613 j io_return 612 j io_return
614 613
614/*
615 * Load idle PSW. The second "half" of this function is in cleanup_idle.
616 */
617ENTRY(psw_idle)
618 st %r4,__SF_EMPTY(%r15)
619 basr %r1,0
620 la %r1,psw_idle_lpsw+4-.(%r1)
621 st %r1,__SF_EMPTY+4(%r15)
622 oi __SF_EMPTY+4(%r15),0x80
623 la %r1,.Lvtimer_max-psw_idle_lpsw-4(%r1)
624 stck __IDLE_ENTER(%r2)
625 ltr %r5,%r5
626 stpt __VQ_IDLE_ENTER(%r3)
627 jz psw_idle_lpsw
628 spt 0(%r1)
629psw_idle_lpsw:
630 lpsw __SF_EMPTY(%r15)
631 br %r14
632psw_idle_end:
633
615__critical_end: 634__critical_end:
616 635
617/* 636/*
@@ -673,7 +692,6 @@ mcck_skip:
673 TRACE_IRQS_ON 692 TRACE_IRQS_ON
674mcck_return: 693mcck_return:
675 mvc __LC_RETURN_MCCK_PSW(8),__PT_PSW(%r11) # move return PSW 694 mvc __LC_RETURN_MCCK_PSW(8),__PT_PSW(%r11) # move return PSW
676 ni __LC_RETURN_MCCK_PSW+1,0xfd # clear wait state bit
677 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? 695 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
678 jno 0f 696 jno 0f
679 lm %r0,%r15,__PT_R0(%r11) 697 lm %r0,%r15,__PT_R0(%r11)
@@ -748,6 +766,8 @@ cleanup_table:
748 .long io_tif + 0x80000000 766 .long io_tif + 0x80000000
749 .long io_restore + 0x80000000 767 .long io_restore + 0x80000000
750 .long io_done + 0x80000000 768 .long io_done + 0x80000000
769 .long psw_idle + 0x80000000
770 .long psw_idle_end + 0x80000000
751 771
752cleanup_critical: 772cleanup_critical:
753 cl %r9,BASED(cleanup_table) # system_call 773 cl %r9,BASED(cleanup_table) # system_call
@@ -766,6 +786,10 @@ cleanup_critical:
766 jl cleanup_io_tif 786 jl cleanup_io_tif
767 cl %r9,BASED(cleanup_table+28) # io_done 787 cl %r9,BASED(cleanup_table+28) # io_done
768 jl cleanup_io_restore 788 jl cleanup_io_restore
789 cl %r9,BASED(cleanup_table+32) # psw_idle
790 jl 0f
791 cl %r9,BASED(cleanup_table+36) # psw_idle_end
792 jl cleanup_idle
7690: br %r14 7930: br %r14
770 794
771cleanup_system_call: 795cleanup_system_call:
@@ -849,7 +873,6 @@ cleanup_io_restore:
849 jhe 0f 873 jhe 0f
850 l %r9,12(%r11) # get saved r11 pointer to pt_regs 874 l %r9,12(%r11) # get saved r11 pointer to pt_regs
851 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9) 875 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9)
852 ni __LC_RETURN_PSW+1,0xfd # clear wait state bit
853 mvc 0(32,%r11),__PT_R8(%r9) 876 mvc 0(32,%r11),__PT_R8(%r9)
854 lm %r0,%r7,__PT_R0(%r9) 877 lm %r0,%r7,__PT_R0(%r9)
8550: lm %r8,%r9,__LC_RETURN_PSW 8780: lm %r8,%r9,__LC_RETURN_PSW
@@ -857,11 +880,52 @@ cleanup_io_restore:
857cleanup_io_restore_insn: 880cleanup_io_restore_insn:
858 .long io_done - 4 + 0x80000000 881 .long io_done - 4 + 0x80000000
859 882
883cleanup_idle:
884 # copy interrupt clock & cpu timer
885 mvc __IDLE_EXIT(8,%r2),__LC_INT_CLOCK
886 mvc __VQ_IDLE_EXIT(8,%r3),__LC_ASYNC_ENTER_TIMER
887 chi %r11,__LC_SAVE_AREA_ASYNC
888 je 0f
889 mvc __IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
890 mvc __VQ_IDLE_EXIT(8,%r3),__LC_MCCK_ENTER_TIMER
8910: # check if stck has been executed
892 cl %r9,BASED(cleanup_idle_insn)
893 jhe 1f
894 mvc __IDLE_ENTER(8,%r2),__IDLE_EXIT(%r2)
895 mvc __VQ_IDLE_ENTER(8,%r3),__VQ_IDLE_EXIT(%r3)
896 j 2f
8971: # check if the cpu timer has been reprogrammed
898 ltr %r5,%r5
899 jz 2f
900 spt __VQ_IDLE_ENTER(%r3)
9012: # account system time going idle
902 lm %r9,%r10,__LC_STEAL_TIMER
903 ADD64 %r9,%r10,__IDLE_ENTER(%r2)
904 SUB64 %r9,%r10,__LC_LAST_UPDATE_CLOCK
905 stm %r9,%r10,__LC_STEAL_TIMER
906 mvc __LC_LAST_UPDATE_CLOCK(8),__IDLE_EXIT(%r2)
907 lm %r9,%r10,__LC_SYSTEM_TIMER
908 ADD64 %r9,%r10,__LC_LAST_UPDATE_TIMER
909 SUB64 %r9,%r10,__VQ_IDLE_ENTER(%r3)
910 stm %r9,%r10,__LC_SYSTEM_TIMER
911 mvc __LC_LAST_UPDATE_TIMER(8),__VQ_IDLE_EXIT(%r3)
912 # prepare return psw
913 n %r8,BASED(cleanup_idle_wait) # clear wait state bit
914 l %r9,24(%r11) # return from psw_idle
915 br %r14
916cleanup_idle_insn:
917 .long psw_idle_lpsw + 0x80000000
918cleanup_idle_wait:
919 .long 0xfffdffff
920
860/* 921/*
861 * Integer constants 922 * Integer constants
862 */ 923 */
863 .align 4 924 .align 4
864.Lnr_syscalls: .long NR_syscalls 925.Lnr_syscalls:
926 .long NR_syscalls
927.Lvtimer_max:
928 .quad 0x7fffffffffffffff
865 929
866/* 930/*
867 * Symbol constants 931 * Symbol constants
diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h
index 92b1617d0c95..4984785e3078 100644
--- a/arch/s390/kernel/entry.h
+++ b/arch/s390/kernel/entry.h
@@ -4,7 +4,8 @@
4#include <linux/types.h> 4#include <linux/types.h>
5#include <linux/signal.h> 5#include <linux/signal.h>
6#include <asm/ptrace.h> 6#include <asm/ptrace.h>
7 7#include <asm/cputime.h>
8#include <asm/timer.h>
8 9
9extern void (*pgm_check_table[128])(struct pt_regs *); 10extern void (*pgm_check_table[128])(struct pt_regs *);
10extern void *restart_stack; 11extern void *restart_stack;
@@ -16,6 +17,8 @@ void io_int_handler(void);
16void mcck_int_handler(void); 17void mcck_int_handler(void);
17void restart_int_handler(void); 18void restart_int_handler(void);
18void restart_call_handler(void); 19void restart_call_handler(void);
20void psw_idle(struct s390_idle_data *, struct vtimer_queue *,
21 unsigned long, int);
19 22
20asmlinkage long do_syscall_trace_enter(struct pt_regs *regs); 23asmlinkage long do_syscall_trace_enter(struct pt_regs *regs);
21asmlinkage void do_syscall_trace_exit(struct pt_regs *regs); 24asmlinkage void do_syscall_trace_exit(struct pt_regs *regs);
diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S
index e33789a45752..4e1c292fa7e3 100644
--- a/arch/s390/kernel/entry64.S
+++ b/arch/s390/kernel/entry64.S
@@ -489,7 +489,6 @@ io_restore:
489 lg %r14,__LC_VDSO_PER_CPU 489 lg %r14,__LC_VDSO_PER_CPU
490 lmg %r0,%r10,__PT_R0(%r11) 490 lmg %r0,%r10,__PT_R0(%r11)
491 mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) 491 mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
492 ni __LC_RETURN_PSW+1,0xfd # clear wait state bit
493 stpt __LC_EXIT_TIMER 492 stpt __LC_EXIT_TIMER
494 mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER 493 mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
495 lmg %r11,%r15,__PT_R11(%r11) 494 lmg %r11,%r15,__PT_R11(%r11)
@@ -631,6 +630,24 @@ ext_skip:
631 brasl %r14,do_extint 630 brasl %r14,do_extint
632 j io_return 631 j io_return
633 632
633/*
634 * Load idle PSW. The second "half" of this function is in cleanup_idle.
635 */
636ENTRY(psw_idle)
637 stg %r4,__SF_EMPTY(%r15)
638 larl %r1,psw_idle_lpsw+4
639 stg %r1,__SF_EMPTY+8(%r15)
640 larl %r1,.Lvtimer_max
641 stck __IDLE_ENTER(%r2)
642 ltr %r5,%r5
643 stpt __VQ_IDLE_ENTER(%r3)
644 jz psw_idle_lpsw
645 spt 0(%r1)
646psw_idle_lpsw:
647 lpswe __SF_EMPTY(%r15)
648 br %r14
649psw_idle_end:
650
634__critical_end: 651__critical_end:
635 652
636/* 653/*
@@ -696,7 +713,6 @@ mcck_return:
696 lg %r14,__LC_VDSO_PER_CPU 713 lg %r14,__LC_VDSO_PER_CPU
697 lmg %r0,%r10,__PT_R0(%r11) 714 lmg %r0,%r10,__PT_R0(%r11)
698 mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW 715 mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
699 ni __LC_RETURN_MCCK_PSW+1,0xfd # clear wait state bit
700 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? 716 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
701 jno 0f 717 jno 0f
702 stpt __LC_EXIT_TIMER 718 stpt __LC_EXIT_TIMER
@@ -770,6 +786,8 @@ cleanup_table:
770 .quad io_tif 786 .quad io_tif
771 .quad io_restore 787 .quad io_restore
772 .quad io_done 788 .quad io_done
789 .quad psw_idle
790 .quad psw_idle_end
773 791
774cleanup_critical: 792cleanup_critical:
775 clg %r9,BASED(cleanup_table) # system_call 793 clg %r9,BASED(cleanup_table) # system_call
@@ -788,6 +806,10 @@ cleanup_critical:
788 jl cleanup_io_tif 806 jl cleanup_io_tif
789 clg %r9,BASED(cleanup_table+56) # io_done 807 clg %r9,BASED(cleanup_table+56) # io_done
790 jl cleanup_io_restore 808 jl cleanup_io_restore
809 clg %r9,BASED(cleanup_table+64) # psw_idle
810 jl 0f
811 clg %r9,BASED(cleanup_table+72) # psw_idle_end
812 jl cleanup_idle
7910: br %r14 8130: br %r14
792 814
793 815
@@ -877,7 +899,6 @@ cleanup_io_restore:
877 je 0f 899 je 0f
878 lg %r9,24(%r11) # get saved r11 pointer to pt_regs 900 lg %r9,24(%r11) # get saved r11 pointer to pt_regs
879 mvc __LC_RETURN_PSW(16),__PT_PSW(%r9) 901 mvc __LC_RETURN_PSW(16),__PT_PSW(%r9)
880 ni __LC_RETURN_PSW+1,0xfd # clear wait state bit
881 mvc 0(64,%r11),__PT_R8(%r9) 902 mvc 0(64,%r11),__PT_R8(%r9)
882 lmg %r0,%r7,__PT_R0(%r9) 903 lmg %r0,%r7,__PT_R0(%r9)
8830: lmg %r8,%r9,__LC_RETURN_PSW 9040: lmg %r8,%r9,__LC_RETURN_PSW
@@ -885,6 +906,42 @@ cleanup_io_restore:
885cleanup_io_restore_insn: 906cleanup_io_restore_insn:
886 .quad io_done - 4 907 .quad io_done - 4
887 908
909cleanup_idle:
910 # copy interrupt clock & cpu timer
911 mvc __IDLE_EXIT(8,%r2),__LC_INT_CLOCK
912 mvc __VQ_IDLE_EXIT(8,%r3),__LC_ASYNC_ENTER_TIMER
913 cghi %r11,__LC_SAVE_AREA_ASYNC
914 je 0f
915 mvc __IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
916 mvc __VQ_IDLE_EXIT(8,%r3),__LC_MCCK_ENTER_TIMER
9170: # check if stck & stpt have been executed
918 clg %r9,BASED(cleanup_idle_insn)
919 jhe 1f
920 mvc __IDLE_ENTER(8,%r2),__IDLE_EXIT(%r2)
921 mvc __VQ_IDLE_ENTER(8,%r3),__VQ_IDLE_EXIT(%r3)
922 j 2f
9231: # check if the cpu timer has been reprogrammed
924 ltr %r5,%r5
925 jz 2f
926 spt __VQ_IDLE_ENTER(%r3)
9272: # account system time going idle
928 lg %r9,__LC_STEAL_TIMER
929 alg %r9,__IDLE_ENTER(%r2)
930 slg %r9,__LC_LAST_UPDATE_CLOCK
931 stg %r9,__LC_STEAL_TIMER
932 mvc __LC_LAST_UPDATE_CLOCK(8),__IDLE_EXIT(%r2)
933 lg %r9,__LC_SYSTEM_TIMER
934 alg %r9,__LC_LAST_UPDATE_TIMER
935 slg %r9,__VQ_IDLE_ENTER(%r3)
936 stg %r9,__LC_SYSTEM_TIMER
937 mvc __LC_LAST_UPDATE_TIMER(8),__VQ_IDLE_EXIT(%r3)
938 # prepare return psw
939 nihh %r8,0xfffd # clear wait state bit
940 lg %r9,48(%r11) # return from psw_idle
941 br %r14
942cleanup_idle_insn:
943 .quad psw_idle_lpsw
944
888/* 945/*
889 * Integer constants 946 * Integer constants
890 */ 947 */
@@ -893,6 +950,8 @@ cleanup_io_restore_insn:
893 .quad __critical_start 950 .quad __critical_start
894.Lcritical_length: 951.Lcritical_length:
895 .quad __critical_end - __critical_start 952 .quad __critical_end - __critical_start
953.Lvtimer_max:
954 .quad 0x7fffffffffffffff
896 955
897 956
898#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) 957#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE)
diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c
index b9a7fdd9c814..09a014c62537 100644
--- a/arch/s390/kernel/irq.c
+++ b/arch/s390/kernel/irq.c
@@ -219,8 +219,6 @@ void __irq_entry do_extint(struct pt_regs *regs, unsigned int ext_int_code,
219 219
220 code = (unsigned short) ext_int_code; 220 code = (unsigned short) ext_int_code;
221 old_regs = set_irq_regs(regs); 221 old_regs = set_irq_regs(regs);
222 s390_idle_check(regs, S390_lowcore.int_clock,
223 S390_lowcore.async_enter_timer);
224 irq_enter(); 222 irq_enter();
225 if (S390_lowcore.int_clock >= S390_lowcore.clock_comparator) 223 if (S390_lowcore.int_clock >= S390_lowcore.clock_comparator)
226 /* Serve timer interrupts first. */ 224 /* Serve timer interrupts first. */
diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c
index 0fd2e863e114..8c372ca61350 100644
--- a/arch/s390/kernel/nmi.c
+++ b/arch/s390/kernel/nmi.c
@@ -254,8 +254,6 @@ void notrace s390_do_machine_check(struct pt_regs *regs)
254 int umode; 254 int umode;
255 255
256 nmi_enter(); 256 nmi_enter();
257 s390_idle_check(regs, S390_lowcore.mcck_clock,
258 S390_lowcore.mcck_enter_timer);
259 kstat_cpu(smp_processor_id()).irqs[NMI_NMI]++; 257 kstat_cpu(smp_processor_id()).irqs[NMI_NMI]++;
260 mci = (struct mci *) &S390_lowcore.mcck_interruption_code; 258 mci = (struct mci *) &S390_lowcore.mcck_interruption_code;
261 mcck = &__get_cpu_var(cpu_mcck); 259 mcck = &__get_cpu_var(cpu_mcck);
diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c
index e795933eb2cb..78b3c149b8b6 100644
--- a/arch/s390/kernel/process.c
+++ b/arch/s390/kernel/process.c
@@ -77,13 +77,8 @@ static void default_idle(void)
77 local_irq_enable(); 77 local_irq_enable();
78 return; 78 return;
79 } 79 }
80 trace_hardirqs_on(); 80 /* Halt the cpu and keep track of cpu time accounting. */
81 /* Don't trace preempt off for idle. */
82 stop_critical_timings();
83 /* Stop virtual timer and halt the cpu. */
84 vtime_stop_cpu(); 81 vtime_stop_cpu();
85 /* Reenable preemption tracer. */
86 start_critical_timings();
87} 82}
88 83
89void cpu_idle(void) 84void cpu_idle(void)
diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index 6db8526a602d..afd6e5113a90 100644
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -972,22 +972,16 @@ static DEVICE_ATTR(capability, 0444, show_capability, NULL);
972static ssize_t show_idle_count(struct device *dev, 972static ssize_t show_idle_count(struct device *dev,
973 struct device_attribute *attr, char *buf) 973 struct device_attribute *attr, char *buf)
974{ 974{
975 struct s390_idle_data *idle; 975 struct s390_idle_data *idle = &per_cpu(s390_idle, dev->id);
976 unsigned long long idle_count; 976 unsigned long long idle_count;
977 unsigned int sequence; 977 unsigned int sequence;
978 978
979 idle = &per_cpu(s390_idle, dev->id); 979 do {
980repeat: 980 sequence = ACCESS_ONCE(idle->sequence);
981 sequence = idle->sequence; 981 idle_count = ACCESS_ONCE(idle->idle_count);
982 smp_rmb(); 982 if (ACCESS_ONCE(idle->idle_enter))
983 if (sequence & 1) 983 idle_count++;
984 goto repeat; 984 } while ((sequence & 1) || (idle->sequence != sequence));
985 idle_count = idle->idle_count;
986 if (idle->idle_enter)
987 idle_count++;
988 smp_rmb();
989 if (idle->sequence != sequence)
990 goto repeat;
991 return sprintf(buf, "%llu\n", idle_count); 985 return sprintf(buf, "%llu\n", idle_count);
992} 986}
993static DEVICE_ATTR(idle_count, 0444, show_idle_count, NULL); 987static DEVICE_ATTR(idle_count, 0444, show_idle_count, NULL);
@@ -995,24 +989,18 @@ static DEVICE_ATTR(idle_count, 0444, show_idle_count, NULL);
995static ssize_t show_idle_time(struct device *dev, 989static ssize_t show_idle_time(struct device *dev,
996 struct device_attribute *attr, char *buf) 990 struct device_attribute *attr, char *buf)
997{ 991{
998 struct s390_idle_data *idle; 992 struct s390_idle_data *idle = &per_cpu(s390_idle, dev->id);
999 unsigned long long now, idle_time, idle_enter; 993 unsigned long long now, idle_time, idle_enter, idle_exit;
1000 unsigned int sequence; 994 unsigned int sequence;
1001 995
1002 idle = &per_cpu(s390_idle, dev->id); 996 do {
1003 now = get_clock(); 997 now = get_clock();
1004repeat: 998 sequence = ACCESS_ONCE(idle->sequence);
1005 sequence = idle->sequence; 999 idle_time = ACCESS_ONCE(idle->idle_time);
1006 smp_rmb(); 1000 idle_enter = ACCESS_ONCE(idle->idle_enter);
1007 if (sequence & 1) 1001 idle_exit = ACCESS_ONCE(idle->idle_exit);
1008 goto repeat; 1002 } while ((sequence & 1) || (idle->sequence != sequence));
1009 idle_time = idle->idle_time; 1003 idle_time += idle_enter ? ((idle_exit ? : now) - idle_enter) : 0;
1010 idle_enter = idle->idle_enter;
1011 if (idle_enter != 0ULL && idle_enter < now)
1012 idle_time += now - idle_enter;
1013 smp_rmb();
1014 if (idle->sequence != sequence)
1015 goto repeat;
1016 return sprintf(buf, "%llu\n", idle_time >> 12); 1004 return sprintf(buf, "%llu\n", idle_time >> 12);
1017} 1005}
1018static DEVICE_ATTR(idle_time_us, 0444, show_idle_time, NULL); 1006static DEVICE_ATTR(idle_time_us, 0444, show_idle_time, NULL);
diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c
index 7bacee9a546f..277ea712b232 100644
--- a/arch/s390/kernel/vtime.c
+++ b/arch/s390/kernel/vtime.c
@@ -26,6 +26,7 @@
26#include <asm/irq_regs.h> 26#include <asm/irq_regs.h>
27#include <asm/cputime.h> 27#include <asm/cputime.h>
28#include <asm/irq.h> 28#include <asm/irq.h>
29#include "entry.h"
29 30
30static DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer); 31static DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer);
31 32
@@ -123,153 +124,53 @@ void account_system_vtime(struct task_struct *tsk)
123} 124}
124EXPORT_SYMBOL_GPL(account_system_vtime); 125EXPORT_SYMBOL_GPL(account_system_vtime);
125 126
126void __kprobes vtime_start_cpu(__u64 int_clock, __u64 enter_timer) 127void __kprobes vtime_stop_cpu(void)
127{ 128{
128 struct s390_idle_data *idle = &__get_cpu_var(s390_idle); 129 struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
129 struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer); 130 struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer);
130 __u64 idle_time, expires; 131 unsigned long long idle_time;
132 unsigned long psw_mask;
131 133
132 if (idle->idle_enter == 0ULL) 134 trace_hardirqs_on();
133 return; 135 /* Don't trace preempt off for idle. */
136 stop_critical_timings();
134 137
135 /* Account time spent with enabled wait psw loaded as idle time. */ 138 /* Wait for external, I/O or machine check interrupt. */
136 idle_time = int_clock - idle->idle_enter; 139 psw_mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_DAT |
137 account_idle_time(idle_time); 140 PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK;
138 S390_lowcore.steal_timer += 141 idle->nohz_delay = 0;
139 idle->idle_enter - S390_lowcore.last_update_clock;
140 S390_lowcore.last_update_clock = int_clock;
141
142 /* Account system time spent going idle. */
143 S390_lowcore.system_timer += S390_lowcore.last_update_timer - vq->idle;
144 S390_lowcore.last_update_timer = enter_timer;
145
146 /* Restart vtime CPU timer */
147 if (vq->do_spt) {
148 /* Program old expire value but first save progress. */
149 expires = vq->idle - enter_timer;
150 expires += get_vtimer();
151 set_vtimer(expires);
152 } else {
153 /* Don't account the CPU timer delta while the cpu was idle. */
154 vq->elapsed -= vq->idle - enter_timer;
155 }
156 142
143 /* Call the assembler magic in entry.S */
144 psw_idle(idle, vq, psw_mask, !list_empty(&vq->list));
145
146 /* Reenable preemption tracer. */
147 start_critical_timings();
148
149 /* Account time spent with enabled wait psw loaded as idle time. */
157 idle->sequence++; 150 idle->sequence++;
158 smp_wmb(); 151 smp_wmb();
152 idle_time = idle->idle_exit - idle->idle_enter;
159 idle->idle_time += idle_time; 153 idle->idle_time += idle_time;
160 idle->idle_enter = 0ULL; 154 idle->idle_enter = idle->idle_exit = 0ULL;
161 idle->idle_count++; 155 idle->idle_count++;
156 account_idle_time(idle_time);
162 smp_wmb(); 157 smp_wmb();
163 idle->sequence++; 158 idle->sequence++;
164} 159}
165 160
166void __kprobes vtime_stop_cpu(void)
167{
168 struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
169 struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer);
170 psw_t psw;
171
172 /* Wait for external, I/O or machine check interrupt. */
173 psw.mask = psw_kernel_bits | PSW_MASK_WAIT |
174 PSW_MASK_DAT | PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK;
175
176 idle->nohz_delay = 0;
177
178 /* Check if the CPU timer needs to be reprogrammed. */
179 if (vq->do_spt) {
180 __u64 vmax = VTIMER_MAX_SLICE;
181 /*
182 * The inline assembly is equivalent to
183 * vq->idle = get_cpu_timer();
184 * set_cpu_timer(VTIMER_MAX_SLICE);
185 * idle->idle_enter = get_clock();
186 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
187 * PSW_MASK_DAT | PSW_MASK_IO |
188 * PSW_MASK_EXT | PSW_MASK_MCHECK);
189 * The difference is that the inline assembly makes sure that
190 * the last three instruction are stpt, stck and lpsw in that
191 * order. This is done to increase the precision.
192 */
193 asm volatile(
194#ifndef CONFIG_64BIT
195 " basr 1,0\n"
196 "0: ahi 1,1f-0b\n"
197 " st 1,4(%2)\n"
198#else /* CONFIG_64BIT */
199 " larl 1,1f\n"
200 " stg 1,8(%2)\n"
201#endif /* CONFIG_64BIT */
202 " stpt 0(%4)\n"
203 " spt 0(%5)\n"
204 " stck 0(%3)\n"
205#ifndef CONFIG_64BIT
206 " lpsw 0(%2)\n"
207#else /* CONFIG_64BIT */
208 " lpswe 0(%2)\n"
209#endif /* CONFIG_64BIT */
210 "1:"
211 : "=m" (idle->idle_enter), "=m" (vq->idle)
212 : "a" (&psw), "a" (&idle->idle_enter),
213 "a" (&vq->idle), "a" (&vmax), "m" (vmax), "m" (psw)
214 : "memory", "cc", "1");
215 } else {
216 /*
217 * The inline assembly is equivalent to
218 * vq->idle = get_cpu_timer();
219 * idle->idle_enter = get_clock();
220 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
221 * PSW_MASK_DAT | PSW_MASK_IO |
222 * PSW_MASK_EXT | PSW_MASK_MCHECK);
223 * The difference is that the inline assembly makes sure that
224 * the last three instruction are stpt, stck and lpsw in that
225 * order. This is done to increase the precision.
226 */
227 asm volatile(
228#ifndef CONFIG_64BIT
229 " basr 1,0\n"
230 "0: ahi 1,1f-0b\n"
231 " st 1,4(%2)\n"
232#else /* CONFIG_64BIT */
233 " larl 1,1f\n"
234 " stg 1,8(%2)\n"
235#endif /* CONFIG_64BIT */
236 " stpt 0(%4)\n"
237 " stck 0(%3)\n"
238#ifndef CONFIG_64BIT
239 " lpsw 0(%2)\n"
240#else /* CONFIG_64BIT */
241 " lpswe 0(%2)\n"
242#endif /* CONFIG_64BIT */
243 "1:"
244 : "=m" (idle->idle_enter), "=m" (vq->idle)
245 : "a" (&psw), "a" (&idle->idle_enter),
246 "a" (&vq->idle), "m" (psw)
247 : "memory", "cc", "1");
248 }
249}
250
251cputime64_t s390_get_idle_time(int cpu) 161cputime64_t s390_get_idle_time(int cpu)
252{ 162{
253 struct s390_idle_data *idle; 163 struct s390_idle_data *idle = &per_cpu(s390_idle, cpu);
254 unsigned long long now, idle_time, idle_enter; 164 unsigned long long now, idle_enter, idle_exit;
255 unsigned int sequence; 165 unsigned int sequence;
256 166
257 idle = &per_cpu(s390_idle, cpu); 167 do {
258 168 now = get_clock();
259 now = get_clock(); 169 sequence = ACCESS_ONCE(idle->sequence);
260repeat: 170 idle_enter = ACCESS_ONCE(idle->idle_enter);
261 sequence = idle->sequence; 171 idle_exit = ACCESS_ONCE(idle->idle_exit);
262 smp_rmb(); 172 } while ((sequence & 1) || (idle->sequence != sequence));
263 if (sequence & 1) 173 return idle_enter ? ((idle_exit ? : now) - idle_enter) : 0;
264 goto repeat;
265 idle_time = 0;
266 idle_enter = idle->idle_enter;
267 if (idle_enter != 0ULL && idle_enter < now)
268 idle_time = now - idle_enter;
269 smp_rmb();
270 if (idle->sequence != sequence)
271 goto repeat;
272 return idle_time;
273} 174}
274 175
275/* 176/*
@@ -346,7 +247,6 @@ static void do_cpu_timer_interrupt(unsigned int ext_int_code,
346 } 247 }
347 spin_unlock(&vq->lock); 248 spin_unlock(&vq->lock);
348 249
349 vq->do_spt = list_empty(&cb_list);
350 do_callbacks(&cb_list); 250 do_callbacks(&cb_list);
351 251
352 /* next event is first in list */ 252 /* next event is first in list */
@@ -355,8 +255,7 @@ static void do_cpu_timer_interrupt(unsigned int ext_int_code,
355 if (!list_empty(&vq->list)) { 255 if (!list_empty(&vq->list)) {
356 event = list_first_entry(&vq->list, struct vtimer_list, entry); 256 event = list_first_entry(&vq->list, struct vtimer_list, entry);
357 next = event->expires; 257 next = event->expires;
358 } else 258 }
359 vq->do_spt = 0;
360 spin_unlock(&vq->lock); 259 spin_unlock(&vq->lock);
361 /* 260 /*
362 * To improve precision add the time spent by the 261 * To improve precision add the time spent by the
diff --git a/arch/s390/lib/delay.c b/arch/s390/lib/delay.c
index db92f044024c..9f1f71e85778 100644
--- a/arch/s390/lib/delay.c
+++ b/arch/s390/lib/delay.c
@@ -13,6 +13,7 @@
13#include <linux/irqflags.h> 13#include <linux/irqflags.h>
14#include <linux/interrupt.h> 14#include <linux/interrupt.h>
15#include <asm/div64.h> 15#include <asm/div64.h>
16#include <asm/timer.h>
16 17
17void __delay(unsigned long loops) 18void __delay(unsigned long loops)
18{ 19{
@@ -28,36 +29,33 @@ void __delay(unsigned long loops)
28 29
29static void __udelay_disabled(unsigned long long usecs) 30static void __udelay_disabled(unsigned long long usecs)
30{ 31{
31 unsigned long mask, cr0, cr0_saved; 32 unsigned long cr0, cr6, new;
32 u64 clock_saved; 33 u64 clock_saved, end;
33 u64 end;
34 34
35 mask = psw_kernel_bits | PSW_MASK_DAT | PSW_MASK_WAIT |
36 PSW_MASK_EXT | PSW_MASK_MCHECK;
37 end = get_clock() + (usecs << 12); 35 end = get_clock() + (usecs << 12);
38 clock_saved = local_tick_disable(); 36 clock_saved = local_tick_disable();
39 __ctl_store(cr0_saved, 0, 0); 37 __ctl_store(cr0, 0, 0);
40 cr0 = (cr0_saved & 0xffff00e0) | 0x00000800; 38 __ctl_store(cr6, 6, 6);
41 __ctl_load(cr0 , 0, 0); 39 new = (cr0 & 0xffff00e0) | 0x00000800;
40 __ctl_load(new , 0, 0);
41 new = 0;
42 __ctl_load(new, 6, 6);
42 lockdep_off(); 43 lockdep_off();
43 do { 44 do {
44 set_clock_comparator(end); 45 set_clock_comparator(end);
45 trace_hardirqs_on(); 46 vtime_stop_cpu();
46 __load_psw_mask(mask);
47 local_irq_disable(); 47 local_irq_disable();
48 } while (get_clock() < end); 48 } while (get_clock() < end);
49 lockdep_on(); 49 lockdep_on();
50 __ctl_load(cr0_saved, 0, 0); 50 __ctl_load(cr0, 0, 0);
51 __ctl_load(cr6, 6, 6);
51 local_tick_enable(clock_saved); 52 local_tick_enable(clock_saved);
52} 53}
53 54
54static void __udelay_enabled(unsigned long long usecs) 55static void __udelay_enabled(unsigned long long usecs)
55{ 56{
56 unsigned long mask; 57 u64 clock_saved, end;
57 u64 clock_saved;
58 u64 end;
59 58
60 mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_EXT | PSW_MASK_IO;
61 end = get_clock() + (usecs << 12); 59 end = get_clock() + (usecs << 12);
62 do { 60 do {
63 clock_saved = 0; 61 clock_saved = 0;
@@ -65,8 +63,7 @@ static void __udelay_enabled(unsigned long long usecs)
65 clock_saved = local_tick_disable(); 63 clock_saved = local_tick_disable();
66 set_clock_comparator(end); 64 set_clock_comparator(end);
67 } 65 }
68 trace_hardirqs_on(); 66 vtime_stop_cpu();
69 __load_psw_mask(mask);
70 local_irq_disable(); 67 local_irq_disable();
71 if (clock_saved) 68 if (clock_saved)
72 local_tick_enable(clock_saved); 69 local_tick_enable(clock_saved);
diff --git a/drivers/s390/cio/cio.c b/drivers/s390/cio/cio.c
index dc67c397449e..a49c46c91983 100644
--- a/drivers/s390/cio/cio.c
+++ b/drivers/s390/cio/cio.c
@@ -601,8 +601,6 @@ void __irq_entry do_IRQ(struct pt_regs *regs)
601 struct pt_regs *old_regs; 601 struct pt_regs *old_regs;
602 602
603 old_regs = set_irq_regs(regs); 603 old_regs = set_irq_regs(regs);
604 s390_idle_check(regs, S390_lowcore.int_clock,
605 S390_lowcore.async_enter_timer);
606 irq_enter(); 604 irq_enter();
607 __this_cpu_write(s390_idle.nohz_delay, 1); 605 __this_cpu_write(s390_idle.nohz_delay, 1);
608 if (S390_lowcore.int_clock >= S390_lowcore.clock_comparator) 606 if (S390_lowcore.int_clock >= S390_lowcore.clock_comparator)
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-12 01:12:57 -0500