aboutsummaryrefslogtreecommitdiffstats
path: root/arch/s390/kernel/vtime.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/s390/kernel/vtime.c')
-rw-r--r--arch/s390/kernel/vtime.c163
1 files changed, 31 insertions, 132 deletions
diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c
index 7bacee9a546f..277ea712b232 100644
--- a/arch/s390/kernel/vtime.c
+++ b/arch/s390/kernel/vtime.c
@@ -26,6 +26,7 @@
26#include <asm/irq_regs.h> 26#include <asm/irq_regs.h>
27#include <asm/cputime.h> 27#include <asm/cputime.h>
28#include <asm/irq.h> 28#include <asm/irq.h>
29#include "entry.h"
29 30
30static DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer); 31static DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer);
31 32
@@ -123,153 +124,53 @@ void account_system_vtime(struct task_struct *tsk)
123} 124}
124EXPORT_SYMBOL_GPL(account_system_vtime); 125EXPORT_SYMBOL_GPL(account_system_vtime);
125 126
126void __kprobes vtime_start_cpu(__u64 int_clock, __u64 enter_timer) 127void __kprobes vtime_stop_cpu(void)
127{ 128{
128 struct s390_idle_data *idle = &__get_cpu_var(s390_idle); 129 struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
129 struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer); 130 struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer);
130 __u64 idle_time, expires; 131 unsigned long long idle_time;
132 unsigned long psw_mask;
131 133
132 if (idle->idle_enter == 0ULL) 134 trace_hardirqs_on();
133 return; 135 /* Don't trace preempt off for idle. */
136 stop_critical_timings();
134 137
135 /* Account time spent with enabled wait psw loaded as idle time. */ 138 /* Wait for external, I/O or machine check interrupt. */
136 idle_time = int_clock - idle->idle_enter; 139 psw_mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_DAT |
137 account_idle_time(idle_time); 140 PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK;
138 S390_lowcore.steal_timer += 141 idle->nohz_delay = 0;
139 idle->idle_enter - S390_lowcore.last_update_clock;
140 S390_lowcore.last_update_clock = int_clock;
141
142 /* Account system time spent going idle. */
143 S390_lowcore.system_timer += S390_lowcore.last_update_timer - vq->idle;
144 S390_lowcore.last_update_timer = enter_timer;
145
146 /* Restart vtime CPU timer */
147 if (vq->do_spt) {
148 /* Program old expire value but first save progress. */
149 expires = vq->idle - enter_timer;
150 expires += get_vtimer();
151 set_vtimer(expires);
152 } else {
153 /* Don't account the CPU timer delta while the cpu was idle. */
154 vq->elapsed -= vq->idle - enter_timer;
155 }
156 142
143 /* Call the assembler magic in entry.S */
144 psw_idle(idle, vq, psw_mask, !list_empty(&vq->list));
145
146 /* Reenable preemption tracer. */
147 start_critical_timings();
148
149 /* Account time spent with enabled wait psw loaded as idle time. */
157 idle->sequence++; 150 idle->sequence++;
158 smp_wmb(); 151 smp_wmb();
152 idle_time = idle->idle_exit - idle->idle_enter;
159 idle->idle_time += idle_time; 153 idle->idle_time += idle_time;
160 idle->idle_enter = 0ULL; 154 idle->idle_enter = idle->idle_exit = 0ULL;
161 idle->idle_count++; 155 idle->idle_count++;
156 account_idle_time(idle_time);
162 smp_wmb(); 157 smp_wmb();
163 idle->sequence++; 158 idle->sequence++;
164} 159}
165 160
166void __kprobes vtime_stop_cpu(void)
167{
168 struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
169 struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer);
170 psw_t psw;
171
172 /* Wait for external, I/O or machine check interrupt. */
173 psw.mask = psw_kernel_bits | PSW_MASK_WAIT |
174 PSW_MASK_DAT | PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK;
175
176 idle->nohz_delay = 0;
177
178 /* Check if the CPU timer needs to be reprogrammed. */
179 if (vq->do_spt) {
180 __u64 vmax = VTIMER_MAX_SLICE;
181 /*
182 * The inline assembly is equivalent to
183 * vq->idle = get_cpu_timer();
184 * set_cpu_timer(VTIMER_MAX_SLICE);
185 * idle->idle_enter = get_clock();
186 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
187 * PSW_MASK_DAT | PSW_MASK_IO |
188 * PSW_MASK_EXT | PSW_MASK_MCHECK);
189 * The difference is that the inline assembly makes sure that
190 * the last three instruction are stpt, stck and lpsw in that
191 * order. This is done to increase the precision.
192 */
193 asm volatile(
194#ifndef CONFIG_64BIT
195 " basr 1,0\n"
196 "0: ahi 1,1f-0b\n"
197 " st 1,4(%2)\n"
198#else /* CONFIG_64BIT */
199 " larl 1,1f\n"
200 " stg 1,8(%2)\n"
201#endif /* CONFIG_64BIT */
202 " stpt 0(%4)\n"
203 " spt 0(%5)\n"
204 " stck 0(%3)\n"
205#ifndef CONFIG_64BIT
206 " lpsw 0(%2)\n"
207#else /* CONFIG_64BIT */
208 " lpswe 0(%2)\n"
209#endif /* CONFIG_64BIT */
210 "1:"
211 : "=m" (idle->idle_enter), "=m" (vq->idle)
212 : "a" (&psw), "a" (&idle->idle_enter),
213 "a" (&vq->idle), "a" (&vmax), "m" (vmax), "m" (psw)
214 : "memory", "cc", "1");
215 } else {
216 /*
217 * The inline assembly is equivalent to
218 * vq->idle = get_cpu_timer();
219 * idle->idle_enter = get_clock();
220 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
221 * PSW_MASK_DAT | PSW_MASK_IO |
222 * PSW_MASK_EXT | PSW_MASK_MCHECK);
223 * The difference is that the inline assembly makes sure that
224 * the last three instruction are stpt, stck and lpsw in that
225 * order. This is done to increase the precision.
226 */
227 asm volatile(
228#ifndef CONFIG_64BIT
229 " basr 1,0\n"
230 "0: ahi 1,1f-0b\n"
231 " st 1,4(%2)\n"
232#else /* CONFIG_64BIT */
233 " larl 1,1f\n"
234 " stg 1,8(%2)\n"
235#endif /* CONFIG_64BIT */
236 " stpt 0(%4)\n"
237 " stck 0(%3)\n"
238#ifndef CONFIG_64BIT
239 " lpsw 0(%2)\n"
240#else /* CONFIG_64BIT */
241 " lpswe 0(%2)\n"
242#endif /* CONFIG_64BIT */
243 "1:"
244 : "=m" (idle->idle_enter), "=m" (vq->idle)
245 : "a" (&psw), "a" (&idle->idle_enter),
246 "a" (&vq->idle), "m" (psw)
247 : "memory", "cc", "1");
248 }
249}
250
251cputime64_t s390_get_idle_time(int cpu) 161cputime64_t s390_get_idle_time(int cpu)
252{ 162{
253 struct s390_idle_data *idle; 163 struct s390_idle_data *idle = &per_cpu(s390_idle, cpu);
254 unsigned long long now, idle_time, idle_enter; 164 unsigned long long now, idle_enter, idle_exit;
255 unsigned int sequence; 165 unsigned int sequence;
256 166
257 idle = &per_cpu(s390_idle, cpu); 167 do {
258 168 now = get_clock();
259 now = get_clock(); 169 sequence = ACCESS_ONCE(idle->sequence);
260repeat: 170 idle_enter = ACCESS_ONCE(idle->idle_enter);
261 sequence = idle->sequence; 171 idle_exit = ACCESS_ONCE(idle->idle_exit);
262 smp_rmb(); 172 } while ((sequence & 1) || (idle->sequence != sequence));
263 if (sequence & 1) 173 return idle_enter ? ((idle_exit ? : now) - idle_enter) : 0;
264 goto repeat;
265 idle_time = 0;
266 idle_enter = idle->idle_enter;
267 if (idle_enter != 0ULL && idle_enter < now)
268 idle_time = now - idle_enter;
269 smp_rmb();
270 if (idle->sequence != sequence)
271 goto repeat;
272 return idle_time;
273} 174}
274 175
275/* 176/*
@@ -346,7 +247,6 @@ static void do_cpu_timer_interrupt(unsigned int ext_int_code,
346 } 247 }
347 spin_unlock(&vq->lock); 248 spin_unlock(&vq->lock);
348 249
349 vq->do_spt = list_empty(&cb_list);
350 do_callbacks(&cb_list); 250 do_callbacks(&cb_list);
351 251
352 /* next event is first in list */ 252 /* next event is first in list */
@@ -355,8 +255,7 @@ static void do_cpu_timer_interrupt(unsigned int ext_int_code,
355 if (!list_empty(&vq->list)) { 255 if (!list_empty(&vq->list)) {
356 event = list_first_entry(&vq->list, struct vtimer_list, entry); 256 event = list_first_entry(&vq->list, struct vtimer_list, entry);
357 next = event->expires; 257 next = event->expires;
358 } else 258 }
359 vq->do_spt = 0;
360 spin_unlock(&vq->lock); 259 spin_unlock(&vq->lock);
361 /* 260 /*
362 * To improve precision add the time spent by the 261 * To improve precision add the time spent by the
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-22 18:26:38 -0500