s390/vtimer: rework virtual timer interface

The current virtual timer interface is inherently per-cpu and hard to
use. The sole user of the interface is appldata which uses it to execute
a function after a specific amount of cputime has been used over all cpus.

Rework the virtual timer interface to hook into the cputime accounting.
This makes the interface independent from the CPU timer interrupts, and
makes the virtual timers global as opposed to per-cpu.
Overall the code is greatly simplified. The downside is that the accuracy
is not as good as the original implementation, but it is still good enough
for appldata.

Reviewed-by: Jan Glauber <jang@linux.vnet.ibm.com>
Reviewed-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

8 files changed, 170 insertions, 302 deletions

diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c
index 0e974ddd156b..45ef1a7b08f9 100644
--- a/arch/s390/kernel/asm-offsets.c
+++ b/arch/s390/kernel/asm-offsets.c
@@ -9,7 +9,6 @@
 #include <linux/kbuild.h>
 #include <linux/sched.h>
 #include <asm/cputime.h>
-#include <asm/timer.h>
 #include <asm/vdso.h>
 #include <asm/pgtable.h>
 
@@ -72,11 +71,10 @@ int main(void)
         DEFINE(__CLOCK_REALTIME_RES, MONOTONIC_RES_NSEC);
         BLANK();
         /* idle data offsets */
-        DEFINE(__IDLE_ENTER, offsetof(struct s390_idle_data, idle_enter));
-        DEFINE(__IDLE_EXIT, offsetof(struct s390_idle_data, idle_exit));
-        /* vtimer queue offsets */
-        DEFINE(__VQ_IDLE_ENTER, offsetof(struct vtimer_queue, idle_enter));
-        DEFINE(__VQ_IDLE_EXIT, offsetof(struct vtimer_queue, idle_exit));
+        DEFINE(__CLOCK_IDLE_ENTER, offsetof(struct s390_idle_data, clock_idle_enter));
+        DEFINE(__CLOCK_IDLE_EXIT, offsetof(struct s390_idle_data, clock_idle_exit));
+        DEFINE(__TIMER_IDLE_ENTER, offsetof(struct s390_idle_data, timer_idle_enter));
+        DEFINE(__TIMER_IDLE_EXIT, offsetof(struct s390_idle_data, timer_idle_exit));
         /* lowcore offsets */
         DEFINE(__LC_EXT_PARAMS, offsetof(struct _lowcore, ext_params));
         DEFINE(__LC_EXT_CPU_ADDR, offsetof(struct _lowcore, ext_cpu_addr));
diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index 2c0eff488875..870bad6d56fc 100644
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -616,17 +616,13 @@ ext_skip:
  * Load idle PSW. The second "half" of this function is in cleanup_idle.
  */
 ENTRY(psw_idle)
-        st      %r4,__SF_EMPTY(%r15)
+        st      %r3,__SF_EMPTY(%r15)
         basr    %r1,0
         la      %r1,psw_idle_lpsw+4-.(%r1)
         st      %r1,__SF_EMPTY+4(%r15)
         oi      __SF_EMPTY+4(%r15),0x80
-        la      %r1,.Lvtimer_max-psw_idle_lpsw-4(%r1)
-        stck    __IDLE_ENTER(%r2)
-        ltr     %r5,%r5
-        stpt    __VQ_IDLE_ENTER(%r3)
-        jz      psw_idle_lpsw
-        spt     0(%r1)
+        stck    __CLOCK_IDLE_ENTER(%r2)
+        stpt    __TIMER_IDLE_ENTER(%r2)
 psw_idle_lpsw:
         lpsw    __SF_EMPTY(%r15)
         br      %r14
@@ -885,33 +881,28 @@ cleanup_io_restore_insn:
 
 cleanup_idle:
         # copy interrupt clock & cpu timer
-        mvc     __IDLE_EXIT(8,%r2),__LC_INT_CLOCK
-        mvc     __VQ_IDLE_EXIT(8,%r3),__LC_ASYNC_ENTER_TIMER
+        mvc     __CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK
+        mvc     __TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER
         chi     %r11,__LC_SAVE_AREA_ASYNC
         je      0f
-        mvc     __IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
-        mvc     __VQ_IDLE_EXIT(8,%r3),__LC_MCCK_ENTER_TIMER
+        mvc     __CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
+        mvc     __TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER
 0:      # check if stck has been executed
         cl      %r9,BASED(cleanup_idle_insn)
         jhe     1f
-        mvc     __IDLE_ENTER(8,%r2),__IDLE_EXIT(%r2)
-        mvc     __VQ_IDLE_ENTER(8,%r3),__VQ_IDLE_EXIT(%r3)
-        j       2f
-1:      # check if the cpu timer has been reprogrammed
-        ltr     %r5,%r5
-        jz      2f
-        spt     __VQ_IDLE_ENTER(%r3)
-2:      # account system time going idle
+        mvc     __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2)
+        mvc     __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r3)
+1:      # account system time going idle
         lm      %r9,%r10,__LC_STEAL_TIMER
-        ADD64   %r9,%r10,__IDLE_ENTER(%r2)
+        ADD64   %r9,%r10,__CLOCK_IDLE_ENTER(%r2)
         SUB64   %r9,%r10,__LC_LAST_UPDATE_CLOCK
         stm     %r9,%r10,__LC_STEAL_TIMER
-        mvc     __LC_LAST_UPDATE_CLOCK(8),__IDLE_EXIT(%r2)
+        mvc     __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2)
         lm      %r9,%r10,__LC_SYSTEM_TIMER
         ADD64   %r9,%r10,__LC_LAST_UPDATE_TIMER
-        SUB64   %r9,%r10,__VQ_IDLE_ENTER(%r3)
+        SUB64   %r9,%r10,__TIMER_IDLE_ENTER(%r2)
         stm     %r9,%r10,__LC_SYSTEM_TIMER
-        mvc     __LC_LAST_UPDATE_TIMER(8),__VQ_IDLE_EXIT(%r3)
+        mvc     __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
         # prepare return psw
         n       %r8,BASED(cleanup_idle_wait)    # clear wait state bit
         l       %r9,24(%r11)                    # return from psw_idle
diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h
index f66a229ab0b3..a5f4dc42a5db 100644
--- a/arch/s390/kernel/entry.h
+++ b/arch/s390/kernel/entry.h
@@ -5,7 +5,6 @@
 #include <linux/signal.h>
 #include <asm/ptrace.h>
 #include <asm/cputime.h>
-#include <asm/timer.h>
 
 extern void (*pgm_check_table[128])(struct pt_regs *);
 extern void *restart_stack;
@@ -17,8 +16,7 @@ void io_int_handler(void);
 void mcck_int_handler(void);
 void restart_int_handler(void);
 void restart_call_handler(void);
-void psw_idle(struct s390_idle_data *, struct vtimer_queue *,
-              unsigned long, int);
+void psw_idle(struct s390_idle_data *, unsigned long);
 
 asmlinkage long do_syscall_trace_enter(struct pt_regs *regs);
 asmlinkage void do_syscall_trace_exit(struct pt_regs *regs);
diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S
index 1983c22a8a99..349b7eeb348a 100644
--- a/arch/s390/kernel/entry64.S
+++ b/arch/s390/kernel/entry64.S
@@ -642,15 +642,11 @@ ext_skip:
  * Load idle PSW. The second "half" of this function is in cleanup_idle.
  */
 ENTRY(psw_idle)
-        stg     %r4,__SF_EMPTY(%r15)
+        stg     %r3,__SF_EMPTY(%r15)
         larl    %r1,psw_idle_lpsw+4
         stg     %r1,__SF_EMPTY+8(%r15)
-        larl    %r1,.Lvtimer_max
-        STCK    __IDLE_ENTER(%r2)
-        ltr     %r5,%r5
-        stpt    __VQ_IDLE_ENTER(%r3)
-        jz      psw_idle_lpsw
-        spt     0(%r1)
+        STCK    __CLOCK_IDLE_ENTER(%r2)
+        stpt    __TIMER_IDLE_ENTER(%r2)
 psw_idle_lpsw:
         lpswe   __SF_EMPTY(%r15)
         br      %r14
@@ -918,33 +914,28 @@ cleanup_io_restore_insn:
 
 cleanup_idle:
         # copy interrupt clock & cpu timer
-        mvc     __IDLE_EXIT(8,%r2),__LC_INT_CLOCK
-        mvc     __VQ_IDLE_EXIT(8,%r3),__LC_ASYNC_ENTER_TIMER
+        mvc     __CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK
+        mvc     __TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER
         cghi    %r11,__LC_SAVE_AREA_ASYNC
         je      0f
-        mvc     __IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
-        mvc     __VQ_IDLE_EXIT(8,%r3),__LC_MCCK_ENTER_TIMER
+        mvc     __CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
+        mvc     __TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER
 0:      # check if stck & stpt have been executed
         clg     %r9,BASED(cleanup_idle_insn)
         jhe     1f
-        mvc     __IDLE_ENTER(8,%r2),__IDLE_EXIT(%r2)
-        mvc     __VQ_IDLE_ENTER(8,%r3),__VQ_IDLE_EXIT(%r3)
-        j       2f
-1:      # check if the cpu timer has been reprogrammed
-        ltr     %r5,%r5
-        jz      2f
-        spt     __VQ_IDLE_ENTER(%r3)
-2:      # account system time going idle
+        mvc     __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2)
+        mvc     __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r2)
+1:      # account system time going idle
         lg      %r9,__LC_STEAL_TIMER
-        alg     %r9,__IDLE_ENTER(%r2)
+        alg     %r9,__CLOCK_IDLE_ENTER(%r2)
         slg     %r9,__LC_LAST_UPDATE_CLOCK
         stg     %r9,__LC_STEAL_TIMER
-        mvc     __LC_LAST_UPDATE_CLOCK(8),__IDLE_EXIT(%r2)
+        mvc     __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2)
         lg      %r9,__LC_SYSTEM_TIMER
         alg     %r9,__LC_LAST_UPDATE_TIMER
-        slg     %r9,__VQ_IDLE_ENTER(%r3)
+        slg     %r9,__TIMER_IDLE_ENTER(%r2)
         stg     %r9,__LC_SYSTEM_TIMER
-        mvc     __LC_LAST_UPDATE_TIMER(8),__VQ_IDLE_EXIT(%r3)
+        mvc     __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
         # prepare return psw
         nihh    %r8,0xfffd              # clear wait state bit
         lg      %r9,48(%r11)            # return from psw_idle
@@ -960,8 +951,6 @@ cleanup_idle_insn:
         .quad   __critical_start
 .Lcritical_length:
         .quad   __critical_end - __critical_start
-.Lvtimer_max:
-        .quad   0x7fffffffffffffff
 
 
 #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE)
diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c
index 7efbfa53d659..733175373a4c 100644
--- a/arch/s390/kernel/process.c
+++ b/arch/s390/kernel/process.c
@@ -25,8 +25,8 @@
 #include <linux/module.h>
 #include <asm/io.h>
 #include <asm/processor.h>
+#include <asm/vtimer.h>
 #include <asm/irq.h>
-#include <asm/timer.h>
 #include <asm/nmi.h>
 #include <asm/smp.h>
 #include <asm/switch_to.h>
diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index eeb441bbddae..5481da80926a 100644
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -38,7 +38,7 @@
 #include <asm/setup.h>
 #include <asm/irq.h>
 #include <asm/tlbflush.h>
-#include <asm/timer.h>
+#include <asm/vtimer.h>
 #include <asm/lowcore.h>
 #include <asm/sclp.h>
 #include <asm/vdso.h>
@@ -917,7 +917,7 @@ static ssize_t show_idle_count(struct device *dev,
         do {
                 sequence = ACCESS_ONCE(idle->sequence);
                 idle_count = ACCESS_ONCE(idle->idle_count);
-                if (ACCESS_ONCE(idle->idle_enter))
+                if (ACCESS_ONCE(idle->clock_idle_enter))
                         idle_count++;
         } while ((sequence & 1) || (idle->sequence != sequence));
         return sprintf(buf, "%llu\n", idle_count);
@@ -935,8 +935,8 @@ static ssize_t show_idle_time(struct device *dev,
                 now = get_clock();
                 sequence = ACCESS_ONCE(idle->sequence);
                 idle_time = ACCESS_ONCE(idle->idle_time);
-                idle_enter = ACCESS_ONCE(idle->idle_enter);
-                idle_exit = ACCESS_ONCE(idle->idle_exit);
+                idle_enter = ACCESS_ONCE(idle->clock_idle_enter);
+                idle_exit = ACCESS_ONCE(idle->clock_idle_exit);
         } while ((sequence & 1) || (idle->sequence != sequence));
         idle_time += idle_enter ? ((idle_exit ? : now) - idle_enter) : 0;
         return sprintf(buf, "%llu\n", idle_time >> 12);
diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c
index 30cf3bdc9b77..dcec960fc724 100644
--- a/arch/s390/kernel/time.c
+++ b/arch/s390/kernel/time.c
@@ -44,7 +44,7 @@
 #include <asm/vdso.h>
 #include <asm/irq.h>
 #include <asm/irq_regs.h>
-#include <asm/timer.h>
+#include <asm/vtimer.h>
 #include <asm/etr.h>
 #include <asm/cio.h>
 #include "entry.h"
diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c
index 39ebff506946..4fc97b40a6e1 100644
--- a/arch/s390/kernel/vtime.c
+++ b/arch/s390/kernel/vtime.c
@@ -1,71 +1,82 @@
 /*
- *  arch/s390/kernel/vtime.c
  *    Virtual cpu timer based timer functions.
  *
- *  S390 version
- *    Copyright (C) 2004 IBM Deutschland Entwicklung GmbH, IBM Corporation
+ *    Copyright IBM Corp. 2004, 2012
  *    Author(s): Jan Glauber <jan.glauber@de.ibm.com>
  */
 
-#include <linux/module.h>
+#include <linux/kernel_stat.h>
+#include <linux/notifier.h>
+#include <linux/kprobes.h>
+#include <linux/export.h>
 #include <linux/kernel.h>
-#include <linux/time.h>
-#include <linux/delay.h>
-#include <linux/init.h>
-#include <linux/smp.h>
-#include <linux/types.h>
 #include <linux/timex.h>
-#include <linux/notifier.h>
-#include <linux/kernel_stat.h>
-#include <linux/rcupdate.h>
-#include <linux/posix-timers.h>
+#include <linux/types.h>
+#include <linux/time.h>
 #include <linux/cpu.h>
-#include <linux/kprobes.h>
+#include <linux/smp.h>
 
-#include <asm/timer.h>
 #include <asm/irq_regs.h>
 #include <asm/cputime.h>
+#include <asm/vtimer.h>
 #include <asm/irq.h>
 #include "entry.h"
 
-static DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer);
+static void virt_timer_expire(void);
 
 DEFINE_PER_CPU(struct s390_idle_data, s390_idle);
 
-static inline __u64 get_vtimer(void)
+static LIST_HEAD(virt_timer_list);
+static DEFINE_SPINLOCK(virt_timer_lock);
+static atomic64_t virt_timer_current;
+static atomic64_t virt_timer_elapsed;
+
+static inline u64 get_vtimer(void)
 {
-        __u64 timer;
+        u64 timer;
 
-        asm volatile("STPT %0" : "=m" (timer));
+        asm volatile("stpt %0" : "=m" (timer));
         return timer;
 }
 
-static inline void set_vtimer(__u64 expires)
+static inline void set_vtimer(u64 expires)
 {
-        __u64 timer;
+        u64 timer;
 
-        asm volatile ("  STPT %0\n"  /* Store current cpu timer value */
-                      "  SPT %1"     /* Set new value immediately afterwards */
-                      : "=m" (timer) : "m" (expires) );
+        asm volatile(
+                "       stpt    %0\n"   /* Store current cpu timer value */
+                "       spt     %1"     /* Set new value imm. afterwards */
+                : "=m" (timer) : "m" (expires));
         S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
         S390_lowcore.last_update_timer = expires;
 }
 
+static inline int virt_timer_forward(u64 elapsed)
+{
+        BUG_ON(!irqs_disabled());
+
+        if (list_empty(&virt_timer_list))
+                return 0;
+        elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
+        return elapsed >= atomic64_read(&virt_timer_current);
+}
+
 /*
  * Update process times based on virtual cpu times stored by entry.S
  * to the lowcore fields user_timer, system_timer & steal_clock.
  */
-static void do_account_vtime(struct task_struct *tsk, int hardirq_offset)
+static int do_account_vtime(struct task_struct *tsk, int hardirq_offset)
 {
         struct thread_info *ti = task_thread_info(tsk);
-        __u64 timer, clock, user, system, steal;
+        u64 timer, clock, user, system, steal;
 
         timer = S390_lowcore.last_update_timer;
         clock = S390_lowcore.last_update_clock;
-        asm volatile ("  STPT %0\n"    /* Store current cpu timer value */
-                      "  STCK %1"      /* Store current tod clock value */
-                      : "=m" (S390_lowcore.last_update_timer),
-                        "=m" (S390_lowcore.last_update_clock) );
+        asm volatile(
+                "       stpt    %0\n"   /* Store current cpu timer value */
+                "       stck    %1"     /* Store current tod clock value */
+                : "=m" (S390_lowcore.last_update_timer),
+                  "=m" (S390_lowcore.last_update_clock));
         S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
         S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock;
 
@@ -84,6 +95,8 @@ static void do_account_vtime(struct task_struct *tsk, int hardirq_offset)
                 S390_lowcore.steal_timer = 0;
                 account_steal_time(steal);
         }
+
+        return virt_timer_forward(user + system);
 }
 
 void account_vtime(struct task_struct *prev, struct task_struct *next)
@@ -101,7 +114,8 @@ void account_vtime(struct task_struct *prev, struct task_struct *next)
 
 void account_process_tick(struct task_struct *tsk, int user_tick)
 {
-        do_account_vtime(tsk, HARDIRQ_OFFSET);
+        if (do_account_vtime(tsk, HARDIRQ_OFFSET))
+                virt_timer_expire();
 }
 
 /*
@@ -111,7 +125,7 @@ void account_process_tick(struct task_struct *tsk, int user_tick)
 void account_system_vtime(struct task_struct *tsk)
 {
         struct thread_info *ti = task_thread_info(tsk);
-        __u64 timer, system;
+        u64 timer, system;
 
         timer = S390_lowcore.last_update_timer;
         S390_lowcore.last_update_timer = get_vtimer();
@@ -121,13 +135,14 @@ void account_system_vtime(struct task_struct *tsk)
         S390_lowcore.steal_timer -= system;
         ti->system_timer = S390_lowcore.system_timer;
         account_system_time(tsk, 0, system, system);
+
+        virt_timer_forward(system);
 }
 EXPORT_SYMBOL_GPL(account_system_vtime);
 
 void __kprobes vtime_stop_cpu(void)
 {
         struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
-        struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer);
         unsigned long long idle_time;
         unsigned long psw_mask;
 
@@ -141,7 +156,7 @@ void __kprobes vtime_stop_cpu(void)
         idle->nohz_delay = 0;
 
         /* Call the assembler magic in entry.S */
-        psw_idle(idle, vq, psw_mask, !list_empty(&vq->list));
+        psw_idle(idle, psw_mask);
 
         /* Reenable preemption tracer. */
         start_critical_timings();
@@ -149,9 +164,9 @@ void __kprobes vtime_stop_cpu(void)
         /* Account time spent with enabled wait psw loaded as idle time. */
         idle->sequence++;
         smp_wmb();
-        idle_time = idle->idle_exit - idle->idle_enter;
+        idle_time = idle->clock_idle_exit - idle->clock_idle_enter;
+        idle->clock_idle_enter = idle->clock_idle_exit = 0ULL;
         idle->idle_time += idle_time;
-        idle->idle_enter = idle->idle_exit = 0ULL;
         idle->idle_count++;
         account_idle_time(idle_time);
         smp_wmb();
@@ -167,10 +182,10 @@ cputime64_t s390_get_idle_time(int cpu)
         do {
                 now = get_clock();
                 sequence = ACCESS_ONCE(idle->sequence);
-                idle_enter = ACCESS_ONCE(idle->idle_enter);
-                idle_exit = ACCESS_ONCE(idle->idle_exit);
+                idle_enter = ACCESS_ONCE(idle->clock_idle_enter);
+                idle_exit = ACCESS_ONCE(idle->clock_idle_exit);
         } while ((sequence & 1) || (idle->sequence != sequence));
-        return idle_enter ? ((idle_exit ? : now) - idle_enter) : 0;
+        return idle_enter ? ((idle_exit ?: now) - idle_enter) : 0;
 }
 
 /*
@@ -179,11 +194,11 @@ cputime64_t s390_get_idle_time(int cpu)
  */
 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
 {
-        struct vtimer_list *event;
+        struct vtimer_list *tmp;
 
-        list_for_each_entry(event, head, entry) {
-                if (event->expires > timer->expires) {
-                        list_add_tail(&timer->entry, &event->entry);
+        list_for_each_entry(tmp, head, entry) {
+                if (tmp->expires > timer->expires) {
+                        list_add_tail(&timer->entry, &tmp->entry);
                         return;
                 }
         }
@@ -191,82 +206,45 @@ static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
 }
 
 /*
- * Do the callback functions of expired vtimer events.
- * Called from within the interrupt handler.
- */
-static void do_callbacks(struct list_head *cb_list)
-{
-        struct vtimer_queue *vq;
-        struct vtimer_list *event, *tmp;
-
-        if (list_empty(cb_list))
-                return;
-
-        vq = &__get_cpu_var(virt_cpu_timer);
-
-        list_for_each_entry_safe(event, tmp, cb_list, entry) {
-                list_del_init(&event->entry);
-                (event->function)(event->data);
-                if (event->interval) {
-                        /* Recharge interval timer */
-                        event->expires = event->interval + vq->elapsed;
-                        spin_lock(&vq->lock);
-                        list_add_sorted(event, &vq->list);
-                        spin_unlock(&vq->lock);
-                }
-        }
-}
-
-/*
- * Handler for the virtual CPU timer.
+ * Handler for expired virtual CPU timer.
  */
-static void do_cpu_timer_interrupt(struct ext_code ext_code,
-                                   unsigned int param32, unsigned long param64)
+static void virt_timer_expire(void)
 {
-        struct vtimer_queue *vq;
-        struct vtimer_list *event, *tmp;
-        struct list_head cb_list;       /* the callback queue */
-        __u64 elapsed, next;
-
-        kstat_cpu(smp_processor_id()).irqs[EXTINT_TMR]++;
-        INIT_LIST_HEAD(&cb_list);
-        vq = &__get_cpu_var(virt_cpu_timer);
-
-        /* walk timer list, fire all expired events */
-        spin_lock(&vq->lock);
-
-        elapsed = vq->elapsed + (vq->timer - S390_lowcore.async_enter_timer);
-        BUG_ON((s64) elapsed < 0);
-        vq->elapsed = 0;
-        list_for_each_entry_safe(event, tmp, &vq->list, entry) {
-                if (event->expires < elapsed)
+        struct vtimer_list *timer, *tmp;
+        unsigned long elapsed;
+        LIST_HEAD(cb_list);
+
+        /* walk timer list, fire all expired timers */
+        spin_lock(&virt_timer_lock);
+        elapsed = atomic64_read(&virt_timer_elapsed);
+        list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
+                if (timer->expires < elapsed)
                         /* move expired timer to the callback queue */
-                        list_move_tail(&event->entry, &cb_list);
+                        list_move_tail(&timer->entry, &cb_list);
                 else
-                        event->expires -= elapsed;
+                        timer->expires -= elapsed;
         }
-        spin_unlock(&vq->lock);
-
-        do_callbacks(&cb_list);
-
-        /* next event is first in list */
-        next = VTIMER_MAX_SLICE;
-        spin_lock(&vq->lock);
-        if (!list_empty(&vq->list)) {
-                event = list_first_entry(&vq->list, struct vtimer_list, entry);
-                next = event->expires;
+        if (!list_empty(&virt_timer_list)) {
+                timer = list_first_entry(&virt_timer_list,
+                                         struct vtimer_list, entry);
+                atomic64_set(&virt_timer_current, timer->expires);
+        }
+        atomic64_sub(elapsed, &virt_timer_elapsed);
+        spin_unlock(&virt_timer_lock);
+
+        /* Do callbacks and recharge periodic timers */
+        list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
+                list_del_init(&timer->entry);
+                timer->function(timer->data);
+                if (timer->interval) {
+                        /* Recharge interval timer */
+                        timer->expires = timer->interval +
+                                atomic64_read(&virt_timer_elapsed);
+                        spin_lock(&virt_timer_lock);
+                        list_add_sorted(timer, &virt_timer_list);
+                        spin_unlock(&virt_timer_lock);
+                }
         }
-        spin_unlock(&vq->lock);
-        /*
-         * To improve precision add the time spent by the
-         * interrupt handler to the elapsed time.
-         * Note: CPU timer counts down and we got an interrupt,
-         *       the current content is negative
-         */
-        elapsed = S390_lowcore.async_enter_timer - get_vtimer();
-        set_vtimer(next - elapsed);
-        vq->timer = next - elapsed;
-        vq->elapsed = elapsed;
 }
 
 void init_virt_timer(struct vtimer_list *timer)
@@ -278,179 +256,108 @@ EXPORT_SYMBOL(init_virt_timer);
 
 static inline int vtimer_pending(struct vtimer_list *timer)
 {
-        return (!list_empty(&timer->entry));
+        return !list_empty(&timer->entry);
 }
 
-/*
- * this function should only run on the specified CPU
- */
 static void internal_add_vtimer(struct vtimer_list *timer)
 {
-        struct vtimer_queue *vq;
-        unsigned long flags;
-        __u64 left, expires;
-
-        vq = &per_cpu(virt_cpu_timer, timer->cpu);
-        spin_lock_irqsave(&vq->lock, flags);
-
-        BUG_ON(timer->cpu != smp_processor_id());
-
-        if (list_empty(&vq->list)) {
-                /* First timer on this cpu, just program it. */
-                list_add(&timer->entry, &vq->list);
-                set_vtimer(timer->expires);
-                vq->timer = timer->expires;
-                vq->elapsed = 0;
+        if (list_empty(&virt_timer_list)) {
+                /* First timer, just program it. */
+                atomic64_set(&virt_timer_current, timer->expires);
+                atomic64_set(&virt_timer_elapsed, 0);
+                list_add(&timer->entry, &virt_timer_list);
         } else {
-                /* Check progress of old timers. */
-                expires = timer->expires;
-                left = get_vtimer();
-                if (likely((s64) expires < (s64) left)) {
+                /* Update timer against current base. */
+                timer->expires += atomic64_read(&virt_timer_elapsed);
+                if (likely((s64) timer->expires <
+                           (s64) atomic64_read(&virt_timer_current)))
                         /* The new timer expires before the current timer. */
-                        set_vtimer(expires);
-                        vq->elapsed += vq->timer - left;
-                        vq->timer = expires;
-                } else {
-                        vq->elapsed += vq->timer - left;
-                        vq->timer = left;
-                }
-                /* Insert new timer into per cpu list. */
-                timer->expires += vq->elapsed;
-                list_add_sorted(timer, &vq->list);
+                        atomic64_set(&virt_timer_current, timer->expires);
+                /* Insert new timer into the list. */
+                list_add_sorted(timer, &virt_timer_list);
         }
-
-        spin_unlock_irqrestore(&vq->lock, flags);
-        /* release CPU acquired in prepare_vtimer or mod_virt_timer() */
-        put_cpu();
 }
 
-static inline void prepare_vtimer(struct vtimer_list *timer)
+static void __add_vtimer(struct vtimer_list *timer, int periodic)
 {
-        BUG_ON(!timer->function);
-        BUG_ON(!timer->expires || timer->expires > VTIMER_MAX_SLICE);
-        BUG_ON(vtimer_pending(timer));
-        timer->cpu = get_cpu();
+        unsigned long flags;
+
+        timer->interval = periodic ? timer->expires : 0;
+        spin_lock_irqsave(&virt_timer_lock, flags);
+        internal_add_vtimer(timer);
+        spin_unlock_irqrestore(&virt_timer_lock, flags);
 }
 
 /*
  * add_virt_timer - add an oneshot virtual CPU timer
  */
-void add_virt_timer(void *new)
+void add_virt_timer(struct vtimer_list *timer)
 {
-        struct vtimer_list *timer;
-
-        timer = (struct vtimer_list *)new;
-        prepare_vtimer(timer);
-        timer->interval = 0;
-        internal_add_vtimer(timer);
+        __add_vtimer(timer, 0);
 }
 EXPORT_SYMBOL(add_virt_timer);
 
 /*
  * add_virt_timer_int - add an interval virtual CPU timer
  */
-void add_virt_timer_periodic(void *new)
+void add_virt_timer_periodic(struct vtimer_list *timer)
 {
-        struct vtimer_list *timer;
-
-        timer = (struct vtimer_list *)new;
-        prepare_vtimer(timer);
-        timer->interval = timer->expires;
-        internal_add_vtimer(timer);
+        __add_vtimer(timer, 1);
 }
 EXPORT_SYMBOL(add_virt_timer_periodic);
 
-static int __mod_vtimer(struct vtimer_list *timer, __u64 expires, int periodic)
+static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
 {
-        struct vtimer_queue *vq;
         unsigned long flags;
-        int cpu;
+        int rc;
 
         BUG_ON(!timer->function);
-        BUG_ON(!expires || expires > VTIMER_MAX_SLICE);
 
         if (timer->expires == expires && vtimer_pending(timer))
                 return 1;
-
-        cpu = get_cpu();
-        vq = &per_cpu(virt_cpu_timer, cpu);
-
-        /* disable interrupts before test if timer is pending */
-        spin_lock_irqsave(&vq->lock, flags);
-
-        /* if timer isn't pending add it on the current CPU */
-        if (!vtimer_pending(timer)) {
-                spin_unlock_irqrestore(&vq->lock, flags);
-
-                if (periodic)
-                        timer->interval = expires;
-                else
-                        timer->interval = 0;
-                timer->expires = expires;
-                timer->cpu = cpu;
-                internal_add_vtimer(timer);
-                return 0;
-        }
-
-        /* check if we run on the right CPU */
-        BUG_ON(timer->cpu != cpu);
-
-        list_del_init(&timer->entry);
+        spin_lock_irqsave(&virt_timer_lock, flags);
+        rc = vtimer_pending(timer);
+        if (rc)
+                list_del_init(&timer->entry);
+        timer->interval = periodic ? expires : 0;
         timer->expires = expires;
-        if (periodic)
-                timer->interval = expires;
-
-        /* the timer can't expire anymore so we can release the lock */
-        spin_unlock_irqrestore(&vq->lock, flags);
         internal_add_vtimer(timer);
-        return 1;
+        spin_unlock_irqrestore(&virt_timer_lock, flags);
+        return rc;
 }
 
 /*
- * If we change a pending timer the function must be called on the CPU
- * where the timer is running on.
- *
  * returns whether it has modified a pending timer (1) or not (0)
  */
-int mod_virt_timer(struct vtimer_list *timer, __u64 expires)
+int mod_virt_timer(struct vtimer_list *timer, u64 expires)
 {
         return __mod_vtimer(timer, expires, 0);
 }
 EXPORT_SYMBOL(mod_virt_timer);
 
 /*
- * If we change a pending timer the function must be called on the CPU
- * where the timer is running on.
- *
  * returns whether it has modified a pending timer (1) or not (0)
  */
-int mod_virt_timer_periodic(struct vtimer_list *timer, __u64 expires)
+int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
 {
         return __mod_vtimer(timer, expires, 1);
 }
 EXPORT_SYMBOL(mod_virt_timer_periodic);
 
 /*
- * delete a virtual timer
+ * Delete a virtual timer.
  *
  * returns whether the deleted timer was pending (1) or not (0)
  */
 int del_virt_timer(struct vtimer_list *timer)
 {
         unsigned long flags;
-        struct vtimer_queue *vq;
 
-        /* check if timer is pending */
         if (!vtimer_pending(timer))
                 return 0;
-
-        vq = &per_cpu(virt_cpu_timer, timer->cpu);
-        spin_lock_irqsave(&vq->lock, flags);
-
-        /* we don't interrupt a running timer, just let it expire! */
+        spin_lock_irqsave(&virt_timer_lock, flags);
         list_del_init(&timer->entry);
-
-        spin_unlock_irqrestore(&vq->lock, flags);
+        spin_unlock_irqrestore(&virt_timer_lock, flags);
         return 1;
 }
 EXPORT_SYMBOL(del_virt_timer);
@@ -458,20 +365,10 @@ EXPORT_SYMBOL(del_virt_timer);
 /*
  * Start the virtual CPU timer on the current CPU.
  */
-void init_cpu_vtimer(void)
+void __cpuinit init_cpu_vtimer(void)
 {
-        struct vtimer_queue *vq;
-
-        /* initialize per cpu vtimer structure */
-        vq = &__get_cpu_var(virt_cpu_timer);
-        INIT_LIST_HEAD(&vq->list);
-        spin_lock_init(&vq->lock);
-
-        /* enable cpu timer interrupts */
-        __ctl_set_bit(0,10);
-
         /* set initial cpu timer */
-        set_vtimer(0x7fffffffffffffffULL);
+        set_vtimer(VTIMER_MAX_SLICE);
 }
 
 static int __cpuinit s390_nohz_notify(struct notifier_block *self,
@@ -493,12 +390,7 @@ static int __cpuinit s390_nohz_notify(struct notifier_block *self,
 
 void __init vtime_init(void)
 {
-        /* request the cpu timer external interrupt */
-        if (register_external_interrupt(0x1005, do_cpu_timer_interrupt))
-                panic("Couldn't request external interrupt 0x1005");
-
         /* Enable cpu timer interrupts on the boot cpu. */
         init_cpu_vtimer();
         cpu_notifier(s390_nohz_notify, 0);
 }
-