s390/rwlock: use directed yield for write-locked rwlocks

Add an owner field to the arch_rwlock_t to be able to pass the timeslice
of a virtual CPU with diagnose 0x9c to the lock owner in case the rwlock
is write-locked. The undirected yield in case the rwlock is acquired
writable but the lock is read-locked is removed.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

5 files changed, 54 insertions, 32 deletions

diff --git a/arch/s390/include/asm/smp.h b/arch/s390/include/asm/smp.h
index 4f1307962a95..762d4f88af5a 100644
--- a/arch/s390/include/asm/smp.h
+++ b/arch/s390/include/asm/smp.h
@@ -29,7 +29,6 @@ extern int smp_find_processor_id(u16 address);
 extern int smp_store_status(int cpu);
 extern int smp_vcpu_scheduled(int cpu);
 extern void smp_yield_cpu(int cpu);
-extern void smp_yield(void);
 extern void smp_cpu_set_polarization(int cpu, int val);
 extern int smp_cpu_get_polarization(int cpu);
 extern void smp_fill_possible_mask(void);
@@ -50,7 +49,6 @@ static inline int smp_find_processor_id(u16 address) { return 0; }
 static inline int smp_store_status(int cpu) { return 0; }
 static inline int smp_vcpu_scheduled(int cpu) { return 1; }
 static inline void smp_yield_cpu(int cpu) { }
-static inline void smp_yield(void) { }
 static inline void smp_fill_possible_mask(void) { }
 
 #endif /* CONFIG_SMP */
diff --git a/arch/s390/include/asm/spinlock.h b/arch/s390/include/asm/spinlock.h
index d26ad2ac7cb2..e98654163e5c 100644
--- a/arch/s390/include/asm/spinlock.h
+++ b/arch/s390/include/asm/spinlock.h
@@ -37,11 +37,17 @@ _raw_compare_and_swap(unsigned int *lock, unsigned int old, unsigned int new)
  * (the type definitions are in asm/spinlock_types.h)
  */
 
+void arch_lock_relax(unsigned int cpu);
+
 void arch_spin_lock_wait(arch_spinlock_t *);
 int arch_spin_trylock_retry(arch_spinlock_t *);
-void arch_spin_relax(arch_spinlock_t *);
 void arch_spin_lock_wait_flags(arch_spinlock_t *, unsigned long flags);
 
+static inline void arch_spin_relax(arch_spinlock_t *lock)
+{
+        arch_lock_relax(lock->lock);
+}
+
 static inline u32 arch_spin_lockval(int cpu)
 {
         return ~cpu;
@@ -170,17 +176,21 @@ static inline void arch_write_lock(arch_rwlock_t *rw)
 {
         if (!arch_write_trylock_once(rw))
                 _raw_write_lock_wait(rw);
+        rw->owner = SPINLOCK_LOCKVAL;
 }
 
 static inline void arch_write_lock_flags(arch_rwlock_t *rw, unsigned long flags)
 {
         if (!arch_write_trylock_once(rw))
                 _raw_write_lock_wait_flags(rw, flags);
+        rw->owner = SPINLOCK_LOCKVAL;
 }
 
 static inline void arch_write_unlock(arch_rwlock_t *rw)
 {
         typecheck(unsigned int, rw->lock);
+
+        rw->owner = 0;
         asm volatile(
                 __ASM_BARRIER
                 "st     %1,%0\n"
@@ -198,12 +208,20 @@ static inline int arch_read_trylock(arch_rwlock_t *rw)
 
 static inline int arch_write_trylock(arch_rwlock_t *rw)
 {
-        if (!arch_write_trylock_once(rw))
-                return _raw_write_trylock_retry(rw);
+        if (!arch_write_trylock_once(rw) && !_raw_write_trylock_retry(rw))
+                return 0;
+        rw->owner = SPINLOCK_LOCKVAL;
         return 1;
 }
 
-#define arch_read_relax(lock)   cpu_relax()
-#define arch_write_relax(lock)  cpu_relax()
+static inline void arch_read_relax(arch_rwlock_t *rw)
+{
+        arch_lock_relax(rw->owner);
+}
+
+static inline void arch_write_relax(arch_rwlock_t *rw)
+{
+        arch_lock_relax(rw->owner);
+}
 
 #endif /* __ASM_SPINLOCK_H */
diff --git a/arch/s390/include/asm/spinlock_types.h b/arch/s390/include/asm/spinlock_types.h
index b2cd6ff7c2c5..d84b6939237c 100644
--- a/arch/s390/include/asm/spinlock_types.h
+++ b/arch/s390/include/asm/spinlock_types.h
@@ -13,6 +13,7 @@ typedef struct {
 
 typedef struct {
         unsigned int lock;
+        unsigned int owner;
 } arch_rwlock_t;
 
 #define __ARCH_RW_LOCK_UNLOCKED         { 0 }
diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index 243c7e512600..abec97b4ddbf 100644
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -333,12 +333,6 @@ int smp_vcpu_scheduled(int cpu)
         return pcpu_running(pcpu_devices + cpu);
 }
 
-void smp_yield(void)
-{
-        if (MACHINE_HAS_DIAG44)
-                asm volatile("diag 0,0,0x44");
-}
-
 void smp_yield_cpu(int cpu)
 {
         if (MACHINE_HAS_DIAG9C)
diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c
index 5b0e445bc3f3..5f63ac5783cb 100644
--- a/arch/s390/lib/spinlock.c
+++ b/arch/s390/lib/spinlock.c
@@ -98,17 +98,6 @@ void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
 }
 EXPORT_SYMBOL(arch_spin_lock_wait_flags);
 
-void arch_spin_relax(arch_spinlock_t *lp)
-{
-        unsigned int cpu = lp->lock;
-        if (cpu != 0) {
-                if (MACHINE_IS_VM || MACHINE_IS_KVM ||
-                    !smp_vcpu_scheduled(~cpu))
-                        smp_yield_cpu(~cpu);
-        }
-}
-EXPORT_SYMBOL(arch_spin_relax);
-
 int arch_spin_trylock_retry(arch_spinlock_t *lp)
 {
         int count;
@@ -122,15 +111,18 @@ EXPORT_SYMBOL(arch_spin_trylock_retry);
 
 void _raw_read_lock_wait(arch_rwlock_t *rw)
 {
-        unsigned int old;
+        unsigned int owner, old;
         int count = spin_retry;
 
+        owner = 0;
         while (1) {
                 if (count-- <= 0) {
-                        smp_yield();
+                        if (owner && !smp_vcpu_scheduled(~owner))
+                                smp_yield_cpu(~owner);
                         count = spin_retry;
                 }
                 old = ACCESS_ONCE(rw->lock);
+                owner = ACCESS_ONCE(rw->owner);
                 if ((int) old < 0)
                         continue;
                 if (_raw_compare_and_swap(&rw->lock, old, old + 1))
@@ -141,16 +133,19 @@ EXPORT_SYMBOL(_raw_read_lock_wait);
 
 void _raw_read_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
 {
-        unsigned int old;
+        unsigned int owner, old;
         int count = spin_retry;
 
         local_irq_restore(flags);
+        owner = 0;
         while (1) {
                 if (count-- <= 0) {
-                        smp_yield();
+                        if (owner && !smp_vcpu_scheduled(~owner))
+                                smp_yield_cpu(~owner);
                         count = spin_retry;
                 }
                 old = ACCESS_ONCE(rw->lock);
+                owner = ACCESS_ONCE(rw->owner);
                 if ((int) old < 0)
                         continue;
                 local_irq_disable();
@@ -179,15 +174,18 @@ EXPORT_SYMBOL(_raw_read_trylock_retry);
 
 void _raw_write_lock_wait(arch_rwlock_t *rw)
 {
-        unsigned int old;
+        unsigned int owner, old;
         int count = spin_retry;
 
+        owner = 0;
         while (1) {
                 if (count-- <= 0) {
-                        smp_yield();
+                        if (owner && !smp_vcpu_scheduled(~owner))
+                                smp_yield_cpu(~owner);
                         count = spin_retry;
                 }
                 old = ACCESS_ONCE(rw->lock);
+                owner = ACCESS_ONCE(rw->owner);
                 if (old)
                         continue;
                 if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
@@ -198,16 +196,19 @@ EXPORT_SYMBOL(_raw_write_lock_wait);
 
 void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
 {
-        unsigned int old;
+        unsigned int owner, old;
         int count = spin_retry;
 
         local_irq_restore(flags);
+        owner = 0;
         while (1) {
                 if (count-- <= 0) {
-                        smp_yield();
+                        if (owner && !smp_vcpu_scheduled(~owner))
+                                smp_yield_cpu(~owner);
                         count = spin_retry;
                 }
                 old = ACCESS_ONCE(rw->lock);
+                owner = ACCESS_ONCE(rw->owner);
                 if (old)
                         continue;
                 local_irq_disable();
@@ -233,3 +234,13 @@ int _raw_write_trylock_retry(arch_rwlock_t *rw)
         return 0;
 }
 EXPORT_SYMBOL(_raw_write_trylock_retry);
+
+void arch_lock_relax(unsigned int cpu)
+{
+        if (!cpu)
+                return;
+        if (MACHINE_IS_LPAR && smp_vcpu_scheduled(~cpu))
+                return;
+        smp_yield_cpu(~cpu);
+}
+EXPORT_SYMBOL(arch_lock_relax);