diff options
-rw-r--r-- | lib/rwsem.c | 28 |
1 files changed, 17 insertions, 11 deletions
diff --git a/lib/rwsem.c b/lib/rwsem.c index 94f2d7a9dc4f..a3e68bf5932e 100644 --- a/lib/rwsem.c +++ b/lib/rwsem.c | |||
@@ -60,7 +60,7 @@ __rwsem_do_wake(struct rw_semaphore *sem, int wake_type) | |||
60 | struct rwsem_waiter *waiter; | 60 | struct rwsem_waiter *waiter; |
61 | struct task_struct *tsk; | 61 | struct task_struct *tsk; |
62 | struct list_head *next; | 62 | struct list_head *next; |
63 | signed long oldcount, woken, loop; | 63 | signed long oldcount, woken, loop, adjustment; |
64 | 64 | ||
65 | waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list); | 65 | waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list); |
66 | if (!(waiter->flags & RWSEM_WAITING_FOR_WRITE)) | 66 | if (!(waiter->flags & RWSEM_WAITING_FOR_WRITE)) |
@@ -73,9 +73,12 @@ __rwsem_do_wake(struct rw_semaphore *sem, int wake_type) | |||
73 | * write lock. However, we only wake this writer if we can transition | 73 | * write lock. However, we only wake this writer if we can transition |
74 | * the active part of the count from 0 -> 1 | 74 | * the active part of the count from 0 -> 1 |
75 | */ | 75 | */ |
76 | adjustment = RWSEM_ACTIVE_WRITE_BIAS; | ||
77 | if (waiter->list.next == &sem->wait_list) | ||
78 | adjustment -= RWSEM_WAITING_BIAS; | ||
79 | |||
76 | try_again_write: | 80 | try_again_write: |
77 | oldcount = rwsem_atomic_update(RWSEM_ACTIVE_BIAS, sem) | 81 | oldcount = rwsem_atomic_update(adjustment, sem) - adjustment; |
78 | - RWSEM_ACTIVE_BIAS; | ||
79 | if (oldcount & RWSEM_ACTIVE_MASK) | 82 | if (oldcount & RWSEM_ACTIVE_MASK) |
80 | /* Someone grabbed the sem already */ | 83 | /* Someone grabbed the sem already */ |
81 | goto undo_write; | 84 | goto undo_write; |
@@ -128,13 +131,15 @@ __rwsem_do_wake(struct rw_semaphore *sem, int wake_type) | |||
128 | 131 | ||
129 | } while (waiter->flags & RWSEM_WAITING_FOR_READ); | 132 | } while (waiter->flags & RWSEM_WAITING_FOR_READ); |
130 | 133 | ||
131 | loop = woken; | 134 | adjustment = woken * RWSEM_ACTIVE_READ_BIAS; |
132 | woken *= RWSEM_ACTIVE_BIAS - RWSEM_WAITING_BIAS; | 135 | if (waiter->flags & RWSEM_WAITING_FOR_READ) |
136 | /* hit end of list above */ | ||
137 | adjustment -= RWSEM_WAITING_BIAS; | ||
133 | 138 | ||
134 | rwsem_atomic_add(woken, sem); | 139 | rwsem_atomic_add(adjustment, sem); |
135 | 140 | ||
136 | next = sem->wait_list.next; | 141 | next = sem->wait_list.next; |
137 | for (; loop > 0; loop--) { | 142 | for (loop = woken; loop > 0; loop--) { |
138 | waiter = list_entry(next, struct rwsem_waiter, list); | 143 | waiter = list_entry(next, struct rwsem_waiter, list); |
139 | next = waiter->list.next; | 144 | next = waiter->list.next; |
140 | tsk = waiter->task; | 145 | tsk = waiter->task; |
@@ -153,7 +158,7 @@ __rwsem_do_wake(struct rw_semaphore *sem, int wake_type) | |||
153 | /* undo the change to the active count, but check for a transition | 158 | /* undo the change to the active count, but check for a transition |
154 | * 1->0 */ | 159 | * 1->0 */ |
155 | undo_write: | 160 | undo_write: |
156 | if (rwsem_atomic_update(-RWSEM_ACTIVE_BIAS, sem) & RWSEM_ACTIVE_MASK) | 161 | if (rwsem_atomic_update(-adjustment, sem) & RWSEM_ACTIVE_MASK) |
157 | goto out; | 162 | goto out; |
158 | goto try_again_write; | 163 | goto try_again_write; |
159 | } | 164 | } |
@@ -175,6 +180,8 @@ rwsem_down_failed_common(struct rw_semaphore *sem, | |||
175 | waiter->task = tsk; | 180 | waiter->task = tsk; |
176 | get_task_struct(tsk); | 181 | get_task_struct(tsk); |
177 | 182 | ||
183 | if (list_empty(&sem->wait_list)) | ||
184 | adjustment += RWSEM_WAITING_BIAS; | ||
178 | list_add_tail(&waiter->list, &sem->wait_list); | 185 | list_add_tail(&waiter->list, &sem->wait_list); |
179 | 186 | ||
180 | /* we're now waiting on the lock, but no longer actively locking */ | 187 | /* we're now waiting on the lock, but no longer actively locking */ |
@@ -208,8 +215,7 @@ rwsem_down_read_failed(struct rw_semaphore *sem) | |||
208 | struct rwsem_waiter waiter; | 215 | struct rwsem_waiter waiter; |
209 | 216 | ||
210 | waiter.flags = RWSEM_WAITING_FOR_READ; | 217 | waiter.flags = RWSEM_WAITING_FOR_READ; |
211 | rwsem_down_failed_common(sem, &waiter, | 218 | rwsem_down_failed_common(sem, &waiter, -RWSEM_ACTIVE_READ_BIAS); |
212 | RWSEM_WAITING_BIAS - RWSEM_ACTIVE_BIAS); | ||
213 | return sem; | 219 | return sem; |
214 | } | 220 | } |
215 | 221 | ||
@@ -222,7 +228,7 @@ rwsem_down_write_failed(struct rw_semaphore *sem) | |||
222 | struct rwsem_waiter waiter; | 228 | struct rwsem_waiter waiter; |
223 | 229 | ||
224 | waiter.flags = RWSEM_WAITING_FOR_WRITE; | 230 | waiter.flags = RWSEM_WAITING_FOR_WRITE; |
225 | rwsem_down_failed_common(sem, &waiter, -RWSEM_ACTIVE_BIAS); | 231 | rwsem_down_failed_common(sem, &waiter, -RWSEM_ACTIVE_WRITE_BIAS); |
226 | 232 | ||
227 | return sem; | 233 | return sem; |
228 | } | 234 | } |