diff options
| author | Ingo Molnar <mingo@elte.hu> | 2009-12-16 12:33:49 -0500 |
|---|---|---|
| committer | Ingo Molnar <mingo@elte.hu> | 2009-12-16 12:33:49 -0500 |
| commit | ee1156c11a1121e118b0a7f2dec240f0d421b1fd (patch) | |
| tree | b8771cc5a9758af9d7410fc519227c036c222130 /kernel | |
| parent | b9f8fcd55bbdb037e5332dbdb7b494f0b70861ac (diff) | |
| parent | 8bea8672edfca7ec5f661cafb218f1205863b343 (diff) | |
Merge branch 'linus' into sched/urgent
Conflicts:
kernel/sched_idletask.c
Merge reason: resolve the conflicts, pick up latest changes.
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'kernel')
53 files changed, 824 insertions, 957 deletions
diff --git a/kernel/acct.c b/kernel/acct.c index 9a4715a2f6bf..a6605ca921b6 100644 --- a/kernel/acct.c +++ b/kernel/acct.c | |||
| @@ -536,7 +536,8 @@ static void do_acct_process(struct bsd_acct_struct *acct, | |||
| 536 | do_div(elapsed, AHZ); | 536 | do_div(elapsed, AHZ); |
| 537 | ac.ac_btime = get_seconds() - elapsed; | 537 | ac.ac_btime = get_seconds() - elapsed; |
| 538 | /* we really need to bite the bullet and change layout */ | 538 | /* we really need to bite the bullet and change layout */ |
| 539 | current_uid_gid(&ac.ac_uid, &ac.ac_gid); | 539 | ac.ac_uid = orig_cred->uid; |
| 540 | ac.ac_gid = orig_cred->gid; | ||
| 540 | #if ACCT_VERSION==2 | 541 | #if ACCT_VERSION==2 |
| 541 | ac.ac_ahz = AHZ; | 542 | ac.ac_ahz = AHZ; |
| 542 | #endif | 543 | #endif |
diff --git a/kernel/exit.c b/kernel/exit.c index 6f50ef55a6f3..5962d7ccf243 100644 --- a/kernel/exit.c +++ b/kernel/exit.c | |||
| @@ -933,7 +933,7 @@ NORET_TYPE void do_exit(long code) | |||
| 933 | * an exiting task cleaning up the robust pi futexes. | 933 | * an exiting task cleaning up the robust pi futexes. |
| 934 | */ | 934 | */ |
| 935 | smp_mb(); | 935 | smp_mb(); |
| 936 | spin_unlock_wait(&tsk->pi_lock); | 936 | raw_spin_unlock_wait(&tsk->pi_lock); |
| 937 | 937 | ||
| 938 | if (unlikely(in_atomic())) | 938 | if (unlikely(in_atomic())) |
| 939 | printk(KERN_INFO "note: %s[%d] exited with preempt_count %d\n", | 939 | printk(KERN_INFO "note: %s[%d] exited with preempt_count %d\n", |
diff --git a/kernel/fork.c b/kernel/fork.c index 1415dc4598ae..9bd91447e052 100644 --- a/kernel/fork.c +++ b/kernel/fork.c | |||
| @@ -939,9 +939,9 @@ SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr) | |||
| 939 | 939 | ||
| 940 | static void rt_mutex_init_task(struct task_struct *p) | 940 | static void rt_mutex_init_task(struct task_struct *p) |
| 941 | { | 941 | { |
| 942 | spin_lock_init(&p->pi_lock); | 942 | raw_spin_lock_init(&p->pi_lock); |
| 943 | #ifdef CONFIG_RT_MUTEXES | 943 | #ifdef CONFIG_RT_MUTEXES |
| 944 | plist_head_init(&p->pi_waiters, &p->pi_lock); | 944 | plist_head_init_raw(&p->pi_waiters, &p->pi_lock); |
| 945 | p->pi_blocked_on = NULL; | 945 | p->pi_blocked_on = NULL; |
| 946 | #endif | 946 | #endif |
| 947 | } | 947 | } |
diff --git a/kernel/futex.c b/kernel/futex.c index d73ef1f3e55d..8e3c3ffe1b9a 100644 --- a/kernel/futex.c +++ b/kernel/futex.c | |||
| @@ -403,9 +403,9 @@ static void free_pi_state(struct futex_pi_state *pi_state) | |||
| 403 | * and has cleaned up the pi_state already | 403 | * and has cleaned up the pi_state already |
| 404 | */ | 404 | */ |
| 405 | if (pi_state->owner) { | 405 | if (pi_state->owner) { |
| 406 | spin_lock_irq(&pi_state->owner->pi_lock); | 406 | raw_spin_lock_irq(&pi_state->owner->pi_lock); |
| 407 | list_del_init(&pi_state->list); | 407 | list_del_init(&pi_state->list); |
| 408 | spin_unlock_irq(&pi_state->owner->pi_lock); | 408 | raw_spin_unlock_irq(&pi_state->owner->pi_lock); |
| 409 | 409 | ||
| 410 | rt_mutex_proxy_unlock(&pi_state->pi_mutex, pi_state->owner); | 410 | rt_mutex_proxy_unlock(&pi_state->pi_mutex, pi_state->owner); |
| 411 | } | 411 | } |
| @@ -470,18 +470,18 @@ void exit_pi_state_list(struct task_struct *curr) | |||
| 470 | * pi_state_list anymore, but we have to be careful | 470 | * pi_state_list anymore, but we have to be careful |
| 471 | * versus waiters unqueueing themselves: | 471 | * versus waiters unqueueing themselves: |
| 472 | */ | 472 | */ |
| 473 | spin_lock_irq(&curr->pi_lock); | 473 | raw_spin_lock_irq(&curr->pi_lock); |
| 474 | while (!list_empty(head)) { | 474 | while (!list_empty(head)) { |
| 475 | 475 | ||
| 476 | next = head->next; | 476 | next = head->next; |
| 477 | pi_state = list_entry(next, struct futex_pi_state, list); | 477 | pi_state = list_entry(next, struct futex_pi_state, list); |
| 478 | key = pi_state->key; | 478 | key = pi_state->key; |
| 479 | hb = hash_futex(&key); | 479 | hb = hash_futex(&key); |
| 480 | spin_unlock_irq(&curr->pi_lock); | 480 | raw_spin_unlock_irq(&curr->pi_lock); |
| 481 | 481 | ||
| 482 | spin_lock(&hb->lock); | 482 | spin_lock(&hb->lock); |
| 483 | 483 | ||
| 484 | spin_lock_irq(&curr->pi_lock); | 484 | raw_spin_lock_irq(&curr->pi_lock); |
| 485 | /* | 485 | /* |
| 486 | * We dropped the pi-lock, so re-check whether this | 486 | * We dropped the pi-lock, so re-check whether this |
| 487 | * task still owns the PI-state: | 487 | * task still owns the PI-state: |
| @@ -495,15 +495,15 @@ void exit_pi_state_list(struct task_struct *curr) | |||
| 495 | WARN_ON(list_empty(&pi_state->list)); | 495 | WARN_ON(list_empty(&pi_state->list)); |
| 496 | list_del_init(&pi_state->list); | 496 | list_del_init(&pi_state->list); |
| 497 | pi_state->owner = NULL; | 497 | pi_state->owner = NULL; |
| 498 | spin_unlock_irq(&curr->pi_lock); | 498 | raw_spin_unlock_irq(&curr->pi_lock); |
| 499 | 499 | ||
| 500 | rt_mutex_unlock(&pi_state->pi_mutex); | 500 | rt_mutex_unlock(&pi_state->pi_mutex); |
| 501 | 501 | ||
| 502 | spin_unlock(&hb->lock); | 502 | spin_unlock(&hb->lock); |
| 503 | 503 | ||
| 504 | spin_lock_irq(&curr->pi_lock); | 504 | raw_spin_lock_irq(&curr->pi_lock); |
| 505 | } | 505 | } |
| 506 | spin_unlock_irq(&curr->pi_lock); | 506 | raw_spin_unlock_irq(&curr->pi_lock); |
| 507 | } | 507 | } |
| 508 | 508 | ||
| 509 | static int | 509 | static int |
| @@ -558,7 +558,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb, | |||
| 558 | * change of the task flags, we do this protected by | 558 | * change of the task flags, we do this protected by |
| 559 | * p->pi_lock: | 559 | * p->pi_lock: |
| 560 | */ | 560 | */ |
| 561 | spin_lock_irq(&p->pi_lock); | 561 | raw_spin_lock_irq(&p->pi_lock); |
| 562 | if (unlikely(p->flags & PF_EXITING)) { | 562 | if (unlikely(p->flags & PF_EXITING)) { |
| 563 | /* | 563 | /* |
| 564 | * The task is on the way out. When PF_EXITPIDONE is | 564 | * The task is on the way out. When PF_EXITPIDONE is |
| @@ -567,7 +567,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb, | |||
| 567 | */ | 567 | */ |
| 568 | int ret = (p->flags & PF_EXITPIDONE) ? -ESRCH : -EAGAIN; | 568 | int ret = (p->flags & PF_EXITPIDONE) ? -ESRCH : -EAGAIN; |
| 569 | 569 | ||
| 570 | spin_unlock_irq(&p->pi_lock); | 570 | raw_spin_unlock_irq(&p->pi_lock); |
| 571 | put_task_struct(p); | 571 | put_task_struct(p); |
| 572 | return ret; | 572 | return ret; |
| 573 | } | 573 | } |
| @@ -586,7 +586,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb, | |||
| 586 | WARN_ON(!list_empty(&pi_state->list)); | 586 | WARN_ON(!list_empty(&pi_state->list)); |
| 587 | list_add(&pi_state->list, &p->pi_state_list); | 587 | list_add(&pi_state->list, &p->pi_state_list); |
| 588 | pi_state->owner = p; | 588 | pi_state->owner = p; |
| 589 | spin_unlock_irq(&p->pi_lock); | 589 | raw_spin_unlock_irq(&p->pi_lock); |
| 590 | 590 | ||
| 591 | put_task_struct(p); | 591 | put_task_struct(p); |
| 592 | 592 | ||
| @@ -760,7 +760,7 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this) | |||
| 760 | if (!pi_state) | 760 | if (!pi_state) |
| 761 | return -EINVAL; | 761 | return -EINVAL; |
| 762 | 762 | ||
| 763 | spin_lock(&pi_state->pi_mutex.wait_lock); | 763 | raw_spin_lock(&pi_state->pi_mutex.wait_lock); |
| 764 | new_owner = rt_mutex_next_owner(&pi_state->pi_mutex); | 764 | new_owner = rt_mutex_next_owner(&pi_state->pi_mutex); |
| 765 | 765 | ||
| 766 | /* | 766 | /* |
| @@ -789,23 +789,23 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this) | |||
| 789 | else if (curval != uval) | 789 | else if (curval != uval) |
| 790 | ret = -EINVAL; | 790 | ret = -EINVAL; |
| 791 | if (ret) { | 791 | if (ret) { |
| 792 | spin_unlock(&pi_state->pi_mutex.wait_lock); | 792 | raw_spin_unlock(&pi_state->pi_mutex.wait_lock); |
| 793 | return ret; | 793 | return ret; |
| 794 | } | 794 | } |
| 795 | } | 795 | } |
| 796 | 796 | ||
| 797 | spin_lock_irq(&pi_state->owner->pi_lock); | 797 | raw_spin_lock_irq(&pi_state->owner->pi_lock); |
| 798 | WARN_ON(list_empty(&pi_state->list)); | 798 | WARN_ON(list_empty(&pi_state->list)); |
| 799 | list_del_init(&pi_state->list); | 799 | list_del_init(&pi_state->list); |
| 800 | spin_unlock_irq(&pi_state->owner->pi_lock); | 800 | raw_spin_unlock_irq(&pi_state->owner->pi_lock); |
| 801 | 801 | ||
| 802 | spin_lock_irq(&new_owner->pi_lock); | 802 | raw_spin_lock_irq(&new_owner->pi_lock); |
| 803 | WARN_ON(!list_empty(&pi_state->list)); | 803 | WARN_ON(!list_empty(&pi_state->list)); |
| 804 | list_add(&pi_state->list, &new_owner->pi_state_list); | 804 | list_add(&pi_state->list, &new_owner->pi_state_list); |
| 805 | pi_state->owner = new_owner; | 805 | pi_state->owner = new_owner; |
| 806 | spin_unlock_irq(&new_owner->pi_lock); | 806 | raw_spin_unlock_irq(&new_owner->pi_lock); |
| 807 | 807 | ||
| 808 | spin_unlock(&pi_state->pi_mutex.wait_lock); | 808 | raw_spin_unlock(&pi_state->pi_mutex.wait_lock); |
| 809 | rt_mutex_unlock(&pi_state->pi_mutex); | 809 | rt_mutex_unlock(&pi_state->pi_mutex); |
| 810 | 810 | ||
| 811 | return 0; | 811 | return 0; |
| @@ -1010,7 +1010,7 @@ void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1, | |||
| 1010 | plist_add(&q->list, &hb2->chain); | 1010 | plist_add(&q->list, &hb2->chain); |
| 1011 | q->lock_ptr = &hb2->lock; | 1011 | q->lock_ptr = &hb2->lock; |
| 1012 | #ifdef CONFIG_DEBUG_PI_LIST | 1012 | #ifdef CONFIG_DEBUG_PI_LIST |
| 1013 | q->list.plist.lock = &hb2->lock; | 1013 | q->list.plist.spinlock = &hb2->lock; |
| 1014 | #endif | 1014 | #endif |
| 1015 | } | 1015 | } |
| 1016 | get_futex_key_refs(key2); | 1016 | get_futex_key_refs(key2); |
| @@ -1046,7 +1046,7 @@ void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key, | |||
| 1046 | 1046 | ||
| 1047 | q->lock_ptr = &hb->lock; | 1047 | q->lock_ptr = &hb->lock; |
| 1048 | #ifdef CONFIG_DEBUG_PI_LIST | 1048 | #ifdef CONFIG_DEBUG_PI_LIST |
| 1049 | q->list.plist.lock = &hb->lock; | 1049 | q->list.plist.spinlock = &hb->lock; |
| 1050 | #endif | 1050 | #endif |
| 1051 | 1051 | ||
| 1052 | wake_up_state(q->task, TASK_NORMAL); | 1052 | wake_up_state(q->task, TASK_NORMAL); |
| @@ -1394,7 +1394,7 @@ static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb) | |||
| 1394 | 1394 | ||
| 1395 | plist_node_init(&q->list, prio); | 1395 | plist_node_init(&q->list, prio); |
| 1396 | #ifdef CONFIG_DEBUG_PI_LIST | 1396 | #ifdef CONFIG_DEBUG_PI_LIST |
| 1397 | q->list.plist.lock = &hb->lock; | 1397 | q->list.plist.spinlock = &hb->lock; |
| 1398 | #endif | 1398 | #endif |
| 1399 | plist_add(&q->list, &hb->chain); | 1399 | plist_add(&q->list, &hb->chain); |
| 1400 | q->task = current; | 1400 | q->task = current; |
| @@ -1529,18 +1529,18 @@ retry: | |||
| 1529 | * itself. | 1529 | * itself. |
| 1530 | */ | 1530 | */ |
| 1531 | if (pi_state->owner != NULL) { | 1531 | if (pi_state->owner != NULL) { |
| 1532 | spin_lock_irq(&pi_state->owner->pi_lock); | 1532 | raw_spin_lock_irq(&pi_state->owner->pi_lock); |
| 1533 | WARN_ON(list_empty(&pi_state->list)); | 1533 | WARN_ON(list_empty(&pi_state->list)); |
| 1534 | list_del_init(&pi_state->list); | 1534 | list_del_init(&pi_state->list); |
| 1535 | spin_unlock_irq(&pi_state->owner->pi_lock); | 1535 | raw_spin_unlock_irq(&pi_state->owner->pi_lock); |
| 1536 | } | 1536 | } |
| 1537 | 1537 | ||
| 1538 | pi_state->owner = newowner; | 1538 | pi_state->owner = newowner; |
| 1539 | 1539 | ||
| 1540 | spin_lock_irq(&newowner->pi_lock); | 1540 | raw_spin_lock_irq(&newowner->pi_lock); |
| 1541 | WARN_ON(!list_empty(&pi_state->list)); | 1541 | WARN_ON(!list_empty(&pi_state->list)); |
| 1542 | list_add(&pi_state->list, &newowner->pi_state_list); | 1542 | list_add(&pi_state->list, &newowner->pi_state_list); |
| 1543 | spin_unlock_irq(&newowner->pi_lock); | 1543 | raw_spin_unlock_irq(&newowner->pi_lock); |
| 1544 | return 0; | 1544 | return 0; |
| 1545 | 1545 | ||
| 1546 | /* | 1546 | /* |
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index d2f9239dc6ba..0086628b6e97 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c | |||
| @@ -127,11 +127,11 @@ struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, | |||
| 127 | for (;;) { | 127 | for (;;) { |
| 128 | base = timer->base; | 128 | base = timer->base; |
| 129 | if (likely(base != NULL)) { | 129 | if (likely(base != NULL)) { |
| 130 | spin_lock_irqsave(&base->cpu_base->lock, *flags); | 130 | raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); |
| 131 | if (likely(base == timer->base)) | 131 | if (likely(base == timer->base)) |
| 132 | return base; | 132 | return base; |
| 133 | /* The timer has migrated to another CPU: */ | 133 | /* The timer has migrated to another CPU: */ |
| 134 | spin_unlock_irqrestore(&base->cpu_base->lock, *flags); | 134 | raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags); |
| 135 | } | 135 | } |
| 136 | cpu_relax(); | 136 | cpu_relax(); |
| 137 | } | 137 | } |
| @@ -208,13 +208,13 @@ again: | |||
| 208 | 208 | ||
| 209 | /* See the comment in lock_timer_base() */ | 209 | /* See the comment in lock_timer_base() */ |
| 210 | timer->base = NULL; | 210 | timer->base = NULL; |
| 211 | spin_unlock(&base->cpu_base->lock); | 211 | raw_spin_unlock(&base->cpu_base->lock); |
| 212 | spin_lock(&new_base->cpu_base->lock); | 212 | raw_spin_lock(&new_base->cpu_base->lock); |
| 213 | 213 | ||
| 214 | if (cpu != this_cpu && hrtimer_check_target(timer, new_base)) { | 214 | if (cpu != this_cpu && hrtimer_check_target(timer, new_base)) { |
| 215 | cpu = this_cpu; | 215 | cpu = this_cpu; |
| 216 | spin_unlock(&new_base->cpu_base->lock); | 216 | raw_spin_unlock(&new_base->cpu_base->lock); |
| 217 | spin_lock(&base->cpu_base->lock); | 217 | raw_spin_lock(&base->cpu_base->lock); |
| 218 | timer->base = base; | 218 | timer->base = base; |
| 219 | goto again; | 219 | goto again; |
| 220 | } | 220 | } |
| @@ -230,7 +230,7 @@ lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) | |||
| 230 | { | 230 | { |
| 231 | struct hrtimer_clock_base *base = timer->base; | 231 | struct hrtimer_clock_base *base = timer->base; |
| 232 | 232 | ||
| 233 | spin_lock_irqsave(&base->cpu_base->lock, *flags); | 233 | raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); |
| 234 | 234 | ||
| 235 | return base; | 235 | return base; |
| 236 | } | 236 | } |
| @@ -628,12 +628,12 @@ static void retrigger_next_event(void *arg) | |||
| 628 | base = &__get_cpu_var(hrtimer_bases); | 628 | base = &__get_cpu_var(hrtimer_bases); |
| 629 | 629 | ||
| 630 | /* Adjust CLOCK_REALTIME offset */ | 630 | /* Adjust CLOCK_REALTIME offset */ |
| 631 | spin_lock(&base->lock); | 631 | raw_spin_lock(&base->lock); |
| 632 | base->clock_base[CLOCK_REALTIME].offset = | 632 | base->clock_base[CLOCK_REALTIME].offset = |
| 633 | timespec_to_ktime(realtime_offset); | 633 | timespec_to_ktime(realtime_offset); |
| 634 | 634 | ||
| 635 | hrtimer_force_reprogram(base, 0); | 635 | hrtimer_force_reprogram(base, 0); |
| 636 | spin_unlock(&base->lock); | 636 | raw_spin_unlock(&base->lock); |
| 637 | } | 637 | } |
| 638 | 638 | ||
| 639 | /* | 639 | /* |
| @@ -694,9 +694,9 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, | |||
| 694 | { | 694 | { |
| 695 | if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) { | 695 | if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) { |
| 696 | if (wakeup) { | 696 | if (wakeup) { |
| 697 | spin_unlock(&base->cpu_base->lock); | 697 | raw_spin_unlock(&base->cpu_base->lock); |
| 698 | raise_softirq_irqoff(HRTIMER_SOFTIRQ); | 698 | raise_softirq_irqoff(HRTIMER_SOFTIRQ); |
| 699 | spin_lock(&base->cpu_base->lock); | 699 | raw_spin_lock(&base->cpu_base->lock); |
| 700 | } else | 700 | } else |
| 701 | __raise_softirq_irqoff(HRTIMER_SOFTIRQ); | 701 | __raise_softirq_irqoff(HRTIMER_SOFTIRQ); |
| 702 | 702 | ||
| @@ -790,7 +790,7 @@ static inline void timer_stats_account_hrtimer(struct hrtimer *timer) | |||
| 790 | static inline | 790 | static inline |
| 791 | void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) | 791 | void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) |
| 792 | { | 792 | { |
| 793 | spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); | 793 | raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); |
| 794 | } | 794 | } |
| 795 | 795 | ||
| 796 | /** | 796 | /** |
| @@ -1123,7 +1123,7 @@ ktime_t hrtimer_get_next_event(void) | |||
| 1123 | unsigned long flags; | 1123 | unsigned long flags; |
| 1124 | int i; | 1124 | int i; |
| 1125 | 1125 | ||
| 1126 | spin_lock_irqsave(&cpu_base->lock, flags); | 1126 | raw_spin_lock_irqsave(&cpu_base->lock, flags); |
| 1127 | 1127 | ||
| 1128 | if (!hrtimer_hres_active()) { | 1128 | if (!hrtimer_hres_active()) { |
| 1129 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) { | 1129 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) { |
| @@ -1140,7 +1140,7 @@ ktime_t hrtimer_get_next_event(void) | |||
| 1140 | } | 1140 | } |
| 1141 | } | 1141 | } |
| 1142 | 1142 | ||
| 1143 | spin_unlock_irqrestore(&cpu_base->lock, flags); | 1143 | raw_spin_unlock_irqrestore(&cpu_base->lock, flags); |
| 1144 | 1144 | ||
| 1145 | if (mindelta.tv64 < 0) | 1145 | if (mindelta.tv64 < 0) |
| 1146 | mindelta.tv64 = 0; | 1146 | mindelta.tv64 = 0; |
| @@ -1222,11 +1222,11 @@ static void __run_hrtimer(struct hrtimer *timer, ktime_t *now) | |||
| 1222 | * they get migrated to another cpu, therefore its safe to unlock | 1222 | * they get migrated to another cpu, therefore its safe to unlock |
| 1223 | * the timer base. | 1223 | * the timer base. |
| 1224 | */ | 1224 | */ |
| 1225 | spin_unlock(&cpu_base->lock); | 1225 | raw_spin_unlock(&cpu_base->lock); |
| 1226 | trace_hrtimer_expire_entry(timer, now); | 1226 | trace_hrtimer_expire_entry(timer, now); |
| 1227 | restart = fn(timer); | 1227 | restart = fn(timer); |
| 1228 | trace_hrtimer_expire_exit(timer); | 1228 | trace_hrtimer_expire_exit(timer); |
| 1229 | spin_lock(&cpu_base->lock); | 1229 | raw_spin_lock(&cpu_base->lock); |
| 1230 | 1230 | ||
| 1231 | /* | 1231 | /* |
| 1232 | * Note: We clear the CALLBACK bit after enqueue_hrtimer and | 1232 | * Note: We clear the CALLBACK bit after enqueue_hrtimer and |
| @@ -1261,7 +1261,7 @@ void hrtimer_interrupt(struct clock_event_device *dev) | |||
| 1261 | retry: | 1261 | retry: |
| 1262 | expires_next.tv64 = KTIME_MAX; | 1262 | expires_next.tv64 = KTIME_MAX; |
| 1263 | 1263 | ||
| 1264 | spin_lock(&cpu_base->lock); | 1264 | raw_spin_lock(&cpu_base->lock); |
| 1265 | /* | 1265 | /* |
| 1266 | * We set expires_next to KTIME_MAX here with cpu_base->lock | 1266 | * We set expires_next to KTIME_MAX here with cpu_base->lock |
| 1267 | * held to prevent that a timer is enqueued in our queue via | 1267 | * held to prevent that a timer is enqueued in our queue via |
| @@ -1317,7 +1317,7 @@ retry: | |||
| 1317 | * against it. | 1317 | * against it. |
| 1318 | */ | 1318 | */ |
| 1319 | cpu_base->expires_next = expires_next; | 1319 | cpu_base->expires_next = expires_next; |
| 1320 | spin_unlock(&cpu_base->lock); | 1320 | raw_spin_unlock(&cpu_base->lock); |
| 1321 | 1321 | ||
| 1322 | /* Reprogramming necessary ? */ | 1322 | /* Reprogramming necessary ? */ |
| 1323 | if (expires_next.tv64 == KTIME_MAX || | 1323 | if (expires_next.tv64 == KTIME_MAX || |
| @@ -1457,7 +1457,7 @@ void hrtimer_run_queues(void) | |||
| 1457 | gettime = 0; | 1457 | gettime = 0; |
| 1458 | } | 1458 | } |
| 1459 | 1459 | ||
| 1460 | spin_lock(&cpu_base->lock); | 1460 | raw_spin_lock(&cpu_base->lock); |
| 1461 | 1461 | ||
| 1462 | while ((node = base->first)) { | 1462 | while ((node = base->first)) { |
| 1463 | struct hrtimer *timer; | 1463 | struct hrtimer *timer; |
| @@ -1469,7 +1469,7 @@ void hrtimer_run_queues(void) | |||
| 1469 | 1469 | ||
| 1470 | __run_hrtimer(timer, &base->softirq_time); | 1470 | __run_hrtimer(timer, &base->softirq_time); |
| 1471 | } | 1471 | } |
| 1472 | spin_unlock(&cpu_base->lock); | 1472 | raw_spin_unlock(&cpu_base->lock); |
| 1473 | } | 1473 | } |
| 1474 | } | 1474 | } |
| 1475 | 1475 | ||
| @@ -1625,7 +1625,7 @@ static void __cpuinit init_hrtimers_cpu(int cpu) | |||
| 1625 | struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); | 1625 | struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); |
| 1626 | int i; | 1626 | int i; |
| 1627 | 1627 | ||
| 1628 | spin_lock_init(&cpu_base->lock); | 1628 | raw_spin_lock_init(&cpu_base->lock); |
| 1629 | 1629 | ||
| 1630 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) | 1630 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) |
| 1631 | cpu_base->clock_base[i].cpu_base = cpu_base; | 1631 | cpu_base->clock_base[i].cpu_base = cpu_base; |
| @@ -1683,16 +1683,16 @@ static void migrate_hrtimers(int scpu) | |||
| 1683 | * The caller is globally serialized and nobody else | 1683 | * The caller is globally serialized and nobody else |
| 1684 | * takes two locks at once, deadlock is not possible. | 1684 | * takes two locks at once, deadlock is not possible. |
| 1685 | */ | 1685 | */ |
| 1686 | spin_lock(&new_base->lock); | 1686 | raw_spin_lock(&new_base->lock); |
| 1687 | spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); | 1687 | raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); |
| 1688 | 1688 | ||
| 1689 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { | 1689 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { |
| 1690 | migrate_hrtimer_list(&old_base->clock_base[i], | 1690 | migrate_hrtimer_list(&old_base->clock_base[i], |
| 1691 | &new_base->clock_base[i]); | 1691 | &new_base->clock_base[i]); |
| 1692 | } | 1692 | } |
| 1693 | 1693 | ||
| 1694 | spin_unlock(&old_base->lock); | 1694 | raw_spin_unlock(&old_base->lock); |
| 1695 | spin_unlock(&new_base->lock); | 1695 | raw_spin_unlock(&new_base->lock); |
| 1696 | 1696 | ||
| 1697 | /* Check, if we got expired work to do */ | 1697 | /* Check, if we got expired work to do */ |
| 1698 | __hrtimer_peek_ahead_timers(); | 1698 | __hrtimer_peek_ahead_timers(); |
diff --git a/kernel/hw_breakpoint.c b/kernel/hw_breakpoint.c index 366eedf949c0..dbcbf6a33a08 100644 --- a/kernel/hw_breakpoint.c +++ b/kernel/hw_breakpoint.c | |||
| @@ -96,7 +96,7 @@ static int task_bp_pinned(struct task_struct *tsk) | |||
| 96 | 96 | ||
| 97 | list = &ctx->event_list; | 97 | list = &ctx->event_list; |
| 98 | 98 | ||
| 99 | spin_lock_irqsave(&ctx->lock, flags); | 99 | raw_spin_lock_irqsave(&ctx->lock, flags); |
| 100 | 100 | ||
| 101 | /* | 101 | /* |
| 102 | * The current breakpoint counter is not included in the list | 102 | * The current breakpoint counter is not included in the list |
| @@ -107,7 +107,7 @@ static int task_bp_pinned(struct task_struct *tsk) | |||
| 107 | count++; | 107 | count++; |
| 108 | } | 108 | } |
| 109 | 109 | ||
| 110 | spin_unlock_irqrestore(&ctx->lock, flags); | 110 | raw_spin_unlock_irqrestore(&ctx->lock, flags); |
| 111 | 111 | ||
| 112 | return count; | 112 | return count; |
| 113 | } | 113 | } |
diff --git a/kernel/irq/autoprobe.c b/kernel/irq/autoprobe.c index 1de9700f416e..2295a31ef110 100644 --- a/kernel/irq/autoprobe.c +++ b/kernel/irq/autoprobe.c | |||
| @@ -45,7 +45,7 @@ unsigned long probe_irq_on(void) | |||
| 45 | * flush such a longstanding irq before considering it as spurious. | 45 | * flush such a longstanding irq before considering it as spurious. |
| 46 | */ | 46 | */ |
| 47 | for_each_irq_desc_reverse(i, desc) { | 47 | for_each_irq_desc_reverse(i, desc) { |
| 48 | spin_lock_irq(&desc->lock); | 48 | raw_spin_lock_irq(&desc->lock); |
| 49 | if (!desc->action && !(desc->status & IRQ_NOPROBE)) { | 49 | if (!desc->action && !(desc->status & IRQ_NOPROBE)) { |
| 50 | /* | 50 | /* |
| 51 | * An old-style architecture might still have | 51 | * An old-style architecture might still have |
| @@ -61,7 +61,7 @@ unsigned long probe_irq_on(void) | |||
| 61 | desc->chip->set_type(i, IRQ_TYPE_PROBE); | 61 | desc->chip->set_type(i, IRQ_TYPE_PROBE); |
| 62 | desc->chip->startup(i); | 62 | desc->chip->startup(i); |
| 63 | } | 63 | } |
| 64 | spin_unlock_irq(&desc->lock); | 64 | raw_spin_unlock_irq(&desc->lock); |
| 65 | } | 65 | } |
| 66 | 66 | ||
| 67 | /* Wait for longstanding interrupts to trigger. */ | 67 | /* Wait for longstanding interrupts to trigger. */ |
| @@ -73,13 +73,13 @@ unsigned long probe_irq_on(void) | |||
| 73 | * happened in the previous stage, it may have masked itself) | 73 | * happened in the previous stage, it may have masked itself) |
| 74 | */ | 74 | */ |
| 75 | for_each_irq_desc_reverse(i, desc) { | 75 | for_each_irq_desc_reverse(i, desc) { |
| 76 | spin_lock_irq(&desc->lock); | 76 | raw_spin_lock_irq(&desc->lock); |
| 77 | if (!desc->action && !(desc->status & IRQ_NOPROBE)) { | 77 | if (!desc->action && !(desc->status & IRQ_NOPROBE)) { |
| 78 | desc->status |= IRQ_AUTODETECT | IRQ_WAITING; | 78 | desc->status |= IRQ_AUTODETECT | IRQ_WAITING; |
| 79 | if (desc->chip->startup(i)) | 79 | if (desc->chip->startup(i)) |
| 80 | desc->status |= IRQ_PENDING; | 80 | desc->status |= IRQ_PENDING; |
| 81 | } | 81 | } |
| 82 | spin_unlock_irq(&desc->lock); | 82 | raw_spin_unlock_irq(&desc->lock); |
| 83 | } | 83 | } |
| 84 | 84 | ||
| 85 | /* | 85 | /* |
| @@ -91,7 +91,7 @@ unsigned long probe_irq_on(void) | |||
| 91 | * Now filter out any obviously spurious interrupts | 91 | * Now filter out any obviously spurious interrupts |
| 92 | */ | 92 | */ |
| 93 | for_each_irq_desc(i, desc) { | 93 | for_each_irq_desc(i, desc) { |
| 94 | spin_lock_irq(&desc->lock); | 94 | raw_spin_lock_irq(&desc->lock); |
| 95 | status = desc->status; | 95 | status = desc->status; |
| 96 | 96 | ||
| 97 | if (status & IRQ_AUTODETECT) { | 97 | if (status & IRQ_AUTODETECT) { |
| @@ -103,7 +103,7 @@ unsigned long probe_irq_on(void) | |||
| 103 | if (i < 32) | 103 | if (i < 32) |
| 104 | mask |= 1 << i; | 104 | mask |= 1 << i; |
| 105 | } | 105 | } |
| 106 | spin_unlock_irq(&desc->lock); | 106 | raw_spin_unlock_irq(&desc->lock); |
| 107 | } | 107 | } |
| 108 | 108 | ||
| 109 | return mask; | 109 | return mask; |
| @@ -129,7 +129,7 @@ unsigned int probe_irq_mask(unsigned long val) | |||
| 129 | int i; | 129 | int i; |
| 130 | 130 | ||
| 131 | for_each_irq_desc(i, desc) { | 131 | for_each_irq_desc(i, desc) { |
| 132 | spin_lock_irq(&desc->lock); | 132 | raw_spin_lock_irq(&desc->lock); |
| 133 | status = desc->status; | 133 | status = desc->status; |
| 134 | 134 | ||
| 135 | if (status & IRQ_AUTODETECT) { | 135 | if (status & IRQ_AUTODETECT) { |
| @@ -139,7 +139,7 @@ unsigned int probe_irq_mask(unsigned long val) | |||
| 139 | desc->status = status & ~IRQ_AUTODETECT; | 139 | desc->status = status & ~IRQ_AUTODETECT; |
| 140 | desc->chip->shutdown(i); | 140 | desc->chip->shutdown(i); |
| 141 | } | 141 | } |
| 142 | spin_unlock_irq(&desc->lock); | 142 | raw_spin_unlock_irq(&desc->lock); |
| 143 | } | 143 | } |
| 144 | mutex_unlock(&probing_active); | 144 | mutex_unlock(&probing_active); |
| 145 | 145 | ||
| @@ -171,7 +171,7 @@ int probe_irq_off(unsigned long val) | |||
| 171 | unsigned int status; | 171 | unsigned int status; |
| 172 | 172 | ||
| 173 | for_each_irq_desc(i, desc) { | 173 | for_each_irq_desc(i, desc) { |
| 174 | spin_lock_irq(&desc->lock); | 174 | raw_spin_lock_irq(&desc->lock); |
| 175 | status = desc->status; | 175 | status = desc->status; |
| 176 | 176 | ||
| 177 | if (status & IRQ_AUTODETECT) { | 177 | if (status & IRQ_AUTODETECT) { |
| @@ -183,7 +183,7 @@ int probe_irq_off(unsigned long val) | |||
| 183 | desc->status = status & ~IRQ_AUTODETECT; | 183 | desc->status = status & ~IRQ_AUTODETECT; |
| 184 | desc->chip->shutdown(i); | 184 | desc->chip->shutdown(i); |
| 185 | } | 185 | } |
| 186 | spin_unlock_irq(&desc->lock); | 186 | raw_spin_unlock_irq(&desc->lock); |
| 187 | } | 187 | } |
| 188 | mutex_unlock(&probing_active); | 188 | mutex_unlock(&probing_active); |
| 189 | 189 | ||
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index ba566c261adc..ecc3fa28f666 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c | |||
| @@ -34,7 +34,7 @@ void dynamic_irq_init(unsigned int irq) | |||
| 34 | } | 34 | } |
| 35 | 35 | ||
| 36 | /* Ensure we don't have left over values from a previous use of this irq */ | 36 | /* Ensure we don't have left over values from a previous use of this irq */ |
| 37 | spin_lock_irqsave(&desc->lock, flags); | 37 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 38 | desc->status = IRQ_DISABLED; | 38 | desc->status = IRQ_DISABLED; |
| 39 | desc->chip = &no_irq_chip; | 39 | desc->chip = &no_irq_chip; |
| 40 | desc->handle_irq = handle_bad_irq; | 40 | desc->handle_irq = handle_bad_irq; |
| @@ -51,7 +51,7 @@ void dynamic_irq_init(unsigned int irq) | |||
| 51 | cpumask_clear(desc->pending_mask); | 51 | cpumask_clear(desc->pending_mask); |
| 52 | #endif | 52 | #endif |
| 53 | #endif | 53 | #endif |
| 54 | spin_unlock_irqrestore(&desc->lock, flags); | 54 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 55 | } | 55 | } |
| 56 | 56 | ||
| 57 | /** | 57 | /** |
| @@ -68,9 +68,9 @@ void dynamic_irq_cleanup(unsigned int irq) | |||
| 68 | return; | 68 | return; |
| 69 | } | 69 | } |
| 70 | 70 | ||
| 71 | spin_lock_irqsave(&desc->lock, flags); | 71 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 72 | if (desc->action) { | 72 | if (desc->action) { |
| 73 | spin_unlock_irqrestore(&desc->lock, flags); | 73 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 74 | WARN(1, KERN_ERR "Destroying IRQ%d without calling free_irq\n", | 74 | WARN(1, KERN_ERR "Destroying IRQ%d without calling free_irq\n", |
| 75 | irq); | 75 | irq); |
| 76 | return; | 76 | return; |
| @@ -82,7 +82,7 @@ void dynamic_irq_cleanup(unsigned int irq) | |||
| 82 | desc->chip = &no_irq_chip; | 82 | desc->chip = &no_irq_chip; |
| 83 | desc->name = NULL; | 83 | desc->name = NULL; |
| 84 | clear_kstat_irqs(desc); | 84 | clear_kstat_irqs(desc); |
| 85 | spin_unlock_irqrestore(&desc->lock, flags); | 85 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 86 | } | 86 | } |
| 87 | 87 | ||
| 88 | 88 | ||
| @@ -104,10 +104,10 @@ int set_irq_chip(unsigned int irq, struct irq_chip *chip) | |||
| 104 | if (!chip) | 104 | if (!chip) |
| 105 | chip = &no_irq_chip; | 105 | chip = &no_irq_chip; |
| 106 | 106 | ||
| 107 | spin_lock_irqsave(&desc->lock, flags); | 107 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 108 | irq_chip_set_defaults(chip); | 108 | irq_chip_set_defaults(chip); |
| 109 | desc->chip = chip; | 109 | desc->chip = chip; |
| 110 | spin_unlock_irqrestore(&desc->lock, flags); | 110 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 111 | 111 | ||
| 112 | return 0; | 112 | return 0; |
| 113 | } | 113 | } |
| @@ -133,9 +133,9 @@ int set_irq_type(unsigned int irq, unsigned int type) | |||
| 133 | if (type == IRQ_TYPE_NONE) | 133 | if (type == IRQ_TYPE_NONE) |
| 134 | return 0; | 134 | return 0; |
| 135 | 135 | ||
| 136 | spin_lock_irqsave(&desc->lock, flags); | 136 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 137 | ret = __irq_set_trigger(desc, irq, type); | 137 | ret = __irq_set_trigger(desc, irq, type); |
| 138 | spin_unlock_irqrestore(&desc->lock, flags); | 138 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 139 | return ret; | 139 | return ret; |
| 140 | } | 140 | } |
| 141 | EXPORT_SYMBOL(set_irq_type); | 141 | EXPORT_SYMBOL(set_irq_type); |
| @@ -158,9 +158,9 @@ int set_irq_data(unsigned int irq, void *data) | |||
| 158 | return -EINVAL; | 158 | return -EINVAL; |
| 159 | } | 159 | } |
| 160 | 160 | ||
| 161 | spin_lock_irqsave(&desc->lock, flags); | 161 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 162 | desc->handler_data = data; | 162 | desc->handler_data = data; |
| 163 | spin_unlock_irqrestore(&desc->lock, flags); | 163 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 164 | return 0; | 164 | return 0; |
| 165 | } | 165 | } |
| 166 | EXPORT_SYMBOL(set_irq_data); | 166 | EXPORT_SYMBOL(set_irq_data); |
| @@ -183,11 +183,11 @@ int set_irq_msi(unsigned int irq, struct msi_desc *entry) | |||
| 183 | return -EINVAL; | 183 | return -EINVAL; |
| 184 | } | 184 | } |
| 185 | 185 | ||
| 186 | spin_lock_irqsave(&desc->lock, flags); | 186 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 187 | desc->msi_desc = entry; | 187 | desc->msi_desc = entry; |
| 188 | if (entry) | 188 | if (entry) |
| 189 | entry->irq = irq; | 189 | entry->irq = irq; |
| 190 | spin_unlock_irqrestore(&desc->lock, flags); | 190 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 191 | return 0; | 191 | return 0; |
| 192 | } | 192 | } |
| 193 | 193 | ||
| @@ -214,9 +214,9 @@ int set_irq_chip_data(unsigned int irq, void *data) | |||
| 214 | return -EINVAL; | 214 | return -EINVAL; |
| 215 | } | 215 | } |
| 216 | 216 | ||
| 217 | spin_lock_irqsave(&desc->lock, flags); | 217 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 218 | desc->chip_data = data; | 218 | desc->chip_data = data; |
| 219 | spin_unlock_irqrestore(&desc->lock, flags); | 219 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 220 | 220 | ||
| 221 | return 0; | 221 | return 0; |
| 222 | } | 222 | } |
| @@ -241,12 +241,12 @@ void set_irq_nested_thread(unsigned int irq, int nest) | |||
| 241 | if (!desc) | 241 | if (!desc) |
| 242 | return; | 242 | return; |
| 243 | 243 | ||
| 244 | spin_lock_irqsave(&desc->lock, flags); | 244 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 245 | if (nest) | 245 | if (nest) |
| 246 | desc->status |= IRQ_NESTED_THREAD; | 246 | desc->status |= IRQ_NESTED_THREAD; |
| 247 | else | 247 | else |
| 248 | desc->status &= ~IRQ_NESTED_THREAD; | 248 | desc->status &= ~IRQ_NESTED_THREAD; |
| 249 | spin_unlock_irqrestore(&desc->lock, flags); | 249 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 250 | } | 250 | } |
| 251 | EXPORT_SYMBOL_GPL(set_irq_nested_thread); | 251 | EXPORT_SYMBOL_GPL(set_irq_nested_thread); |
| 252 | 252 | ||
| @@ -343,7 +343,7 @@ void handle_nested_irq(unsigned int irq) | |||
| 343 | 343 | ||
| 344 | might_sleep(); | 344 | might_sleep(); |
| 345 | 345 | ||
| 346 | spin_lock_irq(&desc->lock); | 346 | raw_spin_lock_irq(&desc->lock); |
| 347 | 347 | ||
| 348 | kstat_incr_irqs_this_cpu(irq, desc); | 348 | kstat_incr_irqs_this_cpu(irq, desc); |
| 349 | 349 | ||
| @@ -352,17 +352,17 @@ void handle_nested_irq(unsigned int irq) | |||
| 352 | goto out_unlock; | 352 | goto out_unlock; |
| 353 | 353 | ||
| 354 | desc->status |= IRQ_INPROGRESS; | 354 | desc->status |= IRQ_INPROGRESS; |
| 355 | spin_unlock_irq(&desc->lock); | 355 | raw_spin_unlock_irq(&desc->lock); |
| 356 | 356 | ||
| 357 | action_ret = action->thread_fn(action->irq, action->dev_id); | 357 | action_ret = action->thread_fn(action->irq, action->dev_id); |
| 358 | if (!noirqdebug) | 358 | if (!noirqdebug) |
| 359 | note_interrupt(irq, desc, action_ret); | 359 | note_interrupt(irq, desc, action_ret); |
| 360 | 360 | ||
| 361 | spin_lock_irq(&desc->lock); | 361 | raw_spin_lock_irq(&desc->lock); |
| 362 | desc->status &= ~IRQ_INPROGRESS; | 362 | desc->status &= ~IRQ_INPROGRESS; |
| 363 | 363 | ||
| 364 | out_unlock: | 364 | out_unlock: |
| 365 | spin_unlock_irq(&desc->lock); | 365 | raw_spin_unlock_irq(&desc->lock); |
| 366 | } | 366 | } |
| 367 | EXPORT_SYMBOL_GPL(handle_nested_irq); | 367 | EXPORT_SYMBOL_GPL(handle_nested_irq); |
| 368 | 368 | ||
| @@ -384,7 +384,7 @@ handle_simple_irq(unsigned int irq, struct irq_desc *desc) | |||
| 384 | struct irqaction *action; | 384 | struct irqaction *action; |
| 385 | irqreturn_t action_ret; | 385 | irqreturn_t action_ret; |
| 386 | 386 | ||
| 387 | spin_lock(&desc->lock); | 387 | raw_spin_lock(&desc->lock); |
| 388 | 388 | ||
| 389 | if (unlikely(desc->status & IRQ_INPROGRESS)) | 389 | if (unlikely(desc->status & IRQ_INPROGRESS)) |
| 390 | goto out_unlock; | 390 | goto out_unlock; |
| @@ -396,16 +396,16 @@ handle_simple_irq(unsigned int irq, struct irq_desc *desc) | |||
| 396 | goto out_unlock; | 396 | goto out_unlock; |
| 397 | 397 | ||
| 398 | desc->status |= IRQ_INPROGRESS; | 398 | desc->status |= IRQ_INPROGRESS; |
| 399 | spin_unlock(&desc->lock); | 399 | raw_spin_unlock(&desc->lock); |
| 400 | 400 | ||
| 401 | action_ret = handle_IRQ_event(irq, action); | 401 | action_ret = handle_IRQ_event(irq, action); |
| 402 | if (!noirqdebug) | 402 | if (!noirqdebug) |
| 403 | note_interrupt(irq, desc, action_ret); | 403 | note_interrupt(irq, desc, action_ret); |
| 404 | 404 | ||
| 405 | spin_lock(&desc->lock); | 405 | raw_spin_lock(&desc->lock); |
| 406 | desc->status &= ~IRQ_INPROGRESS; | 406 | desc->status &= ~IRQ_INPROGRESS; |
| 407 | out_unlock: | 407 | out_unlock: |
| 408 | spin_unlock(&desc->lock); | 408 | raw_spin_unlock(&desc->lock); |
| 409 | } | 409 | } |
| 410 | 410 | ||
| 411 | /** | 411 | /** |
| @@ -424,7 +424,7 @@ handle_level_irq(unsigned int irq, struct irq_desc *desc) | |||
| 424 | struct irqaction *action; | 424 | struct irqaction *action; |
| 425 | irqreturn_t action_ret; | 425 | irqreturn_t action_ret; |
| 426 | 426 | ||
| 427 | spin_lock(&desc->lock); | 427 | raw_spin_lock(&desc->lock); |
| 428 | mask_ack_irq(desc, irq); | 428 | mask_ack_irq(desc, irq); |
| 429 | 429 | ||
| 430 | if (unlikely(desc->status & IRQ_INPROGRESS)) | 430 | if (unlikely(desc->status & IRQ_INPROGRESS)) |
| @@ -441,13 +441,13 @@ handle_level_irq(unsigned int irq, struct irq_desc *desc) | |||
| 441 | goto out_unlock; | 441 | goto out_unlock; |
| 442 | 442 | ||
| 443 | desc->status |= IRQ_INPROGRESS; | 443 | desc->status |= IRQ_INPROGRESS; |
| 444 | spin_unlock(&desc->lock); | 444 | raw_spin_unlock(&desc->lock); |
| 445 | 445 | ||
| 446 | action_ret = handle_IRQ_event(irq, action); | 446 | action_ret = handle_IRQ_event(irq, action); |
| 447 | if (!noirqdebug) | 447 | if (!noirqdebug) |
| 448 | note_interrupt(irq, desc, action_ret); | 448 | note_interrupt(irq, desc, action_ret); |
| 449 | 449 | ||
| 450 | spin_lock(&desc->lock); | 450 | raw_spin_lock(&desc->lock); |
| 451 | desc->status &= ~IRQ_INPROGRESS; | 451 | desc->status &= ~IRQ_INPROGRESS; |
| 452 | 452 | ||
| 453 | if (unlikely(desc->status & IRQ_ONESHOT)) | 453 | if (unlikely(desc->status & IRQ_ONESHOT)) |
| @@ -455,7 +455,7 @@ handle_level_irq(unsigned int irq, struct irq_desc *desc) | |||
| 455 | else if (!(desc->status & IRQ_DISABLED) && desc->chip->unmask) | 455 | else if (!(desc->status & IRQ_DISABLED) && desc->chip->unmask) |
| 456 | desc->chip->unmask(irq); | 456 | desc->chip->unmask(irq); |
| 457 | out_unlock: | 457 | out_unlock: |
| 458 | spin_unlock(&desc->lock); | 458 | raw_spin_unlock(&desc->lock); |
| 459 | } | 459 | } |
| 460 | EXPORT_SYMBOL_GPL(handle_level_irq); | 460 | EXPORT_SYMBOL_GPL(handle_level_irq); |
| 461 | 461 | ||
| @@ -475,7 +475,7 @@ handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc) | |||
| 475 | struct irqaction *action; | 475 | struct irqaction *action; |
| 476 | irqreturn_t action_ret; | 476 | irqreturn_t action_ret; |
| 477 | 477 | ||
| 478 | spin_lock(&desc->lock); | 478 | raw_spin_lock(&desc->lock); |
| 479 | 479 | ||
| 480 | if (unlikely(desc->status & IRQ_INPROGRESS)) | 480 | if (unlikely(desc->status & IRQ_INPROGRESS)) |
| 481 | goto out; | 481 | goto out; |
| @@ -497,18 +497,18 @@ handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc) | |||
| 497 | 497 | ||
| 498 | desc->status |= IRQ_INPROGRESS; | 498 | desc->status |= IRQ_INPROGRESS; |
| 499 | desc->status &= ~IRQ_PENDING; | 499 | desc->status &= ~IRQ_PENDING; |
| 500 | spin_unlock(&desc->lock); | 500 | raw_spin_unlock(&desc->lock); |
| 501 | 501 | ||
| 502 | action_ret = handle_IRQ_event(irq, action); | 502 | action_ret = handle_IRQ_event(irq, action); |
| 503 | if (!noirqdebug) | 503 | if (!noirqdebug) |
| 504 | note_interrupt(irq, desc, action_ret); | 504 | note_interrupt(irq, desc, action_ret); |
| 505 | 505 | ||
| 506 | spin_lock(&desc->lock); | 506 | raw_spin_lock(&desc->lock); |
| 507 | desc->status &= ~IRQ_INPROGRESS; | 507 | desc->status &= ~IRQ_INPROGRESS; |
| 508 | out: | 508 | out: |
| 509 | desc->chip->eoi(irq); | 509 | desc->chip->eoi(irq); |
| 510 | 510 | ||
| 511 | spin_unlock(&desc->lock); | 511 | raw_spin_unlock(&desc->lock); |
| 512 | } | 512 | } |
| 513 | 513 | ||
| 514 | /** | 514 | /** |
| @@ -530,7 +530,7 @@ out: | |||
| 530 | void | 530 | void |
| 531 | handle_edge_irq(unsigned int irq, struct irq_desc *desc) | 531 | handle_edge_irq(unsigned int irq, struct irq_desc *desc) |
| 532 | { | 532 | { |
| 533 | spin_lock(&desc->lock); | 533 | raw_spin_lock(&desc->lock); |
| 534 | 534 | ||
| 535 | desc->status &= ~(IRQ_REPLAY | IRQ_WAITING); | 535 | desc->status &= ~(IRQ_REPLAY | IRQ_WAITING); |
| 536 | 536 | ||
| @@ -576,17 +576,17 @@ handle_edge_irq(unsigned int irq, struct irq_desc *desc) | |||
| 576 | } | 576 | } |
| 577 | 577 | ||
| 578 | desc->status &= ~IRQ_PENDING; | 578 | desc->status &= ~IRQ_PENDING; |
| 579 | spin_unlock(&desc->lock); | 579 | raw_spin_unlock(&desc->lock); |
| 580 | action_ret = handle_IRQ_event(irq, action); | 580 | action_ret = handle_IRQ_event(irq, action); |
| 581 | if (!noirqdebug) | 581 | if (!noirqdebug) |
| 582 | note_interrupt(irq, desc, action_ret); | 582 | note_interrupt(irq, desc, action_ret); |
| 583 | spin_lock(&desc->lock); | 583 | raw_spin_lock(&desc->lock); |
| 584 | 584 | ||
| 585 | } while ((desc->status & (IRQ_PENDING | IRQ_DISABLED)) == IRQ_PENDING); | 585 | } while ((desc->status & (IRQ_PENDING | IRQ_DISABLED)) == IRQ_PENDING); |
| 586 | 586 | ||
| 587 | desc->status &= ~IRQ_INPROGRESS; | 587 | desc->status &= ~IRQ_INPROGRESS; |
| 588 | out_unlock: | 588 | out_unlock: |
| 589 | spin_unlock(&desc->lock); | 589 | raw_spin_unlock(&desc->lock); |
| 590 | } | 590 | } |
| 591 | 591 | ||
| 592 | /** | 592 | /** |
| @@ -643,7 +643,7 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, | |||
| 643 | } | 643 | } |
| 644 | 644 | ||
| 645 | chip_bus_lock(irq, desc); | 645 | chip_bus_lock(irq, desc); |
| 646 | spin_lock_irqsave(&desc->lock, flags); | 646 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 647 | 647 | ||
| 648 | /* Uninstall? */ | 648 | /* Uninstall? */ |
| 649 | if (handle == handle_bad_irq) { | 649 | if (handle == handle_bad_irq) { |
| @@ -661,7 +661,7 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, | |||
| 661 | desc->depth = 0; | 661 | desc->depth = 0; |
| 662 | desc->chip->startup(irq); | 662 | desc->chip->startup(irq); |
| 663 | } | 663 | } |
| 664 | spin_unlock_irqrestore(&desc->lock, flags); | 664 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 665 | chip_bus_sync_unlock(irq, desc); | 665 | chip_bus_sync_unlock(irq, desc); |
| 666 | } | 666 | } |
| 667 | EXPORT_SYMBOL_GPL(__set_irq_handler); | 667 | EXPORT_SYMBOL_GPL(__set_irq_handler); |
| @@ -692,9 +692,9 @@ void __init set_irq_noprobe(unsigned int irq) | |||
| 692 | return; | 692 | return; |
| 693 | } | 693 | } |
| 694 | 694 | ||
| 695 | spin_lock_irqsave(&desc->lock, flags); | 695 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 696 | desc->status |= IRQ_NOPROBE; | 696 | desc->status |= IRQ_NOPROBE; |
| 697 | spin_unlock_irqrestore(&desc->lock, flags); | 697 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 698 | } | 698 | } |
| 699 | 699 | ||
| 700 | void __init set_irq_probe(unsigned int irq) | 700 | void __init set_irq_probe(unsigned int irq) |
| @@ -707,7 +707,7 @@ void __init set_irq_probe(unsigned int irq) | |||
| 707 | return; | 707 | return; |
| 708 | } | 708 | } |
| 709 | 709 | ||
| 710 | spin_lock_irqsave(&desc->lock, flags); | 710 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 711 | desc->status &= ~IRQ_NOPROBE; | 711 | desc->status &= ~IRQ_NOPROBE; |
| 712 | spin_unlock_irqrestore(&desc->lock, flags); | 712 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 713 | } | 713 | } |
diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c index 17c71bb565c6..814940e7f485 100644 --- a/kernel/irq/handle.c +++ b/kernel/irq/handle.c | |||
| @@ -80,7 +80,7 @@ static struct irq_desc irq_desc_init = { | |||
| 80 | .chip = &no_irq_chip, | 80 | .chip = &no_irq_chip, |
| 81 | .handle_irq = handle_bad_irq, | 81 | .handle_irq = handle_bad_irq, |
| 82 | .depth = 1, | 82 | .depth = 1, |
| 83 | .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock), | 83 | .lock = __RAW_SPIN_LOCK_UNLOCKED(irq_desc_init.lock), |
| 84 | }; | 84 | }; |
| 85 | 85 | ||
| 86 | void __ref init_kstat_irqs(struct irq_desc *desc, int node, int nr) | 86 | void __ref init_kstat_irqs(struct irq_desc *desc, int node, int nr) |
| @@ -108,7 +108,7 @@ static void init_one_irq_desc(int irq, struct irq_desc *desc, int node) | |||
| 108 | { | 108 | { |
| 109 | memcpy(desc, &irq_desc_init, sizeof(struct irq_desc)); | 109 | memcpy(desc, &irq_desc_init, sizeof(struct irq_desc)); |
| 110 | 110 | ||
| 111 | spin_lock_init(&desc->lock); | 111 | raw_spin_lock_init(&desc->lock); |
| 112 | desc->irq = irq; | 112 | desc->irq = irq; |
| 113 | #ifdef CONFIG_SMP | 113 | #ifdef CONFIG_SMP |
| 114 | desc->node = node; | 114 | desc->node = node; |
| @@ -130,7 +130,7 @@ static void init_one_irq_desc(int irq, struct irq_desc *desc, int node) | |||
| 130 | /* | 130 | /* |
| 131 | * Protect the sparse_irqs: | 131 | * Protect the sparse_irqs: |
| 132 | */ | 132 | */ |
| 133 | DEFINE_SPINLOCK(sparse_irq_lock); | 133 | DEFINE_RAW_SPINLOCK(sparse_irq_lock); |
| 134 | 134 | ||
| 135 | struct irq_desc **irq_desc_ptrs __read_mostly; | 135 | struct irq_desc **irq_desc_ptrs __read_mostly; |
| 136 | 136 | ||
| @@ -141,7 +141,7 @@ static struct irq_desc irq_desc_legacy[NR_IRQS_LEGACY] __cacheline_aligned_in_sm | |||
| 141 | .chip = &no_irq_chip, | 141 | .chip = &no_irq_chip, |
| 142 | .handle_irq = handle_bad_irq, | 142 | .handle_irq = handle_bad_irq, |
| 143 | .depth = 1, | 143 | .depth = 1, |
| 144 | .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock), | 144 | .lock = __RAW_SPIN_LOCK_UNLOCKED(irq_desc_init.lock), |
| 145 | } | 145 | } |
| 146 | }; | 146 | }; |
| 147 | 147 | ||
| @@ -212,7 +212,7 @@ struct irq_desc * __ref irq_to_desc_alloc_node(unsigned int irq, int node) | |||
| 212 | if (desc) | 212 | if (desc) |
| 213 | return desc; | 213 | return desc; |
| 214 | 214 | ||
| 215 | spin_lock_irqsave(&sparse_irq_lock, flags); | 215 | raw_spin_lock_irqsave(&sparse_irq_lock, flags); |
| 216 | 216 | ||
| 217 | /* We have to check it to avoid races with another CPU */ | 217 | /* We have to check it to avoid races with another CPU */ |
| 218 | desc = irq_desc_ptrs[irq]; | 218 | desc = irq_desc_ptrs[irq]; |
| @@ -234,7 +234,7 @@ struct irq_desc * __ref irq_to_desc_alloc_node(unsigned int irq, int node) | |||
| 234 | irq_desc_ptrs[irq] = desc; | 234 | irq_desc_ptrs[irq] = desc; |
| 235 | 235 | ||
| 236 | out_unlock: | 236 | out_unlock: |
| 237 | spin_unlock_irqrestore(&sparse_irq_lock, flags); | 237 | raw_spin_unlock_irqrestore(&sparse_irq_lock, flags); |
| 238 | 238 | ||
| 239 | return desc; | 239 | return desc; |
| 240 | } | 240 | } |
| @@ -247,7 +247,7 @@ struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = { | |||
| 247 | .chip = &no_irq_chip, | 247 | .chip = &no_irq_chip, |
| 248 | .handle_irq = handle_bad_irq, | 248 | .handle_irq = handle_bad_irq, |
| 249 | .depth = 1, | 249 | .depth = 1, |
| 250 | .lock = __SPIN_LOCK_UNLOCKED(irq_desc->lock), | 250 | .lock = __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock), |
| 251 | } | 251 | } |
| 252 | }; | 252 | }; |
| 253 | 253 | ||
| @@ -473,7 +473,7 @@ unsigned int __do_IRQ(unsigned int irq) | |||
| 473 | return 1; | 473 | return 1; |
| 474 | } | 474 | } |
| 475 | 475 | ||
| 476 | spin_lock(&desc->lock); | 476 | raw_spin_lock(&desc->lock); |
| 477 | if (desc->chip->ack) | 477 | if (desc->chip->ack) |
| 478 | desc->chip->ack(irq); | 478 | desc->chip->ack(irq); |
| 479 | /* | 479 | /* |
| @@ -517,13 +517,13 @@ unsigned int __do_IRQ(unsigned int irq) | |||
| 517 | for (;;) { | 517 | for (;;) { |
| 518 | irqreturn_t action_ret; | 518 | irqreturn_t action_ret; |
| 519 | 519 | ||
| 520 | spin_unlock(&desc->lock); | 520 | raw_spin_unlock(&desc->lock); |
| 521 | 521 | ||
| 522 | action_ret = handle_IRQ_event(irq, action); | 522 | action_ret = handle_IRQ_event(irq, action); |
| 523 | if (!noirqdebug) | 523 | if (!noirqdebug) |
| 524 | note_interrupt(irq, desc, action_ret); | 524 | note_interrupt(irq, desc, action_ret); |
| 525 | 525 | ||
| 526 | spin_lock(&desc->lock); | 526 | raw_spin_lock(&desc->lock); |
| 527 | if (likely(!(desc->status & IRQ_PENDING))) | 527 | if (likely(!(desc->status & IRQ_PENDING))) |
| 528 | break; | 528 | break; |
| 529 | desc->status &= ~IRQ_PENDING; | 529 | desc->status &= ~IRQ_PENDING; |
| @@ -536,7 +536,7 @@ out: | |||
| 536 | * disabled while the handler was running. | 536 | * disabled while the handler was running. |
| 537 | */ | 537 | */ |
| 538 | desc->chip->end(irq); | 538 | desc->chip->end(irq); |
| 539 | spin_unlock(&desc->lock); | 539 | raw_spin_unlock(&desc->lock); |
| 540 | 540 | ||
| 541 | return 1; | 541 | return 1; |
| 542 | } | 542 | } |
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index 1b5d742c6a77..b2821f070a3d 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h | |||
| @@ -18,7 +18,7 @@ extern void __enable_irq(struct irq_desc *desc, unsigned int irq, bool resume); | |||
| 18 | extern struct lock_class_key irq_desc_lock_class; | 18 | extern struct lock_class_key irq_desc_lock_class; |
| 19 | extern void init_kstat_irqs(struct irq_desc *desc, int node, int nr); | 19 | extern void init_kstat_irqs(struct irq_desc *desc, int node, int nr); |
| 20 | extern void clear_kstat_irqs(struct irq_desc *desc); | 20 | extern void clear_kstat_irqs(struct irq_desc *desc); |
| 21 | extern spinlock_t sparse_irq_lock; | 21 | extern raw_spinlock_t sparse_irq_lock; |
| 22 | 22 | ||
| 23 | #ifdef CONFIG_SPARSE_IRQ | 23 | #ifdef CONFIG_SPARSE_IRQ |
| 24 | /* irq_desc_ptrs allocated at boot time */ | 24 | /* irq_desc_ptrs allocated at boot time */ |
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 7305b297d1eb..eb6078ca60c7 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c | |||
| @@ -46,9 +46,9 @@ void synchronize_irq(unsigned int irq) | |||
| 46 | cpu_relax(); | 46 | cpu_relax(); |
| 47 | 47 | ||
| 48 | /* Ok, that indicated we're done: double-check carefully. */ | 48 | /* Ok, that indicated we're done: double-check carefully. */ |
| 49 | spin_lock_irqsave(&desc->lock, flags); | 49 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 50 | status = desc->status; | 50 | status = desc->status; |
| 51 | spin_unlock_irqrestore(&desc->lock, flags); | 51 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 52 | 52 | ||
| 53 | /* Oops, that failed? */ | 53 | /* Oops, that failed? */ |
| 54 | } while (status & IRQ_INPROGRESS); | 54 | } while (status & IRQ_INPROGRESS); |
| @@ -114,7 +114,7 @@ int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask) | |||
| 114 | if (!desc->chip->set_affinity) | 114 | if (!desc->chip->set_affinity) |
| 115 | return -EINVAL; | 115 | return -EINVAL; |
| 116 | 116 | ||
| 117 | spin_lock_irqsave(&desc->lock, flags); | 117 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 118 | 118 | ||
| 119 | #ifdef CONFIG_GENERIC_PENDING_IRQ | 119 | #ifdef CONFIG_GENERIC_PENDING_IRQ |
| 120 | if (desc->status & IRQ_MOVE_PCNTXT) { | 120 | if (desc->status & IRQ_MOVE_PCNTXT) { |
| @@ -134,7 +134,7 @@ int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask) | |||
| 134 | } | 134 | } |
| 135 | #endif | 135 | #endif |
| 136 | desc->status |= IRQ_AFFINITY_SET; | 136 | desc->status |= IRQ_AFFINITY_SET; |
| 137 | spin_unlock_irqrestore(&desc->lock, flags); | 137 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 138 | return 0; | 138 | return 0; |
| 139 | } | 139 | } |
| 140 | 140 | ||
| @@ -181,11 +181,11 @@ int irq_select_affinity_usr(unsigned int irq) | |||
| 181 | unsigned long flags; | 181 | unsigned long flags; |
| 182 | int ret; | 182 | int ret; |
| 183 | 183 | ||
| 184 | spin_lock_irqsave(&desc->lock, flags); | 184 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 185 | ret = setup_affinity(irq, desc); | 185 | ret = setup_affinity(irq, desc); |
| 186 | if (!ret) | 186 | if (!ret) |
| 187 | irq_set_thread_affinity(desc); | 187 | irq_set_thread_affinity(desc); |
| 188 | spin_unlock_irqrestore(&desc->lock, flags); | 188 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 189 | 189 | ||
| 190 | return ret; | 190 | return ret; |
| 191 | } | 191 | } |
| @@ -231,9 +231,9 @@ void disable_irq_nosync(unsigned int irq) | |||
| 231 | return; | 231 | return; |
| 232 | 232 | ||
| 233 | chip_bus_lock(irq, desc); | 233 | chip_bus_lock(irq, desc); |
| 234 | spin_lock_irqsave(&desc->lock, flags); | 234 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 235 | __disable_irq(desc, irq, false); | 235 | __disable_irq(desc, irq, false); |
| 236 | spin_unlock_irqrestore(&desc->lock, flags); | 236 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 237 | chip_bus_sync_unlock(irq, desc); | 237 | chip_bus_sync_unlock(irq, desc); |
| 238 | } | 238 | } |
| 239 | EXPORT_SYMBOL(disable_irq_nosync); | 239 | EXPORT_SYMBOL(disable_irq_nosync); |
| @@ -308,9 +308,9 @@ void enable_irq(unsigned int irq) | |||
| 308 | return; | 308 | return; |
| 309 | 309 | ||
| 310 | chip_bus_lock(irq, desc); | 310 | chip_bus_lock(irq, desc); |
| 311 | spin_lock_irqsave(&desc->lock, flags); | 311 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 312 | __enable_irq(desc, irq, false); | 312 | __enable_irq(desc, irq, false); |
| 313 | spin_unlock_irqrestore(&desc->lock, flags); | 313 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 314 | chip_bus_sync_unlock(irq, desc); | 314 | chip_bus_sync_unlock(irq, desc); |
| 315 | } | 315 | } |
| 316 | EXPORT_SYMBOL(enable_irq); | 316 | EXPORT_SYMBOL(enable_irq); |
| @@ -347,7 +347,7 @@ int set_irq_wake(unsigned int irq, unsigned int on) | |||
| 347 | /* wakeup-capable irqs can be shared between drivers that | 347 | /* wakeup-capable irqs can be shared between drivers that |
| 348 | * don't need to have the same sleep mode behaviors. | 348 | * don't need to have the same sleep mode behaviors. |
| 349 | */ | 349 | */ |
| 350 | spin_lock_irqsave(&desc->lock, flags); | 350 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 351 | if (on) { | 351 | if (on) { |
| 352 | if (desc->wake_depth++ == 0) { | 352 | if (desc->wake_depth++ == 0) { |
| 353 | ret = set_irq_wake_real(irq, on); | 353 | ret = set_irq_wake_real(irq, on); |
| @@ -368,7 +368,7 @@ int set_irq_wake(unsigned int irq, unsigned int on) | |||
| 368 | } | 368 | } |
| 369 | } | 369 | } |
| 370 | 370 | ||
| 371 | spin_unlock_irqrestore(&desc->lock, flags); | 371 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 372 | return ret; | 372 | return ret; |
| 373 | } | 373 | } |
| 374 | EXPORT_SYMBOL(set_irq_wake); | 374 | EXPORT_SYMBOL(set_irq_wake); |
| @@ -484,12 +484,12 @@ static int irq_wait_for_interrupt(struct irqaction *action) | |||
| 484 | static void irq_finalize_oneshot(unsigned int irq, struct irq_desc *desc) | 484 | static void irq_finalize_oneshot(unsigned int irq, struct irq_desc *desc) |
| 485 | { | 485 | { |
| 486 | chip_bus_lock(irq, desc); | 486 | chip_bus_lock(irq, desc); |
| 487 | spin_lock_irq(&desc->lock); | 487 | raw_spin_lock_irq(&desc->lock); |
| 488 | if (!(desc->status & IRQ_DISABLED) && (desc->status & IRQ_MASKED)) { | 488 | if (!(desc->status & IRQ_DISABLED) && (desc->status & IRQ_MASKED)) { |
| 489 | desc->status &= ~IRQ_MASKED; | 489 | desc->status &= ~IRQ_MASKED; |
| 490 | desc->chip->unmask(irq); | 490 | desc->chip->unmask(irq); |
| 491 | } | 491 | } |
| 492 | spin_unlock_irq(&desc->lock); | 492 | raw_spin_unlock_irq(&desc->lock); |
| 493 | chip_bus_sync_unlock(irq, desc); | 493 | chip_bus_sync_unlock(irq, desc); |
| 494 | } | 494 | } |
| 495 | 495 | ||
| @@ -514,9 +514,9 @@ irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) | |||
| 514 | return; | 514 | return; |
| 515 | } | 515 | } |
| 516 | 516 | ||
| 517 | spin_lock_irq(&desc->lock); | 517 | raw_spin_lock_irq(&desc->lock); |
| 518 | cpumask_copy(mask, desc->affinity); | 518 | cpumask_copy(mask, desc->affinity); |
| 519 | spin_unlock_irq(&desc->lock); | 519 | raw_spin_unlock_irq(&desc->lock); |
| 520 | 520 | ||
| 521 | set_cpus_allowed_ptr(current, mask); | 521 | set_cpus_allowed_ptr(current, mask); |
| 522 | free_cpumask_var(mask); | 522 | free_cpumask_var(mask); |
| @@ -545,7 +545,7 @@ static int irq_thread(void *data) | |||
| 545 | 545 | ||
| 546 | atomic_inc(&desc->threads_active); | 546 | atomic_inc(&desc->threads_active); |
| 547 | 547 | ||
| 548 | spin_lock_irq(&desc->lock); | 548 | raw_spin_lock_irq(&desc->lock); |
| 549 | if (unlikely(desc->status & IRQ_DISABLED)) { | 549 | if (unlikely(desc->status & IRQ_DISABLED)) { |
| 550 | /* | 550 | /* |
| 551 | * CHECKME: We might need a dedicated | 551 | * CHECKME: We might need a dedicated |
| @@ -555,9 +555,9 @@ static int irq_thread(void *data) | |||
| 555 | * retriggers the interrupt itself --- tglx | 555 | * retriggers the interrupt itself --- tglx |
| 556 | */ | 556 | */ |
| 557 | desc->status |= IRQ_PENDING; | 557 | desc->status |= IRQ_PENDING; |
| 558 | spin_unlock_irq(&desc->lock); | 558 | raw_spin_unlock_irq(&desc->lock); |
| 559 | } else { | 559 | } else { |
| 560 | spin_unlock_irq(&desc->lock); | 560 | raw_spin_unlock_irq(&desc->lock); |
| 561 | 561 | ||
| 562 | action->thread_fn(action->irq, action->dev_id); | 562 | action->thread_fn(action->irq, action->dev_id); |
| 563 | 563 | ||
| @@ -679,7 +679,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new) | |||
| 679 | /* | 679 | /* |
| 680 | * The following block of code has to be executed atomically | 680 | * The following block of code has to be executed atomically |
| 681 | */ | 681 | */ |
| 682 | spin_lock_irqsave(&desc->lock, flags); | 682 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 683 | old_ptr = &desc->action; | 683 | old_ptr = &desc->action; |
| 684 | old = *old_ptr; | 684 | old = *old_ptr; |
| 685 | if (old) { | 685 | if (old) { |
| @@ -775,7 +775,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new) | |||
| 775 | __enable_irq(desc, irq, false); | 775 | __enable_irq(desc, irq, false); |
| 776 | } | 776 | } |
| 777 | 777 | ||
| 778 | spin_unlock_irqrestore(&desc->lock, flags); | 778 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 779 | 779 | ||
| 780 | /* | 780 | /* |
| 781 | * Strictly no need to wake it up, but hung_task complains | 781 | * Strictly no need to wake it up, but hung_task complains |
| @@ -802,7 +802,7 @@ mismatch: | |||
| 802 | ret = -EBUSY; | 802 | ret = -EBUSY; |
| 803 | 803 | ||
| 804 | out_thread: | 804 | out_thread: |
| 805 | spin_unlock_irqrestore(&desc->lock, flags); | 805 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 806 | if (new->thread) { | 806 | if (new->thread) { |
| 807 | struct task_struct *t = new->thread; | 807 | struct task_struct *t = new->thread; |
| 808 | 808 | ||
| @@ -844,7 +844,7 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id) | |||
| 844 | if (!desc) | 844 | if (!desc) |
| 845 | return NULL; | 845 | return NULL; |
| 846 | 846 | ||
| 847 | spin_lock_irqsave(&desc->lock, flags); | 847 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 848 | 848 | ||
| 849 | /* | 849 | /* |
| 850 | * There can be multiple actions per IRQ descriptor, find the right | 850 | * There can be multiple actions per IRQ descriptor, find the right |
| @@ -856,7 +856,7 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id) | |||
| 856 | 856 | ||
| 857 | if (!action) { | 857 | if (!action) { |
| 858 | WARN(1, "Trying to free already-free IRQ %d\n", irq); | 858 | WARN(1, "Trying to free already-free IRQ %d\n", irq); |
| 859 | spin_unlock_irqrestore(&desc->lock, flags); | 859 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 860 | 860 | ||
| 861 | return NULL; | 861 | return NULL; |
| 862 | } | 862 | } |
| @@ -884,7 +884,7 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id) | |||
| 884 | desc->chip->disable(irq); | 884 | desc->chip->disable(irq); |
| 885 | } | 885 | } |
| 886 | 886 | ||
| 887 | spin_unlock_irqrestore(&desc->lock, flags); | 887 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 888 | 888 | ||
| 889 | unregister_handler_proc(irq, action); | 889 | unregister_handler_proc(irq, action); |
| 890 | 890 | ||
diff --git a/kernel/irq/migration.c b/kernel/irq/migration.c index fcb6c96f2627..241962280836 100644 --- a/kernel/irq/migration.c +++ b/kernel/irq/migration.c | |||
| @@ -27,7 +27,7 @@ void move_masked_irq(int irq) | |||
| 27 | if (!desc->chip->set_affinity) | 27 | if (!desc->chip->set_affinity) |
| 28 | return; | 28 | return; |
| 29 | 29 | ||
| 30 | assert_spin_locked(&desc->lock); | 30 | assert_raw_spin_locked(&desc->lock); |
| 31 | 31 | ||
| 32 | /* | 32 | /* |
| 33 | * If there was a valid mask to work with, please | 33 | * If there was a valid mask to work with, please |
diff --git a/kernel/irq/numa_migrate.c b/kernel/irq/numa_migrate.c index 3fd30197da2e..26bac9d8f860 100644 --- a/kernel/irq/numa_migrate.c +++ b/kernel/irq/numa_migrate.c | |||
| @@ -42,7 +42,7 @@ static bool init_copy_one_irq_desc(int irq, struct irq_desc *old_desc, | |||
| 42 | "for migration.\n", irq); | 42 | "for migration.\n", irq); |
| 43 | return false; | 43 | return false; |
| 44 | } | 44 | } |
| 45 | spin_lock_init(&desc->lock); | 45 | raw_spin_lock_init(&desc->lock); |
| 46 | desc->node = node; | 46 | desc->node = node; |
| 47 | lockdep_set_class(&desc->lock, &irq_desc_lock_class); | 47 | lockdep_set_class(&desc->lock, &irq_desc_lock_class); |
| 48 | init_copy_kstat_irqs(old_desc, desc, node, nr_cpu_ids); | 48 | init_copy_kstat_irqs(old_desc, desc, node, nr_cpu_ids); |
| @@ -67,7 +67,7 @@ static struct irq_desc *__real_move_irq_desc(struct irq_desc *old_desc, | |||
| 67 | 67 | ||
| 68 | irq = old_desc->irq; | 68 | irq = old_desc->irq; |
| 69 | 69 | ||
| 70 | spin_lock_irqsave(&sparse_irq_lock, flags); | 70 | raw_spin_lock_irqsave(&sparse_irq_lock, flags); |
| 71 | 71 | ||
| 72 | /* We have to check it to avoid races with another CPU */ | 72 | /* We have to check it to avoid races with another CPU */ |
| 73 | desc = irq_desc_ptrs[irq]; | 73 | desc = irq_desc_ptrs[irq]; |
| @@ -91,7 +91,7 @@ static struct irq_desc *__real_move_irq_desc(struct irq_desc *old_desc, | |||
| 91 | } | 91 | } |
| 92 | 92 | ||
| 93 | irq_desc_ptrs[irq] = desc; | 93 | irq_desc_ptrs[irq] = desc; |
| 94 | spin_unlock_irqrestore(&sparse_irq_lock, flags); | 94 | raw_spin_unlock_irqrestore(&sparse_irq_lock, flags); |
| 95 | 95 | ||
| 96 | /* free the old one */ | 96 | /* free the old one */ |
| 97 | free_one_irq_desc(old_desc, desc); | 97 | free_one_irq_desc(old_desc, desc); |
| @@ -100,7 +100,7 @@ static struct irq_desc *__real_move_irq_desc(struct irq_desc *old_desc, | |||
| 100 | return desc; | 100 | return desc; |
| 101 | 101 | ||
| 102 | out_unlock: | 102 | out_unlock: |
| 103 | spin_unlock_irqrestore(&sparse_irq_lock, flags); | 103 | raw_spin_unlock_irqrestore(&sparse_irq_lock, flags); |
| 104 | 104 | ||
| 105 | return desc; | 105 | return desc; |
| 106 | } | 106 | } |
diff --git a/kernel/irq/pm.c b/kernel/irq/pm.c index a0bb09e79867..0d4005d85b03 100644 --- a/kernel/irq/pm.c +++ b/kernel/irq/pm.c | |||
| @@ -28,9 +28,9 @@ void suspend_device_irqs(void) | |||
| 28 | for_each_irq_desc(irq, desc) { | 28 | for_each_irq_desc(irq, desc) { |
| 29 | unsigned long flags; | 29 | unsigned long flags; |
| 30 | 30 | ||
| 31 | spin_lock_irqsave(&desc->lock, flags); | 31 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 32 | __disable_irq(desc, irq, true); | 32 | __disable_irq(desc, irq, true); |
| 33 | spin_unlock_irqrestore(&desc->lock, flags); | 33 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 34 | } | 34 | } |
| 35 | 35 | ||
| 36 | for_each_irq_desc(irq, desc) | 36 | for_each_irq_desc(irq, desc) |
| @@ -56,9 +56,9 @@ void resume_device_irqs(void) | |||
| 56 | if (!(desc->status & IRQ_SUSPENDED)) | 56 | if (!(desc->status & IRQ_SUSPENDED)) |
| 57 | continue; | 57 | continue; |
| 58 | 58 | ||
| 59 | spin_lock_irqsave(&desc->lock, flags); | 59 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 60 | __enable_irq(desc, irq, true); | 60 | __enable_irq(desc, irq, true); |
| 61 | spin_unlock_irqrestore(&desc->lock, flags); | 61 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 62 | } | 62 | } |
| 63 | } | 63 | } |
| 64 | EXPORT_SYMBOL_GPL(resume_device_irqs); | 64 | EXPORT_SYMBOL_GPL(resume_device_irqs); |
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 0832145fea97..6f50eccc79c0 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c | |||
| @@ -179,7 +179,7 @@ static int name_unique(unsigned int irq, struct irqaction *new_action) | |||
| 179 | unsigned long flags; | 179 | unsigned long flags; |
| 180 | int ret = 1; | 180 | int ret = 1; |
| 181 | 181 | ||
| 182 | spin_lock_irqsave(&desc->lock, flags); | 182 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 183 | for (action = desc->action ; action; action = action->next) { | 183 | for (action = desc->action ; action; action = action->next) { |
| 184 | if ((action != new_action) && action->name && | 184 | if ((action != new_action) && action->name && |
| 185 | !strcmp(new_action->name, action->name)) { | 185 | !strcmp(new_action->name, action->name)) { |
| @@ -187,7 +187,7 @@ static int name_unique(unsigned int irq, struct irqaction *new_action) | |||
| 187 | break; | 187 | break; |
| 188 | } | 188 | } |
| 189 | } | 189 | } |
| 190 | spin_unlock_irqrestore(&desc->lock, flags); | 190 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 191 | return ret; | 191 | return ret; |
| 192 | } | 192 | } |
| 193 | 193 | ||
diff --git a/kernel/irq/spurious.c b/kernel/irq/spurious.c index e49ea1c5232d..89fb90ae534f 100644 --- a/kernel/irq/spurious.c +++ b/kernel/irq/spurious.c | |||
| @@ -28,7 +28,7 @@ static int try_one_irq(int irq, struct irq_desc *desc) | |||
| 28 | struct irqaction *action; | 28 | struct irqaction *action; |
| 29 | int ok = 0, work = 0; | 29 | int ok = 0, work = 0; |
| 30 | 30 | ||
| 31 | spin_lock(&desc->lock); | 31 | raw_spin_lock(&desc->lock); |
| 32 | /* Already running on another processor */ | 32 | /* Already running on another processor */ |
| 33 | if (desc->status & IRQ_INPROGRESS) { | 33 | if (desc->status & IRQ_INPROGRESS) { |
| 34 | /* | 34 | /* |
| @@ -37,13 +37,13 @@ static int try_one_irq(int irq, struct irq_desc *desc) | |||
| 37 | */ | 37 | */ |
| 38 | if (desc->action && (desc->action->flags & IRQF_SHARED)) | 38 | if (desc->action && (desc->action->flags & IRQF_SHARED)) |
| 39 | desc->status |= IRQ_PENDING; | 39 | desc->status |= IRQ_PENDING; |
| 40 | spin_unlock(&desc->lock); | 40 | raw_spin_unlock(&desc->lock); |
| 41 | return ok; | 41 | return ok; |
| 42 | } | 42 | } |
| 43 | /* Honour the normal IRQ locking */ | 43 | /* Honour the normal IRQ locking */ |
| 44 | desc->status |= IRQ_INPROGRESS; | 44 | desc->status |= IRQ_INPROGRESS; |
| 45 | action = desc->action; | 45 | action = desc->action; |
| 46 | spin_unlock(&desc->lock); | 46 | raw_spin_unlock(&desc->lock); |
| 47 | 47 | ||
| 48 | while (action) { | 48 | while (action) { |
| 49 | /* Only shared IRQ handlers are safe to call */ | 49 | /* Only shared IRQ handlers are safe to call */ |
| @@ -56,7 +56,7 @@ static int try_one_irq(int irq, struct irq_desc *desc) | |||
| 56 | } | 56 | } |
| 57 | local_irq_disable(); | 57 | local_irq_disable(); |
| 58 | /* Now clean up the flags */ | 58 | /* Now clean up the flags */ |
| 59 | spin_lock(&desc->lock); | 59 | raw_spin_lock(&desc->lock); |
| 60 | action = desc->action; | 60 | action = desc->action; |
| 61 | 61 | ||
| 62 | /* | 62 | /* |
| @@ -68,9 +68,9 @@ static int try_one_irq(int irq, struct irq_desc *desc) | |||
| 68 | * Perform real IRQ processing for the IRQ we deferred | 68 | * Perform real IRQ processing for the IRQ we deferred |
| 69 | */ | 69 | */ |
| 70 | work = 1; | 70 | work = 1; |
| 71 | spin_unlock(&desc->lock); | 71 | raw_spin_unlock(&desc->lock); |
| 72 | handle_IRQ_event(irq, action); | 72 | handle_IRQ_event(irq, action); |
| 73 | spin_lock(&desc->lock); | 73 | raw_spin_lock(&desc->lock); |
| 74 | desc->status &= ~IRQ_PENDING; | 74 | desc->status &= ~IRQ_PENDING; |
| 75 | } | 75 | } |
| 76 | desc->status &= ~IRQ_INPROGRESS; | 76 | desc->status &= ~IRQ_INPROGRESS; |
| @@ -80,7 +80,7 @@ static int try_one_irq(int irq, struct irq_desc *desc) | |||
| 80 | */ | 80 | */ |
| 81 | if (work && desc->chip && desc->chip->end) | 81 | if (work && desc->chip && desc->chip->end) |
| 82 | desc->chip->end(irq); | 82 | desc->chip->end(irq); |
| 83 | spin_unlock(&desc->lock); | 83 | raw_spin_unlock(&desc->lock); |
| 84 | 84 | ||
| 85 | return ok; | 85 | return ok; |
| 86 | } | 86 | } |
diff --git a/kernel/lockdep.c b/kernel/lockdep.c index 4f8df01dbe51..5feaddcdbe49 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c | |||
| @@ -73,11 +73,11 @@ module_param(lock_stat, int, 0644); | |||
| 73 | * to use a raw spinlock - we really dont want the spinlock | 73 | * to use a raw spinlock - we really dont want the spinlock |
| 74 | * code to recurse back into the lockdep code... | 74 | * code to recurse back into the lockdep code... |
| 75 | */ | 75 | */ |
| 76 | static raw_spinlock_t lockdep_lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; | 76 | static arch_spinlock_t lockdep_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
| 77 | 77 | ||
| 78 | static int graph_lock(void) | 78 | static int graph_lock(void) |
| 79 | { | 79 | { |
| 80 | __raw_spin_lock(&lockdep_lock); | 80 | arch_spin_lock(&lockdep_lock); |
| 81 | /* | 81 | /* |
| 82 | * Make sure that if another CPU detected a bug while | 82 | * Make sure that if another CPU detected a bug while |
| 83 | * walking the graph we dont change it (while the other | 83 | * walking the graph we dont change it (while the other |
| @@ -85,7 +85,7 @@ static int graph_lock(void) | |||
| 85 | * dropped already) | 85 | * dropped already) |
| 86 | */ | 86 | */ |
| 87 | if (!debug_locks) { | 87 | if (!debug_locks) { |
| 88 | __raw_spin_unlock(&lockdep_lock); | 88 | arch_spin_unlock(&lockdep_lock); |
| 89 | return 0; | 89 | return 0; |
| 90 | } | 90 | } |
| 91 | /* prevent any recursions within lockdep from causing deadlocks */ | 91 | /* prevent any recursions within lockdep from causing deadlocks */ |
| @@ -95,11 +95,11 @@ static int graph_lock(void) | |||
| 95 | 95 | ||
| 96 | static inline int graph_unlock(void) | 96 | static inline int graph_unlock(void) |
| 97 | { | 97 | { |
| 98 | if (debug_locks && !__raw_spin_is_locked(&lockdep_lock)) | 98 | if (debug_locks && !arch_spin_is_locked(&lockdep_lock)) |
| 99 | return DEBUG_LOCKS_WARN_ON(1); | 99 | return DEBUG_LOCKS_WARN_ON(1); |
| 100 | 100 | ||
| 101 | current->lockdep_recursion--; | 101 | current->lockdep_recursion--; |
| 102 | __raw_spin_unlock(&lockdep_lock); | 102 | arch_spin_unlock(&lockdep_lock); |
| 103 | return 0; | 103 | return 0; |
| 104 | } | 104 | } |
| 105 | 105 | ||
| @@ -111,7 +111,7 @@ static inline int debug_locks_off_graph_unlock(void) | |||
| 111 | { | 111 | { |
| 112 | int ret = debug_locks_off(); | 112 | int ret = debug_locks_off(); |
| 113 | 113 | ||
| 114 | __raw_spin_unlock(&lockdep_lock); | 114 | arch_spin_unlock(&lockdep_lock); |
| 115 | 115 | ||
| 116 | return ret; | 116 | return ret; |
| 117 | } | 117 | } |
| @@ -140,7 +140,8 @@ static inline struct lock_class *hlock_class(struct held_lock *hlock) | |||
| 140 | } | 140 | } |
| 141 | 141 | ||
| 142 | #ifdef CONFIG_LOCK_STAT | 142 | #ifdef CONFIG_LOCK_STAT |
| 143 | static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], lock_stats); | 143 | static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], |
| 144 | cpu_lock_stats); | ||
| 144 | 145 | ||
| 145 | static inline u64 lockstat_clock(void) | 146 | static inline u64 lockstat_clock(void) |
| 146 | { | 147 | { |
| @@ -198,7 +199,7 @@ struct lock_class_stats lock_stats(struct lock_class *class) | |||
| 198 | memset(&stats, 0, sizeof(struct lock_class_stats)); | 199 | memset(&stats, 0, sizeof(struct lock_class_stats)); |
| 199 | for_each_possible_cpu(cpu) { | 200 | for_each_possible_cpu(cpu) { |
| 200 | struct lock_class_stats *pcs = | 201 | struct lock_class_stats *pcs = |
| 201 | &per_cpu(lock_stats, cpu)[class - lock_classes]; | 202 | &per_cpu(cpu_lock_stats, cpu)[class - lock_classes]; |
| 202 | 203 | ||
| 203 | for (i = 0; i < ARRAY_SIZE(stats.contention_point); i++) | 204 | for (i = 0; i < ARRAY_SIZE(stats.contention_point); i++) |
| 204 | stats.contention_point[i] += pcs->contention_point[i]; | 205 | stats.contention_point[i] += pcs->contention_point[i]; |
| @@ -225,7 +226,7 @@ void clear_lock_stats(struct lock_class *class) | |||
| 225 | 226 | ||
| 226 | for_each_possible_cpu(cpu) { | 227 | for_each_possible_cpu(cpu) { |
| 227 | struct lock_class_stats *cpu_stats = | 228 | struct lock_class_stats *cpu_stats = |
| 228 | &per_cpu(lock_stats, cpu)[class - lock_classes]; | 229 | &per_cpu(cpu_lock_stats, cpu)[class - lock_classes]; |
| 229 | 230 | ||
| 230 | memset(cpu_stats, 0, sizeof(struct lock_class_stats)); | 231 | memset(cpu_stats, 0, sizeof(struct lock_class_stats)); |
| 231 | } | 232 | } |
| @@ -235,12 +236,12 @@ void clear_lock_stats(struct lock_class *class) | |||
| 235 | 236 | ||
| 236 | static struct lock_class_stats *get_lock_stats(struct lock_class *class) | 237 | static struct lock_class_stats *get_lock_stats(struct lock_class *class) |
| 237 | { | 238 | { |
| 238 | return &get_cpu_var(lock_stats)[class - lock_classes]; | 239 | return &get_cpu_var(cpu_lock_stats)[class - lock_classes]; |
| 239 | } | 240 | } |
| 240 | 241 | ||
| 241 | static void put_lock_stats(struct lock_class_stats *stats) | 242 | static void put_lock_stats(struct lock_class_stats *stats) |
| 242 | { | 243 | { |
| 243 | put_cpu_var(lock_stats); | 244 | put_cpu_var(cpu_lock_stats); |
| 244 | } | 245 | } |
| 245 | 246 | ||
| 246 | static void lock_release_holdtime(struct held_lock *hlock) | 247 | static void lock_release_holdtime(struct held_lock *hlock) |
| @@ -1169,9 +1170,9 @@ unsigned long lockdep_count_forward_deps(struct lock_class *class) | |||
| 1169 | this.class = class; | 1170 | this.class = class; |
| 1170 | 1171 | ||
| 1171 | local_irq_save(flags); | 1172 | local_irq_save(flags); |
| 1172 | __raw_spin_lock(&lockdep_lock); | 1173 | arch_spin_lock(&lockdep_lock); |
| 1173 | ret = __lockdep_count_forward_deps(&this); | 1174 | ret = __lockdep_count_forward_deps(&this); |
| 1174 | __raw_spin_unlock(&lockdep_lock); | 1175 | arch_spin_unlock(&lockdep_lock); |
| 1175 | local_irq_restore(flags); | 1176 | local_irq_restore(flags); |
| 1176 | 1177 | ||
| 1177 | return ret; | 1178 | return ret; |
| @@ -1196,9 +1197,9 @@ unsigned long lockdep_count_backward_deps(struct lock_class *class) | |||
| 1196 | this.class = class; | 1197 | this.class = class; |
| 1197 | 1198 | ||
| 1198 | local_irq_save(flags); | 1199 | local_irq_save(flags); |
| 1199 | __raw_spin_lock(&lockdep_lock); | 1200 | arch_spin_lock(&lockdep_lock); |
| 1200 | ret = __lockdep_count_backward_deps(&this); | 1201 | ret = __lockdep_count_backward_deps(&this); |
| 1201 | __raw_spin_unlock(&lockdep_lock); | 1202 | arch_spin_unlock(&lockdep_lock); |
| 1202 | local_irq_restore(flags); | 1203 | local_irq_restore(flags); |
| 1203 | 1204 | ||
| 1204 | return ret; | 1205 | return ret; |
diff --git a/kernel/module.c b/kernel/module.c index 5842a71cf052..12afc5a3ddd3 100644 --- a/kernel/module.c +++ b/kernel/module.c | |||
| @@ -370,8 +370,6 @@ EXPORT_SYMBOL_GPL(find_module); | |||
| 370 | 370 | ||
| 371 | #ifdef CONFIG_SMP | 371 | #ifdef CONFIG_SMP |
| 372 | 372 | ||
| 373 | #ifndef CONFIG_HAVE_LEGACY_PER_CPU_AREA | ||
| 374 | |||
| 375 | static void *percpu_modalloc(unsigned long size, unsigned long align, | 373 | static void *percpu_modalloc(unsigned long size, unsigned long align, |
| 376 | const char *name) | 374 | const char *name) |
| 377 | { | 375 | { |
| @@ -395,154 +393,6 @@ static void percpu_modfree(void *freeme) | |||
| 395 | free_percpu(freeme); | 393 | free_percpu(freeme); |
| 396 | } | 394 | } |
| 397 | 395 | ||
| 398 | #else /* ... CONFIG_HAVE_LEGACY_PER_CPU_AREA */ | ||
| 399 | |||
| 400 | /* Number of blocks used and allocated. */ | ||
| 401 | static unsigned int pcpu_num_used, pcpu_num_allocated; | ||
| 402 | /* Size of each block. -ve means used. */ | ||
| 403 | static int *pcpu_size; | ||
| 404 | |||
| 405 | static int split_block(unsigned int i, unsigned short size) | ||
| 406 | { | ||
| 407 | /* Reallocation required? */ | ||
| 408 | if (pcpu_num_used + 1 > pcpu_num_allocated) { | ||
| 409 | int *new; | ||
| 410 | |||
| 411 | new = krealloc(pcpu_size, sizeof(new[0])*pcpu_num_allocated*2, | ||
| 412 | GFP_KERNEL); | ||
| 413 | if (!new) | ||
| 414 | return 0; | ||
| 415 | |||
| 416 | pcpu_num_allocated *= 2; | ||
| 417 | pcpu_size = new; | ||
| 418 | } | ||
| 419 | |||
| 420 | /* Insert a new subblock */ | ||
| 421 | memmove(&pcpu_size[i+1], &pcpu_size[i], | ||
| 422 | sizeof(pcpu_size[0]) * (pcpu_num_used - i)); | ||
| 423 | pcpu_num_used++; | ||
| 424 | |||
| 425 | pcpu_size[i+1] -= size; | ||
| 426 | pcpu_size[i] = size; | ||
| 427 | return 1; | ||
| 428 | } | ||
| 429 | |||
| 430 | static inline unsigned int block_size(int val) | ||
| 431 | { | ||
| 432 | if (val < 0) | ||
| 433 | return -val; | ||
| 434 | return val; | ||
| 435 | } | ||
| 436 | |||
| 437 | static void *percpu_modalloc(unsigned long size, unsigned long align, | ||
| 438 | const char *name) | ||
| 439 | { | ||
| 440 | unsigned long extra; | ||
| 441 | unsigned int i; | ||
| 442 | void *ptr; | ||
| 443 | int cpu; | ||
| 444 | |||
| 445 | if (align > PAGE_SIZE) { | ||
| 446 | printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n", | ||
| 447 | name, align, PAGE_SIZE); | ||
| 448 | align = PAGE_SIZE; | ||
| 449 | } | ||
| 450 | |||
| 451 | ptr = __per_cpu_start; | ||
| 452 | for (i = 0; i < pcpu_num_used; ptr += block_size(pcpu_size[i]), i++) { | ||
| 453 | /* Extra for alignment requirement. */ | ||
| 454 | extra = ALIGN((unsigned long)ptr, align) - (unsigned long)ptr; | ||
| 455 | BUG_ON(i == 0 && extra != 0); | ||
| 456 | |||
| 457 | if (pcpu_size[i] < 0 || pcpu_size[i] < extra + size) | ||
| 458 | continue; | ||
| 459 | |||
| 460 | /* Transfer extra to previous block. */ | ||
| 461 | if (pcpu_size[i-1] < 0) | ||
| 462 | pcpu_size[i-1] -= extra; | ||
| 463 | else | ||
| 464 | pcpu_size[i-1] += extra; | ||
| 465 | pcpu_size[i] -= extra; | ||
| 466 | ptr += extra; | ||
| 467 | |||
| 468 | /* Split block if warranted */ | ||
| 469 | if (pcpu_size[i] - size > sizeof(unsigned long)) | ||
| 470 | if (!split_block(i, size)) | ||
| 471 | return NULL; | ||
| 472 | |||
| 473 | /* add the per-cpu scanning areas */ | ||
| 474 | for_each_possible_cpu(cpu) | ||
| 475 | kmemleak_alloc(ptr + per_cpu_offset(cpu), size, 0, | ||
| 476 | GFP_KERNEL); | ||
| 477 | |||
| 478 | /* Mark allocated */ | ||
| 479 | pcpu_size[i] = -pcpu_size[i]; | ||
| 480 | return ptr; | ||
| 481 | } | ||
| 482 | |||
| 483 | printk(KERN_WARNING "Could not allocate %lu bytes percpu data\n", | ||
| 484 | size); | ||
| 485 | return NULL; | ||
| 486 | } | ||
| 487 | |||
| 488 | static void percpu_modfree(void *freeme) | ||
| 489 | { | ||
| 490 | unsigned int i; | ||
| 491 | void *ptr = __per_cpu_start + block_size(pcpu_size[0]); | ||
| 492 | int cpu; | ||
| 493 | |||
| 494 | /* First entry is core kernel percpu data. */ | ||
| 495 | for (i = 1; i < pcpu_num_used; ptr += block_size(pcpu_size[i]), i++) { | ||
| 496 | if (ptr == freeme) { | ||
| 497 | pcpu_size[i] = -pcpu_size[i]; | ||
| 498 | goto free; | ||
| 499 | } | ||
| 500 | } | ||
| 501 | BUG(); | ||
| 502 | |||
| 503 | free: | ||
| 504 | /* remove the per-cpu scanning areas */ | ||
| 505 | for_each_possible_cpu(cpu) | ||
| 506 | kmemleak_free(freeme + per_cpu_offset(cpu)); | ||
| 507 | |||
| 508 | /* Merge with previous? */ | ||
| 509 | if (pcpu_size[i-1] >= 0) { | ||
| 510 | pcpu_size[i-1] += pcpu_size[i]; | ||
| 511 | pcpu_num_used--; | ||
| 512 | memmove(&pcpu_size[i], &pcpu_size[i+1], | ||
| 513 | (pcpu_num_used - i) * sizeof(pcpu_size[0])); | ||
| 514 | i--; | ||
| 515 | } | ||
| 516 | /* Merge with next? */ | ||
| 517 | if (i+1 < pcpu_num_used && pcpu_size[i+1] >= 0) { | ||
| 518 | pcpu_size[i] += pcpu_size[i+1]; | ||
| 519 | pcpu_num_used--; | ||
| 520 | memmove(&pcpu_size[i+1], &pcpu_size[i+2], | ||
| 521 | (pcpu_num_used - (i+1)) * sizeof(pcpu_size[0])); | ||
| 522 | } | ||
| 523 | } | ||
| 524 | |||
| 525 | static int percpu_modinit(void) | ||
| 526 | { | ||
| 527 | pcpu_num_used = 2; | ||
| 528 | pcpu_num_allocated = 2; | ||
| 529 | pcpu_size = kmalloc(sizeof(pcpu_size[0]) * pcpu_num_allocated, | ||
| 530 | GFP_KERNEL); | ||
| 531 | /* Static in-kernel percpu data (used). */ | ||
| 532 | pcpu_size[0] = -(__per_cpu_end-__per_cpu_start); | ||
| 533 | /* Free room. */ | ||
| 534 | pcpu_size[1] = PERCPU_ENOUGH_ROOM + pcpu_size[0]; | ||
| 535 | if (pcpu_size[1] < 0) { | ||
| 536 | printk(KERN_ERR "No per-cpu room for modules.\n"); | ||
| 537 | pcpu_num_used = 1; | ||
| 538 | } | ||
| 539 | |||
| 540 | return 0; | ||
| 541 | } | ||
| 542 | __initcall(percpu_modinit); | ||
| 543 | |||
| 544 | #endif /* CONFIG_HAVE_LEGACY_PER_CPU_AREA */ | ||
| 545 | |||
| 546 | static unsigned int find_pcpusec(Elf_Ehdr *hdr, | 396 | static unsigned int find_pcpusec(Elf_Ehdr *hdr, |
| 547 | Elf_Shdr *sechdrs, | 397 | Elf_Shdr *sechdrs, |
| 548 | const char *secstrings) | 398 | const char *secstrings) |
diff --git a/kernel/mutex-debug.h b/kernel/mutex-debug.h index 6b2d735846a5..57d527a16f9d 100644 --- a/kernel/mutex-debug.h +++ b/kernel/mutex-debug.h | |||
| @@ -43,13 +43,13 @@ static inline void mutex_clear_owner(struct mutex *lock) | |||
| 43 | \ | 43 | \ |
| 44 | DEBUG_LOCKS_WARN_ON(in_interrupt()); \ | 44 | DEBUG_LOCKS_WARN_ON(in_interrupt()); \ |
| 45 | local_irq_save(flags); \ | 45 | local_irq_save(flags); \ |
| 46 | __raw_spin_lock(&(lock)->raw_lock); \ | 46 | arch_spin_lock(&(lock)->rlock.raw_lock);\ |
| 47 | DEBUG_LOCKS_WARN_ON(l->magic != l); \ | 47 | DEBUG_LOCKS_WARN_ON(l->magic != l); \ |
| 48 | } while (0) | 48 | } while (0) |
| 49 | 49 | ||
| 50 | #define spin_unlock_mutex(lock, flags) \ | 50 | #define spin_unlock_mutex(lock, flags) \ |
| 51 | do { \ | 51 | do { \ |
| 52 | __raw_spin_unlock(&(lock)->raw_lock); \ | 52 | arch_spin_unlock(&(lock)->rlock.raw_lock); \ |
| 53 | local_irq_restore(flags); \ | 53 | local_irq_restore(flags); \ |
| 54 | preempt_check_resched(); \ | 54 | preempt_check_resched(); \ |
| 55 | } while (0) | 55 | } while (0) |
diff --git a/kernel/params.c b/kernel/params.c index d656c276508d..cf1b69183127 100644 --- a/kernel/params.c +++ b/kernel/params.c | |||
| @@ -24,6 +24,7 @@ | |||
| 24 | #include <linux/err.h> | 24 | #include <linux/err.h> |
| 25 | #include <linux/slab.h> | 25 | #include <linux/slab.h> |
| 26 | #include <linux/ctype.h> | 26 | #include <linux/ctype.h> |
| 27 | #include <linux/string.h> | ||
| 27 | 28 | ||
| 28 | #if 0 | 29 | #if 0 |
| 29 | #define DEBUGP printk | 30 | #define DEBUGP printk |
| @@ -122,9 +123,7 @@ static char *next_arg(char *args, char **param, char **val) | |||
| 122 | next = args + i; | 123 | next = args + i; |
| 123 | 124 | ||
| 124 | /* Chew up trailing spaces. */ | 125 | /* Chew up trailing spaces. */ |
| 125 | while (isspace(*next)) | 126 | return skip_spaces(next); |
| 126 | next++; | ||
| 127 | return next; | ||
| 128 | } | 127 | } |
| 129 | 128 | ||
| 130 | /* Args looks like "foo=bar,bar2 baz=fuz wiz". */ | 129 | /* Args looks like "foo=bar,bar2 baz=fuz wiz". */ |
| @@ -139,8 +138,7 @@ int parse_args(const char *name, | |||
| 139 | DEBUGP("Parsing ARGS: %s\n", args); | 138 | DEBUGP("Parsing ARGS: %s\n", args); |
| 140 | 139 | ||
| 141 | /* Chew leading spaces */ | 140 | /* Chew leading spaces */ |
| 142 | while (isspace(*args)) | 141 | args = skip_spaces(args); |
| 143 | args++; | ||
| 144 | 142 | ||
| 145 | while (*args) { | 143 | while (*args) { |
| 146 | int ret; | 144 | int ret; |
diff --git a/kernel/perf_event.c b/kernel/perf_event.c index e73e53c7582f..9052d6c8c9fd 100644 --- a/kernel/perf_event.c +++ b/kernel/perf_event.c | |||
| @@ -203,14 +203,14 @@ perf_lock_task_context(struct task_struct *task, unsigned long *flags) | |||
| 203 | * if so. If we locked the right context, then it | 203 | * if so. If we locked the right context, then it |
| 204 | * can't get swapped on us any more. | 204 | * can't get swapped on us any more. |
| 205 | */ | 205 | */ |
| 206 | spin_lock_irqsave(&ctx->lock, *flags); | 206 | raw_spin_lock_irqsave(&ctx->lock, *flags); |
| 207 | if (ctx != rcu_dereference(task->perf_event_ctxp)) { | 207 | if (ctx != rcu_dereference(task->perf_event_ctxp)) { |
| 208 | spin_unlock_irqrestore(&ctx->lock, *flags); | 208 | raw_spin_unlock_irqrestore(&ctx->lock, *flags); |
| 209 | goto retry; | 209 | goto retry; |
| 210 | } | 210 | } |
| 211 | 211 | ||
| 212 | if (!atomic_inc_not_zero(&ctx->refcount)) { | 212 | if (!atomic_inc_not_zero(&ctx->refcount)) { |
| 213 | spin_unlock_irqrestore(&ctx->lock, *flags); | 213 | raw_spin_unlock_irqrestore(&ctx->lock, *flags); |
| 214 | ctx = NULL; | 214 | ctx = NULL; |
| 215 | } | 215 | } |
| 216 | } | 216 | } |
| @@ -231,7 +231,7 @@ static struct perf_event_context *perf_pin_task_context(struct task_struct *task | |||
| 231 | ctx = perf_lock_task_context(task, &flags); | 231 | ctx = perf_lock_task_context(task, &flags); |
| 232 | if (ctx) { | 232 | if (ctx) { |
| 233 | ++ctx->pin_count; | 233 | ++ctx->pin_count; |
| 234 | spin_unlock_irqrestore(&ctx->lock, flags); | 234 | raw_spin_unlock_irqrestore(&ctx->lock, flags); |
| 235 | } | 235 | } |
| 236 | return ctx; | 236 | return ctx; |
| 237 | } | 237 | } |
| @@ -240,9 +240,9 @@ static void perf_unpin_context(struct perf_event_context *ctx) | |||
| 240 | { | 240 | { |
| 241 | unsigned long flags; | 241 | unsigned long flags; |
| 242 | 242 | ||
| 243 | spin_lock_irqsave(&ctx->lock, flags); | 243 | raw_spin_lock_irqsave(&ctx->lock, flags); |
| 244 | --ctx->pin_count; | 244 | --ctx->pin_count; |
| 245 | spin_unlock_irqrestore(&ctx->lock, flags); | 245 | raw_spin_unlock_irqrestore(&ctx->lock, flags); |
| 246 | put_ctx(ctx); | 246 | put_ctx(ctx); |
| 247 | } | 247 | } |
| 248 | 248 | ||
| @@ -427,7 +427,7 @@ static void __perf_event_remove_from_context(void *info) | |||
| 427 | if (ctx->task && cpuctx->task_ctx != ctx) | 427 | if (ctx->task && cpuctx->task_ctx != ctx) |
| 428 | return; | 428 | return; |
| 429 | 429 | ||
| 430 | spin_lock(&ctx->lock); | 430 | raw_spin_lock(&ctx->lock); |
| 431 | /* | 431 | /* |
| 432 | * Protect the list operation against NMI by disabling the | 432 | * Protect the list operation against NMI by disabling the |
| 433 | * events on a global level. | 433 | * events on a global level. |
| @@ -449,7 +449,7 @@ static void __perf_event_remove_from_context(void *info) | |||
| 449 | } | 449 | } |
| 450 | 450 | ||
| 451 | perf_enable(); | 451 | perf_enable(); |
| 452 | spin_unlock(&ctx->lock); | 452 | raw_spin_unlock(&ctx->lock); |
| 453 | } | 453 | } |
| 454 | 454 | ||
| 455 | 455 | ||
| @@ -488,12 +488,12 @@ retry: | |||
| 488 | task_oncpu_function_call(task, __perf_event_remove_from_context, | 488 | task_oncpu_function_call(task, __perf_event_remove_from_context, |
| 489 | event); | 489 | event); |
| 490 | 490 | ||
| 491 | spin_lock_irq(&ctx->lock); | 491 | raw_spin_lock_irq(&ctx->lock); |
| 492 | /* | 492 | /* |
| 493 | * If the context is active we need to retry the smp call. | 493 | * If the context is active we need to retry the smp call. |
| 494 | */ | 494 | */ |
| 495 | if (ctx->nr_active && !list_empty(&event->group_entry)) { | 495 | if (ctx->nr_active && !list_empty(&event->group_entry)) { |
| 496 | spin_unlock_irq(&ctx->lock); | 496 | raw_spin_unlock_irq(&ctx->lock); |
| 497 | goto retry; | 497 | goto retry; |
| 498 | } | 498 | } |
| 499 | 499 | ||
| @@ -504,7 +504,7 @@ retry: | |||
| 504 | */ | 504 | */ |
| 505 | if (!list_empty(&event->group_entry)) | 505 | if (!list_empty(&event->group_entry)) |
| 506 | list_del_event(event, ctx); | 506 | list_del_event(event, ctx); |
| 507 | spin_unlock_irq(&ctx->lock); | 507 | raw_spin_unlock_irq(&ctx->lock); |
| 508 | } | 508 | } |
| 509 | 509 | ||
| 510 | /* | 510 | /* |
| @@ -535,7 +535,7 @@ static void __perf_event_disable(void *info) | |||
| 535 | if (ctx->task && cpuctx->task_ctx != ctx) | 535 | if (ctx->task && cpuctx->task_ctx != ctx) |
| 536 | return; | 536 | return; |
| 537 | 537 | ||
| 538 | spin_lock(&ctx->lock); | 538 | raw_spin_lock(&ctx->lock); |
| 539 | 539 | ||
| 540 | /* | 540 | /* |
| 541 | * If the event is on, turn it off. | 541 | * If the event is on, turn it off. |
| @@ -551,7 +551,7 @@ static void __perf_event_disable(void *info) | |||
| 551 | event->state = PERF_EVENT_STATE_OFF; | 551 | event->state = PERF_EVENT_STATE_OFF; |
| 552 | } | 552 | } |
| 553 | 553 | ||
| 554 | spin_unlock(&ctx->lock); | 554 | raw_spin_unlock(&ctx->lock); |
| 555 | } | 555 | } |
| 556 | 556 | ||
| 557 | /* | 557 | /* |
| @@ -584,12 +584,12 @@ void perf_event_disable(struct perf_event *event) | |||
| 584 | retry: | 584 | retry: |
| 585 | task_oncpu_function_call(task, __perf_event_disable, event); | 585 | task_oncpu_function_call(task, __perf_event_disable, event); |
| 586 | 586 | ||
| 587 | spin_lock_irq(&ctx->lock); | 587 | raw_spin_lock_irq(&ctx->lock); |
| 588 | /* | 588 | /* |
| 589 | * If the event is still active, we need to retry the cross-call. | 589 | * If the event is still active, we need to retry the cross-call. |
| 590 | */ | 590 | */ |
| 591 | if (event->state == PERF_EVENT_STATE_ACTIVE) { | 591 | if (event->state == PERF_EVENT_STATE_ACTIVE) { |
| 592 | spin_unlock_irq(&ctx->lock); | 592 | raw_spin_unlock_irq(&ctx->lock); |
| 593 | goto retry; | 593 | goto retry; |
| 594 | } | 594 | } |
| 595 | 595 | ||
| @@ -602,7 +602,7 @@ void perf_event_disable(struct perf_event *event) | |||
| 602 | event->state = PERF_EVENT_STATE_OFF; | 602 | event->state = PERF_EVENT_STATE_OFF; |
| 603 | } | 603 | } |
| 604 | 604 | ||
| 605 | spin_unlock_irq(&ctx->lock); | 605 | raw_spin_unlock_irq(&ctx->lock); |
| 606 | } | 606 | } |
| 607 | 607 | ||
| 608 | static int | 608 | static int |
| @@ -770,7 +770,7 @@ static void __perf_install_in_context(void *info) | |||
| 770 | cpuctx->task_ctx = ctx; | 770 | cpuctx->task_ctx = ctx; |
| 771 | } | 771 | } |
| 772 | 772 | ||
| 773 | spin_lock(&ctx->lock); | 773 | raw_spin_lock(&ctx->lock); |
| 774 | ctx->is_active = 1; | 774 | ctx->is_active = 1; |
| 775 | update_context_time(ctx); | 775 | update_context_time(ctx); |
| 776 | 776 | ||
| @@ -820,7 +820,7 @@ static void __perf_install_in_context(void *info) | |||
| 820 | unlock: | 820 | unlock: |
| 821 | perf_enable(); | 821 | perf_enable(); |
| 822 | 822 | ||
| 823 | spin_unlock(&ctx->lock); | 823 | raw_spin_unlock(&ctx->lock); |
| 824 | } | 824 | } |
| 825 | 825 | ||
| 826 | /* | 826 | /* |
| @@ -856,12 +856,12 @@ retry: | |||
| 856 | task_oncpu_function_call(task, __perf_install_in_context, | 856 | task_oncpu_function_call(task, __perf_install_in_context, |
| 857 | event); | 857 | event); |
| 858 | 858 | ||
| 859 | spin_lock_irq(&ctx->lock); | 859 | raw_spin_lock_irq(&ctx->lock); |
| 860 | /* | 860 | /* |
| 861 | * we need to retry the smp call. | 861 | * we need to retry the smp call. |
| 862 | */ | 862 | */ |
| 863 | if (ctx->is_active && list_empty(&event->group_entry)) { | 863 | if (ctx->is_active && list_empty(&event->group_entry)) { |
| 864 | spin_unlock_irq(&ctx->lock); | 864 | raw_spin_unlock_irq(&ctx->lock); |
| 865 | goto retry; | 865 | goto retry; |
| 866 | } | 866 | } |
| 867 | 867 | ||
| @@ -872,7 +872,7 @@ retry: | |||
| 872 | */ | 872 | */ |
| 873 | if (list_empty(&event->group_entry)) | 873 | if (list_empty(&event->group_entry)) |
| 874 | add_event_to_ctx(event, ctx); | 874 | add_event_to_ctx(event, ctx); |
| 875 | spin_unlock_irq(&ctx->lock); | 875 | raw_spin_unlock_irq(&ctx->lock); |
| 876 | } | 876 | } |
| 877 | 877 | ||
| 878 | /* | 878 | /* |
| @@ -917,7 +917,7 @@ static void __perf_event_enable(void *info) | |||
| 917 | cpuctx->task_ctx = ctx; | 917 | cpuctx->task_ctx = ctx; |
| 918 | } | 918 | } |
| 919 | 919 | ||
| 920 | spin_lock(&ctx->lock); | 920 | raw_spin_lock(&ctx->lock); |
| 921 | ctx->is_active = 1; | 921 | ctx->is_active = 1; |
| 922 | update_context_time(ctx); | 922 | update_context_time(ctx); |
| 923 | 923 | ||
| @@ -959,7 +959,7 @@ static void __perf_event_enable(void *info) | |||
| 959 | } | 959 | } |
| 960 | 960 | ||
| 961 | unlock: | 961 | unlock: |
| 962 | spin_unlock(&ctx->lock); | 962 | raw_spin_unlock(&ctx->lock); |
| 963 | } | 963 | } |
| 964 | 964 | ||
| 965 | /* | 965 | /* |
| @@ -985,7 +985,7 @@ void perf_event_enable(struct perf_event *event) | |||
| 985 | return; | 985 | return; |
| 986 | } | 986 | } |
| 987 | 987 | ||
| 988 | spin_lock_irq(&ctx->lock); | 988 | raw_spin_lock_irq(&ctx->lock); |
| 989 | if (event->state >= PERF_EVENT_STATE_INACTIVE) | 989 | if (event->state >= PERF_EVENT_STATE_INACTIVE) |
| 990 | goto out; | 990 | goto out; |
| 991 | 991 | ||
| @@ -1000,10 +1000,10 @@ void perf_event_enable(struct perf_event *event) | |||
| 1000 | event->state = PERF_EVENT_STATE_OFF; | 1000 | event->state = PERF_EVENT_STATE_OFF; |
| 1001 | 1001 | ||
| 1002 | retry: | 1002 | retry: |
| 1003 | spin_unlock_irq(&ctx->lock); | 1003 | raw_spin_unlock_irq(&ctx->lock); |
| 1004 | task_oncpu_function_call(task, __perf_event_enable, event); | 1004 | task_oncpu_function_call(task, __perf_event_enable, event); |
| 1005 | 1005 | ||
| 1006 | spin_lock_irq(&ctx->lock); | 1006 | raw_spin_lock_irq(&ctx->lock); |
| 1007 | 1007 | ||
| 1008 | /* | 1008 | /* |
| 1009 | * If the context is active and the event is still off, | 1009 | * If the context is active and the event is still off, |
| @@ -1020,7 +1020,7 @@ void perf_event_enable(struct perf_event *event) | |||
| 1020 | __perf_event_mark_enabled(event, ctx); | 1020 | __perf_event_mark_enabled(event, ctx); |
| 1021 | 1021 | ||
| 1022 | out: | 1022 | out: |
| 1023 | spin_unlock_irq(&ctx->lock); | 1023 | raw_spin_unlock_irq(&ctx->lock); |
| 1024 | } | 1024 | } |
| 1025 | 1025 | ||
| 1026 | static int perf_event_refresh(struct perf_event *event, int refresh) | 1026 | static int perf_event_refresh(struct perf_event *event, int refresh) |
| @@ -1042,7 +1042,7 @@ void __perf_event_sched_out(struct perf_event_context *ctx, | |||
| 1042 | { | 1042 | { |
| 1043 | struct perf_event *event; | 1043 | struct perf_event *event; |
| 1044 | 1044 | ||
| 1045 | spin_lock(&ctx->lock); | 1045 | raw_spin_lock(&ctx->lock); |
| 1046 | ctx->is_active = 0; | 1046 | ctx->is_active = 0; |
| 1047 | if (likely(!ctx->nr_events)) | 1047 | if (likely(!ctx->nr_events)) |
| 1048 | goto out; | 1048 | goto out; |
| @@ -1055,7 +1055,7 @@ void __perf_event_sched_out(struct perf_event_context *ctx, | |||
| 1055 | } | 1055 | } |
| 1056 | perf_enable(); | 1056 | perf_enable(); |
| 1057 | out: | 1057 | out: |
| 1058 | spin_unlock(&ctx->lock); | 1058 | raw_spin_unlock(&ctx->lock); |
| 1059 | } | 1059 | } |
| 1060 | 1060 | ||
| 1061 | /* | 1061 | /* |
| @@ -1193,8 +1193,8 @@ void perf_event_task_sched_out(struct task_struct *task, | |||
| 1193 | * order we take the locks because no other cpu could | 1193 | * order we take the locks because no other cpu could |
| 1194 | * be trying to lock both of these tasks. | 1194 | * be trying to lock both of these tasks. |
| 1195 | */ | 1195 | */ |
| 1196 | spin_lock(&ctx->lock); | 1196 | raw_spin_lock(&ctx->lock); |
| 1197 | spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); | 1197 | raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); |
| 1198 | if (context_equiv(ctx, next_ctx)) { | 1198 | if (context_equiv(ctx, next_ctx)) { |
| 1199 | /* | 1199 | /* |
| 1200 | * XXX do we need a memory barrier of sorts | 1200 | * XXX do we need a memory barrier of sorts |
| @@ -1208,8 +1208,8 @@ void perf_event_task_sched_out(struct task_struct *task, | |||
| 1208 | 1208 | ||
| 1209 | perf_event_sync_stat(ctx, next_ctx); | 1209 | perf_event_sync_stat(ctx, next_ctx); |
| 1210 | } | 1210 | } |
| 1211 | spin_unlock(&next_ctx->lock); | 1211 | raw_spin_unlock(&next_ctx->lock); |
| 1212 | spin_unlock(&ctx->lock); | 1212 | raw_spin_unlock(&ctx->lock); |
| 1213 | } | 1213 | } |
| 1214 | rcu_read_unlock(); | 1214 | rcu_read_unlock(); |
| 1215 | 1215 | ||
| @@ -1251,7 +1251,7 @@ __perf_event_sched_in(struct perf_event_context *ctx, | |||
| 1251 | struct perf_event *event; | 1251 | struct perf_event *event; |
| 1252 | int can_add_hw = 1; | 1252 | int can_add_hw = 1; |
| 1253 | 1253 | ||
| 1254 | spin_lock(&ctx->lock); | 1254 | raw_spin_lock(&ctx->lock); |
| 1255 | ctx->is_active = 1; | 1255 | ctx->is_active = 1; |
| 1256 | if (likely(!ctx->nr_events)) | 1256 | if (likely(!ctx->nr_events)) |
| 1257 | goto out; | 1257 | goto out; |
| @@ -1306,7 +1306,7 @@ __perf_event_sched_in(struct perf_event_context *ctx, | |||
| 1306 | } | 1306 | } |
| 1307 | perf_enable(); | 1307 | perf_enable(); |
| 1308 | out: | 1308 | out: |
| 1309 | spin_unlock(&ctx->lock); | 1309 | raw_spin_unlock(&ctx->lock); |
| 1310 | } | 1310 | } |
| 1311 | 1311 | ||
| 1312 | /* | 1312 | /* |
| @@ -1370,7 +1370,7 @@ static void perf_ctx_adjust_freq(struct perf_event_context *ctx) | |||
| 1370 | struct hw_perf_event *hwc; | 1370 | struct hw_perf_event *hwc; |
| 1371 | u64 interrupts, freq; | 1371 | u64 interrupts, freq; |
| 1372 | 1372 | ||
| 1373 | spin_lock(&ctx->lock); | 1373 | raw_spin_lock(&ctx->lock); |
| 1374 | list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { | 1374 | list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { |
| 1375 | if (event->state != PERF_EVENT_STATE_ACTIVE) | 1375 | if (event->state != PERF_EVENT_STATE_ACTIVE) |
| 1376 | continue; | 1376 | continue; |
| @@ -1425,7 +1425,7 @@ static void perf_ctx_adjust_freq(struct perf_event_context *ctx) | |||
| 1425 | perf_enable(); | 1425 | perf_enable(); |
| 1426 | } | 1426 | } |
| 1427 | } | 1427 | } |
| 1428 | spin_unlock(&ctx->lock); | 1428 | raw_spin_unlock(&ctx->lock); |
| 1429 | } | 1429 | } |
| 1430 | 1430 | ||
| 1431 | /* | 1431 | /* |
| @@ -1438,7 +1438,7 @@ static void rotate_ctx(struct perf_event_context *ctx) | |||
| 1438 | if (!ctx->nr_events) | 1438 | if (!ctx->nr_events) |
| 1439 | return; | 1439 | return; |
| 1440 | 1440 | ||
| 1441 | spin_lock(&ctx->lock); | 1441 | raw_spin_lock(&ctx->lock); |
| 1442 | /* | 1442 | /* |
| 1443 | * Rotate the first entry last (works just fine for group events too): | 1443 | * Rotate the first entry last (works just fine for group events too): |
| 1444 | */ | 1444 | */ |
| @@ -1449,7 +1449,7 @@ static void rotate_ctx(struct perf_event_context *ctx) | |||
| 1449 | } | 1449 | } |
| 1450 | perf_enable(); | 1450 | perf_enable(); |
| 1451 | 1451 | ||
| 1452 | spin_unlock(&ctx->lock); | 1452 | raw_spin_unlock(&ctx->lock); |
| 1453 | } | 1453 | } |
| 1454 | 1454 | ||
| 1455 | void perf_event_task_tick(struct task_struct *curr, int cpu) | 1455 | void perf_event_task_tick(struct task_struct *curr, int cpu) |
| @@ -1498,7 +1498,7 @@ static void perf_event_enable_on_exec(struct task_struct *task) | |||
| 1498 | 1498 | ||
| 1499 | __perf_event_task_sched_out(ctx); | 1499 | __perf_event_task_sched_out(ctx); |
| 1500 | 1500 | ||
| 1501 | spin_lock(&ctx->lock); | 1501 | raw_spin_lock(&ctx->lock); |
| 1502 | 1502 | ||
| 1503 | list_for_each_entry(event, &ctx->group_list, group_entry) { | 1503 | list_for_each_entry(event, &ctx->group_list, group_entry) { |
| 1504 | if (!event->attr.enable_on_exec) | 1504 | if (!event->attr.enable_on_exec) |
| @@ -1516,7 +1516,7 @@ static void perf_event_enable_on_exec(struct task_struct *task) | |||
| 1516 | if (enabled) | 1516 | if (enabled) |
| 1517 | unclone_ctx(ctx); | 1517 | unclone_ctx(ctx); |
| 1518 | 1518 | ||
| 1519 | spin_unlock(&ctx->lock); | 1519 | raw_spin_unlock(&ctx->lock); |
| 1520 | 1520 | ||
| 1521 | perf_event_task_sched_in(task, smp_processor_id()); | 1521 | perf_event_task_sched_in(task, smp_processor_id()); |
| 1522 | out: | 1522 | out: |
| @@ -1542,10 +1542,10 @@ static void __perf_event_read(void *info) | |||
| 1542 | if (ctx->task && cpuctx->task_ctx != ctx) | 1542 | if (ctx->task && cpuctx->task_ctx != ctx) |
| 1543 | return; | 1543 | return; |
| 1544 | 1544 | ||
| 1545 | spin_lock(&ctx->lock); | 1545 | raw_spin_lock(&ctx->lock); |
| 1546 | update_context_time(ctx); | 1546 | update_context_time(ctx); |
| 1547 | update_event_times(event); | 1547 | update_event_times(event); |
| 1548 | spin_unlock(&ctx->lock); | 1548 | raw_spin_unlock(&ctx->lock); |
| 1549 | 1549 | ||
| 1550 | event->pmu->read(event); | 1550 | event->pmu->read(event); |
| 1551 | } | 1551 | } |
| @@ -1563,10 +1563,10 @@ static u64 perf_event_read(struct perf_event *event) | |||
| 1563 | struct perf_event_context *ctx = event->ctx; | 1563 | struct perf_event_context *ctx = event->ctx; |
| 1564 | unsigned long flags; | 1564 | unsigned long flags; |
| 1565 | 1565 | ||
| 1566 | spin_lock_irqsave(&ctx->lock, flags); | 1566 | raw_spin_lock_irqsave(&ctx->lock, flags); |
| 1567 | update_context_time(ctx); | 1567 | update_context_time(ctx); |
| 1568 | update_event_times(event); | 1568 | update_event_times(event); |
| 1569 | spin_unlock_irqrestore(&ctx->lock, flags); | 1569 | raw_spin_unlock_irqrestore(&ctx->lock, flags); |
| 1570 | } | 1570 | } |
| 1571 | 1571 | ||
| 1572 | return atomic64_read(&event->count); | 1572 | return atomic64_read(&event->count); |
| @@ -1579,7 +1579,7 @@ static void | |||
| 1579 | __perf_event_init_context(struct perf_event_context *ctx, | 1579 | __perf_event_init_context(struct perf_event_context *ctx, |
| 1580 | struct task_struct *task) | 1580 | struct task_struct *task) |
| 1581 | { | 1581 | { |
| 1582 | spin_lock_init(&ctx->lock); | 1582 | raw_spin_lock_init(&ctx->lock); |
| 1583 | mutex_init(&ctx->mutex); | 1583 | mutex_init(&ctx->mutex); |
| 1584 | INIT_LIST_HEAD(&ctx->group_list); | 1584 | INIT_LIST_HEAD(&ctx->group_list); |
| 1585 | INIT_LIST_HEAD(&ctx->event_list); | 1585 | INIT_LIST_HEAD(&ctx->event_list); |
| @@ -1649,7 +1649,7 @@ static struct perf_event_context *find_get_context(pid_t pid, int cpu) | |||
| 1649 | ctx = perf_lock_task_context(task, &flags); | 1649 | ctx = perf_lock_task_context(task, &flags); |
| 1650 | if (ctx) { | 1650 | if (ctx) { |
| 1651 | unclone_ctx(ctx); | 1651 | unclone_ctx(ctx); |
| 1652 | spin_unlock_irqrestore(&ctx->lock, flags); | 1652 | raw_spin_unlock_irqrestore(&ctx->lock, flags); |
| 1653 | } | 1653 | } |
| 1654 | 1654 | ||
| 1655 | if (!ctx) { | 1655 | if (!ctx) { |
| @@ -1987,7 +1987,7 @@ static int perf_event_period(struct perf_event *event, u64 __user *arg) | |||
| 1987 | if (!value) | 1987 | if (!value) |
| 1988 | return -EINVAL; | 1988 | return -EINVAL; |
| 1989 | 1989 | ||
| 1990 | spin_lock_irq(&ctx->lock); | 1990 | raw_spin_lock_irq(&ctx->lock); |
| 1991 | if (event->attr.freq) { | 1991 | if (event->attr.freq) { |
| 1992 | if (value > sysctl_perf_event_sample_rate) { | 1992 | if (value > sysctl_perf_event_sample_rate) { |
| 1993 | ret = -EINVAL; | 1993 | ret = -EINVAL; |
| @@ -2000,7 +2000,7 @@ static int perf_event_period(struct perf_event *event, u64 __user *arg) | |||
| 2000 | event->hw.sample_period = value; | 2000 | event->hw.sample_period = value; |
| 2001 | } | 2001 | } |
| 2002 | unlock: | 2002 | unlock: |
| 2003 | spin_unlock_irq(&ctx->lock); | 2003 | raw_spin_unlock_irq(&ctx->lock); |
| 2004 | 2004 | ||
| 2005 | return ret; | 2005 | return ret; |
| 2006 | } | 2006 | } |
| @@ -4992,7 +4992,7 @@ void perf_event_exit_task(struct task_struct *child) | |||
| 4992 | * reading child->perf_event_ctxp, we wait until it has | 4992 | * reading child->perf_event_ctxp, we wait until it has |
| 4993 | * incremented the context's refcount before we do put_ctx below. | 4993 | * incremented the context's refcount before we do put_ctx below. |
| 4994 | */ | 4994 | */ |
| 4995 | spin_lock(&child_ctx->lock); | 4995 | raw_spin_lock(&child_ctx->lock); |
| 4996 | child->perf_event_ctxp = NULL; | 4996 | child->perf_event_ctxp = NULL; |
| 4997 | /* | 4997 | /* |
| 4998 | * If this context is a clone; unclone it so it can't get | 4998 | * If this context is a clone; unclone it so it can't get |
| @@ -5001,7 +5001,7 @@ void perf_event_exit_task(struct task_struct *child) | |||
| 5001 | */ | 5001 | */ |
| 5002 | unclone_ctx(child_ctx); | 5002 | unclone_ctx(child_ctx); |
| 5003 | update_context_time(child_ctx); | 5003 | update_context_time(child_ctx); |
| 5004 | spin_unlock_irqrestore(&child_ctx->lock, flags); | 5004 | raw_spin_unlock_irqrestore(&child_ctx->lock, flags); |
| 5005 | 5005 | ||
| 5006 | /* | 5006 | /* |
| 5007 | * Report the task dead after unscheduling the events so that we | 5007 | * Report the task dead after unscheduling the events so that we |
| @@ -5292,11 +5292,11 @@ perf_set_reserve_percpu(struct sysdev_class *class, | |||
| 5292 | perf_reserved_percpu = val; | 5292 | perf_reserved_percpu = val; |
| 5293 | for_each_online_cpu(cpu) { | 5293 | for_each_online_cpu(cpu) { |
| 5294 | cpuctx = &per_cpu(perf_cpu_context, cpu); | 5294 | cpuctx = &per_cpu(perf_cpu_context, cpu); |
| 5295 | spin_lock_irq(&cpuctx->ctx.lock); | 5295 | raw_spin_lock_irq(&cpuctx->ctx.lock); |
| 5296 | mpt = min(perf_max_events - cpuctx->ctx.nr_events, | 5296 | mpt = min(perf_max_events - cpuctx->ctx.nr_events, |
| 5297 | perf_max_events - perf_reserved_percpu); | 5297 | perf_max_events - perf_reserved_percpu); |
| 5298 | cpuctx->max_pertask = mpt; | 5298 | cpuctx->max_pertask = mpt; |
| 5299 | spin_unlock_irq(&cpuctx->ctx.lock); | 5299 | raw_spin_unlock_irq(&cpuctx->ctx.lock); |
| 5300 | } | 5300 | } |
| 5301 | spin_unlock(&perf_resource_lock); | 5301 | spin_unlock(&perf_resource_lock); |
| 5302 | 5302 | ||
diff --git a/kernel/power/console.c b/kernel/power/console.c index 5187136fe1de..218e5af90156 100644 --- a/kernel/power/console.c +++ b/kernel/power/console.c | |||
| @@ -6,7 +6,7 @@ | |||
| 6 | 6 | ||
| 7 | #include <linux/vt_kern.h> | 7 | #include <linux/vt_kern.h> |
| 8 | #include <linux/kbd_kern.h> | 8 | #include <linux/kbd_kern.h> |
| 9 | #include <linux/console.h> | 9 | #include <linux/vt.h> |
| 10 | #include <linux/module.h> | 10 | #include <linux/module.h> |
| 11 | #include "power.h" | 11 | #include "power.h" |
| 12 | 12 | ||
| @@ -21,8 +21,7 @@ int pm_prepare_console(void) | |||
| 21 | if (orig_fgconsole < 0) | 21 | if (orig_fgconsole < 0) |
| 22 | return 1; | 22 | return 1; |
| 23 | 23 | ||
| 24 | orig_kmsg = kmsg_redirect; | 24 | orig_kmsg = vt_kmsg_redirect(SUSPEND_CONSOLE); |
| 25 | kmsg_redirect = SUSPEND_CONSOLE; | ||
| 26 | return 0; | 25 | return 0; |
| 27 | } | 26 | } |
| 28 | 27 | ||
| @@ -30,7 +29,7 @@ void pm_restore_console(void) | |||
| 30 | { | 29 | { |
| 31 | if (orig_fgconsole >= 0) { | 30 | if (orig_fgconsole >= 0) { |
| 32 | vt_move_to_console(orig_fgconsole, 0); | 31 | vt_move_to_console(orig_fgconsole, 0); |
| 33 | kmsg_redirect = orig_kmsg; | 32 | vt_kmsg_redirect(orig_kmsg); |
| 34 | } | 33 | } |
| 35 | } | 34 | } |
| 36 | #endif | 35 | #endif |
diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c index a621a67ef4e3..9bb52177af02 100644 --- a/kernel/rcutorture.c +++ b/kernel/rcutorture.c | |||
| @@ -763,13 +763,13 @@ static void rcu_torture_timer(unsigned long unused) | |||
| 763 | /* Should not happen, but... */ | 763 | /* Should not happen, but... */ |
| 764 | pipe_count = RCU_TORTURE_PIPE_LEN; | 764 | pipe_count = RCU_TORTURE_PIPE_LEN; |
| 765 | } | 765 | } |
| 766 | ++__get_cpu_var(rcu_torture_count)[pipe_count]; | 766 | __this_cpu_inc(per_cpu_var(rcu_torture_count)[pipe_count]); |
| 767 | completed = cur_ops->completed() - completed; | 767 | completed = cur_ops->completed() - completed; |
| 768 | if (completed > RCU_TORTURE_PIPE_LEN) { | 768 | if (completed > RCU_TORTURE_PIPE_LEN) { |
| 769 | /* Should not happen, but... */ | 769 | /* Should not happen, but... */ |
| 770 | completed = RCU_TORTURE_PIPE_LEN; | 770 | completed = RCU_TORTURE_PIPE_LEN; |
| 771 | } | 771 | } |
| 772 | ++__get_cpu_var(rcu_torture_batch)[completed]; | 772 | __this_cpu_inc(per_cpu_var(rcu_torture_batch)[completed]); |
| 773 | preempt_enable(); | 773 | preempt_enable(); |
| 774 | cur_ops->readunlock(idx); | 774 | cur_ops->readunlock(idx); |
| 775 | } | 775 | } |
| @@ -818,13 +818,13 @@ rcu_torture_reader(void *arg) | |||
| 818 | /* Should not happen, but... */ | 818 | /* Should not happen, but... */ |
| 819 | pipe_count = RCU_TORTURE_PIPE_LEN; | 819 | pipe_count = RCU_TORTURE_PIPE_LEN; |
| 820 | } | 820 | } |
| 821 | ++__get_cpu_var(rcu_torture_count)[pipe_count]; | 821 | __this_cpu_inc(per_cpu_var(rcu_torture_count)[pipe_count]); |
| 822 | completed = cur_ops->completed() - completed; | 822 | completed = cur_ops->completed() - completed; |
| 823 | if (completed > RCU_TORTURE_PIPE_LEN) { | 823 | if (completed > RCU_TORTURE_PIPE_LEN) { |
| 824 | /* Should not happen, but... */ | 824 | /* Should not happen, but... */ |
| 825 | completed = RCU_TORTURE_PIPE_LEN; | 825 | completed = RCU_TORTURE_PIPE_LEN; |
| 826 | } | 826 | } |
| 827 | ++__get_cpu_var(rcu_torture_batch)[completed]; | 827 | __this_cpu_inc(per_cpu_var(rcu_torture_batch)[completed]); |
| 828 | preempt_enable(); | 828 | preempt_enable(); |
| 829 | cur_ops->readunlock(idx); | 829 | cur_ops->readunlock(idx); |
| 830 | schedule(); | 830 | schedule(); |
diff --git a/kernel/rtmutex-debug.c b/kernel/rtmutex-debug.c index 5fcb4fe645e2..ddabb54bb5c8 100644 --- a/kernel/rtmutex-debug.c +++ b/kernel/rtmutex-debug.c | |||
| @@ -37,8 +37,8 @@ do { \ | |||
| 37 | if (rt_trace_on) { \ | 37 | if (rt_trace_on) { \ |
| 38 | rt_trace_on = 0; \ | 38 | rt_trace_on = 0; \ |
| 39 | console_verbose(); \ | 39 | console_verbose(); \ |
| 40 | if (spin_is_locked(¤t->pi_lock)) \ | 40 | if (raw_spin_is_locked(¤t->pi_lock)) \ |
| 41 | spin_unlock(¤t->pi_lock); \ | 41 | raw_spin_unlock(¤t->pi_lock); \ |
| 42 | } \ | 42 | } \ |
| 43 | } while (0) | 43 | } while (0) |
| 44 | 44 | ||
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c index 29bd4baf9e75..a9604815786a 100644 --- a/kernel/rtmutex.c +++ b/kernel/rtmutex.c | |||
| @@ -138,9 +138,9 @@ static void rt_mutex_adjust_prio(struct task_struct *task) | |||
| 138 | { | 138 | { |
| 139 | unsigned long flags; | 139 | unsigned long flags; |
| 140 | 140 | ||
| 141 | spin_lock_irqsave(&task->pi_lock, flags); | 141 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
| 142 | __rt_mutex_adjust_prio(task); | 142 | __rt_mutex_adjust_prio(task); |
| 143 | spin_unlock_irqrestore(&task->pi_lock, flags); | 143 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
| 144 | } | 144 | } |
| 145 | 145 | ||
| 146 | /* | 146 | /* |
| @@ -195,7 +195,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, | |||
| 195 | /* | 195 | /* |
| 196 | * Task can not go away as we did a get_task() before ! | 196 | * Task can not go away as we did a get_task() before ! |
| 197 | */ | 197 | */ |
| 198 | spin_lock_irqsave(&task->pi_lock, flags); | 198 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
| 199 | 199 | ||
| 200 | waiter = task->pi_blocked_on; | 200 | waiter = task->pi_blocked_on; |
| 201 | /* | 201 | /* |
| @@ -231,8 +231,8 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, | |||
| 231 | goto out_unlock_pi; | 231 | goto out_unlock_pi; |
| 232 | 232 | ||
| 233 | lock = waiter->lock; | 233 | lock = waiter->lock; |
| 234 | if (!spin_trylock(&lock->wait_lock)) { | 234 | if (!raw_spin_trylock(&lock->wait_lock)) { |
| 235 | spin_unlock_irqrestore(&task->pi_lock, flags); | 235 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
| 236 | cpu_relax(); | 236 | cpu_relax(); |
| 237 | goto retry; | 237 | goto retry; |
| 238 | } | 238 | } |
| @@ -240,7 +240,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, | |||
| 240 | /* Deadlock detection */ | 240 | /* Deadlock detection */ |
| 241 | if (lock == orig_lock || rt_mutex_owner(lock) == top_task) { | 241 | if (lock == orig_lock || rt_mutex_owner(lock) == top_task) { |
| 242 | debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock); | 242 | debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock); |
| 243 | spin_unlock(&lock->wait_lock); | 243 | raw_spin_unlock(&lock->wait_lock); |
| 244 | ret = deadlock_detect ? -EDEADLK : 0; | 244 | ret = deadlock_detect ? -EDEADLK : 0; |
| 245 | goto out_unlock_pi; | 245 | goto out_unlock_pi; |
| 246 | } | 246 | } |
| @@ -253,13 +253,13 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, | |||
| 253 | plist_add(&waiter->list_entry, &lock->wait_list); | 253 | plist_add(&waiter->list_entry, &lock->wait_list); |
| 254 | 254 | ||
| 255 | /* Release the task */ | 255 | /* Release the task */ |
| 256 | spin_unlock_irqrestore(&task->pi_lock, flags); | 256 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
| 257 | put_task_struct(task); | 257 | put_task_struct(task); |
| 258 | 258 | ||
| 259 | /* Grab the next task */ | 259 | /* Grab the next task */ |
| 260 | task = rt_mutex_owner(lock); | 260 | task = rt_mutex_owner(lock); |
| 261 | get_task_struct(task); | 261 | get_task_struct(task); |
| 262 | spin_lock_irqsave(&task->pi_lock, flags); | 262 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
| 263 | 263 | ||
| 264 | if (waiter == rt_mutex_top_waiter(lock)) { | 264 | if (waiter == rt_mutex_top_waiter(lock)) { |
| 265 | /* Boost the owner */ | 265 | /* Boost the owner */ |
| @@ -277,10 +277,10 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, | |||
| 277 | __rt_mutex_adjust_prio(task); | 277 | __rt_mutex_adjust_prio(task); |
| 278 | } | 278 | } |
| 279 | 279 | ||
| 280 | spin_unlock_irqrestore(&task->pi_lock, flags); | 280 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
| 281 | 281 | ||
| 282 | top_waiter = rt_mutex_top_waiter(lock); | 282 | top_waiter = rt_mutex_top_waiter(lock); |
| 283 | spin_unlock(&lock->wait_lock); | 283 | raw_spin_unlock(&lock->wait_lock); |
| 284 | 284 | ||
| 285 | if (!detect_deadlock && waiter != top_waiter) | 285 | if (!detect_deadlock && waiter != top_waiter) |
| 286 | goto out_put_task; | 286 | goto out_put_task; |
| @@ -288,7 +288,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, | |||
| 288 | goto again; | 288 | goto again; |
| 289 | 289 | ||
| 290 | out_unlock_pi: | 290 | out_unlock_pi: |
| 291 | spin_unlock_irqrestore(&task->pi_lock, flags); | 291 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
| 292 | out_put_task: | 292 | out_put_task: |
| 293 | put_task_struct(task); | 293 | put_task_struct(task); |
| 294 | 294 | ||
| @@ -313,9 +313,9 @@ static inline int try_to_steal_lock(struct rt_mutex *lock, | |||
| 313 | if (pendowner == task) | 313 | if (pendowner == task) |
| 314 | return 1; | 314 | return 1; |
| 315 | 315 | ||
| 316 | spin_lock_irqsave(&pendowner->pi_lock, flags); | 316 | raw_spin_lock_irqsave(&pendowner->pi_lock, flags); |
| 317 | if (task->prio >= pendowner->prio) { | 317 | if (task->prio >= pendowner->prio) { |
| 318 | spin_unlock_irqrestore(&pendowner->pi_lock, flags); | 318 | raw_spin_unlock_irqrestore(&pendowner->pi_lock, flags); |
| 319 | return 0; | 319 | return 0; |
| 320 | } | 320 | } |
| 321 | 321 | ||
| @@ -325,7 +325,7 @@ static inline int try_to_steal_lock(struct rt_mutex *lock, | |||
| 325 | * priority. | 325 | * priority. |
| 326 | */ | 326 | */ |
| 327 | if (likely(!rt_mutex_has_waiters(lock))) { | 327 | if (likely(!rt_mutex_has_waiters(lock))) { |
| 328 | spin_unlock_irqrestore(&pendowner->pi_lock, flags); | 328 | raw_spin_unlock_irqrestore(&pendowner->pi_lock, flags); |
| 329 | return 1; | 329 | return 1; |
| 330 | } | 330 | } |
| 331 | 331 | ||
| @@ -333,7 +333,7 @@ static inline int try_to_steal_lock(struct rt_mutex *lock, | |||
| 333 | next = rt_mutex_top_waiter(lock); | 333 | next = rt_mutex_top_waiter(lock); |
| 334 | plist_del(&next->pi_list_entry, &pendowner->pi_waiters); | 334 | plist_del(&next->pi_list_entry, &pendowner->pi_waiters); |
| 335 | __rt_mutex_adjust_prio(pendowner); | 335 | __rt_mutex_adjust_prio(pendowner); |
| 336 | spin_unlock_irqrestore(&pendowner->pi_lock, flags); | 336 | raw_spin_unlock_irqrestore(&pendowner->pi_lock, flags); |
| 337 | 337 | ||
| 338 | /* | 338 | /* |
| 339 | * We are going to steal the lock and a waiter was | 339 | * We are going to steal the lock and a waiter was |
| @@ -350,10 +350,10 @@ static inline int try_to_steal_lock(struct rt_mutex *lock, | |||
| 350 | * might be task: | 350 | * might be task: |
| 351 | */ | 351 | */ |
| 352 | if (likely(next->task != task)) { | 352 | if (likely(next->task != task)) { |
| 353 | spin_lock_irqsave(&task->pi_lock, flags); | 353 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
| 354 | plist_add(&next->pi_list_entry, &task->pi_waiters); | 354 | plist_add(&next->pi_list_entry, &task->pi_waiters); |
| 355 | __rt_mutex_adjust_prio(task); | 355 | __rt_mutex_adjust_prio(task); |
| 356 | spin_unlock_irqrestore(&task->pi_lock, flags); | 356 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
| 357 | } | 357 | } |
| 358 | return 1; | 358 | return 1; |
| 359 | } | 359 | } |
| @@ -420,7 +420,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, | |||
| 420 | unsigned long flags; | 420 | unsigned long flags; |
| 421 | int chain_walk = 0, res; | 421 | int chain_walk = 0, res; |
| 422 | 422 | ||
| 423 | spin_lock_irqsave(&task->pi_lock, flags); | 423 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
| 424 | __rt_mutex_adjust_prio(task); | 424 | __rt_mutex_adjust_prio(task); |
| 425 | waiter->task = task; | 425 | waiter->task = task; |
| 426 | waiter->lock = lock; | 426 | waiter->lock = lock; |
| @@ -434,17 +434,17 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, | |||
| 434 | 434 | ||
| 435 | task->pi_blocked_on = waiter; | 435 | task->pi_blocked_on = waiter; |
| 436 | 436 | ||
| 437 | spin_unlock_irqrestore(&task->pi_lock, flags); | 437 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
| 438 | 438 | ||
| 439 | if (waiter == rt_mutex_top_waiter(lock)) { | 439 | if (waiter == rt_mutex_top_waiter(lock)) { |
| 440 | spin_lock_irqsave(&owner->pi_lock, flags); | 440 | raw_spin_lock_irqsave(&owner->pi_lock, flags); |
| 441 | plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters); | 441 | plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters); |
| 442 | plist_add(&waiter->pi_list_entry, &owner->pi_waiters); | 442 | plist_add(&waiter->pi_list_entry, &owner->pi_waiters); |
| 443 | 443 | ||
| 444 | __rt_mutex_adjust_prio(owner); | 444 | __rt_mutex_adjust_prio(owner); |
| 445 | if (owner->pi_blocked_on) | 445 | if (owner->pi_blocked_on) |
| 446 | chain_walk = 1; | 446 | chain_walk = 1; |
| 447 | spin_unlock_irqrestore(&owner->pi_lock, flags); | 447 | raw_spin_unlock_irqrestore(&owner->pi_lock, flags); |
| 448 | } | 448 | } |
| 449 | else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock)) | 449 | else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock)) |
| 450 | chain_walk = 1; | 450 | chain_walk = 1; |
| @@ -459,12 +459,12 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, | |||
| 459 | */ | 459 | */ |
| 460 | get_task_struct(owner); | 460 | get_task_struct(owner); |
| 461 | 461 | ||
| 462 | spin_unlock(&lock->wait_lock); | 462 | raw_spin_unlock(&lock->wait_lock); |
| 463 | 463 | ||
| 464 | res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter, | 464 | res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter, |
| 465 | task); | 465 | task); |
| 466 | 466 | ||
| 467 | spin_lock(&lock->wait_lock); | 467 | raw_spin_lock(&lock->wait_lock); |
| 468 | 468 | ||
| 469 | return res; | 469 | return res; |
| 470 | } | 470 | } |
| @@ -483,7 +483,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock) | |||
| 483 | struct task_struct *pendowner; | 483 | struct task_struct *pendowner; |
| 484 | unsigned long flags; | 484 | unsigned long flags; |
| 485 | 485 | ||
| 486 | spin_lock_irqsave(¤t->pi_lock, flags); | 486 | raw_spin_lock_irqsave(¤t->pi_lock, flags); |
| 487 | 487 | ||
| 488 | waiter = rt_mutex_top_waiter(lock); | 488 | waiter = rt_mutex_top_waiter(lock); |
| 489 | plist_del(&waiter->list_entry, &lock->wait_list); | 489 | plist_del(&waiter->list_entry, &lock->wait_list); |
| @@ -500,7 +500,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock) | |||
| 500 | 500 | ||
| 501 | rt_mutex_set_owner(lock, pendowner, RT_MUTEX_OWNER_PENDING); | 501 | rt_mutex_set_owner(lock, pendowner, RT_MUTEX_OWNER_PENDING); |
| 502 | 502 | ||
| 503 | spin_unlock_irqrestore(¤t->pi_lock, flags); | 503 | raw_spin_unlock_irqrestore(¤t->pi_lock, flags); |
| 504 | 504 | ||
| 505 | /* | 505 | /* |
| 506 | * Clear the pi_blocked_on variable and enqueue a possible | 506 | * Clear the pi_blocked_on variable and enqueue a possible |
| @@ -509,7 +509,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock) | |||
| 509 | * waiter with higher priority than pending-owner->normal_prio | 509 | * waiter with higher priority than pending-owner->normal_prio |
| 510 | * is blocked on the unboosted (pending) owner. | 510 | * is blocked on the unboosted (pending) owner. |
| 511 | */ | 511 | */ |
| 512 | spin_lock_irqsave(&pendowner->pi_lock, flags); | 512 | raw_spin_lock_irqsave(&pendowner->pi_lock, flags); |
| 513 | 513 | ||
| 514 | WARN_ON(!pendowner->pi_blocked_on); | 514 | WARN_ON(!pendowner->pi_blocked_on); |
| 515 | WARN_ON(pendowner->pi_blocked_on != waiter); | 515 | WARN_ON(pendowner->pi_blocked_on != waiter); |
| @@ -523,7 +523,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock) | |||
| 523 | next = rt_mutex_top_waiter(lock); | 523 | next = rt_mutex_top_waiter(lock); |
| 524 | plist_add(&next->pi_list_entry, &pendowner->pi_waiters); | 524 | plist_add(&next->pi_list_entry, &pendowner->pi_waiters); |
| 525 | } | 525 | } |
| 526 | spin_unlock_irqrestore(&pendowner->pi_lock, flags); | 526 | raw_spin_unlock_irqrestore(&pendowner->pi_lock, flags); |
| 527 | 527 | ||
| 528 | wake_up_process(pendowner); | 528 | wake_up_process(pendowner); |
| 529 | } | 529 | } |
| @@ -541,15 +541,15 @@ static void remove_waiter(struct rt_mutex *lock, | |||
| 541 | unsigned long flags; | 541 | unsigned long flags; |
| 542 | int chain_walk = 0; | 542 | int chain_walk = 0; |
| 543 | 543 | ||
| 544 | spin_lock_irqsave(¤t->pi_lock, flags); | 544 | raw_spin_lock_irqsave(¤t->pi_lock, flags); |
| 545 | plist_del(&waiter->list_entry, &lock->wait_list); | 545 | plist_del(&waiter->list_entry, &lock->wait_list); |
| 546 | waiter->task = NULL; | 546 | waiter->task = NULL; |
| 547 | current->pi_blocked_on = NULL; | 547 | current->pi_blocked_on = NULL; |
| 548 | spin_unlock_irqrestore(¤t->pi_lock, flags); | 548 | raw_spin_unlock_irqrestore(¤t->pi_lock, flags); |
| 549 | 549 | ||
| 550 | if (first && owner != current) { | 550 | if (first && owner != current) { |
| 551 | 551 | ||
| 552 | spin_lock_irqsave(&owner->pi_lock, flags); | 552 | raw_spin_lock_irqsave(&owner->pi_lock, flags); |
| 553 | 553 | ||
| 554 | plist_del(&waiter->pi_list_entry, &owner->pi_waiters); | 554 | plist_del(&waiter->pi_list_entry, &owner->pi_waiters); |
| 555 | 555 | ||
| @@ -564,7 +564,7 @@ static void remove_waiter(struct rt_mutex *lock, | |||
| 564 | if (owner->pi_blocked_on) | 564 | if (owner->pi_blocked_on) |
| 565 | chain_walk = 1; | 565 | chain_walk = 1; |
| 566 | 566 | ||
| 567 | spin_unlock_irqrestore(&owner->pi_lock, flags); | 567 | raw_spin_unlock_irqrestore(&owner->pi_lock, flags); |
| 568 | } | 568 | } |
| 569 | 569 | ||
| 570 | WARN_ON(!plist_node_empty(&waiter->pi_list_entry)); | 570 | WARN_ON(!plist_node_empty(&waiter->pi_list_entry)); |
| @@ -575,11 +575,11 @@ static void remove_waiter(struct rt_mutex *lock, | |||
| 575 | /* gets dropped in rt_mutex_adjust_prio_chain()! */ | 575 | /* gets dropped in rt_mutex_adjust_prio_chain()! */ |
| 576 | get_task_struct(owner); | 576 | get_task_struct(owner); |
| 577 | 577 | ||
| 578 | spin_unlock(&lock->wait_lock); | 578 | raw_spin_unlock(&lock->wait_lock); |
| 579 | 579 | ||
| 580 | rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current); | 580 | rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current); |
| 581 | 581 | ||
| 582 | spin_lock(&lock->wait_lock); | 582 | raw_spin_lock(&lock->wait_lock); |
| 583 | } | 583 | } |
| 584 | 584 | ||
| 585 | /* | 585 | /* |
| @@ -592,15 +592,15 @@ void rt_mutex_adjust_pi(struct task_struct *task) | |||
| 592 | struct rt_mutex_waiter *waiter; | 592 | struct rt_mutex_waiter *waiter; |
| 593 | unsigned long flags; | 593 | unsigned long flags; |
| 594 | 594 | ||
| 595 | spin_lock_irqsave(&task->pi_lock, flags); | 595 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
| 596 | 596 | ||
| 597 | waiter = task->pi_blocked_on; | 597 | waiter = task->pi_blocked_on; |
| 598 | if (!waiter || waiter->list_entry.prio == task->prio) { | 598 | if (!waiter || waiter->list_entry.prio == task->prio) { |
| 599 | spin_unlock_irqrestore(&task->pi_lock, flags); | 599 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
| 600 | return; | 600 | return; |
| 601 | } | 601 | } |
| 602 | 602 | ||
| 603 | spin_unlock_irqrestore(&task->pi_lock, flags); | 603 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
| 604 | 604 | ||
| 605 | /* gets dropped in rt_mutex_adjust_prio_chain()! */ | 605 | /* gets dropped in rt_mutex_adjust_prio_chain()! */ |
| 606 | get_task_struct(task); | 606 | get_task_struct(task); |
| @@ -672,14 +672,14 @@ __rt_mutex_slowlock(struct rt_mutex *lock, int state, | |||
| 672 | break; | 672 | break; |
| 673 | } | 673 | } |
| 674 | 674 | ||
| 675 | spin_unlock(&lock->wait_lock); | 675 | raw_spin_unlock(&lock->wait_lock); |
| 676 | 676 | ||
| 677 | debug_rt_mutex_print_deadlock(waiter); | 677 | debug_rt_mutex_print_deadlock(waiter); |
| 678 | 678 | ||
| 679 | if (waiter->task) | 679 | if (waiter->task) |
| 680 | schedule_rt_mutex(lock); | 680 | schedule_rt_mutex(lock); |
| 681 | 681 | ||
| 682 | spin_lock(&lock->wait_lock); | 682 | raw_spin_lock(&lock->wait_lock); |
| 683 | set_current_state(state); | 683 | set_current_state(state); |
| 684 | } | 684 | } |
| 685 | 685 | ||
| @@ -700,11 +700,11 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, | |||
| 700 | debug_rt_mutex_init_waiter(&waiter); | 700 | debug_rt_mutex_init_waiter(&waiter); |
| 701 | waiter.task = NULL; | 701 | waiter.task = NULL; |
| 702 | 702 | ||
| 703 | spin_lock(&lock->wait_lock); | 703 | raw_spin_lock(&lock->wait_lock); |
| 704 | 704 | ||
| 705 | /* Try to acquire the lock again: */ | 705 | /* Try to acquire the lock again: */ |
| 706 | if (try_to_take_rt_mutex(lock)) { | 706 | if (try_to_take_rt_mutex(lock)) { |
| 707 | spin_unlock(&lock->wait_lock); | 707 | raw_spin_unlock(&lock->wait_lock); |
| 708 | return 0; | 708 | return 0; |
| 709 | } | 709 | } |
| 710 | 710 | ||
| @@ -731,7 +731,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, | |||
| 731 | */ | 731 | */ |
| 732 | fixup_rt_mutex_waiters(lock); | 732 | fixup_rt_mutex_waiters(lock); |
| 733 | 733 | ||
| 734 | spin_unlock(&lock->wait_lock); | 734 | raw_spin_unlock(&lock->wait_lock); |
| 735 | 735 | ||
| 736 | /* Remove pending timer: */ | 736 | /* Remove pending timer: */ |
| 737 | if (unlikely(timeout)) | 737 | if (unlikely(timeout)) |
| @@ -758,7 +758,7 @@ rt_mutex_slowtrylock(struct rt_mutex *lock) | |||
| 758 | { | 758 | { |
| 759 | int ret = 0; | 759 | int ret = 0; |
| 760 | 760 | ||
| 761 | spin_lock(&lock->wait_lock); | 761 | raw_spin_lock(&lock->wait_lock); |
| 762 | 762 | ||
| 763 | if (likely(rt_mutex_owner(lock) != current)) { | 763 | if (likely(rt_mutex_owner(lock) != current)) { |
| 764 | 764 | ||
| @@ -770,7 +770,7 @@ rt_mutex_slowtrylock(struct rt_mutex *lock) | |||
| 770 | fixup_rt_mutex_waiters(lock); | 770 | fixup_rt_mutex_waiters(lock); |
| 771 | } | 771 | } |
| 772 | 772 | ||
| 773 | spin_unlock(&lock->wait_lock); | 773 | raw_spin_unlock(&lock->wait_lock); |
| 774 | 774 | ||
| 775 | return ret; | 775 | return ret; |
| 776 | } | 776 | } |
| @@ -781,7 +781,7 @@ rt_mutex_slowtrylock(struct rt_mutex *lock) | |||
| 781 | static void __sched | 781 | static void __sched |
| 782 | rt_mutex_slowunlock(struct rt_mutex *lock) | 782 | rt_mutex_slowunlock(struct rt_mutex *lock) |
| 783 | { | 783 | { |
| 784 | spin_lock(&lock->wait_lock); | 784 | raw_spin_lock(&lock->wait_lock); |
| 785 | 785 | ||
| 786 | debug_rt_mutex_unlock(lock); | 786 | debug_rt_mutex_unlock(lock); |
| 787 | 787 | ||
| @@ -789,13 +789,13 @@ rt_mutex_slowunlock(struct rt_mutex *lock) | |||
| 789 | 789 | ||
| 790 | if (!rt_mutex_has_waiters(lock)) { | 790 | if (!rt_mutex_has_waiters(lock)) { |
| 791 | lock->owner = NULL; | 791 | lock->owner = NULL; |
| 792 | spin_unlock(&lock->wait_lock); | 792 | raw_spin_unlock(&lock->wait_lock); |
| 793 | return; | 793 | return; |
| 794 | } | 794 | } |
| 795 | 795 | ||
| 796 | wakeup_next_waiter(lock); | 796 | wakeup_next_waiter(lock); |
| 797 | 797 | ||
| 798 | spin_unlock(&lock->wait_lock); | 798 | raw_spin_unlock(&lock->wait_lock); |
| 799 | 799 | ||
| 800 | /* Undo pi boosting if necessary: */ | 800 | /* Undo pi boosting if necessary: */ |
| 801 | rt_mutex_adjust_prio(current); | 801 | rt_mutex_adjust_prio(current); |
| @@ -970,8 +970,8 @@ EXPORT_SYMBOL_GPL(rt_mutex_destroy); | |||
| 970 | void __rt_mutex_init(struct rt_mutex *lock, const char *name) | 970 | void __rt_mutex_init(struct rt_mutex *lock, const char *name) |
| 971 | { | 971 | { |
| 972 | lock->owner = NULL; | 972 | lock->owner = NULL; |
| 973 | spin_lock_init(&lock->wait_lock); | 973 | raw_spin_lock_init(&lock->wait_lock); |
| 974 | plist_head_init(&lock->wait_list, &lock->wait_lock); | 974 | plist_head_init_raw(&lock->wait_list, &lock->wait_lock); |
| 975 | 975 | ||
| 976 | debug_rt_mutex_init(lock, name); | 976 | debug_rt_mutex_init(lock, name); |
| 977 | } | 977 | } |
| @@ -1032,7 +1032,7 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock, | |||
| 1032 | { | 1032 | { |
| 1033 | int ret; | 1033 | int ret; |
| 1034 | 1034 | ||
| 1035 | spin_lock(&lock->wait_lock); | 1035 | raw_spin_lock(&lock->wait_lock); |
| 1036 | 1036 | ||
| 1037 | mark_rt_mutex_waiters(lock); | 1037 | mark_rt_mutex_waiters(lock); |
| 1038 | 1038 | ||
| @@ -1040,7 +1040,7 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock, | |||
| 1040 | /* We got the lock for task. */ | 1040 | /* We got the lock for task. */ |
| 1041 | debug_rt_mutex_lock(lock); | 1041 | debug_rt_mutex_lock(lock); |
| 1042 | rt_mutex_set_owner(lock, task, 0); | 1042 | rt_mutex_set_owner(lock, task, 0); |
| 1043 | spin_unlock(&lock->wait_lock); | 1043 | raw_spin_unlock(&lock->wait_lock); |
| 1044 | rt_mutex_deadlock_account_lock(lock, task); | 1044 | rt_mutex_deadlock_account_lock(lock, task); |
| 1045 | return 1; | 1045 | return 1; |
| 1046 | } | 1046 | } |
| @@ -1056,7 +1056,7 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock, | |||
| 1056 | */ | 1056 | */ |
| 1057 | ret = 0; | 1057 | ret = 0; |
| 1058 | } | 1058 | } |
| 1059 | spin_unlock(&lock->wait_lock); | 1059 | raw_spin_unlock(&lock->wait_lock); |
| 1060 | 1060 | ||
| 1061 | debug_rt_mutex_print_deadlock(waiter); | 1061 | debug_rt_mutex_print_deadlock(waiter); |
| 1062 | 1062 | ||
| @@ -1106,7 +1106,7 @@ int rt_mutex_finish_proxy_lock(struct rt_mutex *lock, | |||
| 1106 | { | 1106 | { |
| 1107 | int ret; | 1107 | int ret; |
| 1108 | 1108 | ||
| 1109 | spin_lock(&lock->wait_lock); | 1109 | raw_spin_lock(&lock->wait_lock); |
| 1110 | 1110 | ||
| 1111 | set_current_state(TASK_INTERRUPTIBLE); | 1111 | set_current_state(TASK_INTERRUPTIBLE); |
| 1112 | 1112 | ||
| @@ -1124,7 +1124,7 @@ int rt_mutex_finish_proxy_lock(struct rt_mutex *lock, | |||
| 1124 | */ | 1124 | */ |
| 1125 | fixup_rt_mutex_waiters(lock); | 1125 | fixup_rt_mutex_waiters(lock); |
| 1126 | 1126 | ||
| 1127 | spin_unlock(&lock->wait_lock); | 1127 | raw_spin_unlock(&lock->wait_lock); |
| 1128 | 1128 | ||
| 1129 | /* | 1129 | /* |
| 1130 | * Readjust priority, when we did not get the lock. We might have been | 1130 | * Readjust priority, when we did not get the lock. We might have been |
diff --git a/kernel/sched.c b/kernel/sched.c index db5c26692dd5..9c30858b6463 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
| @@ -143,7 +143,7 @@ struct rt_prio_array { | |||
| 143 | 143 | ||
| 144 | struct rt_bandwidth { | 144 | struct rt_bandwidth { |
| 145 | /* nests inside the rq lock: */ | 145 | /* nests inside the rq lock: */ |
| 146 | spinlock_t rt_runtime_lock; | 146 | raw_spinlock_t rt_runtime_lock; |
| 147 | ktime_t rt_period; | 147 | ktime_t rt_period; |
| 148 | u64 rt_runtime; | 148 | u64 rt_runtime; |
| 149 | struct hrtimer rt_period_timer; | 149 | struct hrtimer rt_period_timer; |
| @@ -180,7 +180,7 @@ void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime) | |||
| 180 | rt_b->rt_period = ns_to_ktime(period); | 180 | rt_b->rt_period = ns_to_ktime(period); |
| 181 | rt_b->rt_runtime = runtime; | 181 | rt_b->rt_runtime = runtime; |
| 182 | 182 | ||
| 183 | spin_lock_init(&rt_b->rt_runtime_lock); | 183 | raw_spin_lock_init(&rt_b->rt_runtime_lock); |
| 184 | 184 | ||
| 185 | hrtimer_init(&rt_b->rt_period_timer, | 185 | hrtimer_init(&rt_b->rt_period_timer, |
| 186 | CLOCK_MONOTONIC, HRTIMER_MODE_REL); | 186 | CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| @@ -202,7 +202,7 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b) | |||
| 202 | if (hrtimer_active(&rt_b->rt_period_timer)) | 202 | if (hrtimer_active(&rt_b->rt_period_timer)) |
| 203 | return; | 203 | return; |
| 204 | 204 | ||
| 205 | spin_lock(&rt_b->rt_runtime_lock); | 205 | raw_spin_lock(&rt_b->rt_runtime_lock); |
| 206 | for (;;) { | 206 | for (;;) { |
| 207 | unsigned long delta; | 207 | unsigned long delta; |
| 208 | ktime_t soft, hard; | 208 | ktime_t soft, hard; |
| @@ -219,7 +219,7 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b) | |||
| 219 | __hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta, | 219 | __hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta, |
| 220 | HRTIMER_MODE_ABS_PINNED, 0); | 220 | HRTIMER_MODE_ABS_PINNED, 0); |
| 221 | } | 221 | } |
| 222 | spin_unlock(&rt_b->rt_runtime_lock); | 222 | raw_spin_unlock(&rt_b->rt_runtime_lock); |
| 223 | } | 223 | } |
| 224 | 224 | ||
| 225 | #ifdef CONFIG_RT_GROUP_SCHED | 225 | #ifdef CONFIG_RT_GROUP_SCHED |
| @@ -300,7 +300,7 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct cfs_rq, init_tg_cfs_rq); | |||
| 300 | 300 | ||
| 301 | #ifdef CONFIG_RT_GROUP_SCHED | 301 | #ifdef CONFIG_RT_GROUP_SCHED |
| 302 | static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity); | 302 | static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity); |
| 303 | static DEFINE_PER_CPU_SHARED_ALIGNED(struct rt_rq, init_rt_rq); | 303 | static DEFINE_PER_CPU_SHARED_ALIGNED(struct rt_rq, init_rt_rq_var); |
| 304 | #endif /* CONFIG_RT_GROUP_SCHED */ | 304 | #endif /* CONFIG_RT_GROUP_SCHED */ |
| 305 | #else /* !CONFIG_USER_SCHED */ | 305 | #else /* !CONFIG_USER_SCHED */ |
| 306 | #define root_task_group init_task_group | 306 | #define root_task_group init_task_group |
| @@ -472,7 +472,7 @@ struct rt_rq { | |||
| 472 | u64 rt_time; | 472 | u64 rt_time; |
| 473 | u64 rt_runtime; | 473 | u64 rt_runtime; |
| 474 | /* Nests inside the rq lock: */ | 474 | /* Nests inside the rq lock: */ |
| 475 | spinlock_t rt_runtime_lock; | 475 | raw_spinlock_t rt_runtime_lock; |
| 476 | 476 | ||
| 477 | #ifdef CONFIG_RT_GROUP_SCHED | 477 | #ifdef CONFIG_RT_GROUP_SCHED |
| 478 | unsigned long rt_nr_boosted; | 478 | unsigned long rt_nr_boosted; |
| @@ -527,7 +527,7 @@ static struct root_domain def_root_domain; | |||
| 527 | */ | 527 | */ |
| 528 | struct rq { | 528 | struct rq { |
| 529 | /* runqueue lock: */ | 529 | /* runqueue lock: */ |
| 530 | spinlock_t lock; | 530 | raw_spinlock_t lock; |
| 531 | 531 | ||
| 532 | /* | 532 | /* |
| 533 | * nr_running and cpu_load should be in the same cacheline because | 533 | * nr_running and cpu_load should be in the same cacheline because |
| @@ -687,7 +687,7 @@ inline void update_rq_clock(struct rq *rq) | |||
| 687 | */ | 687 | */ |
| 688 | int runqueue_is_locked(int cpu) | 688 | int runqueue_is_locked(int cpu) |
| 689 | { | 689 | { |
| 690 | return spin_is_locked(&cpu_rq(cpu)->lock); | 690 | return raw_spin_is_locked(&cpu_rq(cpu)->lock); |
| 691 | } | 691 | } |
| 692 | 692 | ||
| 693 | /* | 693 | /* |
| @@ -895,7 +895,7 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) | |||
| 895 | */ | 895 | */ |
| 896 | spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_); | 896 | spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_); |
| 897 | 897 | ||
| 898 | spin_unlock_irq(&rq->lock); | 898 | raw_spin_unlock_irq(&rq->lock); |
| 899 | } | 899 | } |
| 900 | 900 | ||
| 901 | #else /* __ARCH_WANT_UNLOCKED_CTXSW */ | 901 | #else /* __ARCH_WANT_UNLOCKED_CTXSW */ |
| @@ -919,9 +919,9 @@ static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next) | |||
| 919 | next->oncpu = 1; | 919 | next->oncpu = 1; |
| 920 | #endif | 920 | #endif |
| 921 | #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW | 921 | #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW |
| 922 | spin_unlock_irq(&rq->lock); | 922 | raw_spin_unlock_irq(&rq->lock); |
| 923 | #else | 923 | #else |
| 924 | spin_unlock(&rq->lock); | 924 | raw_spin_unlock(&rq->lock); |
| 925 | #endif | 925 | #endif |
| 926 | } | 926 | } |
| 927 | 927 | ||
| @@ -951,10 +951,10 @@ static inline struct rq *__task_rq_lock(struct task_struct *p) | |||
| 951 | { | 951 | { |
| 952 | for (;;) { | 952 | for (;;) { |
| 953 | struct rq *rq = task_rq(p); | 953 | struct rq *rq = task_rq(p); |
| 954 | spin_lock(&rq->lock); | 954 | raw_spin_lock(&rq->lock); |
| 955 | if (likely(rq == task_rq(p))) | 955 | if (likely(rq == task_rq(p))) |
| 956 | return rq; | 956 | return rq; |
| 957 | spin_unlock(&rq->lock); | 957 | raw_spin_unlock(&rq->lock); |
| 958 | } | 958 | } |
| 959 | } | 959 | } |
| 960 | 960 | ||
| @@ -971,10 +971,10 @@ static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags) | |||
| 971 | for (;;) { | 971 | for (;;) { |
| 972 | local_irq_save(*flags); | 972 | local_irq_save(*flags); |
| 973 | rq = task_rq(p); | 973 | rq = task_rq(p); |
| 974 | spin_lock(&rq->lock); | 974 | raw_spin_lock(&rq->lock); |
| 975 | if (likely(rq == task_rq(p))) | 975 | if (likely(rq == task_rq(p))) |
| 976 | return rq; | 976 | return rq; |
| 977 | spin_unlock_irqrestore(&rq->lock, *flags); | 977 | raw_spin_unlock_irqrestore(&rq->lock, *flags); |
| 978 | } | 978 | } |
| 979 | } | 979 | } |
| 980 | 980 | ||
| @@ -983,19 +983,19 @@ void task_rq_unlock_wait(struct task_struct *p) | |||
| 983 | struct rq *rq = task_rq(p); | 983 | struct rq *rq = task_rq(p); |
| 984 | 984 | ||
| 985 | smp_mb(); /* spin-unlock-wait is not a full memory barrier */ | 985 | smp_mb(); /* spin-unlock-wait is not a full memory barrier */ |
| 986 | spin_unlock_wait(&rq->lock); | 986 | raw_spin_unlock_wait(&rq->lock); |
| 987 | } | 987 | } |
| 988 | 988 | ||
| 989 | static void __task_rq_unlock(struct rq *rq) | 989 | static void __task_rq_unlock(struct rq *rq) |
| 990 | __releases(rq->lock) | 990 | __releases(rq->lock) |
| 991 | { | 991 | { |
| 992 | spin_unlock(&rq->lock); | 992 | raw_spin_unlock(&rq->lock); |
| 993 | } | 993 | } |
| 994 | 994 | ||
| 995 | static inline void task_rq_unlock(struct rq *rq, unsigned long *flags) | 995 | static inline void task_rq_unlock(struct rq *rq, unsigned long *flags) |
| 996 | __releases(rq->lock) | 996 | __releases(rq->lock) |
| 997 | { | 997 | { |
| 998 | spin_unlock_irqrestore(&rq->lock, *flags); | 998 | raw_spin_unlock_irqrestore(&rq->lock, *flags); |
| 999 | } | 999 | } |
| 1000 | 1000 | ||
| 1001 | /* | 1001 | /* |
| @@ -1008,7 +1008,7 @@ static struct rq *this_rq_lock(void) | |||
| 1008 | 1008 | ||
| 1009 | local_irq_disable(); | 1009 | local_irq_disable(); |
| 1010 | rq = this_rq(); | 1010 | rq = this_rq(); |
| 1011 | spin_lock(&rq->lock); | 1011 | raw_spin_lock(&rq->lock); |
| 1012 | 1012 | ||
| 1013 | return rq; | 1013 | return rq; |
| 1014 | } | 1014 | } |
| @@ -1055,10 +1055,10 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer) | |||
| 1055 | 1055 | ||
| 1056 | WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); | 1056 | WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); |
| 1057 | 1057 | ||
| 1058 | spin_lock(&rq->lock); | 1058 | raw_spin_lock(&rq->lock); |
| 1059 | update_rq_clock(rq); | 1059 | update_rq_clock(rq); |
| 1060 | rq->curr->sched_class->task_tick(rq, rq->curr, 1); | 1060 | rq->curr->sched_class->task_tick(rq, rq->curr, 1); |
| 1061 | spin_unlock(&rq->lock); | 1061 | raw_spin_unlock(&rq->lock); |
| 1062 | 1062 | ||
| 1063 | return HRTIMER_NORESTART; | 1063 | return HRTIMER_NORESTART; |
| 1064 | } | 1064 | } |
| @@ -1071,10 +1071,10 @@ static void __hrtick_start(void *arg) | |||
| 1071 | { | 1071 | { |
| 1072 | struct rq *rq = arg; | 1072 | struct rq *rq = arg; |
| 1073 | 1073 | ||
| 1074 | spin_lock(&rq->lock); | 1074 | raw_spin_lock(&rq->lock); |
| 1075 | hrtimer_restart(&rq->hrtick_timer); | 1075 | hrtimer_restart(&rq->hrtick_timer); |
| 1076 | rq->hrtick_csd_pending = 0; | 1076 | rq->hrtick_csd_pending = 0; |
| 1077 | spin_unlock(&rq->lock); | 1077 | raw_spin_unlock(&rq->lock); |
| 1078 | } | 1078 | } |
| 1079 | 1079 | ||
| 1080 | /* | 1080 | /* |
| @@ -1181,7 +1181,7 @@ static void resched_task(struct task_struct *p) | |||
| 1181 | { | 1181 | { |
| 1182 | int cpu; | 1182 | int cpu; |
| 1183 | 1183 | ||
| 1184 | assert_spin_locked(&task_rq(p)->lock); | 1184 | assert_raw_spin_locked(&task_rq(p)->lock); |
| 1185 | 1185 | ||
| 1186 | if (test_tsk_need_resched(p)) | 1186 | if (test_tsk_need_resched(p)) |
| 1187 | return; | 1187 | return; |
| @@ -1203,10 +1203,10 @@ static void resched_cpu(int cpu) | |||
| 1203 | struct rq *rq = cpu_rq(cpu); | 1203 | struct rq *rq = cpu_rq(cpu); |
| 1204 | unsigned long flags; | 1204 | unsigned long flags; |
| 1205 | 1205 | ||
| 1206 | if (!spin_trylock_irqsave(&rq->lock, flags)) | 1206 | if (!raw_spin_trylock_irqsave(&rq->lock, flags)) |
| 1207 | return; | 1207 | return; |
| 1208 | resched_task(cpu_curr(cpu)); | 1208 | resched_task(cpu_curr(cpu)); |
| 1209 | spin_unlock_irqrestore(&rq->lock, flags); | 1209 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 1210 | } | 1210 | } |
| 1211 | 1211 | ||
| 1212 | #ifdef CONFIG_NO_HZ | 1212 | #ifdef CONFIG_NO_HZ |
| @@ -1275,7 +1275,7 @@ static void sched_rt_avg_update(struct rq *rq, u64 rt_delta) | |||
| 1275 | #else /* !CONFIG_SMP */ | 1275 | #else /* !CONFIG_SMP */ |
| 1276 | static void resched_task(struct task_struct *p) | 1276 | static void resched_task(struct task_struct *p) |
| 1277 | { | 1277 | { |
| 1278 | assert_spin_locked(&task_rq(p)->lock); | 1278 | assert_raw_spin_locked(&task_rq(p)->lock); |
| 1279 | set_tsk_need_resched(p); | 1279 | set_tsk_need_resched(p); |
| 1280 | } | 1280 | } |
| 1281 | 1281 | ||
| @@ -1602,11 +1602,11 @@ static void update_group_shares_cpu(struct task_group *tg, int cpu, | |||
| 1602 | struct rq *rq = cpu_rq(cpu); | 1602 | struct rq *rq = cpu_rq(cpu); |
| 1603 | unsigned long flags; | 1603 | unsigned long flags; |
| 1604 | 1604 | ||
| 1605 | spin_lock_irqsave(&rq->lock, flags); | 1605 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 1606 | tg->cfs_rq[cpu]->rq_weight = boost ? 0 : rq_weight; | 1606 | tg->cfs_rq[cpu]->rq_weight = boost ? 0 : rq_weight; |
| 1607 | tg->cfs_rq[cpu]->shares = boost ? 0 : shares; | 1607 | tg->cfs_rq[cpu]->shares = boost ? 0 : shares; |
| 1608 | __set_se_shares(tg->se[cpu], shares); | 1608 | __set_se_shares(tg->se[cpu], shares); |
| 1609 | spin_unlock_irqrestore(&rq->lock, flags); | 1609 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 1610 | } | 1610 | } |
| 1611 | } | 1611 | } |
| 1612 | 1612 | ||
| @@ -1708,9 +1708,9 @@ static void update_shares_locked(struct rq *rq, struct sched_domain *sd) | |||
| 1708 | if (root_task_group_empty()) | 1708 | if (root_task_group_empty()) |
| 1709 | return; | 1709 | return; |
| 1710 | 1710 | ||
| 1711 | spin_unlock(&rq->lock); | 1711 | raw_spin_unlock(&rq->lock); |
| 1712 | update_shares(sd); | 1712 | update_shares(sd); |
| 1713 | spin_lock(&rq->lock); | 1713 | raw_spin_lock(&rq->lock); |
| 1714 | } | 1714 | } |
| 1715 | 1715 | ||
| 1716 | static void update_h_load(long cpu) | 1716 | static void update_h_load(long cpu) |
| @@ -1750,7 +1750,7 @@ static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest) | |||
| 1750 | __acquires(busiest->lock) | 1750 | __acquires(busiest->lock) |
| 1751 | __acquires(this_rq->lock) | 1751 | __acquires(this_rq->lock) |
| 1752 | { | 1752 | { |
| 1753 | spin_unlock(&this_rq->lock); | 1753 | raw_spin_unlock(&this_rq->lock); |
| 1754 | double_rq_lock(this_rq, busiest); | 1754 | double_rq_lock(this_rq, busiest); |
| 1755 | 1755 | ||
| 1756 | return 1; | 1756 | return 1; |
| @@ -1771,14 +1771,16 @@ static int _double_lock_balance(struct rq *this_rq, struct rq *busiest) | |||
| 1771 | { | 1771 | { |
| 1772 | int ret = 0; | 1772 | int ret = 0; |
| 1773 | 1773 | ||
| 1774 | if (unlikely(!spin_trylock(&busiest->lock))) { | 1774 | if (unlikely(!raw_spin_trylock(&busiest->lock))) { |
| 1775 | if (busiest < this_rq) { | 1775 | if (busiest < this_rq) { |
| 1776 | spin_unlock(&this_rq->lock); | 1776 | raw_spin_unlock(&this_rq->lock); |
| 1777 | spin_lock(&busiest->lock); | 1777 | raw_spin_lock(&busiest->lock); |
| 1778 | spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING); | 1778 | raw_spin_lock_nested(&this_rq->lock, |
| 1779 | SINGLE_DEPTH_NESTING); | ||
| 1779 | ret = 1; | 1780 | ret = 1; |
| 1780 | } else | 1781 | } else |
| 1781 | spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING); | 1782 | raw_spin_lock_nested(&busiest->lock, |
| 1783 | SINGLE_DEPTH_NESTING); | ||
| 1782 | } | 1784 | } |
| 1783 | return ret; | 1785 | return ret; |
| 1784 | } | 1786 | } |
| @@ -1792,7 +1794,7 @@ static int double_lock_balance(struct rq *this_rq, struct rq *busiest) | |||
| 1792 | { | 1794 | { |
| 1793 | if (unlikely(!irqs_disabled())) { | 1795 | if (unlikely(!irqs_disabled())) { |
| 1794 | /* printk() doesn't work good under rq->lock */ | 1796 | /* printk() doesn't work good under rq->lock */ |
| 1795 | spin_unlock(&this_rq->lock); | 1797 | raw_spin_unlock(&this_rq->lock); |
| 1796 | BUG_ON(1); | 1798 | BUG_ON(1); |
| 1797 | } | 1799 | } |
| 1798 | 1800 | ||
| @@ -1802,7 +1804,7 @@ static int double_lock_balance(struct rq *this_rq, struct rq *busiest) | |||
| 1802 | static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest) | 1804 | static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest) |
| 1803 | __releases(busiest->lock) | 1805 | __releases(busiest->lock) |
| 1804 | { | 1806 | { |
| 1805 | spin_unlock(&busiest->lock); | 1807 | raw_spin_unlock(&busiest->lock); |
| 1806 | lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_); | 1808 | lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_); |
| 1807 | } | 1809 | } |
| 1808 | #endif | 1810 | #endif |
| @@ -2025,13 +2027,13 @@ void kthread_bind(struct task_struct *p, unsigned int cpu) | |||
| 2025 | return; | 2027 | return; |
| 2026 | } | 2028 | } |
| 2027 | 2029 | ||
| 2028 | spin_lock_irqsave(&rq->lock, flags); | 2030 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 2029 | update_rq_clock(rq); | 2031 | update_rq_clock(rq); |
| 2030 | set_task_cpu(p, cpu); | 2032 | set_task_cpu(p, cpu); |
| 2031 | p->cpus_allowed = cpumask_of_cpu(cpu); | 2033 | p->cpus_allowed = cpumask_of_cpu(cpu); |
| 2032 | p->rt.nr_cpus_allowed = 1; | 2034 | p->rt.nr_cpus_allowed = 1; |
| 2033 | p->flags |= PF_THREAD_BOUND; | 2035 | p->flags |= PF_THREAD_BOUND; |
| 2034 | spin_unlock_irqrestore(&rq->lock, flags); | 2036 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 2035 | } | 2037 | } |
| 2036 | EXPORT_SYMBOL(kthread_bind); | 2038 | EXPORT_SYMBOL(kthread_bind); |
| 2037 | 2039 | ||
| @@ -2783,10 +2785,10 @@ static inline void post_schedule(struct rq *rq) | |||
| 2783 | if (rq->post_schedule) { | 2785 | if (rq->post_schedule) { |
| 2784 | unsigned long flags; | 2786 | unsigned long flags; |
| 2785 | 2787 | ||
| 2786 | spin_lock_irqsave(&rq->lock, flags); | 2788 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 2787 | if (rq->curr->sched_class->post_schedule) | 2789 | if (rq->curr->sched_class->post_schedule) |
| 2788 | rq->curr->sched_class->post_schedule(rq); | 2790 | rq->curr->sched_class->post_schedule(rq); |
| 2789 | spin_unlock_irqrestore(&rq->lock, flags); | 2791 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 2790 | 2792 | ||
| 2791 | rq->post_schedule = 0; | 2793 | rq->post_schedule = 0; |
| 2792 | } | 2794 | } |
| @@ -3068,15 +3070,15 @@ static void double_rq_lock(struct rq *rq1, struct rq *rq2) | |||
| 3068 | { | 3070 | { |
| 3069 | BUG_ON(!irqs_disabled()); | 3071 | BUG_ON(!irqs_disabled()); |
| 3070 | if (rq1 == rq2) { | 3072 | if (rq1 == rq2) { |
| 3071 | spin_lock(&rq1->lock); | 3073 | raw_spin_lock(&rq1->lock); |
| 3072 | __acquire(rq2->lock); /* Fake it out ;) */ | 3074 | __acquire(rq2->lock); /* Fake it out ;) */ |
| 3073 | } else { | 3075 | } else { |
| 3074 | if (rq1 < rq2) { | 3076 | if (rq1 < rq2) { |
| 3075 | spin_lock(&rq1->lock); | 3077 | raw_spin_lock(&rq1->lock); |
| 3076 | spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING); | 3078 | raw_spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING); |
| 3077 | } else { | 3079 | } else { |
| 3078 | spin_lock(&rq2->lock); | 3080 | raw_spin_lock(&rq2->lock); |
| 3079 | spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING); | 3081 | raw_spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING); |
| 3080 | } | 3082 | } |
| 3081 | } | 3083 | } |
| 3082 | update_rq_clock(rq1); | 3084 | update_rq_clock(rq1); |
| @@ -3093,9 +3095,9 @@ static void double_rq_unlock(struct rq *rq1, struct rq *rq2) | |||
| 3093 | __releases(rq1->lock) | 3095 | __releases(rq1->lock) |
| 3094 | __releases(rq2->lock) | 3096 | __releases(rq2->lock) |
| 3095 | { | 3097 | { |
| 3096 | spin_unlock(&rq1->lock); | 3098 | raw_spin_unlock(&rq1->lock); |
| 3097 | if (rq1 != rq2) | 3099 | if (rq1 != rq2) |
| 3098 | spin_unlock(&rq2->lock); | 3100 | raw_spin_unlock(&rq2->lock); |
| 3099 | else | 3101 | else |
| 3100 | __release(rq2->lock); | 3102 | __release(rq2->lock); |
| 3101 | } | 3103 | } |
| @@ -4188,14 +4190,15 @@ redo: | |||
| 4188 | 4190 | ||
| 4189 | if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) { | 4191 | if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) { |
| 4190 | 4192 | ||
| 4191 | spin_lock_irqsave(&busiest->lock, flags); | 4193 | raw_spin_lock_irqsave(&busiest->lock, flags); |
| 4192 | 4194 | ||
| 4193 | /* don't kick the migration_thread, if the curr | 4195 | /* don't kick the migration_thread, if the curr |
| 4194 | * task on busiest cpu can't be moved to this_cpu | 4196 | * task on busiest cpu can't be moved to this_cpu |
| 4195 | */ | 4197 | */ |
| 4196 | if (!cpumask_test_cpu(this_cpu, | 4198 | if (!cpumask_test_cpu(this_cpu, |
| 4197 | &busiest->curr->cpus_allowed)) { | 4199 | &busiest->curr->cpus_allowed)) { |
| 4198 | spin_unlock_irqrestore(&busiest->lock, flags); | 4200 | raw_spin_unlock_irqrestore(&busiest->lock, |
| 4201 | flags); | ||
| 4199 | all_pinned = 1; | 4202 | all_pinned = 1; |
| 4200 | goto out_one_pinned; | 4203 | goto out_one_pinned; |
| 4201 | } | 4204 | } |
| @@ -4205,7 +4208,7 @@ redo: | |||
| 4205 | busiest->push_cpu = this_cpu; | 4208 | busiest->push_cpu = this_cpu; |
| 4206 | active_balance = 1; | 4209 | active_balance = 1; |
| 4207 | } | 4210 | } |
| 4208 | spin_unlock_irqrestore(&busiest->lock, flags); | 4211 | raw_spin_unlock_irqrestore(&busiest->lock, flags); |
| 4209 | if (active_balance) | 4212 | if (active_balance) |
| 4210 | wake_up_process(busiest->migration_thread); | 4213 | wake_up_process(busiest->migration_thread); |
| 4211 | 4214 | ||
| @@ -4387,10 +4390,10 @@ redo: | |||
| 4387 | /* | 4390 | /* |
| 4388 | * Should not call ttwu while holding a rq->lock | 4391 | * Should not call ttwu while holding a rq->lock |
| 4389 | */ | 4392 | */ |
| 4390 | spin_unlock(&this_rq->lock); | 4393 | raw_spin_unlock(&this_rq->lock); |
| 4391 | if (active_balance) | 4394 | if (active_balance) |
| 4392 | wake_up_process(busiest->migration_thread); | 4395 | wake_up_process(busiest->migration_thread); |
| 4393 | spin_lock(&this_rq->lock); | 4396 | raw_spin_lock(&this_rq->lock); |
| 4394 | 4397 | ||
| 4395 | } else | 4398 | } else |
| 4396 | sd->nr_balance_failed = 0; | 4399 | sd->nr_balance_failed = 0; |
| @@ -5259,11 +5262,11 @@ void scheduler_tick(void) | |||
| 5259 | 5262 | ||
| 5260 | sched_clock_tick(); | 5263 | sched_clock_tick(); |
| 5261 | 5264 | ||
| 5262 | spin_lock(&rq->lock); | 5265 | raw_spin_lock(&rq->lock); |
| 5263 | update_rq_clock(rq); | 5266 | update_rq_clock(rq); |
| 5264 | update_cpu_load(rq); | 5267 | update_cpu_load(rq); |
| 5265 | curr->sched_class->task_tick(rq, curr, 0); | 5268 | curr->sched_class->task_tick(rq, curr, 0); |
| 5266 | spin_unlock(&rq->lock); | 5269 | raw_spin_unlock(&rq->lock); |
| 5267 | 5270 | ||
| 5268 | perf_event_task_tick(curr, cpu); | 5271 | perf_event_task_tick(curr, cpu); |
| 5269 | 5272 | ||
| @@ -5457,7 +5460,7 @@ need_resched_nonpreemptible: | |||
| 5457 | if (sched_feat(HRTICK)) | 5460 | if (sched_feat(HRTICK)) |
| 5458 | hrtick_clear(rq); | 5461 | hrtick_clear(rq); |
| 5459 | 5462 | ||
| 5460 | spin_lock_irq(&rq->lock); | 5463 | raw_spin_lock_irq(&rq->lock); |
| 5461 | update_rq_clock(rq); | 5464 | update_rq_clock(rq); |
| 5462 | clear_tsk_need_resched(prev); | 5465 | clear_tsk_need_resched(prev); |
| 5463 | 5466 | ||
| @@ -5493,7 +5496,7 @@ need_resched_nonpreemptible: | |||
| 5493 | cpu = smp_processor_id(); | 5496 | cpu = smp_processor_id(); |
| 5494 | rq = cpu_rq(cpu); | 5497 | rq = cpu_rq(cpu); |
| 5495 | } else | 5498 | } else |
| 5496 | spin_unlock_irq(&rq->lock); | 5499 | raw_spin_unlock_irq(&rq->lock); |
| 5497 | 5500 | ||
| 5498 | post_schedule(rq); | 5501 | post_schedule(rq); |
| 5499 | 5502 | ||
| @@ -6324,7 +6327,7 @@ recheck: | |||
| 6324 | * make sure no PI-waiters arrive (or leave) while we are | 6327 | * make sure no PI-waiters arrive (or leave) while we are |
| 6325 | * changing the priority of the task: | 6328 | * changing the priority of the task: |
| 6326 | */ | 6329 | */ |
| 6327 | spin_lock_irqsave(&p->pi_lock, flags); | 6330 | raw_spin_lock_irqsave(&p->pi_lock, flags); |
| 6328 | /* | 6331 | /* |
| 6329 | * To be able to change p->policy safely, the apropriate | 6332 | * To be able to change p->policy safely, the apropriate |
| 6330 | * runqueue lock must be held. | 6333 | * runqueue lock must be held. |
| @@ -6334,7 +6337,7 @@ recheck: | |||
| 6334 | if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { | 6337 | if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { |
| 6335 | policy = oldpolicy = -1; | 6338 | policy = oldpolicy = -1; |
| 6336 | __task_rq_unlock(rq); | 6339 | __task_rq_unlock(rq); |
| 6337 | spin_unlock_irqrestore(&p->pi_lock, flags); | 6340 | raw_spin_unlock_irqrestore(&p->pi_lock, flags); |
| 6338 | goto recheck; | 6341 | goto recheck; |
| 6339 | } | 6342 | } |
| 6340 | update_rq_clock(rq); | 6343 | update_rq_clock(rq); |
| @@ -6358,7 +6361,7 @@ recheck: | |||
| 6358 | check_class_changed(rq, p, prev_class, oldprio, running); | 6361 | check_class_changed(rq, p, prev_class, oldprio, running); |
| 6359 | } | 6362 | } |
| 6360 | __task_rq_unlock(rq); | 6363 | __task_rq_unlock(rq); |
| 6361 | spin_unlock_irqrestore(&p->pi_lock, flags); | 6364 | raw_spin_unlock_irqrestore(&p->pi_lock, flags); |
| 6362 | 6365 | ||
| 6363 | rt_mutex_adjust_pi(p); | 6366 | rt_mutex_adjust_pi(p); |
| 6364 | 6367 | ||
| @@ -6684,7 +6687,7 @@ SYSCALL_DEFINE0(sched_yield) | |||
| 6684 | */ | 6687 | */ |
| 6685 | __release(rq->lock); | 6688 | __release(rq->lock); |
| 6686 | spin_release(&rq->lock.dep_map, 1, _THIS_IP_); | 6689 | spin_release(&rq->lock.dep_map, 1, _THIS_IP_); |
| 6687 | _raw_spin_unlock(&rq->lock); | 6690 | do_raw_spin_unlock(&rq->lock); |
| 6688 | preempt_enable_no_resched(); | 6691 | preempt_enable_no_resched(); |
| 6689 | 6692 | ||
| 6690 | schedule(); | 6693 | schedule(); |
| @@ -6978,7 +6981,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu) | |||
| 6978 | struct rq *rq = cpu_rq(cpu); | 6981 | struct rq *rq = cpu_rq(cpu); |
| 6979 | unsigned long flags; | 6982 | unsigned long flags; |
| 6980 | 6983 | ||
| 6981 | spin_lock_irqsave(&rq->lock, flags); | 6984 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 6982 | 6985 | ||
| 6983 | __sched_fork(idle); | 6986 | __sched_fork(idle); |
| 6984 | idle->se.exec_start = sched_clock(); | 6987 | idle->se.exec_start = sched_clock(); |
| @@ -6990,7 +6993,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu) | |||
| 6990 | #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW) | 6993 | #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW) |
| 6991 | idle->oncpu = 1; | 6994 | idle->oncpu = 1; |
| 6992 | #endif | 6995 | #endif |
| 6993 | spin_unlock_irqrestore(&rq->lock, flags); | 6996 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 6994 | 6997 | ||
| 6995 | /* Set the preempt count _outside_ the spinlocks! */ | 6998 | /* Set the preempt count _outside_ the spinlocks! */ |
| 6996 | #if defined(CONFIG_PREEMPT) | 6999 | #if defined(CONFIG_PREEMPT) |
| @@ -7207,10 +7210,10 @@ static int migration_thread(void *data) | |||
| 7207 | struct migration_req *req; | 7210 | struct migration_req *req; |
| 7208 | struct list_head *head; | 7211 | struct list_head *head; |
| 7209 | 7212 | ||
| 7210 | spin_lock_irq(&rq->lock); | 7213 | raw_spin_lock_irq(&rq->lock); |
| 7211 | 7214 | ||
| 7212 | if (cpu_is_offline(cpu)) { | 7215 | if (cpu_is_offline(cpu)) { |
| 7213 | spin_unlock_irq(&rq->lock); | 7216 | raw_spin_unlock_irq(&rq->lock); |
| 7214 | break; | 7217 | break; |
| 7215 | } | 7218 | } |
| 7216 | 7219 | ||
| @@ -7222,7 +7225,7 @@ static int migration_thread(void *data) | |||
| 7222 | head = &rq->migration_queue; | 7225 | head = &rq->migration_queue; |
| 7223 | 7226 | ||
| 7224 | if (list_empty(head)) { | 7227 | if (list_empty(head)) { |
| 7225 | spin_unlock_irq(&rq->lock); | 7228 | raw_spin_unlock_irq(&rq->lock); |
| 7226 | schedule(); | 7229 | schedule(); |
| 7227 | set_current_state(TASK_INTERRUPTIBLE); | 7230 | set_current_state(TASK_INTERRUPTIBLE); |
| 7228 | continue; | 7231 | continue; |
| @@ -7231,14 +7234,14 @@ static int migration_thread(void *data) | |||
| 7231 | list_del_init(head->next); | 7234 | list_del_init(head->next); |
| 7232 | 7235 | ||
| 7233 | if (req->task != NULL) { | 7236 | if (req->task != NULL) { |
| 7234 | spin_unlock(&rq->lock); | 7237 | raw_spin_unlock(&rq->lock); |
| 7235 | __migrate_task(req->task, cpu, req->dest_cpu); | 7238 | __migrate_task(req->task, cpu, req->dest_cpu); |
| 7236 | } else if (likely(cpu == (badcpu = smp_processor_id()))) { | 7239 | } else if (likely(cpu == (badcpu = smp_processor_id()))) { |
| 7237 | req->dest_cpu = RCU_MIGRATION_GOT_QS; | 7240 | req->dest_cpu = RCU_MIGRATION_GOT_QS; |
| 7238 | spin_unlock(&rq->lock); | 7241 | raw_spin_unlock(&rq->lock); |
| 7239 | } else { | 7242 | } else { |
| 7240 | req->dest_cpu = RCU_MIGRATION_MUST_SYNC; | 7243 | req->dest_cpu = RCU_MIGRATION_MUST_SYNC; |
| 7241 | spin_unlock(&rq->lock); | 7244 | raw_spin_unlock(&rq->lock); |
| 7242 | WARN_ONCE(1, "migration_thread() on CPU %d, expected %d\n", badcpu, cpu); | 7245 | WARN_ONCE(1, "migration_thread() on CPU %d, expected %d\n", badcpu, cpu); |
| 7243 | } | 7246 | } |
| 7244 | local_irq_enable(); | 7247 | local_irq_enable(); |
| @@ -7360,14 +7363,14 @@ void sched_idle_next(void) | |||
| 7360 | * Strictly not necessary since rest of the CPUs are stopped by now | 7363 | * Strictly not necessary since rest of the CPUs are stopped by now |
| 7361 | * and interrupts disabled on the current cpu. | 7364 | * and interrupts disabled on the current cpu. |
| 7362 | */ | 7365 | */ |
| 7363 | spin_lock_irqsave(&rq->lock, flags); | 7366 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 7364 | 7367 | ||
| 7365 | __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1); | 7368 | __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1); |
| 7366 | 7369 | ||
| 7367 | update_rq_clock(rq); | 7370 | update_rq_clock(rq); |
| 7368 | activate_task(rq, p, 0); | 7371 | activate_task(rq, p, 0); |
| 7369 | 7372 | ||
| 7370 | spin_unlock_irqrestore(&rq->lock, flags); | 7373 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 7371 | } | 7374 | } |
| 7372 | 7375 | ||
| 7373 | /* | 7376 | /* |
| @@ -7403,9 +7406,9 @@ static void migrate_dead(unsigned int dead_cpu, struct task_struct *p) | |||
| 7403 | * that's OK. No task can be added to this CPU, so iteration is | 7406 | * that's OK. No task can be added to this CPU, so iteration is |
| 7404 | * fine. | 7407 | * fine. |
| 7405 | */ | 7408 | */ |
| 7406 | spin_unlock_irq(&rq->lock); | 7409 | raw_spin_unlock_irq(&rq->lock); |
| 7407 | move_task_off_dead_cpu(dead_cpu, p); | 7410 | move_task_off_dead_cpu(dead_cpu, p); |
| 7408 | spin_lock_irq(&rq->lock); | 7411 | raw_spin_lock_irq(&rq->lock); |
| 7409 | 7412 | ||
| 7410 | put_task_struct(p); | 7413 | put_task_struct(p); |
| 7411 | } | 7414 | } |
| @@ -7671,13 +7674,13 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) | |||
| 7671 | 7674 | ||
| 7672 | /* Update our root-domain */ | 7675 | /* Update our root-domain */ |
| 7673 | rq = cpu_rq(cpu); | 7676 | rq = cpu_rq(cpu); |
| 7674 | spin_lock_irqsave(&rq->lock, flags); | 7677 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 7675 | if (rq->rd) { | 7678 | if (rq->rd) { |
| 7676 | BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); | 7679 | BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); |
| 7677 | 7680 | ||
| 7678 | set_rq_online(rq); | 7681 | set_rq_online(rq); |
| 7679 | } | 7682 | } |
| 7680 | spin_unlock_irqrestore(&rq->lock, flags); | 7683 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 7681 | break; | 7684 | break; |
| 7682 | 7685 | ||
| 7683 | #ifdef CONFIG_HOTPLUG_CPU | 7686 | #ifdef CONFIG_HOTPLUG_CPU |
| @@ -7702,13 +7705,13 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) | |||
| 7702 | put_task_struct(rq->migration_thread); | 7705 | put_task_struct(rq->migration_thread); |
| 7703 | rq->migration_thread = NULL; | 7706 | rq->migration_thread = NULL; |
| 7704 | /* Idle task back to normal (off runqueue, low prio) */ | 7707 | /* Idle task back to normal (off runqueue, low prio) */ |
| 7705 | spin_lock_irq(&rq->lock); | 7708 | raw_spin_lock_irq(&rq->lock); |
| 7706 | update_rq_clock(rq); | 7709 | update_rq_clock(rq); |
| 7707 | deactivate_task(rq, rq->idle, 0); | 7710 | deactivate_task(rq, rq->idle, 0); |
| 7708 | __setscheduler(rq, rq->idle, SCHED_NORMAL, 0); | 7711 | __setscheduler(rq, rq->idle, SCHED_NORMAL, 0); |
| 7709 | rq->idle->sched_class = &idle_sched_class; | 7712 | rq->idle->sched_class = &idle_sched_class; |
| 7710 | migrate_dead_tasks(cpu); | 7713 | migrate_dead_tasks(cpu); |
| 7711 | spin_unlock_irq(&rq->lock); | 7714 | raw_spin_unlock_irq(&rq->lock); |
| 7712 | cpuset_unlock(); | 7715 | cpuset_unlock(); |
| 7713 | migrate_nr_uninterruptible(rq); | 7716 | migrate_nr_uninterruptible(rq); |
| 7714 | BUG_ON(rq->nr_running != 0); | 7717 | BUG_ON(rq->nr_running != 0); |
| @@ -7718,30 +7721,30 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) | |||
| 7718 | * they didn't take sched_hotcpu_mutex. Just wake up | 7721 | * they didn't take sched_hotcpu_mutex. Just wake up |
| 7719 | * the requestors. | 7722 | * the requestors. |
| 7720 | */ | 7723 | */ |
| 7721 | spin_lock_irq(&rq->lock); | 7724 | raw_spin_lock_irq(&rq->lock); |
| 7722 | while (!list_empty(&rq->migration_queue)) { | 7725 | while (!list_empty(&rq->migration_queue)) { |
| 7723 | struct migration_req *req; | 7726 | struct migration_req *req; |
| 7724 | 7727 | ||
| 7725 | req = list_entry(rq->migration_queue.next, | 7728 | req = list_entry(rq->migration_queue.next, |
| 7726 | struct migration_req, list); | 7729 | struct migration_req, list); |
| 7727 | list_del_init(&req->list); | 7730 | list_del_init(&req->list); |
| 7728 | spin_unlock_irq(&rq->lock); | 7731 | raw_spin_unlock_irq(&rq->lock); |
| 7729 | complete(&req->done); | 7732 | complete(&req->done); |
| 7730 | spin_lock_irq(&rq->lock); | 7733 | raw_spin_lock_irq(&rq->lock); |
| 7731 | } | 7734 | } |
| 7732 | spin_unlock_irq(&rq->lock); | 7735 | raw_spin_unlock_irq(&rq->lock); |
| 7733 | break; | 7736 | break; |
| 7734 | 7737 | ||
| 7735 | case CPU_DYING: | 7738 | case CPU_DYING: |
| 7736 | case CPU_DYING_FROZEN: | 7739 | case CPU_DYING_FROZEN: |
| 7737 | /* Update our root-domain */ | 7740 | /* Update our root-domain */ |
| 7738 | rq = cpu_rq(cpu); | 7741 | rq = cpu_rq(cpu); |
| 7739 | spin_lock_irqsave(&rq->lock, flags); | 7742 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 7740 | if (rq->rd) { | 7743 | if (rq->rd) { |
| 7741 | BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); | 7744 | BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); |
| 7742 | set_rq_offline(rq); | 7745 | set_rq_offline(rq); |
| 7743 | } | 7746 | } |
| 7744 | spin_unlock_irqrestore(&rq->lock, flags); | 7747 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 7745 | break; | 7748 | break; |
| 7746 | #endif | 7749 | #endif |
| 7747 | } | 7750 | } |
| @@ -7965,7 +7968,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd) | |||
| 7965 | struct root_domain *old_rd = NULL; | 7968 | struct root_domain *old_rd = NULL; |
| 7966 | unsigned long flags; | 7969 | unsigned long flags; |
| 7967 | 7970 | ||
| 7968 | spin_lock_irqsave(&rq->lock, flags); | 7971 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 7969 | 7972 | ||
| 7970 | if (rq->rd) { | 7973 | if (rq->rd) { |
| 7971 | old_rd = rq->rd; | 7974 | old_rd = rq->rd; |
| @@ -7991,7 +7994,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd) | |||
| 7991 | if (cpumask_test_cpu(rq->cpu, cpu_active_mask)) | 7994 | if (cpumask_test_cpu(rq->cpu, cpu_active_mask)) |
| 7992 | set_rq_online(rq); | 7995 | set_rq_online(rq); |
| 7993 | 7996 | ||
| 7994 | spin_unlock_irqrestore(&rq->lock, flags); | 7997 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 7995 | 7998 | ||
| 7996 | if (old_rd) | 7999 | if (old_rd) |
| 7997 | free_rootdomain(old_rd); | 8000 | free_rootdomain(old_rd); |
| @@ -8277,14 +8280,14 @@ enum s_alloc { | |||
| 8277 | */ | 8280 | */ |
| 8278 | #ifdef CONFIG_SCHED_SMT | 8281 | #ifdef CONFIG_SCHED_SMT |
| 8279 | static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains); | 8282 | static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains); |
| 8280 | static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus); | 8283 | static DEFINE_PER_CPU(struct static_sched_group, sched_groups); |
| 8281 | 8284 | ||
| 8282 | static int | 8285 | static int |
| 8283 | cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map, | 8286 | cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map, |
| 8284 | struct sched_group **sg, struct cpumask *unused) | 8287 | struct sched_group **sg, struct cpumask *unused) |
| 8285 | { | 8288 | { |
| 8286 | if (sg) | 8289 | if (sg) |
| 8287 | *sg = &per_cpu(sched_group_cpus, cpu).sg; | 8290 | *sg = &per_cpu(sched_groups, cpu).sg; |
| 8288 | return cpu; | 8291 | return cpu; |
| 8289 | } | 8292 | } |
| 8290 | #endif /* CONFIG_SCHED_SMT */ | 8293 | #endif /* CONFIG_SCHED_SMT */ |
| @@ -9347,13 +9350,13 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq) | |||
| 9347 | #ifdef CONFIG_SMP | 9350 | #ifdef CONFIG_SMP |
| 9348 | rt_rq->rt_nr_migratory = 0; | 9351 | rt_rq->rt_nr_migratory = 0; |
| 9349 | rt_rq->overloaded = 0; | 9352 | rt_rq->overloaded = 0; |
| 9350 | plist_head_init(&rt_rq->pushable_tasks, &rq->lock); | 9353 | plist_head_init_raw(&rt_rq->pushable_tasks, &rq->lock); |
| 9351 | #endif | 9354 | #endif |
| 9352 | 9355 | ||
| 9353 | rt_rq->rt_time = 0; | 9356 | rt_rq->rt_time = 0; |
| 9354 | rt_rq->rt_throttled = 0; | 9357 | rt_rq->rt_throttled = 0; |
| 9355 | rt_rq->rt_runtime = 0; | 9358 | rt_rq->rt_runtime = 0; |
| 9356 | spin_lock_init(&rt_rq->rt_runtime_lock); | 9359 | raw_spin_lock_init(&rt_rq->rt_runtime_lock); |
| 9357 | 9360 | ||
| 9358 | #ifdef CONFIG_RT_GROUP_SCHED | 9361 | #ifdef CONFIG_RT_GROUP_SCHED |
| 9359 | rt_rq->rt_nr_boosted = 0; | 9362 | rt_rq->rt_nr_boosted = 0; |
| @@ -9513,7 +9516,7 @@ void __init sched_init(void) | |||
| 9513 | struct rq *rq; | 9516 | struct rq *rq; |
| 9514 | 9517 | ||
| 9515 | rq = cpu_rq(i); | 9518 | rq = cpu_rq(i); |
| 9516 | spin_lock_init(&rq->lock); | 9519 | raw_spin_lock_init(&rq->lock); |
| 9517 | rq->nr_running = 0; | 9520 | rq->nr_running = 0; |
| 9518 | rq->calc_load_active = 0; | 9521 | rq->calc_load_active = 0; |
| 9519 | rq->calc_load_update = jiffies + LOAD_FREQ; | 9522 | rq->calc_load_update = jiffies + LOAD_FREQ; |
| @@ -9573,7 +9576,7 @@ void __init sched_init(void) | |||
| 9573 | #elif defined CONFIG_USER_SCHED | 9576 | #elif defined CONFIG_USER_SCHED |
| 9574 | init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, 0, NULL); | 9577 | init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, 0, NULL); |
| 9575 | init_tg_rt_entry(&init_task_group, | 9578 | init_tg_rt_entry(&init_task_group, |
| 9576 | &per_cpu(init_rt_rq, i), | 9579 | &per_cpu(init_rt_rq_var, i), |
| 9577 | &per_cpu(init_sched_rt_entity, i), i, 1, | 9580 | &per_cpu(init_sched_rt_entity, i), i, 1, |
| 9578 | root_task_group.rt_se[i]); | 9581 | root_task_group.rt_se[i]); |
| 9579 | #endif | 9582 | #endif |
| @@ -9611,7 +9614,7 @@ void __init sched_init(void) | |||
| 9611 | #endif | 9614 | #endif |
| 9612 | 9615 | ||
| 9613 | #ifdef CONFIG_RT_MUTEXES | 9616 | #ifdef CONFIG_RT_MUTEXES |
| 9614 | plist_head_init(&init_task.pi_waiters, &init_task.pi_lock); | 9617 | plist_head_init_raw(&init_task.pi_waiters, &init_task.pi_lock); |
| 9615 | #endif | 9618 | #endif |
| 9616 | 9619 | ||
| 9617 | /* | 9620 | /* |
| @@ -9734,13 +9737,13 @@ void normalize_rt_tasks(void) | |||
| 9734 | continue; | 9737 | continue; |
| 9735 | } | 9738 | } |
| 9736 | 9739 | ||
| 9737 | spin_lock(&p->pi_lock); | 9740 | raw_spin_lock(&p->pi_lock); |
| 9738 | rq = __task_rq_lock(p); | 9741 | rq = __task_rq_lock(p); |
| 9739 | 9742 | ||
| 9740 | normalize_task(rq, p); | 9743 | normalize_task(rq, p); |
| 9741 | 9744 | ||
| 9742 | __task_rq_unlock(rq); | 9745 | __task_rq_unlock(rq); |
| 9743 | spin_unlock(&p->pi_lock); | 9746 | raw_spin_unlock(&p->pi_lock); |
| 9744 | } while_each_thread(g, p); | 9747 | } while_each_thread(g, p); |
| 9745 | 9748 | ||
| 9746 | read_unlock_irqrestore(&tasklist_lock, flags); | 9749 | read_unlock_irqrestore(&tasklist_lock, flags); |
| @@ -10103,9 +10106,9 @@ static void set_se_shares(struct sched_entity *se, unsigned long shares) | |||
| 10103 | struct rq *rq = cfs_rq->rq; | 10106 | struct rq *rq = cfs_rq->rq; |
| 10104 | unsigned long flags; | 10107 | unsigned long flags; |
| 10105 | 10108 | ||
| 10106 | spin_lock_irqsave(&rq->lock, flags); | 10109 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 10107 | __set_se_shares(se, shares); | 10110 | __set_se_shares(se, shares); |
| 10108 | spin_unlock_irqrestore(&rq->lock, flags); | 10111 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 10109 | } | 10112 | } |
| 10110 | 10113 | ||
| 10111 | static DEFINE_MUTEX(shares_mutex); | 10114 | static DEFINE_MUTEX(shares_mutex); |
| @@ -10290,18 +10293,18 @@ static int tg_set_bandwidth(struct task_group *tg, | |||
| 10290 | if (err) | 10293 | if (err) |
| 10291 | goto unlock; | 10294 | goto unlock; |
| 10292 | 10295 | ||
| 10293 | spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock); | 10296 | raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock); |
| 10294 | tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period); | 10297 | tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period); |
| 10295 | tg->rt_bandwidth.rt_runtime = rt_runtime; | 10298 | tg->rt_bandwidth.rt_runtime = rt_runtime; |
| 10296 | 10299 | ||
| 10297 | for_each_possible_cpu(i) { | 10300 | for_each_possible_cpu(i) { |
| 10298 | struct rt_rq *rt_rq = tg->rt_rq[i]; | 10301 | struct rt_rq *rt_rq = tg->rt_rq[i]; |
| 10299 | 10302 | ||
| 10300 | spin_lock(&rt_rq->rt_runtime_lock); | 10303 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 10301 | rt_rq->rt_runtime = rt_runtime; | 10304 | rt_rq->rt_runtime = rt_runtime; |
| 10302 | spin_unlock(&rt_rq->rt_runtime_lock); | 10305 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 10303 | } | 10306 | } |
| 10304 | spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock); | 10307 | raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock); |
| 10305 | unlock: | 10308 | unlock: |
| 10306 | read_unlock(&tasklist_lock); | 10309 | read_unlock(&tasklist_lock); |
| 10307 | mutex_unlock(&rt_constraints_mutex); | 10310 | mutex_unlock(&rt_constraints_mutex); |
| @@ -10406,15 +10409,15 @@ static int sched_rt_global_constraints(void) | |||
| 10406 | if (sysctl_sched_rt_runtime == 0) | 10409 | if (sysctl_sched_rt_runtime == 0) |
| 10407 | return -EBUSY; | 10410 | return -EBUSY; |
| 10408 | 10411 | ||
| 10409 | spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); | 10412 | raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); |
| 10410 | for_each_possible_cpu(i) { | 10413 | for_each_possible_cpu(i) { |
| 10411 | struct rt_rq *rt_rq = &cpu_rq(i)->rt; | 10414 | struct rt_rq *rt_rq = &cpu_rq(i)->rt; |
| 10412 | 10415 | ||
| 10413 | spin_lock(&rt_rq->rt_runtime_lock); | 10416 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 10414 | rt_rq->rt_runtime = global_rt_runtime(); | 10417 | rt_rq->rt_runtime = global_rt_runtime(); |
| 10415 | spin_unlock(&rt_rq->rt_runtime_lock); | 10418 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 10416 | } | 10419 | } |
| 10417 | spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); | 10420 | raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); |
| 10418 | 10421 | ||
| 10419 | return 0; | 10422 | return 0; |
| 10420 | } | 10423 | } |
| @@ -10705,9 +10708,9 @@ static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu) | |||
| 10705 | /* | 10708 | /* |
| 10706 | * Take rq->lock to make 64-bit read safe on 32-bit platforms. | 10709 | * Take rq->lock to make 64-bit read safe on 32-bit platforms. |
| 10707 | */ | 10710 | */ |
| 10708 | spin_lock_irq(&cpu_rq(cpu)->lock); | 10711 | raw_spin_lock_irq(&cpu_rq(cpu)->lock); |
| 10709 | data = *cpuusage; | 10712 | data = *cpuusage; |
| 10710 | spin_unlock_irq(&cpu_rq(cpu)->lock); | 10713 | raw_spin_unlock_irq(&cpu_rq(cpu)->lock); |
| 10711 | #else | 10714 | #else |
| 10712 | data = *cpuusage; | 10715 | data = *cpuusage; |
| 10713 | #endif | 10716 | #endif |
| @@ -10723,9 +10726,9 @@ static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val) | |||
| 10723 | /* | 10726 | /* |
| 10724 | * Take rq->lock to make 64-bit write safe on 32-bit platforms. | 10727 | * Take rq->lock to make 64-bit write safe on 32-bit platforms. |
| 10725 | */ | 10728 | */ |
| 10726 | spin_lock_irq(&cpu_rq(cpu)->lock); | 10729 | raw_spin_lock_irq(&cpu_rq(cpu)->lock); |
| 10727 | *cpuusage = val; | 10730 | *cpuusage = val; |
| 10728 | spin_unlock_irq(&cpu_rq(cpu)->lock); | 10731 | raw_spin_unlock_irq(&cpu_rq(cpu)->lock); |
| 10729 | #else | 10732 | #else |
| 10730 | *cpuusage = val; | 10733 | *cpuusage = val; |
| 10731 | #endif | 10734 | #endif |
| @@ -10959,9 +10962,9 @@ void synchronize_sched_expedited(void) | |||
| 10959 | init_completion(&req->done); | 10962 | init_completion(&req->done); |
| 10960 | req->task = NULL; | 10963 | req->task = NULL; |
| 10961 | req->dest_cpu = RCU_MIGRATION_NEED_QS; | 10964 | req->dest_cpu = RCU_MIGRATION_NEED_QS; |
| 10962 | spin_lock_irqsave(&rq->lock, flags); | 10965 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 10963 | list_add(&req->list, &rq->migration_queue); | 10966 | list_add(&req->list, &rq->migration_queue); |
| 10964 | spin_unlock_irqrestore(&rq->lock, flags); | 10967 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 10965 | wake_up_process(rq->migration_thread); | 10968 | wake_up_process(rq->migration_thread); |
| 10966 | } | 10969 | } |
| 10967 | for_each_online_cpu(cpu) { | 10970 | for_each_online_cpu(cpu) { |
| @@ -10969,11 +10972,11 @@ void synchronize_sched_expedited(void) | |||
| 10969 | req = &per_cpu(rcu_migration_req, cpu); | 10972 | req = &per_cpu(rcu_migration_req, cpu); |
| 10970 | rq = cpu_rq(cpu); | 10973 | rq = cpu_rq(cpu); |
| 10971 | wait_for_completion(&req->done); | 10974 | wait_for_completion(&req->done); |
| 10972 | spin_lock_irqsave(&rq->lock, flags); | 10975 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 10973 | if (unlikely(req->dest_cpu == RCU_MIGRATION_MUST_SYNC)) | 10976 | if (unlikely(req->dest_cpu == RCU_MIGRATION_MUST_SYNC)) |
| 10974 | need_full_sync = 1; | 10977 | need_full_sync = 1; |
| 10975 | req->dest_cpu = RCU_MIGRATION_IDLE; | 10978 | req->dest_cpu = RCU_MIGRATION_IDLE; |
| 10976 | spin_unlock_irqrestore(&rq->lock, flags); | 10979 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 10977 | } | 10980 | } |
| 10978 | rcu_expedited_state = RCU_EXPEDITED_STATE_IDLE; | 10981 | rcu_expedited_state = RCU_EXPEDITED_STATE_IDLE; |
| 10979 | synchronize_sched_expedited_count++; | 10982 | synchronize_sched_expedited_count++; |
diff --git a/kernel/sched_cpupri.c b/kernel/sched_cpupri.c index 0f052fc674d5..597b33099dfa 100644 --- a/kernel/sched_cpupri.c +++ b/kernel/sched_cpupri.c | |||
| @@ -135,26 +135,26 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri) | |||
| 135 | if (likely(newpri != CPUPRI_INVALID)) { | 135 | if (likely(newpri != CPUPRI_INVALID)) { |
| 136 | struct cpupri_vec *vec = &cp->pri_to_cpu[newpri]; | 136 | struct cpupri_vec *vec = &cp->pri_to_cpu[newpri]; |
| 137 | 137 | ||
| 138 | spin_lock_irqsave(&vec->lock, flags); | 138 | raw_spin_lock_irqsave(&vec->lock, flags); |
| 139 | 139 | ||
| 140 | cpumask_set_cpu(cpu, vec->mask); | 140 | cpumask_set_cpu(cpu, vec->mask); |
| 141 | vec->count++; | 141 | vec->count++; |
| 142 | if (vec->count == 1) | 142 | if (vec->count == 1) |
| 143 | set_bit(newpri, cp->pri_active); | 143 | set_bit(newpri, cp->pri_active); |
| 144 | 144 | ||
| 145 | spin_unlock_irqrestore(&vec->lock, flags); | 145 | raw_spin_unlock_irqrestore(&vec->lock, flags); |
| 146 | } | 146 | } |
| 147 | if (likely(oldpri != CPUPRI_INVALID)) { | 147 | if (likely(oldpri != CPUPRI_INVALID)) { |
| 148 | struct cpupri_vec *vec = &cp->pri_to_cpu[oldpri]; | 148 | struct cpupri_vec *vec = &cp->pri_to_cpu[oldpri]; |
| 149 | 149 | ||
| 150 | spin_lock_irqsave(&vec->lock, flags); | 150 | raw_spin_lock_irqsave(&vec->lock, flags); |
| 151 | 151 | ||
| 152 | vec->count--; | 152 | vec->count--; |
| 153 | if (!vec->count) | 153 | if (!vec->count) |
| 154 | clear_bit(oldpri, cp->pri_active); | 154 | clear_bit(oldpri, cp->pri_active); |
| 155 | cpumask_clear_cpu(cpu, vec->mask); | 155 | cpumask_clear_cpu(cpu, vec->mask); |
| 156 | 156 | ||
| 157 | spin_unlock_irqrestore(&vec->lock, flags); | 157 | raw_spin_unlock_irqrestore(&vec->lock, flags); |
| 158 | } | 158 | } |
| 159 | 159 | ||
| 160 | *currpri = newpri; | 160 | *currpri = newpri; |
| @@ -180,7 +180,7 @@ int cpupri_init(struct cpupri *cp, bool bootmem) | |||
| 180 | for (i = 0; i < CPUPRI_NR_PRIORITIES; i++) { | 180 | for (i = 0; i < CPUPRI_NR_PRIORITIES; i++) { |
| 181 | struct cpupri_vec *vec = &cp->pri_to_cpu[i]; | 181 | struct cpupri_vec *vec = &cp->pri_to_cpu[i]; |
| 182 | 182 | ||
| 183 | spin_lock_init(&vec->lock); | 183 | raw_spin_lock_init(&vec->lock); |
| 184 | vec->count = 0; | 184 | vec->count = 0; |
| 185 | if (!zalloc_cpumask_var(&vec->mask, gfp)) | 185 | if (!zalloc_cpumask_var(&vec->mask, gfp)) |
| 186 | goto cleanup; | 186 | goto cleanup; |
diff --git a/kernel/sched_cpupri.h b/kernel/sched_cpupri.h index 9a7e859b8fbf..7cb5bb6b95be 100644 --- a/kernel/sched_cpupri.h +++ b/kernel/sched_cpupri.h | |||
| @@ -12,7 +12,7 @@ | |||
| 12 | /* values 2-101 are RT priorities 0-99 */ | 12 | /* values 2-101 are RT priorities 0-99 */ |
| 13 | 13 | ||
| 14 | struct cpupri_vec { | 14 | struct cpupri_vec { |
| 15 | spinlock_t lock; | 15 | raw_spinlock_t lock; |
| 16 | int count; | 16 | int count; |
| 17 | cpumask_var_t mask; | 17 | cpumask_var_t mask; |
| 18 | }; | 18 | }; |
diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c index 5ae24fc65d75..67f95aada4b9 100644 --- a/kernel/sched_debug.c +++ b/kernel/sched_debug.c | |||
| @@ -184,7 +184,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) | |||
| 184 | SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock", | 184 | SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock", |
| 185 | SPLIT_NS(cfs_rq->exec_clock)); | 185 | SPLIT_NS(cfs_rq->exec_clock)); |
| 186 | 186 | ||
| 187 | spin_lock_irqsave(&rq->lock, flags); | 187 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 188 | if (cfs_rq->rb_leftmost) | 188 | if (cfs_rq->rb_leftmost) |
| 189 | MIN_vruntime = (__pick_next_entity(cfs_rq))->vruntime; | 189 | MIN_vruntime = (__pick_next_entity(cfs_rq))->vruntime; |
| 190 | last = __pick_last_entity(cfs_rq); | 190 | last = __pick_last_entity(cfs_rq); |
| @@ -192,7 +192,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) | |||
| 192 | max_vruntime = last->vruntime; | 192 | max_vruntime = last->vruntime; |
| 193 | min_vruntime = cfs_rq->min_vruntime; | 193 | min_vruntime = cfs_rq->min_vruntime; |
| 194 | rq0_min_vruntime = cpu_rq(0)->cfs.min_vruntime; | 194 | rq0_min_vruntime = cpu_rq(0)->cfs.min_vruntime; |
| 195 | spin_unlock_irqrestore(&rq->lock, flags); | 195 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 196 | SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "MIN_vruntime", | 196 | SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "MIN_vruntime", |
| 197 | SPLIT_NS(MIN_vruntime)); | 197 | SPLIT_NS(MIN_vruntime)); |
| 198 | SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "min_vruntime", | 198 | SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "min_vruntime", |
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 804a411838f1..5bedf6e3ebf3 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c | |||
| @@ -1955,7 +1955,7 @@ static void task_fork_fair(struct task_struct *p) | |||
| 1955 | struct rq *rq = this_rq(); | 1955 | struct rq *rq = this_rq(); |
| 1956 | unsigned long flags; | 1956 | unsigned long flags; |
| 1957 | 1957 | ||
| 1958 | spin_lock_irqsave(&rq->lock, flags); | 1958 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 1959 | 1959 | ||
| 1960 | if (unlikely(task_cpu(p) != this_cpu)) | 1960 | if (unlikely(task_cpu(p) != this_cpu)) |
| 1961 | __set_task_cpu(p, this_cpu); | 1961 | __set_task_cpu(p, this_cpu); |
| @@ -1975,7 +1975,7 @@ static void task_fork_fair(struct task_struct *p) | |||
| 1975 | resched_task(rq->curr); | 1975 | resched_task(rq->curr); |
| 1976 | } | 1976 | } |
| 1977 | 1977 | ||
| 1978 | spin_unlock_irqrestore(&rq->lock, flags); | 1978 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 1979 | } | 1979 | } |
| 1980 | 1980 | ||
| 1981 | /* | 1981 | /* |
diff --git a/kernel/sched_idletask.c b/kernel/sched_idletask.c index b810e22772d5..21b969a28725 100644 --- a/kernel/sched_idletask.c +++ b/kernel/sched_idletask.c | |||
| @@ -34,10 +34,10 @@ static struct task_struct *pick_next_task_idle(struct rq *rq) | |||
| 34 | static void | 34 | static void |
| 35 | dequeue_task_idle(struct rq *rq, struct task_struct *p, int sleep) | 35 | dequeue_task_idle(struct rq *rq, struct task_struct *p, int sleep) |
| 36 | { | 36 | { |
| 37 | spin_unlock_irq(&rq->lock); | 37 | raw_spin_unlock_irq(&rq->lock); |
| 38 | pr_err("bad: scheduling from the idle thread!\n"); | 38 | pr_err("bad: scheduling from the idle thread!\n"); |
| 39 | dump_stack(); | 39 | dump_stack(); |
| 40 | spin_lock_irq(&rq->lock); | 40 | raw_spin_lock_irq(&rq->lock); |
| 41 | } | 41 | } |
| 42 | 42 | ||
| 43 | static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) | 43 | static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) |
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index aecbd9c6b20c..d2ea2828164e 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c | |||
| @@ -327,7 +327,7 @@ static int do_balance_runtime(struct rt_rq *rt_rq) | |||
| 327 | 327 | ||
| 328 | weight = cpumask_weight(rd->span); | 328 | weight = cpumask_weight(rd->span); |
| 329 | 329 | ||
| 330 | spin_lock(&rt_b->rt_runtime_lock); | 330 | raw_spin_lock(&rt_b->rt_runtime_lock); |
| 331 | rt_period = ktime_to_ns(rt_b->rt_period); | 331 | rt_period = ktime_to_ns(rt_b->rt_period); |
| 332 | for_each_cpu(i, rd->span) { | 332 | for_each_cpu(i, rd->span) { |
| 333 | struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); | 333 | struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); |
| @@ -336,7 +336,7 @@ static int do_balance_runtime(struct rt_rq *rt_rq) | |||
| 336 | if (iter == rt_rq) | 336 | if (iter == rt_rq) |
| 337 | continue; | 337 | continue; |
| 338 | 338 | ||
| 339 | spin_lock(&iter->rt_runtime_lock); | 339 | raw_spin_lock(&iter->rt_runtime_lock); |
| 340 | /* | 340 | /* |
| 341 | * Either all rqs have inf runtime and there's nothing to steal | 341 | * Either all rqs have inf runtime and there's nothing to steal |
| 342 | * or __disable_runtime() below sets a specific rq to inf to | 342 | * or __disable_runtime() below sets a specific rq to inf to |
| @@ -358,14 +358,14 @@ static int do_balance_runtime(struct rt_rq *rt_rq) | |||
| 358 | rt_rq->rt_runtime += diff; | 358 | rt_rq->rt_runtime += diff; |
| 359 | more = 1; | 359 | more = 1; |
| 360 | if (rt_rq->rt_runtime == rt_period) { | 360 | if (rt_rq->rt_runtime == rt_period) { |
| 361 | spin_unlock(&iter->rt_runtime_lock); | 361 | raw_spin_unlock(&iter->rt_runtime_lock); |
| 362 | break; | 362 | break; |
| 363 | } | 363 | } |
| 364 | } | 364 | } |
| 365 | next: | 365 | next: |
| 366 | spin_unlock(&iter->rt_runtime_lock); | 366 | raw_spin_unlock(&iter->rt_runtime_lock); |
| 367 | } | 367 | } |
| 368 | spin_unlock(&rt_b->rt_runtime_lock); | 368 | raw_spin_unlock(&rt_b->rt_runtime_lock); |
| 369 | 369 | ||
| 370 | return more; | 370 | return more; |
| 371 | } | 371 | } |
| @@ -386,8 +386,8 @@ static void __disable_runtime(struct rq *rq) | |||
| 386 | s64 want; | 386 | s64 want; |
| 387 | int i; | 387 | int i; |
| 388 | 388 | ||
| 389 | spin_lock(&rt_b->rt_runtime_lock); | 389 | raw_spin_lock(&rt_b->rt_runtime_lock); |
| 390 | spin_lock(&rt_rq->rt_runtime_lock); | 390 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 391 | /* | 391 | /* |
| 392 | * Either we're all inf and nobody needs to borrow, or we're | 392 | * Either we're all inf and nobody needs to borrow, or we're |
| 393 | * already disabled and thus have nothing to do, or we have | 393 | * already disabled and thus have nothing to do, or we have |
| @@ -396,7 +396,7 @@ static void __disable_runtime(struct rq *rq) | |||
| 396 | if (rt_rq->rt_runtime == RUNTIME_INF || | 396 | if (rt_rq->rt_runtime == RUNTIME_INF || |
| 397 | rt_rq->rt_runtime == rt_b->rt_runtime) | 397 | rt_rq->rt_runtime == rt_b->rt_runtime) |
| 398 | goto balanced; | 398 | goto balanced; |
| 399 | spin_unlock(&rt_rq->rt_runtime_lock); | 399 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 400 | 400 | ||
| 401 | /* | 401 | /* |
| 402 | * Calculate the difference between what we started out with | 402 | * Calculate the difference between what we started out with |
| @@ -418,7 +418,7 @@ static void __disable_runtime(struct rq *rq) | |||
| 418 | if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF) | 418 | if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF) |
| 419 | continue; | 419 | continue; |
| 420 | 420 | ||
| 421 | spin_lock(&iter->rt_runtime_lock); | 421 | raw_spin_lock(&iter->rt_runtime_lock); |
| 422 | if (want > 0) { | 422 | if (want > 0) { |
| 423 | diff = min_t(s64, iter->rt_runtime, want); | 423 | diff = min_t(s64, iter->rt_runtime, want); |
| 424 | iter->rt_runtime -= diff; | 424 | iter->rt_runtime -= diff; |
| @@ -427,13 +427,13 @@ static void __disable_runtime(struct rq *rq) | |||
| 427 | iter->rt_runtime -= want; | 427 | iter->rt_runtime -= want; |
| 428 | want -= want; | 428 | want -= want; |
| 429 | } | 429 | } |
| 430 | spin_unlock(&iter->rt_runtime_lock); | 430 | raw_spin_unlock(&iter->rt_runtime_lock); |
| 431 | 431 | ||
| 432 | if (!want) | 432 | if (!want) |
| 433 | break; | 433 | break; |
| 434 | } | 434 | } |
| 435 | 435 | ||
| 436 | spin_lock(&rt_rq->rt_runtime_lock); | 436 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 437 | /* | 437 | /* |
| 438 | * We cannot be left wanting - that would mean some runtime | 438 | * We cannot be left wanting - that would mean some runtime |
| 439 | * leaked out of the system. | 439 | * leaked out of the system. |
| @@ -445,8 +445,8 @@ balanced: | |||
| 445 | * runtime - in which case borrowing doesn't make sense. | 445 | * runtime - in which case borrowing doesn't make sense. |
| 446 | */ | 446 | */ |
| 447 | rt_rq->rt_runtime = RUNTIME_INF; | 447 | rt_rq->rt_runtime = RUNTIME_INF; |
| 448 | spin_unlock(&rt_rq->rt_runtime_lock); | 448 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 449 | spin_unlock(&rt_b->rt_runtime_lock); | 449 | raw_spin_unlock(&rt_b->rt_runtime_lock); |
| 450 | } | 450 | } |
| 451 | } | 451 | } |
| 452 | 452 | ||
| @@ -454,9 +454,9 @@ static void disable_runtime(struct rq *rq) | |||
| 454 | { | 454 | { |
| 455 | unsigned long flags; | 455 | unsigned long flags; |
| 456 | 456 | ||
| 457 | spin_lock_irqsave(&rq->lock, flags); | 457 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 458 | __disable_runtime(rq); | 458 | __disable_runtime(rq); |
| 459 | spin_unlock_irqrestore(&rq->lock, flags); | 459 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 460 | } | 460 | } |
| 461 | 461 | ||
| 462 | static void __enable_runtime(struct rq *rq) | 462 | static void __enable_runtime(struct rq *rq) |
| @@ -472,13 +472,13 @@ static void __enable_runtime(struct rq *rq) | |||
| 472 | for_each_leaf_rt_rq(rt_rq, rq) { | 472 | for_each_leaf_rt_rq(rt_rq, rq) { |
| 473 | struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); | 473 | struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); |
| 474 | 474 | ||
| 475 | spin_lock(&rt_b->rt_runtime_lock); | 475 | raw_spin_lock(&rt_b->rt_runtime_lock); |
| 476 | spin_lock(&rt_rq->rt_runtime_lock); | 476 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 477 | rt_rq->rt_runtime = rt_b->rt_runtime; | 477 | rt_rq->rt_runtime = rt_b->rt_runtime; |
| 478 | rt_rq->rt_time = 0; | 478 | rt_rq->rt_time = 0; |
| 479 | rt_rq->rt_throttled = 0; | 479 | rt_rq->rt_throttled = 0; |
| 480 | spin_unlock(&rt_rq->rt_runtime_lock); | 480 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 481 | spin_unlock(&rt_b->rt_runtime_lock); | 481 | raw_spin_unlock(&rt_b->rt_runtime_lock); |
| 482 | } | 482 | } |
| 483 | } | 483 | } |
| 484 | 484 | ||
| @@ -486,9 +486,9 @@ static void enable_runtime(struct rq *rq) | |||
| 486 | { | 486 | { |
| 487 | unsigned long flags; | 487 | unsigned long flags; |
| 488 | 488 | ||
| 489 | spin_lock_irqsave(&rq->lock, flags); | 489 | raw_spin_lock_irqsave(&rq->lock, flags); |
| 490 | __enable_runtime(rq); | 490 | __enable_runtime(rq); |
| 491 | spin_unlock_irqrestore(&rq->lock, flags); | 491 | raw_spin_unlock_irqrestore(&rq->lock, flags); |
| 492 | } | 492 | } |
| 493 | 493 | ||
| 494 | static int balance_runtime(struct rt_rq *rt_rq) | 494 | static int balance_runtime(struct rt_rq *rt_rq) |
| @@ -496,9 +496,9 @@ static int balance_runtime(struct rt_rq *rt_rq) | |||
| 496 | int more = 0; | 496 | int more = 0; |
| 497 | 497 | ||
| 498 | if (rt_rq->rt_time > rt_rq->rt_runtime) { | 498 | if (rt_rq->rt_time > rt_rq->rt_runtime) { |
| 499 | spin_unlock(&rt_rq->rt_runtime_lock); | 499 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 500 | more = do_balance_runtime(rt_rq); | 500 | more = do_balance_runtime(rt_rq); |
| 501 | spin_lock(&rt_rq->rt_runtime_lock); | 501 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 502 | } | 502 | } |
| 503 | 503 | ||
| 504 | return more; | 504 | return more; |
| @@ -524,11 +524,11 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) | |||
| 524 | struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i); | 524 | struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i); |
| 525 | struct rq *rq = rq_of_rt_rq(rt_rq); | 525 | struct rq *rq = rq_of_rt_rq(rt_rq); |
| 526 | 526 | ||
| 527 | spin_lock(&rq->lock); | 527 | raw_spin_lock(&rq->lock); |
| 528 | if (rt_rq->rt_time) { | 528 | if (rt_rq->rt_time) { |
| 529 | u64 runtime; | 529 | u64 runtime; |
| 530 | 530 | ||
| 531 | spin_lock(&rt_rq->rt_runtime_lock); | 531 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 532 | if (rt_rq->rt_throttled) | 532 | if (rt_rq->rt_throttled) |
| 533 | balance_runtime(rt_rq); | 533 | balance_runtime(rt_rq); |
| 534 | runtime = rt_rq->rt_runtime; | 534 | runtime = rt_rq->rt_runtime; |
| @@ -539,13 +539,13 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) | |||
| 539 | } | 539 | } |
| 540 | if (rt_rq->rt_time || rt_rq->rt_nr_running) | 540 | if (rt_rq->rt_time || rt_rq->rt_nr_running) |
| 541 | idle = 0; | 541 | idle = 0; |
| 542 | spin_unlock(&rt_rq->rt_runtime_lock); | 542 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 543 | } else if (rt_rq->rt_nr_running) | 543 | } else if (rt_rq->rt_nr_running) |
| 544 | idle = 0; | 544 | idle = 0; |
| 545 | 545 | ||
| 546 | if (enqueue) | 546 | if (enqueue) |
| 547 | sched_rt_rq_enqueue(rt_rq); | 547 | sched_rt_rq_enqueue(rt_rq); |
| 548 | spin_unlock(&rq->lock); | 548 | raw_spin_unlock(&rq->lock); |
| 549 | } | 549 | } |
| 550 | 550 | ||
| 551 | return idle; | 551 | return idle; |
| @@ -624,11 +624,11 @@ static void update_curr_rt(struct rq *rq) | |||
| 624 | rt_rq = rt_rq_of_se(rt_se); | 624 | rt_rq = rt_rq_of_se(rt_se); |
| 625 | 625 | ||
| 626 | if (sched_rt_runtime(rt_rq) != RUNTIME_INF) { | 626 | if (sched_rt_runtime(rt_rq) != RUNTIME_INF) { |
| 627 | spin_lock(&rt_rq->rt_runtime_lock); | 627 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 628 | rt_rq->rt_time += delta_exec; | 628 | rt_rq->rt_time += delta_exec; |
| 629 | if (sched_rt_runtime_exceeded(rt_rq)) | 629 | if (sched_rt_runtime_exceeded(rt_rq)) |
| 630 | resched_task(curr); | 630 | resched_task(curr); |
| 631 | spin_unlock(&rt_rq->rt_runtime_lock); | 631 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 632 | } | 632 | } |
| 633 | } | 633 | } |
| 634 | } | 634 | } |
| @@ -1246,7 +1246,7 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq) | |||
| 1246 | task_running(rq, task) || | 1246 | task_running(rq, task) || |
| 1247 | !task->se.on_rq)) { | 1247 | !task->se.on_rq)) { |
| 1248 | 1248 | ||
| 1249 | spin_unlock(&lowest_rq->lock); | 1249 | raw_spin_unlock(&lowest_rq->lock); |
| 1250 | lowest_rq = NULL; | 1250 | lowest_rq = NULL; |
| 1251 | break; | 1251 | break; |
| 1252 | } | 1252 | } |
diff --git a/kernel/smp.c b/kernel/smp.c index a8c76069cf50..de735a6637d0 100644 --- a/kernel/smp.c +++ b/kernel/smp.c | |||
| @@ -16,11 +16,11 @@ static DEFINE_PER_CPU(struct call_single_queue, call_single_queue); | |||
| 16 | 16 | ||
| 17 | static struct { | 17 | static struct { |
| 18 | struct list_head queue; | 18 | struct list_head queue; |
| 19 | spinlock_t lock; | 19 | raw_spinlock_t lock; |
| 20 | } call_function __cacheline_aligned_in_smp = | 20 | } call_function __cacheline_aligned_in_smp = |
| 21 | { | 21 | { |
| 22 | .queue = LIST_HEAD_INIT(call_function.queue), | 22 | .queue = LIST_HEAD_INIT(call_function.queue), |
| 23 | .lock = __SPIN_LOCK_UNLOCKED(call_function.lock), | 23 | .lock = __RAW_SPIN_LOCK_UNLOCKED(call_function.lock), |
| 24 | }; | 24 | }; |
| 25 | 25 | ||
| 26 | enum { | 26 | enum { |
| @@ -35,7 +35,7 @@ struct call_function_data { | |||
| 35 | 35 | ||
| 36 | struct call_single_queue { | 36 | struct call_single_queue { |
| 37 | struct list_head list; | 37 | struct list_head list; |
| 38 | spinlock_t lock; | 38 | raw_spinlock_t lock; |
| 39 | }; | 39 | }; |
| 40 | 40 | ||
| 41 | static DEFINE_PER_CPU(struct call_function_data, cfd_data); | 41 | static DEFINE_PER_CPU(struct call_function_data, cfd_data); |
| @@ -80,7 +80,7 @@ static int __cpuinit init_call_single_data(void) | |||
| 80 | for_each_possible_cpu(i) { | 80 | for_each_possible_cpu(i) { |
| 81 | struct call_single_queue *q = &per_cpu(call_single_queue, i); | 81 | struct call_single_queue *q = &per_cpu(call_single_queue, i); |
| 82 | 82 | ||
| 83 | spin_lock_init(&q->lock); | 83 | raw_spin_lock_init(&q->lock); |
| 84 | INIT_LIST_HEAD(&q->list); | 84 | INIT_LIST_HEAD(&q->list); |
| 85 | } | 85 | } |
| 86 | 86 | ||
| @@ -141,10 +141,10 @@ void generic_exec_single(int cpu, struct call_single_data *data, int wait) | |||
| 141 | unsigned long flags; | 141 | unsigned long flags; |
| 142 | int ipi; | 142 | int ipi; |
| 143 | 143 | ||
| 144 | spin_lock_irqsave(&dst->lock, flags); | 144 | raw_spin_lock_irqsave(&dst->lock, flags); |
| 145 | ipi = list_empty(&dst->list); | 145 | ipi = list_empty(&dst->list); |
| 146 | list_add_tail(&data->list, &dst->list); | 146 | list_add_tail(&data->list, &dst->list); |
| 147 | spin_unlock_irqrestore(&dst->lock, flags); | 147 | raw_spin_unlock_irqrestore(&dst->lock, flags); |
| 148 | 148 | ||
| 149 | /* | 149 | /* |
| 150 | * The list addition should be visible before sending the IPI | 150 | * The list addition should be visible before sending the IPI |
| @@ -171,7 +171,7 @@ void generic_exec_single(int cpu, struct call_single_data *data, int wait) | |||
| 171 | void generic_smp_call_function_interrupt(void) | 171 | void generic_smp_call_function_interrupt(void) |
| 172 | { | 172 | { |
| 173 | struct call_function_data *data; | 173 | struct call_function_data *data; |
| 174 | int cpu = get_cpu(); | 174 | int cpu = smp_processor_id(); |
| 175 | 175 | ||
| 176 | /* | 176 | /* |
| 177 | * Shouldn't receive this interrupt on a cpu that is not yet online. | 177 | * Shouldn't receive this interrupt on a cpu that is not yet online. |
| @@ -201,9 +201,9 @@ void generic_smp_call_function_interrupt(void) | |||
| 201 | refs = atomic_dec_return(&data->refs); | 201 | refs = atomic_dec_return(&data->refs); |
| 202 | WARN_ON(refs < 0); | 202 | WARN_ON(refs < 0); |
| 203 | if (!refs) { | 203 | if (!refs) { |
| 204 | spin_lock(&call_function.lock); | 204 | raw_spin_lock(&call_function.lock); |
| 205 | list_del_rcu(&data->csd.list); | 205 | list_del_rcu(&data->csd.list); |
| 206 | spin_unlock(&call_function.lock); | 206 | raw_spin_unlock(&call_function.lock); |
| 207 | } | 207 | } |
| 208 | 208 | ||
| 209 | if (refs) | 209 | if (refs) |
| @@ -212,7 +212,6 @@ void generic_smp_call_function_interrupt(void) | |||
| 212 | csd_unlock(&data->csd); | 212 | csd_unlock(&data->csd); |
| 213 | } | 213 | } |
| 214 | 214 | ||
| 215 | put_cpu(); | ||
| 216 | } | 215 | } |
| 217 | 216 | ||
| 218 | /* | 217 | /* |
| @@ -230,9 +229,9 @@ void generic_smp_call_function_single_interrupt(void) | |||
| 230 | */ | 229 | */ |
| 231 | WARN_ON_ONCE(!cpu_online(smp_processor_id())); | 230 | WARN_ON_ONCE(!cpu_online(smp_processor_id())); |
| 232 | 231 | ||
| 233 | spin_lock(&q->lock); | 232 | raw_spin_lock(&q->lock); |
| 234 | list_replace_init(&q->list, &list); | 233 | list_replace_init(&q->list, &list); |
| 235 | spin_unlock(&q->lock); | 234 | raw_spin_unlock(&q->lock); |
| 236 | 235 | ||
| 237 | while (!list_empty(&list)) { | 236 | while (!list_empty(&list)) { |
| 238 | struct call_single_data *data; | 237 | struct call_single_data *data; |
| @@ -449,14 +448,14 @@ void smp_call_function_many(const struct cpumask *mask, | |||
| 449 | cpumask_clear_cpu(this_cpu, data->cpumask); | 448 | cpumask_clear_cpu(this_cpu, data->cpumask); |
| 450 | atomic_set(&data->refs, cpumask_weight(data->cpumask)); | 449 | atomic_set(&data->refs, cpumask_weight(data->cpumask)); |
| 451 | 450 | ||
| 452 | spin_lock_irqsave(&call_function.lock, flags); | 451 | raw_spin_lock_irqsave(&call_function.lock, flags); |
| 453 | /* | 452 | /* |
| 454 | * Place entry at the _HEAD_ of the list, so that any cpu still | 453 | * Place entry at the _HEAD_ of the list, so that any cpu still |
| 455 | * observing the entry in generic_smp_call_function_interrupt() | 454 | * observing the entry in generic_smp_call_function_interrupt() |
| 456 | * will not miss any other list entries: | 455 | * will not miss any other list entries: |
| 457 | */ | 456 | */ |
| 458 | list_add_rcu(&data->csd.list, &call_function.queue); | 457 | list_add_rcu(&data->csd.list, &call_function.queue); |
| 459 | spin_unlock_irqrestore(&call_function.lock, flags); | 458 | raw_spin_unlock_irqrestore(&call_function.lock, flags); |
| 460 | 459 | ||
| 461 | /* | 460 | /* |
| 462 | * Make the list addition visible before sending the ipi. | 461 | * Make the list addition visible before sending the ipi. |
| @@ -501,20 +500,20 @@ EXPORT_SYMBOL(smp_call_function); | |||
| 501 | 500 | ||
| 502 | void ipi_call_lock(void) | 501 | void ipi_call_lock(void) |
| 503 | { | 502 | { |
| 504 | spin_lock(&call_function.lock); | 503 | raw_spin_lock(&call_function.lock); |
| 505 | } | 504 | } |
| 506 | 505 | ||
| 507 | void ipi_call_unlock(void) | 506 | void ipi_call_unlock(void) |
| 508 | { | 507 | { |
| 509 | spin_unlock(&call_function.lock); | 508 | raw_spin_unlock(&call_function.lock); |
| 510 | } | 509 | } |
| 511 | 510 | ||
| 512 | void ipi_call_lock_irq(void) | 511 | void ipi_call_lock_irq(void) |
| 513 | { | 512 | { |
| 514 | spin_lock_irq(&call_function.lock); | 513 | raw_spin_lock_irq(&call_function.lock); |
| 515 | } | 514 | } |
| 516 | 515 | ||
| 517 | void ipi_call_unlock_irq(void) | 516 | void ipi_call_unlock_irq(void) |
| 518 | { | 517 | { |
| 519 | spin_unlock_irq(&call_function.lock); | 518 | raw_spin_unlock_irq(&call_function.lock); |
| 520 | } | 519 | } |
diff --git a/kernel/softirq.c b/kernel/softirq.c index 21939d9e830e..a09502e2ef75 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c | |||
| @@ -697,7 +697,7 @@ void __init softirq_init(void) | |||
| 697 | open_softirq(HI_SOFTIRQ, tasklet_hi_action); | 697 | open_softirq(HI_SOFTIRQ, tasklet_hi_action); |
| 698 | } | 698 | } |
| 699 | 699 | ||
| 700 | static int ksoftirqd(void * __bind_cpu) | 700 | static int run_ksoftirqd(void * __bind_cpu) |
| 701 | { | 701 | { |
| 702 | set_current_state(TASK_INTERRUPTIBLE); | 702 | set_current_state(TASK_INTERRUPTIBLE); |
| 703 | 703 | ||
| @@ -810,7 +810,7 @@ static int __cpuinit cpu_callback(struct notifier_block *nfb, | |||
| 810 | switch (action) { | 810 | switch (action) { |
| 811 | case CPU_UP_PREPARE: | 811 | case CPU_UP_PREPARE: |
| 812 | case CPU_UP_PREPARE_FROZEN: | 812 | case CPU_UP_PREPARE_FROZEN: |
| 813 | p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu); | 813 | p = kthread_create(run_ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu); |
| 814 | if (IS_ERR(p)) { | 814 | if (IS_ERR(p)) { |
| 815 | printk("ksoftirqd for %i failed\n", hotcpu); | 815 | printk("ksoftirqd for %i failed\n", hotcpu); |
| 816 | return NOTIFY_BAD; | 816 | return NOTIFY_BAD; |
diff --git a/kernel/softlockup.c b/kernel/softlockup.c index 81324d12eb35..d22579087e27 100644 --- a/kernel/softlockup.c +++ b/kernel/softlockup.c | |||
| @@ -22,9 +22,9 @@ | |||
| 22 | 22 | ||
| 23 | static DEFINE_SPINLOCK(print_lock); | 23 | static DEFINE_SPINLOCK(print_lock); |
| 24 | 24 | ||
| 25 | static DEFINE_PER_CPU(unsigned long, touch_timestamp); | 25 | static DEFINE_PER_CPU(unsigned long, softlockup_touch_ts); /* touch timestamp */ |
| 26 | static DEFINE_PER_CPU(unsigned long, print_timestamp); | 26 | static DEFINE_PER_CPU(unsigned long, softlockup_print_ts); /* print timestamp */ |
| 27 | static DEFINE_PER_CPU(struct task_struct *, watchdog_task); | 27 | static DEFINE_PER_CPU(struct task_struct *, softlockup_watchdog); |
| 28 | 28 | ||
| 29 | static int __read_mostly did_panic; | 29 | static int __read_mostly did_panic; |
| 30 | int __read_mostly softlockup_thresh = 60; | 30 | int __read_mostly softlockup_thresh = 60; |
| @@ -70,12 +70,12 @@ static void __touch_softlockup_watchdog(void) | |||
| 70 | { | 70 | { |
| 71 | int this_cpu = raw_smp_processor_id(); | 71 | int this_cpu = raw_smp_processor_id(); |
| 72 | 72 | ||
| 73 | __raw_get_cpu_var(touch_timestamp) = get_timestamp(this_cpu); | 73 | __raw_get_cpu_var(softlockup_touch_ts) = get_timestamp(this_cpu); |
| 74 | } | 74 | } |
| 75 | 75 | ||
| 76 | void touch_softlockup_watchdog(void) | 76 | void touch_softlockup_watchdog(void) |
| 77 | { | 77 | { |
| 78 | __raw_get_cpu_var(touch_timestamp) = 0; | 78 | __raw_get_cpu_var(softlockup_touch_ts) = 0; |
| 79 | } | 79 | } |
| 80 | EXPORT_SYMBOL(touch_softlockup_watchdog); | 80 | EXPORT_SYMBOL(touch_softlockup_watchdog); |
| 81 | 81 | ||
| @@ -85,7 +85,7 @@ void touch_all_softlockup_watchdogs(void) | |||
| 85 | 85 | ||
| 86 | /* Cause each CPU to re-update its timestamp rather than complain */ | 86 | /* Cause each CPU to re-update its timestamp rather than complain */ |
| 87 | for_each_online_cpu(cpu) | 87 | for_each_online_cpu(cpu) |
| 88 | per_cpu(touch_timestamp, cpu) = 0; | 88 | per_cpu(softlockup_touch_ts, cpu) = 0; |
| 89 | } | 89 | } |
| 90 | EXPORT_SYMBOL(touch_all_softlockup_watchdogs); | 90 | EXPORT_SYMBOL(touch_all_softlockup_watchdogs); |
| 91 | 91 | ||
| @@ -104,28 +104,28 @@ int proc_dosoftlockup_thresh(struct ctl_table *table, int write, | |||
| 104 | void softlockup_tick(void) | 104 | void softlockup_tick(void) |
| 105 | { | 105 | { |
| 106 | int this_cpu = smp_processor_id(); | 106 | int this_cpu = smp_processor_id(); |
| 107 | unsigned long touch_timestamp = per_cpu(touch_timestamp, this_cpu); | 107 | unsigned long touch_ts = per_cpu(softlockup_touch_ts, this_cpu); |
| 108 | unsigned long print_timestamp; | 108 | unsigned long print_ts; |
| 109 | struct pt_regs *regs = get_irq_regs(); | 109 | struct pt_regs *regs = get_irq_regs(); |
| 110 | unsigned long now; | 110 | unsigned long now; |
| 111 | 111 | ||
| 112 | /* Is detection switched off? */ | 112 | /* Is detection switched off? */ |
| 113 | if (!per_cpu(watchdog_task, this_cpu) || softlockup_thresh <= 0) { | 113 | if (!per_cpu(softlockup_watchdog, this_cpu) || softlockup_thresh <= 0) { |
| 114 | /* Be sure we don't false trigger if switched back on */ | 114 | /* Be sure we don't false trigger if switched back on */ |
| 115 | if (touch_timestamp) | 115 | if (touch_ts) |
| 116 | per_cpu(touch_timestamp, this_cpu) = 0; | 116 | per_cpu(softlockup_touch_ts, this_cpu) = 0; |
| 117 | return; | 117 | return; |
| 118 | } | 118 | } |
| 119 | 119 | ||
| 120 | if (touch_timestamp == 0) { | 120 | if (touch_ts == 0) { |
| 121 | __touch_softlockup_watchdog(); | 121 | __touch_softlockup_watchdog(); |
| 122 | return; | 122 | return; |
| 123 | } | 123 | } |
| 124 | 124 | ||
| 125 | print_timestamp = per_cpu(print_timestamp, this_cpu); | 125 | print_ts = per_cpu(softlockup_print_ts, this_cpu); |
| 126 | 126 | ||
| 127 | /* report at most once a second */ | 127 | /* report at most once a second */ |
| 128 | if (print_timestamp == touch_timestamp || did_panic) | 128 | if (print_ts == touch_ts || did_panic) |
| 129 | return; | 129 | return; |
| 130 | 130 | ||
| 131 | /* do not print during early bootup: */ | 131 | /* do not print during early bootup: */ |
| @@ -140,18 +140,18 @@ void softlockup_tick(void) | |||
| 140 | * Wake up the high-prio watchdog task twice per | 140 | * Wake up the high-prio watchdog task twice per |
| 141 | * threshold timespan. | 141 | * threshold timespan. |
| 142 | */ | 142 | */ |
| 143 | if (now > touch_timestamp + softlockup_thresh/2) | 143 | if (now > touch_ts + softlockup_thresh/2) |
| 144 | wake_up_process(per_cpu(watchdog_task, this_cpu)); | 144 | wake_up_process(per_cpu(softlockup_watchdog, this_cpu)); |
| 145 | 145 | ||
| 146 | /* Warn about unreasonable delays: */ | 146 | /* Warn about unreasonable delays: */ |
| 147 | if (now <= (touch_timestamp + softlockup_thresh)) | 147 | if (now <= (touch_ts + softlockup_thresh)) |
| 148 | return; | 148 | return; |
| 149 | 149 | ||
| 150 | per_cpu(print_timestamp, this_cpu) = touch_timestamp; | 150 | per_cpu(softlockup_print_ts, this_cpu) = touch_ts; |
| 151 | 151 | ||
| 152 | spin_lock(&print_lock); | 152 | spin_lock(&print_lock); |
| 153 | printk(KERN_ERR "BUG: soft lockup - CPU#%d stuck for %lus! [%s:%d]\n", | 153 | printk(KERN_ERR "BUG: soft lockup - CPU#%d stuck for %lus! [%s:%d]\n", |
| 154 | this_cpu, now - touch_timestamp, | 154 | this_cpu, now - touch_ts, |
| 155 | current->comm, task_pid_nr(current)); | 155 | current->comm, task_pid_nr(current)); |
| 156 | print_modules(); | 156 | print_modules(); |
| 157 | print_irqtrace_events(current); | 157 | print_irqtrace_events(current); |
| @@ -209,32 +209,32 @@ cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) | |||
| 209 | switch (action) { | 209 | switch (action) { |
| 210 | case CPU_UP_PREPARE: | 210 | case CPU_UP_PREPARE: |
| 211 | case CPU_UP_PREPARE_FROZEN: | 211 | case CPU_UP_PREPARE_FROZEN: |
| 212 | BUG_ON(per_cpu(watchdog_task, hotcpu)); | 212 | BUG_ON(per_cpu(softlockup_watchdog, hotcpu)); |
| 213 | p = kthread_create(watchdog, hcpu, "watchdog/%d", hotcpu); | 213 | p = kthread_create(watchdog, hcpu, "watchdog/%d", hotcpu); |
| 214 | if (IS_ERR(p)) { | 214 | if (IS_ERR(p)) { |
| 215 | printk(KERN_ERR "watchdog for %i failed\n", hotcpu); | 215 | printk(KERN_ERR "watchdog for %i failed\n", hotcpu); |
| 216 | return NOTIFY_BAD; | 216 | return NOTIFY_BAD; |
| 217 | } | 217 | } |
| 218 | per_cpu(touch_timestamp, hotcpu) = 0; | 218 | per_cpu(softlockup_touch_ts, hotcpu) = 0; |
| 219 | per_cpu(watchdog_task, hotcpu) = p; | 219 | per_cpu(softlockup_watchdog, hotcpu) = p; |
| 220 | kthread_bind(p, hotcpu); | 220 | kthread_bind(p, hotcpu); |
| 221 | break; | 221 | break; |
| 222 | case CPU_ONLINE: | 222 | case CPU_ONLINE: |
| 223 | case CPU_ONLINE_FROZEN: | 223 | case CPU_ONLINE_FROZEN: |
| 224 | wake_up_process(per_cpu(watchdog_task, hotcpu)); | 224 | wake_up_process(per_cpu(softlockup_watchdog, hotcpu)); |
| 225 | break; | 225 | break; |
| 226 | #ifdef CONFIG_HOTPLUG_CPU | 226 | #ifdef CONFIG_HOTPLUG_CPU |
| 227 | case CPU_UP_CANCELED: | 227 | case CPU_UP_CANCELED: |
| 228 | case CPU_UP_CANCELED_FROZEN: | 228 | case CPU_UP_CANCELED_FROZEN: |
| 229 | if (!per_cpu(watchdog_task, hotcpu)) | 229 | if (!per_cpu(softlockup_watchdog, hotcpu)) |
| 230 | break; | 230 | break; |
| 231 | /* Unbind so it can run. Fall thru. */ | 231 | /* Unbind so it can run. Fall thru. */ |
| 232 | kthread_bind(per_cpu(watchdog_task, hotcpu), | 232 | kthread_bind(per_cpu(softlockup_watchdog, hotcpu), |
| 233 | cpumask_any(cpu_online_mask)); | 233 | cpumask_any(cpu_online_mask)); |
| 234 | case CPU_DEAD: | 234 | case CPU_DEAD: |
| 235 | case CPU_DEAD_FROZEN: | 235 | case CPU_DEAD_FROZEN: |
| 236 | p = per_cpu(watchdog_task, hotcpu); | 236 | p = per_cpu(softlockup_watchdog, hotcpu); |
| 237 | per_cpu(watchdog_task, hotcpu) = NULL; | 237 | per_cpu(softlockup_watchdog, hotcpu) = NULL; |
| 238 | kthread_stop(p); | 238 | kthread_stop(p); |
| 239 | break; | 239 | break; |
| 240 | #endif /* CONFIG_HOTPLUG_CPU */ | 240 | #endif /* CONFIG_HOTPLUG_CPU */ |
diff --git a/kernel/spinlock.c b/kernel/spinlock.c index 41e042219ff6..be6517fb9c14 100644 --- a/kernel/spinlock.c +++ b/kernel/spinlock.c | |||
| @@ -32,6 +32,8 @@ | |||
| 32 | * include/linux/spinlock_api_smp.h | 32 | * include/linux/spinlock_api_smp.h |
| 33 | */ | 33 | */ |
| 34 | #else | 34 | #else |
| 35 | #define raw_read_can_lock(l) read_can_lock(l) | ||
| 36 | #define raw_write_can_lock(l) write_can_lock(l) | ||
| 35 | /* | 37 | /* |
| 36 | * We build the __lock_function inlines here. They are too large for | 38 | * We build the __lock_function inlines here. They are too large for |
| 37 | * inlining all over the place, but here is only one user per function | 39 | * inlining all over the place, but here is only one user per function |
| @@ -42,49 +44,49 @@ | |||
| 42 | * towards that other CPU that it should break the lock ASAP. | 44 | * towards that other CPU that it should break the lock ASAP. |
| 43 | */ | 45 | */ |
| 44 | #define BUILD_LOCK_OPS(op, locktype) \ | 46 | #define BUILD_LOCK_OPS(op, locktype) \ |
| 45 | void __lockfunc __##op##_lock(locktype##_t *lock) \ | 47 | void __lockfunc __raw_##op##_lock(locktype##_t *lock) \ |
| 46 | { \ | 48 | { \ |
| 47 | for (;;) { \ | 49 | for (;;) { \ |
| 48 | preempt_disable(); \ | 50 | preempt_disable(); \ |
| 49 | if (likely(_raw_##op##_trylock(lock))) \ | 51 | if (likely(do_raw_##op##_trylock(lock))) \ |
| 50 | break; \ | 52 | break; \ |
| 51 | preempt_enable(); \ | 53 | preempt_enable(); \ |
| 52 | \ | 54 | \ |
| 53 | if (!(lock)->break_lock) \ | 55 | if (!(lock)->break_lock) \ |
| 54 | (lock)->break_lock = 1; \ | 56 | (lock)->break_lock = 1; \ |
| 55 | while (!op##_can_lock(lock) && (lock)->break_lock) \ | 57 | while (!raw_##op##_can_lock(lock) && (lock)->break_lock)\ |
| 56 | _raw_##op##_relax(&lock->raw_lock); \ | 58 | arch_##op##_relax(&lock->raw_lock); \ |
| 57 | } \ | 59 | } \ |
| 58 | (lock)->break_lock = 0; \ | 60 | (lock)->break_lock = 0; \ |
| 59 | } \ | 61 | } \ |
| 60 | \ | 62 | \ |
| 61 | unsigned long __lockfunc __##op##_lock_irqsave(locktype##_t *lock) \ | 63 | unsigned long __lockfunc __raw_##op##_lock_irqsave(locktype##_t *lock) \ |
| 62 | { \ | 64 | { \ |
| 63 | unsigned long flags; \ | 65 | unsigned long flags; \ |
| 64 | \ | 66 | \ |
| 65 | for (;;) { \ | 67 | for (;;) { \ |
| 66 | preempt_disable(); \ | 68 | preempt_disable(); \ |
| 67 | local_irq_save(flags); \ | 69 | local_irq_save(flags); \ |
| 68 | if (likely(_raw_##op##_trylock(lock))) \ | 70 | if (likely(do_raw_##op##_trylock(lock))) \ |
| 69 | break; \ | 71 | break; \ |
| 70 | local_irq_restore(flags); \ | 72 | local_irq_restore(flags); \ |
| 71 | preempt_enable(); \ | 73 | preempt_enable(); \ |
| 72 | \ | 74 | \ |
| 73 | if (!(lock)->break_lock) \ | 75 | if (!(lock)->break_lock) \ |
| 74 | (lock)->break_lock = 1; \ | 76 | (lock)->break_lock = 1; \ |
| 75 | while (!op##_can_lock(lock) && (lock)->break_lock) \ | 77 | while (!raw_##op##_can_lock(lock) && (lock)->break_lock)\ |
| 76 | _raw_##op##_relax(&lock->raw_lock); \ | 78 | arch_##op##_relax(&lock->raw_lock); \ |
| 77 | } \ | 79 | } \ |
| 78 | (lock)->break_lock = 0; \ | 80 | (lock)->break_lock = 0; \ |
| 79 | return flags; \ | 81 | return flags; \ |
| 80 | } \ | 82 | } \ |
| 81 | \ | 83 | \ |
| 82 | void __lockfunc __##op##_lock_irq(locktype##_t *lock) \ | 84 | void __lockfunc __raw_##op##_lock_irq(locktype##_t *lock) \ |
| 83 | { \ | 85 | { \ |
| 84 | _##op##_lock_irqsave(lock); \ | 86 | _raw_##op##_lock_irqsave(lock); \ |
| 85 | } \ | 87 | } \ |
| 86 | \ | 88 | \ |
| 87 | void __lockfunc __##op##_lock_bh(locktype##_t *lock) \ | 89 | void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \ |
| 88 | { \ | 90 | { \ |
| 89 | unsigned long flags; \ | 91 | unsigned long flags; \ |
| 90 | \ | 92 | \ |
| @@ -93,7 +95,7 @@ void __lockfunc __##op##_lock_bh(locktype##_t *lock) \ | |||
| 93 | /* irq-disabling. We use the generic preemption-aware */ \ | 95 | /* irq-disabling. We use the generic preemption-aware */ \ |
| 94 | /* function: */ \ | 96 | /* function: */ \ |
| 95 | /**/ \ | 97 | /**/ \ |
| 96 | flags = _##op##_lock_irqsave(lock); \ | 98 | flags = _raw_##op##_lock_irqsave(lock); \ |
| 97 | local_bh_disable(); \ | 99 | local_bh_disable(); \ |
| 98 | local_irq_restore(flags); \ | 100 | local_irq_restore(flags); \ |
| 99 | } \ | 101 | } \ |
| @@ -107,269 +109,269 @@ void __lockfunc __##op##_lock_bh(locktype##_t *lock) \ | |||
| 107 | * __[spin|read|write]_lock_irqsave() | 109 | * __[spin|read|write]_lock_irqsave() |
| 108 | * __[spin|read|write]_lock_bh() | 110 | * __[spin|read|write]_lock_bh() |
| 109 | */ | 111 | */ |
| 110 | BUILD_LOCK_OPS(spin, spinlock); | 112 | BUILD_LOCK_OPS(spin, raw_spinlock); |
| 111 | BUILD_LOCK_OPS(read, rwlock); | 113 | BUILD_LOCK_OPS(read, rwlock); |
| 112 | BUILD_LOCK_OPS(write, rwlock); | 114 | BUILD_LOCK_OPS(write, rwlock); |
| 113 | 115 | ||
| 114 | #endif | 116 | #endif |
| 115 | 117 | ||
| 116 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 118 | #ifndef CONFIG_INLINE_SPIN_TRYLOCK |
| 117 | 119 | int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock) | |
| 118 | void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass) | ||
| 119 | { | 120 | { |
| 120 | preempt_disable(); | 121 | return __raw_spin_trylock(lock); |
| 121 | spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); | ||
| 122 | LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock); | ||
| 123 | } | 122 | } |
| 124 | EXPORT_SYMBOL(_spin_lock_nested); | 123 | EXPORT_SYMBOL(_raw_spin_trylock); |
| 124 | #endif | ||
| 125 | 125 | ||
| 126 | unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, | 126 | #ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH |
| 127 | int subclass) | 127 | int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) |
| 128 | { | 128 | { |
| 129 | unsigned long flags; | 129 | return __raw_spin_trylock_bh(lock); |
| 130 | |||
| 131 | local_irq_save(flags); | ||
| 132 | preempt_disable(); | ||
| 133 | spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); | ||
| 134 | LOCK_CONTENDED_FLAGS(lock, _raw_spin_trylock, _raw_spin_lock, | ||
| 135 | _raw_spin_lock_flags, &flags); | ||
| 136 | return flags; | ||
| 137 | } | 130 | } |
| 138 | EXPORT_SYMBOL(_spin_lock_irqsave_nested); | 131 | EXPORT_SYMBOL(_raw_spin_trylock_bh); |
| 132 | #endif | ||
| 139 | 133 | ||
| 140 | void __lockfunc _spin_lock_nest_lock(spinlock_t *lock, | 134 | #ifndef CONFIG_INLINE_SPIN_LOCK |
| 141 | struct lockdep_map *nest_lock) | 135 | void __lockfunc _raw_spin_lock(raw_spinlock_t *lock) |
| 142 | { | 136 | { |
| 143 | preempt_disable(); | 137 | __raw_spin_lock(lock); |
| 144 | spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_); | ||
| 145 | LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock); | ||
| 146 | } | 138 | } |
| 147 | EXPORT_SYMBOL(_spin_lock_nest_lock); | 139 | EXPORT_SYMBOL(_raw_spin_lock); |
| 148 | |||
| 149 | #endif | 140 | #endif |
| 150 | 141 | ||
| 151 | #ifndef CONFIG_INLINE_SPIN_TRYLOCK | 142 | #ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE |
| 152 | int __lockfunc _spin_trylock(spinlock_t *lock) | 143 | unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) |
| 153 | { | 144 | { |
| 154 | return __spin_trylock(lock); | 145 | return __raw_spin_lock_irqsave(lock); |
| 155 | } | 146 | } |
| 156 | EXPORT_SYMBOL(_spin_trylock); | 147 | EXPORT_SYMBOL(_raw_spin_lock_irqsave); |
| 157 | #endif | 148 | #endif |
| 158 | 149 | ||
| 159 | #ifndef CONFIG_INLINE_READ_TRYLOCK | 150 | #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ |
| 160 | int __lockfunc _read_trylock(rwlock_t *lock) | 151 | void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) |
| 161 | { | 152 | { |
| 162 | return __read_trylock(lock); | 153 | __raw_spin_lock_irq(lock); |
| 163 | } | 154 | } |
| 164 | EXPORT_SYMBOL(_read_trylock); | 155 | EXPORT_SYMBOL(_raw_spin_lock_irq); |
| 165 | #endif | 156 | #endif |
| 166 | 157 | ||
| 167 | #ifndef CONFIG_INLINE_WRITE_TRYLOCK | 158 | #ifndef CONFIG_INLINE_SPIN_LOCK_BH |
| 168 | int __lockfunc _write_trylock(rwlock_t *lock) | 159 | void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) |
| 169 | { | 160 | { |
| 170 | return __write_trylock(lock); | 161 | __raw_spin_lock_bh(lock); |
| 171 | } | 162 | } |
| 172 | EXPORT_SYMBOL(_write_trylock); | 163 | EXPORT_SYMBOL(_raw_spin_lock_bh); |
| 173 | #endif | 164 | #endif |
| 174 | 165 | ||
| 175 | #ifndef CONFIG_INLINE_READ_LOCK | 166 | #ifndef CONFIG_INLINE_SPIN_UNLOCK |
| 176 | void __lockfunc _read_lock(rwlock_t *lock) | 167 | void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) |
| 177 | { | 168 | { |
| 178 | __read_lock(lock); | 169 | __raw_spin_unlock(lock); |
| 179 | } | 170 | } |
| 180 | EXPORT_SYMBOL(_read_lock); | 171 | EXPORT_SYMBOL(_raw_spin_unlock); |
| 181 | #endif | 172 | #endif |
| 182 | 173 | ||
| 183 | #ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE | 174 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE |
| 184 | unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock) | 175 | void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) |
| 185 | { | 176 | { |
| 186 | return __spin_lock_irqsave(lock); | 177 | __raw_spin_unlock_irqrestore(lock, flags); |
| 187 | } | 178 | } |
| 188 | EXPORT_SYMBOL(_spin_lock_irqsave); | 179 | EXPORT_SYMBOL(_raw_spin_unlock_irqrestore); |
| 189 | #endif | 180 | #endif |
| 190 | 181 | ||
| 191 | #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ | 182 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ |
| 192 | void __lockfunc _spin_lock_irq(spinlock_t *lock) | 183 | void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) |
| 193 | { | 184 | { |
| 194 | __spin_lock_irq(lock); | 185 | __raw_spin_unlock_irq(lock); |
| 195 | } | 186 | } |
| 196 | EXPORT_SYMBOL(_spin_lock_irq); | 187 | EXPORT_SYMBOL(_raw_spin_unlock_irq); |
| 197 | #endif | 188 | #endif |
| 198 | 189 | ||
| 199 | #ifndef CONFIG_INLINE_SPIN_LOCK_BH | 190 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH |
| 200 | void __lockfunc _spin_lock_bh(spinlock_t *lock) | 191 | void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) |
| 201 | { | 192 | { |
| 202 | __spin_lock_bh(lock); | 193 | __raw_spin_unlock_bh(lock); |
| 203 | } | 194 | } |
| 204 | EXPORT_SYMBOL(_spin_lock_bh); | 195 | EXPORT_SYMBOL(_raw_spin_unlock_bh); |
| 205 | #endif | 196 | #endif |
| 206 | 197 | ||
| 207 | #ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE | 198 | #ifndef CONFIG_INLINE_READ_TRYLOCK |
| 208 | unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock) | 199 | int __lockfunc _raw_read_trylock(rwlock_t *lock) |
| 209 | { | 200 | { |
| 210 | return __read_lock_irqsave(lock); | 201 | return __raw_read_trylock(lock); |
| 211 | } | 202 | } |
| 212 | EXPORT_SYMBOL(_read_lock_irqsave); | 203 | EXPORT_SYMBOL(_raw_read_trylock); |
| 213 | #endif | 204 | #endif |
| 214 | 205 | ||
| 215 | #ifndef CONFIG_INLINE_READ_LOCK_IRQ | 206 | #ifndef CONFIG_INLINE_READ_LOCK |
| 216 | void __lockfunc _read_lock_irq(rwlock_t *lock) | 207 | void __lockfunc _raw_read_lock(rwlock_t *lock) |
| 217 | { | 208 | { |
| 218 | __read_lock_irq(lock); | 209 | __raw_read_lock(lock); |
| 219 | } | 210 | } |
| 220 | EXPORT_SYMBOL(_read_lock_irq); | 211 | EXPORT_SYMBOL(_raw_read_lock); |
| 221 | #endif | 212 | #endif |
| 222 | 213 | ||
| 223 | #ifndef CONFIG_INLINE_READ_LOCK_BH | 214 | #ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE |
| 224 | void __lockfunc _read_lock_bh(rwlock_t *lock) | 215 | unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock) |
| 225 | { | 216 | { |
| 226 | __read_lock_bh(lock); | 217 | return __raw_read_lock_irqsave(lock); |
| 227 | } | 218 | } |
| 228 | EXPORT_SYMBOL(_read_lock_bh); | 219 | EXPORT_SYMBOL(_raw_read_lock_irqsave); |
| 229 | #endif | 220 | #endif |
| 230 | 221 | ||
| 231 | #ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE | 222 | #ifndef CONFIG_INLINE_READ_LOCK_IRQ |
| 232 | unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock) | 223 | void __lockfunc _raw_read_lock_irq(rwlock_t *lock) |
| 233 | { | 224 | { |
| 234 | return __write_lock_irqsave(lock); | 225 | __raw_read_lock_irq(lock); |
| 235 | } | 226 | } |
| 236 | EXPORT_SYMBOL(_write_lock_irqsave); | 227 | EXPORT_SYMBOL(_raw_read_lock_irq); |
| 237 | #endif | 228 | #endif |
| 238 | 229 | ||
| 239 | #ifndef CONFIG_INLINE_WRITE_LOCK_IRQ | 230 | #ifndef CONFIG_INLINE_READ_LOCK_BH |
| 240 | void __lockfunc _write_lock_irq(rwlock_t *lock) | 231 | void __lockfunc _raw_read_lock_bh(rwlock_t *lock) |
| 241 | { | 232 | { |
| 242 | __write_lock_irq(lock); | 233 | __raw_read_lock_bh(lock); |
| 243 | } | 234 | } |
| 244 | EXPORT_SYMBOL(_write_lock_irq); | 235 | EXPORT_SYMBOL(_raw_read_lock_bh); |
| 245 | #endif | 236 | #endif |
| 246 | 237 | ||
| 247 | #ifndef CONFIG_INLINE_WRITE_LOCK_BH | 238 | #ifndef CONFIG_INLINE_READ_UNLOCK |
| 248 | void __lockfunc _write_lock_bh(rwlock_t *lock) | 239 | void __lockfunc _raw_read_unlock(rwlock_t *lock) |
| 249 | { | 240 | { |
| 250 | __write_lock_bh(lock); | 241 | __raw_read_unlock(lock); |
| 251 | } | 242 | } |
| 252 | EXPORT_SYMBOL(_write_lock_bh); | 243 | EXPORT_SYMBOL(_raw_read_unlock); |
| 253 | #endif | 244 | #endif |
| 254 | 245 | ||
| 255 | #ifndef CONFIG_INLINE_SPIN_LOCK | 246 | #ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE |
| 256 | void __lockfunc _spin_lock(spinlock_t *lock) | 247 | void __lockfunc _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) |
| 257 | { | 248 | { |
| 258 | __spin_lock(lock); | 249 | __raw_read_unlock_irqrestore(lock, flags); |
| 259 | } | 250 | } |
| 260 | EXPORT_SYMBOL(_spin_lock); | 251 | EXPORT_SYMBOL(_raw_read_unlock_irqrestore); |
| 261 | #endif | 252 | #endif |
| 262 | 253 | ||
| 263 | #ifndef CONFIG_INLINE_WRITE_LOCK | 254 | #ifndef CONFIG_INLINE_READ_UNLOCK_IRQ |
| 264 | void __lockfunc _write_lock(rwlock_t *lock) | 255 | void __lockfunc _raw_read_unlock_irq(rwlock_t *lock) |
| 265 | { | 256 | { |
| 266 | __write_lock(lock); | 257 | __raw_read_unlock_irq(lock); |
| 267 | } | 258 | } |
| 268 | EXPORT_SYMBOL(_write_lock); | 259 | EXPORT_SYMBOL(_raw_read_unlock_irq); |
| 269 | #endif | 260 | #endif |
| 270 | 261 | ||
| 271 | #ifndef CONFIG_INLINE_SPIN_UNLOCK | 262 | #ifndef CONFIG_INLINE_READ_UNLOCK_BH |
| 272 | void __lockfunc _spin_unlock(spinlock_t *lock) | 263 | void __lockfunc _raw_read_unlock_bh(rwlock_t *lock) |
| 273 | { | 264 | { |
| 274 | __spin_unlock(lock); | 265 | __raw_read_unlock_bh(lock); |
| 275 | } | 266 | } |
| 276 | EXPORT_SYMBOL(_spin_unlock); | 267 | EXPORT_SYMBOL(_raw_read_unlock_bh); |
| 277 | #endif | 268 | #endif |
| 278 | 269 | ||
| 279 | #ifndef CONFIG_INLINE_WRITE_UNLOCK | 270 | #ifndef CONFIG_INLINE_WRITE_TRYLOCK |
| 280 | void __lockfunc _write_unlock(rwlock_t *lock) | 271 | int __lockfunc _raw_write_trylock(rwlock_t *lock) |
| 281 | { | 272 | { |
| 282 | __write_unlock(lock); | 273 | return __raw_write_trylock(lock); |
| 283 | } | 274 | } |
| 284 | EXPORT_SYMBOL(_write_unlock); | 275 | EXPORT_SYMBOL(_raw_write_trylock); |
| 285 | #endif | 276 | #endif |
| 286 | 277 | ||
| 287 | #ifndef CONFIG_INLINE_READ_UNLOCK | 278 | #ifndef CONFIG_INLINE_WRITE_LOCK |
| 288 | void __lockfunc _read_unlock(rwlock_t *lock) | 279 | void __lockfunc _raw_write_lock(rwlock_t *lock) |
| 289 | { | 280 | { |
| 290 | __read_unlock(lock); | 281 | __raw_write_lock(lock); |
| 291 | } | 282 | } |
| 292 | EXPORT_SYMBOL(_read_unlock); | 283 | EXPORT_SYMBOL(_raw_write_lock); |
| 293 | #endif | 284 | #endif |
| 294 | 285 | ||
| 295 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE | 286 | #ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE |
| 296 | void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) | 287 | unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock) |
| 297 | { | 288 | { |
| 298 | __spin_unlock_irqrestore(lock, flags); | 289 | return __raw_write_lock_irqsave(lock); |
| 299 | } | 290 | } |
| 300 | EXPORT_SYMBOL(_spin_unlock_irqrestore); | 291 | EXPORT_SYMBOL(_raw_write_lock_irqsave); |
| 301 | #endif | 292 | #endif |
| 302 | 293 | ||
| 303 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ | 294 | #ifndef CONFIG_INLINE_WRITE_LOCK_IRQ |
| 304 | void __lockfunc _spin_unlock_irq(spinlock_t *lock) | 295 | void __lockfunc _raw_write_lock_irq(rwlock_t *lock) |
| 305 | { | 296 | { |
| 306 | __spin_unlock_irq(lock); | 297 | __raw_write_lock_irq(lock); |
| 307 | } | 298 | } |
| 308 | EXPORT_SYMBOL(_spin_unlock_irq); | 299 | EXPORT_SYMBOL(_raw_write_lock_irq); |
| 309 | #endif | 300 | #endif |
| 310 | 301 | ||
| 311 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH | 302 | #ifndef CONFIG_INLINE_WRITE_LOCK_BH |
| 312 | void __lockfunc _spin_unlock_bh(spinlock_t *lock) | 303 | void __lockfunc _raw_write_lock_bh(rwlock_t *lock) |
| 313 | { | 304 | { |
| 314 | __spin_unlock_bh(lock); | 305 | __raw_write_lock_bh(lock); |
| 315 | } | 306 | } |
| 316 | EXPORT_SYMBOL(_spin_unlock_bh); | 307 | EXPORT_SYMBOL(_raw_write_lock_bh); |
| 317 | #endif | 308 | #endif |
| 318 | 309 | ||
| 319 | #ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE | 310 | #ifndef CONFIG_INLINE_WRITE_UNLOCK |
| 320 | void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) | 311 | void __lockfunc _raw_write_unlock(rwlock_t *lock) |
| 321 | { | 312 | { |
| 322 | __read_unlock_irqrestore(lock, flags); | 313 | __raw_write_unlock(lock); |
| 323 | } | 314 | } |
| 324 | EXPORT_SYMBOL(_read_unlock_irqrestore); | 315 | EXPORT_SYMBOL(_raw_write_unlock); |
| 325 | #endif | 316 | #endif |
| 326 | 317 | ||
| 327 | #ifndef CONFIG_INLINE_READ_UNLOCK_IRQ | 318 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE |
| 328 | void __lockfunc _read_unlock_irq(rwlock_t *lock) | 319 | void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) |
| 329 | { | 320 | { |
| 330 | __read_unlock_irq(lock); | 321 | __raw_write_unlock_irqrestore(lock, flags); |
| 331 | } | 322 | } |
| 332 | EXPORT_SYMBOL(_read_unlock_irq); | 323 | EXPORT_SYMBOL(_raw_write_unlock_irqrestore); |
| 333 | #endif | 324 | #endif |
| 334 | 325 | ||
| 335 | #ifndef CONFIG_INLINE_READ_UNLOCK_BH | 326 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ |
| 336 | void __lockfunc _read_unlock_bh(rwlock_t *lock) | 327 | void __lockfunc _raw_write_unlock_irq(rwlock_t *lock) |
| 337 | { | 328 | { |
| 338 | __read_unlock_bh(lock); | 329 | __raw_write_unlock_irq(lock); |
| 339 | } | 330 | } |
| 340 | EXPORT_SYMBOL(_read_unlock_bh); | 331 | EXPORT_SYMBOL(_raw_write_unlock_irq); |
| 341 | #endif | 332 | #endif |
| 342 | 333 | ||
| 343 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE | 334 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_BH |
| 344 | void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) | 335 | void __lockfunc _raw_write_unlock_bh(rwlock_t *lock) |
| 345 | { | 336 | { |
| 346 | __write_unlock_irqrestore(lock, flags); | 337 | __raw_write_unlock_bh(lock); |
| 347 | } | 338 | } |
| 348 | EXPORT_SYMBOL(_write_unlock_irqrestore); | 339 | EXPORT_SYMBOL(_raw_write_unlock_bh); |
| 349 | #endif | 340 | #endif |
| 350 | 341 | ||
| 351 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ | 342 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
| 352 | void __lockfunc _write_unlock_irq(rwlock_t *lock) | 343 | |
| 344 | void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass) | ||
| 353 | { | 345 | { |
| 354 | __write_unlock_irq(lock); | 346 | preempt_disable(); |
| 347 | spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); | ||
| 348 | LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); | ||
| 355 | } | 349 | } |
| 356 | EXPORT_SYMBOL(_write_unlock_irq); | 350 | EXPORT_SYMBOL(_raw_spin_lock_nested); |
| 357 | #endif | ||
| 358 | 351 | ||
| 359 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_BH | 352 | unsigned long __lockfunc _raw_spin_lock_irqsave_nested(raw_spinlock_t *lock, |
| 360 | void __lockfunc _write_unlock_bh(rwlock_t *lock) | 353 | int subclass) |
| 361 | { | 354 | { |
| 362 | __write_unlock_bh(lock); | 355 | unsigned long flags; |
| 356 | |||
| 357 | local_irq_save(flags); | ||
| 358 | preempt_disable(); | ||
| 359 | spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); | ||
| 360 | LOCK_CONTENDED_FLAGS(lock, do_raw_spin_trylock, do_raw_spin_lock, | ||
| 361 | do_raw_spin_lock_flags, &flags); | ||
| 362 | return flags; | ||
| 363 | } | 363 | } |
| 364 | EXPORT_SYMBOL(_write_unlock_bh); | 364 | EXPORT_SYMBOL(_raw_spin_lock_irqsave_nested); |
| 365 | #endif | ||
| 366 | 365 | ||
| 367 | #ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH | 366 | void __lockfunc _raw_spin_lock_nest_lock(raw_spinlock_t *lock, |
| 368 | int __lockfunc _spin_trylock_bh(spinlock_t *lock) | 367 | struct lockdep_map *nest_lock) |
| 369 | { | 368 | { |
| 370 | return __spin_trylock_bh(lock); | 369 | preempt_disable(); |
| 370 | spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_); | ||
| 371 | LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); | ||
| 371 | } | 372 | } |
| 372 | EXPORT_SYMBOL(_spin_trylock_bh); | 373 | EXPORT_SYMBOL(_raw_spin_lock_nest_lock); |
| 374 | |||
| 373 | #endif | 375 | #endif |
| 374 | 376 | ||
| 375 | notrace int in_lock_functions(unsigned long addr) | 377 | notrace int in_lock_functions(unsigned long addr) |
diff --git a/kernel/sys.c b/kernel/sys.c index 585d6cd10040..20ccfb5da6af 100644 --- a/kernel/sys.c +++ b/kernel/sys.c | |||
| @@ -189,10 +189,10 @@ SYSCALL_DEFINE3(setpriority, int, which, int, who, int, niceval) | |||
| 189 | !(user = find_user(who))) | 189 | !(user = find_user(who))) |
| 190 | goto out_unlock; /* No processes for this user */ | 190 | goto out_unlock; /* No processes for this user */ |
| 191 | 191 | ||
| 192 | do_each_thread(g, p) | 192 | do_each_thread(g, p) { |
| 193 | if (__task_cred(p)->uid == who) | 193 | if (__task_cred(p)->uid == who) |
| 194 | error = set_one_prio(p, niceval, error); | 194 | error = set_one_prio(p, niceval, error); |
| 195 | while_each_thread(g, p); | 195 | } while_each_thread(g, p); |
| 196 | if (who != cred->uid) | 196 | if (who != cred->uid) |
| 197 | free_uid(user); /* For find_user() */ | 197 | free_uid(user); /* For find_user() */ |
| 198 | break; | 198 | break; |
| @@ -252,13 +252,13 @@ SYSCALL_DEFINE2(getpriority, int, which, int, who) | |||
| 252 | !(user = find_user(who))) | 252 | !(user = find_user(who))) |
| 253 | goto out_unlock; /* No processes for this user */ | 253 | goto out_unlock; /* No processes for this user */ |
| 254 | 254 | ||
| 255 | do_each_thread(g, p) | 255 | do_each_thread(g, p) { |
| 256 | if (__task_cred(p)->uid == who) { | 256 | if (__task_cred(p)->uid == who) { |
| 257 | niceval = 20 - task_nice(p); | 257 | niceval = 20 - task_nice(p); |
| 258 | if (niceval > retval) | 258 | if (niceval > retval) |
| 259 | retval = niceval; | 259 | retval = niceval; |
| 260 | } | 260 | } |
| 261 | while_each_thread(g, p); | 261 | } while_each_thread(g, p); |
| 262 | if (who != cred->uid) | 262 | if (who != cred->uid) |
| 263 | free_uid(user); /* for find_user() */ | 263 | free_uid(user); /* for find_user() */ |
| 264 | break; | 264 | break; |
diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 554ac4894f0f..45e4bef0012a 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c | |||
| @@ -1051,7 +1051,7 @@ static struct ctl_table vm_table[] = { | |||
| 1051 | .extra2 = &one_hundred, | 1051 | .extra2 = &one_hundred, |
| 1052 | }, | 1052 | }, |
| 1053 | #ifdef CONFIG_HUGETLB_PAGE | 1053 | #ifdef CONFIG_HUGETLB_PAGE |
| 1054 | { | 1054 | { |
| 1055 | .procname = "nr_hugepages", | 1055 | .procname = "nr_hugepages", |
| 1056 | .data = NULL, | 1056 | .data = NULL, |
| 1057 | .maxlen = sizeof(unsigned long), | 1057 | .maxlen = sizeof(unsigned long), |
| @@ -1059,7 +1059,18 @@ static struct ctl_table vm_table[] = { | |||
| 1059 | .proc_handler = hugetlb_sysctl_handler, | 1059 | .proc_handler = hugetlb_sysctl_handler, |
| 1060 | .extra1 = (void *)&hugetlb_zero, | 1060 | .extra1 = (void *)&hugetlb_zero, |
| 1061 | .extra2 = (void *)&hugetlb_infinity, | 1061 | .extra2 = (void *)&hugetlb_infinity, |
| 1062 | }, | 1062 | }, |
| 1063 | #ifdef CONFIG_NUMA | ||
| 1064 | { | ||
| 1065 | .procname = "nr_hugepages_mempolicy", | ||
| 1066 | .data = NULL, | ||
| 1067 | .maxlen = sizeof(unsigned long), | ||
| 1068 | .mode = 0644, | ||
| 1069 | .proc_handler = &hugetlb_mempolicy_sysctl_handler, | ||
| 1070 | .extra1 = (void *)&hugetlb_zero, | ||
| 1071 | .extra2 = (void *)&hugetlb_infinity, | ||
| 1072 | }, | ||
| 1073 | #endif | ||
| 1063 | { | 1074 | { |
| 1064 | .procname = "hugetlb_shm_group", | 1075 | .procname = "hugetlb_shm_group", |
| 1065 | .data = &sysctl_hugetlb_shm_group, | 1076 | .data = &sysctl_hugetlb_shm_group, |
| @@ -1120,7 +1131,8 @@ static struct ctl_table vm_table[] = { | |||
| 1120 | .data = &sysctl_max_map_count, | 1131 | .data = &sysctl_max_map_count, |
| 1121 | .maxlen = sizeof(sysctl_max_map_count), | 1132 | .maxlen = sizeof(sysctl_max_map_count), |
| 1122 | .mode = 0644, | 1133 | .mode = 0644, |
| 1123 | .proc_handler = proc_dointvec | 1134 | .proc_handler = proc_dointvec, |
| 1135 | .extra1 = &zero, | ||
| 1124 | }, | 1136 | }, |
| 1125 | #else | 1137 | #else |
| 1126 | { | 1138 | { |
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index 20a8920029ee..3d5fc0fd1cca 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c | |||
| @@ -30,7 +30,7 @@ static LIST_HEAD(clockevents_released); | |||
| 30 | static RAW_NOTIFIER_HEAD(clockevents_chain); | 30 | static RAW_NOTIFIER_HEAD(clockevents_chain); |
| 31 | 31 | ||
| 32 | /* Protection for the above */ | 32 | /* Protection for the above */ |
| 33 | static DEFINE_SPINLOCK(clockevents_lock); | 33 | static DEFINE_RAW_SPINLOCK(clockevents_lock); |
| 34 | 34 | ||
| 35 | /** | 35 | /** |
| 36 | * clockevents_delta2ns - Convert a latch value (device ticks) to nanoseconds | 36 | * clockevents_delta2ns - Convert a latch value (device ticks) to nanoseconds |
| @@ -141,9 +141,9 @@ int clockevents_register_notifier(struct notifier_block *nb) | |||
| 141 | unsigned long flags; | 141 | unsigned long flags; |
| 142 | int ret; | 142 | int ret; |
| 143 | 143 | ||
| 144 | spin_lock_irqsave(&clockevents_lock, flags); | 144 | raw_spin_lock_irqsave(&clockevents_lock, flags); |
| 145 | ret = raw_notifier_chain_register(&clockevents_chain, nb); | 145 | ret = raw_notifier_chain_register(&clockevents_chain, nb); |
| 146 | spin_unlock_irqrestore(&clockevents_lock, flags); | 146 | raw_spin_unlock_irqrestore(&clockevents_lock, flags); |
| 147 | 147 | ||
| 148 | return ret; | 148 | return ret; |
| 149 | } | 149 | } |
| @@ -185,13 +185,13 @@ void clockevents_register_device(struct clock_event_device *dev) | |||
| 185 | BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED); | 185 | BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED); |
| 186 | BUG_ON(!dev->cpumask); | 186 | BUG_ON(!dev->cpumask); |
| 187 | 187 | ||
| 188 | spin_lock_irqsave(&clockevents_lock, flags); | 188 | raw_spin_lock_irqsave(&clockevents_lock, flags); |
| 189 | 189 | ||
| 190 | list_add(&dev->list, &clockevent_devices); | 190 | list_add(&dev->list, &clockevent_devices); |
| 191 | clockevents_do_notify(CLOCK_EVT_NOTIFY_ADD, dev); | 191 | clockevents_do_notify(CLOCK_EVT_NOTIFY_ADD, dev); |
| 192 | clockevents_notify_released(); | 192 | clockevents_notify_released(); |
| 193 | 193 | ||
| 194 | spin_unlock_irqrestore(&clockevents_lock, flags); | 194 | raw_spin_unlock_irqrestore(&clockevents_lock, flags); |
| 195 | } | 195 | } |
| 196 | EXPORT_SYMBOL_GPL(clockevents_register_device); | 196 | EXPORT_SYMBOL_GPL(clockevents_register_device); |
| 197 | 197 | ||
| @@ -241,7 +241,7 @@ void clockevents_notify(unsigned long reason, void *arg) | |||
| 241 | struct list_head *node, *tmp; | 241 | struct list_head *node, *tmp; |
| 242 | unsigned long flags; | 242 | unsigned long flags; |
| 243 | 243 | ||
| 244 | spin_lock_irqsave(&clockevents_lock, flags); | 244 | raw_spin_lock_irqsave(&clockevents_lock, flags); |
| 245 | clockevents_do_notify(reason, arg); | 245 | clockevents_do_notify(reason, arg); |
| 246 | 246 | ||
| 247 | switch (reason) { | 247 | switch (reason) { |
| @@ -256,7 +256,7 @@ void clockevents_notify(unsigned long reason, void *arg) | |||
| 256 | default: | 256 | default: |
| 257 | break; | 257 | break; |
| 258 | } | 258 | } |
| 259 | spin_unlock_irqrestore(&clockevents_lock, flags); | 259 | raw_spin_unlock_irqrestore(&clockevents_lock, flags); |
| 260 | } | 260 | } |
| 261 | EXPORT_SYMBOL_GPL(clockevents_notify); | 261 | EXPORT_SYMBOL_GPL(clockevents_notify); |
| 262 | #endif | 262 | #endif |
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index c2ec25087a35..b3bafd5fc66d 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c | |||
| @@ -31,7 +31,7 @@ static struct tick_device tick_broadcast_device; | |||
| 31 | /* FIXME: Use cpumask_var_t. */ | 31 | /* FIXME: Use cpumask_var_t. */ |
| 32 | static DECLARE_BITMAP(tick_broadcast_mask, NR_CPUS); | 32 | static DECLARE_BITMAP(tick_broadcast_mask, NR_CPUS); |
| 33 | static DECLARE_BITMAP(tmpmask, NR_CPUS); | 33 | static DECLARE_BITMAP(tmpmask, NR_CPUS); |
| 34 | static DEFINE_SPINLOCK(tick_broadcast_lock); | 34 | static DEFINE_RAW_SPINLOCK(tick_broadcast_lock); |
| 35 | static int tick_broadcast_force; | 35 | static int tick_broadcast_force; |
| 36 | 36 | ||
| 37 | #ifdef CONFIG_TICK_ONESHOT | 37 | #ifdef CONFIG_TICK_ONESHOT |
| @@ -96,7 +96,7 @@ int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) | |||
| 96 | unsigned long flags; | 96 | unsigned long flags; |
| 97 | int ret = 0; | 97 | int ret = 0; |
| 98 | 98 | ||
| 99 | spin_lock_irqsave(&tick_broadcast_lock, flags); | 99 | raw_spin_lock_irqsave(&tick_broadcast_lock, flags); |
| 100 | 100 | ||
| 101 | /* | 101 | /* |
| 102 | * Devices might be registered with both periodic and oneshot | 102 | * Devices might be registered with both periodic and oneshot |
| @@ -122,7 +122,7 @@ int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) | |||
| 122 | tick_broadcast_clear_oneshot(cpu); | 122 | tick_broadcast_clear_oneshot(cpu); |
| 123 | } | 123 | } |
| 124 | } | 124 | } |
| 125 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); | 125 | raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); |
| 126 | return ret; | 126 | return ret; |
| 127 | } | 127 | } |
| 128 | 128 | ||
| @@ -161,13 +161,13 @@ static void tick_do_broadcast(struct cpumask *mask) | |||
| 161 | */ | 161 | */ |
| 162 | static void tick_do_periodic_broadcast(void) | 162 | static void tick_do_periodic_broadcast(void) |
| 163 | { | 163 | { |
| 164 | spin_lock(&tick_broadcast_lock); | 164 | raw_spin_lock(&tick_broadcast_lock); |
| 165 | 165 | ||
| 166 | cpumask_and(to_cpumask(tmpmask), | 166 | cpumask_and(to_cpumask(tmpmask), |
| 167 | cpu_online_mask, tick_get_broadcast_mask()); | 167 | cpu_online_mask, tick_get_broadcast_mask()); |
| 168 | tick_do_broadcast(to_cpumask(tmpmask)); | 168 | tick_do_broadcast(to_cpumask(tmpmask)); |
| 169 | 169 | ||
| 170 | spin_unlock(&tick_broadcast_lock); | 170 | raw_spin_unlock(&tick_broadcast_lock); |
| 171 | } | 171 | } |
| 172 | 172 | ||
| 173 | /* | 173 | /* |
| @@ -212,7 +212,7 @@ static void tick_do_broadcast_on_off(unsigned long *reason) | |||
| 212 | unsigned long flags; | 212 | unsigned long flags; |
| 213 | int cpu, bc_stopped; | 213 | int cpu, bc_stopped; |
| 214 | 214 | ||
| 215 | spin_lock_irqsave(&tick_broadcast_lock, flags); | 215 | raw_spin_lock_irqsave(&tick_broadcast_lock, flags); |
| 216 | 216 | ||
| 217 | cpu = smp_processor_id(); | 217 | cpu = smp_processor_id(); |
| 218 | td = &per_cpu(tick_cpu_device, cpu); | 218 | td = &per_cpu(tick_cpu_device, cpu); |
| @@ -263,7 +263,7 @@ static void tick_do_broadcast_on_off(unsigned long *reason) | |||
| 263 | tick_broadcast_setup_oneshot(bc); | 263 | tick_broadcast_setup_oneshot(bc); |
| 264 | } | 264 | } |
| 265 | out: | 265 | out: |
| 266 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); | 266 | raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); |
| 267 | } | 267 | } |
| 268 | 268 | ||
| 269 | /* | 269 | /* |
| @@ -299,7 +299,7 @@ void tick_shutdown_broadcast(unsigned int *cpup) | |||
| 299 | unsigned long flags; | 299 | unsigned long flags; |
| 300 | unsigned int cpu = *cpup; | 300 | unsigned int cpu = *cpup; |
| 301 | 301 | ||
| 302 | spin_lock_irqsave(&tick_broadcast_lock, flags); | 302 | raw_spin_lock_irqsave(&tick_broadcast_lock, flags); |
| 303 | 303 | ||
| 304 | bc = tick_broadcast_device.evtdev; | 304 | bc = tick_broadcast_device.evtdev; |
| 305 | cpumask_clear_cpu(cpu, tick_get_broadcast_mask()); | 305 | cpumask_clear_cpu(cpu, tick_get_broadcast_mask()); |
| @@ -309,7 +309,7 @@ void tick_shutdown_broadcast(unsigned int *cpup) | |||
| 309 | clockevents_shutdown(bc); | 309 | clockevents_shutdown(bc); |
| 310 | } | 310 | } |
| 311 | 311 | ||
| 312 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); | 312 | raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); |
| 313 | } | 313 | } |
| 314 | 314 | ||
| 315 | void tick_suspend_broadcast(void) | 315 | void tick_suspend_broadcast(void) |
| @@ -317,13 +317,13 @@ void tick_suspend_broadcast(void) | |||
| 317 | struct clock_event_device *bc; | 317 | struct clock_event_device *bc; |
| 318 | unsigned long flags; | 318 | unsigned long flags; |
| 319 | 319 | ||
| 320 | spin_lock_irqsave(&tick_broadcast_lock, flags); | 320 | raw_spin_lock_irqsave(&tick_broadcast_lock, flags); |
| 321 | 321 | ||
| 322 | bc = tick_broadcast_device.evtdev; | 322 | bc = tick_broadcast_device.evtdev; |
| 323 | if (bc) | 323 | if (bc) |
| 324 | clockevents_shutdown(bc); | 324 | clockevents_shutdown(bc); |
| 325 | 325 | ||
| 326 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); | 326 | raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); |
| 327 | } | 327 | } |
| 328 | 328 | ||
| 329 | int tick_resume_broadcast(void) | 329 | int tick_resume_broadcast(void) |
| @@ -332,7 +332,7 @@ int tick_resume_broadcast(void) | |||
| 332 | unsigned long flags; | 332 | unsigned long flags; |
| 333 | int broadcast = 0; | 333 | int broadcast = 0; |
| 334 | 334 | ||
| 335 | spin_lock_irqsave(&tick_broadcast_lock, flags); | 335 | raw_spin_lock_irqsave(&tick_broadcast_lock, flags); |
| 336 | 336 | ||
| 337 | bc = tick_broadcast_device.evtdev; | 337 | bc = tick_broadcast_device.evtdev; |
| 338 | 338 | ||
| @@ -351,7 +351,7 @@ int tick_resume_broadcast(void) | |||
| 351 | break; | 351 | break; |
| 352 | } | 352 | } |
| 353 | } | 353 | } |
| 354 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); | 354 | raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); |
| 355 | 355 | ||
| 356 | return broadcast; | 356 | return broadcast; |
| 357 | } | 357 | } |
| @@ -405,7 +405,7 @@ static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) | |||
| 405 | ktime_t now, next_event; | 405 | ktime_t now, next_event; |
| 406 | int cpu; | 406 | int cpu; |
| 407 | 407 | ||
| 408 | spin_lock(&tick_broadcast_lock); | 408 | raw_spin_lock(&tick_broadcast_lock); |
| 409 | again: | 409 | again: |
| 410 | dev->next_event.tv64 = KTIME_MAX; | 410 | dev->next_event.tv64 = KTIME_MAX; |
| 411 | next_event.tv64 = KTIME_MAX; | 411 | next_event.tv64 = KTIME_MAX; |
| @@ -443,7 +443,7 @@ again: | |||
| 443 | if (tick_broadcast_set_event(next_event, 0)) | 443 | if (tick_broadcast_set_event(next_event, 0)) |
| 444 | goto again; | 444 | goto again; |
| 445 | } | 445 | } |
| 446 | spin_unlock(&tick_broadcast_lock); | 446 | raw_spin_unlock(&tick_broadcast_lock); |
| 447 | } | 447 | } |
| 448 | 448 | ||
| 449 | /* | 449 | /* |
| @@ -457,7 +457,7 @@ void tick_broadcast_oneshot_control(unsigned long reason) | |||
| 457 | unsigned long flags; | 457 | unsigned long flags; |
| 458 | int cpu; | 458 | int cpu; |
| 459 | 459 | ||
| 460 | spin_lock_irqsave(&tick_broadcast_lock, flags); | 460 | raw_spin_lock_irqsave(&tick_broadcast_lock, flags); |
| 461 | 461 | ||
| 462 | /* | 462 | /* |
| 463 | * Periodic mode does not care about the enter/exit of power | 463 | * Periodic mode does not care about the enter/exit of power |
| @@ -492,7 +492,7 @@ void tick_broadcast_oneshot_control(unsigned long reason) | |||
| 492 | } | 492 | } |
| 493 | 493 | ||
| 494 | out: | 494 | out: |
| 495 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); | 495 | raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); |
| 496 | } | 496 | } |
| 497 | 497 | ||
| 498 | /* | 498 | /* |
| @@ -563,13 +563,13 @@ void tick_broadcast_switch_to_oneshot(void) | |||
| 563 | struct clock_event_device *bc; | 563 | struct clock_event_device *bc; |
| 564 | unsigned long flags; | 564 | unsigned long flags; |
| 565 | 565 | ||
| 566 | spin_lock_irqsave(&tick_broadcast_lock, flags); | 566 | raw_spin_lock_irqsave(&tick_broadcast_lock, flags); |
| 567 | 567 | ||
| 568 | tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT; | 568 | tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT; |
| 569 | bc = tick_broadcast_device.evtdev; | 569 | bc = tick_broadcast_device.evtdev; |
| 570 | if (bc) | 570 | if (bc) |
| 571 | tick_broadcast_setup_oneshot(bc); | 571 | tick_broadcast_setup_oneshot(bc); |
| 572 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); | 572 | raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); |
| 573 | } | 573 | } |
| 574 | 574 | ||
| 575 | 575 | ||
| @@ -581,7 +581,7 @@ void tick_shutdown_broadcast_oneshot(unsigned int *cpup) | |||
| 581 | unsigned long flags; | 581 | unsigned long flags; |
| 582 | unsigned int cpu = *cpup; | 582 | unsigned int cpu = *cpup; |
| 583 | 583 | ||
| 584 | spin_lock_irqsave(&tick_broadcast_lock, flags); | 584 | raw_spin_lock_irqsave(&tick_broadcast_lock, flags); |
| 585 | 585 | ||
| 586 | /* | 586 | /* |
| 587 | * Clear the broadcast mask flag for the dead cpu, but do not | 587 | * Clear the broadcast mask flag for the dead cpu, but do not |
| @@ -589,7 +589,7 @@ void tick_shutdown_broadcast_oneshot(unsigned int *cpup) | |||
| 589 | */ | 589 | */ |
| 590 | cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask()); | 590 | cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask()); |
| 591 | 591 | ||
| 592 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); | 592 | raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); |
| 593 | } | 593 | } |
| 594 | 594 | ||
| 595 | /* | 595 | /* |
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c index 83c4417b6a3c..b6b898d2eeef 100644 --- a/kernel/time/tick-common.c +++ b/kernel/time/tick-common.c | |||
| @@ -34,7 +34,7 @@ DEFINE_PER_CPU(struct tick_device, tick_cpu_device); | |||
| 34 | ktime_t tick_next_period; | 34 | ktime_t tick_next_period; |
| 35 | ktime_t tick_period; | 35 | ktime_t tick_period; |
| 36 | int tick_do_timer_cpu __read_mostly = TICK_DO_TIMER_BOOT; | 36 | int tick_do_timer_cpu __read_mostly = TICK_DO_TIMER_BOOT; |
| 37 | DEFINE_SPINLOCK(tick_device_lock); | 37 | static DEFINE_RAW_SPINLOCK(tick_device_lock); |
| 38 | 38 | ||
| 39 | /* | 39 | /* |
| 40 | * Debugging: see timer_list.c | 40 | * Debugging: see timer_list.c |
| @@ -209,7 +209,7 @@ static int tick_check_new_device(struct clock_event_device *newdev) | |||
| 209 | int cpu, ret = NOTIFY_OK; | 209 | int cpu, ret = NOTIFY_OK; |
| 210 | unsigned long flags; | 210 | unsigned long flags; |
| 211 | 211 | ||
| 212 | spin_lock_irqsave(&tick_device_lock, flags); | 212 | raw_spin_lock_irqsave(&tick_device_lock, flags); |
| 213 | 213 | ||
| 214 | cpu = smp_processor_id(); | 214 | cpu = smp_processor_id(); |
| 215 | if (!cpumask_test_cpu(cpu, newdev->cpumask)) | 215 | if (!cpumask_test_cpu(cpu, newdev->cpumask)) |
| @@ -268,7 +268,7 @@ static int tick_check_new_device(struct clock_event_device *newdev) | |||
| 268 | if (newdev->features & CLOCK_EVT_FEAT_ONESHOT) | 268 | if (newdev->features & CLOCK_EVT_FEAT_ONESHOT) |
| 269 | tick_oneshot_notify(); | 269 | tick_oneshot_notify(); |
| 270 | 270 | ||
| 271 | spin_unlock_irqrestore(&tick_device_lock, flags); | 271 | raw_spin_unlock_irqrestore(&tick_device_lock, flags); |
| 272 | return NOTIFY_STOP; | 272 | return NOTIFY_STOP; |
| 273 | 273 | ||
| 274 | out_bc: | 274 | out_bc: |
| @@ -278,7 +278,7 @@ out_bc: | |||
| 278 | if (tick_check_broadcast_device(newdev)) | 278 | if (tick_check_broadcast_device(newdev)) |
| 279 | ret = NOTIFY_STOP; | 279 | ret = NOTIFY_STOP; |
| 280 | 280 | ||
| 281 | spin_unlock_irqrestore(&tick_device_lock, flags); | 281 | raw_spin_unlock_irqrestore(&tick_device_lock, flags); |
| 282 | 282 | ||
| 283 | return ret; | 283 | return ret; |
| 284 | } | 284 | } |
| @@ -311,7 +311,7 @@ static void tick_shutdown(unsigned int *cpup) | |||
| 311 | struct clock_event_device *dev = td->evtdev; | 311 | struct clock_event_device *dev = td->evtdev; |
| 312 | unsigned long flags; | 312 | unsigned long flags; |
| 313 | 313 | ||
| 314 | spin_lock_irqsave(&tick_device_lock, flags); | 314 | raw_spin_lock_irqsave(&tick_device_lock, flags); |
| 315 | td->mode = TICKDEV_MODE_PERIODIC; | 315 | td->mode = TICKDEV_MODE_PERIODIC; |
| 316 | if (dev) { | 316 | if (dev) { |
| 317 | /* | 317 | /* |
| @@ -322,7 +322,7 @@ static void tick_shutdown(unsigned int *cpup) | |||
| 322 | clockevents_exchange_device(dev, NULL); | 322 | clockevents_exchange_device(dev, NULL); |
| 323 | td->evtdev = NULL; | 323 | td->evtdev = NULL; |
| 324 | } | 324 | } |
| 325 | spin_unlock_irqrestore(&tick_device_lock, flags); | 325 | raw_spin_unlock_irqrestore(&tick_device_lock, flags); |
| 326 | } | 326 | } |
| 327 | 327 | ||
| 328 | static void tick_suspend(void) | 328 | static void tick_suspend(void) |
| @@ -330,9 +330,9 @@ static void tick_suspend(void) | |||
| 330 | struct tick_device *td = &__get_cpu_var(tick_cpu_device); | 330 | struct tick_device *td = &__get_cpu_var(tick_cpu_device); |
| 331 | unsigned long flags; | 331 | unsigned long flags; |
| 332 | 332 | ||
| 333 | spin_lock_irqsave(&tick_device_lock, flags); | 333 | raw_spin_lock_irqsave(&tick_device_lock, flags); |
| 334 | clockevents_shutdown(td->evtdev); | 334 | clockevents_shutdown(td->evtdev); |
| 335 | spin_unlock_irqrestore(&tick_device_lock, flags); | 335 | raw_spin_unlock_irqrestore(&tick_device_lock, flags); |
| 336 | } | 336 | } |
| 337 | 337 | ||
| 338 | static void tick_resume(void) | 338 | static void tick_resume(void) |
| @@ -341,7 +341,7 @@ static void tick_resume(void) | |||
| 341 | unsigned long flags; | 341 | unsigned long flags; |
| 342 | int broadcast = tick_resume_broadcast(); | 342 | int broadcast = tick_resume_broadcast(); |
| 343 | 343 | ||
| 344 | spin_lock_irqsave(&tick_device_lock, flags); | 344 | raw_spin_lock_irqsave(&tick_device_lock, flags); |
| 345 | clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_RESUME); | 345 | clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_RESUME); |
| 346 | 346 | ||
| 347 | if (!broadcast) { | 347 | if (!broadcast) { |
| @@ -350,7 +350,7 @@ static void tick_resume(void) | |||
| 350 | else | 350 | else |
| 351 | tick_resume_oneshot(); | 351 | tick_resume_oneshot(); |
| 352 | } | 352 | } |
| 353 | spin_unlock_irqrestore(&tick_device_lock, flags); | 353 | raw_spin_unlock_irqrestore(&tick_device_lock, flags); |
| 354 | } | 354 | } |
| 355 | 355 | ||
| 356 | /* | 356 | /* |
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index b1c05bf75ee0..290eefbc1f60 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h | |||
| @@ -6,7 +6,6 @@ | |||
| 6 | #define TICK_DO_TIMER_BOOT -2 | 6 | #define TICK_DO_TIMER_BOOT -2 |
| 7 | 7 | ||
| 8 | DECLARE_PER_CPU(struct tick_device, tick_cpu_device); | 8 | DECLARE_PER_CPU(struct tick_device, tick_cpu_device); |
| 9 | extern spinlock_t tick_device_lock; | ||
| 10 | extern ktime_t tick_next_period; | 9 | extern ktime_t tick_next_period; |
| 11 | extern ktime_t tick_period; | 10 | extern ktime_t tick_period; |
| 12 | extern int tick_do_timer_cpu __read_mostly; | 11 | extern int tick_do_timer_cpu __read_mostly; |
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c index 9d80db4747d4..28265636b6c2 100644 --- a/kernel/time/timer_list.c +++ b/kernel/time/timer_list.c | |||
| @@ -84,7 +84,7 @@ print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base, | |||
| 84 | 84 | ||
| 85 | next_one: | 85 | next_one: |
| 86 | i = 0; | 86 | i = 0; |
| 87 | spin_lock_irqsave(&base->cpu_base->lock, flags); | 87 | raw_spin_lock_irqsave(&base->cpu_base->lock, flags); |
| 88 | 88 | ||
| 89 | curr = base->first; | 89 | curr = base->first; |
| 90 | /* | 90 | /* |
| @@ -100,13 +100,13 @@ next_one: | |||
| 100 | 100 | ||
| 101 | timer = rb_entry(curr, struct hrtimer, node); | 101 | timer = rb_entry(curr, struct hrtimer, node); |
| 102 | tmp = *timer; | 102 | tmp = *timer; |
| 103 | spin_unlock_irqrestore(&base->cpu_base->lock, flags); | 103 | raw_spin_unlock_irqrestore(&base->cpu_base->lock, flags); |
| 104 | 104 | ||
| 105 | print_timer(m, timer, &tmp, i, now); | 105 | print_timer(m, timer, &tmp, i, now); |
| 106 | next++; | 106 | next++; |
| 107 | goto next_one; | 107 | goto next_one; |
| 108 | } | 108 | } |
| 109 | spin_unlock_irqrestore(&base->cpu_base->lock, flags); | 109 | raw_spin_unlock_irqrestore(&base->cpu_base->lock, flags); |
| 110 | } | 110 | } |
| 111 | 111 | ||
| 112 | static void | 112 | static void |
diff --git a/kernel/time/timer_stats.c b/kernel/time/timer_stats.c index ee5681f8d7ec..2f3b585b8d7d 100644 --- a/kernel/time/timer_stats.c +++ b/kernel/time/timer_stats.c | |||
| @@ -86,7 +86,7 @@ static DEFINE_SPINLOCK(table_lock); | |||
| 86 | /* | 86 | /* |
| 87 | * Per-CPU lookup locks for fast hash lookup: | 87 | * Per-CPU lookup locks for fast hash lookup: |
| 88 | */ | 88 | */ |
| 89 | static DEFINE_PER_CPU(spinlock_t, lookup_lock); | 89 | static DEFINE_PER_CPU(raw_spinlock_t, tstats_lookup_lock); |
| 90 | 90 | ||
| 91 | /* | 91 | /* |
| 92 | * Mutex to serialize state changes with show-stats activities: | 92 | * Mutex to serialize state changes with show-stats activities: |
| @@ -238,14 +238,14 @@ void timer_stats_update_stats(void *timer, pid_t pid, void *startf, | |||
| 238 | /* | 238 | /* |
| 239 | * It doesnt matter which lock we take: | 239 | * It doesnt matter which lock we take: |
| 240 | */ | 240 | */ |
| 241 | spinlock_t *lock; | 241 | raw_spinlock_t *lock; |
| 242 | struct entry *entry, input; | 242 | struct entry *entry, input; |
| 243 | unsigned long flags; | 243 | unsigned long flags; |
| 244 | 244 | ||
| 245 | if (likely(!timer_stats_active)) | 245 | if (likely(!timer_stats_active)) |
| 246 | return; | 246 | return; |
| 247 | 247 | ||
| 248 | lock = &per_cpu(lookup_lock, raw_smp_processor_id()); | 248 | lock = &per_cpu(tstats_lookup_lock, raw_smp_processor_id()); |
| 249 | 249 | ||
| 250 | input.timer = timer; | 250 | input.timer = timer; |
| 251 | input.start_func = startf; | 251 | input.start_func = startf; |
| @@ -253,7 +253,7 @@ void timer_stats_update_stats(void *timer, pid_t pid, void *startf, | |||
| 253 | input.pid = pid; | 253 | input.pid = pid; |
| 254 | input.timer_flag = timer_flag; | 254 | input.timer_flag = timer_flag; |
| 255 | 255 | ||
| 256 | spin_lock_irqsave(lock, flags); | 256 | raw_spin_lock_irqsave(lock, flags); |
| 257 | if (!timer_stats_active) | 257 | if (!timer_stats_active) |
| 258 | goto out_unlock; | 258 | goto out_unlock; |
| 259 | 259 | ||
| @@ -264,7 +264,7 @@ void timer_stats_update_stats(void *timer, pid_t pid, void *startf, | |||
| 264 | atomic_inc(&overflow_count); | 264 | atomic_inc(&overflow_count); |
| 265 | 265 | ||
| 266 | out_unlock: | 266 | out_unlock: |
| 267 | spin_unlock_irqrestore(lock, flags); | 267 | raw_spin_unlock_irqrestore(lock, flags); |
| 268 | } | 268 | } |
| 269 | 269 | ||
| 270 | static void print_name_offset(struct seq_file *m, unsigned long addr) | 270 | static void print_name_offset(struct seq_file *m, unsigned long addr) |
| @@ -348,9 +348,11 @@ static void sync_access(void) | |||
| 348 | int cpu; | 348 | int cpu; |
| 349 | 349 | ||
| 350 | for_each_online_cpu(cpu) { | 350 | for_each_online_cpu(cpu) { |
| 351 | spin_lock_irqsave(&per_cpu(lookup_lock, cpu), flags); | 351 | raw_spinlock_t *lock = &per_cpu(tstats_lookup_lock, cpu); |
| 352 | |||
| 353 | raw_spin_lock_irqsave(lock, flags); | ||
| 352 | /* nothing */ | 354 | /* nothing */ |
| 353 | spin_unlock_irqrestore(&per_cpu(lookup_lock, cpu), flags); | 355 | raw_spin_unlock_irqrestore(lock, flags); |
| 354 | } | 356 | } |
| 355 | } | 357 | } |
| 356 | 358 | ||
| @@ -408,7 +410,7 @@ void __init init_timer_stats(void) | |||
| 408 | int cpu; | 410 | int cpu; |
| 409 | 411 | ||
| 410 | for_each_possible_cpu(cpu) | 412 | for_each_possible_cpu(cpu) |
| 411 | spin_lock_init(&per_cpu(lookup_lock, cpu)); | 413 | raw_spin_lock_init(&per_cpu(tstats_lookup_lock, cpu)); |
| 412 | } | 414 | } |
| 413 | 415 | ||
| 414 | static int __init init_tstats_procfs(void) | 416 | static int __init init_tstats_procfs(void) |
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index a1ca4956ab5e..f58c9ad15830 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c | |||
| @@ -423,7 +423,7 @@ struct ring_buffer_per_cpu { | |||
| 423 | int cpu; | 423 | int cpu; |
| 424 | struct ring_buffer *buffer; | 424 | struct ring_buffer *buffer; |
| 425 | spinlock_t reader_lock; /* serialize readers */ | 425 | spinlock_t reader_lock; /* serialize readers */ |
| 426 | raw_spinlock_t lock; | 426 | arch_spinlock_t lock; |
| 427 | struct lock_class_key lock_key; | 427 | struct lock_class_key lock_key; |
| 428 | struct list_head *pages; | 428 | struct list_head *pages; |
| 429 | struct buffer_page *head_page; /* read from head */ | 429 | struct buffer_page *head_page; /* read from head */ |
| @@ -998,7 +998,7 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu) | |||
| 998 | cpu_buffer->buffer = buffer; | 998 | cpu_buffer->buffer = buffer; |
| 999 | spin_lock_init(&cpu_buffer->reader_lock); | 999 | spin_lock_init(&cpu_buffer->reader_lock); |
| 1000 | lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key); | 1000 | lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key); |
| 1001 | cpu_buffer->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; | 1001 | cpu_buffer->lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
| 1002 | 1002 | ||
| 1003 | bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), | 1003 | bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), |
| 1004 | GFP_KERNEL, cpu_to_node(cpu)); | 1004 | GFP_KERNEL, cpu_to_node(cpu)); |
| @@ -2834,7 +2834,7 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) | |||
| 2834 | int ret; | 2834 | int ret; |
| 2835 | 2835 | ||
| 2836 | local_irq_save(flags); | 2836 | local_irq_save(flags); |
| 2837 | __raw_spin_lock(&cpu_buffer->lock); | 2837 | arch_spin_lock(&cpu_buffer->lock); |
| 2838 | 2838 | ||
| 2839 | again: | 2839 | again: |
| 2840 | /* | 2840 | /* |
| @@ -2923,7 +2923,7 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) | |||
| 2923 | goto again; | 2923 | goto again; |
| 2924 | 2924 | ||
| 2925 | out: | 2925 | out: |
| 2926 | __raw_spin_unlock(&cpu_buffer->lock); | 2926 | arch_spin_unlock(&cpu_buffer->lock); |
| 2927 | local_irq_restore(flags); | 2927 | local_irq_restore(flags); |
| 2928 | 2928 | ||
| 2929 | return reader; | 2929 | return reader; |
| @@ -3286,9 +3286,9 @@ ring_buffer_read_start(struct ring_buffer *buffer, int cpu) | |||
| 3286 | synchronize_sched(); | 3286 | synchronize_sched(); |
| 3287 | 3287 | ||
| 3288 | spin_lock_irqsave(&cpu_buffer->reader_lock, flags); | 3288 | spin_lock_irqsave(&cpu_buffer->reader_lock, flags); |
| 3289 | __raw_spin_lock(&cpu_buffer->lock); | 3289 | arch_spin_lock(&cpu_buffer->lock); |
| 3290 | rb_iter_reset(iter); | 3290 | rb_iter_reset(iter); |
| 3291 | __raw_spin_unlock(&cpu_buffer->lock); | 3291 | arch_spin_unlock(&cpu_buffer->lock); |
| 3292 | spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); | 3292 | spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); |
| 3293 | 3293 | ||
| 3294 | return iter; | 3294 | return iter; |
| @@ -3408,11 +3408,11 @@ void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu) | |||
| 3408 | if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing))) | 3408 | if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing))) |
| 3409 | goto out; | 3409 | goto out; |
| 3410 | 3410 | ||
| 3411 | __raw_spin_lock(&cpu_buffer->lock); | 3411 | arch_spin_lock(&cpu_buffer->lock); |
| 3412 | 3412 | ||
| 3413 | rb_reset_cpu(cpu_buffer); | 3413 | rb_reset_cpu(cpu_buffer); |
| 3414 | 3414 | ||
| 3415 | __raw_spin_unlock(&cpu_buffer->lock); | 3415 | arch_spin_unlock(&cpu_buffer->lock); |
| 3416 | 3416 | ||
| 3417 | out: | 3417 | out: |
| 3418 | spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); | 3418 | spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); |
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 88bd9ae2a9ed..bb6b5e7fa2a2 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c | |||
| @@ -86,17 +86,17 @@ static int dummy_set_flag(u32 old_flags, u32 bit, int set) | |||
| 86 | */ | 86 | */ |
| 87 | static int tracing_disabled = 1; | 87 | static int tracing_disabled = 1; |
| 88 | 88 | ||
| 89 | DEFINE_PER_CPU(local_t, ftrace_cpu_disabled); | 89 | DEFINE_PER_CPU(int, ftrace_cpu_disabled); |
| 90 | 90 | ||
| 91 | static inline void ftrace_disable_cpu(void) | 91 | static inline void ftrace_disable_cpu(void) |
| 92 | { | 92 | { |
| 93 | preempt_disable(); | 93 | preempt_disable(); |
| 94 | local_inc(&__get_cpu_var(ftrace_cpu_disabled)); | 94 | __this_cpu_inc(per_cpu_var(ftrace_cpu_disabled)); |
| 95 | } | 95 | } |
| 96 | 96 | ||
| 97 | static inline void ftrace_enable_cpu(void) | 97 | static inline void ftrace_enable_cpu(void) |
| 98 | { | 98 | { |
| 99 | local_dec(&__get_cpu_var(ftrace_cpu_disabled)); | 99 | __this_cpu_dec(per_cpu_var(ftrace_cpu_disabled)); |
| 100 | preempt_enable(); | 100 | preempt_enable(); |
| 101 | } | 101 | } |
| 102 | 102 | ||
| @@ -203,7 +203,7 @@ cycle_t ftrace_now(int cpu) | |||
| 203 | */ | 203 | */ |
| 204 | static struct trace_array max_tr; | 204 | static struct trace_array max_tr; |
| 205 | 205 | ||
| 206 | static DEFINE_PER_CPU(struct trace_array_cpu, max_data); | 206 | static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data); |
| 207 | 207 | ||
| 208 | /* tracer_enabled is used to toggle activation of a tracer */ | 208 | /* tracer_enabled is used to toggle activation of a tracer */ |
| 209 | static int tracer_enabled = 1; | 209 | static int tracer_enabled = 1; |
| @@ -493,15 +493,15 @@ static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) | |||
| 493 | * protected by per_cpu spinlocks. But the action of the swap | 493 | * protected by per_cpu spinlocks. But the action of the swap |
| 494 | * needs its own lock. | 494 | * needs its own lock. |
| 495 | * | 495 | * |
| 496 | * This is defined as a raw_spinlock_t in order to help | 496 | * This is defined as a arch_spinlock_t in order to help |
| 497 | * with performance when lockdep debugging is enabled. | 497 | * with performance when lockdep debugging is enabled. |
| 498 | * | 498 | * |
| 499 | * It is also used in other places outside the update_max_tr | 499 | * It is also used in other places outside the update_max_tr |
| 500 | * so it needs to be defined outside of the | 500 | * so it needs to be defined outside of the |
| 501 | * CONFIG_TRACER_MAX_TRACE. | 501 | * CONFIG_TRACER_MAX_TRACE. |
| 502 | */ | 502 | */ |
| 503 | static raw_spinlock_t ftrace_max_lock = | 503 | static arch_spinlock_t ftrace_max_lock = |
| 504 | (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; | 504 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
| 505 | 505 | ||
| 506 | #ifdef CONFIG_TRACER_MAX_TRACE | 506 | #ifdef CONFIG_TRACER_MAX_TRACE |
| 507 | unsigned long __read_mostly tracing_max_latency; | 507 | unsigned long __read_mostly tracing_max_latency; |
| @@ -555,13 +555,13 @@ update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) | |||
| 555 | return; | 555 | return; |
| 556 | 556 | ||
| 557 | WARN_ON_ONCE(!irqs_disabled()); | 557 | WARN_ON_ONCE(!irqs_disabled()); |
| 558 | __raw_spin_lock(&ftrace_max_lock); | 558 | arch_spin_lock(&ftrace_max_lock); |
| 559 | 559 | ||
| 560 | tr->buffer = max_tr.buffer; | 560 | tr->buffer = max_tr.buffer; |
| 561 | max_tr.buffer = buf; | 561 | max_tr.buffer = buf; |
| 562 | 562 | ||
| 563 | __update_max_tr(tr, tsk, cpu); | 563 | __update_max_tr(tr, tsk, cpu); |
| 564 | __raw_spin_unlock(&ftrace_max_lock); | 564 | arch_spin_unlock(&ftrace_max_lock); |
| 565 | } | 565 | } |
| 566 | 566 | ||
| 567 | /** | 567 | /** |
| @@ -581,7 +581,7 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) | |||
| 581 | return; | 581 | return; |
| 582 | 582 | ||
| 583 | WARN_ON_ONCE(!irqs_disabled()); | 583 | WARN_ON_ONCE(!irqs_disabled()); |
| 584 | __raw_spin_lock(&ftrace_max_lock); | 584 | arch_spin_lock(&ftrace_max_lock); |
| 585 | 585 | ||
| 586 | ftrace_disable_cpu(); | 586 | ftrace_disable_cpu(); |
| 587 | 587 | ||
| @@ -603,7 +603,7 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) | |||
| 603 | WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); | 603 | WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); |
| 604 | 604 | ||
| 605 | __update_max_tr(tr, tsk, cpu); | 605 | __update_max_tr(tr, tsk, cpu); |
| 606 | __raw_spin_unlock(&ftrace_max_lock); | 606 | arch_spin_unlock(&ftrace_max_lock); |
| 607 | } | 607 | } |
| 608 | #endif /* CONFIG_TRACER_MAX_TRACE */ | 608 | #endif /* CONFIG_TRACER_MAX_TRACE */ |
| 609 | 609 | ||
| @@ -802,7 +802,7 @@ static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1]; | |||
| 802 | static unsigned map_cmdline_to_pid[SAVED_CMDLINES]; | 802 | static unsigned map_cmdline_to_pid[SAVED_CMDLINES]; |
| 803 | static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN]; | 803 | static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN]; |
| 804 | static int cmdline_idx; | 804 | static int cmdline_idx; |
| 805 | static raw_spinlock_t trace_cmdline_lock = __RAW_SPIN_LOCK_UNLOCKED; | 805 | static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED; |
| 806 | 806 | ||
| 807 | /* temporary disable recording */ | 807 | /* temporary disable recording */ |
| 808 | static atomic_t trace_record_cmdline_disabled __read_mostly; | 808 | static atomic_t trace_record_cmdline_disabled __read_mostly; |
| @@ -915,7 +915,7 @@ static void trace_save_cmdline(struct task_struct *tsk) | |||
| 915 | * nor do we want to disable interrupts, | 915 | * nor do we want to disable interrupts, |
| 916 | * so if we miss here, then better luck next time. | 916 | * so if we miss here, then better luck next time. |
| 917 | */ | 917 | */ |
| 918 | if (!__raw_spin_trylock(&trace_cmdline_lock)) | 918 | if (!arch_spin_trylock(&trace_cmdline_lock)) |
| 919 | return; | 919 | return; |
| 920 | 920 | ||
| 921 | idx = map_pid_to_cmdline[tsk->pid]; | 921 | idx = map_pid_to_cmdline[tsk->pid]; |
| @@ -940,7 +940,7 @@ static void trace_save_cmdline(struct task_struct *tsk) | |||
| 940 | 940 | ||
| 941 | memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN); | 941 | memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN); |
| 942 | 942 | ||
| 943 | __raw_spin_unlock(&trace_cmdline_lock); | 943 | arch_spin_unlock(&trace_cmdline_lock); |
| 944 | } | 944 | } |
| 945 | 945 | ||
| 946 | void trace_find_cmdline(int pid, char comm[]) | 946 | void trace_find_cmdline(int pid, char comm[]) |
| @@ -958,14 +958,14 @@ void trace_find_cmdline(int pid, char comm[]) | |||
| 958 | } | 958 | } |
| 959 | 959 | ||
| 960 | preempt_disable(); | 960 | preempt_disable(); |
| 961 | __raw_spin_lock(&trace_cmdline_lock); | 961 | arch_spin_lock(&trace_cmdline_lock); |
| 962 | map = map_pid_to_cmdline[pid]; | 962 | map = map_pid_to_cmdline[pid]; |
| 963 | if (map != NO_CMDLINE_MAP) | 963 | if (map != NO_CMDLINE_MAP) |
| 964 | strcpy(comm, saved_cmdlines[map]); | 964 | strcpy(comm, saved_cmdlines[map]); |
| 965 | else | 965 | else |
| 966 | strcpy(comm, "<...>"); | 966 | strcpy(comm, "<...>"); |
| 967 | 967 | ||
| 968 | __raw_spin_unlock(&trace_cmdline_lock); | 968 | arch_spin_unlock(&trace_cmdline_lock); |
| 969 | preempt_enable(); | 969 | preempt_enable(); |
| 970 | } | 970 | } |
| 971 | 971 | ||
| @@ -1085,7 +1085,7 @@ trace_function(struct trace_array *tr, | |||
| 1085 | struct ftrace_entry *entry; | 1085 | struct ftrace_entry *entry; |
| 1086 | 1086 | ||
| 1087 | /* If we are reading the ring buffer, don't trace */ | 1087 | /* If we are reading the ring buffer, don't trace */ |
| 1088 | if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled)))) | 1088 | if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled)))) |
| 1089 | return; | 1089 | return; |
| 1090 | 1090 | ||
| 1091 | event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), | 1091 | event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), |
| @@ -1251,8 +1251,8 @@ ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3) | |||
| 1251 | */ | 1251 | */ |
| 1252 | int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) | 1252 | int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) |
| 1253 | { | 1253 | { |
| 1254 | static raw_spinlock_t trace_buf_lock = | 1254 | static arch_spinlock_t trace_buf_lock = |
| 1255 | (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; | 1255 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
| 1256 | static u32 trace_buf[TRACE_BUF_SIZE]; | 1256 | static u32 trace_buf[TRACE_BUF_SIZE]; |
| 1257 | 1257 | ||
| 1258 | struct ftrace_event_call *call = &event_bprint; | 1258 | struct ftrace_event_call *call = &event_bprint; |
| @@ -1283,7 +1283,7 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) | |||
| 1283 | 1283 | ||
| 1284 | /* Lockdep uses trace_printk for lock tracing */ | 1284 | /* Lockdep uses trace_printk for lock tracing */ |
| 1285 | local_irq_save(flags); | 1285 | local_irq_save(flags); |
| 1286 | __raw_spin_lock(&trace_buf_lock); | 1286 | arch_spin_lock(&trace_buf_lock); |
| 1287 | len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args); | 1287 | len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args); |
| 1288 | 1288 | ||
| 1289 | if (len > TRACE_BUF_SIZE || len < 0) | 1289 | if (len > TRACE_BUF_SIZE || len < 0) |
| @@ -1304,7 +1304,7 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) | |||
| 1304 | ring_buffer_unlock_commit(buffer, event); | 1304 | ring_buffer_unlock_commit(buffer, event); |
| 1305 | 1305 | ||
| 1306 | out_unlock: | 1306 | out_unlock: |
| 1307 | __raw_spin_unlock(&trace_buf_lock); | 1307 | arch_spin_unlock(&trace_buf_lock); |
| 1308 | local_irq_restore(flags); | 1308 | local_irq_restore(flags); |
| 1309 | 1309 | ||
| 1310 | out: | 1310 | out: |
| @@ -1334,7 +1334,7 @@ int trace_array_printk(struct trace_array *tr, | |||
| 1334 | int trace_array_vprintk(struct trace_array *tr, | 1334 | int trace_array_vprintk(struct trace_array *tr, |
| 1335 | unsigned long ip, const char *fmt, va_list args) | 1335 | unsigned long ip, const char *fmt, va_list args) |
| 1336 | { | 1336 | { |
| 1337 | static raw_spinlock_t trace_buf_lock = __RAW_SPIN_LOCK_UNLOCKED; | 1337 | static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED; |
| 1338 | static char trace_buf[TRACE_BUF_SIZE]; | 1338 | static char trace_buf[TRACE_BUF_SIZE]; |
| 1339 | 1339 | ||
| 1340 | struct ftrace_event_call *call = &event_print; | 1340 | struct ftrace_event_call *call = &event_print; |
| @@ -1360,7 +1360,7 @@ int trace_array_vprintk(struct trace_array *tr, | |||
| 1360 | 1360 | ||
| 1361 | pause_graph_tracing(); | 1361 | pause_graph_tracing(); |
| 1362 | raw_local_irq_save(irq_flags); | 1362 | raw_local_irq_save(irq_flags); |
| 1363 | __raw_spin_lock(&trace_buf_lock); | 1363 | arch_spin_lock(&trace_buf_lock); |
| 1364 | len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args); | 1364 | len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args); |
| 1365 | 1365 | ||
| 1366 | size = sizeof(*entry) + len + 1; | 1366 | size = sizeof(*entry) + len + 1; |
| @@ -1378,7 +1378,7 @@ int trace_array_vprintk(struct trace_array *tr, | |||
| 1378 | ring_buffer_unlock_commit(buffer, event); | 1378 | ring_buffer_unlock_commit(buffer, event); |
| 1379 | 1379 | ||
| 1380 | out_unlock: | 1380 | out_unlock: |
| 1381 | __raw_spin_unlock(&trace_buf_lock); | 1381 | arch_spin_unlock(&trace_buf_lock); |
| 1382 | raw_local_irq_restore(irq_flags); | 1382 | raw_local_irq_restore(irq_flags); |
| 1383 | unpause_graph_tracing(); | 1383 | unpause_graph_tracing(); |
| 1384 | out: | 1384 | out: |
| @@ -2279,7 +2279,7 @@ tracing_cpumask_write(struct file *filp, const char __user *ubuf, | |||
| 2279 | mutex_lock(&tracing_cpumask_update_lock); | 2279 | mutex_lock(&tracing_cpumask_update_lock); |
| 2280 | 2280 | ||
| 2281 | local_irq_disable(); | 2281 | local_irq_disable(); |
| 2282 | __raw_spin_lock(&ftrace_max_lock); | 2282 | arch_spin_lock(&ftrace_max_lock); |
| 2283 | for_each_tracing_cpu(cpu) { | 2283 | for_each_tracing_cpu(cpu) { |
| 2284 | /* | 2284 | /* |
| 2285 | * Increase/decrease the disabled counter if we are | 2285 | * Increase/decrease the disabled counter if we are |
| @@ -2294,7 +2294,7 @@ tracing_cpumask_write(struct file *filp, const char __user *ubuf, | |||
| 2294 | atomic_dec(&global_trace.data[cpu]->disabled); | 2294 | atomic_dec(&global_trace.data[cpu]->disabled); |
| 2295 | } | 2295 | } |
| 2296 | } | 2296 | } |
| 2297 | __raw_spin_unlock(&ftrace_max_lock); | 2297 | arch_spin_unlock(&ftrace_max_lock); |
| 2298 | local_irq_enable(); | 2298 | local_irq_enable(); |
| 2299 | 2299 | ||
| 2300 | cpumask_copy(tracing_cpumask, tracing_cpumask_new); | 2300 | cpumask_copy(tracing_cpumask, tracing_cpumask_new); |
| @@ -4307,8 +4307,8 @@ trace_printk_seq(struct trace_seq *s) | |||
| 4307 | 4307 | ||
| 4308 | static void __ftrace_dump(bool disable_tracing) | 4308 | static void __ftrace_dump(bool disable_tracing) |
| 4309 | { | 4309 | { |
| 4310 | static raw_spinlock_t ftrace_dump_lock = | 4310 | static arch_spinlock_t ftrace_dump_lock = |
| 4311 | (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; | 4311 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
| 4312 | /* use static because iter can be a bit big for the stack */ | 4312 | /* use static because iter can be a bit big for the stack */ |
| 4313 | static struct trace_iterator iter; | 4313 | static struct trace_iterator iter; |
| 4314 | unsigned int old_userobj; | 4314 | unsigned int old_userobj; |
| @@ -4318,7 +4318,7 @@ static void __ftrace_dump(bool disable_tracing) | |||
| 4318 | 4318 | ||
| 4319 | /* only one dump */ | 4319 | /* only one dump */ |
| 4320 | local_irq_save(flags); | 4320 | local_irq_save(flags); |
| 4321 | __raw_spin_lock(&ftrace_dump_lock); | 4321 | arch_spin_lock(&ftrace_dump_lock); |
| 4322 | if (dump_ran) | 4322 | if (dump_ran) |
| 4323 | goto out; | 4323 | goto out; |
| 4324 | 4324 | ||
| @@ -4393,7 +4393,7 @@ static void __ftrace_dump(bool disable_tracing) | |||
| 4393 | } | 4393 | } |
| 4394 | 4394 | ||
| 4395 | out: | 4395 | out: |
| 4396 | __raw_spin_unlock(&ftrace_dump_lock); | 4396 | arch_spin_unlock(&ftrace_dump_lock); |
| 4397 | local_irq_restore(flags); | 4397 | local_irq_restore(flags); |
| 4398 | } | 4398 | } |
| 4399 | 4399 | ||
| @@ -4454,7 +4454,7 @@ __init static int tracer_alloc_buffers(void) | |||
| 4454 | /* Allocate the first page for all buffers */ | 4454 | /* Allocate the first page for all buffers */ |
| 4455 | for_each_tracing_cpu(i) { | 4455 | for_each_tracing_cpu(i) { |
| 4456 | global_trace.data[i] = &per_cpu(global_trace_cpu, i); | 4456 | global_trace.data[i] = &per_cpu(global_trace_cpu, i); |
| 4457 | max_tr.data[i] = &per_cpu(max_data, i); | 4457 | max_tr.data[i] = &per_cpu(max_tr_data, i); |
| 4458 | } | 4458 | } |
| 4459 | 4459 | ||
| 4460 | trace_init_cmdlines(); | 4460 | trace_init_cmdlines(); |
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 7fa33cab6962..a52bed2eedd8 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h | |||
| @@ -443,7 +443,7 @@ extern int DYN_FTRACE_TEST_NAME(void); | |||
| 443 | 443 | ||
| 444 | extern int ring_buffer_expanded; | 444 | extern int ring_buffer_expanded; |
| 445 | extern bool tracing_selftest_disabled; | 445 | extern bool tracing_selftest_disabled; |
| 446 | DECLARE_PER_CPU(local_t, ftrace_cpu_disabled); | 446 | DECLARE_PER_CPU(int, ftrace_cpu_disabled); |
| 447 | 447 | ||
| 448 | #ifdef CONFIG_FTRACE_STARTUP_TEST | 448 | #ifdef CONFIG_FTRACE_STARTUP_TEST |
| 449 | extern int trace_selftest_startup_function(struct tracer *trace, | 449 | extern int trace_selftest_startup_function(struct tracer *trace, |
diff --git a/kernel/trace/trace_clock.c b/kernel/trace/trace_clock.c index 878c03f386ba..84a3a7ba072a 100644 --- a/kernel/trace/trace_clock.c +++ b/kernel/trace/trace_clock.c | |||
| @@ -71,10 +71,10 @@ u64 notrace trace_clock(void) | |||
| 71 | /* keep prev_time and lock in the same cacheline. */ | 71 | /* keep prev_time and lock in the same cacheline. */ |
| 72 | static struct { | 72 | static struct { |
| 73 | u64 prev_time; | 73 | u64 prev_time; |
| 74 | raw_spinlock_t lock; | 74 | arch_spinlock_t lock; |
| 75 | } trace_clock_struct ____cacheline_aligned_in_smp = | 75 | } trace_clock_struct ____cacheline_aligned_in_smp = |
| 76 | { | 76 | { |
| 77 | .lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED, | 77 | .lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED, |
| 78 | }; | 78 | }; |
| 79 | 79 | ||
| 80 | u64 notrace trace_clock_global(void) | 80 | u64 notrace trace_clock_global(void) |
| @@ -94,7 +94,7 @@ u64 notrace trace_clock_global(void) | |||
| 94 | if (unlikely(in_nmi())) | 94 | if (unlikely(in_nmi())) |
| 95 | goto out; | 95 | goto out; |
| 96 | 96 | ||
| 97 | __raw_spin_lock(&trace_clock_struct.lock); | 97 | arch_spin_lock(&trace_clock_struct.lock); |
| 98 | 98 | ||
| 99 | /* | 99 | /* |
| 100 | * TODO: if this happens often then maybe we should reset | 100 | * TODO: if this happens often then maybe we should reset |
| @@ -106,7 +106,7 @@ u64 notrace trace_clock_global(void) | |||
| 106 | 106 | ||
| 107 | trace_clock_struct.prev_time = now; | 107 | trace_clock_struct.prev_time = now; |
| 108 | 108 | ||
| 109 | __raw_spin_unlock(&trace_clock_struct.lock); | 109 | arch_spin_unlock(&trace_clock_struct.lock); |
| 110 | 110 | ||
| 111 | out: | 111 | out: |
| 112 | raw_local_irq_restore(flags); | 112 | raw_local_irq_restore(flags); |
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index a43d009c561a..b1342c5d37cf 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c | |||
| @@ -187,7 +187,7 @@ static int __trace_graph_entry(struct trace_array *tr, | |||
| 187 | struct ring_buffer *buffer = tr->buffer; | 187 | struct ring_buffer *buffer = tr->buffer; |
| 188 | struct ftrace_graph_ent_entry *entry; | 188 | struct ftrace_graph_ent_entry *entry; |
| 189 | 189 | ||
| 190 | if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled)))) | 190 | if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled)))) |
| 191 | return 0; | 191 | return 0; |
| 192 | 192 | ||
| 193 | event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT, | 193 | event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT, |
| @@ -251,7 +251,7 @@ static void __trace_graph_return(struct trace_array *tr, | |||
| 251 | struct ring_buffer *buffer = tr->buffer; | 251 | struct ring_buffer *buffer = tr->buffer; |
| 252 | struct ftrace_graph_ret_entry *entry; | 252 | struct ftrace_graph_ret_entry *entry; |
| 253 | 253 | ||
| 254 | if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled)))) | 254 | if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled)))) |
| 255 | return; | 255 | return; |
| 256 | 256 | ||
| 257 | event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET, | 257 | event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET, |
diff --git a/kernel/trace/trace_hw_branches.c b/kernel/trace/trace_hw_branches.c index 69543a905cd5..7b97000745f5 100644 --- a/kernel/trace/trace_hw_branches.c +++ b/kernel/trace/trace_hw_branches.c | |||
| @@ -20,10 +20,10 @@ | |||
| 20 | 20 | ||
| 21 | #define BTS_BUFFER_SIZE (1 << 13) | 21 | #define BTS_BUFFER_SIZE (1 << 13) |
| 22 | 22 | ||
| 23 | static DEFINE_PER_CPU(struct bts_tracer *, tracer); | 23 | static DEFINE_PER_CPU(struct bts_tracer *, hwb_tracer); |
| 24 | static DEFINE_PER_CPU(unsigned char[BTS_BUFFER_SIZE], buffer); | 24 | static DEFINE_PER_CPU(unsigned char[BTS_BUFFER_SIZE], hwb_buffer); |
| 25 | 25 | ||
| 26 | #define this_tracer per_cpu(tracer, smp_processor_id()) | 26 | #define this_tracer per_cpu(hwb_tracer, smp_processor_id()) |
| 27 | 27 | ||
| 28 | static int trace_hw_branches_enabled __read_mostly; | 28 | static int trace_hw_branches_enabled __read_mostly; |
| 29 | static int trace_hw_branches_suspended __read_mostly; | 29 | static int trace_hw_branches_suspended __read_mostly; |
| @@ -32,12 +32,13 @@ static struct trace_array *hw_branch_trace __read_mostly; | |||
| 32 | 32 | ||
| 33 | static void bts_trace_init_cpu(int cpu) | 33 | static void bts_trace_init_cpu(int cpu) |
| 34 | { | 34 | { |
| 35 | per_cpu(tracer, cpu) = | 35 | per_cpu(hwb_tracer, cpu) = |
| 36 | ds_request_bts_cpu(cpu, per_cpu(buffer, cpu), BTS_BUFFER_SIZE, | 36 | ds_request_bts_cpu(cpu, per_cpu(hwb_buffer, cpu), |
| 37 | NULL, (size_t)-1, BTS_KERNEL); | 37 | BTS_BUFFER_SIZE, NULL, (size_t)-1, |
| 38 | BTS_KERNEL); | ||
| 38 | 39 | ||
| 39 | if (IS_ERR(per_cpu(tracer, cpu))) | 40 | if (IS_ERR(per_cpu(hwb_tracer, cpu))) |
| 40 | per_cpu(tracer, cpu) = NULL; | 41 | per_cpu(hwb_tracer, cpu) = NULL; |
| 41 | } | 42 | } |
| 42 | 43 | ||
| 43 | static int bts_trace_init(struct trace_array *tr) | 44 | static int bts_trace_init(struct trace_array *tr) |
| @@ -51,7 +52,7 @@ static int bts_trace_init(struct trace_array *tr) | |||
| 51 | for_each_online_cpu(cpu) { | 52 | for_each_online_cpu(cpu) { |
| 52 | bts_trace_init_cpu(cpu); | 53 | bts_trace_init_cpu(cpu); |
| 53 | 54 | ||
| 54 | if (likely(per_cpu(tracer, cpu))) | 55 | if (likely(per_cpu(hwb_tracer, cpu))) |
| 55 | trace_hw_branches_enabled = 1; | 56 | trace_hw_branches_enabled = 1; |
| 56 | } | 57 | } |
| 57 | trace_hw_branches_suspended = 0; | 58 | trace_hw_branches_suspended = 0; |
| @@ -67,9 +68,9 @@ static void bts_trace_reset(struct trace_array *tr) | |||
| 67 | 68 | ||
| 68 | get_online_cpus(); | 69 | get_online_cpus(); |
| 69 | for_each_online_cpu(cpu) { | 70 | for_each_online_cpu(cpu) { |
| 70 | if (likely(per_cpu(tracer, cpu))) { | 71 | if (likely(per_cpu(hwb_tracer, cpu))) { |
| 71 | ds_release_bts(per_cpu(tracer, cpu)); | 72 | ds_release_bts(per_cpu(hwb_tracer, cpu)); |
| 72 | per_cpu(tracer, cpu) = NULL; | 73 | per_cpu(hwb_tracer, cpu) = NULL; |
| 73 | } | 74 | } |
| 74 | } | 75 | } |
| 75 | trace_hw_branches_enabled = 0; | 76 | trace_hw_branches_enabled = 0; |
| @@ -83,8 +84,8 @@ static void bts_trace_start(struct trace_array *tr) | |||
| 83 | 84 | ||
| 84 | get_online_cpus(); | 85 | get_online_cpus(); |
| 85 | for_each_online_cpu(cpu) | 86 | for_each_online_cpu(cpu) |
| 86 | if (likely(per_cpu(tracer, cpu))) | 87 | if (likely(per_cpu(hwb_tracer, cpu))) |
| 87 | ds_resume_bts(per_cpu(tracer, cpu)); | 88 | ds_resume_bts(per_cpu(hwb_tracer, cpu)); |
| 88 | trace_hw_branches_suspended = 0; | 89 | trace_hw_branches_suspended = 0; |
| 89 | put_online_cpus(); | 90 | put_online_cpus(); |
| 90 | } | 91 | } |
| @@ -95,8 +96,8 @@ static void bts_trace_stop(struct trace_array *tr) | |||
| 95 | 96 | ||
| 96 | get_online_cpus(); | 97 | get_online_cpus(); |
| 97 | for_each_online_cpu(cpu) | 98 | for_each_online_cpu(cpu) |
| 98 | if (likely(per_cpu(tracer, cpu))) | 99 | if (likely(per_cpu(hwb_tracer, cpu))) |
| 99 | ds_suspend_bts(per_cpu(tracer, cpu)); | 100 | ds_suspend_bts(per_cpu(hwb_tracer, cpu)); |
| 100 | trace_hw_branches_suspended = 1; | 101 | trace_hw_branches_suspended = 1; |
| 101 | put_online_cpus(); | 102 | put_online_cpus(); |
| 102 | } | 103 | } |
| @@ -114,16 +115,16 @@ static int __cpuinit bts_hotcpu_handler(struct notifier_block *nfb, | |||
| 114 | bts_trace_init_cpu(cpu); | 115 | bts_trace_init_cpu(cpu); |
| 115 | 116 | ||
| 116 | if (trace_hw_branches_suspended && | 117 | if (trace_hw_branches_suspended && |
| 117 | likely(per_cpu(tracer, cpu))) | 118 | likely(per_cpu(hwb_tracer, cpu))) |
| 118 | ds_suspend_bts(per_cpu(tracer, cpu)); | 119 | ds_suspend_bts(per_cpu(hwb_tracer, cpu)); |
| 119 | } | 120 | } |
| 120 | break; | 121 | break; |
| 121 | 122 | ||
| 122 | case CPU_DOWN_PREPARE: | 123 | case CPU_DOWN_PREPARE: |
| 123 | /* The notification is sent with interrupts enabled. */ | 124 | /* The notification is sent with interrupts enabled. */ |
| 124 | if (likely(per_cpu(tracer, cpu))) { | 125 | if (likely(per_cpu(hwb_tracer, cpu))) { |
| 125 | ds_release_bts(per_cpu(tracer, cpu)); | 126 | ds_release_bts(per_cpu(hwb_tracer, cpu)); |
| 126 | per_cpu(tracer, cpu) = NULL; | 127 | per_cpu(hwb_tracer, cpu) = NULL; |
| 127 | } | 128 | } |
| 128 | } | 129 | } |
| 129 | 130 | ||
| @@ -258,8 +259,8 @@ static void trace_bts_prepare(struct trace_iterator *iter) | |||
| 258 | 259 | ||
| 259 | get_online_cpus(); | 260 | get_online_cpus(); |
| 260 | for_each_online_cpu(cpu) | 261 | for_each_online_cpu(cpu) |
| 261 | if (likely(per_cpu(tracer, cpu))) | 262 | if (likely(per_cpu(hwb_tracer, cpu))) |
| 262 | ds_suspend_bts(per_cpu(tracer, cpu)); | 263 | ds_suspend_bts(per_cpu(hwb_tracer, cpu)); |
| 263 | /* | 264 | /* |
| 264 | * We need to collect the trace on the respective cpu since ftrace | 265 | * We need to collect the trace on the respective cpu since ftrace |
| 265 | * implicitly adds the record for the current cpu. | 266 | * implicitly adds the record for the current cpu. |
| @@ -268,8 +269,8 @@ static void trace_bts_prepare(struct trace_iterator *iter) | |||
| 268 | on_each_cpu(trace_bts_cpu, iter->tr, 1); | 269 | on_each_cpu(trace_bts_cpu, iter->tr, 1); |
| 269 | 270 | ||
| 270 | for_each_online_cpu(cpu) | 271 | for_each_online_cpu(cpu) |
| 271 | if (likely(per_cpu(tracer, cpu))) | 272 | if (likely(per_cpu(hwb_tracer, cpu))) |
| 272 | ds_resume_bts(per_cpu(tracer, cpu)); | 273 | ds_resume_bts(per_cpu(hwb_tracer, cpu)); |
| 273 | put_online_cpus(); | 274 | put_online_cpus(); |
| 274 | } | 275 | } |
| 275 | 276 | ||
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c index 26185d727676..0271742abb8d 100644 --- a/kernel/trace/trace_sched_wakeup.c +++ b/kernel/trace/trace_sched_wakeup.c | |||
| @@ -28,8 +28,8 @@ static int wakeup_current_cpu; | |||
| 28 | static unsigned wakeup_prio = -1; | 28 | static unsigned wakeup_prio = -1; |
| 29 | static int wakeup_rt; | 29 | static int wakeup_rt; |
| 30 | 30 | ||
| 31 | static raw_spinlock_t wakeup_lock = | 31 | static arch_spinlock_t wakeup_lock = |
| 32 | (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; | 32 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
| 33 | 33 | ||
| 34 | static void __wakeup_reset(struct trace_array *tr); | 34 | static void __wakeup_reset(struct trace_array *tr); |
| 35 | 35 | ||
| @@ -143,7 +143,7 @@ probe_wakeup_sched_switch(struct rq *rq, struct task_struct *prev, | |||
| 143 | goto out; | 143 | goto out; |
| 144 | 144 | ||
| 145 | local_irq_save(flags); | 145 | local_irq_save(flags); |
| 146 | __raw_spin_lock(&wakeup_lock); | 146 | arch_spin_lock(&wakeup_lock); |
| 147 | 147 | ||
| 148 | /* We could race with grabbing wakeup_lock */ | 148 | /* We could race with grabbing wakeup_lock */ |
| 149 | if (unlikely(!tracer_enabled || next != wakeup_task)) | 149 | if (unlikely(!tracer_enabled || next != wakeup_task)) |
| @@ -169,7 +169,7 @@ probe_wakeup_sched_switch(struct rq *rq, struct task_struct *prev, | |||
| 169 | 169 | ||
| 170 | out_unlock: | 170 | out_unlock: |
| 171 | __wakeup_reset(wakeup_trace); | 171 | __wakeup_reset(wakeup_trace); |
| 172 | __raw_spin_unlock(&wakeup_lock); | 172 | arch_spin_unlock(&wakeup_lock); |
| 173 | local_irq_restore(flags); | 173 | local_irq_restore(flags); |
| 174 | out: | 174 | out: |
| 175 | atomic_dec(&wakeup_trace->data[cpu]->disabled); | 175 | atomic_dec(&wakeup_trace->data[cpu]->disabled); |
| @@ -193,9 +193,9 @@ static void wakeup_reset(struct trace_array *tr) | |||
| 193 | tracing_reset_online_cpus(tr); | 193 | tracing_reset_online_cpus(tr); |
| 194 | 194 | ||
| 195 | local_irq_save(flags); | 195 | local_irq_save(flags); |
| 196 | __raw_spin_lock(&wakeup_lock); | 196 | arch_spin_lock(&wakeup_lock); |
| 197 | __wakeup_reset(tr); | 197 | __wakeup_reset(tr); |
| 198 | __raw_spin_unlock(&wakeup_lock); | 198 | arch_spin_unlock(&wakeup_lock); |
| 199 | local_irq_restore(flags); | 199 | local_irq_restore(flags); |
| 200 | } | 200 | } |
| 201 | 201 | ||
| @@ -225,7 +225,7 @@ probe_wakeup(struct rq *rq, struct task_struct *p, int success) | |||
| 225 | goto out; | 225 | goto out; |
| 226 | 226 | ||
| 227 | /* interrupts should be off from try_to_wake_up */ | 227 | /* interrupts should be off from try_to_wake_up */ |
| 228 | __raw_spin_lock(&wakeup_lock); | 228 | arch_spin_lock(&wakeup_lock); |
| 229 | 229 | ||
| 230 | /* check for races. */ | 230 | /* check for races. */ |
| 231 | if (!tracer_enabled || p->prio >= wakeup_prio) | 231 | if (!tracer_enabled || p->prio >= wakeup_prio) |
| @@ -255,7 +255,7 @@ probe_wakeup(struct rq *rq, struct task_struct *p, int success) | |||
| 255 | trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, flags, pc); | 255 | trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, flags, pc); |
| 256 | 256 | ||
| 257 | out_locked: | 257 | out_locked: |
| 258 | __raw_spin_unlock(&wakeup_lock); | 258 | arch_spin_unlock(&wakeup_lock); |
| 259 | out: | 259 | out: |
| 260 | atomic_dec(&wakeup_trace->data[cpu]->disabled); | 260 | atomic_dec(&wakeup_trace->data[cpu]->disabled); |
| 261 | } | 261 | } |
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c index dc98309e839a..280fea470d67 100644 --- a/kernel/trace/trace_selftest.c +++ b/kernel/trace/trace_selftest.c | |||
| @@ -67,7 +67,7 @@ static int trace_test_buffer(struct trace_array *tr, unsigned long *count) | |||
| 67 | 67 | ||
| 68 | /* Don't allow flipping of max traces now */ | 68 | /* Don't allow flipping of max traces now */ |
| 69 | local_irq_save(flags); | 69 | local_irq_save(flags); |
| 70 | __raw_spin_lock(&ftrace_max_lock); | 70 | arch_spin_lock(&ftrace_max_lock); |
| 71 | 71 | ||
| 72 | cnt = ring_buffer_entries(tr->buffer); | 72 | cnt = ring_buffer_entries(tr->buffer); |
| 73 | 73 | ||
| @@ -85,7 +85,7 @@ static int trace_test_buffer(struct trace_array *tr, unsigned long *count) | |||
| 85 | break; | 85 | break; |
| 86 | } | 86 | } |
| 87 | tracing_on(); | 87 | tracing_on(); |
| 88 | __raw_spin_unlock(&ftrace_max_lock); | 88 | arch_spin_unlock(&ftrace_max_lock); |
| 89 | local_irq_restore(flags); | 89 | local_irq_restore(flags); |
| 90 | 90 | ||
| 91 | if (count) | 91 | if (count) |
diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c index 8504ac71e4e8..678a5120ee30 100644 --- a/kernel/trace/trace_stack.c +++ b/kernel/trace/trace_stack.c | |||
| @@ -27,8 +27,8 @@ static struct stack_trace max_stack_trace = { | |||
| 27 | }; | 27 | }; |
| 28 | 28 | ||
| 29 | static unsigned long max_stack_size; | 29 | static unsigned long max_stack_size; |
| 30 | static raw_spinlock_t max_stack_lock = | 30 | static arch_spinlock_t max_stack_lock = |
| 31 | (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; | 31 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
| 32 | 32 | ||
| 33 | static int stack_trace_disabled __read_mostly; | 33 | static int stack_trace_disabled __read_mostly; |
| 34 | static DEFINE_PER_CPU(int, trace_active); | 34 | static DEFINE_PER_CPU(int, trace_active); |
| @@ -54,7 +54,7 @@ static inline void check_stack(void) | |||
| 54 | return; | 54 | return; |
| 55 | 55 | ||
| 56 | local_irq_save(flags); | 56 | local_irq_save(flags); |
| 57 | __raw_spin_lock(&max_stack_lock); | 57 | arch_spin_lock(&max_stack_lock); |
| 58 | 58 | ||
| 59 | /* a race could have already updated it */ | 59 | /* a race could have already updated it */ |
| 60 | if (this_size <= max_stack_size) | 60 | if (this_size <= max_stack_size) |
| @@ -103,7 +103,7 @@ static inline void check_stack(void) | |||
| 103 | } | 103 | } |
| 104 | 104 | ||
| 105 | out: | 105 | out: |
| 106 | __raw_spin_unlock(&max_stack_lock); | 106 | arch_spin_unlock(&max_stack_lock); |
| 107 | local_irq_restore(flags); | 107 | local_irq_restore(flags); |
| 108 | } | 108 | } |
| 109 | 109 | ||
| @@ -171,9 +171,9 @@ stack_max_size_write(struct file *filp, const char __user *ubuf, | |||
| 171 | return ret; | 171 | return ret; |
| 172 | 172 | ||
| 173 | local_irq_save(flags); | 173 | local_irq_save(flags); |
| 174 | __raw_spin_lock(&max_stack_lock); | 174 | arch_spin_lock(&max_stack_lock); |
| 175 | *ptr = val; | 175 | *ptr = val; |
| 176 | __raw_spin_unlock(&max_stack_lock); | 176 | arch_spin_unlock(&max_stack_lock); |
| 177 | local_irq_restore(flags); | 177 | local_irq_restore(flags); |
| 178 | 178 | ||
| 179 | return count; | 179 | return count; |
| @@ -207,7 +207,7 @@ t_next(struct seq_file *m, void *v, loff_t *pos) | |||
| 207 | static void *t_start(struct seq_file *m, loff_t *pos) | 207 | static void *t_start(struct seq_file *m, loff_t *pos) |
| 208 | { | 208 | { |
| 209 | local_irq_disable(); | 209 | local_irq_disable(); |
| 210 | __raw_spin_lock(&max_stack_lock); | 210 | arch_spin_lock(&max_stack_lock); |
| 211 | 211 | ||
| 212 | if (*pos == 0) | 212 | if (*pos == 0) |
| 213 | return SEQ_START_TOKEN; | 213 | return SEQ_START_TOKEN; |
| @@ -217,7 +217,7 @@ static void *t_start(struct seq_file *m, loff_t *pos) | |||
| 217 | 217 | ||
| 218 | static void t_stop(struct seq_file *m, void *p) | 218 | static void t_stop(struct seq_file *m, void *p) |
| 219 | { | 219 | { |
| 220 | __raw_spin_unlock(&max_stack_lock); | 220 | arch_spin_unlock(&max_stack_lock); |
| 221 | local_irq_enable(); | 221 | local_irq_enable(); |
| 222 | } | 222 | } |
| 223 | 223 | ||
