diff options
author | Christoph Lameter <cl@linux.com> | 2013-10-21 08:17:08 -0400 |
---|---|---|
committer | Russell King <rmk+kernel@arm.linux.org.uk> | 2013-10-29 07:06:27 -0400 |
commit | 1436c1aa626d0bc0e35c5c5231127086e80ab24a (patch) | |
tree | df8d5d0f892a502f3a5654745f539f3c6db4fe09 /arch/arm64 | |
parent | 39792c7cf3111d69dc4aa0923859d8b929e9039f (diff) |
ARM: 7862/1: pcpu: replace __get_cpu_var_uses
This is the ARM part of Christoph's patchset cleaning up the various
uses of __get_cpu_var across the tree.
The idea is to convert __get_cpu_var into either an explicit address
calculation using this_cpu_ptr() or into a use of this_cpu operations
that use the offset. Thereby address calculations are avoided and fewer
registers are used when code is generated.
[will: fixed debug ref counting checks and pcpu array accesses]
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
Diffstat (limited to 'arch/arm64')
-rw-r--r-- | arch/arm64/kernel/debug-monitors.c | 13 | ||||
-rw-r--r-- | arch/arm64/kernel/hw_breakpoint.c | 22 | ||||
-rw-r--r-- | arch/arm64/kernel/perf_event.c | 4 |
3 files changed, 19 insertions, 20 deletions
diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index cbfacf7fb438..6a0a9b132d7a 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c | |||
@@ -27,7 +27,6 @@ | |||
27 | #include <linux/uaccess.h> | 27 | #include <linux/uaccess.h> |
28 | 28 | ||
29 | #include <asm/debug-monitors.h> | 29 | #include <asm/debug-monitors.h> |
30 | #include <asm/local.h> | ||
31 | #include <asm/cputype.h> | 30 | #include <asm/cputype.h> |
32 | #include <asm/system_misc.h> | 31 | #include <asm/system_misc.h> |
33 | 32 | ||
@@ -89,8 +88,8 @@ early_param("nodebugmon", early_debug_disable); | |||
89 | * Keep track of debug users on each core. | 88 | * Keep track of debug users on each core. |
90 | * The ref counts are per-cpu so we use a local_t type. | 89 | * The ref counts are per-cpu so we use a local_t type. |
91 | */ | 90 | */ |
92 | static DEFINE_PER_CPU(local_t, mde_ref_count); | 91 | static DEFINE_PER_CPU(int, mde_ref_count); |
93 | static DEFINE_PER_CPU(local_t, kde_ref_count); | 92 | static DEFINE_PER_CPU(int, kde_ref_count); |
94 | 93 | ||
95 | void enable_debug_monitors(enum debug_el el) | 94 | void enable_debug_monitors(enum debug_el el) |
96 | { | 95 | { |
@@ -98,11 +97,11 @@ void enable_debug_monitors(enum debug_el el) | |||
98 | 97 | ||
99 | WARN_ON(preemptible()); | 98 | WARN_ON(preemptible()); |
100 | 99 | ||
101 | if (local_inc_return(&__get_cpu_var(mde_ref_count)) == 1) | 100 | if (this_cpu_inc_return(mde_ref_count) == 1) |
102 | enable = DBG_MDSCR_MDE; | 101 | enable = DBG_MDSCR_MDE; |
103 | 102 | ||
104 | if (el == DBG_ACTIVE_EL1 && | 103 | if (el == DBG_ACTIVE_EL1 && |
105 | local_inc_return(&__get_cpu_var(kde_ref_count)) == 1) | 104 | this_cpu_inc_return(kde_ref_count) == 1) |
106 | enable |= DBG_MDSCR_KDE; | 105 | enable |= DBG_MDSCR_KDE; |
107 | 106 | ||
108 | if (enable && debug_enabled) { | 107 | if (enable && debug_enabled) { |
@@ -118,11 +117,11 @@ void disable_debug_monitors(enum debug_el el) | |||
118 | 117 | ||
119 | WARN_ON(preemptible()); | 118 | WARN_ON(preemptible()); |
120 | 119 | ||
121 | if (local_dec_and_test(&__get_cpu_var(mde_ref_count))) | 120 | if (this_cpu_dec_return(mde_ref_count) == 0) |
122 | disable = ~DBG_MDSCR_MDE; | 121 | disable = ~DBG_MDSCR_MDE; |
123 | 122 | ||
124 | if (el == DBG_ACTIVE_EL1 && | 123 | if (el == DBG_ACTIVE_EL1 && |
125 | local_dec_and_test(&__get_cpu_var(kde_ref_count))) | 124 | this_cpu_dec_return(kde_ref_count) == 0) |
126 | disable &= ~DBG_MDSCR_KDE; | 125 | disable &= ~DBG_MDSCR_KDE; |
127 | 126 | ||
128 | if (disable) { | 127 | if (disable) { |
diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c index 329218ca9ffb..ff516f6691e4 100644 --- a/arch/arm64/kernel/hw_breakpoint.c +++ b/arch/arm64/kernel/hw_breakpoint.c | |||
@@ -184,14 +184,14 @@ int arch_install_hw_breakpoint(struct perf_event *bp) | |||
184 | /* Breakpoint */ | 184 | /* Breakpoint */ |
185 | ctrl_reg = AARCH64_DBG_REG_BCR; | 185 | ctrl_reg = AARCH64_DBG_REG_BCR; |
186 | val_reg = AARCH64_DBG_REG_BVR; | 186 | val_reg = AARCH64_DBG_REG_BVR; |
187 | slots = __get_cpu_var(bp_on_reg); | 187 | slots = this_cpu_ptr(bp_on_reg); |
188 | max_slots = core_num_brps; | 188 | max_slots = core_num_brps; |
189 | reg_enable = !debug_info->bps_disabled; | 189 | reg_enable = !debug_info->bps_disabled; |
190 | } else { | 190 | } else { |
191 | /* Watchpoint */ | 191 | /* Watchpoint */ |
192 | ctrl_reg = AARCH64_DBG_REG_WCR; | 192 | ctrl_reg = AARCH64_DBG_REG_WCR; |
193 | val_reg = AARCH64_DBG_REG_WVR; | 193 | val_reg = AARCH64_DBG_REG_WVR; |
194 | slots = __get_cpu_var(wp_on_reg); | 194 | slots = this_cpu_ptr(wp_on_reg); |
195 | max_slots = core_num_wrps; | 195 | max_slots = core_num_wrps; |
196 | reg_enable = !debug_info->wps_disabled; | 196 | reg_enable = !debug_info->wps_disabled; |
197 | } | 197 | } |
@@ -230,12 +230,12 @@ void arch_uninstall_hw_breakpoint(struct perf_event *bp) | |||
230 | if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { | 230 | if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { |
231 | /* Breakpoint */ | 231 | /* Breakpoint */ |
232 | base = AARCH64_DBG_REG_BCR; | 232 | base = AARCH64_DBG_REG_BCR; |
233 | slots = __get_cpu_var(bp_on_reg); | 233 | slots = this_cpu_ptr(bp_on_reg); |
234 | max_slots = core_num_brps; | 234 | max_slots = core_num_brps; |
235 | } else { | 235 | } else { |
236 | /* Watchpoint */ | 236 | /* Watchpoint */ |
237 | base = AARCH64_DBG_REG_WCR; | 237 | base = AARCH64_DBG_REG_WCR; |
238 | slots = __get_cpu_var(wp_on_reg); | 238 | slots = this_cpu_ptr(wp_on_reg); |
239 | max_slots = core_num_wrps; | 239 | max_slots = core_num_wrps; |
240 | } | 240 | } |
241 | 241 | ||
@@ -505,11 +505,11 @@ static void toggle_bp_registers(int reg, enum debug_el el, int enable) | |||
505 | 505 | ||
506 | switch (reg) { | 506 | switch (reg) { |
507 | case AARCH64_DBG_REG_BCR: | 507 | case AARCH64_DBG_REG_BCR: |
508 | slots = __get_cpu_var(bp_on_reg); | 508 | slots = this_cpu_ptr(bp_on_reg); |
509 | max_slots = core_num_brps; | 509 | max_slots = core_num_brps; |
510 | break; | 510 | break; |
511 | case AARCH64_DBG_REG_WCR: | 511 | case AARCH64_DBG_REG_WCR: |
512 | slots = __get_cpu_var(wp_on_reg); | 512 | slots = this_cpu_ptr(wp_on_reg); |
513 | max_slots = core_num_wrps; | 513 | max_slots = core_num_wrps; |
514 | break; | 514 | break; |
515 | default: | 515 | default: |
@@ -546,7 +546,7 @@ static int breakpoint_handler(unsigned long unused, unsigned int esr, | |||
546 | struct debug_info *debug_info; | 546 | struct debug_info *debug_info; |
547 | struct arch_hw_breakpoint_ctrl ctrl; | 547 | struct arch_hw_breakpoint_ctrl ctrl; |
548 | 548 | ||
549 | slots = (struct perf_event **)__get_cpu_var(bp_on_reg); | 549 | slots = this_cpu_ptr(bp_on_reg); |
550 | addr = instruction_pointer(regs); | 550 | addr = instruction_pointer(regs); |
551 | debug_info = ¤t->thread.debug; | 551 | debug_info = ¤t->thread.debug; |
552 | 552 | ||
@@ -596,7 +596,7 @@ unlock: | |||
596 | user_enable_single_step(current); | 596 | user_enable_single_step(current); |
597 | } else { | 597 | } else { |
598 | toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 0); | 598 | toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 0); |
599 | kernel_step = &__get_cpu_var(stepping_kernel_bp); | 599 | kernel_step = this_cpu_ptr(&stepping_kernel_bp); |
600 | 600 | ||
601 | if (*kernel_step != ARM_KERNEL_STEP_NONE) | 601 | if (*kernel_step != ARM_KERNEL_STEP_NONE) |
602 | return 0; | 602 | return 0; |
@@ -623,7 +623,7 @@ static int watchpoint_handler(unsigned long addr, unsigned int esr, | |||
623 | struct arch_hw_breakpoint *info; | 623 | struct arch_hw_breakpoint *info; |
624 | struct arch_hw_breakpoint_ctrl ctrl; | 624 | struct arch_hw_breakpoint_ctrl ctrl; |
625 | 625 | ||
626 | slots = (struct perf_event **)__get_cpu_var(wp_on_reg); | 626 | slots = this_cpu_ptr(wp_on_reg); |
627 | debug_info = ¤t->thread.debug; | 627 | debug_info = ¤t->thread.debug; |
628 | 628 | ||
629 | for (i = 0; i < core_num_wrps; ++i) { | 629 | for (i = 0; i < core_num_wrps; ++i) { |
@@ -698,7 +698,7 @@ unlock: | |||
698 | user_enable_single_step(current); | 698 | user_enable_single_step(current); |
699 | } else { | 699 | } else { |
700 | toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 0); | 700 | toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 0); |
701 | kernel_step = &__get_cpu_var(stepping_kernel_bp); | 701 | kernel_step = this_cpu_ptr(&stepping_kernel_bp); |
702 | 702 | ||
703 | if (*kernel_step != ARM_KERNEL_STEP_NONE) | 703 | if (*kernel_step != ARM_KERNEL_STEP_NONE) |
704 | return 0; | 704 | return 0; |
@@ -722,7 +722,7 @@ int reinstall_suspended_bps(struct pt_regs *regs) | |||
722 | struct debug_info *debug_info = ¤t->thread.debug; | 722 | struct debug_info *debug_info = ¤t->thread.debug; |
723 | int handled_exception = 0, *kernel_step; | 723 | int handled_exception = 0, *kernel_step; |
724 | 724 | ||
725 | kernel_step = &__get_cpu_var(stepping_kernel_bp); | 725 | kernel_step = this_cpu_ptr(&stepping_kernel_bp); |
726 | 726 | ||
727 | /* | 727 | /* |
728 | * Called from single-step exception handler. | 728 | * Called from single-step exception handler. |
diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c index cea1594ff933..6983ed5a351a 100644 --- a/arch/arm64/kernel/perf_event.c +++ b/arch/arm64/kernel/perf_event.c | |||
@@ -1044,7 +1044,7 @@ static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev) | |||
1044 | */ | 1044 | */ |
1045 | regs = get_irq_regs(); | 1045 | regs = get_irq_regs(); |
1046 | 1046 | ||
1047 | cpuc = &__get_cpu_var(cpu_hw_events); | 1047 | cpuc = this_cpu_ptr(&cpu_hw_events); |
1048 | for (idx = 0; idx < cpu_pmu->num_events; ++idx) { | 1048 | for (idx = 0; idx < cpu_pmu->num_events; ++idx) { |
1049 | struct perf_event *event = cpuc->events[idx]; | 1049 | struct perf_event *event = cpuc->events[idx]; |
1050 | struct hw_perf_event *hwc; | 1050 | struct hw_perf_event *hwc; |
@@ -1257,7 +1257,7 @@ device_initcall(register_pmu_driver); | |||
1257 | 1257 | ||
1258 | static struct pmu_hw_events *armpmu_get_cpu_events(void) | 1258 | static struct pmu_hw_events *armpmu_get_cpu_events(void) |
1259 | { | 1259 | { |
1260 | return &__get_cpu_var(cpu_hw_events); | 1260 | return this_cpu_ptr(&cpu_hw_events); |
1261 | } | 1261 | } |
1262 | 1262 | ||
1263 | static void __init cpu_pmu_init(struct arm_pmu *armpmu) | 1263 | static void __init cpu_pmu_init(struct arm_pmu *armpmu) |