diff options
Diffstat (limited to 'arch/arm64/kernel/armv8_deprecated.c')
-rw-r--r-- | arch/arm64/kernel/armv8_deprecated.c | 553 |
1 files changed, 553 insertions, 0 deletions
diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c new file mode 100644 index 000000000000..c363671d7509 --- /dev/null +++ b/arch/arm64/kernel/armv8_deprecated.c | |||
@@ -0,0 +1,553 @@ | |||
1 | /* | ||
2 | * Copyright (C) 2014 ARM Limited | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify | ||
5 | * it under the terms of the GNU General Public License version 2 as | ||
6 | * published by the Free Software Foundation. | ||
7 | */ | ||
8 | |||
9 | #include <linux/cpu.h> | ||
10 | #include <linux/init.h> | ||
11 | #include <linux/list.h> | ||
12 | #include <linux/perf_event.h> | ||
13 | #include <linux/sched.h> | ||
14 | #include <linux/slab.h> | ||
15 | #include <linux/sysctl.h> | ||
16 | |||
17 | #include <asm/insn.h> | ||
18 | #include <asm/opcodes.h> | ||
19 | #include <asm/system_misc.h> | ||
20 | #include <asm/traps.h> | ||
21 | #include <asm/uaccess.h> | ||
22 | |||
23 | #define CREATE_TRACE_POINTS | ||
24 | #include "trace-events-emulation.h" | ||
25 | |||
26 | /* | ||
27 | * The runtime support for deprecated instruction support can be in one of | ||
28 | * following three states - | ||
29 | * | ||
30 | * 0 = undef | ||
31 | * 1 = emulate (software emulation) | ||
32 | * 2 = hw (supported in hardware) | ||
33 | */ | ||
34 | enum insn_emulation_mode { | ||
35 | INSN_UNDEF, | ||
36 | INSN_EMULATE, | ||
37 | INSN_HW, | ||
38 | }; | ||
39 | |||
40 | enum legacy_insn_status { | ||
41 | INSN_DEPRECATED, | ||
42 | INSN_OBSOLETE, | ||
43 | }; | ||
44 | |||
45 | struct insn_emulation_ops { | ||
46 | const char *name; | ||
47 | enum legacy_insn_status status; | ||
48 | struct undef_hook *hooks; | ||
49 | int (*set_hw_mode)(bool enable); | ||
50 | }; | ||
51 | |||
52 | struct insn_emulation { | ||
53 | struct list_head node; | ||
54 | struct insn_emulation_ops *ops; | ||
55 | int current_mode; | ||
56 | int min; | ||
57 | int max; | ||
58 | }; | ||
59 | |||
60 | static LIST_HEAD(insn_emulation); | ||
61 | static int nr_insn_emulated; | ||
62 | static DEFINE_RAW_SPINLOCK(insn_emulation_lock); | ||
63 | |||
64 | static void register_emulation_hooks(struct insn_emulation_ops *ops) | ||
65 | { | ||
66 | struct undef_hook *hook; | ||
67 | |||
68 | BUG_ON(!ops->hooks); | ||
69 | |||
70 | for (hook = ops->hooks; hook->instr_mask; hook++) | ||
71 | register_undef_hook(hook); | ||
72 | |||
73 | pr_notice("Registered %s emulation handler\n", ops->name); | ||
74 | } | ||
75 | |||
76 | static void remove_emulation_hooks(struct insn_emulation_ops *ops) | ||
77 | { | ||
78 | struct undef_hook *hook; | ||
79 | |||
80 | BUG_ON(!ops->hooks); | ||
81 | |||
82 | for (hook = ops->hooks; hook->instr_mask; hook++) | ||
83 | unregister_undef_hook(hook); | ||
84 | |||
85 | pr_notice("Removed %s emulation handler\n", ops->name); | ||
86 | } | ||
87 | |||
88 | static int update_insn_emulation_mode(struct insn_emulation *insn, | ||
89 | enum insn_emulation_mode prev) | ||
90 | { | ||
91 | int ret = 0; | ||
92 | |||
93 | switch (prev) { | ||
94 | case INSN_UNDEF: /* Nothing to be done */ | ||
95 | break; | ||
96 | case INSN_EMULATE: | ||
97 | remove_emulation_hooks(insn->ops); | ||
98 | break; | ||
99 | case INSN_HW: | ||
100 | if (insn->ops->set_hw_mode) { | ||
101 | insn->ops->set_hw_mode(false); | ||
102 | pr_notice("Disabled %s support\n", insn->ops->name); | ||
103 | } | ||
104 | break; | ||
105 | } | ||
106 | |||
107 | switch (insn->current_mode) { | ||
108 | case INSN_UNDEF: | ||
109 | break; | ||
110 | case INSN_EMULATE: | ||
111 | register_emulation_hooks(insn->ops); | ||
112 | break; | ||
113 | case INSN_HW: | ||
114 | if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(true)) | ||
115 | pr_notice("Enabled %s support\n", insn->ops->name); | ||
116 | else | ||
117 | ret = -EINVAL; | ||
118 | break; | ||
119 | } | ||
120 | |||
121 | return ret; | ||
122 | } | ||
123 | |||
124 | static void register_insn_emulation(struct insn_emulation_ops *ops) | ||
125 | { | ||
126 | unsigned long flags; | ||
127 | struct insn_emulation *insn; | ||
128 | |||
129 | insn = kzalloc(sizeof(*insn), GFP_KERNEL); | ||
130 | insn->ops = ops; | ||
131 | insn->min = INSN_UNDEF; | ||
132 | |||
133 | switch (ops->status) { | ||
134 | case INSN_DEPRECATED: | ||
135 | insn->current_mode = INSN_EMULATE; | ||
136 | insn->max = INSN_HW; | ||
137 | break; | ||
138 | case INSN_OBSOLETE: | ||
139 | insn->current_mode = INSN_UNDEF; | ||
140 | insn->max = INSN_EMULATE; | ||
141 | break; | ||
142 | } | ||
143 | |||
144 | raw_spin_lock_irqsave(&insn_emulation_lock, flags); | ||
145 | list_add(&insn->node, &insn_emulation); | ||
146 | nr_insn_emulated++; | ||
147 | raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); | ||
148 | |||
149 | /* Register any handlers if required */ | ||
150 | update_insn_emulation_mode(insn, INSN_UNDEF); | ||
151 | } | ||
152 | |||
153 | static int emulation_proc_handler(struct ctl_table *table, int write, | ||
154 | void __user *buffer, size_t *lenp, | ||
155 | loff_t *ppos) | ||
156 | { | ||
157 | int ret = 0; | ||
158 | struct insn_emulation *insn = (struct insn_emulation *) table->data; | ||
159 | enum insn_emulation_mode prev_mode = insn->current_mode; | ||
160 | |||
161 | table->data = &insn->current_mode; | ||
162 | ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); | ||
163 | |||
164 | if (ret || !write || prev_mode == insn->current_mode) | ||
165 | goto ret; | ||
166 | |||
167 | ret = update_insn_emulation_mode(insn, prev_mode); | ||
168 | if (ret) { | ||
169 | /* Mode change failed, revert to previous mode. */ | ||
170 | insn->current_mode = prev_mode; | ||
171 | update_insn_emulation_mode(insn, INSN_UNDEF); | ||
172 | } | ||
173 | ret: | ||
174 | table->data = insn; | ||
175 | return ret; | ||
176 | } | ||
177 | |||
178 | static struct ctl_table ctl_abi[] = { | ||
179 | { | ||
180 | .procname = "abi", | ||
181 | .mode = 0555, | ||
182 | }, | ||
183 | { } | ||
184 | }; | ||
185 | |||
186 | static void register_insn_emulation_sysctl(struct ctl_table *table) | ||
187 | { | ||
188 | unsigned long flags; | ||
189 | int i = 0; | ||
190 | struct insn_emulation *insn; | ||
191 | struct ctl_table *insns_sysctl, *sysctl; | ||
192 | |||
193 | insns_sysctl = kzalloc(sizeof(*sysctl) * (nr_insn_emulated + 1), | ||
194 | GFP_KERNEL); | ||
195 | |||
196 | raw_spin_lock_irqsave(&insn_emulation_lock, flags); | ||
197 | list_for_each_entry(insn, &insn_emulation, node) { | ||
198 | sysctl = &insns_sysctl[i]; | ||
199 | |||
200 | sysctl->mode = 0644; | ||
201 | sysctl->maxlen = sizeof(int); | ||
202 | |||
203 | sysctl->procname = insn->ops->name; | ||
204 | sysctl->data = insn; | ||
205 | sysctl->extra1 = &insn->min; | ||
206 | sysctl->extra2 = &insn->max; | ||
207 | sysctl->proc_handler = emulation_proc_handler; | ||
208 | i++; | ||
209 | } | ||
210 | raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); | ||
211 | |||
212 | table->child = insns_sysctl; | ||
213 | register_sysctl_table(table); | ||
214 | } | ||
215 | |||
216 | /* | ||
217 | * Implement emulation of the SWP/SWPB instructions using load-exclusive and | ||
218 | * store-exclusive. | ||
219 | * | ||
220 | * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>] | ||
221 | * Where: Rt = destination | ||
222 | * Rt2 = source | ||
223 | * Rn = address | ||
224 | */ | ||
225 | |||
226 | /* | ||
227 | * Error-checking SWP macros implemented using ldxr{b}/stxr{b} | ||
228 | */ | ||
229 | #define __user_swpX_asm(data, addr, res, temp, B) \ | ||
230 | __asm__ __volatile__( \ | ||
231 | " mov %w2, %w1\n" \ | ||
232 | "0: ldxr"B" %w1, [%3]\n" \ | ||
233 | "1: stxr"B" %w0, %w2, [%3]\n" \ | ||
234 | " cbz %w0, 2f\n" \ | ||
235 | " mov %w0, %w4\n" \ | ||
236 | "2:\n" \ | ||
237 | " .pushsection .fixup,\"ax\"\n" \ | ||
238 | " .align 2\n" \ | ||
239 | "3: mov %w0, %w5\n" \ | ||
240 | " b 2b\n" \ | ||
241 | " .popsection" \ | ||
242 | " .pushsection __ex_table,\"a\"\n" \ | ||
243 | " .align 3\n" \ | ||
244 | " .quad 0b, 3b\n" \ | ||
245 | " .quad 1b, 3b\n" \ | ||
246 | " .popsection" \ | ||
247 | : "=&r" (res), "+r" (data), "=&r" (temp) \ | ||
248 | : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT) \ | ||
249 | : "memory") | ||
250 | |||
251 | #define __user_swp_asm(data, addr, res, temp) \ | ||
252 | __user_swpX_asm(data, addr, res, temp, "") | ||
253 | #define __user_swpb_asm(data, addr, res, temp) \ | ||
254 | __user_swpX_asm(data, addr, res, temp, "b") | ||
255 | |||
256 | /* | ||
257 | * Bit 22 of the instruction encoding distinguishes between | ||
258 | * the SWP and SWPB variants (bit set means SWPB). | ||
259 | */ | ||
260 | #define TYPE_SWPB (1 << 22) | ||
261 | |||
262 | /* | ||
263 | * Set up process info to signal segmentation fault - called on access error. | ||
264 | */ | ||
265 | static void set_segfault(struct pt_regs *regs, unsigned long addr) | ||
266 | { | ||
267 | siginfo_t info; | ||
268 | |||
269 | down_read(¤t->mm->mmap_sem); | ||
270 | if (find_vma(current->mm, addr) == NULL) | ||
271 | info.si_code = SEGV_MAPERR; | ||
272 | else | ||
273 | info.si_code = SEGV_ACCERR; | ||
274 | up_read(¤t->mm->mmap_sem); | ||
275 | |||
276 | info.si_signo = SIGSEGV; | ||
277 | info.si_errno = 0; | ||
278 | info.si_addr = (void *) instruction_pointer(regs); | ||
279 | |||
280 | pr_debug("SWP{B} emulation: access caused memory abort!\n"); | ||
281 | arm64_notify_die("Illegal memory access", regs, &info, 0); | ||
282 | } | ||
283 | |||
284 | static int emulate_swpX(unsigned int address, unsigned int *data, | ||
285 | unsigned int type) | ||
286 | { | ||
287 | unsigned int res = 0; | ||
288 | |||
289 | if ((type != TYPE_SWPB) && (address & 0x3)) { | ||
290 | /* SWP to unaligned address not permitted */ | ||
291 | pr_debug("SWP instruction on unaligned pointer!\n"); | ||
292 | return -EFAULT; | ||
293 | } | ||
294 | |||
295 | while (1) { | ||
296 | unsigned long temp; | ||
297 | |||
298 | if (type == TYPE_SWPB) | ||
299 | __user_swpb_asm(*data, address, res, temp); | ||
300 | else | ||
301 | __user_swp_asm(*data, address, res, temp); | ||
302 | |||
303 | if (likely(res != -EAGAIN) || signal_pending(current)) | ||
304 | break; | ||
305 | |||
306 | cond_resched(); | ||
307 | } | ||
308 | |||
309 | return res; | ||
310 | } | ||
311 | |||
312 | /* | ||
313 | * swp_handler logs the id of calling process, dissects the instruction, sanity | ||
314 | * checks the memory location, calls emulate_swpX for the actual operation and | ||
315 | * deals with fixup/error handling before returning | ||
316 | */ | ||
317 | static int swp_handler(struct pt_regs *regs, u32 instr) | ||
318 | { | ||
319 | u32 destreg, data, type, address = 0; | ||
320 | int rn, rt2, res = 0; | ||
321 | |||
322 | perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); | ||
323 | |||
324 | type = instr & TYPE_SWPB; | ||
325 | |||
326 | switch (arm_check_condition(instr, regs->pstate)) { | ||
327 | case ARM_OPCODE_CONDTEST_PASS: | ||
328 | break; | ||
329 | case ARM_OPCODE_CONDTEST_FAIL: | ||
330 | /* Condition failed - return to next instruction */ | ||
331 | goto ret; | ||
332 | case ARM_OPCODE_CONDTEST_UNCOND: | ||
333 | /* If unconditional encoding - not a SWP, undef */ | ||
334 | return -EFAULT; | ||
335 | default: | ||
336 | return -EINVAL; | ||
337 | } | ||
338 | |||
339 | rn = aarch32_insn_extract_reg_num(instr, A32_RN_OFFSET); | ||
340 | rt2 = aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET); | ||
341 | |||
342 | address = (u32)regs->user_regs.regs[rn]; | ||
343 | data = (u32)regs->user_regs.regs[rt2]; | ||
344 | destreg = aarch32_insn_extract_reg_num(instr, A32_RT_OFFSET); | ||
345 | |||
346 | pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n", | ||
347 | rn, address, destreg, | ||
348 | aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data); | ||
349 | |||
350 | /* Check access in reasonable access range for both SWP and SWPB */ | ||
351 | if (!access_ok(VERIFY_WRITE, (address & ~3), 4)) { | ||
352 | pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n", | ||
353 | address); | ||
354 | goto fault; | ||
355 | } | ||
356 | |||
357 | res = emulate_swpX(address, &data, type); | ||
358 | if (res == -EFAULT) | ||
359 | goto fault; | ||
360 | else if (res == 0) | ||
361 | regs->user_regs.regs[destreg] = data; | ||
362 | |||
363 | ret: | ||
364 | if (type == TYPE_SWPB) | ||
365 | trace_instruction_emulation("swpb", regs->pc); | ||
366 | else | ||
367 | trace_instruction_emulation("swp", regs->pc); | ||
368 | |||
369 | pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n", | ||
370 | current->comm, (unsigned long)current->pid, regs->pc); | ||
371 | |||
372 | regs->pc += 4; | ||
373 | return 0; | ||
374 | |||
375 | fault: | ||
376 | set_segfault(regs, address); | ||
377 | |||
378 | return 0; | ||
379 | } | ||
380 | |||
381 | /* | ||
382 | * Only emulate SWP/SWPB executed in ARM state/User mode. | ||
383 | * The kernel must be SWP free and SWP{B} does not exist in Thumb. | ||
384 | */ | ||
385 | static struct undef_hook swp_hooks[] = { | ||
386 | { | ||
387 | .instr_mask = 0x0fb00ff0, | ||
388 | .instr_val = 0x01000090, | ||
389 | .pstate_mask = COMPAT_PSR_MODE_MASK, | ||
390 | .pstate_val = COMPAT_PSR_MODE_USR, | ||
391 | .fn = swp_handler | ||
392 | }, | ||
393 | { } | ||
394 | }; | ||
395 | |||
396 | static struct insn_emulation_ops swp_ops = { | ||
397 | .name = "swp", | ||
398 | .status = INSN_OBSOLETE, | ||
399 | .hooks = swp_hooks, | ||
400 | .set_hw_mode = NULL, | ||
401 | }; | ||
402 | |||
403 | static int cp15barrier_handler(struct pt_regs *regs, u32 instr) | ||
404 | { | ||
405 | perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); | ||
406 | |||
407 | switch (arm_check_condition(instr, regs->pstate)) { | ||
408 | case ARM_OPCODE_CONDTEST_PASS: | ||
409 | break; | ||
410 | case ARM_OPCODE_CONDTEST_FAIL: | ||
411 | /* Condition failed - return to next instruction */ | ||
412 | goto ret; | ||
413 | case ARM_OPCODE_CONDTEST_UNCOND: | ||
414 | /* If unconditional encoding - not a barrier instruction */ | ||
415 | return -EFAULT; | ||
416 | default: | ||
417 | return -EINVAL; | ||
418 | } | ||
419 | |||
420 | switch (aarch32_insn_mcr_extract_crm(instr)) { | ||
421 | case 10: | ||
422 | /* | ||
423 | * dmb - mcr p15, 0, Rt, c7, c10, 5 | ||
424 | * dsb - mcr p15, 0, Rt, c7, c10, 4 | ||
425 | */ | ||
426 | if (aarch32_insn_mcr_extract_opc2(instr) == 5) { | ||
427 | dmb(sy); | ||
428 | trace_instruction_emulation( | ||
429 | "mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs->pc); | ||
430 | } else { | ||
431 | dsb(sy); | ||
432 | trace_instruction_emulation( | ||
433 | "mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs->pc); | ||
434 | } | ||
435 | break; | ||
436 | case 5: | ||
437 | /* | ||
438 | * isb - mcr p15, 0, Rt, c7, c5, 4 | ||
439 | * | ||
440 | * Taking an exception or returning from one acts as an | ||
441 | * instruction barrier. So no explicit barrier needed here. | ||
442 | */ | ||
443 | trace_instruction_emulation( | ||
444 | "mcr p15, 0, Rt, c7, c5, 4 ; isb", regs->pc); | ||
445 | break; | ||
446 | } | ||
447 | |||
448 | ret: | ||
449 | pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n", | ||
450 | current->comm, (unsigned long)current->pid, regs->pc); | ||
451 | |||
452 | regs->pc += 4; | ||
453 | return 0; | ||
454 | } | ||
455 | |||
456 | #define SCTLR_EL1_CP15BEN (1 << 5) | ||
457 | |||
458 | static inline void config_sctlr_el1(u32 clear, u32 set) | ||
459 | { | ||
460 | u32 val; | ||
461 | |||
462 | asm volatile("mrs %0, sctlr_el1" : "=r" (val)); | ||
463 | val &= ~clear; | ||
464 | val |= set; | ||
465 | asm volatile("msr sctlr_el1, %0" : : "r" (val)); | ||
466 | } | ||
467 | |||
468 | static void enable_cp15_ben(void *info) | ||
469 | { | ||
470 | config_sctlr_el1(0, SCTLR_EL1_CP15BEN); | ||
471 | } | ||
472 | |||
473 | static void disable_cp15_ben(void *info) | ||
474 | { | ||
475 | config_sctlr_el1(SCTLR_EL1_CP15BEN, 0); | ||
476 | } | ||
477 | |||
478 | static int cpu_hotplug_notify(struct notifier_block *b, | ||
479 | unsigned long action, void *hcpu) | ||
480 | { | ||
481 | switch (action) { | ||
482 | case CPU_STARTING: | ||
483 | case CPU_STARTING_FROZEN: | ||
484 | enable_cp15_ben(NULL); | ||
485 | return NOTIFY_DONE; | ||
486 | case CPU_DYING: | ||
487 | case CPU_DYING_FROZEN: | ||
488 | disable_cp15_ben(NULL); | ||
489 | return NOTIFY_DONE; | ||
490 | } | ||
491 | |||
492 | return NOTIFY_OK; | ||
493 | } | ||
494 | |||
495 | static struct notifier_block cpu_hotplug_notifier = { | ||
496 | .notifier_call = cpu_hotplug_notify, | ||
497 | }; | ||
498 | |||
499 | static int cp15_barrier_set_hw_mode(bool enable) | ||
500 | { | ||
501 | if (enable) { | ||
502 | register_cpu_notifier(&cpu_hotplug_notifier); | ||
503 | on_each_cpu(enable_cp15_ben, NULL, true); | ||
504 | } else { | ||
505 | unregister_cpu_notifier(&cpu_hotplug_notifier); | ||
506 | on_each_cpu(disable_cp15_ben, NULL, true); | ||
507 | } | ||
508 | |||
509 | return true; | ||
510 | } | ||
511 | |||
512 | static struct undef_hook cp15_barrier_hooks[] = { | ||
513 | { | ||
514 | .instr_mask = 0x0fff0fdf, | ||
515 | .instr_val = 0x0e070f9a, | ||
516 | .pstate_mask = COMPAT_PSR_MODE_MASK, | ||
517 | .pstate_val = COMPAT_PSR_MODE_USR, | ||
518 | .fn = cp15barrier_handler, | ||
519 | }, | ||
520 | { | ||
521 | .instr_mask = 0x0fff0fff, | ||
522 | .instr_val = 0x0e070f95, | ||
523 | .pstate_mask = COMPAT_PSR_MODE_MASK, | ||
524 | .pstate_val = COMPAT_PSR_MODE_USR, | ||
525 | .fn = cp15barrier_handler, | ||
526 | }, | ||
527 | { } | ||
528 | }; | ||
529 | |||
530 | static struct insn_emulation_ops cp15_barrier_ops = { | ||
531 | .name = "cp15_barrier", | ||
532 | .status = INSN_DEPRECATED, | ||
533 | .hooks = cp15_barrier_hooks, | ||
534 | .set_hw_mode = cp15_barrier_set_hw_mode, | ||
535 | }; | ||
536 | |||
537 | /* | ||
538 | * Invoked as late_initcall, since not needed before init spawned. | ||
539 | */ | ||
540 | static int __init armv8_deprecated_init(void) | ||
541 | { | ||
542 | if (IS_ENABLED(CONFIG_SWP_EMULATION)) | ||
543 | register_insn_emulation(&swp_ops); | ||
544 | |||
545 | if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION)) | ||
546 | register_insn_emulation(&cp15_barrier_ops); | ||
547 | |||
548 | register_insn_emulation_sysctl(ctl_abi); | ||
549 | |||
550 | return 0; | ||
551 | } | ||
552 | |||
553 | late_initcall(armv8_deprecated_init); | ||